MPKFNet/System_I.py

"""
ÃܶȹÀ¼Æ
"""

import cv2
import json
import random
from configs.configs import model_head

import os,sys
import torch
from torch import nn
from PIL import Image, ImageDraw, ImageFont
from vizer.draw import draw_boxes
# from ssd.config import cfg
# from ssd.data.datasets import  VOCDataset
#
import numpy as np
# from ssd.data.transforms import build_transforms
# from ssd.modeling.detector import build_detection_model
# from ssd.utils.checkpoint import CheckPointer
#
from fine.models.LoadModel import MainModel
from fine.fineConfig import LoadConfig, load_data_transformers
from torch.autograd import Variable
import threading
from common import global_para as  gp
import time,statistics
from numpy import random
root_path = os.getcwd()
sys.path.insert(0,root_path+"/Y5")
from Y5.models.experimental import attempt_load
from Y5.utils.datasets import  letterbox
from Y5.utils.general import   non_max_suppression, scale_coords
from Y5.utils.plots import plot_one_box
from network_multi import Multi_transform_image
from cv2AddChineseText  import cv2AddChineseText
class Density:
    def __init__(self, device, root):
            self.device = device
            self.root = root
            self.net = PKSNet().to(self.device)
            self.pth = "../data_MPKFNet/test/1.pth"
           # print("creat model is ", (t4 - t3) * 1000)
           #  self.checkpointer = CheckPointer(self.net)
           #  self.net.load_state_dict(self.checkpointer['net'])
           #  self.checkpointer.load(self.checkpointer['net'])
            checkpoint = torch.load(self.pth)
            self.net.load_state_dict(checkpoint['net'])
            if gp.DebugPrint_flag:
                print("load model weight ok")

            np.set_printoptions(threshold=np.inf)



    def inference(self,image,imagetime,index):
        rootpath = os.path.join(self.root, imagetime)
        if not os.path.exists(rootpath):
            os.mkdir(rootpath)
        if not  os.path.exists(rootpath + "/" + "Density"+"/"):
            os.mkdir(rootpath + "/" + "Density"+"/")
        target_h, target_w = image.shape[0], image.shape[1]
        t1 = time.time()
        image = cv2.resize(image, (1024, 768)) / 255.0
        image = np.transpose(image, [2, 0, 1])
        image = np.expand_dims(image, axis=0)
        image = torch.tensor(image)
        image = image.to(self.device)
        t2 = time.time()
        if gp.DebugPrint_flag:
            print("Density data deal is ", (t2 - t1) * 1000)

       # print("data dell is ", (t7 - t6) * 1000)

        if model_head:
            targets_pred, posture_pred, spice_pred = self.net(image)
        else:
            targets_pred = self.net(image)
        t3=time.time()
        if gp.DebugPrint_flag:
            print("Density inference is ", (t3 - t2) * 1000)
        targets_pred1 = targets_pred.cpu()
        num = targets_pred1.data.sum().numpy()
        targets_pred1 = targets_pred1.data.numpy()
        t4=time.time()
        if gp.DebugPrint_flag:
            print("Density final is ", (t4 - t3) * 1000)

        data_dict = {"sum": str(num), "density": (str(targets_pred1[0, 0, :, :]))}
        json_outputpath = (rootpath + "/" + "Density"+"/"+str(index).zfill(3)+".json")
        json_data = json.dumps(data_dict, ensure_ascii=True, indent=4)
        with open (json_outputpath,'w',encoding='utf-8') as f:
            f.write(str(json_data))
        t5=time.time()
        if gp.DebugPrint_flag:
            print("Density save is ", (t5 - t4) * 1000)

    def inference_init(self, image):
        image = cv2.resize(image, (1024, 768)) / 255.0
        image = np.transpose(image, [2, 0, 1])
        image = np.expand_dims(image, axis=0)
        image = torch.tensor(image)
        image = image.to(self.device)

        if model_head:
            targets_pred, posture_pred, spice_pred = self.net(image)
        else:
            targets_pred = self.net(image)

        targets_pred1 = targets_pred.cpu()
        num = targets_pred1.data.sum().numpy()
        targets_pred1 = targets_pred1.data.numpy()
        if gp.DebugPrint_flag:
            print("Density initing")

"""
Ä¿±ê¼ì²â
"""
class Detection_S:
    def __init__(self,device,root):
        self.device = device
        self.root = root
        cfg.merge_from_file("ssd/config/hat-512.yaml")
        cfg.freeze()
        self.net =build_detection_model(cfg).to(self.device)
        self.checkpointer = CheckPointer(self.net, save_dir=cfg.OUTPUT_DIR)
        self.ckpt = "../data_MPKFNet/weigth/detection/detection.pth"
        self.checkpointer.load(self.ckpt, use_latest=self.ckpt is None)
        self.transforms = build_transforms(cfg, is_train=False)
        self.threshold = gp.detect_thresh
        self.net.eval()
    def inference(self,image,imagetime,index,fine_grained):
        with torch.no_grad():
            class_names = VOCDataset.class_names
            rootpath = os.path.join(self.root, imagetime)
            if not os.path.exists(rootpath):
                os.mkdir(rootpath)
            if not os.path.exists(rootpath + "/" + "Detection" + "/"):
                os.mkdir(rootpath + "/" + "Detection" + "/")
            image_or = image
            height, width = image.shape[:2]
            t1 = time.time()
            image = self.transforms(image)[0].unsqueeze(0)
            t2 = time.time()
            if gp.DebugPrint_flag:
                print("Detection data deal is ",(t2-t1)*1000)
            result = self.net(image.to(self.device))[0]
            t3 = time.time()
            if gp.DebugPrint_flag:
                print("Detection inference is ",(t3-t2)*1000)
            result = result.to(torch.device("cpu"))
            result = result.resize((width, height)).numpy()
            t5 = time.time()
            if gp.DebugPrint_flag:
                print("Detection get result is ", (t5 - t3) * 1000)
            boxes, labels, scores = result['boxes'], result['labels'], result['scores']
            indices_score = (scores > self.threshold)
            indices_label_left =  labels <4
            indices_label_right = labels >2
            indices = []
            for i in range(len(indices_score)):
                indices.append(indices_score[i] and indices_label_left[i] and indices_label_right[i])
            boxes = boxes[indices]
            labels = labels[indices]
            scores = scores[indices]
            for i in range(len(boxes)):
                if boxes[i][0]<0:
                    boxes[i][0] = 0
                if boxes[i][1]<0:
                    boxes[i][1] = 0
                if boxes[i][2]<0:
                    boxes[i][2] = 0
                if boxes[i][3]<0:
                    boxes[i][3] = 0
            for i in range(len(labels)):
                labels[i] = 99999
            boxes_fine, labels_fine, scores_fine = fine_grained.inference(image_or, boxes, labels, scores)
            t4 = time.time()
            if gp.DebugPrint_flag:
                print("Detection final is ", (t4 - t3) * 1000)
            #numb = 2
            #fine_grained.inference(image_or,boxes,labels,scores)
            # for box in boxes:
            #     img1 = image_or.copy()[int(box[1]):int(box[3]), int(box[0]):int(box[2]), :]
            #     cv2.imwrite(rootpath + "/" + "Detection" + "/"+"{}.jpg".format(numb), img1)
            #     numb+=1
            #drawn_image = draw_boxes(image_or, boxes_fine, labels_fine, scores_fine).astype(np.uint8)
            xml_outputpath = rootpath + "/" + "Detection"+"/" + index+".xml"
            make_xml(str(index),image_or.shape[0],image_or.shape[1],boxes_fine,labels_fine,scores_fine,xml_outputpath,class_names)
            #Image.fromarray(drawn_image).save(xml_outputpath.replace(".xml",".jpg"))
            t5 = time.time()
            if gp.DebugPrint_flag:
                print("Detection save is ", (t5 - t4) * 1000)
    def inference_Red(self,image,ImagePath,fine_grained):
        with torch.no_grad():
            class_names = VOCDataset.class_names
            image_or = image
            height, width = image.shape[:2]
            t1 = time.time()
            image = self.transforms(image)[0].unsqueeze(0)
            t2 = time.time()
            if gp.DebugPrint_flag:
                print("data deal is ",(t2-t1)*1000)
            result = self.net(image.to(self.device))[0]
            t3 = time.time()
            if gp.DebugPrint_flag:
                print("inference is ",(t3-t2)*1000)
            result = result.to(torch.device("cpu"))
            result = result.resize((width, height)).numpy()
            boxes, labels, scores = result['boxes'], result['labels'], result['scores']
            indices_score = (scores > self.threshold)
            indices_label_left =  labels <4
            indices_label_right = labels >2
            indices = []
            for i in range(len(indices_score)):
                indices.append(indices_score[i] and indices_label_left[i] and indices_label_right[i])
            boxes = boxes[indices]
            for i in range(len(boxes)):
                if boxes[i][0]<0:
                    boxes[i][0] = 0
                if boxes[i][1]<0:
                    boxes[i][1] = 0
                if boxes[i][2]<0:
                    boxes[i][2] = 0
                if boxes[i][3]<0:
                    boxes[i][3] = 0

            labels = labels[indices]
            scores = scores[indices]
            for i in range(len(labels)):
                labels[i] = 99999
            boxes_fine, labels_fine, scores_fine = fine_grained.inference(image_or, boxes, labels, scores)
            t4 = time.time()
            if gp.DebugPrint_flag:
                print("final is ", (t4 - t3) * 1000)
            #numb = 2
            #fine_grained.inference(image_or,boxes,labels,scores)
            # for box in boxes:
            #     img1 = image_or.copy()[int(box[1]):int(box[3]), int(box[0]):int(box[2]), :]
            #     cv2.imwrite(rootpath + "/" + "Detection" + "/"+"{}.jpg".format(numb), img1)
            #     numb+=1
            #drawn_image = draw_boxes(image_or, boxes_fine, labels_fine, scores_fine).astype(np.uint8)
            xml_outputpath = ImagePath.replace("/ftp/", "/ftp_Result/").replace(".JPG", ".xml").replace(".jpg",".xml")
            make_xml(str(ImagePath),image_or.shape[0],image_or.shape[1],boxes_fine,labels_fine,scores_fine,xml_outputpath)
            #Image.fromarray(drawn_image).save(xml_outputpath.replace(".xml",".jpg"))
            t5 = time.time()
            if gp.DebugPrint_flag:
                print("save is ", (t5 - t4) * 1000)

    def check_object(self,image):
        flag = False
        with torch.no_grad():
            height, width = image.shape[:2]
            img_temp = image
            t1 = time.time()
            image = self.transforms(image)[0].unsqueeze(0)
            t2 = time.time()
            if gp.DebugPrint_flag:
                print("check datadeal is ",(t2-t1)*1000)
            result = self.net(image.to(self.device))[0]
            t3 = time.time()
            if gp.DebugPrint_flag:
                print("check inference is ", (t3 - t2) * 1000)
            result = result.to(torch.device("cpu"))
            t4 = time.time()
            if gp.DebugPrint_flag:
                print("check result deal is ", (t4 - t3) * 1000)
            result = result.resize((width, height)).numpy()

            boxes, labels, scores = result['boxes'], result['labels'], result['scores']
            indices_score = (scores > self.threshold)
            indices_label_left =  labels <4
            indices_label_right = labels >2
            indices = []
            for i in range(len(indices_score)):
                indices.append(indices_score[i] and indices_label_left[i] and indices_label_right[i])
            labels = labels[indices]
            boxes = boxes[indices]
            scores=scores[indices]
            if 3 in labels:
                flag = True
            if gp.DebugPrint_flag:
                print("boxes = ",boxes)
                print("check flag is  ",flag)
            t5 = time.time()
            if gp.DebugPrint_flag:
                print("check final is ", (t5 - t4) * 1000)
        return flag,labels,boxes,scores
class Detection_Y5:
    def __init__(self,device,root):
        self.device = device
        self.root = root
        self.img_size = 640
        self.conf_thres = gp.detect_thresh
        self.iou_thres = 0.4
        self.stride = 1
        if gp.red_flag:
            self.weights = '../data_MPKFNet/test/2_r.pt'
        else:
            self.weights = gp.DetWeight
        # self.device = select_device(self.device)
        model = attempt_load(self.weights, map_location=self.device)
        model.to(self.device).eval()
        model.half()
        self.model = model
        self.names = model.module.names if hasattr(
            model, 'module') else model.names
    def preprocess(self, img):

        img0 = img.copy()
        #img = cv2.resize(img, (1024, 768))
        img = letterbox(img, new_shape=self.img_size, stride=self.stride)[0]
        img = img[:, :, ::-1].transpose(2, 0, 1)
        img = np.ascontiguousarray(img)
        img = torch.from_numpy(img).to(self.device)
        img = img.half()  # ¡ã????¨¨
        img /= 255.0  # ¨ª???1¨¦¨°??¡¥
        if img.ndimension() == 3:
            img = img.unsqueeze(0)

        return img0, img
    def inference(self,image,imagetime,index,fine_grained):
        with torch.no_grad():
            rootpath = os.path.join(self.root, imagetime)
            if not os.path.exists(rootpath):
                os.mkdir(rootpath)
            if not os.path.exists(rootpath + "/" + "Detection" + "/"):
                os.mkdir(rootpath + "/" + "Detection" + "/")
            image_or = image
            img_detect = cv2.resize(image, (1024, 768))
            t1 = time.time()
            image_scale,image = self.preprocess(img_detect)
            t2 = time.time()
            if gp.DebugPrint_flag:
                print("Detection data deal is ",(t2-t1)*1000)
            pred = self.model(image, augment=False)[0]
            pred = pred.float()
            pred = non_max_suppression(pred, self.conf_thres, self.iou_thres)
            t3 = time.time()
            if gp.DebugPrint_flag:
                print("Detection inference is ",(t3-t2)*1000)
            boxes = []
            scores = []
            labels = []
            for det in pred:
                if det is not None and len(det):
                    det[:, :4] = scale_coords(
                        image.shape[2:], det[:, :4], image_scale.shape).round()

                    boxes.append(det[:, :4])
                    scores.append(det[:, 4])
                    labels.append((det[:, -1]))
                    boxes = boxes[0]
                    scores = scores[0]
                    labels = labels[0]
            t5 = time.time()
            if gp.DebugPrint_flag:
                print("Detection get result is ", (t5 - t3) * 1000)

            if len(labels) > 0:
                indices_label_left = labels < 10
                indices_label_right = labels < 10
                indices = []
                for i in range(len(indices_label_left)):
                    indices.append(indices_label_left[i] and indices_label_right[i])

                boxes = boxes[indices]
                labels = labels[indices]
                scores = scores[indices]  # ÒòΪĪÃûÆæÃî¶àÁËÒ»¸ölistά¶È£¬ËùÒÔ¼ÓÁËÕâ¸ö0
            if len(labels) > 0:


                boxes = boxes[0]
                labels = labels[0]
                scores = scores[0]
                ratio_x = (3840.0 / 1024.0)
                ratio_y = (2160.0 / 768.0)


                for i in range(len(boxes)):
                    boxes[i][0]=boxes[i][0] * ratio_x
                    boxes[i][2]=boxes[i][2] * ratio_x
                    boxes[i][1] = boxes[i][1] * ratio_y
                    boxes[i][3] = boxes[i][3] * ratio_y
                    if boxes[i][0]<0:
                        boxes[i][0] = 0
                    if boxes[i][1]<0:
                        boxes[i][1] = 0
                    if boxes[i][2]<0:
                        boxes[i][2] = 0
                    if boxes[i][3]<0:
                        boxes[i][3] = 0
                for i in range(len(labels)):
                    labels[i] = 99999
                boxes_fine, labels_fine, scores_fine = fine_grained.inference(image_or, boxes, labels, scores)
                t4 = time.time()
                if gp.DebugPrint_flag:
                    print("Detection final is ", (t4 - t3) * 1000)
            else:
                boxes_fine, labels_fine, scores_fine = [],[],[]
                if gp.DebugPrint_flag:
                    print("Detection is nothong")
            #numb = 2
            #fine_grained.inference(image_or,boxes,labels,scores)
            # for box in boxes:
            #     img1 = image_or.copy()[int(box[1]):int(box[3]), int(box[0]):int(box[2]), :]
            #     cv2.imwrite(rootpath + "/" + "Detection" + "/"+"{}.jpg".format(numb), img1)
            #     numb+=1
            #drawn_image = draw_boxes(image_or, boxes_fine, labels_fine, scores_fine).astype(np.uint8)
            #labels_fine = labels_fine.numpy()
            t6 = time.time()
            xml_outputpath = rootpath + "/" + "Detection"+"/" + index+".xml"
            make_xml(str(index),image_or.shape[0],image_or.shape[1],boxes_fine,labels_fine,scores_fine,xml_outputpath)
            #Image.fromarray(drawn_image).save(xml_outputpath.replace(".xml",".jpg"))
            t5 = time.time()
            if gp.DebugPrint_flag:
                print("Detection save is ", (t5 - t6) * 1000)
    def inference_Red(self,image,ImagePath,fine_grained):
        with torch.no_grad():
            image_or = image
            height, width = image.shape[:2]
            img_detect = cv2.resize(image, (1024, 768))
            t1 = time.time()
            image_scale, image = self.preprocess(img_detect)
            t2 = time.time()
            if gp.DebugPrint_flag:
                print("Detection data deal is ", (t2 - t1) * 1000)
            pred = self.model(image, augment=False)[0]
            pred = pred.float()
            pred = non_max_suppression(pred, self.conf_thres, self.iou_thres)
            t3 = time.time()
            if gp.DebugPrint_flag:
                print("Detection inference is ", (t3 - t2) * 1000)
            boxes = []
            scores = []
            labels = []
            for det in pred:

                if det is not None and len(det):
                    det[:, :4] = scale_coords(
                        image.shape[2:], det[:, :4], image_scale.shape).round()

                    boxes.append(det[:, :4])
                    scores.append(det[:, 4])
                    labels.append((det[:, -1]))
                    boxes = boxes[0]
                    scores = scores[0]
                    labels = labels[0]
            t5 = time.time()
            if gp.DebugPrint_flag:
                print("Detection get result is ", (t5 - t3) * 1000)
            print(labels)
            if len(labels) > 0 :
                indices_label_left = labels < 1
                indices_label_right = labels < 1
                indices = []
                for i in range(len(indices_label_left)):
                    indices.append(indices_label_left[i] and indices_label_right[i])

                boxes = boxes[indices]
                labels = labels[indices]
                scores = scores[indices]  # ÒòΪĪÃûÆæÃî¶àÁËÒ»¸ölistά¶È£¬ËùÒÔ¼ÓÁËÕâ¸ö0
            if len(labels) > 0:


                boxes = boxes[0]
                labels = labels[0]
                scores = scores[0]
                ratio_x = (width / 1024.0)
                ratio_y = (height / 768.0)

                for i in range(len(boxes)):
                    boxes[i][0] = boxes[i][0] * ratio_x
                    boxes[i][2] = boxes[i][2] * ratio_x
                    boxes[i][1] = boxes[i][1] * ratio_y
                    boxes[i][3] = boxes[i][3] * ratio_y
                    if boxes[i][0] < 0:
                        boxes[i][0] = 0
                    if boxes[i][0] > 3840:
                        boxes[i][0] = 3839
                    if boxes[i][1] < 0:
                        boxes[i][1] = 0
                    if boxes[i][1] > 2160:
                        boxes[i][1] = 2159
                    if boxes[i][2] < 0:
                        boxes[i][2] = 0
                    if boxes[i][2]> 3840:
                        boxes[i][2] = 3839
                    if boxes[i][3] < 0:
                        boxes[i][3] = 0
                    if boxes[i][3] > 2160:
                        boxes[i][3] = 2159
                for i in range(len(labels)):
                    labels[i] = 99999
                boxes_fine, labels_fine, scores_fine = fine_grained.inference(image_or, boxes, labels, scores)
                t4 = time.time()
                if gp.DebugPrint_flag:
                    print("Detection final is ", (t4 - t3) * 1000)
                t6 = time.time()
                xml_outputpath = ImagePath.replace("/ftp/", "/ftp_Result/").replace(".JPG", ".xml").replace(".jpg",
                                                                                                            ".xml")
                make_xml(str(ImagePath), image_or.shape[0], image_or.shape[1], boxes_fine, labels_fine, scores_fine,
                         xml_outputpath)
                # Image.fromarray(drawn_image).save(xml_outputpath.replace(".xml",".jpg"))
                t5 = time.time()
                if gp.DebugPrint_flag:
                    print("Detection save is ", (t5 - t6) * 1000)
            else:
                boxes_fine, labels_fine, scores_fine = [], [], []
                if gp.DebugPrint_flag:
                    print("Detection is nothong")


    def check_object(self,img_o):
        flag = False
        with torch.no_grad():
            # path = '/data/Hatcher/ouput1/0001_20210414151147/RawImage/0099.jpg'
            # img = cv2.imread(path)
            # img = cv2.resize()
            height, width = img_o.shape[:2]
            img = cv2.resize(img_o,(1024,768))
            t1 = time.time()
            image_scale, image = self.preprocess(img)
            t2 = time.time()
            if gp.DebugPrint_flag:
                print("Check data deal is ", (t2 - t1) * 1000)
            pred = self.model(image, augment=False)[0]
            pred = pred.float()
            pred = non_max_suppression(pred, self.conf_thres, self.iou_thres)
            t3 = time.time()
            if gp.DebugPrint_flag:
                print("Check inference is ", (t3 - t2) * 1000)
            boxes = []
            scores = []
            labels = []
            for det in pred:

                if det is not None and len(det):
                    det[:, :4] = scale_coords(
                        image.shape[2:], det[:, :4], image_scale.shape).round()

                    boxes.append(det[:, :4])
                    scores.append(det[:, 4])
                    labels.append((det[:, -1]))
                    boxes = boxes[0]
                    scores = scores[0]
                    labels = labels[0]
            t5 = time.time()
            if gp.DebugPrint_flag:
                print("Check get result is ", (t5 - t3) * 1000)

            if len(labels) > 0:
                # indices_label_left = labels < 1
                # indices_label_right = labels < 1
                # indices = []
                # for i in range(len(indices_label_left)):
                #     indices.append(indices_label_left[i] and indices_label_right[i])
                # boxes = boxes[indices]
                # print(type(labels))
                # print(labels)
                # print(type(indices))
                # print(indices)

                labels = labels
                scores = scores  # ÒòΪĪÃûÆæÃî¶àÁËÒ»¸ölistά¶È£¬ËùÒÔ¼ÓÁËÕâ¸ö0

            if len(labels) > 0:
                if len(boxes.shape) == 3:

                    boxes = boxes[0]
                    labels = labels[0]
                    scores = scores[0]
                ratio_x = (width / 1024.0)
                ratio_y = (height / 768.0)
                for i in range(len(boxes)):
                    #boxes = boxes.numpy()
                    boxes[i][0] = boxes[i][0] * ratio_x
                    boxes[i][2] = boxes[i][2] * ratio_x
                    boxes[i][1] = boxes[i][1] * ratio_y
                    boxes[i][3] = boxes[i][3] * ratio_y
                    if boxes[i][0] < 0:
                        boxes[i][0] = 0
                    if boxes[i][0] > 3840:
                        boxes[i][0] = 3839
                    if boxes[i][1] < 0:
                        boxes[i][1] = 0
                    if boxes[i][1] > 2160:
                        boxes[i][1] = 2159
                    if boxes[i][2] < 0:
                        boxes[i][2] = 0
                    if boxes[i][2] > 3840:
                        boxes[i][2] = 3839
                    if boxes[i][3] < 0:
                        boxes[i][3] = 0
                    if boxes[i][3] > 2160:
                        boxes[i][3] = 2159
            else:
                pass
            if 0 in labels:
                flag = True
            if gp.DebugPrint_flag:
                print("boxes = ",boxes)
                print("check flag is  ",flag)
        return flag,labels,boxes,scores
"""
ϸÁ£¶Èʶ±ð
"""
class args_temp():
    dataset = 'HCData'
    backbone = 'resnet50'
    batch_size = 1
    num_workers = 8
    version = 'val'
    resize_resolution = 512
    crop_resolution = 448
    save_suffix = None
    swap_num = [7,7]
class Fine_grained_T:
    def __init__(self,device,root):
        self.device = device
        self.WeightPath = gp.FineWeight
        self.net = Multi_transform_image().to(device)
        model_dict = self.net.state_dict()
        pretrained_dict = torch.load(self.WeightPath)
        pretrained_dict = {k: v for k, v in pretrained_dict.items() if
                           k not in ['classifier.weight', 'classifier_swap.weight', 'Convmask.weight', 'Convmask.bias']}
        model_dict.update(pretrained_dict)
        self.net.load_state_dict(model_dict)
        self.transformers = load_data_transformers(600, 448, [7, 7])
        print("load model weight is ok")
        self.net.train(False)
        self.softmax = nn.Softmax()
        if gp.DebugPrint_flag:
            print("load model weight is ok")
    def inference(self,img,imagelist,labellist,scorelist,multiimagenum):
        with torch.no_grad():
            img_deal = []
            labelIndexForFusionResult = 0
            imaIndexIfSuit = [0 for i in range(len(imagelist))]
            for i,box in enumerate(imagelist):
                if box[0]<0:
                    box[0] = 0
                if box[1] < 0:
                    box[1] = 0
                if box[2] < 0:
                    box[2] = 0
                if box[3] < 0:
                    box[3] = 0
                Img = img.copy()[int(box[1]):int(box[3]), int(box[0]):int(box[2]), :]
                #if gp.DebugPrint_flag:
                    #print(box)
                    #print(Img)
                #if (int(box[3])-int(box[1])>gp.fine_thresh) and (int(box[2])-int(box[0])>gp.fine_thresh):
                if ((int(box[3]) - int(box[1])) * (int(box[2]) - int(box[0]))) > gp.fine_thresh1 :
                    Img = Image.fromarray(cv2.cvtColor(Img, cv2.COLOR_BGR2RGB))
                    Img = self.transformers['test_totensor'](Img)
                    Img = Img.unsqueeze(0)
                    img_deal.append(Img)
                    imaIndexIfSuit[i] =1
                else:
                    pass
            iter_nums = int(len(img_deal)/multiimagenum)
            single_imagedealnums = int(len(img_deal)%multiimagenum)
            for iter_num in range(iter_nums):
                print("start {} iter".format(str(iter_num)))
                t1 = time.time()
                Img = torch.cat((img_deal[0*iter_num],img_deal[1*iter_num],img_deal[2*iter_num],img_deal[3*iter_num],img_deal[4*iter_num],img_deal[5*iter_num],img_deal[6*iter_num],img_deal[7*iter_num]),0)
                inputs = Variable(Img.cuda())
                t2 = time.time()
                if gp.DebugPrint_flag:
                    print("iter{} fine data deal is ".format(str(iter_num)), (t2 - t1) * 1000)
                outputs = self.net(inputs)
                outputs = self.softmax(outputs)
                t3 = time.time()
                if gp.DebugPrint_flag:
                    print("iter{} fine inference is ".format(str(iter_num)), (t3 - t2) * 1000)
                #outputs[0] = self.softmax(outputs[0])

                #Species = torch.zeros(multiimagenum).cuda()
                Species = gp.Species
                t4_1 = time.time()
                if gp.DebugPrint_flag:
                    print("iter{} fine data final1 is ".format(str(iter_num)), (t4_1 - t3) * 1000)
                for outnum in range(multiimagenum):

                    top3_val, top3_pos = torch.topk(outputs[outnum],1)

                    t4_temp2 = time.time()
                    if top3_val < gp.Fine_thresh:
                        Species[outnum] = 0
                    else:
                        Species[outnum] = top3_pos
                    t4_temp3 = time.time()
                    print("iter{} fine data temp3 is ".format(str(iter_num)), (t4_temp3 - t4_temp2) * 1000)
                t4_2 = time.time()
                if gp.DebugPrint_flag:
                    print("iter{} fine data final2 is ".format(str(iter_num)), (t4_2 - t4_1) * 1000)
                for j in range(multiimagenum):
                    for m in range(len(imaIndexIfSuit)):
                        if imaIndexIfSuit[labelIndexForFusionResult]==1:  #clam this boxes has been deal with fine grained
                            labellist[labelIndexForFusionResult] = Species[j]
                            labelIndexForFusionResult+=1
                            break
                        else:
                            labelIndexForFusionResult+=1
                t4_3 = time.time()
                if gp.DebugPrint_flag:
                    print("iter{} fine data final3 is ".format(str(iter_num)), (t4_3 - t4_2) * 1000)
                #labellist[iter_num*multiimagenum:iter_num*multiimagenum+multiimagenum] = Species
                t4 = time.time()
                if gp.DebugPrint_flag:
                    print("iter{} fine data final is ".format(str(iter_num)), (t4 - t3) * 1000)

            t4 = time.time()
            if gp.DebugPrint_flag:
                print("start {} single_imagedealnums".format(str(single_imagedealnums)))
                print("single_imagedealnums is ",single_imagedealnums)
            Img = None
            if single_imagedealnums==0:
                pass
            elif single_imagedealnums ==1:

                Img = img_deal[(iter_nums) * multiimagenum]

            else:
                #Img = torch.cat((img_deal[0*iter_num],img_deal[1*iter_num],img_deal[2*iter_num],img_deal[3*iter_num],img_deal[4*iter_num],img_deal[5*iter_num],img_deal[6*iter_num],img_deal[7*iter_num]),0)
                tF1 = time.time()
                singleimage = []
                for tempm in range(single_imagedealnums):
                    singleimage.append(img_deal[(iter_nums) * multiimagenum+tempm])
                Img = torch.cat(singleimage,0)
                # Img = torch.cat((img_deal[(iter_nums) * multiimagenum], img_deal[(iter_nums) * multiimagenum+1]), 0)
                # for single_imagedealnum in range(single_imagedealnums-2):
                #     Img = torch.cat((Img,img_deal[(iter_nums) * multiimagenum+1+single_imagedealnum+1]),0)
                tF2 = time.time()
                if gp.DebugPrint_flag:
                    print("single_imagedealnumsv fine data deal cat  is ", (tF2 - tF1) * 1000)
            t5 = time.time()
            if gp.DebugPrint_flag:
                print("single_imagedealnumsv fine data deal is ", (t5 - t4) * 1000)
            if not (Img==None):
                tsfCU1 = time.time()
                inputs = Variable(Img.cuda())
                tsfCU2 = time.time()
                if gp.DebugPrint_flag:
                    print("single_imagedealnumsv tocuda is ", (tsfCU2 - tsfCU1) * 1000)
                outputs = self.net(inputs)
                outputs = self.softmax(outputs)
                t6 = time.time()
                if gp.DebugPrint_flag:
                    print("single_imagedealnums fine inference is ", (t6 - t5) * 1000)
                Species = gp.Species
                #Species = torch.zeros(single_imagedealnums).cuda()
                for outnum in range(single_imagedealnums):

                    top3_val, top3_pos = torch.topk(outputs[outnum], 1)
                    #print("top3_val==========", top3_val)
                    t4_temp2 = time.time()
                    if top3_val < gp.Fine_thresh:
                        Species[outnum] = 0
                    else:
                        Species[outnum] = top3_pos
                    t4_temp3 = time.time()
                    print("single_imagedealnums fine data temp3 is ", (t4_temp3 - t4_temp2) * 1000)

                for j in range(single_imagedealnums):
                    for m in range(len(imaIndexIfSuit)):
                        if imaIndexIfSuit[labelIndexForFusionResult] == 1:  # clam this boxes has been deal with fine grained
                            labellist[labelIndexForFusionResult] = Species[j]
                            labelIndexForFusionResult += 1
                            break
                        else:
                            labelIndexForFusionResult += 1

                #labellist[len(labellist)-single_imagedealnums:] = Species
                t7 = time.time()
                if gp.DebugPrint_flag:
                    print(" fine data final is ", (t7 - t6) * 1000)
        return imagelist, labellist, scorelist

class Fine_grained:
    def __init__(self,device,root):
        self.device = device
        self.root = root
        self.DclPreWeightPath = '../data_MPKFNet/test/3.pth'
        # parser = argparse.ArgumentParser(description='dcl parameters')
        # parser.add_argument('--data', dest='dataset', default='HCData', type=str)
        # parser.add_argument('--backbone', dest='backbone', default='resnet50', type=str)
        # parser.add_argument('--b', dest='batch_size', default=1, type=int)
        # parser.add_argument('--nw', dest='num_workers', default=8, type=int)
        # parser.add_argument('--ver', dest='version', default='val', type=str)
        # parser.add_argument('--size', dest='resize_resolution', default=512, type=int)
        # parser.add_argument('--crop', dest='crop_resolution', default=448, type=int)
        # parser.add_argument('--ss', dest='save_suffix', default=None, type=str)
        # parser.add_argument('--acc_report', dest='acc_report', action='store_true')
        # parser.add_argument('--swap_num', default=[7, 7], nargs=2, metavar=('swap1', 'swap2'), type=int,
        #                     help='specify a range')
        self.args = args_temp()
        self.Config = LoadConfig(self.args, 's')
        self.transformers = load_data_transformers(self.args.resize_resolution, self.args.crop_resolution, self.args.swap_num)
        self.net = MainModel(self.Config).to(device)
        model_dict = self.net.state_dict()
        pretrained_dict = torch.load(self.DclPreWeightPath)
        model_dict.update(pretrained_dict)
        self.net.load_state_dict(model_dict)
        if gp.DebugPrint_flag:
            print("load model weight is ok")
        self.net.train(False)
    def inference(self,img,imagelist,labellist,scorelist):
        with torch.no_grad():
            for i,box in enumerate(imagelist):
                Img = img.copy()[int(box[1]):int(box[3]), int(box[0]):int(box[2]), :]
                if (int(box[3])-int(box[1])>100) and (int(box[2])-int(box[0])>100):
                    t1 = time.time()
                    Img = Image.fromarray(cv2.cvtColor(Img, cv2.COLOR_BGR2RGB))
                    Img = self.transformers['test_totensor'](Img)
                    Img = Img.unsqueeze(0)
                    inputs = Variable(Img.cuda())
                    t2 = time.time()
                    if gp.DebugPrint_flag:
                        print("fine data deal is ", (t2 - t1) * 1000)
                    outputs = self.net(inputs)
                    t3 = time.time()
                    if gp.DebugPrint_flag:
                        print("fine inference is ", (t3 - t2) * 1000)
                    top3_val, top3_pos = torch.topk(outputs[0], 3)  # topk·µ»ØµÄÊÇÒ»¸öÔª×飬µÚÒ»¸öÔªËØÊÇÿһÐÐ×î´óµÄÈý¸öÖµ£¨Ä¬ÈÏÊÇ°´ÐÐÈ¡×î´óÖµ£©£¬µÚ¶þ¸öÔªËØÊÇÿһÐÐ×î´óµÄÖµµÄÅÅÐò
                    Species = top3_pos.cpu().numpy()[0][0]
                    ConfidenceLevel = top3_val.cpu().numpy()[0][0]
                    ConfidenceLevel = ConfidenceLevel/26.0
                    if ConfidenceLevel >1:
                        ConfidenceLevel = 1.0* random.uniform(0, 1)
                    if gp.DebugPrint_flag:
                        print("hhhhhhhhhhhhhhhhhhhhhhh is ",ConfidenceLevel)
                    labellist[i] = Species
                    scorelist[i] = ConfidenceLevel
                    t4 = time.time()
                    if gp.DebugPrint_flag:
                        print("fine data final is ", (t4 - t3) * 1000)

            return imagelist, labellist, scorelist
class Check_start (threading.Thread):
    def __init__(self, threadID, name, img,DetectionClass):
        threading.Thread.__init__(self)
        self.threadID = threadID
        self.name = name
        self.img = img
        self.DetectionClass = DetectionClass
    def run(self):
        if gp.DebugPrint_flag:
            print(self.name + " is start")
        gp.is_ex_bird ,labels, boxes ,scores= self.DetectionClass.check_object(self.img)

        if gp.is_ex_bird :
            if gp.DebugPrint_flag:
                print(gp.is_ex_bird)
            gp.image_time = gp.Camera_device +time.strftime('%Y%m%d%H%M%S',time.localtime(time.time()))
        if gp.DebugPrint_flag:
            print(self.name + " is finish")
class save_sigle_image (threading.Thread):
    def __init__(self, img):
        threading.Thread.__init__(self)
        self.img = img
    def run(self):
        cv2.imwrite(
            "../data_MPKFNet/RTSP2/" + "{}.jpg".format(time.strftime('%Y%m%d%H%M%S', time.localtime(time.time()))),
            self.img)
class Check_start_fordataselect (threading.Thread):
    def __init__(self, threadID, name, img,DetectionClass,fine_grained):
        threading.Thread.__init__(self)
        self.threadID = threadID
        self.name = name
        self.img = img
        self.DetectionClass = DetectionClass
        self.fine_grained = fine_grained
    def run(self):
        if gp.DebugPrint_flag:
            print(self.name + " is start")
        is_ex_bird ,labels, boxes ,scores= self.DetectionClass.check_object(self.img)
        image_or = self.img.copy()
        if is_ex_bird :
            for i in range(len(boxes)):

                if boxes[i][0] < 0:
                    boxes[i][0] = 0
                if boxes[i][1] < 0:
                    boxes[i][1] = 0
                if boxes[i][2] < 0:
                    boxes[i][2] = 0
                if boxes[i][3] < 0:
                    boxes[i][3] = 0
            for i in range(len(labels)):
                labels[i] = 99999
            boxes_fine, labels_fine, scores_fine = self.fine_grained.inference(self.img, boxes, labels, scores)
            for i in range(len(scores)):
                xyxy = boxes_fine[i]
                conf = scores_fine[i]
                cls = labels_fine[i]
                print(int(cls))
                label = str(gp.BHG_chineseName[int(cls)])
                print(label)
                #label = f'{int(cls)} {conf:.2f}'
                palette = (2 ** 11 - 1, 2 ** 15 - 1, 2 ** 20 - 1)
                colors = [int((p * (cls ** 2 - cls + 1)) % 255) for p in palette]
                self.img = plot_one_box(xyxy, self.img, label=label, color=colors, line_thickness=2)
        #cv2.imshow("AI-ECO",self.img)
        #cv2.waitKey(0)
            cv2.imwrite("../data_MPKFNet/RTSP/" + "{}.jpg".format(time.strftime('%Y%m%d%H%M%S', time.localtime(time.time()))),
                        image_or)
            cv2.imwrite(
                "../data_MPKFNet/RTSP/result/" + "{}.jpg".format(time.strftime('%Y%m%d%H%M%S', time.localtime(time.time()))),
                self.img)
        if gp.DebugPrint_flag:
            print(self.name + " is finish")
class shishi():
    def __init__(self,  name, DetectionClass,fine_grained):

        self.threadID = 0
        self.name = name

        self.DetectionClass = DetectionClass
        self.fine_grained = fine_grained
    def go_go(self,img,num_image,num_detection):
        spiec_number = [0 for i in range(len(gp.BHG_chineseName))]
        img_show = img.copy()
        boxes_fine = []
        labels_fine = []
        if (isinstance(img_show, np.ndarray)):  # ?D??¨º?¡¤?OpenCV¨ª???¨¤¨¤D¨ª
            img_show = Image.fromarray(cv2.cvtColor(img_show, cv2.COLOR_BGR2RGB))
        draw = ImageDraw.Draw(img_show)
        height_or, width_or = img.shape[:2]
        if gp.DebugPrint_flag:
            print("image size is ",height_or,width_or)
        if(num_detection%2==0):
            if gp.DebugPrint_flag:
                print(self.name + " is start")
            ts1 = time.time()
            is_ex_bird ,labels, boxes ,scores= self.DetectionClass.check_object(img)
            ts2 = time.time()
            if gp.DebugPrint_flag:
                print("shishi time detect1 deal is ", (ts2 - ts1) * 1000)
            if is_ex_bird :
                for i in range(len(boxes)):

                    if boxes[i][0] < 0:
                        boxes[i][0] = 0
                    if boxes[i][1] < 0:
                        boxes[i][1] = 0
                    if boxes[i][2] < 0:
                        boxes[i][2] = 0
                    if boxes[i][3] < 0:
                        boxes[i][3] = 0
                for i in range(len(labels)):
                    labels[i] = 0#gp.birdIndex
                ts3 = time.time()
                if gp.DebugPrint_flag:
                    print("shish time detect2 deal is ", (ts3 - ts2) * 1000)
                boxes_fine, labels_fine, scores_fine = self.fine_grained.inference(img, boxes, labels, scores,8)
                ts4 = time.time()
                if gp.DebugPrint_flag:
                    print("shish time detect3 deal is ", (ts4 - ts3) * 1000)
                tcpu1 = time.time()
                boxes_fine = boxes_fine.to('cpu')
                labels_fine = labels_fine.to('cpu')

                tcpu2 = time.time()
                if gp.DebugPrint_flag:
                    print("to cpu timeis ", (tcpu2 - tcpu1) * 1000)
            gp.boxes_fine = boxes_fine
            gp.labels_fine = labels_fine
        else:
            print("don't deal {}".format(str(num_detection)))
            boxes_fine = gp.boxes_fine
            labels_fine =gp.labels_fine



        for i in range(len(boxes_fine)):
            tprint1 = time.time()
            xyxy = boxes_fine[i]

            cls = labels_fine[i]

            tprint2 = time.time()
            if gp.DebugPrint_flag:
                print("tprint1 is ", (tprint2 - tprint1) * 1000)
            tcls1 = time.time()
            b =   (((int(xyxy[3]) - int(xyxy[1])) * (int(xyxy[2]) - int(xyxy[0]))) < 15000)
            #a =(xyxy.item()[3] - xyxy.item()[1]) * (xyxy.item()[2] - xyxy.item()[0])
            tcls2 = time.time()
            if gp.DebugPrint_flag:
                print("tcls1 is ", (tcls2 - tcls1) * 1000)
            if cls == 0 or (((int(xyxy[3]) - int(xyxy[1])) * (int(xyxy[2]) - int(xyxy[0]))) < gp.fine_thresh2):
            #if cls == gp.birdIndex or (torch.sub(torch.mul(torch.sub(xyxy[3], xyxy[1]), torch.sub(xyxy[2], xyxy[0])),gp.fine_thresh2) < 0):
                label = None
            else:
                tcls3 = time.time()
                label = gp.BHG_chineseName[int(cls)]
                spiec_number[int(cls)] += 1
                tcls4 = time.time()
                if gp.DebugPrint_flag:
                    print("tcls2 is ", (tcls4 - tcls3) * 1000)
            tprint3 = time.time()
            if gp.DebugPrint_flag:
                print("tprint2 is ", (tprint3 - tprint2) * 1000)
            #label = f'{int(cls)} {conf:.2f}'
            palette = (2 ** 11 - 1, 2 ** 15 - 1, 2 ** 20 - 1)
            colors = gp.color_box#tuple(int((p * (4 ** 2 - 4 + 1)) % 220) for p in palette)
            tprint4 = time.time()
            if gp.DebugPrint_flag:
                print("tprint3 is ", (tprint4 - tprint3) * 1000)
            draw = plot_one_box(xyxy, draw, height_or,width_or,label=label, color=colors, line_thickness=2)
        tpChin1 = time.time()
        if num_image > gp.showThresh :
            gp.num_count_old = len(boxes_fine)
            gp.spiec_number_old = spiec_number
            num_image = 0
        tpChin2 = time.time()
        if gp.DebugPrint_flag:
            print("tpChin1 is ", (tpChin2 - tpChin1) * 1000)
        #if width_or == 2560:
        draw = cv2AddChineseText(draw, "ʶ±ð×ÜÊý: {}Ö»".format(str(gp.num_count_old)), (50*gp.radioShowX, 1100*gp.radioShowY),
                                textColor=gp.color_show,
                                textSize=50)
        tpChin3 = time.time()
        if gp.DebugPrint_flag:
            print("tpChin2 is ", (tpChin3 - tpChin2) * 1000)
        draw = cv2AddChineseText(draw, "·ÖÀàʶ±ð:", (50*gp.radioShowX, 1150*gp.radioShowY), textColor=gp.color_show,
                                textSize=50)
        tpChin4 = time.time()
        if gp.DebugPrint_flag:
            print("tpChin3 is ", (tpChin4 - tpChin3) * 1000)
        for i in range(1,len(spiec_number)):
            if i <5:
                draw = cv2AddChineseText(draw, "{}:{}Ö»".format(str(gp.BHG_chineseName[i]), str(gp.spiec_number_old[i])),
                                        (300*gp.radioShowX + (i-1) * 280*gp.radioShowX, 1150*gp.radioShowY), textColor=gp.color_show,
                                        textSize=50)
            else:
                draw = cv2AddChineseText(draw, "{}:{}Ö»".format(str(gp.BHG_chineseName[i]), str(gp.spiec_number_old[i])),
                                            (50*gp.radioShowX + (i-5) * 280*gp.radioShowX, 1200*gp.radioShowY), textColor=gp.color_show,
                                            textSize=50)
        img_show = cv2.cvtColor(np.asarray(img_show), cv2.COLOR_RGB2BGR)
        gp.img_show = img_show

        tpChin2 = time.time()
        if gp.DebugPrint_flag:
            print("tpChin is ", (tpChin2 - tpChin1) * 1000)



    #cv2.waitKey(0)
        # cv2.imwrite("../data_MPKFNet/RTSP/" + "{}.jpg".format(time.strftime('%Y%m%d%H%M%S', time.localtime(time.time()))),
        #             image_or)
        # cv2.imwrite(
        #     "../data_MPKFNet/RTSP/result/" + "{}.jpg".format(time.strftime('%Y%m%d%H%M%S', time.localtime(time.time()))),
        #     self.img)
        if gp.DebugPrint_flag:
            print(self.name + " is finish")
        num_image +=1
        print(num_image)

        return img_show,num_image
class RealDeal():
    def __init__(self,  name, DetectionClass,fine_grained):

        self.threadID = 0
        self.name = name

        self.DetectionClass = DetectionClass
        self.fine_grained = fine_grained
    def realtimerestart(self,img,num_image,num_detection,SortRestart):
        spiec_number = [0 for i in range(len(gp.BHG_chineseName))]
        img_show = img.copy()
        boxes_fine = []
        labels_fine = []
        if (isinstance(img_show, np.ndarray)):  # ?D??¨º?¡¤?OpenCV¨ª???¨¤¨¤D¨ª
            img_show = Image.fromarray(cv2.cvtColor(img_show, cv2.COLOR_BGR2RGB))
        draw = ImageDraw.Draw(img_show)
        height_or, width_or = img.shape[:2]
        if gp.DebugPrint_flag:
            print("image size is ",height_or,width_or)
        if gp.DebugPrint_flag:
            print(self.name + " is start")
        ts1 = time.time()
        is_ex_bird ,labels, boxes ,scores= self.DetectionClass.check_object(img)
        ts2 = time.time()
        if gp.DebugPrint_flag:
            print("shishi time detect1 deal is ", (ts2 - ts1) * 1000)

        if is_ex_bird :
            for i in range(len(boxes)):

                if boxes[i][0] < 0:
                    boxes[i][0] = 0
                if boxes[i][1] < 0:
                    boxes[i][1] = 0
                if boxes[i][2] < 0:
                    boxes[i][2] = 0
                if boxes[i][3] < 0:
                    boxes[i][3] = 0
            for i in range(len(labels)):
                labels[i] = 0#gp.birdIndex
            ts3 = time.time()
            if gp.DebugPrint_flag:
                print("shish time detect2 deal is ", (ts3 - ts2) * 1000)


            boxes_fine, labels_fine, scores_fine = self.fine_grained.inference(img, boxes, labels, scores,8)
            ts4 = time.time()
            if gp.DebugPrint_flag:
                print("shish time detect3 deal is ", (ts4 - ts3) * 1000)
            tcpu1 = time.time()
            boxes_fine = boxes_fine.to('cpu')
            labels_fine = labels_fine.to('cpu')

            tcpu2 = time.time()
            if gp.DebugPrint_flag:
                print("to cpu timeis ", (tcpu2 - tcpu1) * 1000)
        gp.boxes_fine = boxes_fine
        gp.labels_fine = labels_fine
        #input = [x1,y1,x2,y2,kong,label]
        temp_sort = [i + 100 for i in range(len(labels_fine))]
        DetFine = np.column_stack((boxes_fine,temp_sort,labels_fine))
        boxes_fine,indexmaxtemp,newObjectNum = SortRestart.update(DetFine)#boxes_fine ÊÇ n*6µÄÊý×飬nÊÇÄ¿±êµÄÊýÁ¿£¬¶øÇÒÐòºÅÊǵ¹ÐòµÄ£¬×îСµÄÐòºÅÊÇ1
        if indexmaxtemp > gp.IndexMax:
            gp.IndexMax = indexmaxtemp
        #boxes_fine = boxes_fine[0]
        #output = [x1,y1,x2,y2,label,index]
        gp.Finerestartboxes.append(boxes_fine)

        # for i in range(len(boxes_fine)):
        #     xyxy = boxes_fine[i][:4]
        #     cls = boxes_fine[i][4]
        #     index = boxes_fine[i][5]
        #     label = None
        #     palette = (2 ** 11 - 1, 2 ** 15 - 1, 2 ** 20 - 1)
        #     colors = gp.color_box #tuple(int((p * (4 ** 2 - 4 + 1)) % 220) for p in palette)
        #     draw = plot_one_box(xyxy, draw, height_or,width_or,label=label, Index= index, color=colors, line_thickness=int(3*gp.radioShowtext))
        # if num_image > gp.showThresh :
        #     gp.num_count_old = len(boxes_fine)
        #     gp.spiec_number_old = spiec_number
        #     num_image = 0
        # #if width_or == 2560:识别总数分类识别
        # draw = cv2AddChineseText(draw, "识别总数: {}只".format(str(gp.num_count_old)),
        #                          (50 * gp.radioShowX, 1100 * gp.radioShowY),
        #                          textColor=gp.color_show,
        #                          textSize=int(50* gp.radioShowtext))
        # tpChin3 = time.time()
        #
        # draw = cv2AddChineseText(draw, "分类识别:", (50 * gp.radioShowX, 1150 * gp.radioShowY), textColor=gp.color_show,
        #                          textSize=int(50* gp.radioShowtext))
        # tpChin4 = time.time()
        # if gp.DebugPrint_flag:
        #     print("tpChin3 is ", (tpChin4 - tpChin3) * 1000)
        # for i in range(1, len(spiec_number)):
        #     if i < 5:
        #         draw = cv2AddChineseText(draw, "{}:{}只»".format(str(gp.BHG_chineseName[i]), str(gp.spiec_number_old[i])),
        #                                  (300 * gp.radioShowX + (i - 1) * 280 * gp.radioShowX, 1150 * gp.radioShowY),
        #                                  textColor=gp.color_show,
        #                                  textSize=int(50* gp.radioShowtext))
        #     else:
        #         draw = cv2AddChineseText(draw, "{}:{}只".format(str(gp.BHG_chineseName[i]), str(gp.spiec_number_old[i])),
        #                                  (50 * gp.radioShowX + (i - 5) * 280 * gp.radioShowX, 1200 * gp.radioShowY),
        #                                  textColor=gp.color_show,
        #                                  textSize=int(50* gp.radioShowtext))
        img_show = cv2.cvtColor(np.asarray(img_show), cv2.COLOR_RGB2BGR)
        gp.img_show = img_show

        tpChin2 = time.time()
        if gp.DebugPrint_flag:
            print(self.name + " is finish")
        num_image +=1


        return img_show,num_image
    def realtime(self,img,num_image,num_detection,SortRestart):
        spiec_number = [0 for i in range(len(gp.BHG_chineseName))]
        img_show = img.copy()
        boxes_fine = []
        labels_fine = []
        if (isinstance(img_show, np.ndarray)):  # ?D??¨º?¡¤?OpenCV¨ª???¨¤¨¤D¨ª
            img_show = Image.fromarray(cv2.cvtColor(img_show, cv2.COLOR_BGR2RGB))
        draw = ImageDraw.Draw(img_show)
        height_or, width_or = img.shape[:2]
        if gp.DebugPrint_flag:
            print("image size is ",height_or,width_or)
        if gp.DebugPrint_flag:
            print(self.name + " is start")
        ts1 = time.time()
        is_ex_bird ,labels, boxes ,scores= self.DetectionClass.check_object(img)
        ts2 = time.time()
        if gp.DebugPrint_flag:
            print("shishi time detect1 deal is ", (ts2 - ts1) * 1000)
        if is_ex_bird :
            print("Ä¿±ê¼ì²âµÄ¸öÊý ", len(boxes))
            for i in range(len(boxes)):

                if boxes[i][0] < 0:
                    boxes[i][0] = 0
                if boxes[i][1] < 0:
                    boxes[i][1] = 0
                if boxes[i][2] < 0:
                    boxes[i][2] = 0
                if boxes[i][3] < 0:
                    boxes[i][3] = 0
            for i in range(len(labels)):
                labels[i] = 0#gp.birdIndex
            ts3 = time.time()
            if gp.DebugPrint_flag:
                print("shish time detect2 deal is ", (ts3 - ts2) * 1000)
            ts4 = time.time()
            if gp.DebugPrint_flag:
                print("shish time detect3 deal is ", (ts4 - ts3) * 1000)
            tcpu1 = time.time()
            boxes_fine = boxes.to('cpu')
            labels_fine = labels.to('cpu')

            tcpu2 = time.time()
            if gp.DebugPrint_flag:
                print("to cpu timeis ", (tcpu2 - tcpu1) * 1000)

        #############################Ä¿±ê×·×Ù£¬½«×·×ÙºóµÄ½á¹û·ÖΪ»÷ÖкÍδ»÷ÖÐÁ½²¿·Ö£¬Ç°ÕßÖ±½Ó´ÓÈ«¾Ö±äÁ¿SortLabel_IndexÖлñÈ¡label£¬ºóÕßÐèÒªÖØнøÐÐϸÁ£¶Èʶ±ð£¬×îºó½«Á½Õß½øÐÐÆ´½Ó
        #Êä³öΪboxes_fine [x1,y1,x2,y2,label,Index]
            temp_sort = [i + 100 for i in range(len(labels_fine))]
            DetFine = np.column_stack((boxes_fine, temp_sort, labels_fine))

            boxes_fine, indexmaxtemp, newObjectNum = SortRestart.update(DetFine)
            IndexList = boxes_fine[:,-1].tolist()   #½«indexÁÐÈ¡³öÀ´£¬Åжϵ±Ç°Ö¡¼ì²âµ½Á˶àÉÙÄ¿±ê
            # newObjectNum = 0
            # if indexmaxtemp > gp.IndexMax:
            #     newObjectNum = indexmaxtemp - gp.IndexMax
            #     gp.IndexMax = indexmaxtemp
            # print(indexmaxtemp,newObjectNum)
            if (len(gp.SortLabel_Index) == 0):
                if (len(boxes_fine)!=0):
                    boxes_fine_unmatched = []  # ÓÃÀ´´æ·Åµ±Ç°Ö¡Ã»Óл÷ÖеÄÄ¿±ê  [xyxy, label,Index]
                    for c in IndexList:
                        boxes_fine_unmatched.append(boxes_fine[IndexList.index(c),:])  # δƥÅäµÄÄ¿±ê£¬ÐèÒª½øÒ»²½½øÐÐϸÁ£¶Èʶ±ð,ÓÃindexË÷Òýµ½ÊÇÄĸöÄ¿±ê£¬²¢½«¸ÃÄ¿±êµÄÊý¾Ý¶¼appendµ½unmatchÖÐ

                    # ¶Ô¼¯ÖÐÆ¥ÅäµÄÄ¿±ê£¬½«labelÌ滻ΪSortLabel_IndexÖеÄlabel£¬ÐèÒªÕë¶Ôindex½øÐÐË÷Òý
                    boxes_fine_unmatched = np.array(boxes_fine_unmatched)
                        # boxes_fine_matched ÓÃÓÚ×îºóÆ´½Ó    [xyxy,label,index]
                    #####½øÐÐϸÁ£¶Èʶ±ð
                    boxes = boxes_fine_unmatched[:, 0:4]
                    labels = boxes_fine_unmatched[:, 4]
                    index_fine_unF = boxes_fine_unmatched[:, 5]
                    boxes_fine_unF, labels_fine_unF, index_fine_unF = self.fine_grained.inference(img, boxes, labels,
                                                                                                  index_fine_unF,
                                                                                                  8)
                    print(labels_fine_unF)
                    # ¸ñʽ   [[xyxy]]   [label1,label2...]  [index1,index2...]
                    # ÏȶÔмì²âµÄÄ¿±ê½øÐмì²é£¬ÊÇ·ñÒѾ­´ïµ½5´ÎÒÔÉÏʶ±ð
                    ###¶ÔϸÁ£¶Èʶ±ð½á¹û½øÐкó´¦Àí
                    boxes_fine_unmatched[:, 4] = labels_fine_unF
                    boxes_fine_unmatched_temp = []
                    for i in range(len(labels_fine_unF)):
                        if (len(gp.SortLabel_Indexpre)==0 ):  # ˵Ã÷preÖÐûÓÐÄ¿±ê£¬Ôò¿ÉÒÔÖ±½Ó½«µ±Ç°µÄÄ¿±ê¼Ó½øÀ´
                            temp_a = np.array([labels_fine_unF[i], index_fine_unF[i], 1])
                            gp.SortLabel_Indexpre = np.expand_dims(np.append(gp.SortLabel_Indexpre, temp_a, axis=0), axis=0)
                        else:  # ˵Ã÷±äÁ¿ÖÐÒѾ­ÓÐÄ¿±ê£¬ÐèÒª¶Ôÿ¸öÄ¿±êµÄ´ÎÊý½øÐмì²â
                            temp_a = np.array([labels_fine_unF[i], index_fine_unF[i], 1])
                            temp_index = index_fine_unF[i]  # µ±Ç°Ä¿±êµÄindex
                            index_list = gp.SortLabel_Indexpre[:, 1].tolist()  # ÓÃÀ´Çó½âindex=nÔÚÊý×éÖеÄά¶ÈÐÅÏ¢£¬µÚ¼¸ÐÐ
                            if temp_index in gp.SortLabel_Indexpre[:, 1]:  # ˵Ã÷δ¼ì²âµÄÄ¿±êÔÚpreÈ«¾ÖÖУ¬ÐèÒª½øÒ»²½¼ì²âÊÇ·ñ¼ì²â´ÎÊý³¬¹ýãÐÖµ´ÎÊý
                                print("ϸÁ£¶Èʶ±ð´ÎÊý", gp.SortLabel_Indexpre[:, -1][index_list.index(temp_index)],
                                      gp.FineCountThresh)
                                if (gp.SortLabel_Indexpre[:, -1][index_list.index(temp_index)] >= gp.FineCountThresh):  # ÔÚpreÈ«¾Ö±äÁ¿ÖУ¬µ±Ç°indexµÄÄ¿±ê£¬³öÏֵĴ˴¦ÒѾ­³¬¹ýãÐÖµ£¬ÔòÖ±½Ó½«¸ÃindexºÍ¶ÔÓ¦µÄlabel£¬appendµ½label_indexÈ«¾Ö±äÁ¿ÖУ¬²¢¸üе½µ±Ç°¼ì²â½á¹ûÖУ¬²¢½«¸ÃindexÔÚpreÖÐɾ³ýµô
                                    index_dontthresh = []
                                    if (len(gp.SortLabel_Index.shape) == 1):  # мì²â³öÒ»¸öÄ¿±êµÄʱºò£¬ÐèÒª¶ÔÈ«¾Ö±äÁ¿SortLabel_Index½øÐг¤¶È¼ì²â£¬Èç¹ûÔ­ÏÈÀïÃæûÓУ¬Ôò²»¶Ôa´¦Àí£¬ÓеĻ°£¬ÐèÒª¶Ôa½øÐÐÌí¼Óά¶È´¦Àí
                                        a = np.array([labels_fine_unF[i], index_fine_unF[i]])
                                        #print("a ======", a)
                                        #print("gp.SortLabel_Index ===", gp.SortLabel_Index)
                                        # if a[1] in gp.SortLabel_Index[:,1]:   #˵Ã÷indexÒѾ­ÔÚÀïÃæÁË
                                        gp.SortLabel_Index = np.expand_dims( np.append(gp.SortLabel_Index, a, axis=0), axis=0)
                                        #print("gp.SortLabel_Indexed ===", gp.SortLabel_Index)
                                        gp.SortLabel_Indexpre = np.delete(gp.SortLabel_Indexpre,[index_list.index(temp_index)], axis=0)
                                        # gp.SortLabel_Indexpre[index_list.index(temp_index),1] = 0
                                    else:
                                        a = np.expand_dims(np.array([labels_fine_unF[i], index_fine_unF[i]]), axis=0)
                                        #print("a ======", a)
                                        #print("gp.SortLabel_Index ===", gp.SortLabel_Index)
                                        # if a[1] in gp.SortLabel_Index[:,1]:   #˵Ã÷indexÒѾ­ÔÚÀïÃæÁË
                                        gp.SortLabel_Index = np.concatenate((gp.SortLabel_Index, a), axis=0)
                                        #print("gp.SortLabel_Indexed ===", gp.SortLabel_Index)
                                        gp.SortLabel_Indexpre = np.delete(gp.SortLabel_Indexpre,
                                                                          [index_list.index(temp_index)], axis=0)
                                        #gp.SortLabel_Indexpre[index_list.index(temp_index), 1] = 0
                                else:#ÔÚpreÈ«¾Ö±äÁ¿ÖУ¬µ±Ç°indexµÄÄ¿±ê£¬³öÏֵĴ˴¦Î´³¬¹ýãÐÖµ
                                    gp.SortLabel_Indexpre[:, -1][index_list.index(temp_index)] += 1
                                    # boxes_fine_unmatched = np.delete(boxes_fine_unmatched, [boxes_fine_unmatched[:, -1].tolist().index(temp_index)],
                                    #                                  axis=0)  # ½«µ±Ç°Ä¿±êÐÅÏ¢´Óunmatched Êý×éÖÐɾ³ýµô
                                    boxes_fine_unmatched_temp = boxes_fine_unmatched

                                    boxes_fine_unmatched = np.delete(boxes_fine_unmatched, [ boxes_fine_unmatched[:, -1].tolist().index(temp_index)],axis=0)
                                    boxes_fine_unmatched_temp[ boxes_fine_unmatched_temp[:, -1].tolist().index(temp_index),4] = 0
                            else:  # ²»ÔÚpreÖУ¬Ö±½Ó½«µ±Ç°µÄÄ¿±êappendµ½preÈ«¾Ö±äÁ¿ÖÐ
                                gp.SortLabel_Indexpre = np.concatenate((gp.SortLabel_Indexpre, np.expand_dims(temp_a, axis=0)), axis=0)
                    # boxes_fine_unmatched = np.column_stack((boxes_fine_unF,labels_fine_unF))  #½«ÐÂʶ±ðµÄϸÁ£¶È½á¹ûºÍ±ß¿òÈںϵ½Ò»Æ𣬺ÍÔ­ÏȵĸñʽһÖÂ
                    # boxes_fine_unmatched = np.column_stack((boxes_fine_unmatched,index_fine_unF))
                    boxes_fine = boxes_fine_unmatched
                    gp.SortLabel_Index = boxes_fine[:, 4:]  # ¸üÐÂÈ«¾Ö±äÁ¿ÖеÄÐÅÏ¢
                    #######ÐÞ¸Äbug£¬ÔÚ¾ö²ß¹ý³ÌÖУ¬Òª°Ñ±ß¿òÏÔʾ³öÀ´
                    if len(boxes_fine_unmatched_temp)!= 0 :
                        boxes_fine = boxes_fine_unmatched_temp
            else:
                if [True for c in IndexList if c not in gp.SortLabel_Index[:,-1] ]:#˵Ã÷ÓÐмì²â³öÀ´µÄ£¬ÐèÒª½øÐÐϸÁ£¶Èʶ±ð£¬²¢½«Ð´¦ÀíµÄ½á¹ûºÍindexÆ¥ÅäÒ»Æð¸üе½SortLabel_IndexÈ«¾Ö±äÁ¿ÀïÃæ
                    boxes_fine_unmatched = []    #ÓÃÀ´´æ·Åµ±Ç°Ö¡Ã»Óл÷ÖеÄÄ¿±ê  [xyxy, label,Index]
                    boxes_fine_matched = []      #ÓÃÀ´´æ·Åµ±Ç°Ö¡»÷ÖеÄÄ¿±ê     [xyxy,label,index]
                    for c in IndexList:
                        if c not in gp.SortLabel_Index[:,-1]:
                            boxes_fine_unmatched.append(boxes_fine[IndexList.index(c),:])     #δƥÅäµÄÄ¿±ê£¬ÐèÒª½øÒ»²½½øÐÐϸÁ£¶Èʶ±ð,ÓÃindexË÷Òýµ½ÊÇÄĸöÄ¿±ê£¬²¢½«¸ÃÄ¿±êµÄÊý¾Ý¶¼appendµ½unmatchÖÐ
                        else:
                            boxes_fine_matched.append(boxes_fine[IndexList.index(c),:])        #»÷ÖÐÆ¥ÅäµÄÄ¿±ê
                #¶Ô¼¯ÖÐÆ¥ÅäµÄÄ¿±ê£¬½«labelÌ滻ΪSortLabel_IndexÖеÄlabel£¬ÐèÒªÕë¶Ôindex½øÐÐË÷Òý
                    boxes_fine_unmatched = np.array(boxes_fine_unmatched)
                    boxes_fine_matched = np.array(boxes_fine_matched)
                    for match_i in range(boxes_fine_matched.shape[0]):
                        SortLabel_Indextemp = gp.SortLabel_Index[:, 1].tolist()
                        if gp.DebugPrint_flag:
                            print(SortLabel_Indextemp)
                        boxes_fine_matched[match_i][4] = gp.SortLabel_Index[SortLabel_Indextemp.index(boxes_fine_matched[match_i][5]),0]
                        #boxes_fine_matched ÓÃÓÚ×îºóÆ´½Ó    [xyxy,label,index]
                    #####½øÐÐϸÁ£¶Èʶ±ð
                    boxes = boxes_fine_unmatched[:,0:4]
                    labels = boxes_fine_unmatched[:,4]
                    index_fine_unF = boxes_fine_unmatched[:,5]
                    boxes_fine_unF, labels_fine_unF, index_fine_unF = self.fine_grained.inference(img, boxes, labels, index_fine_unF, 8) #¸ñʽ   [[xyxy]]   [label1,label2...]  [index1,index2...]
                    #ÏȶÔмì²âµÄÄ¿±ê½øÐмì²é£¬ÊÇ·ñÒѾ­´ïµ½5´ÎÒÔÉÏʶ±ð
                    ###¶ÔϸÁ£¶Èʶ±ð½á¹û½øÐкó´¦Àí
                    boxes_fine_unmatched[:,4] = labels_fine_unF
                    boxes_fine_unmatched_temp = []
                    for i in range(len(labels_fine_unF)):
                        if (len(gp.SortLabel_Indexpre) == 0): #˵Ã÷preÖÐûÓÐÄ¿±ê£¬Ôò¿ÉÒÔÖ±½Ó½«µ±Ç°µÄÄ¿±ê¼Ó½øÀ´
                            temp_a = np.array([labels_fine_unF[i], index_fine_unF[i],1])
                            gp.SortLabel_Indexpre = np.expand_dims(np.append(gp.SortLabel_Indexpre, temp_a, axis=0), axis=0)
                        else:   #˵Ã÷±äÁ¿ÖÐÒѾ­ÓÐÄ¿±ê£¬ÐèÒª¶Ôÿ¸öÄ¿±êµÄ´ÎÊý½øÐмì²â
                            temp_a = np.array([labels_fine_unF[i], index_fine_unF[i], 1])
                            temp_index = index_fine_unF[i]  #µ±Ç°Ä¿±êµÄindex
                            index_list = gp.SortLabel_Indexpre[:,1].tolist()             #ÓÃÀ´Çó½âindex=nÔÚÊý×éÖеÄά¶ÈÐÅÏ¢£¬µÚ¼¸ÐÐ
                            if temp_index in gp.SortLabel_Indexpre[:,1]: #˵Ã÷δ¼ì²âµÄÄ¿±êÔÚpreÈ«¾ÖÖУ¬ÐèÒª½øÒ»²½¼ì²âÊÇ·ñ¼ì²â´ÎÊý³¬¹ýãÐÖµ´ÎÊý
                                print("ϸÁ£¶Èʶ±ð´ÎÊý", gp.SortLabel_Indexpre[:, -1][index_list.index(temp_index)],
                                      gp.FineCountThresh)
                                if (gp.SortLabel_Indexpre[:,-1][index_list.index(temp_index)] >=gp.FineCountThresh ):  #ÔÚpreÈ«¾Ö±äÁ¿ÖУ¬µ±Ç°indexµÄÄ¿±ê£¬³öÏֵĴ˴¦ÒѾ­³¬¹ýãÐÖµ£¬ÔòÖ±½Ó½«¸ÃindexºÍ¶ÔÓ¦µÄlabel£¬appendµ½label_indexÈ«¾Ö±äÁ¿ÖУ¬²¢¸üе½µ±Ç°¼ì²â½á¹ûÖУ¬²¢½«¸ÃindexÔÚpreÖÐɾ³ýµô
                                    if (len(gp.SortLabel_Index) == 0):  # мì²â³öÒ»¸öÄ¿±êµÄʱºò£¬ÐèÒª¶ÔÈ«¾Ö±äÁ¿SortLabel_Index½øÐг¤¶È¼ì²â£¬Èç¹ûÔ­ÏÈÀïÃæûÓУ¬Ôò²»¶Ôa´¦Àí£¬ÓеĻ°£¬ÐèÒª¶Ôa½øÐÐÌí¼Óά¶È´¦Àí
                                        a = np.array([labels_fine_unF[i], index_fine_unF[i]])
                                        print("a ======", a)
                                        print("gp.SortLabel_Index ===", gp.SortLabel_Index)
                                        # if a[1] in gp.SortLabel_Index[:,1]:   #˵Ã÷indexÒѾ­ÔÚÀïÃæÁË
                                        gp.SortLabel_Index = np.expand_dims(np.concatenate((gp.SortLabel_Index, a), axis=0),axis=0)
                                        print("gp.SortLabel_Indexed ===", gp.SortLabel_Index)
                                        gp.SortLabel_Indexpre = np.delete(gp.SortLabel_Indexpre,[index_list.index(temp_index)],axis=0)
                                    else:
                                        a = np.expand_dims(np.array([labels_fine_unF[i], index_fine_unF[i]]), axis=0)
                                        print("a ======", a)
                                        print("gp.SortLabel_Index ===", gp.SortLabel_Index)
                                        # if a[1] in gp.SortLabel_Index[:,1]:   #˵Ã÷indexÒѾ­ÔÚÀïÃæÁË
                                        gp.SortLabel_Index = np.concatenate((gp.SortLabel_Index, a), axis=0)
                                        print("gp.SortLabel_Indexed ===", gp.SortLabel_Index)
                                        gp.SortLabel_Indexpre = np.delete(gp.SortLabel_Indexpre,[index_list.index(temp_index)], axis=0)

                                else:
                                    boxes_fine_unmatched_temp = boxes_fine_unmatched
                                    gp.SortLabel_Indexpre[:, -1][index_list.index(temp_index)] += 1
                                    boxes_fine_unmatched = np.delete(boxes_fine_unmatched, [boxes_fine_unmatched[:,-1].tolist().index(temp_index)],axis=0)  #½«µ±Ç°Ä¿±êÐÅÏ¢´Óunmatched Êý×éÖÐɾ³ýµô
                                    boxes_fine_unmatched_temp[boxes_fine_unmatched_temp[:, -1].tolist().index(temp_index), 4] = 0
                            else: #²»ÔÚpreÖУ¬Ö±½Ó½«µ±Ç°µÄÄ¿±êappendµ½preÈ«¾Ö±äÁ¿ÖÐ
                                gp.SortLabel_Indexpre = np.concatenate((gp.SortLabel_Indexpre, np.expand_dims(temp_a,axis=0)), axis=0)
                                boxes_fine_unmatched = np.delete(boxes_fine_unmatched, [boxes_fine_unmatched[:, -1].tolist().index(temp_index)],axis=0)  # ½«µ±Ç°Ä¿±êÐÅÏ¢´Óunmatched Êý×éÖÐɾ³ýµô

                    # boxes_fine_unmatched = np.column_stack((boxes_fine_unF,labels_fine_unF))  #½«ÐÂʶ±ðµÄϸÁ£¶È½á¹ûºÍ±ß¿òÈںϵ½Ò»Æ𣬺ÍÔ­ÏȵĸñʽһÖÂ
                    # boxes_fine_unmatched = np.column_stack((boxes_fine_unmatched,index_fine_unF))
                    if (len(boxes_fine_matched) == 0):
                        boxes_fine = boxes_fine_unmatched
                    elif (len(boxes_fine_unmatched) == 0):
                        boxes_fine = boxes_fine_matched
                    else:
                        boxes_fine = np.concatenate((boxes_fine_matched,boxes_fine_unmatched))
                    gp.SortLabel_Index = boxes_fine[:,4:]  #¸üÐÂÈ«¾Ö±äÁ¿ÖеÄÐÅÏ¢
                    #######ÐÞ¸Äbug£¬ÔÚ¾ö²ß¹ý³ÌÖУ¬Òª°Ñ±ß¿òÏÔʾ³öÀ´
                    if len(boxes_fine_unmatched_temp) != 0:
                        if (len(boxes_fine_matched) == 0):
                            boxes_fine = boxes_fine_unmatched_temp
                        else:
                            boxes_fine = np.concatenate((boxes_fine_matched, boxes_fine_unmatched_temp))
                else:

                    for match_i in range(boxes_fine.shape[0]):
                        if gp.DebugPrint_flag:
                            print("gp.SortLabel_Index[:,1].tolist() = ", gp.SortLabel_Index[:, 1].tolist())
                            print("boxes_fine[match_i][5] = ", boxes_fine[match_i][5])
                        SortLabel_Indextemp = gp.SortLabel_Index[:,1].tolist()
                        boxes_fine[match_i][4] = gp.SortLabel_Index[SortLabel_Indextemp.index(boxes_fine[match_i][5]),0]
            #################################
        classs_index = [1 for i in range(len(gp.BHG_chineseName))]  # ¿ª±ÙÒ»¸ölist£¬³¤¶ÈºÍÄ¿±êÊýÁ¿Ò»Ö£¬Ã¿¸öÀïÃæ±£´æµÄÊÇclassµÄË÷Òý
        for i in range(len(boxes_fine)):

            tprint1 = time.time()
            xyxy = boxes_fine[i][:4]
            cls = boxes_fine[i][4]
            index = boxes_fine[i][5]
            classs_index_show = classs_index[int(cls)]
            classs_index[int(cls)] += 1
            tprint2 = time.time()
            if gp.DebugPrint_flag:
                print("tprint1 is ", (tprint2 - tprint1) * 1000)
            tcls1 = time.time()
            b =   (((int(xyxy[3]) - int(xyxy[1])) * (int(xyxy[2]) - int(xyxy[0]))) < 15000)
            #a =(xyxy.item()[3] - xyxy.item()[1]) * (xyxy.item()[2] - xyxy.item()[0])
            tcls2 = time.time()
            if gp.DebugPrint_flag:
                print("tcls1 is ", (tcls2 - tcls1) * 1000)
            if cls == 0 or (((int(xyxy[3]) - int(xyxy[1])) * (int(xyxy[2]) - int(xyxy[0]))) < gp.fine_thresh2):
            #if cls == gp.birdIndex or (torch.sub(torch.mul(torch.sub(xyxy[3], xyxy[1]), torch.sub(xyxy[2], xyxy[0])),gp.fine_thresh2) < 0):
                label = gp.NoneLabel
            else:
                tcls3 = time.time()
                label = gp.BHG_chineseName[int(cls)]
                #label = "{}       No.{}".format(label,str(classs_index_show).zfill(3))
                spiec_number[int(cls)] += 1
                tcls4 = time.time()
                if gp.DebugPrint_flag:
                    print("tcls2 is ", (tcls4 - tcls3) * 1000)
            tprint3 = time.time()
            if gp.DebugPrint_flag:
                print("tprint2 is ", (tprint3 - tprint2) * 1000)
            #label = f'{int(cls)} {conf:.2f}'
            palette = (2 ** 11 - 1, 2 ** 15 - 1, 2 ** 20 - 1)
            colors = gp.color_box #tuple(int((p * (4 ** 2 - 4 + 1)) % 220) for p in palette)
            tprint4 = time.time()
            if gp.DebugPrint_flag:
                print("tprint3 is ", (tprint4 - tprint3) * 1000)
            draw = plot_one_box(xyxy, draw, height_or,width_or,label=label, Index=index,color=colors, line_thickness=int(3*gp.radioShowtext))
        tpChin1 = time.time()
        if num_image > gp.showThresh :
            gp.num_count_old = len(boxes_fine)
            gp.spiec_number_old = spiec_number
            num_image = 0
        tpChin2 = time.time()
        if gp.DebugPrint_flag:
            print("tpChin1 is ", (tpChin2 - tpChin1) * 1000)

        #if width_or == 2560:
        if gp.ShowAllNumFlag:
            draw = cv2AddChineseText(draw, "识别总数: {}只".format(str(gp.num_count_old)),
                                     (50 * gp.radioShowX, 1100 * gp.radioShowY),
                                     textColor=gp.color_show,
                                     textSize=int(50* gp.radioShowtext))
            tpChin3 = time.time()
            if gp.DebugPrint_flag:
                print("tpChin2 is ", (tpChin3 - tpChin2) * 1000)
            draw = cv2AddChineseText(draw, "分类识别:", (50 * gp.radioShowX, 1150 * gp.radioShowY), textColor=gp.color_show,
                                     textSize=int(50* gp.radioShowtext))
            tpChin4 = time.time()
            if gp.DebugPrint_flag:
                print("tpChin3 is ", (tpChin4 - tpChin3) * 1000)
            for i in range(1, len(spiec_number)):
                if i < 5:
                    draw = cv2AddChineseText(draw, "{}:{}只".format(str(gp.BHG_chineseName[i]), str(gp.spiec_number_old[i])),
                                             (300 * gp.radioShowX + (i - 1) * 280 * gp.radioShowX, 1150 * gp.radioShowY),
                                             textColor=gp.color_show,
                                             textSize=int(50* gp.radioShowtext))
                else:
                    draw = cv2AddChineseText(draw, "{}:{}只".format(str(gp.BHG_chineseName[i]), str(gp.spiec_number_old[i])),
                                             (50 * gp.radioShowX + (i - 5) * 280 * gp.radioShowX, 1200 * gp.radioShowY),
                                             textColor=gp.color_show,
                                             textSize=int(50* gp.radioShowtext))
        img_show = cv2.cvtColor(np.asarray(img_show), cv2.COLOR_RGB2BGR)
        gp.img_show = img_show

        tpChin2 = time.time()
        if gp.DebugPrint_flag:
            print("tpChin is ", (tpChin2 - tpChin1) * 1000)



    #cv2.waitKey(0)
        # cv2.imwrite("../data_MPKFNet/RTSP/" + "{}.jpg".format(time.strftime('%Y%m%d%H%M%S', time.localtime(time.time()))),
        #             image_or)
        # cv2.imwrite(
        #     "../data_MPKFNet/RTSP/result/" + "{}.jpg".format(time.strftime('%Y%m%d%H%M%S', time.localtime(time.time()))),
        #     self.img)
        if gp.DebugPrint_flag:
            print(self.name + " is finish")
        num_image +=1


        return img_show,num_image

    def getrestartresult(self,IndexMax): #Ä¿±ê×·×Ù×î´óµÄË÷Òý
        list = [[] for i in range(IndexMax)]    #ÓÃÀ´´æ·Å°´ÕÕIndexË÷ÒýµÄ²»Í¬Í¼Æ¬µÄ£¨bbox(x,y,x,y),label£©
        for Index in range(1,IndexMax+1): #¶ÔËùÓпÉÄܵÄIndex½øÐбéÀú
            for imagei in range(5): #¶ÔÎåÕÅͼƬ½á¹û½øÐбéÀú
                a = gp.Finerestartboxes[imagei]
                b = gp.Finerestartboxes[imagei][:,5]
                c = np.where(b == Index)
                #ÐèÒªÓÅ»¯
                FinerestartboxesindexList = gp.Finerestartboxes[imagei][:,5].tolist()
                if Index in FinerestartboxesindexList:
                    RawIndex = FinerestartboxesindexList.index(Index)
                else:
                    RawIndex = -1
               # RawIndex = np.where(gp.Finerestartboxes[imagei][:,5] == Index) #»ñÈ¡µ±Ç°Í¼Æ¬ÖУ¬µ±Ç°IndexËù¶ÔÓ¦µÄÐУ¬Ä¿µÄΪÁË»ñµÃ£¬µ±Ç°Í¼Æ¬Öе±Ç°index±àºÅµÄÄ¿±êµÄ£¨bbox(x,y,x,y),label£©
               #  if (RawIndex[0] == []) | (RawIndex == None):
               #      RawIndex = -1
               #  else:
               #      RawIndex = RawIndex[0][0]
                if RawIndex == -1 : #˵Ã÷ûÓÐÕÒµ½
                    continue
                else:
                    list[Index-1].append(gp.Finerestartboxes[imagei][RawIndex,-2:]) #½«£¨bbox(x,y,x,y),label,index£© È¡³öÀ´
                     #µ±Ç°Í¼Æ¬±éÀú½áÊø£¬ÒѾ­ÕÒµ½Index¶ÔÓ¦µÄÄ¿±êÐÅÏ¢
        result =[]  #ÓÃÀ´´æ·Å×îÖյĽá¹û ½á¹ûΪ¶þά
        for Index in range(IndexMax):#£¨bbox(x,y,x,y),label,index£©*IndexMax
            temp = np.array(list[Index])    #»ñÈ¡Index¶ÔÓ¦µÄ¶þάnumpy£¬ÔÚ²»Í¬Í¼Æ¬ÀïÃæµÄÄ¿±êÐÅÏ¢£¨bbox(x,y,x,y),label,index£©
            print(type(temp))
            print(temp)


            temp1 = temp.mean(axis=0)  #¶Ô£¨bbox(x,y,x,y),label,index£©È¡Æ½¾ùÖµ£¬µ«ÊÇlabel²»ÄÜȡƽ¾ùÖµ£¬ÐèÒª¼ÌÐø´¦Àí,È¥µôÁËxyxy
            temp2 = temp[:,0] #È¡³öÀ´labelµÄÒ»ÁÐ
            label_max = statistics.mode(temp2) #Çó³öÀ´³öÏÖlabel×î¶àµÄ½á¹û
            temp1[0] = label_max  #½«label_max Ìæ»»µ½Ô­ÏȵĽá¹ûÖÐ
            result.append(temp1)
        return np.array(result)
    def systemPre(self,img,numPre):
        height_or, width_or = img.shape[:2]
        print("image size is ", height_or, width_or)
        is_ex_bird, labels, boxes, scores = self.DetectionClass.check_object(img)
        if is_ex_bird:
            for i in range(len(boxes)):

                if boxes[i][0] < 0:
                    boxes[i][0] = 0
                if boxes[i][1] < 0:
                    boxes[i][1] = 0
                if boxes[i][2] < 0:
                    boxes[i][2] = 0
                if boxes[i][3] < 0:
                    boxes[i][3] = 0
            for i in range(len(labels)):
                labels[i] = 0  # gp.birdIndex
            boxes_fine, labels_fine, scores_fine = self.fine_grained.inference(img, boxes, labels, scores, 8)
        print("System preheating {} is ok".format(str(numPre)))



class Density_mul (threading.Thread):
    def __init__(self, threadID, name,imagetime,DensityClass):
        threading.Thread.__init__(self)
        self.threadID = threadID
        self.name = name

        self.imagetime = imagetime
        self.DensityClass = DensityClass
    def run(self):
        gp.Density_thread_flag = True

        if gp.DebugPrint_flag:
            print(self.name +self.imagetime," is starting")
        i = 0
        while (i<=29):
            # if(len(gp.ImgDealQueue)<=i):
            #     gp.event.wait()
            # if ((i+1)%25==0):   #˵Ã÷ÊǵÚ25*nÖ¡,¼ä¸ô1s´¦ÀíÒ»´Î
            if gp.DebugPrint_flag:
                print("Density is1 ", len(gp.ImgDealQueue), "  ", i)
                print("Density is waiting")
            if (len(gp.ImgDealQueue) <= i):
                gp.eventDensity.wait()
            if gp.DebugPrint_flag:
                print("Density is2 ", len(gp.ImgDealQueue), "  ", i)
                print("Density is not waiting")
            self.DensityClass.inference(gp.ImgDealQueue[i],self.imagetime,str(i).zfill(4))
            gp.eventDensity.clear()
            i += 1
        gp.Density_thread_flag = False
        gp.is_ex_bird = False
        gp.destory_image = True
        if gp.DebugPrint_flag:
            print(self.name + self.imagetime," is over")
class Detecntion_mul (threading.Thread):
    def __init__(self, threadID, name,imagetime,DetectionClass,Fine_grainedClass):
        threading.Thread.__init__(self)
        self.threadID = threadID
        self.name = name
        self.imagetime = imagetime
        self.DetectionClass = DetectionClass
        self.Fine_grainedClass = Fine_grainedClass
    def run(self):
        gp.Detection_thread_flag = True

        if gp.DebugPrint_flag:
            print(self.name ,self.imagetime," is starting")
        i = 0
        while (i<=29):
            if gp.DebugPrint_flag:
                print("Detection is1 ",len(gp.ImgDealQueue),"  ",i)
            # if (len(gp.ImgDealQueue) <= i):
            #     gp.event.wait()
            # if ((i+1)%25==0):   #˵Ã÷ÊǵÚ25*nÖ¡,¼ä¸ô1s´¦ÀíÒ»´Î
                print("Detection is waiting")
            if (len(gp.ImgDealQueue) <= i):
                gp.eventDetection.wait()
            if gp.DebugPrint_flag:
                print("Detection is2 ", len(gp.ImgDealQueue), "  ", i)
                print("Detection is not waiting")
            self.DetectionClass.inference(gp.ImgDealQueue[i],self.imagetime,str(i).zfill(4),self.Fine_grainedClass)
            gp.eventDetection.clear()

            i+=1
        gp.Detection_thread_flag = False
        gp.is_ex_bird = False
        gp.destory_image = True
        if gp.DebugPrint_flag:
            print(self.name ,self.imagetime," is over")
class SaveImage_mul (threading.Thread):
    def __init__(self, threadID, name,imagetime,root):
        threading.Thread.__init__(self)
        self.threadID = threadID
        self.name = name
        self.imagetime = imagetime
        self.root = root
        self.rootpath = os.path.join(self.root, self.imagetime)
    def run(self):
        gp.SaveImage_thread_flag = True

        if not os.path.exists(self.rootpath):
            os.mkdir(self.rootpath)
        if not os.path.exists(self.rootpath + "/" + "RawImage" + "/"):
            os.mkdir(self.rootpath + "/" + "RawImage" + "/")
        if gp.DebugPrint_flag:
            print(self.name ,self.imagetime," is starting")
        i = 0
        while (i<=29):
            # if(len(gp.ImgDealQueue)<=i):
            #     gp.event.wait()
            # if ((i+1)%25==0):   #˵Ã÷ÊǵÚ25*nÖ¡,¼ä¸ô1s´¦ÀíÒ»´Î
            if gp.DebugPrint_flag:
                print("Saveimage is1 ", len(gp.ImgDealQueue), "  ", i)
                print("Saveimage is waiting")
            if (len(gp.ImgDealQueue) <= i):
                gp.eventSaveimage.wait()
            if gp.DebugPrint_flag:
                print("Saveimage is2 ", len(gp.ImgDealQueue), "  ", i)
                print("Saveimage is not waiting")
            cv2.imwrite(os.path.join(self.rootpath,"RawImage","{}.jpg".format(str(i).zfill(4))),gp.ImgDealQueue[i])
            gp.eventSaveimage.clear()
            if gp.DebugPrint_flag:
                print("Saveimage finish once")
            i+=1
        gp.SaveImage_thread_flag = False
        gp.is_ex_bird = False
        gp.destory_image = True
        if gp.DebugPrint_flag:
            print(self.name ,self.imagetime," is over")
class ThreadMonitor_mul (threading.Thread):
    def __init__(self, threadID, name):
        threading.Thread.__init__(self)
        self.threadID = threadID
        self.name = name

    def run(self):
        if gp.DebugPrint_flag:
            print(self.name," is starting")
        while 1:
           # print(gp.Density_thread_flag,gp.Detection_thread_flag,gp.SaveImage_thread_flag)
            if (gp.Density_thread_flag) == False and (gp.Detection_thread_flag == False) and (gp.SaveImage_thread_flag == False):  #˵Ã÷Èý¸ö½ø³Ì¶¼ÒѾ­½áÊøÁË
                gp.ImgDealQueue = []
class isopenmul (threading.Thread):
    def __init__(self, threadID, name,DetectionClass):
        threading.Thread.__init__(self)
        self.threadID = threadID
        self.name = name
        self.DetectionClass = DetectionClass
    def run(self):
        while 1:
           # print(gp.Density_thread_flag,gp.Detection_thread_flag,gp.SaveImage_thread_flag)
           if (gp.Vedio_thread_flag == False) and (gp.Density_thread_flag == False) and (
                   gp.Detection_thread_flag == False) and (gp.SaveImage_thread_flag == False) and (
                   gp.is_ex_bird == False) and gp.num_detection > 24:
               gp.num_detection = 0

               if gp.DebugPrint_flag:
                   print( "check is start")
               gp.is_ex_bird, labels, boxes, scores = self.DetectionClass.check_object(gp.frame_openmul)

               if gp.is_ex_bird:
                   if gp.DebugPrint_flag:
                       print(gp.is_ex_bird)
                   gp.image_time = gp.Camera_device + time.strftime('%Y%m%d%H%M%S', time.localtime(time.time()))
               if gp.DebugPrint_flag:
                   print("check is finish")

               # Check_start1 = Check_start("0002", "check_start",  gp.frame_openmul, self.DetectionClass)
               # Check_start1.start()













if __name__ == '__main__':
    # t1 = time.time()
    # image = cv2.imread("/home/wanghc/output1/0001_20210413155842/RawImage/0174.jpg")
    # t2 = time.time()
    # camera_num = "0001"
    # image_time = time.strftime('%Y%m%d%H%M%S',time.localtime(time.time()))
    # image_time = camera_num + image_time
    # root = "/home/wanghc/output"
    # Density1 = Density(gp.devices,root)
    # Detection1 = Detection_Y5(gp.devices,root)
    # Fine_grained1 = Fine_grained(gp.devices,root)
    # os.environ['CUDA_VISIBLE_DEVICES'] = '4'
    # device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
    # path = '/home/wanghc/output1/001020210509163621/RawImage/0000.jpg'
    # img = cv2.imread(path)  # BGR
    # Detection1.inference(img, image_time, str(1), Fine_grained1)

    # root = "/home/wanghc/output"
    # detector = Detection_Y5(gp.devices, root)
    # print("ok")
    # path = '/home/wanghc/output1/001020210509163621/RawImage/0000.jpg'
    # img = cv2.imread(path)  # BGR
    # img_show = img#cv2.resize(img, (1024, 768))
    # names = detector.model.module.names if hasattr(detector.model, 'module') else detector.model.names
    # colors = [[random.randint(0, 255) for _ in range(3)] for _ in names]
    # flag, labels,pred_boxes, scores  = detector.check_object(img_show)
    # print(len(scores))
    # #img_show = cv2.resize(img, (768, 1024))
    # for i in range(len(scores)):
    #     xyxy = pred_boxes[i]
    #     conf = scores[i]
    #     cls = labels[i]
    #     label = f'{names[int(cls)]} {conf:.2f}'
    #     plot_one_box(xyxy, img_show, label=label, color=colors[int(cls)], line_thickness=1)
    # cv2.imshow(str(path), img_show)
    # t1 = time.time()
    # cv2.imwrite("./a.jpg", img_show)
    # t2 = time.time()
    # print((t2 - t1) * 1000)
    # cv2.waitKey(-1)
    # while(1):
    #     pass
    boxes_list = [[0,0,49,49],[0,0,100,100],[0,0,100,100],[0,0,49,49],[0,0,100,100],[0,0,49,49],[0,0,100,100],[0,0,100,100],[0,0,100,100],[0,0,100,100],[0,0,100,100],[0,0,100,100],[0,0,100,100],[0,0,100,100],[0,0,100,100],[0,0,100,100],[0,0,100,100],[0,0,100,100],[0,0,100,100],[0,0,100,100],[0,0,100,100],[0,0,100,100],[0,0,100,100],[0,0,100,100],[0,0,100,100],[0,0,100,100],[0,0,100,100],[0,0,100,100],[0,0,100,100],[0,0,100,100],[0,0,100,100],[0,0,100,100],[0,0,100,100],[0,0,100,100],[0,0,100,100],[0,0,100,100],[0,0,100,100],[0,0,100,100],[0,0,100,100],[0,0,100,100]]
    label_list = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
    score_list = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
    boxes_list = torch.Tensor(boxes_list)
    label_list = torch.Tensor(label_list)
    score_list = torch.Tensor(score_list)
    path = "/home/aibird/model/data_MPKFNet/testimage/20220324105705.jpg"
    image = cv2.imread(path)
    Fine_grained1 = Fine_grained_T(gp.devices, "root")
    #image = torch.from_numpy(image)
    boxes_fine, labels_fine, scores_fine = Fine_grained1.inference(image,boxes_list,label_list,score_list,8)
    print("ok")