# -- coding: utf-8 -- # @Author : Calvin # @Email : 934055521@qq.com # @File : xld.py # @use : # @Software: PyCharm # @Time : 27/5/2022 3:32 PM """ python3 FinegrainedIdentificationDatasetGenerate.py -rp /data/DataManagement/Project/004_kunmingcuihu/0003_20220507_800/CAS_KMCuihuBird/CMB_KMCuihuBird -blp /data/DataManagement/Project/004_kunmingcuihu/0003_20220507_800/CAS_KMCuihuBird/CMB_KMCuihuBird/birdList/KMCuihuBird.xlsx """ import os import shutil import argparse import random import pandas as pd import numpy as np import cv2 from tqdm import tqdm BHG_chineseName = ['bird', 'Anser_albifrons', 'Fulica_atra', 'Egretta_garzetta', 'Platalea_leucorodia', 'Mergellus_albellus', 'Tringa_ochropus', 'Anser_indicus', 'Anas_zonorhyncha', 'Ardea_cinerea', 'Mareca_strepera', 'Mareca_penelope', 'Ardea_alba', 'Botaurus_stellaris', 'Anser_fabalis', 'Recurvirostra_avosetta', 'Podiceps_cristatus', 'Vanellus_vanellus', 'Aythya_fuligula', 'Ciconia_nigra', 'Gallinula_chloropus', 'Chroicocephalus_ridibundus', 'Anser_cygnoides', 'Grus_grus', 'Anser_anser', 'Anas_crecca', 'Anas_platyrhynchos', 'Spatula_clypeata', 'Phalacrocorax_carbo', 'Mergus_merganser', 'Tadorna_tadorna', 'Bucephala_clangula', 'Gallinago_gallinago', 'Anser_erythropus', 'Cygnus_columbianus', 'Anas_acuta', 'Ardea_intermedia', 'Zosterops_simplex', 'Acridotheres_cristatellus', 'Circus_spilonotus', 'Motacilla_alba', 'Aythya_nyroca', 'Ardea_purpurea', 'Ardeola_bacchus', 'Upupa_epops', 'Himantopus_himantopus', 'Milvus_migrans', 'Falco_tinnunculus', 'Sibirionetta_formosa', 'Phasianus_colchicus', 'Picus_canus', 'Vanellus_cinereus', 'Actitis_hypoleucos', 'Ixobrychus_cinnamomeus', 'Mareca_falcata', 'Bubulcus_coromandus', 'Alcedo_atthis', 'Accipiter_nisus', 'Pica_serica', 'Nycticorax_nycticorax', 'Aix_galericulata', 'Lanius_schach', 'Mergus_serrator', 'Sterna_hirundo', 'Sternula_antillarum', 'Scolopax_rusticola', 'Pernis_ptilorhynchus', 'Cygnus_cygnus', 'Otis_tarda', 'Zapornia_pusilla', 'Streptopelia_orientalis', 'Aythya_valisineria', 'Accipiter_gularis', 'Buteo_japonicus', 'Accipiter_virgatus', 'Buteo_lagopus', 'Streptopelia_decaocto', 'Circus_aeruginosus', 'Circus_cyaneus', 'Spatula_querquedula', 'Pelecanus_onocrotalus', 'Aythya_ferina', 'Accipiter_gentilis', 'Larus_cachinnans', 'Rallus_aquaticus', 'Netta_rufina', 'Calidris_falcinellus', 'Bubo_bubo', 'Coturnix_japonica', 'Pandion_haliaetus', 'Ixobrychus_sinensis', 'Grus_nigricollis', 'Tachybaptus_ruficollis', 'Tadorna_ferruginea', 'Leucogeranus_leucogeranus', 'Grus_monacha', 'Antigone_vipio', 'Cyanopica_cyanus', 'Passer_montanus', 'Hirundo_rustica'] ##################################### Number_last = 0 LabelSpecies = [] DataNOlineFeed = [] WrongImages = [] DataSpeciesClassier = dict() ##################################### def parse(): arg = argparse.ArgumentParser() arg.add_argument('--root_path', '-rp', default='/data/DataManagement/Project/CAS_ProjectBird/CMB_ProjectBird0615/', type=str) arg.add_argument('--division_ratio', '-dr', default='0.7', type=float) arg.add_argument('--bird_list_path', '-blp', default='/data/DataManagement/Project/CAS_ProjectBird/CMB_ProjectBird0615/birdList/ProjectBirdList.xlsx') args = arg.parse_args() return args if __name__ == '__main__': args = parse() dataset_txt_path = os.path.join(args.root_path, 'DatasetTxt') species_txt_path = f'{args.root_path}/DatasetTxt/Species.txt' dataset_spectrum_path = f'{args.root_path}/OrginalData' dataset_allspectrum_path = f'{args.root_path}/FinalData/data' dataset_allspectrum_path_train = f'{args.root_path}/FinalData/train' dataset_allspectrum_path_test = f'{args.root_path}/FinalData/test' dataset_allspectrum_txt_path = f'{args.root_path}/FinalData/anno/all.txt' if not os.path.exists(dataset_txt_path): os.makedirs(dataset_txt_path) # for data in tqdm(os.listdir(dataset_spectrum_path), ncols=80, ascii=True, desc="OrginalData images path"): # # for data in os.listdir(dataset_spectrum_path): # for image in os.listdir(os.path.join(dataset_spectrum_path, data)): # try: # im = cv2.imread(os.path.join(dataset_spectrum_path, data, image)) # except: # WrongImages.append(os.path.join(dataset_spectrum_path, data, image)) # os.remove(os.path.join(dataset_spectrum_path, data, image)) # tqdm.write('Wrong images are {:}'.format(WrongImages)) bird_list = pd.read_excel(args.bird_list_path, usecols=['拉丁名']) bird_list = np.array(bird_list.stack()).tolist() for num, data in tqdm(enumerate(bird_list), ncols=80, ascii=True, desc="writing species.txt"): # for num, data in enumerate(bird_list): with open(species_txt_path, 'a') as f: filename = data file_final_name = data + ' ' + str(num) f.write(file_final_name) f.write('\n') for speices in tqdm(os.listdir(dataset_spectrum_path), ncols=80, ascii=True, desc="writing species.txt"): # for speices in os.listdir(dataset_spectrum_path): for data in os.listdir(os.path.join(dataset_spectrum_path, speices)): if data.split('___')[0] != speices: os.rename(os.path.join(dataset_spectrum_path, speices, data), os.path.join(dataset_spectrum_path, speices, '{:}___{:}'.format(speices, data))) tqdm.write('The pictures naming is done!') with open(species_txt_path, 'r') as file: Data = file.readlines() for line in Data: line = line.strip('\n') species = line.split(' ')[0] Number = line.split(' ')[1] #Judge Number if Number != Number_last: DataSpeciesClassier[species] = Number Number_last = Number tqdm.write(str(DataSpeciesClassier)) tqdm.write('The species have been obtained') # transfer single files to all files if not os.path.exists(dataset_allspectrum_path): os.makedirs(dataset_allspectrum_path) if not os.path.exists(dataset_allspectrum_path_train): os.makedirs(dataset_allspectrum_path_train) if not os.path.exists(dataset_allspectrum_path_test): os.makedirs(dataset_allspectrum_path_test) for data in tqdm(os.listdir(dataset_spectrum_path), ncols=80, ascii=True, desc="transfer single files"): # for data in os.listdir(dataset_spectrum_path): for file in os.listdir(os.path.join(dataset_spectrum_path, data)): shutil.copy(os.path.join(dataset_spectrum_path, data, file), dataset_allspectrum_path) tqdm.write('Transfer single files to all files is done') # generate the all.txt if not os.path.exists('/'.join(dataset_allspectrum_txt_path.split('/')[:-1])): os.makedirs('/'.join(dataset_allspectrum_txt_path.split('/')[:-1])) for data in tqdm(os.listdir(dataset_allspectrum_path), ncols=80, ascii=True, desc="transfer single files"): # for data in os.listdir(dataset_allspectrum_path): with open(dataset_allspectrum_txt_path, 'a') as f: file_new_name = data.split('___')[0] # file_new_name = data # file_new_name = '_'.join(file_new_name.split('_')[:-1]) num = DataSpeciesClassier[file_new_name] file_final_name = data + ' ' + num f.write(file_final_name) f.write('\n') tqdm.write('Generate the all.txt is done') # Spilit dataset DataPath = dataset_allspectrum_txt_path # delete file_txt if os.path.exists(DataPath.replace(os.path.basename(DataPath), 'train.txt')) : os.remove(DataPath.replace(os.path.basename(DataPath), 'train.txt')) if os.path.exists(DataPath.replace(os.path.basename(DataPath), 'test.txt')): os.remove(DataPath.replace(os.path.basename(DataPath), 'test.txt')) with open(DataPath, 'r')as f: Data = f.readlines() LabelSpecies = [] for RowData in Data: RowData = RowData.strip() # find picture-label RowLabels = RowData.split(' ')[1] DataNOlineFeed.append(RowData.split(' ')) if RowLabels not in LabelSpecies: LabelSpecies.append(RowLabels) print(LabelSpecies) tqdm.write("classes num of Dataset is : {:}".format(len(LabelSpecies))) for Labels in tqdm(LabelSpecies, ncols=80, ascii=True, desc="transfer single files"): # for Labels in LabelSpecies: Num = 0 TrainData = [] TestData = [] TotalData = [] for SpeciesData in DataNOlineFeed: if Labels == SpeciesData[1]: Num += 1 TotalData.append(SpeciesData) TrainNum = int(Num * args.division_ratio) TestNum = Num - TrainNum random.shuffle(TotalData) with open(DataPath.replace(os.path.basename(DataPath), 'train.txt'), 'a') as f: for i in range(TrainNum): f.write(' '.join(TotalData[i]) + '\n') label_classname = "{}.{}".format(str(Labels).zfill(4),BHG_chineseName[int(Labels)]) soursepathTrain = os.path.join(dataset_spectrum_path,label_classname) destpathTrain = os.path.join(dataset_allspectrum_path_train,label_classname) if not os.path.exists(destpathTrain): os.makedirs(destpathTrain) image_name = TotalData[i][0] shutil.copy(os.path.join(soursepathTrain,str(image_name)),os.path.join(destpathTrain,str(image_name))) with open(DataPath.replace(os.path.basename(DataPath), 'test.txt'), 'a') as f: for i in range(TrainNum, Num): f.write(' '.join(TotalData[i]) + '\n') label_classname = "{}.{}".format(str(Labels).zfill(4), BHG_chineseName[int(Labels)]) soursepathTest = os.path.join(dataset_spectrum_path, label_classname) destpathTest = os.path.join(dataset_allspectrum_path_test, label_classname) if not os.path.exists(destpathTest): os.makedirs(destpathTest) image_name = TotalData[i][0] shutil.copy(os.path.join(soursepathTest, str(image_name)), os.path.join(destpathTest, str(image_name)))