177 lines
10 KiB
Python
Executable File
177 lines
10 KiB
Python
Executable File
# -- coding: utf-8 --
|
|
# @Author : Calvin
|
|
# @Email : 934055521@qq.com
|
|
# @File : xld.py
|
|
# @use :
|
|
# @Software: PyCharm
|
|
# @Time : 27/5/2022 3:32 PM
|
|
"""
|
|
python3 FinegrainedIdentificationDatasetGenerate.py
|
|
-rp /data/DataManagement/Project/004_kunmingcuihu/0003_20220507_800/CAS_KMCuihuBird/CMB_KMCuihuBird
|
|
-blp /data/DataManagement/Project/004_kunmingcuihu/0003_20220507_800/CAS_KMCuihuBird/CMB_KMCuihuBird/birdList/KMCuihuBird.xlsx
|
|
"""
|
|
import os
|
|
import shutil
|
|
import argparse
|
|
import random
|
|
import pandas as pd
|
|
import numpy as np
|
|
import cv2
|
|
from tqdm import tqdm
|
|
BHG_chineseName = ['bird', 'Anser_albifrons', 'Fulica_atra', 'Egretta_garzetta', 'Platalea_leucorodia', 'Mergellus_albellus', 'Tringa_ochropus', 'Anser_indicus', 'Anas_zonorhyncha', 'Ardea_cinerea', 'Mareca_strepera', 'Mareca_penelope', 'Ardea_alba', 'Botaurus_stellaris', 'Anser_fabalis', 'Recurvirostra_avosetta', 'Podiceps_cristatus', 'Vanellus_vanellus', 'Aythya_fuligula', 'Ciconia_nigra', 'Gallinula_chloropus', 'Chroicocephalus_ridibundus', 'Anser_cygnoides', 'Grus_grus', 'Anser_anser', 'Anas_crecca', 'Anas_platyrhynchos', 'Spatula_clypeata', 'Phalacrocorax_carbo', 'Mergus_merganser', 'Tadorna_tadorna', 'Bucephala_clangula', 'Gallinago_gallinago', 'Anser_erythropus', 'Cygnus_columbianus', 'Anas_acuta', 'Ardea_intermedia', 'Zosterops_simplex', 'Acridotheres_cristatellus', 'Circus_spilonotus', 'Motacilla_alba', 'Aythya_nyroca', 'Ardea_purpurea', 'Ardeola_bacchus', 'Upupa_epops', 'Himantopus_himantopus', 'Milvus_migrans', 'Falco_tinnunculus', 'Sibirionetta_formosa', 'Phasianus_colchicus', 'Picus_canus', 'Vanellus_cinereus', 'Actitis_hypoleucos', 'Ixobrychus_cinnamomeus', 'Mareca_falcata', 'Bubulcus_coromandus', 'Alcedo_atthis', 'Accipiter_nisus', 'Pica_serica', 'Nycticorax_nycticorax', 'Aix_galericulata', 'Lanius_schach', 'Mergus_serrator', 'Sterna_hirundo', 'Sternula_antillarum', 'Scolopax_rusticola', 'Pernis_ptilorhynchus', 'Cygnus_cygnus', 'Otis_tarda', 'Zapornia_pusilla', 'Streptopelia_orientalis', 'Aythya_valisineria', 'Accipiter_gularis', 'Buteo_japonicus', 'Accipiter_virgatus', 'Buteo_lagopus', 'Streptopelia_decaocto', 'Circus_aeruginosus', 'Circus_cyaneus', 'Spatula_querquedula', 'Pelecanus_onocrotalus', 'Aythya_ferina', 'Accipiter_gentilis', 'Larus_cachinnans', 'Rallus_aquaticus', 'Netta_rufina', 'Calidris_falcinellus', 'Bubo_bubo', 'Coturnix_japonica', 'Pandion_haliaetus', 'Ixobrychus_sinensis', 'Grus_nigricollis', 'Tachybaptus_ruficollis', 'Tadorna_ferruginea', 'Leucogeranus_leucogeranus', 'Grus_monacha', 'Antigone_vipio', 'Cyanopica_cyanus', 'Passer_montanus', 'Hirundo_rustica']
|
|
|
|
|
|
#####################################
|
|
Number_last = 0
|
|
LabelSpecies = []
|
|
DataNOlineFeed = []
|
|
WrongImages = []
|
|
DataSpeciesClassier = dict()
|
|
#####################################
|
|
|
|
|
|
def parse():
|
|
arg = argparse.ArgumentParser()
|
|
arg.add_argument('--root_path', '-rp', default='/data/DataManagement/Project/CAS_ProjectBird/CMB_ProjectBird0615/', type=str)
|
|
arg.add_argument('--division_ratio', '-dr', default='0.7', type=float)
|
|
arg.add_argument('--bird_list_path', '-blp', default='/data/DataManagement/Project/CAS_ProjectBird/CMB_ProjectBird0615/birdList/ProjectBirdList.xlsx')
|
|
args = arg.parse_args()
|
|
return args
|
|
|
|
|
|
if __name__ == '__main__':
|
|
args = parse()
|
|
dataset_txt_path = os.path.join(args.root_path, 'DatasetTxt')
|
|
species_txt_path = f'{args.root_path}/DatasetTxt/Species.txt'
|
|
dataset_spectrum_path = f'{args.root_path}/OrginalData'
|
|
dataset_allspectrum_path = f'{args.root_path}/FinalData/data'
|
|
dataset_allspectrum_path_train = f'{args.root_path}/FinalData/train'
|
|
dataset_allspectrum_path_test = f'{args.root_path}/FinalData/test'
|
|
dataset_allspectrum_txt_path = f'{args.root_path}/FinalData/anno/all.txt'
|
|
if not os.path.exists(dataset_txt_path):
|
|
os.makedirs(dataset_txt_path)
|
|
# for data in tqdm(os.listdir(dataset_spectrum_path), ncols=80, ascii=True, desc="OrginalData images path"):
|
|
# # for data in os.listdir(dataset_spectrum_path):
|
|
# for image in os.listdir(os.path.join(dataset_spectrum_path, data)):
|
|
# try:
|
|
# im = cv2.imread(os.path.join(dataset_spectrum_path, data, image))
|
|
# except:
|
|
# WrongImages.append(os.path.join(dataset_spectrum_path, data, image))
|
|
# os.remove(os.path.join(dataset_spectrum_path, data, image))
|
|
# tqdm.write('Wrong images are {:}'.format(WrongImages))
|
|
bird_list = pd.read_excel(args.bird_list_path, usecols=['拉丁名'])
|
|
bird_list = np.array(bird_list.stack()).tolist()
|
|
for num, data in tqdm(enumerate(bird_list), ncols=80, ascii=True, desc="writing species.txt"):
|
|
# for num, data in enumerate(bird_list):
|
|
with open(species_txt_path, 'a') as f:
|
|
filename = data
|
|
file_final_name = data + ' ' + str(num)
|
|
f.write(file_final_name)
|
|
f.write('\n')
|
|
for speices in tqdm(os.listdir(dataset_spectrum_path), ncols=80, ascii=True, desc="writing species.txt"):
|
|
# for speices in os.listdir(dataset_spectrum_path):
|
|
for data in os.listdir(os.path.join(dataset_spectrum_path, speices)):
|
|
if data.split('___')[0] != speices:
|
|
os.rename(os.path.join(dataset_spectrum_path, speices, data), os.path.join(dataset_spectrum_path, speices, '{:}___{:}'.format(speices, data)))
|
|
tqdm.write('The pictures naming is done!')
|
|
|
|
with open(species_txt_path, 'r') as file:
|
|
Data = file.readlines()
|
|
for line in Data:
|
|
line = line.strip('\n')
|
|
species = line.split(' ')[0]
|
|
Number = line.split(' ')[1]
|
|
#Judge Number
|
|
if Number != Number_last:
|
|
DataSpeciesClassier[species] = Number
|
|
Number_last = Number
|
|
|
|
tqdm.write(str(DataSpeciesClassier))
|
|
tqdm.write('The species have been obtained')
|
|
|
|
# transfer single files to all files
|
|
if not os.path.exists(dataset_allspectrum_path):
|
|
os.makedirs(dataset_allspectrum_path)
|
|
if not os.path.exists(dataset_allspectrum_path_train):
|
|
os.makedirs(dataset_allspectrum_path_train)
|
|
if not os.path.exists(dataset_allspectrum_path_test):
|
|
os.makedirs(dataset_allspectrum_path_test)
|
|
for data in tqdm(os.listdir(dataset_spectrum_path), ncols=80, ascii=True, desc="transfer single files"):
|
|
# for data in os.listdir(dataset_spectrum_path):
|
|
for file in os.listdir(os.path.join(dataset_spectrum_path, data)):
|
|
shutil.copy(os.path.join(dataset_spectrum_path, data, file), dataset_allspectrum_path)
|
|
tqdm.write('Transfer single files to all files is done')
|
|
|
|
# generate the all.txt
|
|
if not os.path.exists('/'.join(dataset_allspectrum_txt_path.split('/')[:-1])):
|
|
os.makedirs('/'.join(dataset_allspectrum_txt_path.split('/')[:-1]))
|
|
for data in tqdm(os.listdir(dataset_allspectrum_path), ncols=80, ascii=True, desc="transfer single files"):
|
|
# for data in os.listdir(dataset_allspectrum_path):
|
|
with open(dataset_allspectrum_txt_path, 'a') as f:
|
|
file_new_name = data.split('___')[0]
|
|
# file_new_name = data
|
|
# file_new_name = '_'.join(file_new_name.split('_')[:-1])
|
|
num = DataSpeciesClassier[file_new_name]
|
|
file_final_name = data + ' ' + num
|
|
f.write(file_final_name)
|
|
f.write('\n')
|
|
tqdm.write('Generate the all.txt is done')
|
|
|
|
# Spilit dataset
|
|
DataPath = dataset_allspectrum_txt_path
|
|
# delete file_txt
|
|
if os.path.exists(DataPath.replace(os.path.basename(DataPath), 'train.txt')) :
|
|
os.remove(DataPath.replace(os.path.basename(DataPath), 'train.txt'))
|
|
if os.path.exists(DataPath.replace(os.path.basename(DataPath), 'test.txt')):
|
|
os.remove(DataPath.replace(os.path.basename(DataPath), 'test.txt'))
|
|
|
|
with open(DataPath, 'r')as f:
|
|
Data = f.readlines()
|
|
LabelSpecies = []
|
|
for RowData in Data:
|
|
RowData = RowData.strip()
|
|
# find picture-label
|
|
RowLabels = RowData.split(' ')[1]
|
|
DataNOlineFeed.append(RowData.split(' '))
|
|
if RowLabels not in LabelSpecies:
|
|
LabelSpecies.append(RowLabels)
|
|
print(LabelSpecies)
|
|
tqdm.write("classes num of Dataset is : {:}".format(len(LabelSpecies)))
|
|
for Labels in tqdm(LabelSpecies, ncols=80, ascii=True, desc="transfer single files"):
|
|
# for Labels in LabelSpecies:
|
|
Num = 0
|
|
TrainData = []
|
|
TestData = []
|
|
TotalData = []
|
|
for SpeciesData in DataNOlineFeed:
|
|
if Labels == SpeciesData[1]:
|
|
Num += 1
|
|
TotalData.append(SpeciesData)
|
|
|
|
TrainNum = int(Num * args.division_ratio)
|
|
TestNum = Num - TrainNum
|
|
random.shuffle(TotalData)
|
|
with open(DataPath.replace(os.path.basename(DataPath), 'train.txt'), 'a') as f:
|
|
for i in range(TrainNum):
|
|
f.write(' '.join(TotalData[i]) + '\n')
|
|
label_classname = "{}.{}".format(str(Labels).zfill(4),BHG_chineseName[int(Labels)])
|
|
soursepathTrain = os.path.join(dataset_spectrum_path,label_classname)
|
|
destpathTrain = os.path.join(dataset_allspectrum_path_train,label_classname)
|
|
if not os.path.exists(destpathTrain):
|
|
os.makedirs(destpathTrain)
|
|
image_name = TotalData[i][0]
|
|
shutil.copy(os.path.join(soursepathTrain,str(image_name)),os.path.join(destpathTrain,str(image_name)))
|
|
|
|
with open(DataPath.replace(os.path.basename(DataPath), 'test.txt'), 'a') as f:
|
|
for i in range(TrainNum, Num):
|
|
f.write(' '.join(TotalData[i]) + '\n')
|
|
label_classname = "{}.{}".format(str(Labels).zfill(4), BHG_chineseName[int(Labels)])
|
|
soursepathTest = os.path.join(dataset_spectrum_path, label_classname)
|
|
destpathTest = os.path.join(dataset_allspectrum_path_test, label_classname)
|
|
if not os.path.exists(destpathTest):
|
|
os.makedirs(destpathTest)
|
|
image_name = TotalData[i][0]
|
|
shutil.copy(os.path.join(soursepathTest, str(image_name)), os.path.join(destpathTest, str(image_name)))
|
|
|
|
|