first commit
This commit is contained in:
hatcher committed
1 parent
ae53e9eac5
commit
91cbd01126
514 files changed
+55625
-2203
No files matched your search
Executable
+176
@@ -0,0 +1,176 @@
|
||||
# -- coding: utf-8 --
|
||||
# @Author : Calvin
|
||||
# @Email : 934055521@qq.com
|
||||
# @File : xld.py
|
||||
# @use :
|
||||
# @Software: PyCharm
|
||||
# @Time : 27/5/2022 3:32 PM
|
||||
"""
|
||||
python3 FinegrainedIdentificationDatasetGenerate.py
|
||||
-rp /data/DataManagement/Project/004_kunmingcuihu/0003_20220507_800/CAS_KMCuihuBird/CMB_KMCuihuBird
|
||||
-blp /data/DataManagement/Project/004_kunmingcuihu/0003_20220507_800/CAS_KMCuihuBird/CMB_KMCuihuBird/birdList/KMCuihuBird.xlsx
|
||||
"""
|
||||
import os
|
||||
import shutil
|
||||
import argparse
|
||||
import random
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
import cv2
|
||||
from tqdm import tqdm
|
||||
BHG_chineseName = ['bird', 'Anser_albifrons', 'Fulica_atra', 'Egretta_garzetta', 'Platalea_leucorodia', 'Mergellus_albellus', 'Tringa_ochropus', 'Anser_indicus', 'Anas_zonorhyncha', 'Ardea_cinerea', 'Mareca_strepera', 'Mareca_penelope', 'Ardea_alba', 'Botaurus_stellaris', 'Anser_fabalis', 'Recurvirostra_avosetta', 'Podiceps_cristatus', 'Vanellus_vanellus', 'Aythya_fuligula', 'Ciconia_nigra', 'Gallinula_chloropus', 'Chroicocephalus_ridibundus', 'Anser_cygnoides', 'Grus_grus', 'Anser_anser', 'Anas_crecca', 'Anas_platyrhynchos', 'Spatula_clypeata', 'Phalacrocorax_carbo', 'Mergus_merganser', 'Tadorna_tadorna', 'Bucephala_clangula', 'Gallinago_gallinago', 'Anser_erythropus', 'Cygnus_columbianus', 'Anas_acuta', 'Ardea_intermedia', 'Zosterops_simplex', 'Acridotheres_cristatellus', 'Circus_spilonotus', 'Motacilla_alba', 'Aythya_nyroca', 'Ardea_purpurea', 'Ardeola_bacchus', 'Upupa_epops', 'Himantopus_himantopus', 'Milvus_migrans', 'Falco_tinnunculus', 'Sibirionetta_formosa', 'Phasianus_colchicus', 'Picus_canus', 'Vanellus_cinereus', 'Actitis_hypoleucos', 'Ixobrychus_cinnamomeus', 'Mareca_falcata', 'Bubulcus_coromandus', 'Alcedo_atthis', 'Accipiter_nisus', 'Pica_serica', 'Nycticorax_nycticorax', 'Aix_galericulata', 'Lanius_schach', 'Mergus_serrator', 'Sterna_hirundo', 'Sternula_antillarum', 'Scolopax_rusticola', 'Pernis_ptilorhynchus', 'Cygnus_cygnus', 'Otis_tarda', 'Zapornia_pusilla', 'Streptopelia_orientalis', 'Aythya_valisineria', 'Accipiter_gularis', 'Buteo_japonicus', 'Accipiter_virgatus', 'Buteo_lagopus', 'Streptopelia_decaocto', 'Circus_aeruginosus', 'Circus_cyaneus', 'Spatula_querquedula', 'Pelecanus_onocrotalus', 'Aythya_ferina', 'Accipiter_gentilis', 'Larus_cachinnans', 'Rallus_aquaticus', 'Netta_rufina', 'Calidris_falcinellus', 'Bubo_bubo', 'Coturnix_japonica', 'Pandion_haliaetus', 'Ixobrychus_sinensis', 'Grus_nigricollis', 'Tachybaptus_ruficollis', 'Tadorna_ferruginea', 'Leucogeranus_leucogeranus', 'Grus_monacha', 'Antigone_vipio', 'Cyanopica_cyanus', 'Passer_montanus', 'Hirundo_rustica']
|
||||
|
||||
|
||||
#####################################
|
||||
Number_last = 0
|
||||
LabelSpecies = []
|
||||
DataNOlineFeed = []
|
||||
WrongImages = []
|
||||
DataSpeciesClassier = dict()
|
||||
#####################################
|
||||
|
||||
|
||||
def parse():
|
||||
arg = argparse.ArgumentParser()
|
||||
arg.add_argument('--root_path', '-rp', default='/data/DataManagement/Project/CAS_ProjectBird/CMB_ProjectBird0615/', type=str)
|
||||
arg.add_argument('--division_ratio', '-dr', default='0.7', type=float)
|
||||
arg.add_argument('--bird_list_path', '-blp', default='/data/DataManagement/Project/CAS_ProjectBird/CMB_ProjectBird0615/birdList/ProjectBirdList.xlsx')
|
||||
args = arg.parse_args()
|
||||
return args
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
args = parse()
|
||||
dataset_txt_path = os.path.join(args.root_path, 'DatasetTxt')
|
||||
species_txt_path = f'{args.root_path}/DatasetTxt/Species.txt'
|
||||
dataset_spectrum_path = f'{args.root_path}/OrginalData'
|
||||
dataset_allspectrum_path = f'{args.root_path}/FinalData/data'
|
||||
dataset_allspectrum_path_train = f'{args.root_path}/FinalData/train'
|
||||
dataset_allspectrum_path_test = f'{args.root_path}/FinalData/test'
|
||||
dataset_allspectrum_txt_path = f'{args.root_path}/FinalData/anno/all.txt'
|
||||
if not os.path.exists(dataset_txt_path):
|
||||
os.makedirs(dataset_txt_path)
|
||||
# for data in tqdm(os.listdir(dataset_spectrum_path), ncols=80, ascii=True, desc="OrginalData images path"):
|
||||
# # for data in os.listdir(dataset_spectrum_path):
|
||||
# for image in os.listdir(os.path.join(dataset_spectrum_path, data)):
|
||||
# try:
|
||||
# im = cv2.imread(os.path.join(dataset_spectrum_path, data, image))
|
||||
# except:
|
||||
# WrongImages.append(os.path.join(dataset_spectrum_path, data, image))
|
||||
# os.remove(os.path.join(dataset_spectrum_path, data, image))
|
||||
# tqdm.write('Wrong images are {:}'.format(WrongImages))
|
||||
bird_list = pd.read_excel(args.bird_list_path, usecols=['拉丁名'])
|
||||
bird_list = np.array(bird_list.stack()).tolist()
|
||||
for num, data in tqdm(enumerate(bird_list), ncols=80, ascii=True, desc="writing species.txt"):
|
||||
# for num, data in enumerate(bird_list):
|
||||
with open(species_txt_path, 'a') as f:
|
||||
filename = data
|
||||
file_final_name = data + ' ' + str(num)
|
||||
f.write(file_final_name)
|
||||
f.write('\n')
|
||||
for speices in tqdm(os.listdir(dataset_spectrum_path), ncols=80, ascii=True, desc="writing species.txt"):
|
||||
# for speices in os.listdir(dataset_spectrum_path):
|
||||
for data in os.listdir(os.path.join(dataset_spectrum_path, speices)):
|
||||
if data.split('___')[0] != speices:
|
||||
os.rename(os.path.join(dataset_spectrum_path, speices, data), os.path.join(dataset_spectrum_path, speices, '{:}___{:}'.format(speices, data)))
|
||||
tqdm.write('The pictures naming is done!')
|
||||
|
||||
with open(species_txt_path, 'r') as file:
|
||||
Data = file.readlines()
|
||||
for line in Data:
|
||||
line = line.strip('\n')
|
||||
species = line.split(' ')[0]
|
||||
Number = line.split(' ')[1]
|
||||
#Judge Number
|
||||
if Number != Number_last:
|
||||
DataSpeciesClassier[species] = Number
|
||||
Number_last = Number
|
||||
|
||||
tqdm.write(str(DataSpeciesClassier))
|
||||
tqdm.write('The species have been obtained')
|
||||
|
||||
# transfer single files to all files
|
||||
if not os.path.exists(dataset_allspectrum_path):
|
||||
os.makedirs(dataset_allspectrum_path)
|
||||
if not os.path.exists(dataset_allspectrum_path_train):
|
||||
os.makedirs(dataset_allspectrum_path_train)
|
||||
if not os.path.exists(dataset_allspectrum_path_test):
|
||||
os.makedirs(dataset_allspectrum_path_test)
|
||||
for data in tqdm(os.listdir(dataset_spectrum_path), ncols=80, ascii=True, desc="transfer single files"):
|
||||
# for data in os.listdir(dataset_spectrum_path):
|
||||
for file in os.listdir(os.path.join(dataset_spectrum_path, data)):
|
||||
shutil.copy(os.path.join(dataset_spectrum_path, data, file), dataset_allspectrum_path)
|
||||
tqdm.write('Transfer single files to all files is done')
|
||||
|
||||
# generate the all.txt
|
||||
if not os.path.exists('/'.join(dataset_allspectrum_txt_path.split('/')[:-1])):
|
||||
os.makedirs('/'.join(dataset_allspectrum_txt_path.split('/')[:-1]))
|
||||
for data in tqdm(os.listdir(dataset_allspectrum_path), ncols=80, ascii=True, desc="transfer single files"):
|
||||
# for data in os.listdir(dataset_allspectrum_path):
|
||||
with open(dataset_allspectrum_txt_path, 'a') as f:
|
||||
file_new_name = data.split('___')[0]
|
||||
# file_new_name = data
|
||||
# file_new_name = '_'.join(file_new_name.split('_')[:-1])
|
||||
num = DataSpeciesClassier[file_new_name]
|
||||
file_final_name = data + ' ' + num
|
||||
f.write(file_final_name)
|
||||
f.write('\n')
|
||||
tqdm.write('Generate the all.txt is done')
|
||||
|
||||
# Spilit dataset
|
||||
DataPath = dataset_allspectrum_txt_path
|
||||
# delete file_txt
|
||||
if os.path.exists(DataPath.replace(os.path.basename(DataPath), 'train.txt')) :
|
||||
os.remove(DataPath.replace(os.path.basename(DataPath), 'train.txt'))
|
||||
if os.path.exists(DataPath.replace(os.path.basename(DataPath), 'test.txt')):
|
||||
os.remove(DataPath.replace(os.path.basename(DataPath), 'test.txt'))
|
||||
|
||||
with open(DataPath, 'r')as f:
|
||||
Data = f.readlines()
|
||||
LabelSpecies = []
|
||||
for RowData in Data:
|
||||
RowData = RowData.strip()
|
||||
# find picture-label
|
||||
RowLabels = RowData.split(' ')[1]
|
||||
DataNOlineFeed.append(RowData.split(' '))
|
||||
if RowLabels not in LabelSpecies:
|
||||
LabelSpecies.append(RowLabels)
|
||||
print(LabelSpecies)
|
||||
tqdm.write("classes num of Dataset is : {:}".format(len(LabelSpecies)))
|
||||
for Labels in tqdm(LabelSpecies, ncols=80, ascii=True, desc="transfer single files"):
|
||||
# for Labels in LabelSpecies:
|
||||
Num = 0
|
||||
TrainData = []
|
||||
TestData = []
|
||||
TotalData = []
|
||||
for SpeciesData in DataNOlineFeed:
|
||||
if Labels == SpeciesData[1]:
|
||||
Num += 1
|
||||
TotalData.append(SpeciesData)
|
||||
|
||||
TrainNum = int(Num * args.division_ratio)
|
||||
TestNum = Num - TrainNum
|
||||
random.shuffle(TotalData)
|
||||
with open(DataPath.replace(os.path.basename(DataPath), 'train.txt'), 'a') as f:
|
||||
for i in range(TrainNum):
|
||||
f.write(' '.join(TotalData[i]) + '\n')
|
||||
label_classname = "{}.{}".format(str(Labels).zfill(4),BHG_chineseName[int(Labels)])
|
||||
soursepathTrain = os.path.join(dataset_spectrum_path,label_classname)
|
||||
destpathTrain = os.path.join(dataset_allspectrum_path_train,label_classname)
|
||||
if not os.path.exists(destpathTrain):
|
||||
os.makedirs(destpathTrain)
|
||||
image_name = TotalData[i][0]
|
||||
shutil.copy(os.path.join(soursepathTrain,str(image_name)),os.path.join(destpathTrain,str(image_name)))
|
||||
|
||||
with open(DataPath.replace(os.path.basename(DataPath), 'test.txt'), 'a') as f:
|
||||
for i in range(TrainNum, Num):
|
||||
f.write(' '.join(TotalData[i]) + '\n')
|
||||
label_classname = "{}.{}".format(str(Labels).zfill(4), BHG_chineseName[int(Labels)])
|
||||
soursepathTest = os.path.join(dataset_spectrum_path, label_classname)
|
||||
destpathTest = os.path.join(dataset_allspectrum_path_test, label_classname)
|
||||
if not os.path.exists(destpathTest):
|
||||
os.makedirs(destpathTest)
|
||||
image_name = TotalData[i][0]
|
||||
shutil.copy(os.path.join(soursepathTest, str(image_name)), os.path.join(destpathTest, str(image_name)))
|
||||
|
||||
|
||||
Reference in new issue
Block a user