Multi-threaded VisDrone and VOC downloads (#7108)

* Multi-threaded VOC download

* Update VOC.yaml

* Update

* Update general.py

* Update general.py
This commit is contained in:
Glenn Jocher authored and GitHub committed 2022-03-23 01:19:37 +01:00
1 parent ecc2c7ba73
commit c3ae4e4af6
7 files changed
+13 -6

No files matched your search

+7 -4
View File
@@ -449,8 +449,9 @@ def check_dataset(data, autodownload=True):
if val:
val = [Path(x).resolve() for x in (val if isinstance(val, list) else [val])] # val path
if not all(x.exists() for x in val):
LOGGER.info('\nDataset not found, missing paths: %s' % [str(x) for x in val if not x.exists()])
LOGGER.info(emojis('\nDataset not found ⚠️, missing paths %s' % [str(x) for x in val if not x.exists()]))
if s and autodownload: # download script
t = time.time()
root = path.parent if 'path' in data else '..' # unzip directory i.e. '../'
if s.startswith('http') and s.endswith('.zip'): # URL
f = Path(s).name # filename
@@ -465,9 +466,11 @@ def check_dataset(data, autodownload=True):
r = os.system(s)
else: # python script
r = exec(s, {'yaml': data}) # return None
LOGGER.info(f"Dataset autodownload {f'success, saved to {root}' if r in (0, None) else 'failure'}\n")
dt = f'({round(time.time() - t, 1)}s)'
s = f"success ✅ {dt}, saved to {colorstr('bold', root)}" if r in (0, None) else f"failure {dt} ❌"
LOGGER.info(emojis(f"Dataset download {s}"))
else:
raise Exception('Dataset not found.')
raise Exception(emojis('Dataset not found ❌'))
return data # dictionary
@@ -491,7 +494,7 @@ def download(url, dir='.', unzip=True, delete=True, curl=False, threads=1):
if curl:
os.system(f"curl -L '{url}' -o '{f}' --retry 9 -C -") # curl download, retry and resume on fail
else:
torch.hub.download_url_to_file(url, f, progress=True) # torch download
torch.hub.download_url_to_file(url, f, progress=threads == 1) # torch download
if unzip and f.suffix in ('.zip', '.gz'):
LOGGER.info(f'Unzipping {f}...')
if f.suffix == '.zip':