Multi-threaded VisDrone and VOC downloads (#7108)
* Multi-threaded VOC download * Update VOC.yaml * Update * Update general.py * Update general.py
This commit is contained in:
1 parent
ecc2c7ba73
commit
c3ae4e4af6
7 files changed
+13
-6
No files matched your search
+7
-4
@@ -449,8 +449,9 @@ def check_dataset(data, autodownload=True):
|
||||
if val:
|
||||
val = [Path(x).resolve() for x in (val if isinstance(val, list) else [val])] # val path
|
||||
if not all(x.exists() for x in val):
|
||||
LOGGER.info('\nDataset not found, missing paths: %s' % [str(x) for x in val if not x.exists()])
|
||||
LOGGER.info(emojis('\nDataset not found ⚠️, missing paths %s' % [str(x) for x in val if not x.exists()]))
|
||||
if s and autodownload: # download script
|
||||
t = time.time()
|
||||
root = path.parent if 'path' in data else '..' # unzip directory i.e. '../'
|
||||
if s.startswith('http') and s.endswith('.zip'): # URL
|
||||
f = Path(s).name # filename
|
||||
@@ -465,9 +466,11 @@ def check_dataset(data, autodownload=True):
|
||||
r = os.system(s)
|
||||
else: # python script
|
||||
r = exec(s, {'yaml': data}) # return None
|
||||
LOGGER.info(f"Dataset autodownload {f'success, saved to {root}' if r in (0, None) else 'failure'}\n")
|
||||
dt = f'({round(time.time() - t, 1)}s)'
|
||||
s = f"success ✅ {dt}, saved to {colorstr('bold', root)}" if r in (0, None) else f"failure {dt} ❌"
|
||||
LOGGER.info(emojis(f"Dataset download {s}"))
|
||||
else:
|
||||
raise Exception('Dataset not found.')
|
||||
raise Exception(emojis('Dataset not found ❌'))
|
||||
|
||||
return data # dictionary
|
||||
|
||||
@@ -491,7 +494,7 @@ def download(url, dir='.', unzip=True, delete=True, curl=False, threads=1):
|
||||
if curl:
|
||||
os.system(f"curl -L '{url}' -o '{f}' --retry 9 -C -") # curl download, retry and resume on fail
|
||||
else:
|
||||
torch.hub.download_url_to_file(url, f, progress=True) # torch download
|
||||
torch.hub.download_url_to_file(url, f, progress=threads == 1) # torch download
|
||||
if unzip and f.suffix in ('.zip', '.gz'):
|
||||
LOGGER.info(f'Unzipping {f}...')
|
||||
if f.suffix == '.zip':
|
||||
|
||||
Reference in new issue
Block a user