download() ThreadPool update (#3027)

* download() ThreadPool update

* update train image count

* cid + 1
This commit is contained in:
Glenn Jocher authored and GitHub committed 2021-05-04 23:37:31 +02:00
1 parent 5189b3addb
commit 8cab907f60
2 files changed
+12 -4

No files matched your search

+4 -1
View File
@@ -217,7 +217,10 @@ def download(url, dir='.', unzip=True, delete=True, curl=False, threads=1):
dir = Path(dir)
dir.mkdir(parents=True, exist_ok=True) # make directory
if threads > 1:
ThreadPool(threads).imap(lambda x: download_one(*x), zip(url, repeat(dir))) # multi-threaded
pool = ThreadPool(threads)
pool.imap(lambda x: download_one(*x), zip(url, repeat(dir))) # multi-threaded
pool.close()
pool.join()
else:
for u in tuple(url) if isinstance(url, str) else url:
download_one(u, dir)