Subprocess improvements (#10973)

* Use list-form arguments for subprocess.run calls where possible

Augments #10944

* Deduplicate curl code

* Avoid eval() to parse integer

---------

Signed-off-by: Glenn Jocher <glenn.jocher@ultralytics.com>
Co-authored-by: Glenn Jocher <glenn.jocher@ultralytics.com>
This commit is contained in:
Aarni KoskelaandGlenn Jocher authored and GitHub committed 2023-02-13 18:00:31 +04:00
1 parent 1a2eb532ce
commit a2de5c5bf6
8 files changed
+68 -24

No files matched your search

+25 -4
View File
@@ -26,8 +26,10 @@ def is_url(url, check=True):
def gsutil_getsize(url=''):
# gs://bucket/file size https://cloud.google.com/storage/docs/gsutil/commands/du
s = subprocess.check_output(f'gsutil du {url}', shell=True).decode('utf-8')
return eval(s.split(' ')[0]) if len(s) else 0 # bytes
output = subprocess.check_output(['gsutil', 'du', url], shell=True, encoding='utf-8')
if output:
return int(output.split()[0])
return 0
def url_getsize(url='https://ultralytics.com/images/bus.jpg'):
@@ -36,6 +38,25 @@ def url_getsize(url='https://ultralytics.com/images/bus.jpg'):
return int(response.headers.get('content-length', -1))
def curl_download(url, filename, *, silent: bool = False) -> bool:
"""
Download a file from a url to a filename using curl.
"""
silent_option = 'sS' if silent else '' # silent
proc = subprocess.run([
'curl',
'-#',
f'-{silent_option}L',
url,
'--output',
filename,
'--retry',
'9',
'-C',
'-',])
return proc.returncode == 0
def safe_download(file, url, url2=None, min_bytes=1E0, error_msg=''):
# Attempts to download file from url or url2, checks and removes incomplete downloads < min_bytes
from utils.general import LOGGER
@@ -50,8 +71,8 @@ def safe_download(file, url, url2=None, min_bytes=1E0, error_msg=''):
if file.exists():
file.unlink() # remove partial downloads
LOGGER.info(f'ERROR: {e}\nRe-attempting {url2 or url} to {file}...')
subprocess.run(
f"curl -# -L '{url2 or url}' -o '{file}' --retry 3 -C -".split()) # curl download, retry and resume on fail
# curl download, retry and resume on fail
curl_download(url2 or url, file)
finally:
if not file.exists() or file.stat().st_size < min_bytes: # check
if file.exists():