Subprocess improvements (#10973)
* Use list-form arguments for subprocess.run calls where possible Augments #10944 * Deduplicate curl code * Avoid eval() to parse integer --------- Signed-off-by: Glenn Jocher <glenn.jocher@ultralytics.com> Co-authored-by: Glenn Jocher <glenn.jocher@ultralytics.com>
This commit is contained in:
1 parent
1a2eb532ce
commit
a2de5c5bf6
8 files changed
+68
-24
No files matched your search
+25
-4
@@ -26,8 +26,10 @@ def is_url(url, check=True):
|
||||
|
||||
def gsutil_getsize(url=''):
|
||||
# gs://bucket/file size https://cloud.google.com/storage/docs/gsutil/commands/du
|
||||
s = subprocess.check_output(f'gsutil du {url}', shell=True).decode('utf-8')
|
||||
return eval(s.split(' ')[0]) if len(s) else 0 # bytes
|
||||
output = subprocess.check_output(['gsutil', 'du', url], shell=True, encoding='utf-8')
|
||||
if output:
|
||||
return int(output.split()[0])
|
||||
return 0
|
||||
|
||||
|
||||
def url_getsize(url='https://ultralytics.com/images/bus.jpg'):
|
||||
@@ -36,6 +38,25 @@ def url_getsize(url='https://ultralytics.com/images/bus.jpg'):
|
||||
return int(response.headers.get('content-length', -1))
|
||||
|
||||
|
||||
def curl_download(url, filename, *, silent: bool = False) -> bool:
|
||||
"""
|
||||
Download a file from a url to a filename using curl.
|
||||
"""
|
||||
silent_option = 'sS' if silent else '' # silent
|
||||
proc = subprocess.run([
|
||||
'curl',
|
||||
'-#',
|
||||
f'-{silent_option}L',
|
||||
url,
|
||||
'--output',
|
||||
filename,
|
||||
'--retry',
|
||||
'9',
|
||||
'-C',
|
||||
'-',])
|
||||
return proc.returncode == 0
|
||||
|
||||
|
||||
def safe_download(file, url, url2=None, min_bytes=1E0, error_msg=''):
|
||||
# Attempts to download file from url or url2, checks and removes incomplete downloads < min_bytes
|
||||
from utils.general import LOGGER
|
||||
@@ -50,8 +71,8 @@ def safe_download(file, url, url2=None, min_bytes=1E0, error_msg=''):
|
||||
if file.exists():
|
||||
file.unlink() # remove partial downloads
|
||||
LOGGER.info(f'ERROR: {e}\nRe-attempting {url2 or url} to {file}...')
|
||||
subprocess.run(
|
||||
f"curl -# -L '{url2 or url}' -o '{file}' --retry 3 -C -".split()) # curl download, retry and resume on fail
|
||||
# curl download, retry and resume on fail
|
||||
curl_download(url2 or url, file)
|
||||
finally:
|
||||
if not file.exists() or file.stat().st_size < min_bytes: # check
|
||||
if file.exists():
|
||||
|
||||
Reference in new issue
Block a user