From bc1fd13a7a7730dfe7ac6287eb9eb0d2eae70f19 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Thu, 3 Sep 2020 20:29:08 -0700 Subject: [PATCH] gsutil cp hyp evolution bug fix (#876) --- utils/general.py | 5 ++++- utils/google_utils.py | 7 +++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/utils/general.py b/utils/general.py index 686e682..2cbe517 100755 --- a/utils/general.py +++ b/utils/general.py @@ -22,6 +22,7 @@ from scipy.cluster.vq import kmeans from scipy.signal import butter, filtfilt from tqdm import tqdm +from utils.google_utils import gsutil_getsize from utils.torch_utils import init_seeds as init_torch_seeds from utils.torch_utils import is_parallel @@ -854,7 +855,9 @@ def print_mutation(hyp, results, yaml_file='hyp_evolved.yaml', bucket=''): print('\n%s\n%s\nEvolved fitness: %s\n' % (a, b, c)) if bucket: - os.system('gsutil cp gs://%s/evolve.txt .' % bucket) # download evolve.txt + url = 'gs://%s/evolve.txt' % bucket + if gsutil_getsize(url) > (os.path.getsize('evolve.txt') if os.path.exists('evolve.txt') else 0): + os.system('gsutil cp %s .' % url) # download evolve.txt if larger than local with open('evolve.txt', 'a') as f: # append result f.write(c + b + '\n') diff --git a/utils/google_utils.py b/utils/google_utils.py index 6012f14..1630ab7 100644 --- a/utils/google_utils.py +++ b/utils/google_utils.py @@ -4,12 +4,19 @@ import os import platform +import subprocess import time from pathlib import Path import torch +def gsutil_getsize(url=''): + # gs://bucket/file size https://cloud.google.com/storage/docs/gsutil/commands/du + s = subprocess.check_output('gsutil du %s' % url, shell=True).decode('utf-8') + return eval(s.split(' ')[0]) if len(s) else 0 # bytes + + def attempt_download(weights): # Attempt to download pretrained weights if not found locally weights = weights.strip().replace("'", '')