diff --git a/tools/get.py b/tools/get.py index 10af3d14a9d..592af0d1bdf 100755 --- a/tools/get.py +++ b/tools/get.py @@ -22,6 +22,12 @@ import tarfile import zipfile import re +import time +from datetime import timedelta +import functools + +# Initialize start_time globally +start_time = -1 if sys.version_info[0] == 3: from urllib.request import urlretrieve @@ -58,51 +64,161 @@ def mkdir_p(path): if exc.errno != errno.EEXIST or not os.path.isdir(path): raise -def report_progress(count, blockSize, totalSize): +def format_time(seconds): + minutes, seconds = divmod(seconds, 60) + return "{:02}:{:05.2f}".format(int(minutes), seconds) + +def report_progress(block_count, block_size, total_size, start_time): + downloaded_size = block_count * block_size + time_elapsed = time.time() - start_time + current_speed = downloaded_size / (time_elapsed) + if sys.stdout.isatty(): - if totalSize > 0: - percent = int(count*blockSize*100/totalSize) - percent = min(100, percent) - sys.stdout.write("\r%d%%" % percent) + if total_size > 0: + percent_complete = min((downloaded_size / total_size) * 100, 100) + sys.stdout.write(f"\rDownloading... {percent_complete:.2f}% - {downloaded_size / 1024 / 1024:.2f} MB downloaded - Elapsed Time: {format_time(time_elapsed)} - Speed: {current_speed / 1024 / 1024:.2f} MB/s") else: - sofar = (count*blockSize) / 1024 - if sofar >= 1000: - sofar /= 1024 - sys.stdout.write("\r%dMB " % (sofar)) - else: - sys.stdout.write("\r%dKB" % (sofar)) + sys.stdout.write(f"\rDownloading... {downloaded_size / 1024 / 1024:.2f} MB downloaded - Elapsed Time: {format_time(time_elapsed)} - Speed: {current_speed / 1024 / 1024:.2f} MB/s") + sys.stdout.flush() + +def print_verification_progress(total_files, i, t1): + if sys.stdout.isatty(): + sys.stdout.write(f'\rElapsed time {format_time(time.time() - t1)}') sys.stdout.flush() -def unpack(filename, destination): +def verify_files(filename, destination, rename_to): + # Set the path of the extracted directory + extracted_dir_path = destination + t1 = time.time() + if filename.endswith(".zip"): + try: + with zipfile.ZipFile(filename, 'r') as archive: + first_dir = archive.namelist()[0].split('/')[0] + total_files = len(archive.namelist()) + for i, zipped_file in enumerate(archive.namelist(), 1): + local_path = os.path.join(extracted_dir_path, zipped_file.replace(first_dir, rename_to, 1)) + if not os.path.exists(local_path): + #print(f'\nMissing {zipped_file} on location: {extracted_dir_path}') + print(f"Verification failed; aborted in {format_time(time.time() - t1)}") + return False + print_verification_progress(total_files, i , t1) + except zipfile.BadZipFile: + print(f"Verification failed; aborted in {format_time(time.time() - t1)}") + return False + elif filename.endswith(".tar.gz"): + try: + with tarfile.open(filename, 'r:gz') as archive: + first_dir = archive.getnames()[0].split('/')[0] + total_files = len(archive.getnames()) + for i, zipped_file in enumerate(archive.getnames(), 1): + local_path = os.path.join(extracted_dir_path, zipped_file.replace(first_dir, rename_to, 1)) + if not os.path.exists(local_path): + #print(f'\nMissing {zipped_file} on location: {extracted_dir_path}') + print(f"Verification failed; aborted in {format_time(time.time() - t1)}") + return False + print_verification_progress(total_files, i , t1) + except tarfile.ReadError: + print(f"Verification failed; aborted in {format_time(time.time() - t1)}") + return False + elif filename.endswith(".tar.xz"): + try: + with tarfile.open(filename, 'r:xz') as archive: + first_dir = archive.getnames()[0].split('/')[0] + total_files = len(archive.getnames()) + for i, zipped_file in enumerate(archive.getnames(), 1): + local_path = os.path.join(extracted_dir_path, zipped_file.replace(first_dir, rename_to, 1)) + if not os.path.exists(local_path): + #print(f'\nMissing {zipped_file} on location: {extracted_dir_path}') + print(f"Verification failed; aborted in {format_time(time.time() - t1)}") + return False + print_verification_progress(total_files, i , t1) + except tarfile.ReadError: + print(f"Verification failed; aborted in {format_time(time.time() - t1)}") + return False + else: + raise NotImplementedError('Unsupported archive type') + + #if(verbose): + #print(f"\nVerification passed; completed in {format_time(time.time() - t1)}") + return True + + +def unpack(filename, destination, force_extract): dirname = '' - print('Extracting {0} ...'.format(os.path.basename(filename))) - sys.stdout.flush() + file_is_corrupted=False + if(not force_extract): + print(f' > Verify archive... ', end="", flush=True) + + try: + if filename.endswith('tar.gz'): + if tarfile.is_tarfile(filename): + tfile = tarfile.open(filename, 'r:gz') + dirname = tfile.getnames()[0] + else: + print('File corrupted!') + file_is_corrupted=True + elif filename.endswith('tar.xz'): + if tarfile.is_tarfile(filename): + tfile = tarfile.open(filename, 'r:xz') + dirname = tfile.getnames()[0] + else: + print('File corrupted!') + file_is_corrupted=True + elif filename.endswith('zip'): + if zipfile.is_zipfile(filename): + zfile = zipfile.ZipFile(filename) + dirname = zfile.namelist()[0] + else: + print('File corrupted!') + file_is_corrupted=True + else: + raise NotImplementedError('Unsupported archive type') + except EOFError: + print(f'File corrupted or incomplete!') + file_is_corrupted=True + + if(file_is_corrupted): + corrupted_filename = filename + ".corrupted" + os.rename(filename, corrupted_filename) + if(verbose): + print(f'Renaming corrupted archive to {corrupted_filename}') + return False + + # A little trick to rename tool directories so they don't contain version number + rename_to = re.match(r'^([a-z][^\-]*\-*)+', dirname).group(0).strip('-') + if rename_to == dirname and dirname.startswith('esp32-arduino-libs-'): + rename_to = 'esp32-arduino-libs' + + if not force_extract: + if(verify_files(filename, destination, rename_to)): + print(" Files ok. Skipping Extraction") + return True + else: + print(" Extracting archive...") + else: + print(" Forcing extraction") + if filename.endswith('tar.gz'): tfile = tarfile.open(filename, 'r:gz') tfile.extractall(destination) - dirname = tfile.getnames()[0] elif filename.endswith('tar.xz'): tfile = tarfile.open(filename, 'r:xz') tfile.extractall(destination) - dirname = tfile.getnames()[0] elif filename.endswith('zip'): zfile = zipfile.ZipFile(filename) zfile.extractall(destination) - dirname = zfile.namelist()[0] else: raise NotImplementedError('Unsupported archive type') - # a little trick to rename tool directories so they don't contain version number - rename_to = re.match(r'^([a-z][^\-]*\-*)+', dirname).group(0).strip('-') - if rename_to == dirname and dirname.startswith('esp32-arduino-libs-'): - rename_to = 'esp32-arduino-libs' if rename_to != dirname: print('Renaming {0} to {1} ...'.format(dirname, rename_to)) if os.path.isdir(rename_to): shutil.rmtree(rename_to) shutil.move(dirname, rename_to) -def download_file_with_progress(url,filename): + return True + +def download_file_with_progress(url,filename, start_time): import ssl import contextlib ctx = ssl.create_default_context() @@ -117,16 +233,16 @@ def download_file_with_progress(url,filename): with open(filename,'wb') as out_file: out_file.write(block) block_count += 1 - report_progress(block_count, block_size, total_size) + report_progress(block_count, block_size, total_size, start_time) while True: block = fp.read(block_size) if not block: break out_file.write(block) block_count += 1 - report_progress(block_count, block_size, total_size) + report_progress(block_count, block_size, total_size, start_time) else: - raise Exception ('nonexisting file or connection error') + raise Exception('Non-existing file or connection error') def download_file(url,filename): import ssl @@ -146,15 +262,19 @@ def download_file(url,filename): break out_file.write(block) else: - raise Exception ('nonexisting file or connection error') + raise Exception ('Non-existing file or connection error') -def get_tool(tool): +def get_tool(tool, force_download, force_extract): sys_name = platform.system() archive_name = tool['archiveFileName'] local_path = dist_dir + archive_name url = tool['url'] - if not os.path.isfile(local_path): - print('Downloading ' + archive_name + ' ...') + start_time = time.time() + if not os.path.isfile(local_path) or force_download: + if verbose: + print('Downloading \'' + archive_name + '\' to \'' + local_path + '\'') + else: + print('Downloading \'' + archive_name + '\' ...') sys.stdout.flush() if 'CYGWIN_NT' in sys_name: import ssl @@ -175,13 +295,13 @@ def get_tool(tool): try: urlretrieve(url, local_path, report_progress) except: - download_file_with_progress(url, local_path) - sys.stdout.write("\rDone \n") + download_file_with_progress(url, local_path, start_time) + sys.stdout.write(" - Done\n") sys.stdout.flush() else: print('Tool {0} already downloaded'.format(archive_name)) sys.stdout.flush() - unpack(local_path, '.') + return unpack(local_path, '.', force_extract) def load_tools_list(filename, platform): tools_info = json.load(open(filename))['packages'][0]['tools'] @@ -224,10 +344,34 @@ def identify_platform(): print('System: %s, Bits: %d, Info: %s' % (sys_name, bits, sys_platform)) return arduino_platform_names[sys_name][bits] +def print_help(): + print("TODO help") + if __name__ == '__main__': - is_test = (len(sys.argv) > 1 and sys.argv[1] == '-h') + option_print_help = "-h" in sys.argv + verbose = "-v" in sys.argv + force_download = "-d" in sys.argv + force_extract = "-e" in sys.argv + force_all = "-f" in sys.argv + is_test = "-t" in sys.argv + + print(f'Debug: options values: option_print_help={option_print_help}; verbose={verbose}; force_download={force_download}; force_extract={force_extract}; force_all={force_all}; is_test={is_test};') + if option_print_help: + print_help() + sys.exit() + + if is_test and (force_download or force_extract or force_all): + print('Cannot combine test (-t) and forced execution (-d | -e | -f)') + print_help() + sys.exit() + if is_test: print('Test run!') + + if force_all: + force_download = True + force_extract = True + identified_platform = identify_platform() print('Platform: {0}'.format(identified_platform)) tools_to_download = load_tools_list(current_dir + '/../package/package_esp32_index.template.json', identified_platform) @@ -236,5 +380,9 @@ def identify_platform(): if is_test: print('Would install: {0}'.format(tool['archiveFileName'])) else: - get_tool(tool) + if(not get_tool(tool, force_download, force_extract)): + if(verbose): + print(f"Tool {tool['archiveFileName']} was corrupted. Re-downloading...\n") + if(not get_tool(tool, True, force_extract)): # Corrupted file was renamed, will not be found and will be re-downloaded + print(f"Tool {tool['archiveFileName']} was corrupted, but re-downloading did not help!\n") print('Platform Tools Installed')