From 9034f087568d2c291b168c7fa19be0c376f882ef Mon Sep 17 00:00:00 2001 From: Jonas Bardino Date: Thu, 13 Feb 2025 09:53:12 +0100 Subject: [PATCH 1/5] Move addheader to ./bin/ with various refactoring and lint style fixes. * Change constants to `UPPER_CASE` as suggested by linters and PEP8 * Wrap action in main function for better isolation * Switch to fileio helpers instead of raw open+read/write+close * Automatic python module load path lookup from cmd parent path if insuffient * Simplify some `if X: return True` constructs as suggested by linters * Normalize path patterns to actually make the ../X ones work Adjust the addition of shebang interpreter and encoding lines to actually do what the label says. Add new bin, sbin and mig/lib dirs to projcode paths. --- addheader.py => bin/addheader.py | 137 +++++++++++++++++-------------- mig/shared/projcode.py | 5 +- 2 files changed, 80 insertions(+), 62 deletions(-) rename addheader.py => bin/addheader.py (59%) diff --git a/addheader.py b/bin/addheader.py similarity index 59% rename from addheader.py rename to bin/addheader.py index 38e30f8da..ccaac53f1 100755 --- a/addheader.py +++ b/bin/addheader.py @@ -4,7 +4,7 @@ # --- BEGIN_HEADER --- # # addheader - add license header to all code modules. -# Copyright (C) 2009-2024 The MiG Project by the Science HPC Center at UCPH +# Copyright (C) 2009-2025 The MiG Project by the Science HPC Center at UCPH # # This file is part of MiG. # @@ -26,7 +26,8 @@ # --- END_HEADER --- # -"""Search code tree and add the required header to all python modules""" +"""Search code tree and add the required header to all python modules.""" + from __future__ import print_function from __future__ import absolute_import @@ -35,25 +36,39 @@ import os import sys +# Try to import mig to assure we have a suitable python module load path +try: + import mig +except ImportError: + mig = None # type: ignore[assignment] + +if mig is None: + # NOTE: include cmd parent path in search path for mig.X imports to work + MIG_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0]))) + print("Using mig installation in %s" % MIG_ROOT) + sys.path.append(MIG_ROOT) + +from mig.shared.fileio import read_head_lines, read_file_lines, write_file_lines from mig.shared.projcode import code_root, py_code_files, sh_code_files, \ js_code_files # Modify these to fit actual project -proj_vars = {} -proj_vars['project_name'] = "MiG" -proj_vars['authors'] = 'The MiG Project by the Science HPC Center at UCPH' +PROJ_CONSTS = {} +PROJ_CONSTS['project_name'] = "MiG" +PROJ_CONSTS['authors'] = 'The MiG Project by the Science HPC Center at UCPH' -proj_vars['copyright_year'] = '2003-%d' % datetime.date.today().year +PROJ_CONSTS['copyright_year'] = '2003-%d' % datetime.date.today().year # Set interpreter path and file encoding if not already set in source files # Use empty string to leave them alone. -proj_vars['interpreter_path'] = '/usr/bin/python' -proj_vars['module_encoding'] = 'utf-8' +PROJ_CONSTS['interpreter_path'] = '/usr/bin/env python' +PROJ_CONSTS['module_encoding'] = 'utf-8' -begin_marker, end_marker = "--- BEGIN_HEADER ---", "--- END_HEADER ---" +BEGIN_MARKER, END_MARKER = "--- BEGIN_HEADER ---", "--- END_HEADER ---" +BACKUP_MARKER = ".unlicensed" # Mandatory copyright notice for any license -license_text = """# +LICENSE_TEXT = """# # %(module_name)s - [optionally add short module description on this line] # Copyright (C) %(copyright_year)s %(authors)s """ @@ -65,7 +80,7 @@ # with a verbatim copy of the license text: # wget -O COPYING http://www.gnu.org/licenses/gpl2.txt -license_text += """# +LICENSE_TEXT += """# # This file is part of %(project_name)s. # # %(project_name)s is free software: you can redistribute it and/or modify @@ -84,18 +99,13 @@ # USA.""" -def check_header(path, var_dict, preamble_size=4096): +def check_header(path, var_dict, preamble_lines=100): """Check if path already has a credible license header. Only looks inside the first preamble_size bytes of the file. """ - module_fd = open(path, 'r') - module_preamble = module_fd.read(4096) - module_fd.close() - if begin_marker in module_preamble or \ - proj_vars['authors'] in module_preamble: - return True - else: - return False + module_preamble = '\n'.join(read_head_lines(path, preamble_lines, None)) + return (BEGIN_MARKER in module_preamble or + var_dict['authors'] in module_preamble) def add_header(path, var_dict, explicit_border=True, block_wrap=False): @@ -109,33 +119,29 @@ def add_header(path, var_dict, explicit_border=True, block_wrap=False): Creates a '.unlicensed' backup copy of each file changed. """ - module_fd = open(path, 'r') - module_lines = module_fd.readlines() - module_fd.close() - backup_fd = open(path + '.unlicensed', 'w') - backup_fd.writelines(module_lines) - backup_fd.close() - # Do not truncate any existing unix executable hint and encoding - act = '#!%(interpreter_path)s' % var_dict + module_lines = read_file_lines(path, None) + if not write_file_lines(module_lines, path + BACKUP_MARKER, None): + print("Failed to create backup of %s - skip!" % path) + return False + # Do not truncate any existing unix executable hint (shebang) and encoding + act = '#!%(interpreter_path)s\n' % var_dict if block_wrap: enc = '' else: enc = '# -*- coding: %(module_encoding)s -*-' % var_dict - lic = license_text % var_dict + lic = LICENSE_TEXT % var_dict module_header = [] - if var_dict['interpreter_path']: + if module_lines and module_lines[0].startswith("#!"): + module_header.append(module_lines[0]) + module_lines = module_lines[1:] + elif var_dict['interpreter_path']: module_header.append(act) - if module_lines and module_lines[0].startswith("#!"): - module_lines = module_lines[1:] - else: - if module_lines and module_lines[0].startswith("#!"): - module_header.append(module_lines[0].strip()) - module_lines = module_lines[1:] - if var_dict['module_encoding']: + if module_lines and module_lines[0].startswith("# -*- coding"): + module_header.append(module_lines[0]) + module_lines = module_lines[1:] + elif var_dict['module_encoding']: module_header.append(enc) - if module_lines and module_lines[0].startswith("# -*- coding"): - module_lines = module_lines[1:] if explicit_border: lic = """ @@ -146,7 +152,7 @@ def add_header(path, var_dict, explicit_border=True, block_wrap=False): # # %s # -""" % (begin_marker, lic, end_marker) +""" % (BEGIN_MARKER, lic, END_MARKER) if block_wrap: lic = """ /* @@ -155,23 +161,26 @@ def add_header(path, var_dict, explicit_border=True, block_wrap=False): """ % lic module_header.append(lic) - module_text = '\n'.join(module_header) % var_dict + '\n'\ - + ''.join(module_lines) - module_fd = open(path, 'w') - module_fd.write(module_text) - module_fd.close() + updated_lines = [i % var_dict for i in module_header + [''] + module_lines] + + if not write_file_lines(updated_lines, path, None): + print("Failed to write %s with added headers!" % path) + return False + #print("DEBUG: wrote %s with added headers!" % path) + return True -if __name__ == '__main__': +def main(argv): + """Run header addition for given argv""" target = os.getcwd() - if len(sys.argv) > 1: - target = os.path.abspath(sys.argv[1]) + if len(argv) > 1: + target = os.path.abspath(argv[1]) mig_code_base = target - if len(sys.argv) > 2: - mig_code_base = os.path.abspath(sys.argv[2]) + if len(argv) > 2: + mig_code_base = os.path.abspath(argv[2]) - for (root, dirs, files) in os.walk(target): + for (root, _, files) in os.walk(target): # skip all dot dirs - they are from repos etc and _not_ jobs @@ -181,27 +190,33 @@ def add_header(path, var_dict, explicit_border=True, block_wrap=False): src_path = os.path.join(root, name) if os.path.islink(src_path): continue + if src_path.endswith(BACKUP_MARKER): + continue print('Inspecting %s' % src_path) for pattern in py_code_files + sh_code_files + js_code_files: - if pattern in js_code_files: - needs_block = True - else: - needs_block = False - - pattern = os.path.join(mig_code_base, code_root, pattern) + needs_block = (pattern in js_code_files) + pattern = os.path.normpath(os.path.join( + mig_code_base, code_root, pattern)) - # print "Testing %s against %s" % (src_path, pattern) + #print("DEBUG: Testing %s against %s" % (src_path, pattern)) if src_path == pattern or fnmatch.fnmatch(src_path, pattern): print('Matched %s against %s' % (src_path, pattern)) - proj_vars['module_name'] = name.replace('.py', '') - if check_header(src_path, proj_vars): + PROJ_CONSTS['module_name'] = name.replace('.py', '') + if check_header(src_path, PROJ_CONSTS): print('Skip %s with existing header' % src_path) continue - add_header(src_path, proj_vars, block_wrap=needs_block) + add_header(src_path, PROJ_CONSTS, block_wrap=needs_block) + # else: + # print('DEBUG: %s does not match %s' % (src_path, pattern)) + print() print("Added license headers to code in %s" % target) print() print("Don't forget to include COPYING file in root of source, e.g. run:") print("wget -O COPYING http://www.gnu.org/licenses/gpl2.txt") print("if using the default GPL v2 license here.") + + +if __name__ == '__main__': + main(sys.argv) diff --git a/mig/shared/projcode.py b/mig/shared/projcode.py index f2159db43..4095c1825 100644 --- a/mig/shared/projcode.py +++ b/mig/shared/projcode.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- # # projcode - simple helpers for transforming or searching project code -# Copyright (C) 2003-2021 The MiG Project lead by Brian Vinter +# Copyright (C) 2009-2025 The MiG Project by the Science HPC Center at UCPH # # This file is part of MiG. # @@ -31,7 +31,10 @@ py_code_files = [ # a few scripts are in parent dir of code_root '../%s' % plain, + '../bin/%s' % plain, + '../sbin/%s' % plain, '%s' % plain, + 'lib/%s' % plain, 'cgi-bin/%s' % plain, 'cgi-sid/%s' % plain, 'install/%s' % plain, From bfc85c6b57d3de582ecc707e9e074c8c96ed60fe Mon Sep 17 00:00:00 2001 From: Jonas Bardino Date: Thu, 13 Feb 2025 10:13:40 +0100 Subject: [PATCH 2/5] Reformat with `black` and fix import order with `isort`. Rewrapped a bit and changed strings to use double quotes. Applied `isort` last to revert from the `black` import mangling and preserve our usual hanging indent imports. --- bin/addheader.py | 78 +++++++++++++++++++++++++++--------------------- 1 file changed, 44 insertions(+), 34 deletions(-) diff --git a/bin/addheader.py b/bin/addheader.py index ccaac53f1..eee033280 100755 --- a/bin/addheader.py +++ b/bin/addheader.py @@ -28,8 +28,7 @@ """Search code tree and add the required header to all python modules.""" -from __future__ import print_function -from __future__ import absolute_import +from __future__ import absolute_import, print_function import datetime import fnmatch @@ -48,21 +47,22 @@ print("Using mig installation in %s" % MIG_ROOT) sys.path.append(MIG_ROOT) -from mig.shared.fileio import read_head_lines, read_file_lines, write_file_lines -from mig.shared.projcode import code_root, py_code_files, sh_code_files, \ - js_code_files +from mig.shared.fileio import read_file_lines, read_head_lines, \ + write_file_lines +from mig.shared.projcode import code_root, js_code_files, py_code_files, \ + sh_code_files # Modify these to fit actual project PROJ_CONSTS = {} -PROJ_CONSTS['project_name'] = "MiG" -PROJ_CONSTS['authors'] = 'The MiG Project by the Science HPC Center at UCPH' +PROJ_CONSTS["project_name"] = "MiG" +PROJ_CONSTS["authors"] = "The MiG Project by the Science HPC Center at UCPH" -PROJ_CONSTS['copyright_year'] = '2003-%d' % datetime.date.today().year +PROJ_CONSTS["copyright_year"] = "2003-%d" % datetime.date.today().year # Set interpreter path and file encoding if not already set in source files # Use empty string to leave them alone. -PROJ_CONSTS['interpreter_path'] = '/usr/bin/env python' -PROJ_CONSTS['module_encoding'] = 'utf-8' +PROJ_CONSTS["interpreter_path"] = "/usr/bin/env python" +PROJ_CONSTS["module_encoding"] = "utf-8" BEGIN_MARKER, END_MARKER = "--- BEGIN_HEADER ---", "--- END_HEADER ---" BACKUP_MARKER = ".unlicensed" @@ -103,9 +103,11 @@ def check_header(path, var_dict, preamble_lines=100): """Check if path already has a credible license header. Only looks inside the first preamble_size bytes of the file. """ - module_preamble = '\n'.join(read_head_lines(path, preamble_lines, None)) - return (BEGIN_MARKER in module_preamble or - var_dict['authors'] in module_preamble) + module_preamble = "\n".join(read_head_lines(path, preamble_lines, None)) + return ( + BEGIN_MARKER in module_preamble + or var_dict["authors"] in module_preamble + ) def add_header(path, var_dict, explicit_border=True, block_wrap=False): @@ -124,23 +126,23 @@ def add_header(path, var_dict, explicit_border=True, block_wrap=False): print("Failed to create backup of %s - skip!" % path) return False # Do not truncate any existing unix executable hint (shebang) and encoding - act = '#!%(interpreter_path)s\n' % var_dict + act = "#!%(interpreter_path)s\n" % var_dict if block_wrap: - enc = '' + enc = "" else: - enc = '# -*- coding: %(module_encoding)s -*-' % var_dict + enc = "# -*- coding: %(module_encoding)s -*-" % var_dict lic = LICENSE_TEXT % var_dict module_header = [] if module_lines and module_lines[0].startswith("#!"): module_header.append(module_lines[0]) module_lines = module_lines[1:] - elif var_dict['interpreter_path']: + elif var_dict["interpreter_path"]: module_header.append(act) if module_lines and module_lines[0].startswith("# -*- coding"): module_header.append(module_lines[0]) module_lines = module_lines[1:] - elif var_dict['module_encoding']: + elif var_dict["module_encoding"]: module_header.append(enc) if explicit_border: @@ -152,22 +154,29 @@ def add_header(path, var_dict, explicit_border=True, block_wrap=False): # # %s # -""" % (BEGIN_MARKER, lic, END_MARKER) +""" % ( + BEGIN_MARKER, + lic, + END_MARKER, + ) if block_wrap: - lic = """ + lic = ( + """ /* %s */ -""" % lic +""" + % lic + ) module_header.append(lic) - updated_lines = [i % var_dict for i in module_header + [''] + module_lines] + updated_lines = [i % var_dict for i in module_header + [""] + module_lines] if not write_file_lines(updated_lines, path, None): print("Failed to write %s with added headers!" % path) return False - #print("DEBUG: wrote %s with added headers!" % path) + # print("DEBUG: wrote %s with added headers!" % path) return True @@ -180,11 +189,11 @@ def main(argv): if len(argv) > 2: mig_code_base = os.path.abspath(argv[2]) - for (root, _, files) in os.walk(target): + for root, _, files in os.walk(target): # skip all dot dirs - they are from repos etc and _not_ jobs - if root.find(os.sep + '.') != -1: + if root.find(os.sep + ".") != -1: continue for name in files: src_path = os.path.join(root, name) @@ -192,19 +201,20 @@ def main(argv): continue if src_path.endswith(BACKUP_MARKER): continue - print('Inspecting %s' % src_path) + print("Inspecting %s" % src_path) for pattern in py_code_files + sh_code_files + js_code_files: - needs_block = (pattern in js_code_files) - pattern = os.path.normpath(os.path.join( - mig_code_base, code_root, pattern)) + needs_block = pattern in js_code_files + pattern = os.path.normpath( + os.path.join(mig_code_base, code_root, pattern) + ) - #print("DEBUG: Testing %s against %s" % (src_path, pattern)) + # print("DEBUG: Testing %s against %s" % (src_path, pattern)) if src_path == pattern or fnmatch.fnmatch(src_path, pattern): - print('Matched %s against %s' % (src_path, pattern)) - PROJ_CONSTS['module_name'] = name.replace('.py', '') + print("Matched %s against %s" % (src_path, pattern)) + PROJ_CONSTS["module_name"] = name.replace(".py", "") if check_header(src_path, PROJ_CONSTS): - print('Skip %s with existing header' % src_path) + print("Skip %s with existing header" % src_path) continue add_header(src_path, PROJ_CONSTS, block_wrap=needs_block) # else: @@ -218,5 +228,5 @@ def main(argv): print("if using the default GPL v2 license here.") -if __name__ == '__main__': +if __name__ == "__main__": main(sys.argv) From dc09a93db51a0af8d11d4f39fb514e658cbd8060 Mon Sep 17 00:00:00 2001 From: Jonas Bardino Date: Thu, 13 Feb 2025 11:11:20 +0100 Subject: [PATCH 3/5] Refactor projcode to mainly expose a list_code_files helper function which optionally takes a list of languages to filter on or all by default. Make sure that there's at least a single blank line between license header and actual module content. Minor adjustments to fix style check warnings. --- bin/addheader.py | 21 ++++++------ mig/shared/projcode.py | 77 ++++++++++++++++++++++++++++-------------- 2 files changed, 62 insertions(+), 36 deletions(-) diff --git a/bin/addheader.py b/bin/addheader.py index eee033280..ab731ad0a 100755 --- a/bin/addheader.py +++ b/bin/addheader.py @@ -47,10 +47,8 @@ print("Using mig installation in %s" % MIG_ROOT) sys.path.append(MIG_ROOT) -from mig.shared.fileio import read_file_lines, read_head_lines, \ - write_file_lines -from mig.shared.projcode import code_root, js_code_files, py_code_files, \ - sh_code_files +from mig.shared.fileio import read_file_lines, read_head_lines, write_file_lines +from mig.shared.projcode import CODE_ROOT, JAVASCRIPT, list_code_files # Modify these to fit actual project PROJ_CONSTS = {} @@ -112,6 +110,7 @@ def check_header(path, var_dict, preamble_lines=100): def add_header(path, var_dict, explicit_border=True, block_wrap=False): """Add the required copyright and license header to module in path. + The optional explicit_border argument can be set to wrap the license text in begin and end lines that are easy to find, so that license can be updated or replaced later. @@ -120,7 +119,6 @@ def add_header(path, var_dict, explicit_border=True, block_wrap=False): JavaScript where the per-line commenting using hash (#) won't work. Creates a '.unlicensed' backup copy of each file changed. """ - module_lines = read_file_lines(path, None) if not write_file_lines(module_lines, path + BACKUP_MARKER, None): print("Failed to create backup of %s - skip!" % path) @@ -170,8 +168,11 @@ def add_header(path, var_dict, explicit_border=True, block_wrap=False): ) module_header.append(lic) + # Make sure there's a blank line between license header and code + if module_lines and module_lines[0].strip(): + module_header.append("\n") - updated_lines = [i % var_dict for i in module_header + [""] + module_lines] + updated_lines = [i % var_dict for i in module_header + module_lines] if not write_file_lines(updated_lines, path, None): print("Failed to write %s with added headers!" % path) @@ -181,7 +182,7 @@ def add_header(path, var_dict, explicit_border=True, block_wrap=False): def main(argv): - """Run header addition for given argv""" + """Run header addition for given argv.""" target = os.getcwd() if len(argv) > 1: target = os.path.abspath(argv[1]) @@ -202,10 +203,10 @@ def main(argv): if src_path.endswith(BACKUP_MARKER): continue print("Inspecting %s" % src_path) - for pattern in py_code_files + sh_code_files + js_code_files: - needs_block = pattern in js_code_files + for pattern in list_code_files(): + needs_block = pattern in list_code_files(JAVASCRIPT) pattern = os.path.normpath( - os.path.join(mig_code_base, code_root, pattern) + os.path.join(mig_code_base, CODE_ROOT, pattern) ) # print("DEBUG: Testing %s against %s" % (src_path, pattern)) diff --git a/mig/shared/projcode.py b/mig/shared/projcode.py index 4095c1825..89627f944 100644 --- a/mig/shared/projcode.py +++ b/mig/shared/projcode.py @@ -21,38 +21,42 @@ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA # 02110-1301, USA. -"""Helpers to transform or search project code files""" +"""Helpers to transform or search project code files.""" + +from mig.shared.defaults import keyword_all + +# TODO: phase out lowercase names once all scripts switched to get_code_files # Top dir with all code -code_root = 'mig' +CODE_ROOT = code_root = 'mig' # Ignore backup and dot files in wild card match -plain = '[a-zA-Z0-9]*.py' +PLAIN = '[a-zA-Z0-9]*.py' py_code_files = [ # a few scripts are in parent dir of code_root - '../%s' % plain, - '../bin/%s' % plain, - '../sbin/%s' % plain, - '%s' % plain, - 'lib/%s' % plain, - 'cgi-bin/%s' % plain, - 'cgi-sid/%s' % plain, - 'install/%s' % plain, - 'migfs-fuse/%s' % plain, - 'resource/bin/%s' % plain, - 'resource/image-scripts/%s' % plain, - 'resource/keepalive-scripts/%s' % plain, - 'server/%s' % plain, - 'shared/%s' % plain, - 'shared/functionality/%s' % plain, - 'shared/distos/%s' % plain, - 'shared/gdp/%s' % plain, - 'shared/griddaemons/%s' % plain, - 'simulation/%s' % plain, - 'user/%s' % plain, - 'vm-proxy/%s' % plain, - 'webserver/%s' % plain, - 'wsgi-bin/%s' % plain, + '../%s' % PLAIN, + '../bin/%s' % PLAIN, + '../sbin/%s' % PLAIN, + '%s' % PLAIN, + 'lib/%s' % PLAIN, + 'cgi-bin/%s' % PLAIN, + 'cgi-sid/%s' % PLAIN, + 'install/%s' % PLAIN, + 'migfs-fuse/%s' % PLAIN, + 'resource/bin/%s' % PLAIN, + 'resource/image-scripts/%s' % PLAIN, + 'resource/keepalive-scripts/%s' % PLAIN, + 'server/%s' % PLAIN, + 'shared/%s' % PLAIN, + 'shared/functionality/%s' % PLAIN, + 'shared/distos/%s' % PLAIN, + 'shared/gdp/%s' % PLAIN, + 'shared/griddaemons/%s' % PLAIN, + 'simulation/%s' % PLAIN, + 'user/%s' % PLAIN, + 'vm-proxy/%s' % PLAIN, + 'webserver/%s' % PLAIN, + 'wsgi-bin/%s' % PLAIN, ] py_code_files += ['cgi-sid/%s' % name for name in ['requestnewjob', 'putrespgid']] @@ -68,12 +72,16 @@ 'walk', 'getrespgid', ]] +PY_CODE_FILES = py_code_files + sh_code_files = [ 'resource/frontend_script.sh', 'resource/master_node_script.sh', 'resource/leader_node_script.sh', 'resource/dummy_node_script.sh', ] +SH_CODE_FILES = sh_code_files + js_code_files = [ 'images/js/jquery.accountform.js', 'images/js/jquery.ajaxhelpers.js', @@ -89,4 +97,21 @@ 'assets/js/V3/ui-global.js', 'assets/js/V3/ui-extra.js', ] +JS_CODE_FILES = js_code_files + code_files = py_code_files + sh_code_files + js_code_files +CODE_FILES = code_files + +PYTHON, SHELL, JAVASCRIPT = "PYTHON", "SHELL", "JAVASCRIPT" +LANG_MAP = {keyword_all: CODE_FILES, PYTHON: PY_CODE_FILES, + JAVASCRIPT: JS_CODE_FILES, SHELL: SH_CODE_FILES} + + +def list_code_files(code_langs=[keyword_all]): + """Get list of all code files.""" + match = [] + for lang in code_langs: + if not lang in code_langs: + print("Warning: no such code lang: %s" % lang) + match += LANG_MAP.get(lang, []) + return match From 741c03158f7e3a4c0e2ccbe8f4991d2dbeb4c869 Mon Sep 17 00:00:00 2001 From: Jonas Bardino Date: Thu, 13 Feb 2025 11:17:13 +0100 Subject: [PATCH 4/5] Really request JAVASCRIPT as lang (requires list). Fix language lookup and skip append if missing. --- bin/addheader.py | 2 +- mig/shared/projcode.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/bin/addheader.py b/bin/addheader.py index ab731ad0a..8c1d380a6 100755 --- a/bin/addheader.py +++ b/bin/addheader.py @@ -204,7 +204,7 @@ def main(argv): continue print("Inspecting %s" % src_path) for pattern in list_code_files(): - needs_block = pattern in list_code_files(JAVASCRIPT) + needs_block = pattern in list_code_files([JAVASCRIPT]) pattern = os.path.normpath( os.path.join(mig_code_base, CODE_ROOT, pattern) ) diff --git a/mig/shared/projcode.py b/mig/shared/projcode.py index 89627f944..6090086fa 100644 --- a/mig/shared/projcode.py +++ b/mig/shared/projcode.py @@ -111,7 +111,8 @@ def list_code_files(code_langs=[keyword_all]): """Get list of all code files.""" match = [] for lang in code_langs: - if not lang in code_langs: + if not lang in LANG_MAP: print("Warning: no such code lang: %s" % lang) - match += LANG_MAP.get(lang, []) + else: + match += LANG_MAP[lang] return match From 9f9edb48ecfbeacb12be82ee5bfeb232109b4e8a Mon Sep 17 00:00:00 2001 From: Jonas Bardino Date: Thu, 13 Feb 2025 11:34:51 +0100 Subject: [PATCH 5/5] Let black and ruff reformat and fix projcode. Minor style adjustments and comment updates. --- bin/addheader.py | 7 ++- mig/shared/projcode.py | 127 ++++++++++++++++++++++------------------- 2 files changed, 73 insertions(+), 61 deletions(-) diff --git a/bin/addheader.py b/bin/addheader.py index 8c1d380a6..0376d048e 100755 --- a/bin/addheader.py +++ b/bin/addheader.py @@ -98,8 +98,11 @@ def check_header(path, var_dict, preamble_lines=100): - """Check if path already has a credible license header. Only looks inside - the first preamble_size bytes of the file. + """Check if path has a credible license header and otherwise adds one. + + Only looks inside the first preamble_lines of the file and if it doesn't + find an existing license header there it adds a standard header populated + with project variables from var_dict. """ module_preamble = "\n".join(read_head_lines(path, preamble_lines, None)) return ( diff --git a/mig/shared/projcode.py b/mig/shared/projcode.py index 6090086fa..2b8b06166 100644 --- a/mig/shared/projcode.py +++ b/mig/shared/projcode.py @@ -25,77 +25,82 @@ from mig.shared.defaults import keyword_all -# TODO: phase out lowercase names once all scripts switched to get_code_files +# TODO: phase out lowercase names once all scripts switched to list_code_files # Top dir with all code -CODE_ROOT = code_root = 'mig' +code_root = "mig" +CODE_ROOT = code_root # Ignore backup and dot files in wild card match -PLAIN = '[a-zA-Z0-9]*.py' +PLAIN = "[a-zA-Z0-9]*.py" py_code_files = [ # a few scripts are in parent dir of code_root - '../%s' % PLAIN, - '../bin/%s' % PLAIN, - '../sbin/%s' % PLAIN, - '%s' % PLAIN, - 'lib/%s' % PLAIN, - 'cgi-bin/%s' % PLAIN, - 'cgi-sid/%s' % PLAIN, - 'install/%s' % PLAIN, - 'migfs-fuse/%s' % PLAIN, - 'resource/bin/%s' % PLAIN, - 'resource/image-scripts/%s' % PLAIN, - 'resource/keepalive-scripts/%s' % PLAIN, - 'server/%s' % PLAIN, - 'shared/%s' % PLAIN, - 'shared/functionality/%s' % PLAIN, - 'shared/distos/%s' % PLAIN, - 'shared/gdp/%s' % PLAIN, - 'shared/griddaemons/%s' % PLAIN, - 'simulation/%s' % PLAIN, - 'user/%s' % PLAIN, - 'vm-proxy/%s' % PLAIN, - 'webserver/%s' % PLAIN, - 'wsgi-bin/%s' % PLAIN, + "../%s" % PLAIN, + "../bin/%s" % PLAIN, + "../sbin/%s" % PLAIN, + "%s" % PLAIN, + "lib/%s" % PLAIN, + "cgi-bin/%s" % PLAIN, + "cgi-sid/%s" % PLAIN, + "install/%s" % PLAIN, + "migfs-fuse/%s" % PLAIN, + "resource/bin/%s" % PLAIN, + "resource/image-scripts/%s" % PLAIN, + "resource/keepalive-scripts/%s" % PLAIN, + "server/%s" % PLAIN, + "shared/%s" % PLAIN, + "shared/functionality/%s" % PLAIN, + "shared/distos/%s" % PLAIN, + "shared/gdp/%s" % PLAIN, + "shared/griddaemons/%s" % PLAIN, + "simulation/%s" % PLAIN, + "user/%s" % PLAIN, + "vm-proxy/%s" % PLAIN, + "webserver/%s" % PLAIN, + "wsgi-bin/%s" % PLAIN, +] +py_code_files += [ + "cgi-sid/%s" % name for name in ["requestnewjob", "putrespgid"] ] -py_code_files += ['cgi-sid/%s' % name for name in ['requestnewjob', - 'putrespgid']] -py_code_files += ['cgi-bin/%s' % name for name in [ - 'listdir', - 'mkdir', - 'put', - 'remove', - 'rename', - 'rmdir', - 'stat', - 'walk', - 'getrespgid', -]] +py_code_files += [ + "cgi-bin/%s" % name + for name in [ + "listdir", + "mkdir", + "put", + "remove", + "rename", + "rmdir", + "stat", + "walk", + "getrespgid", + ] +] PY_CODE_FILES = py_code_files sh_code_files = [ - 'resource/frontend_script.sh', - 'resource/master_node_script.sh', - 'resource/leader_node_script.sh', - 'resource/dummy_node_script.sh', + "resource/frontend_script.sh", + "resource/master_node_script.sh", + "resource/leader_node_script.sh", + "resource/dummy_node_script.sh", ] SH_CODE_FILES = sh_code_files js_code_files = [ - 'images/js/jquery.accountform.js', - 'images/js/jquery.ajaxhelpers.js', - 'images/js/jquery.confirm.js', - 'images/js/jquery.filemanager.js', - 'images/js/jquery.jobmanager.js', - 'images/js/jquery.migtools.js', - 'images/js/jquery.prettyprint.js', - 'images/js/preview-caman.js', - 'images/js/preview.js', - 'images/js/preview-paraview.js', - 'assets/js/shared/ui-dynamic.js', - 'assets/js/V3/ui-global.js', - 'assets/js/V3/ui-extra.js', + "images/js/jquery.accountform.js", + "images/js/jquery.ajaxhelpers.js", + "images/js/jquery.confirm.js", + "images/js/jquery.filemanager.js", + "images/js/jquery.jobmanager.js", + "images/js/jquery.migtools.js", + "images/js/jquery.prettyprint.js", + "images/js/preview-caman.js", + "images/js/preview.js", + "images/js/preview-paraview.js", + "assets/js/shared/ui-dynamic.js", + "assets/js/V3/ui-global.js", + "assets/js/V3/ui-extra.js", ] JS_CODE_FILES = js_code_files @@ -103,15 +108,19 @@ CODE_FILES = code_files PYTHON, SHELL, JAVASCRIPT = "PYTHON", "SHELL", "JAVASCRIPT" -LANG_MAP = {keyword_all: CODE_FILES, PYTHON: PY_CODE_FILES, - JAVASCRIPT: JS_CODE_FILES, SHELL: SH_CODE_FILES} +LANG_MAP = { + keyword_all: CODE_FILES, + PYTHON: PY_CODE_FILES, + JAVASCRIPT: JS_CODE_FILES, + SHELL: SH_CODE_FILES, +} def list_code_files(code_langs=[keyword_all]): """Get list of all code files.""" match = [] for lang in code_langs: - if not lang in LANG_MAP: + if lang not in LANG_MAP: print("Warning: no such code lang: %s" % lang) else: match += LANG_MAP[lang]