Refactor projcode to mainly expose a list_code_files helper function which

optionally takes a list of languages to filter on or all by default.

Make sure that there's at least a single blank line between license header and
actual module content.

Minor adjustments to fix style check warnings.

Author: jonasbardino

bin/addheader.py

@@ -47,10 +47,8 @@
     print("Using mig installation in %s" % MIG_ROOT)
     sys.path.append(MIG_ROOT)
 
-from mig.shared.fileio import read_file_lines, read_head_lines, \
-    write_file_lines
-from mig.shared.projcode import code_root, js_code_files, py_code_files, \
-    sh_code_files
+from mig.shared.fileio import read_file_lines, read_head_lines, write_file_lines
+from mig.shared.projcode import CODE_ROOT, JAVASCRIPT, list_code_files
 
 # Modify these to fit actual project
 PROJ_CONSTS = {}
@@ -112,6 +110,7 @@ def check_header(path, var_dict, preamble_lines=100):
 
 def add_header(path, var_dict, explicit_border=True, block_wrap=False):
     """Add the required copyright and license header to module in path.
+
     The optional explicit_border argument can be set to wrap the license
     text in begin and end lines that are easy to find, so that license can
     be updated or replaced later.
@@ -120,7 +119,6 @@ def add_header(path, var_dict, explicit_border=True, block_wrap=False):
     JavaScript where the per-line commenting using hash (#) won't work.
     Creates a '.unlicensed' backup copy of each file changed.
     """
-
     module_lines = read_file_lines(path, None)
     if not write_file_lines(module_lines, path + BACKUP_MARKER, None):
         print("Failed to create backup of %s - skip!" % path)
@@ -170,8 +168,11 @@ def add_header(path, var_dict, explicit_border=True, block_wrap=False):
         )
 
     module_header.append(lic)
+    # Make sure there's a blank line between license header and code
+    if module_lines and module_lines[0].strip():
+        module_header.append("\n")
 
-    updated_lines = [i % var_dict for i in module_header + [""] + module_lines]
+    updated_lines = [i % var_dict for i in module_header + module_lines]
 
     if not write_file_lines(updated_lines, path, None):
         print("Failed to write %s with added headers!" % path)
@@ -181,7 +182,7 @@ def add_header(path, var_dict, explicit_border=True, block_wrap=False):
 
 
 def main(argv):
-    """Run header addition for given argv"""
+    """Run header addition for given argv."""
     target = os.getcwd()
     if len(argv) > 1:
         target = os.path.abspath(argv[1])
@@ -202,10 +203,10 @@ def main(argv):
             if src_path.endswith(BACKUP_MARKER):
                 continue
             print("Inspecting %s" % src_path)
-            for pattern in py_code_files + sh_code_files + js_code_files:
-                needs_block = pattern in js_code_files
+            for pattern in list_code_files():
+                needs_block = pattern in list_code_files(JAVASCRIPT)
                 pattern = os.path.normpath(
-                    os.path.join(mig_code_base, code_root, pattern)
+                    os.path.join(mig_code_base, CODE_ROOT, pattern)
                 )
 
                 # print("DEBUG: Testing %s against %s" % (src_path, pattern))

mig/shared/projcode.py

@@ -21,38 +21,42 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
 # 02110-1301, USA.
 
-"""Helpers to transform or search project code files"""
+"""Helpers to transform or search project code files."""
+
+from mig.shared.defaults import keyword_all
+
+# TODO: phase out lowercase names once all scripts switched to get_code_files
 
 # Top dir with all code
-code_root = 'mig'
+CODE_ROOT = code_root = 'mig'
 
 # Ignore backup and dot files in wild card match
-plain = '[a-zA-Z0-9]*.py'
+PLAIN = '[a-zA-Z0-9]*.py'
 py_code_files = [
     # a few scripts are in parent dir of code_root
-    '../%s' % plain,
-    '../bin/%s' % plain,
-    '../sbin/%s' % plain,
-    '%s' % plain,
-    'lib/%s' % plain,
-    'cgi-bin/%s' % plain,
-    'cgi-sid/%s' % plain,
-    'install/%s' % plain,
-    'migfs-fuse/%s' % plain,
-    'resource/bin/%s' % plain,
-    'resource/image-scripts/%s' % plain,
-    'resource/keepalive-scripts/%s' % plain,
-    'server/%s' % plain,
-    'shared/%s' % plain,
-    'shared/functionality/%s' % plain,
-    'shared/distos/%s' % plain,
-    'shared/gdp/%s' % plain,
-    'shared/griddaemons/%s' % plain,
-    'simulation/%s' % plain,
-    'user/%s' % plain,
-    'vm-proxy/%s' % plain,
-    'webserver/%s' % plain,
-    'wsgi-bin/%s' % plain,
+    '../%s' % PLAIN,
+    '../bin/%s' % PLAIN,
+    '../sbin/%s' % PLAIN,
+    '%s' % PLAIN,
+    'lib/%s' % PLAIN,
+    'cgi-bin/%s' % PLAIN,
+    'cgi-sid/%s' % PLAIN,
+    'install/%s' % PLAIN,
+    'migfs-fuse/%s' % PLAIN,
+    'resource/bin/%s' % PLAIN,
+    'resource/image-scripts/%s' % PLAIN,
+    'resource/keepalive-scripts/%s' % PLAIN,
+    'server/%s' % PLAIN,
+    'shared/%s' % PLAIN,
+    'shared/functionality/%s' % PLAIN,
+    'shared/distos/%s' % PLAIN,
+    'shared/gdp/%s' % PLAIN,
+    'shared/griddaemons/%s' % PLAIN,
+    'simulation/%s' % PLAIN,
+    'user/%s' % PLAIN,
+    'vm-proxy/%s' % PLAIN,
+    'webserver/%s' % PLAIN,
+    'wsgi-bin/%s' % PLAIN,
 ]
 py_code_files += ['cgi-sid/%s' % name for name in ['requestnewjob',
                                                    'putrespgid']]
@@ -68,12 +72,16 @@
     'walk',
     'getrespgid',
 ]]
+PY_CODE_FILES = py_code_files
+
 sh_code_files = [
     'resource/frontend_script.sh',
     'resource/master_node_script.sh',
     'resource/leader_node_script.sh',
     'resource/dummy_node_script.sh',
 ]
+SH_CODE_FILES = sh_code_files
+
 js_code_files = [
     'images/js/jquery.accountform.js',
     'images/js/jquery.ajaxhelpers.js',
@@ -89,4 +97,21 @@
     'assets/js/V3/ui-global.js',
     'assets/js/V3/ui-extra.js',
 ]
+JS_CODE_FILES = js_code_files
+
 code_files = py_code_files + sh_code_files + js_code_files
+CODE_FILES = code_files
+
+PYTHON, SHELL, JAVASCRIPT = "PYTHON", "SHELL", "JAVASCRIPT"
+LANG_MAP = {keyword_all: CODE_FILES, PYTHON: PY_CODE_FILES,
+            JAVASCRIPT: JS_CODE_FILES, SHELL: SH_CODE_FILES}
+
+
+def list_code_files(code_langs=[keyword_all]):
+    """Get list of all code files."""
+    match = []
+    for lang in code_langs:
+        if not lang in code_langs:
+            print("Warning: no such code lang: %s" % lang)
+        match += LANG_MAP.get(lang, [])
+    return match