[stm32variant] Aggregate generated variant if they are the same

To avoid having duplicated variants with all the same files.

Signed-off-by: Frederic Pillon <frederic.pillon@st.com>

Author: fpistm

CI/utils/stm32variant.py

@@ -6,6 +6,7 @@
 import sys
 import textwrap
 from argparse import RawTextHelpFormatter
+from collections import OrderedDict
 from jinja2 import Environment, FileSystemLoader
 from pathlib import Path
 from xml.dom.minidom import parse, Node
@@ -64,6 +65,36 @@
 """
 
 
+def rm_tree(pth: Path):
+    if pth.exists():
+        for child in pth.iterdir():
+            if child.is_file():
+                child.unlink()
+            else:
+                rm_tree(child)
+        pth.rmdir()
+
+
+def update_file(filePath, compile_pattern, subs):
+    with open(filePath, "r+", newline="\n") as file:
+        fileContents = file.read()
+        fileContents = compile_pattern.sub(subs, fileContents)
+        file.seek(0)
+        file.truncate()
+        file.write(fileContents)
+
+
+# get xml family
+def get_mcu_family(mcu_file):
+    xml_mcu = parse(str(mcu_file))
+    mcu_node = xml_mcu.getElementsByTagName("Mcu")[0]
+    mcu_family = mcu_node.attributes["Family"].value
+    if mcu_family.endswith("+"):
+        mcu_family = mcu_family[:-1]
+    xml_mcu.unlink()
+    return mcu_family
+
+
 # mcu file parsing
 def parse_mcu_file():
     global gpiofile
@@ -1274,7 +1305,7 @@ def print_boards_entry():
             / "CMSIS"
             / "Device"
             / "ST"
-            / mcu_dir
+            / mcu_family_dir
             / "Source"
             / "Templates"
             / "gcc"
@@ -1297,8 +1328,8 @@ def print_boards_entry():
         boards_entry_template.render(
             generic_list=generic_list,
             gen_entry=gen_entry,
-            mcu_name=mcu_refname,
-            mcu_dir=mcu_dir,
+            mcu_dir=mcu_refname.replace("STM32", ""),
+            mcu_family_dir=mcu_family_dir,
             mcu_ram=mcu_ram,
             product_line=mcu_line,
         )
@@ -1621,10 +1652,13 @@ def manage_repo():
 
 # main
 cur_dir = Path.cwd()
+tmp_dir = cur_dir / "variants"
 root_dir = cur_dir.parents[1]
 system_path = root_dir / "system"
 templates_dir = cur_dir / "templates"
-mcu_dir = ""
+mcu_family_dir = ""
+filtered_family = ""
+filtered_mcu_file = ""
 periph_c_filename = "PeripheralPins.c"
 pinvar_h_filename = "PinNamesVar.h"
 config_filename = Path("variant_config.json")
@@ -1766,9 +1800,13 @@ def manage_repo():
         print("\nCheck in " + dirMCU + " the correct name of this file")
         print("\nYou may use double quotes for file containing special characters")
         quit()
-    mcu_list.append(dirMCU / args.mcu)
-else:
-    mcu_list = dirMCU.glob("STM32*.xml")
+    # Get the family of the desired mcu file
+    filtered_mcu_file = dirMCU / args.mcu
+    filtered_family = get_mcu_family(filtered_mcu_file)
+if args.family:
+    filtered_family = args.family.upper()
+# Get all xml files
+mcu_list = sorted(dirMCU.glob("STM32*.xml"))
 
 if args.list:
     print("Available xml files description:")
@@ -1781,12 +1819,15 @@ def manage_repo():
     loader=FileSystemLoader(str(templates_dir)), trim_blocks=True, lstrip_blocks=True
 )
 
+# Clean temporary dir
+rm_tree(tmp_dir)
+
 for mcu_file in mcu_list:
     # Open input file
     xml_mcu = parse(str(mcu_file))
     parse_mcu_file()
     # Generate only for one family
-    if args.family and args.family.upper() not in mcu_family:
+    if filtered_family and filtered_family not in mcu_family:
         xml_mcu.unlink()
         continue
 
@@ -1796,31 +1837,20 @@ def manage_repo():
         quit()
     xml_gpio = parse(str(dirIP / ("GPIO-" + gpiofile + "_Modes.xml")))
 
-    mcu_dir = mcu_family + "xx"
-    out_path = root_dir / "variants" / mcu_dir / mcu_file.stem
-    periph_c_filepath = out_path / periph_c_filename
-    pinvar_h_filepath = out_path / pinvar_h_filename
-    variant_cpp_filepath = out_path / variant_cpp_filename
-    variant_h_filepath = out_path / variant_h_filename
-    boards_entry_filepath = out_path / boards_entry_filename
-    out_path.mkdir(parents=True, exist_ok=True)
+    mcu_family_dir = mcu_family + "xx"
+    out_temp_path = tmp_dir / mcu_family_dir / mcu_file.stem.replace("STM32", "")
+    periph_c_filepath = out_temp_path / periph_c_filename
+    pinvar_h_filepath = out_temp_path / pinvar_h_filename
+    variant_cpp_filepath = out_temp_path / variant_cpp_filename
+    variant_h_filepath = out_temp_path / variant_h_filename
+    boards_entry_filepath = out_temp_path / boards_entry_filename
+    out_temp_path.mkdir(parents=True, exist_ok=True)
 
     # open output file
-    # unlink(missing_ok=True) supported since python version 3.8
-    if periph_c_filepath.exists():
-        periph_c_filepath.unlink()
     periph_c_file = open(periph_c_filepath, "w", newline="\n")
-    if pinvar_h_filepath.exists():
-        pinvar_h_filepath.unlink()
     pinvar_h_file = open(pinvar_h_filepath, "w", newline="\n")
-    if variant_cpp_filepath.exists():
-        variant_cpp_filepath.unlink()
     variant_cpp_file = open(variant_cpp_filepath, "w", newline="\n")
-    if variant_h_filepath.exists():
-        variant_h_filepath.unlink()
     variant_h_file = open(variant_h_filepath, "w", newline="\n")
-    if boards_entry_filepath.exists():
-        boards_entry_filepath.unlink()
     boards_entry_file = open(boards_entry_filepath, "w", newline="\n")
 
     parse_pins()
@@ -1856,3 +1886,151 @@ def manage_repo():
     boards_entry_file.close()
     xml_mcu.unlink()
     xml_gpio.unlink()
+
+# Aggregating all genertaed files
+print("Aggregating all generated files...")
+periperalpins_regex = re.compile(r"\S+\.xml")
+variant_regex = re.compile(r"defined\(ARDUINO_GENERIC_[^\s&|]*\)")
+update_regex = re.compile(r"defined\(ARDUINO_GENERIC_.+\)")
+board_entry_regex = re.compile(r"(Gen.+\..+variant=STM32.+xx/)\S+")
+
+# Get mcu_family directory
+out_temp_path = cur_dir / "variants"
+mcu_families = sorted(out_temp_path.glob("STM32*/"))
+# Compare per family
+for mcu_family in mcu_families:
+    # Generate only for one family
+    if filtered_family and filtered_family not in mcu_family.name:
+        continue
+    out_family_path = root_dir / "variants" / mcu_family.name
+    # Get all mcu_dir
+    mcu_dirs = sorted(mcu_family.glob("*/"))
+    # Group mcu directories when only pexpressions and xml file name are differents
+    while mcu_dirs:
+        # Pop first item
+        group_mcu_dir = [mcu_dirs.pop(0)]
+        index = 0
+        mcu_dir = group_mcu_dir[0]
+        mcu_dir_files_list = [
+            mcu_dir / periph_c_filename,
+            mcu_dir / pinvar_h_filename,
+            mcu_dir / variant_cpp_filename,
+            mcu_dir / variant_h_filename,
+        ]
+        periph_xml = []
+        variant_exp = []
+        # Compare the first directory to all other directories
+        while mcu_dirs and index < len(mcu_dirs):
+            # Compare all the variant file except the generic_boards.txt
+            mcu_dir2_files_list = [
+                mcu_dirs[index] / periph_c_filename,
+                mcu_dirs[index] / pinvar_h_filename,
+                mcu_dirs[index] / variant_cpp_filename,
+                mcu_dirs[index] / variant_h_filename,
+            ]
+            for index2, fname in enumerate(mcu_dir_files_list):
+                with open(fname, "r") as f1:
+                    with open(mcu_dir2_files_list[index2], "r") as f2:
+                        diff = set(f1).symmetric_difference(f2)
+                        diff.discard("\n")
+                        if not diff or len(diff) == 2:
+                            if index2 == 0:
+                                for line in diff:
+                                    periph_xml += periperalpins_regex.findall(line)
+                            elif index2 == 2:
+                                for line in diff:
+                                    variant_exp += variant_regex.findall(line)
+                            continue
+                        else:
+                            # Not the same directory compare with the next one
+                            index += 1
+                            break
+            # All files compared and matched
+            else:
+                # Matched files append to the group list
+                group_mcu_dir.append(mcu_dirs.pop(index))
+
+        group_mcu_dir.sort()
+        mcu_dir = group_mcu_dir.pop(0)
+        # Merge if needed
+        if group_mcu_dir:
+            new_mcu_dirname = mcu_dir.stem
+            board_entry = ""
+            mcu_dir_name = []
+            for dir_name in group_mcu_dir:
+                # Remove package information
+                mcu_dir_name.append(dir_name.stem)
+
+            if sum(new_mcu_dirname[:-2] in dname for dname in mcu_dir_name) > 1:
+                new_mcu_dirname = new_mcu_dirname[:-2]
+            # Concatenate names
+            for dir_name in group_mcu_dir:
+                nb = sum(dir_name.stem[:-2] in dname for dname in mcu_dir_name) == 1
+                # Only one occurence
+                if nb == 1:
+                    new_mcu_dirname += "_" + dir_name.stem
+                else:
+                    # Concatenate once without package information
+                    if dir_name.stem[:-2] not in new_mcu_dirname:
+                        new_mcu_dirname += "_" + dir_name.stem[:-2]
+            # Handle files
+            for dir_name in group_mcu_dir:
+                # Save board entry
+                with open(dir_name / boards_entry_filename) as fp:
+                    for index, line in enumerate(fp):
+                        # Skip first line
+                        if index > 4:
+                            board_entry += line
+                # Delete directory
+                for filepath in dir_name.glob("*.*"):
+                    filepath.unlink()
+                dir_name.rmdir()
+            new_mcu_dir = out_temp_path / mcu_family.name / new_mcu_dirname
+            # Rename it
+            # With python 3.8 and above: mcu_dir = mcu_dir.replace(new_mcu_dir)
+            mcu_dir.replace(new_mcu_dir)
+            mcu_dir = new_mcu_dir
+            # Update files
+            periph_xml.sort()
+            periph_xml = list(OrderedDict.fromkeys(periph_xml))
+            new_line_c = periph_xml.pop(0)
+            for index, xml in enumerate(periph_xml, 1):
+                if index % 2 == 0:
+                    new_line_c += "\n * {}".format(xml)
+                else:
+                    new_line_c += ", {}".format(xml)
+
+            update_file(mcu_dir / periph_c_filename, periperalpins_regex, new_line_c)
+
+            variant_exp.sort()
+            variant_exp = list(OrderedDict.fromkeys(variant_exp))
+            new_line_c = variant_exp[0]
+            new_line_h = "{}".format(variant_exp.pop(0))
+            for index, pre in enumerate(variant_exp, 1):
+                if index % 2 == 0:
+                    new_line_c += " ||\\\n    {}".format(pre)
+                    new_line_h += " &&\\\n    !{}".format(pre)
+                else:
+                    new_line_c += " || {}".format(pre)
+                    new_line_h += " && !{}".format(pre)
+            update_file(mcu_dir / variant_cpp_filename, update_regex, new_line_c)
+
+            update_file(mcu_dir / variant_h_filename, update_regex, new_line_h)
+
+            # Appending to board_entry file
+            with open(mcu_dir / boards_entry_filename, "a", newline="\n") as fp:
+                fp.write(board_entry)
+
+            update_file(
+                mcu_dir / boards_entry_filename,
+                board_entry_regex,
+                rf"\g<1>{mcu_dir.name}",
+            )
+
+        # Move to variants/ folder
+        out_path = out_family_path / mcu_dir.stem
+        out_path.mkdir(parents=True, exist_ok=True)
+        for fname in mcu_dir.glob("*.*"):
+            fname.replace(out_path / fname.name)
+# Clean temporary dir
+rm_tree(tmp_dir)

CI/utils/templates/PeripheralPins.c

@@ -14,6 +14,7 @@
  * Automatically generated from {{mcu_file}}
  * CubeMX DB release {{db_release}}
  */
+#if !defined(CUSTOM_PERIPHERAL_PINS)
 #include "Arduino.h"
 #include "PeripheralPins.h"
 
@@ -73,3 +74,5 @@ WEAK const PinMap PinMap_{{periph.aname}}[] = {
   {% endfor %}
 {% endif %}
 
+#endif /* !CUSTOM_PERIPHERAL_PINS */
+

CI/utils/templates/boards_entry.txt

@@ -10,7 +10,6 @@
 {{gen_entry}}.menu.pnum.GENERIC_{{generic.board}}.upload.maximum_data_size={{mcu_ram}}
 {{gen_entry}}.menu.pnum.GENERIC_{{generic.board}}.build.board=GENERIC_{{generic.board}}
 {{gen_entry}}.menu.pnum.GENERIC_{{generic.board}}.build.product_line={{product_line}}
-{{gen_entry}}.menu.pnum.GENERIC_{{generic.board}}.build.mcu_name={{mcu_name}}
-{{gen_entry}}.menu.pnum.GENERIC_{{generic.board}}.build.variant={{mcu_dir}}/{{mcu_name}}
+{{gen_entry}}.menu.pnum.GENERIC_{{generic.board}}.build.variant={{mcu_family_dir}}/{{mcu_dir}}
 
 {% endfor %}