Merge branch 'simplify_createstubs' of https://github.com/Josverl/micropython-stubber into simplify_createstubs

Signed-off-by: Jos Verlinde <jos_verlinde@hotmail.com>

Author: Josverl

mip/v5/createstubs.py

@@ -7,6 +7,7 @@
 import logging
 import os
 import sys
+from time import sleep
 
 try:
     from ujson import dumps
@@ -23,11 +24,26 @@
 except ImportError:
     from ucollections import OrderedDict  # type: ignore
 
+try:
+    from nope_machine import WDT
+
+    wdt = WDT()
+
+except ImportError:
+
+    class _WDT:
+        def feed(self):
+            pass
+
+    wdt = _WDT()
+
+
+wdt.feed()
+
 __version__ = "v1.16.2"
 ENOENT = 2
 _MAX_CLASS_LEVEL = 2  # Max class nesting
 LIBS = [".", "/lib", "/sd/lib", "/flash/lib", "lib"]
-from time import sleep
 
 
 class Stubber:
@@ -45,11 +61,12 @@ def __init__(self, path: str = None, firmware_id: str = None):  # type: ignore
         self.log.info("Port: {}".format(self.info["port"]))
         self.log.info("Board: {}".format(self.info["board"]))
         gc.collect()
+        wdt.feed()
         if firmware_id:
             self._fwid = firmware_id.lower()
         else:
             if self.info["family"] == "micropython":
-                self._fwid = "{family}-v{version}-{port}-{board}".format(**self.info)
+                self._fwid = "{family}-v{version}-{port}-{board}".format(**self.info).rstrip("-")
             else:
                 self._fwid = "{family}-v{version}-{port}".format(**self.info)
         self._start_free = gc.mem_free()  # type: ignore
@@ -98,6 +115,7 @@ def get_obj_attributes(self, item_instance: object):
             try:
                 val = getattr(item_instance, name)
                 # name , item_repr(value) , type as text, item_instance, order
+                self.log.debug("attribute {}:{}".format(name, val))
                 try:
                     type_text = repr(type(val)).split("'")[1]
                 except IndexError:
@@ -137,6 +155,7 @@ def create_all_stubs(self):
         self.log.info("Finally done")
 
     def create_one_stub(self, module_name: str):
+        wdt.feed()
         if module_name in self.problematic:
             self.log.warning("Skip module: {:<25}        : Known problematic".format(module_name))
             return False
@@ -191,7 +210,7 @@ def create_module_stub(self, module_name: str, file_name: str = None) -> bool:
                 module_name, self._fwid, info_, __version__
             )
             fp.write(s)
-            fp.write("from typing import Any\nfrom _typeshed import Incomplete\n\n")
+            fp.write("from __future__ import annotations\nfrom typing import Any\nfrom _typeshed import Incomplete\n\n")
             self.write_object_stub(fp, new_module, module_name, "")
 
         self._report.append('{{"module": "{}", "file": "{}"}}'.format(module_name, file_name.replace("\\", "/")))
@@ -202,10 +221,11 @@ def create_module_stub(self, module_name: str, file_name: str = None) -> bool:
                 del new_module
             except (OSError, KeyError):  # lgtm [py/unreachable-statement]
                 self.log.warning("could not del new_module")
-            try:
-                del sys.modules[module_name]
-            except KeyError:
-                self.log.debug("could not del sys.modules[{}]".format(module_name))
+            # lets not try - most times it does not work anyway
+            # try:
+            #     del sys.modules[module_name]
+            # except KeyError:
+            #     self.log.warning("could not del sys.modules[{}]".format(module_name))
         gc.collect()
         return True
 
@@ -231,8 +251,13 @@ def write_object_stub(self, fp, object_expr: object, obj_name: str, indent: str,
                 self.log.warning("NameError: invalid name {}".format(item_name))
                 continue
             # Class expansion only on first 3 levels (bit of a hack)
-            if item_type_txt == "<class 'type'>" and len(indent) <= _MAX_CLASS_LEVEL * 4:
-                self.log.debug("{0}class {1}:".format(indent, item_name))
+            if (
+                item_type_txt == "<class 'type'>"
+                and len(indent) <= _MAX_CLASS_LEVEL * 4
+                # and not obj_name.endswith(".Pin")
+                # avoid expansion of Pin.cpu / Pin.board to avoid crashes on most platforms
+            ):
+                self.log.info("{0}class {1}:".format(indent, item_name))
                 superclass = ""
                 is_exception = (
                     item_name.endswith("Exception")
@@ -306,12 +331,14 @@ def write_object_stub(self, fp, object_expr: object, obj_name: str, indent: str,
                     s = "{0}{1} = {2} # type: {3}\n".format(indent, item_name, ev[t], t)
                 else:
                     # something else
-                    if t not in ["object", "set", "frozenset"]:
-                        # Possibly default others to item_instance object ?
+                    if t in ["object", "set", "frozenset", "Pin", "FileIO"]:
                         # https://docs.python.org/3/tutorial/classes.html#item_instance-objects
+                        #  use these types for the attribute
+                        s = "{0}{1} : {2} ## = {4}\n".format(indent, item_name, t, item_type_txt, item_repr)
+                    else:
+                        # Requires Python 3.6 syntax, which is OK for the stubs/pyi
                         t = "Incomplete"
-                    # Requires Python 3.6 syntax, which is OK for the stubs/pyi
-                    s = "{0}{1} : {2} ## {3} = {4}\n".format(indent, item_name, t, item_type_txt, item_repr)
+                        s = "{0}{1} : {2} ## {3} = {4}\n".format(indent, item_name, t, item_type_txt, item_repr)
                 fp.write(s)
                 self.log.debug("\n" + s)
             else:
@@ -321,12 +348,12 @@ def write_object_stub(self, fp, object_expr: object, obj_name: str, indent: str,
 
                 fp.write(indent + item_name + " # type: Incomplete\n")
 
-        del items
-        del errors
-        try:
-            del item_name, item_repr, item_type_txt, item_instance  # type: ignore
-        except (OSError, KeyError, NameError):
-            pass
+        # del items
+        # del errors
+        # try:
+        #     del item_name, item_repr, item_type_txt, item_instance  # type: ignore
+        # except (OSError, KeyError, NameError):
+        #     pass
 
     @property
     def flat_fwid(self):
@@ -340,6 +367,7 @@ def flat_fwid(self):
 
     def clean(self, path: str = None):  # type: ignore
         "Remove all files from the stub folder"
+        wdt.feed()
         if path is None:
             path = self.path
         self.log.info("Clean/remove files in folder: {}".format(path))
@@ -362,6 +390,7 @@ def clean(self, path: str = None):  # type: ignore
 
     def report(self, filename: str = "modules.json"):
         "create json with list of exported modules"
+        wdt.feed()
         self.log.info(
             "Created stubs for {} modules on board {}\nPath: {}".format(len(self._report), self._fwid, self.path)
         )
@@ -423,13 +452,19 @@ def ensure_folder(path: str):
 
 def _build(s):
     # extract build from sys.version or os.uname().version if available
-    # 'v1.13-103-gb137d064e'
-    # 'MicroPython v1.23.0-preview.6.g3d0b6276f'
+    # sys.version: 'MicroPython v1.23.0-preview.6.g3d0b6276f'
+    # sys.implementation.version: 'v1.13-103-gb137d064e'
     if not s:
         return ""
     s = s.split(" on ", 1)[0] if " on " in s else s
-    s = s.split("; ", 1)[1] if "; " in s else s
-    b = s.split("-")[1] if s.startswith("v") else s.split("-", 1)[-1].split(".")[1]
+    if s.startswith("v"):
+        if not "-" in s:
+            return ""
+        b = s.split("-")[1]
+        return b
+    if not "-preview" in s:
+        return ""
+    b = s.split("-preview")[1].split(".")[1]
     return b
 
 
@@ -565,56 +600,78 @@ def version_str(version: tuple):  #  -> str:
 
 
 def read_boardname(info, desc: str = ""):
+    info["board"] = info["board"].replace(" ", "_")
     found = False
-    for filename in [d + "/board_info.csv" for d in LIBS]:
+    for filename in [d + "/board_name.txt" for d in LIBS]:
+        wdt.feed()
         # print("look up the board name in the file", filename)
         if file_exists(filename):
-            descr = desc or info["board"].strip()
-            pos = descr.rfind(" with")
-            if pos != -1:
-                short_descr = descr[:pos].strip()
-            else:
-                short_descr = ""
-            print("searching info file: {} for: '{}' or '{}'".format(filename, descr, short_descr))
-            if find_board(info, descr, filename, short_descr):
+            with open(filename, "r") as file:
+                data = file.read()
+            if data:
+                info["board"] = data.strip()
                 found = True
                 break
     if not found:
         print("Board not found, guessing board name")
-        descr = desc or info["board"].strip()
-        if "with " + info["cpu"].upper() in descr:
-            # remove the with cpu part
-            descr = descr.split("with " + info["cpu"].upper())[0].strip()
+        descr = ""
+        # descr = desc or info["board"].strip()
+        # if "with " + info["cpu"].upper() in descr:
+        #     # remove the with cpu part
+        #     descr = descr.split("with " + info["cpu"].upper())[0].strip()
         info["board"] = descr
-    info["board"] = info["board"].replace(" ", "_")
-    gc.collect()
 
 
-def find_board(info: dict, descr: str, filename: str, short_descr: str):
-    "Find the board in the provided board_info.csv file"
-    short_hit = ""
-    with open(filename, "r") as file:
-        # ugly code to make testable in python and micropython
-        # TODO: This is VERY slow on micropython whith MPREMOTE mount on esp32 (2-3 minutes to read file)
-        while 1:
-            line = file.readline()
-            if not line:
-                break
-            descr_, board_ = line.split(",")[0].strip(), line.split(",")[1].strip()
-            if descr_ == descr:
-                info["board"] = board_
-                return True
-            elif short_descr and descr_ == short_descr:
-                if "with" in short_descr:
-                    # Good enough - no need to trawl the entire file
-                    info["board"] = board_
-                    return True
-                # good enough if not found in the rest of the file (but slow)
-                short_hit = board_
-    if short_hit:
-        info["board"] = short_hit
-        return True
-    return False
+# def read_boardname(info, desc: str = ""):
+#         wdt.feed()
+#         # print("look up the board name in the file", filename)
+#         if file_exists(filename):
+#             descr = desc or info["board"].strip()
+#             pos = descr.rfind(" with")
+#             if pos != -1:
+#                 short_descr = descr[:pos].strip()
+#             else:
+#                 short_descr = ""
+#             print("searching info file: {} for: '{}' or '{}'".format(filename, descr, short_descr))
+#             if find_board(info, descr, filename, short_descr):
+#                 found = True
+#                 break
+#     if not found:
+#         print("Board not found, guessing board name")
+#         descr = desc or info["board"].strip()
+#         if "with " + info["cpu"].upper() in descr:
+#             # remove the with cpu part
+#             descr = descr.split("with " + info["cpu"].upper())[0].strip()
+#         info["board"] = descr
+#     info["board"] = info["board"].replace(" ", "_")
+#     gc.collect()
+
+
+# def find_board(info: dict, descr: str, filename: str, short_descr: str):
+#     "Find the board in the provided board_info.csv file"
+#     short_hit = ""
+#     with open(filename, "r") as file:
+#         # ugly code to make testable in python and micropython
+#         # TODO: This is VERY slow on micropython whith MPREMOTE mount on esp32 (2-3 minutes to read file)
+#         while 1:
+#             line = file.readline()
+#             if not line:
+#                 break
+#             descr_, board_ = line.split(",")[0].strip(), line.split(",")[1].strip()
+#             if descr_ == descr:
+#                 info["board"] = board_
+#                 return True
+#             elif short_descr and descr_ == short_descr:
+#                 if "with" in short_descr:
+#                     # Good enough - no need to trawl the entire file
+#                     info["board"] = board_
+#                     return True
+#                 # good enough if not found in the rest of the file (but slow)
+#                 short_hit = board_
+#     if short_hit:
+#         info["board"] = short_hit
+#         return True
+#     return False
 
 
 def get_root() -> str:  # sourcery skip: use-assigned-variable