100% Test Success (#5)

Author: SamuelMarks

cdd/__init__.py

@@ -9,7 +9,7 @@
 from logging import getLogger as get_logger
 
 __author__ = "Samuel Marks"  # type: str
-__version__ = "0.0.99rc46"  # type: str
+__version__ = "0.0.99rc47"  # type: str
 __description__ = (
     "Open API to/fro routes, models, and tests. "
     "Convert between docstrings, classes, methods, argparse, pydantic, and SQLalchemy."

cdd/docstring/utils/parse_utils.py

@@ -13,6 +13,7 @@
 from cdd.shared.ast_utils import deduplicate
 from cdd.shared.pure_utils import (
     count_iter_items,
+    pp,
     simple_types,
     sliding_window,
     type_to_name,
@@ -56,6 +57,7 @@
     ("List", " ", "of"): "List",
     ("Tuple", " ", "of"): "Tuple",
     ("Dictionary", " ", "of"): "Mapping",
+    ("One", " ", "of"): "Union",
 }
 
 
@@ -369,8 +371,19 @@ def _parse_adhoc_doc_for_typ_phase0(doc, words):
     word_chars: str = "{0}{1}`'\"/|".format(string.digits, string.ascii_letters)
     sentence_ends: int = -1
     break_the_union: bool = False  # lincoln
-    for i, ch in enumerate(doc):
-        if (
+    counter = Counter(doc)  # Imperfect because won't catch escaped quote marks
+    balanced_single: bool = counter["'"] > 0 and counter["'"] & 1 == 0
+    balanced_double: bool = counter['"'] > 0 and counter['"'] & 1 == 0
+
+    i: int = 0
+    n: int = len(doc)
+    while i < n:
+        ch: str = doc[i]
+        if (ch == "'" and balanced_single or ch == '"' and balanced_double) and (
+            i == 0 or doc[i - 1] != "\\"
+        ):
+            i = eat_quoted(ch, doc, i, words, n)
+        elif (
             ch in word_chars
             or ch == "."
             and len(doc) > (i + 1)
@@ -380,14 +393,20 @@ def _parse_adhoc_doc_for_typ_phase0(doc, words):
         ):
             words[-1].append(ch)
         elif ch in frozenset((".", ";", ",")) or ch.isspace():
-            words[-1] = "".join(words[-1])
-            words.append(ch)
+            if words[-1]:
+                words[-1] = "".join(words[-1])
+                words.append(ch)
+            else:
+                words[-1] = ch
             if ch == "." and sentence_ends == -1:
                 sentence_ends: int = len(words)
             elif ch == ";":
                 break_the_union = True
             words.append([])
+        i += 1
     words[-1] = "".join(words[-1])
+    if not words[-1]:
+        del words[-1]
     candidate_type: Optional[str] = next(
         map(
             adhoc_type_to_type.__getitem__,
@@ -414,4 +433,47 @@ def _parse_adhoc_doc_for_typ_phase0(doc, words):
     return candidate_type, fst_sentence, sentence
 
 
+def eat_quoted(ch, doc, chomp_start_idx, words, n):
+    """
+    Chomp from quoted character `ch` to quoted character `ch`
+
+    :param ch: Character of `'` or `"`
+    :type ch: ```Literal["'", '"']```
+
+    :param doc: Possibly ambiguous docstring for argument, that *might* hint as to the type
+    :type doc: ```str```
+
+    :param chomp_start_idx: chomp_start_idx
+    :type chomp_start_idx: ```int```
+
+    :param words: Words
+    :type words: ```List[Union[List[str], str]]```
+
+    :param n: Length of `doc`
+    :type n: ```int```
+
+    :return: chomp_end_idx
+    :rtype: ```int```
+    """
+    chomp_end_idx: int = next(
+        filter(
+            lambda _chomp_end_idx: doc[_chomp_end_idx + 1] != ch
+            or doc[_chomp_end_idx] == "\\",
+            range(chomp_start_idx, n),
+        ),
+        chomp_start_idx,
+    )
+    quoted_str: str = doc[chomp_start_idx:chomp_end_idx]
+    from operator import iadd
+
+    pp({"b4::words": words, '"".join(words[-1])': "".join(words[-1])})
+    (
+        iadd(words, (quoted_str, []))
+        if len(words[-1]) == 1 and words[-1][-1] == "`"
+        else iadd(words, ("".join(words[-1]), quoted_str, []))
+    )
+    pp({"words": words})
+    return chomp_end_idx
+
+
 __all__ = ["parse_adhoc_doc_for_typ"]  # type: list[str]

cdd/shared/ast_utils.py

@@ -427,8 +427,10 @@ def get_default_val(val):
             )
         elif _param.get("typ") == "Str":
             _param["typ"] = "str"
-        elif _param.get("typ") in frozenset(("Constant", "NameConstant", "Num")):
+        elif _param.get("typ") in frozenset(("Constant", "NameConstant")):
             _param["typ"] = "object"
+        elif _param.get("typ") == "Num":
+            _param["typ"] = "float"
     if "typ" in _param and needs_quoting(_param["typ"]):
         default = (
             _param.get("default")

cdd/shared/defaults_utils.py

@@ -228,27 +228,38 @@ def _parse_out_default_and_doc(
     :rtype: Tuple[str, Optional[str]]
     """
     if typ is not None and typ in simple_types and default not in none_types:
-        lit = (
-            ast.AST()
-            if typ != "str"
-            and any(
-                map(
-                    partial(contains, frozenset(("*", "^", "&", "|", "$", "@", "!"))),
-                    default,
+        keep_default: bool = False
+        try:
+            lit = (
+                ast.AST()
+                if typ != "str"
+                and any(
+                    map(
+                        partial(
+                            contains, frozenset(("*", "^", "&", "|", "$", "@", "!"))
+                        ),
+                        default,
+                    )
                 )
+                else literal_eval("({default})".format(default=default))
             )
-            else literal_eval("({default})".format(default=default))
-        )
+        except ValueError as e:
+            assert e.args[0].startswith("malformed node or string"), e
+            keep_default = True
         default = (
-            "```{default}```".format(default=default)
-            if isinstance(lit, ast.AST)
-            else {
-                "bool": bool,
-                "int": int,
-                "float": float,
-                "complex": complex,
-                "str": str,
-            }[typ](lit)
+            default
+            if keep_default
+            else (
+                "```{default}```".format(default=default)
+                if isinstance(lit, ast.AST)
+                else {
+                    "bool": bool,
+                    "int": int,
+                    "float": float,
+                    "complex": complex,
+                    "str": str,
+                }[typ](lit)
+            )
         )
     elif default.isdecimal():
         default = int(default)

cdd/sqlalchemy/utils/emit_utils.py

@@ -38,7 +38,6 @@
 from json import dumps
 from operator import attrgetter, eq, methodcaller
 from os import path
-from platform import system
 from typing import Any, Dict, List, Optional
 
 import cdd.shared.ast_utils
@@ -79,8 +78,7 @@ def param_to_sqlalchemy_column_calls(name_param, include_name):
     :return: Iterable of elements in form of: `Column(…)`
     :rtype: ```Iterable[Call]```
     """
-    if system() == "Darwin":
-        print("param_to_sqlalchemy_column_calls::include_name:", include_name, ";")
+    # if system() == "Darwin": print("param_to_sqlalchemy_column_calls::include_name:", include_name, ";")
     name, _param = name_param
     del name_param
 

cdd/sqlalchemy/utils/shared_utils.py

@@ -87,9 +87,12 @@ def update_args_infer_typ_sqlalchemy(_param, args, name, nullable, x_typ_sql):
         parsed_typ: Call = cast(
             Call, cdd.shared.ast_utils.get_value(ast.parse(_param["typ"]).body[0])
         )
-        assert parsed_typ.value.id == "Literal", "Expected `Literal` got: {!r}".format(
-            parsed_typ.value.id
-        )
+        try:
+            assert (
+                parsed_typ.value.id == "Literal"
+            ), "Expected `Literal` got: {!r}".format(parsed_typ.value.id)
+        except AssertionError:
+            raise
         val = cdd.shared.ast_utils.get_value(parsed_typ.slice)
         (
             args.append(

cdd/tests/mocks/docstrings.py

@@ -500,20 +500,22 @@
 )
 docstring_google_pytorch_lbfgs_str: str = "\n".join(docstring_google_pytorch_lbfgs)
 
-docstring_google_str: str = (
-    """{docstring_header_str}
-Args:
+docstring_google_args_str: str = """Args:
   dataset_name (str): name of dataset. Defaults to "mnist"
   tfds_dir (str): directory to look for models in. Defaults to "~/tensorflow_datasets"
   K (Literal['np', 'tf']): backend engine, e.g., `np` or `tf`. Defaults to "np"
   as_numpy (Optional[bool]): Convert to numpy ndarrays
   data_loader_kwargs (Optional[dict]): pass this as arguments to data_loader function
-
-Returns:
+"""
+docstring_google_footer_return_str: str = """Returns:
   Union[Tuple[tf.data.Dataset, tf.data.Dataset], Tuple[np.ndarray, np.ndarray]]:
    Train and tests dataset splits. Defaults to (np.empty(0), np.empty(0))
-""".format(
-        docstring_header_str=docstring_header_str
+"""
+docstring_google_str: str = "\n".join(
+    (
+        docstring_header_str,
+        docstring_google_args_str,
+        docstring_google_footer_return_str,
     )
 )
 

cdd/tests/test_compound/test_exmod_utils.py

@@ -57,6 +57,7 @@ def test_emit_file_on_hierarchy(self) -> None:
         with patch(
             "cdd.compound.exmod_utils.EXMOD_OUT_STREAM", new_callable=StringIO
         ), TemporaryDirectory() as tempdir:
+            output_directory: str = path.join(tempdir, "haz")
             open(path.join(tempdir, INIT_FILENAME), "a").close()
             emit_file_on_hierarchy(
                 ("foo.bar", "foo_dir", ir),
@@ -65,13 +66,13 @@ def test_emit_file_on_hierarchy(self) -> None:
                 "",
                 True,
                 filesystem_layout="as_input",
-                output_directory=tempdir,
+                output_directory=output_directory,
                 first_output_directory=tempdir,
                 no_word_wrap=None,
                 dry_run=False,
                 extra_modules_to_all=None,
             )
-            self.assertTrue(path.isdir(tempdir))
+            self.assertTrue(path.isdir(output_directory))
 
     def test__emit_symbols_isfile_emit_filename_true(self) -> None:
         """Test `_emit_symbol` when `isfile_emit_filename is True`"""

cdd/tests/test_marshall_docstring.py

@@ -248,6 +248,25 @@ def test_from_docstring_google_str(self) -> None:
         _intermediate_repr_no_default_doc["doc"] = docstring_header_str
         self.assertDictEqual(ir, _intermediate_repr_no_default_doc)
 
+    def test_from_docstring_google_str_extra_after(self) -> None:
+        """
+        Tests whether `parse_docstring` produces `intermediate_repr_no_default_doc`  + "\n\n\ntrailer"
+              from `docstring_google_str` + "\n\n\ntrailer"
+        """
+        ir: IntermediateRepr = parse_docstring(
+            "{}\n\n\ntrailer".format(docstring_google_str)
+        )
+        _intermediate_repr_no_default_doc = deepcopy(
+            intermediate_repr_no_default_with_nones_doc
+        )
+        _intermediate_repr_no_default_doc["doc"] = "{0}\n\n\n\ntrailer".format(
+            docstring_header_str
+        )
+
+        self.assertDictEqual(ir, _intermediate_repr_no_default_doc)
+
+    maxDiff = None
+
     def test_from_docstring_google_keras_squared_hinge(self) -> None:
         """
         Tests whether `parse_docstring` produces the right IR

cdd/tests/test_shared/test_ast_utils.py

@@ -1011,6 +1011,39 @@ def test_param2ast_with_wrapped_default(self) -> None:
             ),
         )
 
+    def test_param2ast_with_simple_types(self) -> None:
+        """Check that `param2ast` behaves correctly with simple types"""
+        deque(
+            map(
+                lambda typ_res: run_ast_test(
+                    self,
+                    param2ast(
+                        ("zion", {"typ": typ_res[0], "default": NoneStr}),
+                    ),
+                    gold=AnnAssign(
+                        annotation=Name(
+                            typ_res[1], Load(), lineno=None, col_offset=None
+                        ),
+                        simple=1,
+                        target=Name("zion", Store(), lineno=None, col_offset=None),
+                        value=set_value(None),
+                        expr=None,
+                        expr_target=None,
+                        expr_annotation=None,
+                        col_offset=None,
+                        lineno=None,
+                    ),
+                ),
+                (
+                    ("Str", "str"),
+                    ("Constant", "object"),
+                    ("NameConstant", "object"),
+                    ("Num", "float"),
+                ),
+            ),
+            maxlen=0,
+        )
+
     def test_param2argparse_param_none_default(self) -> None:
         """
         Tests that param2argparse_param works to reparse the default

cdd/tests/test_sqlalchemy/test_emit_sqlalchemy_utils.py

@@ -337,6 +337,34 @@ def test_update_args_infer_typ_sqlalchemy_early_exit(self) -> None:
             (True, None),
         )
 
+    def test_update_args_infer_typ_sqlalchemy_list_struct(self) -> None:
+        """Tests that `update_args_infer_typ_sqlalchemy` behaves correctly on List[struct]"""
+        args = []
+        self.assertTupleEqual(
+            update_args_infer_typ_sqlalchemy(
+                {"typ": "List[struct]"},
+                args=args,
+                name="",
+                nullable=True,
+                x_typ_sql={},
+            ),
+            (True, None),
+        )
+        self.assertEqual(len(args), 1)
+        run_ast_test(
+            self,
+            args[0],
+            Call(
+                func=Name("ARRAY", Load(), lineno=None, col_offset=None),
+                args=[Name("JSON", Load(), lineno=None, col_offset=None)],
+                keywords=[],
+                expr=None,
+                expr_func=None,
+                lineno=None,
+                col_offset=None,
+            ),
+        )
+
     def test_update_with_imports_from_columns(self) -> None:
         """
         Tests basic `cdd.sqlalchemy.utils.emit_utils.update_with_imports_from_columns` usage