Merge pull request #1194 from google/google_sync

rchen152 · web-flow · commit f892ff916624 · 2022-04-26T14:08:59.000-07:00
Google sync
diff --git a/CHANGELOG b/CHANGELOG
@@ -1,3 +1,15 @@
+Version 2022.04.26:
+
+Updates:
+* Enable --enable-typed-dicts by default.
+
+Bug fixes:
+* Preserve the full name of imported TypedDicts.
+* Stop dropping TypedDict annotations from local assignments.
+* Don't mark fields of anonymous namedtuple parents as ClassVars.
+* Instantiate imported TypedDicts to typed_dict.TypedDict, not Instance.
+* Fix more typeshed circular dependency issues.
+
 Version 2022.04.22:
 
 Updates:
diff --git a/docs/support.md b/docs/support.md
@@ -13,7 +13,7 @@ of pytype.
          * [Typing](#typing)
          * [Third-Party Libraries](#third-party-libraries)
 
-<!-- Added by: rechen, at: 2022-04-06T14:32-07:00 -->
+<!-- Added by: rechen, at: 2022-04-22T15:56-07:00 -->
 
 <!--te-->
 
@@ -66,7 +66,7 @@ Feature
 [PEP 563 -- Postponed Evaluation of Annotations][563]                                   | 3.7     | ✅        |
 [PEP 585 -- Type Hinting Generics in Standard Collections][585]                         | 3.9     | ✅        |
 [PEP 586 -- Literal Types][586]                                                         | 3.8     | ✅        |
-[PEP 589 -- TypedDict][589]                                                             | 3.8     | ✅        | Requires `--enable-typed-dicts` flag externally
+[PEP 589 -- TypedDict][589]                                                             | 3.8     | ✅        |
 [PEP 591 -- Adding a Final Qualifier to Typing][591]                                    | 3.8     | ✅        |
 [PEP 593 -- Flexible Function and Variable Annotations][593]                            | 3.9     | ✅        |
 [PEP 604 -- Allow Writing Union Types as X \| Y][604]                                   | 3.10    | ❌        | [#785][union-pipe]
diff --git a/pytype/__version__.py b/pytype/__version__.py
@@ -1,2 +1,2 @@
 # pylint: skip-file
-__version__ = '2022.04.22'
+__version__ = '2022.04.26'
diff --git a/pytype/config.py b/pytype/config.py
@@ -160,7 +160,7 @@ def add_basic_options(o):
      "Enable exhaustive checking of function parameter types."),
     ("--enable-nested-classes", False,
      "Enable support for nested classes in .py files."),
-    ("--enable-typed-dicts", False,
+    ("--enable-typed-dicts", True,
      "Enable support for TypedDicts."),
     ("--strict-primitive-comparisons", False,
      "Emit errors for comparisons between incompatible primitive types."),
diff --git a/pytype/convert.py b/pytype/convert.py
@@ -824,7 +824,10 @@ def _constant_to_value(self, pyval, subst, get_node):
             instance = self._create_new_unknown_value("type")
           else:
             mycls = self.constant_to_value(cls, subst, self.ctx.root_node)
-            instance = abstract.Instance(mycls, self.ctx)
+            if isinstance(mycls, typed_dict.TypedDictClass):
+              instance = mycls.instantiate_value(self.ctx.root_node, None)
+            else:
+              instance = abstract.Instance(mycls, self.ctx)
           log.info("New pytd instance for %s: %r", cls.name, instance)
           self._convert_cache[key] = instance
         return self._convert_cache[key]
diff --git a/pytype/load_pytd.py b/pytype/load_pytd.py
@@ -312,8 +312,7 @@ def resolve_builtin_types(self, mod_ast, *, lookup_ast=None):
         visitors.ExpandCompatibleBuiltins(self.builtins_ast))
     return mod_ast
 
-  def resolve_external_types(self, mod_ast, module_map, aliases, *,
-                             mod_name=None):
+  def resolve_external_types(self, mod_ast, module_map, aliases, *, mod_name):
     name = mod_name or mod_ast.name
     try:
       mod_ast = mod_ast.Visit(visitors.LookupExternalTypes(
@@ -616,10 +615,15 @@ def finish_and_verify_ast(self, mod_ast):
     if mod_ast:
       try:
         self._resolver.verify(mod_ast)
-      except BadDependencyError:
+      except (BadDependencyError, visitors.ContainerError):
         # In the case of a circular import, an external type may be left
-        # unresolved. As long as the module containing the unresolved type does
-        # not also contain a circular import, an extra lookup should resolve it.
+        # unresolved, so we re-resolve lookups in this module and its direct
+        # dependencies. Technically speaking, we should re-resolve all
+        # transitive imports, but lookups are expensive.
+        dependencies = self._resolver.collect_dependencies(mod_ast)
+        for k in dependencies:
+          self._modules[k].ast = self._resolve_external_types(
+              self._modules[k].ast)
         mod_ast = self._resolve_external_types(mod_ast)
         self._resolver.verify(mod_ast)
     return mod_ast
diff --git a/pytype/output.py b/pytype/output.py
@@ -863,13 +863,19 @@ def add_attributes_from(instance):
       if not any(x.name == "typing.NamedTuple" for x in bases):
         bases.append(pytd.NamedType("typing.NamedTuple"))
 
-    if any(isinstance(x, named_tuple.NamedTupleClass)
-           for x in missing_bases):
+    has_namedtuple_parent = False
+    parent_field_names = set()
+    for x in missing_bases:
+      if isinstance(x, named_tuple.NamedTupleClass):
+        has_namedtuple_parent = True
+        parent_field_names.update(field.name for field in x.props.fields)
+    if has_namedtuple_parent:
       # If inheriting from an anonymous namedtuple, mark all derived class
       # constants as ClassVars, otherwise MergeBaseClasses will convert them
       # into namedtuple fields.
-      for c in constants.values():
-        c.wrap("typing.ClassVar")
+      for k, c in constants.items():
+        if k not in parent_field_names:
+          c.wrap("typing.ClassVar")
 
     final_constants = []
     if isinstance(v, named_tuple.NamedTupleClass):
diff --git a/pytype/overlays/typed_dict.py b/pytype/overlays/typed_dict.py
@@ -178,12 +178,15 @@ def _new_instance(self, container, node, args):
       ret.set_str_item(node, k, v)
     return ret
 
-  def instantiate(self, node, container=None):
+  def instantiate_value(self, node, container):
     args = function.Args(())
     for name, typ in self.props.fields.items():
       args.namedargs[name] = self.ctx.join_variables(
           node, [t.instantiate(node) for t in typ.data])
-    return self._new_instance(container, node, args).to_variable(node)
+    return self._new_instance(container, node, args)
+
+  def instantiate(self, node, container=None):
+    return self.instantiate_value(node, container).to_variable(node)
 
   def make_class(self, *args, **kwargs):
     return self._base_cls.make_class(*args, **kwargs)
diff --git a/pytype/pytd/pytd.py b/pytype/pytd/pytd.py
@@ -644,10 +644,7 @@ def IsContainer(t):
   for p in t.bases:
     if isinstance(p, GenericType):
       base = p.base_type
-      # We need to check for Generic and Protocol again here because base may
-      # not yet have been resolved to a ClassType.
-      if (base.name in ('typing.Generic', 'typing.Protocol') or
-          isinstance(base, ClassType) and IsContainer(base.cls)):
+      if isinstance(base, ClassType) and IsContainer(base.cls):
         return True
   return False
 
diff --git a/pytype/pytd/visitors.py b/pytype/pytd/visitors.py
@@ -1594,16 +1594,7 @@ def EnterGenericType(self, node):
           raise ContainerError("Name %s must be defined as a TypeVar" % t.name)
     elif not isinstance(node, (pytd.CallableType, pytd.TupleType)):
       actual_param_count = len(node.parameters)
-      if actual_param_count and not base_type.cls.template:
-        # This AdjustTypeParameters() call is needed because we validate nodes
-        # before their type parameters have been adjusted in some circular
-        # import cases. The result of this adjustment is not saved because it
-        # may not be accurate if the container is only partially resolved, but
-        # it's good enough to avoid some spurious container validation errors.
-        cls = base_type.cls.Visit(AdjustTypeParameters())
-      else:
-        cls = base_type.cls
-      max_param_count = len(cls.template)
+      max_param_count = len(base_type.cls.template)
       if actual_param_count > max_param_count:
         raise ContainerError(
             "Too many parameters on %s: expected %s, got %s" % (
diff --git a/pytype/tests/test_namedtuple2.py b/pytype/tests/test_namedtuple2.py
@@ -92,6 +92,21 @@ def test_namedtuple_defaults(self):
       X('a', 'b')
     """)
 
+  def test_variable_annotations(self):
+    ty = self.Infer("""
+      import collections
+      class X(collections.namedtuple('X', ['a', 'b'])):
+        a: int
+        b: str
+    """)
+    self.assertTypesMatchPytd(ty, """
+      import collections
+      from typing import NamedTuple
+      class X(NamedTuple):
+        a: int
+        b: str
+    """)
+
 
 if __name__ == "__main__":
   test_base.main()
diff --git a/pytype/tests/test_typed_dict.py b/pytype/tests/test_typed_dict.py
@@ -417,6 +417,34 @@ def g() -> foo.A:
           "Expected", "x: A", "Actual", "x: foo.A"
       ]})
 
+  def test_setitem(self):
+    with self.DepTree([("foo.pyi", """
+      from typing import TypedDict
+      class Foo(TypedDict):
+        x: int
+    """), ("bar.pyi", """
+      import foo
+      def f() -> foo.Foo: ...
+    """)]):
+      self.Check("""
+        import bar
+        foo = bar.f()
+        foo['x'] = 42
+      """)
+
+  def test_match(self):
+    with self.DepTree([("foo.pyi", """
+      from typing import TypedDict
+      class Foo(TypedDict):
+        x: int
+      def f(x: Foo) -> None: ...
+      def g() -> Foo: ...
+    """)]):
+      self.Check("""
+        import foo
+        foo.f(foo.g())
+      """)
+
 
 if __name__ == "__main__":
   test_base.main()
diff --git a/pytype/tools/analyze_project/pytype_runner_test.py b/pytype/tools/analyze_project/pytype_runner_test.py
@@ -145,7 +145,7 @@ def setUp(self):
   def assertFlags(self, flags, expected_flags):
     # Add temporary flags that are set to true by default here, so that they are
     # filtered out of tests.
-    temporary_flags = set()
+    temporary_flags = {'--enable-typed-dicts'}
     self.assertEqual(flags - temporary_flags, expected_flags)
 
   # --disable tests a flag with a string value.

Original file line number	Diff line number	Diff line change
`@@ -1,2 +1,2 @@`
`1`	`1`	`# pylint: skip-file`
`2`		`-__version__ = '2022.04.22'`
	`2`	`+__version__ = '2022.04.26'`