-
-
Notifications
You must be signed in to change notification settings - Fork 3k
Draft: Solve intermediate variable bug #19399
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Draft
randolf-scholz
wants to merge
13
commits into
python:master
Choose a base branch
from
randolf-scholz:solve_intermediate_variable_bug
base: master
Could not load branches
Branch not found: {{ refName }}
Loading
Could not load tags
Nothing to show
Loading
Are you sure you want to change the base?
Some commits from the old base branch may be removed from the timeline,
and old review comments may become outdated.
+172
−64
Draft
Changes from all commits
Commits
Show all changes
13 commits
Select commit
Hold shift + click to select a range
2de93ae
inital commit
randolf-scholz e4238c7
11 failures remain
randolf-scholz db7ed35
3 test remain
randolf-scholz 40f8ca8
1 check remaining
randolf-scholz d7145d9
checks pass
randolf-scholz ce4956d
some simplifications
randolf-scholz e704f6e
revert fixture modification
randolf-scholz 26e8da2
simplify
randolf-scholz 925a3ea
Simplified away early inner_solution computation
randolf-scholz 307aa36
Merge branch 'master' into solve_intermediate_variable_bug
randolf-scholz 507c439
Merge branch 'master' into solve_intermediate_variable_bug
randolf-scholz b38dcc6
Merge branch 'master' into solve_intermediate_variable_bug
randolf-scholz 76f3571
added testLiteralMappingContext
randolf-scholz File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -18,6 +18,12 @@ | |
from mypy.checker_shared import ExpressionCheckerSharedApi | ||
from mypy.checkmember import analyze_member_access, has_operator | ||
from mypy.checkstrformat import StringFormatterChecker | ||
from mypy.constraints import ( | ||
SUBTYPE_OF, | ||
Constraint, | ||
infer_constraints, | ||
infer_constraints_for_callable, | ||
) | ||
from mypy.erasetype import erase_type, remove_instance_last_known_values, replace_meta_vars | ||
from mypy.errors import ErrorWatcher, report_internal_error | ||
from mypy.expandtype import ( | ||
|
@@ -26,7 +32,7 @@ | |
freshen_all_functions_type_vars, | ||
freshen_function_type_vars, | ||
) | ||
from mypy.infer import ArgumentInferContext, infer_function_type_arguments, infer_type_arguments | ||
from mypy.infer import ArgumentInferContext, infer_function_type_arguments | ||
from mypy.literals import literal | ||
from mypy.maptype import map_instance_to_supertype | ||
from mypy.meet import is_overlapping_types, narrow_declared_type | ||
|
@@ -110,6 +116,7 @@ | |
Plugin, | ||
) | ||
from mypy.semanal_enum import ENUM_BASES | ||
from mypy.solve import solve_constraints | ||
from mypy.state import state | ||
from mypy.subtypes import ( | ||
find_member, | ||
|
@@ -191,12 +198,7 @@ | |
is_named_instance, | ||
split_with_prefix_and_suffix, | ||
) | ||
from mypy.types_utils import ( | ||
is_generic_instance, | ||
is_overlapping_none, | ||
is_self_type_like, | ||
remove_optional, | ||
) | ||
from mypy.types_utils import is_generic_instance, is_self_type_like, remove_optional | ||
from mypy.typestate import type_state | ||
from mypy.typevars import fill_typevars | ||
from mypy.util import split_module_names | ||
|
@@ -1774,18 +1776,6 @@ def check_callable_call( | |
isinstance(v, (ParamSpecType, TypeVarTupleType)) for v in callee.variables | ||
) | ||
callee = freshen_function_type_vars(callee) | ||
callee = self.infer_function_type_arguments_using_context(callee, context) | ||
if need_refresh: | ||
# Argument kinds etc. may have changed due to | ||
# ParamSpec or TypeVarTuple variables being replaced with an arbitrary | ||
# number of arguments; recalculate actual-to-formal map | ||
formal_to_actual = map_actuals_to_formals( | ||
arg_kinds, | ||
arg_names, | ||
callee.arg_kinds, | ||
callee.arg_names, | ||
lambda i: self.accept(args[i]), | ||
) | ||
callee = self.infer_function_type_arguments( | ||
callee, args, arg_kinds, arg_names, formal_to_actual, need_refresh, context | ||
) | ||
|
@@ -2000,9 +1990,9 @@ def infer_arg_types_in_context( | |
assert all(tp is not None for tp in res) | ||
return cast(list[Type], res) | ||
|
||
def infer_function_type_arguments_using_context( | ||
self, callable: CallableType, error_context: Context | ||
) -> CallableType: | ||
def infer_constraints_from_context( | ||
self, callee: CallableType, error_context: Context | ||
) -> list[Constraint]: | ||
"""Unify callable return type to type context to infer type vars. | ||
|
||
For example, if the return type is set[t] where 't' is a type variable | ||
|
@@ -2011,23 +2001,23 @@ def infer_function_type_arguments_using_context( | |
""" | ||
ctx = self.type_context[-1] | ||
if not ctx: | ||
return callable | ||
return [] | ||
# The return type may have references to type metavariables that | ||
# we are inferring right now. We must consider them as indeterminate | ||
# and they are not potential results; thus we replace them with the | ||
# special ErasedType type. On the other hand, class type variables are | ||
# valid results. | ||
erased_ctx = replace_meta_vars(ctx, ErasedType()) | ||
ret_type = callable.ret_type | ||
if is_overlapping_none(ret_type) and is_overlapping_none(ctx): | ||
# If both the context and the return type are optional, unwrap the optional, | ||
# since in 99% cases this is what a user expects. In other words, we replace | ||
# Optional[T] <: Optional[int] | ||
# with | ||
# T <: int | ||
# while the former would infer T <: Optional[int]. | ||
ret_type = remove_optional(ret_type) | ||
erased_ctx = remove_optional(erased_ctx) | ||
erased_ctx = get_proper_type(replace_meta_vars(ctx, ErasedType())) | ||
proper_ret = get_proper_type(callee.ret_type) | ||
if isinstance(proper_ret, UnionType) and isinstance(erased_ctx, UnionType): | ||
# If both the context and the return type are unions, we simplify shared items | ||
# e.g. T | None <: int | None => T <: int | ||
# since the former would infer T <: int | None. | ||
# whereas the latter would infer the more precise T <: int. | ||
new_ret = [val for val in proper_ret.items if val not in erased_ctx.items] | ||
new_ctx = [val for val in erased_ctx.items if val not in proper_ret.items] | ||
proper_ret = make_simplified_union(new_ret) | ||
erased_ctx = make_simplified_union(new_ctx) | ||
# | ||
# TODO: Instead of this hack and the one below, we need to use outer and | ||
# inner contexts at the same time. This is however not easy because of two | ||
|
@@ -2038,7 +2028,6 @@ def infer_function_type_arguments_using_context( | |
# variables in an expression are inferred at the same time. | ||
# (And this is hard, also we need to be careful with lambdas that require | ||
# two passes.) | ||
proper_ret = get_proper_type(ret_type) | ||
if ( | ||
isinstance(proper_ret, TypeVarType) | ||
or isinstance(proper_ret, UnionType) | ||
|
@@ -2068,22 +2057,9 @@ def infer_function_type_arguments_using_context( | |
# TODO: we may want to add similar exception if all arguments are lambdas, since | ||
# in this case external context is almost everything we have. | ||
if not is_generic_instance(ctx) and not is_literal_type_like(ctx): | ||
return callable.copy_modified() | ||
args = infer_type_arguments( | ||
callable.variables, ret_type, erased_ctx, skip_unsatisfied=True | ||
) | ||
# Only substitute non-Uninhabited and non-erased types. | ||
new_args: list[Type | None] = [] | ||
for arg in args: | ||
if has_uninhabited_component(arg) or has_erased_component(arg): | ||
new_args.append(None) | ||
else: | ||
new_args.append(arg) | ||
# Don't show errors after we have only used the outer context for inference. | ||
# We will use argument context to infer more variables. | ||
return self.apply_generic_arguments( | ||
callable, new_args, error_context, skip_unsatisfied=True | ||
) | ||
return [] | ||
constraints = infer_constraints(proper_ret, erased_ctx, SUBTYPE_OF) | ||
return constraints | ||
|
||
def infer_function_type_arguments( | ||
self, | ||
|
@@ -2122,15 +2098,110 @@ def infer_function_type_arguments( | |
else: | ||
pass1_args.append(arg) | ||
|
||
inferred_args, _ = infer_function_type_arguments( | ||
callee_type, | ||
pass1_args, | ||
arg_kinds, | ||
arg_names, | ||
formal_to_actual, | ||
context=self.argument_infer_context(), | ||
strict=self.chk.in_checked_function(), | ||
) | ||
if True: # NEW CODE | ||
# compute the inner constraints | ||
inner_constraints = infer_constraints_for_callable( | ||
callee_type, | ||
pass1_args, | ||
arg_kinds, | ||
arg_names, | ||
formal_to_actual, | ||
context=self.argument_infer_context(), | ||
) | ||
|
||
# compute the outer solution | ||
outer_constraints = self.infer_constraints_from_context(callee_type, context) | ||
outer_solution = solve_constraints( | ||
callee_type.variables, | ||
outer_constraints, | ||
strict=self.chk.in_checked_function(), | ||
allow_polymorphic=False, | ||
) | ||
outer_args = [ | ||
None if has_uninhabited_component(arg) or has_erased_component(arg) else arg | ||
for arg in outer_solution[0] | ||
] | ||
outer_solution = (outer_args, outer_solution[1]) | ||
outer_callee = self.apply_generic_arguments( | ||
callee_type, outer_solution[0], context, skip_unsatisfied=True | ||
) | ||
outer_ret_type = get_proper_type(outer_callee.ret_type) | ||
|
||
# compute the joint solution using both inner and outer constraints. | ||
# NOTE: The order of constraints is important here! | ||
# solve(outer + inner) and solve(inner + outer) may yield different results. | ||
# we need to use outer first. | ||
joint_solution = solve_constraints( | ||
callee_type.variables, | ||
outer_constraints + inner_constraints, | ||
strict=self.chk.in_checked_function(), | ||
allow_polymorphic=False, | ||
) | ||
joint_args = [ | ||
None if has_uninhabited_component(arg) or has_erased_component(arg) else arg | ||
for arg in joint_solution[0] | ||
] | ||
joint_solution = (joint_args, joint_solution[1]) | ||
joint_callee = self.apply_generic_arguments( | ||
callee_type, joint_solution[0], context, skip_unsatisfied=True | ||
) | ||
joint_ret_type = get_proper_type(joint_callee.ret_type) | ||
|
||
if ( # determine which solution to take | ||
# joint constraints failed to produce a complete solution | ||
None in joint_solution[0] | ||
# If the outer solution is more concrete than the joint solution, prefer the outer solution. | ||
or is_subtype(outer_ret_type, joint_ret_type) | ||
or ( # HACK to fix testLiteralAndGenericWithUnion | ||
isinstance(outer_ret_type, UnionType) | ||
and any(is_subtype(val, joint_ret_type) for val in outer_ret_type.items) | ||
) | ||
): | ||
use_joint = False | ||
else: | ||
use_joint = True | ||
|
||
if use_joint: | ||
inferred_args = joint_solution[0] | ||
else: | ||
# If we cannot use the joint solution, fallback to outer_solution | ||
inferred_args = outer_solution[0] | ||
# Don't show errors after we have only used the outer context for inference. | ||
# We will use argument context to infer more variables. | ||
callee_type = self.apply_generic_arguments( | ||
callee_type, inferred_args, context, skip_unsatisfied=True | ||
) | ||
if need_refresh: | ||
# Argument kinds etc. may have changed due to | ||
# ParamSpec or TypeVarTuple variables being replaced with an arbitrary | ||
# number of arguments; recalculate actual-to-formal map | ||
formal_to_actual = map_actuals_to_formals( | ||
arg_kinds, | ||
arg_names, | ||
callee_type.arg_kinds, | ||
callee_type.arg_names, | ||
lambda i: self.accept(args[i]), | ||
) | ||
|
||
# ??? QUESTION: Do we need to recompute arg_types and pass1_args here??? | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. It seems to work both with/without recomputing |
||
# recompute and apply inner solution. | ||
inner_constraints = infer_constraints_for_callable( | ||
callee_type, | ||
pass1_args, | ||
arg_kinds, | ||
arg_names, | ||
formal_to_actual, | ||
context=self.argument_infer_context(), | ||
) | ||
inner_solution = solve_constraints( | ||
callee_type.variables, | ||
inner_constraints, | ||
strict=self.chk.in_checked_function(), | ||
allow_polymorphic=False, | ||
) | ||
inferred_args = inner_solution[0] | ||
else: # END NEW CODE | ||
pass | ||
|
||
if 2 in arg_pass_nums: | ||
# Second pass of type inference. | ||
|
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Without this hack,
testLiteralAndGenericWithUnion
fails (see footnote 1 in OP). Not sure if this is the appropriate way to do it