Skip to content

Commit cb1786f

Browse files
ddfisherddfisher
committed
Pr/strict optional (#1562)
First pass strict Optional checking. Adds the experimental `--strict-optional` flag. Fixes #1450. With --strict-optional: - "None" in type annotations refers to NoneTyp, except as a return value, where it still refers to Void - None and List[None] will now be inferred as a type if no other information is available. - Class variables may be initialized to None without having an Optional type. Mypy does not currently check that they're assigned to in __init__ or elsewhere before use. See #1450 for more details. This also fixes the bug where mypy didn't understand that x.y = "foo" implied that x.y would be a str for the remaineder of that block. This isn't entirely sound, but is in line with the way we do isinstance checks.
1 parent 56d75b8 commit cb1786f

24 files changed

+590
-107
lines changed

‎mypy/checker.py‎

Lines changed: 124 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@
3232
from mypy.types import (
3333
Type, AnyType, CallableType, Void, FunctionLike, Overloaded, TupleType,
3434
Instance, NoneTyp, ErrorType, strip_type,
35-
UnionType, TypeVarType, PartialType, DeletedType
35+
UnionType, TypeVarType, PartialType, DeletedType, UninhabitedType
3636
)
3737
from mypy.sametypes import is_same_type
3838
from mypy.messages import MessageBuilder
@@ -53,6 +53,8 @@
5353
from mypy.treetransform import TransformVisitor
5454
from mypy.meet import meet_simple, nearest_builtin_ancestor, is_overlapping_types
5555

56+
from mypy import experiments
57+
5658

5759
T = TypeVar('T')
5860

@@ -223,14 +225,14 @@ def get_declaration(self, expr: Any) -> Type:
223225
else:
224226
return self.frames[0].get(expr.literal_hash)
225227

226-
def assign_type(self, expr: Node, type: Type,
228+
def assign_type(self, expr: Node,
229+
type: Type,
230+
declared_type: Type,
227231
restrict_any: bool = False) -> None:
228232
if not expr.literal:
229233
return
230234
self.invalidate_dependencies(expr)
231235

232-
declared_type = self.get_declaration(expr)
233-
234236
if declared_type is None:
235237
# Not sure why this happens. It seems to mainly happen in
236238
# member initialization.
@@ -1200,18 +1202,31 @@ def check_assignment(self, lvalue: Node, rvalue: Node, infer_lvalue_type: bool =
12001202
partial_types = self.find_partial_types(var)
12011203
if partial_types is not None:
12021204
if not self.current_node_deferred:
1203-
var.type = rvalue_type
1205+
if experiments.STRICT_OPTIONAL:
1206+
var.type = UnionType.make_simplified_union(
1207+
[rvalue_type, NoneTyp()])
1208+
else:
1209+
var.type = rvalue_type
12041210
else:
12051211
var.type = None
12061212
del partial_types[var]
12071213
# Try to infer a partial type. No need to check the return value, as
12081214
# an error will be reported elsewhere.
12091215
self.infer_partial_type(lvalue_type.var, lvalue, rvalue_type)
12101216
return
1211-
rvalue_type = self.check_simple_assignment(lvalue_type, rvalue, lvalue)
1217+
if (is_literal_none(rvalue) and
1218+
isinstance(lvalue, NameExpr) and
1219+
isinstance(lvalue.node, Var) and
1220+
lvalue.node.is_initialized_in_class):
1221+
# Allow None's to be assigned to class variables with non-Optional types.
1222+
rvalue_type = lvalue_type
1223+
else:
1224+
rvalue_type = self.check_simple_assignment(lvalue_type, rvalue, lvalue)
12121225

12131226
if rvalue_type and infer_lvalue_type:
1214-
self.binder.assign_type(lvalue, rvalue_type,
1227+
self.binder.assign_type(lvalue,
1228+
rvalue_type,
1229+
lvalue_type,
12151230
self.typing_mode_weak())
12161231
elif index_lvalue:
12171232
self.check_indexed_assignment(index_lvalue, rvalue, rvalue)
@@ -1444,7 +1459,7 @@ def infer_variable_type(self, name: Var, lvalue: Node,
14441459
"""Infer the type of initialized variables from initializer type."""
14451460
if self.typing_mode_weak():
14461461
self.set_inferred_type(name, lvalue, AnyType())
1447-
self.binder.assign_type(lvalue, init_type, True)
1462+
self.binder.assign_type(lvalue, init_type, self.binder.get_declaration(lvalue), True)
14481463
elif isinstance(init_type, Void):
14491464
self.check_not_void(init_type, context)
14501465
self.set_inference_error_fallback_type(name, lvalue, init_type, context)
@@ -1467,16 +1482,16 @@ def infer_variable_type(self, name: Var, lvalue: Node,
14671482
self.set_inferred_type(name, lvalue, init_type)
14681483

14691484
def infer_partial_type(self, name: Var, lvalue: Node, init_type: Type) -> bool:
1470-
if isinstance(init_type, NoneTyp):
1471-
partial_type = PartialType(None, name)
1485+
if isinstance(init_type, (NoneTyp, UninhabitedType)):
1486+
partial_type = PartialType(None, name, [init_type])
14721487
elif isinstance(init_type, Instance):
14731488
fullname = init_type.type.fullname()
1474-
if ((fullname == 'builtins.list' or fullname == 'builtins.set' or
1475-
fullname == 'builtins.dict')
1476-
and isinstance(init_type.args[0], NoneTyp)
1477-
and (fullname != 'builtins.dict' or isinstance(init_type.args[1], NoneTyp))
1478-
and isinstance(lvalue, NameExpr)):
1479-
partial_type = PartialType(init_type.type, name)
1489+
if (isinstance(lvalue, NameExpr) and
1490+
(fullname == 'builtins.list' or
1491+
fullname == 'builtins.set' or
1492+
fullname == 'builtins.dict') and
1493+
all(isinstance(t, (NoneTyp, UninhabitedType)) for t in init_type.args)):
1494+
partial_type = PartialType(init_type.type, name, init_type.args)
14801495
else:
14811496
return False
14821497
else:
@@ -1559,8 +1574,8 @@ def try_infer_partial_type_from_indexed_assignment(
15591574
self, lvalue: IndexExpr, rvalue: Node) -> None:
15601575
# TODO: Should we share some of this with try_infer_partial_type?
15611576
if isinstance(lvalue.base, RefExpr) and isinstance(lvalue.base.node, Var):
1562-
var = cast(Var, lvalue.base.node)
1563-
if var is not None and isinstance(var.type, PartialType):
1577+
var = lvalue.base.node
1578+
if isinstance(var.type, PartialType):
15641579
type_type = var.type.type
15651580
if type_type is None:
15661581
return # The partial type is None.
@@ -1572,10 +1587,15 @@ def try_infer_partial_type_from_indexed_assignment(
15721587
# TODO: Don't infer things twice.
15731588
key_type = self.accept(lvalue.index)
15741589
value_type = self.accept(rvalue)
1575-
if is_valid_inferred_type(key_type) and is_valid_inferred_type(value_type):
1590+
full_key_type = UnionType.make_simplified_union(
1591+
[key_type, var.type.inner_types[0]])
1592+
full_value_type = UnionType.make_simplified_union(
1593+
[value_type, var.type.inner_types[1]])
1594+
if (is_valid_inferred_type(full_key_type) and
1595+
is_valid_inferred_type(full_value_type)):
15761596
if not self.current_node_deferred:
15771597
var.type = self.named_generic_type('builtins.dict',
1578-
[key_type, value_type])
1598+
[full_key_type, full_value_type])
15791599
del partial_types[var]
15801600

15811601
def visit_expression_stmt(self, s: ExpressionStmt) -> Type:
@@ -1881,7 +1901,10 @@ def analyze_iterable_item_type(self, expr: Node) -> Type:
18811901

18821902
self.check_not_void(iterable, expr)
18831903
if isinstance(iterable, TupleType):
1884-
joined = NoneTyp() # type: Type
1904+
if experiments.STRICT_OPTIONAL:
1905+
joined = UninhabitedType() # type: Type
1906+
else:
1907+
joined = NoneTyp()
18851908
for item in iterable.items:
18861909
joined = join_types(joined, item)
18871910
if isinstance(joined, ErrorType):
@@ -1932,7 +1955,9 @@ def flatten(t: Node) -> List[Node]:
19321955
s.expr.accept(self)
19331956
for elt in flatten(s.expr):
19341957
if isinstance(elt, NameExpr):
1935-
self.binder.assign_type(elt, DeletedType(source=elt.name),
1958+
self.binder.assign_type(elt,
1959+
DeletedType(source=elt.name),
1960+
self.binder.get_declaration(elt),
19361961
self.typing_mode_weak())
19371962
return None
19381963

@@ -2311,8 +2336,12 @@ def leave_partial_types(self) -> None:
23112336
partial_types = self.partial_types.pop()
23122337
if not self.current_node_deferred:
23132338
for var, context in partial_types.items():
2314-
self.msg.fail(messages.NEED_ANNOTATION_FOR_VAR, context)
2315-
var.type = AnyType()
2339+
if experiments.STRICT_OPTIONAL and cast(PartialType, var.type).type is None:
2340+
# None partial type: assume variable is intended to have type None
2341+
var.type = NoneTyp()
2342+
else:
2343+
self.msg.fail(messages.NEED_ANNOTATION_FOR_VAR, context)
2344+
var.type = AnyType()
23162345

23172346
def find_partial_types(self, var: Var) -> Optional[Dict[Var, Context]]:
23182347
for partial_types in reversed(self.partial_types):
@@ -2356,11 +2385,48 @@ def method_type(self, func: FuncBase) -> FunctionLike:
23562385
return method_type_with_fallback(func, self.named_type('builtins.function'))
23572386

23582387

2388+
def conditional_type_map(expr: Node,
2389+
current_type: Optional[Type],
2390+
proposed_type: Optional[Type],
2391+
*,
2392+
weak: bool = False
2393+
) -> Tuple[Optional[Dict[Node, Type]], Optional[Dict[Node, Type]]]:
2394+
"""Takes in an expression, the current type of the expression, and a
2395+
proposed type of that expression.
2396+
2397+
Returns a 2-tuple: The first element is a map from the expression to
2398+
the proposed type, if the expression can be the proposed type. The
2399+
second element is a map from the expression to the type it would hold
2400+
if it was not the proposed type, if any."""
2401+
if proposed_type:
2402+
if current_type:
2403+
if is_proper_subtype(current_type, proposed_type):
2404+
return {expr: proposed_type}, None
2405+
elif not is_overlapping_types(current_type, proposed_type):
2406+
return None, {expr: current_type}
2407+
else:
2408+
remaining_type = restrict_subtype_away(current_type, proposed_type)
2409+
return {expr: proposed_type}, {expr: remaining_type}
2410+
else:
2411+
return {expr: proposed_type}, {}
2412+
else:
2413+
# An isinstance check, but we don't understand the type
2414+
if weak:
2415+
return {expr: AnyType()}, {expr: current_type}
2416+
else:
2417+
return {}, {}
2418+
2419+
2420+
def is_literal_none(n: Node) -> bool:
2421+
return isinstance(n, NameExpr) and n.fullname == 'builtins.None'
2422+
2423+
23592424
def find_isinstance_check(node: Node,
23602425
type_map: Dict[Node, Type],
2361-
weak: bool=False) \
2362-
-> Tuple[Optional[Dict[Node, Type]], Optional[Dict[Node, Type]]]:
2363-
"""Find any isinstance checks (within a chain of ands).
2426+
weak: bool=False
2427+
) -> Tuple[Optional[Dict[Node, Type]], Optional[Dict[Node, Type]]]:
2428+
"""Find any isinstance checks (within a chain of ands). Includes
2429+
implicit and explicit checks for None.
23642430
23652431
Return value is a map of variables to their types if the condition
23662432
is true and a map of variables to their types if the condition is false.
@@ -2376,20 +2442,31 @@ def find_isinstance_check(node: Node,
23762442
if expr.literal == LITERAL_TYPE:
23772443
vartype = type_map[expr]
23782444
type = get_isinstance_type(node.args[1], type_map)
2379-
if type:
2380-
elsetype = vartype
2381-
if vartype:
2382-
if is_proper_subtype(vartype, type):
2383-
return {expr: type}, None
2384-
elif not is_overlapping_types(vartype, type):
2385-
return None, {expr: elsetype}
2386-
else:
2387-
elsetype = restrict_subtype_away(vartype, type)
2388-
return {expr: type}, {expr: elsetype}
2389-
else:
2390-
# An isinstance check, but we don't understand the type
2391-
if weak:
2392-
return {expr: AnyType()}, {expr: vartype}
2445+
return conditional_type_map(expr, vartype, type, weak=weak)
2446+
elif (isinstance(node, ComparisonExpr) and any(is_literal_none(n) for n in node.operands) and
2447+
experiments.STRICT_OPTIONAL):
2448+
# Check for `x is None` and `x is not None`.
2449+
is_not = node.operators == ['is not']
2450+
if is_not or node.operators == ['is']:
2451+
if_vars = {} # type: Dict[Node, Type]
2452+
else_vars = {} # type: Dict[Node, Type]
2453+
for expr in node.operands:
2454+
if expr.literal == LITERAL_TYPE and not is_literal_none(expr) and expr in type_map:
2455+
# This should only be true at most once: there should be
2456+
# two elements in node.operands, and at least one of them
2457+
# should represent a None.
2458+
vartype = type_map[expr]
2459+
if_vars, else_vars = conditional_type_map(expr, vartype, NoneTyp(), weak=weak)
2460+
break
2461+
2462+
if is_not:
2463+
if_vars, else_vars = else_vars, if_vars
2464+
return if_vars, else_vars
2465+
elif isinstance(node, RefExpr) and experiments.STRICT_OPTIONAL:
2466+
# The type could be falsy, so we can't deduce anything new about the else branch
2467+
vartype = type_map[node]
2468+
_, if_vars = conditional_type_map(node, vartype, NoneTyp(), weak=weak)
2469+
return if_vars, {}
23932470
elif isinstance(node, OpExpr) and node.op == 'and':
23942471
left_if_vars, right_else_vars = find_isinstance_check(
23952472
node.left,
@@ -2571,6 +2648,12 @@ def is_valid_inferred_type(typ: Type) -> bool:
25712648
Examples of invalid types include the None type or a type with a None component.
25722649
"""
25732650
if is_same_type(typ, NoneTyp()):
2651+
# With strict Optional checking, we *may* eventually infer NoneTyp, but
2652+
# we only do that if we can't infer a specific Optional type. This
2653+
# resolution happens in leave_partial_types when we pop a partial types
2654+
# scope.
2655+
return False
2656+
if is_same_type(typ, UninhabitedType()):
25742657
return False
25752658
elif isinstance(typ, Instance):
25762659
for arg in typ.args:

0 commit comments

Comments
 (0)