From 9be68f56c99dc250bca90bbb80d4f9d306bac53c Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Wed, 8 Dec 2021 08:30:10 -0800 Subject: [PATCH 01/13] support overloads from typeshed --- pyanalyze/annotations.py | 3 ++- pyanalyze/test_signature.py | 24 ++++++++++++++++++++++++ pyanalyze/typeshed.py | 29 +++++++++++++++++++++++------ 3 files changed, 49 insertions(+), 7 deletions(-) diff --git a/pyanalyze/annotations.py b/pyanalyze/annotations.py index 1f307c13..1c382114 100644 --- a/pyanalyze/annotations.py +++ b/pyanalyze/annotations.py @@ -30,6 +30,7 @@ import ast import builtins from collections.abc import Callable, Iterable +from typed_ast import ast3 from typing import ( Any, Container, @@ -745,7 +746,7 @@ def visit_Expr(self, node: ast.Expr) -> Value: return self.visit(node.value) def visit_BinOp(self, node: ast.BinOp) -> Optional[Value]: - if isinstance(node.op, ast.BitOr): + if isinstance(node.op, (ast.BitOr, ast3.BitOr)): return _SubscriptedValue( KnownValue(Union), (self.visit(node.left), self.visit(node.right)) ) diff --git a/pyanalyze/test_signature.py b/pyanalyze/test_signature.py index 0562fd0c..d5506a71 100644 --- a/pyanalyze/test_signature.py +++ b/pyanalyze/test_signature.py @@ -1106,3 +1106,27 @@ def capybara( assert_is_value(val2, AnyValue(AnySource.multiple_overload_matches)) val3 = overloaded2("x", int_or_str_or_float) # E: incompatible_argument assert_is_value(val3, AnyValue(AnySource.error)) + + @assert_passes() + def test_typeshed_overload(self): + class SupportsWrite: + def write(self, s: str) -> None: + pass + + class SupportsWriteAndFlush(SupportsWrite): + def flush(self) -> None: + pass + + def capybara(): + print() # ok + print("x", file=SupportsWrite()) + print("x", file=SupportsWrite(), flush=True) # E: incompatible_argument + print("x", file=SupportsWriteAndFlush(), flush=True) + print("x", file=SupportsWriteAndFlush()) + print("x", file="not a file") # E: incompatible_call + + def pacarana(f: float): + assert_is_value(f.__round__(), TypedValue(int)) + assert_is_value(f.__round__(None), TypedValue(int)) + f.__round__(ndigits=None) # E: incompatible_call + assert_is_value(f.__round__(1), TypedValue(float)) diff --git a/pyanalyze/typeshed.py b/pyanalyze/typeshed.py index 595683e2..e322905c 100644 --- a/pyanalyze/typeshed.py +++ b/pyanalyze/typeshed.py @@ -8,7 +8,7 @@ from .error_code import ErrorCode from .safe import is_typing_name from .stacked_scopes import uniq_chain -from .signature import SigParameter, Signature +from .signature import ConcreteSignature, OverloadedSignature, SigParameter, Signature from .value import ( AnySource, AnyValue, @@ -52,6 +52,7 @@ Callable, List, TypeVar, + overload, ) from typing_extensions import Protocol, TypedDict import typeshed_client @@ -110,7 +111,7 @@ def log(self, message: str, obj: object) -> None: return print("%s: %r" % (message, obj)) - def get_argspec(self, obj: object) -> Optional[Signature]: + def get_argspec(self, obj: object) -> Optional[ConcreteSignature]: if inspect.ismethoddescriptor(obj) and hasattr(obj, "__objclass__"): objclass = obj.__objclass__ fq_name = self._get_fq_name(objclass) @@ -152,7 +153,7 @@ def get_argspec(self, obj: object) -> Optional[Signature]: def get_argspec_for_fully_qualified_name( self, fq_name: str, obj: object - ) -> Optional[Signature]: + ) -> Optional[ConcreteSignature]: info = self._get_info_for_name(fq_name) mod, _ = fq_name.rsplit(".", maxsplit=1) sig = self._get_signature_from_info(info, obj, fq_name, mod) @@ -502,7 +503,7 @@ def _get_method_signature_from_info( fq_name: str, mod: str, objclass: type, - ) -> Optional[Signature]: + ) -> Optional[ConcreteSignature]: if info is None: return None elif isinstance(info, typeshed_client.ImportedInfo): @@ -554,10 +555,24 @@ def _get_signature_from_info( fq_name: str, mod: str, objclass: Optional[type] = None, - ) -> Optional[Signature]: + ) -> Optional[ConcreteSignature]: if isinstance(info, typeshed_client.NameInfo): if isinstance(info.ast, (ast3.FunctionDef, ast3.AsyncFunctionDef)): return self._get_signature_from_func_def(info.ast, obj, mod, objclass) + elif isinstance(info.ast, typeshed_client.OverloadedName): + sigs = [] + for defn in info.ast.definitions: + if not isinstance(defn, (ast3.FunctionDef, ast3.AsyncFunctionDef)): + self.log( + "Ignoring unrecognized AST in overload", (fq_name, info) + ) + return None + sig = self._get_signature_from_func_def(defn, obj, mod, objclass) + if sig is None: + self.log("Could not get sig for overload member", (defn,)) + return None + sigs.append(sig) + return OverloadedSignature(sigs) else: self.log("Ignoring unrecognized AST", (fq_name, info)) return None @@ -587,7 +602,9 @@ def _get_signature_from_func_def( is_classmethod = is_staticmethod = False for decorator_ast in node.decorator_list: decorator = self._parse_expr(decorator_ast, mod) - if decorator == KnownValue(abstractmethod): + if decorator == KnownValue(abstractmethod) or decorator == KnownValue( + overload + ): continue elif decorator == KnownValue(classmethod): is_classmethod = True From df212c16235a7118478ccfa75a3e9332ebc2fa3a Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 9 Dec 2021 20:30:25 -0800 Subject: [PATCH 02/13] fix one of the errors --- pyanalyze/node_visitor.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pyanalyze/node_visitor.py b/pyanalyze/node_visitor.py index ef59ae9b..cc6b861a 100644 --- a/pyanalyze/node_visitor.py +++ b/pyanalyze/node_visitor.py @@ -473,10 +473,11 @@ def _apply_changes(cls, changes: Dict[str, List[Replacement]]) -> None: @classmethod def _apply_changes_to_lines( - cls, changes: List[Replacement], lines: Sequence[str] + cls, changes: List[Replacement], input_lines: Sequence[str] ) -> Sequence[str]: # only apply the first change because that change might affect other fixes # that test_scope came up for that file. So we break after finding first applicable fix. + lines = list(input_lines) if changes: change = changes[0] additions = change.lines_to_add @@ -484,7 +485,7 @@ def _apply_changes_to_lines( lines_to_remove = change.linenos_to_delete max_line = max(lines_to_remove) # add the additions after the max_line - lines = lines[:max_line] + additions + lines[max_line:] + lines = [*lines[:max_line], *additions, *lines[max_line:]] lines_to_remove = sorted(lines_to_remove, reverse=True) for lineno in lines_to_remove: del lines[lineno - 1] From 2f230d082fd7493387446c4ed1a183eb65acb973 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 9 Dec 2021 19:30:57 -0800 Subject: [PATCH 03/13] make test context more powerful --- pyanalyze/test_value.py | 31 ++++--------------------------- 1 file changed, 4 insertions(+), 27 deletions(-) diff --git a/pyanalyze/test_value.py b/pyanalyze/test_value.py index 2e8348fb..dbf087c5 100644 --- a/pyanalyze/test_value.py +++ b/pyanalyze/test_value.py @@ -3,7 +3,7 @@ import enum import io import pickle -from typing import NewType, Sequence, Union +from typing import NewType from typing_extensions import Protocol, runtime_checkable import typing import types @@ -14,17 +14,15 @@ from . import value from .checker import Checker from .name_check_visitor import NameCheckVisitor -from .signature import Signature, MaybeSignature +from .signature import Signature from .stacked_scopes import Composite from .test_config import TestConfig -from .type_object import TypeObject from .value import ( AnnotatedValue, AnySource, AnyValue, CallableValue, CanAssignError, - GenericBases, KVPair, Value, GenericValue, @@ -32,34 +30,13 @@ TypedValue, MultiValuedValue, SubclassValue, - CanAssignContext, SequenceIncompleteValue, TypeVarMap, concrete_values_from_iterable, ) - -class Context(CanAssignContext): - def __init__(self) -> None: - self.checker = Checker(TestConfig()) - self.visitor = NameCheckVisitor("", "", ast.parse(""), checker=self.checker) - - def make_type_object(self, typ: Union[type, super]) -> TypeObject: - return self.checker.make_type_object(typ) - - def get_generic_bases( - self, typ: Union[type, str], generic_args: Sequence[Value] = () - ) -> GenericBases: - return self.checker.arg_spec_cache.get_generic_bases(typ, generic_args) - - def signature_from_value(self, value: Value) -> MaybeSignature: - return self.visitor.signature_from_value(value) - - def get_attribute_from_value(self, root_value: Value, attribute: str) -> Value: - return self.visitor.get_attribute(Composite(root_value), attribute) - - -CTX = Context() +_checker = Checker(TestConfig()) +CTX = NameCheckVisitor("", "", ast.parse(""), checker=_checker) def assert_cannot_assign(left: Value, right: Value) -> None: From 39a15a8d7610d3b353f3bcf6bb82d69eacf05245 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 9 Dec 2021 19:31:13 -0800 Subject: [PATCH 04/13] support multiple subscripting --- pyanalyze/annotations.py | 242 +++++++++++++++++++-------------------- 1 file changed, 120 insertions(+), 122 deletions(-) diff --git a/pyanalyze/annotations.py b/pyanalyze/annotations.py index d3a734de..62100825 100644 --- a/pyanalyze/annotations.py +++ b/pyanalyze/annotations.py @@ -495,137 +495,135 @@ def _type_from_value(value: Value, ctx: Context, is_typeddict: bool = False) -> elif isinstance(value, AnnotatedValue): return _type_from_value(value.value, ctx) elif isinstance(value, _SubscriptedValue): - if isinstance(value.root, GenericValue): - if len(value.root.args) == len(value.members): - return GenericValue( - value.root.typ, - [_type_from_value(member, ctx) for member in value.members], - ) - if isinstance(value.root, _SubscriptedValue): - root_type = _type_from_value(value.root, ctx) - if isinstance(root_type, GenericValue) and len(root_type.args) == len( - value.members - ): - return GenericValue( - root_type.typ, - [_type_from_value(member, ctx) for member in value.members], - ) - if isinstance(value.root, TypedValue) and isinstance(value.root.typ, str): + return _type_from_subscripted_value( + value.root, value.members, ctx, is_typeddict=is_typeddict + ) + elif isinstance(value, AnyValue): + return value + elif isinstance(value, TypedValue) and isinstance(value.typ, str): + # Synthetic type + return value + else: + ctx.show_error(f"Unrecognized annotation {value}") + return AnyValue(AnySource.error) + + +def _type_from_subscripted_value( + root: Optional[Value], + members: Sequence[Value], + ctx: Context, + is_typeddict: bool = False, +) -> Value: + if isinstance(root, GenericValue): + if len(root.args) == len(members): return GenericValue( - value.root.typ, [_type_from_value(elt, ctx) for elt in value.members] + root.typ, [_type_from_value(member, ctx) for member in members] ) + if isinstance(root, _SubscriptedValue): + root_type = _type_from_value(root, ctx) + return _type_from_subscripted_value(root_type, members, ctx) + elif isinstance(root, MultiValuedValue): + return unite_values( + *[ + _type_from_subscripted_value(subval, members, ctx, is_typeddict) + for subval in root.vals + ] + ) + if isinstance(root, TypedValue) and isinstance(root.typ, str): + return GenericValue(root.typ, [_type_from_value(elt, ctx) for elt in members]) - if not isinstance(value.root, KnownValue): - ctx.show_error(f"Cannot resolve subscripted annotation: {value.root}") + if not isinstance(root, KnownValue): + ctx.show_error(f"Cannot resolve subscripted annotation: {root}") + return AnyValue(AnySource.error) + root = root.val + if root is typing.Union: + return unite_values(*[_type_from_value(elt, ctx) for elt in members]) + elif is_typing_name(root, "Literal"): + # Note that in Python 3.8, the way typing's internal cache works means that + # Literal[1] and Literal[True] are cached to the same value, so if you use + # both, you'll get whichever one was used first in later calls. There's nothing + # we can do about that. + if all(isinstance(elt, KnownValue) for elt in members): + return unite_values(*members) + else: + ctx.show_error(f"Arguments to Literal[] must be literals, not {members}") return AnyValue(AnySource.error) - root = value.root.val - if root is typing.Union: - return unite_values(*[_type_from_value(elt, ctx) for elt in value.members]) - elif is_typing_name(root, "Literal"): - # Note that in Python 3.8, the way typing's internal cache works means that - # Literal[1] and Literal[True] are cached to the same value, so if you use - # both, you'll get whichever one was used first in later calls. There's nothing - # we can do about that. - if all(isinstance(elt, KnownValue) for elt in value.members): - return unite_values(*value.members) - else: - ctx.show_error( - f"Arguments to Literal[] must be literals, not {value.members}" - ) - return AnyValue(AnySource.error) - elif root is typing.Tuple or root is tuple: - if len(value.members) == 2 and value.members[1] == KnownValue(Ellipsis): - return GenericValue(tuple, [_type_from_value(value.members[0], ctx)]) - elif len(value.members) == 1 and value.members[0] == KnownValue(()): - return SequenceIncompleteValue(tuple, []) - else: - return SequenceIncompleteValue( - tuple, [_type_from_value(arg, ctx) for arg in value.members] - ) - elif root is typing.Optional: - if len(value.members) != 1: - ctx.show_error("Optional[] takes only one argument") - return AnyValue(AnySource.error) - return unite_values( - KnownValue(None), _type_from_value(value.members[0], ctx) - ) - elif root is typing.Type or root is type: - if len(value.members) != 1: - ctx.show_error("Type[] takes only one argument") - return AnyValue(AnySource.error) - argument = _type_from_value(value.members[0], ctx) - return SubclassValue.make(argument) - elif is_typing_name(root, "Annotated"): - origin, *metadata = value.members - return _make_annotated(_type_from_value(origin, ctx), metadata, ctx) - elif is_typing_name(root, "TypeGuard"): - if len(value.members) != 1: - ctx.show_error("TypeGuard requires a single argument") - return AnyValue(AnySource.error) - return AnnotatedValue( - TypedValue(bool), - [TypeGuardExtension(_type_from_value(value.members[0], ctx))], + elif root is typing.Tuple or root is tuple: + if len(members) == 2 and members[1] == KnownValue(Ellipsis): + return GenericValue(tuple, [_type_from_value(members[0], ctx)]) + elif len(members) == 1 and members[0] == KnownValue(()): + return SequenceIncompleteValue(tuple, []) + else: + return SequenceIncompleteValue( + tuple, [_type_from_value(arg, ctx) for arg in members] ) - elif is_typing_name(root, "Required"): - if not is_typeddict: - ctx.show_error("Required[] used in unsupported context") - return AnyValue(AnySource.error) - if len(value.members) != 1: - ctx.show_error("Required[] requires a single argument") - return AnyValue(AnySource.error) - return _Pep655Value(True, _type_from_value(value.members[0], ctx)) - elif is_typing_name(root, "NotRequired"): - if not is_typeddict: - ctx.show_error("NotRequired[] used in unsupported context") - return AnyValue(AnySource.error) - if len(value.members) != 1: - ctx.show_error("NotRequired[] requires a single argument") - return AnyValue(AnySource.error) - return _Pep655Value(False, _type_from_value(value.members[0], ctx)) - elif root is Callable or root is typing.Callable: - if len(value.members) == 2: - args, return_value = value.members - return _make_callable_from_value(args, return_value, ctx) - ctx.show_error("Callable requires exactly two arguments") + elif root is typing.Optional: + if len(members) != 1: + ctx.show_error("Optional[] takes only one argument") return AnyValue(AnySource.error) - elif root is AsynqCallable: - if len(value.members) == 2: - args, return_value = value.members - return _make_callable_from_value(args, return_value, ctx, is_asynq=True) - ctx.show_error("AsynqCallable requires exactly two arguments") + return unite_values(KnownValue(None), _type_from_value(members[0], ctx)) + elif root is typing.Type or root is type: + if len(members) != 1: + ctx.show_error("Type[] takes only one argument") return AnyValue(AnySource.error) - elif typing_inspect.is_generic_type(root): - origin = typing_inspect.get_origin(root) - if origin is None: - # On Python 3.9 at least, get_origin() of a class that inherits - # from Generic[T] is None. - origin = root - if getattr(origin, "__extra__", None) is not None: - origin = origin.__extra__ - return GenericValue( - origin, [_type_from_value(elt, ctx) for elt in value.members] - ) - elif isinstance(root, type): - return GenericValue( - root, [_type_from_value(elt, ctx) for elt in value.members] - ) - else: - # In Python 3.9, generics are implemented differently and typing.get_origin - # can help. - origin = get_origin(root) - if isinstance(origin, type): - return GenericValue( - origin, [_type_from_value(elt, ctx) for elt in value.members] - ) - ctx.show_error(f"Unrecognized subscripted annotation: {root}") + argument = _type_from_value(members[0], ctx) + return SubclassValue.make(argument) + elif is_typing_name(root, "Annotated"): + origin, *metadata = members + return _make_annotated(_type_from_value(origin, ctx), metadata, ctx) + elif is_typing_name(root, "TypeGuard"): + if len(members) != 1: + ctx.show_error("TypeGuard requires a single argument") return AnyValue(AnySource.error) - elif isinstance(value, AnyValue): - return value - elif isinstance(value, TypedValue) and isinstance(value.typ, str): - # Synthetic type - return value + return AnnotatedValue( + TypedValue(bool), [TypeGuardExtension(_type_from_value(members[0], ctx))] + ) + elif is_typing_name(root, "Required"): + if not is_typeddict: + ctx.show_error("Required[] used in unsupported context") + return AnyValue(AnySource.error) + if len(members) != 1: + ctx.show_error("Required[] requires a single argument") + return AnyValue(AnySource.error) + return _Pep655Value(True, _type_from_value(members[0], ctx)) + elif is_typing_name(root, "NotRequired"): + if not is_typeddict: + ctx.show_error("NotRequired[] used in unsupported context") + return AnyValue(AnySource.error) + if len(members) != 1: + ctx.show_error("NotRequired[] requires a single argument") + return AnyValue(AnySource.error) + return _Pep655Value(False, _type_from_value(members[0], ctx)) + elif root is Callable or root is typing.Callable: + if len(members) == 2: + args, return_value = members + return _make_callable_from_value(args, return_value, ctx) + ctx.show_error("Callable requires exactly two arguments") + return AnyValue(AnySource.error) + elif root is AsynqCallable: + if len(members) == 2: + args, return_value = members + return _make_callable_from_value(args, return_value, ctx, is_asynq=True) + ctx.show_error("AsynqCallable requires exactly two arguments") + return AnyValue(AnySource.error) + elif typing_inspect.is_generic_type(root): + origin = typing_inspect.get_origin(root) + if origin is None: + # On Python 3.9 at least, get_origin() of a class that inherits + # from Generic[T] is None. + origin = root + if getattr(origin, "__extra__", None) is not None: + origin = origin.__extra__ + return GenericValue(origin, [_type_from_value(elt, ctx) for elt in members]) + elif isinstance(root, type): + return GenericValue(root, [_type_from_value(elt, ctx) for elt in members]) else: - ctx.show_error(f"Unrecognized annotation {value}") + # In Python 3.9, generics are implemented differently and typing.get_origin + # can help. + origin = get_origin(root) + if isinstance(origin, type): + return GenericValue(origin, [_type_from_value(elt, ctx) for elt in members]) + ctx.show_error(f"Unrecognized subscripted annotation: {root}") return AnyValue(AnySource.error) From 8a6870be1b27bb5784e8f180c33d720e43d91841 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 9 Dec 2021 19:31:22 -0800 Subject: [PATCH 05/13] typeshed integration test --- pyanalyze/test_typeshed.py | 45 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/pyanalyze/test_typeshed.py b/pyanalyze/test_typeshed.py index 9aba670e..d5405681 100644 --- a/pyanalyze/test_typeshed.py +++ b/pyanalyze/test_typeshed.py @@ -8,7 +8,10 @@ import sys import tempfile import time +from pyanalyze.test_value import CTX +import typeshed_client from typeshed_client import Resolver, get_search_context +from typed_ast import ast3 import typing from typing import Dict, Generic, List, TypeVar, NewType, Union from urllib.error import HTTPError @@ -392,3 +395,45 @@ def test_property(self) -> None: def test_http_error(self) -> None: tsf = TypeshedFinder(verbose=True) assert True is tsf.has_attribute(HTTPError, "read") + + +class TestIntegration: + """Tests that all files in typeshed are parsed without error.""" + + def test(self): + finder = CTX.arg_spec_cache.ts_finder + # finder.verbose = True + resolver = finder.resolver + print(resolver.ctx) + for module_name, module_path in sorted( + typeshed_client.get_all_stub_files(resolver.ctx) + ): + if module_name in ("this", "antigravity"): + continue # please stop opening my browser + names = typeshed_client.get_stub_names( + module_name, search_context=resolver.ctx + ) + for name, info in names.items(): + is_function = isinstance( + info.ast, + ( + ast3.FunctionDef, + ast3.AsyncFunctionDef, + typeshed_client.OverloadedName, + ), + ) + fq_name = f"{module_name}.{name}" + if is_function: + sig = finder.get_argspec_for_fully_qualified_name(fq_name, None) + if sig is None: + print(fq_name) + val = finder.resolve_name(module_name, name) + if val == AnyValue(AnySource.inference): + if is_function: + # Probably a function that only exists on another OS. + continue + if isinstance(info.ast, typeshed_client.ImportedName): + # Some legitimate triggers on installed stubs + continue + print(module_name, name, info) + assert False From 5dd120d3cfbfc1b62f4b5c9294e0b740448ad8fc Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Sun, 12 Dec 2021 08:10:20 -0800 Subject: [PATCH 06/13] stubwalk --- pyanalyze/stubwalk.py | 196 +++++++++++++++++++++++++++++++++++++ pyanalyze/test_stubwalk.py | 8 ++ 2 files changed, 204 insertions(+) create mode 100644 pyanalyze/stubwalk.py create mode 100644 pyanalyze/test_stubwalk.py diff --git a/pyanalyze/stubwalk.py b/pyanalyze/stubwalk.py new file mode 100644 index 00000000..88a14c3d --- /dev/null +++ b/pyanalyze/stubwalk.py @@ -0,0 +1,196 @@ +""" + +A tool for walking over stubs and checking that pyanalyze +can handle them. + +""" +import ast +from ast_decompiler import decompile +from dataclasses import dataclass +import enum +from pathlib import Path +import textwrap +from pyanalyze.signature import ConcreteSignature +import typeshed_client +from typed_ast import ast3 +from typing import Collection, Container, Iterable, Optional, Sequence, Union + +from .config import Config +from .value import AnySource, AnyValue, TypedValue, Value +from .checker import Checker +from .name_check_visitor import NameCheckVisitor + +_checker = Checker(Config()) +CTX = NameCheckVisitor("", "", ast.parse(""), checker=_checker) + + +class ErrorCode(enum.Enum): + unresolved_import = 1 + unresolved_function = 2 + unresolved_object = 3 + signature_failed = 4 + unresolved_type_in_signature = 5 + unused_allowlist_entry = 6 + unresolved_bases = 7 + + +DISABLED_BY_DEFAULT = { + # False positives with imports that only exist on another OS + ErrorCode.unresolved_import, + # Happens with functions that only exist on another OS + ErrorCode.unresolved_function, +} + + +def _try_decompile(node: ast3.AST) -> str: + try: + return decompile(node) + except Exception as e: + return f"could not decompile {ast3.dump(node)} due to {e}\n" + + +@dataclass +class Error: + code: ErrorCode + message: str + fully_qualified_name: str + ast: Union[ast3.AST, typeshed_client.OverloadedName, None] = None + + def display(self) -> str: + heading = f"{self.fully_qualified_name}: {self.message} ({self.code.name})\n" + if isinstance(self.ast, ast3.AST): + decompiled = _try_decompile(self.ast) + heading += textwrap.indent(decompiled, " ") + elif isinstance(self.ast, typeshed_client.OverloadedName): + lines = [ + textwrap.indent(_try_decompile(node), " ") + for node in self.ast.definitions + ] + heading += "".join(lines) + return heading + + +def stubwalk( + typeshed_path: Optional[Path] = None, + search_path: Sequence[Path] = (), + allowlist: Collection[str] = (), + disabled_codes: Container[ErrorCode] = DISABLED_BY_DEFAULT, + verbose: bool = True, +) -> Sequence[Error]: + search_context = CTX.arg_spec_cache.ts_finder.resolver.ctx + if typeshed_path is not None: + search_context = search_context._replace(typeshed=typeshed_path) + search_context = search_context._replace(search_path=search_path) + final_errors = [] + used_allowlist_entries = set() + for error in _stubwalk(search_context): + if verbose: + print(error.display(), end="") + if error.code in disabled_codes: + continue + if error.fully_qualified_name in allowlist: + used_allowlist_entries.add(error.fully_qualified_name) + continue + final_errors.append(error) + if ErrorCode.unused_allowlist_entry not in disabled_codes: + for unused_allowlist in set(allowlist) - used_allowlist_entries: + final_errors.append( + Error( + ErrorCode.unused_allowlist_entry, + "Unused allowlist entry", + unused_allowlist, + ) + ) + return final_errors + + +def _stubwalk(search_context: typeshed_client.SearchContext) -> Iterable[Error]: + finder = CTX.arg_spec_cache.ts_finder + resolver = finder.resolver + for module_name, _ in sorted(typeshed_client.get_all_stub_files(search_context)): + if module_name in ("this", "antigravity"): + continue # please stop opening my browser + names = typeshed_client.get_stub_names(module_name, search_context=resolver.ctx) + for name, info in names.items(): + is_function = isinstance( + info.ast, + ( + ast3.FunctionDef, + ast3.AsyncFunctionDef, + typeshed_client.OverloadedName, + ), + ) + fq_name = f"{module_name}.{name}" + if is_function: + sig = finder.get_argspec_for_fully_qualified_name(fq_name, None) + if sig is None: + yield Error( + ErrorCode.signature_failed, + "Cannot get signature for function", + fq_name, + info.ast, + ) + else: + yield from _error_on_nested_any(sig, "Signature", fq_name, info) + if isinstance(info.ast, ast3.ClassDef): + bases = finder.get_bases_for_fq_name(fq_name) + if bases is None: + yield Error( + ErrorCode.unresolved_bases, + "Cannot resolve bases", + fq_name, + info.ast, + ) + else: + for base in bases: + if not isinstance(base, TypedValue): + yield Error( + ErrorCode.unresolved_bases, + "Cannot resolve one of the bases", + fq_name, + info.ast, + ) + else: + yield from _error_on_nested_any(base, "Base", fq_name, info) + # TODO: + # - Loop over all attributes and assert their values don't contain Any + # - Loop over all methods and check their signatures + val = finder.resolve_name(module_name, name) + if val == AnyValue(AnySource.inference): + if is_function: + yield Error( + ErrorCode.unresolved_function, + "Cannot resolve function", + fq_name, + info.ast, + ) + elif isinstance(info.ast, typeshed_client.ImportedName): + yield Error( + ErrorCode.unresolved_import, + "Cannot resolve imported name", + fq_name, + info.ast, + ) + else: + yield Error( + ErrorCode.unresolved_object, + "Cannot resolve name", + fq_name, + info.ast, + ) + + +def _error_on_nested_any( + sig_or_val: Union[ConcreteSignature, Value], + label: str, + fq_name: str, + info: typeshed_client.NameInfo, +) -> Iterable[Error]: + for val in sig_or_val.walk_values(): + if val == AnyValue(AnySource.inference): + yield Error( + ErrorCode.unresolved_type_in_signature, + f"{label} {sig_or_val} contains unresolved type", + fq_name, + info.ast, + ) diff --git a/pyanalyze/test_stubwalk.py b/pyanalyze/test_stubwalk.py new file mode 100644 index 00000000..65be0b1f --- /dev/null +++ b/pyanalyze/test_stubwalk.py @@ -0,0 +1,8 @@ +from .stubwalk import stubwalk + + +def test_stubwalk() -> None: + errors = stubwalk(allowlist={"typing._promote"}) + if errors: + message = "".join(error.display() for error in errors) + raise AssertionError(message) From 55e753699ce271225f326d5c3bf554708c79fd38 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Wed, 22 Dec 2021 15:54:42 -0800 Subject: [PATCH 07/13] use ast --- pyanalyze/annotations.py | 6 ++-- pyanalyze/stubwalk.py | 17 ++++----- pyanalyze/test_typeshed.py | 6 ++-- pyanalyze/typeshed.py | 74 +++++++++++++++++++------------------- 4 files changed, 47 insertions(+), 56 deletions(-) diff --git a/pyanalyze/annotations.py b/pyanalyze/annotations.py index a45cbbad..d6e53d7b 100644 --- a/pyanalyze/annotations.py +++ b/pyanalyze/annotations.py @@ -29,10 +29,8 @@ import typing_inspect import qcore import ast -from typed_ast import ast3 import builtins from collections.abc import Callable, Iterable -from typed_ast import ast3 from typing import ( Any, Container, @@ -794,7 +792,7 @@ def visit_Expr(self, node: ast.Expr) -> Value: return self.visit(node.value) def visit_BinOp(self, node: ast.BinOp) -> Optional[Value]: - if isinstance(node.op, (ast.BitOr, ast3.BitOr)): + if isinstance(node.op, ast.BitOr): return _SubscriptedValue( KnownValue(Union), (self.visit(node.left), self.visit(node.right)) ) @@ -803,7 +801,7 @@ def visit_BinOp(self, node: ast.BinOp) -> Optional[Value]: def visit_UnaryOp(self, node: ast.UnaryOp) -> Optional[Value]: # Only int and float negation on literals are supported. - if isinstance(node.op, (ast.USub, ast3.USub)): + if isinstance(node.op, ast.USub): operand = self.visit(node.operand) if isinstance(operand, KnownValue) and isinstance( operand.val, (int, float) diff --git a/pyanalyze/stubwalk.py b/pyanalyze/stubwalk.py index 88a14c3d..7b0440fd 100644 --- a/pyanalyze/stubwalk.py +++ b/pyanalyze/stubwalk.py @@ -12,7 +12,6 @@ import textwrap from pyanalyze.signature import ConcreteSignature import typeshed_client -from typed_ast import ast3 from typing import Collection, Container, Iterable, Optional, Sequence, Union from .config import Config @@ -42,11 +41,11 @@ class ErrorCode(enum.Enum): } -def _try_decompile(node: ast3.AST) -> str: +def _try_decompile(node: ast.AST) -> str: try: return decompile(node) except Exception as e: - return f"could not decompile {ast3.dump(node)} due to {e}\n" + return f"could not decompile {ast.dump(node)} due to {e}\n" @dataclass @@ -54,11 +53,11 @@ class Error: code: ErrorCode message: str fully_qualified_name: str - ast: Union[ast3.AST, typeshed_client.OverloadedName, None] = None + ast: Union[ast.AST, typeshed_client.OverloadedName, None] = None def display(self) -> str: heading = f"{self.fully_qualified_name}: {self.message} ({self.code.name})\n" - if isinstance(self.ast, ast3.AST): + if isinstance(self.ast, ast.AST): decompiled = _try_decompile(self.ast) heading += textwrap.indent(decompiled, " ") elif isinstance(self.ast, typeshed_client.OverloadedName): @@ -114,11 +113,7 @@ def _stubwalk(search_context: typeshed_client.SearchContext) -> Iterable[Error]: for name, info in names.items(): is_function = isinstance( info.ast, - ( - ast3.FunctionDef, - ast3.AsyncFunctionDef, - typeshed_client.OverloadedName, - ), + (ast.FunctionDef, ast.AsyncFunctionDef, typeshed_client.OverloadedName), ) fq_name = f"{module_name}.{name}" if is_function: @@ -132,7 +127,7 @@ def _stubwalk(search_context: typeshed_client.SearchContext) -> Iterable[Error]: ) else: yield from _error_on_nested_any(sig, "Signature", fq_name, info) - if isinstance(info.ast, ast3.ClassDef): + if isinstance(info.ast, ast.ClassDef): bases = finder.get_bases_for_fq_name(fq_name) if bases is None: yield Error( diff --git a/pyanalyze/test_typeshed.py b/pyanalyze/test_typeshed.py index 6966de23..68acda87 100644 --- a/pyanalyze/test_typeshed.py +++ b/pyanalyze/test_typeshed.py @@ -11,7 +11,7 @@ from pyanalyze.test_value import CTX import typeshed_client from typeshed_client import Resolver, get_search_context -from typed_ast import ast3 +import ast import typing from typing import Dict, Generic, List, TypeVar, NewType, Union from urllib.error import HTTPError @@ -444,8 +444,8 @@ def test(self): is_function = isinstance( info.ast, ( - ast3.FunctionDef, - ast3.AsyncFunctionDef, + ast.FunctionDef, + ast.AsyncFunctionDef, typeshed_client.OverloadedName, ), ) diff --git a/pyanalyze/typeshed.py b/pyanalyze/typeshed.py index 5147cf5f..6ea1e5bd 100644 --- a/pyanalyze/typeshed.py +++ b/pyanalyze/typeshed.py @@ -50,7 +50,6 @@ Dict, Set, Tuple, - cast, Any, Generic, Iterable, @@ -63,7 +62,6 @@ ) from typing_extensions import Protocol, TypedDict import typeshed_client -from typed_ast import ast3 try: # 3.7+ @@ -112,7 +110,7 @@ def get_name(self, node: ast.Name) -> Value: class TypeshedFinder: verbose: bool = True resolver: typeshed_client.Resolver = field(default_factory=typeshed_client.Resolver) - _assignment_cache: Dict[Tuple[str, ast3.AST], Value] = field( + _assignment_cache: Dict[Tuple[str, ast.AST], Value] = field( default_factory=dict, repr=False, init=False ) _attribute_cache: Dict[Tuple[str, str, bool], Value] = field( @@ -364,13 +362,13 @@ def _get_attribute_from_info( info.info, ".".join(info.source_module), attr, on_class=on_class ) elif isinstance(info, typeshed_client.NameInfo): - if isinstance(info.ast, ast3.ClassDef): + if isinstance(info.ast, ast.ClassDef): if info.child_nodes and attr in info.child_nodes: child_info = info.child_nodes[attr] if isinstance(child_info, typeshed_client.NameInfo): - if isinstance(child_info.ast, ast3.AnnAssign): + if isinstance(child_info.ast, ast.AnnAssign): return self._parse_type(child_info.ast.annotation, mod) - elif isinstance(child_info.ast, ast3.FunctionDef): + elif isinstance(child_info.ast, ast.FunctionDef): decorators = [ self._parse_expr(decorator, mod) for decorator in child_info.ast.decorator_list @@ -386,13 +384,13 @@ def _get_attribute_from_info( return AnyValue(AnySource.inference) else: return CallableValue(sig) - elif isinstance(child_info.ast, ast3.AsyncFunctionDef): + elif isinstance(child_info.ast, ast.AsyncFunctionDef): return UNINITIALIZED_VALUE - elif isinstance(child_info.ast, ast3.Assign): + elif isinstance(child_info.ast, ast.Assign): return UNINITIALIZED_VALUE assert False, repr(child_info) return UNINITIALIZED_VALUE - elif isinstance(info.ast, ast3.Assign): + elif isinstance(info.ast, ast.Assign): val = self._parse_type(info.ast.value, mod) if isinstance(val, KnownValue) and isinstance(val.val, type): return self.get_attribute(val.val, attr, on_class=on_class) @@ -410,11 +408,11 @@ def _get_child_info( elif isinstance(info, typeshed_client.ImportedInfo): return self._get_child_info(info.info, attr, ".".join(info.source_module)) elif isinstance(info, typeshed_client.NameInfo): - if isinstance(info.ast, ast3.ClassDef): + if isinstance(info.ast, ast.ClassDef): if info.child_nodes and attr in info.child_nodes: return info.child_nodes[attr], mod return None - elif isinstance(info.ast, ast3.Assign): + elif isinstance(info.ast, ast.Assign): return None # TODO maybe we need this for aliased methods else: return None @@ -444,10 +442,10 @@ def _get_all_attributes_from_info( info.info, ".".join(info.source_module) ) elif isinstance(info, typeshed_client.NameInfo): - if isinstance(info.ast, ast3.ClassDef): + if isinstance(info.ast, ast.ClassDef): if info.child_nodes is not None: return set(info.child_nodes) - elif isinstance(info.ast, ast3.Assign): + elif isinstance(info.ast, ast.Assign): val = self._parse_expr(info.ast.value, mod) if isinstance(val, KnownValue) and isinstance(val.val, type): return self.get_all_attributes(val.val) @@ -467,11 +465,11 @@ def _has_attribute_from_info( info.info, ".".join(info.source_module), attr ) elif isinstance(info, typeshed_client.NameInfo): - if isinstance(info.ast, ast3.ClassDef): + if isinstance(info.ast, ast.ClassDef): if info.child_nodes and attr in info.child_nodes: return True return False - elif isinstance(info.ast, ast3.Assign): + elif isinstance(info.ast, ast.Assign): val = self._parse_expr(info.ast.value, mod) if isinstance(val, KnownValue) and isinstance(val.val, type): return self.has_attribute(val.val, attr) @@ -489,10 +487,10 @@ def _get_bases_from_info( elif isinstance(info, typeshed_client.ImportedInfo): return self._get_bases_from_info(info.info, ".".join(info.source_module)) elif isinstance(info, typeshed_client.NameInfo): - if isinstance(info.ast, ast3.ClassDef): + if isinstance(info.ast, ast.ClassDef): bases = info.ast.bases return [self._parse_type(base, mod) for base in bases] - elif isinstance(info.ast, ast3.Assign): + elif isinstance(info.ast, ast.Assign): val = self._parse_type(info.ast.value, mod) if isinstance(val, KnownValue) and isinstance(val.val, type): return self.get_bases(val.val) @@ -505,12 +503,12 @@ def _get_bases_from_info( typeshed_client.OverloadedName, typeshed_client.ImportedName, # typeshed pretends the class is a function - ast3.FunctionDef, + ast.FunctionDef, ), ): return None else: - raise NotImplementedError(ast3.dump(info.ast)) + raise NotImplementedError(ast.dump(info.ast)) return None def _get_method_signature_from_info( @@ -583,14 +581,14 @@ def _get_signature_from_info( allow_call: bool = False, ) -> Optional[ConcreteSignature]: if isinstance(info, typeshed_client.NameInfo): - if isinstance(info.ast, (ast3.FunctionDef, ast3.AsyncFunctionDef)): + if isinstance(info.ast, (ast.FunctionDef, ast.AsyncFunctionDef)): return self._get_signature_from_func_def( info.ast, obj, mod, objclass, allow_call=allow_call ) elif isinstance(info.ast, typeshed_client.OverloadedName): sigs = [] for defn in info.ast.definitions: - if not isinstance(defn, (ast3.FunctionDef, ast3.AsyncFunctionDef)): + if not isinstance(defn, (ast.FunctionDef, ast.AsyncFunctionDef)): self.log( "Ignoring unrecognized AST in overload", (fq_name, info) ) @@ -627,7 +625,7 @@ def _get_info_for_name(self, fq_name: str) -> typeshed_client.resolver.ResolvedN def _get_signature_from_func_def( self, - node: Union[ast3.FunctionDef, ast3.AsyncFunctionDef], + node: Union[ast.FunctionDef, ast.AsyncFunctionDef], obj: object, mod: str, objclass: Optional[type] = None, @@ -705,15 +703,15 @@ def _get_signature_from_func_def( cleaned_arguments, callable=obj, return_annotation=GenericValue(Awaitable, [return_value]) - if isinstance(node, ast3.AsyncFunctionDef) + if isinstance(node, ast.AsyncFunctionDef) else return_value, allow_call=allow_call, ) def _parse_param_list( self, - args: Iterable[ast3.arg], - defaults: Iterable[Optional[ast3.AST]], + args: Iterable[ast.arg], + defaults: Iterable[Optional[ast.AST]], module: str, kind: ParameterKind, objclass: Optional[type] = None, @@ -725,8 +723,8 @@ def _parse_param_list( def _parse_param( self, - arg: ast3.arg, - default: Optional[ast3.arg], + arg: ast.arg, + default: Optional[ast.arg], module: str, kind: ParameterKind, objclass: Optional[type] = None, @@ -763,16 +761,16 @@ def _parse_param( default_value = AnyValue(AnySource.unannotated) return SigParameter(name, kind, annotation=typ, default=default_value) - def _parse_expr(self, node: ast3.AST, module: str) -> Value: + def _parse_expr(self, node: ast.AST, module: str) -> Value: ctx = _AnnotationContext(finder=self, module=module) - return value_from_ast(cast(ast.AST, node), ctx=ctx) + return value_from_ast(node, ctx=ctx) - def _parse_type(self, node: ast3.AST, module: str) -> Value: + def _parse_type(self, node: ast.AST, module: str) -> Value: val = self._parse_expr(node, module) ctx = _AnnotationContext(finder=self, module=module) typ = type_from_value(val, ctx=ctx) if self.verbose and isinstance(typ, AnyValue): - self.log("Got Any", (ast3.dump(node), module)) + self.log("Got Any", (ast.dump(node), module)) return typ def _parse_call_assignment( @@ -785,12 +783,12 @@ def _parse_call_assignment( except Exception: pass - if not isinstance(info.ast, ast3.Assign) or not isinstance( - info.ast.value, ast3.Call + if not isinstance(info.ast, ast.Assign) or not isinstance( + info.ast.value, ast.Call ): return AnyValue(AnySource.inference) ctx = _AnnotationContext(finder=self, module=module) - return value_from_ast(cast(ast.AST, info.ast.value), ctx=ctx) + return value_from_ast(info.ast.value, ctx=ctx) def _value_from_info( self, info: typeshed_client.resolver.ResolvedName, module: str @@ -808,11 +806,11 @@ def _value_from_info( elif IS_PRE_38: if fq_name in ("typing.Protocol", "typing_extensions.Protocol"): return KnownValue(Protocol) - if isinstance(info.ast, ast3.Assign): + if isinstance(info.ast, ast.Assign): key = (module, info.ast) if key in self._assignment_cache: return self._assignment_cache[key] - if isinstance(info.ast.value, ast3.Call): + if isinstance(info.ast.value, ast.Call): value = self._parse_call_assignment(info, module) else: value = self._parse_expr(info.ast.value, module) @@ -823,9 +821,9 @@ def _value_from_info( mod = sys.modules[module] return KnownValue(getattr(mod, info.name)) except Exception: - if isinstance(info.ast, ast3.ClassDef): + if isinstance(info.ast, ast.ClassDef): return TypedValue(f"{module}.{info.name}") - elif isinstance(info.ast, ast3.AnnAssign): + elif isinstance(info.ast, ast.AnnAssign): return self._parse_type(info.ast.annotation, module) self.log("Unable to import", (module, info)) return AnyValue(AnySource.inference) From d102ef82fb3454d12577e1b35d67b6412f0e9bec Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Wed, 22 Dec 2021 20:33:15 -0800 Subject: [PATCH 08/13] remove redundant test --- pyanalyze/stubwalk.py | 10 ++++----- pyanalyze/test_typeshed.py | 42 -------------------------------------- 2 files changed, 5 insertions(+), 47 deletions(-) diff --git a/pyanalyze/stubwalk.py b/pyanalyze/stubwalk.py index 7b0440fd..42d08d2d 100644 --- a/pyanalyze/stubwalk.py +++ b/pyanalyze/stubwalk.py @@ -53,17 +53,17 @@ class Error: code: ErrorCode message: str fully_qualified_name: str - ast: Union[ast.AST, typeshed_client.OverloadedName, None] = None + node: Union[ast.AST, typeshed_client.OverloadedName, None] = None def display(self) -> str: heading = f"{self.fully_qualified_name}: {self.message} ({self.code.name})\n" - if isinstance(self.ast, ast.AST): - decompiled = _try_decompile(self.ast) + if isinstance(self.node, ast.AST): + decompiled = _try_decompile(self.node) heading += textwrap.indent(decompiled, " ") - elif isinstance(self.ast, typeshed_client.OverloadedName): + elif isinstance(self.node, typeshed_client.OverloadedName): lines = [ textwrap.indent(_try_decompile(node), " ") - for node in self.ast.definitions + for node in self.node.definitions ] heading += "".join(lines) return heading diff --git a/pyanalyze/test_typeshed.py b/pyanalyze/test_typeshed.py index 68acda87..454069b1 100644 --- a/pyanalyze/test_typeshed.py +++ b/pyanalyze/test_typeshed.py @@ -424,48 +424,6 @@ def test_http_error(self) -> None: assert True is tsf.has_attribute(HTTPError, "read") -class TestIntegration: - """Tests that all files in typeshed are parsed without error.""" - - def test(self): - finder = CTX.arg_spec_cache.ts_finder - # finder.verbose = True - resolver = finder.resolver - print(resolver.ctx) - for module_name, module_path in sorted( - typeshed_client.get_all_stub_files(resolver.ctx) - ): - if module_name in ("this", "antigravity"): - continue # please stop opening my browser - names = typeshed_client.get_stub_names( - module_name, search_context=resolver.ctx - ) - for name, info in names.items(): - is_function = isinstance( - info.ast, - ( - ast.FunctionDef, - ast.AsyncFunctionDef, - typeshed_client.OverloadedName, - ), - ) - fq_name = f"{module_name}.{name}" - if is_function: - sig = finder.get_argspec_for_fully_qualified_name(fq_name, None) - if sig is None: - print(fq_name) - val = finder.resolve_name(module_name, name) - if val == AnyValue(AnySource.inference): - if is_function: - # Probably a function that only exists on another OS. - continue - if isinstance(info.ast, typeshed_client.ImportedName): - # Some legitimate triggers on installed stubs - continue - print(module_name, name, info) - assert False - - class TestRange(TestNameCheckVisitorBase): @assert_passes() def test_iteration(self): From ec084ec3db4f724e60284325d1df928839a9108f Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Wed, 22 Dec 2021 20:41:36 -0800 Subject: [PATCH 09/13] support stub-only functions --- pyanalyze/typeshed.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pyanalyze/typeshed.py b/pyanalyze/typeshed.py index 6ea1e5bd..70e3bc3b 100644 --- a/pyanalyze/typeshed.py +++ b/pyanalyze/typeshed.py @@ -825,6 +825,10 @@ def _value_from_info( return TypedValue(f"{module}.{info.name}") elif isinstance(info.ast, ast.AnnAssign): return self._parse_type(info.ast.annotation, module) + elif isinstance(info.ast, (ast.FunctionDef, ast.AsyncFunctionDef)): + sig = self._get_signature_from_func_def(info.ast, None, module) + if sig is not None: + return CallableValue(sig) self.log("Unable to import", (module, info)) return AnyValue(AnySource.inference) elif isinstance(info, tuple): From 181401ee28600529d4e249819d25fbc972a1d46e Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Wed, 22 Dec 2021 20:46:49 -0800 Subject: [PATCH 10/13] remove unused imports --- pyanalyze/test_typeshed.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/pyanalyze/test_typeshed.py b/pyanalyze/test_typeshed.py index 454069b1..f8d4aff9 100644 --- a/pyanalyze/test_typeshed.py +++ b/pyanalyze/test_typeshed.py @@ -8,10 +8,7 @@ import sys import tempfile import time -from pyanalyze.test_value import CTX -import typeshed_client from typeshed_client import Resolver, get_search_context -import ast import typing from typing import Dict, Generic, List, TypeVar, NewType, Union from urllib.error import HTTPError From c4aa6e92b5f33e928131a505c258054a864e4fa5 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 11 Jan 2022 16:37:02 -0800 Subject: [PATCH 11/13] fix self check --- pyanalyze/stubwalk.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/pyanalyze/stubwalk.py b/pyanalyze/stubwalk.py index 42d08d2d..b7498fee 100644 --- a/pyanalyze/stubwalk.py +++ b/pyanalyze/stubwalk.py @@ -31,6 +31,7 @@ class ErrorCode(enum.Enum): unresolved_type_in_signature = 5 unused_allowlist_entry = 6 unresolved_bases = 7 + unresolved_module = 8 DISABLED_BY_DEFAULT = { @@ -53,7 +54,9 @@ class Error: code: ErrorCode message: str fully_qualified_name: str - node: Union[ast.AST, typeshed_client.OverloadedName, None] = None + node: Union[ + ast.AST, typeshed_client.OverloadedName, typeshed_client.ImportedName, None + ] = None def display(self) -> str: heading = f"{self.fully_qualified_name}: {self.message} ({self.code.name})\n" @@ -66,6 +69,11 @@ def display(self) -> str: for node in self.node.definitions ] heading += "".join(lines) + elif isinstance(self.node, typeshed_client.ImportedName): + heading += ( + f" imported from: {'.'.join(self.node.module_name)} with name" + f" {self.node.name}" + ) return heading @@ -110,6 +118,13 @@ def _stubwalk(search_context: typeshed_client.SearchContext) -> Iterable[Error]: if module_name in ("this", "antigravity"): continue # please stop opening my browser names = typeshed_client.get_stub_names(module_name, search_context=resolver.ctx) + if names is None: + yield Error( + ErrorCode.unresolved_module, + f"Failed to find stub for module {module_name}", + module_name, + ) + continue for name, info in names.items(): is_function = isinstance( info.ast, From 6140c555b71e19b80bd5631db4056a95e3f7b431 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 11 Jan 2022 16:57:21 -0800 Subject: [PATCH 12/13] fix infinite recursion --- pyanalyze/typeshed.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/pyanalyze/typeshed.py b/pyanalyze/typeshed.py index e825a9b0..bafbbcad 100644 --- a/pyanalyze/typeshed.py +++ b/pyanalyze/typeshed.py @@ -155,6 +155,9 @@ class TypeshedFinder: _attribute_cache: Dict[Tuple[str, str, bool], Value] = field( default_factory=dict, repr=False, init=False ) + _active_infos: List[typeshed_client.resolver.ResolvedName] = field( + default_factory=list, repr=False, init=False + ) @classmethod def make(cls, options: Options, *, verbose: bool = False) -> "TypeshedFinder": @@ -858,7 +861,7 @@ def _parse_call_assignment( def make_synthetic_type(self, module: str, info: typeshed_client.NameInfo) -> Value: fq_name = f"{module}.{info.name}" - bases = self.get_bases_for_fq_name(fq_name) + bases = self._get_bases_from_info(info, module) typ = TypedValue(fq_name) if bases is not None: if any( @@ -906,6 +909,19 @@ def _make_td_value(self, field: Value, total: bool) -> Tuple[bool, Value]: def _value_from_info( self, info: typeshed_client.resolver.ResolvedName, module: str + ) -> Value: + # This guard against infinite recursion if a type refers to itself + # (real-world example: os._ScandirIterator). + if info in self._active_infos: + return AnyValue(AnySource.inference) + self._active_infos.append(info) + try: + return self._value_from_info_inner(info, module) + finally: + self._active_infos.pop() + + def _value_from_info_inner( + self, info: typeshed_client.resolver.ResolvedName, module: str ) -> Value: if isinstance(info, typeshed_client.ImportedInfo): return self._value_from_info(info.info, ".".join(info.source_module)) From 9866a083ef3f5faea8577c96e9e6ea247034199f Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 11 Jan 2022 17:07:44 -0800 Subject: [PATCH 13/13] fix TypedDicts --- pyanalyze/stubwalk.py | 48 ++++++++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 21 deletions(-) diff --git a/pyanalyze/stubwalk.py b/pyanalyze/stubwalk.py index b7498fee..498f51b2 100644 --- a/pyanalyze/stubwalk.py +++ b/pyanalyze/stubwalk.py @@ -15,7 +15,7 @@ from typing import Collection, Container, Iterable, Optional, Sequence, Union from .config import Config -from .value import AnySource, AnyValue, TypedValue, Value +from .value import AnySource, AnyValue, SubclassValue, TypedDictValue, TypedValue, Value from .checker import Checker from .name_check_visitor import NameCheckVisitor @@ -131,6 +131,7 @@ def _stubwalk(search_context: typeshed_client.SearchContext) -> Iterable[Error]: (ast.FunctionDef, ast.AsyncFunctionDef, typeshed_client.OverloadedName), ) fq_name = f"{module_name}.{name}" + val = finder.resolve_name(module_name, name) if is_function: sig = finder.get_argspec_for_fully_qualified_name(fq_name, None) if sig is None: @@ -143,29 +144,34 @@ def _stubwalk(search_context: typeshed_client.SearchContext) -> Iterable[Error]: else: yield from _error_on_nested_any(sig, "Signature", fq_name, info) if isinstance(info.ast, ast.ClassDef): - bases = finder.get_bases_for_fq_name(fq_name) - if bases is None: - yield Error( - ErrorCode.unresolved_bases, - "Cannot resolve bases", - fq_name, - info.ast, - ) - else: - for base in bases: - if not isinstance(base, TypedValue): - yield Error( - ErrorCode.unresolved_bases, - "Cannot resolve one of the bases", - fq_name, - info.ast, - ) - else: - yield from _error_on_nested_any(base, "Base", fq_name, info) + if not ( + isinstance(val, SubclassValue) + and isinstance(val.typ, TypedDictValue) + ): + bases = finder.get_bases_for_fq_name(fq_name) + if bases is None: + yield Error( + ErrorCode.unresolved_bases, + "Cannot resolve bases", + fq_name, + info.ast, + ) + else: + for base in bases: + if not isinstance(base, TypedValue): + yield Error( + ErrorCode.unresolved_bases, + "Cannot resolve one of the bases", + fq_name, + info.ast, + ) + else: + yield from _error_on_nested_any( + base, "Base", fq_name, info + ) # TODO: # - Loop over all attributes and assert their values don't contain Any # - Loop over all methods and check their signatures - val = finder.resolve_name(module_name, name) if val == AnyValue(AnySource.inference): if is_function: yield Error(