Add mypy to the pipeline (#7383)

Kludex · atombrella · pre-commit-ci[bot] · web-flow · commit fbda0089f08d · 2022-04-02T14:49:56.000+06:00
* Add typing to Celery This is a simple bootstrap of the process, adding some types to a few selected functions, based on comment annotations. MyPy is chosen as the default static analyzer for the types. * Add mypy to the pipeline * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Remove unused command from tox * Install mypy only on CPython * Remove wrong annotations * Update celery/utils/saferepr.py Co-authored-by: Mads Jensen <mje@inducks.org> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
diff --git a/.github/workflows/lint_python.yml b/.github/workflows/lint_python.yml
@@ -9,9 +9,22 @@ jobs:
       - uses: pre-commit/action@v2.0.3
       - run: pip install --upgrade pip wheel
       - run: pip install bandit codespell flake8 isort pytest pyupgrade tox
-      - run: bandit -r . || true
-      - run: codespell --ignore-words-list="brane,gool,ist,sherif,wil" --quiet-level=2 --skip="*.key" || true
+
+      - name: bandit
+        run: bandit -r . || true
+
+      - name: Run CodeSpell
+        run: codespell --ignore-words-list="brane,gool,ist,sherif,wil" --quiet-level=2 --skip="*.key" || true
       - run: pip install -r requirements.txt || true
-      - run: tox || true
-      - run: pytest . || true
-      - run: pytest --doctest-modules . || true
+
+      - name: Run tox
+        run: tox || true
+
+      - name: Run pytest
+        run: pytest . || true
+
+      - name: Test pytest with doctest
+        run: pytest --doctest-modules . || true
+
+      - name: MyPy
+        run: tox -e mypy
diff --git a/celery/__main__.py b/celery/__main__.py
@@ -7,7 +7,7 @@
 __all__ = ('main',)
 
 
-def main():
+def main() -> None:
     """Entrypoint to the ``celery`` umbrella command."""
     if 'multi' not in sys.argv:
         maybe_patch_concurrency()
diff --git a/celery/contrib/testing/worker.py b/celery/contrib/testing/worker.py
@@ -167,7 +167,7 @@ def _start_worker_process(app,
     cluster.stopwait()
 
 
-def setup_app_for_worker(app, loglevel, logfile):
+def setup_app_for_worker(app, loglevel, logfile) -> None:
     # type: (Celery, Union[str, int], str) -> None
     """Setup the app to be used for starting an embedded worker."""
     app.finalize()
diff --git a/celery/events/state.py b/celery/events/state.py
@@ -22,7 +22,7 @@
 from itertools import islice
 from operator import itemgetter
 from time import time
-from typing import Mapping
+from typing import Mapping, Optional
 from weakref import WeakSet, ref
 
 from kombu.clocks import timetuple
@@ -452,7 +452,7 @@ def clear_tasks(self, ready=True):
         with self._mutex:
             return self._clear_tasks(ready)
 
-    def _clear_tasks(self, ready=True):
+    def _clear_tasks(self, ready: bool = True):
         if ready:
             in_progress = {
                 uuid: task for uuid, task in self.itertasks()
@@ -470,7 +470,7 @@ def _clear(self, ready=True):
         self.event_count = 0
         self.task_count = 0
 
-    def clear(self, ready=True):
+    def clear(self, ready: bool = True):
         with self._mutex:
             return self._clear(ready)
 
@@ -647,13 +647,13 @@ def rebuild_taskheap(self, timetuple=timetuple):
         ]
         heap.sort()
 
-    def itertasks(self, limit=None):
+    def itertasks(self, limit: Optional[int] = None):
         for index, row in enumerate(self.tasks.items()):
             yield row
             if limit and index + 1 >= limit:
                 break
 
-    def tasks_by_time(self, limit=None, reverse=True):
+    def tasks_by_time(self, limit=None, reverse: bool = True):
         """Generator yielding tasks ordered by time.
 
         Yields:
diff --git a/celery/utils/collections.py b/celery/utils/collections.py
@@ -113,8 +113,7 @@ def __getattr__(self, k):
             raise AttributeError(
                 f'{type(self).__name__!r} object has no attribute {k!r}')
 
-    def __setattr__(self, key, value):
-        # type: (str, Any) -> None
+    def __setattr__(self, key: str, value) -> None:
         """`d[key] = value -> d.key = value`."""
         self[key] = value
 
@@ -595,7 +594,7 @@ def purge(self, now=None):
                     break  # oldest item hasn't expired yet
                 self.pop()
 
-    def pop(self, default=None):
+    def pop(self, default=None) -> Any:
         # type: (Any) -> Any
         """Remove and return the oldest item, or :const:`None` when empty."""
         while self._heap:
@@ -671,20 +670,17 @@ class Evictable:
 
     Empty = Empty
 
-    def evict(self):
-        # type: () -> None
+    def evict(self) -> None:
         """Force evict until maxsize is enforced."""
         self._evict(range=count)
 
-    def _evict(self, limit=100, range=range):
-        # type: (int) -> None
+    def _evict(self, limit: int = 100, range=range) -> None:
         try:
             [self._evict1() for _ in range(limit)]
         except IndexError:
             pass
 
-    def _evict1(self):
-        # type: () -> None
+    def _evict1(self) -> None:
         if self._evictcount <= self.maxsize:
             raise IndexError()
         try:
@@ -746,8 +742,7 @@ def __len__(self):
         # type: () -> int
         return self._len()
 
-    def __contains__(self, item):
-        # type: () -> bool
+    def __contains__(self, item) -> bool:
         return item in self.data
 
     def __reversed__(self):
diff --git a/pyproject.toml b/pyproject.toml
@@ -4,3 +4,15 @@ testpaths = "t/unit/"
 python_classes = "test_*"
 xdfail_strict=true
 markers = ["sleepdeprived_patched_module", "masked_modules", "patched_environ", "patched_module"]
+
+[tool.mypy]
+warn_unused_configs = true
+strict = false
+warn_return_any = true
+follow_imports = "skip"
+show_error_codes = true
+disallow_untyped_defs = true
+ignore_missing_imports = true
+files = [
+    "celery/__main__.py",
+]
diff --git a/requirements/test.txt b/requirements/test.txt
@@ -4,6 +4,8 @@ pytest-subtests
 pytest-timeout~=2.1.0
 boto3>=1.9.178
 moto>=2.2.6
+# typing extensions
+mypy; platform_python_implementation=="CPython"
 pre-commit
 -r extras/yaml.txt
 -r extras/msgpack.txt
diff --git a/tox.ini b/tox.ini
@@ -78,9 +78,12 @@ basepython =
     3.9: python3.9
     3.10: python3.10
     pypy3: pypy3
+    mypy: python3.8
     lint,apicheck,linkcheck,configcheck,bandit: python3.9
 usedevelop = True
 
+[testenv:mypy]
+commands = python -m mypy --config-file pyproject.toml
 
 [testenv:apicheck]
 setenv =
