Skip to content

Commit edd9dc4

Browse files
committed
Add typing to Celery
This is a simple bootstrap of the process, adding some types to a few selected functions, based on comment annotations. MyPy is chosen as the default static analyzer for the types.
1 parent 095cd78 commit edd9dc4

7 files changed

Lines changed: 54 additions & 34 deletions

File tree

.github/workflows/lint_python.yml

Lines changed: 18 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,22 @@ jobs:
99
- uses: pre-commit/action@v2.0.3
1010
- run: pip install --upgrade pip wheel
1111
- run: pip install bandit codespell flake8 isort pytest pyupgrade tox
12-
- run: bandit -r . || true
13-
- run: codespell --ignore-words-list="brane,gool,ist,sherif,wil" --quiet-level=2 --skip="*.key" || true
12+
13+
- name: bandit
14+
run: bandit -r . || true
15+
16+
- name: Run CodeSpell
17+
run: codespell --ignore-words-list="brane,gool,ist,sherif,wil" --quiet-level=2 --skip="*.key" || true
1418
- run: pip install -r requirements.txt || true
15-
- run: tox || true
16-
- run: pytest . || true
17-
- run: pytest --doctest-modules . || true
19+
20+
- name: Run tox
21+
run: tox || true
22+
23+
- name: Run pytest
24+
run: pytest . || true
25+
26+
- name: Test pytest with doctest
27+
run: pytest --doctest-modules . || true
28+
29+
- name: MyPy
30+
run: tox -e mypy

celery/contrib/testing/worker.py

Lines changed: 9 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -90,16 +90,15 @@ def start_worker(
9090

9191

9292
@contextmanager
93-
def _start_worker_thread(app,
94-
concurrency=1,
95-
pool='solo',
93+
def _start_worker_thread(app: Celery,
94+
concurrency: int = 1,
95+
pool: str = 'solo',
9696
loglevel=WORKER_LOGLEVEL,
97-
logfile=None,
98-
WorkController=TestWorkController,
99-
perform_ping_check=True,
100-
shutdown_timeout=10.0,
101-
**kwargs):
102-
# type: (Celery, int, str, Union[str, int], str, Any, **Any) -> Iterable
97+
logfile: Union[str, int] = None,
98+
WorkController: Any = TestWorkController,
99+
perform_ping_check: bool = True,
100+
shutdown_timeout: float = 10.0,
101+
**kwargs: Any) -> Iterable:
103102
"""Start Celery worker in a thread.
104103
105104
Yields:
@@ -167,8 +166,7 @@ def _start_worker_process(app,
167166
cluster.stopwait()
168167

169168

170-
def setup_app_for_worker(app, loglevel, logfile):
171-
# type: (Celery, Union[str, int], str) -> None
169+
def setup_app_for_worker(app: Celery, loglevel: Union[str, int], logfile: str) -> None:
172170
"""Setup the app to be used for starting an embedded worker."""
173171
app.finalize()
174172
app.set_current()

celery/events/state.py

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222
from itertools import islice
2323
from operator import itemgetter
2424
from time import time
25-
from typing import Mapping
25+
from typing import Mapping, Optional
2626
from weakref import WeakSet, ref
2727

2828
from kombu.clocks import timetuple
@@ -457,7 +457,7 @@ def clear_tasks(self, ready=True):
457457
with self._mutex:
458458
return self._clear_tasks(ready)
459459

460-
def _clear_tasks(self, ready=True):
460+
def _clear_tasks(self, ready: bool = True):
461461
if ready:
462462
in_progress = {
463463
uuid: task for uuid, task in self.itertasks()
@@ -475,7 +475,7 @@ def _clear(self, ready=True):
475475
self.event_count = 0
476476
self.task_count = 0
477477

478-
def clear(self, ready=True):
478+
def clear(self, ready: bool = True):
479479
with self._mutex:
480480
return self._clear(ready)
481481

@@ -495,7 +495,7 @@ def get_or_create_worker(self, hostname, **kwargs):
495495
hostname, **kwargs)
496496
return worker, True
497497

498-
def get_or_create_task(self, uuid):
498+
def get_or_create_task(self, uuid: str):
499499
"""Get or create task by uuid."""
500500
try:
501501
return self.tasks[uuid], False
@@ -652,13 +652,13 @@ def rebuild_taskheap(self, timetuple=timetuple):
652652
]
653653
heap.sort()
654654

655-
def itertasks(self, limit=None):
655+
def itertasks(self, limit: Optional[int] = None):
656656
for index, row in enumerate(self.tasks.items()):
657657
yield row
658658
if limit and index + 1 >= limit:
659659
break
660660

661-
def tasks_by_time(self, limit=None, reverse=True):
661+
def tasks_by_time(self, limit=None, reverse: bool = True):
662662
"""Generator yielding tasks ordered by time.
663663
664664
Yields:

celery/utils/collections.py

Lines changed: 6 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -113,8 +113,7 @@ def __getattr__(self, k):
113113
raise AttributeError(
114114
f'{type(self).__name__!r} object has no attribute {k!r}')
115115

116-
def __setattr__(self, key, value):
117-
# type: (str, Any) -> None
116+
def __setattr__(self, key: str, value: Any) -> None:
118117
"""`d[key] = value -> d.key = value`."""
119118
self[key] = value
120119

@@ -595,8 +594,7 @@ def purge(self, now=None):
595594
break # oldest item hasn't expired yet
596595
self.pop()
597596

598-
def pop(self, default=None):
599-
# type: (Any) -> Any
597+
def pop(self, default: Any = None) -> Any:
600598
"""Remove and return the oldest item, or :const:`None` when empty."""
601599
while self._heap:
602600
_, item = heappop(self._heap)
@@ -608,8 +606,7 @@ def pop(self, default=None):
608606
return item
609607
return default
610608

611-
def as_dict(self):
612-
# type: () -> Dict
609+
def as_dict(self) -> Dict:
613610
"""Whole set as serializable dictionary.
614611
615612
Example:
@@ -675,20 +672,17 @@ class Evictable:
675672

676673
Empty = Empty
677674

678-
def evict(self):
679-
# type: () -> None
675+
def evict(self) -> None:
680676
"""Force evict until maxsize is enforced."""
681677
self._evict(range=count)
682678

683-
def _evict(self, limit=100, range=range):
684-
# type: (int) -> None
679+
def _evict(self, limit: int = 100, range=range) -> None:
685680
try:
686681
[self._evict1() for _ in range(limit)]
687682
except IndexError:
688683
pass
689684

690-
def _evict1(self):
691-
# type: () -> None
685+
def _evict1(self) -> None:
692686
if self._evictcount <= self.maxsize:
693687
raise IndexError()
694688
try:

mypy.ini

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
[mypy]
2+
warn_unused_configs = True
3+
follow_imports = skip
4+
strict = False
5+
warn_return_any = True
6+
show_error_codes = True
7+
# True is the goal for disallow_untyped_defs
8+
disallow_untyped_defs = False
9+
ignore_missing_imports = True
10+
allowlist_externals = True

requirements/test.txt

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@ pytest-subtests
44
pytest-timeout~=1.4.2
55
boto3>=1.9.178
66
moto>=2.2.6
7+
# typing extensions
8+
mypy
79
pre-commit
810
-r extras/yaml.txt
911
-r extras/msgpack.txt

tox.ini

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,9 +78,12 @@ basepython =
7878
3.9: python3.9
7979
3.10: python3.10
8080
pypy3: pypy3
81+
mypy: python3.8
8182
lint,apicheck,linkcheck,configcheck,bandit: python3.9
8283
usedevelop = True
8384

85+
[testenv:mypy]
86+
commands = mypy celery
8487

8588
[testenv:apicheck]
8689
setenv =

0 commit comments

Comments
 (0)