fix: include positional args in the deduplication fingerprint (#81)

This commit is contained in:
d3vyce
2026-07-29 10:13:58 +02:00
committed by GitHub
parent 56e82689f8
commit e75b9f3f20
6 changed files with 51 additions and 7 deletions
+1 -1
View File
@@ -55,7 +55,7 @@ except DuplicateTaskError:
- **Configurable TTL** — set a global default or override per task with the `deduplication_ttl` label.
- **Lock heartbeat** — a background task re-extends the lock TTL while the task runs, so long-running tasks keep their lock instead of expiring mid-execution and admitting a duplicate.
- **Explicit lock key** — pin any task to a fixed Redis key with `deduplication_key`, bypassing fingerprint computation entirely.
- **Partial fingerprint** — deduplicate on a subset of kwargs with `deduplication_key_fields`, ignoring irrelevant arguments.
- **Partial fingerprint** — deduplicate on a subset of kwargs with `deduplication_key_fields`, ignoring irrelevant arguments (positional arguments are excluded).
- **Per-task opt-out** — disable deduplication for individual tasks with the `deduplication` label.
- **Startup resilience** — automatic reconnection with exponential backoff if Redis is unavailable at broker startup.
+1 -1
View File
@@ -55,7 +55,7 @@ except DuplicateTaskError:
- **Configurable TTL** — set a global default or override per task with the `deduplication_ttl` label.
- **Lock heartbeat** — a background task re-extends the lock TTL while the task runs, so long-running tasks keep their lock instead of expiring mid-execution and admitting a duplicate.
- **Explicit lock key** — pin any task to a fixed Redis key with `deduplication_key`, bypassing fingerprint computation entirely.
- **Partial fingerprint** — deduplicate on a subset of kwargs with `deduplication_key_fields`, ignoring irrelevant arguments.
- **Partial fingerprint** — deduplicate on a subset of kwargs with `deduplication_key_fields`, ignoring irrelevant arguments (positional arguments are excluded).
- **Per-task opt-out** — disable deduplication for individual tasks with the `deduplication` label.
- **Startup resilience** — automatic reconnection with exponential backoff if Redis is unavailable at broker startup.
+14 -2
View File
@@ -147,11 +147,19 @@ await my_task.kicker().with_labels(deduplication_ttl=60).kiq(user_id=42)
| `deduplication` | `bool` | Set `False` to opt out of deduplication entirely for this task. |
| `deduplication_ttl` | `int` | Lock TTL in seconds. Overrides the middleware `default_ttl`. |
| `deduplication_key` | `str` | Explicit lock key. Skips fingerprint computation entirely. |
| `deduplication_key_fields` | `list[str]` | Subset of kwargs to include in the fingerprint. Ignored if `deduplication_key` is set. |
| `deduplication_key_fields` | `list[str]` | Subset of kwargs to include in the fingerprint. Positional arguments are excluded. Ignored if `deduplication_key` is set. |
## Fingerprint and key customisation
By default the lock key is a SHA-256 fingerprint of the task name and all kwargs.
By default the lock key is a SHA-256 fingerprint of the task name, its positional
arguments and all kwargs.
!!! warning "Positional and keyword calls fingerprint differently"
taskiq serialises arguments as they were passed, without binding them to the
task signature. `my_task.kiq(42)` and `my_task.kiq(user_id=42)` are therefore
*not* recognised as duplicates of each other. Call a deduplicated task
consistently, preferably always with keyword arguments.
### Explicit key
@@ -182,6 +190,10 @@ If a listed field is absent from a task's kwargs, it is dropped from the
fingerprint and a warning is logged, since this can make genuinely different
calls collide on the same lock.
Positional arguments are excluded from the fingerprint entirely when this label is
set: you asked to deduplicate on named fields, so pass them as keyword arguments.
A warning is logged if the task is called with positional arguments anyway.
## Opting out per task
```python
+10 -1
View File
@@ -154,12 +154,21 @@ class RedisDeduplicationMiddleware(TaskiqMiddleware):
message.task_name,
missing,
)
if message.args:
logger.warning(
"Task %s was called with positional arguments but uses "
"deduplication_key_fields; positional arguments are excluded "
"from the fingerprint.",
message.task_name,
)
kwargs = {k: v for k, v in message.kwargs.items() if k in key_fields}
args: list[Any] = []
else:
kwargs = message.kwargs
args = message.args
try:
payload = json.dumps(
{"task": message.task_name, "kwargs": kwargs},
{"task": message.task_name, "args": args, "kwargs": kwargs},
sort_keys=True,
)
except TypeError:
+4 -2
View File
@@ -33,13 +33,15 @@ async def real_redis():
@pytest.fixture
def make_message():
def _make(task_name="my_task", task_id="task-1", labels=None, kwargs=None):
def _make(
task_name="my_task", task_id="task-1", labels=None, kwargs=None, args=None
):
return TaskiqMessage(
task_id=task_id,
task_name=task_name,
labels=labels or {},
labels_types={},
args=[],
args=args or [],
kwargs=kwargs or {},
)
+21
View File
@@ -86,6 +86,27 @@ class TestDefaultBuildDeduplicationKey:
m1
) == middleware._build_deduplication_key(m2)
def test_different_args_different_key(self, middleware, make_message):
m1 = make_message(args=["a"])
m2 = make_message(args=["b"])
assert middleware._build_deduplication_key(
m1
) != middleware._build_deduplication_key(m2)
def test_arg_order_matters(self, middleware, make_message):
m1 = make_message(args=["a", "b"])
m2 = make_message(args=["b", "a"])
assert middleware._build_deduplication_key(
m1
) != middleware._build_deduplication_key(m2)
def test_key_fields_ignores_args(self, middleware, make_message):
m1 = make_message(args=["a"], labels={DEDUP_KEY_FIELDS_LABEL: ["x"]})
m2 = make_message(args=["b"], labels={DEDUP_KEY_FIELDS_LABEL: ["x"]})
assert middleware._build_deduplication_key(
m1
) == middleware._build_deduplication_key(m2)
def test_key_prefix_in_output(self, make_message):
mw = RedisDeduplicationMiddleware(
redis_url="redis://localhost", key_prefix="myapp:locks"