mirror of
https://github.com/d3vyce/taskiq-deduplication.git
synced 2026-08-04 19:14:07 +00:00
Version 1.0.0 (#2)
* feat: add taskiq deduplication * doc: rework class comment + update README * fix: make build_deduplication_key private * fix: raise_on_duplicate is now False by default * chore: remove pre_execute * chore: add documentation
This commit is contained in:
d3vyce
@@ -0,0 +1,14 @@
|
|||||||
|
version: 2
|
||||||
|
updates:
|
||||||
|
- package-ecosystem: "github-actions"
|
||||||
|
directory: "/"
|
||||||
|
schedule:
|
||||||
|
interval: "weekly"
|
||||||
|
commit-message:
|
||||||
|
prefix: ⬆
|
||||||
|
- package-ecosystem: "uv"
|
||||||
|
directory: "/"
|
||||||
|
schedule:
|
||||||
|
interval: "weekly"
|
||||||
|
commit-message:
|
||||||
|
prefix: ⬆
|
||||||
@@ -0,0 +1,29 @@
|
|||||||
|
name: Build Package
|
||||||
|
|
||||||
|
on:
|
||||||
|
release:
|
||||||
|
types: [published]
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
build-package:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
environment: pypi
|
||||||
|
permissions:
|
||||||
|
id-token: write
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v6
|
||||||
|
|
||||||
|
- name: Install uv
|
||||||
|
uses: astral-sh/setup-uv@v7
|
||||||
|
|
||||||
|
- name: Set up Python
|
||||||
|
run: uv python install 3.14
|
||||||
|
|
||||||
|
- name: Install dependencies
|
||||||
|
run: uv sync
|
||||||
|
|
||||||
|
- name: Build
|
||||||
|
run: uv build
|
||||||
|
|
||||||
|
- name: Publish package distributions to PyPI
|
||||||
|
uses: pypa/gh-action-pypi-publish@release/v1
|
||||||
@@ -0,0 +1,93 @@
|
|||||||
|
name: CI
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches: [main]
|
||||||
|
pull_request:
|
||||||
|
branches: [main]
|
||||||
|
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
|
||||||
|
concurrency:
|
||||||
|
group: ${{ github.workflow }}-${{ github.ref }}
|
||||||
|
cancel-in-progress: true
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
lint:
|
||||||
|
name: Lint (Ruff)
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v6
|
||||||
|
|
||||||
|
- name: Install uv
|
||||||
|
uses: astral-sh/setup-uv@v7
|
||||||
|
|
||||||
|
- name: Set up Python
|
||||||
|
run: uv python install 3.13
|
||||||
|
|
||||||
|
- name: Install dependencies
|
||||||
|
run: uv sync --group dev
|
||||||
|
|
||||||
|
- name: Run Ruff linter
|
||||||
|
run: uv run ruff check .
|
||||||
|
|
||||||
|
- name: Run Ruff formatter check
|
||||||
|
run: uv run ruff format --check .
|
||||||
|
|
||||||
|
typecheck:
|
||||||
|
name: Type Check (ty)
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v6
|
||||||
|
|
||||||
|
- name: Install uv
|
||||||
|
uses: astral-sh/setup-uv@v7
|
||||||
|
|
||||||
|
- name: Set up Python
|
||||||
|
run: uv python install 3.13
|
||||||
|
|
||||||
|
- name: Install dependencies
|
||||||
|
run: uv sync --group dev
|
||||||
|
|
||||||
|
- name: Run ty
|
||||||
|
run: uv run ty check
|
||||||
|
|
||||||
|
test:
|
||||||
|
name: Test (Python ${{ matrix.python-version }})
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
strategy:
|
||||||
|
fail-fast: false
|
||||||
|
matrix:
|
||||||
|
python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v6
|
||||||
|
|
||||||
|
- name: Install uv
|
||||||
|
uses: astral-sh/setup-uv@v7
|
||||||
|
|
||||||
|
- name: Set up Python ${{ matrix.python-version }}
|
||||||
|
run: uv python install ${{ matrix.python-version }}
|
||||||
|
|
||||||
|
- name: Install dependencies
|
||||||
|
run: uv sync --group dev
|
||||||
|
|
||||||
|
- name: Run tests with coverage
|
||||||
|
run: uv run pytest --cov --cov-report=xml --cov-report=term-missing --junitxml=junit.xml -o junit_family=legacy
|
||||||
|
|
||||||
|
- name: Upload coverage to Codecov
|
||||||
|
if: matrix.python-version == '3.14'
|
||||||
|
uses: codecov/codecov-action@v6
|
||||||
|
with:
|
||||||
|
token: ${{ secrets.CODECOV_TOKEN }}
|
||||||
|
report_type: coverage
|
||||||
|
files: ./coverage.xml
|
||||||
|
fail_ci_if_error: false
|
||||||
|
|
||||||
|
- name: Upload test results to Codecov
|
||||||
|
if: matrix.python-version == '3.14'
|
||||||
|
uses: codecov/codecov-action@v6
|
||||||
|
with:
|
||||||
|
token: ${{ secrets.CODECOV_TOKEN }}
|
||||||
|
report_type: test_results
|
||||||
@@ -0,0 +1,38 @@
|
|||||||
|
name: Documentation
|
||||||
|
|
||||||
|
on:
|
||||||
|
release:
|
||||||
|
types: [published]
|
||||||
|
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
pages: write
|
||||||
|
id-token: write
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
deploy:
|
||||||
|
environment:
|
||||||
|
name: github-pages
|
||||||
|
url: ${{ steps.deployment.outputs.page_url }}
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/configure-pages@v5
|
||||||
|
|
||||||
|
- uses: actions/checkout@v6
|
||||||
|
|
||||||
|
- name: Install uv
|
||||||
|
uses: astral-sh/setup-uv@v7
|
||||||
|
|
||||||
|
- name: Set up Python
|
||||||
|
run: uv python install 3.13
|
||||||
|
|
||||||
|
- run: uv sync --group dev
|
||||||
|
|
||||||
|
- run: uv run zensical build --clean
|
||||||
|
|
||||||
|
- uses: actions/upload-pages-artifact@v4
|
||||||
|
with:
|
||||||
|
path: site
|
||||||
|
|
||||||
|
- uses: actions/deploy-pages@v4
|
||||||
|
id: deployment
|
||||||
+207
@@ -0,0 +1,207 @@
|
|||||||
|
# Byte-compiled / optimized / DLL files
|
||||||
|
__pycache__/
|
||||||
|
*.py[codz]
|
||||||
|
*$py.class
|
||||||
|
|
||||||
|
# C extensions
|
||||||
|
*.so
|
||||||
|
|
||||||
|
# Distribution / packaging
|
||||||
|
.Python
|
||||||
|
build/
|
||||||
|
develop-eggs/
|
||||||
|
dist/
|
||||||
|
downloads/
|
||||||
|
eggs/
|
||||||
|
.eggs/
|
||||||
|
lib/
|
||||||
|
lib64/
|
||||||
|
parts/
|
||||||
|
sdist/
|
||||||
|
var/
|
||||||
|
wheels/
|
||||||
|
share/python-wheels/
|
||||||
|
*.egg-info/
|
||||||
|
.installed.cfg
|
||||||
|
*.egg
|
||||||
|
MANIFEST
|
||||||
|
|
||||||
|
# PyInstaller
|
||||||
|
# Usually these files are written by a python script from a template
|
||||||
|
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
||||||
|
*.manifest
|
||||||
|
*.spec
|
||||||
|
|
||||||
|
# Installer logs
|
||||||
|
pip-log.txt
|
||||||
|
pip-delete-this-directory.txt
|
||||||
|
|
||||||
|
# Unit test / coverage reports
|
||||||
|
htmlcov/
|
||||||
|
.tox/
|
||||||
|
.nox/
|
||||||
|
.coverage
|
||||||
|
.coverage.*
|
||||||
|
.cache
|
||||||
|
nosetests.xml
|
||||||
|
coverage.xml
|
||||||
|
*.cover
|
||||||
|
*.py.cover
|
||||||
|
.hypothesis/
|
||||||
|
.pytest_cache/
|
||||||
|
cover/
|
||||||
|
|
||||||
|
# Translations
|
||||||
|
*.mo
|
||||||
|
*.pot
|
||||||
|
|
||||||
|
# Django stuff:
|
||||||
|
*.log
|
||||||
|
local_settings.py
|
||||||
|
db.sqlite3
|
||||||
|
db.sqlite3-journal
|
||||||
|
|
||||||
|
# Flask stuff:
|
||||||
|
instance/
|
||||||
|
.webassets-cache
|
||||||
|
|
||||||
|
# Scrapy stuff:
|
||||||
|
.scrapy
|
||||||
|
|
||||||
|
# Sphinx documentation
|
||||||
|
docs/_build/
|
||||||
|
|
||||||
|
# PyBuilder
|
||||||
|
.pybuilder/
|
||||||
|
target/
|
||||||
|
|
||||||
|
# Jupyter Notebook
|
||||||
|
.ipynb_checkpoints
|
||||||
|
|
||||||
|
# IPython
|
||||||
|
profile_default/
|
||||||
|
ipython_config.py
|
||||||
|
|
||||||
|
# pyenv
|
||||||
|
# For a library or package, you might want to ignore these files since the code is
|
||||||
|
# intended to run in multiple environments; otherwise, check them in:
|
||||||
|
# .python-version
|
||||||
|
|
||||||
|
# pipenv
|
||||||
|
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
||||||
|
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
||||||
|
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
||||||
|
# install all needed dependencies.
|
||||||
|
#Pipfile.lock
|
||||||
|
|
||||||
|
# UV
|
||||||
|
# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
|
||||||
|
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
||||||
|
# commonly ignored for libraries.
|
||||||
|
#uv.lock
|
||||||
|
|
||||||
|
# poetry
|
||||||
|
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
|
||||||
|
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
||||||
|
# commonly ignored for libraries.
|
||||||
|
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
|
||||||
|
#poetry.lock
|
||||||
|
#poetry.toml
|
||||||
|
|
||||||
|
# pdm
|
||||||
|
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
|
||||||
|
# pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
|
||||||
|
# https://pdm-project.org/en/latest/usage/project/#working-with-version-control
|
||||||
|
#pdm.lock
|
||||||
|
#pdm.toml
|
||||||
|
.pdm-python
|
||||||
|
.pdm-build/
|
||||||
|
|
||||||
|
# pixi
|
||||||
|
# Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
|
||||||
|
#pixi.lock
|
||||||
|
# Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
|
||||||
|
# in the .venv directory. It is recommended not to include this directory in version control.
|
||||||
|
.pixi
|
||||||
|
|
||||||
|
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
|
||||||
|
__pypackages__/
|
||||||
|
|
||||||
|
# Celery stuff
|
||||||
|
celerybeat-schedule
|
||||||
|
celerybeat.pid
|
||||||
|
|
||||||
|
# SageMath parsed files
|
||||||
|
*.sage.py
|
||||||
|
|
||||||
|
# Environments
|
||||||
|
.env
|
||||||
|
.envrc
|
||||||
|
.venv
|
||||||
|
env/
|
||||||
|
venv/
|
||||||
|
ENV/
|
||||||
|
env.bak/
|
||||||
|
venv.bak/
|
||||||
|
|
||||||
|
# Spyder project settings
|
||||||
|
.spyderproject
|
||||||
|
.spyproject
|
||||||
|
|
||||||
|
# Rope project settings
|
||||||
|
.ropeproject
|
||||||
|
|
||||||
|
# mkdocs documentation
|
||||||
|
/site
|
||||||
|
|
||||||
|
# mypy
|
||||||
|
.mypy_cache/
|
||||||
|
.dmypy.json
|
||||||
|
dmypy.json
|
||||||
|
|
||||||
|
# Pyre type checker
|
||||||
|
.pyre/
|
||||||
|
|
||||||
|
# pytype static type analyzer
|
||||||
|
.pytype/
|
||||||
|
|
||||||
|
# Cython debug symbols
|
||||||
|
cython_debug/
|
||||||
|
|
||||||
|
# PyCharm
|
||||||
|
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
|
||||||
|
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
|
||||||
|
# and can be added to the global gitignore or merged into this file. For a more nuclear
|
||||||
|
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
||||||
|
#.idea/
|
||||||
|
|
||||||
|
# Abstra
|
||||||
|
# Abstra is an AI-powered process automation framework.
|
||||||
|
# Ignore directories containing user credentials, local state, and settings.
|
||||||
|
# Learn more at https://abstra.io/docs
|
||||||
|
.abstra/
|
||||||
|
|
||||||
|
# Visual Studio Code
|
||||||
|
# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore
|
||||||
|
# that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
|
||||||
|
# and can be added to the global gitignore or merged into this file. However, if you prefer,
|
||||||
|
# you could uncomment the following to ignore the entire vscode folder
|
||||||
|
# .vscode/
|
||||||
|
|
||||||
|
# Ruff stuff:
|
||||||
|
.ruff_cache/
|
||||||
|
|
||||||
|
# PyPI configuration file
|
||||||
|
.pypirc
|
||||||
|
|
||||||
|
# Cursor
|
||||||
|
# Cursor is an AI-powered code editor. `.cursorignore` specifies files/directories to
|
||||||
|
# exclude from AI features like autocomplete and code analysis. Recommended for sensitive data
|
||||||
|
# refer to https://docs.cursor.com/context/ignore-files
|
||||||
|
.cursorignore
|
||||||
|
.cursorindexingignore
|
||||||
|
|
||||||
|
# Marimo
|
||||||
|
marimo/_static/
|
||||||
|
marimo/_lsp/
|
||||||
|
__marimo__/
|
||||||
@@ -0,0 +1,34 @@
|
|||||||
|
# See https://pre-commit.com for more information
|
||||||
|
# See https://pre-commit.com/hooks.html for more hooks
|
||||||
|
repos:
|
||||||
|
- repo: https://github.com/pre-commit/pre-commit-hooks
|
||||||
|
rev: v6.0.0
|
||||||
|
hooks:
|
||||||
|
- id: check-added-large-files
|
||||||
|
args: ["--maxkb=750"]
|
||||||
|
exclude: ^uv.lock$
|
||||||
|
- id: end-of-file-fixer
|
||||||
|
- id: trailing-whitespace
|
||||||
|
|
||||||
|
- repo: local
|
||||||
|
hooks:
|
||||||
|
- id: local-ruff-check
|
||||||
|
name: ruff check
|
||||||
|
entry: uv run ruff check --force-exclude --fix --exit-non-zero-on-fix .
|
||||||
|
require_serial: true
|
||||||
|
language: unsupported
|
||||||
|
types: [python]
|
||||||
|
|
||||||
|
- id: local-ruff-format
|
||||||
|
name: ruff format
|
||||||
|
entry: uv run ruff format --force-exclude --exit-non-zero-on-format .
|
||||||
|
require_serial: true
|
||||||
|
language: unsupported
|
||||||
|
types: [python]
|
||||||
|
|
||||||
|
- id: local-ty
|
||||||
|
name: ty check
|
||||||
|
entry: uv run ty check
|
||||||
|
require_serial: true
|
||||||
|
language: unsupported
|
||||||
|
pass_filenames: false
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
3.14
|
||||||
@@ -0,0 +1,21 @@
|
|||||||
|
MIT License
|
||||||
|
|
||||||
|
Copyright (c) 2026 d3vyce
|
||||||
|
|
||||||
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
|
of this software and associated documentation files (the "Software"), to deal
|
||||||
|
in the Software without restriction, including without limitation the rights
|
||||||
|
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
copies of the Software, and to permit persons to whom the Software is
|
||||||
|
furnished to do so, subject to the following conditions:
|
||||||
|
|
||||||
|
The above copyright notice and this permission notice shall be included in all
|
||||||
|
copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
|
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||||
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||||
|
SOFTWARE.
|
||||||
@@ -1,2 +1,66 @@
|
|||||||
# Taskiq deduplication
|
# Taskiq Deduplication
|
||||||
|
|
||||||
|
Redis-backed deduplication middleware for Taskiq that prevents duplicate tasks from being queued or executed concurrently.
|
||||||
|
|
||||||
|
[](https://github.com/d3vyce/taskiq-deduplication/actions/workflows/ci.yml)
|
||||||
|
[](https://codecov.io/gh/d3vyce/taskiq-deduplication)
|
||||||
|
[](https://github.com/astral-sh/ty)
|
||||||
|
[](https://github.com/astral-sh/uv)
|
||||||
|
[](https://github.com/astral-sh/ruff)
|
||||||
|
[](https://www.python.org/downloads/)
|
||||||
|
[](https://opensource.org/licenses/MIT)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
**Documentation**: [https://taskiq-deduplication.d3vyce.fr](https://taskiq-deduplication.d3vyce.fr)
|
||||||
|
|
||||||
|
**Source Code**: [https://github.com/d3vyce/taskiq-deduplication](https://github.com/d3vyce/taskiq-deduplication)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Installation
|
||||||
|
|
||||||
|
```bash
|
||||||
|
uv add taskiq-deduplication
|
||||||
|
```
|
||||||
|
|
||||||
|
## Quick Start
|
||||||
|
|
||||||
|
```python
|
||||||
|
from taskiq_redis import ListQueueBroker
|
||||||
|
from taskiq_deduplication import RedisDeduplicationMiddleware, DuplicateTaskError
|
||||||
|
|
||||||
|
broker = ListQueueBroker("redis://localhost:6379").with_middlewares(
|
||||||
|
RedisDeduplicationMiddleware(redis_url="redis://localhost:6379"),
|
||||||
|
)
|
||||||
|
|
||||||
|
@broker.task
|
||||||
|
async def send_report(user_id: int) -> None:
|
||||||
|
...
|
||||||
|
|
||||||
|
# First dispatch acquires the lock — succeeds.
|
||||||
|
await send_report.kiq(user_id=42)
|
||||||
|
|
||||||
|
# Second dispatch while the first is queued or running — raises.
|
||||||
|
try:
|
||||||
|
await send_report.kiq(user_id=42)
|
||||||
|
except DuplicateTaskError:
|
||||||
|
pass # already queued or running
|
||||||
|
```
|
||||||
|
|
||||||
|
## Features
|
||||||
|
|
||||||
|
- **Sender-side deduplication** — rejects duplicate tasks at dispatch time via a Redis queue lock, before they reach the broker.
|
||||||
|
- **Worker-side detection** — logs concurrent duplicate executions without raising, keeping `SmartRetryMiddleware` safe from retry storms.
|
||||||
|
- **Configurable TTL** — set a global default or override per task with the `deduplication_ttl` label.
|
||||||
|
- **Explicit lock key** — pin any task to a fixed Redis key with `deduplication_key`, bypassing fingerprint computation entirely.
|
||||||
|
- **Partial fingerprint** — deduplicate on a subset of kwargs with `deduplication_key_fields`, ignoring irrelevant arguments.
|
||||||
|
- **Per-task opt-out** — disable deduplication for individual tasks with the `deduplication` label.
|
||||||
|
|
||||||
|
## License
|
||||||
|
|
||||||
|
MIT License - see [LICENSE](LICENSE) for details.
|
||||||
|
|
||||||
|
## Contributing
|
||||||
|
|
||||||
|
Contributions are welcome! Please feel free to submit issues and pull requests.
|
||||||
|
|||||||
@@ -0,0 +1,66 @@
|
|||||||
|
# Taskiq Deduplication
|
||||||
|
|
||||||
|
Redis-backed deduplication middleware for Taskiq that prevents duplicate tasks from being queued or executed concurrently.
|
||||||
|
|
||||||
|
[](https://github.com/d3vyce/taskiq-deduplication/actions/workflows/ci.yml)
|
||||||
|
[](https://codecov.io/gh/d3vyce/taskiq-deduplication)
|
||||||
|
[](https://github.com/astral-sh/ty)
|
||||||
|
[](https://github.com/astral-sh/uv)
|
||||||
|
[](https://github.com/astral-sh/ruff)
|
||||||
|
[](https://www.python.org/downloads/)
|
||||||
|
[](https://opensource.org/licenses/MIT)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
**Documentation**: [https://taskiq-deduplication.d3vyce.fr](https://taskiq-deduplication.d3vyce.fr)
|
||||||
|
|
||||||
|
**Source Code**: [https://github.com/d3vyce/taskiq-deduplication](https://github.com/d3vyce/taskiq-deduplication)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Installation
|
||||||
|
|
||||||
|
```bash
|
||||||
|
uv add "taskiq-deduplication"
|
||||||
|
```
|
||||||
|
|
||||||
|
## Quick Start
|
||||||
|
|
||||||
|
```python
|
||||||
|
from taskiq_redis import ListQueueBroker
|
||||||
|
from taskiq_deduplication import RedisDeduplicationMiddleware, DuplicateTaskError
|
||||||
|
|
||||||
|
broker = ListQueueBroker("redis://localhost:6379").with_middlewares(
|
||||||
|
RedisDeduplicationMiddleware(redis_url="redis://localhost:6379"),
|
||||||
|
)
|
||||||
|
|
||||||
|
@broker.task
|
||||||
|
async def send_report(user_id: int) -> None:
|
||||||
|
...
|
||||||
|
|
||||||
|
# First dispatch acquires the lock — succeeds.
|
||||||
|
await send_report.kiq(user_id=42)
|
||||||
|
|
||||||
|
# Second dispatch while the first is queued or running — raises.
|
||||||
|
try:
|
||||||
|
await send_report.kiq(user_id=42)
|
||||||
|
except DuplicateTaskError:
|
||||||
|
pass # already queued or running
|
||||||
|
```
|
||||||
|
|
||||||
|
## Features
|
||||||
|
|
||||||
|
- **Sender-side deduplication** — rejects duplicate tasks at dispatch time via a Redis queue lock, before they reach the broker.
|
||||||
|
- **Worker-side detection** — logs concurrent duplicate executions without raising, keeping `SmartRetryMiddleware` safe from retry storms.
|
||||||
|
- **Configurable TTL** — set a global default or override per task with the `deduplication_ttl` label.
|
||||||
|
- **Explicit lock key** — pin any task to a fixed Redis key with `deduplication_key`, bypassing fingerprint computation entirely.
|
||||||
|
- **Partial fingerprint** — deduplicate on a subset of kwargs with `deduplication_key_fields`, ignoring irrelevant arguments.
|
||||||
|
- **Per-task opt-out** — disable deduplication for individual tasks with the `deduplication` label.
|
||||||
|
|
||||||
|
## License
|
||||||
|
|
||||||
|
MIT License - see [LICENSE](LICENSE) for details.
|
||||||
|
|
||||||
|
## Contributing
|
||||||
|
|
||||||
|
Contributions are welcome! Please feel free to submit issues and pull requests.
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
# API Reference
|
||||||
|
|
||||||
|
## Middleware
|
||||||
|
|
||||||
|
::: taskiq_deduplication.RedisDeduplicationMiddleware
|
||||||
|
options:
|
||||||
|
show_source: false
|
||||||
|
|
||||||
|
## Exceptions
|
||||||
|
|
||||||
|
::: taskiq_deduplication.DuplicateTaskError
|
||||||
|
options:
|
||||||
|
show_source: false
|
||||||
+120
@@ -0,0 +1,120 @@
|
|||||||
|
# Usage
|
||||||
|
|
||||||
|
## Setup
|
||||||
|
|
||||||
|
Register `RedisDeduplicationMiddleware` on your broker before the application starts:
|
||||||
|
|
||||||
|
```python
|
||||||
|
from taskiq_redis import ListQueueBroker
|
||||||
|
from taskiq_deduplication import RedisDeduplicationMiddleware
|
||||||
|
|
||||||
|
broker = ListQueueBroker("redis://localhost:6379").with_middlewares(
|
||||||
|
RedisDeduplicationMiddleware(redis_url="redis://localhost:6379"),
|
||||||
|
)
|
||||||
|
```
|
||||||
|
|
||||||
|
## Middleware options
|
||||||
|
|
||||||
|
| Parameter | Type | Default | Description |
|
||||||
|
|---|---|---|---|
|
||||||
|
| `redis_url` | `str` | — | Redis connection URL passed to `Redis.from_url`. |
|
||||||
|
| `default_deduplication` | `bool` | `True` | Whether deduplication is enabled for all tasks by default. Set `False` to opt-in per task instead of opting out. |
|
||||||
|
| `default_ttl` | `int` | `300` | Default lock TTL in seconds. Overridden per task with the `deduplication_ttl` label. |
|
||||||
|
| `key_prefix` | `str` | `"taskiq:deduplication"` | Prefix for all Redis lock keys. |
|
||||||
|
|
||||||
|
```python
|
||||||
|
broker = ListQueueBroker("redis://localhost:6379").with_middlewares(
|
||||||
|
RedisDeduplicationMiddleware(
|
||||||
|
redis_url="redis://localhost:6379",
|
||||||
|
default_deduplication=True,
|
||||||
|
default_ttl=60,
|
||||||
|
key_prefix="myapp:dedup",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
```
|
||||||
|
|
||||||
|
## How it works
|
||||||
|
|
||||||
|
When a task is dispatched, the middleware acquires a Redis lock keyed on the task's
|
||||||
|
fingerprint. Any subsequent dispatch with the same fingerprint raises
|
||||||
|
`DuplicateTaskError` while the lock is held. The lock is released automatically when
|
||||||
|
the task completes or fails.
|
||||||
|
|
||||||
|
## Handling duplicates
|
||||||
|
|
||||||
|
When a duplicate is detected, the middleware logs a warning and raises
|
||||||
|
`DuplicateTaskError`, which prevents the task from reaching the broker.
|
||||||
|
Catch it at the call site if you need to handle it explicitly:
|
||||||
|
|
||||||
|
```python
|
||||||
|
from taskiq_deduplication import DuplicateTaskError
|
||||||
|
|
||||||
|
try:
|
||||||
|
await my_task.kiq(user_id=42)
|
||||||
|
except DuplicateTaskError:
|
||||||
|
pass # task is already queued or running
|
||||||
|
```
|
||||||
|
|
||||||
|
## Per-task label overrides
|
||||||
|
|
||||||
|
Labels can be set at the task level (applied to every call) or at call time.
|
||||||
|
|
||||||
|
### Task-level (decorator)
|
||||||
|
|
||||||
|
```python
|
||||||
|
@broker.task(deduplication_ttl=60)
|
||||||
|
async def my_task(user_id: int) -> None:
|
||||||
|
...
|
||||||
|
```
|
||||||
|
|
||||||
|
### Call-level (kicker)
|
||||||
|
|
||||||
|
```python
|
||||||
|
await my_task.kicker().with_labels(deduplication_ttl=60).kiq(user_id=42)
|
||||||
|
```
|
||||||
|
|
||||||
|
### Available labels
|
||||||
|
|
||||||
|
| Label | Type | Description |
|
||||||
|
|---|---|---|
|
||||||
|
| `deduplication` | `bool` | Set `False` to opt out of deduplication entirely for this task. |
|
||||||
|
| `deduplication_ttl` | `int` | Lock TTL in seconds. Overrides the middleware `default_ttl`. |
|
||||||
|
| `deduplication_key` | `str` | Explicit lock key. Skips fingerprint computation entirely. |
|
||||||
|
| `deduplication_key_fields` | `list[str]` | Subset of kwargs to include in the fingerprint. Ignored if `deduplication_key` is set. |
|
||||||
|
|
||||||
|
## Fingerprint and key customisation
|
||||||
|
|
||||||
|
By default the lock key is a SHA-256 fingerprint of the task name and all kwargs.
|
||||||
|
|
||||||
|
### Explicit key
|
||||||
|
|
||||||
|
Use `deduplication_key` when you want full control over the lock key, regardless of
|
||||||
|
the kwargs:
|
||||||
|
|
||||||
|
```python
|
||||||
|
@broker.task(deduplication_key="send-welcome-email")
|
||||||
|
async def send_welcome_email(user_id: int, locale: str) -> None:
|
||||||
|
...
|
||||||
|
```
|
||||||
|
|
||||||
|
All calls to this task share a single lock, no matter what arguments are passed.
|
||||||
|
|
||||||
|
### Partial key (key fields)
|
||||||
|
|
||||||
|
Use `deduplication_key_fields` to deduplicate only on a subset of kwargs.
|
||||||
|
Here, two calls with the same `user_id` but different `locale` are treated as
|
||||||
|
duplicates:
|
||||||
|
|
||||||
|
```python
|
||||||
|
@broker.task(deduplication_key_fields=["user_id"])
|
||||||
|
async def send_welcome_email(user_id: int, locale: str) -> None:
|
||||||
|
...
|
||||||
|
```
|
||||||
|
|
||||||
|
## Opting out per task
|
||||||
|
|
||||||
|
```python
|
||||||
|
@broker.task(deduplication=False)
|
||||||
|
async def always_run(payload: str) -> None:
|
||||||
|
...
|
||||||
|
```
|
||||||
@@ -0,0 +1,80 @@
|
|||||||
|
[project]
|
||||||
|
name = "taskiq-deduplication"
|
||||||
|
version = "0.1.0"
|
||||||
|
description = "Production-ready utilities for FastAPI applications"
|
||||||
|
readme = "README.md"
|
||||||
|
license = "MIT"
|
||||||
|
license-files = ["LICENSE"]
|
||||||
|
requires-python = ">=3.10"
|
||||||
|
authors = [
|
||||||
|
{ name = "d3vyce", email = "contact@d3vyce.fr" }
|
||||||
|
]
|
||||||
|
keywords = ["fastapi", "sqlalchemy", "postgresql"]
|
||||||
|
classifiers = [
|
||||||
|
"Development Status :: 5 - Production/Stable",
|
||||||
|
"Framework :: AsyncIO",
|
||||||
|
"Intended Audience :: Developers",
|
||||||
|
"Operating System :: OS Independent",
|
||||||
|
"Programming Language :: Python :: 3 :: Only",
|
||||||
|
"Programming Language :: Python :: 3.10",
|
||||||
|
"Programming Language :: Python :: 3.11",
|
||||||
|
"Programming Language :: Python :: 3.12",
|
||||||
|
"Programming Language :: Python :: 3.13",
|
||||||
|
"Programming Language :: Python :: 3.14",
|
||||||
|
"Topic :: Software Development :: Libraries :: Python Modules",
|
||||||
|
"Topic :: Software Development :: Libraries",
|
||||||
|
"Topic :: Software Development",
|
||||||
|
"Typing :: Typed",
|
||||||
|
]
|
||||||
|
dependencies = [
|
||||||
|
"redis>=7.0.0",
|
||||||
|
"taskiq>=0.12.0",
|
||||||
|
]
|
||||||
|
|
||||||
|
[project.urls]
|
||||||
|
Homepage = "https://github.com/d3vyce/taskiq-deduplication"
|
||||||
|
Documentation = "https://taskiq-deduplication.d3vyce.fr/"
|
||||||
|
Repository = "https://github.com/d3vyce/taskiq-deduplication"
|
||||||
|
Issues = "https://github.com/d3vyce/taskiq-deduplication/issues"
|
||||||
|
|
||||||
|
[dependency-groups]
|
||||||
|
dev = [
|
||||||
|
{include-group = "tests"},
|
||||||
|
{include-group = "docs"},
|
||||||
|
"taskiq_deduplication",
|
||||||
|
"prek>=0.3.8",
|
||||||
|
"ruff>=0.1.0",
|
||||||
|
"ty>=0.0.1a0",
|
||||||
|
]
|
||||||
|
tests = [
|
||||||
|
"coverage>=7.0.0",
|
||||||
|
"fakeredis[lua]>=2.0.0",
|
||||||
|
"pytest-anyio>=0.0.0",
|
||||||
|
"pytest-cov>=4.0.0",
|
||||||
|
"pytest>=8.0.0",
|
||||||
|
]
|
||||||
|
docs = [
|
||||||
|
"mkdocstrings-python>=2.0.2",
|
||||||
|
"zensical>=0.0.30",
|
||||||
|
]
|
||||||
|
|
||||||
|
[build-system]
|
||||||
|
requires = ["uv_build>=0.10,<0.12.0"]
|
||||||
|
build-backend = "uv_build"
|
||||||
|
|
||||||
|
[tool.pytest.ini_options]
|
||||||
|
testpaths = ["tests"]
|
||||||
|
filterwarnings = [
|
||||||
|
"ignore::DeprecationWarning",
|
||||||
|
]
|
||||||
|
|
||||||
|
[tool.coverage.run]
|
||||||
|
source = ["src/taskiq_deduplication"]
|
||||||
|
branch = true
|
||||||
|
|
||||||
|
[tool.coverage.report]
|
||||||
|
exclude_lines = [
|
||||||
|
"pragma: no cover",
|
||||||
|
"if TYPE_CHECKING:",
|
||||||
|
"raise NotImplementedError",
|
||||||
|
]
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
"""FastAPI utilities package."""
|
||||||
|
|
||||||
|
from .middleware import DuplicateTaskError, RedisDeduplicationMiddleware
|
||||||
|
|
||||||
|
__version__ = "3.1.1"
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"DuplicateTaskError",
|
||||||
|
"RedisDeduplicationMiddleware",
|
||||||
|
]
|
||||||
@@ -0,0 +1,135 @@
|
|||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from redis.asyncio import Redis
|
||||||
|
from taskiq import TaskiqMessage, TaskiqResult
|
||||||
|
from taskiq.abc.middleware import TaskiqMiddleware
|
||||||
|
|
||||||
|
from .utils import check_and_delete
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
DEDUP_LABEL = "deduplication"
|
||||||
|
DEDUP_TTL_LABEL = "deduplication_ttl"
|
||||||
|
DEDUP_KEY_FIELDS_LABEL = "deduplication_key_fields"
|
||||||
|
DEDUP_EXPLICIT_KEY_LABEL = "deduplication_key"
|
||||||
|
|
||||||
|
|
||||||
|
class DuplicateTaskError(Exception):
|
||||||
|
"""Raised when a task with identical name and kwargs is already queued or running."""
|
||||||
|
|
||||||
|
|
||||||
|
class RedisDeduplicationMiddleware(TaskiqMiddleware):
|
||||||
|
"""Prevents duplicate tasks from being queued.
|
||||||
|
|
||||||
|
When a task is dispatched, a Redis lock is acquired for the duration of its
|
||||||
|
execution. Any subsequent task with the same fingerprint is rejected with
|
||||||
|
``DuplicateTaskError`` while the lock is held. The lock is released automatically
|
||||||
|
on completion or error.
|
||||||
|
|
||||||
|
Attributes:
|
||||||
|
redis_url: Redis connection URL passed to ``Redis.from_url``.
|
||||||
|
default_deduplication: Whether deduplication is enabled by default.
|
||||||
|
default_ttl: Default lock TTL in seconds.
|
||||||
|
key_prefix: Prefix for all Redis lock keys.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
redis_url: str,
|
||||||
|
default_deduplication: bool = True,
|
||||||
|
default_ttl: int = 300,
|
||||||
|
key_prefix: str = "taskiq:deduplication",
|
||||||
|
) -> None:
|
||||||
|
self.redis_url = redis_url
|
||||||
|
self.default_deduplication = default_deduplication
|
||||||
|
self.default_ttl = default_ttl
|
||||||
|
self.key_prefix = key_prefix
|
||||||
|
self._redis: Redis | None = None
|
||||||
|
|
||||||
|
async def startup(self) -> None:
|
||||||
|
self._redis = Redis.from_url(self.redis_url)
|
||||||
|
|
||||||
|
async def shutdown(self) -> None:
|
||||||
|
if self._redis is not None:
|
||||||
|
await self._redis.aclose()
|
||||||
|
|
||||||
|
def _build_deduplication_key(self, message: TaskiqMessage) -> str:
|
||||||
|
explicit_key: str | None = message.labels.get(DEDUP_EXPLICIT_KEY_LABEL)
|
||||||
|
if explicit_key is not None:
|
||||||
|
return f"{self.key_prefix}:{explicit_key}"
|
||||||
|
|
||||||
|
key_fields: list[str] | None = message.labels.get(DEDUP_KEY_FIELDS_LABEL)
|
||||||
|
kwargs = (
|
||||||
|
{k: v for k, v in message.kwargs.items() if k in key_fields}
|
||||||
|
if key_fields is not None
|
||||||
|
else message.kwargs
|
||||||
|
)
|
||||||
|
payload = json.dumps(
|
||||||
|
{"task": message.task_name, "kwargs": kwargs},
|
||||||
|
sort_keys=True,
|
||||||
|
)
|
||||||
|
fingerprint = hashlib.sha256(payload.encode()).hexdigest()[:16]
|
||||||
|
return f"{self.key_prefix}:{fingerprint}"
|
||||||
|
|
||||||
|
def _is_enabled(self, labels: dict[str, Any]) -> bool:
|
||||||
|
return bool(labels.get(DEDUP_LABEL, self.default_deduplication))
|
||||||
|
|
||||||
|
def _get_ttl(self, labels: dict[str, Any]) -> int:
|
||||||
|
return int(labels.get(DEDUP_TTL_LABEL, self.default_ttl))
|
||||||
|
|
||||||
|
async def _release_if_owned(self, key: str, task_id: str) -> None:
|
||||||
|
assert self._redis is not None
|
||||||
|
released = await check_and_delete(self._redis, key, task_id)
|
||||||
|
if released:
|
||||||
|
logger.debug("Released lock %s", key)
|
||||||
|
else:
|
||||||
|
logger.debug("Skipped release of lock %s: not owned by this task", key)
|
||||||
|
|
||||||
|
async def pre_send(self, message: TaskiqMessage) -> TaskiqMessage:
|
||||||
|
if not self._is_enabled(message.labels):
|
||||||
|
return message
|
||||||
|
|
||||||
|
assert self._redis is not None
|
||||||
|
key = self._build_deduplication_key(message)
|
||||||
|
ttl = self._get_ttl(message.labels)
|
||||||
|
|
||||||
|
logger.debug("Acquiring lock %s for task %s", key, message.task_name)
|
||||||
|
acquired = await self._redis.set(key, message.task_id, ex=ttl, nx=True)
|
||||||
|
if not acquired:
|
||||||
|
logger.warning(
|
||||||
|
"Duplicate task %s dropped (key=%s).",
|
||||||
|
message.task_name,
|
||||||
|
key,
|
||||||
|
)
|
||||||
|
raise DuplicateTaskError(
|
||||||
|
f"Task {message.task_name!r} with the same arguments is already queued or running."
|
||||||
|
)
|
||||||
|
|
||||||
|
logger.debug("Lock %s acquired for task %s", key, message.task_name)
|
||||||
|
return message
|
||||||
|
|
||||||
|
async def post_execute(
|
||||||
|
self,
|
||||||
|
message: TaskiqMessage,
|
||||||
|
result: TaskiqResult,
|
||||||
|
) -> None:
|
||||||
|
if not self._is_enabled(message.labels):
|
||||||
|
return
|
||||||
|
await self._release_if_owned(
|
||||||
|
self._build_deduplication_key(message), message.task_id
|
||||||
|
)
|
||||||
|
|
||||||
|
async def on_error(
|
||||||
|
self,
|
||||||
|
message: TaskiqMessage,
|
||||||
|
result: TaskiqResult,
|
||||||
|
exception: BaseException,
|
||||||
|
) -> None:
|
||||||
|
if not self._is_enabled(message.labels):
|
||||||
|
return
|
||||||
|
await self._release_if_owned(
|
||||||
|
self._build_deduplication_key(message), message.task_id
|
||||||
|
)
|
||||||
@@ -0,0 +1,18 @@
|
|||||||
|
from collections.abc import Awaitable
|
||||||
|
from typing import cast
|
||||||
|
|
||||||
|
from redis.asyncio import Redis
|
||||||
|
|
||||||
|
|
||||||
|
async def check_and_delete(redis: Redis, key: str, owner: str) -> bool:
|
||||||
|
"""Delete *key* only if its value equals *owner*. Returns True if deleted."""
|
||||||
|
release_script = """
|
||||||
|
if redis.call('get', KEYS[1]) == ARGV[1] then
|
||||||
|
return redis.call('del', KEYS[1])
|
||||||
|
else
|
||||||
|
return 0
|
||||||
|
end
|
||||||
|
"""
|
||||||
|
|
||||||
|
released = await cast(Awaitable[int], redis.eval(release_script, 1, key, owner))
|
||||||
|
return bool(released)
|
||||||
@@ -0,0 +1,43 @@
|
|||||||
|
import pytest
|
||||||
|
import fakeredis.aioredis
|
||||||
|
from taskiq import TaskiqMessage, TaskiqResult
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def anyio_backend():
|
||||||
|
return "asyncio"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
async def fake_redis():
|
||||||
|
client = fakeredis.aioredis.FakeRedis()
|
||||||
|
yield client
|
||||||
|
await client.aclose()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def make_message():
|
||||||
|
def _make(task_name="my_task", task_id="task-1", labels=None, kwargs=None):
|
||||||
|
return TaskiqMessage(
|
||||||
|
task_id=task_id,
|
||||||
|
task_name=task_name,
|
||||||
|
labels=labels or {},
|
||||||
|
labels_types={},
|
||||||
|
args=[],
|
||||||
|
kwargs=kwargs or {},
|
||||||
|
)
|
||||||
|
|
||||||
|
return _make
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def make_result():
|
||||||
|
def _make(is_err=False):
|
||||||
|
return TaskiqResult(
|
||||||
|
is_err=is_err,
|
||||||
|
log="",
|
||||||
|
return_value=None,
|
||||||
|
execution_time=0.0,
|
||||||
|
)
|
||||||
|
|
||||||
|
return _make
|
||||||
@@ -0,0 +1,242 @@
|
|||||||
|
from unittest.mock import AsyncMock, patch
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from taskiq_deduplication import DuplicateTaskError, RedisDeduplicationMiddleware
|
||||||
|
from taskiq_deduplication.middleware import (
|
||||||
|
DEDUP_EXPLICIT_KEY_LABEL,
|
||||||
|
DEDUP_KEY_FIELDS_LABEL,
|
||||||
|
DEDUP_LABEL,
|
||||||
|
DEDUP_TTL_LABEL,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def middleware(fake_redis):
|
||||||
|
mw = RedisDeduplicationMiddleware(redis_url="redis://localhost")
|
||||||
|
mw._redis = fake_redis
|
||||||
|
return mw
|
||||||
|
|
||||||
|
|
||||||
|
class TestDefaultBuildDeduplicationKey:
|
||||||
|
def test_same_kwargs_same_key(self, middleware, make_message):
|
||||||
|
m1 = make_message(kwargs={"a": 1, "b": 2})
|
||||||
|
m2 = make_message(kwargs={"a": 1, "b": 2})
|
||||||
|
assert middleware._build_deduplication_key(
|
||||||
|
m1
|
||||||
|
) == middleware._build_deduplication_key(m2)
|
||||||
|
|
||||||
|
def test_different_kwargs_different_key(self, middleware, make_message):
|
||||||
|
m1 = make_message(kwargs={"a": 1})
|
||||||
|
m2 = make_message(kwargs={"a": 2})
|
||||||
|
assert middleware._build_deduplication_key(
|
||||||
|
m1
|
||||||
|
) != middleware._build_deduplication_key(m2)
|
||||||
|
|
||||||
|
def test_kwarg_order_invariant(self, middleware, make_message):
|
||||||
|
m1 = make_message(kwargs={"a": 1, "b": 2})
|
||||||
|
m2 = make_message(kwargs={"b": 2, "a": 1})
|
||||||
|
assert middleware._build_deduplication_key(
|
||||||
|
m1
|
||||||
|
) == middleware._build_deduplication_key(m2)
|
||||||
|
|
||||||
|
def test_different_task_names_different_keys(self, middleware, make_message):
|
||||||
|
m1 = make_message(task_name="task_a", kwargs={"x": 1})
|
||||||
|
m2 = make_message(task_name="task_b", kwargs={"x": 1})
|
||||||
|
assert middleware._build_deduplication_key(
|
||||||
|
m1
|
||||||
|
) != middleware._build_deduplication_key(m2)
|
||||||
|
|
||||||
|
def test_explicit_key_label(self, middleware, make_message):
|
||||||
|
m = make_message(labels={DEDUP_EXPLICIT_KEY_LABEL: "my-lock"})
|
||||||
|
key = middleware._build_deduplication_key(m)
|
||||||
|
assert key == "taskiq:deduplication:my-lock"
|
||||||
|
|
||||||
|
def test_explicit_key_ignores_kwargs(self, middleware, make_message):
|
||||||
|
m1 = make_message(kwargs={"a": 1}, labels={DEDUP_EXPLICIT_KEY_LABEL: "fixed"})
|
||||||
|
m2 = make_message(kwargs={"a": 99}, labels={DEDUP_EXPLICIT_KEY_LABEL: "fixed"})
|
||||||
|
assert middleware._build_deduplication_key(
|
||||||
|
m1
|
||||||
|
) == middleware._build_deduplication_key(m2)
|
||||||
|
|
||||||
|
def test_key_fields_filters_kwargs(self, middleware, make_message):
|
||||||
|
m1 = make_message(
|
||||||
|
kwargs={"a": 1, "b": 2, "c": 3},
|
||||||
|
labels={DEDUP_KEY_FIELDS_LABEL: ["a", "b"]},
|
||||||
|
)
|
||||||
|
m2 = make_message(
|
||||||
|
kwargs={"a": 1, "b": 2, "c": 999},
|
||||||
|
labels={DEDUP_KEY_FIELDS_LABEL: ["a", "b"]},
|
||||||
|
)
|
||||||
|
assert middleware._build_deduplication_key(
|
||||||
|
m1
|
||||||
|
) == middleware._build_deduplication_key(m2)
|
||||||
|
|
||||||
|
def test_key_fields_different_included_fields(self, middleware, make_message):
|
||||||
|
m1 = make_message(
|
||||||
|
kwargs={"a": 1, "b": 2},
|
||||||
|
labels={DEDUP_KEY_FIELDS_LABEL: ["a"]},
|
||||||
|
)
|
||||||
|
m2 = make_message(
|
||||||
|
kwargs={"a": 1, "b": 99},
|
||||||
|
labels={DEDUP_KEY_FIELDS_LABEL: ["a"]},
|
||||||
|
)
|
||||||
|
assert middleware._build_deduplication_key(
|
||||||
|
m1
|
||||||
|
) == middleware._build_deduplication_key(m2)
|
||||||
|
|
||||||
|
def test_key_prefix_in_output(self, make_message):
|
||||||
|
mw = RedisDeduplicationMiddleware(
|
||||||
|
redis_url="redis://localhost", key_prefix="myapp:locks"
|
||||||
|
)
|
||||||
|
mw._redis = None
|
||||||
|
m = make_message()
|
||||||
|
key = mw._build_deduplication_key(m)
|
||||||
|
assert key.startswith("myapp:locks:")
|
||||||
|
|
||||||
|
|
||||||
|
class TestPreSend:
|
||||||
|
@pytest.mark.anyio
|
||||||
|
async def test_first_send_passes(self, middleware, make_message):
|
||||||
|
msg = make_message()
|
||||||
|
result = await middleware.pre_send(msg)
|
||||||
|
assert result is msg
|
||||||
|
|
||||||
|
@pytest.mark.anyio
|
||||||
|
async def test_duplicate_raises(self, middleware, make_message):
|
||||||
|
msg = make_message()
|
||||||
|
await middleware.pre_send(msg)
|
||||||
|
with pytest.raises(DuplicateTaskError):
|
||||||
|
await middleware.pre_send(make_message())
|
||||||
|
|
||||||
|
@pytest.mark.anyio
|
||||||
|
async def test_deduplication_disabled_label(self, middleware, make_message):
|
||||||
|
msg1 = make_message(labels={DEDUP_LABEL: False})
|
||||||
|
msg2 = make_message(labels={DEDUP_LABEL: False})
|
||||||
|
await middleware.pre_send(msg1)
|
||||||
|
await middleware.pre_send(msg2) # should not raise
|
||||||
|
|
||||||
|
@pytest.mark.anyio
|
||||||
|
async def test_deduplication_disabled_by_default_init(
|
||||||
|
self, fake_redis, make_message
|
||||||
|
):
|
||||||
|
mw = RedisDeduplicationMiddleware(
|
||||||
|
redis_url="redis://localhost", default_deduplication=False
|
||||||
|
)
|
||||||
|
mw._redis = fake_redis
|
||||||
|
await mw.pre_send(make_message())
|
||||||
|
await mw.pre_send(make_message()) # should not raise
|
||||||
|
|
||||||
|
@pytest.mark.anyio
|
||||||
|
async def test_ttl_applied(self, middleware, fake_redis, make_message):
|
||||||
|
msg = make_message(labels={DEDUP_TTL_LABEL: 42})
|
||||||
|
await middleware.pre_send(msg)
|
||||||
|
key = middleware._build_deduplication_key(msg)
|
||||||
|
ttl = await fake_redis.ttl(key)
|
||||||
|
assert 0 < ttl <= 42
|
||||||
|
|
||||||
|
@pytest.mark.anyio
|
||||||
|
async def test_different_kwargs_both_pass(self, middleware, make_message):
|
||||||
|
await middleware.pre_send(make_message(kwargs={"x": 1}))
|
||||||
|
await middleware.pre_send(make_message(kwargs={"x": 2}))
|
||||||
|
|
||||||
|
|
||||||
|
class TestPostExecute:
|
||||||
|
@pytest.mark.anyio
|
||||||
|
async def test_releases_lock(
|
||||||
|
self, middleware, fake_redis, make_message, make_result
|
||||||
|
):
|
||||||
|
msg = make_message()
|
||||||
|
await middleware.pre_send(msg)
|
||||||
|
key = middleware._build_deduplication_key(msg)
|
||||||
|
assert await fake_redis.exists(key)
|
||||||
|
|
||||||
|
await middleware.post_execute(msg, make_result())
|
||||||
|
assert not await fake_redis.exists(key)
|
||||||
|
|
||||||
|
@pytest.mark.anyio
|
||||||
|
async def test_deduplication_disabled_noop(
|
||||||
|
self, middleware, fake_redis, make_message, make_result
|
||||||
|
):
|
||||||
|
msg = make_message()
|
||||||
|
await middleware.pre_send(msg)
|
||||||
|
key = middleware._build_deduplication_key(msg)
|
||||||
|
|
||||||
|
disabled_msg = make_message(labels={DEDUP_LABEL: False})
|
||||||
|
await middleware.post_execute(disabled_msg, make_result())
|
||||||
|
assert await fake_redis.exists(key)
|
||||||
|
|
||||||
|
|
||||||
|
class TestOnError:
|
||||||
|
@pytest.mark.anyio
|
||||||
|
async def test_releases_lock_on_error(
|
||||||
|
self, middleware, fake_redis, make_message, make_result
|
||||||
|
):
|
||||||
|
msg = make_message()
|
||||||
|
await middleware.pre_send(msg)
|
||||||
|
key = middleware._build_deduplication_key(msg)
|
||||||
|
assert await fake_redis.exists(key)
|
||||||
|
|
||||||
|
await middleware.on_error(msg, make_result(is_err=True), RuntimeError("boom"))
|
||||||
|
assert not await fake_redis.exists(key)
|
||||||
|
|
||||||
|
@pytest.mark.anyio
|
||||||
|
async def test_deduplication_disabled_noop(
|
||||||
|
self, middleware, fake_redis, make_message, make_result
|
||||||
|
):
|
||||||
|
msg = make_message()
|
||||||
|
await middleware.pre_send(msg)
|
||||||
|
key = middleware._build_deduplication_key(msg)
|
||||||
|
|
||||||
|
disabled_msg = make_message(labels={DEDUP_LABEL: False})
|
||||||
|
await middleware.on_error(
|
||||||
|
disabled_msg, make_result(is_err=True), RuntimeError("x")
|
||||||
|
)
|
||||||
|
assert await fake_redis.exists(key)
|
||||||
|
|
||||||
|
|
||||||
|
class TestAtomicRelease:
|
||||||
|
@pytest.mark.anyio
|
||||||
|
async def test_only_owner_can_release(self, middleware, fake_redis, make_message):
|
||||||
|
owner_msg = make_message(task_id="owner-task")
|
||||||
|
key = middleware._build_deduplication_key(owner_msg)
|
||||||
|
|
||||||
|
await fake_redis.set(key, "owner-task", ex=300)
|
||||||
|
|
||||||
|
await middleware._release_if_owned(key, "other-task")
|
||||||
|
assert await fake_redis.exists(key)
|
||||||
|
|
||||||
|
await middleware._release_if_owned(key, "owner-task")
|
||||||
|
assert not await fake_redis.exists(key)
|
||||||
|
|
||||||
|
@pytest.mark.anyio
|
||||||
|
async def test_release_missing_key_is_noop(self, middleware, fake_redis):
|
||||||
|
await middleware._release_if_owned(
|
||||||
|
"taskiq:deduplication:nonexistent", "some-task"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class TestLifecycle:
|
||||||
|
@pytest.mark.anyio
|
||||||
|
async def test_startup_creates_redis_client(self):
|
||||||
|
mw = RedisDeduplicationMiddleware(redis_url="redis://localhost")
|
||||||
|
assert mw._redis is None
|
||||||
|
with patch("redis.asyncio.Redis.from_url") as mock_from_url:
|
||||||
|
mock_client = AsyncMock()
|
||||||
|
mock_from_url.return_value = mock_client
|
||||||
|
await mw.startup()
|
||||||
|
mock_from_url.assert_called_once_with("redis://localhost")
|
||||||
|
assert mw._redis is mock_client
|
||||||
|
|
||||||
|
@pytest.mark.anyio
|
||||||
|
async def test_shutdown_closes_redis_client(self):
|
||||||
|
mw = RedisDeduplicationMiddleware(redis_url="redis://localhost")
|
||||||
|
mock_client = AsyncMock()
|
||||||
|
mw._redis = mock_client
|
||||||
|
await mw.shutdown()
|
||||||
|
mock_client.aclose.assert_called_once()
|
||||||
|
|
||||||
|
@pytest.mark.anyio
|
||||||
|
async def test_shutdown_without_startup_is_safe(self):
|
||||||
|
mw = RedisDeduplicationMiddleware(redis_url="redis://localhost")
|
||||||
|
await mw.shutdown()
|
||||||
+112
@@ -0,0 +1,112 @@
|
|||||||
|
[project]
|
||||||
|
site_name = "Taskiq Deduplication"
|
||||||
|
site_description = "Redis-backed deduplication middleware for Taskiq."
|
||||||
|
site_author = "d3vyce"
|
||||||
|
site_url = "https://taskiq-deduplication.d3vyce.fr/"
|
||||||
|
copyright = "Copyright © 2026 d3vyce"
|
||||||
|
repo_url = "https://github.com/d3vyce/taskiq-deduplication"
|
||||||
|
|
||||||
|
[project.theme]
|
||||||
|
language = "en"
|
||||||
|
features = [
|
||||||
|
"announce.dismiss",
|
||||||
|
"content.action.view",
|
||||||
|
"content.code.annotate",
|
||||||
|
"content.code.copy",
|
||||||
|
"content.code.select",
|
||||||
|
"content.footnote.tooltips",
|
||||||
|
"content.tabs.link",
|
||||||
|
"content.tooltips",
|
||||||
|
"navigation.footer",
|
||||||
|
"navigation.indexes",
|
||||||
|
"navigation.instant",
|
||||||
|
"navigation.instant.prefetch",
|
||||||
|
"navigation.path",
|
||||||
|
"navigation.sections",
|
||||||
|
"navigation.tabs",
|
||||||
|
"navigation.top",
|
||||||
|
"navigation.tracking",
|
||||||
|
"search.highlight",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[project.theme.palette]]
|
||||||
|
scheme = "default"
|
||||||
|
toggle.icon = "lucide/sun"
|
||||||
|
toggle.name = "Switch to dark mode"
|
||||||
|
|
||||||
|
[[project.theme.palette]]
|
||||||
|
scheme = "slate"
|
||||||
|
toggle.icon = "lucide/moon"
|
||||||
|
toggle.name = "Switch to light mode"
|
||||||
|
|
||||||
|
[project.theme.font]
|
||||||
|
text = "Inter"
|
||||||
|
code = "Jetbrains Mono"
|
||||||
|
|
||||||
|
[project.theme.icon]
|
||||||
|
repo = "fontawesome/brands/github"
|
||||||
|
|
||||||
|
[project.plugins.mkdocstrings.handlers.python]
|
||||||
|
inventories = ["https://docs.python.org/3/objects.inv"]
|
||||||
|
paths = ["src"]
|
||||||
|
|
||||||
|
[project.plugins.mkdocstrings.handlers.python.options]
|
||||||
|
docstring_style = "google"
|
||||||
|
inherited_members = true
|
||||||
|
show_source = false
|
||||||
|
show_root_heading = true
|
||||||
|
|
||||||
|
[project.markdown_extensions]
|
||||||
|
abbr = {}
|
||||||
|
admonition = {}
|
||||||
|
attr_list = {}
|
||||||
|
def_list = {}
|
||||||
|
footnotes = {}
|
||||||
|
md_in_html = {}
|
||||||
|
"pymdownx.arithmatex" = {generic = true}
|
||||||
|
"pymdownx.betterem" = {}
|
||||||
|
"pymdownx.caret" = {}
|
||||||
|
"pymdownx.details" = {}
|
||||||
|
"pymdownx.emoji" = {}
|
||||||
|
"pymdownx.inlinehilite" = {}
|
||||||
|
"pymdownx.keys" = {}
|
||||||
|
"pymdownx.magiclink" = {}
|
||||||
|
"pymdownx.mark" = {}
|
||||||
|
"pymdownx.smartsymbols" = {}
|
||||||
|
"pymdownx.tasklist" = {custom_checkbox = true}
|
||||||
|
"pymdownx.tilde" = {}
|
||||||
|
|
||||||
|
[project.markdown_extensions.pymdownx.emoji]
|
||||||
|
emoji_index = "zensical.extensions.emoji.twemoji"
|
||||||
|
emoji_generator = "zensical.extensions.emoji.to_svg"
|
||||||
|
|
||||||
|
[project.markdown_extensions."pymdownx.highlight"]
|
||||||
|
anchor_linenums = true
|
||||||
|
line_spans = "__span"
|
||||||
|
pygments_lang_class = true
|
||||||
|
|
||||||
|
[project.markdown_extensions."pymdownx.superfences"]
|
||||||
|
custom_fences = [{name = "mermaid", class = "mermaid"}]
|
||||||
|
|
||||||
|
[project.markdown_extensions."pymdownx.tabbed"]
|
||||||
|
alternate_style = true
|
||||||
|
combine_header_slug = true
|
||||||
|
|
||||||
|
[project.markdown_extensions."toc"]
|
||||||
|
permalink = true
|
||||||
|
|
||||||
|
[project.markdown_extensions."pymdownx.snippets"]
|
||||||
|
base_path = ["."]
|
||||||
|
check_paths = true
|
||||||
|
|
||||||
|
[[project.nav]]
|
||||||
|
Home = "index.md"
|
||||||
|
|
||||||
|
[[project.nav]]
|
||||||
|
Usage = "usage.md"
|
||||||
|
|
||||||
|
[[project.nav]]
|
||||||
|
"API Reference" = "reference.md"
|
||||||
|
|
||||||
|
[[project.nav]]
|
||||||
|
"Changelog ↗" = "https://github.com/d3vyce/taskiq-deduplication/releases"
|
||||||
Reference in New Issue
Block a user