mirror of
https://github.com/d3vyce/taskiq-deduplication.git
synced 2026-08-04 19:14:07 +00:00
Version 1.0.0 (#2)
* feat: add taskiq deduplication * doc: rework class comment + update README * fix: make build_deduplication_key private * fix: raise_on_duplicate is now False by default * chore: remove pre_execute * chore: add documentation
This commit is contained in:
d3vyce
@@ -0,0 +1,14 @@
|
||||
version: 2
|
||||
updates:
|
||||
- package-ecosystem: "github-actions"
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
commit-message:
|
||||
prefix: ⬆
|
||||
- package-ecosystem: "uv"
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
commit-message:
|
||||
prefix: ⬆
|
||||
@@ -0,0 +1,29 @@
|
||||
name: Build Package
|
||||
|
||||
on:
|
||||
release:
|
||||
types: [published]
|
||||
|
||||
jobs:
|
||||
build-package:
|
||||
runs-on: ubuntu-latest
|
||||
environment: pypi
|
||||
permissions:
|
||||
id-token: write
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v7
|
||||
|
||||
- name: Set up Python
|
||||
run: uv python install 3.14
|
||||
|
||||
- name: Install dependencies
|
||||
run: uv sync
|
||||
|
||||
- name: Build
|
||||
run: uv build
|
||||
|
||||
- name: Publish package distributions to PyPI
|
||||
uses: pypa/gh-action-pypi-publish@release/v1
|
||||
@@ -0,0 +1,93 @@
|
||||
name: CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
pull_request:
|
||||
branches: [main]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
lint:
|
||||
name: Lint (Ruff)
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v7
|
||||
|
||||
- name: Set up Python
|
||||
run: uv python install 3.13
|
||||
|
||||
- name: Install dependencies
|
||||
run: uv sync --group dev
|
||||
|
||||
- name: Run Ruff linter
|
||||
run: uv run ruff check .
|
||||
|
||||
- name: Run Ruff formatter check
|
||||
run: uv run ruff format --check .
|
||||
|
||||
typecheck:
|
||||
name: Type Check (ty)
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v7
|
||||
|
||||
- name: Set up Python
|
||||
run: uv python install 3.13
|
||||
|
||||
- name: Install dependencies
|
||||
run: uv sync --group dev
|
||||
|
||||
- name: Run ty
|
||||
run: uv run ty check
|
||||
|
||||
test:
|
||||
name: Test (Python ${{ matrix.python-version }})
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v7
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
run: uv python install ${{ matrix.python-version }}
|
||||
|
||||
- name: Install dependencies
|
||||
run: uv sync --group dev
|
||||
|
||||
- name: Run tests with coverage
|
||||
run: uv run pytest --cov --cov-report=xml --cov-report=term-missing --junitxml=junit.xml -o junit_family=legacy
|
||||
|
||||
- name: Upload coverage to Codecov
|
||||
if: matrix.python-version == '3.14'
|
||||
uses: codecov/codecov-action@v6
|
||||
with:
|
||||
token: ${{ secrets.CODECOV_TOKEN }}
|
||||
report_type: coverage
|
||||
files: ./coverage.xml
|
||||
fail_ci_if_error: false
|
||||
|
||||
- name: Upload test results to Codecov
|
||||
if: matrix.python-version == '3.14'
|
||||
uses: codecov/codecov-action@v6
|
||||
with:
|
||||
token: ${{ secrets.CODECOV_TOKEN }}
|
||||
report_type: test_results
|
||||
@@ -0,0 +1,38 @@
|
||||
name: Documentation
|
||||
|
||||
on:
|
||||
release:
|
||||
types: [published]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pages: write
|
||||
id-token: write
|
||||
|
||||
jobs:
|
||||
deploy:
|
||||
environment:
|
||||
name: github-pages
|
||||
url: ${{ steps.deployment.outputs.page_url }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/configure-pages@v5
|
||||
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v7
|
||||
|
||||
- name: Set up Python
|
||||
run: uv python install 3.13
|
||||
|
||||
- run: uv sync --group dev
|
||||
|
||||
- run: uv run zensical build --clean
|
||||
|
||||
- uses: actions/upload-pages-artifact@v4
|
||||
with:
|
||||
path: site
|
||||
|
||||
- uses: actions/deploy-pages@v4
|
||||
id: deployment
|
||||
+207
@@ -0,0 +1,207 @@
|
||||
# Byte-compiled / optimized / DLL files
|
||||
__pycache__/
|
||||
*.py[codz]
|
||||
*$py.class
|
||||
|
||||
# C extensions
|
||||
*.so
|
||||
|
||||
# Distribution / packaging
|
||||
.Python
|
||||
build/
|
||||
develop-eggs/
|
||||
dist/
|
||||
downloads/
|
||||
eggs/
|
||||
.eggs/
|
||||
lib/
|
||||
lib64/
|
||||
parts/
|
||||
sdist/
|
||||
var/
|
||||
wheels/
|
||||
share/python-wheels/
|
||||
*.egg-info/
|
||||
.installed.cfg
|
||||
*.egg
|
||||
MANIFEST
|
||||
|
||||
# PyInstaller
|
||||
# Usually these files are written by a python script from a template
|
||||
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
||||
*.manifest
|
||||
*.spec
|
||||
|
||||
# Installer logs
|
||||
pip-log.txt
|
||||
pip-delete-this-directory.txt
|
||||
|
||||
# Unit test / coverage reports
|
||||
htmlcov/
|
||||
.tox/
|
||||
.nox/
|
||||
.coverage
|
||||
.coverage.*
|
||||
.cache
|
||||
nosetests.xml
|
||||
coverage.xml
|
||||
*.cover
|
||||
*.py.cover
|
||||
.hypothesis/
|
||||
.pytest_cache/
|
||||
cover/
|
||||
|
||||
# Translations
|
||||
*.mo
|
||||
*.pot
|
||||
|
||||
# Django stuff:
|
||||
*.log
|
||||
local_settings.py
|
||||
db.sqlite3
|
||||
db.sqlite3-journal
|
||||
|
||||
# Flask stuff:
|
||||
instance/
|
||||
.webassets-cache
|
||||
|
||||
# Scrapy stuff:
|
||||
.scrapy
|
||||
|
||||
# Sphinx documentation
|
||||
docs/_build/
|
||||
|
||||
# PyBuilder
|
||||
.pybuilder/
|
||||
target/
|
||||
|
||||
# Jupyter Notebook
|
||||
.ipynb_checkpoints
|
||||
|
||||
# IPython
|
||||
profile_default/
|
||||
ipython_config.py
|
||||
|
||||
# pyenv
|
||||
# For a library or package, you might want to ignore these files since the code is
|
||||
# intended to run in multiple environments; otherwise, check them in:
|
||||
# .python-version
|
||||
|
||||
# pipenv
|
||||
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
||||
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
||||
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
||||
# install all needed dependencies.
|
||||
#Pipfile.lock
|
||||
|
||||
# UV
|
||||
# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
|
||||
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
||||
# commonly ignored for libraries.
|
||||
#uv.lock
|
||||
|
||||
# poetry
|
||||
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
|
||||
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
||||
# commonly ignored for libraries.
|
||||
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
|
||||
#poetry.lock
|
||||
#poetry.toml
|
||||
|
||||
# pdm
|
||||
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
|
||||
# pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
|
||||
# https://pdm-project.org/en/latest/usage/project/#working-with-version-control
|
||||
#pdm.lock
|
||||
#pdm.toml
|
||||
.pdm-python
|
||||
.pdm-build/
|
||||
|
||||
# pixi
|
||||
# Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
|
||||
#pixi.lock
|
||||
# Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
|
||||
# in the .venv directory. It is recommended not to include this directory in version control.
|
||||
.pixi
|
||||
|
||||
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
|
||||
__pypackages__/
|
||||
|
||||
# Celery stuff
|
||||
celerybeat-schedule
|
||||
celerybeat.pid
|
||||
|
||||
# SageMath parsed files
|
||||
*.sage.py
|
||||
|
||||
# Environments
|
||||
.env
|
||||
.envrc
|
||||
.venv
|
||||
env/
|
||||
venv/
|
||||
ENV/
|
||||
env.bak/
|
||||
venv.bak/
|
||||
|
||||
# Spyder project settings
|
||||
.spyderproject
|
||||
.spyproject
|
||||
|
||||
# Rope project settings
|
||||
.ropeproject
|
||||
|
||||
# mkdocs documentation
|
||||
/site
|
||||
|
||||
# mypy
|
||||
.mypy_cache/
|
||||
.dmypy.json
|
||||
dmypy.json
|
||||
|
||||
# Pyre type checker
|
||||
.pyre/
|
||||
|
||||
# pytype static type analyzer
|
||||
.pytype/
|
||||
|
||||
# Cython debug symbols
|
||||
cython_debug/
|
||||
|
||||
# PyCharm
|
||||
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
|
||||
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
|
||||
# and can be added to the global gitignore or merged into this file. For a more nuclear
|
||||
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
||||
#.idea/
|
||||
|
||||
# Abstra
|
||||
# Abstra is an AI-powered process automation framework.
|
||||
# Ignore directories containing user credentials, local state, and settings.
|
||||
# Learn more at https://abstra.io/docs
|
||||
.abstra/
|
||||
|
||||
# Visual Studio Code
|
||||
# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore
|
||||
# that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
|
||||
# and can be added to the global gitignore or merged into this file. However, if you prefer,
|
||||
# you could uncomment the following to ignore the entire vscode folder
|
||||
# .vscode/
|
||||
|
||||
# Ruff stuff:
|
||||
.ruff_cache/
|
||||
|
||||
# PyPI configuration file
|
||||
.pypirc
|
||||
|
||||
# Cursor
|
||||
# Cursor is an AI-powered code editor. `.cursorignore` specifies files/directories to
|
||||
# exclude from AI features like autocomplete and code analysis. Recommended for sensitive data
|
||||
# refer to https://docs.cursor.com/context/ignore-files
|
||||
.cursorignore
|
||||
.cursorindexingignore
|
||||
|
||||
# Marimo
|
||||
marimo/_static/
|
||||
marimo/_lsp/
|
||||
__marimo__/
|
||||
@@ -0,0 +1,34 @@
|
||||
# See https://pre-commit.com for more information
|
||||
# See https://pre-commit.com/hooks.html for more hooks
|
||||
repos:
|
||||
- repo: https://github.com/pre-commit/pre-commit-hooks
|
||||
rev: v6.0.0
|
||||
hooks:
|
||||
- id: check-added-large-files
|
||||
args: ["--maxkb=750"]
|
||||
exclude: ^uv.lock$
|
||||
- id: end-of-file-fixer
|
||||
- id: trailing-whitespace
|
||||
|
||||
- repo: local
|
||||
hooks:
|
||||
- id: local-ruff-check
|
||||
name: ruff check
|
||||
entry: uv run ruff check --force-exclude --fix --exit-non-zero-on-fix .
|
||||
require_serial: true
|
||||
language: unsupported
|
||||
types: [python]
|
||||
|
||||
- id: local-ruff-format
|
||||
name: ruff format
|
||||
entry: uv run ruff format --force-exclude --exit-non-zero-on-format .
|
||||
require_serial: true
|
||||
language: unsupported
|
||||
types: [python]
|
||||
|
||||
- id: local-ty
|
||||
name: ty check
|
||||
entry: uv run ty check
|
||||
require_serial: true
|
||||
language: unsupported
|
||||
pass_filenames: false
|
||||
@@ -0,0 +1 @@
|
||||
3.14
|
||||
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2026 d3vyce
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
@@ -1,2 +1,66 @@
|
||||
# Taskiq deduplication
|
||||
# Taskiq Deduplication
|
||||
|
||||
Redis-backed deduplication middleware for Taskiq that prevents duplicate tasks from being queued or executed concurrently.
|
||||
|
||||
[](https://github.com/d3vyce/taskiq-deduplication/actions/workflows/ci.yml)
|
||||
[](https://codecov.io/gh/d3vyce/taskiq-deduplication)
|
||||
[](https://github.com/astral-sh/ty)
|
||||
[](https://github.com/astral-sh/uv)
|
||||
[](https://github.com/astral-sh/ruff)
|
||||
[](https://www.python.org/downloads/)
|
||||
[](https://opensource.org/licenses/MIT)
|
||||
|
||||
---
|
||||
|
||||
**Documentation**: [https://taskiq-deduplication.d3vyce.fr](https://taskiq-deduplication.d3vyce.fr)
|
||||
|
||||
**Source Code**: [https://github.com/d3vyce/taskiq-deduplication](https://github.com/d3vyce/taskiq-deduplication)
|
||||
|
||||
---
|
||||
|
||||
## Installation
|
||||
|
||||
```bash
|
||||
uv add taskiq-deduplication
|
||||
```
|
||||
|
||||
## Quick Start
|
||||
|
||||
```python
|
||||
from taskiq_redis import ListQueueBroker
|
||||
from taskiq_deduplication import RedisDeduplicationMiddleware, DuplicateTaskError
|
||||
|
||||
broker = ListQueueBroker("redis://localhost:6379").with_middlewares(
|
||||
RedisDeduplicationMiddleware(redis_url="redis://localhost:6379"),
|
||||
)
|
||||
|
||||
@broker.task
|
||||
async def send_report(user_id: int) -> None:
|
||||
...
|
||||
|
||||
# First dispatch acquires the lock — succeeds.
|
||||
await send_report.kiq(user_id=42)
|
||||
|
||||
# Second dispatch while the first is queued or running — raises.
|
||||
try:
|
||||
await send_report.kiq(user_id=42)
|
||||
except DuplicateTaskError:
|
||||
pass # already queued or running
|
||||
```
|
||||
|
||||
## Features
|
||||
|
||||
- **Sender-side deduplication** — rejects duplicate tasks at dispatch time via a Redis queue lock, before they reach the broker.
|
||||
- **Worker-side detection** — logs concurrent duplicate executions without raising, keeping `SmartRetryMiddleware` safe from retry storms.
|
||||
- **Configurable TTL** — set a global default or override per task with the `deduplication_ttl` label.
|
||||
- **Explicit lock key** — pin any task to a fixed Redis key with `deduplication_key`, bypassing fingerprint computation entirely.
|
||||
- **Partial fingerprint** — deduplicate on a subset of kwargs with `deduplication_key_fields`, ignoring irrelevant arguments.
|
||||
- **Per-task opt-out** — disable deduplication for individual tasks with the `deduplication` label.
|
||||
|
||||
## License
|
||||
|
||||
MIT License - see [LICENSE](LICENSE) for details.
|
||||
|
||||
## Contributing
|
||||
|
||||
Contributions are welcome! Please feel free to submit issues and pull requests.
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
# Taskiq Deduplication
|
||||
|
||||
Redis-backed deduplication middleware for Taskiq that prevents duplicate tasks from being queued or executed concurrently.
|
||||
|
||||
[](https://github.com/d3vyce/taskiq-deduplication/actions/workflows/ci.yml)
|
||||
[](https://codecov.io/gh/d3vyce/taskiq-deduplication)
|
||||
[](https://github.com/astral-sh/ty)
|
||||
[](https://github.com/astral-sh/uv)
|
||||
[](https://github.com/astral-sh/ruff)
|
||||
[](https://www.python.org/downloads/)
|
||||
[](https://opensource.org/licenses/MIT)
|
||||
|
||||
---
|
||||
|
||||
**Documentation**: [https://taskiq-deduplication.d3vyce.fr](https://taskiq-deduplication.d3vyce.fr)
|
||||
|
||||
**Source Code**: [https://github.com/d3vyce/taskiq-deduplication](https://github.com/d3vyce/taskiq-deduplication)
|
||||
|
||||
---
|
||||
|
||||
## Installation
|
||||
|
||||
```bash
|
||||
uv add "taskiq-deduplication"
|
||||
```
|
||||
|
||||
## Quick Start
|
||||
|
||||
```python
|
||||
from taskiq_redis import ListQueueBroker
|
||||
from taskiq_deduplication import RedisDeduplicationMiddleware, DuplicateTaskError
|
||||
|
||||
broker = ListQueueBroker("redis://localhost:6379").with_middlewares(
|
||||
RedisDeduplicationMiddleware(redis_url="redis://localhost:6379"),
|
||||
)
|
||||
|
||||
@broker.task
|
||||
async def send_report(user_id: int) -> None:
|
||||
...
|
||||
|
||||
# First dispatch acquires the lock — succeeds.
|
||||
await send_report.kiq(user_id=42)
|
||||
|
||||
# Second dispatch while the first is queued or running — raises.
|
||||
try:
|
||||
await send_report.kiq(user_id=42)
|
||||
except DuplicateTaskError:
|
||||
pass # already queued or running
|
||||
```
|
||||
|
||||
## Features
|
||||
|
||||
- **Sender-side deduplication** — rejects duplicate tasks at dispatch time via a Redis queue lock, before they reach the broker.
|
||||
- **Worker-side detection** — logs concurrent duplicate executions without raising, keeping `SmartRetryMiddleware` safe from retry storms.
|
||||
- **Configurable TTL** — set a global default or override per task with the `deduplication_ttl` label.
|
||||
- **Explicit lock key** — pin any task to a fixed Redis key with `deduplication_key`, bypassing fingerprint computation entirely.
|
||||
- **Partial fingerprint** — deduplicate on a subset of kwargs with `deduplication_key_fields`, ignoring irrelevant arguments.
|
||||
- **Per-task opt-out** — disable deduplication for individual tasks with the `deduplication` label.
|
||||
|
||||
## License
|
||||
|
||||
MIT License - see [LICENSE](LICENSE) for details.
|
||||
|
||||
## Contributing
|
||||
|
||||
Contributions are welcome! Please feel free to submit issues and pull requests.
|
||||
@@ -0,0 +1,13 @@
|
||||
# API Reference
|
||||
|
||||
## Middleware
|
||||
|
||||
::: taskiq_deduplication.RedisDeduplicationMiddleware
|
||||
options:
|
||||
show_source: false
|
||||
|
||||
## Exceptions
|
||||
|
||||
::: taskiq_deduplication.DuplicateTaskError
|
||||
options:
|
||||
show_source: false
|
||||
+120
@@ -0,0 +1,120 @@
|
||||
# Usage
|
||||
|
||||
## Setup
|
||||
|
||||
Register `RedisDeduplicationMiddleware` on your broker before the application starts:
|
||||
|
||||
```python
|
||||
from taskiq_redis import ListQueueBroker
|
||||
from taskiq_deduplication import RedisDeduplicationMiddleware
|
||||
|
||||
broker = ListQueueBroker("redis://localhost:6379").with_middlewares(
|
||||
RedisDeduplicationMiddleware(redis_url="redis://localhost:6379"),
|
||||
)
|
||||
```
|
||||
|
||||
## Middleware options
|
||||
|
||||
| Parameter | Type | Default | Description |
|
||||
|---|---|---|---|
|
||||
| `redis_url` | `str` | — | Redis connection URL passed to `Redis.from_url`. |
|
||||
| `default_deduplication` | `bool` | `True` | Whether deduplication is enabled for all tasks by default. Set `False` to opt-in per task instead of opting out. |
|
||||
| `default_ttl` | `int` | `300` | Default lock TTL in seconds. Overridden per task with the `deduplication_ttl` label. |
|
||||
| `key_prefix` | `str` | `"taskiq:deduplication"` | Prefix for all Redis lock keys. |
|
||||
|
||||
```python
|
||||
broker = ListQueueBroker("redis://localhost:6379").with_middlewares(
|
||||
RedisDeduplicationMiddleware(
|
||||
redis_url="redis://localhost:6379",
|
||||
default_deduplication=True,
|
||||
default_ttl=60,
|
||||
key_prefix="myapp:dedup",
|
||||
),
|
||||
)
|
||||
```
|
||||
|
||||
## How it works
|
||||
|
||||
When a task is dispatched, the middleware acquires a Redis lock keyed on the task's
|
||||
fingerprint. Any subsequent dispatch with the same fingerprint raises
|
||||
`DuplicateTaskError` while the lock is held. The lock is released automatically when
|
||||
the task completes or fails.
|
||||
|
||||
## Handling duplicates
|
||||
|
||||
When a duplicate is detected, the middleware logs a warning and raises
|
||||
`DuplicateTaskError`, which prevents the task from reaching the broker.
|
||||
Catch it at the call site if you need to handle it explicitly:
|
||||
|
||||
```python
|
||||
from taskiq_deduplication import DuplicateTaskError
|
||||
|
||||
try:
|
||||
await my_task.kiq(user_id=42)
|
||||
except DuplicateTaskError:
|
||||
pass # task is already queued or running
|
||||
```
|
||||
|
||||
## Per-task label overrides
|
||||
|
||||
Labels can be set at the task level (applied to every call) or at call time.
|
||||
|
||||
### Task-level (decorator)
|
||||
|
||||
```python
|
||||
@broker.task(deduplication_ttl=60)
|
||||
async def my_task(user_id: int) -> None:
|
||||
...
|
||||
```
|
||||
|
||||
### Call-level (kicker)
|
||||
|
||||
```python
|
||||
await my_task.kicker().with_labels(deduplication_ttl=60).kiq(user_id=42)
|
||||
```
|
||||
|
||||
### Available labels
|
||||
|
||||
| Label | Type | Description |
|
||||
|---|---|---|
|
||||
| `deduplication` | `bool` | Set `False` to opt out of deduplication entirely for this task. |
|
||||
| `deduplication_ttl` | `int` | Lock TTL in seconds. Overrides the middleware `default_ttl`. |
|
||||
| `deduplication_key` | `str` | Explicit lock key. Skips fingerprint computation entirely. |
|
||||
| `deduplication_key_fields` | `list[str]` | Subset of kwargs to include in the fingerprint. Ignored if `deduplication_key` is set. |
|
||||
|
||||
## Fingerprint and key customisation
|
||||
|
||||
By default the lock key is a SHA-256 fingerprint of the task name and all kwargs.
|
||||
|
||||
### Explicit key
|
||||
|
||||
Use `deduplication_key` when you want full control over the lock key, regardless of
|
||||
the kwargs:
|
||||
|
||||
```python
|
||||
@broker.task(deduplication_key="send-welcome-email")
|
||||
async def send_welcome_email(user_id: int, locale: str) -> None:
|
||||
...
|
||||
```
|
||||
|
||||
All calls to this task share a single lock, no matter what arguments are passed.
|
||||
|
||||
### Partial key (key fields)
|
||||
|
||||
Use `deduplication_key_fields` to deduplicate only on a subset of kwargs.
|
||||
Here, two calls with the same `user_id` but different `locale` are treated as
|
||||
duplicates:
|
||||
|
||||
```python
|
||||
@broker.task(deduplication_key_fields=["user_id"])
|
||||
async def send_welcome_email(user_id: int, locale: str) -> None:
|
||||
...
|
||||
```
|
||||
|
||||
## Opting out per task
|
||||
|
||||
```python
|
||||
@broker.task(deduplication=False)
|
||||
async def always_run(payload: str) -> None:
|
||||
...
|
||||
```
|
||||
@@ -0,0 +1,80 @@
|
||||
[project]
|
||||
name = "taskiq-deduplication"
|
||||
version = "0.1.0"
|
||||
description = "Production-ready utilities for FastAPI applications"
|
||||
readme = "README.md"
|
||||
license = "MIT"
|
||||
license-files = ["LICENSE"]
|
||||
requires-python = ">=3.10"
|
||||
authors = [
|
||||
{ name = "d3vyce", email = "contact@d3vyce.fr" }
|
||||
]
|
||||
keywords = ["fastapi", "sqlalchemy", "postgresql"]
|
||||
classifiers = [
|
||||
"Development Status :: 5 - Production/Stable",
|
||||
"Framework :: AsyncIO",
|
||||
"Intended Audience :: Developers",
|
||||
"Operating System :: OS Independent",
|
||||
"Programming Language :: Python :: 3 :: Only",
|
||||
"Programming Language :: Python :: 3.10",
|
||||
"Programming Language :: Python :: 3.11",
|
||||
"Programming Language :: Python :: 3.12",
|
||||
"Programming Language :: Python :: 3.13",
|
||||
"Programming Language :: Python :: 3.14",
|
||||
"Topic :: Software Development :: Libraries :: Python Modules",
|
||||
"Topic :: Software Development :: Libraries",
|
||||
"Topic :: Software Development",
|
||||
"Typing :: Typed",
|
||||
]
|
||||
dependencies = [
|
||||
"redis>=7.0.0",
|
||||
"taskiq>=0.12.0",
|
||||
]
|
||||
|
||||
[project.urls]
|
||||
Homepage = "https://github.com/d3vyce/taskiq-deduplication"
|
||||
Documentation = "https://taskiq-deduplication.d3vyce.fr/"
|
||||
Repository = "https://github.com/d3vyce/taskiq-deduplication"
|
||||
Issues = "https://github.com/d3vyce/taskiq-deduplication/issues"
|
||||
|
||||
[dependency-groups]
|
||||
dev = [
|
||||
{include-group = "tests"},
|
||||
{include-group = "docs"},
|
||||
"taskiq_deduplication",
|
||||
"prek>=0.3.8",
|
||||
"ruff>=0.1.0",
|
||||
"ty>=0.0.1a0",
|
||||
]
|
||||
tests = [
|
||||
"coverage>=7.0.0",
|
||||
"fakeredis[lua]>=2.0.0",
|
||||
"pytest-anyio>=0.0.0",
|
||||
"pytest-cov>=4.0.0",
|
||||
"pytest>=8.0.0",
|
||||
]
|
||||
docs = [
|
||||
"mkdocstrings-python>=2.0.2",
|
||||
"zensical>=0.0.30",
|
||||
]
|
||||
|
||||
[build-system]
|
||||
requires = ["uv_build>=0.10,<0.12.0"]
|
||||
build-backend = "uv_build"
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
testpaths = ["tests"]
|
||||
filterwarnings = [
|
||||
"ignore::DeprecationWarning",
|
||||
]
|
||||
|
||||
[tool.coverage.run]
|
||||
source = ["src/taskiq_deduplication"]
|
||||
branch = true
|
||||
|
||||
[tool.coverage.report]
|
||||
exclude_lines = [
|
||||
"pragma: no cover",
|
||||
"if TYPE_CHECKING:",
|
||||
"raise NotImplementedError",
|
||||
]
|
||||
@@ -0,0 +1,10 @@
|
||||
"""FastAPI utilities package."""
|
||||
|
||||
from .middleware import DuplicateTaskError, RedisDeduplicationMiddleware
|
||||
|
||||
__version__ = "3.1.1"
|
||||
|
||||
__all__ = [
|
||||
"DuplicateTaskError",
|
||||
"RedisDeduplicationMiddleware",
|
||||
]
|
||||
@@ -0,0 +1,135 @@
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from redis.asyncio import Redis
|
||||
from taskiq import TaskiqMessage, TaskiqResult
|
||||
from taskiq.abc.middleware import TaskiqMiddleware
|
||||
|
||||
from .utils import check_and_delete
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
DEDUP_LABEL = "deduplication"
|
||||
DEDUP_TTL_LABEL = "deduplication_ttl"
|
||||
DEDUP_KEY_FIELDS_LABEL = "deduplication_key_fields"
|
||||
DEDUP_EXPLICIT_KEY_LABEL = "deduplication_key"
|
||||
|
||||
|
||||
class DuplicateTaskError(Exception):
|
||||
"""Raised when a task with identical name and kwargs is already queued or running."""
|
||||
|
||||
|
||||
class RedisDeduplicationMiddleware(TaskiqMiddleware):
|
||||
"""Prevents duplicate tasks from being queued.
|
||||
|
||||
When a task is dispatched, a Redis lock is acquired for the duration of its
|
||||
execution. Any subsequent task with the same fingerprint is rejected with
|
||||
``DuplicateTaskError`` while the lock is held. The lock is released automatically
|
||||
on completion or error.
|
||||
|
||||
Attributes:
|
||||
redis_url: Redis connection URL passed to ``Redis.from_url``.
|
||||
default_deduplication: Whether deduplication is enabled by default.
|
||||
default_ttl: Default lock TTL in seconds.
|
||||
key_prefix: Prefix for all Redis lock keys.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
redis_url: str,
|
||||
default_deduplication: bool = True,
|
||||
default_ttl: int = 300,
|
||||
key_prefix: str = "taskiq:deduplication",
|
||||
) -> None:
|
||||
self.redis_url = redis_url
|
||||
self.default_deduplication = default_deduplication
|
||||
self.default_ttl = default_ttl
|
||||
self.key_prefix = key_prefix
|
||||
self._redis: Redis | None = None
|
||||
|
||||
async def startup(self) -> None:
|
||||
self._redis = Redis.from_url(self.redis_url)
|
||||
|
||||
async def shutdown(self) -> None:
|
||||
if self._redis is not None:
|
||||
await self._redis.aclose()
|
||||
|
||||
def _build_deduplication_key(self, message: TaskiqMessage) -> str:
|
||||
explicit_key: str | None = message.labels.get(DEDUP_EXPLICIT_KEY_LABEL)
|
||||
if explicit_key is not None:
|
||||
return f"{self.key_prefix}:{explicit_key}"
|
||||
|
||||
key_fields: list[str] | None = message.labels.get(DEDUP_KEY_FIELDS_LABEL)
|
||||
kwargs = (
|
||||
{k: v for k, v in message.kwargs.items() if k in key_fields}
|
||||
if key_fields is not None
|
||||
else message.kwargs
|
||||
)
|
||||
payload = json.dumps(
|
||||
{"task": message.task_name, "kwargs": kwargs},
|
||||
sort_keys=True,
|
||||
)
|
||||
fingerprint = hashlib.sha256(payload.encode()).hexdigest()[:16]
|
||||
return f"{self.key_prefix}:{fingerprint}"
|
||||
|
||||
def _is_enabled(self, labels: dict[str, Any]) -> bool:
|
||||
return bool(labels.get(DEDUP_LABEL, self.default_deduplication))
|
||||
|
||||
def _get_ttl(self, labels: dict[str, Any]) -> int:
|
||||
return int(labels.get(DEDUP_TTL_LABEL, self.default_ttl))
|
||||
|
||||
async def _release_if_owned(self, key: str, task_id: str) -> None:
|
||||
assert self._redis is not None
|
||||
released = await check_and_delete(self._redis, key, task_id)
|
||||
if released:
|
||||
logger.debug("Released lock %s", key)
|
||||
else:
|
||||
logger.debug("Skipped release of lock %s: not owned by this task", key)
|
||||
|
||||
async def pre_send(self, message: TaskiqMessage) -> TaskiqMessage:
|
||||
if not self._is_enabled(message.labels):
|
||||
return message
|
||||
|
||||
assert self._redis is not None
|
||||
key = self._build_deduplication_key(message)
|
||||
ttl = self._get_ttl(message.labels)
|
||||
|
||||
logger.debug("Acquiring lock %s for task %s", key, message.task_name)
|
||||
acquired = await self._redis.set(key, message.task_id, ex=ttl, nx=True)
|
||||
if not acquired:
|
||||
logger.warning(
|
||||
"Duplicate task %s dropped (key=%s).",
|
||||
message.task_name,
|
||||
key,
|
||||
)
|
||||
raise DuplicateTaskError(
|
||||
f"Task {message.task_name!r} with the same arguments is already queued or running."
|
||||
)
|
||||
|
||||
logger.debug("Lock %s acquired for task %s", key, message.task_name)
|
||||
return message
|
||||
|
||||
async def post_execute(
|
||||
self,
|
||||
message: TaskiqMessage,
|
||||
result: TaskiqResult,
|
||||
) -> None:
|
||||
if not self._is_enabled(message.labels):
|
||||
return
|
||||
await self._release_if_owned(
|
||||
self._build_deduplication_key(message), message.task_id
|
||||
)
|
||||
|
||||
async def on_error(
|
||||
self,
|
||||
message: TaskiqMessage,
|
||||
result: TaskiqResult,
|
||||
exception: BaseException,
|
||||
) -> None:
|
||||
if not self._is_enabled(message.labels):
|
||||
return
|
||||
await self._release_if_owned(
|
||||
self._build_deduplication_key(message), message.task_id
|
||||
)
|
||||
@@ -0,0 +1,18 @@
|
||||
from collections.abc import Awaitable
|
||||
from typing import cast
|
||||
|
||||
from redis.asyncio import Redis
|
||||
|
||||
|
||||
async def check_and_delete(redis: Redis, key: str, owner: str) -> bool:
|
||||
"""Delete *key* only if its value equals *owner*. Returns True if deleted."""
|
||||
release_script = """
|
||||
if redis.call('get', KEYS[1]) == ARGV[1] then
|
||||
return redis.call('del', KEYS[1])
|
||||
else
|
||||
return 0
|
||||
end
|
||||
"""
|
||||
|
||||
released = await cast(Awaitable[int], redis.eval(release_script, 1, key, owner))
|
||||
return bool(released)
|
||||
@@ -0,0 +1,43 @@
|
||||
import pytest
|
||||
import fakeredis.aioredis
|
||||
from taskiq import TaskiqMessage, TaskiqResult
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def anyio_backend():
|
||||
return "asyncio"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def fake_redis():
|
||||
client = fakeredis.aioredis.FakeRedis()
|
||||
yield client
|
||||
await client.aclose()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def make_message():
|
||||
def _make(task_name="my_task", task_id="task-1", labels=None, kwargs=None):
|
||||
return TaskiqMessage(
|
||||
task_id=task_id,
|
||||
task_name=task_name,
|
||||
labels=labels or {},
|
||||
labels_types={},
|
||||
args=[],
|
||||
kwargs=kwargs or {},
|
||||
)
|
||||
|
||||
return _make
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def make_result():
|
||||
def _make(is_err=False):
|
||||
return TaskiqResult(
|
||||
is_err=is_err,
|
||||
log="",
|
||||
return_value=None,
|
||||
execution_time=0.0,
|
||||
)
|
||||
|
||||
return _make
|
||||
@@ -0,0 +1,242 @@
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from taskiq_deduplication import DuplicateTaskError, RedisDeduplicationMiddleware
|
||||
from taskiq_deduplication.middleware import (
|
||||
DEDUP_EXPLICIT_KEY_LABEL,
|
||||
DEDUP_KEY_FIELDS_LABEL,
|
||||
DEDUP_LABEL,
|
||||
DEDUP_TTL_LABEL,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def middleware(fake_redis):
|
||||
mw = RedisDeduplicationMiddleware(redis_url="redis://localhost")
|
||||
mw._redis = fake_redis
|
||||
return mw
|
||||
|
||||
|
||||
class TestDefaultBuildDeduplicationKey:
|
||||
def test_same_kwargs_same_key(self, middleware, make_message):
|
||||
m1 = make_message(kwargs={"a": 1, "b": 2})
|
||||
m2 = make_message(kwargs={"a": 1, "b": 2})
|
||||
assert middleware._build_deduplication_key(
|
||||
m1
|
||||
) == middleware._build_deduplication_key(m2)
|
||||
|
||||
def test_different_kwargs_different_key(self, middleware, make_message):
|
||||
m1 = make_message(kwargs={"a": 1})
|
||||
m2 = make_message(kwargs={"a": 2})
|
||||
assert middleware._build_deduplication_key(
|
||||
m1
|
||||
) != middleware._build_deduplication_key(m2)
|
||||
|
||||
def test_kwarg_order_invariant(self, middleware, make_message):
|
||||
m1 = make_message(kwargs={"a": 1, "b": 2})
|
||||
m2 = make_message(kwargs={"b": 2, "a": 1})
|
||||
assert middleware._build_deduplication_key(
|
||||
m1
|
||||
) == middleware._build_deduplication_key(m2)
|
||||
|
||||
def test_different_task_names_different_keys(self, middleware, make_message):
|
||||
m1 = make_message(task_name="task_a", kwargs={"x": 1})
|
||||
m2 = make_message(task_name="task_b", kwargs={"x": 1})
|
||||
assert middleware._build_deduplication_key(
|
||||
m1
|
||||
) != middleware._build_deduplication_key(m2)
|
||||
|
||||
def test_explicit_key_label(self, middleware, make_message):
|
||||
m = make_message(labels={DEDUP_EXPLICIT_KEY_LABEL: "my-lock"})
|
||||
key = middleware._build_deduplication_key(m)
|
||||
assert key == "taskiq:deduplication:my-lock"
|
||||
|
||||
def test_explicit_key_ignores_kwargs(self, middleware, make_message):
|
||||
m1 = make_message(kwargs={"a": 1}, labels={DEDUP_EXPLICIT_KEY_LABEL: "fixed"})
|
||||
m2 = make_message(kwargs={"a": 99}, labels={DEDUP_EXPLICIT_KEY_LABEL: "fixed"})
|
||||
assert middleware._build_deduplication_key(
|
||||
m1
|
||||
) == middleware._build_deduplication_key(m2)
|
||||
|
||||
def test_key_fields_filters_kwargs(self, middleware, make_message):
|
||||
m1 = make_message(
|
||||
kwargs={"a": 1, "b": 2, "c": 3},
|
||||
labels={DEDUP_KEY_FIELDS_LABEL: ["a", "b"]},
|
||||
)
|
||||
m2 = make_message(
|
||||
kwargs={"a": 1, "b": 2, "c": 999},
|
||||
labels={DEDUP_KEY_FIELDS_LABEL: ["a", "b"]},
|
||||
)
|
||||
assert middleware._build_deduplication_key(
|
||||
m1
|
||||
) == middleware._build_deduplication_key(m2)
|
||||
|
||||
def test_key_fields_different_included_fields(self, middleware, make_message):
|
||||
m1 = make_message(
|
||||
kwargs={"a": 1, "b": 2},
|
||||
labels={DEDUP_KEY_FIELDS_LABEL: ["a"]},
|
||||
)
|
||||
m2 = make_message(
|
||||
kwargs={"a": 1, "b": 99},
|
||||
labels={DEDUP_KEY_FIELDS_LABEL: ["a"]},
|
||||
)
|
||||
assert middleware._build_deduplication_key(
|
||||
m1
|
||||
) == middleware._build_deduplication_key(m2)
|
||||
|
||||
def test_key_prefix_in_output(self, make_message):
|
||||
mw = RedisDeduplicationMiddleware(
|
||||
redis_url="redis://localhost", key_prefix="myapp:locks"
|
||||
)
|
||||
mw._redis = None
|
||||
m = make_message()
|
||||
key = mw._build_deduplication_key(m)
|
||||
assert key.startswith("myapp:locks:")
|
||||
|
||||
|
||||
class TestPreSend:
|
||||
@pytest.mark.anyio
|
||||
async def test_first_send_passes(self, middleware, make_message):
|
||||
msg = make_message()
|
||||
result = await middleware.pre_send(msg)
|
||||
assert result is msg
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_duplicate_raises(self, middleware, make_message):
|
||||
msg = make_message()
|
||||
await middleware.pre_send(msg)
|
||||
with pytest.raises(DuplicateTaskError):
|
||||
await middleware.pre_send(make_message())
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_deduplication_disabled_label(self, middleware, make_message):
|
||||
msg1 = make_message(labels={DEDUP_LABEL: False})
|
||||
msg2 = make_message(labels={DEDUP_LABEL: False})
|
||||
await middleware.pre_send(msg1)
|
||||
await middleware.pre_send(msg2) # should not raise
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_deduplication_disabled_by_default_init(
|
||||
self, fake_redis, make_message
|
||||
):
|
||||
mw = RedisDeduplicationMiddleware(
|
||||
redis_url="redis://localhost", default_deduplication=False
|
||||
)
|
||||
mw._redis = fake_redis
|
||||
await mw.pre_send(make_message())
|
||||
await mw.pre_send(make_message()) # should not raise
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_ttl_applied(self, middleware, fake_redis, make_message):
|
||||
msg = make_message(labels={DEDUP_TTL_LABEL: 42})
|
||||
await middleware.pre_send(msg)
|
||||
key = middleware._build_deduplication_key(msg)
|
||||
ttl = await fake_redis.ttl(key)
|
||||
assert 0 < ttl <= 42
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_different_kwargs_both_pass(self, middleware, make_message):
|
||||
await middleware.pre_send(make_message(kwargs={"x": 1}))
|
||||
await middleware.pre_send(make_message(kwargs={"x": 2}))
|
||||
|
||||
|
||||
class TestPostExecute:
|
||||
@pytest.mark.anyio
|
||||
async def test_releases_lock(
|
||||
self, middleware, fake_redis, make_message, make_result
|
||||
):
|
||||
msg = make_message()
|
||||
await middleware.pre_send(msg)
|
||||
key = middleware._build_deduplication_key(msg)
|
||||
assert await fake_redis.exists(key)
|
||||
|
||||
await middleware.post_execute(msg, make_result())
|
||||
assert not await fake_redis.exists(key)
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_deduplication_disabled_noop(
|
||||
self, middleware, fake_redis, make_message, make_result
|
||||
):
|
||||
msg = make_message()
|
||||
await middleware.pre_send(msg)
|
||||
key = middleware._build_deduplication_key(msg)
|
||||
|
||||
disabled_msg = make_message(labels={DEDUP_LABEL: False})
|
||||
await middleware.post_execute(disabled_msg, make_result())
|
||||
assert await fake_redis.exists(key)
|
||||
|
||||
|
||||
class TestOnError:
|
||||
@pytest.mark.anyio
|
||||
async def test_releases_lock_on_error(
|
||||
self, middleware, fake_redis, make_message, make_result
|
||||
):
|
||||
msg = make_message()
|
||||
await middleware.pre_send(msg)
|
||||
key = middleware._build_deduplication_key(msg)
|
||||
assert await fake_redis.exists(key)
|
||||
|
||||
await middleware.on_error(msg, make_result(is_err=True), RuntimeError("boom"))
|
||||
assert not await fake_redis.exists(key)
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_deduplication_disabled_noop(
|
||||
self, middleware, fake_redis, make_message, make_result
|
||||
):
|
||||
msg = make_message()
|
||||
await middleware.pre_send(msg)
|
||||
key = middleware._build_deduplication_key(msg)
|
||||
|
||||
disabled_msg = make_message(labels={DEDUP_LABEL: False})
|
||||
await middleware.on_error(
|
||||
disabled_msg, make_result(is_err=True), RuntimeError("x")
|
||||
)
|
||||
assert await fake_redis.exists(key)
|
||||
|
||||
|
||||
class TestAtomicRelease:
|
||||
@pytest.mark.anyio
|
||||
async def test_only_owner_can_release(self, middleware, fake_redis, make_message):
|
||||
owner_msg = make_message(task_id="owner-task")
|
||||
key = middleware._build_deduplication_key(owner_msg)
|
||||
|
||||
await fake_redis.set(key, "owner-task", ex=300)
|
||||
|
||||
await middleware._release_if_owned(key, "other-task")
|
||||
assert await fake_redis.exists(key)
|
||||
|
||||
await middleware._release_if_owned(key, "owner-task")
|
||||
assert not await fake_redis.exists(key)
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_release_missing_key_is_noop(self, middleware, fake_redis):
|
||||
await middleware._release_if_owned(
|
||||
"taskiq:deduplication:nonexistent", "some-task"
|
||||
)
|
||||
|
||||
|
||||
class TestLifecycle:
|
||||
@pytest.mark.anyio
|
||||
async def test_startup_creates_redis_client(self):
|
||||
mw = RedisDeduplicationMiddleware(redis_url="redis://localhost")
|
||||
assert mw._redis is None
|
||||
with patch("redis.asyncio.Redis.from_url") as mock_from_url:
|
||||
mock_client = AsyncMock()
|
||||
mock_from_url.return_value = mock_client
|
||||
await mw.startup()
|
||||
mock_from_url.assert_called_once_with("redis://localhost")
|
||||
assert mw._redis is mock_client
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_shutdown_closes_redis_client(self):
|
||||
mw = RedisDeduplicationMiddleware(redis_url="redis://localhost")
|
||||
mock_client = AsyncMock()
|
||||
mw._redis = mock_client
|
||||
await mw.shutdown()
|
||||
mock_client.aclose.assert_called_once()
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_shutdown_without_startup_is_safe(self):
|
||||
mw = RedisDeduplicationMiddleware(redis_url="redis://localhost")
|
||||
await mw.shutdown()
|
||||
+112
@@ -0,0 +1,112 @@
|
||||
[project]
|
||||
site_name = "Taskiq Deduplication"
|
||||
site_description = "Redis-backed deduplication middleware for Taskiq."
|
||||
site_author = "d3vyce"
|
||||
site_url = "https://taskiq-deduplication.d3vyce.fr/"
|
||||
copyright = "Copyright © 2026 d3vyce"
|
||||
repo_url = "https://github.com/d3vyce/taskiq-deduplication"
|
||||
|
||||
[project.theme]
|
||||
language = "en"
|
||||
features = [
|
||||
"announce.dismiss",
|
||||
"content.action.view",
|
||||
"content.code.annotate",
|
||||
"content.code.copy",
|
||||
"content.code.select",
|
||||
"content.footnote.tooltips",
|
||||
"content.tabs.link",
|
||||
"content.tooltips",
|
||||
"navigation.footer",
|
||||
"navigation.indexes",
|
||||
"navigation.instant",
|
||||
"navigation.instant.prefetch",
|
||||
"navigation.path",
|
||||
"navigation.sections",
|
||||
"navigation.tabs",
|
||||
"navigation.top",
|
||||
"navigation.tracking",
|
||||
"search.highlight",
|
||||
]
|
||||
|
||||
[[project.theme.palette]]
|
||||
scheme = "default"
|
||||
toggle.icon = "lucide/sun"
|
||||
toggle.name = "Switch to dark mode"
|
||||
|
||||
[[project.theme.palette]]
|
||||
scheme = "slate"
|
||||
toggle.icon = "lucide/moon"
|
||||
toggle.name = "Switch to light mode"
|
||||
|
||||
[project.theme.font]
|
||||
text = "Inter"
|
||||
code = "Jetbrains Mono"
|
||||
|
||||
[project.theme.icon]
|
||||
repo = "fontawesome/brands/github"
|
||||
|
||||
[project.plugins.mkdocstrings.handlers.python]
|
||||
inventories = ["https://docs.python.org/3/objects.inv"]
|
||||
paths = ["src"]
|
||||
|
||||
[project.plugins.mkdocstrings.handlers.python.options]
|
||||
docstring_style = "google"
|
||||
inherited_members = true
|
||||
show_source = false
|
||||
show_root_heading = true
|
||||
|
||||
[project.markdown_extensions]
|
||||
abbr = {}
|
||||
admonition = {}
|
||||
attr_list = {}
|
||||
def_list = {}
|
||||
footnotes = {}
|
||||
md_in_html = {}
|
||||
"pymdownx.arithmatex" = {generic = true}
|
||||
"pymdownx.betterem" = {}
|
||||
"pymdownx.caret" = {}
|
||||
"pymdownx.details" = {}
|
||||
"pymdownx.emoji" = {}
|
||||
"pymdownx.inlinehilite" = {}
|
||||
"pymdownx.keys" = {}
|
||||
"pymdownx.magiclink" = {}
|
||||
"pymdownx.mark" = {}
|
||||
"pymdownx.smartsymbols" = {}
|
||||
"pymdownx.tasklist" = {custom_checkbox = true}
|
||||
"pymdownx.tilde" = {}
|
||||
|
||||
[project.markdown_extensions.pymdownx.emoji]
|
||||
emoji_index = "zensical.extensions.emoji.twemoji"
|
||||
emoji_generator = "zensical.extensions.emoji.to_svg"
|
||||
|
||||
[project.markdown_extensions."pymdownx.highlight"]
|
||||
anchor_linenums = true
|
||||
line_spans = "__span"
|
||||
pygments_lang_class = true
|
||||
|
||||
[project.markdown_extensions."pymdownx.superfences"]
|
||||
custom_fences = [{name = "mermaid", class = "mermaid"}]
|
||||
|
||||
[project.markdown_extensions."pymdownx.tabbed"]
|
||||
alternate_style = true
|
||||
combine_header_slug = true
|
||||
|
||||
[project.markdown_extensions."toc"]
|
||||
permalink = true
|
||||
|
||||
[project.markdown_extensions."pymdownx.snippets"]
|
||||
base_path = ["."]
|
||||
check_paths = true
|
||||
|
||||
[[project.nav]]
|
||||
Home = "index.md"
|
||||
|
||||
[[project.nav]]
|
||||
Usage = "usage.md"
|
||||
|
||||
[[project.nav]]
|
||||
"API Reference" = "reference.md"
|
||||
|
||||
[[project.nav]]
|
||||
"Changelog ↗" = "https://github.com/d3vyce/taskiq-deduplication/releases"
|
||||
Reference in New Issue
Block a user