# dedup_me
dedup_me is a simple library for concurrent in-flight deduplication of functions.
This can be useful for e.g. API calls or DB access. Instead of querying the same data multiple times,
this library allows you to query once and share the result.
Note: This library does not cache results. After the result is returned, new consumers will call the function(API, DB, ...) again.
# Usage
## AsyncIO
```python
import asyncio
import random
from dedup_me import async_dedup
# @async_dedup('static-key') # if the arguments don't matter, you can use a static key
@async_dedup(key = lambda x: f'dynamic-key-{x}') # or pass in a key function that accepts all arguments
async def expensive_function(x: int) -> int:
print('expensive function called')
await asyncio.sleep(x)
return random.randint(0, 10)
async def main() -> None:
# call the function 10 times
# this should only print 'expensive function called' once
results = await asyncio.gather(
*(
expensive_function(1)
for _ in range(10)
)
)
print(results)
# all results should be the same
results_list = list(results)
assert results_list == [results_list[0]] * 10
```
Alternatively, without the decorator:
```python
import asyncio
import random
from dedup_me import AsyncDedup
async def expensive_function() -> int:
print('expensive function called')
await asyncio.sleep(1)
return random.randint(0, 10)
async def main() -> None:
dedup = AsyncDedup()
await asyncio.gather(
*(
# functions are grouped by the key, choose something that represents the function and its arguments
# the second argument must be a function without arguments that returns an awaitable
dedup.run('some-key', lambda: expensive_function())
for _ in range(10)
)
)
```
## Threads
For threading just use the `threading_dedup` decorator or `ThreadingDedup`.
```python
from time import sleep
from dedup_me import threading_dedup, ThreadingDedup
@threading_dedup('key')
def expensive_function() -> None:
sleep(1)
expensive_function()
# or
dedup = ThreadingDedup()
dedup.run('key', lambda: sleep(1))
```
Raw data
{
"_id": null,
"home_page": "https://github.com/ottermata/dedup_me",
"name": "dedup-me",
"maintainer": "",
"docs_url": null,
"requires_python": ">=3.10,<4.0",
"maintainer_email": "",
"keywords": "dedup,deduplication,request",
"author": "Christopher Krause",
"author_email": "ottermata@pm.me",
"download_url": "https://files.pythonhosted.org/packages/90/9a/bc02f82dc51edeeecb5c8e4b5ce8ecbca05ce1d3f179ebb26d6094f5a3ae/dedup_me-0.1.0.tar.gz",
"platform": null,
"description": "# dedup_me\n\ndedup_me is a simple library for concurrent in-flight deduplication of functions.\nThis can be useful for e.g. API calls or DB access. Instead of querying the same data multiple times,\nthis library allows you to query once and share the result.\n\nNote: This library does not cache results. After the result is returned, new consumers will call the function(API, DB, ...) again.\n\n# Usage\n\n## AsyncIO\n\n```python\nimport asyncio\nimport random\n\nfrom dedup_me import async_dedup\n\n# @async_dedup('static-key') # if the arguments don't matter, you can use a static key\n@async_dedup(key = lambda x: f'dynamic-key-{x}') # or pass in a key function that accepts all arguments\nasync def expensive_function(x: int) -> int:\n print('expensive function called')\n await asyncio.sleep(x)\n return random.randint(0, 10)\n\n\nasync def main() -> None:\n # call the function 10 times\n # this should only print 'expensive function called' once\n results = await asyncio.gather(\n *(\n expensive_function(1)\n for _ in range(10)\n )\n )\n print(results)\n # all results should be the same\n results_list = list(results)\n assert results_list == [results_list[0]] * 10\n```\n\nAlternatively, without the decorator:\n```python\nimport asyncio\nimport random\n\nfrom dedup_me import AsyncDedup\n\n\nasync def expensive_function() -> int:\n print('expensive function called')\n await asyncio.sleep(1)\n return random.randint(0, 10)\n\n\nasync def main() -> None:\n dedup = AsyncDedup()\n await asyncio.gather(\n *(\n # functions are grouped by the key, choose something that represents the function and its arguments\n # the second argument must be a function without arguments that returns an awaitable\n dedup.run('some-key', lambda: expensive_function())\n for _ in range(10)\n )\n )\n```\n\n## Threads\n\nFor threading just use the `threading_dedup` decorator or `ThreadingDedup`.\n```python\nfrom time import sleep\nfrom dedup_me import threading_dedup, ThreadingDedup\n\n\n@threading_dedup('key')\ndef expensive_function() -> None:\n sleep(1)\n\nexpensive_function()\n\n# or\ndedup = ThreadingDedup()\ndedup.run('key', lambda: sleep(1))\n```",
"bugtrack_url": null,
"license": "MIT",
"summary": "Deduplicate concurrent function calls.",
"version": "0.1.0",
"split_keywords": [
"dedup",
"deduplication",
"request"
],
"urls": [
{
"comment_text": "",
"digests": {
"blake2b_256": "f7413cd8fc63580481392fccef6c9f6b594ea8f053bc2ef0bcc941950223859f",
"md5": "1595fa2570d05c77a4bddd91f20b93a0",
"sha256": "9d85d8a58e85b0aa909a1eb66f2b92f8ff1d19b1aa3c9dad71312550184eee45"
},
"downloads": -1,
"filename": "dedup_me-0.1.0-py3-none-any.whl",
"has_sig": false,
"md5_digest": "1595fa2570d05c77a4bddd91f20b93a0",
"packagetype": "bdist_wheel",
"python_version": "py3",
"requires_python": ">=3.10,<4.0",
"size": 6579,
"upload_time": "2023-02-11T11:34:17",
"upload_time_iso_8601": "2023-02-11T11:34:17.563842Z",
"url": "https://files.pythonhosted.org/packages/f7/41/3cd8fc63580481392fccef6c9f6b594ea8f053bc2ef0bcc941950223859f/dedup_me-0.1.0-py3-none-any.whl",
"yanked": false,
"yanked_reason": null
},
{
"comment_text": "",
"digests": {
"blake2b_256": "909abc02f82dc51edeeecb5c8e4b5ce8ecbca05ce1d3f179ebb26d6094f5a3ae",
"md5": "af49c36498e9be47a087948ceb280eda",
"sha256": "e4cb9498388667fb8eaa28fed1c14cabcfb4acdbd8f4d4c3f64dd8af59baddd8"
},
"downloads": -1,
"filename": "dedup_me-0.1.0.tar.gz",
"has_sig": false,
"md5_digest": "af49c36498e9be47a087948ceb280eda",
"packagetype": "sdist",
"python_version": "source",
"requires_python": ">=3.10,<4.0",
"size": 5201,
"upload_time": "2023-02-11T11:34:19",
"upload_time_iso_8601": "2023-02-11T11:34:19.264156Z",
"url": "https://files.pythonhosted.org/packages/90/9a/bc02f82dc51edeeecb5c8e4b5ce8ecbca05ce1d3f179ebb26d6094f5a3ae/dedup_me-0.1.0.tar.gz",
"yanked": false,
"yanked_reason": null
}
],
"upload_time": "2023-02-11 11:34:19",
"github": true,
"gitlab": false,
"bitbucket": false,
"github_user": "ottermata",
"github_project": "dedup_me",
"travis_ci": false,
"coveralls": false,
"github_actions": false,
"lcname": "dedup-me"
}
JSON
