# Timed Batch Worker
Python library for efficiently batch-processing workloads asynchronously with batch-size- and time-based flushes using short-circuiting events.
## Installation
```bash
pip install timed-batch-worker
```
## Usage
```python
from timed_batch_worker import TimedBatchWorker
# Handle the batched items
def handle_batch(items):
http.post('/api/logs/bulk', data=items)
# Create a worker that flushes the queue when:
# 1. The batch contains 20 objects, or
# 2. Five seconds have passed since the last flush
worker = TimedBatchWorker(
handler=handle_batch,
flush_interval=5,
flush_batch_size=20,
)
# Worker started in separate thread to avoid throttling main event loop
# in case the batch handler includes synchronous workloads.
worker.start()
# Add objects to the current batch at any time
@app.get('/users/create')
def create_user():
...
# Hand off an object to the worker
worker.enqueue({
'event': 'USER_CREATED',
'data': user
})
...
```
Raw data
{
"_id": null,
"home_page": "https://github.com/Minibrams/timed-batch-worker",
"name": "timed-batch-worker",
"maintainer": "",
"docs_url": null,
"requires_python": ">=3",
"maintainer_email": "",
"keywords": "batch,worker,asynchronous,threading,timed",
"author": "Anders Brams",
"author_email": "anders@brams.dk",
"download_url": "https://files.pythonhosted.org/packages/e2/99/47a6a4a28848779c0013fe79e1989bd25ee355848c684e741bfa3709dc5e/timed-batch-worker-0.0.5.tar.gz",
"platform": null,
"description": "# Timed Batch Worker\n\nPython library for efficiently batch-processing workloads asynchronously with batch-size- and time-based flushes using short-circuiting events.\n\n## Installation\n```bash\npip install timed-batch-worker\n```\n\n## Usage\n```python\nfrom timed_batch_worker import TimedBatchWorker\n\n\n# Handle the batched items\ndef handle_batch(items):\n http.post('/api/logs/bulk', data=items)\n\n\n# Create a worker that flushes the queue when:\n# 1. The batch contains 20 objects, or\n# 2. Five seconds have passed since the last flush\nworker = TimedBatchWorker(\n handler=handle_batch,\n flush_interval=5,\n flush_batch_size=20,\n)\n\n# Worker started in separate thread to avoid throttling main event loop\n# in case the batch handler includes synchronous workloads.\nworker.start()\n\n\n# Add objects to the current batch at any time\n@app.get('/users/create')\ndef create_user():\n ...\n\n # Hand off an object to the worker\n worker.enqueue({\n 'event': 'USER_CREATED',\n 'data': user\n })\n\n ...\n```\n\n\n\n",
"bugtrack_url": null,
"license": "",
"summary": "Python library for efficiently batch-processing workloads asynchronously with batch-size- and time-based flushes.",
"version": "0.0.5",
"project_urls": {
"Homepage": "https://github.com/Minibrams/timed-batch-worker"
},
"split_keywords": [
"batch",
"worker",
"asynchronous",
"threading",
"timed"
],
"urls": [
{
"comment_text": "",
"digests": {
"blake2b_256": "9cd514865be26e7ade6dbf14a880d7ec7824c9bb9c2d67b2be0161d41cea1528",
"md5": "a941d02efd883a3e978cd9a31a1868d2",
"sha256": "fd87d810c3e01d603541ff6f289996b20b46f71e364e1aac7b060b0a198f6020"
},
"downloads": -1,
"filename": "timed_batch_worker-0.0.5-py3-none-any.whl",
"has_sig": false,
"md5_digest": "a941d02efd883a3e978cd9a31a1868d2",
"packagetype": "bdist_wheel",
"python_version": "py3",
"requires_python": ">=3",
"size": 4537,
"upload_time": "2023-07-11T21:17:22",
"upload_time_iso_8601": "2023-07-11T21:17:22.236551Z",
"url": "https://files.pythonhosted.org/packages/9c/d5/14865be26e7ade6dbf14a880d7ec7824c9bb9c2d67b2be0161d41cea1528/timed_batch_worker-0.0.5-py3-none-any.whl",
"yanked": false,
"yanked_reason": null
},
{
"comment_text": "",
"digests": {
"blake2b_256": "e29947a6a4a28848779c0013fe79e1989bd25ee355848c684e741bfa3709dc5e",
"md5": "8856d08c2753a43a942ec99b68067fa2",
"sha256": "e7a70c1ccea2ed51fbccde8a002d07982511ef7255d231f29f129c315a8fd374"
},
"downloads": -1,
"filename": "timed-batch-worker-0.0.5.tar.gz",
"has_sig": false,
"md5_digest": "8856d08c2753a43a942ec99b68067fa2",
"packagetype": "sdist",
"python_version": "source",
"requires_python": ">=3",
"size": 4220,
"upload_time": "2023-07-11T21:17:23",
"upload_time_iso_8601": "2023-07-11T21:17:23.979840Z",
"url": "https://files.pythonhosted.org/packages/e2/99/47a6a4a28848779c0013fe79e1989bd25ee355848c684e741bfa3709dc5e/timed-batch-worker-0.0.5.tar.gz",
"yanked": false,
"yanked_reason": null
}
],
"upload_time": "2023-07-11 21:17:23",
"github": true,
"gitlab": false,
"bitbucket": false,
"codeberg": false,
"github_user": "Minibrams",
"github_project": "timed-batch-worker",
"travis_ci": false,
"coveralls": false,
"github_actions": false,
"lcname": "timed-batch-worker"
}
JSON
