# Download/upload images from/to s3 very quickly
## Setup
```
poetry install
```
## Usage
Download image files
```python
from PIL import Image
from fast_s3 import Fetcher, Status
large_list_of_image_paths = [...]
fetcher = Fetcher(
paths=large_list_of_image_paths,
endpoint_url="https://s3.my-path-to-s3",
aws_access_key_id="my-key-id",
aws_secret_access_key="my-secret-key",
region_name="my-region",
bucket_name="my-bucket",
ordered=True, # returns files in the same order as paths
buffer_size=1024,
n_workers=32,
)
for file in fetcher:
if file.status != Status.error:
Image.open(file.buffer).save(file.path)
fetcher.close()
```
Upload files
```python
from fast_s3 import Uploader
large_list_of_files = [...]
large_list_of_paths = [...]
uploader = Uploader(
endpoint_url="https://s3.my-path-to-s3",
aws_access_key_id="my-key-id",
aws_secret_access_key="my-secret-key",
region_name="my-region",
bucket_name="my-bucket",
)
uploader.queue_upload(
source=large_list_of_files,
destination=large_list_of_paths,
)
uploader.await_upload()
uploader.close()
```
Raw data
{
"_id": null,
"home_page": "https://github.com/nextml-code/fast-s3",
"name": "fast-s3",
"maintainer": null,
"docs_url": null,
"requires_python": "<3.11,>=3.8",
"maintainer_email": null,
"keywords": null,
"author": "NextML AB",
"author_email": null,
"download_url": "https://files.pythonhosted.org/packages/ee/99/df258d292cfa069fa5c6764ca6af78774495362021ae6fa65e7ad9f02aa3/fast_s3-3.0.1.tar.gz",
"platform": null,
"description": "# Download/upload images from/to s3 very quickly\n\n## Setup\n\n```\npoetry install\n```\n\n## Usage\n\nDownload image files\n\n```python\nfrom PIL import Image\nfrom fast_s3 import Fetcher, Status\n\n\nlarge_list_of_image_paths = [...]\n\nfetcher = Fetcher(\n paths=large_list_of_image_paths,\n endpoint_url=\"https://s3.my-path-to-s3\",\n aws_access_key_id=\"my-key-id\",\n aws_secret_access_key=\"my-secret-key\",\n region_name=\"my-region\",\n bucket_name=\"my-bucket\",\n ordered=True, # returns files in the same order as paths\n buffer_size=1024,\n n_workers=32,\n)\n\nfor file in fetcher:\n if file.status != Status.error:\n Image.open(file.buffer).save(file.path)\n\nfetcher.close()\n```\n\nUpload files\n\n```python\nfrom fast_s3 import Uploader\n\n\nlarge_list_of_files = [...]\nlarge_list_of_paths = [...]\n\nuploader = Uploader(\n endpoint_url=\"https://s3.my-path-to-s3\",\n aws_access_key_id=\"my-key-id\",\n aws_secret_access_key=\"my-secret-key\",\n region_name=\"my-region\",\n bucket_name=\"my-bucket\",\n)\n\nuploader.queue_upload(\n source=large_list_of_files,\n destination=large_list_of_paths,\n)\nuploader.await_upload()\nuploader.close()\n```\n",
"bugtrack_url": null,
"license": null,
"summary": "Download images from s3 fast",
"version": "3.0.1",
"project_urls": {
"Homepage": "https://github.com/nextml-code/fast-s3",
"Repository": "https://github.com/nextml-code/fast-s3"
},
"split_keywords": [],
"urls": [
{
"comment_text": "",
"digests": {
"blake2b_256": "ac242f3df2679152f160add96371c72b0b5b82176c360c8d3b196d39ca6c41d1",
"md5": "36f3336fa6fface4918e16caa49810aa",
"sha256": "7d55fed6162e9972b98761fcc9ed4357d5800b25846e090cdaabdb2ec72d07f5"
},
"downloads": -1,
"filename": "fast_s3-3.0.1-py3-none-any.whl",
"has_sig": false,
"md5_digest": "36f3336fa6fface4918e16caa49810aa",
"packagetype": "bdist_wheel",
"python_version": "py3",
"requires_python": "<3.11,>=3.8",
"size": 5228,
"upload_time": "2024-08-13T11:57:03",
"upload_time_iso_8601": "2024-08-13T11:57:03.012876Z",
"url": "https://files.pythonhosted.org/packages/ac/24/2f3df2679152f160add96371c72b0b5b82176c360c8d3b196d39ca6c41d1/fast_s3-3.0.1-py3-none-any.whl",
"yanked": false,
"yanked_reason": null
},
{
"comment_text": "",
"digests": {
"blake2b_256": "ee99df258d292cfa069fa5c6764ca6af78774495362021ae6fa65e7ad9f02aa3",
"md5": "c7db5dd8aac09b6bbc4316a9ece5a678",
"sha256": "e9cc3e31d37105397f41c58430ac1c0b9ad5e56c0cec7a7c5538b066cf314a4a"
},
"downloads": -1,
"filename": "fast_s3-3.0.1.tar.gz",
"has_sig": false,
"md5_digest": "c7db5dd8aac09b6bbc4316a9ece5a678",
"packagetype": "sdist",
"python_version": "source",
"requires_python": "<3.11,>=3.8",
"size": 4236,
"upload_time": "2024-08-13T11:57:04",
"upload_time_iso_8601": "2024-08-13T11:57:04.449212Z",
"url": "https://files.pythonhosted.org/packages/ee/99/df258d292cfa069fa5c6764ca6af78774495362021ae6fa65e7ad9f02aa3/fast_s3-3.0.1.tar.gz",
"yanked": false,
"yanked_reason": null
}
],
"upload_time": "2024-08-13 11:57:04",
"github": true,
"gitlab": false,
"bitbucket": false,
"codeberg": false,
"github_user": "nextml-code",
"github_project": "fast-s3",
"travis_ci": false,
"coveralls": false,
"github_actions": true,
"lcname": "fast-s3"
}
JSON
