| Name | modal_or_local JSON |
| Version |
0.1.1
JSON |
| download |
| home_page | None |
| Summary | Utilities for working with files and directories in modal volumes or the local filesystem |
| upload_time | 2024-07-26 03:27:51 |
| maintainer | None |
| docs_url | None |
| author | None |
| requires_python | >=3.10 |
| license | MIT |
| keywords |
modal
|
| VCS |
 |
| bugtrack_url |
|
| requirements |
No requirements were recorded.
|
| Travis-CI |
No Travis.
|
| coveralls test coverage |
No coveralls.
|
# modal_or_local
File and directory utilities for working with modal volumes. The modal_or_local package gives the ability to use the same code to control files and directories from either a local run or remote run. This is done essentially by pairing the [Modal](https://modal.com/) API with the corresponding os commands and using either based on whether working with a volume, local filesystem, remotely, or locally.
See scripts/example.py and tests/ for code samples.
This package is expected to work on any Linux type system and has been tested on WSL. Working from Windows will have issues primarily because of the extensive use of os.path tools that are presumed to give Linux-style paths to work on Modal.
## Usage
```python
# Run this with 'modal run scripts/example.py'
#
import modal
from modal_or_local import setup_image, ModalOrLocal, ModalOrLocalDir
image = setup_image()
app = modal.App("myapp")
mvol1 = ModalOrLocal(volume_name="my_modal_volume1", volume_mount_dir="/volume_mnt_dir1")
mvol2 = ModalOrLocal(volume_name="my_modal_volume2", volume_mount_dir="/volume_mnt_dir2")
@app.function(image=image, volumes={mvol1.volume_mount_dir: mvol1.volume})
def do_some_stuff_locally():
# Set local directory to work with.
mdir_local = ModalOrLocalDir(dir_full_path="/tmp/my_local_dir")
# Set remote directory to work with
mdir_on_volume = ModalOrLocalDir(dir_full_path="/volume_mnt_dir1/my_volume_dir", modal_or_local=mvol1)
# Create a json file on the volume and a text file locally
mdir_on_volume.write_json_file("created_on_volume.json", {"a":1})
mdir_local.write_file("created_locally.txt", str("this is some text").encode(), force=True)
# Copy the files from the volume (created_on_volume.json) that are newer than whats in (or does not exist in) the local directory
mdir_local.copy_changed_files_from(mdir_on_volume)
# Give time for the files to age (mtime from modal api is only 1s precision)
from time import sleep
sleep(1)
# Create a new file on the volume
mdir_on_volume.write_file("newer_file_on_volume.txt", str("this is some text").encode(), force=True)
# Get the mtime of the file created on the volume, we will get any new files created since then
mtime = mdir_on_volume.get_mtime("created_on_volume.json")
# Get the files from the volume that were created after created_on_volume.json was created (this will copy newer_file_on_volume.txt to the local dir)
from datetime import datetime
copied_files = mdir_local.copy_changed_files_from(mdir_on_volume, since_date=datetime.fromtimestamp(mtime))
print(f"Copied files: {copied_files} to {mdir_local.dir_full_path}. Full list is now: {mdir_local.listdir()}")
# Clean up
#mdir_local.remove_own_directory()
#mdir_on_volume.remove_own_directory()
@app.function(image=image, volumes={mvol1.volume_mount_dir: mvol1.volume, mvol2.volume_mount_dir: mvol2.volume})
def do_some_stuff_locally_or_remotely():
# Set directories to work with on two different volumes
mdir_on_volume1 = ModalOrLocalDir(dir_full_path="/volume_mnt_dir1/my_dir_on_volume_one", modal_or_local=mvol1)
mdir_on_volume2 = ModalOrLocalDir(dir_full_path="/volume_mnt_dir2/my_dir_on_volume_two", modal_or_local=mvol2)
# Create a file on each volume
mdir_on_volume1.write_json_file("file1.json", {"a":1})
mdir_on_volume2.write_file("file2.txt", str("this is some text").encode(), force=True)
# Read the json file
metadata = mdir_on_volume1.read_json_file("file1.json")
print(f"Metadata from file1.json is {metadata}")
# Read the text file
content = mdir_on_volume2.read_file("file2.txt")
print("Text from file2.txt is:", content.decode('utf-8'))
# Copy a file1.json from volume1 to volume2
mdir_on_volume2.copy_file(mdir_on_volume1, "file1.json")
print(f"Files on volume one are: {mdir_on_volume1.listdir()}")
print(f"Files on volume two are: {mdir_on_volume2.listdir()}")
# Create or remove a directory on the volume - similar to os.mkdirs() and shutil.rmtree()
mvol1.create_directory("/volume_mnt_dir1/my/set/of/created/directories")
mvol1.remove_file_or_directory("/volume_mnt_dir1/my/set/of/created/directories")
# List items in a directory on the volume
filenames = mvol1.listdir("/volume_mnt_dir1")
print("filenames in /volume_mnt_dir1 are", filenames)
filenames_full_path = mvol1.listdir("/volume_mnt_dir1", return_full_paths=True)
print("filenames with full path in /volume_mnt_dir1 are", filenames_full_path)
# Create / overwrite a json file on the volume
metadata = {"name": "Heather", "age": None}
json_file_full_path = "/volume_mnt_dir1/myfile.json"
mvol1.write_json_file(json_file_full_path, metadata)
# Read a json file on the volume
json_data = mvol1.read_json_file(json_file_full_path)
print("json_data is", json_data)
# Clean up
#mdir_on_volume1.remove_own_directory()
#mdir_on_volume2.remove_own_directory()
# See tests/ for more examples
@app.local_entrypoint()
def main():
# The methods in modal_or_local will work for a modal volume whether running locally or remotely.
# Of course if you need the local filesystem you will need to run locally
do_some_stuff_locally.local()
do_some_stuff_locally_or_remotely.local()
do_some_stuff_locally_or_remotely.remote()
Raw data
{
"_id": null,
"home_page": null,
"name": "modal_or_local",
"maintainer": null,
"docs_url": null,
"requires_python": ">=3.10",
"maintainer_email": null,
"keywords": "modal",
"author": null,
"author_email": "Paul Wessling <paul@nowfree.org>",
"download_url": "https://files.pythonhosted.org/packages/03/37/68f22029b7f2b7923909504a39515c466738969d8996b5eaefdbac513fb1/modal_or_local-0.1.1.tar.gz",
"platform": null,
"description": "# modal_or_local\nFile and directory utilities for working with modal volumes. The modal_or_local package gives the ability to use the same code to control files and directories from either a local run or remote run. This is done essentially by pairing the [Modal](https://modal.com/) API with the corresponding os commands and using either based on whether working with a volume, local filesystem, remotely, or locally.\n\nSee scripts/example.py and tests/ for code samples.\n\nThis package is expected to work on any Linux type system and has been tested on WSL. Working from Windows will have issues primarily because of the extensive use of os.path tools that are presumed to give Linux-style paths to work on Modal.\n\n## Usage\n```python\n# Run this with 'modal run scripts/example.py'\n# \nimport modal\nfrom modal_or_local import setup_image, ModalOrLocal, ModalOrLocalDir\n\nimage = setup_image()\napp = modal.App(\"myapp\")\nmvol1 = ModalOrLocal(volume_name=\"my_modal_volume1\", volume_mount_dir=\"/volume_mnt_dir1\")\nmvol2 = ModalOrLocal(volume_name=\"my_modal_volume2\", volume_mount_dir=\"/volume_mnt_dir2\")\n\n@app.function(image=image, volumes={mvol1.volume_mount_dir: mvol1.volume})\ndef do_some_stuff_locally():\n # Set local directory to work with.\n mdir_local = ModalOrLocalDir(dir_full_path=\"/tmp/my_local_dir\") \n # Set remote directory to work with\n mdir_on_volume = ModalOrLocalDir(dir_full_path=\"/volume_mnt_dir1/my_volume_dir\", modal_or_local=mvol1)\n\n # Create a json file on the volume and a text file locally\n mdir_on_volume.write_json_file(\"created_on_volume.json\", {\"a\":1})\n mdir_local.write_file(\"created_locally.txt\", str(\"this is some text\").encode(), force=True)\n\n # Copy the files from the volume (created_on_volume.json) that are newer than whats in (or does not exist in) the local directory\n mdir_local.copy_changed_files_from(mdir_on_volume)\n\n # Give time for the files to age (mtime from modal api is only 1s precision)\n from time import sleep\n sleep(1)\n\n # Create a new file on the volume\n mdir_on_volume.write_file(\"newer_file_on_volume.txt\", str(\"this is some text\").encode(), force=True)\n\n # Get the mtime of the file created on the volume, we will get any new files created since then\n mtime = mdir_on_volume.get_mtime(\"created_on_volume.json\")\n \n # Get the files from the volume that were created after created_on_volume.json was created (this will copy newer_file_on_volume.txt to the local dir)\n from datetime import datetime\n copied_files = mdir_local.copy_changed_files_from(mdir_on_volume, since_date=datetime.fromtimestamp(mtime))\n print(f\"Copied files: {copied_files} to {mdir_local.dir_full_path}. Full list is now: {mdir_local.listdir()}\")\n \n # Clean up\n #mdir_local.remove_own_directory()\n #mdir_on_volume.remove_own_directory()\n \n\n@app.function(image=image, volumes={mvol1.volume_mount_dir: mvol1.volume, mvol2.volume_mount_dir: mvol2.volume})\ndef do_some_stuff_locally_or_remotely():\n\n # Set directories to work with on two different volumes\n mdir_on_volume1 = ModalOrLocalDir(dir_full_path=\"/volume_mnt_dir1/my_dir_on_volume_one\", modal_or_local=mvol1)\n mdir_on_volume2 = ModalOrLocalDir(dir_full_path=\"/volume_mnt_dir2/my_dir_on_volume_two\", modal_or_local=mvol2)\n\n # Create a file on each volume\n mdir_on_volume1.write_json_file(\"file1.json\", {\"a\":1})\n mdir_on_volume2.write_file(\"file2.txt\", str(\"this is some text\").encode(), force=True)\n\n # Read the json file\n metadata = mdir_on_volume1.read_json_file(\"file1.json\")\n print(f\"Metadata from file1.json is {metadata}\")\n\n # Read the text file\n content = mdir_on_volume2.read_file(\"file2.txt\")\n print(\"Text from file2.txt is:\", content.decode('utf-8'))\n\n # Copy a file1.json from volume1 to volume2\n mdir_on_volume2.copy_file(mdir_on_volume1, \"file1.json\")\n\n print(f\"Files on volume one are: {mdir_on_volume1.listdir()}\")\n print(f\"Files on volume two are: {mdir_on_volume2.listdir()}\")\n\n # Create or remove a directory on the volume - similar to os.mkdirs() and shutil.rmtree()\n mvol1.create_directory(\"/volume_mnt_dir1/my/set/of/created/directories\")\n mvol1.remove_file_or_directory(\"/volume_mnt_dir1/my/set/of/created/directories\")\n\n # List items in a directory on the volume\n filenames = mvol1.listdir(\"/volume_mnt_dir1\")\n print(\"filenames in /volume_mnt_dir1 are\", filenames)\n filenames_full_path = mvol1.listdir(\"/volume_mnt_dir1\", return_full_paths=True)\n print(\"filenames with full path in /volume_mnt_dir1 are\", filenames_full_path)\n\n # Create / overwrite a json file on the volume\n metadata = {\"name\": \"Heather\", \"age\": None}\n json_file_full_path = \"/volume_mnt_dir1/myfile.json\"\n mvol1.write_json_file(json_file_full_path, metadata)\n\n # Read a json file on the volume\n json_data = mvol1.read_json_file(json_file_full_path)\n print(\"json_data is\", json_data)\n\n # Clean up\n #mdir_on_volume1.remove_own_directory()\n #mdir_on_volume2.remove_own_directory()\n\n # See tests/ for more examples\n\n@app.local_entrypoint()\ndef main():\n # The methods in modal_or_local will work for a modal volume whether running locally or remotely. \n # Of course if you need the local filesystem you will need to run locally\n do_some_stuff_locally.local()\n do_some_stuff_locally_or_remotely.local()\n do_some_stuff_locally_or_remotely.remote()\n ",
"bugtrack_url": null,
"license": "MIT",
"summary": "Utilities for working with files and directories in modal volumes or the local filesystem",
"version": "0.1.1",
"project_urls": {
"Issues": "https://github.com/eyecantell/modal_or_local/issues",
"Repository": "https://github.com/eyecantell/modal_or_local"
},
"split_keywords": [
"modal"
],
"urls": [
{
"comment_text": "",
"digests": {
"blake2b_256": "24aadc32f3c1e3968c3d77c4032499a2350ba764b2ac788fec02535413f0b1be",
"md5": "70b682a2ba42f525086d23c01a6f1cda",
"sha256": "4b313ac4f310b3fbc1d27fee38c2b5639a6c58cd48eff35396ed3011642a67bb"
},
"downloads": -1,
"filename": "modal_or_local-0.1.1-py3-none-any.whl",
"has_sig": false,
"md5_digest": "70b682a2ba42f525086d23c01a6f1cda",
"packagetype": "bdist_wheel",
"python_version": "py3",
"requires_python": ">=3.10",
"size": 15931,
"upload_time": "2024-07-26T03:27:49",
"upload_time_iso_8601": "2024-07-26T03:27:49.705912Z",
"url": "https://files.pythonhosted.org/packages/24/aa/dc32f3c1e3968c3d77c4032499a2350ba764b2ac788fec02535413f0b1be/modal_or_local-0.1.1-py3-none-any.whl",
"yanked": false,
"yanked_reason": null
},
{
"comment_text": "",
"digests": {
"blake2b_256": "033768f22029b7f2b7923909504a39515c466738969d8996b5eaefdbac513fb1",
"md5": "935b605431187f2034d8647773e1bb9c",
"sha256": "392ccb955ecd7526681b3755b9e75fd879cc8e06abbbd65d5cf7180a33fdfda6"
},
"downloads": -1,
"filename": "modal_or_local-0.1.1.tar.gz",
"has_sig": false,
"md5_digest": "935b605431187f2034d8647773e1bb9c",
"packagetype": "sdist",
"python_version": "source",
"requires_python": ">=3.10",
"size": 23536,
"upload_time": "2024-07-26T03:27:51",
"upload_time_iso_8601": "2024-07-26T03:27:51.036549Z",
"url": "https://files.pythonhosted.org/packages/03/37/68f22029b7f2b7923909504a39515c466738969d8996b5eaefdbac513fb1/modal_or_local-0.1.1.tar.gz",
"yanked": false,
"yanked_reason": null
}
],
"upload_time": "2024-07-26 03:27:51",
"github": true,
"gitlab": false,
"bitbucket": false,
"codeberg": false,
"github_user": "eyecantell",
"github_project": "modal_or_local",
"travis_ci": false,
"coveralls": false,
"github_actions": true,
"requirements": [],
"lcname": "modal_or_local"
}
