[](https://badge.fury.io/py/experimaestro)
[](https://experimaestro-python.readthedocs.io)
Experimaestro helps in designing and managing complex workflows. It allows for the definition of tasks and their dependencies, ensuring that each step in a workflow is executed in the correct order. Some key aspects of Experimaestro are:
- **Task Automation**: The tool automates repetitive tasks, making it easier to run large-scale experiments. It's particularly useful in scenarios where experiments need to be repeated with different parameters or datasets.
- **Resource Management**: It efficiently manages computational resources, which is critical when dealing with data-intensive tasks or when running multiple experiments in parallel.
- **Extensibility**: Experimaestro is designed to be flexible and extensible, allowing users to integrate it with various programming languages and tools commonly used in data science and research.
- **Reproducibility**: By keeping a detailed record of experiments, including parameters and environments, it aids in ensuring the reproducibility of scientific experiments, which is a fundamental requirement in research.
- **User Interface**: While primarily a back-end tool, Experimaestro also offers a user interface to help in managing and visualizing workflows.
The full documentation can be read by going to the following URL: [https://experimaestro-python.readthedocs.io](https://experimaestro-python.readthedocs.io)
# Install
## With pip
You can then install the package using `pip install experimaestro`
## Develop
Checkout the git directory, then
```
pip install -e .
```
# Example
This very simple example shows how to submit two tasks that concatenate two strings.
Under the curtain,
- A directory is created for each task (in `workdir/jobs/helloworld.add/HASHID`)
based on a unique ID computed from the parameters
- Two processes for `Say` are launched (there are no dependencies, so they will be run in parallel)
- A tag `y` is created for the main task
<!-- SNIPPET: MAIN ARGS[%WORKDIR% --port 0 --sleeptime=0.0001] -->
```python
# --- Task and types definitions
import logging
logging.basicConfig(level=logging.DEBUG)
from pathlib import Path
from experimaestro import Task, Param, experiment, progress
import click
import time
import os
from typing import List
# --- Just to be able to monitor the tasks
def slowdown(sleeptime: int, N: int):
logging.info("Sleeping %ds after each step", sleeptime)
for i in range(N):
time.sleep(sleeptime)
progress((i+1)/N)
# --- Define the tasks
class Say(Task):
word: Param[str]
sleeptime: Param[float]
def execute(self):
slowdown(self.sleeptime, len(self.word))
print(self.word.upper(),)
class Concat(Task):
strings: Param[List[Say]]
sleeptime: Param[float]
def execute(self):
says = []
slowdown(self.sleeptime, len(self.strings))
for string in self.strings:
with open(string.__xpm_stdout__) as fp:
says.append(fp.read().strip())
print(" ".join(says))
# --- Defines the experiment
@click.option("--port", type=int, default=12345, help="Port for monitoring")
@click.option("--sleeptime", type=float, default=2, help="Sleep time")
@click.argument("workdir", type=Path)
@click.command()
def cli(port, workdir, sleeptime):
"""Runs an experiment"""
# Sets the working directory and the name of the xp
with experiment(workdir, "helloworld", port=port) as xp:
# Submit the tasks
hello = Say(word="hello", sleeptime=sleeptime).submit()
world = Say(word="world", sleeptime=sleeptime).submit()
# Concat will depend on the two first tasks
Concat(strings=[hello, world], sleeptime=sleeptime).tag("y", 1).submit()
if __name__ == "__main__":
cli()
```
which can be launched with `python test.py /tmp/helloworld-workdir`
Raw data
{
"_id": null,
"home_page": "https://github.com/experimaestro/experimaestro-python",
"name": "experimaestro",
"maintainer": null,
"docs_url": null,
"requires_python": "<4.0,>=3.8",
"maintainer_email": null,
"keywords": "experiment manager",
"author": "Benjamin Piwowarski",
"author_email": "benjamin@piwowarski.fr",
"download_url": "https://files.pythonhosted.org/packages/46/5b/6b069200c09d2c6bc9b5e00468945e7a1715ca88e47371be17ab96ca35ba/experimaestro-1.6.2.tar.gz",
"platform": null,
"description": "[](https://badge.fury.io/py/experimaestro)\n[](https://experimaestro-python.readthedocs.io)\n\nExperimaestro helps in designing and managing complex workflows. It allows for the definition of tasks and their dependencies, ensuring that each step in a workflow is executed in the correct order. Some key aspects of Experimaestro are:\n\n- **Task Automation**: The tool automates repetitive tasks, making it easier to run large-scale experiments. It's particularly useful in scenarios where experiments need to be repeated with different parameters or datasets.\n- **Resource Management**: It efficiently manages computational resources, which is critical when dealing with data-intensive tasks or when running multiple experiments in parallel.\n- **Extensibility**: Experimaestro is designed to be flexible and extensible, allowing users to integrate it with various programming languages and tools commonly used in data science and research.\n- **Reproducibility**: By keeping a detailed record of experiments, including parameters and environments, it aids in ensuring the reproducibility of scientific experiments, which is a fundamental requirement in research.\n- **User Interface**: While primarily a back-end tool, Experimaestro also offers a user interface to help in managing and visualizing workflows.\n\nThe full documentation can be read by going to the following URL: [https://experimaestro-python.readthedocs.io](https://experimaestro-python.readthedocs.io)\n\n\n# Install\n\n## With pip\n\nYou can then install the package using `pip install experimaestro`\n\n## Develop\n\nCheckout the git directory, then\n\n```\npip install -e .\n```\n\n# Example\n\nThis very simple example shows how to submit two tasks that concatenate two strings.\nUnder the curtain,\n\n- A directory is created for each task (in `workdir/jobs/helloworld.add/HASHID`)\n based on a unique ID computed from the parameters\n- Two processes for `Say` are launched (there are no dependencies, so they will be run in parallel)\n- A tag `y` is created for the main task\n\n<!-- SNIPPET: MAIN ARGS[%WORKDIR% --port 0 --sleeptime=0.0001] -->\n\n```python\n# --- Task and types definitions\n\nimport logging\nlogging.basicConfig(level=logging.DEBUG)\nfrom pathlib import Path\nfrom experimaestro import Task, Param, experiment, progress\nimport click\nimport time\nimport os\nfrom typing import List\n\n# --- Just to be able to monitor the tasks\n\ndef slowdown(sleeptime: int, N: int):\n logging.info(\"Sleeping %ds after each step\", sleeptime)\n for i in range(N):\n time.sleep(sleeptime)\n progress((i+1)/N)\n\n\n# --- Define the tasks\n\nclass Say(Task):\n word: Param[str]\n sleeptime: Param[float]\n\n def execute(self):\n slowdown(self.sleeptime, len(self.word))\n print(self.word.upper(),)\n\nclass Concat(Task):\n strings: Param[List[Say]]\n sleeptime: Param[float]\n\n def execute(self):\n says = []\n slowdown(self.sleeptime, len(self.strings))\n for string in self.strings:\n with open(string.__xpm_stdout__) as fp:\n says.append(fp.read().strip())\n print(\" \".join(says))\n\n\n# --- Defines the experiment\n\n@click.option(\"--port\", type=int, default=12345, help=\"Port for monitoring\")\n@click.option(\"--sleeptime\", type=float, default=2, help=\"Sleep time\")\n@click.argument(\"workdir\", type=Path)\n@click.command()\ndef cli(port, workdir, sleeptime):\n \"\"\"Runs an experiment\"\"\"\n # Sets the working directory and the name of the xp\n with experiment(workdir, \"helloworld\", port=port) as xp:\n # Submit the tasks\n hello = Say(word=\"hello\", sleeptime=sleeptime).submit()\n world = Say(word=\"world\", sleeptime=sleeptime).submit()\n\n # Concat will depend on the two first tasks\n Concat(strings=[hello, world], sleeptime=sleeptime).tag(\"y\", 1).submit()\n\n\nif __name__ == \"__main__\":\n cli()\n```\n\nwhich can be launched with `python test.py /tmp/helloworld-workdir`\n",
"bugtrack_url": null,
"license": "GPL-3",
"summary": "\"Experimaestro is a computer science experiment manager\"",
"version": "1.6.2",
"project_urls": {
"Documentation": "https://experimaestro-python.readthedocs.io/",
"Homepage": "https://github.com/experimaestro/experimaestro-python",
"Repository": "https://github.com/experimaestro/experimaestro-python"
},
"split_keywords": [
"experiment",
"manager"
],
"urls": [
{
"comment_text": "",
"digests": {
"blake2b_256": "1717f3b823967680bc2f2c25a612011455b36969667956d4275a1f6bde3446a0",
"md5": "5b65308cbee418de7ea2bf2fe3968e19",
"sha256": "6373afe5f8797088994b2846d62a2dc241914854238f65dfb662ed84e1024a77"
},
"downloads": -1,
"filename": "experimaestro-1.6.2-py3-none-any.whl",
"has_sig": false,
"md5_digest": "5b65308cbee418de7ea2bf2fe3968e19",
"packagetype": "bdist_wheel",
"python_version": "py3",
"requires_python": "<4.0,>=3.8",
"size": 173145,
"upload_time": "2025-01-24T17:25:49",
"upload_time_iso_8601": "2025-01-24T17:25:49.520246Z",
"url": "https://files.pythonhosted.org/packages/17/17/f3b823967680bc2f2c25a612011455b36969667956d4275a1f6bde3446a0/experimaestro-1.6.2-py3-none-any.whl",
"yanked": false,
"yanked_reason": null
},
{
"comment_text": "",
"digests": {
"blake2b_256": "465b6b069200c09d2c6bc9b5e00468945e7a1715ca88e47371be17ab96ca35ba",
"md5": "2cfe3ac9a02f6b11334fda310631c0c2",
"sha256": "b1c0f03a57e6e4f44e2ed1bdf2da52414293206d3235838f4583e3058c952f95"
},
"downloads": -1,
"filename": "experimaestro-1.6.2.tar.gz",
"has_sig": false,
"md5_digest": "2cfe3ac9a02f6b11334fda310631c0c2",
"packagetype": "sdist",
"python_version": "source",
"requires_python": "<4.0,>=3.8",
"size": 4286135,
"upload_time": "2025-01-24T17:25:51",
"upload_time_iso_8601": "2025-01-24T17:25:51.258453Z",
"url": "https://files.pythonhosted.org/packages/46/5b/6b069200c09d2c6bc9b5e00468945e7a1715ca88e47371be17ab96ca35ba/experimaestro-1.6.2.tar.gz",
"yanked": false,
"yanked_reason": null
}
],
"upload_time": "2025-01-24 17:25:51",
"github": true,
"gitlab": false,
"bitbucket": false,
"codeberg": false,
"github_user": "experimaestro",
"github_project": "experimaestro-python",
"travis_ci": false,
"coveralls": false,
"github_actions": true,
"circle": true,
"requirements": [
{
"name": "click",
"specs": []
},
{
"name": "omegaconf",
"specs": []
},
{
"name": "typing_extensions",
"specs": [
[
">=",
"4.2"
]
]
},
{
"name": "cached_property",
"specs": []
},
{
"name": "attrs",
"specs": []
},
{
"name": "fasteners",
"specs": []
},
{
"name": "pyyaml",
"specs": []
},
{
"name": "psutil",
"specs": []
},
{
"name": "pytools",
"specs": []
},
{
"name": "tqdm",
"specs": []
},
{
"name": "docstring_parser",
"specs": []
},
{
"name": "termcolor",
"specs": [
[
">=",
"2.3.0"
]
]
},
{
"name": "requests",
"specs": []
},
{
"name": "sortedcontainers",
"specs": []
},
{
"name": "pyparsing",
"specs": []
},
{
"name": "humanfriendly",
"specs": []
},
{
"name": "huggingface_hub",
"specs": []
},
{
"name": "gevent",
"specs": []
},
{
"name": "gevent-websocket",
"specs": []
},
{
"name": "flask",
"specs": []
},
{
"name": "flask-socketio",
"specs": []
},
{
"name": "Arpeggio",
"specs": [
[
">=",
"2.0"
]
]
},
{
"name": "watchdog",
"specs": [
[
">",
"2.0.0"
]
]
},
{
"name": "marshmallow",
"specs": []
},
{
"name": "fabric",
"specs": []
},
{
"name": "decorator",
"specs": []
},
{
"name": "rpyc",
"specs": []
}
],
"tox": true,
"lcname": "experimaestro"
}
JSON
