miniwdl-slurm
=============
Extends miniwdl to run workflows on SLURM clusters in singularity containers.
This `SLURM backend
<https://miniwdl.readthedocs.io/en/latest/runner_backends.html>`_ plugin for
`miniwdl <https://github.com/chanzuckerberg/miniwdl>`_ runs WDL task containers
by creating a job script that is submitted to a SLURM cluster. In case the job
description has a container, singularity will be used as container runtime.
Installation
------------
For the latest stable version::
pip install miniwdl-slurm
For the development version::
pip install git+https://github.com/miniwdl-ext/miniwdl-slurm.git
Configuration
--------------
The following `miniwdl configuration
<https://miniwdl.readthedocs.io/en/latest/runner_reference.html#configuration>`_
example can be used to use miniwdl on a SLURM cluster:
.. code-block:: ini
[scheduler]
container_backend=slurm_singularity
# task_concurrency defaults to the number of processors on the system.
# since we submit the jobs to SLURM this is not necessary.
# higher numbers means miniwdl has to monitor more processes simultaneously
# which might impact performance.
task_concurrency=200
# This setting allows running tasks to continue, even if one other tasks fails.
# Useful in combination with call caching. Prevents wasting resources by
# cancelling jobs half-way that would probably succeed.
fail_fast = false
[call_cache]
# The following settings create a call cache under the current directory.
# This prevents wasting unnecessary resources on the cluster by rerunning
# jobs that have already succeeded.
put = true
get = true
dir = "$PWD/miniwdl_call_cache"
[task_runtime]
# Setting a 'maxRetries' default allows jobs that fail due to intermittent
# errors on the cluster to be retried.
defaults = {
"maxRetries": 2,
"docker": "ubuntu:20.04"
}
[singularity]
# This plugin wraps the singularity backend. Make sure the settings are
# appropriate for your cluster.
exe = ["singularity"]
# the miniwdl default options contain options to run as a fake root, which
# is not available on most clusters.
run_options = [
"--containall"
]
# Location of the singularity images (optional). The miniwdl-slurm plugin
# will set it to a directory inside $PWD. This location must be reachable
# for the submit nodes.
image_cache = "$PWD/miniwdl_singularity_cache"
[slurm]
# extra arguments passed to the srun command (optional).
extra_args="--partition heavy_users,gpu --comment 'run with miniwdl'"
Raw data
{
"_id": null,
"home_page": "https://github.com/miniwdl-ext/miniwdl-slurm",
"name": "miniwdl-slurm",
"maintainer": "",
"docs_url": null,
"requires_python": ">=3.6",
"maintainer_email": "",
"keywords": "WDL miniwdl slurm backend singularity",
"author": "Leiden University Medical Center",
"author_email": "sasc@lumc.nl",
"download_url": "https://files.pythonhosted.org/packages/5e/6a/c245d037a286458873761d1f8b45024aab28a46de0b3b7efca5302e4ea68/miniwdl-slurm-0.2.0.tar.gz",
"platform": null,
"description": "miniwdl-slurm\n=============\nExtends miniwdl to run workflows on SLURM clusters in singularity containers.\n\nThis `SLURM backend\n<https://miniwdl.readthedocs.io/en/latest/runner_backends.html>`_ plugin for\n`miniwdl <https://github.com/chanzuckerberg/miniwdl>`_ runs WDL task containers\nby creating a job script that is submitted to a SLURM cluster. In case the job\ndescription has a container, singularity will be used as container runtime.\n\nInstallation\n------------\nFor the latest stable version::\n\n pip install miniwdl-slurm\n\nFor the development version::\n\n pip install git+https://github.com/miniwdl-ext/miniwdl-slurm.git\n\nConfiguration\n--------------\nThe following `miniwdl configuration\n<https://miniwdl.readthedocs.io/en/latest/runner_reference.html#configuration>`_\nexample can be used to use miniwdl on a SLURM cluster:\n\n.. code-block:: ini\n\n [scheduler]\n container_backend=slurm_singularity\n # task_concurrency defaults to the number of processors on the system.\n # since we submit the jobs to SLURM this is not necessary.\n # higher numbers means miniwdl has to monitor more processes simultaneously\n # which might impact performance.\n task_concurrency=200\n \n # This setting allows running tasks to continue, even if one other tasks fails. \n # Useful in combination with call caching. Prevents wasting resources by\n # cancelling jobs half-way that would probably succeed.\n fail_fast = false\n\n [call_cache]\n # The following settings create a call cache under the current directory.\n # This prevents wasting unnecessary resources on the cluster by rerunning \n # jobs that have already succeeded.\n put = true \n get = true \n dir = \"$PWD/miniwdl_call_cache\"\n\n [task_runtime]\n # Setting a 'maxRetries' default allows jobs that fail due to intermittent\n # errors on the cluster to be retried.\n defaults = {\n \"maxRetries\": 2,\n \"docker\": \"ubuntu:20.04\"\n }\n \n [singularity]\n # This plugin wraps the singularity backend. Make sure the settings are\n # appropriate for your cluster.\n exe = [\"singularity\"]\n\n # the miniwdl default options contain options to run as a fake root, which\n # is not available on most clusters.\n run_options = [\n \"--containall\"\n ]\n\n # Location of the singularity images (optional). The miniwdl-slurm plugin\n # will set it to a directory inside $PWD. This location must be reachable\n # for the submit nodes.\n image_cache = \"$PWD/miniwdl_singularity_cache\"\n\n [slurm]\n # extra arguments passed to the srun command (optional).\n extra_args=\"--partition heavy_users,gpu --comment 'run with miniwdl'\"\n",
"bugtrack_url": null,
"license": "MIT",
"summary": "miniwdl slurm backend using singularity",
"version": "0.2.0",
"split_keywords": [
"wdl",
"miniwdl",
"slurm",
"backend",
"singularity"
],
"urls": [
{
"comment_text": "",
"digests": {
"blake2b_256": "509d5afd88340606bdf6440deb1bcd8525bf78914c4887a2ea29c2d5b45ceb54",
"md5": "d2f6b3625e9c53791c7563cd9748e139",
"sha256": "e2bff7fe4cb13d7d9374507699773f4d7a1d527b19794f635bca3bd1d205d872"
},
"downloads": -1,
"filename": "miniwdl_slurm-0.2.0-py3-none-any.whl",
"has_sig": false,
"md5_digest": "d2f6b3625e9c53791c7563cd9748e139",
"packagetype": "bdist_wheel",
"python_version": "py3",
"requires_python": ">=3.6",
"size": 5929,
"upload_time": "2023-04-04T08:19:47",
"upload_time_iso_8601": "2023-04-04T08:19:47.831933Z",
"url": "https://files.pythonhosted.org/packages/50/9d/5afd88340606bdf6440deb1bcd8525bf78914c4887a2ea29c2d5b45ceb54/miniwdl_slurm-0.2.0-py3-none-any.whl",
"yanked": false,
"yanked_reason": null
},
{
"comment_text": "",
"digests": {
"blake2b_256": "5e6ac245d037a286458873761d1f8b45024aab28a46de0b3b7efca5302e4ea68",
"md5": "91e493c9bdd0fb70d0aa6bc82d5828b7",
"sha256": "ac1635e0f57c8e600f84074d940050cf261ade0ff619f2ed09d273a438f2fc02"
},
"downloads": -1,
"filename": "miniwdl-slurm-0.2.0.tar.gz",
"has_sig": false,
"md5_digest": "91e493c9bdd0fb70d0aa6bc82d5828b7",
"packagetype": "sdist",
"python_version": "source",
"requires_python": ">=3.6",
"size": 4994,
"upload_time": "2023-04-04T08:19:50",
"upload_time_iso_8601": "2023-04-04T08:19:50.033411Z",
"url": "https://files.pythonhosted.org/packages/5e/6a/c245d037a286458873761d1f8b45024aab28a46de0b3b7efca5302e4ea68/miniwdl-slurm-0.2.0.tar.gz",
"yanked": false,
"yanked_reason": null
}
],
"upload_time": "2023-04-04 08:19:50",
"github": true,
"gitlab": false,
"bitbucket": false,
"github_user": "miniwdl-ext",
"github_project": "miniwdl-slurm",
"travis_ci": false,
"coveralls": false,
"github_actions": true,
"tox": true,
"lcname": "miniwdl-slurm"
}
JSON
