compspec-modules


Namecompspec-modules JSON
Version 0.0.1 PyPI version JSON
download
home_pagehttps://github.com/compspec/compspec-modules
SummarySubsystem graph generation for environment modules
upload_time2024-06-05 23:31:38
maintainerVanessa Sochat
docs_urlNone
authorVanessa Sochat
requires_pythonNone
licenseLICENSE
keywords compatibility compspec modules subsystem scheduling hpc
VCS
bugtrack_url
requirements No requirements were recorded.
Travis-CI No Travis.
coveralls test coverage No coveralls.
            # Compspec Modules

<p align="center">
  <img height="300" src="https://raw.githubusercontent.com/compspec/spec/main/img/compspec-circle.png">
</p>

[![PyPI version](https://badge.fury.io/py/compspec-modules.svg)](https://badge.fury.io/py/compspec-modules)

A compspec (Composition spec) is a specification and model for comparing things. Compspec modules is
a plugin for extraction of environment modules subsystem metadata about software installed
on a system. This can be for the entire system or a user-specific install. The plugin exports json graph format (JGF) that can be used for scheduling and other purposes. To learn more:

 - [Compspec](https://github.com/compspec/compspec): the Python library that discovers and loads this plugin.

Note that I'm currently parsing environment modules, and very simply. We can extend to more complex / LMOD if needed.

## Usage

Install compspec and the plugin here:

```bash
pip install compspec
pip install compspec-modules
```

Here is an easy way (in the devcontainer) to get a bunch of testing modules.

```bash
git clone https://github.com/shawfdong/modulefiles /tmp/modulefiles
export MODULEPATH=$MODULEPATH:/tmp/modulefiles
```

Then run an extraction with spack. You will want to provide the root to the spack install to describe:

```bash
compspec extract modules
```

To save to file:

```bash
compspec extract --outfile module-subsystem.json modules
```


<details>

<summary>compspec-modules output</summary>

```console
{
    "graph": {
        "nodes": {
            "environment-modules0": {
                "metadata": {
                    "type": "environment-modules",
                    "basename": "environment-modules",
                    "name": "environment-modules0",
                    "id": 0,
                    "uniq_id": 0,
                    "containment": {
                        "paths": "/environment-modules0"
                    },
                    "size": 1,
                    "unit": "",
                    "rank": 0,
                    "exclusive": false
                },
                "label": "environment-modules0"
            },
            "module1": {
                "metadata": {
                    "type": "module",
                    "basename": "module",
                    "name": "module0",
                    "id": 1,
                    "uniq_id": 1,
                    "containment": {
                        "paths": "/environment-modules0/module0"
                    },
                    "size": 1,
                    "unit": "",
                    "rank": 0,
                    "exclusive": false,
                    "attributes": {
                        "name": "dot",
                        "software": "dot"
                    }
                },
                "label": "module1"
            },
            "module2": {
                "metadata": {
                    "type": "module",
                    "basename": "module",
                    "name": "module1",
                    "id": 2,
                    "uniq_id": 2,
                    "containment": {
                        "paths": "/environment-modules0/module1"
                    },
                    "size": 1,
                    "unit": "",
                    "rank": 0,
                    "exclusive": false,
                    "attributes": {
                        "name": "module-info",
                        "software": "module-info"
                    }
                },
                "label": "module2"
            },
            "module3": {
                "metadata": {
                    "type": "module",
                    "basename": "module",
                    "name": "module2",
                    "id": 3,
                    "uniq_id": 3,
                    "containment": {
                        "paths": "/environment-modules0/module2"
                    },
                    "size": 1,
                    "unit": "",
                    "rank": 0,
                    "exclusive": false,
                    "attributes": {
                        "name": "modules",
                        "software": "modules",
                        "version": "5.4.0"
                    }
                },
                "label": "module3"
            },
            "module4": {
                "metadata": {
                    "type": "module",
                    "basename": "module",
                    "name": "module3",
                    "id": 4,
                    "uniq_id": 4,
                    "containment": {
                        "paths": "/environment-modules0/module3"
                    },
                    "size": 1,
                    "unit": "",
                    "rank": 0,
                    "exclusive": false,
                    "attributes": {
                        "name": "module-git",
                        "software": "module-git"
                    }
                },
                "label": "module4"
            },
            "module5": {
                "metadata": {
                    "type": "module",
                    "basename": "module",
                    "name": "module4",
                    "id": 5,
                    "uniq_id": 5,
                    "containment": {
                        "paths": "/environment-modules0/module4"
                    },
                    "size": 1,
                    "unit": "",
                    "rank": 0,
                    "exclusive": false,
                    "attributes": {
                        "name": "use.own",
                        "software": "use.own"
                    }
                },
                "label": "module5"
            },
            "module6": {
                "metadata": {
                    "type": "module",
                    "basename": "module",
                    "name": "module5",
                    "id": 6,
                    "uniq_id": 6,
                    "containment": {
                        "paths": "/environment-modules0/module5"
                    },
                    "size": 1,
                    "unit": "",
                    "rank": 0,
                    "exclusive": false,
                    "attributes": {
                        "name": "null",
                        "software": "null"
                    }
                },
                "label": "module6"
            },
            "module7": {
                "metadata": {
                    "type": "module",
                    "basename": "module",
                    "name": "module6",
                    "id": 7,
                    "uniq_id": 7,
                    "containment": {
                        "paths": "/environment-modules0/module6"
                    },
                    "size": 1,
                    "unit": "",
                    "rank": 0,
                    "exclusive": false,
                    "attributes": {
                        "name": "python/intelpython2",
                        "software": "python"
                    }
                },
                "label": "module7"
            },
            "module8": {
                "metadata": {
                    "type": "module",
                    "basename": "module",
                    "name": "module7",
                    "id": 8,
                    "uniq_id": 8,
                    "containment": {
                        "paths": "/environment-modules0/module7"
                    },
                    "size": 1,
                    "unit": "",
                    "rank": 0,
                    "exclusive": false,
                    "attributes": {
                        "name": "python/anaconda3",
                        "software": "python"
                    }
                },
                "label": "module8"
            },
            "module9": {
                "metadata": {
                    "type": "module",
                    "basename": "module",
                    "name": "module8",
                    "id": 9,
                    "uniq_id": 9,
                    "containment": {
                        "paths": "/environment-modules0/module8"
                    },
                    "size": 1,
                    "unit": "",
                    "rank": 0,
                    "exclusive": false,
                    "attributes": {
                        "name": "python/anaconda2",
                        "software": "python"
                    }
                },
                "label": "module9"
            },
            "module10": {
                "metadata": {
                    "type": "module",
                    "basename": "module",
                    "name": "module9",
                    "id": 10,
                    "uniq_id": 10,
                    "containment": {
                        "paths": "/environment-modules0/module9"
                    },
                    "size": 1,
                    "unit": "",
                    "rank": 0,
                    "exclusive": false,
                    "attributes": {
                        "name": "python/intelpython3",
                        "software": "python"
                    }
                },
                "label": "module10"
            },
            "module11": {
                "metadata": {
                    "type": "module",
                    "basename": "module",
                    "name": "module10",
                    "id": 11,
                    "uniq_id": 11,
                    "containment": {
                        "paths": "/environment-modules0/module10"
                    },
                    "size": 1,
                    "unit": "",
                    "rank": 0,
                    "exclusive": false,
                    "attributes": {
                        "name": "python/rh-python36",
                        "software": "python"
                    }
                },
                "label": "module11"
            },
            "module12": {
                "metadata": {
                    "type": "module",
                    "basename": "module",
                    "name": "module11",
                    "id": 12,
                    "uniq_id": 12,
                    "containment": {
                        "paths": "/environment-modules0/module11"
                    },
                    "size": 1,
                    "unit": "",
                    "rank": 0,
                    "exclusive": false,
                    "attributes": {
                        "name": "PrgEnv-intel/2019.1.053",
                        "software": "PrgEnv-intel",
                        "version": "2019.1.053"
                    }
                },
                "label": "module12"
            },
            "module13": {
                "metadata": {
                    "type": "module",
                    "basename": "module",
                    "name": "module12",
                    "id": 13,
                    "uniq_id": 13,
                    "containment": {
                        "paths": "/environment-modules0/module12"
                    },
                    "size": 1,
                    "unit": "",
                    "rank": 0,
                    "exclusive": false,
                    "attributes": {
                        "name": "PrgEnv-pgi/llvm_18.10",
                        "software": "PrgEnv-pgi",
                        "version": "18.10"
                    }
                },
                "label": "module13"
            },

...
    },
    "metadata": {
        "install_name": "compat-experiment"
    }
}
```
</details>


### Development

If you open the [Development container](.devcontainer) in VSCode, you'll find environment modules are installed:

```bash
$ module avail
```

This allows us to easily develop and test the compatibility plugin.


## License

HPCIC DevTools is distributed under the terms of the MIT license.
All new contributions must be made under this license.

See [LICENSE](https://github.com/converged-computing/cloud-select/blob/main/LICENSE),
[COPYRIGHT](https://github.com/converged-computing/cloud-select/blob/main/COPYRIGHT), and
[NOTICE](https://github.com/converged-computing/cloud-select/blob/main/NOTICE) for details.

SPDX-License-Identifier: (MIT)

LLNL-CODE- 842614

            

Raw data

            {
    "_id": null,
    "home_page": "https://github.com/compspec/compspec-modules",
    "name": "compspec-modules",
    "maintainer": "Vanessa Sochat",
    "docs_url": null,
    "requires_python": null,
    "maintainer_email": null,
    "keywords": "compatibility, compspec, modules, subsystem, scheduling, hpc",
    "author": "Vanessa Sochat",
    "author_email": "vsoch@users.noreply.github.com",
    "download_url": "https://files.pythonhosted.org/packages/46/1f/0c99e2c64980c5e34115d5f7c51b4d0ca68ba13ce0e4b51ad42f8003ee4b/compspec-modules-0.0.1.tar.gz",
    "platform": null,
    "description": "# Compspec Modules\n\n<p align=\"center\">\n  <img height=\"300\" src=\"https://raw.githubusercontent.com/compspec/spec/main/img/compspec-circle.png\">\n</p>\n\n[![PyPI version](https://badge.fury.io/py/compspec-modules.svg)](https://badge.fury.io/py/compspec-modules)\n\nA compspec (Composition spec) is a specification and model for comparing things. Compspec modules is\na plugin for extraction of environment modules subsystem metadata about software installed\non a system. This can be for the entire system or a user-specific install. The plugin exports json graph format (JGF) that can be used for scheduling and other purposes. To learn more:\n\n - [Compspec](https://github.com/compspec/compspec): the Python library that discovers and loads this plugin.\n\nNote that I'm currently parsing environment modules, and very simply. We can extend to more complex / LMOD if needed.\n\n## Usage\n\nInstall compspec and the plugin here:\n\n```bash\npip install compspec\npip install compspec-modules\n```\n\nHere is an easy way (in the devcontainer) to get a bunch of testing modules.\n\n```bash\ngit clone https://github.com/shawfdong/modulefiles /tmp/modulefiles\nexport MODULEPATH=$MODULEPATH:/tmp/modulefiles\n```\n\nThen run an extraction with spack. You will want to provide the root to the spack install to describe:\n\n```bash\ncompspec extract modules\n```\n\nTo save to file:\n\n```bash\ncompspec extract --outfile module-subsystem.json modules\n```\n\n\n<details>\n\n<summary>compspec-modules output</summary>\n\n```console\n{\n    \"graph\": {\n        \"nodes\": {\n            \"environment-modules0\": {\n                \"metadata\": {\n                    \"type\": \"environment-modules\",\n                    \"basename\": \"environment-modules\",\n                    \"name\": \"environment-modules0\",\n                    \"id\": 0,\n                    \"uniq_id\": 0,\n                    \"containment\": {\n                        \"paths\": \"/environment-modules0\"\n                    },\n                    \"size\": 1,\n                    \"unit\": \"\",\n                    \"rank\": 0,\n                    \"exclusive\": false\n                },\n                \"label\": \"environment-modules0\"\n            },\n            \"module1\": {\n                \"metadata\": {\n                    \"type\": \"module\",\n                    \"basename\": \"module\",\n                    \"name\": \"module0\",\n                    \"id\": 1,\n                    \"uniq_id\": 1,\n                    \"containment\": {\n                        \"paths\": \"/environment-modules0/module0\"\n                    },\n                    \"size\": 1,\n                    \"unit\": \"\",\n                    \"rank\": 0,\n                    \"exclusive\": false,\n                    \"attributes\": {\n                        \"name\": \"dot\",\n                        \"software\": \"dot\"\n                    }\n                },\n                \"label\": \"module1\"\n            },\n            \"module2\": {\n                \"metadata\": {\n                    \"type\": \"module\",\n                    \"basename\": \"module\",\n                    \"name\": \"module1\",\n                    \"id\": 2,\n                    \"uniq_id\": 2,\n                    \"containment\": {\n                        \"paths\": \"/environment-modules0/module1\"\n                    },\n                    \"size\": 1,\n                    \"unit\": \"\",\n                    \"rank\": 0,\n                    \"exclusive\": false,\n                    \"attributes\": {\n                        \"name\": \"module-info\",\n                        \"software\": \"module-info\"\n                    }\n                },\n                \"label\": \"module2\"\n            },\n            \"module3\": {\n                \"metadata\": {\n                    \"type\": \"module\",\n                    \"basename\": \"module\",\n                    \"name\": \"module2\",\n                    \"id\": 3,\n                    \"uniq_id\": 3,\n                    \"containment\": {\n                        \"paths\": \"/environment-modules0/module2\"\n                    },\n                    \"size\": 1,\n                    \"unit\": \"\",\n                    \"rank\": 0,\n                    \"exclusive\": false,\n                    \"attributes\": {\n                        \"name\": \"modules\",\n                        \"software\": \"modules\",\n                        \"version\": \"5.4.0\"\n                    }\n                },\n                \"label\": \"module3\"\n            },\n            \"module4\": {\n                \"metadata\": {\n                    \"type\": \"module\",\n                    \"basename\": \"module\",\n                    \"name\": \"module3\",\n                    \"id\": 4,\n                    \"uniq_id\": 4,\n                    \"containment\": {\n                        \"paths\": \"/environment-modules0/module3\"\n                    },\n                    \"size\": 1,\n                    \"unit\": \"\",\n                    \"rank\": 0,\n                    \"exclusive\": false,\n                    \"attributes\": {\n                        \"name\": \"module-git\",\n                        \"software\": \"module-git\"\n                    }\n                },\n                \"label\": \"module4\"\n            },\n            \"module5\": {\n                \"metadata\": {\n                    \"type\": \"module\",\n                    \"basename\": \"module\",\n                    \"name\": \"module4\",\n                    \"id\": 5,\n                    \"uniq_id\": 5,\n                    \"containment\": {\n                        \"paths\": \"/environment-modules0/module4\"\n                    },\n                    \"size\": 1,\n                    \"unit\": \"\",\n                    \"rank\": 0,\n                    \"exclusive\": false,\n                    \"attributes\": {\n                        \"name\": \"use.own\",\n                        \"software\": \"use.own\"\n                    }\n                },\n                \"label\": \"module5\"\n            },\n            \"module6\": {\n                \"metadata\": {\n                    \"type\": \"module\",\n                    \"basename\": \"module\",\n                    \"name\": \"module5\",\n                    \"id\": 6,\n                    \"uniq_id\": 6,\n                    \"containment\": {\n                        \"paths\": \"/environment-modules0/module5\"\n                    },\n                    \"size\": 1,\n                    \"unit\": \"\",\n                    \"rank\": 0,\n                    \"exclusive\": false,\n                    \"attributes\": {\n                        \"name\": \"null\",\n                        \"software\": \"null\"\n                    }\n                },\n                \"label\": \"module6\"\n            },\n            \"module7\": {\n                \"metadata\": {\n                    \"type\": \"module\",\n                    \"basename\": \"module\",\n                    \"name\": \"module6\",\n                    \"id\": 7,\n                    \"uniq_id\": 7,\n                    \"containment\": {\n                        \"paths\": \"/environment-modules0/module6\"\n                    },\n                    \"size\": 1,\n                    \"unit\": \"\",\n                    \"rank\": 0,\n                    \"exclusive\": false,\n                    \"attributes\": {\n                        \"name\": \"python/intelpython2\",\n                        \"software\": \"python\"\n                    }\n                },\n                \"label\": \"module7\"\n            },\n            \"module8\": {\n                \"metadata\": {\n                    \"type\": \"module\",\n                    \"basename\": \"module\",\n                    \"name\": \"module7\",\n                    \"id\": 8,\n                    \"uniq_id\": 8,\n                    \"containment\": {\n                        \"paths\": \"/environment-modules0/module7\"\n                    },\n                    \"size\": 1,\n                    \"unit\": \"\",\n                    \"rank\": 0,\n                    \"exclusive\": false,\n                    \"attributes\": {\n                        \"name\": \"python/anaconda3\",\n                        \"software\": \"python\"\n                    }\n                },\n                \"label\": \"module8\"\n            },\n            \"module9\": {\n                \"metadata\": {\n                    \"type\": \"module\",\n                    \"basename\": \"module\",\n                    \"name\": \"module8\",\n                    \"id\": 9,\n                    \"uniq_id\": 9,\n                    \"containment\": {\n                        \"paths\": \"/environment-modules0/module8\"\n                    },\n                    \"size\": 1,\n                    \"unit\": \"\",\n                    \"rank\": 0,\n                    \"exclusive\": false,\n                    \"attributes\": {\n                        \"name\": \"python/anaconda2\",\n                        \"software\": \"python\"\n                    }\n                },\n                \"label\": \"module9\"\n            },\n            \"module10\": {\n                \"metadata\": {\n                    \"type\": \"module\",\n                    \"basename\": \"module\",\n                    \"name\": \"module9\",\n                    \"id\": 10,\n                    \"uniq_id\": 10,\n                    \"containment\": {\n                        \"paths\": \"/environment-modules0/module9\"\n                    },\n                    \"size\": 1,\n                    \"unit\": \"\",\n                    \"rank\": 0,\n                    \"exclusive\": false,\n                    \"attributes\": {\n                        \"name\": \"python/intelpython3\",\n                        \"software\": \"python\"\n                    }\n                },\n                \"label\": \"module10\"\n            },\n            \"module11\": {\n                \"metadata\": {\n                    \"type\": \"module\",\n                    \"basename\": \"module\",\n                    \"name\": \"module10\",\n                    \"id\": 11,\n                    \"uniq_id\": 11,\n                    \"containment\": {\n                        \"paths\": \"/environment-modules0/module10\"\n                    },\n                    \"size\": 1,\n                    \"unit\": \"\",\n                    \"rank\": 0,\n                    \"exclusive\": false,\n                    \"attributes\": {\n                        \"name\": \"python/rh-python36\",\n                        \"software\": \"python\"\n                    }\n                },\n                \"label\": \"module11\"\n            },\n            \"module12\": {\n                \"metadata\": {\n                    \"type\": \"module\",\n                    \"basename\": \"module\",\n                    \"name\": \"module11\",\n                    \"id\": 12,\n                    \"uniq_id\": 12,\n                    \"containment\": {\n                        \"paths\": \"/environment-modules0/module11\"\n                    },\n                    \"size\": 1,\n                    \"unit\": \"\",\n                    \"rank\": 0,\n                    \"exclusive\": false,\n                    \"attributes\": {\n                        \"name\": \"PrgEnv-intel/2019.1.053\",\n                        \"software\": \"PrgEnv-intel\",\n                        \"version\": \"2019.1.053\"\n                    }\n                },\n                \"label\": \"module12\"\n            },\n            \"module13\": {\n                \"metadata\": {\n                    \"type\": \"module\",\n                    \"basename\": \"module\",\n                    \"name\": \"module12\",\n                    \"id\": 13,\n                    \"uniq_id\": 13,\n                    \"containment\": {\n                        \"paths\": \"/environment-modules0/module12\"\n                    },\n                    \"size\": 1,\n                    \"unit\": \"\",\n                    \"rank\": 0,\n                    \"exclusive\": false,\n                    \"attributes\": {\n                        \"name\": \"PrgEnv-pgi/llvm_18.10\",\n                        \"software\": \"PrgEnv-pgi\",\n                        \"version\": \"18.10\"\n                    }\n                },\n                \"label\": \"module13\"\n            },\n\n...\n    },\n    \"metadata\": {\n        \"install_name\": \"compat-experiment\"\n    }\n}\n```\n</details>\n\n\n### Development\n\nIf you open the [Development container](.devcontainer) in VSCode, you'll find environment modules are installed:\n\n```bash\n$ module avail\n```\n\nThis allows us to easily develop and test the compatibility plugin.\n\n\n## License\n\nHPCIC DevTools is distributed under the terms of the MIT license.\nAll new contributions must be made under this license.\n\nSee [LICENSE](https://github.com/converged-computing/cloud-select/blob/main/LICENSE),\n[COPYRIGHT](https://github.com/converged-computing/cloud-select/blob/main/COPYRIGHT), and\n[NOTICE](https://github.com/converged-computing/cloud-select/blob/main/NOTICE) for details.\n\nSPDX-License-Identifier: (MIT)\n\nLLNL-CODE- 842614\n",
    "bugtrack_url": null,
    "license": "LICENSE",
    "summary": "Subsystem graph generation for environment modules",
    "version": "0.0.1",
    "project_urls": {
        "Homepage": "https://github.com/compspec/compspec-modules"
    },
    "split_keywords": [
        "compatibility",
        " compspec",
        " modules",
        " subsystem",
        " scheduling",
        " hpc"
    ],
    "urls": [
        {
            "comment_text": "",
            "digests": {
                "blake2b_256": "f40041a855f21ce1ec7d1cb2ce6e5e1a58ddf62e971323b3535c171b053eb09c",
                "md5": "fca2993f4c9c4120f99da437365c4420",
                "sha256": "86ee0cd9b128f4c74fa33f3adc75c606d12856c912cd5a9bf84da120debe7ae9"
            },
            "downloads": -1,
            "filename": "compspec_modules-0.0.1-py3-none-any.whl",
            "has_sig": false,
            "md5_digest": "fca2993f4c9c4120f99da437365c4420",
            "packagetype": "bdist_wheel",
            "python_version": "py3",
            "requires_python": null,
            "size": 8337,
            "upload_time": "2024-06-05T23:31:36",
            "upload_time_iso_8601": "2024-06-05T23:31:36.864152Z",
            "url": "https://files.pythonhosted.org/packages/f4/00/41a855f21ce1ec7d1cb2ce6e5e1a58ddf62e971323b3535c171b053eb09c/compspec_modules-0.0.1-py3-none-any.whl",
            "yanked": false,
            "yanked_reason": null
        },
        {
            "comment_text": "",
            "digests": {
                "blake2b_256": "461f0c99e2c64980c5e34115d5f7c51b4d0ca68ba13ce0e4b51ad42f8003ee4b",
                "md5": "ff8919fe2e73e494594cd879c1c96042",
                "sha256": "16e1bb28b8d28ee1b4377ac5acae257d65ec61740e26ed72b77c1471596e29dd"
            },
            "downloads": -1,
            "filename": "compspec-modules-0.0.1.tar.gz",
            "has_sig": false,
            "md5_digest": "ff8919fe2e73e494594cd879c1c96042",
            "packagetype": "sdist",
            "python_version": "source",
            "requires_python": null,
            "size": 10251,
            "upload_time": "2024-06-05T23:31:38",
            "upload_time_iso_8601": "2024-06-05T23:31:38.766608Z",
            "url": "https://files.pythonhosted.org/packages/46/1f/0c99e2c64980c5e34115d5f7c51b4d0ca68ba13ce0e4b51ad42f8003ee4b/compspec-modules-0.0.1.tar.gz",
            "yanked": false,
            "yanked_reason": null
        }
    ],
    "upload_time": "2024-06-05 23:31:38",
    "github": true,
    "gitlab": false,
    "bitbucket": false,
    "codeberg": false,
    "github_user": "compspec",
    "github_project": "compspec-modules",
    "travis_ci": false,
    "coveralls": false,
    "github_actions": false,
    "lcname": "compspec-modules"
}
        
Elapsed time: 0.81040s