| Name | alchemy-cat JSON |
| Version |
0.0.8
JSON |
| download |
| home_page | None |
| Summary | Alchemy Cat ββ π₯Config System for SOTA |
| upload_time | 2024-07-29 13:27:04 |
| maintainer | None |
| docs_url | None |
| author | None |
| requires_python | >=3.9 |
| license | Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "{}" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright {yyyy} {name of copyright owner} Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. |
| keywords |
config
deep learning
parameter tuning
hyperparameter
|
| VCS |
 |
| bugtrack_url |
|
| requirements |
No requirements were recorded.
|
| Travis-CI |
No Travis.
|
| coveralls test coverage |
No coveralls.
|
# Alchemy Cat ββ π₯Config System for SOTA
<div align="center">
<img src="https://img.shields.io/github/stars/HAL-42/AlchemyCat?color=yellow" alt="Stars">
<img src="https://img.shields.io/github/issues/HAL-42/AlchemyCat?color=red" alt="Issues">
<br>
[](https://badge.fury.io/py/alchemy-cat)
<img src="https://img.shields.io/badge/python-3.9-purple.svg" alt="Python"> <br>
</div>
<div align="center">
<a href="README.md">English</a> | <a href="README_CN.md">δΈζ</a>
</div>
<br>
<div align="center">
[πIntroduction](https://github.com/HAL-42/AlchemyCat/blob/master/README.md#-introduction) | [π¦Installation](https://github.com/HAL-42/AlchemyCat/blob/master/README.md#-installation) | [πMigration](https://github.com/HAL-42/AlchemyCat/blob/master/README.md#-migration) | [πDocumentation](https://github.com/HAL-42/AlchemyCat/blob/master/README.md#-documentation-)
</div>
# <div align="center">π Introduction</div>
When developing machine learning algorithms, we often bother with:
* Config files (YAML / YACS / MMCV) are lengthy and complex. If entries are interdependent, extra caution is needed to avoid errors when modifying them.
* Parameter tuning requires rewriting the config for each parameter set, leading to code duplication and difficulty in tracking changes.
* Manually traversing the parameter space and summarizing results during parameter tuning is time-consuming and inefficient.
* Insufficient parameter tuning can obscure effective designs.
* Effective methods may not achieve SOTA due to insufficient parameter tuning, reducing persuasiveness.
AlchemyCat is a config system designed for machine learning research to address such issues. It helps researchers to fully explore the parameter tuning potential by simplifying repetitive tasks like reproduction, modifying configs, and hyperparameter tuning
The table below compares AlchemyCat with existing config systems (π‘ not support, π€ limited support, π₯³ supported):
| Feature | argparse | yaml | YACS | mmcv | AlchemyCat |
|----------------------------|----------|------|------|------|------------|
| Reproducible | π‘ | π₯³ | π₯³ | π₯³ | π₯³ |
| IDE Jump | π‘ | π‘ | π₯³ | π₯³ | π₯³ |
| Inheritance | π‘ | π‘ | π€ | π€ | π₯³ |
| Composition | π‘ | π‘ | π€ | π€ | π₯³ |
| dependency | π‘ | π‘ | π‘ | π‘ | π₯³ |
| Automatic Parameter Tuning | π‘ | π‘ | π‘ | π‘ | π₯³ |
AlchemyCat implements all features of current popular config systems, while fully considering various special cases, ensuring stability. AlchemyCat distinguishes itself by:
* Readable: The syntax is simple, elegant, and Pythonic.
* Reusable: Supports **inheritance** and **composition** of configs, reducing redundancy and enhancing reusability.
* Maintainable: Allows for establishing **dependency** between config items, enabling global synchronization with a single change.
* Supports auto parameter tuning and result summarization without needing to modify original configs or training codes.
[Migrate](https://github.com/HAL-42/AlchemyCat/blob/master/README.md#-migration) from config systems listed above to AlchemyCat is effortless. Just spend 15 minutes reading the [documentation](https://github.com/HAL-42/AlchemyCat/blob/master/README.md#-documentation-) and apply AlchemyCat to your project, and your GPU will never be idle again!
## Quick Glance
Deep learning relies on numerous empirical hyperparameters, such as learning rate, loss weights, max iterations, sliding window size, drop probability, thresholds, and even random seeds.
The relationship between hyperparameters and performance is hard to predict theoretically. The only certainty is that arbitrarily chosen hyperparameters are unlikely to be optimal. Practice has shown that grid search through the hyperparameter space can significantly enhance model performance; sometimes its effect even surpasses so-called "contributions." Achieving SOTA often depends on this!
AlchemyCat offers an auto parameter-tuner that seamlessly integrates with existing config systems to explore the hyperparameter space and summarize experiment results automatically. Using this tool requires no modifications to the original config or training code.
For example, with [MMSeg](https://github.com/open-mmlab/mmsegmentation) users only need to write a tunable config inherited from MMSeg's base config and define the parameter search space:
```python
# -- configs/deeplabv3plus/tune_bs,iter/cfg.py --
from alchemy_cat import Cfg2Tune, Param2Tune
# Inherit from standard mmcv config.
cfg = Cfg2Tune(caps='configs/deeplabv3plus/deeplabv3plus_r50-d8_4xb4-40k_voc12aug-512x512.py')
# Inherit and override
cfg.model.auxiliary_head.loss_decode.loss_weight = 0.2
# Tuning parameters: grid search batch_size and max_iters
cfg.train_dataloader.batch_size = Param2Tune([4, 8])
cfg.train_cfg.max_iters = Param2Tune([20_000, 40_000])
# ...
```
Next, write a script specifying how to run a single config and read its results:
```python
# -- tools/tune_dist_train.py --
import argparse, subprocess
from alchemy_cat.dl_config import Cfg2TuneRunner, Config
from alchemy_cat.dl_config.examples.read_mmcv_metric import get_metric
parser = argparse.ArgumentParser()
parser.add_argument('--cfg2tune', type=str) # Path to the tunable config
parser.add_argument('--num_gpu', type=int, default=2) # Number of GPUs for each task
args = parser.parse_args()
runner = Cfg2TuneRunner(args.cfg2tune, experiment_root='work_dirs', work_gpu_num=args.num_gpu)
@runner.register_work_fn # Run experiment for each param combination with mmcv official train script
def work(pkl_idx: int, cfg: Config, cfg_pkl: str, cfg_rslt_dir: str, cuda_env: dict[str, str]):
mmcv_cfg = cfg.save_mmcv(cfg_rslt_dir + '/mmcv_config.py')
subprocess.run(f'./tools/dist_train.sh {mmcv_cfg} {args.num_gpu}', env=cuda_env, shell=True)
@runner.register_gather_metric_fn # Optional, gather metric of each config
def gather_metric(cfg: Config, cfg_rslt_dir: str, run_rslt, param_comb) -> dict[str, float]:
return get_metric(cfg_rslt_dir) # {'aAcc': xxx, 'mIoU': xxx, 'mAcc': xxx}
runner.tuning()
```
Run `CUDA_VISIBLE_DEVICES=0,1,2,3 python tools/tune_dist_train.py --cfg2tune configs/deeplabv3plus/tune_bs,iter/cfg.py`, which will automatically search the parameter space in parallel and summarize the experiment results as follows:
<div align = "center">
<img src="https://raw.githubusercontent.com/HAL-42/AlchemyCat/master/docs/figs/readme-teaser-excel.png" width="500" />
</div>
In fact, the above config is still incomplete for some hyperparameters are interdependent and need to be adjusted together. For instance, the learning rate should scale with the batch size. AlchemyCat uses **dependency** to manage these relationships; when a dependency source changes, related dependencies automatically update for consistency. The complete config with dependencies is:
```python
# -- configs/deeplabv3plus/tune_bs,iter/cfg.py --
from alchemy_cat import Cfg2Tune, Param2Tune, P_DEP
# Inherit from standard mmcv config.
cfg = Cfg2Tune(caps='configs/deeplabv3plus/deeplabv3plus_r50-d8_4xb4-40k_voc12aug-512x512.py')
# Inherit and override
cfg.model.auxiliary_head.loss_decode.loss_weight = 0.2
# Tuning parameters: grid search batch_size and max_iters
cfg.train_dataloader.batch_size = Param2Tune([4, 8])
cfg.train_cfg.max_iters = Param2Tune([20_000, 40_000])
# Dependencies:
# 1) learning rate increase with batch_size
cfg.optim_wrapper.optimizer.lr = P_DEP(lambda c: (c.train_dataloader.batch_size / 8) * 0.01)
# 2) end of param_scheduler increase with max_iters
@cfg.set_DEP()
def param_scheduler(c):
return dict(
type='PolyLR',
eta_min=1e-4,
power=0.9,
begin=0,
end=c.train_cfg.max_iters,
by_epoch=False)
```
> [!NOTE]
> In the example above, defining dependencies might seem needless since they can be computed directly. However, when combined with **inheritance**, setting dependencies in the base config allows tunable configs to focus on key hyperparameters without worrying about trivial dependency details. Refer to the [documentation](#dependency) for details.
# <div align="center">π¦ Installation</div>
```bash
pip install alchemy-cat
```
# <div align="center">π Migration</div>
<details>
<summary> How to migrate from YAML / YACS / MMCV </summary>
Ο`βΒ΄)Ο Just kidding! No migration is needed. AlchemyCat can direct read and write YAML / YACS / MMCV config files:
```python
from alchemy_cat.dl_config import load_config, Config
# READ YAML / YACS / MMCV config to alchemy_cat.Config
cfg = load_config('path/to/yaml_config.yaml or yacs_config.py or mmcv_config.py')
# Init alchemy_cat.Config with YAML / YACS / MMCV config
cfg = Config('path/to/yaml_config.yaml or yacs_config.py or mmcv_config.py')
# alchemy_cat.Config inherits from YAML / YACS / MMCV config
cfg = Config(caps='path/to/yaml_config.yaml or yacs_config.py or mmcv_config.py')
print(cfg.model.backbone) # Access config item
cfg.save_yaml('path/to/save.yaml') # Save to YAML config
cfg.save_mmcv('path/to/save.py') # Save to MMCV config
cfg.save_py('path/to/save.py') # Save to AlchemyCat config
```
We also provide a script to convert between different config formats:
```bash
python -m alchemy_cat.dl_config.from_x_to_y --x X --y Y --y_type=yaml/mmcv/alchemy-cat
```
where:
* `--x`: Source config file path, can be YAML / YACS / MMCV / AlchemyCat config.
* `--y`: Target config file path.
* `--y_type`: Target config format, can be `yaml`, `mmcv`, or `alchemy-cat`.
</details>
# <div align="center">π Documentation </div>
## Basic Usage
AlchemyCat ensures a one-to-one correspondence between each configuration and its unique experimental record, with the bijective relationship ensuring the experiment's reproducibility.
```text
config C + algorithm code A ββ> reproducible experiment E(C, A)
```
The experimental directory is automatically generated, mirroring the relative path of the configuration file. This path can include multi-level directories and special characters such as spaces, commas, and equal signs. Such flexibility aids in categorizing experiments for clear management. For instance:
```text
.
βββ configs
βΒ Β βββ MNIST
βΒ Β βΒ Β βββ resnet18,wd=1e-5@run2
βΒ Β βΒ Β βΒ Β βββ cfg.py
βΒ Β βΒ Β βββ vgg,lr=1e-2
βΒ Β βΒ Β βββ cfg.py
βΒ Β βββ VOC2012
βΒ Β βββ swin-T,Ξ³=10
βΒ Β βββ 10 epoch
βΒ Β βββ cfg.py
βββ experiment
βββ MNIST
βΒ Β βββ resnet18,wd=1e-5@run2
βΒ Β βΒ Β βββ xxx.log
βΒ Β βββ vgg,lr=1e-2
βΒ Β βββ xxx.log
βββ VOC2012
βββ swin-T,Ξ³=10
βββ 10 epoch
βββ xxx.log
```
> [!TIP]
> **Best Practice: Create a `__init__.py` next to `cfg.py`(usually will be auto created by IDE), and avoid paths containing '.'. This can help IDE to debug and allow relative import in `cfg.py`.**
Let's begin with an incomplete example to demonstrate writing and loading a config. First, create the [config file](alchemy_cat/dl_config/examples/configs/mnist/plain_usage/cfg.py):
```python
# -- [INCOMPLETE] configs/mnist/plain_usage/cfg.py --
from torchvision.datasets import MNIST
from alchemy_cat.dl_config import Config
cfg = Config()
cfg.rand_seed = 0
cfg.dt.cls = MNIST
cfg.dt.ini.root = '/tmp/data'
cfg.dt.ini.train = True
# ... Code Omitted.
```
Here, we first instantiate a `Config` object `cfg`, and then add config items through attribute operator `.`. Config items can be any Python objects, including functions, methods, and classes.
> [!TIP]
> **Best Practice: We prefer specifying functions or classes directly in config over using strings/semaphores to control the program behavior. This enables IDE navigation, simplifying reading and debugging.**
`Config` is a subclass of Python's `dict`. The above code defines a nested dictionary with a **tree structure**:
```text
>>> print(cfg.to_dict())
{'rand_seed': 0,
'dt': {'cls': <class 'torchvision.datasets.mnist.MNIST'>,
'ini': {'root': '/tmp/data', 'train': True}}}
```
`Config` implements all API of Python `dict`:
```test
>>> cfg.keys()
dict_keys(['rand_seed', 'dt'])
>>> cfg['dt']['ini']['root']
'/tmp/data'
>>> {**cfg['dt']['ini'], 'download': True}
{'root': '/tmp/data', 'train': True, 'download': True}
```
You can initialize a `Config` object using `dict` (yaml, json) or its subclasses (YACS, mmcv.Config).
```text
>>> Config({'rand_seed': 0, 'dt': {'cls': MNIST, 'ini': {'root': '/tmp/data', 'train': True}}})
{'rand_seed': 0, 'dt': {'cls': <class 'torchvision.datasets.mnist.MNIST'>, 'ini': {'root': '/tmp/data', 'train': True}}}
```
Using operator `.` to read and write `cfg` will be clearer. For instance, the following code creates and initializes the `MNIST` dataset based on the config:
```text
>>> dataset = cfg.dt.cls(**cfg.dt.ini)
>>> dataset
Dataset MNIST
Number of datapoints: 60000
Root location: /tmp/data
Split: Train
```
Accessing a non-existent key returns an empty dictionary, which should be treated as `False`:
```text
>>> cfg.not_exit
{}
```
In the [main code](alchemy_cat/dl_config/examples/train.py), use the following code to load the config:
```python
# # [INCOMPLETE] -- train.py --
from alchemy_cat.dl_config import load_config
cfg = load_config('configs/mnist/base/cfg.py', experiments_root='/tmp/experiment', config_root='configs')
# ... Code Omitted.
torch.save(model.state_dict(), f"{cfg.rslt_dir}/model_{epoch}.pth") # Save all experiment results to cfg.rslt_dir.
```
The `load_config` imports `cfg` from `configs/mnist/base/cfg.py`, handling inheritance and dependencies. Given the experiment root directory `experiments_root` and config root directory `config_root`, it auto creates an experiment directory at `experiment/mnist/base` and assign it to `cfg.rslt_dir`. All experimental results should be saved to `cfg.rslt_dir`.
The loaded `cfg` is read-only by default (`cfg.is_frozen == True`). To modify, unfreeze `cfg` with `cfg.unfreeze()`.
### Summary of This Chapter
* The config file offers a `Config` object `cfg`, a nested dictionary with a tree structure, allowing read and write via the `.` operator.
* Accessing non-existent keys in `cfg` returns a one-time empty dictionary considered as `False`.
* Use `load_config` to load the config file. The experiment path will be auto created and assigned to `cfg.rslt_dir`.
## Inheritance
The new config can inherit the existing base config, written as `cfg = Config(caps='base_cfg.py')`. The new config only needs to override or add items, with rest items reusing the base config. For example, with [base config](alchemy_cat/dl_config/examples/configs/mnist/plain_usage/cfg.py):
```python
# -- [INCOMPLETE] configs/mnist/plain_usage/cfg.py --
# ... Code Omitted.
cfg.loader.ini.batch_size = 128
cfg.loader.ini.num_workers = 2
cfg.opt.cls = optim.AdamW
cfg.opt.ini.lr = 0.01
# ... Code Omitted.
```
To double the batch size, [new config](alchemy_cat/dl_config/examples/configs/mnist/plain_usage,2xbs/cfg.py) can be written as:
```python
# -- configs/mnist/plain_usage,2xbs/cfg.py --
from alchemy_cat.dl_config import Config
cfg = Config(caps='configs/mnist/plain_usage/cfg.py') # Inherit from base config.
cfg.loader.ini.batch_size = 128 * 2 # Double batch size.
cfg.opt.ini.lr = 0.01 * 2 # Linear scaling rule, see https://arxiv.org/abs/1706.02677
```
Inheritance behaves like `dict.update`. The key difference is that if both config have keys with the same name and their values are `Config` instance (naming config subtree), we recursively update within these subtrees. Thus, the new config can modify `cfg.loader.ini.batch_size` while inheriting `cfg.loader.ini.num_workers`.
```text
>>> base_cfg = load_config('configs/mnist/plain_usage/cfg.py', create_rslt_dir=False)
>>> new_cfg = load_config('configs/mnist/plain_usage,2xbs/cfg.py', create_rslt_dir=False)
>>> base_cfg.loader.ini
{'batch_size': 128, 'num_workers': 2}
>>> new_cfg.loader.ini
{'batch_size': 256, 'num_workers': 2}
```
To overwrite the entire config subtree in the new config, set this subtree to "override", [e.g.](alchemy_cat/dl_config/examples/configs/mnist/plain_usage,override_loader/cfg.py) :
```python
# -- configs/mnist/plain_usage,override_loader/cfg.py --
from alchemy_cat.dl_config import Config
cfg = Config(caps='configs/mnist/plain_usage/cfg.py') # Inherit from base config.
cfg.loader.ini.override() # Set subtree as whole.
cfg.loader.ini.shuffle = False
cfg.loader.ini.drop_last = False
```
`cfg.loader.ini` will now be solely defined by the new config:
```text
>>> base_cfg = load_config('configs/mnist/plain_usage/cfg.py', create_rslt_dir=False)
>>> new_cfg = load_config('configs/mnist/plain_usage,2xbs/cfg.py', create_rslt_dir=False)
>>> base_cfg.loader.ini
{'batch_size': 128, 'num_workers': 2}
>>> new_cfg.loader.ini
{'shuffle': False, 'drop_last': False}
```
Naturally, a base config can inherit from another base config, known as chain inheritance.
Multiple inheritance is also supported, written as `cfg = Config(caps=('base.py', 'patch1.py', 'patch2.py', ...))`, creating an inheritance chain of `base -> patch1 -> patch2 -> current cfg`. The base configs on the right are often used patches to batch add config items. For example, this [patch](alchemy_cat/dl_config/examples/configs/patches/cifar10.py) includes CIFAR10 dataset configurations:
```python
# -- configs/patches/cifar10.py --
import torchvision.transforms as T
from torchvision.datasets import CIFAR10
from alchemy_cat.dl_config import Config
cfg = Config()
cfg.dt.override()
cfg.dt.cls = CIFAR10
cfg.dt.ini.root = '/tmp/data'
cfg.dt.ini.transform = T.Compose([T.ToTensor(), T.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])
```
To switch to CIFAR10, [new config](alchemy_cat/dl_config/examples/configs/mnist/plain_usage,cifar10/cfg.py) only need to inherit the patch:
```python
# -- configs/mnist/plain_usage,cifar10/cfg.py --
from alchemy_cat.dl_config import Config
cfg = Config(caps=('configs/mnist/plain_usage/cfg.py', 'alchemy_cat/dl_config/examples/configs/patches/cifar10.py'))
```
```text
>>> cfg = load_config('configs/mnist/plain_usage,cifar10/cfg.py', create_rslt_dir=False)
>>> cfg.dt
{'cls': torchvision.datasets.cifar.CIFAR10,
'ini': {'root': '/tmp/data',
'transform': Compose(
ToTensor()
Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5))
)}}
```
> _Inheritance Implementation Details_
>
> We copy the base config tree and update it with the new config, ensuring isolation between them. This means changes to the new config do not affect the base. Complex inheritance like diamond inheritance is supported but not recommended due to readability issues. \
> Note that leaf node values are passed by reference; modifying them inplace will affect the entire inheritance chain.
### Summary of This Chapter
* The new config can leverage inheritance to reuse the base config and modifies or adds some items.
* The new config updates the base config recursively. Use `Config.override` to revert to the `dict.update` method for updates.
* `Config` supports chain and multiple inheritance, allowing for more fine-grained reuse.
## Dependency
In the [previous](#inheritance) example, changing the batch size in the new configuration also alters the learning rate. This interdependence is called "dependency."
When modifying a config item, it's common to forget its dependencies. AlchemyCat lets you define dependencies, changing the dependency source updates all dependent items automatically. [For example](alchemy_cat/dl_config/examples/configs/mnist/base/cfg.py):
```python
# -- [INCOMPLETE] configs/mnist/base/cfg.py --
from alchemy_cat.dl_config import Config, DEP
# ... Code Omitted.
cfg.loader.ini.batch_size = 128
# ... Code Omitted.
cfg.opt.ini.lr = DEP(lambda c: c.loader.ini.batch_size // 128 * 0.01) # Linear scaling rule.
# ... Code Omitted.
```
The learning rate `cfg.opt.ini.lr` is calculated as a dependency `DEP` using the batch size `cfg.loader.ini.batch_size`. `DEP` takes a function with `cfg` as an argument and returns the dependency value.
In the [new config](alchemy_cat/dl_config/examples/configs/mnist/base,2xbs/cfg.py), we only need to modify the batch size, and the learning rate will update automatically:
```python
# -- configs/mnist/base,2xbs/cfg.py --
from alchemy_cat.dl_config import Config
cfg = Config(caps='configs/mnist/base/cfg.py')
cfg.loader.ini.batch_size = 128 * 2 # Double batch size, learning rate will be doubled automatically.
```
```text
>>> cfg = load_config('configs/mnist/base,2xbs/cfg.py', create_rslt_dir=False)
>>> cfg.loader.ini.batch_size
256
>>> cfg.opt.ini.lr
0.02
```
Below is a more complex [example](alchemy_cat/dl_config/examples/configs/mnist/base/cfg.py):
```python
# -- configs/mnist/base/cfg.py --
# ... Code Omitted.
cfg.sched.epochs = 30
@cfg.sched.set_DEP(name='warm_epochs', priority=0) # kwarg `name` is not necessary
def warm_epochs(c: Config) -> int: # warm_epochs = 10% of total epochs
return round(0.1 * c.sched.epochs)
cfg.sched.warm.cls = sched.LinearLR
cfg.sched.warm.ini.total_iters = DEP(lambda c: c.sched.warm_epochs, priority=1)
cfg.sched.warm.ini.start_factor = 1e-5
cfg.sched.warm.ini.end_factor = 1.
cfg.sched.main.cls = sched.CosineAnnealingLR
cfg.sched.main.ini.T_max = DEP(lambda c: c.sched.epochs - c.sched.warm.ini.total_iters,
priority=2) # main_epochs = total_epochs - warm_epochs
# ... Code Omitted.
```
```text
>>> print(cfg.sched.to_txt(prefix='cfg.sched.')) # A pretty print of the config tree.
cfg.sched = Config()
# ------- β LEAVES β ------- #
cfg.sched.epochs = 30
cfg.sched.warm_epochs = 3
cfg.sched.warm.cls = <class 'torch.optim.lr_scheduler.LinearLR'>
cfg.sched.warm.ini.total_iters = 3
cfg.sched.warm.ini.start_factor = 1e-05
cfg.sched.warm.ini.end_factor = 1.0
cfg.sched.main.cls = <class 'torch.optim.lr_scheduler.CosineAnnealingLR'>
cfg.sched.main.ini.T_max = 27
```
In the code, `cfg.sched.epochs` determines total training epochs, which is also the dependency source. Warm-up epochs `cfg.sched.warm_epochs` are 10% of this total, and main epochs `cfg.sched.main.ini.T_max` is the remainder. Adjusting total training epochs updates both warm-up and main epochs automatically.
The dependency `cfg.sched.warm_epochs` is defined using the `Config.set_DEP` decorator. The decorated function, passed as the first parameter of `DEP`, computes the dependency. The key name of dependency can be specified via the keyword argument `name`; if omitted, it defaults to the function's name. For complex computations, using a decorator for definition is recommended.
When a dependency relies on another dependency, they must be computed in the correct order. By default, this is the defined order. The `priority` parameter can specify computation order: smaller `priority` compute earlier. For instance, `cfg.sched.warm_epochs` depended by `cfg.sched.warm.ini.total_iters`, which is depended by `cfg.sched.main.ini.T_max`, so their `priority` increase sequentially.
### Summary of This Chapter
* A dependency is defined when one config item relies on another. Changing the dependency source will automatically recalculate the dependency based on the calculation function.
* Dependencies can be defined by `DEP(...)` or the `Config.set_DEP` decorator.
* If dependencies are interdependent, use the `priority` parameter to specify the computation order; otherwise, they resolve in the order of definition.
## Composition
Composition allows reusing configs by compose predefined config subtrees to form a complete config. For instance, the following [config subtree](alchemy_cat/dl_config/examples/configs/addons/linear_warm_cos_sched.py) defines a learning rate strategy:
```python
# -- configs/addons/linear_warm_cos_sched.py --
import torch.optim.lr_scheduler as sched
from alchemy_cat.dl_config import Config, DEP
cfg = Config()
cfg.epochs = 30
@cfg.set_DEP(priority=0) # warm_epochs = 10% of total epochs
def warm_epochs(c: Config) -> int:
return round(0.1 * c.epochs)
cfg.warm.cls = sched.LinearLR
cfg.warm.ini.total_iters = DEP(lambda c: c.warm_epochs, priority=1)
cfg.warm.ini.start_factor = 1e-5
cfg.warm.ini.end_factor = 1.
cfg.main.cls = sched.CosineAnnealingLR
cfg.main.ini.T_max = DEP(lambda c: c.epochs - c.warm.ini.total_iters,
priority=2) # main_epochs = total_epochs - warm_epochs
```
In the [final config](alchemy_cat/dl_config/examples/configs/mnist/base,sched_from_addon/cfg.py), we compose this set of learning rate strategy:
```python
# -- configs/mnist/base,sched_from_addon/cfg.py --
# ... Code Omitted.
cfg.sched = Config('configs/addons/linear_warm_cos_sched.py')
# ... Code Omitted.
```
```text
>>> print(cfg.sched.to_txt(prefix='cfg.sched.')) # A pretty print of the config tree.
cfg.sched = Config()
# ------- β LEAVES β ------- #
cfg.sched.epochs = 30
cfg.sched.warm_epochs = 3
cfg.sched.warm.cls = <class 'torch.optim.lr_scheduler.LinearLR'>
cfg.sched.warm.ini.total_iters = 3
cfg.sched.warm.ini.start_factor = 1e-05
cfg.sched.warm.ini.end_factor = 1.0
cfg.sched.main.cls = <class 'torch.optim.lr_scheduler.CosineAnnealingLR'>
cfg.sched.main.ini.T_max = 27
```
It looks very simple! Just assign/mount the predefined config sub-subtree to the final config. `Config('path/to/cfg.py')` returns a copy of the `cfg` object in the config file, ensuring modifications before and after copying are isolated.
> _Implementation Details of Composition and Dependency_
>
> Attentive readers might wonder how `DEP` determines the parameter `c` for the dependency computation function, specifically which Config object is passed. In this chapter's example, `c` is the config subtree of learning rate; thus, the calculation function for `cfg.warm.ini.total_iters` is `lambda c: c.warm_epochs`. However, in the [previous chapter's](#dependency) example, `c` is the final config; hence, the calculation function for `cfg.sched.warm.ini.total_iters` is `lambda c: c.sched.warm_epochs`.
>
> In fact, `c` is the root node of the configuration tree where `DEP` was first mounted. The `Config` is a bidirectional tree. When `DEP` is first mounted, it records its relative distance to the root. During computation, it traces back this distance to find and pass the corresponding config tree into the computation function.
>
> To prevent this default behavior, set `DEP(lambda c: ..., rel=False)`, ensuring `c` is always the complete configuration.
**Best Practice: Both composition and inheritance aim to reuse config. Composition is more flexible and loosely coupled, so it should be prioritized over inheritance.**
### Summary of This Chapter
* Define config subtree and compose them to create a complete config.
## Full Example
<details>
<summary> Expand full example </summary>
[Config subtree](alchemy_cat/dl_config/examples/configs/addons/linear_warm_cos_sched.py) related to learning rate:
```python
# -- configs/addons/linear_warm_cos_sched.py --
import torch.optim.lr_scheduler as sched
from alchemy_cat.dl_config import Config, DEP
cfg = Config()
cfg.epochs = 30
@cfg.set_DEP(priority=0) # warm_epochs = 10% of total epochs
def warm_epochs(c: Config) -> int:
return round(0.1 * c.epochs)
cfg.warm.cls = sched.LinearLR
cfg.warm.ini.total_iters = DEP(lambda c: c.warm_epochs, priority=1)
cfg.warm.ini.start_factor = 1e-5
cfg.warm.ini.end_factor = 1.
cfg.main.cls = sched.CosineAnnealingLR
cfg.main.ini.T_max = DEP(lambda c: c.epochs - c.warm.ini.total_iters,
priority=2) # main_epochs = total_epochs - warm_epochs
```
The composed [base config](alchemy_cat/dl_config/examples/configs/mnist/base,sched_from_addon/cfg.py):
```python
# -- configs/mnist/base/cfg.py --
import torchvision.models as model
import torchvision.transforms as T
from torch import optim
from torchvision.datasets import MNIST
from alchemy_cat.dl_config import Config, DEP
cfg = Config()
cfg.rand_seed = 0
# -* Set datasets.
cfg.dt.cls = MNIST
cfg.dt.ini.root = '/tmp/data'
cfg.dt.ini.transform = T.Compose([T.Grayscale(3), T.ToTensor(), T.Normalize((0.1307,), (0.3081,)),])
# -* Set data loader.
cfg.loader.ini.batch_size = 128
cfg.loader.ini.num_workers = 2
# -* Set model.
cfg.model.cls = model.resnet18
cfg.model.ini.num_classes = DEP(lambda c: len(c.dt.cls.classes))
# -* Set optimizer.
cfg.opt.cls = optim.AdamW
cfg.opt.ini.lr = DEP(lambda c: c.loader.ini.batch_size // 128 * 0.01) # Linear scaling rule.
# -* Set scheduler.
cfg.sched = Config('configs/addons/linear_warm_cos_sched.py')
# -* Set logger.
cfg.log.save_interval = DEP(lambda c: c.sched.epochs // 5, priority=1) # Save model at every 20% of total epochs.
```
Inherited from the base config, batch size doubled, number of epochs halved [new config](alchemy_cat/dl_config/examples/configs/mnist/base,sched_from_addon,2xbs,2Γ·epo/cfg.py):
```python
# -- configs/mnist/base,sched_from_addon,2xbs,2Γ·epo/cfg.py --
from alchemy_cat.dl_config import Config
cfg = Config(caps='configs/mnist/base,sched_from_addon/cfg.py')
cfg.loader.ini.batch_size = 256
cfg.sched.epochs = 15
```
Note that dependencies such as learning rate, warm-up epochs, and main epochs will be automatically updated:
```text
>>> cfg = load_config('configs/mnist/base,sched_from_addon,2xbs,2Γ·epo/cfg.py', create_rslt_dir=False)
>>> print(cfg)
cfg = Config()
cfg.override(False).set_attribute('_cfgs_update_at_parser', ('configs/mnist/base,sched_from_addon/cfg.py',))
# ------- β LEAVES β ------- #
cfg.rand_seed = 0
cfg.dt.cls = <class 'torchvision.datasets.mnist.MNIST'>
cfg.dt.ini.root = '/tmp/data'
cfg.dt.ini.transform = Compose(
Grayscale(num_output_channels=3)
ToTensor()
Normalize(mean=(0.1307,), std=(0.3081,))
)
cfg.loader.ini.batch_size = 256
cfg.loader.ini.num_workers = 2
cfg.model.cls = <function resnet18 at 0x7f5bcda68a40>
cfg.model.ini.num_classes = 10
cfg.opt.cls = <class 'torch.optim.adamw.AdamW'>
cfg.opt.ini.lr = 0.02
cfg.sched.epochs = 15
cfg.sched.warm_epochs = 2
cfg.sched.warm.cls = <class 'torch.optim.lr_scheduler.LinearLR'>
cfg.sched.warm.ini.total_iters = 2
cfg.sched.warm.ini.start_factor = 1e-05
cfg.sched.warm.ini.end_factor = 1.0
cfg.sched.main.cls = <class 'torch.optim.lr_scheduler.CosineAnnealingLR'>
cfg.sched.main.ini.T_max = 13
cfg.log.save_interval = 3
cfg.rslt_dir = 'mnist/base,sched_from_addon,2xbs,2Γ·epo'
```
[Training code](alchemy_cat/dl_config/examples/train.py):
```python
# -- train.py --
import argparse
import json
import torch
import torch.nn.functional as F
from rich.progress import track
from torch.optim.lr_scheduler import SequentialLR
from alchemy_cat.dl_config import load_config
from utils import eval_model
parser = argparse.ArgumentParser(description='AlchemyCat MNIST Example')
parser.add_argument('-c', '--config', type=str, default='configs/mnist/base,sched_from_addon,2xbs,2Γ·epo/cfg.py')
args = parser.parse_args()
# Folder 'experiment/mnist/base' will be auto created by `load` and assigned to `cfg.rslt_dir`
cfg = load_config(args.config, experiments_root='/tmp/experiment', config_root='configs')
print(cfg)
torch.manual_seed(cfg.rand_seed) # Use `cfg` to set random seed
dataset = cfg.dt.cls(**cfg.dt.ini) # Use `cfg` to set dataset type and its initial parameters
# Use `cfg` to set changeable parameters of loader,
# other fixed parameter like `shuffle` is set in main code
loader = torch.utils.data.DataLoader(dataset, shuffle=True, **cfg.loader.ini)
model = cfg.model.cls(**cfg.model.ini).train().to('cuda') # Use `cfg` to set model
# Use `cfg` to set optimizer, and get `model.parameters()` in run time
opt = cfg.opt.cls(model.parameters(), **cfg.opt.ini, weight_decay=0.)
# Use `cfg` to set warm and main scheduler, and `SequentialLR` to combine them
warm_sched = cfg.sched.warm.cls(opt, **cfg.sched.warm.ini)
main_sched = cfg.sched.main.cls(opt, **cfg.sched.main.ini)
sched = SequentialLR(opt, [warm_sched, main_sched], [cfg.sched.warm_epochs])
for epoch in range(1, cfg.sched.epochs + 1): # train `cfg.sched.epochs` epochs
for data, target in track(loader, description=f"Epoch {epoch}/{cfg.sched.epochs}"):
F.cross_entropy(model(data.to('cuda')), target.to('cuda')).backward()
opt.step()
opt.zero_grad()
sched.step()
# If cfg.log is defined, save model to `cfg.rslt_dir` at every `cfg.log.save_interval`
if cfg.log and epoch % cfg.log.save_interval == 0:
torch.save(model.state_dict(), f"{cfg.rslt_dir}/model_{epoch}.pth")
eval_model(model)
if cfg.log:
eval_ret = eval_model(model)
with open(f"{cfg.rslt_dir}/eval.json", 'w') as json_f:
json.dump(eval_ret, json_f)
```
Run `python train.py --config 'configs/mnist/base,sched_from_addon,2xbs,2Γ·epo/cfg.py'`, and it will use the settings in the config file to train with `train.py` and save the results to the `/tmp/experiment/mnist/base,sched_from_addon,2xbs,2Γ·epo` directory.
</details>
## Auto Parameter Tuning
In the [example above](#full-example), running `python train.py --config path/to/cfg.py` each time yields an experimental result for a set of parameters.
However, we often need to perform grid search over the parameter space to find the optimal parameter combination. Writing a config for each combination is laborious and error-prone. Can we define the entire parameter space in a "tunable config"? Then let the program automatically traverse all combinations, generate configs, run them, and summarize results for comparison.
The auto-tuner traverses through tunable config's parameter combinations, generates `N` sub-configs, runs them to obtain `N` experimental records, and summarizes all experimental results into an Excel sheet:
```text
config to be tuned T βββ> config C1 + algorithm code A βββ> reproducible experiment E1(C1, A) βββ> summary table S(T,A)
βΒ βΒ
βββ> config C2 + algorithm code A βββ> reproducible experiment E1(C2, A) βββΒ
... ...
```
### Tunable Config
To use the auto-tuner, we first need to write a tunable config:
```python
# -- configs/tune/tune_bs_epoch/cfg.py --
from alchemy_cat.dl_config import Cfg2Tune, Param2Tune
cfg = Cfg2Tune(caps='configs/mnist/base,sched_from_addon/cfg.py')
cfg.loader.ini.batch_size = Param2Tune([128, 256, 512])
cfg.sched.epochs = Param2Tune([5, 15])
```
Its writing style is similar to the [normal configuration](alchemy_cat/dl_config/examples/configs/mnist/base,sched_from_addon,2xbs,2Γ·epo/cfg.py) in the previous chapter. It supports attribute reading and writing, inheritance, dependency, and combination. The difference lies in:
* The type of config is `Cfg2Tune`, a subclass of `Config`.
* For grid search parameters, use `Param2Tune([v1, v2, ...])` with optional values `v1, v2, ...`.
The tunable config above will search a parameter space of size 3Γ2=6 and generate these 6 sub-configs:
```text
batch_size epochs child_configs
128 5 configs/tune/tune_bs_epoch/batch_size=128,epochs=5/cfg.pkl
15 configs/tune/tune_bs_epoch/batch_size=128,epochs=15/cfg.pkl
256 5 configs/tune/tune_bs_epoch/batch_size=256,epochs=5/cfg.pkl
15 configs/tune/tune_bs_epoch/batch_size=256,epochs=15/cfg.pkl
512 5 configs/tune/tune_bs_epoch/batch_size=512,epochs=5/cfg.pkl
15 configs/tune/tune_bs_epoch/batch_size=512,epochs=15/cfg.pkl
```
Set the `priority` parameter of `Param2Tune` to specify the search order. The default is the defined order. Use `optional_value_names` to assign readable names to parameter values. [For example](alchemy_cat/dl_config/examples/configs/tune/tune_bs_epoch,pri,name/cfg.py):
```python
# -- configs/tune/tune_bs_epoch,pri,name/cfg.py --
from alchemy_cat.dl_config import Cfg2Tune, Param2Tune
cfg = Cfg2Tune(caps='configs/mnist/base,sched_from_addon/cfg.py')
cfg.loader.ini.batch_size = Param2Tune([128, 256, 512], optional_value_names=['1xbs', '2xbs', '4xbs'], priority=1)
cfg.sched.epochs = Param2Tune([5, 15], priority=0)
```
whose search space is:
```text
epochs batch_size child_configs
5 1xbs configs/tune/tune_bs_epoch,pri,name/epochs=5,batch_size=1xbs/cfg.pkl
2xbs configs/tune/tune_bs_epoch,pri,name/epochs=5,batch_size=2xbs/cfg.pkl
4xbs configs/tune/tune_bs_epoch,pri,name/epochs=5,batch_size=4xbs/cfg.pkl
15 1xbs configs/tune/tune_bs_epoch,pri,name/epochs=15,batch_size=1xbs/cfg.pkl
2xbs configs/tune/tune_bs_epoch,pri,name/epochs=15,batch_size=2xbs/cfg.pkl
4xbs configs/tune/tune_bs_epoch,pri,name/epochs=15,batch_size=4xbs/cfg.pk
```
We can set constraints between parameters to eliminate unnecessary combinations. For example, the following [example](alchemy_cat/dl_config/examples/configs/tune/tune_bs_epoch,subject_to/cfg.py) limits total iterations to a maximum of 15Γ128:
```python
# -- configs/tune/tune_bs_epoch,subject_to/cfg.py --
from alchemy_cat.dl_config import Cfg2Tune, Param2Tune
cfg = Cfg2Tune(caps='configs/mnist/base,sched_from_addon/cfg.py')
cfg.loader.ini.batch_size = Param2Tune([128, 256, 512])
cfg.sched.epochs = Param2Tune([5, 15],
subject_to=lambda cur_val: cur_val * cfg.loader.ini.batch_size.cur_val <= 15 * 128)
```
whose search space is:
```text
batch_size epochs child_configs
128 5 configs/tune/tune_bs_epoch,subject_to/batch_size=128,epochs=5/cfg.pkl
15 configs/tune/tune_bs_epoch,subject_to/batch_size=128,epochs=15/cfg.pkl
256 5 configs/tune/tune_bs_epoch,subject_to/batch_size=256,epochs=5/cfg.pkl
```
### Running auto-tuner
We also need to write a small script to run the auto-tuner:
```python
# -- tune_train.py --
import argparse, json, os, subprocess, sys
from alchemy_cat.dl_config import Config, Cfg2TuneRunner
parser = argparse.ArgumentParser(description='Tuning AlchemyCat MNIST Example')
parser.add_argument('-c', '--cfg2tune', type=str)
args = parser.parse_args()
# Will run `torch.cuda.device_count() // work_gpu_num` of configs in parallel
runner = Cfg2TuneRunner(args.cfg2tune, experiment_root='/tmp/experiment', work_gpu_num=1)
@runner.register_work_fn # How to run config
def work(pkl_idx: int, cfg: Config, cfg_pkl: str, cfg_rslt_dir: str, cuda_env: dict[str, str]) -> ...:
subprocess.run([sys.executable, 'train.py', '-c', cfg_pkl], env=cuda_env)
@runner.register_gather_metric_fn # How to gather metric for summary
def gather_metric(cfg: Config, cfg_rslt_dir: str, run_rslt: ..., param_comb: dict[str, tuple[..., str]]) -> dict[str, ...]:
return json.load(open(os.path.join(cfg_rslt_dir, 'eval.json')))
runner.tuning()
```
The script performs these operations:
* Instantiates the auto-tuner with `runner = Cfg2TuneRunner(...)`, passing in the tunable config path. By default, it runs sub-configs sequentially. Set the parameter `work_gpu_num` to run `len(os.environ['CUDA_VISIBLE_DEVICES']) // work_gpu_num` sub-configs in parallel.
* Registers a worker that executes each sub-config. The function parameters are:
- `pkl_idx`: index of the sub-config
- `cfg`: the sub-config
- `cfg_pkl`: pickle save path for this sub-config
- `cfg_rslt_dir`: experiment directory.
- `cuda_env`: If `work_gpu_num` is set, then `cuda_env` will allocate non-overlapping `CUDA_VISIBLE_DEVICES` environment variables for parallel sub-configs.
Commonly, we only need to pass `cfg_pkl` as the config file into the training script, since `load_cfg` supports reading config in pickle format. For deep learning tasks, different `CUDA_VISIBLE_DEVICES` are needed for each sub-config.
* Registers a summary function that returns an experimental result as a `{metric_name: metric_value}` dictionary. The auto-tunner will traverse all experimental results and summary into a table. The summary function accepts these parameters:
- `cfg`: the sub-configuration
- `cfg_rslt_dir`: experiment directory
- `run_rslt`: returned from working functions
- `param_comb`: parameter combinations for that particular sub-configuration.
Generally, only need to read results from `cfg_rslt_dir` and return them.
* Calls `runner.tuning()` to start automatic tuning.
After tuning, the tuning results will be printed:
```text
Metric Frame:
test_loss acc
batch_size epochs
128 5 1.993285 32.63
15 0.016772 99.48
256 5 1.889874 37.11
15 0.020811 99.49
512 5 1.790593 41.74
15 0.024695 99.33
Saving Metric Frame at /tmp/experiment/tune/tune_bs_epoch/metric_frame.xlsx
```
As the prompt says, the tuning results will also be saved to the `/tmp/experiment/tune/tune_bs_epoch/metric_frame.xlsx` table:
<div align = "center">
<img src="https://github.com/HAL-42/AlchemyCat/raw/master/docs/figs/readme-cfg2tune-excel.png" width="400" />
</div>
> [!TIP]
> **Best Practice: The auto-tuner is separate from the standard workflow. Write configs and code without considering it. When tuning, add extra code to define parameter space, specify invocation and result methods. After tuning, remove the auto-tuner, keeping only the best config and algorithm.**
### Summary of This Chapter
* Define a tunable config `Cfg2Tune` with `Param2Tune` to specify the parameter space.
* Use the auto-tuner `Cfg2TuneRunner` to traverse the parameter space, generate sub-configs, run them, and summarize the results.
## Advanced Usage
<details>
<summary> Expand advanced usage </summary>
### Pretty Print
The `__str__` method of `Config` is overloaded to print the tree structure with keys separated by `.`:
```text
>>> cfg = Config()
>>> cfg.foo.bar.a = 1
>>> cfg.bar.foo.b = ['str1', 'str2']
>>> cfg.whole.override()
>>> print(cfg)
cfg = Config()
cfg.whole.override(True)
# ------- β LEAVES β ------- #
cfg.foo.bar.a = 1
cfg.bar.foo.b = ['str1', 'str2']
```
When all leaf nodes are built-in types, the pretty print output of `Config` can be executed as Python code to get the same configuration:
```text
>>> exec(cfg.to_txt(prefix='new_cfg.'), globals(), (l_dict := {}))
>>> l_dict['new_cfg'] == cfg
True
```
For invalid attribute names, `Config` will fall back to the print format of `dict`:
```text
>>> cfg = Config()
>>> cfg['Invalid Attribute Name'].foo = 10
>>> cfg.bar['def'] = {'a': 1, 'b': 2}
>>> print(cfg)
cfg = Config()
# ------- β LEAVES β ------- #
cfg['Invalid Attribute Name'].foo = 10
cfg.bar['def'] = {'a': 1, 'b': 2}
```
### Auto Capture Experiment Logs
For deep learning tasks, we recommend using `init_env` instead of `load_config`. In addition to loading the config, `init_env` can also initialize the deep learning environment, such as setting the torch device, gradient, random seed, and distributed training:
```python
from alchemy_cat.torch_tools import init_env
if __name__ == '__main__':
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('-c', '--config', type=str)
parser.add_argument('--local_rank', type=int, default=-1)
args = parser.parse_args()
device, cfg = init_env(config_path=args.config, # config file pathοΌread to `cfg`
is_cuda=True, # if TrueοΌ`device` is cudaοΌelse cpu
is_benchmark=bool(args.benchmark), # torch.backends.cudnn.benchmark = is_benchmark
is_train=True, # torch.set_grad_enabled(is_train)
experiments_root="experiment", # root of experiment dir
rand_seed=True, # set python, numpy, torch rand seed. If True, read cfg.rand_seed as seed, else use actual parameter as rand seed.
cv2_num_threads=0, # set cv2 num threads
verbosity=True, # print more env init info
log_stdout=True, # where fork stdout to log file
loguru_ini=True, # config a pretty loguru format
reproducibility=False, # set pytorch to reproducible mode
local_rank=..., # dist.init_process_group(..., local_rank=local_rank)
silence_non_master_rank=True, # if True, non-master rank will not print to stdout, but only log to file
is_debug=bool(args.is_debug)) # is debug mode
```
If `log_stdout=True`, `init_env` will fork `sys.stdout` and `sys.stderr` to the log file `cfg.rslt_dir/{local-time}.log`. This will not interfere with normal `print`, but all screen output will be recorded in the log. Therefore, there is no need to manually write logs, what you see on the screen is what you get in the log.
Details can be found in the docstring of `init_env`.
### Attribute Dict
If you are a user of [addict](https://github.com/mewwts/addict), our `ADict` can be used as a drop-in replacement for `addict.Dict`: `from alchemy_cat.dl_config import ADict as Dict`.
`ADict` has all the interfaces of `addict.Dict`. However, all methods are re-implemented to optimize execution efficiency and cover more corner cases (such as circular references). `Config` is actually a subclass of `ADict`.
If you haven't used `addict` before, read this [documentation](https://github.com/mewwts/addict). Research code often involves complex dictionaries. `addict.Dict` or `ADict` supports attribute-style access for nested dictionaries.
### Circular References
The initialization, inheritance, and composition of `ADict` and `Config` require a `branch_copy` operation, which is between shallow and deep copy, that is, copying the tree structure but not the leaf nodes. `ADict.copy`, `Config.copy`, and `copy.copy(cfg)` all call `branch_copy`, not the `copy` method of `dict`.
In theory, `ADict.branch_copy` can handle circular references, such as:
```text
>>> dic = {'num': 0,
'lst': [1, 'str'],
'sub_dic': {'sub_num': 3}}
>>> dic['lst'].append(dic['sub_dic'])
>>> dic['sub_dic']['parent'] = dic
>>> dic
{'num': 0,
'lst': [1, 'str', {'sub_num': 3, 'parent': {...}}],
'sub_dic': {'sub_num': 3, 'parent': {...}}}
>>> adic = ADict(dic)
>>> adic.sub_dic.parent is adic is not dic
True
>>> adic.lst[-1] is adic.sub_dic is not dic['sub_dic']
True
```
Different from `ADict`, the data model of `Config` is a bidirectional tree, and circular references will form a cycle. To avoid cycles, if a subtree is mounted to different parent configs multiple times, the subtree will be copied to an independent config tree before mounting. In normal use, circular references should not appear in the config tree.
In summary, although circular references are supported, they are neither necessary nor recommended.
### Traverse the Config Tree
`Config.named_branchs` and `Config.named_ckl` respectively traverse all branches and leaves of the config tree (the branch, key name, and value they are in):
```text
>>> list(cfg.named_branches)
[('', {'foo': {'bar': {'a': 1}},
'bar': {'foo': {'b': ['str1', 'str2']}},
'whole': {}}),
('foo', {'bar': {'a': 1}}),
('foo.bar', {'a': 1}),
('bar', {'foo': {'b': ['str1', 'str2']}}),
('bar.foo', {'b': ['str1', 'str2']}),
('whole', {})]
>>> list(cfg.ckl)
[({'a': 1}, 'a', 1), ({'b': ['str1', 'str2']}, 'b', ['str1', 'str2'])]
```
### Lazy Inheritance
```text
>>> from alchemy_cat.dl_config import Config
>>> cfg = Config(caps='configs/mnist/base,sched_from_addon/cfg.py')
>>> cfg.loader.ini.batch_size = 256
>>> cfg.sched.epochs = 15
>>> print(cfg)
cfg = Config()
cfg.override(False).set_attribute('_cfgs_update_at_parser', ('configs/mnist/base,sched_from_addon/cfg.py',))
# ------- β LEAVES β ------- #
cfg.loader.ini.batch_size = 256
cfg.sched.epochs = 15
```
When inheriting, the parent configs `caps` is not immediately updated, but is loaded when `load_config` is called. Lazy inheritance allows the config system to have an eager-view of the entire inheritance chain, and a few features rely on this.
### Work with Git
For `config C + algorithm code A ββ> reproducible experiment E(C, A)`, meaning that when the config `C` and the algorithm code `A` are determined, the experiment `E` can always be reproduced. Therefore, it is recommended to submit the configuration file and algorithm code to the Git repository together for reproducibility.
We also provide a [script](alchemy_cat/torch_tools/scripts/tag_exps.py) that runs `pyhon -m alchemy_cat.torch_tools.scripts.tag_exps -s commit_ID -a commit_ID`, interactively lists the new configs added by the commit, and tags the commit according to the config path. This helps quickly trace back the config and algorithm of a historical experiment.
### Allocate GPU for Child Processes Manually
The `work` function receives the idle GPU automatically allocated by `Cfg2TuneRunner` through the `cuda_env` parameter. We can further control the definition of 'idle GPU':
```python
runner = Cfg2TuneRunner(args.cfg2tune, experiment_root='/tmp/experiment', work_gpu_num=1,
block=True, # Try to allocate idle GPU
memory_need=10 * 1024, # Need 10 GB memory
max_process=2) # Max 2 process already ran on each GPU
```
where:
- `block`: Defaults is `True`. If set to `False`, GPUs are allocated sequentially, regardless of whether they are idle.
- `memory_need`: The amount of GPU memory required for each sub-config, in MB. The free memory on an idle GPU must be β₯ `memory_need`. Default is `-1.`, indicating need all memory.
- `max_process`: Maximum number of existing processes. The number of existing processes on an idle GPU must be β€ `max_process`. Default value is `-1`, indicating no limit.
### Pickling Lambda Functions
Sub-configs generated by `Cfg2Tune` will be saved using pickle. However, if `Cfg2Tune` defines dependencies as `DEP(lambda c: ...)`, these lambda functions cannot be pickled. Workarounds include:
* Using the decorator `@Config.set_DEP` to define the dependency's computation function.
* Defining the dependency's calculation function in a separate module and passing it to `DEP`.
* Defining dependencies in the parent configs since inheritance is handled lazily, so sub-configs temporarily exclude dependencies.
* If the dependency source is a tunable parameter, use `P_DEP`, which resolves after generating sub-configs of `Cfg2Tune` but before saving them as pickle.
### More Inheritance Tricks
#### Deleting During Inheritance
The `Config.empty_leaf()` combines `Config.clear()` and `Config.override()` to get an empty and "override" subtree. This is commonly used to represent the "delete" semantics during inheritance, that is, using an empty config to override a subtree of the base config.
#### `update` Method
Let `cfg` be a `Config` instance and `base_cfg` be a `dict` instance. The effects of `cfg.dict_update(base_cfg)`, `cfg.update(base_cfg)`, and `cfg |= base_cfg` are similar to inheriting `Config(base_cfg)` from `cfg`.
Run `cfg.dict_update(base_cfg, incremental=True)` to ensure only incremental updates, that is, only add keys that do not exist in `cfg` without overwriting existing keys.
</details>
Raw data
{
"_id": null,
"home_page": null,
"name": "alchemy-cat",
"maintainer": null,
"docs_url": null,
"requires_python": ">=3.9",
"maintainer_email": null,
"keywords": "config, deep learning, parameter tuning, hyperparameter",
"author": null,
"author_email": "HAL-42 <hal_42@zju.edu.cn>",
"download_url": "https://files.pythonhosted.org/packages/3d/e2/a1338fcca70b4caf5b82bfd2f75c2d5328f7351f367283a56a6d1912543f/alchemy_cat-0.0.8.tar.gz",
"platform": null,
"description": "# Alchemy Cat \u2014\u2014 \ud83d\udd25Config System for SOTA\n\n<div align=\"center\">\n\n\n<img src=\"https://img.shields.io/github/stars/HAL-42/AlchemyCat?color=yellow\" alt=\"Stars\">\n<img src=\"https://img.shields.io/github/issues/HAL-42/AlchemyCat?color=red\" alt=\"Issues\">\n\n<br>\n[](https://badge.fury.io/py/alchemy-cat)\n\n<img src=\"https://img.shields.io/badge/python-3.9-purple.svg\" alt=\"Python\"> <br>\n\n</div>\n\n<div align=\"center\">\n<a href=\"README.md\">English</a> | <a href=\"README_CN.md\">\u4e2d\u6587</a>\n</div>\n<br>\n\n\n\n<div align=\"center\">\n\n[\ud83d\ude80Introduction](https://github.com/HAL-42/AlchemyCat/blob/master/README.md#-introduction) | [\ud83d\udce6Installation](https://github.com/HAL-42/AlchemyCat/blob/master/README.md#-installation) | [\ud83d\ude9aMigration](https://github.com/HAL-42/AlchemyCat/blob/master/README.md#-migration) | [\ud83d\udcd6Documentation](https://github.com/HAL-42/AlchemyCat/blob/master/README.md#-documentation-)\n\n</div>\n\n# <div align=\"center\">\ud83d\ude80 Introduction</div>\n\nWhen developing machine learning algorithms, we often bother with:\n* Config files (YAML / YACS / MMCV) are lengthy and complex. If entries are interdependent, extra caution is needed to avoid errors when modifying them.\n* Parameter tuning requires rewriting the config for each parameter set, leading to code duplication and difficulty in tracking changes.\n* Manually traversing the parameter space and summarizing results during parameter tuning is time-consuming and inefficient.\n* Insufficient parameter tuning can obscure effective designs.\n* Effective methods may not achieve SOTA due to insufficient parameter tuning, reducing persuasiveness.\n\nAlchemyCat is a config system designed for machine learning research to address such issues. It helps researchers to fully explore the parameter tuning potential by simplifying repetitive tasks like reproduction, modifying configs, and hyperparameter tuning\n\nThe table below compares AlchemyCat with existing config systems (\ud83d\ude21 not support, \ud83e\udd14 limited support, \ud83e\udd73 supported):\n\n| Feature | argparse | yaml | YACS | mmcv | AlchemyCat |\n|----------------------------|----------|------|------|------|------------|\n| Reproducible | \ud83d\ude21 | \ud83e\udd73 | \ud83e\udd73 | \ud83e\udd73 | \ud83e\udd73 |\n| IDE Jump | \ud83d\ude21 | \ud83d\ude21 | \ud83e\udd73 | \ud83e\udd73 | \ud83e\udd73 |\n| Inheritance | \ud83d\ude21 | \ud83d\ude21 | \ud83e\udd14 | \ud83e\udd14 | \ud83e\udd73 |\n| Composition | \ud83d\ude21 | \ud83d\ude21 | \ud83e\udd14 | \ud83e\udd14 | \ud83e\udd73 |\n| dependency | \ud83d\ude21 | \ud83d\ude21 | \ud83d\ude21 | \ud83d\ude21 | \ud83e\udd73 |\n| Automatic Parameter Tuning | \ud83d\ude21 | \ud83d\ude21 | \ud83d\ude21 | \ud83d\ude21 | \ud83e\udd73 |\n\nAlchemyCat implements all features of current popular config systems, while fully considering various special cases, ensuring stability. AlchemyCat distinguishes itself by:\n* Readable: The syntax is simple, elegant, and Pythonic.\n* Reusable: Supports **inheritance** and **composition** of configs, reducing redundancy and enhancing reusability.\n* Maintainable: Allows for establishing **dependency** between config items, enabling global synchronization with a single change.\n* Supports auto parameter tuning and result summarization without needing to modify original configs or training codes.\n\n[Migrate](https://github.com/HAL-42/AlchemyCat/blob/master/README.md#-migration) from config systems listed above to AlchemyCat is effortless. Just spend 15 minutes reading the [documentation](https://github.com/HAL-42/AlchemyCat/blob/master/README.md#-documentation-) and apply AlchemyCat to your project, and your GPU will never be idle again!\n\n## Quick Glance\nDeep learning relies on numerous empirical hyperparameters, such as learning rate, loss weights, max iterations, sliding window size, drop probability, thresholds, and even random seeds. \n\nThe relationship between hyperparameters and performance is hard to predict theoretically. The only certainty is that arbitrarily chosen hyperparameters are unlikely to be optimal. Practice has shown that grid search through the hyperparameter space can significantly enhance model performance; sometimes its effect even surpasses so-called \"contributions.\" Achieving SOTA often depends on this!\n\nAlchemyCat offers an auto parameter-tuner that seamlessly integrates with existing config systems to explore the hyperparameter space and summarize experiment results automatically. Using this tool requires no modifications to the original config or training code.\n\nFor example, with [MMSeg](https://github.com/open-mmlab/mmsegmentation) users only need to write a tunable config inherited from MMSeg's base config and define the parameter search space:\n```python\n# -- configs/deeplabv3plus/tune_bs,iter/cfg.py --\nfrom alchemy_cat import Cfg2Tune, Param2Tune\n\n# Inherit from standard mmcv config.\ncfg = Cfg2Tune(caps='configs/deeplabv3plus/deeplabv3plus_r50-d8_4xb4-40k_voc12aug-512x512.py')\n\n# Inherit and override\ncfg.model.auxiliary_head.loss_decode.loss_weight = 0.2\n\n# Tuning parameters: grid search batch_size and max_iters\ncfg.train_dataloader.batch_size = Param2Tune([4, 8])\ncfg.train_cfg.max_iters = Param2Tune([20_000, 40_000])\n# ... \n```\nNext, write a script specifying how to run a single config and read its results:\n```python\n# -- tools/tune_dist_train.py --\nimport argparse, subprocess\nfrom alchemy_cat.dl_config import Cfg2TuneRunner, Config\nfrom alchemy_cat.dl_config.examples.read_mmcv_metric import get_metric\n\nparser = argparse.ArgumentParser()\nparser.add_argument('--cfg2tune', type=str) # Path to the tunable config\nparser.add_argument('--num_gpu', type=int, default=2) # Number of GPUs for each task\nargs = parser.parse_args()\n\nrunner = Cfg2TuneRunner(args.cfg2tune, experiment_root='work_dirs', work_gpu_num=args.num_gpu)\n\n@runner.register_work_fn # Run experiment for each param combination with mmcv official train script\ndef work(pkl_idx: int, cfg: Config, cfg_pkl: str, cfg_rslt_dir: str, cuda_env: dict[str, str]):\n mmcv_cfg = cfg.save_mmcv(cfg_rslt_dir + '/mmcv_config.py')\n subprocess.run(f'./tools/dist_train.sh {mmcv_cfg} {args.num_gpu}', env=cuda_env, shell=True)\n\n@runner.register_gather_metric_fn # Optional, gather metric of each config\ndef gather_metric(cfg: Config, cfg_rslt_dir: str, run_rslt, param_comb) -> dict[str, float]:\n return get_metric(cfg_rslt_dir) # {'aAcc': xxx, 'mIoU': xxx, 'mAcc': xxx}\n\nrunner.tuning()\n```\nRun `CUDA_VISIBLE_DEVICES=0,1,2,3 python tools/tune_dist_train.py --cfg2tune configs/deeplabv3plus/tune_bs,iter/cfg.py`, which will automatically search the parameter space in parallel and summarize the experiment results as follows:\n\n<div align = \"center\">\n<img src=\"https://raw.githubusercontent.com/HAL-42/AlchemyCat/master/docs/figs/readme-teaser-excel.png\" width=\"500\" />\n</div>\n\nIn fact, the above config is still incomplete for some hyperparameters are interdependent and need to be adjusted together. For instance, the learning rate should scale with the batch size. AlchemyCat uses **dependency** to manage these relationships; when a dependency source changes, related dependencies automatically update for consistency. The complete config with dependencies is:\n```python\n# -- configs/deeplabv3plus/tune_bs,iter/cfg.py --\nfrom alchemy_cat import Cfg2Tune, Param2Tune, P_DEP\n\n# Inherit from standard mmcv config.\ncfg = Cfg2Tune(caps='configs/deeplabv3plus/deeplabv3plus_r50-d8_4xb4-40k_voc12aug-512x512.py')\n\n# Inherit and override\ncfg.model.auxiliary_head.loss_decode.loss_weight = 0.2\n\n# Tuning parameters: grid search batch_size and max_iters\ncfg.train_dataloader.batch_size = Param2Tune([4, 8])\ncfg.train_cfg.max_iters = Param2Tune([20_000, 40_000])\n\n# Dependencies:\n# 1) learning rate increase with batch_size\ncfg.optim_wrapper.optimizer.lr = P_DEP(lambda c: (c.train_dataloader.batch_size / 8) * 0.01)\n\n# 2) end of param_scheduler increase with max_iters\n@cfg.set_DEP()\ndef param_scheduler(c):\n return dict(\n type='PolyLR',\n eta_min=1e-4,\n power=0.9,\n begin=0,\n end=c.train_cfg.max_iters,\n by_epoch=False)\n```\n> [!NOTE]\n> In the example above, defining dependencies might seem needless since they can be computed directly. However, when combined with **inheritance**, setting dependencies in the base config allows tunable configs to focus on key hyperparameters without worrying about trivial dependency details. Refer to the [documentation](#dependency) for details.\n\n# <div align=\"center\">\ud83d\udce6 Installation</div>\n```bash\npip install alchemy-cat\n```\n\n# <div align=\"center\">\ud83d\ude9a Migration</div>\n<details>\n<summary> How to migrate from YAML / YACS / MMCV </summary>\n\n\u03c3`\u2200\u00b4)\u03c3 Just kidding! No migration is needed. AlchemyCat can direct read and write YAML / YACS / MMCV config files:\n\n```python\nfrom alchemy_cat.dl_config import load_config, Config\n\n# READ YAML / YACS / MMCV config to alchemy_cat.Config\ncfg = load_config('path/to/yaml_config.yaml or yacs_config.py or mmcv_config.py')\n# Init alchemy_cat.Config with YAML / YACS / MMCV config\ncfg = Config('path/to/yaml_config.yaml or yacs_config.py or mmcv_config.py')\n# alchemy_cat.Config inherits from YAML / YACS / MMCV config\ncfg = Config(caps='path/to/yaml_config.yaml or yacs_config.py or mmcv_config.py')\n\nprint(cfg.model.backbone) # Access config item\n\ncfg.save_yaml('path/to/save.yaml') # Save to YAML config\ncfg.save_mmcv('path/to/save.py') # Save to MMCV config\ncfg.save_py('path/to/save.py') # Save to AlchemyCat config\n```\nWe also provide a script to convert between different config formats:\n```bash\npython -m alchemy_cat.dl_config.from_x_to_y --x X --y Y --y_type=yaml/mmcv/alchemy-cat\n```\nwhere: \n* `--x`: Source config file path, can be YAML / YACS / MMCV / AlchemyCat config.\n* `--y`: Target config file path.\n* `--y_type`: Target config format, can be `yaml`, `mmcv`, or `alchemy-cat`.\n\n</details>\n\n# <div align=\"center\">\ud83d\udcd6 Documentation </div>\n\n## Basic Usage\nAlchemyCat ensures a one-to-one correspondence between each configuration and its unique experimental record, with the bijective relationship ensuring the experiment's reproducibility.\n```text\nconfig C + algorithm code A \u2014\u2014> reproducible experiment E(C, A)\n```\nThe experimental directory is automatically generated, mirroring the relative path of the configuration file. This path can include multi-level directories and special characters such as spaces, commas, and equal signs. Such flexibility aids in categorizing experiments for clear management. For instance:\n```text\n.\n\u251c\u2500\u2500 configs\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 MNIST\n\u2502\u00a0\u00a0 \u2502\u00a0\u00a0 \u251c\u2500\u2500 resnet18,wd=1e-5@run2\n\u2502\u00a0\u00a0 \u2502\u00a0\u00a0 \u2502\u00a0\u00a0 \u2514\u2500\u2500 cfg.py\n\u2502\u00a0\u00a0 \u2502\u00a0\u00a0 \u2514\u2500\u2500 vgg,lr=1e-2\n\u2502\u00a0\u00a0 \u2502\u00a0\u00a0 \u2514\u2500\u2500 cfg.py\n\u2502\u00a0\u00a0 \u2514\u2500\u2500 VOC2012\n\u2502\u00a0\u00a0 \u2514\u2500\u2500 swin-T,\u03b3=10\n\u2502\u00a0\u00a0 \u2514\u2500\u2500 10 epoch\n\u2502\u00a0\u00a0 \u2514\u2500\u2500 cfg.py\n\u2514\u2500\u2500 experiment\n \u251c\u2500\u2500 MNIST\n \u2502\u00a0\u00a0 \u251c\u2500\u2500 resnet18,wd=1e-5@run2\n \u2502\u00a0\u00a0 \u2502\u00a0\u00a0 \u2514\u2500\u2500 xxx.log\n \u2502\u00a0\u00a0 \u2514\u2500\u2500 vgg,lr=1e-2\n \u2502\u00a0\u00a0 \u2514\u2500\u2500 xxx.log\n \u2514\u2500\u2500 VOC2012\n \u2514\u2500\u2500 swin-T,\u03b3=10\n \u2514\u2500\u2500 10 epoch\n \u2514\u2500\u2500 xxx.log\n```\n> [!TIP]\n> **Best Practice: Create a `__init__.py` next to `cfg.py`(usually will be auto created by IDE), and avoid paths containing '.'. This can help IDE to debug and allow relative import in `cfg.py`.**\n\n\nLet's begin with an incomplete example to demonstrate writing and loading a config. First, create the [config file](alchemy_cat/dl_config/examples/configs/mnist/plain_usage/cfg.py):\n```python\n# -- [INCOMPLETE] configs/mnist/plain_usage/cfg.py --\n\nfrom torchvision.datasets import MNIST\nfrom alchemy_cat.dl_config import Config\n\ncfg = Config()\n\ncfg.rand_seed = 0\n\ncfg.dt.cls = MNIST\ncfg.dt.ini.root = '/tmp/data'\ncfg.dt.ini.train = True\n\n# ... Code Omitted.\n```\nHere, we first instantiate a `Config` object `cfg`, and then add config items through attribute operator `.`. Config items can be any Python objects, including functions, methods, and classes.\n\n> [!TIP]\n> **Best Practice: We prefer specifying functions or classes directly in config over using strings/semaphores to control the program behavior. This enables IDE navigation, simplifying reading and debugging.**\n\n`Config` is a subclass of Python's `dict`. The above code defines a nested dictionary with a **tree structure**:\n```text\n>>> print(cfg.to_dict())\n{'rand_seed': 0,\n 'dt': {'cls': <class 'torchvision.datasets.mnist.MNIST'>,\n 'ini': {'root': '/tmp/data', 'train': True}}}\n```\n`Config` implements all API of Python `dict`:\n```test\n>>> cfg.keys()\ndict_keys(['rand_seed', 'dt'])\n\n>>> cfg['dt']['ini']['root']\n'/tmp/data'\n\n>>> {**cfg['dt']['ini'], 'download': True}\n{'root': '/tmp/data', 'train': True, 'download': True}\n```\n\nYou can initialize a `Config` object using `dict` (yaml, json) or its subclasses (YACS, mmcv.Config).\n```text\n>>> Config({'rand_seed': 0, 'dt': {'cls': MNIST, 'ini': {'root': '/tmp/data', 'train': True}}})\n{'rand_seed': 0, 'dt': {'cls': <class 'torchvision.datasets.mnist.MNIST'>, 'ini': {'root': '/tmp/data', 'train': True}}}\n```\n\nUsing operator `.` to read and write `cfg` will be clearer. For instance, the following code creates and initializes the `MNIST` dataset based on the config:\n```text\n>>> dataset = cfg.dt.cls(**cfg.dt.ini)\n>>> dataset\nDataset MNIST\n Number of datapoints: 60000\n Root location: /tmp/data\n Split: Train\n```\nAccessing a non-existent key returns an empty dictionary, which should be treated as `False`:\n```text\n>>> cfg.not_exit\n{}\n```\n\nIn the [main code](alchemy_cat/dl_config/examples/train.py), use the following code to load the config:\n```python\n# # [INCOMPLETE] -- train.py --\n\nfrom alchemy_cat.dl_config import load_config\ncfg = load_config('configs/mnist/base/cfg.py', experiments_root='/tmp/experiment', config_root='configs')\n# ... Code Omitted.\ntorch.save(model.state_dict(), f\"{cfg.rslt_dir}/model_{epoch}.pth\") # Save all experiment results to cfg.rslt_dir.\n```\n\nThe `load_config` imports `cfg` from `configs/mnist/base/cfg.py`, handling inheritance and dependencies. Given the experiment root directory `experiments_root` and config root directory `config_root`, it auto creates an experiment directory at `experiment/mnist/base` and assign it to `cfg.rslt_dir`. All experimental results should be saved to `cfg.rslt_dir`.\n\nThe loaded `cfg` is read-only by default (`cfg.is_frozen == True`). To modify, unfreeze `cfg` with `cfg.unfreeze()`.\n\n### Summary of This Chapter\n* The config file offers a `Config` object `cfg`, a nested dictionary with a tree structure, allowing read and write via the `.` operator.\n* Accessing non-existent keys in `cfg` returns a one-time empty dictionary considered as `False`.\n* Use `load_config` to load the config file. The experiment path will be auto created and assigned to `cfg.rslt_dir`.\n\n## Inheritance\nThe new config can inherit the existing base config, written as `cfg = Config(caps='base_cfg.py')`. The new config only needs to override or add items, with rest items reusing the base config. For example, with [base config](alchemy_cat/dl_config/examples/configs/mnist/plain_usage/cfg.py):\n```python\n# -- [INCOMPLETE] configs/mnist/plain_usage/cfg.py --\n\n# ... Code Omitted.\n\ncfg.loader.ini.batch_size = 128\ncfg.loader.ini.num_workers = 2\n\ncfg.opt.cls = optim.AdamW\ncfg.opt.ini.lr = 0.01\n\n# ... Code Omitted.\n```\nTo double the batch size, [new config](alchemy_cat/dl_config/examples/configs/mnist/plain_usage,2xbs/cfg.py) can be written as:\n```python\n# -- configs/mnist/plain_usage,2xbs/cfg.py --\n\nfrom alchemy_cat.dl_config import Config\n\ncfg = Config(caps='configs/mnist/plain_usage/cfg.py') # Inherit from base config.\n\ncfg.loader.ini.batch_size = 128 * 2 # Double batch size.\n\ncfg.opt.ini.lr = 0.01 * 2 # Linear scaling rule, see https://arxiv.org/abs/1706.02677\n```\nInheritance behaves like `dict.update`. The key difference is that if both config have keys with the same name and their values are `Config` instance (naming config subtree), we recursively update within these subtrees. Thus, the new config can modify `cfg.loader.ini.batch_size` while inheriting `cfg.loader.ini.num_workers`.\n```text\n>>> base_cfg = load_config('configs/mnist/plain_usage/cfg.py', create_rslt_dir=False)\n>>> new_cfg = load_config('configs/mnist/plain_usage,2xbs/cfg.py', create_rslt_dir=False)\n>>> base_cfg.loader.ini\n{'batch_size': 128, 'num_workers': 2}\n>>> new_cfg.loader.ini\n{'batch_size': 256, 'num_workers': 2}\n```\nTo overwrite the entire config subtree in the new config, set this subtree to \"override\", [e.g.](alchemy_cat/dl_config/examples/configs/mnist/plain_usage,override_loader/cfg.py) :\n```python\n# -- configs/mnist/plain_usage,override_loader/cfg.py --\n\nfrom alchemy_cat.dl_config import Config\n\ncfg = Config(caps='configs/mnist/plain_usage/cfg.py') # Inherit from base config.\n\ncfg.loader.ini.override() # Set subtree as whole.\ncfg.loader.ini.shuffle = False\ncfg.loader.ini.drop_last = False\n```\n`cfg.loader.ini` will now be solely defined by the new config:\n```text\n>>> base_cfg = load_config('configs/mnist/plain_usage/cfg.py', create_rslt_dir=False)\n>>> new_cfg = load_config('configs/mnist/plain_usage,2xbs/cfg.py', create_rslt_dir=False)\n>>> base_cfg.loader.ini\n{'batch_size': 128, 'num_workers': 2}\n>>> new_cfg.loader.ini\n{'shuffle': False, 'drop_last': False}\n```\nNaturally, a base config can inherit from another base config, known as chain inheritance.\n\nMultiple inheritance is also supported, written as `cfg = Config(caps=('base.py', 'patch1.py', 'patch2.py', ...))`, creating an inheritance chain of `base -> patch1 -> patch2 -> current cfg`. The base configs on the right are often used patches to batch add config items. For example, this [patch](alchemy_cat/dl_config/examples/configs/patches/cifar10.py) includes CIFAR10 dataset configurations:\n```python\n# -- configs/patches/cifar10.py --\n\nimport torchvision.transforms as T\nfrom torchvision.datasets import CIFAR10\n\nfrom alchemy_cat.dl_config import Config\n\ncfg = Config()\n\ncfg.dt.override()\ncfg.dt.cls = CIFAR10\ncfg.dt.ini.root = '/tmp/data'\ncfg.dt.ini.transform = T.Compose([T.ToTensor(), T.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])\n```\nTo switch to CIFAR10, [new config](alchemy_cat/dl_config/examples/configs/mnist/plain_usage,cifar10/cfg.py) only need to inherit the patch:\n```python\n# -- configs/mnist/plain_usage,cifar10/cfg.py --\n\nfrom alchemy_cat.dl_config import Config\n\ncfg = Config(caps=('configs/mnist/plain_usage/cfg.py', 'alchemy_cat/dl_config/examples/configs/patches/cifar10.py'))\n```\n```text\n>>> cfg = load_config('configs/mnist/plain_usage,cifar10/cfg.py', create_rslt_dir=False)\n>>> cfg.dt\n{'cls': torchvision.datasets.cifar.CIFAR10,\n 'ini': {'root': '/tmp/data',\n 'transform': Compose(\n ToTensor()\n Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5))\n )}}\n```\n> _Inheritance Implementation Details_\n> \n> We copy the base config tree and update it with the new config, ensuring isolation between them. This means changes to the new config do not affect the base. Complex inheritance like diamond inheritance is supported but not recommended due to readability issues. \\\n> Note that leaf node values are passed by reference; modifying them inplace will affect the entire inheritance chain.\n\n### Summary of This Chapter\n* The new config can leverage inheritance to reuse the base config and modifies or adds some items.\n* The new config updates the base config recursively. Use `Config.override` to revert to the `dict.update` method for updates.\n* `Config` supports chain and multiple inheritance, allowing for more fine-grained reuse.\n\n## Dependency\nIn the [previous](#inheritance) example, changing the batch size in the new configuration also alters the learning rate. This interdependence is called \"dependency.\"\n\nWhen modifying a config item, it's common to forget its dependencies. AlchemyCat lets you define dependencies, changing the dependency source updates all dependent items automatically. [For example](alchemy_cat/dl_config/examples/configs/mnist/base/cfg.py):\n\n```python\n# -- [INCOMPLETE] configs/mnist/base/cfg.py --\n\nfrom alchemy_cat.dl_config import Config, DEP\n# ... Code Omitted.\n\ncfg.loader.ini.batch_size = 128\n# ... Code Omitted.\ncfg.opt.ini.lr = DEP(lambda c: c.loader.ini.batch_size // 128 * 0.01) # Linear scaling rule.\n\n# ... Code Omitted.\n```\nThe learning rate `cfg.opt.ini.lr` is calculated as a dependency `DEP` using the batch size `cfg.loader.ini.batch_size`. `DEP` takes a function with `cfg` as an argument and returns the dependency value.\n\nIn the [new config](alchemy_cat/dl_config/examples/configs/mnist/base,2xbs/cfg.py), we only need to modify the batch size, and the learning rate will update automatically:\n```python\n# -- configs/mnist/base,2xbs/cfg.py --\n\nfrom alchemy_cat.dl_config import Config\n\ncfg = Config(caps='configs/mnist/base/cfg.py')\n\ncfg.loader.ini.batch_size = 128 * 2 # Double batch size, learning rate will be doubled automatically.\n```\n```text\n>>> cfg = load_config('configs/mnist/base,2xbs/cfg.py', create_rslt_dir=False)\n>>> cfg.loader.ini.batch_size\n256\n>>> cfg.opt.ini.lr\n0.02\n```\nBelow is a more complex [example](alchemy_cat/dl_config/examples/configs/mnist/base/cfg.py):\n```python\n# -- configs/mnist/base/cfg.py --\n\n# ... Code Omitted.\n\ncfg.sched.epochs = 30\n@cfg.sched.set_DEP(name='warm_epochs', priority=0) # kwarg `name` is not necessary\ndef warm_epochs(c: Config) -> int: # warm_epochs = 10% of total epochs\n return round(0.1 * c.sched.epochs)\n\ncfg.sched.warm.cls = sched.LinearLR\ncfg.sched.warm.ini.total_iters = DEP(lambda c: c.sched.warm_epochs, priority=1)\ncfg.sched.warm.ini.start_factor = 1e-5\ncfg.sched.warm.ini.end_factor = 1.\n\ncfg.sched.main.cls = sched.CosineAnnealingLR\ncfg.sched.main.ini.T_max = DEP(lambda c: c.sched.epochs - c.sched.warm.ini.total_iters,\n priority=2) # main_epochs = total_epochs - warm_epochs\n\n# ... Code Omitted.\n```\n```text\n>>> print(cfg.sched.to_txt(prefix='cfg.sched.')) # A pretty print of the config tree.\ncfg.sched = Config()\n# ------- \u2193 LEAVES \u2193 ------- #\ncfg.sched.epochs = 30\ncfg.sched.warm_epochs = 3\ncfg.sched.warm.cls = <class 'torch.optim.lr_scheduler.LinearLR'>\ncfg.sched.warm.ini.total_iters = 3\ncfg.sched.warm.ini.start_factor = 1e-05\ncfg.sched.warm.ini.end_factor = 1.0\ncfg.sched.main.cls = <class 'torch.optim.lr_scheduler.CosineAnnealingLR'>\ncfg.sched.main.ini.T_max = 27\n```\nIn the code, `cfg.sched.epochs` determines total training epochs, which is also the dependency source. Warm-up epochs `cfg.sched.warm_epochs` are 10% of this total, and main epochs `cfg.sched.main.ini.T_max` is the remainder. Adjusting total training epochs updates both warm-up and main epochs automatically.\n\nThe dependency `cfg.sched.warm_epochs` is defined using the `Config.set_DEP` decorator. The decorated function, passed as the first parameter of `DEP`, computes the dependency. The key name of dependency can be specified via the keyword argument `name`; if omitted, it defaults to the function's name. For complex computations, using a decorator for definition is recommended.\n\nWhen a dependency relies on another dependency, they must be computed in the correct order. By default, this is the defined order. The `priority` parameter can specify computation order: smaller `priority` compute earlier. For instance, `cfg.sched.warm_epochs` depended by `cfg.sched.warm.ini.total_iters`, which is depended by `cfg.sched.main.ini.T_max`, so their `priority` increase sequentially.\n\n### Summary of This Chapter\n* A dependency is defined when one config item relies on another. Changing the dependency source will automatically recalculate the dependency based on the calculation function.\n* Dependencies can be defined by `DEP(...)` or the `Config.set_DEP` decorator.\n* If dependencies are interdependent, use the `priority` parameter to specify the computation order; otherwise, they resolve in the order of definition.\n\n## Composition\nComposition allows reusing configs by compose predefined config subtrees to form a complete config. For instance, the following [config subtree](alchemy_cat/dl_config/examples/configs/addons/linear_warm_cos_sched.py) defines a learning rate strategy: \n\n```python\n# -- configs/addons/linear_warm_cos_sched.py --\nimport torch.optim.lr_scheduler as sched\n\nfrom alchemy_cat.dl_config import Config, DEP\n\ncfg = Config()\n\ncfg.epochs = 30\n\n@cfg.set_DEP(priority=0) # warm_epochs = 10% of total epochs\ndef warm_epochs(c: Config) -> int:\n return round(0.1 * c.epochs)\n\ncfg.warm.cls = sched.LinearLR\ncfg.warm.ini.total_iters = DEP(lambda c: c.warm_epochs, priority=1)\ncfg.warm.ini.start_factor = 1e-5\ncfg.warm.ini.end_factor = 1.\n\ncfg.main.cls = sched.CosineAnnealingLR\ncfg.main.ini.T_max = DEP(lambda c: c.epochs - c.warm.ini.total_iters,\n priority=2) # main_epochs = total_epochs - warm_epochs\n\n```\nIn the [final config](alchemy_cat/dl_config/examples/configs/mnist/base,sched_from_addon/cfg.py), we compose this set of learning rate strategy: \n```python\n# -- configs/mnist/base,sched_from_addon/cfg.py --\n# ... Code Omitted.\n\ncfg.sched = Config('configs/addons/linear_warm_cos_sched.py')\n\n# ... Code Omitted.\n```\n```text\n>>> print(cfg.sched.to_txt(prefix='cfg.sched.')) # A pretty print of the config tree.\ncfg.sched = Config()\n# ------- \u2193 LEAVES \u2193 ------- #\ncfg.sched.epochs = 30\ncfg.sched.warm_epochs = 3\ncfg.sched.warm.cls = <class 'torch.optim.lr_scheduler.LinearLR'>\ncfg.sched.warm.ini.total_iters = 3\ncfg.sched.warm.ini.start_factor = 1e-05\ncfg.sched.warm.ini.end_factor = 1.0\ncfg.sched.main.cls = <class 'torch.optim.lr_scheduler.CosineAnnealingLR'>\ncfg.sched.main.ini.T_max = 27\n```\n\nIt looks very simple! Just assign/mount the predefined config sub-subtree to the final config. `Config('path/to/cfg.py')` returns a copy of the `cfg` object in the config file, ensuring modifications before and after copying are isolated. \n\n> _Implementation Details of Composition and Dependency_\n> \n> Attentive readers might wonder how `DEP` determines the parameter `c` for the dependency computation function, specifically which Config object is passed. In this chapter's example, `c` is the config subtree of learning rate; thus, the calculation function for `cfg.warm.ini.total_iters` is `lambda c: c.warm_epochs`. However, in the [previous chapter's](#dependency) example, `c` is the final config; hence, the calculation function for `cfg.sched.warm.ini.total_iters` is `lambda c: c.sched.warm_epochs`.\n> \n> In fact, `c` is the root node of the configuration tree where `DEP` was first mounted. The `Config` is a bidirectional tree. When `DEP` is first mounted, it records its relative distance to the root. During computation, it traces back this distance to find and pass the corresponding config tree into the computation function.\n> \n> To prevent this default behavior, set `DEP(lambda c: ..., rel=False)`, ensuring `c` is always the complete configuration.\n\n**Best Practice: Both composition and inheritance aim to reuse config. Composition is more flexible and loosely coupled, so it should be prioritized over inheritance.**\n\n### Summary of This Chapter\n* Define config subtree and compose them to create a complete config.\n\n## Full Example\n\n<details>\n<summary> Expand full example </summary>\n\n[Config subtree](alchemy_cat/dl_config/examples/configs/addons/linear_warm_cos_sched.py) related to learning rate:\n```python\n# -- configs/addons/linear_warm_cos_sched.py --\n\nimport torch.optim.lr_scheduler as sched\n\nfrom alchemy_cat.dl_config import Config, DEP\n\ncfg = Config()\n\ncfg.epochs = 30\n\n@cfg.set_DEP(priority=0) # warm_epochs = 10% of total epochs\ndef warm_epochs(c: Config) -> int:\n return round(0.1 * c.epochs)\n\ncfg.warm.cls = sched.LinearLR\ncfg.warm.ini.total_iters = DEP(lambda c: c.warm_epochs, priority=1)\ncfg.warm.ini.start_factor = 1e-5\ncfg.warm.ini.end_factor = 1.\n\ncfg.main.cls = sched.CosineAnnealingLR\ncfg.main.ini.T_max = DEP(lambda c: c.epochs - c.warm.ini.total_iters,\n priority=2) # main_epochs = total_epochs - warm_epochs\n```\nThe composed [base config](alchemy_cat/dl_config/examples/configs/mnist/base,sched_from_addon/cfg.py):\n```python\n# -- configs/mnist/base/cfg.py --\n\nimport torchvision.models as model\nimport torchvision.transforms as T\nfrom torch import optim\nfrom torchvision.datasets import MNIST\n\nfrom alchemy_cat.dl_config import Config, DEP\n\ncfg = Config()\n\ncfg.rand_seed = 0\n\n# -* Set datasets.\ncfg.dt.cls = MNIST\ncfg.dt.ini.root = '/tmp/data'\ncfg.dt.ini.transform = T.Compose([T.Grayscale(3), T.ToTensor(), T.Normalize((0.1307,), (0.3081,)),])\n\n# -* Set data loader.\ncfg.loader.ini.batch_size = 128\ncfg.loader.ini.num_workers = 2\n\n# -* Set model.\ncfg.model.cls = model.resnet18\ncfg.model.ini.num_classes = DEP(lambda c: len(c.dt.cls.classes))\n\n# -* Set optimizer.\ncfg.opt.cls = optim.AdamW\ncfg.opt.ini.lr = DEP(lambda c: c.loader.ini.batch_size // 128 * 0.01) # Linear scaling rule.\n\n# -* Set scheduler.\ncfg.sched = Config('configs/addons/linear_warm_cos_sched.py')\n\n# -* Set logger.\ncfg.log.save_interval = DEP(lambda c: c.sched.epochs // 5, priority=1) # Save model at every 20% of total epochs.\n```\nInherited from the base config, batch size doubled, number of epochs halved [new config](alchemy_cat/dl_config/examples/configs/mnist/base,sched_from_addon,2xbs,2\u00f7epo/cfg.py):\n\n```python\n# -- configs/mnist/base,sched_from_addon,2xbs,2\u00f7epo/cfg.py --\n\nfrom alchemy_cat.dl_config import Config\n\ncfg = Config(caps='configs/mnist/base,sched_from_addon/cfg.py')\n\ncfg.loader.ini.batch_size = 256\n\ncfg.sched.epochs = 15\n```\nNote that dependencies such as learning rate, warm-up epochs, and main epochs will be automatically updated:\n```text\n>>> cfg = load_config('configs/mnist/base,sched_from_addon,2xbs,2\u00f7epo/cfg.py', create_rslt_dir=False)\n>>> print(cfg)\ncfg = Config()\ncfg.override(False).set_attribute('_cfgs_update_at_parser', ('configs/mnist/base,sched_from_addon/cfg.py',))\n# ------- \u2193 LEAVES \u2193 ------- #\ncfg.rand_seed = 0\ncfg.dt.cls = <class 'torchvision.datasets.mnist.MNIST'>\ncfg.dt.ini.root = '/tmp/data'\ncfg.dt.ini.transform = Compose(\n Grayscale(num_output_channels=3)\n ToTensor()\n Normalize(mean=(0.1307,), std=(0.3081,))\n)\ncfg.loader.ini.batch_size = 256\ncfg.loader.ini.num_workers = 2\ncfg.model.cls = <function resnet18 at 0x7f5bcda68a40>\ncfg.model.ini.num_classes = 10\ncfg.opt.cls = <class 'torch.optim.adamw.AdamW'>\ncfg.opt.ini.lr = 0.02\ncfg.sched.epochs = 15\ncfg.sched.warm_epochs = 2\ncfg.sched.warm.cls = <class 'torch.optim.lr_scheduler.LinearLR'>\ncfg.sched.warm.ini.total_iters = 2\ncfg.sched.warm.ini.start_factor = 1e-05\ncfg.sched.warm.ini.end_factor = 1.0\ncfg.sched.main.cls = <class 'torch.optim.lr_scheduler.CosineAnnealingLR'>\ncfg.sched.main.ini.T_max = 13\ncfg.log.save_interval = 3\ncfg.rslt_dir = 'mnist/base,sched_from_addon,2xbs,2\u00f7epo'\n```\n[Training code](alchemy_cat/dl_config/examples/train.py):\n```python\n# -- train.py --\nimport argparse\nimport json\n\nimport torch\nimport torch.nn.functional as F\nfrom rich.progress import track\nfrom torch.optim.lr_scheduler import SequentialLR\n\nfrom alchemy_cat.dl_config import load_config\nfrom utils import eval_model\n\nparser = argparse.ArgumentParser(description='AlchemyCat MNIST Example')\nparser.add_argument('-c', '--config', type=str, default='configs/mnist/base,sched_from_addon,2xbs,2\u00f7epo/cfg.py')\nargs = parser.parse_args()\n\n# Folder 'experiment/mnist/base' will be auto created by `load` and assigned to `cfg.rslt_dir`\ncfg = load_config(args.config, experiments_root='/tmp/experiment', config_root='configs')\nprint(cfg)\n\ntorch.manual_seed(cfg.rand_seed) # Use `cfg` to set random seed\n\ndataset = cfg.dt.cls(**cfg.dt.ini) # Use `cfg` to set dataset type and its initial parameters\n\n# Use `cfg` to set changeable parameters of loader,\n# other fixed parameter like `shuffle` is set in main code\nloader = torch.utils.data.DataLoader(dataset, shuffle=True, **cfg.loader.ini)\n\nmodel = cfg.model.cls(**cfg.model.ini).train().to('cuda') # Use `cfg` to set model\n\n# Use `cfg` to set optimizer, and get `model.parameters()` in run time\nopt = cfg.opt.cls(model.parameters(), **cfg.opt.ini, weight_decay=0.)\n\n# Use `cfg` to set warm and main scheduler, and `SequentialLR` to combine them\nwarm_sched = cfg.sched.warm.cls(opt, **cfg.sched.warm.ini)\nmain_sched = cfg.sched.main.cls(opt, **cfg.sched.main.ini)\nsched = SequentialLR(opt, [warm_sched, main_sched], [cfg.sched.warm_epochs])\n\nfor epoch in range(1, cfg.sched.epochs + 1): # train `cfg.sched.epochs` epochs\n for data, target in track(loader, description=f\"Epoch {epoch}/{cfg.sched.epochs}\"):\n F.cross_entropy(model(data.to('cuda')), target.to('cuda')).backward()\n opt.step()\n opt.zero_grad()\n\n sched.step()\n\n # If cfg.log is defined, save model to `cfg.rslt_dir` at every `cfg.log.save_interval`\n if cfg.log and epoch % cfg.log.save_interval == 0:\n torch.save(model.state_dict(), f\"{cfg.rslt_dir}/model_{epoch}.pth\")\n\n eval_model(model)\n\nif cfg.log:\n eval_ret = eval_model(model)\n with open(f\"{cfg.rslt_dir}/eval.json\", 'w') as json_f:\n json.dump(eval_ret, json_f)\n```\nRun `python train.py --config 'configs/mnist/base,sched_from_addon,2xbs,2\u00f7epo/cfg.py'`, and it will use the settings in the config file to train with `train.py` and save the results to the `/tmp/experiment/mnist/base,sched_from_addon,2xbs,2\u00f7epo` directory.\n</details>\n\n## Auto Parameter Tuning\nIn the [example above](#full-example), running `python train.py --config path/to/cfg.py` each time yields an experimental result for a set of parameters.\n\nHowever, we often need to perform grid search over the parameter space to find the optimal parameter combination. Writing a config for each combination is laborious and error-prone. Can we define the entire parameter space in a \"tunable config\"? Then let the program automatically traverse all combinations, generate configs, run them, and summarize results for comparison.\n\nThe auto-tuner traverses through tunable config's parameter combinations, generates `N` sub-configs, runs them to obtain `N` experimental records, and summarizes all experimental results into an Excel sheet:\n\n```text\nconfig to be tuned T \u2500\u2500\u2500> config C1 + algorithm code A \u2500\u2500\u2500> reproducible experiment E1(C1, A) \u2500\u2500\u2500> summary table S(T,A)\n \u2502\u00a0 \u2502\u00a0 \n \u251c\u2500\u2500> config C2 + algorithm code A \u2500\u2500\u2500> reproducible experiment E1(C2, A) \u2500\u2500\u2502\u00a0\n ... ...\n```\n### Tunable Config\nTo use the auto-tuner, we first need to write a tunable config:\n```python\n# -- configs/tune/tune_bs_epoch/cfg.py --\n\nfrom alchemy_cat.dl_config import Cfg2Tune, Param2Tune\n\ncfg = Cfg2Tune(caps='configs/mnist/base,sched_from_addon/cfg.py')\n\ncfg.loader.ini.batch_size = Param2Tune([128, 256, 512])\n\ncfg.sched.epochs = Param2Tune([5, 15])\n```\nIts writing style is similar to the [normal configuration](alchemy_cat/dl_config/examples/configs/mnist/base,sched_from_addon,2xbs,2\u00f7epo/cfg.py) in the previous chapter. It supports attribute reading and writing, inheritance, dependency, and combination. The difference lies in:\n* The type of config is `Cfg2Tune`, a subclass of `Config`.\n* For grid search parameters, use `Param2Tune([v1, v2, ...])` with optional values `v1, v2, ...`.\n\nThe tunable config above will search a parameter space of size 3\u00d72=6 and generate these 6 sub-configs:\n```text\nbatch_size epochs child_configs \n128 5 configs/tune/tune_bs_epoch/batch_size=128,epochs=5/cfg.pkl\n 15 configs/tune/tune_bs_epoch/batch_size=128,epochs=15/cfg.pkl\n256 5 configs/tune/tune_bs_epoch/batch_size=256,epochs=5/cfg.pkl\n 15 configs/tune/tune_bs_epoch/batch_size=256,epochs=15/cfg.pkl\n512 5 configs/tune/tune_bs_epoch/batch_size=512,epochs=5/cfg.pkl\n 15 configs/tune/tune_bs_epoch/batch_size=512,epochs=15/cfg.pkl\n```\nSet the `priority` parameter of `Param2Tune` to specify the search order. The default is the defined order. Use `optional_value_names` to assign readable names to parameter values. [For example](alchemy_cat/dl_config/examples/configs/tune/tune_bs_epoch,pri,name/cfg.py):\n\n```python\n# -- configs/tune/tune_bs_epoch,pri,name/cfg.py --\n\nfrom alchemy_cat.dl_config import Cfg2Tune, Param2Tune\n\ncfg = Cfg2Tune(caps='configs/mnist/base,sched_from_addon/cfg.py')\n\ncfg.loader.ini.batch_size = Param2Tune([128, 256, 512], optional_value_names=['1xbs', '2xbs', '4xbs'], priority=1)\n\ncfg.sched.epochs = Param2Tune([5, 15], priority=0)\n```\nwhose search space is:\n```text\nepochs batch_size child_configs \n5 1xbs configs/tune/tune_bs_epoch,pri,name/epochs=5,batch_size=1xbs/cfg.pkl\n 2xbs configs/tune/tune_bs_epoch,pri,name/epochs=5,batch_size=2xbs/cfg.pkl\n 4xbs configs/tune/tune_bs_epoch,pri,name/epochs=5,batch_size=4xbs/cfg.pkl\n15 1xbs configs/tune/tune_bs_epoch,pri,name/epochs=15,batch_size=1xbs/cfg.pkl\n 2xbs configs/tune/tune_bs_epoch,pri,name/epochs=15,batch_size=2xbs/cfg.pkl\n 4xbs configs/tune/tune_bs_epoch,pri,name/epochs=15,batch_size=4xbs/cfg.pk\n```\n\nWe can set constraints between parameters to eliminate unnecessary combinations. For example, the following [example](alchemy_cat/dl_config/examples/configs/tune/tune_bs_epoch,subject_to/cfg.py) limits total iterations to a maximum of 15\u00d7128:\n```python\n# -- configs/tune/tune_bs_epoch,subject_to/cfg.py --\n\nfrom alchemy_cat.dl_config import Cfg2Tune, Param2Tune\n\ncfg = Cfg2Tune(caps='configs/mnist/base,sched_from_addon/cfg.py')\n\ncfg.loader.ini.batch_size = Param2Tune([128, 256, 512])\n\ncfg.sched.epochs = Param2Tune([5, 15],\n subject_to=lambda cur_val: cur_val * cfg.loader.ini.batch_size.cur_val <= 15 * 128)\n```\nwhose search space is:\n```text\nbatch_size epochs child_configs \n128 5 configs/tune/tune_bs_epoch,subject_to/batch_size=128,epochs=5/cfg.pkl \n 15 configs/tune/tune_bs_epoch,subject_to/batch_size=128,epochs=15/cfg.pkl\n256 5 configs/tune/tune_bs_epoch,subject_to/batch_size=256,epochs=5/cfg.pkl\n```\n\n### Running auto-tuner\nWe also need to write a small script to run the auto-tuner:\n```python\n# -- tune_train.py --\nimport argparse, json, os, subprocess, sys\nfrom alchemy_cat.dl_config import Config, Cfg2TuneRunner\n\nparser = argparse.ArgumentParser(description='Tuning AlchemyCat MNIST Example')\nparser.add_argument('-c', '--cfg2tune', type=str)\nargs = parser.parse_args()\n\n# Will run `torch.cuda.device_count() // work_gpu_num` of configs in parallel\nrunner = Cfg2TuneRunner(args.cfg2tune, experiment_root='/tmp/experiment', work_gpu_num=1)\n\n@runner.register_work_fn # How to run config\ndef work(pkl_idx: int, cfg: Config, cfg_pkl: str, cfg_rslt_dir: str, cuda_env: dict[str, str]) -> ...:\n subprocess.run([sys.executable, 'train.py', '-c', cfg_pkl], env=cuda_env)\n\n@runner.register_gather_metric_fn # How to gather metric for summary\ndef gather_metric(cfg: Config, cfg_rslt_dir: str, run_rslt: ..., param_comb: dict[str, tuple[..., str]]) -> dict[str, ...]:\n return json.load(open(os.path.join(cfg_rslt_dir, 'eval.json')))\n\nrunner.tuning()\n```\nThe script performs these operations:\n* Instantiates the auto-tuner with `runner = Cfg2TuneRunner(...)`, passing in the tunable config path. By default, it runs sub-configs sequentially. Set the parameter `work_gpu_num` to run `len(os.environ['CUDA_VISIBLE_DEVICES']) // work_gpu_num` sub-configs in parallel.\n* Registers a worker that executes each sub-config. The function parameters are:\n - `pkl_idx`: index of the sub-config\n - `cfg`: the sub-config\n - `cfg_pkl`: pickle save path for this sub-config\n - `cfg_rslt_dir`: experiment directory.\n - `cuda_env`: If `work_gpu_num` is set, then `cuda_env` will allocate non-overlapping `CUDA_VISIBLE_DEVICES` environment variables for parallel sub-configs.\n \n Commonly, we only need to pass `cfg_pkl` as the config file into the training script, since `load_cfg` supports reading config in pickle format. For deep learning tasks, different `CUDA_VISIBLE_DEVICES` are needed for each sub-config.\n* Registers a summary function that returns an experimental result as a `{metric_name: metric_value}` dictionary. The auto-tunner will traverse all experimental results and summary into a table. The summary function accepts these parameters:\n - `cfg`: the sub-configuration\n - `cfg_rslt_dir`: experiment directory\n - `run_rslt`: returned from working functions\n - `param_comb`: parameter combinations for that particular sub-configuration.\n \n Generally, only need to read results from `cfg_rslt_dir` and return them.\n* Calls `runner.tuning()` to start automatic tuning.\n\nAfter tuning, the tuning results will be printed:\n```text\nMetric Frame: \n test_loss acc\nbatch_size epochs \n128 5 1.993285 32.63\n 15 0.016772 99.48\n256 5 1.889874 37.11\n 15 0.020811 99.49\n512 5 1.790593 41.74\n 15 0.024695 99.33\n\nSaving Metric Frame at /tmp/experiment/tune/tune_bs_epoch/metric_frame.xlsx\n```\nAs the prompt says, the tuning results will also be saved to the `/tmp/experiment/tune/tune_bs_epoch/metric_frame.xlsx` table:\n<div align = \"center\">\n<img src=\"https://github.com/HAL-42/AlchemyCat/raw/master/docs/figs/readme-cfg2tune-excel.png\" width=\"400\" />\n</div>\n\n> [!TIP]\n> **Best Practice: The auto-tuner is separate from the standard workflow. Write configs and code without considering it. When tuning, add extra code to define parameter space, specify invocation and result methods. After tuning, remove the auto-tuner, keeping only the best config and algorithm.**\n\n### Summary of This Chapter\n* Define a tunable config `Cfg2Tune` with `Param2Tune` to specify the parameter space.\n* Use the auto-tuner `Cfg2TuneRunner` to traverse the parameter space, generate sub-configs, run them, and summarize the results.\n\n## Advanced Usage\n\n<details>\n<summary> Expand advanced usage </summary>\n\n### Pretty Print\nThe `__str__` method of `Config` is overloaded to print the tree structure with keys separated by `.`:\n\n```text\n>>> cfg = Config()\n>>> cfg.foo.bar.a = 1\n>>> cfg.bar.foo.b = ['str1', 'str2']\n>>> cfg.whole.override()\n>>> print(cfg)\ncfg = Config()\ncfg.whole.override(True)\n# ------- \u2193 LEAVES \u2193 ------- #\ncfg.foo.bar.a = 1\ncfg.bar.foo.b = ['str1', 'str2']\n```\n\nWhen all leaf nodes are built-in types, the pretty print output of `Config` can be executed as Python code to get the same configuration:\n```text\n>>> exec(cfg.to_txt(prefix='new_cfg.'), globals(), (l_dict := {}))\n>>> l_dict['new_cfg'] == cfg\nTrue\n```\n\nFor invalid attribute names, `Config` will fall back to the print format of `dict`:\n```text\n>>> cfg = Config()\n>>> cfg['Invalid Attribute Name'].foo = 10\n>>> cfg.bar['def'] = {'a': 1, 'b': 2}\n>>> print(cfg)\ncfg = Config()\n# ------- \u2193 LEAVES \u2193 ------- #\ncfg['Invalid Attribute Name'].foo = 10\ncfg.bar['def'] = {'a': 1, 'b': 2}\n```\n\n### Auto Capture Experiment Logs\nFor deep learning tasks, we recommend using `init_env` instead of `load_config`. In addition to loading the config, `init_env` can also initialize the deep learning environment, such as setting the torch device, gradient, random seed, and distributed training:\n\n```python\nfrom alchemy_cat.torch_tools import init_env\n\nif __name__ == '__main__':\n import argparse\n\n parser = argparse.ArgumentParser()\n parser.add_argument('-c', '--config', type=str)\n parser.add_argument('--local_rank', type=int, default=-1)\n args = parser.parse_args()\n \n device, cfg = init_env(config_path=args.config, # config file path\uff0cread to `cfg`\n is_cuda=True, # if True\uff0c`device` is cuda\uff0celse cpu\n is_benchmark=bool(args.benchmark), # torch.backends.cudnn.benchmark = is_benchmark\n is_train=True, # torch.set_grad_enabled(is_train)\n experiments_root=\"experiment\", # root of experiment dir\n rand_seed=True, # set python, numpy, torch rand seed. If True, read cfg.rand_seed as seed, else use actual parameter as rand seed. \n cv2_num_threads=0, # set cv2 num threads\n verbosity=True, # print more env init info\n log_stdout=True, # where fork stdout to log file\n loguru_ini=True, # config a pretty loguru format\n reproducibility=False, # set pytorch to reproducible mode\n local_rank=..., # dist.init_process_group(..., local_rank=local_rank)\n silence_non_master_rank=True, # if True, non-master rank will not print to stdout, but only log to file\n is_debug=bool(args.is_debug)) # is debug mode\n```\nIf `log_stdout=True`, `init_env` will fork `sys.stdout` and `sys.stderr` to the log file `cfg.rslt_dir/{local-time}.log`. This will not interfere with normal `print`, but all screen output will be recorded in the log. Therefore, there is no need to manually write logs, what you see on the screen is what you get in the log.\n\nDetails can be found in the docstring of `init_env`.\n\n### Attribute Dict\nIf you are a user of [addict](https://github.com/mewwts/addict), our `ADict` can be used as a drop-in replacement for `addict.Dict`: `from alchemy_cat.dl_config import ADict as Dict`.\n\n`ADict` has all the interfaces of `addict.Dict`. However, all methods are re-implemented to optimize execution efficiency and cover more corner cases (such as circular references). `Config` is actually a subclass of `ADict`.\n\nIf you haven't used `addict` before, read this [documentation](https://github.com/mewwts/addict). Research code often involves complex dictionaries. `addict.Dict` or `ADict` supports attribute-style access for nested dictionaries.\n\n### Circular References\nThe initialization, inheritance, and composition of `ADict` and `Config` require a `branch_copy` operation, which is between shallow and deep copy, that is, copying the tree structure but not the leaf nodes. `ADict.copy`, `Config.copy`, and `copy.copy(cfg)` all call `branch_copy`, not the `copy` method of `dict`.\n\nIn theory, `ADict.branch_copy` can handle circular references, such as:\n```text\n>>> dic = {'num': 0,\n 'lst': [1, 'str'],\n 'sub_dic': {'sub_num': 3}}\n>>> dic['lst'].append(dic['sub_dic'])\n>>> dic['sub_dic']['parent'] = dic\n>>> dic\n{'num': 0,\n 'lst': [1, 'str', {'sub_num': 3, 'parent': {...}}],\n 'sub_dic': {'sub_num': 3, 'parent': {...}}}\n\n>>> adic = ADict(dic)\n>>> adic.sub_dic.parent is adic is not dic\nTrue\n>>> adic.lst[-1] is adic.sub_dic is not dic['sub_dic']\nTrue\n```\nDifferent from `ADict`, the data model of `Config` is a bidirectional tree, and circular references will form a cycle. To avoid cycles, if a subtree is mounted to different parent configs multiple times, the subtree will be copied to an independent config tree before mounting. In normal use, circular references should not appear in the config tree.\n\nIn summary, although circular references are supported, they are neither necessary nor recommended.\n\n### Traverse the Config Tree\n`Config.named_branchs` and `Config.named_ckl` respectively traverse all branches and leaves of the config tree (the branch, key name, and value they are in):\n```text\n>>> list(cfg.named_branches) \n[('', {'foo': {'bar': {'a': 1}}, \n 'bar': {'foo': {'b': ['str1', 'str2']}}, \n 'whole': {}}),\n ('foo', {'bar': {'a': 1}}),\n ('foo.bar', {'a': 1}),\n ('bar', {'foo': {'b': ['str1', 'str2']}}),\n ('bar.foo', {'b': ['str1', 'str2']}),\n ('whole', {})]\n \n>>> list(cfg.ckl)\n[({'a': 1}, 'a', 1), ({'b': ['str1', 'str2']}, 'b', ['str1', 'str2'])]\n```\n\n### Lazy Inheritance\n```text\n>>> from alchemy_cat.dl_config import Config\n>>> cfg = Config(caps='configs/mnist/base,sched_from_addon/cfg.py')\n>>> cfg.loader.ini.batch_size = 256\n>>> cfg.sched.epochs = 15\n>>> print(cfg)\n\ncfg = Config()\ncfg.override(False).set_attribute('_cfgs_update_at_parser', ('configs/mnist/base,sched_from_addon/cfg.py',))\n# ------- \u2193 LEAVES \u2193 ------- #\ncfg.loader.ini.batch_size = 256\ncfg.sched.epochs = 15\n```\nWhen inheriting, the parent configs `caps` is not immediately updated, but is loaded when `load_config` is called. Lazy inheritance allows the config system to have an eager-view of the entire inheritance chain, and a few features rely on this.\n\n### Work with Git\n\nFor `config C + algorithm code A \u2014\u2014> reproducible experiment E(C, A)`, meaning that when the config `C` and the algorithm code `A` are determined, the experiment `E` can always be reproduced. Therefore, it is recommended to submit the configuration file and algorithm code to the Git repository together for reproducibility.\n\nWe also provide a [script](alchemy_cat/torch_tools/scripts/tag_exps.py) that runs `pyhon -m alchemy_cat.torch_tools.scripts.tag_exps -s commit_ID -a commit_ID`, interactively lists the new configs added by the commit, and tags the commit according to the config path. This helps quickly trace back the config and algorithm of a historical experiment.\n\n### Allocate GPU for Child Processes Manually\nThe `work` function receives the idle GPU automatically allocated by `Cfg2TuneRunner` through the `cuda_env` parameter. We can further control the definition of 'idle GPU':\n```python\nrunner = Cfg2TuneRunner(args.cfg2tune, experiment_root='/tmp/experiment', work_gpu_num=1, \n block=True, # Try to allocate idle GPU\n memory_need=10 * 1024, # Need 10 GB memory\n max_process=2) # Max 2 process already ran on each GPU\n```\nwhere:\n- `block`: Defaults is `True`. If set to `False`, GPUs are allocated sequentially, regardless of whether they are idle.\n- `memory_need`: The amount of GPU memory required for each sub-config, in MB. The free memory on an idle GPU must be \u2265 `memory_need`. Default is `-1.`, indicating need all memory.\n- `max_process`: Maximum number of existing processes. The number of existing processes on an idle GPU must be \u2264 `max_process`. Default value is `-1`, indicating no limit.\n\n### Pickling Lambda Functions\nSub-configs generated by `Cfg2Tune` will be saved using pickle. However, if `Cfg2Tune` defines dependencies as `DEP(lambda c: ...)`, these lambda functions cannot be pickled. Workarounds include:\n* Using the decorator `@Config.set_DEP` to define the dependency's computation function.\n* Defining the dependency's calculation function in a separate module and passing it to `DEP`.\n* Defining dependencies in the parent configs since inheritance is handled lazily, so sub-configs temporarily exclude dependencies.\n* If the dependency source is a tunable parameter, use `P_DEP`, which resolves after generating sub-configs of `Cfg2Tune` but before saving them as pickle.\n\n### More Inheritance Tricks\n\n#### Deleting During Inheritance\nThe `Config.empty_leaf()` combines `Config.clear()` and `Config.override()` to get an empty and \"override\" subtree. This is commonly used to represent the \"delete\" semantics during inheritance, that is, using an empty config to override a subtree of the base config.\n\n#### `update` Method\nLet `cfg` be a `Config` instance and `base_cfg` be a `dict` instance. The effects of `cfg.dict_update(base_cfg)`, `cfg.update(base_cfg)`, and `cfg |= base_cfg` are similar to inheriting `Config(base_cfg)` from `cfg`.\n\nRun `cfg.dict_update(base_cfg, incremental=True)` to ensure only incremental updates, that is, only add keys that do not exist in `cfg` without overwriting existing keys.\n\n</details>\n",
"bugtrack_url": null,
"license": "Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. \"License\" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. \"Licensor\" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. \"Legal Entity\" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, \"control\" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. \"You\" (or \"Your\") shall mean an individual or Legal Entity exercising permissions granted by this License. \"Source\" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. \"Object\" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. \"Work\" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). \"Derivative Works\" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. \"Contribution\" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, \"submitted\" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as \"Not a Contribution.\" \"Contributor\" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a \"NOTICE\" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets \"{}\" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same \"printed page\" as the copyright notice for easier identification within third-party archives. Copyright {yyyy} {name of copyright owner} Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.",
"summary": "Alchemy Cat \u2014\u2014 \ud83d\udd25Config System for SOTA",
"version": "0.0.8",
"project_urls": {
"Homepage": "https://github.com/HAL-42/AlchemyCat",
"Issues": "https://github.com/HAL-42/AlchemyCat/issues"
},
"split_keywords": [
"config",
" deep learning",
" parameter tuning",
" hyperparameter"
],
"urls": [
{
"comment_text": "",
"digests": {
"blake2b_256": "129ffffc41175e191d1fcbe08054620e7fe951e4bfe012dcadb7e39f5593e8de",
"md5": "256c0b92085456c39aa0c546d80b3545",
"sha256": "490e9fbd6d5758af0d84ec443842bb6154a6e3dbde0cbb1f0eb6a5ca6064a468"
},
"downloads": -1,
"filename": "alchemy_cat-0.0.8-py3-none-any.whl",
"has_sig": false,
"md5_digest": "256c0b92085456c39aa0c546d80b3545",
"packagetype": "bdist_wheel",
"python_version": "py3",
"requires_python": ">=3.9",
"size": 348932,
"upload_time": "2024-07-29T13:27:00",
"upload_time_iso_8601": "2024-07-29T13:27:00.629481Z",
"url": "https://files.pythonhosted.org/packages/12/9f/fffc41175e191d1fcbe08054620e7fe951e4bfe012dcadb7e39f5593e8de/alchemy_cat-0.0.8-py3-none-any.whl",
"yanked": false,
"yanked_reason": null
},
{
"comment_text": "",
"digests": {
"blake2b_256": "3de2a1338fcca70b4caf5b82bfd2f75c2d5328f7351f367283a56a6d1912543f",
"md5": "fc83e8d063b17b23ad753251dfdc2b17",
"sha256": "e9bf8f31ea1bfbca4048e3fa9cc3581091a7aa6953341de0f81d93c23b0b7b96"
},
"downloads": -1,
"filename": "alchemy_cat-0.0.8.tar.gz",
"has_sig": false,
"md5_digest": "fc83e8d063b17b23ad753251dfdc2b17",
"packagetype": "sdist",
"python_version": "source",
"requires_python": ">=3.9",
"size": 260069,
"upload_time": "2024-07-29T13:27:04",
"upload_time_iso_8601": "2024-07-29T13:27:04.496203Z",
"url": "https://files.pythonhosted.org/packages/3d/e2/a1338fcca70b4caf5b82bfd2f75c2d5328f7351f367283a56a6d1912543f/alchemy_cat-0.0.8.tar.gz",
"yanked": false,
"yanked_reason": null
}
],
"upload_time": "2024-07-29 13:27:04",
"github": true,
"gitlab": false,
"bitbucket": false,
"codeberg": false,
"github_user": "HAL-42",
"github_project": "AlchemyCat",
"travis_ci": false,
"coveralls": false,
"github_actions": false,
"lcname": "alchemy-cat"
}
