| Name | config_wrangler JSON |
| Version |
1.1.0
JSON |
| download |
| home_page | https://github.com/arcann/config_wrangler |
| Summary | pydantic based configuration wrangler. Handles reading multiple ini or toml files with inheritance rules and variable expansions. |
| upload_time | 2024-03-12 18:54:44 |
| maintainer | |
| docs_url | None |
| author | Derek Wood |
| requires_python | >=3.9,<4.0.0 |
| license | MIT |
| keywords |
|
| VCS |
 |
| bugtrack_url |
|
| requirements |
No requirements were recorded.
|
| Travis-CI |
No Travis.
|
| coveralls test coverage |
No coveralls.
|
# Config Wrangler
[](https://pypi.org/project/config-wrangler/)
[](https://github.com/arcann/config_wrangler/blob/master/LICENSE)
pydantic based configuration wrangler. Handles reading multiple ini or toml files with inheritance rules and variable expansions.
## Installation
Install using your package manager of choice:
- `poetry add config-wrangler`
- `pip install -U config-wrangler`
- `conda install config-wrangler -c conda-forge`.
## A Simple Example
config.ini
```ini
[S3_Source]
bucket_name=my.exmple-bucket
key_prefixes=processed/
user_id=AK123456789ABC
# Not a secure way to store the password, but OK for local prototype or examples.
# See KEYRING or KEEPASS for better options
password_source=CONFIG_FILE
raw_password=My secret password
[target_database]
dialect=sqlite
database_name=${test_section:my_environment:source_data_dir}/example_db
[test_section]
my_int=123
my_float=123.45
my_bool=Yes
my_str=ABC☕
my_bytes=ABCⓁⓄⓋ☕
my_list_auto_c=a,b,c
my_list_auto_nl=
a
b
c
my_list_auto_pipe=a|b|c
my_list_c=a,b,c
my_list_python=['x','y','z']
my_list_json=["J","S","O","N"]
my_list_nl=
a
b
c
my_list_int_c=1,2,3
my_tuple_c=a,b,c
my_tuple_nl=
a
b
c
my_tuple_int_c=1,2,3
my_dict={1: "One", 2: "Two"}
my_dict_str_int={"one": 1, "two": 2}
my_set={'A','B','C'}
my_set_int=1,2,3
my_frozenset=A,B,C
my_date=2021-05-31
my_time=11:55:23
my_datetime=2021-05-31 11:23:53
my_url=https://localhost:6553/
[test_section.my_environment]
name=dev
# For example to run we'll make both paths relative to current
temp_data_dir=.\temp_data\${test_section:my_environment:name}
source_data_dir=.
```
python code
```py
import typing
from datetime import date, time, datetime
from pydantic import BaseModel, DirectoryPath, Field, AnyHttpUrl
from config_wrangler.config_data_loaders.base_config_data_loader import BaseConfigDataLoader
from config_wrangler.config_from_ini_env import ConfigFromIniEnv
from config_wrangler.config_from_loaders import ConfigFromLoaders
from config_wrangler.config_templates.config_hierarchy import ConfigHierarchy
from config_wrangler.config_templates.aws.s3_bucket import S3_Bucket
from config_wrangler.config_templates.sqlalchemy_database import SQLAlchemyDatabase
from config_wrangler.config_types.path_types import AutoCreateDirectoryPath
from config_wrangler.config_types.delimited_field import DelimitedListField
class S3_Bucket_KeyPrefixes(S3_Bucket):
key_prefixes: typing.List[str]
class Environment(ConfigHierarchy):
name: str = Field(..., env='env_name')
temp_data_dir: AutoCreateDirectoryPath
source_data_dir: DirectoryPath
class TestSection(BaseModel):
my_int: int
my_float: float
my_bool: bool
my_str: str
my_bytes: bytes
my_list_auto_c: list
my_list_auto_nl: list
my_list_auto_pipe: list
my_list_python: list
my_list_json: list
my_list_c: list = DelimitedListField(delimiter=',')
my_list_nl: list = DelimitedListField(delimiter='\n')
my_list_int_c: typing.List[int] = DelimitedListField(delimiter=',')
my_tuple_c: tuple = DelimitedListField(delimiter=',')
my_tuple_nl: tuple = DelimitedListField(delimiter='\n')
my_tuple_int_c: typing.Tuple[int, int, int] = DelimitedListField(delimiter=',')
my_dict: dict
my_dict_str_int: typing.Dict[str, int]
my_set: set
my_set_int: typing.Set[int]
my_frozenset: frozenset
my_date: date
my_time: time
my_datetime: datetime
my_url: AnyHttpUrl
my_environment: Environment
class ETLConfig(ConfigFromIniEnv):
class Config:
validate_default = True
validate_assignment = True
target_database: SQLAlchemyDatabase
s3_source: S3_Bucket_KeyPrefixes
test_section: TestSection
class ETLConfigAnyLoaders(ETLConfig):
def __init__(
self,
_config_data_loaders: typing.List[BaseConfigDataLoader],
**kwargs: typing.Dict[str, typing.Any]
) -> None:
# Skip super and call the next higher class
ConfigFromLoaders.__init__(
self,
_config_data_loaders=_config_data_loaders,
**kwargs
)
def main():
config = ETLConfig(file_name='simple_example.ini')
print(f"Temp data dir = {config.test_section.my_environment.temp_data_dir}")
# > Temp data dir = temp_data\dev
print(f"Source data dir = {config.test_section.my_environment.source_data_dir}")
# > Source data dir = .
print(f"my_int = {config.test_section.my_int}")
# > my_int = 123
print(f"my_float = {config.test_section.my_float}")
# > my_float = 123.45
print(f"my_str = {config.test_section.my_str}")
# > my_str = ABC☕
print(f"my_list_auto_c = {config.test_section.my_list_auto_c}")
# > my_list_auto_c = ['a', 'b', 'c']
print(f"my_list_auto_nl = {config.test_section.my_list_auto_nl}")
# > my_list_auto_c = ['a', 'b', 'c']
print(f"my_dict = {config.test_section.my_dict}")
# > my_dict = {1: 'One', 2: 'Two'}
print(f"my_set = {config.test_section.my_set}")
# > my_set = {'C', 'A', 'B'}
print(f"my_time = {config.test_section.my_time}")
# > my_time = 11:55:23
print(f"my_datetime = {config.test_section.my_datetime}")
# > my_datetime = 2021-05-31 11:23:53
print(f"my_url = {config.test_section.my_url}")
# > my_url = https://localhost:6553/
# Getting DB engine (requires sqlalchemy optional install
engine = config.target_database.get_engine()
print(f"target_database.engine = {engine}")
# > target_database.engine = Engine(sqlite:///.example_db)
print("Getting S3 Data")
bucket = config.s3_source.get_bucket()
print(f"S3 bucket definition = {bucket}")
for prefix in config.s3_source.key_prefixes:
print(f" bucket search prefix = {prefix}")
# > Getting S3 Data
# > credentials.py:56: UserWarning: Passwords stored directly in config or worse in code are not safe. Please make sure to fix this before deploying.
# > S3 bucket definitition = s3.Bucket(name='my.exmple-bucket')
# > bucket search prefix = processed/
if __name__ == '__main__':
main()
```
Raw data
{
"_id": null,
"home_page": "https://github.com/arcann/config_wrangler",
"name": "config_wrangler",
"maintainer": "",
"docs_url": null,
"requires_python": ">=3.9,<4.0.0",
"maintainer_email": "",
"keywords": "",
"author": "Derek Wood",
"author_email": "",
"download_url": "https://files.pythonhosted.org/packages/28/63/83f5296c3c17ee1a2b39dfbd7b32b12887e6ab0e185d54a24a3fc11fd848/config_wrangler-1.1.0.tar.gz",
"platform": null,
"description": "# Config Wrangler\n\n[](https://pypi.org/project/config-wrangler/)\n[](https://github.com/arcann/config_wrangler/blob/master/LICENSE)\n\npydantic based configuration wrangler. Handles reading multiple ini or toml files with inheritance rules and variable expansions.\n\n## Installation\n\nInstall using your package manager of choice:\n - `poetry add config-wrangler`\n - `pip install -U config-wrangler` \n - `conda install config-wrangler -c conda-forge`.\n\n## A Simple Example\n\nconfig.ini\n```ini\n[S3_Source]\nbucket_name=my.exmple-bucket\nkey_prefixes=processed/\nuser_id=AK123456789ABC\n# Not a secure way to store the password, but OK for local prototype or examples.\n# See KEYRING or KEEPASS for better options\npassword_source=CONFIG_FILE\nraw_password=My secret password\n\n[target_database]\ndialect=sqlite\ndatabase_name=${test_section:my_environment:source_data_dir}/example_db\n\n[test_section]\nmy_int=123\nmy_float=123.45\nmy_bool=Yes\nmy_str=ABC\u2615\nmy_bytes=ABC\u24c1\u24c4\u24cb\u2615\nmy_list_auto_c=a,b,c\nmy_list_auto_nl=\n a\n b\n c\nmy_list_auto_pipe=a|b|c\nmy_list_c=a,b,c\nmy_list_python=['x','y','z']\nmy_list_json=[\"J\",\"S\",\"O\",\"N\"]\nmy_list_nl=\n a\n b\n c\nmy_list_int_c=1,2,3\nmy_tuple_c=a,b,c\nmy_tuple_nl=\n a\n b\n c\nmy_tuple_int_c=1,2,3\nmy_dict={1: \"One\", 2: \"Two\"}\nmy_dict_str_int={\"one\": 1, \"two\": 2}\nmy_set={'A','B','C'}\nmy_set_int=1,2,3\nmy_frozenset=A,B,C\nmy_date=2021-05-31\nmy_time=11:55:23\nmy_datetime=2021-05-31 11:23:53\nmy_url=https://localhost:6553/\n\n[test_section.my_environment]\nname=dev\n# For example to run we'll make both paths relative to current\ntemp_data_dir=.\\temp_data\\${test_section:my_environment:name}\nsource_data_dir=.\n```\n\npython code\n\n```py\nimport typing\nfrom datetime import date, time, datetime\n\nfrom pydantic import BaseModel, DirectoryPath, Field, AnyHttpUrl\n\nfrom config_wrangler.config_data_loaders.base_config_data_loader import BaseConfigDataLoader\nfrom config_wrangler.config_from_ini_env import ConfigFromIniEnv\nfrom config_wrangler.config_from_loaders import ConfigFromLoaders\nfrom config_wrangler.config_templates.config_hierarchy import ConfigHierarchy\nfrom config_wrangler.config_templates.aws.s3_bucket import S3_Bucket\nfrom config_wrangler.config_templates.sqlalchemy_database import SQLAlchemyDatabase\nfrom config_wrangler.config_types.path_types import AutoCreateDirectoryPath\nfrom config_wrangler.config_types.delimited_field import DelimitedListField\n\n\nclass S3_Bucket_KeyPrefixes(S3_Bucket):\n key_prefixes: typing.List[str]\n\n\nclass Environment(ConfigHierarchy):\n name: str = Field(..., env='env_name')\n temp_data_dir: AutoCreateDirectoryPath\n source_data_dir: DirectoryPath\n\n\nclass TestSection(BaseModel):\n my_int: int\n my_float: float\n my_bool: bool\n my_str: str\n my_bytes: bytes\n my_list_auto_c: list\n my_list_auto_nl: list\n my_list_auto_pipe: list\n my_list_python: list\n my_list_json: list\n my_list_c: list = DelimitedListField(delimiter=',')\n my_list_nl: list = DelimitedListField(delimiter='\\n')\n my_list_int_c: typing.List[int] = DelimitedListField(delimiter=',')\n my_tuple_c: tuple = DelimitedListField(delimiter=',')\n my_tuple_nl: tuple = DelimitedListField(delimiter='\\n')\n my_tuple_int_c: typing.Tuple[int, int, int] = DelimitedListField(delimiter=',')\n my_dict: dict\n my_dict_str_int: typing.Dict[str, int]\n my_set: set\n my_set_int: typing.Set[int]\n my_frozenset: frozenset\n my_date: date\n my_time: time\n my_datetime: datetime\n my_url: AnyHttpUrl\n my_environment: Environment\n\n\nclass ETLConfig(ConfigFromIniEnv):\n class Config:\n validate_default = True\n validate_assignment = True\n\n target_database: SQLAlchemyDatabase\n\n s3_source: S3_Bucket_KeyPrefixes\n\n test_section: TestSection\n\n\nclass ETLConfigAnyLoaders(ETLConfig):\n def __init__(\n self,\n _config_data_loaders: typing.List[BaseConfigDataLoader],\n **kwargs: typing.Dict[str, typing.Any]\n ) -> None:\n # Skip super and call the next higher class\n ConfigFromLoaders.__init__(\n self,\n _config_data_loaders=_config_data_loaders,\n **kwargs\n )\n\n\ndef main():\n config = ETLConfig(file_name='simple_example.ini')\n\n print(f\"Temp data dir = {config.test_section.my_environment.temp_data_dir}\")\n # > Temp data dir = temp_data\\dev\n\n print(f\"Source data dir = {config.test_section.my_environment.source_data_dir}\")\n # > Source data dir = .\n\n print(f\"my_int = {config.test_section.my_int}\")\n # > my_int = 123\n\n print(f\"my_float = {config.test_section.my_float}\")\n # > my_float = 123.45\n\n print(f\"my_str = {config.test_section.my_str}\")\n # > my_str = ABC\u2615\n\n print(f\"my_list_auto_c = {config.test_section.my_list_auto_c}\")\n # > my_list_auto_c = ['a', 'b', 'c']\n\n print(f\"my_list_auto_nl = {config.test_section.my_list_auto_nl}\")\n # > my_list_auto_c = ['a', 'b', 'c']\n\n print(f\"my_dict = {config.test_section.my_dict}\")\n # > my_dict = {1: 'One', 2: 'Two'}\n\n print(f\"my_set = {config.test_section.my_set}\")\n # > my_set = {'C', 'A', 'B'}\n\n print(f\"my_time = {config.test_section.my_time}\")\n # > my_time = 11:55:23\n\n print(f\"my_datetime = {config.test_section.my_datetime}\")\n # > my_datetime = 2021-05-31 11:23:53\n\n print(f\"my_url = {config.test_section.my_url}\")\n # > my_url = https://localhost:6553/\n\n # Getting DB engine (requires sqlalchemy optional install\n engine = config.target_database.get_engine()\n print(f\"target_database.engine = {engine}\")\n # > target_database.engine = Engine(sqlite:///.example_db)\n\n print(\"Getting S3 Data\")\n bucket = config.s3_source.get_bucket()\n print(f\"S3 bucket definition = {bucket}\")\n for prefix in config.s3_source.key_prefixes:\n print(f\" bucket search prefix = {prefix}\")\n # > Getting S3 Data\n # > credentials.py:56: UserWarning: Passwords stored directly in config or worse in code are not safe. Please make sure to fix this before deploying.\n # > S3 bucket definitition = s3.Bucket(name='my.exmple-bucket')\n # > bucket search prefix = processed/\n\n\nif __name__ == '__main__':\n main()\n\n```\n",
"bugtrack_url": null,
"license": "MIT",
"summary": "pydantic based configuration wrangler. Handles reading multiple ini or toml files with inheritance rules and variable expansions.",
"version": "1.1.0",
"project_urls": {
"Documentation": "https://bietl.dev/config_wrangler/",
"Homepage": "https://github.com/arcann/config_wrangler",
"Repository": "https://github.com/arcann/config_wrangler"
},
"split_keywords": [],
"urls": [
{
"comment_text": "",
"digests": {
"blake2b_256": "7a1b9bb4c90b0d289a6ea245a26ad3a2c3d191e6f901f96e6142a0d8ac8f81d1",
"md5": "2feb857565432431eac111b3c42f22fe",
"sha256": "b4231e74d969719bfeb92e0cfb5eb4dd9414aa98e3261006cc7888ec588c0c79"
},
"downloads": -1,
"filename": "config_wrangler-1.1.0-py3-none-any.whl",
"has_sig": false,
"md5_digest": "2feb857565432431eac111b3c42f22fe",
"packagetype": "bdist_wheel",
"python_version": "py3",
"requires_python": ">=3.9,<4.0.0",
"size": 63490,
"upload_time": "2024-03-12T18:54:41",
"upload_time_iso_8601": "2024-03-12T18:54:41.737345Z",
"url": "https://files.pythonhosted.org/packages/7a/1b/9bb4c90b0d289a6ea245a26ad3a2c3d191e6f901f96e6142a0d8ac8f81d1/config_wrangler-1.1.0-py3-none-any.whl",
"yanked": false,
"yanked_reason": null
},
{
"comment_text": "",
"digests": {
"blake2b_256": "286383f5296c3c17ee1a2b39dfbd7b32b12887e6ab0e185d54a24a3fc11fd848",
"md5": "86b86e4f6f30607bd24db36e9cf9fb19",
"sha256": "9c3ce4a176cf0ba980c1ea2e02b8e72c160ee3988c8179a1332e67460e081111"
},
"downloads": -1,
"filename": "config_wrangler-1.1.0.tar.gz",
"has_sig": false,
"md5_digest": "86b86e4f6f30607bd24db36e9cf9fb19",
"packagetype": "sdist",
"python_version": "source",
"requires_python": ">=3.9,<4.0.0",
"size": 194340,
"upload_time": "2024-03-12T18:54:44",
"upload_time_iso_8601": "2024-03-12T18:54:44.331504Z",
"url": "https://files.pythonhosted.org/packages/28/63/83f5296c3c17ee1a2b39dfbd7b32b12887e6ab0e185d54a24a3fc11fd848/config_wrangler-1.1.0.tar.gz",
"yanked": false,
"yanked_reason": null
}
],
"upload_time": "2024-03-12 18:54:44",
"github": true,
"gitlab": false,
"bitbucket": false,
"codeberg": false,
"github_user": "arcann",
"github_project": "config_wrangler",
"travis_ci": false,
"coveralls": false,
"github_actions": false,
"tox": true,
"lcname": "config_wrangler"
}
