# syncany
简单易用的数据同步转换导出框架。
在构建ETL和报表系统的过程中，存在大量需要从各个子系统收集整理数据的过程，常规写同步脚本非常繁琐。
syncany支持mysql、mongodb、postgresql、redis、elasticsearch、influxdb、clickhouse、execl、beanstalk等数据源读入或写结果数据，同时支持从不同DB不同数据库类型读入关联数据，并对数据进行转换计算，之后配合superset等图形报表框架，快速完成复杂报表分析系统搭建。
[https://pysyncany.readthedocs.io/](https://pysyncany.readthedocs.io/)
# 安装
```
pip3 install syncany
```
# 配置示例
```json
{
"extends": ["examples/demo/json/database.json", "examples/demo/json/log.json"],
"name": "demo",
"input": "<<&.data.demo.json::_id",
"output": ">>&.stdio.&1::site_id use I",
"querys": {
"start_date": {">=": "2021-01-01"}
},
"schema": {
"site_id": ["#yield", "$.sites", [
":#aggregate", "$.*|int", "$$.*|int"
]],
"site_name": ["#yield", "$.sites", [
":#aggregate", "$.*|int", [
"$$.*|int", ["&.data.sites.json::site_id", {"status|int": {">=": 0}}], ":$.name"
]
]],
"site_amount": ["#yield", "$.sites", [
":#aggregate", "$.*|int", [
"$$.*|int", "&.data.orders.json::site_id", [
":#foreach|int", "$.*|array", [
"#if", ["@lte", "$.status", 0], ["#make", {"value": "$.amount"}], "#continue"
], [
":@sum", "$.*|array", "value"
]
]
]
]],
"timeout_at": ["#yield", "$.sites", [
":#aggregate", "$.*|int", {
"#case": "$$$.vip_type",
"1": "$.timeout_at",
"#end": "$$$.rules.:0.timeout_time"
}
]],
"vip_timeout_at": ["#yield", "$.sites", [
":#aggregate", "$.*|int", {
"#match": "$$$.vip_type",
"/2/": "$$.vip_timeout_at",
"#end": "$$$.rules.:0.timeout_time"
}
]]
}
}
```
## 运行示例
```bash
# 克隆并在项目目录下执行即可看到输出，具体输入数据请查看examples/demo/data
python3 ./bin/syncany examples/demo/demo.json --start_date__gte="2021-01-01"
┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
┃ site_id ┃site_name ┃site_amount ┃timeout_at ┃vip_timeout_at ┃
┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩
│ 8 │黄豆网 │17.04 │16:00:00 │11:00:00 │
│ 15 │青菜网 │7.2 │15:00:00 │10:00:00 │
│ 21 │去啥网 │0 │16:00:00 │11:00:00 │
│ 26 │汽车网 │0 │16:00:00 │11:00:00 │
│ 28 │火箭网 │0 │15:00:00 │10:00:00 │
│ 34 │卫星网 │11.2 │16:40:00 │11:20:00 │
└─────────────────────────────┴────────────────────────────────┴───────────────────────────────────────┴───────────────────────────────────┴───────────────────────────────────────────────┘
2021-03-18 17:33:54,570 2377 INFO demo loader: DBLoader <- &.data.demo.json::_id loader_querys: 1 loader_rows: 6
2021-03-18 17:33:54,570 2377 INFO demo join_count: 2 join_querys: 14 join_rows: 13
2021-03-18 17:33:54,571 2377 INFO demo outputer: DBInsertOutputer -> &.stdio.&1::site_id outputer_querys: 0 outputer_operators: 1 outputer_load_rows: 0 outputer_rows: 6
2021-03-18 17:33:54,571 2377 INFO demo finish examples/demo/demo.json demo 96.28ms
```
# License
syncany uses the MIT license, see LICENSE file for the details.
Raw data
{
"_id": null,
"home_page": "https://github.com/snower/syncany",
"name": "syncany",
"maintainer": "",
"docs_url": null,
"requires_python": "",
"maintainer_email": "",
"keywords": "",
"author": "snower",
"author_email": "sujian199@gmail.com",
"download_url": "https://files.pythonhosted.org/packages/f8/4a/fbfb84d7dc3d1a99bb6b22d05406f063a81112e84b9cda609bb1fe91aa4d/syncany-0.2.10.tar.gz",
"platform": null,
"description": "# syncany\n\n\u7b80\u5355\u6613\u7528\u7684\u6570\u636e\u540c\u6b65\u8f6c\u6362\u5bfc\u51fa\u6846\u67b6\u3002\n\n\u5728\u6784\u5efaETL\u548c\u62a5\u8868\u7cfb\u7edf\u7684\u8fc7\u7a0b\u4e2d\uff0c\u5b58\u5728\u5927\u91cf\u9700\u8981\u4ece\u5404\u4e2a\u5b50\u7cfb\u7edf\u6536\u96c6\u6574\u7406\u6570\u636e\u7684\u8fc7\u7a0b\uff0c\u5e38\u89c4\u5199\u540c\u6b65\u811a\u672c\u975e\u5e38\u7e41\u7410\u3002\n\nsyncany\u652f\u6301mysql\u3001mongodb\u3001postgresql\u3001redis\u3001elasticsearch\u3001influxdb\u3001clickhouse\u3001execl\u3001beanstalk\u7b49\u6570\u636e\u6e90\u8bfb\u5165\u6216\u5199\u7ed3\u679c\u6570\u636e\uff0c\u540c\u65f6\u652f\u6301\u4ece\u4e0d\u540cDB\u4e0d\u540c\u6570\u636e\u5e93\u7c7b\u578b\u8bfb\u5165\u5173\u8054\u6570\u636e\uff0c\u5e76\u5bf9\u6570\u636e\u8fdb\u884c\u8f6c\u6362\u8ba1\u7b97\uff0c\u4e4b\u540e\u914d\u5408superset\u7b49\u56fe\u5f62\u62a5\u8868\u6846\u67b6\uff0c\u5feb\u901f\u5b8c\u6210\u590d\u6742\u62a5\u8868\u5206\u6790\u7cfb\u7edf\u642d\u5efa\u3002\n\n[https://pysyncany.readthedocs.io/](https://pysyncany.readthedocs.io/)\n\n# \u5b89\u88c5\n\n```\npip3 install syncany\n```\n\n# \u914d\u7f6e\u793a\u4f8b\n\n\n```json\n{\n \"extends\": [\"examples/demo/json/database.json\", \"examples/demo/json/log.json\"],\n \"name\": \"demo\",\n \"input\": \"<<&.data.demo.json::_id\",\n \"output\": \">>&.stdio.&1::site_id use I\",\n \"querys\": {\n \"start_date\": {\">=\": \"2021-01-01\"}\n },\n \"schema\": {\n \"site_id\": [\"#yield\", \"$.sites\", [\n \":#aggregate\", \"$.*|int\", \"$$.*|int\"\n ]],\n \"site_name\": [\"#yield\", \"$.sites\", [\n \":#aggregate\", \"$.*|int\", [\n \"$$.*|int\", [\"&.data.sites.json::site_id\", {\"status|int\": {\">=\": 0}}], \":$.name\"\n ]\n ]],\n \"site_amount\": [\"#yield\", \"$.sites\", [\n \":#aggregate\", \"$.*|int\", [\n \"$$.*|int\", \"&.data.orders.json::site_id\", [\n \":#foreach|int\", \"$.*|array\", [\n \"#if\", [\"@lte\", \"$.status\", 0], [\"#make\", {\"value\": \"$.amount\"}], \"#continue\"\n ], [\n \":@sum\", \"$.*|array\", \"value\"\n ]\n ]\n ]\n ]],\n \"timeout_at\": [\"#yield\", \"$.sites\", [\n \":#aggregate\", \"$.*|int\", {\n \"#case\": \"$$$.vip_type\",\n \"1\": \"$.timeout_at\",\n \"#end\": \"$$$.rules.:0.timeout_time\"\n }\n ]],\n \"vip_timeout_at\": [\"#yield\", \"$.sites\", [\n \":#aggregate\", \"$.*|int\", {\n \"#match\": \"$$$.vip_type\",\n \"/2/\": \"$$.vip_timeout_at\",\n \"#end\": \"$$$.rules.:0.timeout_time\"\n }\n ]]\n }\n}\n```\n\n## \u8fd0\u884c\u793a\u4f8b\n\n```bash\n# \u514b\u9686\u5e76\u5728\u9879\u76ee\u76ee\u5f55\u4e0b\u6267\u884c\u5373\u53ef\u770b\u5230\u8f93\u51fa\uff0c\u5177\u4f53\u8f93\u5165\u6570\u636e\u8bf7\u67e5\u770bexamples/demo/data\npython3 ./bin/syncany examples/demo/demo.json --start_date__gte=\"2021-01-01\"\n\u250f\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2513\n\u2503 site_id \u2503site_name \u2503site_amount \u2503timeout_at \u2503vip_timeout_at \u2503\n\u2521\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2529\n\u2502 8 \u2502\u9ec4\u8c46\u7f51 \u250217.04 \u250216:00:00 \u250211:00:00 \u2502\n\u2502 15 \u2502\u9752\u83dc\u7f51 \u25027.2 \u250215:00:00 \u250210:00:00 \u2502\n\u2502 21 \u2502\u53bb\u5565\u7f51 \u25020 \u250216:00:00 \u250211:00:00 \u2502\n\u2502 26 \u2502\u6c7d\u8f66\u7f51 \u25020 \u250216:00:00 \u250211:00:00 \u2502\n\u2502 28 \u2502\u706b\u7bad\u7f51 \u25020 \u250215:00:00 \u250210:00:00 \u2502\n\u2502 34 \u2502\u536b\u661f\u7f51 \u250211.2 \u250216:40:00 \u250211:20:00 \u2502\n\u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518\n2021-03-18 17:33:54,570 2377 INFO demo loader: DBLoader <- &.data.demo.json::_id loader_querys: 1 loader_rows: 6\n2021-03-18 17:33:54,570 2377 INFO demo join_count: 2 join_querys: 14 join_rows: 13\n2021-03-18 17:33:54,571 2377 INFO demo outputer: DBInsertOutputer -> &.stdio.&1::site_id outputer_querys: 0 outputer_operators: 1 outputer_load_rows: 0 outputer_rows: 6\n2021-03-18 17:33:54,571 2377 INFO demo finish examples/demo/demo.json demo 96.28ms\n```\n\n# License\n\nsyncany uses the MIT license, see LICENSE file for the details.",
"bugtrack_url": null,
"license": "MIT",
"summary": "\u7b80\u5355\u6613\u7528\u7684\u6570\u636e\u540c\u6b65\u8f6c\u6362\u5bfc\u51fa\u6846\u67b6",
"version": "0.2.10",
"project_urls": {
"Homepage": "https://github.com/snower/syncany"
},
"split_keywords": [],
"urls": [
{
"comment_text": "",
"digests": {
"blake2b_256": "f84afbfb84d7dc3d1a99bb6b22d05406f063a81112e84b9cda609bb1fe91aa4d",
"md5": "a3387dec4f843f92cdefd6d035341d7e",
"sha256": "8b978c7d8a74dd10b10bdc6322d23c49a3e1f3767e9159a7167156ac225d0781"
},
"downloads": -1,
"filename": "syncany-0.2.10.tar.gz",
"has_sig": false,
"md5_digest": "a3387dec4f843f92cdefd6d035341d7e",
"packagetype": "sdist",
"python_version": "source",
"requires_python": null,
"size": 104880,
"upload_time": "2023-05-26T10:43:51",
"upload_time_iso_8601": "2023-05-26T10:43:51.697225Z",
"url": "https://files.pythonhosted.org/packages/f8/4a/fbfb84d7dc3d1a99bb6b22d05406f063a81112e84b9cda609bb1fe91aa4d/syncany-0.2.10.tar.gz",
"yanked": false,
"yanked_reason": null
}
],
"upload_time": "2023-05-26 10:43:51",
"github": true,
"gitlab": false,
"bitbucket": false,
"codeberg": false,
"github_user": "snower",
"github_project": "syncany",
"travis_ci": false,
"coveralls": false,
"github_actions": false,
"lcname": "syncany"
}
JSON
