syncany


Namesyncany JSON
Version 0.2.21 PyPI version JSON
download
home_pagehttps://github.com/snower/syncany
Summary简单易用的数据同步转换导出框架
upload_time2024-03-22 10:45:27
maintainerNone
docs_urlNone
authorsnower
requires_pythonNone
licenseMIT
keywords
VCS
bugtrack_url
requirements No requirements were recorded.
Travis-CI No Travis.
coveralls test coverage No coveralls.
            # syncany

简单易用的数据同步转换导出框架。

在构建ETL和报表系统的过程中,存在大量需要从各个子系统收集整理数据的过程,常规写同步脚本非常繁琐。

syncany支持mysql、mongodb、postgresql、redis、elasticsearch、influxdb、clickhouse、execl、beanstalk等数据源读入或写结果数据,同时支持从不同DB不同数据库类型读入关联数据,并对数据进行转换计算,之后配合superset等图形报表框架,快速完成复杂报表分析系统搭建。

[https://pysyncany.readthedocs.io/](https://pysyncany.readthedocs.io/)

# 安装

```
pip3 install syncany
```

# 配置示例


```json
{
  "extends": ["examples/demo/json/database.json", "examples/demo/json/log.json"],
  "name": "demo",
  "input": "<<&.data.demo.json::_id",
  "output": ">>&.stdio.&1::site_id use I",
  "querys": {
    "start_date": {">=": "2021-01-01"}
  },
  "schema": {
    "site_id": ["#yield", "$.sites", [
      ":#aggregate", "$.*|int", "$$.*|int"
    ]],
    "site_name": ["#yield", "$.sites", [
      ":#aggregate", "$.*|int", [
        "$$.*|int", ["&.data.sites.json::site_id", {"status|int": {">=": 0}}], ":$.name"
      ]
    ]],
    "site_amount": ["#yield", "$.sites", [
      ":#aggregate", "$.*|int", [
        "$$.*|int", "&.data.orders.json::site_id", [
          ":#foreach|int", "$.*|array", [
            "#if", ["@lte", "$.status", 0], ["#make", {"value": "$.amount"}], "#continue"
          ], [
            ":@sum", "$.*|array", "value"
          ]
        ]
      ]
    ]],
    "timeout_at": ["#yield", "$.sites", [
      ":#aggregate", "$.*|int", {
        "#case": "$$$.vip_type",
        "1": "$.timeout_at",
        "#end": "$$$.rules.:0.timeout_time"
      }
    ]],
    "vip_timeout_at": ["#yield", "$.sites", [
      ":#aggregate", "$.*|int", {
        "#match": "$$$.vip_type",
        "/2/": "$$.vip_timeout_at",
        "#end": "$$$.rules.:0.timeout_time"
      }
    ]]
  }
}
```

## 运行示例

```bash
# 克隆并在项目目录下执行即可看到输出,具体输入数据请查看examples/demo/data
python3 ./bin/syncany examples/demo/demo.json --start_date__gte="2021-01-01"
┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
┃ site_id                     ┃site_name                       ┃site_amount                            ┃timeout_at                         ┃vip_timeout_at                                 ┃
┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩
│ 8                           │黄豆网                          │17.04                                  │16:00:00                           │11:00:00                                       │
│ 15                          │青菜网                          │7.2                                    │15:00:00                           │10:00:00                                       │
│ 21                          │去啥网                          │0                                      │16:00:00                           │11:00:00                                       │
│ 26                          │汽车网                          │0                                      │16:00:00                           │11:00:00                                       │
│ 28                          │火箭网                          │0                                      │15:00:00                           │10:00:00                                       │
│ 34                          │卫星网                          │11.2                                   │16:40:00                           │11:20:00                                       │
└─────────────────────────────┴────────────────────────────────┴───────────────────────────────────────┴───────────────────────────────────┴───────────────────────────────────────────────┘
2021-03-18 17:33:54,570 2377 INFO demo loader: DBLoader <- &.data.demo.json::_id loader_querys: 1 loader_rows: 6
2021-03-18 17:33:54,570 2377 INFO demo join_count: 2 join_querys: 14 join_rows: 13
2021-03-18 17:33:54,571 2377 INFO demo outputer: DBInsertOutputer -> &.stdio.&1::site_id outputer_querys: 0 outputer_operators: 1 outputer_load_rows: 0 outputer_rows: 6
2021-03-18 17:33:54,571 2377 INFO demo finish examples/demo/demo.json demo 96.28ms
```

# License

syncany uses the MIT license, see LICENSE file for the details.
            

Raw data

            {
    "_id": null,
    "home_page": "https://github.com/snower/syncany",
    "name": "syncany",
    "maintainer": null,
    "docs_url": null,
    "requires_python": null,
    "maintainer_email": null,
    "keywords": null,
    "author": "snower",
    "author_email": "sujian199@gmail.com",
    "download_url": "https://files.pythonhosted.org/packages/57/e1/1e5aa26baa0d58b3b0610a34a9dee51e7f4bae2797640a9701203e028f82/syncany-0.2.21.tar.gz",
    "platform": null,
    "description": "# syncany\n\n\u7b80\u5355\u6613\u7528\u7684\u6570\u636e\u540c\u6b65\u8f6c\u6362\u5bfc\u51fa\u6846\u67b6\u3002\n\n\u5728\u6784\u5efaETL\u548c\u62a5\u8868\u7cfb\u7edf\u7684\u8fc7\u7a0b\u4e2d\uff0c\u5b58\u5728\u5927\u91cf\u9700\u8981\u4ece\u5404\u4e2a\u5b50\u7cfb\u7edf\u6536\u96c6\u6574\u7406\u6570\u636e\u7684\u8fc7\u7a0b\uff0c\u5e38\u89c4\u5199\u540c\u6b65\u811a\u672c\u975e\u5e38\u7e41\u7410\u3002\n\nsyncany\u652f\u6301mysql\u3001mongodb\u3001postgresql\u3001redis\u3001elasticsearch\u3001influxdb\u3001clickhouse\u3001execl\u3001beanstalk\u7b49\u6570\u636e\u6e90\u8bfb\u5165\u6216\u5199\u7ed3\u679c\u6570\u636e\uff0c\u540c\u65f6\u652f\u6301\u4ece\u4e0d\u540cDB\u4e0d\u540c\u6570\u636e\u5e93\u7c7b\u578b\u8bfb\u5165\u5173\u8054\u6570\u636e\uff0c\u5e76\u5bf9\u6570\u636e\u8fdb\u884c\u8f6c\u6362\u8ba1\u7b97\uff0c\u4e4b\u540e\u914d\u5408superset\u7b49\u56fe\u5f62\u62a5\u8868\u6846\u67b6\uff0c\u5feb\u901f\u5b8c\u6210\u590d\u6742\u62a5\u8868\u5206\u6790\u7cfb\u7edf\u642d\u5efa\u3002\n\n[https://pysyncany.readthedocs.io/](https://pysyncany.readthedocs.io/)\n\n# \u5b89\u88c5\n\n```\npip3 install syncany\n```\n\n# \u914d\u7f6e\u793a\u4f8b\n\n\n```json\n{\n  \"extends\": [\"examples/demo/json/database.json\", \"examples/demo/json/log.json\"],\n  \"name\": \"demo\",\n  \"input\": \"<<&.data.demo.json::_id\",\n  \"output\": \">>&.stdio.&1::site_id use I\",\n  \"querys\": {\n    \"start_date\": {\">=\": \"2021-01-01\"}\n  },\n  \"schema\": {\n    \"site_id\": [\"#yield\", \"$.sites\", [\n      \":#aggregate\", \"$.*|int\", \"$$.*|int\"\n    ]],\n    \"site_name\": [\"#yield\", \"$.sites\", [\n      \":#aggregate\", \"$.*|int\", [\n        \"$$.*|int\", [\"&.data.sites.json::site_id\", {\"status|int\": {\">=\": 0}}], \":$.name\"\n      ]\n    ]],\n    \"site_amount\": [\"#yield\", \"$.sites\", [\n      \":#aggregate\", \"$.*|int\", [\n        \"$$.*|int\", \"&.data.orders.json::site_id\", [\n          \":#foreach|int\", \"$.*|array\", [\n            \"#if\", [\"@lte\", \"$.status\", 0], [\"#make\", {\"value\": \"$.amount\"}], \"#continue\"\n          ], [\n            \":@sum\", \"$.*|array\", \"value\"\n          ]\n        ]\n      ]\n    ]],\n    \"timeout_at\": [\"#yield\", \"$.sites\", [\n      \":#aggregate\", \"$.*|int\", {\n        \"#case\": \"$$$.vip_type\",\n        \"1\": \"$.timeout_at\",\n        \"#end\": \"$$$.rules.:0.timeout_time\"\n      }\n    ]],\n    \"vip_timeout_at\": [\"#yield\", \"$.sites\", [\n      \":#aggregate\", \"$.*|int\", {\n        \"#match\": \"$$$.vip_type\",\n        \"/2/\": \"$$.vip_timeout_at\",\n        \"#end\": \"$$$.rules.:0.timeout_time\"\n      }\n    ]]\n  }\n}\n```\n\n## \u8fd0\u884c\u793a\u4f8b\n\n```bash\n# \u514b\u9686\u5e76\u5728\u9879\u76ee\u76ee\u5f55\u4e0b\u6267\u884c\u5373\u53ef\u770b\u5230\u8f93\u51fa\uff0c\u5177\u4f53\u8f93\u5165\u6570\u636e\u8bf7\u67e5\u770bexamples/demo/data\npython3 ./bin/syncany examples/demo/demo.json --start_date__gte=\"2021-01-01\"\n\u250f\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2513\n\u2503 site_id                     \u2503site_name                       \u2503site_amount                            \u2503timeout_at                         \u2503vip_timeout_at                                 \u2503\n\u2521\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2529\n\u2502 8                           \u2502\u9ec4\u8c46\u7f51                          \u250217.04                                  \u250216:00:00                           \u250211:00:00                                       \u2502\n\u2502 15                          \u2502\u9752\u83dc\u7f51                          \u25027.2                                    \u250215:00:00                           \u250210:00:00                                       \u2502\n\u2502 21                          \u2502\u53bb\u5565\u7f51                          \u25020                                      \u250216:00:00                           \u250211:00:00                                       \u2502\n\u2502 26                          \u2502\u6c7d\u8f66\u7f51                          \u25020                                      \u250216:00:00                           \u250211:00:00                                       \u2502\n\u2502 28                          \u2502\u706b\u7bad\u7f51                          \u25020                                      \u250215:00:00                           \u250210:00:00                                       \u2502\n\u2502 34                          \u2502\u536b\u661f\u7f51                          \u250211.2                                   \u250216:40:00                           \u250211:20:00                                       \u2502\n\u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518\n2021-03-18 17:33:54,570 2377 INFO demo loader: DBLoader <- &.data.demo.json::_id loader_querys: 1 loader_rows: 6\n2021-03-18 17:33:54,570 2377 INFO demo join_count: 2 join_querys: 14 join_rows: 13\n2021-03-18 17:33:54,571 2377 INFO demo outputer: DBInsertOutputer -> &.stdio.&1::site_id outputer_querys: 0 outputer_operators: 1 outputer_load_rows: 0 outputer_rows: 6\n2021-03-18 17:33:54,571 2377 INFO demo finish examples/demo/demo.json demo 96.28ms\n```\n\n# License\n\nsyncany uses the MIT license, see LICENSE file for the details.",
    "bugtrack_url": null,
    "license": "MIT",
    "summary": "\u7b80\u5355\u6613\u7528\u7684\u6570\u636e\u540c\u6b65\u8f6c\u6362\u5bfc\u51fa\u6846\u67b6",
    "version": "0.2.21",
    "project_urls": {
        "Homepage": "https://github.com/snower/syncany"
    },
    "split_keywords": [],
    "urls": [
        {
            "comment_text": "",
            "digests": {
                "blake2b_256": "57e11e5aa26baa0d58b3b0610a34a9dee51e7f4bae2797640a9701203e028f82",
                "md5": "5973e2c555af9d1845f2f218beb1beb8",
                "sha256": "6711b30e519d27c101c56e943cc63fc70257d57db8d4c6ba653ed92a1e571137"
            },
            "downloads": -1,
            "filename": "syncany-0.2.21.tar.gz",
            "has_sig": false,
            "md5_digest": "5973e2c555af9d1845f2f218beb1beb8",
            "packagetype": "sdist",
            "python_version": "source",
            "requires_python": null,
            "size": 114337,
            "upload_time": "2024-03-22T10:45:27",
            "upload_time_iso_8601": "2024-03-22T10:45:27.711911Z",
            "url": "https://files.pythonhosted.org/packages/57/e1/1e5aa26baa0d58b3b0610a34a9dee51e7f4bae2797640a9701203e028f82/syncany-0.2.21.tar.gz",
            "yanked": false,
            "yanked_reason": null
        }
    ],
    "upload_time": "2024-03-22 10:45:27",
    "github": true,
    "gitlab": false,
    "bitbucket": false,
    "codeberg": false,
    "github_user": "snower",
    "github_project": "syncany",
    "travis_ci": false,
    "coveralls": false,
    "github_actions": false,
    "lcname": "syncany"
}
        
Elapsed time: 0.34164s