Replaces the built-in robotsparser with a
RFC-conformant implementation that supports modern robots.txt constructs like
Sitemaps, Allow, and Crawl-delay. Main features:
- Memoization of fetched robots.txt
- Expiration taken from the `Expires` header
- Batch queries
- Configurable user agent for fetching robots.txt
- Automatic refetching basing on expiration
This is a patched fork of the last pure Python version that
works on Python 2 and 3.
Raw data
{
"_id": null,
"home_page": "http://github.com/seomoz/reppy",
"name": "reppy2",
"maintainer": "",
"docs_url": null,
"requires_python": "",
"maintainer_email": "",
"keywords": "utilities",
"author": "Dan Lecocq",
"author_email": "dan@moz.com",
"download_url": "https://files.pythonhosted.org/packages/5d/f7/26cea5ae51939790a449fd70ca2a85376eda1ceaa9970bac18dced21b0ef/reppy2-0.3.6.tar.gz",
"platform": "",
"description": "Replaces the built-in robotsparser with a\nRFC-conformant implementation that supports modern robots.txt constructs like\nSitemaps, Allow, and Crawl-delay. Main features:\n\n- Memoization of fetched robots.txt\n- Expiration taken from the `Expires` header\n- Batch queries\n- Configurable user agent for fetching robots.txt\n- Automatic refetching basing on expiration\n\nThis is a patched fork of the last pure Python version that\nworks on Python 2 and 3.\n\n",
"bugtrack_url": null,
"license": "MIT License",
"summary": "Replacement robots.txt Parser in pure Python",
"version": "0.3.6",
"project_urls": {
"Homepage": "http://github.com/seomoz/reppy"
},
"split_keywords": [
"utilities"
],
"urls": [
{
"comment_text": "",
"digests": {
"blake2b_256": "d981b5d136977d6b297ff7256a96b171fd15165121888f822d9643ff30d5c5b9",
"md5": "fced09a4f38aa974b995802ce1fcfd67",
"sha256": "b05d6a600834c663a9603658ca3b2f440c9b0353372f728e4dd5f81f37eeb61c"
},
"downloads": -1,
"filename": "reppy2-0.3.6-py3-none-any.whl",
"has_sig": false,
"md5_digest": "fced09a4f38aa974b995802ce1fcfd67",
"packagetype": "bdist_wheel",
"python_version": "py3",
"requires_python": null,
"size": 12233,
"upload_time": "2021-10-04T23:31:04",
"upload_time_iso_8601": "2021-10-04T23:31:04.682918Z",
"url": "https://files.pythonhosted.org/packages/d9/81/b5d136977d6b297ff7256a96b171fd15165121888f822d9643ff30d5c5b9/reppy2-0.3.6-py3-none-any.whl",
"yanked": false,
"yanked_reason": null
},
{
"comment_text": "",
"digests": {
"blake2b_256": "5df726cea5ae51939790a449fd70ca2a85376eda1ceaa9970bac18dced21b0ef",
"md5": "bf1d531ff152f8ad8d9719e2789ca579",
"sha256": "e1113935d24f01aaca474b513655660bd1d45dda96b0300163ec32d0c98ca5fc"
},
"downloads": -1,
"filename": "reppy2-0.3.6.tar.gz",
"has_sig": false,
"md5_digest": "bf1d531ff152f8ad8d9719e2789ca579",
"packagetype": "sdist",
"python_version": "source",
"requires_python": null,
"size": 72687,
"upload_time": "2021-10-04T23:31:06",
"upload_time_iso_8601": "2021-10-04T23:31:06.317789Z",
"url": "https://files.pythonhosted.org/packages/5d/f7/26cea5ae51939790a449fd70ca2a85376eda1ceaa9970bac18dced21b0ef/reppy2-0.3.6.tar.gz",
"yanked": false,
"yanked_reason": null
}
],
"upload_time": "2021-10-04 23:31:06",
"github": true,
"gitlab": false,
"bitbucket": false,
"codeberg": false,
"github_user": "seomoz",
"github_project": "reppy",
"travis_ci": true,
"coveralls": true,
"github_actions": false,
"requirements": [
{
"name": "cachetools",
"specs": [
[
"==",
"3.0.0"
]
]
},
{
"name": "requests",
"specs": [
[
"==",
"2.10.0"
]
]
},
{
"name": "six",
"specs": [
[
"==",
"1.10.0"
]
]
},
{
"name": "python-dateutil",
"specs": [
[
"==",
"2.5.3"
]
]
},
{
"name": "Cython",
"specs": [
[
"==",
"0.29.14"
]
]
},
{
"name": "mock",
"specs": [
[
"==",
"4.0.1"
]
]
},
{
"name": "requests_mock",
"specs": [
[
"==",
"1.7.0"
]
]
},
{
"name": "nose",
"specs": [
[
"==",
"1.3.7"
]
]
},
{
"name": "colorama",
"specs": [
[
"==",
"0.4.3"
]
]
},
{
"name": "python-termstyle",
"specs": [
[
"==",
"0.1.10"
]
]
},
{
"name": "rednose",
"specs": [
[
"==",
"1.2.1"
]
]
}
],
"lcname": "reppy2"
}
JSON
