robotexclusionrulesparser

Name	robotexclusionrulesparser JSON
Version	1.7.1 JSON
	download
home_page	http://nikitathespider.com/python/rerp/
Summary	A robots.txt parser alternative to Python's robotparser module
upload_time	2016-08-12 15:53:17
maintainer	None
docs_url	None
author	Philip Semanchuk
requires_python	None
license	http://creativecommons.org/licenses/BSD/
keywords	robots.txt robot parser
VCS
bugtrack_url
requirements	No requirements were recorded.
Travis-CI	No Travis.
coveralls test coverage	No coveralls.

            Robotexclusionrulesparser is an alternative to the Python standard library
module robotparser. It fetches and parses robots.txt files and can answer
questions as to whether or not a given user agent is permitted to visit a 
certain URL.

This module has some features that the standard library module robotparser 
does not, including the ability to decode non-ASCII robots.txt files, respect
for Expires headers and understanding of Crawl-delay and Sitemap directives 
and wildcard syntax in path names.

Complete documentation (including a comparison with the standard library
module robotparser) is available in ReadMe.html.

Robotexclusionrulesparser is released under a BSD license.
            

Raw data

            {
    "_id": null,
    "home_page": "http://nikitathespider.com/python/rerp/",
    "name": "robotexclusionrulesparser",
    "maintainer": null,
    "docs_url": null,
    "requires_python": null,
    "maintainer_email": null,
    "keywords": "robots.txt robot parser",
    "author": "Philip Semanchuk",
    "author_email": "philip@pyspoken.com",
    "download_url": "https://files.pythonhosted.org/packages/39/97/74634de03a0856160a8c2fa92f03cdf1827c3b1d3d42378d4b79119cd9fa/robotexclusionrulesparser-1.7.1.tar.gz",
    "platform": "UNKNOWN",
    "description": "Robotexclusionrulesparser is an alternative to the Python standard library\nmodule robotparser. It fetches and parses robots.txt files and can answer\nquestions as to whether or not a given user agent is permitted to visit a \ncertain URL.\n\nThis module has some features that the standard library module robotparser \ndoes not, including the ability to decode non-ASCII robots.txt files, respect\nfor Expires headers and understanding of Crawl-delay and Sitemap directives \nand wildcard syntax in path names.\n\nComplete documentation (including a comparison with the standard library\nmodule robotparser) is available in ReadMe.html.\n\nRobotexclusionrulesparser is released under a BSD license.",
    "bugtrack_url": null,
    "license": "http://creativecommons.org/licenses/BSD/",
    "summary": "A robots.txt parser alternative to Python's robotparser module",
    "version": "1.7.1",
    "project_urls": {
        "Download": "http://nikitathespider.com/python/rerp/robotexclusionrulesparser-1.7.1.tar.gz",
        "Homepage": "http://nikitathespider.com/python/rerp/"
    },
    "split_keywords": [
        "robots.txt",
        "robot",
        "parser"
    ],
    "urls": [
        {
            "comment_text": "",
            "digests": {
                "blake2b_256": "399774634de03a0856160a8c2fa92f03cdf1827c3b1d3d42378d4b79119cd9fa",
                "md5": "f11ccefc9ec9397db8fc8e62b79c93ef",
                "sha256": "d23aa14ae8145c13c95612d696736bad52a4bd0819ce8c9437ee745098fb8388"
            },
            "downloads": -1,
            "filename": "robotexclusionrulesparser-1.7.1.tar.gz",
            "has_sig": false,
            "md5_digest": "f11ccefc9ec9397db8fc8e62b79c93ef",
            "packagetype": "sdist",
            "python_version": "source",
            "requires_python": null,
            "size": 31494,
            "upload_time": "2016-08-12T15:53:17",
            "upload_time_iso_8601": "2016-08-12T15:53:17.737520Z",
            "url": "https://files.pythonhosted.org/packages/39/97/74634de03a0856160a8c2fa92f03cdf1827c3b1d3d42378d4b79119cd9fa/robotexclusionrulesparser-1.7.1.tar.gz",
            "yanked": false,
            "yanked_reason": null
        }
    ],
    "upload_time": "2016-08-12 15:53:17",
    "github": false,
    "gitlab": false,
    "bitbucket": false,
    "codeberg": false,
    "lcname": "robotexclusionrulesparser"
}