.. contents:: **sqlitebiter**
:backlinks: top
:depth: 2
Summary
=========
`sqlitebiter <https://github.com/thombashi/sqlitebiter>`__ is a CLI tool to convert CSV / Excel / HTML / JSON / Jupyter Notebook / LDJSON / LTSV / Markdown / SQLite / SSV / TSV / Google-Sheets to a SQLite database file.
.. image:: https://badge.fury.io/py/sqlitebiter.svg
:target: https://badge.fury.io/py/sqlitebiter
:alt: PyPI package version
.. image:: https://img.shields.io/pypi/pyversions/sqlitebiter.svg
:target: https://pypi.org/project/sqlitebiter
:alt: Supported Python versions
.. image:: https://github.com/thombashi/sqlitebiter/actions/workflows/tests.yml/badge.svg
:target: https://github.com/thombashi/sqlitebiter/actions/workflows/tests.yml
:alt: CI status of Linux/macOS/Windows
.. image:: https://github.com/thombashi/sqlitebiter/actions/workflows/github-code-scanning/codeql/badge.svg
:target: https://github.com/thombashi/sqlitebiter/actions/workflows/github-code-scanning/codeql
:alt: CodeQL
Features
--------
- Create a SQLite database file from:
- File(s):
- CSV / Tab separated values (TSV) / Space separated values (SSV)
- Microsoft Excel :superscript:`TM`
- HTML
- JSON
- See also: `Limitations of JSON conversion <https://sqlitebiter.readthedocs.io/en/latest/pages/usage/limitations.html>`_
- `Jupyter Notebook <https://jupyter.org/>`__
- `Labeled Tab-separated Values (LTSV) <http://ltsv.org/>`__
- `Line-delimited JSON(LDJSON) <https://en.wikipedia.org/wiki/JSON_streaming#Line-delimited_JSON>`__ / NDJSON / JSON Lines
- Markdown
- Mediawiki
- SQLite
- `Google Sheets <https://www.google.com/intl/en_us/sheets/about/>`_
- URL (scrape tabular data from web pages)
- Multi-byte character support
- Automatic file encoding detection
Usage
=======
Create SQLite database from files
-----------------------------------
.. image:: https://cdn.jsdelivr.net/gh/thombashi/sqlitebiter@master/docs/svg/usage_example.svg
Create SQLite database from URL
---------------------------------
Following is an example that converts HTML table tags within a web page to SQLite tables by the web page URL.
:Example:
.. code-block:: console
$ sqlitebiter url "https://en.wikipedia.org/wiki/Comparison_of_firewalls"
[INFO] sqlitebiter url: convert 'Comparison_of_firewalls' to 'Comparison_of_firewalls_Wikipedia_html1' table
[INFO] sqlitebiter url: convert 'Comparison_of_firewalls' to 'Comparison_of_firewalls_Wikipedia_html2' table
[INFO] sqlitebiter url: convert 'Comparison_of_firewalls' to 'Comparison_of_firewalls_Wikipedia_html3' table
[INFO] sqlitebiter url: convert 'Comparison_of_firewalls' to 'Comparison_of_firewalls_Wikipedia_html4' table
[INFO] sqlitebiter url: convert 'Comparison_of_firewalls' to 'Comparison_of_firewalls_Wikipedia_html5' table
[INFO] sqlitebiter url: convert 'Comparison_of_firewalls' to 'Comparison_of_firewalls_Wikipedia_html6' table
[INFO] sqlitebiter url: convert 'Comparison_of_firewalls' to 'Comparison_of_firewalls_Wikipedia_html7' table
[INFO] sqlitebiter url: convert 'Comparison_of_firewalls' to 'Comparison_of_firewalls_Wikipedia_html8' table
[INFO] sqlitebiter url: convert 'Comparison_of_firewalls' to 'Comparison_of_firewalls_Wikipedia_html9' table
[INFO] sqlitebiter url: convert 'Comparison_of_firewalls' to 'Comparison_of_firewalls_Wikipedia_html10' table
[INFO] sqlitebiter url: convert 'Comparison_of_firewalls' to 'Comparison_of_firewalls_Wikipedia_html11' table
[INFO] sqlitebiter url: converted results: source=1, success=11, created-table=11
[INFO] sqlitebiter url: database path: out.sqlite
:Output:
.. code-block:: sql
$ sqlite3 out.sqlite .schema
CREATE TABLE IF NOT EXISTS '_source_info_' ("source_id" INTEGER NOT NULL, "dir_name" TEXT, "base_name" TEXT NOT NULL, "format_name" TEXT NOT NULL, "dst_table" TEXT NOT NULL, size INTEGER, mtime INTEGER);
CREATE TABLE IF NOT EXISTS 'Comparison_of_firewalls_Wikipedia_html1' (Firewall TEXT, License TEXT, [Cost and usage limits] TEXT, OS TEXT);
CREATE TABLE IF NOT EXISTS 'Comparison_of_firewalls_Wikipedia_html2' (Firewall TEXT, License TEXT, Cost TEXT, OS TEXT);
CREATE TABLE IF NOT EXISTS 'Comparison_of_firewalls_Wikipedia_html3' ([Can Target:] TEXT, [Changing default policy to accept/reject (by issuing a single rule)] TEXT, [IP destination address(es)] TEXT, [IP source address(es)] TEXT, [TCP/UDP destination port(s)] TEXT, [TCP/UDP source port(s)] TEXT, [Ethernet MAC destination address] TEXT, [Ethernet MAC source address] TEXT, [Inbound firewall (ingress)] TEXT, [Outbound firewall (egress)] TEXT);
CREATE TABLE IF NOT EXISTS 'Comparison_of_firewalls_Wikipedia_html4' ([Can:] TEXT, [work at OSI Layer 4 (stateful firewall)] TEXT, [work at OSI Layer 7 (application inspection)] TEXT, [Change TTL? (Transparent to traceroute)] TEXT, [Configure REJECT-with answer] TEXT, [DMZ (de-militarized zone) - allows for single/several hosts not to be firewalled.] TEXT, [Filter according to time of day] TEXT, [Redirect TCP/UDP ports (port forwarding)] TEXT, [Redirect IP addresses (forwarding)] TEXT, [Filter according to User Authorization] TEXT, [Traffic rate-limit / QoS] TEXT, Tarpit TEXT, Log TEXT);
CREATE TABLE IF NOT EXISTS 'Comparison_of_firewalls_Wikipedia_html5' ([Features:] TEXT, "Configuration: GUI_ text or both modes?" TEXT, "Remote Access: Web (HTTP)_ Telnet_ SSH_ RDP_ Serial COM RS232_ ..." TEXT, [Change rules without requiring restart?] TEXT, [Ability to centrally manage all firewalls together] TEXT);
CREATE TABLE IF NOT EXISTS 'Comparison_of_firewalls_Wikipedia_html6' ([Features:] TEXT, [Modularity: supports third-party modules to extend functionality?] TEXT, [IPS : Intrusion prevention system] TEXT, [Open-Source License?] TEXT, [supports IPv6 ?] TEXT, [Class: Home / Professional] TEXT, [Operating Systems on which it runs?] TEXT);
CREATE TABLE IF NOT EXISTS 'Comparison_of_firewalls_Wikipedia_html7' ([Can:] TEXT, "NAT44 (static_ dynamic w/o ports_ PAT)" TEXT, "NAT64_ NPTv6" TEXT, [IDS (Intrusion Detection System)] TEXT, [VPN (Virtual Private Network)] TEXT, [AV (Anti-Virus)] TEXT, Sniffer TEXT, [Profile selection] TEXT);
CREATE TABLE IF NOT EXISTS 'Comparison_of_firewalls_Wikipedia_html8' ([vteFirewall software] TEXT);
CREATE TABLE IF NOT EXISTS 'Comparison_of_firewalls_Wikipedia_html9' (A TEXT, B TEXT);
CREATE TABLE IF NOT EXISTS 'Comparison_of_firewalls_Wikipedia_html10' (A TEXT, B TEXT);
CREATE TABLE IF NOT EXISTS 'Comparison_of_firewalls_Wikipedia_html11' (A TEXT, B TEXT);
The attributes within the converted SQLite database may include symbols as the above.
Symbols within attributes can be replaced by using ``--replace-symbol`` option.
The following example shows replacing symbols with underscores.
:Example:
.. code-block:: console
$ sqlitebiter --replace-symbol _ -q url "https://en.wikipedia.org/wiki/Comparison_of_firewalls"
:Output:
.. code-block:: sql
$ sqlite3 out.sqlite .schema
CREATE TABLE IF NOT EXISTS '_source_info_' ("source_id" INTEGER NOT NULL, "dir_name" TEXT, "base_name" TEXT NOT NULL, "format_name" TEXT NOT NULL, "dst_table" TEXT NOT NULL, size INTEGER, mtime INTEGER);
CREATE TABLE IF NOT EXISTS 'Comparison_of_firewalls_Wikipedia_html1' (Firewall TEXT, License TEXT, "Cost_and_usage_limits" TEXT, OS TEXT);
CREATE TABLE IF NOT EXISTS 'Comparison_of_firewalls_Wikipedia_html2' (Firewall TEXT, License TEXT, Cost TEXT, OS TEXT);
CREATE TABLE IF NOT EXISTS 'Comparison_of_firewalls_Wikipedia_html3' ("Can_Target" TEXT, "Changing_default_policy_to_accept_reject_by_issuing_a_single_rule" TEXT, "IP_destination_address_es" TEXT, "IP_source_address_es" TEXT, "TCP_UDP_destination_port_s" TEXT, "TCP_UDP_source_port_s" TEXT, "Ethernet_MAC_destination_address" TEXT, "Ethernet_MAC_source_address" TEXT, "Inbound_firewall_ingress" TEXT, "Outbound_firewall_egress" TEXT);
CREATE TABLE IF NOT EXISTS 'Comparison_of_firewalls_Wikipedia_html4' (Can TEXT, "work_at_OSI_Layer_4_stateful_firewall" TEXT, "work_at_OSI_Layer_7_application_inspection" TEXT, "Change_TTL_Transparent_to_traceroute" TEXT, "Configure_REJECT_with_answer" TEXT, "DMZ_de_militarized_zone_allows_for_single_several_hosts_not_to_be_firewalled" TEXT, "Filter_according_to_time_of_day" TEXT, "Redirect_TCP_UDP_ports_port_forwarding" TEXT, "Redirect_IP_addresses_forwarding" TEXT, "Filter_according_to_User_Authorization" TEXT, "Traffic_rate_limit_QoS" TEXT, Tarpit TEXT, Log TEXT);
CREATE TABLE IF NOT EXISTS 'Comparison_of_firewalls_Wikipedia_html5' (Features TEXT, "Configuration_GUI_text_or_both_modes" TEXT, "Remote_Access_Web_HTTP_Telnet_SSH_RDP_Serial_COM_RS232" TEXT, "Change_rules_without_requiring_restart" TEXT, "Ability_to_centrally_manage_all_firewalls_together" TEXT);
CREATE TABLE IF NOT EXISTS 'Comparison_of_firewalls_Wikipedia_html6' (Features TEXT, "Modularity_supports_third_party_modules_to_extend_functionality" TEXT, "IPS _Intrusion_prevention_system" TEXT, "Open_Source_License" TEXT, "supports_IPv6" TEXT, "Class_Home_Professional" TEXT, "Operating_Systems_on_which_it_runs" TEXT);
CREATE TABLE IF NOT EXISTS 'Comparison_of_firewalls_Wikipedia_html7' (Can TEXT, "NAT44_static_dynamic_w_o_ports_PAT" TEXT, "NAT64_NPTv6" TEXT, "IDS_Intrusion_Detection_System" TEXT, "VPN_Virtual_Private_Network" TEXT, "AV_Anti_Virus" TEXT, Sniffer TEXT, "Profile_selection" TEXT);
CREATE TABLE IF NOT EXISTS 'Comparison_of_firewalls_Wikipedia_html8' ("vteFirewall_software" TEXT);
CREATE TABLE IF NOT EXISTS 'Comparison_of_firewalls_Wikipedia_html9' (A TEXT, B TEXT);
CREATE TABLE IF NOT EXISTS 'Comparison_of_firewalls_Wikipedia_html10' (A TEXT, B TEXT);
CREATE TABLE IF NOT EXISTS 'Comparison_of_firewalls_Wikipedia_html11' (A TEXT, B TEXT);
Create SQLite database from stdin
-----------------------------------
Examples
~~~~~~~~
A data format is a mandatory argument for sqlitebiter ``stdin subcommand``:
:Example:
.. code-block:: console
$ echo '[{"hoge": 4, "foo": "abc"}, {"hoge": 12, "foo": "xyz"}]' | sqlitebiter stdin json
[INFO] convert 'stdin' to 'json1' table
[INFO] converted results: source=1, success=1, created-table=1
[INFO] database path: out.sqlite
Command help
--------------
::
Usage: sqlitebiter [OPTIONS] COMMAND [ARGS]...
Options:
-o, --output-path PATH Output path of the SQLite database file.
[default: out.sqlite]
-a, --append Append table(s) to existing database.
--add-primary-key PRIMARY_KEY_NAME
Add 'PRIMARY KEY AUTOINCREMENT' column to a
converted table with the specified name.
--convert-config TEXT [experimental] Configurations for data
conversion. The option can be used only for
url subcommand.
-i, --index INDEX_ATTR Comma separated attribute names to create
indices.
--no-type-inference All of the columns assume as TEXT data type
in creating tables.
--type-hint-header Use header suffixes as type hints. If there
are type hints, convert columns by datatype
corresponding with type hints. The following
suffixes can be recognized as type hints
(case insensitive): "text": TEXT datatype.
"integer": INTEGER datatype. "real": REAL
datatype.
--matrix-formatting [header_aligned|trim]
header_aligned: fitting table data to header
columns. trim: fitting table data to minimum
column size. [default: header_aligned]
--replace-symbol TEXT Replace symbols in attributes.
-v, --verbose Verbosity level. [default: 0]
--max-workers WORKERS Specify the maximum number of workers that
the command may use. [default: 1]
--debug For debug print.
-q, --quiet Suppress execution log messages.
-h, --help Show this message and exit.
Commands:
completion A helper command to setup command completion.
configure Configure the following application settings:
file Convert tabular data within CSV/Excel/HTML/JSON/Jupyter...
gs Convert a spreadsheet in Google Sheets to a SQLite database...
stdin Convert tabular data within CSV/HTML/JSON/Jupyter...
url Scrape tabular data from a URL and convert data to a SQLite...
version Show version information
For more information
~~~~~~~~~~~~~~~~~~~~~~
More examples are available at
https://sqlitebiter.rtfd.io/en/latest/pages/usage/index.html
Installation
============
Installation: pip (Python package manager)
----------------------------------------------------------
::
pip install sqlitebiter
Installation: apt
----------------------------------------------------------------------------
You can install the package by ``apt`` via a Personal Package Archive (`PPA <https://launchpad.net/~thombashi/+archive/ubuntu/ppa>`__):
::
sudo add-apt-repository ppa:thombashi/ppa
sudo apt update
sudo apt install sqlitebiter
Installation: dpkg (``.deb`` package)
----------------------------------------------------------------------------
The following commands will download the latest ``.deb`` package from the `release page <https://github.com/thombashi/sqlitebiter/releases>`__, and install it.
::
curl -sSL https://raw.githubusercontent.com/thombashi/sqlitebiter/master/scripts/installer.sh | sudo bash
Installation: Windows
----------------------------------------------------------
``sqlitebiter`` can be used in Windows environments without Python installation as follows:
#. Navigate to https://github.com/thombashi/sqlitebiter/releases
#. Download the latest version of the ``sqlitebiter_win_x64.zip``
#. Unzip the file
#. Execute ``sqlitebiter.exe`` in either Command Prompt or PowerShell
Installation: Windows (PowerShell)
----------------------------------------------------------
The following commands will download the latest execution binary from the `release page <https://github.com/thombashi/sqlitebiter/releases>`__ to the current directory.
::
wget https://github.com/thombashi/sqlitebiter/raw/master/scripts/get-sqlitebiter.ps1 -OutFile get-sqlitebiter.ps1
Set-ExecutionPolicy Unrestricted -Scope Process -Force; .\get-sqlitebiter.ps1
Installation: brew for macOS
----------------------------------------------------------
.. code:: console
$ brew tap thombashi/sqlitebiter
$ brew install sqlitebiter
- `Homebrew Formula <https://github.com/thombashi/homebrew-sqlitebiter>`__
Command Completion (bash/zsh)
----------------------------------------------------------
.. code:: console
setup command completion for bash:
sqlitebiter completion bash >> ~/.bashrc
setup command completion for zsh:
sqlitebiter completion zsh >> ~/.zshrc
Dependencies
============
Python 3.7+
Python package dependencies
------------------------------------------------------------
- `Mandatory dependencies (automatically installed) <https://github.com/thombashi/DateTimeRange/network/dependencies>`__
Google Sheets dependencies (Optional)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Extra Python packages are required to install to use the Google Sheets feature (`gs` subcommand):
- `gspread <https://github.com/burnash/gspread>`_
- `oauth2client <https://github.com/google/oauth2client/>`_
- `pyOpenSSL <https://pyopenssl.readthedocs.io/en/stable/>`_
The extra packages can be installed with the following `pip` command;
.. code:: console
$ pip install sqlitebiter[gs]
note: binary packages include these dependencies
Misc dependencies (Optional)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
- `lxml <https://lxml.de/installation.html>`__
- `pypandoc <https://github.com/bebraw/pypandoc>`__
- required when converting MediaWiki files
Dependencies other than Python packages (Optional)
------------------------------------------------------------
- ``libxml2`` (faster HTML/Markdown conversion)
- `pandoc <https://pandoc.org/>`__ (required when converting MediaWiki files)
Documentation
===============
https://sqlitebiter.rtfd.io/
Sponsors
====================================
.. image:: https://avatars.githubusercontent.com/u/44389260?s=48&u=6da7176e51ae2654bcfd22564772ef8a3bb22318&v=4
:target: https://github.com/chasbecker
:alt: Charles Becker (chasbecker)
.. image:: https://avatars.githubusercontent.com/u/46711571?s=48&u=57687c0e02d5d6e8eeaf9177f7b7af4c9f275eb5&v=4
:target: https://github.com/Arturi0
:alt: onetime: Arturi0
.. image:: https://avatars.githubusercontent.com/u/3658062?s=48&v=4
:target: https://github.com/b4tman
:alt: onetime: Dmitry Belyaev (b4tman)
`Become a sponsor <https://github.com/sponsors/thombashi>`__
Raw data
{
"_id": null,
"home_page": "https://github.com/thombashi/sqlitebiter",
"name": "sqlitebiter",
"maintainer": "",
"docs_url": null,
"requires_python": ">=3.7",
"maintainer_email": "",
"keywords": "SQLite,converter,CSV,Excel,Google Sheets,HTML,JSON,LTSV,TSV",
"author": "Tsuyoshi Hombashi",
"author_email": "tsuyoshi.hombashi@gmail.com",
"download_url": "https://files.pythonhosted.org/packages/af/df/671b132f798f5732c22a9ad59767039165de8387dcaaba38024165b84f52/sqlitebiter-0.36.3.tar.gz",
"platform": null,
"description": ".. contents:: **sqlitebiter**\n :backlinks: top\n :depth: 2\n\nSummary\n=========\n`sqlitebiter <https://github.com/thombashi/sqlitebiter>`__ is a CLI tool to convert CSV / Excel / HTML / JSON / Jupyter Notebook / LDJSON / LTSV / Markdown / SQLite / SSV / TSV / Google-Sheets to a SQLite database file.\n\n.. image:: https://badge.fury.io/py/sqlitebiter.svg\n :target: https://badge.fury.io/py/sqlitebiter\n :alt: PyPI package version\n\n.. image:: https://img.shields.io/pypi/pyversions/sqlitebiter.svg\n :target: https://pypi.org/project/sqlitebiter\n :alt: Supported Python versions\n\n.. image:: https://github.com/thombashi/sqlitebiter/actions/workflows/tests.yml/badge.svg\n :target: https://github.com/thombashi/sqlitebiter/actions/workflows/tests.yml\n :alt: CI status of Linux/macOS/Windows\n\n.. image:: https://github.com/thombashi/sqlitebiter/actions/workflows/github-code-scanning/codeql/badge.svg\n :target: https://github.com/thombashi/sqlitebiter/actions/workflows/github-code-scanning/codeql\n :alt: CodeQL\n\nFeatures\n--------\n- Create a SQLite database file from:\n - File(s):\n - CSV / Tab separated values (TSV) / Space separated values (SSV)\n - Microsoft Excel :superscript:`TM`\n - HTML\n - JSON\n - See also: `Limitations of JSON conversion <https://sqlitebiter.readthedocs.io/en/latest/pages/usage/limitations.html>`_\n - `Jupyter Notebook <https://jupyter.org/>`__\n - `Labeled Tab-separated Values (LTSV) <http://ltsv.org/>`__\n - `Line-delimited JSON(LDJSON) <https://en.wikipedia.org/wiki/JSON_streaming#Line-delimited_JSON>`__ / NDJSON / JSON Lines\n - Markdown\n - Mediawiki\n - SQLite\n - `Google Sheets <https://www.google.com/intl/en_us/sheets/about/>`_\n - URL (scrape tabular data from web pages)\n- Multi-byte character support\n- Automatic file encoding detection\n\nUsage\n=======\nCreate SQLite database from files\n-----------------------------------\n.. image:: https://cdn.jsdelivr.net/gh/thombashi/sqlitebiter@master/docs/svg/usage_example.svg\n\nCreate SQLite database from URL\n---------------------------------\nFollowing is an example that converts HTML table tags within a web page to SQLite tables by the web page URL.\n\n:Example:\n .. code-block:: console\n\n $ sqlitebiter url \"https://en.wikipedia.org/wiki/Comparison_of_firewalls\"\n [INFO] sqlitebiter url: convert 'Comparison_of_firewalls' to 'Comparison_of_firewalls_Wikipedia_html1' table\n [INFO] sqlitebiter url: convert 'Comparison_of_firewalls' to 'Comparison_of_firewalls_Wikipedia_html2' table\n [INFO] sqlitebiter url: convert 'Comparison_of_firewalls' to 'Comparison_of_firewalls_Wikipedia_html3' table\n [INFO] sqlitebiter url: convert 'Comparison_of_firewalls' to 'Comparison_of_firewalls_Wikipedia_html4' table\n [INFO] sqlitebiter url: convert 'Comparison_of_firewalls' to 'Comparison_of_firewalls_Wikipedia_html5' table\n [INFO] sqlitebiter url: convert 'Comparison_of_firewalls' to 'Comparison_of_firewalls_Wikipedia_html6' table\n [INFO] sqlitebiter url: convert 'Comparison_of_firewalls' to 'Comparison_of_firewalls_Wikipedia_html7' table\n [INFO] sqlitebiter url: convert 'Comparison_of_firewalls' to 'Comparison_of_firewalls_Wikipedia_html8' table\n [INFO] sqlitebiter url: convert 'Comparison_of_firewalls' to 'Comparison_of_firewalls_Wikipedia_html9' table\n [INFO] sqlitebiter url: convert 'Comparison_of_firewalls' to 'Comparison_of_firewalls_Wikipedia_html10' table\n [INFO] sqlitebiter url: convert 'Comparison_of_firewalls' to 'Comparison_of_firewalls_Wikipedia_html11' table\n [INFO] sqlitebiter url: converted results: source=1, success=11, created-table=11\n [INFO] sqlitebiter url: database path: out.sqlite\n\n:Output:\n .. code-block:: sql\n\n $ sqlite3 out.sqlite .schema\n CREATE TABLE IF NOT EXISTS '_source_info_' (\"source_id\" INTEGER NOT NULL, \"dir_name\" TEXT, \"base_name\" TEXT NOT NULL, \"format_name\" TEXT NOT NULL, \"dst_table\" TEXT NOT NULL, size INTEGER, mtime INTEGER);\n CREATE TABLE IF NOT EXISTS 'Comparison_of_firewalls_Wikipedia_html1' (Firewall TEXT, License TEXT, [Cost and usage limits] TEXT, OS TEXT);\n CREATE TABLE IF NOT EXISTS 'Comparison_of_firewalls_Wikipedia_html2' (Firewall TEXT, License TEXT, Cost TEXT, OS TEXT);\n CREATE TABLE IF NOT EXISTS 'Comparison_of_firewalls_Wikipedia_html3' ([Can Target:] TEXT, [Changing default policy to accept/reject (by issuing a single rule)] TEXT, [IP destination address(es)] TEXT, [IP source address(es)] TEXT, [TCP/UDP destination port(s)] TEXT, [TCP/UDP source port(s)] TEXT, [Ethernet MAC destination address] TEXT, [Ethernet MAC source address] TEXT, [Inbound firewall (ingress)] TEXT, [Outbound firewall (egress)] TEXT);\n CREATE TABLE IF NOT EXISTS 'Comparison_of_firewalls_Wikipedia_html4' ([Can:] TEXT, [work at OSI Layer 4 (stateful firewall)] TEXT, [work at OSI Layer 7 (application inspection)] TEXT, [Change TTL? (Transparent to traceroute)] TEXT, [Configure REJECT-with answer] TEXT, [DMZ (de-militarized zone) - allows for single/several hosts not to be firewalled.] TEXT, [Filter according to time of day] TEXT, [Redirect TCP/UDP ports (port forwarding)] TEXT, [Redirect IP addresses (forwarding)] TEXT, [Filter according to User Authorization] TEXT, [Traffic rate-limit / QoS] TEXT, Tarpit TEXT, Log TEXT);\n CREATE TABLE IF NOT EXISTS 'Comparison_of_firewalls_Wikipedia_html5' ([Features:] TEXT, \"Configuration: GUI_ text or both modes?\" TEXT, \"Remote Access: Web (HTTP)_ Telnet_ SSH_ RDP_ Serial COM RS232_ ...\" TEXT, [Change rules without requiring restart?] TEXT, [Ability to centrally manage all firewalls together] TEXT);\n CREATE TABLE IF NOT EXISTS 'Comparison_of_firewalls_Wikipedia_html6' ([Features:] TEXT, [Modularity: supports third-party modules to extend functionality?] TEXT, [IPS\u00a0: Intrusion prevention system] TEXT, [Open-Source License?] TEXT, [supports IPv6\u00a0?] TEXT, [Class: Home / Professional] TEXT, [Operating Systems on which it runs?] TEXT);\n CREATE TABLE IF NOT EXISTS 'Comparison_of_firewalls_Wikipedia_html7' ([Can:] TEXT, \"NAT44 (static_ dynamic w/o ports_ PAT)\" TEXT, \"NAT64_ NPTv6\" TEXT, [IDS (Intrusion Detection System)] TEXT, [VPN (Virtual Private Network)] TEXT, [AV (Anti-Virus)] TEXT, Sniffer TEXT, [Profile selection] TEXT);\n CREATE TABLE IF NOT EXISTS 'Comparison_of_firewalls_Wikipedia_html8' ([vteFirewall software] TEXT);\n CREATE TABLE IF NOT EXISTS 'Comparison_of_firewalls_Wikipedia_html9' (A TEXT, B TEXT);\n CREATE TABLE IF NOT EXISTS 'Comparison_of_firewalls_Wikipedia_html10' (A TEXT, B TEXT);\n CREATE TABLE IF NOT EXISTS 'Comparison_of_firewalls_Wikipedia_html11' (A TEXT, B TEXT);\n\nThe attributes within the converted SQLite database may include symbols as the above.\nSymbols within attributes can be replaced by using ``--replace-symbol`` option.\nThe following example shows replacing symbols with underscores.\n\n:Example:\n .. code-block:: console\n\n $ sqlitebiter --replace-symbol _ -q url \"https://en.wikipedia.org/wiki/Comparison_of_firewalls\"\n\n:Output:\n .. code-block:: sql\n\n $ sqlite3 out.sqlite .schema\n CREATE TABLE IF NOT EXISTS '_source_info_' (\"source_id\" INTEGER NOT NULL, \"dir_name\" TEXT, \"base_name\" TEXT NOT NULL, \"format_name\" TEXT NOT NULL, \"dst_table\" TEXT NOT NULL, size INTEGER, mtime INTEGER);\n CREATE TABLE IF NOT EXISTS 'Comparison_of_firewalls_Wikipedia_html1' (Firewall TEXT, License TEXT, \"Cost_and_usage_limits\" TEXT, OS TEXT);\n CREATE TABLE IF NOT EXISTS 'Comparison_of_firewalls_Wikipedia_html2' (Firewall TEXT, License TEXT, Cost TEXT, OS TEXT);\n CREATE TABLE IF NOT EXISTS 'Comparison_of_firewalls_Wikipedia_html3' (\"Can_Target\" TEXT, \"Changing_default_policy_to_accept_reject_by_issuing_a_single_rule\" TEXT, \"IP_destination_address_es\" TEXT, \"IP_source_address_es\" TEXT, \"TCP_UDP_destination_port_s\" TEXT, \"TCP_UDP_source_port_s\" TEXT, \"Ethernet_MAC_destination_address\" TEXT, \"Ethernet_MAC_source_address\" TEXT, \"Inbound_firewall_ingress\" TEXT, \"Outbound_firewall_egress\" TEXT);\n CREATE TABLE IF NOT EXISTS 'Comparison_of_firewalls_Wikipedia_html4' (Can TEXT, \"work_at_OSI_Layer_4_stateful_firewall\" TEXT, \"work_at_OSI_Layer_7_application_inspection\" TEXT, \"Change_TTL_Transparent_to_traceroute\" TEXT, \"Configure_REJECT_with_answer\" TEXT, \"DMZ_de_militarized_zone_allows_for_single_several_hosts_not_to_be_firewalled\" TEXT, \"Filter_according_to_time_of_day\" TEXT, \"Redirect_TCP_UDP_ports_port_forwarding\" TEXT, \"Redirect_IP_addresses_forwarding\" TEXT, \"Filter_according_to_User_Authorization\" TEXT, \"Traffic_rate_limit_QoS\" TEXT, Tarpit TEXT, Log TEXT);\n CREATE TABLE IF NOT EXISTS 'Comparison_of_firewalls_Wikipedia_html5' (Features TEXT, \"Configuration_GUI_text_or_both_modes\" TEXT, \"Remote_Access_Web_HTTP_Telnet_SSH_RDP_Serial_COM_RS232\" TEXT, \"Change_rules_without_requiring_restart\" TEXT, \"Ability_to_centrally_manage_all_firewalls_together\" TEXT);\n CREATE TABLE IF NOT EXISTS 'Comparison_of_firewalls_Wikipedia_html6' (Features TEXT, \"Modularity_supports_third_party_modules_to_extend_functionality\" TEXT, \"IPS\u00a0_Intrusion_prevention_system\" TEXT, \"Open_Source_License\" TEXT, \"supports_IPv6\" TEXT, \"Class_Home_Professional\" TEXT, \"Operating_Systems_on_which_it_runs\" TEXT);\n CREATE TABLE IF NOT EXISTS 'Comparison_of_firewalls_Wikipedia_html7' (Can TEXT, \"NAT44_static_dynamic_w_o_ports_PAT\" TEXT, \"NAT64_NPTv6\" TEXT, \"IDS_Intrusion_Detection_System\" TEXT, \"VPN_Virtual_Private_Network\" TEXT, \"AV_Anti_Virus\" TEXT, Sniffer TEXT, \"Profile_selection\" TEXT);\n CREATE TABLE IF NOT EXISTS 'Comparison_of_firewalls_Wikipedia_html8' (\"vteFirewall_software\" TEXT);\n CREATE TABLE IF NOT EXISTS 'Comparison_of_firewalls_Wikipedia_html9' (A TEXT, B TEXT);\n CREATE TABLE IF NOT EXISTS 'Comparison_of_firewalls_Wikipedia_html10' (A TEXT, B TEXT);\n CREATE TABLE IF NOT EXISTS 'Comparison_of_firewalls_Wikipedia_html11' (A TEXT, B TEXT);\n\nCreate SQLite database from stdin\n-----------------------------------\nExamples\n~~~~~~~~\nA data format is a mandatory argument for sqlitebiter ``stdin subcommand``:\n\n:Example:\n .. code-block:: console\n\n $ echo '[{\"hoge\": 4, \"foo\": \"abc\"}, {\"hoge\": 12, \"foo\": \"xyz\"}]' | sqlitebiter stdin json\n [INFO] convert 'stdin' to 'json1' table\n [INFO] converted results: source=1, success=1, created-table=1\n [INFO] database path: out.sqlite\n\nCommand help\n--------------\n::\n\n Usage: sqlitebiter [OPTIONS] COMMAND [ARGS]...\n\n Options:\n -o, --output-path PATH Output path of the SQLite database file.\n [default: out.sqlite]\n -a, --append Append table(s) to existing database.\n --add-primary-key PRIMARY_KEY_NAME\n Add 'PRIMARY KEY AUTOINCREMENT' column to a\n converted table with the specified name.\n --convert-config TEXT [experimental] Configurations for data\n conversion. The option can be used only for\n url subcommand.\n -i, --index INDEX_ATTR Comma separated attribute names to create\n indices.\n --no-type-inference All of the columns assume as TEXT data type\n in creating tables.\n --type-hint-header Use header suffixes as type hints. If there\n are type hints, convert columns by datatype\n corresponding with type hints. The following\n suffixes can be recognized as type hints\n (case insensitive): \"text\": TEXT datatype.\n \"integer\": INTEGER datatype. \"real\": REAL\n datatype.\n --matrix-formatting [header_aligned|trim]\n header_aligned: fitting table data to header\n columns. trim: fitting table data to minimum\n column size. [default: header_aligned]\n --replace-symbol TEXT Replace symbols in attributes.\n -v, --verbose Verbosity level. [default: 0]\n --max-workers WORKERS Specify the maximum number of workers that\n the command may use. [default: 1]\n --debug For debug print.\n -q, --quiet Suppress execution log messages.\n -h, --help Show this message and exit.\n\n Commands:\n completion A helper command to setup command completion.\n configure Configure the following application settings:\n file Convert tabular data within CSV/Excel/HTML/JSON/Jupyter...\n gs Convert a spreadsheet in Google Sheets to a SQLite database...\n stdin Convert tabular data within CSV/HTML/JSON/Jupyter...\n url Scrape tabular data from a URL and convert data to a SQLite...\n version Show version information\n\nFor more information\n~~~~~~~~~~~~~~~~~~~~~~\nMore examples are available at \nhttps://sqlitebiter.rtfd.io/en/latest/pages/usage/index.html\n\nInstallation\n============\n\nInstallation: pip (Python package manager)\n----------------------------------------------------------\n::\n\n pip install sqlitebiter\n\n\nInstallation: apt\n----------------------------------------------------------------------------\nYou can install the package by ``apt`` via a Personal Package Archive (`PPA <https://launchpad.net/~thombashi/+archive/ubuntu/ppa>`__):\n\n::\n\n sudo add-apt-repository ppa:thombashi/ppa\n sudo apt update\n sudo apt install sqlitebiter\n\n\nInstallation: dpkg (``.deb`` package)\n----------------------------------------------------------------------------\nThe following commands will download the latest ``.deb`` package from the `release page <https://github.com/thombashi/sqlitebiter/releases>`__, and install it.\n\n::\n\n curl -sSL https://raw.githubusercontent.com/thombashi/sqlitebiter/master/scripts/installer.sh | sudo bash\n\n\nInstallation: Windows\n----------------------------------------------------------\n``sqlitebiter`` can be used in Windows environments without Python installation as follows:\n\n#. Navigate to https://github.com/thombashi/sqlitebiter/releases\n#. Download the latest version of the ``sqlitebiter_win_x64.zip``\n#. Unzip the file\n#. Execute ``sqlitebiter.exe`` in either Command Prompt or PowerShell\n\n\nInstallation: Windows (PowerShell)\n----------------------------------------------------------\nThe following commands will download the latest execution binary from the `release page <https://github.com/thombashi/sqlitebiter/releases>`__ to the current directory.\n\n::\n\n wget https://github.com/thombashi/sqlitebiter/raw/master/scripts/get-sqlitebiter.ps1 -OutFile get-sqlitebiter.ps1\n Set-ExecutionPolicy Unrestricted -Scope Process -Force; .\\get-sqlitebiter.ps1\n\n\nInstallation: brew for macOS\n----------------------------------------------------------\n.. code:: console\n\n $ brew tap thombashi/sqlitebiter\n $ brew install sqlitebiter\n\n- `Homebrew Formula <https://github.com/thombashi/homebrew-sqlitebiter>`__\n\n\nCommand Completion (bash/zsh)\n----------------------------------------------------------\n.. code:: console\n\n setup command completion for bash:\n\n sqlitebiter completion bash >> ~/.bashrc\n\n setup command completion for zsh:\n\n sqlitebiter completion zsh >> ~/.zshrc\n\n\nDependencies\n============\nPython 3.7+\n\nPython package dependencies\n------------------------------------------------------------\n- `Mandatory dependencies (automatically installed) <https://github.com/thombashi/DateTimeRange/network/dependencies>`__\n\nGoogle Sheets dependencies (Optional)\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\nExtra Python packages are required to install to use the Google Sheets feature (`gs` subcommand):\n\n- `gspread <https://github.com/burnash/gspread>`_\n- `oauth2client <https://github.com/google/oauth2client/>`_\n- `pyOpenSSL <https://pyopenssl.readthedocs.io/en/stable/>`_\n\nThe extra packages can be installed with the following `pip` command;\n\n.. code:: console\n\n $ pip install sqlitebiter[gs]\n\nnote: binary packages include these dependencies\n\nMisc dependencies (Optional)\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n- `lxml <https://lxml.de/installation.html>`__\n- `pypandoc <https://github.com/bebraw/pypandoc>`__\n - required when converting MediaWiki files\n\n\nDependencies other than Python packages (Optional)\n------------------------------------------------------------\n- ``libxml2`` (faster HTML/Markdown conversion)\n- `pandoc <https://pandoc.org/>`__ (required when converting MediaWiki files)\n\nDocumentation\n===============\nhttps://sqlitebiter.rtfd.io/\n\nSponsors\n====================================\n.. image:: https://avatars.githubusercontent.com/u/44389260?s=48&u=6da7176e51ae2654bcfd22564772ef8a3bb22318&v=4\n :target: https://github.com/chasbecker\n :alt: Charles Becker (chasbecker)\n.. image:: https://avatars.githubusercontent.com/u/46711571?s=48&u=57687c0e02d5d6e8eeaf9177f7b7af4c9f275eb5&v=4\n :target: https://github.com/Arturi0\n :alt: onetime: Arturi0\n.. image:: https://avatars.githubusercontent.com/u/3658062?s=48&v=4\n :target: https://github.com/b4tman\n :alt: onetime: Dmitry Belyaev (b4tman)\n\n`Become a sponsor <https://github.com/sponsors/thombashi>`__\n\n",
"bugtrack_url": null,
"license": "MIT License",
"summary": "sqlitebiter is a CLI tool to convert CSV / Excel / HTML / JSON / Jupyter Notebook / LDJSON / LTSV / Markdown / SQLite / SSV / TSV / Google-Sheets to a SQLite database file.",
"version": "0.36.3",
"project_urls": {
"Changlog": "https://github.com/thombashi/sqlitebiter/releases",
"Documentation": "https://sqlitebiter.rtfd.io/",
"Download": "https://github.com/thombashi/sqlitebiter/releases",
"Funding": "https://github.com/sponsors/thombashi",
"Homepage": "https://github.com/thombashi/sqlitebiter",
"Source": "https://github.com/thombashi/sqlitebiter",
"Tracker": "https://github.com/thombashi/sqlitebiter/issues"
},
"split_keywords": [
"sqlite",
"converter",
"csv",
"excel",
"google sheets",
"html",
"json",
"ltsv",
"tsv"
],
"urls": [
{
"comment_text": "",
"digests": {
"blake2b_256": "96e9b9b0c21bf519c9efb1c4cbf0cfc2fde6961ac799c9560da4b07d6b59bd7a",
"md5": "49fdbb294b3e3805c6700ba60c7b761a",
"sha256": "fed73ff356429acba335fdd46a9c7a164f96124b7caae828fa5271b3dacb4419"
},
"downloads": -1,
"filename": "sqlitebiter-0.36.3-py3-none-any.whl",
"has_sig": false,
"md5_digest": "49fdbb294b3e3805c6700ba60c7b761a",
"packagetype": "bdist_wheel",
"python_version": "py3",
"requires_python": ">=3.7",
"size": 33205,
"upload_time": "2023-12-02T17:34:16",
"upload_time_iso_8601": "2023-12-02T17:34:16.375297Z",
"url": "https://files.pythonhosted.org/packages/96/e9/b9b0c21bf519c9efb1c4cbf0cfc2fde6961ac799c9560da4b07d6b59bd7a/sqlitebiter-0.36.3-py3-none-any.whl",
"yanked": false,
"yanked_reason": null
},
{
"comment_text": "",
"digests": {
"blake2b_256": "afdf671b132f798f5732c22a9ad59767039165de8387dcaaba38024165b84f52",
"md5": "ecda0f7c17cf830dd22ed6efa3f15585",
"sha256": "9c810741bd6578c4370e4b01f5398b68f42eb26184bc0735d7cafc511f18195a"
},
"downloads": -1,
"filename": "sqlitebiter-0.36.3.tar.gz",
"has_sig": false,
"md5_digest": "ecda0f7c17cf830dd22ed6efa3f15585",
"packagetype": "sdist",
"python_version": "source",
"requires_python": ">=3.7",
"size": 50541,
"upload_time": "2023-12-02T17:34:19",
"upload_time_iso_8601": "2023-12-02T17:34:19.737426Z",
"url": "https://files.pythonhosted.org/packages/af/df/671b132f798f5732c22a9ad59767039165de8387dcaaba38024165b84f52/sqlitebiter-0.36.3.tar.gz",
"yanked": false,
"yanked_reason": null
}
],
"upload_time": "2023-12-02 17:34:19",
"github": true,
"gitlab": false,
"bitbucket": false,
"codeberg": false,
"github_user": "thombashi",
"github_project": "sqlitebiter",
"travis_ci": false,
"coveralls": false,
"github_actions": true,
"tox": true,
"lcname": "sqlitebiter"
}
JSON
