| Name | twscrapeplus JSON |
| Version |
0.2.0
JSON |
| download |
| home_page | None |
| Summary | Twitter GraphQL and Search API implementation with SNScrape data models |
| upload_time | 2025-03-19 02:52:58 |
| maintainer | None |
| docs_url | None |
| author | None |
| requires_python | >=3.10 |
| license | MIT |
| keywords |
api
scrape
scrapper
snscrape
twitter
|
| VCS |
 |
| bugtrack_url |
|
| requirements |
No requirements were recorded.
|
| Travis-CI |
No Travis.
|
| coveralls test coverage |
No coveralls.
|
# twscrape
<div align="center">
[<img src="https://badges.ws/pypi/v/twscrape" alt="version" />](https://pypi.org/project/twscrape)
[<img src="https://badges.ws/pypi/python/twscrape" alt="py versions" />](https://pypi.org/project/twscrape)
[<img src="https://badges.ws/pypi/dm/twscrape" alt="downloads" />](https://pypi.org/project/twscrape)
[<img src="https://badges.ws/github/license/vladkens/twscrape" alt="license" />](https://github.com/vladkens/twscrape/blob/main/LICENSE)
[<img src="https://badges.ws/badge/-/buy%20me%20a%20coffee/ff813f?icon=buymeacoffee&label" alt="donate" />](https://buymeacoffee.com/vladkens)
</div>
Twitter GraphQL API implementation with [SNScrape](https://github.com/JustAnotherArchivist/snscrape) data models.
<div align="center">
<img src=".github/example.png" alt="example of cli usage" height="400px">
</div>
## Install
```bash
pip install twscrape
```
Or development version:
```bash
pip install git+https://github.com/vladkens/twscrape.git
```
## Features
- Support both Search & GraphQL Twitter API
- Async/Await functions (can run multiple scrapers in parallel at the same time)
- Login flow (with receiving verification code from email)
- Saving/restoring account sessions
- Raw Twitter API responses & SNScrape models
- Automatic account switching to smooth Twitter API rate limits
- Direct messages (DM) support
- Media upload support for tweets and DMs
- Account status monitoring
## Usage
This project requires authorized X/Twitter accounts to work with the API. You have two options:
1. **Create Your Own Account**: While you can register a new account on X/Twitter yourself, it's can be difficult due to strict verification processes and high ban rates.
2. **Use Ready Accounts**: For immediate access, you can get ready-to-use accounts with cookies from [our recommended provider](https://kutt.it/ueeM5f). Cookie-based accounts typically have fewer login issues.
For optimal performance and to avoid IP-based restrictions, we also recommend using proxies from [our provider](https://kutt.it/eb3rXk).
**Disclaimer**: While X/Twitter's Terms of Service discourage using multiple accounts, this is a common practice for data collection and research purposes. Use responsibly and at your own discretion.
```python
import asyncio
from twscrape import API, gather
from twscrape.logger import set_log_level
async def main():
api = API() # or API("path-to.db") – default is `accounts.db`
# ADD ACCOUNTS (for CLI usage see next readme section)
# Option 1. Adding account with cookies (more stable)
cookies = "abc=12; ct0=xyz" # or '{"abc": "12", "ct0": "xyz"}'
await api.pool.add_account("user3", "pass3", "u3@mail.com", "mail_pass3", cookies=cookies)
# Option2. Adding account with login / password (less stable)
# email login / password required to receive the verification code via IMAP protocol
# (not all email providers are supported, e.g. ProtonMail)
await api.pool.add_account("user1", "pass1", "u1@example.com", "mail_pass1")
await api.pool.add_account("user2", "pass2", "u2@example.com", "mail_pass2")
await api.pool.login_all() # try to login to receive account cookies
# API USAGE
# search (latest tab)
await gather(api.search("elon musk", limit=20)) # list[Tweet]
# change search tab (product), can be: Top, Latest (default), Media
await gather(api.search("elon musk", limit=20, kv={"product": "Top"}))
# tweet info
tweet_id = 20
await api.tweet_details(tweet_id) # Tweet
await gather(api.retweeters(tweet_id, limit=20)) # list[User]
# Note: this method have small pagination from X side, like 5 tweets per query
await gather(api.tweet_replies(tweet_id, limit=20)) # list[Tweet]
# get user by login
user_login = "xdevelopers"
await api.user_by_login(user_login) # User
# user info
user_id = 2244994945
await api.user_by_id(user_id) # User
await gather(api.following(user_id, limit=20)) # list[User]
await gather(api.followers(user_id, limit=20)) # list[User]
await gather(api.verified_followers(user_id, limit=20)) # list[User]
await gather(api.subscriptions(user_id, limit=20)) # list[User]
await gather(api.user_tweets(user_id, limit=20)) # list[Tweet]
await gather(api.user_tweets_and_replies(user_id, limit=20)) # list[Tweet]
await gather(api.user_media(user_id, limit=20)) # list[Tweet]
# list info
await gather(api.list_timeline(list_id=123456789))
# trends
await gather(api.trends("news")) # list[Trend]
await gather(api.trends("sport")) # list[Trend]
await gather(api.trends("VGltZWxpbmU6DAC2CwABAAAACHRyZW5kaW5nAAA")) # list[Trend]
# Send direct message (DM)
# 123456789 is the user ID of the recipient
dm_result = await api.dm("Hello from twscrape!", [123456789])
print(dm_result)
# Send DM with media
dm_with_media = await api.dm("Check this image!", [123456789], media="path/to/image.jpg")
print(dm_with_media)
# Upload media for other purposes
media_id = await api._upload_media("path/to/image.jpg")
print(f"Uploaded media ID: {media_id}")
# Add alternative text to media
alt_text_result = await api._add_alt_text(media_id, "Description of the image")
print(alt_text_result)
# Check account status
status = await api.account_status()
print(f"Total accounts: {status['total']}")
print(f"Active accounts: {status['active']}")
print(f"Locks: {status['locks']}")
# Reset account locks if needed
await api.reset_locks()
# NOTE 1: gather is a helper function to receive all data as list, FOR can be used as well:
async for tweet in api.search("elon musk"):
print(tweet.id, tweet.user.username, tweet.rawContent) # tweet is `Tweet` object
# NOTE 2: all methods have `raw` version (returns `httpx.Response` object):
async for rep in api.search_raw("elon musk"):
print(rep.status_code, rep.json()) # rep is `httpx.Response` object
# change log level, default info
set_log_level("DEBUG")
# Tweet & User model can be converted to regular dict or json, e.g.:
doc = await api.user_by_id(user_id) # User
doc.dict() # -> python dict
doc.json() # -> json string
if __name__ == "__main__":
asyncio.run(main())
```
### Stoping iteration with break
In order to correctly release an account in case of `break` in loop, a special syntax must be used. Otherwise, Python's events loop will release lock on the account sometime in the future. See explanation [here](https://github.com/vladkens/twscrape/issues/27#issuecomment-1623395424).
```python
from contextlib import aclosing
async with aclosing(api.search("elon musk")) as gen:
async for tweet in gen:
if tweet.id < 200:
break
```
## CLI
### Get help on CLI commands
```sh
# show all commands
twscrape
# help on specific comand
twscrape search --help
```
### Add accounts
To add accounts use `add_accounts` command. Command syntax is:
```sh
twscrape add_accounts <file_path> <line_format>
```
Where:
`<line_format>` is format of line if accounts file splited by delimeter. Possible tokens:
- `username` – required
- `password` – required
- `email` – required
- `email_password` – to receive email code (you can use `--manual` mode to get code)
- `cookies` – can be any parsable format (string, json, base64 string, etc)
- `_` – skip column from parse
Tokens should be splited by delimeter, usually "`:`" used.
Example:
I have account files named `order-12345.txt` with format:
```text
username:password:email:email password:user_agent:cookies
```
Command to add accounts will be (user_agent column skiped with `_`):
```sh
twscrape add_accounts ./order-12345.txt username:password:email:email_password:_:cookies
```
### Login accounts
_Note:_ If you added accounts with cookies, login not required.
Run:
```sh
twscrape login_accounts
```
`twscrape` will start login flow for each new account. If X will ask to verify email and you provided `email_password` in `add_account`, then `twscrape` will try to receive verification code by IMAP protocol. After success login account cookies will be saved to db file for future use.
#### Manual email verification
In case your email provider not support IMAP protocol (ProtonMail, Tutanota, etc) or IMAP is disabled in settings, you can enter email verification code manually. To do this run login command with `--manual` flag.
Example:
```sh
twscrape login_accounts --manual
twscrape relogin user1 user2 --manual
twscrape relogin_failed --manual
```
### Get list of accounts and their statuses
```sh
twscrape accounts
# Output:
# username logged_in active last_used total_req error_msg
# user1 True True 2023-05-20 03:20:40 100 None
# user2 True True 2023-05-20 03:25:45 120 None
# user3 False False None 120 Login error
```
### Re-login accounts
It is possible to re-login specific accounts:
```sh
twscrape relogin user1 user2
```
Or retry login for all failed logins:
```sh
twscrape relogin_failed
```
### Use different accounts file
Useful if using a different set of accounts for different actions
```
twscrape --db test-accounts.db <command>
```
### Search commands
```sh
twscrape search "QUERY" --limit=20
twscrape tweet_details TWEET_ID
twscrape tweet_replies TWEET_ID --limit=20
twscrape retweeters TWEET_ID --limit=20
twscrape user_by_id USER_ID
twscrape user_by_login USERNAME
twscrape user_media USER_ID --limit=20
twscrape following USER_ID --limit=20
twscrape followers USER_ID --limit=20
twscrape verified_followers USER_ID --limit=20
twscrape subscriptions USER_ID --limit=20
twscrape user_tweets USER_ID --limit=20
twscrape user_tweets_and_replies USER_ID --limit=20
twscrape trends sport
```
The default output is in the console (stdout), one document per line. So it can be redirected to the file.
```sh
twscrape search "elon mask lang:es" --limit=20 > data.txt
```
By default, parsed data is returned. The original tweet responses can be retrieved with `--raw` flag.
```sh
twscrape search "elon mask lang:es" --limit=20 --raw
```
### About `limit` param
X API works through pagination, each API method can have different defaults for per page parameter (and this parameter can't be changed by caller). So `limit` param in `twscrape` is the desired number of objects (tweets or users, depending on the method). `twscrape` tries to return NO LESS objects than requested. If the X API returns less or more objects, `twscrape` will return whatever X gives.
## Proxy
There are few options to use proxies.
1. You can add proxy per account
```py
proxy = "http://login:pass@example.com:8080"
await api.pool.add_account("user4", "pass4", "u4@mail.com", "mail_pass4", proxy=proxy)
```
2. You can use global proxy for all accounts
```py
proxy = "http://login:pass@example.com:8080"
api = API(proxy=proxy)
doc = await api.user_by_login("elonmusk")
```
3. Use can set proxy with environemt variable `TWS_RPOXY`:
```sh
TWS_PROXY=socks5://user:pass@127.0.0.1:1080 twscrape user_by_login elonmusk
```
4. You can change proxy any time like:
```py
api.proxy = "socks5://user:pass@127.0.0.1:1080"
doc = await api.user_by_login("elonmusk") # new proxy will be used
api.proxy = None
doc = await api.user_by_login("elonmusk") # no proxy used
```
5. Proxy priorities
- `api.proxy` have top priority
- `env.proxy` will be used if `api.proxy` is None
- `acc.proxy` have lowest priotity
So if you want to use proxy PER ACCOUNT, do NOT override proxy with env variable or by passing proxy param to API.
_Note:_ If proxy not working, exception will be raised from API class.
## Environment Variables
- `TWS_PROXY` - global proxy for all accounts (e.g. `socks5://user:pass@127.0.0.1:1080`)
- `TWS_WAIT_EMAIL_CODE` - timeout for email verification code during login (default: `30`, in seconds)
- `TWS_RAISE_WHEN_NO_ACCOUNT` - raise `NoAccountError` exception when no available accounts, instead of waiting (default: `false`, values: `false`/`0`/`true`/`1`)
## Limitations
X/Twitter regularly [updates](https://x.com/elonmusk/status/1675187969420828672) their rate limits. Current basic behavior:
- Request limits reset every 15 minutes for each endpoint individually
- Each account has separate limits for different operations (search, profile views, etc.)
API data limitations:
- `user_tweets` & `user_tweets_and_replies` - can return ~3200 tweets maximum
- Rate limits may vary based on account age and status
## Articles
- [How to still scrape millions of tweets in 2023](https://medium.com/@vladkens/how-to-still-scrape-millions-of-tweets-in-2023-using-twscrape-97f5d3881434)
- [_(Add Article)_](https://github.com/vladkens/twscrape/edit/main/readme.md)
## See also
- [twitter-advanced-search](https://github.com/igorbrigadir/twitter-advanced-search) – guide on search filters
- [twitter-api-client](https://github.com/trevorhobenshield/twitter-api-client) – Implementation of Twitter's v1, v2, and GraphQL APIs
- [snscrape](https://github.com/JustAnotherArchivist/snscrape) – is a scraper for social networking services (SNS)
- [twint](https://github.com/twintproject/twint) – Twitter Intelligence Tool
## 新功能
### 私信和媒体上传
现在可以使用以下方法发送私信和上传媒体文件：
```python
# 发送简单私信
await api.dm(text="你好", receivers=[1234567890])
# 发送带图片的私信
await api.dm(text="查看这张图片", receivers=[1234567890], media="path/to/image.jpg")
# 内部上传媒体API（通常不需要直接调用）
media_id = await api._upload_media(filename="path/to/media.jpg", is_dm=False)
# 添加参数wait_for_account=True可以在没有可用账号时等待
await api.dm(text="你好", receivers=[1234567890], wait_for_account=True)
# 检查账号池状态
account_status = await api.account_status()
print(f"总账号数: {account_status['total']}")
print(f"活跃账号数: {account_status['active']}")
print(f"锁定情况: {account_status['locks']}")
# 重置所有锁
await api.reset_locks()
# 不支持重置特定队列的锁
```
上传媒体支持以下格式：
- 图片：JPG、PNG、GIF等 (最大5MB)
- 动图：GIF (最大15MB)
- 视频：MP4等 (最大530MB)
注意：以上API都会自动使用账户池中的账户，并应用代理设置。
#### 常见问题排查
1. 如果收到错误 `AttributeError: 'QueueClient' object has no attribute 'post'`，请确保您使用的是最新版本的twscrape库。旧版本可能存在这个问题。
2. 如果收到错误 `No account available for queue "useSendMessageMutation". Next available at 15:14:23`，表示所有账号都在冷却期。可以选择：
- 等待直到指定时间
- 添加更多账号到池中：`await api.pool.add_account(email, username, password)`
- 使用`wait_for_account=True`参数等待账号可用
- 使用`await api.reset_locks()`重置所有锁（谨慎使用，可能导致Twitter限制账号）
3. 如果收到 `403 Forbidden` 错误，可能是账号没有私信权限或媒体上传权限。常见原因：
- 新账号可能需要验证手机号才能发送私信
- 账号被Twitter限制
- 媒体格式不受支持
- 目标用户的隐私设置不允许收到私信
4. 为了避免上传错误，请确保：
- 媒体文件存在且可读
- 文件格式支持且未损坏
- 文件大小未超过限制
- 您的账号有权限发送私信和上传媒体
5. 如果上传大文件时失败，可以尝试先压缩媒体文件再上传
#### 完整示例：发送私信
以下是一个检查账号状态、重置锁并发送私信的完整示例：
```python
import asyncio
from twscrape import API
async def main():
# 初始化API
api = API("accounts.db") # 使用已有的账号数据库
api.debug = True # 启用调试模式，查看详细日志
# 检查账号状态
status = await api.account_status()
print(f"总账号数: {status['total']}")
print(f"活跃账号数: {status['active']}")
if status['total'] == 0:
print("没有账号，需要添加账号")
await api.pool.add_account("your_email", "your_username", "your_password")
# 如果所有账号都被锁定，重置锁
if 'useSendMessageMutation' in status['locks']:
print("检测到账号被锁定，正在重置...")
await api.reset_locks()
# 发送私信
try:
# 不带媒体的私信
dm_response = await api.dm(
text="你好，这是一条测试消息",
receivers=[1234567890], # 接收者ID
wait_for_account=True # 如果没有账号可用，等待直到有账号可用
)
if "error" in dm_response:
print(f"发送失败: {dm_response['error']}")
else:
print("私信发送成功!")
# 带媒体的私信
dm_with_media = await api.dm(
text="查看这张图片",
receivers=[1234567890],
media="path/to/image.jpg"
)
if "error" in dm_with_media:
print(f"带媒体的私信发送失败: {dm_with_media['error']}")
else:
print("带媒体的私信发送成功!")
except Exception as e:
print(f"发送私信时出错: {e}")
if __name__ == "__main__":
asyncio.run(main())
```
Raw data
{
"_id": null,
"home_page": null,
"name": "twscrapeplus",
"maintainer": null,
"docs_url": null,
"requires_python": ">=3.10",
"maintainer_email": null,
"keywords": "api, scrape, scrapper, snscrape, twitter",
"author": null,
"author_email": "vladkens <Richard@mail.com>",
"download_url": "https://files.pythonhosted.org/packages/ab/11/45b874a57700e27bacd4db0fd21024d92c9b9ecba5265cc0fecc50d7cf06/twscrapeplus-0.2.0.tar.gz",
"platform": null,
"description": "# twscrape\n\n<div align=\"center\">\n\n[<img src=\"https://badges.ws/pypi/v/twscrape\" alt=\"version\" />](https://pypi.org/project/twscrape)\n[<img src=\"https://badges.ws/pypi/python/twscrape\" alt=\"py versions\" />](https://pypi.org/project/twscrape)\n[<img src=\"https://badges.ws/pypi/dm/twscrape\" alt=\"downloads\" />](https://pypi.org/project/twscrape)\n[<img src=\"https://badges.ws/github/license/vladkens/twscrape\" alt=\"license\" />](https://github.com/vladkens/twscrape/blob/main/LICENSE)\n[<img src=\"https://badges.ws/badge/-/buy%20me%20a%20coffee/ff813f?icon=buymeacoffee&label\" alt=\"donate\" />](https://buymeacoffee.com/vladkens)\n\n</div>\n\nTwitter GraphQL API implementation with [SNScrape](https://github.com/JustAnotherArchivist/snscrape) data models.\n\n<div align=\"center\">\n <img src=\".github/example.png\" alt=\"example of cli usage\" height=\"400px\">\n</div>\n\n## Install\n\n```bash\npip install twscrape\n```\nOr development version:\n```bash\npip install git+https://github.com/vladkens/twscrape.git\n```\n\n## Features\n- Support both Search & GraphQL Twitter API\n- Async/Await functions (can run multiple scrapers in parallel at the same time)\n- Login flow (with receiving verification code from email)\n- Saving/restoring account sessions\n- Raw Twitter API responses & SNScrape models\n- Automatic account switching to smooth Twitter API rate limits\n- Direct messages (DM) support\n- Media upload support for tweets and DMs\n- Account status monitoring\n\n## Usage\n\nThis project requires authorized X/Twitter accounts to work with the API. You have two options:\n\n1. **Create Your Own Account**: While you can register a new account on X/Twitter yourself, it's can be difficult due to strict verification processes and high ban rates.\n\n2. **Use Ready Accounts**: For immediate access, you can get ready-to-use accounts with cookies from [our recommended provider](https://kutt.it/ueeM5f). Cookie-based accounts typically have fewer login issues.\n\nFor optimal performance and to avoid IP-based restrictions, we also recommend using proxies from [our provider](https://kutt.it/eb3rXk).\n\n**Disclaimer**: While X/Twitter's Terms of Service discourage using multiple accounts, this is a common practice for data collection and research purposes. Use responsibly and at your own discretion.\n\n```python\nimport asyncio\nfrom twscrape import API, gather\nfrom twscrape.logger import set_log_level\n\nasync def main():\n api = API() # or API(\"path-to.db\") \u2013 default is `accounts.db`\n\n # ADD ACCOUNTS (for CLI usage see next readme section)\n\n # Option 1. Adding account with cookies (more stable)\n cookies = \"abc=12; ct0=xyz\" # or '{\"abc\": \"12\", \"ct0\": \"xyz\"}'\n await api.pool.add_account(\"user3\", \"pass3\", \"u3@mail.com\", \"mail_pass3\", cookies=cookies)\n\n # Option2. Adding account with login / password (less stable)\n # email login / password required to receive the verification code via IMAP protocol\n # (not all email providers are supported, e.g. ProtonMail)\n await api.pool.add_account(\"user1\", \"pass1\", \"u1@example.com\", \"mail_pass1\")\n await api.pool.add_account(\"user2\", \"pass2\", \"u2@example.com\", \"mail_pass2\")\n await api.pool.login_all() # try to login to receive account cookies\n\n # API USAGE\n\n # search (latest tab)\n await gather(api.search(\"elon musk\", limit=20)) # list[Tweet]\n # change search tab (product), can be: Top, Latest (default), Media\n await gather(api.search(\"elon musk\", limit=20, kv={\"product\": \"Top\"}))\n\n # tweet info\n tweet_id = 20\n await api.tweet_details(tweet_id) # Tweet\n await gather(api.retweeters(tweet_id, limit=20)) # list[User]\n\n # Note: this method have small pagination from X side, like 5 tweets per query\n await gather(api.tweet_replies(tweet_id, limit=20)) # list[Tweet]\n\n # get user by login\n user_login = \"xdevelopers\"\n await api.user_by_login(user_login) # User\n\n # user info\n user_id = 2244994945\n await api.user_by_id(user_id) # User\n await gather(api.following(user_id, limit=20)) # list[User]\n await gather(api.followers(user_id, limit=20)) # list[User]\n await gather(api.verified_followers(user_id, limit=20)) # list[User]\n await gather(api.subscriptions(user_id, limit=20)) # list[User]\n await gather(api.user_tweets(user_id, limit=20)) # list[Tweet]\n await gather(api.user_tweets_and_replies(user_id, limit=20)) # list[Tweet]\n await gather(api.user_media(user_id, limit=20)) # list[Tweet]\n\n # list info\n await gather(api.list_timeline(list_id=123456789))\n\n # trends\n await gather(api.trends(\"news\")) # list[Trend]\n await gather(api.trends(\"sport\")) # list[Trend]\n await gather(api.trends(\"VGltZWxpbmU6DAC2CwABAAAACHRyZW5kaW5nAAA\")) # list[Trend]\n \n # Send direct message (DM)\n # 123456789 is the user ID of the recipient\n dm_result = await api.dm(\"Hello from twscrape!\", [123456789])\n print(dm_result)\n \n # Send DM with media\n dm_with_media = await api.dm(\"Check this image!\", [123456789], media=\"path/to/image.jpg\")\n print(dm_with_media)\n \n # Upload media for other purposes\n media_id = await api._upload_media(\"path/to/image.jpg\")\n print(f\"Uploaded media ID: {media_id}\")\n \n # Add alternative text to media\n alt_text_result = await api._add_alt_text(media_id, \"Description of the image\")\n print(alt_text_result)\n \n # Check account status\n status = await api.account_status()\n print(f\"Total accounts: {status['total']}\")\n print(f\"Active accounts: {status['active']}\")\n print(f\"Locks: {status['locks']}\")\n \n # Reset account locks if needed\n await api.reset_locks()\n\n # NOTE 1: gather is a helper function to receive all data as list, FOR can be used as well:\n async for tweet in api.search(\"elon musk\"):\n print(tweet.id, tweet.user.username, tweet.rawContent) # tweet is `Tweet` object\n\n # NOTE 2: all methods have `raw` version (returns `httpx.Response` object):\n async for rep in api.search_raw(\"elon musk\"):\n print(rep.status_code, rep.json()) # rep is `httpx.Response` object\n\n # change log level, default info\n set_log_level(\"DEBUG\")\n\n # Tweet & User model can be converted to regular dict or json, e.g.:\n doc = await api.user_by_id(user_id) # User\n doc.dict() # -> python dict\n doc.json() # -> json string\n\nif __name__ == \"__main__\":\n asyncio.run(main())\n```\n\n### Stoping iteration with break\n\nIn order to correctly release an account in case of `break` in loop, a special syntax must be used. Otherwise, Python's events loop will release lock on the account sometime in the future. See explanation [here](https://github.com/vladkens/twscrape/issues/27#issuecomment-1623395424).\n\n```python\nfrom contextlib import aclosing\n\nasync with aclosing(api.search(\"elon musk\")) as gen:\n async for tweet in gen:\n if tweet.id < 200:\n break\n```\n\n## CLI\n\n### Get help on CLI commands\n\n```sh\n# show all commands\ntwscrape\n\n# help on specific comand\ntwscrape search --help\n```\n\n### Add accounts\n\nTo add accounts use `add_accounts` command. Command syntax is:\n```sh\ntwscrape add_accounts <file_path> <line_format>\n```\n\nWhere:\n`<line_format>` is format of line if accounts file splited by delimeter. Possible tokens:\n- `username` \u2013 required\n- `password` \u2013 required\n- `email` \u2013 required\n- `email_password` \u2013 to receive email code (you can use `--manual` mode to get code)\n- `cookies` \u2013 can be any parsable format (string, json, base64 string, etc)\n- `_` \u2013 skip column from parse\n\nTokens should be splited by delimeter, usually \"`:`\" used.\n\nExample:\n\nI have account files named `order-12345.txt` with format:\n```text\nusername:password:email:email password:user_agent:cookies\n```\n\nCommand to add accounts will be (user_agent column skiped with `_`):\n```sh\ntwscrape add_accounts ./order-12345.txt username:password:email:email_password:_:cookies\n```\n\n### Login accounts\n\n_Note:_ If you added accounts with cookies, login not required.\n\nRun:\n\n```sh\ntwscrape login_accounts\n```\n\n`twscrape` will start login flow for each new account. If X will ask to verify email and you provided `email_password` in `add_account`, then `twscrape` will try to receive verification code by IMAP protocol. After success login account cookies will be saved to db file for future use.\n\n#### Manual email verification\n\nIn case your email provider not support IMAP protocol (ProtonMail, Tutanota, etc) or IMAP is disabled in settings, you can enter email verification code manually. To do this run login command with `--manual` flag.\n\nExample:\n\n```sh\ntwscrape login_accounts --manual\ntwscrape relogin user1 user2 --manual\ntwscrape relogin_failed --manual\n```\n\n### Get list of accounts and their statuses\n\n```sh\ntwscrape accounts\n\n# Output:\n# username logged_in active last_used total_req error_msg\n# user1 True True 2023-05-20 03:20:40 100 None\n# user2 True True 2023-05-20 03:25:45 120 None\n# user3 False False None 120 Login error\n```\n\n### Re-login accounts\n\nIt is possible to re-login specific accounts:\n\n```sh\ntwscrape relogin user1 user2\n```\n\nOr retry login for all failed logins:\n\n```sh\ntwscrape relogin_failed\n```\n\n### Use different accounts file\n\nUseful if using a different set of accounts for different actions\n\n```\ntwscrape --db test-accounts.db <command>\n```\n\n### Search commands\n\n```sh\ntwscrape search \"QUERY\" --limit=20\ntwscrape tweet_details TWEET_ID\ntwscrape tweet_replies TWEET_ID --limit=20\ntwscrape retweeters TWEET_ID --limit=20\ntwscrape user_by_id USER_ID\ntwscrape user_by_login USERNAME\ntwscrape user_media USER_ID --limit=20\ntwscrape following USER_ID --limit=20\ntwscrape followers USER_ID --limit=20\ntwscrape verified_followers USER_ID --limit=20\ntwscrape subscriptions USER_ID --limit=20\ntwscrape user_tweets USER_ID --limit=20\ntwscrape user_tweets_and_replies USER_ID --limit=20\ntwscrape trends sport\n```\n\nThe default output is in the console (stdout), one document per line. So it can be redirected to the file.\n\n```sh\ntwscrape search \"elon mask lang:es\" --limit=20 > data.txt\n```\n\nBy default, parsed data is returned. The original tweet responses can be retrieved with `--raw` flag.\n\n```sh\ntwscrape search \"elon mask lang:es\" --limit=20 --raw\n```\n\n### About `limit` param\n\nX API works through pagination, each API method can have different defaults for per page parameter (and this parameter can't be changed by caller). So `limit` param in `twscrape` is the desired number of objects (tweets or users, depending on the method). `twscrape` tries to return NO LESS objects than requested. If the X API returns less or more objects, `twscrape` will return whatever X gives.\n\n## Proxy\n\nThere are few options to use proxies.\n\n1. You can add proxy per account\n\n```py\nproxy = \"http://login:pass@example.com:8080\"\nawait api.pool.add_account(\"user4\", \"pass4\", \"u4@mail.com\", \"mail_pass4\", proxy=proxy)\n```\n\n2. You can use global proxy for all accounts\n\n```py\nproxy = \"http://login:pass@example.com:8080\"\napi = API(proxy=proxy)\ndoc = await api.user_by_login(\"elonmusk\")\n```\n\n3. Use can set proxy with environemt variable `TWS_RPOXY`:\n\n```sh\nTWS_PROXY=socks5://user:pass@127.0.0.1:1080 twscrape user_by_login elonmusk\n```\n\n4. You can change proxy any time like:\n\n```py\napi.proxy = \"socks5://user:pass@127.0.0.1:1080\"\ndoc = await api.user_by_login(\"elonmusk\") # new proxy will be used\napi.proxy = None\ndoc = await api.user_by_login(\"elonmusk\") # no proxy used\n```\n\n5. Proxy priorities\n\n- `api.proxy` have top priority\n- `env.proxy` will be used if `api.proxy` is None\n- `acc.proxy` have lowest priotity\n\nSo if you want to use proxy PER ACCOUNT, do NOT override proxy with env variable or by passing proxy param to API.\n\n_Note:_ If proxy not working, exception will be raised from API class.\n\n## Environment Variables\n\n- `TWS_PROXY` - global proxy for all accounts (e.g. `socks5://user:pass@127.0.0.1:1080`)\n- `TWS_WAIT_EMAIL_CODE` - timeout for email verification code during login (default: `30`, in seconds)\n- `TWS_RAISE_WHEN_NO_ACCOUNT` - raise `NoAccountError` exception when no available accounts, instead of waiting (default: `false`, values: `false`/`0`/`true`/`1`)\n\n## Limitations\n\nX/Twitter regularly [updates](https://x.com/elonmusk/status/1675187969420828672) their rate limits. Current basic behavior:\n- Request limits reset every 15 minutes for each endpoint individually\n- Each account has separate limits for different operations (search, profile views, etc.)\n\nAPI data limitations:\n- `user_tweets` & `user_tweets_and_replies` - can return ~3200 tweets maximum\n- Rate limits may vary based on account age and status\n\n## Articles\n- [How to still scrape millions of tweets in 2023](https://medium.com/@vladkens/how-to-still-scrape-millions-of-tweets-in-2023-using-twscrape-97f5d3881434)\n- [_(Add Article)_](https://github.com/vladkens/twscrape/edit/main/readme.md)\n\n## See also\n- [twitter-advanced-search](https://github.com/igorbrigadir/twitter-advanced-search) \u2013 guide on search filters\n- [twitter-api-client](https://github.com/trevorhobenshield/twitter-api-client) \u2013 Implementation of Twitter's v1, v2, and GraphQL APIs\n- [snscrape](https://github.com/JustAnotherArchivist/snscrape) \u2013 is a scraper for social networking services (SNS)\n- [twint](https://github.com/twintproject/twint) \u2013 Twitter Intelligence Tool\n\n## \u65b0\u529f\u80fd\n\n### \u79c1\u4fe1\u548c\u5a92\u4f53\u4e0a\u4f20\n\n\u73b0\u5728\u53ef\u4ee5\u4f7f\u7528\u4ee5\u4e0b\u65b9\u6cd5\u53d1\u9001\u79c1\u4fe1\u548c\u4e0a\u4f20\u5a92\u4f53\u6587\u4ef6\uff1a\n\n```python\n# \u53d1\u9001\u7b80\u5355\u79c1\u4fe1\nawait api.dm(text=\"\u4f60\u597d\", receivers=[1234567890])\n\n# \u53d1\u9001\u5e26\u56fe\u7247\u7684\u79c1\u4fe1\nawait api.dm(text=\"\u67e5\u770b\u8fd9\u5f20\u56fe\u7247\", receivers=[1234567890], media=\"path/to/image.jpg\")\n\n# \u5185\u90e8\u4e0a\u4f20\u5a92\u4f53API\uff08\u901a\u5e38\u4e0d\u9700\u8981\u76f4\u63a5\u8c03\u7528\uff09\nmedia_id = await api._upload_media(filename=\"path/to/media.jpg\", is_dm=False)\n\n# \u6dfb\u52a0\u53c2\u6570wait_for_account=True\u53ef\u4ee5\u5728\u6ca1\u6709\u53ef\u7528\u8d26\u53f7\u65f6\u7b49\u5f85\nawait api.dm(text=\"\u4f60\u597d\", receivers=[1234567890], wait_for_account=True)\n\n# \u68c0\u67e5\u8d26\u53f7\u6c60\u72b6\u6001\naccount_status = await api.account_status()\nprint(f\"\u603b\u8d26\u53f7\u6570: {account_status['total']}\")\nprint(f\"\u6d3b\u8dc3\u8d26\u53f7\u6570: {account_status['active']}\")\nprint(f\"\u9501\u5b9a\u60c5\u51b5: {account_status['locks']}\")\n\n# \u91cd\u7f6e\u6240\u6709\u9501\nawait api.reset_locks()\n\n# \u4e0d\u652f\u6301\u91cd\u7f6e\u7279\u5b9a\u961f\u5217\u7684\u9501\n```\n\n\u4e0a\u4f20\u5a92\u4f53\u652f\u6301\u4ee5\u4e0b\u683c\u5f0f\uff1a\n- \u56fe\u7247\uff1aJPG\u3001PNG\u3001GIF\u7b49 (\u6700\u59275MB)\n- \u52a8\u56fe\uff1aGIF (\u6700\u592715MB)\n- \u89c6\u9891\uff1aMP4\u7b49 (\u6700\u5927530MB)\n\n\u6ce8\u610f\uff1a\u4ee5\u4e0aAPI\u90fd\u4f1a\u81ea\u52a8\u4f7f\u7528\u8d26\u6237\u6c60\u4e2d\u7684\u8d26\u6237\uff0c\u5e76\u5e94\u7528\u4ee3\u7406\u8bbe\u7f6e\u3002\n\n#### \u5e38\u89c1\u95ee\u9898\u6392\u67e5\n\n1. \u5982\u679c\u6536\u5230\u9519\u8bef `AttributeError: 'QueueClient' object has no attribute 'post'`\uff0c\u8bf7\u786e\u4fdd\u60a8\u4f7f\u7528\u7684\u662f\u6700\u65b0\u7248\u672c\u7684twscrape\u5e93\u3002\u65e7\u7248\u672c\u53ef\u80fd\u5b58\u5728\u8fd9\u4e2a\u95ee\u9898\u3002\n\n2. \u5982\u679c\u6536\u5230\u9519\u8bef `No account available for queue \"useSendMessageMutation\". Next available at 15:14:23`\uff0c\u8868\u793a\u6240\u6709\u8d26\u53f7\u90fd\u5728\u51b7\u5374\u671f\u3002\u53ef\u4ee5\u9009\u62e9\uff1a\n - \u7b49\u5f85\u76f4\u5230\u6307\u5b9a\u65f6\u95f4\n - \u6dfb\u52a0\u66f4\u591a\u8d26\u53f7\u5230\u6c60\u4e2d\uff1a`await api.pool.add_account(email, username, password)`\n - \u4f7f\u7528`wait_for_account=True`\u53c2\u6570\u7b49\u5f85\u8d26\u53f7\u53ef\u7528\n - \u4f7f\u7528`await api.reset_locks()`\u91cd\u7f6e\u6240\u6709\u9501\uff08\u8c28\u614e\u4f7f\u7528\uff0c\u53ef\u80fd\u5bfc\u81f4Twitter\u9650\u5236\u8d26\u53f7\uff09\n\n3. \u5982\u679c\u6536\u5230 `403 Forbidden` \u9519\u8bef\uff0c\u53ef\u80fd\u662f\u8d26\u53f7\u6ca1\u6709\u79c1\u4fe1\u6743\u9650\u6216\u5a92\u4f53\u4e0a\u4f20\u6743\u9650\u3002\u5e38\u89c1\u539f\u56e0\uff1a\n - \u65b0\u8d26\u53f7\u53ef\u80fd\u9700\u8981\u9a8c\u8bc1\u624b\u673a\u53f7\u624d\u80fd\u53d1\u9001\u79c1\u4fe1\n - \u8d26\u53f7\u88abTwitter\u9650\u5236\n - \u5a92\u4f53\u683c\u5f0f\u4e0d\u53d7\u652f\u6301\n - \u76ee\u6807\u7528\u6237\u7684\u9690\u79c1\u8bbe\u7f6e\u4e0d\u5141\u8bb8\u6536\u5230\u79c1\u4fe1\n\n4. \u4e3a\u4e86\u907f\u514d\u4e0a\u4f20\u9519\u8bef\uff0c\u8bf7\u786e\u4fdd\uff1a\n - \u5a92\u4f53\u6587\u4ef6\u5b58\u5728\u4e14\u53ef\u8bfb\n - \u6587\u4ef6\u683c\u5f0f\u652f\u6301\u4e14\u672a\u635f\u574f\n - \u6587\u4ef6\u5927\u5c0f\u672a\u8d85\u8fc7\u9650\u5236\n - \u60a8\u7684\u8d26\u53f7\u6709\u6743\u9650\u53d1\u9001\u79c1\u4fe1\u548c\u4e0a\u4f20\u5a92\u4f53\n\n5. \u5982\u679c\u4e0a\u4f20\u5927\u6587\u4ef6\u65f6\u5931\u8d25\uff0c\u53ef\u4ee5\u5c1d\u8bd5\u5148\u538b\u7f29\u5a92\u4f53\u6587\u4ef6\u518d\u4e0a\u4f20\n\n#### \u5b8c\u6574\u793a\u4f8b\uff1a\u53d1\u9001\u79c1\u4fe1\n\n\u4ee5\u4e0b\u662f\u4e00\u4e2a\u68c0\u67e5\u8d26\u53f7\u72b6\u6001\u3001\u91cd\u7f6e\u9501\u5e76\u53d1\u9001\u79c1\u4fe1\u7684\u5b8c\u6574\u793a\u4f8b\uff1a\n\n```python\nimport asyncio\nfrom twscrape import API\n\nasync def main():\n # \u521d\u59cb\u5316API\n api = API(\"accounts.db\") # \u4f7f\u7528\u5df2\u6709\u7684\u8d26\u53f7\u6570\u636e\u5e93\n api.debug = True # \u542f\u7528\u8c03\u8bd5\u6a21\u5f0f\uff0c\u67e5\u770b\u8be6\u7ec6\u65e5\u5fd7\n \n # \u68c0\u67e5\u8d26\u53f7\u72b6\u6001\n status = await api.account_status()\n print(f\"\u603b\u8d26\u53f7\u6570: {status['total']}\")\n print(f\"\u6d3b\u8dc3\u8d26\u53f7\u6570: {status['active']}\")\n \n if status['total'] == 0:\n print(\"\u6ca1\u6709\u8d26\u53f7\uff0c\u9700\u8981\u6dfb\u52a0\u8d26\u53f7\")\n await api.pool.add_account(\"your_email\", \"your_username\", \"your_password\")\n \n # \u5982\u679c\u6240\u6709\u8d26\u53f7\u90fd\u88ab\u9501\u5b9a\uff0c\u91cd\u7f6e\u9501\n if 'useSendMessageMutation' in status['locks']:\n print(\"\u68c0\u6d4b\u5230\u8d26\u53f7\u88ab\u9501\u5b9a\uff0c\u6b63\u5728\u91cd\u7f6e...\")\n await api.reset_locks()\n \n # \u53d1\u9001\u79c1\u4fe1\n try:\n # \u4e0d\u5e26\u5a92\u4f53\u7684\u79c1\u4fe1\n dm_response = await api.dm(\n text=\"\u4f60\u597d\uff0c\u8fd9\u662f\u4e00\u6761\u6d4b\u8bd5\u6d88\u606f\", \n receivers=[1234567890], # \u63a5\u6536\u8005ID\n wait_for_account=True # \u5982\u679c\u6ca1\u6709\u8d26\u53f7\u53ef\u7528\uff0c\u7b49\u5f85\u76f4\u5230\u6709\u8d26\u53f7\u53ef\u7528\n )\n \n if \"error\" in dm_response:\n print(f\"\u53d1\u9001\u5931\u8d25: {dm_response['error']}\")\n else:\n print(\"\u79c1\u4fe1\u53d1\u9001\u6210\u529f!\")\n \n # \u5e26\u5a92\u4f53\u7684\u79c1\u4fe1\n dm_with_media = await api.dm(\n text=\"\u67e5\u770b\u8fd9\u5f20\u56fe\u7247\", \n receivers=[1234567890],\n media=\"path/to/image.jpg\"\n )\n \n if \"error\" in dm_with_media:\n print(f\"\u5e26\u5a92\u4f53\u7684\u79c1\u4fe1\u53d1\u9001\u5931\u8d25: {dm_with_media['error']}\")\n else:\n print(\"\u5e26\u5a92\u4f53\u7684\u79c1\u4fe1\u53d1\u9001\u6210\u529f!\")\n \n except Exception as e:\n print(f\"\u53d1\u9001\u79c1\u4fe1\u65f6\u51fa\u9519: {e}\")\n\nif __name__ == \"__main__\":\n asyncio.run(main())\n```\n",
"bugtrack_url": null,
"license": "MIT",
"summary": "Twitter GraphQL and Search API implementation with SNScrape data models",
"version": "0.2.0",
"project_urls": {
"repository": "https://github.com/vladkens/twscrape"
},
"split_keywords": [
"api",
" scrape",
" scrapper",
" snscrape",
" twitter"
],
"urls": [
{
"comment_text": null,
"digests": {
"blake2b_256": "eec59bc5baebbb29121420eb9613c4f4388a1d3b7fe7da74d5b3d1c1f8d7144f",
"md5": "0a6f9c8862b67603fac245e4e5745d60",
"sha256": "915f4f8f534f8c9cb659cc32a07b154f77e04281e986b09680958e8b5f87f4f9"
},
"downloads": -1,
"filename": "twscrapeplus-0.2.0-py3-none-any.whl",
"has_sig": false,
"md5_digest": "0a6f9c8862b67603fac245e4e5745d60",
"packagetype": "bdist_wheel",
"python_version": "py3",
"requires_python": ">=3.10",
"size": 42351,
"upload_time": "2025-03-19T02:52:55",
"upload_time_iso_8601": "2025-03-19T02:52:55.155827Z",
"url": "https://files.pythonhosted.org/packages/ee/c5/9bc5baebbb29121420eb9613c4f4388a1d3b7fe7da74d5b3d1c1f8d7144f/twscrapeplus-0.2.0-py3-none-any.whl",
"yanked": false,
"yanked_reason": null
},
{
"comment_text": null,
"digests": {
"blake2b_256": "ab1145b874a57700e27bacd4db0fd21024d92c9b9ecba5265cc0fecc50d7cf06",
"md5": "a89fc684049a91d34317aeda8e0db778",
"sha256": "ef95d5fd19d1b8f2db0d6336917ae27b046a69dd8aba316d99cea39679f129f2"
},
"downloads": -1,
"filename": "twscrapeplus-0.2.0.tar.gz",
"has_sig": false,
"md5_digest": "a89fc684049a91d34317aeda8e0db778",
"packagetype": "sdist",
"python_version": "source",
"requires_python": ">=3.10",
"size": 614538,
"upload_time": "2025-03-19T02:52:58",
"upload_time_iso_8601": "2025-03-19T02:52:58.583019Z",
"url": "https://files.pythonhosted.org/packages/ab/11/45b874a57700e27bacd4db0fd21024d92c9b9ecba5265cc0fecc50d7cf06/twscrapeplus-0.2.0.tar.gz",
"yanked": false,
"yanked_reason": null
}
],
"upload_time": "2025-03-19 02:52:58",
"github": true,
"gitlab": false,
"bitbucket": false,
"codeberg": false,
"github_user": "vladkens",
"github_project": "twscrape",
"travis_ci": false,
"coveralls": false,
"github_actions": true,
"lcname": "twscrapeplus"
}
