# hfmirror
[](https://pypi.org/project/hfmirror/)
[](https://github.com/narugo1992/hfmirror/actions?query=workflow%3A%22Code+Test%22)
[](https://github.com/narugo1992/hfmirror/actions?query=workflow%3A%22Package+Release%22)
[](https://codecov.io/gh/narugo1992/hfmirror)
[](https://github.com/narugo1992/hfmirror/stargazers)
[](https://github.com/narugo1992/hfmirror/network)
[](https://github.com/narugo1992/hfmirror/issues)
[](https://github.com/narugo1992/hfmirror/pulls)
[](https://github.com/narugo1992/hfmirror/graphs/contributors)
[](https://github.com/narugo1992/hfmirror/blob/master/LICENSE)
Mirror for resources to local and huggingface.
## Installation
You can simply install it with `pip` command line from the official PyPI site.
```bash
pip install hfmirror
```
For more information about installation, you can refer to [Installation](https://narugo1992.github.io/hfmirror/main/tutorials/installation/index.html).
## Quick Start
### Mirror Github Releases to Your Disk
The following code can mirror the release files of repository `narugo1992/gchar` to your local directory `test_releases`
```python
from hfmirror.resource import GithubReleaseResource
from hfmirror.storage import LocalStorage
from hfmirror.sync import SyncTask
if __name__ == '__main__':
github = GithubReleaseResource(
# the github repository
repo='narugo1992/gchar',
# access_token of github client (if needed)
access_token='my_github_access_token',
# add files like LATEST_RELEASE to mark the versions
add_version_attachment=True,
)
storage = LocalStorage('test_releases')
task = SyncTask(github, storage)
task.sync()
```
This is the `test_releases`
```
test_releases
├── LATEST_RELEASE
├── LATEST_RELEASE_0
├── LATEST_RELEASE_0.0
├── LATEST_RELEASE_0.0.1
├── LATEST_RELEASE_0.0.2
├── LATEST_RELEASE_0.0.3
├── LATEST_RELEASE_0.0.4
├── LATEST_RELEASE_0.0.5
├── LATEST_RELEASE_0.0.6
├── LATEST_RELEASE_0.0.8
├── v0.0.1
│ ├── gchar-0.0.1-py3-none-any.whl
│ └── gchar-0.0.1.tar.gz
├── v0.0.2
│ ├── gchar-0.0.2-py3-none-any.whl
│ └── gchar-0.0.2.tar.gz
├── v0.0.3
│ ├── gchar-0.0.3-py3-none-any.whl
│ └── gchar-0.0.3.tar.gz
├── v0.0.4
│ ├── gchar-0.0.4-py3-none-any.whl
│ └── gchar-0.0.4.tar.gz
├── v0.0.5
│ ├── gchar-0.0.5-py3-none-any.whl
│ └── gchar-0.0.5.tar.gz
├── v0.0.6
│ ├── gchar-0.0.6-py3-none-any.whl
│ └── gchar-0.0.6.tar.gz
└── v0.0.8
├── gchar-0.0.8-py3-none-any.whl
└── gchar-0.0.8.tar.gz
```
### Mirror Game Skins to HuggingFace
Your can mirror the skins of genshin impact based on `gchar` to huggingface repo, using the following code with custom resource and huggingface repository storage:
```python
import mimetypes
import os
import re
from typing import Iterable, Union, Tuple, Any, Mapping, List, Type
from gchar.games.base import Character as BaseCharacter
from gchar.games.genshin import Character
from hbutils.system import urlsplit
from huggingface_hub import HfApi
from tqdm.auto import tqdm
from hfmirror.resource import SyncResource
from hfmirror.resource.resource import TargetPathType
from hfmirror.storage import HuggingfaceStorage
from hfmirror.sync import SyncTask
from hfmirror.utils import srequest, get_requests_session
class ArknightsSkinResource(SyncResource):
def __init__(self, chs: List[Character], ch_type: Type[BaseCharacter]):
self.characters = chs
self.ch_type = ch_type
self.session = get_requests_session()
def grab(self) -> Iterable[Union[
Tuple[str, Any, TargetPathType, Mapping],
Tuple[str, Any, TargetPathType],
]]:
yield 'metadata', {'game': self.ch_type.__game_name__}, ''
_exist_ids = set()
for ch in tqdm(self.characters):
if ch.index in _exist_ids:
continue
metadata = {
'id': ch.index,
'cnname': str(ch.cnname) if ch.cnname else None,
'jpname': str(ch.jpname) if ch.jpname else None,
'enname': str(ch.enname) if ch.enname else None,
'alias': list(map(str, ch.alias_names)),
}
yield 'metadata', metadata, f'{ch.index}'
_exist_ids.add(ch.index)
for skin in ch.skins:
_, ext = os.path.splitext(urlsplit(skin.url).filename)
if not ext:
resp = srequest(self.session, 'HEAD', skin.url)
ext = mimetypes.guess_extension(resp.headers['Content-Type'])
filename = re.sub(r'\W+', '_', skin.name).strip('_') + ext
yield 'remote', skin.url, f'{ch.index}/{filename}', {'name': skin.name}
if __name__ == '__main__':
resource = ArknightsSkinResource(
Character.all(contains_extra=False),
Character
)
api = HfApi(token=os.environ['HF_TOKEN'])
api.create_repo('narugo/test_repo', repo_type='dataset', exist_ok=True)
storage = HuggingfaceStorage(
repo='narugo/test_repo',
hf_client=api,
namespace='genshin',
)
task = SyncTask(resource, storage)
task.sync()
```
Raw data
{
"_id": null,
"home_page": "https://github.com/narugo1992/hfmirror",
"name": "hfmirror",
"maintainer": "",
"docs_url": null,
"requires_python": ">=3.7",
"maintainer_email": "",
"keywords": "A simple tool for automatic parameter tuning.",
"author": "narugo1992",
"author_email": "narugo@126.com",
"download_url": "https://files.pythonhosted.org/packages/08/fb/3b3baa81f04bbfbb25944340300a71d7620f6a52f62ca93475672d9f5663/hfmirror-0.0.7.tar.gz",
"platform": null,
"description": "# hfmirror\n\n[](https://pypi.org/project/hfmirror/)\n\n\n\n\n[](https://github.com/narugo1992/hfmirror/actions?query=workflow%3A%22Code+Test%22)\n[](https://github.com/narugo1992/hfmirror/actions?query=workflow%3A%22Package+Release%22)\n[](https://codecov.io/gh/narugo1992/hfmirror)\n\n\n[](https://github.com/narugo1992/hfmirror/stargazers)\n[](https://github.com/narugo1992/hfmirror/network)\n\n[](https://github.com/narugo1992/hfmirror/issues)\n[](https://github.com/narugo1992/hfmirror/pulls)\n[](https://github.com/narugo1992/hfmirror/graphs/contributors)\n[](https://github.com/narugo1992/hfmirror/blob/master/LICENSE)\n\nMirror for resources to local and huggingface.\n\n## Installation\n\nYou can simply install it with `pip` command line from the official PyPI site.\n\n```bash\npip install hfmirror\n```\n\nFor more information about installation, you can refer to [Installation](https://narugo1992.github.io/hfmirror/main/tutorials/installation/index.html).\n\n## Quick Start\n\n### Mirror Github Releases to Your Disk\n\nThe following code can mirror the release files of repository `narugo1992/gchar` to your local directory `test_releases`\n\n```python\nfrom hfmirror.resource import GithubReleaseResource\nfrom hfmirror.storage import LocalStorage\nfrom hfmirror.sync import SyncTask\n\nif __name__ == '__main__':\n github = GithubReleaseResource(\n # the github repository\n repo='narugo1992/gchar',\n\n # access_token of github client (if needed)\n access_token='my_github_access_token',\n\n # add files like LATEST_RELEASE to mark the versions\n add_version_attachment=True,\n )\n\n storage = LocalStorage('test_releases')\n\n task = SyncTask(github, storage)\n task.sync()\n\n```\n\nThis is the `test_releases`\n\n```\ntest_releases\n\u251c\u2500\u2500 LATEST_RELEASE\n\u251c\u2500\u2500 LATEST_RELEASE_0\n\u251c\u2500\u2500 LATEST_RELEASE_0.0\n\u251c\u2500\u2500 LATEST_RELEASE_0.0.1\n\u251c\u2500\u2500 LATEST_RELEASE_0.0.2\n\u251c\u2500\u2500 LATEST_RELEASE_0.0.3\n\u251c\u2500\u2500 LATEST_RELEASE_0.0.4\n\u251c\u2500\u2500 LATEST_RELEASE_0.0.5\n\u251c\u2500\u2500 LATEST_RELEASE_0.0.6\n\u251c\u2500\u2500 LATEST_RELEASE_0.0.8\n\u251c\u2500\u2500 v0.0.1\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 gchar-0.0.1-py3-none-any.whl\n\u2502\u00a0\u00a0 \u2514\u2500\u2500 gchar-0.0.1.tar.gz\n\u251c\u2500\u2500 v0.0.2\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 gchar-0.0.2-py3-none-any.whl\n\u2502\u00a0\u00a0 \u2514\u2500\u2500 gchar-0.0.2.tar.gz\n\u251c\u2500\u2500 v0.0.3\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 gchar-0.0.3-py3-none-any.whl\n\u2502\u00a0\u00a0 \u2514\u2500\u2500 gchar-0.0.3.tar.gz\n\u251c\u2500\u2500 v0.0.4\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 gchar-0.0.4-py3-none-any.whl\n\u2502\u00a0\u00a0 \u2514\u2500\u2500 gchar-0.0.4.tar.gz\n\u251c\u2500\u2500 v0.0.5\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 gchar-0.0.5-py3-none-any.whl\n\u2502\u00a0\u00a0 \u2514\u2500\u2500 gchar-0.0.5.tar.gz\n\u251c\u2500\u2500 v0.0.6\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 gchar-0.0.6-py3-none-any.whl\n\u2502\u00a0\u00a0 \u2514\u2500\u2500 gchar-0.0.6.tar.gz\n\u2514\u2500\u2500 v0.0.8\n \u251c\u2500\u2500 gchar-0.0.8-py3-none-any.whl\n \u2514\u2500\u2500 gchar-0.0.8.tar.gz\n```\n\n\n\n### Mirror Game Skins to HuggingFace\n\nYour can mirror the skins of genshin impact based on `gchar` to huggingface repo, using the following code with custom resource and huggingface repository storage:\n\n```python\nimport mimetypes\nimport os\nimport re\nfrom typing import Iterable, Union, Tuple, Any, Mapping, List, Type\n\nfrom gchar.games.base import Character as BaseCharacter\nfrom gchar.games.genshin import Character\nfrom hbutils.system import urlsplit\nfrom huggingface_hub import HfApi\nfrom tqdm.auto import tqdm\n\nfrom hfmirror.resource import SyncResource\nfrom hfmirror.resource.resource import TargetPathType\nfrom hfmirror.storage import HuggingfaceStorage\nfrom hfmirror.sync import SyncTask\nfrom hfmirror.utils import srequest, get_requests_session\n\n\nclass ArknightsSkinResource(SyncResource):\n def __init__(self, chs: List[Character], ch_type: Type[BaseCharacter]):\n self.characters = chs\n self.ch_type = ch_type\n self.session = get_requests_session()\n\n def grab(self) -> Iterable[Union[\n Tuple[str, Any, TargetPathType, Mapping],\n Tuple[str, Any, TargetPathType],\n ]]:\n yield 'metadata', {'game': self.ch_type.__game_name__}, ''\n _exist_ids = set()\n for ch in tqdm(self.characters):\n if ch.index in _exist_ids:\n continue\n\n metadata = {\n 'id': ch.index,\n 'cnname': str(ch.cnname) if ch.cnname else None,\n 'jpname': str(ch.jpname) if ch.jpname else None,\n 'enname': str(ch.enname) if ch.enname else None,\n 'alias': list(map(str, ch.alias_names)),\n }\n yield 'metadata', metadata, f'{ch.index}'\n _exist_ids.add(ch.index)\n\n for skin in ch.skins:\n _, ext = os.path.splitext(urlsplit(skin.url).filename)\n if not ext:\n resp = srequest(self.session, 'HEAD', skin.url)\n ext = mimetypes.guess_extension(resp.headers['Content-Type'])\n\n filename = re.sub(r'\\W+', '_', skin.name).strip('_') + ext\n yield 'remote', skin.url, f'{ch.index}/{filename}', {'name': skin.name}\n\n\nif __name__ == '__main__':\n resource = ArknightsSkinResource(\n Character.all(contains_extra=False),\n Character\n )\n\n api = HfApi(token=os.environ['HF_TOKEN'])\n api.create_repo('narugo/test_repo', repo_type='dataset', exist_ok=True)\n storage = HuggingfaceStorage(\n repo='narugo/test_repo',\n hf_client=api,\n namespace='genshin',\n )\n\n task = SyncTask(resource, storage)\n task.sync()\n\n```\n\n\n\n\n\n",
"bugtrack_url": null,
"license": "Apache License, Version 2.0",
"summary": "Mirror for resources to local and huggingface.",
"version": "0.0.7",
"project_urls": {
"Homepage": "https://github.com/narugo1992/hfmirror"
},
"split_keywords": [
"a",
"simple",
"tool",
"for",
"automatic",
"parameter",
"tuning."
],
"urls": [
{
"comment_text": "",
"digests": {
"blake2b_256": "f64f9f7d918dc3c51b8d7b05eb9c1e33fbc44042fcbfa2e9d97b7d23b3854b72",
"md5": "3aa942b08fe251754fdd83580febfe2f",
"sha256": "408b66bd5c20bfe321e3342f0b8f13d3db3c69e72a6620cbe3b84db09840d64c"
},
"downloads": -1,
"filename": "hfmirror-0.0.7-py3-none-any.whl",
"has_sig": false,
"md5_digest": "3aa942b08fe251754fdd83580febfe2f",
"packagetype": "bdist_wheel",
"python_version": "py3",
"requires_python": ">=3.7",
"size": 27971,
"upload_time": "2023-09-20T14:44:22",
"upload_time_iso_8601": "2023-09-20T14:44:22.456781Z",
"url": "https://files.pythonhosted.org/packages/f6/4f/9f7d918dc3c51b8d7b05eb9c1e33fbc44042fcbfa2e9d97b7d23b3854b72/hfmirror-0.0.7-py3-none-any.whl",
"yanked": false,
"yanked_reason": null
},
{
"comment_text": "",
"digests": {
"blake2b_256": "08fb3b3baa81f04bbfbb25944340300a71d7620f6a52f62ca93475672d9f5663",
"md5": "28607f59a3ac0650238820c802ae3ebc",
"sha256": "f8f5badef1eb87655d3d5d48ea42315ce7299b661f3b34dd7016b8a0d106ce4c"
},
"downloads": -1,
"filename": "hfmirror-0.0.7.tar.gz",
"has_sig": false,
"md5_digest": "28607f59a3ac0650238820c802ae3ebc",
"packagetype": "sdist",
"python_version": "source",
"requires_python": ">=3.7",
"size": 24704,
"upload_time": "2023-09-20T14:44:23",
"upload_time_iso_8601": "2023-09-20T14:44:23.995280Z",
"url": "https://files.pythonhosted.org/packages/08/fb/3b3baa81f04bbfbb25944340300a71d7620f6a52f62ca93475672d9f5663/hfmirror-0.0.7.tar.gz",
"yanked": false,
"yanked_reason": null
}
],
"upload_time": "2023-09-20 14:44:23",
"github": true,
"gitlab": false,
"bitbucket": false,
"codeberg": false,
"github_user": "narugo1992",
"github_project": "hfmirror",
"travis_ci": false,
"coveralls": false,
"github_actions": true,
"requirements": [],
"lcname": "hfmirror"
}
JSON
