# openai-agent-llm
- 为本地私有部署大模型提供openai接口兼容
- 开发者只需要定义推理接口内容即可
- 安装
```bash
pip install openai_agent_llm
```
```python
import json
import os
import uuid
import uvicorn
from starlette.middleware.cors import CORSMiddleware
from openai_agent_llm import LLMAPI
from openai_agent_llm import ModelServer
from websockets.sync.client import connect
from loguru import logger
server = os.environ.get("server", "wss://mikeee-chatglm2-6b-4bit.hf.space/queue/join")
class ChatGLMInt4SpiderServer(ModelServer):
http_request = None
def deal_prompt(self, params):
prompt = params.get("prompt")
if type(prompt) == str:
prompt = prompt.replace("Human:", "问:").replace("AI:", "答:")
if len(prompt) > 2 and prompt[-2:0] == "答:":
prompt = prompt[:-2]
elif type(prompt) == list:
prompt = ""
for i in params.get("prompt", []):
role = i.get("role", "")
if role == "system":
pass
else:
prompt = prompt + i.get("content", "") + "\n"
else:
prompt = "hi"
# if len(prompt) * CutWordMultiple > 1024:
# prompt = prompt[int(len(prompt) * 2) / 3:]
return prompt
def send_init(self, websocket, session_hash):
value = websocket.recv()
logger.info(value)
websocket.send(
json.dumps({"fn_index": 1, "session_hash": session_hash}, ensure_ascii=False))
for i in range(2):
value = websocket.recv()
logger.info(value)
def infer(self, params):
prompt = self.deal_prompt(params)
url = server
request_id = str(uuid.uuid4())
with connect(url) as websocket:
self.send_init(websocket=websocket, session_hash=request_id)
websocket.send(
json.dumps({"data": [False, prompt, [], 8192, 0.85, 0.95, None, None],
"event_data": None, "fn_index": 1, "session_hash": request_id}, ensure_ascii=False))
text = ""
while True:
value = websocket.recv()
try:
data = json.loads(value)
except:
continue
msg = data.get("msg")
if data.get("msg") == "process_completed":
finish = True
else:
finish = False
if msg == "process_starts":
continue
elif msg == "process_generating":
data_ = data.get("output", {}).get("data", [])[0][0][1][3:-4].replace("<br>", "\n")
token = data_[len(text):]
text = data_
stop = None
if finish is True:
stop = "stop"
yield {
"text": token,
"usage": {
"prompt_tokens": 0,
"completion_tokens": 0,
"total_tokens": 0,
},
"finish_reason": stop,
}
if finish is True:
print()
break
app = LLMAPI()
app.init(model_server_class=ChatGLMInt4SpiderServer)
app.add_middleware(
CORSMiddleware,
allow_origins=["*"],
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
if __name__ == '__main__':
uvicorn.run(app=app, port=9000)
```
Raw data
{
"_id": null,
"home_page": "https://github.com/lijiacaigit/openai-agent",
"name": "openai-agent-llm",
"maintainer": "",
"docs_url": null,
"requires_python": ">=3.7.1",
"maintainer_email": "",
"keywords": "",
"author": "Lijiacai",
"author_email": "1050518702@qq.com",
"download_url": "https://files.pythonhosted.org/packages/50/46/1f943c32d3c80debabaed8bcd26b47b5793e13447b8362cd636646ab769f/openai-agent-llm-0.0.2.tar.gz",
"platform": null,
"description": "# openai-agent-llm\r\n\r\n- \u4e3a\u672c\u5730\u79c1\u6709\u90e8\u7f72\u5927\u6a21\u578b\u63d0\u4f9bopenai\u63a5\u53e3\u517c\u5bb9\r\n- \u5f00\u53d1\u8005\u53ea\u9700\u8981\u5b9a\u4e49\u63a8\u7406\u63a5\u53e3\u5185\u5bb9\u5373\u53ef\r\n- \u5b89\u88c5\r\n\r\n```bash\r\npip install openai_agent_llm\r\n```\r\n\r\n```python\r\nimport json\r\nimport os\r\nimport uuid\r\n\r\nimport uvicorn\r\nfrom starlette.middleware.cors import CORSMiddleware\r\n\r\nfrom openai_agent_llm import LLMAPI\r\nfrom openai_agent_llm import ModelServer\r\nfrom websockets.sync.client import connect\r\nfrom loguru import logger\r\n\r\nserver = os.environ.get(\"server\", \"wss://mikeee-chatglm2-6b-4bit.hf.space/queue/join\")\r\n\r\n\r\nclass ChatGLMInt4SpiderServer(ModelServer):\r\n http_request = None\r\n\r\n def deal_prompt(self, params):\r\n prompt = params.get(\"prompt\")\r\n if type(prompt) == str:\r\n prompt = prompt.replace(\"Human:\", \"\u95ee:\").replace(\"AI:\", \"\u7b54:\")\r\n if len(prompt) > 2 and prompt[-2:0] == \"\u7b54:\":\r\n prompt = prompt[:-2]\r\n elif type(prompt) == list:\r\n prompt = \"\"\r\n for i in params.get(\"prompt\", []):\r\n role = i.get(\"role\", \"\")\r\n if role == \"system\":\r\n pass\r\n else:\r\n prompt = prompt + i.get(\"content\", \"\") + \"\\n\"\r\n else:\r\n prompt = \"hi\"\r\n # if len(prompt) * CutWordMultiple > 1024:\r\n # prompt = prompt[int(len(prompt) * 2) / 3:]\r\n return prompt\r\n\r\n def send_init(self, websocket, session_hash):\r\n value = websocket.recv()\r\n logger.info(value)\r\n websocket.send(\r\n json.dumps({\"fn_index\": 1, \"session_hash\": session_hash}, ensure_ascii=False))\r\n for i in range(2):\r\n value = websocket.recv()\r\n logger.info(value)\r\n\r\n def infer(self, params):\r\n prompt = self.deal_prompt(params)\r\n url = server\r\n request_id = str(uuid.uuid4())\r\n with connect(url) as websocket:\r\n self.send_init(websocket=websocket, session_hash=request_id)\r\n websocket.send(\r\n json.dumps({\"data\": [False, prompt, [], 8192, 0.85, 0.95, None, None],\r\n \"event_data\": None, \"fn_index\": 1, \"session_hash\": request_id}, ensure_ascii=False))\r\n text = \"\"\r\n while True:\r\n value = websocket.recv()\r\n try:\r\n data = json.loads(value)\r\n except:\r\n continue\r\n msg = data.get(\"msg\")\r\n if data.get(\"msg\") == \"process_completed\":\r\n finish = True\r\n else:\r\n finish = False\r\n if msg == \"process_starts\":\r\n continue\r\n elif msg == \"process_generating\":\r\n data_ = data.get(\"output\", {}).get(\"data\", [])[0][0][1][3:-4].replace(\"<br>\", \"\\n\")\r\n token = data_[len(text):]\r\n text = data_\r\n stop = None\r\n if finish is True:\r\n stop = \"stop\"\r\n yield {\r\n \"text\": token,\r\n \"usage\": {\r\n \"prompt_tokens\": 0,\r\n \"completion_tokens\": 0,\r\n \"total_tokens\": 0,\r\n },\r\n \"finish_reason\": stop,\r\n }\r\n if finish is True:\r\n print()\r\n break\r\n\r\n\r\napp = LLMAPI()\r\n\r\napp.init(model_server_class=ChatGLMInt4SpiderServer)\r\n\r\napp.add_middleware(\r\n CORSMiddleware,\r\n allow_origins=[\"*\"],\r\n allow_credentials=True,\r\n allow_methods=[\"*\"],\r\n allow_headers=[\"*\"],\r\n)\r\n\r\nif __name__ == '__main__':\r\n uvicorn.run(app=app, port=9000)\r\n\r\n```\r\n\r\n",
"bugtrack_url": null,
"license": "",
"summary": "Customize the LLM interface to make it compatible with the openai interface",
"version": "0.0.2",
"project_urls": {
"Homepage": "https://github.com/lijiacaigit/openai-agent"
},
"split_keywords": [],
"urls": [
{
"comment_text": "",
"digests": {
"blake2b_256": "50461f943c32d3c80debabaed8bcd26b47b5793e13447b8362cd636646ab769f",
"md5": "b08156fb76393f631c0a234558f03666",
"sha256": "16ce76a2045813be78eb83cff61c0807b7a38bc2877cafd330db2fbda2152616"
},
"downloads": -1,
"filename": "openai-agent-llm-0.0.2.tar.gz",
"has_sig": false,
"md5_digest": "b08156fb76393f631c0a234558f03666",
"packagetype": "sdist",
"python_version": "source",
"requires_python": ">=3.7.1",
"size": 8805,
"upload_time": "2023-07-17T03:28:28",
"upload_time_iso_8601": "2023-07-17T03:28:28.639924Z",
"url": "https://files.pythonhosted.org/packages/50/46/1f943c32d3c80debabaed8bcd26b47b5793e13447b8362cd636646ab769f/openai-agent-llm-0.0.2.tar.gz",
"yanked": false,
"yanked_reason": null
}
],
"upload_time": "2023-07-17 03:28:28",
"github": true,
"gitlab": false,
"bitbucket": false,
"codeberg": false,
"github_user": "lijiacaigit",
"github_project": "openai-agent",
"github_not_found": true,
"lcname": "openai-agent-llm"
}
JSON
