人工智能 | 搭建企业内部的大语言模型系统

ollama pull llama2ollama listollama run llama2 "Summarize this file: $(cat README.md)"
ollama serve
curl http://localhost:11434/api/generate -d '{  "model": "llama2",  "prompt":"Why is the sky blue?"}'curl http://localhost:11434/api/chat -d '{  "model": "mistral",  "messages": [    { "role": "user", "content": "why is the sky blue?" }  ]}'

docker run -p 3000:3000 ghcr.io/ivanfioravanti/chatbot-ollama:main## http://localhost:3000

git clone https://github.com/imartinez/privateGPTcd privateGPT#不支持3.11之前的版本python3.11 -m venv .venvsource .venv/bin/activatepip install --upgrade pip poetry
#虽然官网只说了要安装少部分的依赖，但是那些依赖管理不是那么完善，容易有遗漏#所以我们的策略就是全都要。poetry install --extras "ui llms-llama-cpp llms-openai llms-openai-like llms-ollama llms-sagemaker llms-azopenai embeddings-ollama embeddings-huggingface embeddings-openai embeddings-sagemaker embeddings-azopenai vector-stores-qdrant vector-stores-chroma vector-stores-postgres storage-nodestore-postgres"
#或者用这个安装脚本#poetry install --extras "$(sed -n '/tool.poetry.extras/,/^$/p'  pyproject.toml | awk -F= 'NR>1{print $1}' | xargs)"

ollama pull mistralollama pull nomic-embed-textollama serve
#官方这个依赖不够，还需要额外安装torch，所以尽量采用上面提到的全部安装的策略poetry install --extras "ui llms-ollama embeddings-ollama vector-stores-qdrant"PGPT_PROFILES=ollama poetry run python -m private_gpt

server:  env_name: ${APP_ENV:ollama}
llm:  mode: ollama  max_new_tokens: 512  context_window: 3900  temperature: 0.1 #The temperature of the model. Increasing the temperature will make the model answer more creatively. A value of 0.1 would be more factual. (Default: 0.1)
embedding:  mode: ollama
ollama:  llm_model: mistral  embedding_model: nomic-embed-text  api_base: http://localhost:11434  tfs_z: 1.0 ## Tail free sampling is used to reduce the impact of less probable tokens from the output. A higher value (e.g., 2.0) will reduce the impact more, while a value of 1.0 disables this setting.  top_k: 40 ## Reduces the probability of generating nonsense. A higher value (e.g. 100) will give more diverse answers, while a lower value (e.g. 10) will be more conservative. (Default: 40)  top_p: 0.9 ## Works together with top-k. A higher value (e.g., 0.95) will lead to more diverse text, while a lower value (e.g., 0.5) will generate more focused and conservative text. (Default: 0.9)  repeat_last_n: 64 ## Sets how far back for the model to look back to prevent repetition. (Default: 64, 0 = disabled, -1 = num_ctx)  repeat_penalty: 1.2 ## Sets how strongly to penalize repetitions. A higher value (e.g., 1.5) will penalize repetitions more strongly, while a lower value (e.g., 0.9) will be more lenient. (Default: 1.1)
vectorstore:  database: qdrant
qdrant:  path: local_data/private_gpt/qdrant

PGPT_PROFILES=ollama poetry run python -m private_gpt
poetry run python -m private_gpt02:36:06.928 [INFO    ] private_gpt.settings.settings_loader - Starting application with profiles=['default', 'ollama']02:36:46.567 [INFO    ] private_gpt.components.llm.llm_component - Initializing the LLM in mode=ollama02:36:47.405 [INFO    ] private_gpt.components.embedding.embedding_component - Initializing the embedding model in mode=ollama02:36:47.414 [INFO    ] llama_index.core.indices.loading - Loading all indices.02:36:47.571 [INFO    ]         private_gpt.ui.ui - Mounting the gradio UI, at path=/02:36:47.620 [INFO    ]             uvicorn.error - Started server process [72677]02:36:47.620 [INFO    ]             uvicorn.error - Waiting for application startup.02:36:47.620 [INFO    ]             uvicorn.error - Application startup complete.02:36:47.620 [INFO    ]             uvicorn.error - Uvicorn running on http://0.0.0.0:8001 (Press CTRL+C to quit)

#todo: 需要安装llama-cpp，每个平台的安装方式都不同，参考官方文档
poetry run python scripts/setupPGPT_PROFILES=local poetry run python -m private_gpt

server:  env_name: ${APP_ENV:local}
llm:  mode: llamacpp  ## Should be matching the selected model  max_new_tokens: 512  context_window: 3900  tokenizer: mistralai/Mistral-7B-Instruct-v0.2
llamacpp:  prompt_style: "mistral"  llm_hf_repo_id: TheBloke/Mistral-7B-Instruct-v0.2-GGUF  llm_hf_model_file: mistral-7b-instruct-v0.2.Q4_K_M.gguf
embedding:  mode: huggingface
huggingface:  embedding_hf_model_name: BAAI/bge-small-en-v1.5
vectorstore:  database: qdrant
qdrant:  path: local_data/private_gpt/qdrant

poetry install --extras "ui llms-openai embeddings-openai vector-stores-qdrant"PGPT_PROFILES=openai poetry run python -m private_gpt

server:  env_name: ${APP_ENV:openai}
llm:  mode: openai
embedding:  mode: openai
openai:  api_key: ${OPENAI_API_KEY:}  model: gpt-3.5-turbo

curl -X POST http://localhost:8000/v1/completions \     -H "Content-Type: application/json" \     -d '{  "prompt": "string",  "stream": true
}'