大模型之RAG：关键字检索的认识与实战（混合检索进阶储备）

# 安装ES客户端!pip install elasticsearch7
# 安装 NLTK（文本处理方法库）!pip install nltk

from elasticsearch7 import Elasticsearch, helpersfrom nltk.stem import PorterStemmerfrom nltk.tokenize import word_tokenizefrom nltk.corpus import stopwordsimport nltkimport re
import warnings# 屏蔽 ES 的一些Warningswarnings.simplefilter("ignore")  
# 英文切词、词根、切句等方法nltk.download('punkt')  # 英文停用词库nltk.download('stopwords')  
def to_keywords(input_string):    '''（英文）文本只保留关键字'''    # 使用正则表达式替换所有非字母数字的字符为空格    no_symbols = re.sub(r'[^a-zA-Z0-9\s]', ' ', input_string)    word_tokens = word_tokenize(no_symbols)    # 加载停用词表    stop_words = set(stopwords.words('english'))    ps = PorterStemmer()    # 去停用词，取词根    filtered_sentence = [ps.stem(w)                         for w in word_tokens if not w.lower() in stop_words]    return ' '.join(filtered_sentence)

import reimport jiebaimport nltkfrom nltk.corpus import stopwords
nltk.download('stopwords')  
def to_keywords(input_string):    """将句子转成检索关键词序列"""    # 按搜索引擎模式分词    word_tokens = jieba.cut_for_search(input_string)    # 加载停用词表    stop_words = set(stopwords.words('chinese'))    # 去除停用词    filtered_sentence = [w for w in word_tokens if not w in stop_words]    return ' '.join(filtered_sentence)
def sent_tokenize(input_string):    """按标点断句"""    # 按标点切分    sentences = re.split(r'(?<=[。！？；?!])', input_string)    # 去掉空字符串    return [sentence for sentence in sentences if sentence.strip()]

import os, time
# 引入配置文件ELASTICSEARCH_BASE_URL = os.getenv('ELASTICSEARCH_BASE_URL')ELASTICSEARCH_PASSWORD = os.getenv('ELASTICSEARCH_PASSWORD')ELASTICSEARCH_NAME= os.getenv('ELASTICSEARCH_NAME')
# tips: 如果想在本地运行，请在下面一行 print(ELASTICSEARCH_BASE_URL) 获取真实的配置
# 1. 创建Elasticsearch连接es = Elasticsearch(    hosts=[ELASTICSEARCH_BASE_URL],  # 服务地址与端口    http_auth=(ELASTICSEARCH_NAME, ELASTICSEARCH_PASSWORD),  # 用户名，密码)
# 2. 定义索引名称index_name = "teacher_demo_index"
# 3. 如果索引已存在，删除它（仅供演示，实际应用时不需要这步）if es.indices.exists(index=index_name):    es.indices.delete(index=index_name)
# 4. 创建索引es.indices.create(index=index_name)
# 5. 灌库指令actions = [    {        "_index": index_name,        "_source": {            "keywords": to_keywords(para),            "text": para        }    }    for para in paragraphs]
# 6. 文本灌库helpers.bulk(es, actions)
# 灌库是异步的time.sleep(2)

def search(query_string, top_n=3):    # ES 的查询语言    search_query = {        "match": {            "keywords": to_keywords(query_string)        }    }    res = es.search(index=index_name, query=search_query, size=top_n)    return [hit["_source"]["text"] for hit in res["hits"]["hits"]]    results = search("how many parameters does llama 2 have?", 2)for r in results:    print(r+"\n")

from openai import OpenAIimport os# 加载环境变量from dotenv import load_dotenv, find_dotenv_ = load_dotenv(find_dotenv())  # 读取本地 .env 文件，里面定义了 OPENAI_API_KEY
client = OpenAI()
def get_completion(prompt, model="gpt-3.5-turbo"):    '''封装 openai 接口'''    messages = [{"role": "user", "content": prompt}]    response = client.chat.completions.create(        model=model,        messages=messages,        temperature=0,  # 模型输出的随机性，0 表示随机性最小    )    return response.choices[0].message.content

def build_prompt(prompt_template, **kwargs):    '''将 Prompt 模板赋值'''    inputs = {}    for k, v in kwargs.items():        if isinstance(v, list) and all(isinstance(elem, str) for elem in v):            val = '\n\n'.join(v)        else:            val = v        inputs[k] = val    return prompt_template.format(**inputs)

prompt_template = """你是一个问答机器人。你的任务是根据下述给定的已知信息回答用户问题。
已知信息:{context}
用户问：{query}
如果已知信息不包含用户问题的答案，或者已知信息不足以回答用户的问题，请直接回复"我无法回答您的问题"。请不要输出已知信息中不包含的信息或答案。请用中文回答用户问题。"""

user_query = "how many parameters does llama 2 have?"
# 1. 检索search_results = search(user_query, 2)
# 2. 构建 Promptprompt = build_prompt(prompt_template, context=search_results, query=user_query)print("===Prompt===")print(prompt)
# 3. 调用 LLMresponse = get_completion(prompt)
print("===回复===")print(response)

===Prompt===
你是一个问答机器人。你的任务是根据下述给定的已知信息回答用户问题。
已知信息: 1. Llama 2, an updated version of Llama 1, trained on a new mix of publicly available data. We also increased the size of the pretraining corpus by 40%, doubled the context length of the model, and adopted grouped-query attention (Ainslie et al., 2023). We are releasing variants of Llama 2 with 7B, 13B, and 70B parameters. We have also trained 34B variants, which we report on in this paper but are not releasing.§
 In this work, we develop and release Llama 2, a collection of pretrained and fine-tuned large language models (LLMs) ranging in scale from 7 billion to 70 billion parameters. Our fine-tuned LLMs, called Llama 2-Chat, are optimized for dialogue use cases. Our models outperform open-source chat models on most benchmarks we tested, and based onour human evaluations for helpfulness and safety, may be a suitable substitute for closed source models. We provide a detailed description of our approach to fine-tuning and safety improvements of Llama 2-Chat in order to enable the community to build on our work and contribute to the responsible development of LLMs.
用户问：how many parameters does llama 2 have?
如果已知信息不包含用户问题的答案，或者已知信息不足以回答用户的问题，请直接回复"我无法回答您的问题"。请不要输出已知信息中不包含的信息或答案。请用中文回答用户问题。
===回复===Llama 2有7B, 13B和70B参数。