C#使用词嵌入向量与向量数据库为大语言模型(LLM)赋能长期记忆实现私域问答机器人落地之openai接口平替

from fastapi import FastAPI
from pydantic import BaseModel
from typing import List
from transformers import AutoTokenizer, AutoModel
import torch

app = FastAPI()

# Load the model and tokenizer
model = AutoModel.from_pretrained("/app").half().cuda()
tokenizer = AutoTokenizer.from_pretrained("/app")


# Request body
class Sentence(BaseModel):
    sentence: str


@app.post("/embed")
async def embed(sentence: Sentence):
    # Tokenize the sentence and get the input tensors
    inputs = tokenizer(sentence.sentence, return_tensors='pt', padding=True, truncation=True, max_length=512)

    # Move inputs to GPU
    for key in inputs.keys():
        inputs[key] = inputs[key].to('cuda')

    # Run the model
    with torch.no_grad():
        outputs = model(**inputs)

    # Get the embeddings
    embeddings = outputs.last_hidden_state[0].cpu().numpy()

    # Return the embeddings as a JSON response
    return embeddings.tolist()

from fastapi import FastAPI
from pydantic import BaseModel
from typing import List
from transformers import AutoTokenizer, AutoModel
import torch

app = FastAPI()

# Load the model and tokenizer
model = AutoModel.from_pretrained("/app").half()
tokenizer = AutoTokenizer.from_pretrained("/app")

# Request body
class Sentence(BaseModel):
    sentence: str

@app.post("/embed")
async def embed(sentence: Sentence):
    # Tokenize the sentence and get the input tensors
    inputs = tokenizer(sentence.sentence, return_tensors='pt', padding=True, truncation=True, max_length=512)

    # No need to move inputs to GPU as we are using CPU

    # Run the model
    with torch.no_grad():
        outputs = model(**inputs)

    # Get the embeddings
    embeddings = outputs.last_hidden_state[0].cpu().numpy()

    # Return the embeddings as a JSON response
    return embeddings.tolist()

torch
transformers
fastapi
uvicorn

FROM python:3.8-slim-buster

# Set the working directory to /app
WORKDIR /app

# Copy the current directory contents into the container at /app
ADD . /app

# Install any needed packages specified in requirements.txt
RUN pip install --trusted-host pypi.python.org -r requirements.txt

# Run app.py when the container launches
ENV MODULE_NAME=web 
ENV VARIABLE_NAME=app 
ENV HOST=0.0.0.0 
ENV PORT=80

# Run the application: 
CMD uvicorn ${MODULE_NAME}:${VARIABLE_NAME} --host ${HOST} --port ${PORT}

git clone http://github.com/THUDM/ChatGLM-6B.git

from fastapi import FastAPI, Request
from fastapi.responses import StreamingResponse
from transformers import AutoTokenizer, AutoModel
import uvicorn, json, datetime
import torch
import asyncio

DEVICE = "cuda"
DEVICE_ID = "0"
CUDA_DEVICE = f"{DEVICE}:{DEVICE_ID}" if DEVICE_ID else DEVICE


def torch_gc():
    if torch.cuda.is_available():
        with torch.cuda.device(CUDA_DEVICE):
            torch.cuda.empty_cache()
            torch.cuda.ipc_collect()

app = FastAPI()

@app.post("/chat", response_class=StreamingResponse)
async def create_item(request: Request):
    global model, tokenizer
    json_post_raw = await request.json()
    json_post = json.dumps(json_post_raw)
    json_post_list = json.loads(json_post)
    prompt = json_post_list.get('prompt')
    history = json_post_list.get('history')
    max_length = json_post_list.get('max_length')
    top_p = json_post_list.get('top_p')
    temperature = json_post_list.get('temperature')
    
    last_response = ''
    async def stream_chat():
        nonlocal last_response,history
        for response, history in model.stream_chat(tokenizer,
                                                prompt,
                                                history=history,
                                                max_length=max_length if max_length else 2048,
                                                top_p=top_p if top_p else 0.7,
                                                temperature=temperature if temperature else 0.95):
            new_part = response[len(last_response):]
            last_response = response
            yield json.dumps(new_part,ensure_ascii=False)
            
    return StreamingResponse(stream_chat(), media_type="text/plain")


if __name__ == '__main__':
    tokenizer = AutoTokenizer.from_pretrained("/app", trust_remote_code=True)
    model = AutoModel.from_pretrained("/app", trust_remote_code=True).half().cuda()
    model.eval()
    uvicorn.run(app, host='0.0.0.0', port=80, workers=1)

model = AutoModel.from_pretrained("/app", trust_remote_code=True).half().cuda()

model = AutoModel.from_pretrained("/app", trust_remote_code=True)

FROM python:3.8-slim-buster
WORKDIR /app
ADD . /app
RUN pip install -r requirements.txt -i http://pypi.tuna.tsinghua.edu.cn/simple
CMD ["python", "api.py"]

///用于模拟向量库    
private Dictionary<string, List<double>> MemoryList = new Dictionary<string, List<double>>();
///用于计算相似度
double Compute(List<double> vector1, List<double> vector2) => vector1.Zip(vector2, (a, b) => a * b).Sum() / (Math.Sqrt(vector1.Sum(a => a * a)) * Math.Sqrt(vector2.Sum(b => b * b)));
...
    [HttpPost("/api/save")]
    public async Task<int> SaveMemory(string str)
    {
        if (!string.IsNullOrEmpty(str))
        {
            foreach (var x in memory.Split("\n").ToList())
            {
                if (!MemoryList.ContainsKey(x))
                {
                    MemoryList.Add(x, await GetEmbeding(x));
                    StateHasChanged();
                }
            }
        }
        return MemoryList.Count; 
    }
...
    [HttpPost("/api/chat")]
    public async IAsyncEnumerable<string> SendData(string content)
    {
        if (!string.IsNullOrEmpty(content))
        {
            var userquestionEmbeding = await GetEmbeding(content);
            var prompt = "";
            if (MemoryList.Any())
            {  //这里从向量库中获取到第一条，你可以根据实际情况设置比如相似度阈值或者返回多条等等
                prompt = MemoryList.OrderByDescending(x => Compute(userquestionEmbeding, x.Value)).FirstOrDefault().Key;
                prompt = $"你是一个问答小助手，你需要基于以下事实依据回答问题，事实依据如下：{prompt}。用户的问题如下：{Content}。不要编造事实依据，请回答：";
            }
            else
                prompt = Content;
            await foreach (var item in ChatStream(prompt))
            {
                yield return item;
            }
        }
    }

async IAsyncEnumerable<string> ChatStream(string x)
    {
        HttpClient hc = new HttpClient();
        var reqcontent = new StringContent(System.Text.Json.JsonSerializer.Serialize(new { prompt = x }));
        reqcontent.Headers.ContentType = new System.Net.Http.Headers.MediaTypeHeaderValue("application/json");
        var response = await hc.PostAsync("http://192.168.1.100:8081/chat", reqcontent);
        if (response.IsSuccessStatusCode)
        {
            var responseStream = await response.Content.ReadAsStreamAsync();
            using (var reader = new StreamReader(responseStream, Encoding.UTF8))
            {
                string line;
                while ((line = await reader.ReadLineAsync()) != null)
                {
                    yield return line;
                }
            }
        }
    }
    async Task<List<double>> GetEmbeding(string x)
    {
        HttpClient hc = new HttpClient();
        var reqcontent = new StringContent(System.Text.Json.JsonSerializer.Serialize(new { sentence = x }));
        reqcontent.Headers.ContentType = new System.Net.Http.Headers.MediaTypeHeaderValue("application/json");
        var result = await hc.PostAsync("http://192.168.1.100:8080/embed", reqcontent);
        var content = await result.Content.ReadAsStringAsync();
        var embed = System.Text.Json.JsonSerializer.Deserialize<List<List<double>>>(content);
        var embedresult = new List<double>();
        for (var i = 0; i < 1024; i++)
        {
            double sum = 0;
            foreach (List<double> sublist in embed)
            {
                sum += (sublist[i]);
            }
            embedresult.Add(sum / 1024);
        }
        return embedresult;
    }