Chatbot Deployment System

from fastapi import FastAPI
from fastapi.responses import StreamingResponse
from openai import OpenAI

app = FastAPI()
client = OpenAI()

@app.post("/chat/stream")
async def chat_stream(req: ChatRequest):
    history = await load_history(req.session_id)
    history.append({"role": "user", "content": req.message})

    def gen():
        full = []
        stream = client.chat.completions.create(
            model="gpt-4o-mini",
            messages=history,
            stream=True,
        )
        for chunk in stream:
            tok = chunk.choices[0].delta.content or ""
            full.append(tok)
            yield f"data: {tok}\n\n"
        save_turn(req.session_id, req.message, "".join(full))
        yield "data: [DONE]\n\n"

    return StreamingResponse(gen(), media_type="text/event-stream")

MAX_TOKENS = 8000
RESERVED_REPLY = 1000

def trim_history(messages):
    while count_tokens(messages) > MAX_TOKENS - RESERVED_REPLY:
        # 1) drop oldest non-system turn
        for i, m in enumerate(messages):
            if m["role"] != "system":
                messages.pop(i)
                break
    return messages

# Better: summarize older turns
def compress(messages):
    if count_tokens(messages) < MAX_TOKENS * 0.7:
        return messages
    old, recent = messages[1:-6], messages[-6:]
    summary = summarize(old)
    return [messages[0],
            {"role": "system", "content": f"Earlier summary: {summary}"},
            *recent]

from sentence_transformers import SentenceTransformer
import chromadb

embedder = SentenceTransformer("all-MiniLM-L6-v2")
db = chromadb.PersistentClient("./vdb").get_collection("docs")

def augment(question: str, k: int = 4) -> list[dict]:
    qv = embedder.encode([question]).tolist()
    hits = db.query(query_embeddings=qv, n_results=k)
    context = "\n---\n".join(hits["documents"][0])
    return [
        {"role": "system", "content":
            "Answer ONLY from the context. If unknown, say so.\n\n" + context},
        {"role": "user", "content": question},
    ]