Local RAG with Ollama embeddings

Embedding models such as nomic-embed-text, batched /api/embed calls, storing vectors, sensible chunking, and answering from your own documents offline.

Embedding with /api/embed

import requests
import numpy as np

BASE = "http://localhost:11434"

def embed(texts, model="nomic-embed-text:v1.5"):
    """Batch embedding: one HTTP call per batch, not per document."""
    response = requests.post(
        f"{BASE}/api/embed",
        json={"model": model, "input": texts, "truncate": True},
        timeout=300,
    )
    response.raise_for_status()
    return np.asarray(response.json()["embeddings"], dtype="float32")

documents = [
    "Refunds are available within 30 days of delivery.",
    "Gift cards cannot be refunded or exchanged for cash.",
    "Enterprise plans include a one-hour response target.",
]

vectors = embed(documents)
print(vectors.shape)                    # (3, 768) for nomic-embed-text

def cosine(query_vec, matrix):
    q = query_vec / np.linalg.norm(query_vec)
    m = matrix / np.linalg.norm(matrix, axis=1, keepdims=True)
    return m @ q

query = embed(["can I get my money back for a gift card?"])[0]
scores = cosine(query, vectors)
for text, score in sorted(zip(documents, scores), key=lambda p: -p[1]):
    print(round(float(score), 4), text)

• Send many inputs in one input array. Calling the endpoint once per document is the most common performance mistake in a local RAG pipeline.
• Use an embedding model, not a chat model. nomic-embed-text is designed for retrieval; a chat model's hidden states are not.
• Normalise the vectors if you intend to use cosine similarity, and be consistent: normalise at index time and query time, or neither.
• Store the embedding model name with the index. Upgrading the model without re-embedding leaves a corpus split across two incompatible vector spaces.

A complete local pipeline

import json
import re
import sqlite3
from pathlib import Path

import numpy as np
import requests

BASE = "http://localhost:11434"
EMBED_MODEL = "nomic-embed-text:v1.5"
CHAT_MODEL = "llama3.2:3b-instruct-q4_K_M"

def chunk(text, size=700, overlap=100):
    """Split on paragraph boundaries, then pack to roughly the target size."""
    paragraphs = [p.strip() for p in re.split(r"\n\s*\n", text) if p.strip()]
    chunks, current = [], ""
    for paragraph in paragraphs:
        if len(current) + len(paragraph) + 1 <= size:
            current = (current + "\n" + paragraph).strip()
        else:
            if current:
                chunks.append(current)
            tail = current[-overlap:] if current else ""
            current = (tail + "\n" + paragraph).strip()
    if current:
        chunks.append(current)
    return chunks

def init_db(path="index.sqlite"):
    con = sqlite3.connect(path)
    con.execute("CREATE TABLE IF NOT EXISTS chunks "
                "(id INTEGER PRIMARY KEY, source TEXT, text TEXT, vector BLOB)")
    con.commit()
    return con

def index_file(con, path: Path):
    text = path.read_text(encoding="utf-8", errors="ignore")
    pieces = [f"[{path.name}] {c}" for c in chunk(text)]
    if not pieces:
        return 0
    vectors = embed(pieces, model=EMBED_MODEL)
    con.executemany(
        "INSERT INTO chunks (source, text, vector) VALUES (?, ?, ?)",
        [(str(path), piece, vec.tobytes()) for piece, vec in zip(pieces, vectors)],
    )
    con.commit()
    return len(pieces)

def retrieve(con, question, k=4):
    rows = con.execute("SELECT text, vector FROM chunks").fetchall()
    if not rows:
        return []
    matrix = np.vstack([np.frombuffer(r[1], dtype="float32") for r in rows])
    query = embed([question], model=EMBED_MODEL)[0]
    scores = cosine(query, matrix)
    order = np.argsort(scores)[::-1][:k]
    return [(rows[i][0], float(scores[i])) for i in order]

def answer(con, question, model=CHAT_MODEL):
    passages = retrieve(con, question)
    context = "\n\n".join(text for text, _ in passages)
    prompt = (f"Answer the question using only the context below. "
              f"If the context does not contain the answer, reply NOT_IN_CONTEXT.\n\n"
              f"Context:\n{context}\n\nQuestion: {question}\nAnswer:")
    reply = requests.post(f"{BASE}/api/generate",
                          json={"model": model, "prompt": prompt, "stream": False,
                                "options": {"temperature": 0.0, "num_ctx": 4096}},
                          timeout=180).json()["response"]
    return {"answer": reply.strip(), "sources": [t[:60] for t, _ in passages]}

con = init_db()
print(index_file(con, Path("policy.txt")), "chunks indexed")
print(answer(con, "What is the refund window for a gift card?"))

Improving retrieval quality

# run the retrieval stage on its own and count how often the right chunk appears
CASES = [
    {"question": "How long do I have to return an item?", "expected": "30 days"},
    {"question": "Can I refund a gift card?", "expected": "gift card"},
]

def recall_at_k(con, cases, k=4):
    hits = 0
    for case in cases:
        passages = retrieve(con, case["question"], k=k)
        if any(case["expected"].lower() in text.lower() for text, _ in passages):
            hits += 1
    return hits / len(cases)

print("recall@4 =", recall_at_k(con, CASES))

# an embedding-model comparison keeps everything else fixed
for model in ["nomic-embed-text:v1.5", "mxbai-embed-large"]:
    EMBED_MODEL = model
    print(model, recall_at_k(con, CASES))

• Measure retrieval recall separately from answer quality. A wrong answer from a correct passage and a wrong answer from a missing passage need completely different fixes.
• Hybrid retrieval matters more with small local models, which are weaker at matching paraphrases than large hosted embedding models.
• Rerank the top twenty candidates with a cross-encoder, locally if you have the compute. On a local stack this is often the single largest quality gain.
• Keep the whole pipeline offline and reproducible: the same documents, the same embedding model, the same chunking parameters. A RAG system you cannot rebuild is one you cannot debug.

FAQ

Related

Calling Ollama from code Structured output and tool calling

Last refreshed 2026-09-18.