Chunking strategies

Embedding models

Workflow

Step 1: Document chunking

type Chunk = {
  id: string;
  text: string;
  metadata: {
    source: string;
    title: string;
    section: string;
    chunkIndex: number;
    tokenCount: number;
  };
};

function chunkDocument(
  doc: { title: string; content: string; source: string },
  maxTokens = 500,
  overlap = 50
): Chunk[] {
  const paragraphs = doc.content.split(/\n\n+/);
  const chunks: Chunk[] = [];
  let currentChunk = "";
  let chunkIndex = 0;

  for (const paragraph of paragraphs) {
    const combined = currentChunk
      ? \`\${currentChunk}\n\n\${paragraph}\`
      : paragraph;

    if (estimateTokens(combined) > maxTokens && currentChunk) {
      chunks.push({
        id: \`\${doc.source}-\${chunkIndex}\`,
        text: currentChunk,
        metadata: {
          source: doc.source,
          title: doc.title,
          section: extractSection(currentChunk),
          chunkIndex,
          tokenCount: estimateTokens(currentChunk),
        },
      });
      chunkIndex++;

      // Overlap: keep last N tokens from previous chunk
      const sentences = currentChunk.split(/\.\s+/);
      currentChunk = sentences.slice(-2).join(". ") + ". " + paragraph;
    } else {
      currentChunk = combined;
    }
  }

  if (currentChunk) {
    chunks.push({
      id: \`\${doc.source}-\${chunkIndex}\`,
      text: currentChunk,
      metadata: {
        source: doc.source,
        title: doc.title,
        section: extractSection(currentChunk),
        chunkIndex,
        tokenCount: estimateTokens(currentChunk),
      },
    });
  }

  return chunks;
}

Step 2: Generate embeddings

import OpenAI from "openai";

const openai = new OpenAI();

async function embedChunks(chunks: Chunk[]): Promise<Array<Chunk & { embedding: number[] }>> {
  const batchSize = 100;
  const results: Array<Chunk & { embedding: number[] }> = [];

  for (let i = 0; i < chunks.length; i += batchSize) {
    const batch = chunks.slice(i, i + batchSize);
    const response = await openai.embeddings.create({
      model: "text-embedding-3-small",
      input: batch.map(c => c.text),
    });

    for (let j = 0; j < batch.length; j++) {
      results.push({
        ...batch[j],
        embedding: response.data[j].embedding,
      });
    }
  }

  return results;
}

async function embedQuery(query: string): Promise<number[]> {
  const response = await openai.embeddings.create({
    model: "text-embedding-3-small",
    input: query,
  });
  return response.data[0].embedding;
}

Step 3: Store in vector database (Pinecone)

import { Pinecone } from "@pinecone-database/pinecone";

const pinecone = new Pinecone();
const index = pinecone.index("knowledge-base");

async function upsertChunks(chunks: Array<Chunk & { embedding: number[] }>) {
  const vectors = chunks.map(chunk => ({
    id: chunk.id,
    values: chunk.embedding,
    metadata: {
      text: chunk.text,
      source: chunk.metadata.source,
      title: chunk.metadata.title,
      section: chunk.metadata.section,
    },
  }));

  // Upsert in batches of 100
  for (let i = 0; i < vectors.length; i += 100) {
    await index.upsert(vectors.slice(i, i + 100));
  }
}

async function queryVectors(
  embedding: number[],
  topK = 5,
  filter?: Record<string, string>
) {
  const result = await index.query({
    vector: embedding,
    topK,
    includeMetadata: true,
    filter,
  });

  return result.matches?.map(match => ({
    id: match.id,
    score: match.score ?? 0,
    text: (match.metadata?.text as string) ?? "",
    source: (match.metadata?.source as string) ?? "",
    title: (match.metadata?.title as string) ?? "",
  })) ?? [];
}

Step 4: Alternative — Supabase pgvector

import { createClient } from "@supabase/supabase-js";

const supabase = createClient(
  process.env.SUPABASE_URL!,
  process.env.SUPABASE_SERVICE_KEY!
);

async function upsertChunksPgvector(chunks: Array<Chunk & { embedding: number[] }>) {
  const rows = chunks.map(chunk => ({
    id: chunk.id,
    content: chunk.text,
    embedding: chunk.embedding,
    metadata: chunk.metadata,
  }));

  const { error } = await supabase.from("documents").upsert(rows);
  if (error) throw error;
}

async function queryPgvector(embedding: number[], topK = 5) {
  const { data, error } = await supabase.rpc("match_documents", {
    query_embedding: embedding,
    match_threshold: 0.7,
    match_count: topK,
  });

  if (error) throw error;
  return data;
}

// Required SQL function for pgvector similarity search:
// CREATE FUNCTION match_documents(
//   query_embedding vector(1536),
//   match_threshold float,
//   match_count int
// ) RETURNS TABLE (id text, content text, similarity float)
// LANGUAGE plpgsql AS $$
// BEGIN
//   RETURN QUERY
//   SELECT d.id, d.content,
//     1 - (d.embedding <=> query_embedding) as similarity
//   FROM documents d
//   WHERE 1 - (d.embedding <=> query_embedding) > match_threshold
//   ORDER BY d.embedding <=> query_embedding
//   LIMIT match_count;
// END;
// $$;

Step 5: Build the RAG prompt

function buildRagPrompt(
  query: string,
  retrievedChunks: Array<{ text: string; source: string; title: string; score: number }>
): string {
  const contextBlock = retrievedChunks
    .map((chunk, i) =>
      \`[Source \${i + 1}: \${chunk.title} (\${chunk.source})]\n\${chunk.text}\`
    )
    .join("\n\n---\n\n");

  return \`Answer the user's question based on the provided context.

## Rules
- Only use information from the provided context
- Cite sources using [Source N] notation
- If the context doesn't contain the answer, say "I don't have enough information"
- Do not make up information not present in the context
- Prefer the most relevant source when multiple sources agree

## Context
\${contextBlock}

## Question
\${query}\`;
}

Step 6: End-to-end RAG query

async function ragQuery(userQuestion: string) {
  // 1. Embed the question
  const queryEmbedding = await embedQuery(userQuestion);

  // 2. Retrieve relevant chunks
  const chunks = await queryVectors(queryEmbedding, 5);

  // 3. Build the augmented prompt
  const augmentedPrompt = buildRagPrompt(userQuestion, chunks);

  // 4. Generate the response
  const response = await openai.responses.create({
    model: "gpt-4o",
    instructions: "You are a helpful assistant that answers questions based on provided context.",
    input: augmentedPrompt,
  });

  return {
    answer: response.output_text,
    sources: chunks.map(c => ({ title: c.title, source: c.source, score: c.score })),
  };
}

Examples

Example 1: Hybrid search (semantic + keyword)

async function hybridSearch(query: string, topK = 5) {
  // Semantic search
  const embedding = await embedQuery(query);
  const semanticResults = await queryVectors(embedding, topK * 2);

  // Keyword search (BM25 via Supabase full-text search)
  const { data: keywordResults } = await supabase
    .from("documents")
    .select("id, content, metadata")
    .textSearch("content", query, { type: "websearch" })
    .limit(topK * 2);

  // Reciprocal Rank Fusion (RRF)
  const scores = new Map<string, number>();
  const k = 60; // RRF constant

  semanticResults.forEach((result, rank) => {
    const current = scores.get(result.id) ?? 0;
    scores.set(result.id, current + 1 / (k + rank + 1));
  });

  (keywordResults ?? []).forEach((result: { id: string }, rank: number) => {
    const current = scores.get(result.id) ?? 0;
    scores.set(result.id, current + 1 / (k + rank + 1));
  });

  // Sort by combined score and return top K
  return Array.from(scores.entries())
    .sort((a, b) => b[1] - a[1])
    .slice(0, topK)
    .map(([id, score]) => ({ id, score }));
}

Example 2: Context window management

function fitChunksToWindow(
  chunks: Array<{ text: string; score: number }>,
  maxContextTokens: number
): string[] {
  const selected: string[] = [];
  let totalTokens = 0;

  // Sort by relevance score (highest first)
  const sorted = [...chunks].sort((a, b) => b.score - a.score);

  for (const chunk of sorted) {
    const chunkTokens = estimateTokens(chunk.text);
    if (totalTokens + chunkTokens > maxContextTokens) break;
    selected.push(chunk.text);
    totalTokens += chunkTokens;
  }

  return selected;
}

// Budget: model context - system prompt - output reserve
const maxContext = 128000 - 2000 - 4000; // for gpt-4o

Example 3: Incremental ingestion

async function ingestNewDocuments(docs: Document[]) {
  // Only process documents not already in the vector store
  const existingIds = new Set(await getStoredDocumentIds());
  const newDocs = docs.filter(d => !existingIds.has(d.id));

  if (newDocs.length === 0) return { ingested: 0 };

  const chunks = newDocs.flatMap(doc => chunkDocument(doc));
  const embedded = await embedChunks(chunks);
  await upsertChunks(embedded);

  return { ingested: newDocs.length, chunks: embedded.length };
}

Decision tree

Do you need external knowledge in the LLM response?
├── No → Standard prompting (no RAG needed)
└── Yes
    ├── How much data?
    │   ├── < 50 docs / < 100K tokens → Put it all in context (no vector DB)
    │   ├── 50-10K docs → Single vector index with metadata filters
    │   └── 10K+ docs → Partitioned indexes or hybrid search
    ├── Data freshness?
    │   ├── Static → One-time ingestion
    │   ├── Weekly updates → Batch re-ingestion
    │   └── Real-time → Incremental ingestion + cache invalidation
    ├── Search type?
    │   ├── Semantic similarity → Vector search only
    │   ├── Exact keyword match → Full-text search only
    │   └── Both → Hybrid search with RRF
    └── Vector store?
        ├── Already using Supabase → pgvector extension
        ├── Need managed service → Pinecone
        ├── Need open source → Qdrant, Weaviate, Milvus
        └── Prototyping → In-memory with cosine similarity

Edge cases and gotchas

• Chunk boundary artifacts: Important information split across two chunks — use overlap to mitigate
• Embedding model mismatch: Query and document embeddings must use the same model — mixing models breaks similarity
• Metadata filtering: Filter before vector search, not after — otherwise you get irrelevant results within the top K
• Stale embeddings: When documents update, old embeddings remain — implement a re-indexing strategy
• Score calibration: Cosine similarity scores are not comparable across queries — use rank, not absolute score
• Context window overflow: Retrieved chunks + prompt + expected output must fit in the model's context window
• Hallucination despite RAG: The model may still hallucinate even with retrieved context — add "only use provided context" instructions
• Multi-language: Embedding models work best in English — multilingual retrieval needs multilingual embedding models
• Code vs. prose: Code chunks need different splitting strategies (by function/class, not by paragraph)
• Duplicate chunks: Near-duplicate documents create redundant retrieval — deduplicate before ingestion

Evaluation criteria

Research-backed changes

The biggest delta is News. Fold the concrete changes into the operating notes, then discard the fluff.

Fresh signals

• News · Anthropic News
• Research · Anthropic News
• Economic Futures · Anthropic News
• Try Claude · Anthropic News

Next edits

• Add one new tactic based on News.
• Rewrite one stale section with today's source language.
• Turn the top signal into a reusable agent prompt.