RAG

import { sniffMime } from '@deuz-sdk/core/rag';

const sniff = sniffMime(bytes, { filename: 'report.pdf' });
// { mime: 'application/pdf', confidence: 'magic', container: 'none' }

import { parse, createParserRegistry } from '@deuz-sdk/core/rag';

// Empty registry is fine for pure-text formats.
const registry = createParserRegistry();

const doc = await parse(bytes, registry, {
  hint: { filename: 'notes.md' },
});
console.log(doc.text);

import { parse } from '@deuz-sdk/core/rag';
import { defaultNodeParserRegistry } from '@deuz-sdk/core/rag/node';

const registry = defaultNodeParserRegistry();
const doc = await parse(bytes, registry, {
  hint: { filename: 'whitepaper.pdf' },
});
console.log(doc.pages, doc.text.length);

import { chunkFixed, chunkRecursive, chunkBlocks } from '@deuz-sdk/core/rag';

// Fixed sliding window with overlap; records startOffset / endOffset.
const a = chunkFixed(doc.text, { size: 400, overlap: 50 });

// Structure-aware: only recurses into the separator hierarchy when a piece
// exceeds the budget. Accepts an extra `separators` array.
const b = chunkRecursive(doc.text, { size: 400, overlap: 50 });

// Pre-parsed blocks: never packs across a heading boundary.
const c = chunkBlocks(doc.structure ?? [], { size: 400 });

import { embedMany } from '@deuz-sdk/core';
import { createVoyage } from '@deuz-sdk/core/voyage';
import type { Embedder } from '@deuz-sdk/core/rag';

const model = createVoyage({ apiKey: process.env.VOYAGE_API_KEY! })('voyage-3.5');

const embedder: Embedder = {
  dims: 1024,
  async embed(texts) {
    const { embeddings } = await embedMany({ model, values: texts });
    return embeddings;
  },
};

import {
  chunkRecursive,
  indexChunks,
  retrieve,
  createMemoryVectorStore,
} from '@deuz-sdk/core/rag';

const store = createMemoryVectorStore();
const chunks = chunkRecursive(doc.text, { size: 400, overlap: 50 });

await indexChunks(chunks, { embedder, store });

const hits = await retrieve('how do refunds work?', { embedder, store }, {
  topK: 8,
  topN: 4,
});
// hits: ScoredChunk[] — Chunk + a `score`

import {
  indexChunks,
  hybridRetrieve,
  createBm25Index,
  createMemoryVectorStore,
} from '@deuz-sdk/core/rag';

const store = createMemoryVectorStore();
await indexChunks(chunks, { embedder, store });

// Lexical index over the SAME chunk set (Chunk.index must match).
const bm25 = createBm25Index(chunks);

const hits = await hybridRetrieve(
  'warm-loving animal and GDPR clause 17',
  { embedder, store, bm25 },
  { topK: 8, topN: 4 },
);

import {
  createBm25Index,
  createMemoryVectorStore,
  reciprocalRankFusion,
  type Chunk,
  type Embedder,
} from '@deuz-sdk/core/rag';

declare const chunks: Chunk[];
declare const embedder: Embedder;
declare const store: ReturnType<typeof createMemoryVectorStore>;

const bm25 = createBm25Index(chunks);
const lexical = bm25.search('clause 17 erasure', 10);

const [queryVector] = await embedder.embed(['clause 17 erasure']);
const dense = queryVector ? await store.query(queryVector, 10) : [];

const fused = reciprocalRankFusion([dense, lexical], { k: 60, topN: 5 });

import { retrieve, createCohereReranker } from '@deuz-sdk/core/rag';
import { createVoyageReranker } from '@deuz-sdk/core/voyage';

const reranker = createCohereReranker({ apiKey: process.env.COHERE_API_KEY });
// or: createVoyageReranker({ apiKey: process.env.VOYAGE_API_KEY, model: 'rerank-2.5-lite' })

// Over-fetch with the cheap retriever, let the cross-encoder pick the best few.
const hits = await retrieve(query, { embedder, store, reranker }, { topK: 40, topN: 5 });

import type { Reranker } from '@deuz-sdk/core/rag';

const crossEncoder: Reranker = {
  async rerank(query, candidates, topN) {
    const scored = await scoreWithCrossEncoder(query, candidates); // your call
    return scored.sort((a, b) => b.score - a.score).slice(0, topN);
  },
};

const hits = await retrieve(query, { embedder, store, reranker: crossEncoder });

import {
  estimatePdfTokens,
  modelSupportsDocuments,
  shouldSendWhole,
  toNativeDocumentPart,
} from '@deuz-sdk/core/rag';

const estTokens = estimatePdfTokens(pages); // ~700 tokens/page

const sendWhole = shouldSendWhole({
  estTokens,
  modelSupportsDocuments: modelSupportsDocuments({ nativePdf: true }),
  thresholdTokens: 6000, // default
});

if (sendWhole) {
  const part = toNativeDocumentPart({ bytes, mime: 'application/pdf' });
  // PDFs ride as an ImagePart with mediaType 'application/pdf'; text → TextPart.
}

import { embedMany, generateText } from '@deuz-sdk/core';
import { createAnthropic } from '@deuz-sdk/core/anthropic';
import { createVoyage } from '@deuz-sdk/core/voyage';
import {
  parse,
  chunkRecursive,
  indexChunks,
  hybridRetrieve,
  createBm25Index,
  createMemoryVectorStore,
  type Embedder,
} from '@deuz-sdk/core/rag';
import { defaultNodeParserRegistry } from '@deuz-sdk/core/rag/node';

// 1. Embedder seam over a real embedding model.
const embedModel = createVoyage({ apiKey: process.env.VOYAGE_API_KEY! })('voyage-3.5');
const embedder: Embedder = {
  dims: 1024,
  embed: async (texts) => (await embedMany({ model: embedModel, values: texts })).embeddings,
};

// 2. Parse the PDF (Node parsers) and chunk it.
const doc = await parse(bytes, defaultNodeParserRegistry(), {
  hint: { filename: 'handbook.pdf' },
});
const chunks = chunkRecursive(doc.text, { size: 400, overlap: 50 });

// 3. Index dense + lexical over the SAME chunk set.
const store = createMemoryVectorStore();
await indexChunks(chunks, { embedder, store });
const bm25 = createBm25Index(chunks);

// 4. Retrieve and build a grounded prompt.
const question = 'What is the refund window for clause 17 purchases?';
const hits = await hybridRetrieve(question, { embedder, store, bm25 }, { topK: 8, topN: 4 });
const context = hits.map((h) => h.text).join('\n\n---\n\n');

const { text } = await generateText({
  model: createAnthropic({ apiKey: process.env.ANTHROPIC_API_KEY! })('claude-opus-4-8'),
  messages: [
    { role: 'system', content: 'Answer using only the provided context. Cite nothing you cannot find there.' },
    { role: 'user', content: `Context:\n${context}\n\nQuestion: ${question}` },
  ],
});

import {
  parse,
  createParserRegistry,
  chunkRecursive,
  indexChunks,
  hybridRetrieve,
  createBm25Index,
  createMemoryVectorStore,
  type Embedder,
} from '@deuz-sdk/core/rag';

export async function ragOnEdge(bytes: Uint8Array, question: string, embedder: Embedder) {
  // Text/markdown/CSV parse purely in core — empty registry is enough.
  const doc = await parse(bytes, createParserRegistry(), { hint: { filename: 'kb.md' } });
  const chunks = chunkRecursive(doc.text, { size: 400, overlap: 50 });

  const store = createMemoryVectorStore();
  await indexChunks(chunks, { embedder, store });
  const bm25 = createBm25Index(chunks);

  return hybridRetrieve(question, { embedder, store, bm25 }, { topK: 6, topN: 3 });
}

import { createExecutionContext } from '@deuz-sdk/core/agent';
import { createMemoryVectorStore, indexChunks, retrieve } from '@deuz-sdk/core/rag';
import type { CallWarning } from '@deuz-sdk/core';

const vectorOf = (text: string) => [text.length % 7, text.includes('deploy') ? 1 : 0, 1];
const store = createMemoryVectorStore();
const execution = createExecutionContext({ budget: { tokens: 10_000 } });
const warnings: CallWarning[] = [];

// An embedder that reports its usage: `accounted: true`, and vectors with usage and modelId.
const accounted = {
  dims: 3,
  accounted: true as const,
  async embed(texts: string[]) {
    const usage = { inputTokens: 12, outputTokens: 0, reasoningTokens: 0, cachedReadTokens: 0, cacheWriteTokens: 0, cacheWrite1hTokens: 0, totalTokens: 12 };
    return { vectors: texts.map(vectorOf), usage, modelId: 'text-embedding-3-small' };
  },
};
await indexChunks(
  [{ text: 'Every deploy needs a green CI run.', index: 0 }],
  { embedder: accounted, store },
  { accounting: { execution, requestId: 'index-1' } },
);

// A plain embedder (bare vectors) is charged at its estimate and reported as unaccounted.
const plain = { dims: 3, embed: async (texts: string[]) => texts.map(vectorOf) };
const hits = await retrieve('How do I deploy?', { embedder: plain, store }, {
  accounting: { execution, requestId: 'query-1' },
  onWarning: (warning) => warnings.push(warning),
});
console.log(hits.length, warnings.map((w) => w.type)); // 1 [ 'unaccounted-usage' ]
const totals = execution.ledger.totals();
console.log(totals.spent.tokens, totals.reserved.tokens, totals.unaccounted); // 12 5 1: the query's estimate stays held