mirror of
https://github.com/instructkr/claude-code.git
synced 2026-05-27 07:56:46 +00:00
feat(rag): add claw-rag-service
Adds claw-rag-service for repository indexing and semantic search.
This commit is contained in:
30
rust/crates/claw-rag-service/Cargo.toml
Normal file
30
rust/crates/claw-rag-service/Cargo.toml
Normal file
@@ -0,0 +1,30 @@
|
||||
[package]
|
||||
name = "claw-rag-service"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
publish.workspace = true
|
||||
description = "Workspace RAG service: SQLite index, OpenAI-compatible embeddings, query API."
|
||||
|
||||
[dependencies]
|
||||
axum = "0.8"
|
||||
clap = { version = "4", features = ["derive", "env"] }
|
||||
dotenvy = "0.15"
|
||||
reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] }
|
||||
rusqlite = { version = "0.32", features = ["bundled"] }
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json.workspace = true
|
||||
tokio = { version = "1", features = ["macros", "net", "rt-multi-thread", "signal"] }
|
||||
walkdir = "2"
|
||||
qdrant-client = { version = "1.17", optional = true }
|
||||
blake3 = "1"
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = "3"
|
||||
|
||||
[features]
|
||||
default = []
|
||||
qdrant-index = ["dep:qdrant-client"]
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
20
rust/crates/claw-rag-service/Dockerfile
Normal file
20
rust/crates/claw-rag-service/Dockerfile
Normal file
@@ -0,0 +1,20 @@
|
||||
# qdrant-client currently requires a fairly recent stable Rust.
|
||||
# Keep this pinned to avoid surprise breaks from `rust:latest`.
|
||||
FROM rust:1.91-bookworm AS builder
|
||||
|
||||
WORKDIR /repo
|
||||
COPY . /repo/rust/
|
||||
|
||||
WORKDIR /repo/rust
|
||||
# Sanity check toolchain version (helps debug CI/Docker Desktop issues).
|
||||
RUN rustc --version && cargo --version
|
||||
# Build the service with qdrant support enabled (works even if you don't use qdrant).
|
||||
RUN cargo build -p claw-rag-service --release --features qdrant-index
|
||||
|
||||
FROM debian:bookworm-slim
|
||||
|
||||
WORKDIR /app
|
||||
COPY --from=builder /repo/rust/target/release/claw-rag-service /app/claw-rag-service
|
||||
|
||||
EXPOSE 8787
|
||||
ENTRYPOINT ["/app/claw-rag-service"]
|
||||
41
rust/crates/claw-rag-service/src/chunk.rs
Normal file
41
rust/crates/claw-rag-service/src/chunk.rs
Normal file
@@ -0,0 +1,41 @@
|
||||
//! Split file text into overlapping windows (character-based UTF-8).
|
||||
|
||||
#[must_use]
|
||||
pub fn chunk_text(text: &str, max_chars: usize, overlap: usize) -> Vec<String> {
|
||||
if max_chars == 0 {
|
||||
return Vec::new();
|
||||
}
|
||||
let overlap = overlap.min(max_chars.saturating_sub(1));
|
||||
let mut out = Vec::new();
|
||||
let chars: Vec<char> = text.chars().collect();
|
||||
if chars.is_empty() {
|
||||
return out;
|
||||
}
|
||||
let mut start = 0;
|
||||
loop {
|
||||
let end = (start + max_chars).min(chars.len());
|
||||
let piece: String = chars[start..end].iter().collect();
|
||||
if !piece.trim().is_empty() {
|
||||
out.push(piece);
|
||||
}
|
||||
if end >= chars.len() {
|
||||
break;
|
||||
}
|
||||
let step = max_chars.saturating_sub(overlap).max(1);
|
||||
start += step;
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn chunks_non_empty() {
|
||||
let c = chunk_text("hello world test", 5, 2);
|
||||
assert!(!c.is_empty());
|
||||
let joined: String = c.join("");
|
||||
assert!(joined.contains("hello"));
|
||||
}
|
||||
}
|
||||
210
rust/crates/claw-rag-service/src/db.rs
Normal file
210
rust/crates/claw-rag-service/src/db.rs
Normal file
@@ -0,0 +1,210 @@
|
||||
//! `SQLite` storage for chunks and embedding vectors.
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
use rusqlite::{params, Connection};
|
||||
|
||||
const SCHEMA: &str = r"
|
||||
CREATE TABLE IF NOT EXISTS chunks (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
path TEXT NOT NULL,
|
||||
ordinal INTEGER NOT NULL,
|
||||
text TEXT NOT NULL,
|
||||
UNIQUE(path, ordinal)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS embeddings (
|
||||
chunk_id INTEGER PRIMARY KEY,
|
||||
dim INTEGER NOT NULL,
|
||||
vec BLOB NOT NULL,
|
||||
FOREIGN KEY (chunk_id) REFERENCES chunks(id) ON DELETE CASCADE
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS files (
|
||||
path TEXT PRIMARY KEY,
|
||||
content_hash TEXT NOT NULL,
|
||||
size_bytes INTEGER NOT NULL,
|
||||
mtime_ms INTEGER NOT NULL,
|
||||
indexed_at_ms INTEGER NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_chunks_path ON chunks(path);
|
||||
";
|
||||
|
||||
pub fn open_db(path: &Path) -> Result<Connection, String> {
|
||||
if let Some(parent) = path.parent() {
|
||||
if !parent.as_os_str().is_empty() {
|
||||
std::fs::create_dir_all(parent).map_err(|e| e.to_string())?;
|
||||
}
|
||||
}
|
||||
|
||||
let conn = Connection::open(path).map_err(|e| e.to_string())?;
|
||||
conn.execute_batch(
|
||||
r"
|
||||
PRAGMA foreign_keys = ON;
|
||||
PRAGMA journal_mode = WAL;
|
||||
",
|
||||
)
|
||||
.map_err(|e| e.to_string())?;
|
||||
conn.execute_batch(SCHEMA).map_err(|e| e.to_string())?;
|
||||
|
||||
Ok(conn)
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub fn truncate_index(conn: &Connection) -> Result<(), String> {
|
||||
conn.execute_batch("DELETE FROM embeddings; DELETE FROM chunks; DELETE FROM files;")
|
||||
.map_err(|e| e.to_string())?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn file_is_unchanged(
|
||||
conn: &Connection,
|
||||
path: &str,
|
||||
content_hash: &str,
|
||||
size_bytes: i64,
|
||||
mtime_ms: i64,
|
||||
) -> Result<bool, String> {
|
||||
let mut stmt = conn
|
||||
.prepare("SELECT content_hash, size_bytes, mtime_ms FROM files WHERE path=?1 LIMIT 1")
|
||||
.map_err(|e| e.to_string())?;
|
||||
let mut rows = stmt.query(params![path]).map_err(|e| e.to_string())?;
|
||||
if let Some(r) = rows.next().map_err(|e| e.to_string())? {
|
||||
let h: String = r.get(0).map_err(|e| e.to_string())?;
|
||||
let sz: i64 = r.get(1).map_err(|e| e.to_string())?;
|
||||
let mt: i64 = r.get(2).map_err(|e| e.to_string())?;
|
||||
return Ok(h == content_hash && sz == size_bytes && mt == mtime_ms);
|
||||
}
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
pub fn upsert_file_meta(
|
||||
conn: &Connection,
|
||||
path: &str,
|
||||
content_hash: &str,
|
||||
size_bytes: i64,
|
||||
mtime_ms: i64,
|
||||
indexed_at_ms: i64,
|
||||
) -> Result<(), String> {
|
||||
conn.execute(
|
||||
r"
|
||||
INSERT INTO files(path, content_hash, size_bytes, mtime_ms, indexed_at_ms)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5)
|
||||
ON CONFLICT(path) DO UPDATE SET
|
||||
content_hash=excluded.content_hash,
|
||||
size_bytes=excluded.size_bytes,
|
||||
mtime_ms=excluded.mtime_ms,
|
||||
indexed_at_ms=excluded.indexed_at_ms
|
||||
",
|
||||
params![path, content_hash, size_bytes, mtime_ms, indexed_at_ms],
|
||||
)
|
||||
.map_err(|e| e.to_string())?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn delete_file_and_chunks(conn: &Connection, path: &str) -> Result<(), String> {
|
||||
// Delete chunks first (embeddings cascade); then remove file meta.
|
||||
conn.execute("DELETE FROM chunks WHERE path=?1", params![path])
|
||||
.map_err(|e| e.to_string())?;
|
||||
conn.execute("DELETE FROM files WHERE path=?1", params![path])
|
||||
.map_err(|e| e.to_string())?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn list_all_files(conn: &Connection) -> Result<Vec<String>, String> {
|
||||
let mut stmt = conn
|
||||
.prepare("SELECT path FROM files")
|
||||
.map_err(|e| e.to_string())?;
|
||||
let rows = stmt
|
||||
.query_map([], |r| r.get::<_, String>(0))
|
||||
.map_err(|e| e.to_string())?;
|
||||
let mut out = Vec::new();
|
||||
for r in rows {
|
||||
out.push(r.map_err(|e| e.to_string())?);
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
pub fn insert_chunk(
|
||||
conn: &Connection,
|
||||
path: &str,
|
||||
ordinal: i32,
|
||||
text: &str,
|
||||
) -> Result<i64, String> {
|
||||
conn.execute(
|
||||
"INSERT INTO chunks (path, ordinal, text) VALUES (?1, ?2, ?3)",
|
||||
params![path, ordinal, text],
|
||||
)
|
||||
.map_err(|e| e.to_string())?;
|
||||
Ok(conn.last_insert_rowid())
|
||||
}
|
||||
|
||||
pub fn insert_embedding(
|
||||
conn: &Connection,
|
||||
chunk_id: i64,
|
||||
dim: usize,
|
||||
vec: &[f32],
|
||||
) -> Result<(), String> {
|
||||
let bytes = f32_slice_to_blob(vec);
|
||||
let dim_i64 = i64::try_from(dim).map_err(|_| "embedding dim too large".to_string())?;
|
||||
conn.execute(
|
||||
"INSERT INTO embeddings (chunk_id, dim, vec) VALUES (?1, ?2, ?3)",
|
||||
params![chunk_id, dim_i64, bytes],
|
||||
)
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn f32_slice_to_blob(v: &[f32]) -> Vec<u8> {
|
||||
let mut b = Vec::with_capacity(v.len() * 4);
|
||||
for x in v {
|
||||
b.extend_from_slice(&x.to_le_bytes());
|
||||
}
|
||||
b
|
||||
}
|
||||
|
||||
pub fn blob_to_f32_vec(blob: &[u8], dim: usize) -> Option<Vec<f32>> {
|
||||
if blob.len() != dim * 4 {
|
||||
return None;
|
||||
}
|
||||
let mut v = Vec::with_capacity(dim);
|
||||
for chunk in blob.chunks_exact(4) {
|
||||
v.push(f32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]]));
|
||||
}
|
||||
Some(v)
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ChunkRow {
|
||||
pub path: String,
|
||||
pub text: String,
|
||||
pub vec: Vec<f32>,
|
||||
}
|
||||
|
||||
pub fn load_all_indexed(conn: &Connection) -> Result<Vec<ChunkRow>, String> {
|
||||
let mut stmt = conn
|
||||
.prepare(
|
||||
"SELECT c.path, c.text, e.dim, e.vec FROM chunks c
|
||||
INNER JOIN embeddings e ON e.chunk_id = c.id",
|
||||
)
|
||||
.map_err(|e| e.to_string())?;
|
||||
let mut rows = stmt.query([]).map_err(|e| e.to_string())?;
|
||||
let mut out = Vec::new();
|
||||
while let Some(r) = rows.next().map_err(|e| e.to_string())? {
|
||||
let path: String = r.get(0).map_err(|e| e.to_string())?;
|
||||
let text: String = r.get(1).map_err(|e| e.to_string())?;
|
||||
let dim: i64 = r.get(2).map_err(|e| e.to_string())?;
|
||||
let blob: Vec<u8> = r.get(3).map_err(|e| e.to_string())?;
|
||||
let dim = usize::try_from(dim).map_err(|_| "invalid embedding dim in db".to_string())?;
|
||||
let Some(vec) = blob_to_f32_vec(&blob, dim) else {
|
||||
continue;
|
||||
};
|
||||
out.push(ChunkRow { path, text, vec });
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
pub fn chunk_count(conn: &Connection) -> Result<i64, String> {
|
||||
let n: i64 = conn
|
||||
.query_row("SELECT COUNT(*) FROM chunks", [], |r| r.get(0))
|
||||
.map_err(|e| e.to_string())?;
|
||||
Ok(n)
|
||||
}
|
||||
129
rust/crates/claw-rag-service/src/embed.rs
Normal file
129
rust/crates/claw-rag-service/src/embed.rs
Normal file
@@ -0,0 +1,129 @@
|
||||
//! OpenAI-compatible embeddings HTTP client.
|
||||
|
||||
use reqwest::Client;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct EmbedConfig {
|
||||
pub api_key: String,
|
||||
pub base_url: String,
|
||||
pub model: String,
|
||||
}
|
||||
|
||||
impl EmbedConfig {
|
||||
pub fn from_env() -> Result<Self, String> {
|
||||
let api_key = std::env::var("CLAW_RAG_OPENAI_API_KEY")
|
||||
.or_else(|_| std::env::var("OPENAI_API_KEY"))
|
||||
.map_err(|_| {
|
||||
"set CLAW_RAG_OPENAI_API_KEY or OPENAI_API_KEY for embeddings".to_string()
|
||||
})?;
|
||||
let base_url = std::env::var("CLAW_RAG_EMBEDDING_BASE_URL")
|
||||
.unwrap_or_else(|_| "https://api.openai.com/v1".into());
|
||||
let model = std::env::var("CLAW_RAG_EMBEDDING_MODEL")
|
||||
.unwrap_or_else(|_| "text-embedding-3-small".into());
|
||||
Ok(Self {
|
||||
api_key,
|
||||
base_url: base_url.trim_end_matches('/').to_string(),
|
||||
model,
|
||||
})
|
||||
}
|
||||
|
||||
/// Deterministic fake vectors for tests / dry-run (1536 dims match common `OpenAI` models;
|
||||
/// truncated scan still works if dim mismatches — ingest uses same mock for all).
|
||||
#[must_use]
|
||||
pub fn mock_from_env() -> Option<Self> {
|
||||
if std::env::var("CLAW_RAG_MOCK_PROVIDERS").ok().as_deref() != Some("1") {
|
||||
return None;
|
||||
}
|
||||
Some(Self {
|
||||
api_key: "mock".into(),
|
||||
base_url: "mock://".into(),
|
||||
model: "mock-embedding".into(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct EmbeddingsRequest<'a> {
|
||||
model: &'a str,
|
||||
input: Vec<&'a str>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct EmbeddingsResponse {
|
||||
data: Vec<EmbeddingItem>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct EmbeddingItem {
|
||||
embedding: Vec<f32>,
|
||||
}
|
||||
|
||||
pub async fn embed_batch(
|
||||
client: &Client,
|
||||
cfg: &EmbedConfig,
|
||||
texts: &[String],
|
||||
) -> Result<Vec<Vec<f32>>, String> {
|
||||
if cfg.base_url.starts_with("mock://") {
|
||||
return Ok(texts
|
||||
.iter()
|
||||
.map(|s| mock_vector_for_text(s.as_str()))
|
||||
.collect());
|
||||
}
|
||||
|
||||
let url = format!("{}/embeddings", cfg.base_url);
|
||||
let inputs: Vec<&str> = texts.iter().map(String::as_str).collect();
|
||||
let body = EmbeddingsRequest {
|
||||
model: &cfg.model,
|
||||
input: inputs,
|
||||
};
|
||||
let res = client
|
||||
.post(&url)
|
||||
.header("Authorization", format!("Bearer {}", cfg.api_key))
|
||||
.header("Content-Type", "application/json")
|
||||
.json(&body)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
if !res.status().is_success() {
|
||||
let t = res.text().await.unwrap_or_default();
|
||||
return Err(format!("embeddings HTTP error: {t}"));
|
||||
}
|
||||
let parsed: EmbeddingsResponse = res.json().await.map_err(|e| e.to_string())?;
|
||||
if parsed.data.len() != texts.len() {
|
||||
return Err(format!(
|
||||
"embeddings count mismatch: got {} for {} inputs",
|
||||
parsed.data.len(),
|
||||
texts.len()
|
||||
));
|
||||
}
|
||||
Ok(parsed.data.into_iter().map(|d| d.embedding).collect())
|
||||
}
|
||||
|
||||
fn mock_vector_for_text(s: &str) -> Vec<f32> {
|
||||
const DIM: usize = 16;
|
||||
let mut v = vec![0f32; DIM];
|
||||
for (i, b) in s.bytes().enumerate().take(DIM * 4) {
|
||||
v[i % DIM] += f32::from(b) / 255.0;
|
||||
}
|
||||
let norm: f32 = v.iter().map(|x| x * x).sum::<f32>().sqrt();
|
||||
if norm > 0.0 {
|
||||
for x in &mut v {
|
||||
*x /= norm;
|
||||
}
|
||||
}
|
||||
v
|
||||
}
|
||||
|
||||
pub fn cosine_similarity(a: &[f32], b: &[f32]) -> f32 {
|
||||
if a.len() != b.len() || a.is_empty() {
|
||||
return 0.0;
|
||||
}
|
||||
let dot: f32 = a.iter().zip(b.iter()).map(|(x, y)| x * y).sum();
|
||||
let na: f32 = a.iter().map(|x| x * x).sum::<f32>().sqrt();
|
||||
let nb: f32 = b.iter().map(|x| x * x).sum::<f32>().sqrt();
|
||||
if na == 0.0 || nb == 0.0 {
|
||||
return 0.0;
|
||||
}
|
||||
dot / (na * nb)
|
||||
}
|
||||
219
rust/crates/claw-rag-service/src/ingest.rs
Normal file
219
rust/crates/claw-rag-service/src/ingest.rs
Normal file
@@ -0,0 +1,219 @@
|
||||
//! Walk workspace and fill `SQLite` + embeddings.
|
||||
|
||||
use std::path::Path;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use reqwest::Client;
|
||||
use walkdir::WalkDir;
|
||||
|
||||
use crate::chunk::chunk_text;
|
||||
use crate::db::{
|
||||
delete_file_and_chunks, file_is_unchanged, insert_chunk, insert_embedding, list_all_files,
|
||||
open_db, upsert_file_meta,
|
||||
};
|
||||
use crate::embed::{embed_batch, EmbedConfig};
|
||||
#[cfg(feature = "qdrant-index")]
|
||||
use crate::qdrant_index::{upsert_points, ChunkPoint};
|
||||
|
||||
const DEFAULT_MAX_FILE_BYTES: u64 = 2 * 1024 * 1024;
|
||||
const CHUNK_CHARS: usize = 900;
|
||||
const CHUNK_OVERLAP: usize = 120;
|
||||
const EMBED_BATCH: usize = 16;
|
||||
|
||||
static SKIP_DIR_NAMES: &[&str] = &[".git", "target", "node_modules", "__pycache__", ".claw-rag"];
|
||||
|
||||
static TEXT_EXTENSIONS: &[&str] = &[
|
||||
"rs", "md", "toml", "txt", "json", "yaml", "yml", "js", "ts", "tsx", "jsx", "py", "go", "c",
|
||||
"h", "cpp", "hpp", "cs", "java", "kt", "swift", "rb", "php", "sh", "ps1", "html", "css", "sql",
|
||||
];
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct IngestStats {
|
||||
pub files_indexed: usize,
|
||||
pub chunks_total: usize,
|
||||
pub embeddings_written: usize,
|
||||
}
|
||||
|
||||
fn should_skip_dir(path: &Path) -> bool {
|
||||
path.file_name()
|
||||
.and_then(std::ffi::OsStr::to_str)
|
||||
.is_some_and(|n| SKIP_DIR_NAMES.contains(&n))
|
||||
}
|
||||
|
||||
fn is_text_extension(path: &Path) -> bool {
|
||||
path.extension()
|
||||
.and_then(std::ffi::OsStr::to_str)
|
||||
.is_some_and(|e| TEXT_EXTENSIONS.contains(&e.to_ascii_lowercase().as_str()))
|
||||
}
|
||||
|
||||
async fn flush_path_batch(
|
||||
conn: &rusqlite::Connection,
|
||||
path: &str,
|
||||
batch: &mut Vec<(i32, String)>,
|
||||
client: &Client,
|
||||
cfg: &EmbedConfig,
|
||||
stats: &mut IngestStats,
|
||||
) -> Result<(), String> {
|
||||
if batch.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
let texts: Vec<String> = batch.iter().map(|(_, t)| t.clone()).collect();
|
||||
let vecs = embed_batch(client, cfg, &texts).await?;
|
||||
if vecs.len() != batch.len() {
|
||||
return Err("embed batch size mismatch".into());
|
||||
}
|
||||
|
||||
#[cfg(feature = "qdrant-index")]
|
||||
let mut qdrant_points: Vec<ChunkPoint> = Vec::with_capacity(batch.len());
|
||||
|
||||
for ((ord, t), vec) in batch.drain(..).zip(vecs.into_iter()) {
|
||||
let dim = vec.len();
|
||||
let cid = insert_chunk(conn, path, ord, &t)?;
|
||||
insert_embedding(conn, cid, dim, &vec)?;
|
||||
stats.embeddings_written += 1;
|
||||
|
||||
#[cfg(feature = "qdrant-index")]
|
||||
{
|
||||
qdrant_points.push(ChunkPoint {
|
||||
id: cid,
|
||||
vec,
|
||||
path: path.to_string(),
|
||||
text: t,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "qdrant-index")]
|
||||
upsert_points(qdrant_points).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn run_ingest(
|
||||
workspaces: &[PathBuf],
|
||||
db_path: &Path,
|
||||
cfg: &EmbedConfig,
|
||||
client: &Client,
|
||||
) -> Result<IngestStats, String> {
|
||||
let conn = open_db(db_path)?;
|
||||
|
||||
let mut all_files: Vec<(String, PathBuf)> = Vec::new();
|
||||
let mut seen_paths: Vec<String> = Vec::new();
|
||||
|
||||
for ws in workspaces {
|
||||
let workspace = ws
|
||||
.canonicalize()
|
||||
.map_err(|e| format!("workspace: {}: {e}", ws.display()))?;
|
||||
let ws_prefix = workspace.clone();
|
||||
let repo_id = repo_id_for_workspace(&workspace);
|
||||
|
||||
for entry in WalkDir::new(&workspace)
|
||||
.into_iter()
|
||||
.filter_entry(|e| !should_skip_dir(e.path()))
|
||||
{
|
||||
let entry = entry.map_err(|e| e.to_string())?;
|
||||
if !entry.file_type().is_file() {
|
||||
continue;
|
||||
}
|
||||
let path = entry.path();
|
||||
if !is_text_extension(path) {
|
||||
continue;
|
||||
}
|
||||
let meta = entry.metadata().map_err(|e| e.to_string())?;
|
||||
if meta.len() > DEFAULT_MAX_FILE_BYTES {
|
||||
continue;
|
||||
}
|
||||
let rel = path
|
||||
.strip_prefix(&ws_prefix)
|
||||
.unwrap_or(path)
|
||||
.to_string_lossy()
|
||||
.replace('\\', "/");
|
||||
let key = format!("{repo_id}:{rel}");
|
||||
seen_paths.push(key.clone());
|
||||
all_files.push((key, path.to_path_buf()));
|
||||
}
|
||||
}
|
||||
|
||||
all_files.sort_by(|a, b| a.0.cmp(&b.0));
|
||||
seen_paths.sort();
|
||||
|
||||
let mut stats = IngestStats {
|
||||
files_indexed: all_files.len(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
for (rel, file) in all_files {
|
||||
let Ok(meta) = std::fs::metadata(&file) else {
|
||||
continue;
|
||||
};
|
||||
let size_bytes =
|
||||
i64::try_from(meta.len()).map_err(|_| "file size too large".to_string())?;
|
||||
let mtime_ms = meta
|
||||
.modified()
|
||||
.ok()
|
||||
.and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok())
|
||||
.and_then(|d| i64::try_from(d.as_millis()).ok())
|
||||
.unwrap_or(0);
|
||||
|
||||
let Ok(raw) = std::fs::read_to_string(&file) else {
|
||||
continue;
|
||||
};
|
||||
|
||||
let content_hash = blake3::hash(raw.as_bytes()).to_hex().to_string();
|
||||
if file_is_unchanged(&conn, &rel, &content_hash, size_bytes, mtime_ms)? {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Re-index this file: delete previous chunks (and embeddings) for path.
|
||||
delete_file_and_chunks(&conn, &rel)?;
|
||||
|
||||
let pieces = chunk_text(&raw, CHUNK_CHARS, CHUNK_OVERLAP);
|
||||
if pieces.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let mut batch: Vec<(i32, String)> = Vec::new();
|
||||
for (ord, piece) in pieces.into_iter().enumerate() {
|
||||
stats.chunks_total += 1;
|
||||
let ord_i32 =
|
||||
i32::try_from(ord).map_err(|_| "file produced too many chunks".to_string())?;
|
||||
batch.push((ord_i32, piece));
|
||||
if batch.len() >= EMBED_BATCH {
|
||||
flush_path_batch(&conn, &rel, &mut batch, client, cfg, &mut stats).await?;
|
||||
}
|
||||
}
|
||||
flush_path_batch(&conn, &rel, &mut batch, client, cfg, &mut stats).await?;
|
||||
|
||||
let now_ms = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map(|d| i64::try_from(d.as_millis()).unwrap_or(0))
|
||||
.unwrap_or(0);
|
||||
upsert_file_meta(&conn, &rel, &content_hash, size_bytes, mtime_ms, now_ms)?;
|
||||
}
|
||||
|
||||
// Delete entries for files that no longer exist.
|
||||
// (We compare against file list from DB to avoid needing a SQL "NOT IN" temp table.)
|
||||
let mut seen_set = std::collections::BTreeSet::new();
|
||||
for p in &seen_paths {
|
||||
seen_set.insert(p.as_str());
|
||||
}
|
||||
for p in list_all_files(&conn)? {
|
||||
if !seen_set.contains(p.as_str()) {
|
||||
delete_file_and_chunks(&conn, &p)?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(stats)
|
||||
}
|
||||
|
||||
fn repo_id_for_workspace(workspace: &Path) -> String {
|
||||
let name = workspace
|
||||
.file_name()
|
||||
.and_then(std::ffi::OsStr::to_str)
|
||||
.filter(|s| !s.is_empty())
|
||||
.unwrap_or("workspace");
|
||||
let hash = blake3::hash(workspace.to_string_lossy().as_bytes())
|
||||
.to_hex()
|
||||
.to_string();
|
||||
format!("{name}-{h}", name = name, h = &hash[..8])
|
||||
}
|
||||
111
rust/crates/claw-rag-service/src/lib.rs
Normal file
111
rust/crates/claw-rag-service/src/lib.rs
Normal file
@@ -0,0 +1,111 @@
|
||||
//! Workspace RAG: ingest files → `SQLite` + embeddings, query via cosine similarity (linear scan MVP).
|
||||
#![forbid(unsafe_code)]
|
||||
|
||||
mod chunk;
|
||||
mod db;
|
||||
mod embed;
|
||||
mod ingest;
|
||||
#[cfg(feature = "qdrant-index")]
|
||||
mod qdrant_index;
|
||||
mod search;
|
||||
|
||||
pub use db::{chunk_count, open_db};
|
||||
pub use embed::EmbedConfig;
|
||||
pub use ingest::{run_ingest, IngestStats};
|
||||
pub use search::query_index;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// One retrieved chunk for the model or UI.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct RagHit {
|
||||
pub path: String,
|
||||
pub snippet: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub score: Option<f32>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
pub struct QueryRequest {
|
||||
pub query: String,
|
||||
#[serde(default = "default_top_k")]
|
||||
pub top_k: u32,
|
||||
}
|
||||
|
||||
fn default_top_k() -> u32 {
|
||||
8
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct QueryResponse {
|
||||
pub hits: Vec<RagHit>,
|
||||
/// `0-stub` (legacy), `1-sqlite`, `1-sqlite-empty`, `1-sqlite-no-db`
|
||||
pub phase: &'static str,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::path::Path;
|
||||
|
||||
use reqwest::Client;
|
||||
use tempfile::tempdir;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn query_missing_db_reports_phase() {
|
||||
let client = Client::new();
|
||||
let cfg = EmbedConfig {
|
||||
api_key: "x".into(),
|
||||
base_url: "mock://".into(),
|
||||
model: "m".into(),
|
||||
};
|
||||
let r = query_index(
|
||||
Path::new("/no/such/claw_rag.sqlite"),
|
||||
&client,
|
||||
&cfg,
|
||||
&QueryRequest {
|
||||
query: "hello".into(),
|
||||
top_k: 3,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(r.phase, "1-sqlite-no-db");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn ingest_and_query_roundtrip_mock() {
|
||||
std::env::set_var("CLAW_RAG_MOCK_PROVIDERS", "1");
|
||||
let dir = tempdir().unwrap();
|
||||
let ws1 = dir.path().join("ws1");
|
||||
let ws2 = dir.path().join("ws2");
|
||||
std::fs::create_dir_all(&ws1).unwrap();
|
||||
std::fs::create_dir_all(&ws2).unwrap();
|
||||
std::fs::write(ws1.join("note.md"), "hello RAG service test content").unwrap();
|
||||
std::fs::write(ws2.join("docs.md"), "secondary repo doc about embeddings").unwrap();
|
||||
let db = dir.path().join("idx.sqlite");
|
||||
let client = Client::new();
|
||||
let cfg = EmbedConfig::mock_from_env().expect("mock");
|
||||
let st = run_ingest(&[ws1.clone(), ws2.clone()], &db, &cfg, &client)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(st.embeddings_written >= 1);
|
||||
|
||||
let r = query_index(
|
||||
&db,
|
||||
&client,
|
||||
&cfg,
|
||||
&QueryRequest {
|
||||
query: "RAG service".into(),
|
||||
top_k: 4,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(r.phase, "1-sqlite");
|
||||
assert!(!r.hits.is_empty());
|
||||
assert!(r.hits.iter().all(|h| h.path.contains(':')));
|
||||
std::env::remove_var("CLAW_RAG_MOCK_PROVIDERS");
|
||||
}
|
||||
}
|
||||
175
rust/crates/claw-rag-service/src/main.rs
Normal file
175
rust/crates/claw-rag-service/src/main.rs
Normal file
@@ -0,0 +1,175 @@
|
||||
//! `claw-rag-service` — HTTP API + `ingest` subcommand.
|
||||
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
|
||||
use axum::{
|
||||
extract::State,
|
||||
http::StatusCode,
|
||||
response::Html,
|
||||
routing::{get, post},
|
||||
Json, Router,
|
||||
};
|
||||
use clap::{Parser, Subcommand};
|
||||
use claw_rag_service::{
|
||||
chunk_count, open_db, query_index, run_ingest, EmbedConfig, QueryRequest, QueryResponse,
|
||||
};
|
||||
|
||||
#[derive(Parser)]
|
||||
#[command(
|
||||
name = "claw-rag-service",
|
||||
about = "Workspace RAG index + HTTP query API"
|
||||
)]
|
||||
struct Cli {
|
||||
#[command(subcommand)]
|
||||
command: Option<Cmd>,
|
||||
}
|
||||
|
||||
#[derive(Subcommand)]
|
||||
enum Cmd {
|
||||
/// Run HTTP server (default when no subcommand).
|
||||
Serve(ServeArgs),
|
||||
/// Index a workspace into `SQLite` (calls embedding API).
|
||||
Ingest(IngestArgs),
|
||||
}
|
||||
|
||||
#[derive(Parser)]
|
||||
struct ServeArgs {
|
||||
#[arg(long, env = "CLAW_RAG_DB", default_value = ".claw-rag/index.sqlite")]
|
||||
db: PathBuf,
|
||||
}
|
||||
|
||||
#[derive(Parser)]
|
||||
struct IngestArgs {
|
||||
/// Workspace roots to ingest. Repeat `--workspace` to ingest multiple repos (cross-repo RAG).
|
||||
#[arg(short, long)]
|
||||
workspace: Vec<PathBuf>,
|
||||
#[arg(long, env = "CLAW_RAG_DB", default_value = ".claw-rag/index.sqlite")]
|
||||
db: PathBuf,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct AppState {
|
||||
db_path: PathBuf,
|
||||
client: reqwest::Client,
|
||||
cfg: EmbedConfig,
|
||||
}
|
||||
|
||||
/// Single-page UI for phase 3 (served at `GET /`).
|
||||
static INDEX_HTML: &str = include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/static/index.html"));
|
||||
|
||||
async fn ui_index() -> Html<&'static str> {
|
||||
Html(INDEX_HTML)
|
||||
}
|
||||
|
||||
fn rag_router(state: Arc<AppState>) -> Router {
|
||||
Router::new()
|
||||
.route("/", get(ui_index))
|
||||
.route("/health", get(|| async { "ok" }))
|
||||
.route("/v1/stats", get(stats))
|
||||
.route("/v1/query", post(query))
|
||||
.with_state(state)
|
||||
}
|
||||
|
||||
fn resolve_embed_config() -> Result<EmbedConfig, String> {
|
||||
if let Some(c) = EmbedConfig::mock_from_env() {
|
||||
return Ok(c);
|
||||
}
|
||||
EmbedConfig::from_env()
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
// Load `.env` if present (walks up parent directories).
|
||||
// This is a convenience for local development; CI/production should set real env vars.
|
||||
let _ = dotenvy::dotenv();
|
||||
|
||||
let cli = Cli::parse();
|
||||
|
||||
if let Some(Cmd::Ingest(a)) = cli.command {
|
||||
let cfg = resolve_embed_config()?;
|
||||
let client = reqwest::Client::new();
|
||||
let st = run_ingest(&a.workspace, &a.db, &cfg, &client).await?;
|
||||
eprintln!(
|
||||
"ingest: files={} chunks={} embeddings={}",
|
||||
st.files_indexed, st.chunks_total, st.embeddings_written
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let db = if let Some(Cmd::Serve(s)) = cli.command {
|
||||
s.db
|
||||
} else {
|
||||
PathBuf::from(
|
||||
std::env::var("CLAW_RAG_DB").unwrap_or_else(|_| ".claw-rag/index.sqlite".into()),
|
||||
)
|
||||
};
|
||||
|
||||
let cfg = resolve_embed_config()?;
|
||||
let state = Arc::new(AppState {
|
||||
db_path: db,
|
||||
client: reqwest::Client::new(),
|
||||
cfg,
|
||||
});
|
||||
|
||||
let app = rag_router(state.clone());
|
||||
|
||||
let port: u16 = std::env::var("CLAW_RAG_PORT")
|
||||
.ok()
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(8787);
|
||||
let host: std::net::IpAddr = std::env::var("CLAW_RAG_HOST")
|
||||
.ok()
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(std::net::IpAddr::V4(std::net::Ipv4Addr::LOCALHOST));
|
||||
let addr = std::net::SocketAddr::from((host, port));
|
||||
eprintln!(
|
||||
"claw-rag-service db={} listen=http://{addr}",
|
||||
state.db_path.display()
|
||||
);
|
||||
let listener = tokio::net::TcpListener::bind(addr).await?;
|
||||
axum::serve(listener, app).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn stats(State(state): State<Arc<AppState>>) -> Result<Json<serde_json::Value>, StatusCode> {
|
||||
let path = state.db_path.clone();
|
||||
if !path.is_file() {
|
||||
return Ok(Json(serde_json::json!({
|
||||
"chunks": 0,
|
||||
"phase": "1-sqlite-no-db"
|
||||
})));
|
||||
}
|
||||
let res = tokio::task::spawn_blocking(move || {
|
||||
let conn = open_db(&path).map_err(|_| ())?;
|
||||
chunk_count(&conn).map_err(|_| ())
|
||||
})
|
||||
.await
|
||||
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?
|
||||
.map_err(|()| StatusCode::INTERNAL_SERVER_ERROR)?;
|
||||
Ok(Json(serde_json::json!({
|
||||
"chunks": res,
|
||||
"phase": "1-sqlite"
|
||||
})))
|
||||
}
|
||||
|
||||
async fn query(
|
||||
State(state): State<Arc<AppState>>,
|
||||
Json(req): Json<QueryRequest>,
|
||||
) -> Result<Json<QueryResponse>, (StatusCode, String)> {
|
||||
query_index(&state.db_path, &state.client, &state.cfg, &req)
|
||||
.await
|
||||
.map(Json)
|
||||
.map_err(|e| (StatusCode::BAD_REQUEST, e))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::INDEX_HTML;
|
||||
|
||||
#[test]
|
||||
fn index_html_wires_api_paths() {
|
||||
assert!(INDEX_HTML.contains("/v1/stats"));
|
||||
assert!(INDEX_HTML.contains("/v1/query"));
|
||||
}
|
||||
}
|
||||
177
rust/crates/claw-rag-service/src/qdrant_index.rs
Normal file
177
rust/crates/claw-rag-service/src/qdrant_index.rs
Normal file
@@ -0,0 +1,177 @@
|
||||
use crate::{QueryResponse, RagHit};
|
||||
use serde_json::json;
|
||||
|
||||
async fn ensure_collection(
|
||||
client: &qdrant_client::Qdrant,
|
||||
collection: &str,
|
||||
dim: usize,
|
||||
) -> Result<(), String> {
|
||||
let dim_u64 = u64::try_from(dim).map_err(|_| "embedding dim too large".to_string())?;
|
||||
|
||||
// Try to create the collection; if it already exists, Qdrant will error.
|
||||
// We treat "already exists" as success to keep ingest idempotent.
|
||||
let res = client
|
||||
.create_collection(
|
||||
qdrant_client::qdrant::CreateCollectionBuilder::new(collection).vectors_config(
|
||||
qdrant_client::qdrant::VectorParamsBuilder::new(
|
||||
dim_u64,
|
||||
qdrant_client::qdrant::Distance::Cosine,
|
||||
),
|
||||
),
|
||||
)
|
||||
.await;
|
||||
|
||||
match res {
|
||||
Ok(_) => Ok(()),
|
||||
Err(e) => {
|
||||
let msg = e.to_string();
|
||||
if msg.contains("already exists") || msg.contains("Already exists") {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(format!("qdrant create_collection: {e}"))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct QdrantConfig {
|
||||
pub url: String,
|
||||
pub api_key: Option<String>,
|
||||
pub collection: String,
|
||||
}
|
||||
|
||||
impl QdrantConfig {
|
||||
pub fn from_env() -> Option<Self> {
|
||||
let url = std::env::var("CLAW_RAG_QDRANT_URL").ok()?;
|
||||
let collection = std::env::var("CLAW_RAG_QDRANT_COLLECTION")
|
||||
.ok()
|
||||
.unwrap_or_else(|| "claw_rag_chunks".to_string());
|
||||
let api_key = std::env::var("CLAW_RAG_QDRANT_API_KEY").ok();
|
||||
Some(Self {
|
||||
url,
|
||||
api_key,
|
||||
collection,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn query_qdrant(q: &[f32], top_k: u32) -> Result<Option<QueryResponse>, String> {
|
||||
let Some(cfg) = QdrantConfig::from_env() else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
let limit = top_k.min(64);
|
||||
let mut client = qdrant_client::Qdrant::from_url(&cfg.url);
|
||||
if let Some(key) = &cfg.api_key {
|
||||
client = client.api_key(key.clone());
|
||||
}
|
||||
let client = client.build().map_err(|e| format!("qdrant client: {e}"))?;
|
||||
|
||||
// If collection doesn't exist yet, treat it as "no results" and fall back.
|
||||
// (We avoid creating it on query because ingest controls dimension/model.)
|
||||
if let Err(e) = client.collection_info(&cfg.collection).await {
|
||||
let msg = e.to_string();
|
||||
if msg.contains("doesn't exist") || msg.contains("Not found") {
|
||||
return Ok(None);
|
||||
}
|
||||
return Err(format!("qdrant collection_info: {e}"));
|
||||
}
|
||||
|
||||
let res = client
|
||||
.query(
|
||||
qdrant_client::qdrant::QueryPointsBuilder::new(&cfg.collection)
|
||||
.query(q.to_vec())
|
||||
.limit(u64::from(limit))
|
||||
.with_payload(true),
|
||||
)
|
||||
.await
|
||||
.map_err(|e| format!("qdrant query: {e}"))?;
|
||||
|
||||
let mut hits = Vec::new();
|
||||
for p in res.result {
|
||||
let payload = p.payload;
|
||||
let path = payload
|
||||
.get("path")
|
||||
.and_then(|v| v.as_str())
|
||||
.map(ToString::to_string)
|
||||
.unwrap_or_default();
|
||||
let text = payload
|
||||
.get("text")
|
||||
.and_then(|v| v.as_str())
|
||||
.map(ToString::to_string)
|
||||
.unwrap_or_default();
|
||||
let score = p.score;
|
||||
if !path.is_empty() {
|
||||
hits.push(RagHit {
|
||||
path,
|
||||
snippet: truncate_snippet(&text, 480),
|
||||
score: Some(score),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
Ok(Some(QueryResponse {
|
||||
hits,
|
||||
phase: "2-qdrant",
|
||||
}))
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ChunkPoint {
|
||||
pub id: i64,
|
||||
pub vec: Vec<f32>,
|
||||
pub path: String,
|
||||
pub text: String,
|
||||
}
|
||||
|
||||
pub async fn upsert_points(points: Vec<ChunkPoint>) -> Result<(), String> {
|
||||
let Some(cfg) = QdrantConfig::from_env() else {
|
||||
return Ok(());
|
||||
};
|
||||
if points.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let mut client = qdrant_client::Qdrant::from_url(&cfg.url);
|
||||
if let Some(key) = &cfg.api_key {
|
||||
client = client.api_key(key.clone());
|
||||
}
|
||||
let client = client.build().map_err(|e| format!("qdrant client: {e}"))?;
|
||||
|
||||
let dim = points[0].vec.len();
|
||||
ensure_collection(&client, &cfg.collection, dim).await?;
|
||||
|
||||
let mut qpoints = Vec::with_capacity(points.len());
|
||||
for p in points {
|
||||
if p.vec.len() != dim {
|
||||
return Err("qdrant upsert: embedding dimension mismatch within batch".to_string());
|
||||
}
|
||||
let id = u64::try_from(p.id).map_err(|_| "chunk id must be non-negative".to_string())?;
|
||||
let payload_map = serde_json::Map::from_iter([
|
||||
("path".to_string(), json!(p.path)),
|
||||
("text".to_string(), json!(p.text)),
|
||||
]);
|
||||
let payload: qdrant_client::Payload = payload_map.into();
|
||||
|
||||
qpoints.push(qdrant_client::qdrant::PointStruct::new(id, p.vec, payload));
|
||||
}
|
||||
|
||||
client
|
||||
.upsert_points(qdrant_client::qdrant::UpsertPointsBuilder::new(
|
||||
&cfg.collection,
|
||||
qpoints,
|
||||
))
|
||||
.await
|
||||
.map_err(|e| format!("qdrant upsert: {e}"))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn truncate_snippet(s: &str, max_chars: usize) -> String {
|
||||
let n = s.chars().count();
|
||||
if n <= max_chars {
|
||||
return s.to_string();
|
||||
}
|
||||
s.chars().take(max_chars).collect::<String>() + "…"
|
||||
}
|
||||
87
rust/crates/claw-rag-service/src/search.rs
Normal file
87
rust/crates/claw-rag-service/src/search.rs
Normal file
@@ -0,0 +1,87 @@
|
||||
//! Vector search over indexed chunks (linear scan MVP).
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
use reqwest::Client;
|
||||
|
||||
use crate::db::{load_all_indexed, open_db};
|
||||
use crate::embed::{cosine_similarity, embed_batch, EmbedConfig};
|
||||
use crate::{QueryRequest, QueryResponse, RagHit};
|
||||
|
||||
pub async fn query_index(
|
||||
db_path: &Path,
|
||||
client: &Client,
|
||||
cfg: &EmbedConfig,
|
||||
req: &QueryRequest,
|
||||
) -> Result<QueryResponse, String> {
|
||||
if !db_path.is_file() {
|
||||
return Ok(QueryResponse {
|
||||
hits: Vec::new(),
|
||||
phase: "1-sqlite-no-db",
|
||||
});
|
||||
}
|
||||
|
||||
let conn = open_db(db_path)?;
|
||||
let qvecs = embed_batch(client, cfg, std::slice::from_ref(&req.query)).await?;
|
||||
let q = qvecs
|
||||
.into_iter()
|
||||
.next()
|
||||
.ok_or_else(|| "no query embedding".to_string())?;
|
||||
|
||||
#[cfg(feature = "qdrant-index")]
|
||||
if let Ok(Some(r)) = crate::qdrant_index::query_qdrant(&q, req.top_k).await {
|
||||
return Ok(r);
|
||||
}
|
||||
|
||||
let rows = load_all_indexed(&conn)?;
|
||||
drop(conn);
|
||||
|
||||
if rows.is_empty() {
|
||||
return Ok(QueryResponse {
|
||||
hits: Vec::new(),
|
||||
phase: "1-sqlite-empty",
|
||||
});
|
||||
}
|
||||
|
||||
let expected = rows[0].vec.len();
|
||||
if q.len() != expected {
|
||||
return Err(format!(
|
||||
"embedding dimension mismatch: index uses dim {} but query embedding has {} (same model/env as ingest required)",
|
||||
expected, q.len()
|
||||
));
|
||||
}
|
||||
|
||||
let mut scored: Vec<(f32, usize)> = rows
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, r)| (cosine_similarity(&q, &r.vec), i))
|
||||
.collect();
|
||||
scored.sort_by(|a, b| b.0.partial_cmp(&a.0).unwrap_or(std::cmp::Ordering::Equal));
|
||||
|
||||
let top = req.top_k.min(64) as usize;
|
||||
let hits: Vec<RagHit> = scored
|
||||
.into_iter()
|
||||
.take(top)
|
||||
.map(|(score, i)| {
|
||||
let r = &rows[i];
|
||||
RagHit {
|
||||
path: r.path.clone(),
|
||||
snippet: truncate_snippet(&r.text, 480),
|
||||
score: Some(score),
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
Ok(QueryResponse {
|
||||
hits,
|
||||
phase: "1-sqlite",
|
||||
})
|
||||
}
|
||||
|
||||
fn truncate_snippet(s: &str, max_chars: usize) -> String {
|
||||
let n = s.chars().count();
|
||||
if n <= max_chars {
|
||||
return s.to_string();
|
||||
}
|
||||
s.chars().take(max_chars).collect::<String>() + "…"
|
||||
}
|
||||
233
rust/crates/claw-rag-service/static/index.html
Normal file
233
rust/crates/claw-rag-service/static/index.html
Normal file
@@ -0,0 +1,233 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>claw-rag</title>
|
||||
<style>
|
||||
:root {
|
||||
--bg: #12141a;
|
||||
--surface: #1a1d26;
|
||||
--border: #2a3140;
|
||||
--text: #e8eaef;
|
||||
--muted: #8b93a8;
|
||||
--accent: #e8a035;
|
||||
--ok: #6daf8a;
|
||||
--err: #d97b7b;
|
||||
}
|
||||
* { box-sizing: border-box; }
|
||||
body {
|
||||
font-family: ui-sans-serif, system-ui, "Segoe UI", Roboto, sans-serif;
|
||||
margin: 0;
|
||||
min-height: 100vh;
|
||||
background: var(--bg);
|
||||
color: var(--text);
|
||||
line-height: 1.5;
|
||||
}
|
||||
header {
|
||||
padding: 1rem 1.25rem;
|
||||
border-bottom: 1px solid var(--border);
|
||||
background: var(--surface);
|
||||
}
|
||||
header h1 {
|
||||
margin: 0;
|
||||
font-size: 1.1rem;
|
||||
font-weight: 600;
|
||||
letter-spacing: 0.02em;
|
||||
}
|
||||
header p { margin: 0.35rem 0 0; font-size: 0.85rem; color: var(--muted); }
|
||||
main { max-width: 52rem; margin: 0 auto; padding: 1.25rem; }
|
||||
.stats {
|
||||
display: flex;
|
||||
gap: 1rem;
|
||||
flex-wrap: wrap;
|
||||
margin-bottom: 1.25rem;
|
||||
font-size: 0.9rem;
|
||||
}
|
||||
.stats span { color: var(--muted); }
|
||||
.stats strong { color: var(--accent); }
|
||||
form {
|
||||
display: grid;
|
||||
gap: 0.75rem;
|
||||
margin-bottom: 1.5rem;
|
||||
padding: 1rem;
|
||||
background: var(--surface);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 6px;
|
||||
}
|
||||
label { font-size: 0.8rem; color: var(--muted); }
|
||||
textarea, input[type="number"] {
|
||||
width: 100%;
|
||||
padding: 0.5rem 0.65rem;
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 4px;
|
||||
background: var(--bg);
|
||||
color: var(--text);
|
||||
font: inherit;
|
||||
}
|
||||
textarea { min-height: 5rem; resize: vertical; }
|
||||
.row { display: flex; gap: 1rem; align-items: end; flex-wrap: wrap; }
|
||||
.row > div:first-child { flex: 1; min-width: 12rem; }
|
||||
button {
|
||||
padding: 0.55rem 1.1rem;
|
||||
background: var(--accent);
|
||||
color: #1a1206;
|
||||
border: none;
|
||||
border-radius: 4px;
|
||||
font-weight: 600;
|
||||
cursor: pointer;
|
||||
}
|
||||
button:disabled { opacity: 0.5; cursor: not-allowed; }
|
||||
button:not(:disabled):hover { filter: brightness(1.05); }
|
||||
.status { font-size: 0.85rem; min-height: 1.25rem; }
|
||||
.status.err { color: var(--err); }
|
||||
.status.ok { color: var(--ok); }
|
||||
.hits { display: flex; flex-direction: column; gap: 1rem; }
|
||||
.hit {
|
||||
padding: 0.85rem 1rem;
|
||||
background: var(--surface);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 6px;
|
||||
border-left: 3px solid var(--accent);
|
||||
}
|
||||
.hit header {
|
||||
padding: 0;
|
||||
border: none;
|
||||
background: transparent;
|
||||
margin-bottom: 0.5rem;
|
||||
}
|
||||
.hit .path { font-family: ui-monospace, monospace; font-size: 0.85rem; color: var(--accent); }
|
||||
.hit .score { font-size: 0.75rem; color: var(--muted); }
|
||||
pre {
|
||||
margin: 0;
|
||||
white-space: pre-wrap;
|
||||
word-break: break-word;
|
||||
font-size: 0.82rem;
|
||||
color: var(--muted);
|
||||
}
|
||||
footer {
|
||||
margin-top: 2rem;
|
||||
padding-top: 1rem;
|
||||
border-top: 1px solid var(--border);
|
||||
font-size: 0.75rem;
|
||||
color: var(--muted);
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<header>
|
||||
<h1>claw-rag-service</h1>
|
||||
<p>Local index · same-origin <code>/v1/*</code> API</p>
|
||||
</header>
|
||||
<main>
|
||||
<div class="stats" id="stats">
|
||||
<span>chunks: <strong id="chunks">—</strong></span>
|
||||
<span>phase: <strong id="phase">—</strong></span>
|
||||
<button type="button" id="refresh" style="margin-left:auto">Refresh stats</button>
|
||||
</div>
|
||||
|
||||
<form id="qform">
|
||||
<div>
|
||||
<label for="query">Query</label>
|
||||
<textarea id="query" name="query" placeholder="Natural language search…" required></textarea>
|
||||
</div>
|
||||
<div class="row">
|
||||
<div>
|
||||
<label for="top_k">top_k</label>
|
||||
<input type="number" id="top_k" name="top_k" value="8" min="1" max="64" />
|
||||
</div>
|
||||
<button type="submit" id="submit">Search</button>
|
||||
</div>
|
||||
</form>
|
||||
|
||||
<div class="status" id="status"></div>
|
||||
<div class="hits" id="hits"></div>
|
||||
|
||||
<footer>
|
||||
Index is read-only here; run <code>claw-rag-service ingest</code> to (re)build. Phase 3 UI — no auth; bind to loopback only in production.
|
||||
</footer>
|
||||
</main>
|
||||
<script>
|
||||
async function loadStats() {
|
||||
const elC = document.getElementById('chunks');
|
||||
const elP = document.getElementById('phase');
|
||||
try {
|
||||
const r = await fetch('/v1/stats');
|
||||
const j = await r.json();
|
||||
elC.textContent = j.chunks ?? '?';
|
||||
elP.textContent = j.phase ?? '?';
|
||||
} catch (e) {
|
||||
elC.textContent = '?';
|
||||
elP.textContent = 'error';
|
||||
}
|
||||
}
|
||||
|
||||
function setStatus(msg, cls) {
|
||||
const s = document.getElementById('status');
|
||||
s.textContent = msg || '';
|
||||
s.className = 'status' + (cls ? ' ' + cls : '');
|
||||
}
|
||||
|
||||
function renderHits(data) {
|
||||
const root = document.getElementById('hits');
|
||||
root.innerHTML = '';
|
||||
const hits = data.hits || [];
|
||||
if (hits.length === 0) {
|
||||
setStatus('No hits (phase: ' + (data.phase || '?') + ')', 'ok');
|
||||
return;
|
||||
}
|
||||
setStatus(hits.length + ' hit(s) · phase: ' + (data.phase || '?'), 'ok');
|
||||
for (const h of hits) {
|
||||
const card = document.createElement('article');
|
||||
card.className = 'hit';
|
||||
const hdr = document.createElement('header');
|
||||
const path = document.createElement('div');
|
||||
path.className = 'path';
|
||||
path.textContent = h.path || '';
|
||||
hdr.appendChild(path);
|
||||
if (h.score != null) {
|
||||
const sc = document.createElement('div');
|
||||
sc.className = 'score';
|
||||
sc.textContent = 'score: ' + h.score;
|
||||
hdr.appendChild(sc);
|
||||
}
|
||||
card.appendChild(hdr);
|
||||
const pre = document.createElement('pre');
|
||||
pre.textContent = h.snippet || '';
|
||||
card.appendChild(pre);
|
||||
root.appendChild(card);
|
||||
}
|
||||
}
|
||||
|
||||
document.getElementById('refresh').addEventListener('click', loadStats);
|
||||
document.getElementById('qform').addEventListener('submit', async (ev) => {
|
||||
ev.preventDefault();
|
||||
const query = document.getElementById('query').value.trim();
|
||||
const top_k = Math.min(64, Math.max(1, parseInt(document.getElementById('top_k').value, 10) || 8));
|
||||
const btn = document.getElementById('submit');
|
||||
btn.disabled = true;
|
||||
setStatus('Searching…', '');
|
||||
document.getElementById('hits').innerHTML = '';
|
||||
try {
|
||||
const r = await fetch('/v1/query', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ query, top_k }),
|
||||
});
|
||||
const text = await r.text();
|
||||
if (!r.ok) {
|
||||
setStatus('HTTP ' + r.status + ': ' + text, 'err');
|
||||
return;
|
||||
}
|
||||
renderHits(JSON.parse(text));
|
||||
} catch (e) {
|
||||
setStatus(String(e), 'err');
|
||||
} finally {
|
||||
btn.disabled = false;
|
||||
}
|
||||
});
|
||||
|
||||
loadStats();
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
Reference in New Issue
Block a user