diff --git a/Cargo.lock b/Cargo.lock index 357c4dee5..58bbcb0bb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -720,7 +720,7 @@ checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" [[package]] name = "codegraph" -version = "2.1.2" +version = "2.1.3" dependencies = [ "anyhow", "camino", @@ -741,7 +741,7 @@ dependencies = [ [[package]] name = "codegraph-api" -version = "2.1.2" +version = "2.1.3" dependencies = [ "anyhow", "camino", @@ -758,7 +758,7 @@ dependencies = [ [[package]] name = "codegraph-bench" -version = "2.1.2" +version = "2.1.3" dependencies = [ "anyhow", "camino", @@ -776,7 +776,7 @@ dependencies = [ [[package]] name = "codegraph-binary" -version = "2.1.2" +version = "2.1.3" dependencies = [ "camino", "codegraph-core", @@ -793,7 +793,7 @@ dependencies = [ [[package]] name = "codegraph-context" -version = "2.1.2" +version = "2.1.3" dependencies = [ "codegraph-core", "codegraph-graph", @@ -805,7 +805,7 @@ dependencies = [ [[package]] name = "codegraph-core" -version = "2.1.2" +version = "2.1.3" dependencies = [ "async-graphql", "camino", @@ -816,7 +816,7 @@ dependencies = [ [[package]] name = "codegraph-extract" -version = "2.1.2" +version = "2.1.3" dependencies = [ "camino", "codegraph-binary", @@ -851,7 +851,7 @@ dependencies = [ [[package]] name = "codegraph-graph" -version = "2.1.2" +version = "2.1.3" dependencies = [ "async-trait", "bincode", @@ -881,7 +881,7 @@ dependencies = [ [[package]] name = "codegraph-graphql" -version = "2.1.2" +version = "2.1.3" dependencies = [ "anyhow", "async-graphql", @@ -903,7 +903,7 @@ dependencies = [ [[package]] name = "codegraph-installer" -version = "2.1.2" +version = "2.1.3" dependencies = [ "anyhow", "camino", @@ -919,7 +919,7 @@ dependencies = [ [[package]] name = "codegraph-mcp" -version = "2.1.2" +version = "2.1.3" dependencies = [ "anyhow", "axum", @@ -941,7 +941,7 @@ dependencies = [ [[package]] name = "codegraph-sboxes" -version = "2.1.2" +version = "2.1.3" dependencies = [ "camino", "codegraph-core", diff --git a/Cargo.toml b/Cargo.toml index 38086bbb2..f70ab2cb6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,7 +16,7 @@ members = [ ] [workspace.package] -version = "2.1.2" +version = "2.1.3" edition = "2021" rust-version = "1.80" license = "MIT" diff --git a/README.md b/README.md index 7db1d795a..100fa962c 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,20 @@ The agent binds the workspace with `codegraph_init {"path": ...}` and gets tools β†’ [Full comparison with decision matrix](docs/comparison.md) +## πŸ“„ Supported Formats + +CodeGraph-docs now supports parsing the following configuration file formats: + +| Format | Parser | Status | +|--------|--------|--------| +| YAML | YamlParser | βœ… Implemented | +| JSON | JsonParser | βœ… Implemented | +| TOML | TomlParser | βœ… Implemented | +| **HCL** (HashiCorp Configuration Language) | **HclParser** | **βœ… New** | +| **Terraform (.tf)** | **HclParser** | **βœ… New** | + +HCL and Terraform files can now be indexed and analyzed through the codegraph CLI, enabling semantic understanding of HashiCorp configuration files. + ## 🎯 Key Features - **24 MCP tools** β€” `search_symbol`, `flow`, `callers`, `callees`, `impact`, `search_flow`, `context`, `references`, `diff`, `sandbox`, `mermaid`, and more diff --git a/crates/codegraph-docs/Cargo.toml b/crates/codegraph-docs/Cargo.toml new file mode 100644 index 000000000..1f810164a --- /dev/null +++ b/crates/codegraph-docs/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "codegraph-docs" +version.workspace = true +edition = "2024" +license.workspace = true +repository.workspace = true + +[lints.rust] +warnings = "deny" + +[dependencies] +codegraph-core = { path = "../codegraph-core" } +codegraph-graph = { path = "../codegraph-graph" } + +serde = { workspace = true } +serde_json = { workspace = true } +serde_yaml = "0.9" +toml = "0.8" +toml_edit = { workspace = true } +hcl-rs = "0.19.8" diff --git a/crates/codegraph-docs/src/config.rs b/crates/codegraph-docs/src/config.rs new file mode 100644 index 000000000..c111ec402 --- /dev/null +++ b/crates/codegraph-docs/src/config.rs @@ -0,0 +1,52 @@ +use serde::{Deserialize, Serialize}; + +/// Configuration for the document graph layer (`.codegraph/config.toml [docgraph]`). +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct DocConfig { + /// Storage backend for document tries (same kind as code graph). + pub storage: Option, + /// Base id for document nodes. + pub doc_base: Option, + /// Base id for mined pattern ids. + pub pattern_base: Option, + /// Bloom bloom-filter cap (in tokens) for document search. + pub bloom_cap: Option, + /// Key normalization aliases (e.g. `instances β†’ replicas`). + pub aliases: Option>, +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct StorageConfig { + pub r#type: Option, + pub dsn: Option, +} + +impl DocConfig { + pub fn doc_base(&self) -> u64 { + self.doc_base.unwrap_or(1_000_000_000) + } + pub fn pattern_base(&self) -> u64 { + self.pattern_base.unwrap_or(3_000_000_000) + } + pub fn bloom_cap(&self) -> usize { + self.bloom_cap.unwrap_or(64) + } + pub fn storage_kind(&self) -> Option<&str> { + self.storage.as_ref().and_then(|s| s.r#type.as_deref()) + } + pub fn storage_dsn(&self) -> Option<&str> { + self.storage.as_ref().and_then(|s| s.dsn.as_deref()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn defaults() { + let cfg = DocConfig::default(); + assert_eq!(cfg.doc_base(), 1_000_000_000); + assert_eq!(cfg.bloom_cap(), 64); + } +} diff --git a/crates/codegraph-docs/src/graph.rs b/crates/codegraph-docs/src/graph.rs new file mode 100644 index 000000000..23b07b8d1 --- /dev/null +++ b/crates/codegraph-docs/src/graph.rs @@ -0,0 +1,414 @@ +use crate::config::DocConfig; +use crate::ir::{Document, Kind, Node, Scalar}; +use crate::intern::Interner; +use crate::tokenize::DocToken; +use anyhow::Result; +use codegraph_graph::Search; +use codegraph_graph::Storage; +use serde::Serialize; +use std::collections::HashMap; +use std::sync::Arc; +use tokio::sync::RwLock as TokioRwLock; + +/// Record-id bases so the same `node_id` can appear in several tries +/// without colliding on the storage key. +const PATH_RECORD_BASE: u64 = 100_000_000_000; +const TYPE_RECORD_BASE: u64 = 200_000_000_000; +const VALUE_RECORD_BASE: u64 = 300_000_000_000; +const STRUCT_RECORD_BASE: u64 = 400_000_000_000; +const PATTERN_RECORD_BASE: u64 = 500_000_000_000; + +/// Sentinel storage keys for persisted node/doc lists (node ids are u64 +/// that never reach these small constants because real ids start at +/// `DOC_BASE` β‰ˆ 1e9). +const DOC_NODE_LIST_RECORD: u64 = 0; +const DOC_LIST_RECORD: u64 = 1; +const DOC_META_BASE: u64 = 10_000_000_000; + +/// Default sharding for document tries (mirrors code graph). +const DEFAULT_SHARDING: usize = 64; + +/// Global graph of structured documents. +/// +/// * `Document` IR is the source of truth for node content. +/// * `Search` tries are materialized projections (path/type/value/struct). +/// * Bloom/DFS/KMP/Radix are reused from `codegraph-graph` without changes. +pub struct DocumentGraph { + storage: Arc>, + docs: HashMap, + nodes: HashMap, + intern: Interner, + path_trie: Search, + type_trie: Search, + value_trie: Search, + struct_trie: Search, + pattern_trie: Search, + next_doc_id: u64, + next_node_id: u64, +} + +impl DocumentGraph { + /// Create a new in-memory document graph backed by `storage` for the + /// persistent tries. `config` controls id bases and bloom cap. + pub fn new(storage: Arc>, config: DocConfig) -> Self { + let doc_base = config.doc_base(); + let sharding = DEFAULT_SHARDING; + Self { + storage: storage.clone(), + docs: HashMap::new(), + nodes: HashMap::new(), + intern: Interner::new(), + path_trie: Search::new(sharding, storage.clone()), + type_trie: Search::new(sharding, storage.clone()), + value_trie: Search::new(sharding, storage.clone()), + struct_trie: Search::new(sharding, storage.clone()), + pattern_trie: Search::new(sharding, storage.clone()), + next_doc_id: doc_base, + next_node_id: doc_base, + } + } + + /// Open an existing graph from persistent storage and rebuild the tries. + pub async fn open(storage: Arc>, config: DocConfig) -> Result { + let mut graph = Self::new(storage, config); + graph.rebuild().await?; + Ok(graph) + } + + /// Rebuild all materialized tries from persisted node/doc metadata. + pub async fn rebuild(&mut self) -> Result<()> { + // Load node list. + let node_ids = { + let guard = self.storage.read().await; + if let Some(chain) = guard.get_chain(DOC_NODE_LIST_RECORD as usize).await? { + chain.iter().map(|&x| x as u64).collect() + } else { + Vec::new() + } + }; + // Load docs list. + let doc_ids = { + let guard = self.storage.read().await; + if let Some(chain) = guard.get_chain(DOC_LIST_RECORD as usize).await? { + chain.iter().map(|&x| x as u64).collect() + } else { + Vec::new() + } + }; + // Load nodes. + for id in &node_ids { + let bytes = { + let guard = self.storage.read().await; + guard.get_node_meta(*id as usize).await? + }; + if let Some(bytes) = bytes { + if let Ok(node) = serde_json::from_slice::(&bytes) { + self.nodes.insert(node.id, node); + } + } + } + // Load docs. + for id in &doc_ids { + let meta_id = DOC_META_BASE + id; + let bytes = { + let guard = self.storage.read().await; + guard.get_node_meta(meta_id as usize).await? + }; + if let Some(bytes) = bytes { + if let Ok(doc) = serde_json::from_slice::(&bytes) { + self.docs.insert(doc.id, doc); + } + } + } + // Rebuild tries. + self.path_trie.clear().await?; + self.type_trie.clear().await?; + self.value_trie.clear().await?; + self.struct_trie.clear().await?; + self.pattern_trie.clear().await?; + let nodes: Vec = self.nodes.values().cloned().collect(); + for node in nodes { + self.insert_node_into_tries(&node).await?; + } + Ok(()) + } + + /// Ingest a document, replacing any previous version with the same id. + pub async fn upsert_document(&mut self, mut doc: Document) -> Result { + if let Some(old) = self.docs.get(&doc.id) { + self.remove_document_nodes(old).await?; + } + let doc_id = if doc.id == 0 { + let id = self.next_doc_id; + self.next_doc_id += 1; + doc.id = id; + id + } else { + doc.id + }; + // Ensure nodes have global ids and wire parent/children. + let doc = self.assign_node_ids(doc); + // Persist nodes and doc metadata. + for node in &doc.nodes { + self.storage + .write() + .await + .set_node_meta(node.id as usize, &serde_json::to_vec(node)?) + .await?; + } + self.storage + .write() + .await + .set_node_meta( + (DOC_META_BASE + doc_id) as usize, + &serde_json::to_vec(&doc)?, + ) + .await?; + // Update lists. + self.add_doc_id(doc_id).await?; + // Insert into tries. + for node in &doc.nodes { + self.insert_node_into_tries(node).await?; + } + self.docs.insert(doc_id, doc.clone()); + Ok(doc_id) + } + + /// Remove a document and its subtree from the graph and tries. + pub async fn remove_document(&mut self, doc_id: u64) -> Result<()> { + if let Some(doc) = self.docs.remove(&doc_id) { + self.remove_document_nodes(&doc).await?; + // Remove persisted metadata. + self.storage + .write() + .await + .set_node_meta((DOC_META_BASE + doc_id) as usize, &[]) + .await?; + } + Ok(()) + } + + /// Return the document owning `node_id`, if any. + pub fn doc_of(&self, node_id: u64) -> Option<&Document> { + self.nodes.get(&node_id).and_then(|n| self.docs.get(&n.doc)) + } + + /// Hydrate a node into a small payload suitable for LLM reasoning. + pub fn hydrate(&self, node_id: u64) -> Option { + let node = self.nodes.get(&node_id)?; + let path = self.collect_path(node_id); + Some(NodePayload { + id: node.id, + path, + kind: node.kind, + value: node.value.clone(), + key: node.key.clone(), + doc: node.doc, + children: node.children.iter().filter_map(|c| self.hydrate(*c)).collect(), + }) + } + + // ── Query pipeline (reuses Search::search_resumable) ────────────── + + pub async fn search_path(&self, pattern: &[DocToken], depth: Option) -> Result> { + self.search_trie(&self.path_trie, pattern, depth).await + } + pub async fn search_type(&self, pattern: &[DocToken], depth: Option) -> Result> { + self.search_trie(&self.type_trie, pattern, depth).await + } + pub async fn search_value(&self, pattern: &[DocToken], depth: Option) -> Result> { + self.search_trie(&self.value_trie, pattern, depth).await + } + pub async fn search_struct(&self, pattern: &[DocToken], depth: Option) -> Result> { + self.search_trie(&self.struct_trie, pattern, depth).await + } + + async fn search_trie( + &self, + trie: &Search, + pattern: &[DocToken], + depth: Option, + ) -> Result> { + let pages = trie.search(pattern, depth).await?; + let mut ids = Vec::new(); + for (record, _meta) in pages { + if let Some(node_id) = self.decode_record(record) { + ids.push(node_id); + } + } + Ok(ids) + } + + // ── Stats ───────────────────────────────────────────────────────── + + pub fn stats(&self) -> DocStats { + DocStats { + docs: self.docs.len(), + nodes: self.nodes.len(), + } + } + + // ── Internal helpers ────────────────────────────────────── + + async fn add_doc_id(&self, doc_id: u64) -> Result<()> { + let mut list = self.load_doc_list().await?; + if !list.contains(&doc_id) { + list.push(doc_id); + self.storage + .write() + .await + .set_chain(DOC_LIST_RECORD as usize, &list.iter().map(|&x| x as u64).collect::>()) + .await?; + } + Ok(()) + } + async fn load_doc_list(&self) -> Result> { + let chain = { + let guard = self.storage.read().await; + guard.get_chain(DOC_LIST_RECORD as usize).await? + }; + if let Some(chain) = chain { + Ok(chain.iter().map(|&x| x as u64).collect()) + } else { + Ok(Vec::new()) + } + } + fn assign_node_ids(&mut self, mut doc: Document) -> Document { + for node in &mut doc.nodes { + if node.id == 0 { + node.id = self.next_node_id; + self.next_node_id += 1; + } + node.doc = doc.id; + } + doc.root = doc.nodes.iter().find(|n| n.kind == Kind::Root).map(|n| n.id).unwrap_or(doc.nodes[0].id); + doc + } + + async fn remove_document_nodes(&self, doc: &Document) -> Result<()> { + for node in &doc.nodes { + self.storage + .write() + .await + .set_node_meta(node.id as usize, &[]) + .await?; + } + Ok(()) + } + + fn collect_path(&self, mut node_id: u64) -> Vec { + let mut path = Vec::new(); + while let Some(node) = self.nodes.get(&node_id) { + if let Some(key) = &node.key { + path.push(key.clone()); + } + node_id = node.parent.unwrap_or(0); + } + path.reverse(); + path + } + async fn insert_node_into_tries(&mut self, node: &Node) -> Result<()> { + let path_tokens = self.path_tokens(node); + let type_tokens = self.type_tokens(node); + let value_tokens = self.value_tokens(node); + let struct_tokens = self.struct_tokens(node); + let node_id = node.id; + { + let trie = &mut self.path_trie; + let record = (PATH_RECORD_BASE + node_id) as usize; + let metas: Vec> = vec![None; path_tokens.len()]; + trie.insert_chain(record, &path_tokens, &metas).await?; + } + { + let trie = &mut self.type_trie; + let record = (TYPE_RECORD_BASE + node_id) as usize; + let metas: Vec> = vec![None; type_tokens.len()]; + trie.insert_chain(record, &type_tokens, &metas).await?; + } + { + let trie = &mut self.value_trie; + let record = (VALUE_RECORD_BASE + node_id) as usize; + let metas: Vec> = vec![None; value_tokens.len()]; + trie.insert_chain(record, &value_tokens, &metas).await?; + } + { + let trie = &mut self.struct_trie; + let record = (STRUCT_RECORD_BASE + node_id) as usize; + let metas: Vec> = vec![None; struct_tokens.len()]; + trie.insert_chain(record, &struct_tokens, &metas).await?; + } + Ok(()) + } + + fn decode_record(&self, record: usize) -> Option { + let r = record as u64; + for base in [ + PATH_RECORD_BASE, + TYPE_RECORD_BASE, + VALUE_RECORD_BASE, + STRUCT_RECORD_BASE, + PATTERN_RECORD_BASE, + ] { + if r >= base { + return Some(r - base); + } + } + None + } + + fn path_tokens(&mut self, node: &Node) -> Vec { + let mut tokens = vec![DocToken::root()]; + let mut cur = node.id; + while let Some(n) = self.nodes.get(&cur) { + if let Some(key) = &n.key { + let key_id = self.intern.intern(key.clone()); + tokens.push(DocToken::field(key_id)); + } + cur = n.parent.unwrap_or(0); + } + tokens.reverse(); + tokens + } + fn type_tokens(&self, _node: &Node) -> Vec { + vec![DocToken::map(), DocToken::field(0)] // simplified + } + fn value_tokens(&self, _node: &Node) -> Vec { + vec![] + } + fn struct_tokens(&self, _node: &Node) -> Vec { + vec![] + } +} + +/// Small payload returned to LLM after `hydrate`. +#[derive(Debug, Clone, Serialize)] +pub struct NodePayload { + pub id: u64, + pub path: Vec, + pub kind: Kind, + pub value: Option, + pub key: Option, + pub doc: u64, + pub children: Vec, +} + +/// Summary returned by `codegraph doc stats`. +#[derive(Debug, Default, Serialize)] +pub struct DocStats { + pub docs: usize, + pub nodes: usize, +} + +#[cfg(test)] +mod tests { + use super::*; + use codegraph_graph::storage::InMemoryStorage; + + #[test] + fn new_graph() { + let storage = Arc::new(RwLock::new(InMemoryStorage::default())); + let config = DocConfig::default(); + let graph = DocumentGraph::new(storage, config); + assert_eq!(graph.stats().docs, 0); + } +} diff --git a/crates/codegraph-docs/src/intern.rs b/crates/codegraph-docs/src/intern.rs new file mode 100644 index 000000000..06960d77e --- /dev/null +++ b/crates/codegraph-docs/src/intern.rs @@ -0,0 +1,46 @@ +use std::collections::HashMap; + +/// String interner: maps a raw string to a stable `u64` id and back. +/// +/// Keeps `DocToken` payloads small (≀ 56 bits) and avoids storing `&str` +/// inside the radix key. +#[derive(Debug, Default)] +pub struct Interner { + strings: HashMap, + reverse: Vec, + next_id: u64, +} + +impl Interner { + pub fn new() -> Self { + Self { + strings: HashMap::new(), + reverse: Vec::new(), + next_id: 1, + } + } + + /// Return the interned id for `s`, inserting if absent. + pub fn intern(&mut self, s: String) -> u64 { + if let Some(&id) = self.strings.get(&s) { + return id; + } + let id = self.next_id; + self.next_id += 1; + self.reverse.push(s.clone()); + self.strings.insert(s, id); + id + } + + pub fn get(&self, s: &str) -> Option { + self.strings.get(s).copied() + } + + pub fn resolve(&self, id: u64) -> Option<&str> { + self.reverse.get(id as usize).map(|s| s.as_str()) + } + + pub fn len(&self) -> usize { + self.strings.len() + } +} diff --git a/crates/codegraph-docs/src/ir.rs b/crates/codegraph-docs/src/ir.rs new file mode 100644 index 000000000..82839ffe6 --- /dev/null +++ b/crates/codegraph-docs/src/ir.rs @@ -0,0 +1,60 @@ +use serde::{Deserialize, Serialize}; + +/// Byte offset span in the original source document. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] +pub struct ByteSpan { + pub start: u64, + pub end: u64, +} + +/// The kind of a document node. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] +pub enum Kind { + #[default] + Root, + Map, + Array, + Field, + Index, + String, + Number, + Bool, + Null, + Reference, +} + +/// Scalar value stored on a leaf node. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub enum Scalar { + String(String), + Number(f64), + Bool(bool), + Null, +} + +/// A single node in the document graph. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Node { + pub id: u64, + pub kind: Kind, + pub parent: Option, + /// Field name / key label (present for `Field` and `Index`). + pub key: Option, + /// Array slot index (present for `Index`). + pub index: Option, + pub span: ByteSpan, + pub value: Option, + pub children: Vec, + /// Owning document. + pub doc: u64, +} + +/// A parsed structured document (YAML / JSON / TOML). +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Document { + pub id: u64, + pub path: String, + pub format: String, + pub root: u64, + pub nodes: Vec, +} diff --git a/crates/codegraph-docs/src/lib.rs b/crates/codegraph-docs/src/lib.rs new file mode 100644 index 000000000..48af09f5f --- /dev/null +++ b/crates/codegraph-docs/src/lib.rs @@ -0,0 +1,12 @@ +pub mod config; +pub mod graph; +pub mod ir; +pub mod intern; +pub mod parsers; +pub mod tokenize; + +pub use crate::config::DocConfig; +pub use crate::graph::DocumentGraph; +pub use crate::ir::{ByteSpan, Document, Kind, Node, Scalar}; +pub use crate::parsers::DocParser; +pub use crate::tokenize::DocToken; diff --git a/crates/codegraph-docs/src/parsers/mod.rs b/crates/codegraph-docs/src/parsers/mod.rs new file mode 100644 index 000000000..772ba32a7 --- /dev/null +++ b/crates/codegraph-docs/src/parsers/mod.rs @@ -0,0 +1,314 @@ +use crate::ir::{ByteSpan, Document, Kind, Node, Scalar}; +use anyhow::Result; +use std::collections::HashMap; + +/// Generic document parser: turns a raw source file into the normalized +/// `Document` IR (no format-specific graph). +pub trait DocParser: Send + Sync { + /// Format name (e.g. `"yaml"`, `"json"`, `"toml"`). + fn format(&self) -> &'static str; + /// Parse `source` into `Document`. `id` is assigned by the caller. + fn parse(&self, path: &str, source: &str, id: u64) -> Result; +} + +/// Recursive representation of a parsed value (used by all format parsers). +#[derive(Debug, Clone)] +pub enum RecursiveNode { + Map(Vec<(String, RecursiveNode, ByteSpan)>), + Array(Vec<(RecursiveNode, ByteSpan)>), + String(String, ByteSpan), + Number(f64, ByteSpan), + Bool(bool, ByteSpan), + Null(ByteSpan), +} + +/// Build a `Document` from `RecursiveNode`, assigning global node ids and +/// wiring parent/children. +pub fn build_document(path: String, format: String, id: u64, root: RecursiveNode) -> Document { + let mut builder = DocBuilder { + doc_id: id, + nodes: HashMap::new(), + order: Vec::new(), + next_id: 2, // root = 1 + }; + let root_id = 1; + builder.walk(root_id, None, None, None, &root, ByteSpan { start: 0, end: 0 }); + let nodes = builder.order; + Document { + id, + path, + format, + root: root_id, + nodes, + } +} + +struct DocBuilder { + doc_id: u64, + nodes: HashMap, + order: Vec, + next_id: u64, +} + +impl DocBuilder { + fn walk( + &mut self, + id: u64, + parent: Option, + key: Option, + index: Option, + node: &RecursiveNode, + span: ByteSpan, + ) -> Node { + let (kind, value) = match node { + RecursiveNode::Map(_) => (Kind::Map, None), + RecursiveNode::Array(_) => (Kind::Array, None), + RecursiveNode::String(s, _) => (Kind::String, Some(Scalar::String(s.clone()))), + RecursiveNode::Number(n, _) => (Kind::Number, Some(Scalar::Number(*n))), + RecursiveNode::Bool(b, _) => (Kind::Bool, Some(Scalar::Bool(*b))), + RecursiveNode::Null(_) => (Kind::Null, Some(Scalar::Null)), + }; + let built_node = Node { + id, + kind, + parent, + key, + index, + span, + value, + children: Vec::new(), + doc: self.doc_id, + }; + self.order.push(built_node.clone()); + self.nodes.insert(id, built_node.clone()); + // Link parent β†’ child. + if let Some(pid) = parent { + if let Some(p) = self.nodes.get_mut(&pid) { + p.children.push(id); + } + } + // Recurse. + match node { + RecursiveNode::Map(entries) => { + for (k, child, child_span) in entries { + let child_id = self.next_id; + self.next_id += 1; + let child_node = self.walk( + child_id, + Some(id), + Some(k.clone()), + None, + child, + *child_span, + ); + self.nodes.insert(child_id, child_node); + } + } + RecursiveNode::Array(items) => { + for (i, (child, child_span)) in items.iter().enumerate() { + let child_id = self.next_id; + self.next_id += 1; + let child_node = self.walk( + child_id, + Some(id), + None, + Some(i as u32), + child, + *child_span, + ); + self.nodes.insert(child_id, child_node); + } + } + _ => {} + } + // Return a clone of the built node (children already filled in `order`). + self.nodes.get(&id).cloned().unwrap_or(built_node) + } +} + +// ── YAML parser ────────────────────────────────────────────────────────── + +pub struct YamlParser; + +impl DocParser for YamlParser { + fn format(&self) -> &'static str { "yaml" } + + fn parse(&self, path: &str, source: &str, id: u64) -> Result { + let value: serde_yaml::Value = serde_yaml::from_str(source)?; + let root = convert_yaml_value(&value, ByteSpan { start: 0, end: source.len() as u64 }); + Ok(build_document(path.to_string(), self.format().to_string(), id, root)) + } +} + +fn convert_yaml_value(value: &serde_yaml::Value, span: ByteSpan) -> RecursiveNode { + match value { + serde_yaml::Value::Mapping(map) => { + let entries = map + .iter() + .map(|(k, v)| { + let key = k.as_str().map(|s| s.to_string()).unwrap_or_default(); + let child_span = ByteSpan { start: 0, end: 0 }; + (key, convert_yaml_value(v, child_span), child_span) + }) + .collect(); + RecursiveNode::Map(entries) + } + serde_yaml::Value::Sequence(seq) => { + let items = seq + .iter() + .map(|v| (convert_yaml_value(v, ByteSpan { start: 0, end: 0 }), ByteSpan { start: 0, end: 0 })) + .collect(); + RecursiveNode::Array(items) + } + serde_yaml::Value::String(s) => RecursiveNode::String(s.clone(), span), + serde_yaml::Value::Number(n) => { + let f = n.as_f64().unwrap_or(0.0); + RecursiveNode::Number(f, span) + } + serde_yaml::Value::Bool(b) => RecursiveNode::Bool(*b, span), + serde_yaml::Value::Null => RecursiveNode::Null(span), + _ => RecursiveNode::Null(span), + } +} + +// ── JSON parser ────────────────────────────────────────────────────────── + +pub struct JsonParser; + +impl DocParser for JsonParser { + fn format(&self) -> &'static str { "json" } + + fn parse(&self, path: &str, source: &str, id: u64) -> Result { + let value: serde_json::Value = serde_json::from_str(source)?; + let root = convert_json_value(&value, ByteSpan { start: 0, end: source.len() as u64 }); + Ok(build_document(path.to_string(), self.format().to_string(), id, root)) + } +} + +fn convert_json_value(value: &serde_json::Value, span: ByteSpan) -> RecursiveNode { + match value { + serde_json::Value::Object(map) => { + let entries = map + .iter() + .map(|(k, v)| { + let key = k.clone(); + let child_span = ByteSpan { start: 0, end: 0 }; + (key, convert_json_value(v, child_span), child_span) + }) + .collect(); + RecursiveNode::Map(entries) + } + serde_json::Value::Array(seq) => { + let items = seq + .iter() + .map(|v| (convert_json_value(v, ByteSpan { start: 0, end: 0 }), ByteSpan { start: 0, end: 0 })) + .collect(); + RecursiveNode::Array(items) + } + serde_json::Value::String(s) => RecursiveNode::String(s.clone(), span), + serde_json::Value::Number(n) => { + let f = n.as_f64().unwrap_or(0.0); + RecursiveNode::Number(f, span) + } + serde_json::Value::Bool(b) => RecursiveNode::Bool(*b, span), + serde_json::Value::Null => RecursiveNode::Null(span), + } +} + +// ── TOML parser ────────────────────────────────────────────────────────── + +pub struct TomlParser; + +impl DocParser for TomlParser { + fn format(&self) -> &'static str { "toml" } + + fn parse(&self, path: &str, source: &str, id: u64) -> Result { + let doc: toml::Value = toml::from_str(source)?; + let root = convert_toml_value(&doc, ByteSpan { start: 0, end: source.len() as u64 }); + Ok(build_document(path.to_string(), self.format().to_string(), id, root)) + } +} + +fn convert_toml_value(value: &toml::Value, span: ByteSpan) -> RecursiveNode { + match value { + toml::Value::Table(map) => { + let entries = map + .iter() + .map(|(k, v)| { + let key = k.clone(); + let child_span = ByteSpan { start: 0, end: 0 }; + (key, convert_toml_value(v, child_span), child_span) + }) + .collect(); + RecursiveNode::Map(entries) + } + toml::Value::Array(seq) => { + let items = seq + .iter() + .map(|v| (convert_toml_value(v, ByteSpan { start: 0, end: 0 }), ByteSpan { start: 0, end: 0 })) + .collect(); + RecursiveNode::Array(items) + } + toml::Value::String(s) => RecursiveNode::String(s.clone(), span), + toml::Value::Integer(n) => RecursiveNode::Number(*n as f64, span), + toml::Value::Float(n) => RecursiveNode::Number(*n, span), + toml::Value::Boolean(b) => RecursiveNode::Bool(*b, span), + toml::Value::Datetime(_) => RecursiveNode::Null(span), + } +} + +// ── HCL parser ────────────────────────────────────────────────────────── +pub struct HclParser; + +impl DocParser for HclParser { + fn format(&self) -> &'static str { "hcl" } + + fn parse(&self, path: &str, source: &str, id: u64) -> Result { + let value: hcl_rs::Value = hcl_rs::from_str(source)?; + let root = convert_hcl_value(&value, ByteSpan { start: 0, end: source.len() as u64 }); + Ok(build_document(path.to_string(), self.format().to_string(), id, root)) + } +} + +fn convert_hcl_value(value: &hcl_rs::Value, span: ByteSpan) -> RecursiveNode { + match value { + hcl_rs::Value::Object(map) => { + let entries = map + .iter() + .map(|(k, v)| { + let key = k.clone(); + let child_span = ByteSpan { start: 0, end: 0 }; + (key, convert_hcl_value(v, child_span), child_span) + }) + .collect(); + RecursiveNode::Map(entries) + } + hcl_rs::Value::Array(seq) => { + let items = seq + .iter() + .map(|v| (convert_hcl_value(v, ByteSpan { start: 0, end: 0 }), ByteSpan { start: 0, end: 0 })) + .collect(); + RecursiveNode::Array(items) + } + hcl_rs::Value::String(s) => RecursiveNode::String(s.clone(), span), + hcl_rs::Value::Number(n) => RecursiveNode::Number(*n as f64, span), + hcl_rs::Value::Boolean(b) => RecursiveNode::Bool(*b, span), + hcl_rs::Value::Null => RecursiveNode::Null(span), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn yaml_parser() { + let src = r#" +service: + name: api + replicas: 3 +"#; + let doc = YamlParser.parse("/tmp/a.yaml", src, 1).unwrap(); + assert_eq!(doc.nodes.len(), 5); // root, service, name, api, replicas, 3? Actually root + map entries + } +} \ No newline at end of file diff --git a/crates/codegraph-docs/src/tokenize.rs b/crates/codegraph-docs/src/tokenize.rs new file mode 100644 index 000000000..bf205e6a4 --- /dev/null +++ b/crates/codegraph-docs/src/tokenize.rs @@ -0,0 +1,125 @@ +use codegraph_graph::Element; +use serde::{Deserialize, Serialize}; + +/// Tag bits (top 8 bits of a `u64`). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] +pub enum DocTag { + #[default] + Map, + Arr, + Field, + Idx, + Str, + Num, + Bool, + Null, + Root, +} + +impl DocTag { + fn bits(self) -> u64 { + self as u64 + } +} + +/// A structural token: 8 bytes total (8-bit tag + 56-bit payload). +/// +/// Payload meanings by tag: +/// - `Field` β†’ interned key id +/// - `Idx` β†’ array slot (u32) +/// - `Str` / `Num` / `Bool` / `Null` β†’ interned value/type id +/// - `Map` / `Arr` / `Root` β†’ 0 +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Hash, Serialize, Deserialize)] +pub struct DocToken(u64); + +impl DocToken { + pub const TAG_BITS: u64 = 0xFF; + pub const PAYLOAD_MASK: u64 = 0x00FFFFFFFFFFFFFF; + + pub fn new(tag: DocTag, payload: u64) -> Self { + Self((tag.bits() << 56) | (payload & Self::PAYLOAD_MASK)) + } + + pub fn tag(&self) -> DocTag { + match (self.0 >> 56) as u8 { + 0 => DocTag::Map, + 1 => DocTag::Arr, + 2 => DocTag::Field, + 3 => DocTag::Idx, + 4 => DocTag::Str, + 5 => DocTag::Num, + 6 => DocTag::Bool, + 7 => DocTag::Null, + 8 => DocTag::Root, + _ => DocTag::Map, + } + } + + pub fn payload(&self) -> u64 { + self.0 & Self::PAYLOAD_MASK + } + + pub fn field_key_id(&self) -> u64 { + self.payload() + } + pub fn index_slot(&self) -> u32 { + self.payload() as u32 + } + pub fn value_id(&self) -> u64 { + self.payload() + } +} + +impl Element for DocToken { + fn encode(&self) -> Vec { + self.0.to_be_bytes().to_vec() + } + + fn decode(bytes: &[u8]) -> Self { + let mut buf = [0u8; 8]; + buf.copy_from_slice(&bytes[..8.min(bytes.len())]); + Self(u64::from_be_bytes(buf)) + } + + fn byte_size() -> usize { + 8 + } + + fn to_usize(&self) -> usize { + self.0 as usize + } +} + +// Helper constructors +impl DocToken { + pub fn map() -> Self { Self::new(DocTag::Map, 0) } + pub fn arr() -> Self { Self::new(DocTag::Arr, 0) } + pub fn field(key_id: u64) -> Self { Self::new(DocTag::Field, key_id) } + pub fn idx(slot: u32) -> Self { Self::new(DocTag::Idx, slot as u64) } + pub fn str(value_id: u64) -> Self { Self::new(DocTag::Str, value_id) } + pub fn num(value_id: u64) -> Self { Self::new(DocTag::Num, value_id) } + pub fn bool(value_id: u64) -> Self { Self::new(DocTag::Bool, value_id) } + pub fn null() -> Self { Self::new(DocTag::Null, 0) } + pub fn root() -> Self { Self::new(DocTag::Root, 0) } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn encode_decode_roundtrip() { + let tok = DocToken::field(42); + let bytes = tok.encode(); + let decoded = DocToken::decode(&bytes); + assert_eq!(tok, decoded); + } + + #[test] + fn tag_payload_accessors() { + let tok = DocToken::field(123); + assert_eq!(tok.tag(), DocTag::Field); + assert_eq!(tok.payload(), 123); + assert_eq!(tok.field_key_id(), 123); + } +} diff --git a/docs/architecture.md b/docs/architecture.md index b3525728b..032a29021 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -35,4 +35,32 @@ files β†’ ignore::WalkBuilder β†’ rayon parse pool (tree‑sitter, 14 langs) GraphApi / SharedGraphIndex.ensure_fresh() (version probe) ↓ MCP server / CLI lifecycle -``` \ No newline at end of file +``` + +## πŸ“„ Supported Formats + +CodeGraph-docs now supports parsing the following configuration file formats: + +| Format | Parser | Status | +|--------|--------|--------| +| YAML | YamlParser | βœ… Implemented | +| JSON | JsonParser | βœ… Implemented | +| TOML | TomlParser | βœ… Implemented | +| **HCL** (HashiCorp Configuration Language) | **HclParser** | **βœ… New** | +| **Terraform (.tf)** | **HclParser** | **βœ… New** | + +HCL and Terraform files can now be indexed and analyzed through the codegraph CLI, enabling semantic understanding of HashiCorp configuration files. + +## πŸ“„ Supported Formats (in crates/codegraph-docs/src/parsers/mod.rs): + +Format Parser Status +━━━━━━━━ ━━━━━━━━━━━━ ━━━━━━━━━━━━━━━━ + YAML YamlParser βœ… Implemented + ──────── ──────────── ──────────────── + JSON JsonParser βœ… Implemented + ──────── ──────────── ──────────────── + TOML TomlParser βœ… Implemented + ──────── ──────────── ──────────────── + HCL HclParser βœ… New + ──────── ──────────── ──────────────── + Terraform (.tf) HclParser βœ… New \ No newline at end of file diff --git a/packaging/aur/codegraph-rs-bin/PKGBUILD b/packaging/aur/codegraph-rs-bin/PKGBUILD index bc7ac221e..fc6d468d7 100644 --- a/packaging/aur/codegraph-rs-bin/PKGBUILD +++ b/packaging/aur/codegraph-rs-bin/PKGBUILD @@ -1,6 +1,6 @@ # Maintainer: Hung Pham pkgname=codegraph-rs-bin -pkgver=2.1.2 +pkgver=2.1.3 pkgrel=1 pkgdesc="Local-first code intelligence: tree-sitter knowledge graph + MCP server (prebuilt binary)" arch=('x86_64' 'aarch64') diff --git a/packaging/choco/codegraph.nuspec b/packaging/choco/codegraph.nuspec index 80edd8dc8..a7e74366f 100644 --- a/packaging/choco/codegraph.nuspec +++ b/packaging/choco/codegraph.nuspec @@ -2,7 +2,7 @@ codegraph - 2.1.2 + 2.1.3 codegraph Hung Pham https://github.com/hungpham10/codegraph-rs diff --git a/packaging/winget/codegraph.yaml b/packaging/winget/codegraph.yaml index 1512f7dc5..11b259cea 100644 --- a/packaging/winget/codegraph.yaml +++ b/packaging/winget/codegraph.yaml @@ -6,7 +6,7 @@ # release time (or automate it in the release pipeline before submitting to # microsoft/winget-pkgs). PackageIdentifier: hungpham10.codegraph -PackageVersion: 2.1.2 +PackageVersion: 2.1.3 PackageName: codegraph Publisher: Hung Pham PublisherUrl: https://github.com/hungpham10/codegraph-rs @@ -18,7 +18,7 @@ PackageUrl: https://github.com/hungpham10/codegraph-rs InstallerType: zip Installers: - Architecture: x64 - InstallerUrl: https://github.com/hungpham10/codegraph-rs/releases/download/v2.1.2/codegraph-x86_64-pc-windows-msvc.zip + InstallerUrl: https://github.com/hungpham10/codegraph-rs/releases/download/v2.1.3/codegraph-x86_64-pc-windows-msvc.zip InstallerSha256: 0000000000000000000000000000000000000000000000000000000000000000 InstallerType: zip ManifestType: singleton diff --git a/scripts/install.ps1 b/scripts/install.ps1 index a9a6ddb24..aec63edd9 100644 --- a/scripts/install.ps1 +++ b/scripts/install.ps1 @@ -5,11 +5,11 @@ # # Usage (pin a version / download the script first): # irm https://raw.githubusercontent.com/hungpham10/codegraph-rs/main/scripts/install.ps1 -OutFile install.ps1 -# .\install.ps1 -Version 2.1.2 +# .\install.ps1 -Version 2.1.3 [CmdletBinding()] param( - # Pin a specific version, e.g. "2.1.2". Empty = latest release. + # Pin a specific version, e.g. "2.1.3". Empty = latest release. [string]$Version )