feat/shallow index (#40)

* feat(index): add shallow index support (INDEX_FILE_SHALLOW); builder method for file list; manager load/build shallow with find_files fallback

* feat(index): default to shallow index; add ShallowIndexManager; deep rebuild tool; remove build_shallow_index tool; watcher+refresh use shallow; fix server context type

* feat(find_files): enforce true glob semantics (* no dir, ** recursive) and unify to shallow index; watcher verified for add/delete

* feat(file_summary): return needs_deep_index hint when deep index not available

Author: johnhuang316

src/code_index_mcp/constants.py

@@ -5,7 +5,8 @@
 # Directory and file names
 SETTINGS_DIR = "code_indexer"
 CONFIG_FILE = "config.json"
-INDEX_FILE = "index.json"  # JSON index file
+INDEX_FILE = "index.json"  # JSON index file (deep index)
+INDEX_FILE_SHALLOW = "index.shallow.json"  # Minimal shallow index (file list)
 
 # Supported file extensions for code analysis
 # This is the authoritative list used by both old and new indexing systems

src/code_index_mcp/indexing/__init__.py

@@ -13,6 +13,8 @@
 # New JSON-based indexing system
 from .json_index_builder import JSONIndexBuilder, IndexMetadata
 from .json_index_manager import JSONIndexManager, get_index_manager
+from .shallow_index_manager import ShallowIndexManager, get_shallow_index_manager
+from .deep_index_manager import DeepIndexManager
 from .models import SymbolInfo, FileInfo
 
 __all__ = [
@@ -21,6 +23,9 @@
     'JSONIndexBuilder',
     'JSONIndexManager',
     'get_index_manager',
+    'ShallowIndexManager',
+    'get_shallow_index_manager',
+    'DeepIndexManager',
     'SymbolInfo',
     'FileInfo', 
     'IndexMetadata'

src/code_index_mcp/indexing/deep_index_manager.py

@@ -0,0 +1,46 @@
+"""
+Deep Index Manager - Wrapper around JSONIndexManager for deep indexing.
+
+This class provides a clear semantic separation from the shallow manager.
+It delegates to the existing JSONIndexManager (symbols + files JSON index).
+"""
+
+from __future__ import annotations
+
+from typing import Optional, Dict, Any, List
+
+from .json_index_manager import JSONIndexManager
+
+
+class DeepIndexManager:
+    """Thin wrapper over JSONIndexManager to expose deep-index API."""
+
+    def __init__(self) -> None:
+        self._mgr = JSONIndexManager()
+
+    # Expose a subset of API to keep callers simple
+    def set_project_path(self, project_path: str) -> bool:
+        return self._mgr.set_project_path(project_path)
+
+    def build_index(self, force_rebuild: bool = False) -> bool:
+        return self._mgr.build_index(force_rebuild=force_rebuild)
+
+    def load_index(self) -> bool:
+        return self._mgr.load_index()
+
+    def refresh_index(self) -> bool:
+        return self._mgr.refresh_index()
+
+    def find_files(self, pattern: str = "*") -> List[str]:
+        return self._mgr.find_files(pattern)
+
+    def get_file_summary(self, file_path: str) -> Optional[Dict[str, Any]]:
+        return self._mgr.get_file_summary(file_path)
+
+    def get_index_stats(self) -> Dict[str, Any]:
+        return self._mgr.get_index_stats()
+
+    def cleanup(self) -> None:
+        self._mgr.cleanup()
+
+

src/code_index_mcp/indexing/json_index_builder.py

@@ -274,6 +274,31 @@ def _get_supported_files(self) -> List[str]:
         logger.debug(f"Found {len(supported_files)} supported files")
         return supported_files
 
+    def build_shallow_file_list(self) -> List[str]:
+        """
+        Build a minimal shallow index consisting of relative file paths only.
+
+        This method does not read file contents. It enumerates supported files
+        using centralized filtering and returns normalized relative paths with
+        forward slashes for cross-platform consistency.
+
+        Returns:
+            List of relative file paths (using '/').
+        """
+        try:
+            absolute_files = self._get_supported_files()
+            result: List[str] = []
+            for abs_path in absolute_files:
+                rel_path = os.path.relpath(abs_path, self.project_path).replace('\\', '/')
+                # Normalize leading './'
+                if rel_path.startswith('./'):
+                    rel_path = rel_path[2:]
+                result.append(rel_path)
+            return result
+        except Exception as e:
+            logger.error(f"Failed to build shallow file list: {e}")
+            return []
+
     def save_index(self, index: Dict[str, Any], index_path: str) -> bool:
         """
         Save index to disk.

src/code_index_mcp/indexing/json_index_manager.py

@@ -9,14 +9,15 @@
 import json
 import logging
 import os
+import re
 import tempfile
 import threading
 import fnmatch
 from pathlib import Path
 from typing import Dict, List, Optional, Any
 
 from .json_index_builder import JSONIndexBuilder
-from ..constants import SETTINGS_DIR, INDEX_FILE
+from ..constants import SETTINGS_DIR, INDEX_FILE, INDEX_FILE_SHALLOW
 
 logger = logging.getLogger(__name__)
 
@@ -29,6 +30,8 @@ def __init__(self):
         self.index_builder: Optional[JSONIndexBuilder] = None
         self.temp_dir: Optional[str] = None
         self.index_path: Optional[str] = None
+        self.shallow_index_path: Optional[str] = None
+        self._shallow_file_list: Optional[List[str]] = None
         self._lock = threading.RLock()
         logger.info("Initialized JSON Index Manager")
 
@@ -59,6 +62,7 @@ def set_project_path(self, project_path: str) -> bool:
                 os.makedirs(self.temp_dir, exist_ok=True)
 
                 self.index_path = os.path.join(self.temp_dir, INDEX_FILE)
+                self.shallow_index_path = os.path.join(self.temp_dir, INDEX_FILE_SHALLOW)
 
                 logger.info(f"Set project path: {project_path}")
                 logger.info(f"Index storage: {self.index_path}")
@@ -114,6 +118,52 @@ def load_index(self) -> bool:
                 logger.error(f"Failed to load index: {e}")
                 return False
 
+    def build_shallow_index(self) -> bool:
+        """Build and save the minimal shallow index (file list)."""
+        with self._lock:
+            if not self.index_builder or not self.project_path or not self.shallow_index_path:
+                logger.error("Index builder not initialized for shallow index")
+                return False
+
+            try:
+                file_list = self.index_builder.build_shallow_file_list()
+                # Persist as a JSON array for minimal overhead
+                with open(self.shallow_index_path, 'w', encoding='utf-8') as f:
+                    json.dump(file_list, f, ensure_ascii=False)
+                self._shallow_file_list = file_list
+                logger.info(f"Saved shallow index with {len(file_list)} files to {self.shallow_index_path}")
+                return True
+            except Exception as e:
+                logger.error(f"Failed to build shallow index: {e}")
+                return False
+
+    def load_shallow_index(self) -> bool:
+        """Load shallow index (file list) from disk into memory."""
+        with self._lock:
+            try:
+                if not self.shallow_index_path or not os.path.exists(self.shallow_index_path):
+                    logger.warning("No existing shallow index found")
+                    return False
+                with open(self.shallow_index_path, 'r', encoding='utf-8') as f:
+                    data = json.load(f)
+                    if not isinstance(data, list):
+                        logger.error("Shallow index format invalid (expected list)")
+                        return False
+                    # Normalize paths
+                    normalized = []
+                    for p in data:
+                        if isinstance(p, str):
+                            q = p.replace('\\\\', '/').replace('\\', '/')
+                            if q.startswith('./'):
+                                q = q[2:]
+                            normalized.append(q)
+                    self._shallow_file_list = normalized
+                    logger.info(f"Loaded shallow index with {len(normalized)} files")
+                    return True
+            except Exception as e:
+                logger.error(f"Failed to load shallow index: {e}")
+                return False
+
     def refresh_index(self) -> bool:
         """Refresh the index (rebuild and reload)."""
         with self._lock:
@@ -123,7 +173,14 @@ def refresh_index(self) -> bool:
             return False
 
     def find_files(self, pattern: str = "*") -> List[str]:
-        """Find files matching a pattern."""
+        """
+        Find files matching a glob pattern using the SHALLOW file list only.
+
+        Notes:
+            - '*' does not cross '/'
+            - '**' matches across directories
+            - Always sources from the shallow index for consistency and speed
+        """
         with self._lock:
             # Input validation
             if not isinstance(pattern, str):
@@ -134,18 +191,27 @@ def find_files(self, pattern: str = "*") -> List[str]:
             if not pattern:
                 pattern = "*"
 
-            if not self.index_builder or not self.index_builder.in_memory_index:
-                logger.warning("Index not loaded")
-                return []
+            # Normalize to forward slashes
+            norm_pattern = pattern.replace('\\\\', '/').replace('\\', '/')
+
+            # Build glob regex: '*' does not cross '/', '**' crosses directories
+            regex = self._compile_glob_regex(norm_pattern)
 
+            # Always use shallow index for file discovery
             try:
-                files = list(self.index_builder.in_memory_index["files"].keys())
+                if self._shallow_file_list is None:
+                    # Try load existing shallow index; if missing, build then load
+                    if not self.load_shallow_index():
+                        # If still not available, attempt to build
+                        if self.build_shallow_index():
+                            self.load_shallow_index()
 
-                if pattern == "*":
+                files = list(self._shallow_file_list or [])
+
+                if norm_pattern == "*":
                     return files
 
-                # Simple pattern matching
-                return [f for f in files if fnmatch.fnmatch(f, pattern)]
+                return [f for f in files if regex.match(f) is not None]
 
             except Exception as e:
                 logger.error(f"Error finding files: {e}")
@@ -356,6 +422,39 @@ def cleanup(self):
             self.index_path = None
             logger.info("Cleaned up JSON Index Manager")
 
+    @staticmethod
+    def _compile_glob_regex(pattern: str) -> re.Pattern:
+        """
+        Compile a glob pattern where '*' does not match '/', and '**' matches across directories.
+
+        Examples:
+            src/*.py  -> direct children .py under src
+            **/*.py   -> .py at any depth
+        """
+        # Translate glob to regex
+        i = 0
+        out = []
+        special = ".^$+{}[]|()"
+        while i < len(pattern):
+            c = pattern[i]
+            if c == '*':
+                if i + 1 < len(pattern) and pattern[i + 1] == '*':
+                    # '**' -> match across directories
+                    out.append('.*')
+                    i += 2
+                    continue
+                else:
+                    out.append('[^/]*')
+            elif c == '?':
+                out.append('[^/]')
+            elif c in special:
+                out.append('\\' + c)
+            else:
+                out.append(c)
+            i += 1
+        regex_str = '^' + ''.join(out) + '$'
+        return re.compile(regex_str)
+
 
 # Global instance
 _index_manager = JSONIndexManager()
@@ -364,4 +463,3 @@ def cleanup(self):
 def get_index_manager() -> JSONIndexManager:
     """Get the global index manager instance."""
     return _index_manager
-