From e63466845f1aaf08ff3b076592107474e3f1a741 Mon Sep 17 00:00:00 2001
From: Illya <assad.illya@gmail.com>
Date: Mon, 8 Dec 2025 23:33:33 -0500
Subject: [PATCH] fix: Skip node_modules and common build directories during
 ingestion

The ingest_directory function was recursively indexing ALL files including
node_modules, which caused:
- Massive index bloat (thousands of irrelevant library files)
- Polluted search results with third-party code
- Very slow indexing times

Added SKIP_DIRECTORIES set with common directories to exclude:
- Package managers: node_modules, vendor, Pods
- Build outputs: dist, build, target, .next, .nuxt
- VCS: .git, .svn, .hg
- Caches: __pycache__, .pytest_cache, .mypy_cache, .cache
- IDE: .vscode, .idea
- Our own: .claude-os

Also added should_skip_path() helper function with wildcard support
for patterns like *.egg-info.
---
 app/core/ingestion.py | 49 ++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 48 insertions(+), 1 deletion(-)

diff --git a/app/core/ingestion.py b/app/core/ingestion.py
index 60d072c..7fcf88f 100644
--- a/app/core/ingestion.py
+++ b/app/core/ingestion.py
@@ -323,12 +323,56 @@ def ingest_documents(
         }
 
 
+# Directories to always skip during ingestion
+SKIP_DIRECTORIES = {
+    'node_modules',
+    '.git',
+    '.svn',
+    '.hg',
+    '__pycache__',
+    '.pytest_cache',
+    '.mypy_cache',
+    '.tox',
+    '.nox',
+    '.eggs',
+    '*.egg-info',
+    'dist',
+    'build',
+    '.next',
+    '.nuxt',
+    '.output',
+    'coverage',
+    '.nyc_output',
+    '.cache',
+    'vendor',
+    'target',  # Rust/Java
+    'Pods',  # iOS
+    '.gradle',
+    '.idea',
+    '.vscode',
+    '.claude-os',  # Our own cache
+}
+
+
+def should_skip_path(file_path: Path) -> bool:
+    """Check if a file path should be skipped based on directory exclusions."""
+    for part in file_path.parts:
+        if part in SKIP_DIRECTORIES:
+            return True
+        # Handle wildcard patterns like *.egg-info
+        for pattern in SKIP_DIRECTORIES:
+            if '*' in pattern and part.endswith(pattern.replace('*', '')):
+                return True
+    return False
+
+
 def ingest_directory(
     dir_path: str,
     collection_name: str
 ) -> List[Dict[str, any]]:
     """
     Recursively ingest all supported files from a directory.
+    Automatically skips common non-source directories like node_modules, .git, etc.
 
     Args:
         dir_path: Path to directory
@@ -347,8 +391,11 @@ def ingest_directory(
             "error": f"Directory not found: {dir_path}"
         }]
 
-    # Recursively find all supported files
+    # Recursively find all supported files, skipping excluded directories
     for file_path in dir_path.rglob("*"):
+        # Skip excluded directories
+        if should_skip_path(file_path):
+            continue
         if file_path.is_file() and Config.is_supported_file(file_path.name):
             result = ingest_file(
                 str(file_path),