From 403b4e8c5f284f931373e73854ee37a6be6e5f67 Mon Sep 17 00:00:00 2001 From: Rosie Val Date: Sat, 22 Aug 2026 20:14:45 -0400 Subject: [PATCH] Fix selective semantic indexing pulling in .venv/node_modules garbage _run_semantic_indexing_background and _run_semantic_indexing_sync both select doc files with Path(project_path).rglob("*.md"/"*.txt"/"*.rst") and no directory exclusion at all - unlike the tree-sitter code-file selection right next to it, which already filters via TreeSitterIndexer.SKIP_DIRS. For a project with a .venv checked into its own directory, this pulled in hundreds of unrelated site-packages files (LICENSE.txt, dist-info top_level.txt, vendored library .rst docs) as "important" project knowledge, drowning out the actual project's docs and code in every KB built with selective indexing. Confirmed live: Ody-project_index had .venv/lib/python3.11/site-packages/... entries making up most of its 67 documents. Filter with the already-existing should_skip_path() from app.core.ingestion (same SKIP_DIRECTORIES used by ingest_directory) instead of writing a second exclusion list. --- mcp_server/server.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/mcp_server/server.py b/mcp_server/server.py index bf8fc6b..f428616 100644 --- a/mcp_server/server.py +++ b/mcp_server/server.py @@ -1000,7 +1000,7 @@ def _run_semantic_indexing_background(job_id: str, kb_name: str, project_path: s Updates INDEXING_JOBS with progress. """ from app.core.tree_sitter_indexer import TreeSitterIndexer - from app.core.ingestion import ingest_directory, ingest_file + from app.core.ingestion import ingest_directory, ingest_file, should_skip_path def update_job(status: str, progress: int = 0, message: str = "", error: str = None): with INDEXING_JOBS_LOCK: @@ -1037,7 +1037,9 @@ def _run_semantic_indexing_background(job_id: str, kb_name: str, project_path: s docs_patterns = ["*.md", "*.txt", "*.rst"] doc_files = [] for pattern in docs_patterns: - doc_files.extend(Path(project_path).rglob(pattern)) + doc_files.extend( + f for f in Path(project_path).rglob(pattern) if not should_skip_path(f) + ) all_files = list(set(important_files + [str(f.relative_to(project_path)) for f in doc_files])) total_files = len(all_files) @@ -1091,7 +1093,7 @@ def _run_semantic_indexing_sync(kb_name: str, project_path: str, selective: bool Returns the final result dict. """ from app.core.tree_sitter_indexer import TreeSitterIndexer - from app.core.ingestion import ingest_directory, ingest_file + from app.core.ingestion import ingest_directory, ingest_file, should_skip_path start_time = time.time() @@ -1108,7 +1110,9 @@ def _run_semantic_indexing_sync(kb_name: str, project_path: str, selective: bool docs_patterns = ["*.md", "*.txt", "*.rst"] doc_files = [] for pattern in docs_patterns: - doc_files.extend(Path(project_path).rglob(pattern)) + doc_files.extend( + f for f in Path(project_path).rglob(pattern) if not should_skip_path(f) + ) all_files = list(set(important_files + [str(f.relative_to(project_path)) for f in doc_files]))