Fix selective semantic indexing pulling in .venv/node_modules garbage

_run_semantic_indexing_background and _run_semantic_indexing_sync both
select doc files with Path(project_path).rglob("*.md"/"*.txt"/"*.rst")
and no directory exclusion at all - unlike the tree-sitter code-file
selection right next to it, which already filters via
TreeSitterIndexer.SKIP_DIRS.

For a project with a .venv checked into its own directory, this pulled
in hundreds of unrelated site-packages files (LICENSE.txt, dist-info
top_level.txt, vendored library .rst docs) as "important" project
knowledge, drowning out the actual project's docs and code in every
KB built with selective indexing. Confirmed live: Ody-project_index
had .venv/lib/python3.11/site-packages/... entries making up most of
its 67 documents.

Filter with the already-existing should_skip_path() from
app.core.ingestion (same SKIP_DIRECTORIES used by ingest_directory)
instead of writing a second exclusion list.
This commit is contained in:
Rosie Val 2026-08-22 20:14:45 -04:00
parent e374440c67
commit 403b4e8c5f

View file

@ -1000,7 +1000,7 @@ def _run_semantic_indexing_background(job_id: str, kb_name: str, project_path: s
Updates INDEXING_JOBS with progress.
"""
from app.core.tree_sitter_indexer import TreeSitterIndexer
from app.core.ingestion import ingest_directory, ingest_file
from app.core.ingestion import ingest_directory, ingest_file, should_skip_path
def update_job(status: str, progress: int = 0, message: str = "", error: str = None):
with INDEXING_JOBS_LOCK:
@ -1037,7 +1037,9 @@ def _run_semantic_indexing_background(job_id: str, kb_name: str, project_path: s
docs_patterns = ["*.md", "*.txt", "*.rst"]
doc_files = []
for pattern in docs_patterns:
doc_files.extend(Path(project_path).rglob(pattern))
doc_files.extend(
f for f in Path(project_path).rglob(pattern) if not should_skip_path(f)
)
all_files = list(set(important_files + [str(f.relative_to(project_path)) for f in doc_files]))
total_files = len(all_files)
@ -1091,7 +1093,7 @@ def _run_semantic_indexing_sync(kb_name: str, project_path: str, selective: bool
Returns the final result dict.
"""
from app.core.tree_sitter_indexer import TreeSitterIndexer
from app.core.ingestion import ingest_directory, ingest_file
from app.core.ingestion import ingest_directory, ingest_file, should_skip_path
start_time = time.time()
@ -1108,7 +1110,9 @@ def _run_semantic_indexing_sync(kb_name: str, project_path: str, selective: bool
docs_patterns = ["*.md", "*.txt", "*.rst"]
doc_files = []
for pattern in docs_patterns:
doc_files.extend(Path(project_path).rglob(pattern))
doc_files.extend(
f for f in Path(project_path).rglob(pattern) if not should_skip_path(f)
)
all_files = list(set(important_files + [str(f.relative_to(project_path)) for f in doc_files]))