Fix selective semantic indexing pulling in .venv/node_modules garbage
_run_semantic_indexing_background and _run_semantic_indexing_sync both
select doc files with Path(project_path).rglob("*.md"/"*.txt"/"*.rst")
and no directory exclusion at all - unlike the tree-sitter code-file
selection right next to it, which already filters via
TreeSitterIndexer.SKIP_DIRS.
For a project with a .venv checked into its own directory, this pulled
in hundreds of unrelated site-packages files (LICENSE.txt, dist-info
top_level.txt, vendored library .rst docs) as "important" project
knowledge, drowning out the actual project's docs and code in every
KB built with selective indexing. Confirmed live: Ody-project_index
had .venv/lib/python3.11/site-packages/... entries making up most of
its 67 documents.
Filter with the already-existing should_skip_path() from
app.core.ingestion (same SKIP_DIRECTORIES used by ingest_directory)
instead of writing a second exclusion list.
This commit is contained in:
parent
e374440c67
commit
403b4e8c5f
1 changed files with 8 additions and 4 deletions
|
|
@ -1000,7 +1000,7 @@ def _run_semantic_indexing_background(job_id: str, kb_name: str, project_path: s
|
|||
Updates INDEXING_JOBS with progress.
|
||||
"""
|
||||
from app.core.tree_sitter_indexer import TreeSitterIndexer
|
||||
from app.core.ingestion import ingest_directory, ingest_file
|
||||
from app.core.ingestion import ingest_directory, ingest_file, should_skip_path
|
||||
|
||||
def update_job(status: str, progress: int = 0, message: str = "", error: str = None):
|
||||
with INDEXING_JOBS_LOCK:
|
||||
|
|
@ -1037,7 +1037,9 @@ def _run_semantic_indexing_background(job_id: str, kb_name: str, project_path: s
|
|||
docs_patterns = ["*.md", "*.txt", "*.rst"]
|
||||
doc_files = []
|
||||
for pattern in docs_patterns:
|
||||
doc_files.extend(Path(project_path).rglob(pattern))
|
||||
doc_files.extend(
|
||||
f for f in Path(project_path).rglob(pattern) if not should_skip_path(f)
|
||||
)
|
||||
|
||||
all_files = list(set(important_files + [str(f.relative_to(project_path)) for f in doc_files]))
|
||||
total_files = len(all_files)
|
||||
|
|
@ -1091,7 +1093,7 @@ def _run_semantic_indexing_sync(kb_name: str, project_path: str, selective: bool
|
|||
Returns the final result dict.
|
||||
"""
|
||||
from app.core.tree_sitter_indexer import TreeSitterIndexer
|
||||
from app.core.ingestion import ingest_directory, ingest_file
|
||||
from app.core.ingestion import ingest_directory, ingest_file, should_skip_path
|
||||
|
||||
start_time = time.time()
|
||||
|
||||
|
|
@ -1108,7 +1110,9 @@ def _run_semantic_indexing_sync(kb_name: str, project_path: str, selective: bool
|
|||
docs_patterns = ["*.md", "*.txt", "*.rst"]
|
||||
doc_files = []
|
||||
for pattern in docs_patterns:
|
||||
doc_files.extend(Path(project_path).rglob(pattern))
|
||||
doc_files.extend(
|
||||
f for f in Path(project_path).rglob(pattern) if not should_skip_path(f)
|
||||
)
|
||||
|
||||
all_files = list(set(important_files + [str(f.relative_to(project_path)) for f in doc_files]))
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue