Merge pull request #19 from illAssad/feat/background-indexing-jobs-ui
Add non-blocking semantic indexing with Jobs Dashboard UI
This commit is contained in:
Bob Roberts
GitHub
commit
046f18949c
5 changed files with 653 additions and 87 deletions
348
frontend/src/components/JobsDashboard.tsx
Normal file
348
frontend/src/components/JobsDashboard.tsx
Normal file
|
|
@ -0,0 +1,348 @@
|
|||
import { useState } from 'react';
|
||||
import { useQuery } from '@tanstack/react-query';
|
||||
import { motion } from 'framer-motion';
|
||||
import {
|
||||
Activity,
|
||||
Clock,
|
||||
CheckCircle2,
|
||||
XCircle,
|
||||
AlertCircle,
|
||||
RefreshCw,
|
||||
Loader2,
|
||||
FileSearch,
|
||||
Database,
|
||||
Trash2
|
||||
} from 'lucide-react';
|
||||
import axios from 'axios';
|
||||
|
||||
interface IndexingJob {
|
||||
job_id: string;
|
||||
status: 'queued' | 'running' | 'completed' | 'failed';
|
||||
progress: number;
|
||||
message: string;
|
||||
started_at: number | string | null;
|
||||
completed_at: number | string | null;
|
||||
error: string | null;
|
||||
kb_name?: string;
|
||||
type?: string;
|
||||
project_path?: string;
|
||||
}
|
||||
|
||||
interface JobsResponse {
|
||||
jobs: IndexingJob[];
|
||||
count: number;
|
||||
}
|
||||
|
||||
const STATUS_CONFIG = {
|
||||
queued: {
|
||||
bg: 'bg-yellow-500/20',
|
||||
border: 'border-yellow-500',
|
||||
text: 'text-yellow-400',
|
||||
icon: Clock,
|
||||
label: 'Queued'
|
||||
},
|
||||
running: {
|
||||
bg: 'bg-electric-teal/20',
|
||||
border: 'border-electric-teal',
|
||||
text: 'text-electric-teal',
|
||||
icon: Loader2,
|
||||
label: 'Running'
|
||||
},
|
||||
completed: {
|
||||
bg: 'bg-green-500/20',
|
||||
border: 'border-green-500',
|
||||
text: 'text-green-400',
|
||||
icon: CheckCircle2,
|
||||
label: 'Completed'
|
||||
},
|
||||
failed: {
|
||||
bg: 'bg-red-500/20',
|
||||
border: 'border-red-500',
|
||||
text: 'text-red-400',
|
||||
icon: XCircle,
|
||||
label: 'Failed'
|
||||
}
|
||||
};
|
||||
|
||||
function formatTime(timestamp: number | string | null): string {
|
||||
if (!timestamp) return '-';
|
||||
try {
|
||||
// Handle Unix timestamp (seconds) or ISO string
|
||||
const ts = typeof timestamp === 'number' ? timestamp * 1000 : new Date(timestamp).getTime();
|
||||
return new Date(ts).toLocaleString();
|
||||
} catch {
|
||||
return String(timestamp);
|
||||
}
|
||||
}
|
||||
|
||||
function formatDuration(startedAt: number | string | null, completedAt: number | string | null): string {
|
||||
if (!startedAt) return '-';
|
||||
|
||||
// Handle Unix timestamp (seconds) or ISO string
|
||||
const start = typeof startedAt === 'number' ? startedAt * 1000 : new Date(startedAt).getTime();
|
||||
const end = completedAt
|
||||
? (typeof completedAt === 'number' ? completedAt * 1000 : new Date(completedAt).getTime())
|
||||
: Date.now();
|
||||
const durationMs = end - start;
|
||||
|
||||
if (durationMs < 1000) return `${durationMs}ms`;
|
||||
if (durationMs < 60000) return `${(durationMs / 1000).toFixed(1)}s`;
|
||||
if (durationMs < 3600000) return `${Math.floor(durationMs / 60000)}m ${Math.floor((durationMs % 60000) / 1000)}s`;
|
||||
return `${Math.floor(durationMs / 3600000)}h ${Math.floor((durationMs % 3600000) / 60000)}m`;
|
||||
}
|
||||
|
||||
function extractKbName(jobId: string): string {
|
||||
// Job IDs are formatted as "semantic-{kb_name}-{uuid}"
|
||||
const match = jobId.match(/^semantic-(.+)-[a-f0-9-]+$/);
|
||||
return match ? match[1] : jobId;
|
||||
}
|
||||
|
||||
export default function JobsDashboard() {
|
||||
const [autoRefresh, setAutoRefresh] = useState(true);
|
||||
|
||||
// Fetch jobs list
|
||||
const { data: jobsData, isLoading, refetch } = useQuery<JobsResponse>({
|
||||
queryKey: ['indexing-jobs'],
|
||||
queryFn: async () => {
|
||||
const response = await axios.get('/api/jobs');
|
||||
return response.data;
|
||||
},
|
||||
refetchInterval: autoRefresh ? 2000 : false, // Refresh every 2 seconds when enabled
|
||||
});
|
||||
|
||||
const jobs = jobsData?.jobs || [];
|
||||
|
||||
// Count jobs by status
|
||||
const runningJobs = jobs.filter(j => j.status === 'running').length;
|
||||
const queuedJobs = jobs.filter(j => j.status === 'queued').length;
|
||||
const completedJobs = jobs.filter(j => j.status === 'completed').length;
|
||||
const failedJobs = jobs.filter(j => j.status === 'failed').length;
|
||||
|
||||
return (
|
||||
<div className="space-y-6">
|
||||
{/* Header */}
|
||||
<div className="flex items-center justify-between">
|
||||
<div>
|
||||
<h2 className="text-2xl font-bold gradient-text flex items-center gap-2">
|
||||
<FileSearch className="w-6 h-6" />
|
||||
Indexing Jobs
|
||||
</h2>
|
||||
<p className="text-light-grey text-sm mt-1">
|
||||
Monitor semantic indexing operations in real-time
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div className="flex items-center gap-3">
|
||||
{/* Auto-refresh toggle */}
|
||||
<button
|
||||
onClick={() => setAutoRefresh(!autoRefresh)}
|
||||
className={`flex items-center gap-2 px-3 py-2 rounded-lg transition-colors ${
|
||||
autoRefresh
|
||||
? 'bg-electric-teal/20 text-electric-teal border border-electric-teal/50'
|
||||
: 'bg-cool-blue/10 text-light-grey border border-electric-teal/20'
|
||||
}`}
|
||||
>
|
||||
<RefreshCw className={`w-4 h-4 ${autoRefresh ? 'animate-spin' : ''}`} />
|
||||
<span className="text-sm font-medium">
|
||||
{autoRefresh ? 'Live' : 'Paused'}
|
||||
</span>
|
||||
</button>
|
||||
|
||||
{/* Manual refresh */}
|
||||
<button
|
||||
onClick={() => refetch()}
|
||||
disabled={isLoading}
|
||||
className="btn-secondary flex items-center gap-2"
|
||||
>
|
||||
<RefreshCw className={`w-4 h-4 ${isLoading ? 'animate-spin' : ''}`} />
|
||||
Refresh
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Summary Cards */}
|
||||
<div className="grid grid-cols-4 gap-4">
|
||||
<motion.div
|
||||
initial={{ opacity: 0, y: 20 }}
|
||||
animate={{ opacity: 1, y: 0 }}
|
||||
className="bg-cool-blue/10 border border-electric-teal/30 rounded-lg p-4"
|
||||
>
|
||||
<div className="flex items-center justify-between">
|
||||
<div>
|
||||
<div className="text-light-grey text-sm">Running</div>
|
||||
<div className="text-3xl font-bold text-electric-teal mt-1">{runningJobs}</div>
|
||||
</div>
|
||||
<Loader2 className={`w-8 h-8 text-electric-teal opacity-50 ${runningJobs > 0 ? 'animate-spin' : ''}`} />
|
||||
</div>
|
||||
</motion.div>
|
||||
|
||||
<motion.div
|
||||
initial={{ opacity: 0, y: 20 }}
|
||||
animate={{ opacity: 1, y: 0 }}
|
||||
transition={{ delay: 0.1 }}
|
||||
className="bg-cool-blue/10 border border-electric-teal/30 rounded-lg p-4"
|
||||
>
|
||||
<div className="flex items-center justify-between">
|
||||
<div>
|
||||
<div className="text-light-grey text-sm">Queued</div>
|
||||
<div className="text-3xl font-bold text-yellow-400 mt-1">{queuedJobs}</div>
|
||||
</div>
|
||||
<Clock className="w-8 h-8 text-yellow-400 opacity-50" />
|
||||
</div>
|
||||
</motion.div>
|
||||
|
||||
<motion.div
|
||||
initial={{ opacity: 0, y: 20 }}
|
||||
animate={{ opacity: 1, y: 0 }}
|
||||
transition={{ delay: 0.2 }}
|
||||
className="bg-cool-blue/10 border border-electric-teal/30 rounded-lg p-4"
|
||||
>
|
||||
<div className="flex items-center justify-between">
|
||||
<div>
|
||||
<div className="text-light-grey text-sm">Completed</div>
|
||||
<div className="text-3xl font-bold text-green-400 mt-1">{completedJobs}</div>
|
||||
</div>
|
||||
<CheckCircle2 className="w-8 h-8 text-green-400 opacity-50" />
|
||||
</div>
|
||||
</motion.div>
|
||||
|
||||
<motion.div
|
||||
initial={{ opacity: 0, y: 20 }}
|
||||
animate={{ opacity: 1, y: 0 }}
|
||||
transition={{ delay: 0.3 }}
|
||||
className="bg-cool-blue/10 border border-electric-teal/30 rounded-lg p-4"
|
||||
>
|
||||
<div className="flex items-center justify-between">
|
||||
<div>
|
||||
<div className="text-light-grey text-sm">Failed</div>
|
||||
<div className="text-3xl font-bold text-red-400 mt-1">{failedJobs}</div>
|
||||
</div>
|
||||
<XCircle className="w-8 h-8 text-red-400 opacity-50" />
|
||||
</div>
|
||||
</motion.div>
|
||||
</div>
|
||||
|
||||
{/* Jobs List */}
|
||||
<div className="space-y-4">
|
||||
{isLoading && jobs.length === 0 ? (
|
||||
<div className="flex items-center justify-center py-12">
|
||||
<Loader2 className="w-8 h-8 text-electric-teal animate-spin" />
|
||||
<span className="ml-3 text-light-grey">Loading jobs...</span>
|
||||
</div>
|
||||
) : jobs.length === 0 ? (
|
||||
<div className="text-center py-12">
|
||||
<Database className="w-16 h-16 text-electric-teal/30 mx-auto mb-4" />
|
||||
<h3 className="text-xl font-bold text-light-grey mb-2">No Indexing Jobs</h3>
|
||||
<p className="text-light-grey text-sm">
|
||||
Jobs will appear here when you run semantic indexing on a knowledge base
|
||||
</p>
|
||||
</div>
|
||||
) : (
|
||||
jobs.map((job, index) => {
|
||||
const config = STATUS_CONFIG[job.status] || STATUS_CONFIG.queued;
|
||||
const StatusIcon = config.icon;
|
||||
const kbName = job.kb_name || extractKbName(job.job_id);
|
||||
|
||||
return (
|
||||
<motion.div
|
||||
key={job.job_id}
|
||||
initial={{ opacity: 0, x: -20 }}
|
||||
animate={{ opacity: 1, x: 0 }}
|
||||
transition={{ delay: index * 0.05 }}
|
||||
className={`bg-cool-blue/10 border-2 rounded-lg p-5 ${config.border}/30`}
|
||||
>
|
||||
{/* Job Header */}
|
||||
<div className="flex items-start justify-between mb-4">
|
||||
<div className="flex items-center gap-3">
|
||||
<div className={`p-2 rounded-lg ${config.bg}`}>
|
||||
<StatusIcon className={`w-5 h-5 ${config.text} ${job.status === 'running' ? 'animate-spin' : ''}`} />
|
||||
</div>
|
||||
<div>
|
||||
<h3 className="font-bold text-white">{kbName}</h3>
|
||||
<p className="text-xs text-light-grey font-mono">{job.job_id}</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Status Badge */}
|
||||
<div className={`flex items-center gap-1 px-3 py-1 rounded-full ${config.bg} border ${config.border}`}>
|
||||
<StatusIcon className={`w-3 h-3 ${config.text} ${job.status === 'running' ? 'animate-spin' : ''}`} />
|
||||
<span className={`text-sm font-medium ${config.text}`}>
|
||||
{config.label}
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Progress Bar (only for running/completed jobs) */}
|
||||
{(job.status === 'running' || job.status === 'completed') && (
|
||||
<div className="mb-4">
|
||||
<div className="flex items-center justify-between text-sm mb-2">
|
||||
<span className="text-light-grey">Progress</span>
|
||||
<span className={`font-bold ${config.text}`}>{job.progress}%</span>
|
||||
</div>
|
||||
<div className="h-3 bg-deep-night rounded-full overflow-hidden border border-electric-teal/20">
|
||||
<motion.div
|
||||
initial={{ width: 0 }}
|
||||
animate={{ width: `${job.progress}%` }}
|
||||
transition={{ duration: 0.5, ease: 'easeOut' }}
|
||||
className={`h-full ${job.status === 'completed' ? 'bg-green-500' : 'bg-gradient-to-r from-electric-teal to-cool-blue'}`}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Message */}
|
||||
{job.message && (
|
||||
<div className="mb-4 p-3 bg-deep-night/50 rounded-lg border border-electric-teal/10">
|
||||
<p className="text-sm text-light-grey">{job.message}</p>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Error */}
|
||||
{job.error && (
|
||||
<div className="mb-4 p-3 bg-red-500/10 rounded-lg border border-red-500/30">
|
||||
<div className="flex items-start gap-2">
|
||||
<AlertCircle className="w-4 h-4 text-red-400 flex-shrink-0 mt-0.5" />
|
||||
<p className="text-sm text-red-400">{job.error}</p>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Timestamps */}
|
||||
<div className="grid grid-cols-3 gap-4 text-sm">
|
||||
<div>
|
||||
<span className="text-light-grey block">Started</span>
|
||||
<span className="text-white font-mono text-xs">{formatTime(job.started_at)}</span>
|
||||
</div>
|
||||
<div>
|
||||
<span className="text-light-grey block">Completed</span>
|
||||
<span className="text-white font-mono text-xs">{formatTime(job.completed_at)}</span>
|
||||
</div>
|
||||
<div>
|
||||
<span className="text-light-grey block">Duration</span>
|
||||
<span className={`font-mono text-xs ${config.text}`}>
|
||||
{formatDuration(job.started_at, job.completed_at)}
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
</motion.div>
|
||||
);
|
||||
})
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Help Text */}
|
||||
<div className="bg-cool-blue/10 border border-electric-teal/30 rounded-lg p-4">
|
||||
<div className="flex items-start gap-3">
|
||||
<AlertCircle className="w-5 h-5 text-electric-teal flex-shrink-0 mt-0.5" />
|
||||
<div>
|
||||
<h4 className="font-semibold text-white mb-1">About Indexing Jobs</h4>
|
||||
<p className="text-sm text-light-grey">
|
||||
Semantic indexing creates vector embeddings for your documents, enabling AI-powered semantic search.
|
||||
Jobs run in the background so the server stays responsive. Progress updates every few seconds.
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
import { useState } from 'react';
|
||||
import { Link } from 'react-router-dom';
|
||||
import { Home, Settings, Database, MessageSquare, Plus, Trash2, FolderOpen, Zap, Edit, Activity, Trello } from 'lucide-react';
|
||||
import { Home, Settings, Database, MessageSquare, Plus, Trash2, FolderOpen, Zap, Edit, Activity, Trello, FileSearch } from 'lucide-react';
|
||||
import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query';
|
||||
import { motion, AnimatePresence } from 'framer-motion';
|
||||
import axios from 'axios';
|
||||
|
|
@ -10,6 +10,7 @@ import ProjectSetup from '../components/ProjectSetup';
|
|||
import DirectoryPicker from '../components/DirectoryPicker';
|
||||
import ServiceDashboard from '../components/ServiceDashboard';
|
||||
import KanbanBoard from '../components/KanbanBoard';
|
||||
import JobsDashboard from '../components/JobsDashboard';
|
||||
|
||||
const API_URL = import.meta.env.VITE_API_URL || 'http://localhost:8051';
|
||||
|
||||
|
|
@ -33,7 +34,7 @@ interface ProjectMCP {
|
|||
export default function MainApp() {
|
||||
const [selectedProject, setSelectedProject] = useState<Project | null>(null);
|
||||
const [selectedMCP, setSelectedMCP] = useState<string | null>(null);
|
||||
const [activeTab, setActiveTab] = useState<'overview' | 'mcps' | 'chat' | 'services' | 'kanban'>('overview');
|
||||
const [activeTab, setActiveTab] = useState<'overview' | 'mcps' | 'chat' | 'services' | 'kanban' | 'jobs'>('overview');
|
||||
const [showCreateProject, setShowCreateProject] = useState(false);
|
||||
const [showProjectSetup, setShowProjectSetup] = useState(false);
|
||||
const [newProjectName, setNewProjectName] = useState('');
|
||||
|
|
@ -346,6 +347,17 @@ export default function MainApp() {
|
|||
<Activity className="w-4 h-4 inline mr-2" />
|
||||
Services
|
||||
</button>
|
||||
<button
|
||||
onClick={() => setActiveTab('jobs')}
|
||||
className={`px-4 py-2 rounded-lg font-semibold transition-all ${
|
||||
activeTab === 'jobs'
|
||||
? 'bg-electric-teal text-deep-night'
|
||||
: 'text-light-grey hover:text-white'
|
||||
}`}
|
||||
>
|
||||
<FileSearch className="w-4 h-4 inline mr-2" />
|
||||
Jobs
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{/* Tab Content */}
|
||||
|
|
@ -438,6 +450,8 @@ export default function MainApp() {
|
|||
)
|
||||
) : activeTab === 'services' ? (
|
||||
<ServiceDashboard />
|
||||
) : activeTab === 'jobs' ? (
|
||||
<JobsDashboard />
|
||||
) : (
|
||||
selectedMCP ? (
|
||||
<ChatInterface
|
||||
|
|
|
|||
|
|
@ -202,13 +202,16 @@ async def list_tools() -> list[Tool]:
|
|||
),
|
||||
Tool(
|
||||
name="index_semantic",
|
||||
description="Run semantic embedding indexing on a knowledge base. Creates vector embeddings for similarity search.",
|
||||
description="Run semantic embedding indexing on a knowledge base. Creates vector embeddings for similarity search. Runs in background by default - returns job_id to check progress at /api/jobs/{job_id}.",
|
||||
inputSchema={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"kb_name": {"type": "string", "description": "Name of the knowledge base"}
|
||||
"kb_name": {"type": "string", "description": "Name of the knowledge base"},
|
||||
"project_path": {"type": "string", "description": "Path to project to index"},
|
||||
"selective": {"type": "boolean", "description": "If true, only index top 20% most important files + docs (default: true)"},
|
||||
"background": {"type": "boolean", "description": "If true, run in background (default: true). Set false to block until complete."}
|
||||
},
|
||||
"required": ["kb_name"]
|
||||
"required": ["kb_name", "project_path"]
|
||||
}
|
||||
),
|
||||
|
||||
|
|
@ -331,7 +334,11 @@ async def _execute_tool(name: str, args: dict[str, Any]) -> dict:
|
|||
})
|
||||
|
||||
elif name == "index_semantic":
|
||||
return await api_post(f"/api/kb/{args['kb_name']}/index-semantic")
|
||||
return await api_post(f"/api/kb/{args['kb_name']}/index-semantic", {
|
||||
"project_path": args["project_path"],
|
||||
"selective": args.get("selective", True),
|
||||
"background": args.get("background", True)
|
||||
})
|
||||
|
||||
# Document Management
|
||||
elif name == "upload_document":
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ from typing import Any, Dict, List
|
|||
# Add parent directory to path for imports
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
from fastapi import FastAPI, Request, UploadFile, File, HTTPException, Depends
|
||||
from fastapi import FastAPI, Request, UploadFile, File, HTTPException, Depends, BackgroundTasks
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.responses import JSONResponse
|
||||
from pydantic import BaseModel
|
||||
|
|
@ -49,6 +49,45 @@ RAG_ENGINE_CACHE = {}
|
|||
RAG_ENGINE_CACHE_LOCK = Lock()
|
||||
RAG_ENGINE_CACHE_TTL = 3600 # 1 hour TTL with plenty of RAM
|
||||
|
||||
# Background job status tracking for long-running operations
|
||||
INDEXING_JOBS = {} # job_id -> {status, progress, message, started_at, completed_at, error}
|
||||
INDEXING_JOBS_LOCK = Lock()
|
||||
MAX_STORED_JOBS = 100 # Limit stored jobs to prevent memory leak
|
||||
JOB_RETENTION_SECONDS = 3600 * 24 # Keep completed jobs for 24 hours
|
||||
|
||||
|
||||
def _cleanup_old_jobs():
|
||||
"""Remove old completed jobs to prevent memory leak."""
|
||||
with INDEXING_JOBS_LOCK:
|
||||
if len(INDEXING_JOBS) <= MAX_STORED_JOBS:
|
||||
return
|
||||
|
||||
current_time = time.time()
|
||||
# Find jobs to remove (completed/failed and older than retention period)
|
||||
jobs_to_remove = []
|
||||
for job_id, job in INDEXING_JOBS.items():
|
||||
if job["status"] in ["completed", "failed"]:
|
||||
completed_at = job.get("completed_at", job.get("started_at", 0))
|
||||
if current_time - completed_at > JOB_RETENTION_SECONDS:
|
||||
jobs_to_remove.append(job_id)
|
||||
|
||||
# If still over limit, remove oldest completed jobs
|
||||
if len(INDEXING_JOBS) - len(jobs_to_remove) > MAX_STORED_JOBS:
|
||||
completed_jobs = [
|
||||
(jid, j) for jid, j in INDEXING_JOBS.items()
|
||||
if j["status"] in ["completed", "failed"] and jid not in jobs_to_remove
|
||||
]
|
||||
completed_jobs.sort(key=lambda x: x[1].get("completed_at", 0))
|
||||
excess = len(INDEXING_JOBS) - len(jobs_to_remove) - MAX_STORED_JOBS
|
||||
for jid, _ in completed_jobs[:excess]:
|
||||
jobs_to_remove.append(jid)
|
||||
|
||||
for job_id in jobs_to_remove:
|
||||
del INDEXING_JOBS[job_id]
|
||||
|
||||
if jobs_to_remove:
|
||||
logger.debug(f"Cleaned up {len(jobs_to_remove)} old indexing jobs")
|
||||
|
||||
# Initialize FastAPI
|
||||
app = FastAPI(title="Claude OS MCP Server")
|
||||
|
||||
|
|
@ -738,6 +777,7 @@ class SemanticIndexRequest(BaseModel):
|
|||
project_path: str
|
||||
selective: bool = True # Only index top 20% + docs
|
||||
personalization: Optional[Dict[str, float]] = None
|
||||
background: bool = True # Run in background (True) or sync/blocking (False)
|
||||
|
||||
|
||||
@app.post("/api/kb/{kb_name}/index-structural")
|
||||
|
|
@ -843,15 +883,174 @@ async def api_index_structural(kb_name: str, request: StructuralIndexRequest):
|
|||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
|
||||
@app.post("/api/kb/{kb_name}/index-semantic")
|
||||
async def api_index_semantic(kb_name: str, request: SemanticIndexRequest):
|
||||
def _run_semantic_indexing_background(job_id: str, kb_name: str, project_path: str, selective: bool, personalization: dict = None):
|
||||
"""
|
||||
REST API: Semantic indexing with selective embedding.
|
||||
Phase 2 of hybrid indexing - optional, runs in background.
|
||||
Background worker for semantic indexing. Runs in a separate thread to avoid blocking.
|
||||
Updates INDEXING_JOBS with progress.
|
||||
"""
|
||||
from app.core.tree_sitter_indexer import TreeSitterIndexer
|
||||
from app.core.ingestion import ingest_directory
|
||||
import json
|
||||
from app.core.ingestion import ingest_directory, ingest_file
|
||||
|
||||
def update_job(status: str, progress: int = 0, message: str = "", error: str = None):
|
||||
with INDEXING_JOBS_LOCK:
|
||||
INDEXING_JOBS[job_id].update({
|
||||
"status": status,
|
||||
"progress": progress,
|
||||
"message": message,
|
||||
"error": error,
|
||||
"updated_at": time.time()
|
||||
})
|
||||
if status in ["completed", "failed"]:
|
||||
INDEXING_JOBS[job_id]["completed_at"] = time.time()
|
||||
|
||||
try:
|
||||
logger.info(f"[Job {job_id}] Starting semantic indexing for {kb_name} at {project_path}")
|
||||
start_time = time.time()
|
||||
update_job("running", 0, "Starting indexing...")
|
||||
|
||||
if selective:
|
||||
# Load repo map if exists
|
||||
cache_path = str(Path(project_path) / ".claude-os" / "tree_sitter_cache.db")
|
||||
update_job("running", 5, "Loading structural index...")
|
||||
|
||||
indexer = TreeSitterIndexer(cache_path)
|
||||
repo_map = indexer.index_directory(project_path, personalization)
|
||||
indexer.close()
|
||||
|
||||
# Select top 20% files by importance
|
||||
top_20_percent = max(1, int(len(repo_map.tags) * 0.2))
|
||||
important_tags = repo_map.tags[:top_20_percent]
|
||||
important_files = list(set(tag.file for tag in important_tags))
|
||||
|
||||
# Add all documentation files
|
||||
docs_patterns = ["*.md", "*.txt", "*.rst"]
|
||||
doc_files = []
|
||||
for pattern in docs_patterns:
|
||||
doc_files.extend(Path(project_path).rglob(pattern))
|
||||
|
||||
all_files = list(set(important_files + [str(f.relative_to(project_path)) for f in doc_files]))
|
||||
total_files = len(all_files)
|
||||
|
||||
update_job("running", 10, f"Found {total_files} files to index (top 20% + docs)")
|
||||
logger.info(f"[Job {job_id}] Selective indexing: {total_files} files")
|
||||
|
||||
# Ingest selected files only
|
||||
results = []
|
||||
for i, file_rel_path in enumerate(all_files):
|
||||
file_path = Path(project_path) / file_rel_path
|
||||
if file_path.exists() and file_path.is_file():
|
||||
try:
|
||||
result = ingest_file(str(file_path), kb_name, str(file_rel_path))
|
||||
results.append({"status": "success", "file": str(file_rel_path)})
|
||||
logger.debug(f"[Job {job_id}] Ingested: {file_rel_path}")
|
||||
except Exception as e:
|
||||
logger.warning(f"[Job {job_id}] Failed to ingest {file_rel_path}: {e}")
|
||||
results.append({"status": "error", "file": str(file_rel_path), "error": str(e)})
|
||||
|
||||
# Update progress every 5 files or at end
|
||||
if (i + 1) % 5 == 0 or (i + 1) == total_files:
|
||||
progress = 10 + int(((i + 1) / total_files) * 90)
|
||||
update_job("running", progress, f"Indexed {i + 1}/{total_files} files...")
|
||||
|
||||
successes = [r for r in results if r.get("status") == "success"]
|
||||
elapsed = time.time() - start_time
|
||||
|
||||
logger.info(f"[Job {job_id}] Selective semantic indexing complete in {elapsed:.1f}s")
|
||||
update_job("completed", 100, f"Complete: {len(successes)}/{total_files} files indexed in {elapsed:.1f}s")
|
||||
|
||||
else:
|
||||
# Full indexing (all files)
|
||||
update_job("running", 10, "Starting full directory indexing...")
|
||||
results = ingest_directory(project_path, kb_name)
|
||||
successes = [r for r in results if r.get("status") == "success"]
|
||||
|
||||
elapsed = time.time() - start_time
|
||||
logger.info(f"[Job {job_id}] Full semantic indexing complete in {elapsed:.1f}s")
|
||||
update_job("completed", 100, f"Complete: {len(successes)} files indexed in {elapsed:.1f}s")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"[Job {job_id}] Semantic indexing failed: {e}")
|
||||
update_job("failed", 0, "", str(e))
|
||||
|
||||
|
||||
def _run_semantic_indexing_sync(kb_name: str, project_path: str, selective: bool, personalization: dict = None) -> dict:
|
||||
"""
|
||||
Synchronous semantic indexing for backward compatibility.
|
||||
WARNING: This blocks the server! Use background=true for production.
|
||||
Returns the final result dict.
|
||||
"""
|
||||
from app.core.tree_sitter_indexer import TreeSitterIndexer
|
||||
from app.core.ingestion import ingest_directory, ingest_file
|
||||
|
||||
start_time = time.time()
|
||||
|
||||
if selective:
|
||||
cache_path = str(Path(project_path) / ".claude-os" / "tree_sitter_cache.db")
|
||||
indexer = TreeSitterIndexer(cache_path)
|
||||
repo_map = indexer.index_directory(project_path, personalization)
|
||||
indexer.close()
|
||||
|
||||
top_20_percent = max(1, int(len(repo_map.tags) * 0.2))
|
||||
important_tags = repo_map.tags[:top_20_percent]
|
||||
important_files = list(set(tag.file for tag in important_tags))
|
||||
|
||||
docs_patterns = ["*.md", "*.txt", "*.rst"]
|
||||
doc_files = []
|
||||
for pattern in docs_patterns:
|
||||
doc_files.extend(Path(project_path).rglob(pattern))
|
||||
|
||||
all_files = list(set(important_files + [str(f.relative_to(project_path)) for f in doc_files]))
|
||||
|
||||
results = []
|
||||
for file_rel_path in all_files:
|
||||
file_path = Path(project_path) / file_rel_path
|
||||
if file_path.exists() and file_path.is_file():
|
||||
try:
|
||||
result = ingest_file(str(file_path), kb_name, str(file_rel_path))
|
||||
results.append({"status": "success", "file": str(file_rel_path)})
|
||||
except Exception as e:
|
||||
results.append({"status": "error", "file": str(file_rel_path), "error": str(e)})
|
||||
|
||||
successes = [r for r in results if r.get("status") == "success"]
|
||||
elapsed = time.time() - start_time
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"kb_name": kb_name,
|
||||
"mode": "selective",
|
||||
"files_selected": len(all_files),
|
||||
"files_indexed": len(successes),
|
||||
"time_taken_seconds": elapsed,
|
||||
"message": f"Selective semantic indexing complete: {len(successes)}/{len(all_files)} files indexed"
|
||||
}
|
||||
else:
|
||||
results = ingest_directory(project_path, kb_name)
|
||||
successes = [r for r in results if r.get("status") == "success"]
|
||||
elapsed = time.time() - start_time
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"kb_name": kb_name,
|
||||
"mode": "full",
|
||||
"total_files": len(results),
|
||||
"successful": len(successes),
|
||||
"time_taken_seconds": elapsed,
|
||||
"message": f"Full semantic indexing complete: {len(successes)} files indexed"
|
||||
}
|
||||
|
||||
|
||||
@app.post("/api/kb/{kb_name}/index-semantic")
|
||||
async def api_index_semantic(kb_name: str, request: SemanticIndexRequest, background_tasks: BackgroundTasks):
|
||||
"""
|
||||
REST API: Semantic indexing with selective embedding.
|
||||
Phase 2 of hybrid indexing.
|
||||
|
||||
By default (background=true), runs in background and returns immediately with job_id.
|
||||
Check progress at GET /api/jobs/{job_id}
|
||||
|
||||
Set background=false for synchronous execution (WARNING: blocks server, use for small projects only).
|
||||
"""
|
||||
import uuid
|
||||
|
||||
# Validate KB exists
|
||||
db_manager = get_sqlite_manager()
|
||||
|
|
@ -862,87 +1061,80 @@ async def api_index_semantic(kb_name: str, request: SemanticIndexRequest):
|
|||
if not Path(request.project_path).exists():
|
||||
raise HTTPException(status_code=404, detail=f"Project path not found: {request.project_path}")
|
||||
|
||||
try:
|
||||
logger.info(f"Starting semantic indexing for {kb_name} at {request.project_path}")
|
||||
start_time = time.time()
|
||||
# Cleanup old jobs periodically
|
||||
_cleanup_old_jobs()
|
||||
|
||||
if request.selective:
|
||||
# Get structural index first
|
||||
structure_kb = f"{kb_name.split('-')[0]}-code_structure"
|
||||
# Synchronous mode (backward compatible, but blocks server)
|
||||
if not request.background:
|
||||
logger.warning(f"Running semantic indexing synchronously for {kb_name} - this will block the server!")
|
||||
try:
|
||||
result = _run_semantic_indexing_sync(
|
||||
kb_name,
|
||||
request.project_path,
|
||||
request.selective,
|
||||
request.personalization
|
||||
)
|
||||
return result
|
||||
except Exception as e:
|
||||
logger.error(f"Semantic indexing failed: {e}")
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
# Load repo map if exists
|
||||
cache_path = str(Path(request.project_path) / ".claude-os" / "tree_sitter_cache.db")
|
||||
indexer = TreeSitterIndexer(cache_path)
|
||||
repo_map = indexer.index_directory(request.project_path, request.personalization)
|
||||
indexer.close()
|
||||
# Background mode (default, non-blocking)
|
||||
job_id = f"semantic-{kb_name}-{uuid.uuid4().hex[:8]}"
|
||||
with INDEXING_JOBS_LOCK:
|
||||
INDEXING_JOBS[job_id] = {
|
||||
"job_id": job_id,
|
||||
"type": "semantic_indexing",
|
||||
"kb_name": kb_name,
|
||||
"project_path": request.project_path,
|
||||
"selective": request.selective,
|
||||
"status": "queued",
|
||||
"progress": 0,
|
||||
"message": "Job queued",
|
||||
"started_at": time.time(),
|
||||
"completed_at": None,
|
||||
"error": None
|
||||
}
|
||||
|
||||
# Select top 20% files by importance
|
||||
top_20_percent = int(len(repo_map.tags) * 0.2)
|
||||
important_tags = repo_map.tags[:top_20_percent]
|
||||
# Queue the background task
|
||||
background_tasks.add_task(
|
||||
_run_semantic_indexing_background,
|
||||
job_id,
|
||||
kb_name,
|
||||
request.project_path,
|
||||
request.selective,
|
||||
request.personalization
|
||||
)
|
||||
|
||||
# Get unique files
|
||||
important_files = list(set(tag.file for tag in important_tags))
|
||||
logger.info(f"Queued semantic indexing job {job_id} for {kb_name}")
|
||||
|
||||
# Add all documentation files
|
||||
docs_patterns = ["*.md", "*.txt", "*.rst"]
|
||||
doc_files = []
|
||||
for pattern in docs_patterns:
|
||||
doc_files.extend(Path(request.project_path).rglob(pattern))
|
||||
return {
|
||||
"success": True,
|
||||
"job_id": job_id,
|
||||
"kb_name": kb_name,
|
||||
"mode": "selective" if request.selective else "full",
|
||||
"status": "queued",
|
||||
"message": f"Semantic indexing started in background. Check progress at GET /api/jobs/{job_id}"
|
||||
}
|
||||
|
||||
all_files = important_files + [str(f.relative_to(request.project_path)) for f in doc_files]
|
||||
|
||||
logger.info(f"Selective indexing: {len(all_files)} files (top 20% + docs)")
|
||||
@app.get("/api/jobs/{job_id}")
|
||||
async def api_get_job_status(job_id: str):
|
||||
"""Get the status of a background indexing job."""
|
||||
with INDEXING_JOBS_LOCK:
|
||||
if job_id not in INDEXING_JOBS:
|
||||
raise HTTPException(status_code=404, detail=f"Job '{job_id}' not found")
|
||||
return INDEXING_JOBS[job_id].copy()
|
||||
|
||||
# Ingest selected files only
|
||||
from app.core.ingestion import ingest_file
|
||||
results = []
|
||||
|
||||
for file_rel_path in all_files:
|
||||
file_path = Path(request.project_path) / file_rel_path
|
||||
if file_path.exists() and file_path.is_file():
|
||||
try:
|
||||
result = ingest_file(str(file_path), kb_name, str(file_rel_path))
|
||||
results.append({"status": "success", "file": str(file_rel_path)})
|
||||
logger.debug(f"Ingested: {file_rel_path}")
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to ingest {file_rel_path}: {e}")
|
||||
results.append({"status": "error", "file": str(file_rel_path), "error": str(e)})
|
||||
|
||||
successes = [r for r in results if r.get("status") == "success"]
|
||||
elapsed = time.time() - start_time
|
||||
|
||||
logger.info(f"Selective semantic indexing complete for {kb_name} in {elapsed:.1f}s")
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"kb_name": kb_name,
|
||||
"mode": "selective",
|
||||
"files_selected": len(all_files),
|
||||
"files_indexed": len(successes),
|
||||
"time_taken_seconds": elapsed,
|
||||
"message": f"Selective semantic indexing complete: {len(successes)}/{len(all_files)} files indexed"
|
||||
}
|
||||
else:
|
||||
# Full indexing (all files)
|
||||
results = ingest_directory(request.project_path, kb_name)
|
||||
successes = [r for r in results if r.get("status") == "success"]
|
||||
|
||||
elapsed = time.time() - start_time
|
||||
logger.info(f"Full semantic indexing complete for {kb_name} in {elapsed:.1f}s")
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"kb_name": kb_name,
|
||||
"mode": "full",
|
||||
"total_files": len(results),
|
||||
"successful": len(successes),
|
||||
"time_taken_seconds": elapsed,
|
||||
"message": f"Full semantic indexing complete: {len(successes)} files indexed"
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Semantic indexing failed: {e}")
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
@app.get("/api/jobs")
|
||||
async def api_list_jobs():
|
||||
"""List all indexing jobs (recent first)."""
|
||||
with INDEXING_JOBS_LOCK:
|
||||
jobs = list(INDEXING_JOBS.values())
|
||||
# Sort by started_at descending
|
||||
jobs.sort(key=lambda x: x.get("started_at", 0), reverse=True)
|
||||
return {"jobs": jobs[:50]} # Return last 50 jobs
|
||||
|
||||
|
||||
@app.get("/api/kb/{kb_name}/repo-map")
|
||||
|
|
|
|||
|
|
@ -351,7 +351,7 @@ class TestMCPServerIndexingOperations:
|
|||
async def test_index_semantic_calls_correct_endpoint(self, mock_httpx):
|
||||
"""Test that index_semantic calls the correct endpoint."""
|
||||
mock_response = MagicMock()
|
||||
mock_response.json.return_value = {"embedded": 100}
|
||||
mock_response.json.return_value = {"job_id": "semantic-docs_kb-abc123", "status": "queued"}
|
||||
mock_response.raise_for_status = MagicMock()
|
||||
|
||||
mock_client = AsyncMock()
|
||||
|
|
@ -362,9 +362,14 @@ class TestMCPServerIndexingOperations:
|
|||
|
||||
from mcp_server.claude_code_mcp import _execute_tool
|
||||
result = await _execute_tool("index_semantic", {
|
||||
"kb_name": "docs_kb"
|
||||
"kb_name": "docs_kb",
|
||||
"project_path": "/path/to/project"
|
||||
})
|
||||
|
||||
mock_client.post.assert_called_once()
|
||||
call_args = mock_client.post.call_args
|
||||
assert "/api/kb/docs_kb/index-semantic" in str(call_args)
|
||||
# Verify the payload includes required fields
|
||||
payload = call_args[1].get("json", {})
|
||||
assert payload.get("project_path") == "/path/to/project"
|
||||
assert payload.get("background") == True
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue