diff --git a/app/core/config.py b/app/core/config.py index 0c07fbd..0dcf715 100644 --- a/app/core/config.py +++ b/app/core/config.py @@ -31,7 +31,7 @@ class Config(metaclass=ConfigMeta): # ═══════════════════════════════════════════════════════════════════════ # PROVIDER CONFIGURATION # ═══════════════════════════════════════════════════════════════════════ - # Provider: "local" (Ollama) or "openai" (OpenAI API) + # Provider: "local" (Ollama), "openai" (OpenAI API), or "groq" (Groq API) PROVIDER: str = os.getenv("CLAUDE_OS_PROVIDER", "local") # ═══════════════════════════════════════════════════════════════════════ @@ -49,6 +49,13 @@ class Config(metaclass=ConfigMeta): OPENAI_EMBED_MODEL: str = os.getenv("OPENAI_EMBED_MODEL", "text-embedding-3-small") OPENAI_EMBED_DIMENSIONS: int = 1536 # OpenAI text-embedding-3-small dimensions + # ═══════════════════════════════════════════════════════════════════════ + # GROQ CONFIGURATION (for groq provider) + # ═══════════════════════════════════════════════════════════════════════ + # Groq has no embeddings API - embeddings always stay on Ollama/OpenAI when using this provider. + GROQ_API_KEY: str = os.getenv("GROQ_API_KEY", "") + GROQ_MODEL: str = os.getenv("GROQ_MODEL", "openai/gpt-oss-120b") + # ═══════════════════════════════════════════════════════════════════════ # MODEL CONFIGURATION (auto-selected based on provider) # ═══════════════════════════════════════════════════════════════════════ @@ -217,9 +224,9 @@ class Config(metaclass=ConfigMeta): @classmethod def get_provider(cls) -> str: - """Get the active provider (local or openai).""" + """Get the active provider (local, openai, or groq).""" provider = os.getenv("CLAUDE_OS_PROVIDER", "local").lower() - if provider not in ("local", "openai", "custom"): + if provider not in ("local", "openai", "groq", "custom"): return "local" return provider @@ -233,6 +240,11 @@ class Config(metaclass=ConfigMeta): """Check if using OpenAI provider.""" return cls.get_provider() == "openai" + @classmethod + def is_groq_provider(cls) -> bool: + """Check if using Groq provider.""" + return cls.get_provider() == "groq" + @classmethod def get_active_llm_model(cls) -> str: """Get the active LLM model based on provider.""" @@ -243,6 +255,8 @@ class Config(metaclass=ConfigMeta): if cls.is_openai_provider(): return cls.OPENAI_LLM_MODEL + if cls.is_groq_provider(): + return cls.GROQ_MODEL return cls.OLLAMA_MODEL @classmethod @@ -271,6 +285,12 @@ class Config(metaclass=ConfigMeta): key = os.getenv("OPENAI_API_KEY", "") return key if key else None + @classmethod + def get_groq_api_key(cls) -> Optional[str]: + """Get the Groq API key if configured.""" + key = os.getenv("GROQ_API_KEY", "") + return key if key else None + @classmethod def validate_provider_config(cls) -> List[str]: """Validate provider-specific configuration. Returns list of errors.""" @@ -281,6 +301,10 @@ class Config(metaclass=ConfigMeta): if not cls.get_openai_api_key(): errors.append("OPENAI_API_KEY is required when using OpenAI provider") + elif provider == "groq": + if not cls.get_groq_api_key(): + errors.append("GROQ_API_KEY is required when using Groq provider") + elif provider == "local": # Could add Ollama connectivity check here pass diff --git a/app/core/insight_extractor.py b/app/core/insight_extractor.py index f27c258..48e333a 100644 --- a/app/core/insight_extractor.py +++ b/app/core/insight_extractor.py @@ -102,9 +102,12 @@ class InsightExtractor: # Build prompt prompt = EXTRACTION_PROMPT.format(transcript=session_summary) - # Call Ollama + # Call the configured LLM provider try: - insights = await self._call_ollama(prompt) + if Config.is_groq_provider(): + insights = await self._call_groq(prompt) + else: + insights = await self._call_ollama(prompt) # Filter by requested types filtered_insights = [ @@ -152,25 +155,7 @@ class InsightExtractor: result = response.json() response_text = result.get("response", "") - # Parse JSON response - insights_data = self._parse_llm_response(response_text) - - # Convert to Insight objects - insights = [] - for item in insights_data.get("insights", []): - try: - insight = Insight( - type=item.get("type", "pattern"), - title=item.get("title", "Untitled"), - content=item.get("content", ""), - confidence=float(item.get("confidence", 0.5)) - ) - insights.append(insight) - except (ValueError, KeyError) as e: - logger.warning(f"Failed to parse insight item: {e}") - continue - - return insights + return self._insights_from_response_text(response_text) except httpx.HTTPError as e: logger.error(f"Ollama API error: {e}") @@ -179,6 +164,65 @@ class InsightExtractor: logger.error(f"Unexpected error calling Ollama: {e}") raise + async def _call_groq(self, prompt: str) -> List[Insight]: + """ + Call Groq's OpenAI-compatible chat completions API to extract insights. + + Args: + prompt: Extraction prompt with session transcript + + Returns: + List of Insight objects + """ + url = "https://api.groq.com/openai/v1/chat/completions" + + payload = { + "model": self.model, + "messages": [{"role": "user", "content": prompt}], + "temperature": 0.3, # Lower temperature for more focused extraction + "max_tokens": 2048, # Allow longer responses for multiple insights + "top_p": 0.9, + "response_format": {"type": "json_object"}, # Request JSON response + } + headers = {"Authorization": f"Bearer {Config.GROQ_API_KEY}"} + + try: + async with httpx.AsyncClient(timeout=120.0) as client: + response = await client.post(url, json=payload, headers=headers) + response.raise_for_status() + + result = response.json() + response_text = result["choices"][0]["message"]["content"] + + return self._insights_from_response_text(response_text) + + except httpx.HTTPError as e: + logger.error(f"Groq API error: {e}") + raise + except Exception as e: + logger.error(f"Unexpected error calling Groq: {e}") + raise + + def _insights_from_response_text(self, response_text: str) -> List[Insight]: + """Parse a raw LLM response string into Insight objects.""" + insights_data = self._parse_llm_response(response_text) + + insights = [] + for item in insights_data.get("insights", []): + try: + insight = Insight( + type=item.get("type", "pattern"), + title=item.get("title", "Untitled"), + content=item.get("content", ""), + confidence=float(item.get("confidence", 0.5)) + ) + insights.append(insight) + except (ValueError, KeyError) as e: + logger.warning(f"Failed to parse insight item: {e}") + continue + + return insights + def _parse_llm_response(self, response_text: str) -> Dict[str, Any]: """ Parse LLM response JSON. diff --git a/app/core/knowledge_lifecycle.py b/app/core/knowledge_lifecycle.py index c7163c4..056dddd 100644 --- a/app/core/knowledge_lifecycle.py +++ b/app/core/knowledge_lifecycle.py @@ -343,10 +343,25 @@ Source documents: Write the consolidated document:""" try: + if Config.is_groq_provider(): + response = requests.post( + "https://api.groq.com/openai/v1/chat/completions", + json={ + "model": Config.get_active_llm_model(), + "messages": [{"role": "user", "content": prompt}], + "temperature": 0.3, + "max_tokens": 2048 + }, + headers={"Authorization": f"Bearer {Config.GROQ_API_KEY}"}, + timeout=120 + ) + response.raise_for_status() + return response.json()["choices"][0]["message"]["content"] + response = requests.post( f"{Config.OLLAMA_HOST}/api/generate", json={ - "model": Config.OLLAMA_MODEL, + "model": Config.get_active_llm_model(), "prompt": prompt, "stream": False, "options": { diff --git a/app/core/rag_engine.py b/app/core/rag_engine.py index 36c93bb..8c9a951 100644 --- a/app/core/rag_engine.py +++ b/app/core/rag_engine.py @@ -85,25 +85,33 @@ class RAGEngine: """ self.collection_name = collection_name - # Initialize Ollama LLM optimized for Apple M4 Pro (14 cores, 48GB RAM) - # Using native Ollama with Metal GPU acceleration - self.llm = Ollama( - model="llama3.1:latest", # 8B model runs great on M4 Pro - base_url="http://localhost:11434", # Native Ollama (not Docker) - request_timeout=120.0, # Increased to 120s for LLM generation - context_window=8192, # Larger context with 48GB RAM - num_ctx=8192, # Match context_window - temperature=0.1, # Very low for factual RAG responses - num_predict=512, # Can handle more tokens with M4 Pro - top_k=40, # Better quality sampling - top_p=0.9, # Nucleus sampling for better quality - num_thread=12, # Use most cores (leave 2 for system) - num_gpu=99, # Use all GPU layers on Apple Silicon - repeat_penalty=1.1, # Avoid repetition - num_batch=512, # Larger batch size for M4 Pro - use_mmap=True, # Memory-map model for faster loading - use_mlock=True, # Lock model in RAM (plenty available) - ) + if Config.is_groq_provider(): + from llama_index.llms.groq import Groq + self.llm = Groq( + model=Config.get_active_llm_model(), + api_key=Config.GROQ_API_KEY, + temperature=0.1, # Very low for factual RAG responses + ) + else: + # Initialize Ollama LLM optimized for Apple M4 Pro (14 cores, 48GB RAM) + # Using native Ollama with Metal GPU acceleration + self.llm = Ollama( + model="llama3.1:latest", # 8B model runs great on M4 Pro + base_url="http://localhost:11434", # Native Ollama (not Docker) + request_timeout=120.0, # Increased to 120s for LLM generation + context_window=8192, # Larger context with 48GB RAM + num_ctx=8192, # Match context_window + temperature=0.1, # Very low for factual RAG responses + num_predict=512, # Can handle more tokens with M4 Pro + top_k=40, # Better quality sampling + top_p=0.9, # Nucleus sampling for better quality + num_thread=12, # Use most cores (leave 2 for system) + num_gpu=99, # Use all GPU layers on Apple Silicon + repeat_penalty=1.1, # Avoid repetition + num_batch=512, # Larger batch size for M4 Pro + use_mmap=True, # Memory-map model for faster loading + use_mlock=True, # Lock model in RAM (plenty available) + ) Settings.llm = self.llm # Initialize Ollama embeddings diff --git a/requirements.txt b/requirements.txt index 1cac88e..0e9412d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,7 @@ llama-index>=0.10.0 llama-index-llms-ollama>=0.1.0 llama-index-embeddings-ollama>=0.1.0 +llama-index-llms-groq>=0.5.0 # Vector Database (SQLite-based) sqlite-vec>=0.1.0