Add Groq as an LLM provider
Config.PROVIDER was never actually consulted by the LLM call sites - rag_engine.py, insight_extractor.py, and knowledge_lifecycle.py all hardcoded Ollama directly. This wires up a real "groq" provider: - config.py: GROQ_API_KEY/GROQ_MODEL settings, is_groq_provider(), provider-aware get_active_llm_model() and validate_provider_config() - rag_engine.py: uses llama_index's Groq LLM when provider is groq - insight_extractor.py / knowledge_lifecycle.py: branch to Groq's OpenAI-compatible chat completions API instead of Ollama's /api/generate - requirements.txt: add llama-index-llms-groq Embeddings are untouched and stay on Ollama - Groq has no embeddings API. Default model is openai/gpt-oss-120b.
This commit is contained in:
Rosie Val
parent
ee7b62bc5b
commit
614a109378
5 changed files with 136 additions and 44 deletions
|
|
@ -31,7 +31,7 @@ class Config(metaclass=ConfigMeta):
|
|||
# ═══════════════════════════════════════════════════════════════════════
|
||||
# PROVIDER CONFIGURATION
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
# Provider: "local" (Ollama) or "openai" (OpenAI API)
|
||||
# Provider: "local" (Ollama), "openai" (OpenAI API), or "groq" (Groq API)
|
||||
PROVIDER: str = os.getenv("CLAUDE_OS_PROVIDER", "local")
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
|
|
@ -49,6 +49,13 @@ class Config(metaclass=ConfigMeta):
|
|||
OPENAI_EMBED_MODEL: str = os.getenv("OPENAI_EMBED_MODEL", "text-embedding-3-small")
|
||||
OPENAI_EMBED_DIMENSIONS: int = 1536 # OpenAI text-embedding-3-small dimensions
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
# GROQ CONFIGURATION (for groq provider)
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
# Groq has no embeddings API - embeddings always stay on Ollama/OpenAI when using this provider.
|
||||
GROQ_API_KEY: str = os.getenv("GROQ_API_KEY", "")
|
||||
GROQ_MODEL: str = os.getenv("GROQ_MODEL", "openai/gpt-oss-120b")
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
# MODEL CONFIGURATION (auto-selected based on provider)
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
|
|
@ -217,9 +224,9 @@ class Config(metaclass=ConfigMeta):
|
|||
|
||||
@classmethod
|
||||
def get_provider(cls) -> str:
|
||||
"""Get the active provider (local or openai)."""
|
||||
"""Get the active provider (local, openai, or groq)."""
|
||||
provider = os.getenv("CLAUDE_OS_PROVIDER", "local").lower()
|
||||
if provider not in ("local", "openai", "custom"):
|
||||
if provider not in ("local", "openai", "groq", "custom"):
|
||||
return "local"
|
||||
return provider
|
||||
|
||||
|
|
@ -233,6 +240,11 @@ class Config(metaclass=ConfigMeta):
|
|||
"""Check if using OpenAI provider."""
|
||||
return cls.get_provider() == "openai"
|
||||
|
||||
@classmethod
|
||||
def is_groq_provider(cls) -> bool:
|
||||
"""Check if using Groq provider."""
|
||||
return cls.get_provider() == "groq"
|
||||
|
||||
@classmethod
|
||||
def get_active_llm_model(cls) -> str:
|
||||
"""Get the active LLM model based on provider."""
|
||||
|
|
@ -243,6 +255,8 @@ class Config(metaclass=ConfigMeta):
|
|||
|
||||
if cls.is_openai_provider():
|
||||
return cls.OPENAI_LLM_MODEL
|
||||
if cls.is_groq_provider():
|
||||
return cls.GROQ_MODEL
|
||||
return cls.OLLAMA_MODEL
|
||||
|
||||
@classmethod
|
||||
|
|
@ -271,6 +285,12 @@ class Config(metaclass=ConfigMeta):
|
|||
key = os.getenv("OPENAI_API_KEY", "")
|
||||
return key if key else None
|
||||
|
||||
@classmethod
|
||||
def get_groq_api_key(cls) -> Optional[str]:
|
||||
"""Get the Groq API key if configured."""
|
||||
key = os.getenv("GROQ_API_KEY", "")
|
||||
return key if key else None
|
||||
|
||||
@classmethod
|
||||
def validate_provider_config(cls) -> List[str]:
|
||||
"""Validate provider-specific configuration. Returns list of errors."""
|
||||
|
|
@ -281,6 +301,10 @@ class Config(metaclass=ConfigMeta):
|
|||
if not cls.get_openai_api_key():
|
||||
errors.append("OPENAI_API_KEY is required when using OpenAI provider")
|
||||
|
||||
elif provider == "groq":
|
||||
if not cls.get_groq_api_key():
|
||||
errors.append("GROQ_API_KEY is required when using Groq provider")
|
||||
|
||||
elif provider == "local":
|
||||
# Could add Ollama connectivity check here
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -102,9 +102,12 @@ class InsightExtractor:
|
|||
# Build prompt
|
||||
prompt = EXTRACTION_PROMPT.format(transcript=session_summary)
|
||||
|
||||
# Call Ollama
|
||||
# Call the configured LLM provider
|
||||
try:
|
||||
insights = await self._call_ollama(prompt)
|
||||
if Config.is_groq_provider():
|
||||
insights = await self._call_groq(prompt)
|
||||
else:
|
||||
insights = await self._call_ollama(prompt)
|
||||
|
||||
# Filter by requested types
|
||||
filtered_insights = [
|
||||
|
|
@ -152,25 +155,7 @@ class InsightExtractor:
|
|||
result = response.json()
|
||||
response_text = result.get("response", "")
|
||||
|
||||
# Parse JSON response
|
||||
insights_data = self._parse_llm_response(response_text)
|
||||
|
||||
# Convert to Insight objects
|
||||
insights = []
|
||||
for item in insights_data.get("insights", []):
|
||||
try:
|
||||
insight = Insight(
|
||||
type=item.get("type", "pattern"),
|
||||
title=item.get("title", "Untitled"),
|
||||
content=item.get("content", ""),
|
||||
confidence=float(item.get("confidence", 0.5))
|
||||
)
|
||||
insights.append(insight)
|
||||
except (ValueError, KeyError) as e:
|
||||
logger.warning(f"Failed to parse insight item: {e}")
|
||||
continue
|
||||
|
||||
return insights
|
||||
return self._insights_from_response_text(response_text)
|
||||
|
||||
except httpx.HTTPError as e:
|
||||
logger.error(f"Ollama API error: {e}")
|
||||
|
|
@ -179,6 +164,65 @@ class InsightExtractor:
|
|||
logger.error(f"Unexpected error calling Ollama: {e}")
|
||||
raise
|
||||
|
||||
async def _call_groq(self, prompt: str) -> List[Insight]:
|
||||
"""
|
||||
Call Groq's OpenAI-compatible chat completions API to extract insights.
|
||||
|
||||
Args:
|
||||
prompt: Extraction prompt with session transcript
|
||||
|
||||
Returns:
|
||||
List of Insight objects
|
||||
"""
|
||||
url = "https://api.groq.com/openai/v1/chat/completions"
|
||||
|
||||
payload = {
|
||||
"model": self.model,
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
"temperature": 0.3, # Lower temperature for more focused extraction
|
||||
"max_tokens": 2048, # Allow longer responses for multiple insights
|
||||
"top_p": 0.9,
|
||||
"response_format": {"type": "json_object"}, # Request JSON response
|
||||
}
|
||||
headers = {"Authorization": f"Bearer {Config.GROQ_API_KEY}"}
|
||||
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=120.0) as client:
|
||||
response = await client.post(url, json=payload, headers=headers)
|
||||
response.raise_for_status()
|
||||
|
||||
result = response.json()
|
||||
response_text = result["choices"][0]["message"]["content"]
|
||||
|
||||
return self._insights_from_response_text(response_text)
|
||||
|
||||
except httpx.HTTPError as e:
|
||||
logger.error(f"Groq API error: {e}")
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Unexpected error calling Groq: {e}")
|
||||
raise
|
||||
|
||||
def _insights_from_response_text(self, response_text: str) -> List[Insight]:
|
||||
"""Parse a raw LLM response string into Insight objects."""
|
||||
insights_data = self._parse_llm_response(response_text)
|
||||
|
||||
insights = []
|
||||
for item in insights_data.get("insights", []):
|
||||
try:
|
||||
insight = Insight(
|
||||
type=item.get("type", "pattern"),
|
||||
title=item.get("title", "Untitled"),
|
||||
content=item.get("content", ""),
|
||||
confidence=float(item.get("confidence", 0.5))
|
||||
)
|
||||
insights.append(insight)
|
||||
except (ValueError, KeyError) as e:
|
||||
logger.warning(f"Failed to parse insight item: {e}")
|
||||
continue
|
||||
|
||||
return insights
|
||||
|
||||
def _parse_llm_response(self, response_text: str) -> Dict[str, Any]:
|
||||
"""
|
||||
Parse LLM response JSON.
|
||||
|
|
|
|||
|
|
@ -343,10 +343,25 @@ Source documents:
|
|||
Write the consolidated document:"""
|
||||
|
||||
try:
|
||||
if Config.is_groq_provider():
|
||||
response = requests.post(
|
||||
"https://api.groq.com/openai/v1/chat/completions",
|
||||
json={
|
||||
"model": Config.get_active_llm_model(),
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
"temperature": 0.3,
|
||||
"max_tokens": 2048
|
||||
},
|
||||
headers={"Authorization": f"Bearer {Config.GROQ_API_KEY}"},
|
||||
timeout=120
|
||||
)
|
||||
response.raise_for_status()
|
||||
return response.json()["choices"][0]["message"]["content"]
|
||||
|
||||
response = requests.post(
|
||||
f"{Config.OLLAMA_HOST}/api/generate",
|
||||
json={
|
||||
"model": Config.OLLAMA_MODEL,
|
||||
"model": Config.get_active_llm_model(),
|
||||
"prompt": prompt,
|
||||
"stream": False,
|
||||
"options": {
|
||||
|
|
|
|||
|
|
@ -85,25 +85,33 @@ class RAGEngine:
|
|||
"""
|
||||
self.collection_name = collection_name
|
||||
|
||||
# Initialize Ollama LLM optimized for Apple M4 Pro (14 cores, 48GB RAM)
|
||||
# Using native Ollama with Metal GPU acceleration
|
||||
self.llm = Ollama(
|
||||
model="llama3.1:latest", # 8B model runs great on M4 Pro
|
||||
base_url="http://localhost:11434", # Native Ollama (not Docker)
|
||||
request_timeout=120.0, # Increased to 120s for LLM generation
|
||||
context_window=8192, # Larger context with 48GB RAM
|
||||
num_ctx=8192, # Match context_window
|
||||
temperature=0.1, # Very low for factual RAG responses
|
||||
num_predict=512, # Can handle more tokens with M4 Pro
|
||||
top_k=40, # Better quality sampling
|
||||
top_p=0.9, # Nucleus sampling for better quality
|
||||
num_thread=12, # Use most cores (leave 2 for system)
|
||||
num_gpu=99, # Use all GPU layers on Apple Silicon
|
||||
repeat_penalty=1.1, # Avoid repetition
|
||||
num_batch=512, # Larger batch size for M4 Pro
|
||||
use_mmap=True, # Memory-map model for faster loading
|
||||
use_mlock=True, # Lock model in RAM (plenty available)
|
||||
)
|
||||
if Config.is_groq_provider():
|
||||
from llama_index.llms.groq import Groq
|
||||
self.llm = Groq(
|
||||
model=Config.get_active_llm_model(),
|
||||
api_key=Config.GROQ_API_KEY,
|
||||
temperature=0.1, # Very low for factual RAG responses
|
||||
)
|
||||
else:
|
||||
# Initialize Ollama LLM optimized for Apple M4 Pro (14 cores, 48GB RAM)
|
||||
# Using native Ollama with Metal GPU acceleration
|
||||
self.llm = Ollama(
|
||||
model="llama3.1:latest", # 8B model runs great on M4 Pro
|
||||
base_url="http://localhost:11434", # Native Ollama (not Docker)
|
||||
request_timeout=120.0, # Increased to 120s for LLM generation
|
||||
context_window=8192, # Larger context with 48GB RAM
|
||||
num_ctx=8192, # Match context_window
|
||||
temperature=0.1, # Very low for factual RAG responses
|
||||
num_predict=512, # Can handle more tokens with M4 Pro
|
||||
top_k=40, # Better quality sampling
|
||||
top_p=0.9, # Nucleus sampling for better quality
|
||||
num_thread=12, # Use most cores (leave 2 for system)
|
||||
num_gpu=99, # Use all GPU layers on Apple Silicon
|
||||
repeat_penalty=1.1, # Avoid repetition
|
||||
num_batch=512, # Larger batch size for M4 Pro
|
||||
use_mmap=True, # Memory-map model for faster loading
|
||||
use_mlock=True, # Lock model in RAM (plenty available)
|
||||
)
|
||||
Settings.llm = self.llm
|
||||
|
||||
# Initialize Ollama embeddings
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
llama-index>=0.10.0
|
||||
llama-index-llms-ollama>=0.1.0
|
||||
llama-index-embeddings-ollama>=0.1.0
|
||||
llama-index-llms-groq>=0.5.0
|
||||
|
||||
# Vector Database (SQLite-based)
|
||||
sqlite-vec>=0.1.0
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue