Add Groq model support to LLMClient (#1977)

docs/open_source/setting_up/index.md

@@ -2,6 +2,35 @@
 
 This guide focuses primarily on configuring and using various LLM clients supported to run Giskard's LLM-assisted functionalities. We are using [LiteLLM](https://github.com/BerriAI/litellm) to handle the model calls, you can see the list of supported models in the [LiteLLM documentation](https://docs.litellm.ai/docs/providers).
 
+
+## Groq Client Setup
+
+More information on [Groq LiteLLM documentation](https://docs.litellm.ai/docs/providers/groq)
+
+### Setup using .env variables
+
+```python
+import os
+import giskard
+
+os.environ["GROQ_API_KEY"] = "" # "my-groq-api-key"
+
+# Optional, setup a model (default LLM is llama-3.3-70b-versatile)
+giskard.llm.set_llm_model("groq/llama-3.3-70b-versatile")
+
+# Note: Groq does not currently support embedding models
+# Use another provider for embeddings if needed
+```
+
+### Setup using completion params
+
+```python
+import giskard
+
+api_key = "" # "my-groq-api-key"
+giskard.llm.set_llm_model("groq/llama-3.3-70b-versatile", api_key=api_key)
+```
+
 ## OpenAI Client Setup
 
 More information on [OpenAI LiteLLM documentation](https://docs.litellm.ai/docs/providers/openai)

giskard/llm/client/__init__.py

@@ -36,12 +36,13 @@ def get_default_llm_api() -> str:
     global _default_llm_api
     if _default_llm_api is None:
         _default_llm_api = os.getenv(
-            "GSK_LLM_API", "azure" if "AZURE_OPENAI_API_KEY" in os.environ else "openai"
+            "GSK_LLM_API",
+            "azure" if "AZURE_OPENAI_API_KEY" in os.environ else "groq" if "GROQ_API_KEY" in os.environ else "openai",
         ).lower()
 
-        if _default_llm_api not in {"azure", "openai"}:
+        if _default_llm_api not in {"azure", "openai", "groq"}:
             logging.warning(
-                f"LLM-based evaluation is only working with `azure` and `openai`. Found {_default_llm_api} in GSK_LLM_API, falling back to `openai`"
+                f"LLM-based evaluation is only working with `azure`, `openai`, 'groq'. Found {_default_llm_api} in GSK_LLM_API, falling back to `openai`"
             )
             _default_llm_api = "openai"
 
@@ -90,9 +91,6 @@ def get_default_client() -> LLMClient:
     global _default_llm_model
     global _disable_structured_output
 
-    if _default_client is not None:
-        return _default_client
-
     try:
         from .litellm import LiteLLMClient
 

giskard/llm/client/groq_client.py

@@ -0,0 +1,97 @@
+from typing import Optional, Sequence
+
+import logging
+from dataclasses import asdict
+
+from ..config import LLMConfigurationError
+from ..errors import LLMImportError
+from . import LLMClient
+from .base import ChatMessage
+
+try:
+    import groq
+    from groq import Groq
+except ImportError as err:
+    raise LLMImportError(flavor="llm") from err
+
+AUTH_ERROR_MESSAGE = (
+    "Could not authenticate with Groq API. Please make sure you have configured the API key by "
+    "setting GROQ_API_KEY in the environment."
+)
+
+JSON_MODE_GUIDANCE = (
+    "To use JSON mode, make sure:\n"
+    "1. Pass format='json' or format='json_object' to the `complete()` method.\n"
+    "2. You describe the expected JSON structure clearly in the system prompt.\n"
+    "3. The selected model supports JSON output.\n"
+    "See: https://console.groq.com/docs/text-chat#json-mode"
+)
+
+logger = logging.getLogger(__name__)
+
+
+class GroqClient(LLMClient):
+    def __init__(
+        self,
+        model: str = "llama-3.3-70b-versatile",  # Default model for Groq
+        client: Groq = None,
+        # json_mode: Optional[bool] = None
+    ):
+        logger.info(f"Initializing GroqClient with model: {model}")
+        self.model = model
+        self._client = client or Groq()
+        logger.info("GroqClient initialized successfully")
+
+    def get_config(self) -> dict:
+        """Return the configuration of the LLM client."""
+        return {"client_type": self.__class__.__name__, "model": self.model}
+
+    def complete(
+        self,
+        messages: Sequence[ChatMessage],
+        temperature: float = 1.0,
+        max_tokens: Optional[int] = None,
+        caller_id: Optional[str] = None,
+        seed: Optional[int] = None,
+        format: Optional[str] = None,
+    ) -> ChatMessage:
+        logger.info(f"GroqClient.complete called with model: {self.model}")
+        logger.info(f"Messages: {messages}")
+
+        extra_params = dict()
+
+        extra_params["seed"] = seed
+
+        if format in {"json", "json_object"}:
+            extra_params["response_format"] = {"type": "json_object"}
+
+        try:
+            completion = self._client.chat.completions.create(
+                model=self.model,
+                messages=[asdict(m) for m in messages],
+                temperature=temperature,
+                max_tokens=max_tokens,
+                **extra_params,
+            )
+
+        except groq.AuthenticationError as err:
+            raise LLMConfigurationError(AUTH_ERROR_MESSAGE) from err
+
+        except groq.BadRequestError as err:
+            if format in {"json", "json_object"}:
+                raise LLMConfigurationError(
+                    f"Model '{self.model}' does not support JSON output or the request format is incorrect.\n\n{JSON_MODE_GUIDANCE}"
+                ) from err
+            raise
+
+        self.logger.log_call(
+            prompt_tokens=completion.usage.prompt_tokens,
+            sampled_tokens=completion.usage.completion_tokens,
+            model=self.model,
+            client_class=self.__class__.__name__,
+            caller_id=caller_id,
+        )
+
+        msg = completion.choices[0].message
+
+        return ChatMessage(role=msg.role, content=msg.content)