-
-
Notifications
You must be signed in to change notification settings - Fork 338
Add Groq model support to LLMClient (#1977) #2165
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 11 commits
930099f
10a4b68
97ac6c6
b9d3f01
97dc016
1659045
3bc5b74
4f27293
ef817bf
3a6362a
ff02a9e
8af4202
a70fbb6
1fb5174
7c86562
ff8d141
c0f3a5b
9ad79ec
92981aa
7c5b277
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I don't think we should set There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Thank you for the feedback. Before I make any changes, I want to ensure I understand your request correctly:
Is this understanding correct? If you'd prefer I completely remove all Groq-related changes, please let me know. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Hey @kunjanshah0811, yes exactly! There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
@henchaves I have made suggested changes with the recent commit. Please have a look and let me know. Thanks a lot 🚀. |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,97 @@ | ||
from typing import Optional, Sequence | ||
|
||
import logging | ||
from dataclasses import asdict | ||
|
||
from ..config import LLMConfigurationError | ||
from ..errors import LLMImportError | ||
from . import LLMClient | ||
from .base import ChatMessage | ||
|
||
try: | ||
import groq | ||
from groq import Groq | ||
except ImportError as err: | ||
raise LLMImportError(flavor="llm") from err | ||
|
||
AUTH_ERROR_MESSAGE = ( | ||
"Could not authenticate with Groq API. Please make sure you have configured the API key by " | ||
"setting GROQ_API_KEY in the environment." | ||
) | ||
|
||
JSON_MODE_GUIDANCE = ( | ||
"To use JSON mode, make sure:\n" | ||
"1. Pass format='json' or format='json_object' to the `complete()` method.\n" | ||
"2. You describe the expected JSON structure clearly in the system prompt.\n" | ||
"3. The selected model supports JSON output.\n" | ||
"See: https://console.groq.com/docs/text-chat#json-mode" | ||
) | ||
|
||
logger = logging.getLogger(__name__) | ||
|
||
|
||
class GroqClient(LLMClient): | ||
def __init__( | ||
self, | ||
model: str = "llama-3.3-70b-versatile", # Default model for Groq | ||
davidberenstein1957 marked this conversation as resolved.
Show resolved
Hide resolved
|
||
client: Groq = None, | ||
# json_mode: Optional[bool] = None | ||
): | ||
logger.info(f"Initializing GroqClient with model: {model}") | ||
self.model = model | ||
self._client = client or Groq() | ||
logger.info("GroqClient initialized successfully") | ||
|
||
def get_config(self) -> dict: | ||
"""Return the configuration of the LLM client.""" | ||
return {"client_type": self.__class__.__name__, "model": self.model} | ||
|
||
def complete( | ||
self, | ||
messages: Sequence[ChatMessage], | ||
temperature: float = 1.0, | ||
max_tokens: Optional[int] = None, | ||
caller_id: Optional[str] = None, | ||
seed: Optional[int] = None, | ||
format: Optional[str] = None, | ||
) -> ChatMessage: | ||
logger.info(f"GroqClient.complete called with model: {self.model}") | ||
logger.info(f"Messages: {messages}") | ||
|
||
extra_params = dict() | ||
|
||
extra_params["seed"] = seed | ||
|
||
if format in {"json", "json_object"}: | ||
extra_params["response_format"] = {"type": "json_object"} | ||
|
||
try: | ||
completion = self._client.chat.completions.create( | ||
model=self.model, | ||
messages=[asdict(m) for m in messages], | ||
temperature=temperature, | ||
max_tokens=max_tokens, | ||
**extra_params, | ||
) | ||
|
||
except groq.AuthenticationError as err: | ||
raise LLMConfigurationError(AUTH_ERROR_MESSAGE) from err | ||
|
||
except groq.BadRequestError as err: | ||
if format in {"json", "json_object"}: | ||
raise LLMConfigurationError( | ||
f"Model '{self.model}' does not support JSON output or the request format is incorrect.\n\n{JSON_MODE_GUIDANCE}" | ||
) from err | ||
raise | ||
|
||
self.logger.log_call( | ||
prompt_tokens=completion.usage.prompt_tokens, | ||
sampled_tokens=completion.usage.completion_tokens, | ||
model=self.model, | ||
client_class=self.__class__.__name__, | ||
caller_id=caller_id, | ||
) | ||
|
||
msg = completion.choices[0].message | ||
|
||
return ChatMessage(role=msg.role, content=msg.content) |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Perfect! Thanks.
Could you just mention to access https://docs.litellm.ai/docs/embedding/supported_embedding to see the full list of supported providers?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
docs/open_source/setting_up/index.md
linking to LiteLLM's supported embedding providersThere was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
yes, thanks!