$ python distill.py

Provider List: https://docs.litellm.ai/docs/providers


Provider List: https://docs.litellm.ai/docs/providers

Traceback (most recent call last):
  File "/Users/corey.zumar/mlflowrepos/mlflow3/distill.py", line 14, in <module>
    df = generate_seed_conversations(
  File "/Users/corey.zumar/mlflowrepos/mlflow3/mlflow/genai/simulators/distillation.py", line 94, in generate_seed_conversations
    results = [_distill_goal_and_persona(session, model=model) for session in sessions]
  File "/Users/corey.zumar/mlflowrepos/mlflow3/mlflow/genai/simulators/distillation.py", line 94, in <listcomp>
    results = [_distill_goal_and_persona(session, model=model) for session in sessions]
  File "/Users/corey.zumar/mlflowrepos/mlflow3/mlflow/genai/simulators/distillation.py", line 36, in _distill_goal_and_persona
    response = invoke_model_without_tracing(
  File "/Users/corey.zumar/mlflowrepos/mlflow3/mlflow/genai/simulators/utils.py", line 90, in invoke_model_without_tracing
    response = litellm.completion(**kwargs)
  File "/Users/corey.zumar/miniconda3/envs/mlflow/lib/python3.10/site-packages/litellm/utils.py", line 1739, in wrapper
    raise e
  File "/Users/corey.zumar/miniconda3/envs/mlflow/lib/python3.10/site-packages/litellm/utils.py", line 1560, in wrapper
    result = original_function(*args, **kwargs)
  File "/Users/corey.zumar/miniconda3/envs/mlflow/lib/python3.10/site-packages/litellm/main.py", line 4205, in completion
    raise exception_type(
  File "/Users/corey.zumar/miniconda3/envs/mlflow/lib/python3.10/site-packages/litellm/main.py", line 1289, in completion
    model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider(
  File "/Users/corey.zumar/miniconda3/envs/mlflow/lib/python3.10/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 476, in get_llm_provider
    raise e
  File "/Users/corey.zumar/miniconda3/envs/mlflow/lib/python3.10/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 453, in get_llm_provider
    raise litellm.exceptions.BadRequestError(  # type: ignore
litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=gateway/corey-gemini-flash
 Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers

# Define jarvis agent's predict function
def jarvis_agent(input: list[dict], **kwargs) -> dict:
    """
    Jarvis agent that uses process_query to handle conversations.

    Args:
        input: List of conversation messages in OpenAI format

    Returns:
        Dict with the assistant's response
    """
    # Create a thread for this conversation
    # (I can't really do anything else, since I'm not sure when a new conversation
    # should actually start / end <- THIS IS THE PROBLEM)
    thread = create_thread(include_briefing=True)

    # Get the last user message
    last_message = input[-1]
    query = "Hey Jarvis, " + last_message["content"]

    # Process the query using jarvis
    response = process_query(query=query, thread=thread)

    # Return in the format expected by the simulator
    return {
        "choices": [{
            "message": {
                "role": "assistant",
                "content": response.text or "I apologize, but I couldn't process that request."
            }
        }]
    }

test_cases = [{'goal': 'Provide the current major stock market index levels and their percentage change, using the most recent trading session when the market is closed.', 'persona': 'You are a frustrated, impatient user seeking quick market performance information; you communicate in short, emphatic all-caps demands, have basic financial context (e.g., knowing markets are closed on weekends), and focus on percent moves rather than raw index values.'}]


custom_scorer = mlflow.genai.make_judge(
    name="correct_terminology",
    instructions="Read the {{ conversation }} and evaluate whether the assistant used correct financial terminology in its responses.",
    feedback_value_type=bool,
    model="openai:/gpt-5-mini",
)

# Create simulator
simulator = ConversationSimulator(
    test_cases=test_cases,
    max_turns=8,  # Maximum conversation length
    user_model="openai:/gpt-5-mini",
)

# Evaluate - the simulator generates conversations and evaluates them
results = mlflow.genai.evaluate(
    data=simulator,
    predict_fn=jarvis_agent,
    scorers=[
        custom_scorer,
        ConversationCompleteness(),
        UserFrustration(),
    ],
)

[{'goal': 'Provide the current major stock market index levels and their percentage change, using the most recent trading session when the market is closed.', 'persona': 'You are a frustrated, impatient user seeking quick market performance information; you communicate in short, emphatic all-caps demands, have basic financial context (e.g., knowing markets are closed on weekends), and focus on percent moves rather than raw index values.'}]

test_cases = [{'goal': 'Provide the current major stock market index levels. If markets are closed, expect the agent to provide information for the most recent trading day without you needing to ask (DO *NOT* ASK EXPLICITLY)', 'persona': 'You are a frustrated, impatient user seeking quick market performance information; you communicate in short, emphatic all-caps demands, have basic financial context (e.g., knowing markets are closed on weekends), and focus on percent moves rather than raw index values.'}]

{
    "expectations": [
        "If markets are closed, the agent should indicate this and provide information for the most recent trading day"
    ]
}