src/system_tests/profiling/config/default_experiment.yaml

# Default Profiling Experiment Configuration
# Override these settings by creating your own YAML file

profiling:
  # Available configurations to test
  configs:
    - "settings.openai.toml"
    # - "settings.azure.toml"
    # - "settings.watsonx.toml"
  
  # Available modes to test
  modes:
    - "fast"
    - "balanced"
    # - "accurate"
  
  # Test tasks to run
  tasks:
    - "test_get_top_account_by_revenue_stream"
    # - "test_list_my_accounts"
    # - "test_find_vp_sales_active_high_value_accounts"
  
  # Number of runs per configuration
  runs: 3
  
  # Output settings
  output:
    directory: "reports"
    filename_prefix: "profiling_report"
    timestamp_format: "%Y%m%d_%H%M%S"
  
  # Langfuse settings (can be overridden by environment variables)
  langfuse:
    host: "https://cloud.langfuse.com"
    # public_key and secret_key should be set via environment variables:
    # LANGFUSE_PUBLIC_KEY and LANGFUSE_SECRET_KEY
    
    # Retry settings for fetching trace data
    # Langfuse data takes time to propagate, so we retry with exponential backoff
    retry:
      max_attempts: 10
      initial_delay: 2.0
      max_delay: 30.0

# Experiment configuration for comparing different modes
experiment:
  name: "fast_vs_balanced"
  description: "Compare fast and balanced modes"
  
  # Runs to perform
  runs:
    - name: "fast_mode"
      test_id: "settings.openai.toml:fast:test_get_top_account_by_revenue_stream"
      iterations: 3
      output: "experiments/fast_{{timestamp}}.json"
      # Optional: define environment variables for this run
      # env:
      #   MODEL_NAME: "Azure/gpt-4o"
    
    - name: "balanced_mode"
      test_id: "settings.openai.toml:balanced:test_get_top_account_by_revenue_stream"
      iterations: 3
      output: "experiments/balanced_{{timestamp}}.json"
      # Optional: define environment variables for this run
      # env:
      #   MODEL_NAME: null  # Use null to unset a variable
  
  # Comparison settings
  comparison:
    generate_html: true
    html_output: "experiments/comparison.html"
    auto_open: false