# Example alerting rules for Gurubase Siper. Thresholds are starting points; tune
# them to your traffic. Alerts are on symptoms (errors, slowness, unavailability).
groups:
  - name: gurubase
    rules:
      # Requests stopped by an API key's own limit are left out on both sides: they have
      # their own alert (GurubaseKeyLimitReached) and are not a failing service.
      - alert: GurubaseHighErrorRate
        expr: |
          sum(rate(gurubase_llm_requests_total{status!~"2xx|429_key_limit"}[5m]))
            / sum(rate(gurubase_llm_requests_total{status!="429_key_limit"}[5m])) > 0.05
          and sum(rate(gurubase_llm_requests_total{status!="429_key_limit"}[5m])) > 0.01
        for: 10m
        labels:
          severity: critical
        annotations:
          summary: More than 5% of gateway LLM requests are failing.

      # The model provider is out of capacity (HTTP 429 after the gateway's retries).
      - alert: GurubaseRateLimited
        expr: sum(rate(gurubase_llm_requests_total{status="429_provider"}[5m])) > 0
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: The model provider is rejecting gateway LLM requests as busy (HTTP 429).

      # Requests above an API key's own limit, which your organization set in the panel
      # (API Keys). Expected when a client sends bursts; raise the limit if it is too low.
      - alert: GurubaseKeyLimitReached
        expr: sum by (key_id) (rate(gurubase_llm_requests_total{status="429_key_limit"}[5m])) > 0
        for: 10m
        labels:
          severity: info
        annotations:
          summary: "API key {{ $labels.key_id }} is hitting its own request or token limit (HTTP 429)."

      - alert: GurubaseSlowResponses
        expr: |
          histogram_quantile(0.95,
            sum by (le) (rate(gurubase_llm_request_duration_seconds_bucket[5m]))) > 10
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: 95th percentile LLM response time is above 10 seconds.

      - alert: GurubaseDataStale
        expr: max(gurubase_last_ingest_age_seconds) > 300
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: Gurubase usage data has not been refreshed for more than 5 minutes.

      - alert: GurubaseMetricsUnreachable
        expr: up{job="gurubase"} == 0
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: Prometheus cannot read the Gurubase metrics endpoint.

      - alert: GurubaseUnavailable
        expr: probe_success{job=~"gurubase-health|gurubase-mask-probe"} == 0
        for: 3m
        labels:
          severity: critical
        annotations:
          summary: "Gurubase availability probe is failing ({{ $labels.job }})."