# AIO Recovery — Alert Definitions (W2-C)
#
# These alerts cover the five observability gates required by §13.2 of
# docs/plan-ai-overview-click-recovery.md and verified by Wave 5-F.
#
# Source of thresholds: config/aio_recovery_monitoring.php (tunable via env).
# Datadog monitor API: POST /api/v1/monitor
#
# Validation in staging:
#   1. Run a synthetic AioRecoveryRun that produces 0 items (cliff trigger).
#   2. Force-fail RunAioRecoveryJob via `php artisan queue:fail --queue=background`
#      and verify the failure-rate monitor fires within `window_minutes`.
#   3. Seed platform_serp_spend.usd_spent above the warn threshold and confirm
#      the SERP spend monitor fires on the next 5-min evaluation.
#   4. Enqueue >200 synthetic background jobs and confirm the backpressure
#      monitor fires.
#
# See monitoring/aio_recovery_dashboard.json for the companion dashboard tiles.
#
# IMPORTANT: Credentials (DD_API_KEY, DD_APP_KEY) are stored in the secrets
# vault — never committed. Apply with: ./deploy/apply-monitoring.sh

version: "1"

alerts:

  # ── 1. Detection-Volume Anomaly ────────────────────────────────────────────
  #
  # A sudden spike or cliff in pages_flagged per run is the leading signal for
  # false-positive precision regression or a silent detector failure.
  #
  # Log source: aio_recovery.run.completed → pages_flagged (integer field)
  # Metric:     rankwiz.aio_recovery.pages_flagged (gauge, emitted by run)
  # Datadog monitor type: metric alert

  - name: "AIO Recovery — Detection Volume Anomaly"
    type: metric_alert
    query: |
      anomalies(avg:rankwiz.aio_recovery.pages_flagged{*}.rollup(avg, 3600), 'agile', 3)
    message: |
      **AIO Recovery: detection volume anomaly detected.**

      pages_flagged per run deviates by more than 3σ from the 7-day rolling average.
      This may indicate:
        - Precision regression (spike): review `EXPLAIN` on the coarse SQL filter and
          confirm `DeclineCauseClassifier` is still the sole authority (§12.1).
        - Silent detector failure (cliff): confirm `RunAioRecoveryJob` completed runs and
          check for GSC data staleness.

      Runbook: docs/plan-ai-overview-click-recovery.md §13.2
      @pagerduty-rankwiz-oncall
    thresholds:
      critical: 1
      warning: 0.75
    options:
      notify_no_data: true
      no_data_timeframe: 120        # 2-hour window; detection jobs run hourly per site
      evaluation_delay: 300
      renotify_interval: 60
      include_tags: true
      tags:
        - "service:aio-recovery"
        - "alert:detection-volume"
        - "env:production"
    # Staging synthetic test:
    #   php artisan tinker --execute="App\Models\AioRecoveryRun::create([...]) + items = 0"
    #   Expect: alert fires within 5 minutes of metric evaluation.

  # ── 2. Job Failure-Rate (RunAioRecoveryJob) ────────────────────────────────
  #
  # A sustained failure rate above 20% in a 60-min window indicates a systemic
  # issue (circuit breaker open, GSC API outage, schema drift).
  #
  # Log source: level=ERROR, job class = RunAioRecoveryJob
  # Metric:     rankwiz.aio_recovery.job_failure_rate (gauge, 0.0–1.0)
  # Datadog monitor type: metric alert (log-derived custom metric)

  - name: "AIO Recovery — Job Failure Rate"
    type: metric_alert
    query: |
      avg(last_1h):avg:rankwiz.aio_recovery.job_failure_rate{*} > 0.20
    message: |
      **AIO Recovery: RunAioRecoveryJob failure rate exceeded 20% in the last hour.**

      More than 1 in 5 recovery detection jobs failed. Check:
        - Horizon dashboard for `background` queue errors.
        - Sentry for `RunAioRecoveryJob` exception grouping.
        - GSC connection health (`php artisan gsc:health-check`).
        - Circuit breaker status (`php artisan circuit-breaker:status`).

      Runbook: docs/plan-ai-overview-click-recovery.md §13.2
      @pagerduty-rankwiz-oncall
    thresholds:
      critical: 0.20
      warning: 0.10
    options:
      notify_no_data: false          # zero failures = zero rate = no data = OK
      evaluation_delay: 60
      renotify_interval: 30
      include_tags: true
      tags:
        - "service:aio-recovery"
        - "alert:job-failure-rate"
        - "env:production"
    # Staging synthetic test:
    #   php artisan queue:fail --queue=background (force 3 out of 5 jobs failed)
    #   Confirm Sentry captures RunAioRecoveryJob exception with site_id tag.

  # ── 3. SERP Spend Alert ────────────────────────────────────────────────────
  #
  # AIO SERP confirmation routes through PlatformSerpBudgetService. These alerts
  # fire before the hard $25/mo cap is reached so the operator can act.
  #
  # Metric: rankwiz.serp.platform_spend_mtd_usd (gauge, from platform_serp_spend table)
  # Log key: serp.platform_fetch.budget_blocked_in_lock (hard cap hit)

  - name: "AIO Recovery — SERP Platform Spend Warning"
    type: metric_alert
    query: |
      avg(last_5m):max:rankwiz.serp.platform_spend_mtd_usd{*} > 20.0
    message: |
      **SERP platform spend reached ${{value}} MTD — above the $20 warning threshold.**

      The hard cap is $25/mo. AIO recovery SERP confirmation will be blocked at $25.
      Consider:
        - Reducing AIO SERP confirmation sample size (config/aio_recovery.php).
        - Checking for unexpected SERP confirmation dispatch loops.
        - Verifying free-tier users are correctly skipping SERP confirmation (paid-only).

      Runbook: docs/plan-ai-overview-click-recovery.md §14.2
      @slack-rankwiz-alerts
    thresholds:
      critical: 23.75
      warning: 20.0
    options:
      notify_no_data: false
      evaluation_delay: 300
      renotify_interval: 1440        # once per day; spend grows monotonically MTD
      include_tags: true
      tags:
        - "service:aio-recovery"
        - "service:serp"
        - "alert:spend"
        - "env:production"
    # Staging synthetic test:
    #   DB::table('platform_serp_spend')->updateOrInsert(['month' => now()->format('Y-m')], ['usd_spent' => 21.00])
    #   Confirm Datadog metric rankwiz.serp.platform_spend_mtd_usd = 21.00
    #   Expect: warning fires on next 5-min evaluation.

  # ── 4. OpenAI / Bundled AI Spend Alert ────────────────────────────────────
  #
  # Recovery draft generation uses BYOK keys (no platform spend), but when
  # bundled AI is active, this watches MonitorBundledAiCostJob's thresholds.
  #
  # Log key: bundled_ai_cost.hourly_threshold_exceeded (warning)
  #          bundled_ai_cost.daily_threshold_exceeded  (critical)
  # Metric:  rankwiz.ai.bundled_cost_daily_usd (gauge, from MonitorBundledAiCostJob)

  - name: "AIO Recovery — OpenAI Bundled AI Daily Spend"
    type: metric_alert
    query: |
      avg(last_5m):max:rankwiz.ai.bundled_cost_daily_usd{*} > 25.0
    message: |
      **Bundled AI daily spend ${{value}} exceeded the $25/day critical threshold.**

      AIO recovery batch draft generation may be driving elevated spend.
      IMMEDIATE ACTION: Set FEATURE_BUNDLED_AI=false to engage the kill switch.

      Recovery draft generation only uses BYOK keys when the user has a key configured;
      bundled AI is used as a fallback. Check BatchCostEstimator usage for recovery batches.

      Runbook: docs/plan-ai-overview-click-recovery.md §11.2
      Emergency kill switch: docs/plan-ai-overview-click-recovery.md §13.1
      @pagerduty-rankwiz-oncall
    thresholds:
      critical: 25.0
      warning: 5.0            # $5/hr hourly warning mirrors MonitorBundledAiCostJob
    options:
      notify_no_data: false
      evaluation_delay: 120
      renotify_interval: 60
      include_tags: true
      tags:
        - "service:aio-recovery"
        - "service:openai"
        - "alert:spend"
        - "env:production"
    # Staging synthetic test:
    #   php artisan tinker → MonitorBundledAiCostJob::dispatch() after seeding
    #   ai_drafts with bundled ai_provider_type rows totalling > $25 token cost.

  # ── 5. Background Queue Backpressure ──────────────────────────────────────
  #
  # RunAioRecoveryJob runs on the 'background' queue. When depth > threshold,
  # new detection runs are deferred by LimitService::isQueueBacklogged('background').
  #
  # Metric: rankwiz.queue.background_depth (gauge, from Horizon / queue:size)
  # Log key: queue.backpressure.background (logged when a run is shed)

  - name: "AIO Recovery — Background Queue Backpressure"
    type: metric_alert
    query: |
      avg(last_5m):max:rankwiz.queue.background_depth{*} > 200
    message: |
      **Background queue depth {{value}} exceeded 200 — AIO recovery runs are being shed.**

      `LimitService::isQueueBacklogged('background')` returns true above this depth.
      New AIO recovery detection runs will not be dispatched until the backlog clears.

      Investigate:
        - Horizon dashboard for stuck/slow jobs on the `background` queue.
        - Worker count: `ps aux | grep queue:work` on the production server.
        - Recent large job dispatches (cannibalization, freshness, aio-recovery).

      Runbook: docs/plan-ai-overview-click-recovery.md §13.2
      @slack-rankwiz-alerts
    thresholds:
      critical: 500
      warning: 200
    options:
      notify_no_data: false
      evaluation_delay: 60
      renotify_interval: 30
      include_tags: true
      tags:
        - "service:aio-recovery"
        - "service:queue"
        - "alert:backpressure"
        - "env:production"
    # Staging synthetic test:
    #   php artisan tinker → Artisan::call('queue:push', [...]) × 201 jobs on 'background'
    #   Confirm Horizon reports 200+ pending, Datadog metric crosses 200.
    #   Expect: warning fires on next 5-min evaluation.
