{
  "$schema": "https://docs.datadoghq.com/api/latest/dashboards/",
  "title": "AIO Click Recovery — Observability (W2-C)",
  "description": "Detection volume, recoverability buckets, SERP/OpenAI spend, ROI realized. GA gate: all tiles green before launch. See docs/plan-ai-overview-click-recovery.md §13.2 and §4.",
  "layout_type": "ordered",
  "tags": ["service:aio-recovery", "team:rankwiz"],
  "template_variables": [
    {
      "name": "env",
      "prefix": "env",
      "default": "production"
    }
  ],
  "widgets": [

    {
      "id": 1,
      "definition": {
        "type": "group",
        "title": "Detection Health",
        "layout_type": "ordered",
        "widgets": [

          {
            "id": 101,
            "definition": {
              "type": "timeseries",
              "title": "Pages Flagged per Run",
              "title_size": "16",
              "title_align": "left",
              "show_legend": true,
              "legend_layout": "horizontal",
              "requests": [
                {
                  "q": "avg:rankwiz.aio_recovery.pages_flagged{$env}",
                  "display_type": "bars",
                  "style": {
                    "palette": "dog_classic",
                    "line_type": "solid",
                    "line_width": "normal"
                  },
                  "metadata": [
                    {
                      "expression": "avg:rankwiz.aio_recovery.pages_flagged{$env}",
                      "alias_name": "Pages Flagged"
                    }
                  ]
                },
                {
                  "q": "week_before(avg:rankwiz.aio_recovery.pages_flagged{$env})",
                  "display_type": "line",
                  "style": {
                    "palette": "gray",
                    "line_type": "dashed",
                    "line_width": "thin"
                  },
                  "metadata": [
                    {
                      "expression": "week_before(avg:rankwiz.aio_recovery.pages_flagged{$env})",
                      "alias_name": "Prior Week"
                    }
                  ]
                }
              ],
              "yaxis": {
                "min": "0"
              },
              "markers": [
                {
                  "value": "y > 0",
                  "display_type": "ok dashed",
                  "label": "Normal range (±3σ computed by anomaly monitor)"
                }
              ],
              "time": {
                "live_span": "1w"
              }
            },
            "layout": {
              "x": 0, "y": 0, "width": 6, "height": 3
            }
          },

          {
            "id": 102,
            "definition": {
              "type": "query_value",
              "title": "Completed Runs (Last 24h)",
              "title_size": "16",
              "title_align": "center",
              "requests": [
                {
                  "q": "sum:rankwiz.aio_recovery.runs_completed{$env}.as_count()",
                  "aggregator": "sum",
                  "conditional_formats": [
                    {
                      "comparator": ">",
                      "value": 0,
                      "palette": "white_on_green"
                    },
                    {
                      "comparator": "<=",
                      "value": 0,
                      "palette": "white_on_yellow"
                    }
                  ]
                }
              ],
              "time": {
                "live_span": "1d"
              },
              "precision": 0,
              "text_align": "center"
            },
            "layout": {
              "x": 6, "y": 0, "width": 3, "height": 2
            }
          },

          {
            "id": 103,
            "definition": {
              "type": "query_value",
              "title": "Failed Runs (Last 24h)",
              "title_size": "16",
              "title_align": "center",
              "requests": [
                {
                  "q": "sum:rankwiz.aio_recovery.runs_failed{$env}.as_count()",
                  "aggregator": "sum",
                  "conditional_formats": [
                    {
                      "comparator": ">",
                      "value": 0,
                      "palette": "white_on_red"
                    },
                    {
                      "comparator": "<=",
                      "value": 0,
                      "palette": "white_on_green"
                    }
                  ]
                }
              ],
              "time": {
                "live_span": "1d"
              },
              "precision": 0,
              "text_align": "center"
            },
            "layout": {
              "x": 9, "y": 0, "width": 3, "height": 2
            }
          }
        ]
      }
    },

    {
      "id": 2,
      "definition": {
        "type": "group",
        "title": "Items by Recoverability Bucket",
        "layout_type": "ordered",
        "widgets": [

          {
            "id": 201,
            "definition": {
              "type": "timeseries",
              "title": "Detection Items — Recoverability Breakdown",
              "title_size": "16",
              "title_align": "left",
              "show_legend": true,
              "legend_layout": "horizontal",
              "requests": [
                {
                  "q": "avg:rankwiz.aio_recovery.items_by_bucket{$env,bucket:high}",
                  "display_type": "bars",
                  "style": {
                    "palette": "purple",
                    "line_type": "solid"
                  },
                  "metadata": [
                    {
                      "expression": "avg:rankwiz.aio_recovery.items_by_bucket{$env,bucket:high}",
                      "alias_name": "High Recoverability"
                    }
                  ]
                },
                {
                  "q": "avg:rankwiz.aio_recovery.items_by_bucket{$env,bucket:medium}",
                  "display_type": "bars",
                  "style": {
                    "palette": "cool",
                    "line_type": "solid"
                  },
                  "metadata": [
                    {
                      "expression": "avg:rankwiz.aio_recovery.items_by_bucket{$env,bucket:medium}",
                      "alias_name": "Medium Recoverability"
                    }
                  ]
                },
                {
                  "q": "avg:rankwiz.aio_recovery.items_by_bucket{$env,bucket:low}",
                  "display_type": "bars",
                  "style": {
                    "palette": "warm",
                    "line_type": "solid"
                  },
                  "metadata": [
                    {
                      "expression": "avg:rankwiz.aio_recovery.items_by_bucket{$env,bucket:low}",
                      "alias_name": "Low Recoverability"
                    }
                  ]
                }
              ],
              "yaxis": {
                "min": "0"
              },
              "time": {
                "live_span": "2w"
              }
            },
            "layout": {
              "x": 0, "y": 0, "width": 8, "height": 3
            }
          },

          {
            "id": 202,
            "definition": {
              "type": "toplist",
              "title": "Top Sites by Est. Clicks Recoverable",
              "title_size": "16",
              "title_align": "left",
              "requests": [
                {
                  "q": "top(sum:rankwiz.aio_recovery.est_clicks_recoverable{$env} by {site_id}, 10, 'sum', 'desc')",
                  "style": {
                    "palette": "dog_classic"
                  }
                }
              ],
              "time": {
                "live_span": "1w"
              }
            },
            "layout": {
              "x": 8, "y": 0, "width": 4, "height": 3
            }
          }
        ]
      }
    },

    {
      "id": 3,
      "definition": {
        "type": "group",
        "title": "Spend — SERP + OpenAI",
        "layout_type": "ordered",
        "widgets": [

          {
            "id": 301,
            "definition": {
              "type": "timeseries",
              "title": "SERP Platform Spend — MTD (USD)",
              "title_size": "16",
              "title_align": "left",
              "show_legend": false,
              "requests": [
                {
                  "q": "max:rankwiz.serp.platform_spend_mtd_usd{$env}",
                  "display_type": "line",
                  "style": {
                    "palette": "warm",
                    "line_type": "solid",
                    "line_width": "thick"
                  },
                  "metadata": [
                    {
                      "expression": "max:rankwiz.serp.platform_spend_mtd_usd{$env}",
                      "alias_name": "SERP MTD Spend (USD)"
                    }
                  ]
                }
              ],
              "yaxis": {
                "min": "0",
                "max": "25"
              },
              "markers": [
                {
                  "value": "y = 20",
                  "display_type": "warning dashed",
                  "label": "Warn ($20)"
                },
                {
                  "value": "y = 23.75",
                  "display_type": "error dashed",
                  "label": "Critical ($23.75)"
                },
                {
                  "value": "y = 25",
                  "display_type": "error solid",
                  "label": "Hard Cap ($25)"
                }
              ],
              "time": {
                "live_span": "1mo"
              }
            },
            "layout": {
              "x": 0, "y": 0, "width": 6, "height": 3
            }
          },

          {
            "id": 302,
            "definition": {
              "type": "timeseries",
              "title": "OpenAI Bundled AI Cost — Daily (USD)",
              "title_size": "16",
              "title_align": "left",
              "show_legend": false,
              "requests": [
                {
                  "q": "max:rankwiz.ai.bundled_cost_daily_usd{$env}",
                  "display_type": "line",
                  "style": {
                    "palette": "purple",
                    "line_type": "solid",
                    "line_width": "thick"
                  },
                  "metadata": [
                    {
                      "expression": "max:rankwiz.ai.bundled_cost_daily_usd{$env}",
                      "alias_name": "AI Daily Cost (USD)"
                    }
                  ]
                }
              ],
              "yaxis": {
                "min": "0"
              },
              "markers": [
                {
                  "value": "y = 5",
                  "display_type": "warning dashed",
                  "label": "Hourly warn ($5/hr → daily proxy)"
                },
                {
                  "value": "y = 25",
                  "display_type": "error solid",
                  "label": "Daily Critical ($25)"
                }
              ],
              "time": {
                "live_span": "1w"
              }
            },
            "layout": {
              "x": 6, "y": 0, "width": 6, "height": 3
            }
          }
        ]
      }
    },

    {
      "id": 4,
      "definition": {
        "type": "group",
        "title": "ROI Realized (v1-ready)",
        "description": "Shows measured clicks_delta from completed AIO recovery recommendations. Data populates after Wave 3 (run-linkage) and Wave 4 (ROI surfacing) ship. Tiles render with zero values until then — this is intentional. Copy: 'observed change', not 'caused'.",
        "layout_type": "ordered",
        "widgets": [

          {
            "id": 401,
            "definition": {
              "type": "query_value",
              "title": "Total Clicks Recovered (Observed, MTD)",
              "title_size": "16",
              "title_align": "center",
              "requests": [
                {
                  "q": "sum:rankwiz.aio_recovery.clicks_recovered_mtd{$env}",
                  "aggregator": "last",
                  "conditional_formats": [
                    {
                      "comparator": ">",
                      "value": 0,
                      "palette": "white_on_green"
                    },
                    {
                      "comparator": "<=",
                      "value": 0,
                      "palette": "white_on_gray"
                    }
                  ]
                }
              ],
              "custom_unit": "clicks",
              "time": {
                "live_span": "1mo"
              },
              "precision": 0,
              "text_align": "center"
            },
            "layout": {
              "x": 0, "y": 0, "width": 4, "height": 2
            }
          },

          {
            "id": 402,
            "definition": {
              "type": "query_value",
              "title": "Active Recovery Items (All Sites)",
              "title_size": "16",
              "title_align": "center",
              "requests": [
                {
                  "q": "sum:rankwiz.aio_recovery.items_active{$env}",
                  "aggregator": "last",
                  "conditional_formats": [
                    {
                      "comparator": ">",
                      "value": 0,
                      "palette": "white_on_green"
                    },
                    {
                      "comparator": "<=",
                      "value": 0,
                      "palette": "white_on_gray"
                    }
                  ]
                }
              ],
              "time": {
                "live_span": "1d"
              },
              "precision": 0,
              "text_align": "center"
            },
            "layout": {
              "x": 4, "y": 0, "width": 4, "height": 2
            }
          },

          {
            "id": 403,
            "definition": {
              "type": "timeseries",
              "title": "analytics.aio_recovery_* Event Volume",
              "title_size": "16",
              "title_align": "left",
              "show_legend": true,
              "legend_layout": "horizontal",
              "requests": [
                {
                  "q": "sum:rankwiz.analytics.aio_recovery_detected{$env}.as_count()",
                  "display_type": "bars",
                  "metadata": [{"expression": "sum:rankwiz.analytics.aio_recovery_detected{$env}.as_count()", "alias_name": "Detected"}]
                },
                {
                  "q": "sum:rankwiz.analytics.aio_recovery_viewed{$env}.as_count()",
                  "display_type": "bars",
                  "metadata": [{"expression": "sum:rankwiz.analytics.aio_recovery_viewed{$env}.as_count()", "alias_name": "Viewed"}]
                },
                {
                  "q": "sum:rankwiz.analytics.aio_recovery_draft_generated{$env}.as_count()",
                  "display_type": "bars",
                  "metadata": [{"expression": "sum:rankwiz.analytics.aio_recovery_draft_generated{$env}.as_count()", "alias_name": "Draft Generated"}]
                },
                {
                  "q": "sum:rankwiz.analytics.aio_recovery_applied{$env}.as_count()",
                  "display_type": "bars",
                  "metadata": [{"expression": "sum:rankwiz.analytics.aio_recovery_applied{$env}.as_count()", "alias_name": "Applied"}]
                },
                {
                  "q": "sum:rankwiz.analytics.aio_recovery_realized{$env}.as_count()",
                  "display_type": "bars",
                  "metadata": [{"expression": "sum:rankwiz.analytics.aio_recovery_realized{$env}.as_count()", "alias_name": "ROI Realized"}]
                }
              ],
              "yaxis": {
                "min": "0"
              },
              "time": {
                "live_span": "2w"
              }
            },
            "layout": {
              "x": 8, "y": 0, "width": 4, "height": 3
            }
          }
        ]
      }
    },

    {
      "id": 5,
      "definition": {
        "type": "group",
        "title": "Queue Health",
        "layout_type": "ordered",
        "widgets": [

          {
            "id": 501,
            "definition": {
              "type": "timeseries",
              "title": "Background Queue Depth",
              "title_size": "16",
              "title_align": "left",
              "show_legend": false,
              "requests": [
                {
                  "q": "max:rankwiz.queue.background_depth{$env}",
                  "display_type": "line",
                  "style": {
                    "palette": "dog_classic",
                    "line_type": "solid",
                    "line_width": "normal"
                  },
                  "metadata": [
                    {
                      "expression": "max:rankwiz.queue.background_depth{$env}",
                      "alias_name": "Background Queue Depth"
                    }
                  ]
                }
              ],
              "markers": [
                {
                  "value": "y = 200",
                  "display_type": "warning dashed",
                  "label": "Backpressure warn (200)"
                },
                {
                  "value": "y = 500",
                  "display_type": "error dashed",
                  "label": "Critical (500)"
                }
              ],
              "time": {
                "live_span": "4h"
              }
            },
            "layout": {
              "x": 0, "y": 0, "width": 6, "height": 3
            }
          },

          {
            "id": 502,
            "definition": {
              "type": "timeseries",
              "title": "RunAioRecoveryJob — Throughput",
              "title_size": "16",
              "title_align": "left",
              "show_legend": true,
              "legend_layout": "horizontal",
              "requests": [
                {
                  "q": "sum:rankwiz.aio_recovery.runs_completed{$env}.as_rate()",
                  "display_type": "bars",
                  "style": {"palette": "cool"},
                  "metadata": [{"expression": "sum:rankwiz.aio_recovery.runs_completed{$env}.as_rate()", "alias_name": "Completed / min"}]
                },
                {
                  "q": "sum:rankwiz.aio_recovery.runs_failed{$env}.as_rate()",
                  "display_type": "bars",
                  "style": {"palette": "warm"},
                  "metadata": [{"expression": "sum:rankwiz.aio_recovery.runs_failed{$env}.as_rate()", "alias_name": "Failed / min"}]
                }
              ],
              "yaxis": {
                "min": "0"
              },
              "time": {
                "live_span": "4h"
              }
            },
            "layout": {
              "x": 6, "y": 0, "width": 6, "height": 3
            }
          }
        ]
      }
    }
  ],

  "_notes": {
    "custom_metrics_required": [
      "rankwiz.aio_recovery.pages_flagged",
      "rankwiz.aio_recovery.runs_completed",
      "rankwiz.aio_recovery.runs_failed",
      "rankwiz.aio_recovery.items_by_bucket (tag: bucket:high|medium|low)",
      "rankwiz.aio_recovery.est_clicks_recoverable (tag: site_id)",
      "rankwiz.serp.platform_spend_mtd_usd",
      "rankwiz.ai.bundled_cost_daily_usd",
      "rankwiz.aio_recovery.clicks_recovered_mtd",
      "rankwiz.aio_recovery.items_active",
      "rankwiz.analytics.aio_recovery_detected",
      "rankwiz.analytics.aio_recovery_viewed",
      "rankwiz.analytics.aio_recovery_draft_generated",
      "rankwiz.analytics.aio_recovery_applied",
      "rankwiz.analytics.aio_recovery_realized",
      "rankwiz.queue.background_depth",
      "rankwiz.aio_recovery.job_failure_rate"
    ],
    "emission_source": "Custom metrics are emitted by RunAioRecoveryJob via DogStatsd or Datadog log-based metric rules. See monitoring/aio_recovery_metrics.md for the emission guide.",
    "staging_validation": "See monitoring/aio_recovery_alerts.yaml — each alert has a '# Staging synthetic test' comment with the exact command to trigger the threshold.",
    "ga_gate": "All 5 alert monitors must be green and both dashboard widget groups must show real data (non-zero from at least one beta site) before GA. See docs/plan-ai-overview-click-recovery.md §15 DoD."
  }
}