{
  "apiVersion": "dashboard.grafana.app/v2",
  "kind": "Dashboard",
  "metadata": {
    "name": "5J4pyKEnka",
    "namespace": "default",
    "uid": "f075f359-3350-4060-9ca9-8ea9694b89cc",
    "resourceVersion": "1784812819751004",
    "generation": 1,
    "creationTimestamp": "2026-07-23T13:20:19Z",
    "labels": {
      "grafana.app/deprecatedInternalID": "4122431297622016"
    },
    "annotations": {
      "grafana.app/createdBy": "access-policy:service",
      "grafana.app/managedBy": "classic-file-provisioning",
      "grafana.app/managerId": "sidecarProvider",
      "grafana.app/sourceChecksum": "9327c8db4f041328c75cbd67e8675c9e",
      "grafana.app/sourcePath": "/tmp/dashboards/namespace_capsule-system.configmap_capsule-controller-runtime-dashboard.controller-runtime-dashboard.json",
      "grafana.app/sourceTimestamp": "1784812715000",
      "grafana.app/folder": ""
    }
  },
  "spec": {
    "annotations": [
      {
        "kind": "AnnotationQuery",
        "spec": {
          "builtIn": true,
          "enable": true,
          "hide": true,
          "iconColor": "rgba(0, 211, 255, 1)",
          "name": "Annotations & Alerts",
          "query": {
            "group": "grafana",
            "kind": "DataQuery",
            "spec": {
              "limit": 100,
              "matchAny": false,
              "tags": [],
              "type": "dashboard"
            },
            "version": "v0"
          }
        }
      }
    ],
    "cursorSync": "Off",
    "description": "Profiling performance-related metrics based on controller-runtime.",
    "editable": true,
    "elements": {
      "panel-10": {
        "kind": "Panel",
        "spec": {
          "data": {
            "kind": "QueryGroup",
            "spec": {
              "queries": [
                {
                  "kind": "PanelQuery",
                  "spec": {
                    "hidden": false,
                    "query": {
                      "datasource": {
                        "name": "prometheus"
                      },
                      "group": "prometheus",
                      "kind": "DataQuery",
                      "spec": {
                        "editorMode": "code",
                        "exemplar": true,
                        "expr": "sum by (result) (\n  increase(controller_runtime_reconcile_total{job=~\"$job\", controller=~\"$Controller\"}[$__rate_interval])\n)\n",
                        "interval": "",
                        "legendFormat": "{{result}}",
                        "range": true
                      },
                      "version": "v0"
                    },
                    "refId": "A"
                  }
                }
              ],
              "queryOptions": {},
              "transformations": []
            }
          },
          "description": "",
          "id": 10,
          "links": [],
          "title": "Reconcile Rate",
          "vizConfig": {
            "group": "timeseries",
            "kind": "VizConfig",
            "spec": {
              "fieldConfig": {
                "defaults": {
                  "color": {
                    "mode": "palette-classic"
                  },
                  "custom": {
                    "axisBorderShow": false,
                    "axisCenteredZero": false,
                    "axisColorMode": "text",
                    "axisLabel": "",
                    "axisPlacement": "auto",
                    "barAlignment": 0,
                    "barWidthFactor": 0.6,
                    "drawStyle": "line",
                    "fillOpacity": 0,
                    "gradientMode": "none",
                    "hideFrom": {
                      "legend": false,
                      "tooltip": false,
                      "viz": false
                    },
                    "insertNulls": false,
                    "lineInterpolation": "linear",
                    "lineWidth": 1,
                    "pointSize": 5,
                    "scaleDistribution": {
                      "type": "linear"
                    },
                    "showPoints": "auto",
                    "showValues": false,
                    "spanNulls": false,
                    "stacking": {
                      "group": "A",
                      "mode": "none"
                    },
                    "thresholdsStyle": {
                      "mode": "off"
                    }
                  },
                  "thresholds": {
                    "mode": "absolute",
                    "steps": [
                      {
                        "color": "green",
                        "value": 0
                      },
                      {
                        "color": "red",
                        "value": 80
                      }
                    ]
                  }
                },
                "overrides": []
              },
              "options": {
                "annotations": {
                  "clustering": -1,
                  "multiLane": false
                },
                "legend": {
                  "calcs": [],
                  "displayMode": "list",
                  "enableFacetedFilter": false,
                  "overflow": "ellipsis",
                  "placement": "bottom",
                  "showLegend": true
                },
                "tooltip": {
                  "hideZeros": false,
                  "mode": "single",
                  "sort": "none"
                }
              }
            },
            "version": "13.1.0"
          }
        }
      },
      "panel-12": {
        "kind": "Panel",
        "spec": {
          "data": {
            "kind": "QueryGroup",
            "spec": {
              "queries": [
                {
                  "kind": "PanelQuery",
                  "spec": {
                    "hidden": false,
                    "query": {
                      "datasource": {
                        "name": "prometheus"
                      },
                      "group": "prometheus",
                      "kind": "DataQuery",
                      "spec": {
                        "editorMode": "code",
                        "exemplar": true,
                        "expr": "rate(controller_runtime_reconcile_time_seconds_bucket{job=~'$job', controller=~'$Controller'}[$__rate_interval])",
                        "format": "heatmap",
                        "interval": "",
                        "legendFormat": "{{le}}",
                        "range": true
                      },
                      "version": "v0"
                    },
                    "refId": "A"
                  }
                }
              ],
              "queryOptions": {},
              "transformations": []
            }
          },
          "description": "",
          "id": 12,
          "links": [],
          "title": "Reconcile Time Buckets",
          "vizConfig": {
            "group": "heatmap",
            "kind": "VizConfig",
            "spec": {
              "fieldConfig": {
                "defaults": {
                  "custom": {
                    "hideFrom": {
                      "legend": false,
                      "tooltip": false,
                      "viz": false
                    },
                    "scaleDistribution": {
                      "type": "linear"
                    }
                  }
                },
                "overrides": []
              },
              "options": {
                "annotations": {
                  "clustering": -1,
                  "multiLane": false
                },
                "calculate": false,
                "calculation": {},
                "cellGap": 2,
                "cellValues": {},
                "color": {
                  "exponent": 0.5,
                  "fill": "#b4ff00",
                  "mode": "scheme",
                  "reverse": false,
                  "scale": "exponential",
                  "scheme": "Oranges",
                  "steps": 128
                },
                "exemplars": {
                  "color": "rgba(255,0,255,0.7)"
                },
                "filterValues": {
                  "le": 1e-9
                },
                "legend": {
                  "show": false
                },
                "rowsFrame": {
                  "layout": "auto"
                },
                "showValue": "never",
                "tooltip": {
                  "mode": "single",
                  "showColorScale": false,
                  "yHistogram": false
                },
                "yAxis": {
                  "axisPlacement": "left",
                  "reverse": false,
                  "unit": "short"
                }
              }
            },
            "version": "13.1.0"
          }
        }
      },
      "panel-13": {
        "kind": "Panel",
        "spec": {
          "data": {
            "kind": "QueryGroup",
            "spec": {
              "queries": [
                {
                  "kind": "PanelQuery",
                  "spec": {
                    "hidden": false,
                    "query": {
                      "datasource": {
                        "name": "prometheus"
                      },
                      "group": "prometheus",
                      "kind": "DataQuery",
                      "spec": {
                        "editorMode": "code",
                        "expr": "sum by (name) (\n  workqueue_depth{job=~\"$job\"}\n)",
                        "legendFormat": "__auto",
                        "range": true
                      },
                      "version": "v0"
                    },
                    "refId": "A"
                  }
                }
              ],
              "queryOptions": {},
              "transformations": []
            }
          },
          "description": "",
          "id": 13,
          "links": [],
          "title": "Workqueue Depth",
          "vizConfig": {
            "group": "timeseries",
            "kind": "VizConfig",
            "spec": {
              "fieldConfig": {
                "defaults": {
                  "color": {
                    "mode": "palette-classic"
                  },
                  "custom": {
                    "axisBorderShow": false,
                    "axisCenteredZero": false,
                    "axisColorMode": "text",
                    "axisLabel": "",
                    "axisPlacement": "auto",
                    "barAlignment": 0,
                    "barWidthFactor": 0.6,
                    "drawStyle": "line",
                    "fillOpacity": 0,
                    "gradientMode": "none",
                    "hideFrom": {
                      "legend": false,
                      "tooltip": false,
                      "viz": false
                    },
                    "insertNulls": false,
                    "lineInterpolation": "linear",
                    "lineWidth": 1,
                    "pointSize": 5,
                    "scaleDistribution": {
                      "type": "linear"
                    },
                    "showPoints": "auto",
                    "showValues": false,
                    "spanNulls": false,
                    "stacking": {
                      "group": "A",
                      "mode": "none"
                    },
                    "thresholdsStyle": {
                      "mode": "dashed"
                    }
                  },
                  "thresholds": {
                    "mode": "absolute",
                    "steps": [
                      {
                        "color": "green",
                        "value": 0
                      },
                      {
                        "color": "orange",
                        "value": 100
                      },
                      {
                        "color": "red",
                        "value": 500
                      }
                    ]
                  },
                  "unit": "short"
                },
                "overrides": []
              },
              "options": {
                "annotations": {
                  "clustering": -1,
                  "multiLane": false
                },
                "legend": {
                  "calcs": [
                    "lastNotNull",
                    "max",
                    "mean"
                  ],
                  "displayMode": "table",
                  "enableFacetedFilter": false,
                  "overflow": "ellipsis",
                  "placement": "right",
                  "showLegend": true,
                  "sortBy": "Max",
                  "sortDesc": true
                },
                "tooltip": {
                  "hideZeros": false,
                  "mode": "single",
                  "sort": "none"
                }
              }
            },
            "version": "13.1.0"
          }
        }
      },
      "panel-14": {
        "kind": "Panel",
        "spec": {
          "data": {
            "kind": "QueryGroup",
            "spec": {
              "queries": [
                {
                  "kind": "PanelQuery",
                  "spec": {
                    "hidden": false,
                    "query": {
                      "datasource": {
                        "name": "prometheus"
                      },
                      "group": "prometheus",
                      "kind": "DataQuery",
                      "spec": {
                        "editorMode": "code",
                        "expr": "max by (name) (\n  max_over_time(workqueue_depth{job=~\"$job\"}[15m])\n)",
                        "legendFormat": "__auto",
                        "range": true
                      },
                      "version": "v0"
                    },
                    "refId": "A"
                  }
                }
              ],
              "queryOptions": {},
              "transformations": []
            }
          },
          "description": "Max Queue Depth, 15m",
          "id": 14,
          "links": [],
          "title": "Workqueue Depth Max",
          "vizConfig": {
            "group": "timeseries",
            "kind": "VizConfig",
            "spec": {
              "fieldConfig": {
                "defaults": {
                  "color": {
                    "mode": "palette-classic"
                  },
                  "custom": {
                    "axisBorderShow": false,
                    "axisCenteredZero": false,
                    "axisColorMode": "text",
                    "axisLabel": "",
                    "axisPlacement": "auto",
                    "barAlignment": 0,
                    "barWidthFactor": 0.6,
                    "drawStyle": "line",
                    "fillOpacity": 0,
                    "gradientMode": "none",
                    "hideFrom": {
                      "legend": false,
                      "tooltip": false,
                      "viz": false
                    },
                    "insertNulls": false,
                    "lineInterpolation": "linear",
                    "lineWidth": 1,
                    "pointSize": 5,
                    "scaleDistribution": {
                      "type": "linear"
                    },
                    "showPoints": "auto",
                    "showValues": false,
                    "spanNulls": false,
                    "stacking": {
                      "group": "A",
                      "mode": "none"
                    },
                    "thresholdsStyle": {
                      "mode": "dashed"
                    }
                  },
                  "thresholds": {
                    "mode": "absolute",
                    "steps": [
                      {
                        "color": "green",
                        "value": 0
                      },
                      {
                        "color": "orange",
                        "value": 100
                      },
                      {
                        "color": "red",
                        "value": 500
                      }
                    ]
                  }
                },
                "overrides": []
              },
              "options": {
                "annotations": {
                  "clustering": -1,
                  "multiLane": false
                },
                "legend": {
                  "calcs": [
                    "lastNotNull",
                    "max",
                    "mean"
                  ],
                  "displayMode": "table",
                  "enableFacetedFilter": false,
                  "overflow": "ellipsis",
                  "placement": "right",
                  "showLegend": true,
                  "sortBy": "Max",
                  "sortDesc": true
                },
                "tooltip": {
                  "hideZeros": false,
                  "mode": "single",
                  "sort": "none"
                }
              }
            },
            "version": "13.1.0"
          }
        }
      },
      "panel-15": {
        "kind": "Panel",
        "spec": {
          "data": {
            "kind": "QueryGroup",
            "spec": {
              "queries": [
                {
                  "kind": "PanelQuery",
                  "spec": {
                    "hidden": false,
                    "query": {
                      "datasource": {
                        "name": "prometheus"
                      },
                      "group": "prometheus",
                      "kind": "DataQuery",
                      "spec": {
                        "editorMode": "code",
                        "expr": "histogram_quantile(\n  0.95,\n  sum by (name, le) (\n    rate(workqueue_queue_duration_seconds_bucket{job=~\"$job\"}[$__rate_interval])\n  )\n)",
                        "legendFormat": "__auto",
                        "range": true
                      },
                      "version": "v0"
                    },
                    "refId": "A"
                  }
                }
              ],
              "queryOptions": {},
              "transformations": []
            }
          },
          "description": "Queue Wait Duration p95",
          "id": 15,
          "links": [],
          "title": "Queue Wait Duration (p95)",
          "vizConfig": {
            "group": "timeseries",
            "kind": "VizConfig",
            "spec": {
              "fieldConfig": {
                "defaults": {
                  "color": {
                    "mode": "palette-classic"
                  },
                  "custom": {
                    "axisBorderShow": false,
                    "axisCenteredZero": false,
                    "axisColorMode": "text",
                    "axisLabel": "",
                    "axisPlacement": "auto",
                    "barAlignment": 0,
                    "barWidthFactor": 0.6,
                    "drawStyle": "line",
                    "fillOpacity": 0,
                    "gradientMode": "none",
                    "hideFrom": {
                      "legend": false,
                      "tooltip": false,
                      "viz": false
                    },
                    "insertNulls": false,
                    "lineInterpolation": "linear",
                    "lineWidth": 1,
                    "pointSize": 5,
                    "scaleDistribution": {
                      "type": "linear"
                    },
                    "showPoints": "auto",
                    "showValues": false,
                    "spanNulls": false,
                    "stacking": {
                      "group": "A",
                      "mode": "none"
                    },
                    "thresholdsStyle": {
                      "mode": "dashed"
                    }
                  },
                  "thresholds": {
                    "mode": "absolute",
                    "steps": [
                      {
                        "color": "green",
                        "value": 0
                      },
                      {
                        "color": "orange",
                        "value": 30
                      },
                      {
                        "color": "red",
                        "value": 300
                      }
                    ]
                  },
                  "unit": "s"
                },
                "overrides": [
                  {
                    "__systemRef": "hideSeriesFrom",
                    "matcher": {
                      "id": "byNames",
                      "options": {
                        "mode": "exclude",
                        "names": [
                          "capsule/tenants"
                        ],
                        "prefix": "All except:",
                        "readOnly": true
                      }
                    },
                    "properties": [
                      {
                        "id": "custom.hideFrom",
                        "value": {
                          "legend": false,
                          "tooltip": true,
                          "viz": true
                        }
                      }
                    ]
                  }
                ]
              },
              "options": {
                "annotations": {
                  "clustering": -1,
                  "multiLane": false
                },
                "legend": {
                  "calcs": [
                    "lastNotNull",
                    "max",
                    "mean"
                  ],
                  "displayMode": "table",
                  "enableFacetedFilter": false,
                  "overflow": "ellipsis",
                  "placement": "right",
                  "showLegend": true,
                  "sortBy": "Max",
                  "sortDesc": true
                },
                "tooltip": {
                  "hideZeros": false,
                  "mode": "single",
                  "sort": "none"
                }
              }
            },
            "version": "13.1.0"
          }
        }
      },
      "panel-16": {
        "kind": "Panel",
        "spec": {
          "data": {
            "kind": "QueryGroup",
            "spec": {
              "queries": [
                {
                  "kind": "PanelQuery",
                  "spec": {
                    "hidden": false,
                    "query": {
                      "datasource": {
                        "name": "prometheus"
                      },
                      "group": "prometheus",
                      "kind": "DataQuery",
                      "spec": {
                        "editorMode": "code",
                        "expr": "histogram_quantile(\n  0.99,\n  sum by (name, le) (\n    rate(workqueue_queue_duration_seconds_bucket{job=~\"$job\"}[$__rate_interval])\n  )\n)",
                        "legendFormat": "__auto",
                        "range": true
                      },
                      "version": "v0"
                    },
                    "refId": "A"
                  }
                }
              ],
              "queryOptions": {},
              "transformations": []
            }
          },
          "description": "",
          "id": 16,
          "links": [],
          "title": "Queue Wait Duration (p99)",
          "vizConfig": {
            "group": "timeseries",
            "kind": "VizConfig",
            "spec": {
              "fieldConfig": {
                "defaults": {
                  "color": {
                    "mode": "palette-classic"
                  },
                  "custom": {
                    "axisBorderShow": false,
                    "axisCenteredZero": false,
                    "axisColorMode": "text",
                    "axisLabel": "",
                    "axisPlacement": "auto",
                    "barAlignment": 0,
                    "barWidthFactor": 0.6,
                    "drawStyle": "line",
                    "fillOpacity": 0,
                    "gradientMode": "none",
                    "hideFrom": {
                      "legend": false,
                      "tooltip": false,
                      "viz": false
                    },
                    "insertNulls": false,
                    "lineInterpolation": "linear",
                    "lineWidth": 1,
                    "pointSize": 5,
                    "scaleDistribution": {
                      "type": "linear"
                    },
                    "showPoints": "auto",
                    "showValues": false,
                    "spanNulls": false,
                    "stacking": {
                      "group": "A",
                      "mode": "none"
                    },
                    "thresholdsStyle": {
                      "mode": "dashed"
                    }
                  },
                  "thresholds": {
                    "mode": "absolute",
                    "steps": [
                      {
                        "color": "green",
                        "value": 0
                      },
                      {
                        "color": "orange",
                        "value": 60
                      },
                      {
                        "color": "red",
                        "value": 300
                      }
                    ]
                  },
                  "unit": "s"
                },
                "overrides": []
              },
              "options": {
                "annotations": {
                  "clustering": -1,
                  "multiLane": false
                },
                "legend": {
                  "calcs": [
                    "lastNotNull",
                    "max",
                    "mean"
                  ],
                  "displayMode": "table",
                  "enableFacetedFilter": false,
                  "overflow": "ellipsis",
                  "placement": "right",
                  "showLegend": true,
                  "sortBy": "Max",
                  "sortDesc": true
                },
                "tooltip": {
                  "hideZeros": false,
                  "mode": "single",
                  "sort": "none"
                }
              }
            },
            "version": "13.1.0"
          }
        }
      },
      "panel-17": {
        "kind": "Panel",
        "spec": {
          "data": {
            "kind": "QueryGroup",
            "spec": {
              "queries": [
                {
                  "kind": "PanelQuery",
                  "spec": {
                    "hidden": false,
                    "query": {
                      "datasource": {
                        "name": "prometheus"
                      },
                      "group": "prometheus",
                      "kind": "DataQuery",
                      "spec": {
                        "editorMode": "code",
                        "expr": "histogram_quantile(\n  0.95,\n  sum by (name, le) (\n    rate(workqueue_work_duration_seconds_bucket{job=~\"$job\"}[$__rate_interval])\n  )\n)",
                        "legendFormat": "__auto",
                        "range": true
                      },
                      "version": "v0"
                    },
                    "refId": "A"
                  }
                }
              ],
              "queryOptions": {},
              "transformations": []
            }
          },
          "description": "",
          "id": 17,
          "links": [],
          "title": "Work Duration p95",
          "vizConfig": {
            "group": "timeseries",
            "kind": "VizConfig",
            "spec": {
              "fieldConfig": {
                "defaults": {
                  "color": {
                    "mode": "palette-classic"
                  },
                  "custom": {
                    "axisBorderShow": false,
                    "axisCenteredZero": false,
                    "axisColorMode": "text",
                    "axisLabel": "",
                    "axisPlacement": "auto",
                    "barAlignment": 0,
                    "barWidthFactor": 0.6,
                    "drawStyle": "line",
                    "fillOpacity": 0,
                    "gradientMode": "none",
                    "hideFrom": {
                      "legend": false,
                      "tooltip": false,
                      "viz": false
                    },
                    "insertNulls": false,
                    "lineInterpolation": "linear",
                    "lineWidth": 1,
                    "pointSize": 5,
                    "scaleDistribution": {
                      "type": "linear"
                    },
                    "showPoints": "auto",
                    "showValues": false,
                    "spanNulls": false,
                    "stacking": {
                      "group": "A",
                      "mode": "none"
                    },
                    "thresholdsStyle": {
                      "mode": "dashed"
                    }
                  },
                  "thresholds": {
                    "mode": "absolute",
                    "steps": [
                      {
                        "color": "green",
                        "value": 0
                      },
                      {
                        "color": "orange",
                        "value": 5
                      },
                      {
                        "color": "red",
                        "value": 15
                      }
                    ]
                  },
                  "unit": "s"
                },
                "overrides": []
              },
              "options": {
                "annotations": {
                  "clustering": -1,
                  "multiLane": false
                },
                "legend": {
                  "calcs": [
                    "lastNotNull",
                    "max",
                    "mean"
                  ],
                  "displayMode": "table",
                  "enableFacetedFilter": false,
                  "overflow": "ellipsis",
                  "placement": "right",
                  "showLegend": true,
                  "sortBy": "Max",
                  "sortDesc": true
                },
                "tooltip": {
                  "hideZeros": false,
                  "mode": "single",
                  "sort": "none"
                }
              }
            },
            "version": "13.1.0"
          }
        }
      },
      "panel-18": {
        "kind": "Panel",
        "spec": {
          "data": {
            "kind": "QueryGroup",
            "spec": {
              "queries": [
                {
                  "kind": "PanelQuery",
                  "spec": {
                    "hidden": false,
                    "query": {
                      "datasource": {
                        "name": "prometheus"
                      },
                      "group": "prometheus",
                      "kind": "DataQuery",
                      "spec": {
                        "editorMode": "code",
                        "expr": "histogram_quantile(\n  0.99,\n  sum by (name, le) (\n    rate(workqueue_work_duration_seconds_bucket{job=~\"$job\"}[$__rate_interval])\n  )\n)",
                        "legendFormat": "__auto",
                        "range": true
                      },
                      "version": "v0"
                    },
                    "refId": "A"
                  }
                }
              ],
              "queryOptions": {},
              "transformations": []
            }
          },
          "description": "",
          "id": 18,
          "links": [],
          "title": "Work Duration p99",
          "vizConfig": {
            "group": "timeseries",
            "kind": "VizConfig",
            "spec": {
              "fieldConfig": {
                "defaults": {
                  "color": {
                    "mode": "palette-classic"
                  },
                  "custom": {
                    "axisBorderShow": false,
                    "axisCenteredZero": false,
                    "axisColorMode": "text",
                    "axisLabel": "",
                    "axisPlacement": "auto",
                    "barAlignment": 0,
                    "barWidthFactor": 0.6,
                    "drawStyle": "line",
                    "fillOpacity": 0,
                    "gradientMode": "none",
                    "hideFrom": {
                      "legend": false,
                      "tooltip": false,
                      "viz": false
                    },
                    "insertNulls": false,
                    "lineInterpolation": "linear",
                    "lineWidth": 1,
                    "pointSize": 5,
                    "scaleDistribution": {
                      "type": "linear"
                    },
                    "showPoints": "auto",
                    "showValues": false,
                    "spanNulls": false,
                    "stacking": {
                      "group": "A",
                      "mode": "none"
                    },
                    "thresholdsStyle": {
                      "mode": "dashed"
                    }
                  },
                  "thresholds": {
                    "mode": "absolute",
                    "steps": [
                      {
                        "color": "green",
                        "value": 0
                      },
                      {
                        "color": "orange",
                        "value": 5
                      },
                      {
                        "color": "red",
                        "value": 15
                      }
                    ]
                  },
                  "unit": "s"
                },
                "overrides": []
              },
              "options": {
                "annotations": {
                  "clustering": -1,
                  "multiLane": false
                },
                "legend": {
                  "calcs": [
                    "lastNotNull",
                    "max",
                    "mean"
                  ],
                  "displayMode": "table",
                  "enableFacetedFilter": false,
                  "overflow": "ellipsis",
                  "placement": "right",
                  "showLegend": true,
                  "sortBy": "Max",
                  "sortDesc": true
                },
                "tooltip": {
                  "hideZeros": false,
                  "mode": "single",
                  "sort": "none"
                }
              }
            },
            "version": "13.1.0"
          }
        }
      },
      "panel-19": {
        "kind": "Panel",
        "spec": {
          "data": {
            "kind": "QueryGroup",
            "spec": {
              "queries": [
                {
                  "kind": "PanelQuery",
                  "spec": {
                    "hidden": false,
                    "query": {
                      "datasource": {
                        "name": "prometheus"
                      },
                      "group": "prometheus",
                      "kind": "DataQuery",
                      "spec": {
                        "editorMode": "code",
                        "expr": "sum by (name) (\n  rate(workqueue_work_duration_seconds_sum{job=~\"$job\"}[$__rate_interval])\n)\n/\nsum by (name) (\n  rate(workqueue_work_duration_seconds_count{job=~\"$job\"}[$__rate_interval])\n)",
                        "legendFormat": "__auto",
                        "range": true
                      },
                      "version": "v0"
                    },
                    "refId": "A"
                  }
                }
              ],
              "queryOptions": {},
              "transformations": []
            }
          },
          "description": "",
          "id": 19,
          "links": [],
          "title": "Work Duration Avg",
          "vizConfig": {
            "group": "timeseries",
            "kind": "VizConfig",
            "spec": {
              "fieldConfig": {
                "defaults": {
                  "color": {
                    "mode": "palette-classic"
                  },
                  "custom": {
                    "axisBorderShow": false,
                    "axisCenteredZero": false,
                    "axisColorMode": "text",
                    "axisLabel": "",
                    "axisPlacement": "auto",
                    "barAlignment": 0,
                    "barWidthFactor": 0.6,
                    "drawStyle": "line",
                    "fillOpacity": 0,
                    "gradientMode": "none",
                    "hideFrom": {
                      "legend": false,
                      "tooltip": false,
                      "viz": false
                    },
                    "insertNulls": false,
                    "lineInterpolation": "linear",
                    "lineWidth": 1,
                    "pointSize": 5,
                    "scaleDistribution": {
                      "type": "linear"
                    },
                    "showPoints": "auto",
                    "showValues": false,
                    "spanNulls": false,
                    "stacking": {
                      "group": "A",
                      "mode": "none"
                    },
                    "thresholdsStyle": {
                      "mode": "dashed"
                    }
                  },
                  "thresholds": {
                    "mode": "absolute",
                    "steps": [
                      {
                        "color": "green",
                        "value": 0
                      },
                      {
                        "color": "orange",
                        "value": 5
                      },
                      {
                        "color": "red",
                        "value": 15
                      }
                    ]
                  },
                  "unit": "s"
                },
                "overrides": []
              },
              "options": {
                "annotations": {
                  "clustering": -1,
                  "multiLane": false
                },
                "legend": {
                  "calcs": [
                    "lastNotNull",
                    "max",
                    "mean"
                  ],
                  "displayMode": "table",
                  "enableFacetedFilter": false,
                  "overflow": "ellipsis",
                  "placement": "right",
                  "showLegend": true,
                  "sortBy": "Max",
                  "sortDesc": true
                },
                "tooltip": {
                  "hideZeros": false,
                  "mode": "single",
                  "sort": "none"
                }
              }
            },
            "version": "13.1.0"
          }
        }
      },
      "panel-2": {
        "kind": "Panel",
        "spec": {
          "data": {
            "kind": "QueryGroup",
            "spec": {
              "queries": [
                {
                  "kind": "PanelQuery",
                  "spec": {
                    "hidden": false,
                    "query": {
                      "datasource": {
                        "name": "prometheus"
                      },
                      "group": "prometheus",
                      "kind": "DataQuery",
                      "spec": {
                        "editorMode": "code",
                        "exemplar": true,
                        "expr": "controller_runtime_active_workers{job=~\"$job\"}",
                        "interval": "",
                        "legendFormat": "{{controller}}",
                        "range": true
                      },
                      "version": "v0"
                    },
                    "refId": "A"
                  }
                },
                {
                  "kind": "PanelQuery",
                  "spec": {
                    "hidden": false,
                    "query": {
                      "datasource": {
                        "name": "prometheus"
                      },
                      "group": "prometheus",
                      "kind": "DataQuery",
                      "spec": {
                        "editorMode": "code",
                        "expr": "sum (controller_runtime_active_workers{job=~\"$job\"})",
                        "instant": false,
                        "legendFormat": "Total",
                        "range": true
                      },
                      "version": "v0"
                    },
                    "refId": "B"
                  }
                }
              ],
              "queryOptions": {},
              "transformations": []
            }
          },
          "description": "",
          "id": 2,
          "links": [],
          "title": "Active Workers",
          "vizConfig": {
            "group": "timeseries",
            "kind": "VizConfig",
            "spec": {
              "fieldConfig": {
                "defaults": {
                  "color": {
                    "mode": "palette-classic"
                  },
                  "custom": {
                    "axisBorderShow": false,
                    "axisCenteredZero": false,
                    "axisColorMode": "text",
                    "axisLabel": "",
                    "axisPlacement": "auto",
                    "barAlignment": 0,
                    "barWidthFactor": 0.6,
                    "drawStyle": "line",
                    "fillOpacity": 0,
                    "gradientMode": "none",
                    "hideFrom": {
                      "legend": false,
                      "tooltip": false,
                      "viz": false
                    },
                    "insertNulls": false,
                    "lineInterpolation": "linear",
                    "lineWidth": 1,
                    "pointSize": 5,
                    "scaleDistribution": {
                      "type": "linear"
                    },
                    "showPoints": "auto",
                    "showValues": false,
                    "spanNulls": false,
                    "stacking": {
                      "group": "A",
                      "mode": "none"
                    },
                    "thresholdsStyle": {
                      "mode": "off"
                    }
                  },
                  "thresholds": {
                    "mode": "absolute",
                    "steps": [
                      {
                        "color": "green",
                        "value": 0
                      },
                      {
                        "color": "red",
                        "value": 80
                      }
                    ]
                  }
                },
                "overrides": []
              },
              "options": {
                "annotations": {
                  "clustering": -1,
                  "multiLane": false
                },
                "legend": {
                  "calcs": [
                    "lastNotNull",
                    "max",
                    "mean"
                  ],
                  "displayMode": "table",
                  "enableFacetedFilter": false,
                  "overflow": "ellipsis",
                  "placement": "right",
                  "showLegend": true,
                  "sortBy": "Max",
                  "sortDesc": true
                },
                "tooltip": {
                  "hideZeros": false,
                  "mode": "single",
                  "sort": "none"
                }
              }
            },
            "version": "13.1.0"
          }
        }
      },
      "panel-20": {
        "kind": "Panel",
        "spec": {
          "data": {
            "kind": "QueryGroup",
            "spec": {
              "queries": [
                {
                  "kind": "PanelQuery",
                  "spec": {
                    "hidden": false,
                    "query": {
                      "datasource": {
                        "name": "prometheus"
                      },
                      "group": "prometheus",
                      "kind": "DataQuery",
                      "spec": {
                        "editorMode": "code",
                        "expr": "sum by (name) (\n  rate(workqueue_work_duration_seconds_count{job=~\"$job\"}[$__rate_interval])\n)",
                        "legendFormat": "__auto",
                        "range": true
                      },
                      "version": "v0"
                    },
                    "refId": "A"
                  }
                }
              ],
              "queryOptions": {},
              "transformations": []
            }
          },
          "description": "",
          "id": 20,
          "links": [],
          "title": "Processed Work Items / Second",
          "vizConfig": {
            "group": "timeseries",
            "kind": "VizConfig",
            "spec": {
              "fieldConfig": {
                "defaults": {
                  "color": {
                    "mode": "palette-classic"
                  },
                  "custom": {
                    "axisBorderShow": false,
                    "axisCenteredZero": false,
                    "axisColorMode": "text",
                    "axisLabel": "",
                    "axisPlacement": "auto",
                    "barAlignment": 0,
                    "barWidthFactor": 0.6,
                    "drawStyle": "line",
                    "fillOpacity": 0,
                    "gradientMode": "none",
                    "hideFrom": {
                      "legend": false,
                      "tooltip": false,
                      "viz": false
                    },
                    "insertNulls": false,
                    "lineInterpolation": "linear",
                    "lineWidth": 1,
                    "pointSize": 5,
                    "scaleDistribution": {
                      "type": "linear"
                    },
                    "showPoints": "auto",
                    "showValues": false,
                    "spanNulls": false,
                    "stacking": {
                      "group": "A",
                      "mode": "none"
                    },
                    "thresholdsStyle": {
                      "mode": "off"
                    }
                  },
                  "thresholds": {
                    "mode": "absolute",
                    "steps": [
                      {
                        "color": "green",
                        "value": 0
                      },
                      {
                        "color": "red",
                        "value": 80
                      }
                    ]
                  },
                  "unit": "ops"
                },
                "overrides": []
              },
              "options": {
                "annotations": {
                  "clustering": -1,
                  "multiLane": false
                },
                "legend": {
                  "calcs": [
                    "lastNotNull",
                    "max",
                    "mean"
                  ],
                  "displayMode": "table",
                  "enableFacetedFilter": false,
                  "overflow": "ellipsis",
                  "placement": "right",
                  "showLegend": true,
                  "sortBy": "Max",
                  "sortDesc": true
                },
                "tooltip": {
                  "hideZeros": false,
                  "mode": "single",
                  "sort": "none"
                }
              }
            },
            "version": "13.1.0"
          }
        }
      },
      "panel-21": {
        "kind": "Panel",
        "spec": {
          "data": {
            "kind": "QueryGroup",
            "spec": {
              "queries": [
                {
                  "kind": "PanelQuery",
                  "spec": {
                    "hidden": false,
                    "query": {
                      "datasource": {
                        "name": "prometheus"
                      },
                      "group": "prometheus",
                      "kind": "DataQuery",
                      "spec": {
                        "editorMode": "code",
                        "expr": "sum by (name) (\n  rate(workqueue_adds_total{job=~\"$job\"}[$__rate_interval])\n)",
                        "legendFormat": "__auto",
                        "range": true
                      },
                      "version": "v0"
                    },
                    "refId": "A"
                  }
                }
              ],
              "queryOptions": {},
              "transformations": []
            }
          },
          "description": "",
          "id": 21,
          "links": [],
          "title": "Queue Adds / Second",
          "vizConfig": {
            "group": "timeseries",
            "kind": "VizConfig",
            "spec": {
              "fieldConfig": {
                "defaults": {
                  "color": {
                    "mode": "palette-classic"
                  },
                  "custom": {
                    "axisBorderShow": false,
                    "axisCenteredZero": false,
                    "axisColorMode": "text",
                    "axisLabel": "",
                    "axisPlacement": "auto",
                    "barAlignment": 0,
                    "barWidthFactor": 0.6,
                    "drawStyle": "line",
                    "fillOpacity": 0,
                    "gradientMode": "none",
                    "hideFrom": {
                      "legend": false,
                      "tooltip": false,
                      "viz": false
                    },
                    "insertNulls": false,
                    "lineInterpolation": "linear",
                    "lineWidth": 1,
                    "pointSize": 5,
                    "scaleDistribution": {
                      "type": "linear"
                    },
                    "showPoints": "auto",
                    "showValues": false,
                    "spanNulls": false,
                    "stacking": {
                      "group": "A",
                      "mode": "none"
                    },
                    "thresholdsStyle": {
                      "mode": "off"
                    }
                  },
                  "thresholds": {
                    "mode": "absolute",
                    "steps": [
                      {
                        "color": "green",
                        "value": 0
                      },
                      {
                        "color": "red",
                        "value": 80
                      }
                    ]
                  },
                  "unit": "ops"
                },
                "overrides": []
              },
              "options": {
                "annotations": {
                  "clustering": -1,
                  "multiLane": false
                },
                "legend": {
                  "calcs": [
                    "lastNotNull",
                    "max",
                    "mean"
                  ],
                  "displayMode": "table",
                  "enableFacetedFilter": false,
                  "overflow": "ellipsis",
                  "placement": "right",
                  "showLegend": true,
                  "sortBy": "Max",
                  "sortDesc": true
                },
                "tooltip": {
                  "hideZeros": false,
                  "mode": "single",
                  "sort": "none"
                }
              }
            },
            "version": "13.1.0"
          }
        }
      },
      "panel-22": {
        "kind": "Panel",
        "spec": {
          "data": {
            "kind": "QueryGroup",
            "spec": {
              "queries": [
                {
                  "kind": "PanelQuery",
                  "spec": {
                    "hidden": false,
                    "query": {
                      "datasource": {
                        "name": "prometheus"
                      },
                      "group": "prometheus",
                      "kind": "DataQuery",
                      "spec": {
                        "editorMode": "code",
                        "expr": "sum by (name) (\n  rate(workqueue_adds_total{job=~\"$job\"}[$__rate_interval])\n)\n-\nsum by (name) (\n  rate(workqueue_work_duration_seconds_count{job=~\"$job\"}[$__rate_interval])\n)",
                        "legendFormat": "__auto",
                        "range": true
                      },
                      "version": "v0"
                    },
                    "refId": "A"
                  }
                }
              ],
              "queryOptions": {},
              "transformations": []
            }
          },
          "description": "Positive means the queue is growing. Negative means the controller is draining backlog.",
          "id": 22,
          "links": [],
          "title": "Backlog Pressure",
          "vizConfig": {
            "group": "timeseries",
            "kind": "VizConfig",
            "spec": {
              "fieldConfig": {
                "defaults": {
                  "color": {
                    "mode": "palette-classic"
                  },
                  "custom": {
                    "axisBorderShow": false,
                    "axisCenteredZero": false,
                    "axisColorMode": "text",
                    "axisLabel": "",
                    "axisPlacement": "auto",
                    "barAlignment": 0,
                    "barWidthFactor": 0.6,
                    "drawStyle": "line",
                    "fillOpacity": 0,
                    "gradientMode": "none",
                    "hideFrom": {
                      "legend": false,
                      "tooltip": false,
                      "viz": false
                    },
                    "insertNulls": false,
                    "lineInterpolation": "linear",
                    "lineWidth": 1,
                    "pointSize": 5,
                    "scaleDistribution": {
                      "type": "linear"
                    },
                    "showPoints": "auto",
                    "showValues": false,
                    "spanNulls": false,
                    "stacking": {
                      "group": "A",
                      "mode": "none"
                    },
                    "thresholdsStyle": {
                      "mode": "dashed"
                    }
                  },
                  "thresholds": {
                    "mode": "absolute",
                    "steps": [
                      {
                        "color": "green",
                        "value": 0
                      },
                      {
                        "color": "orange",
                        "value": 0
                      },
                      {
                        "color": "red",
                        "value": 1
                      }
                    ]
                  },
                  "unit": "ops"
                },
                "overrides": []
              },
              "options": {
                "annotations": {
                  "clustering": -1,
                  "multiLane": false
                },
                "legend": {
                  "calcs": [
                    "lastNotNull",
                    "max",
                    "mean"
                  ],
                  "displayMode": "table",
                  "enableFacetedFilter": false,
                  "overflow": "ellipsis",
                  "placement": "right",
                  "showLegend": true,
                  "sortBy": "Max",
                  "sortDesc": true
                },
                "tooltip": {
                  "hideZeros": false,
                  "mode": "single",
                  "sort": "none"
                }
              }
            },
            "version": "13.1.0"
          }
        }
      },
      "panel-23": {
        "kind": "Panel",
        "spec": {
          "data": {
            "kind": "QueryGroup",
            "spec": {
              "queries": [
                {
                  "kind": "PanelQuery",
                  "spec": {
                    "hidden": false,
                    "query": {
                      "datasource": {
                        "name": "prometheus"
                      },
                      "group": "prometheus",
                      "kind": "DataQuery",
                      "spec": {
                        "editorMode": "code",
                        "expr": "sum by (name) (\n  rate(workqueue_queue_duration_seconds_sum{job=~\"$job\"}[$__rate_interval])\n)\n/\nsum by (name) (\n  rate(workqueue_queue_duration_seconds_count{job=~\"$job\"}[$__rate_interval])\n)",
                        "legendFormat": "__auto",
                        "range": true
                      },
                      "version": "v0"
                    },
                    "refId": "A"
                  }
                }
              ],
              "queryOptions": {},
              "transformations": []
            }
          },
          "description": "Good next to p95/p99 to distinguish general slowness from spikes.",
          "id": 23,
          "links": [],
          "title": "Average Queue Wait Duration",
          "vizConfig": {
            "group": "timeseries",
            "kind": "VizConfig",
            "spec": {
              "fieldConfig": {
                "defaults": {
                  "color": {
                    "mode": "palette-classic"
                  },
                  "custom": {
                    "axisBorderShow": false,
                    "axisCenteredZero": false,
                    "axisColorMode": "text",
                    "axisLabel": "",
                    "axisPlacement": "auto",
                    "barAlignment": 0,
                    "barWidthFactor": 0.6,
                    "drawStyle": "line",
                    "fillOpacity": 0,
                    "gradientMode": "none",
                    "hideFrom": {
                      "legend": false,
                      "tooltip": false,
                      "viz": false
                    },
                    "insertNulls": false,
                    "lineInterpolation": "linear",
                    "lineWidth": 1,
                    "pointSize": 5,
                    "scaleDistribution": {
                      "type": "linear"
                    },
                    "showPoints": "auto",
                    "showValues": false,
                    "spanNulls": false,
                    "stacking": {
                      "group": "A",
                      "mode": "none"
                    },
                    "thresholdsStyle": {
                      "mode": "dashed"
                    }
                  },
                  "thresholds": {
                    "mode": "absolute",
                    "steps": [
                      {
                        "color": "green",
                        "value": 0
                      },
                      {
                        "color": "orange",
                        "value": 30
                      },
                      {
                        "color": "red",
                        "value": 120
                      }
                    ]
                  },
                  "unit": "s"
                },
                "overrides": []
              },
              "options": {
                "annotations": {
                  "clustering": -1,
                  "multiLane": false
                },
                "legend": {
                  "calcs": [
                    "lastNotNull",
                    "max",
                    "mean"
                  ],
                  "displayMode": "table",
                  "enableFacetedFilter": false,
                  "overflow": "ellipsis",
                  "placement": "right",
                  "showLegend": true,
                  "sortBy": "Max",
                  "sortDesc": true
                },
                "tooltip": {
                  "hideZeros": false,
                  "mode": "single",
                  "sort": "none"
                }
              }
            },
            "version": "13.1.0"
          }
        }
      },
      "panel-24": {
        "kind": "Panel",
        "spec": {
          "data": {
            "kind": "QueryGroup",
            "spec": {
              "queries": [
                {
                  "kind": "PanelQuery",
                  "spec": {
                    "hidden": false,
                    "query": {
                      "datasource": {
                        "name": "prometheus"
                      },
                      "group": "prometheus",
                      "kind": "DataQuery",
                      "spec": {
                        "editorMode": "code",
                        "expr": "sum by (name) (\n  rate(workqueue_retries_total{job=~\"$job\"}[$__rate_interval])\n)",
                        "legendFormat": "__auto",
                        "range": true
                      },
                      "version": "v0"
                    },
                    "refId": "A"
                  }
                }
              ],
              "queryOptions": {},
              "transformations": []
            }
          },
          "description": "Spikes here usually mean reconcile errors, conflicts, or rate-limited requeues.",
          "id": 24,
          "links": [],
          "title": "Queue Retries / Second",
          "vizConfig": {
            "group": "timeseries",
            "kind": "VizConfig",
            "spec": {
              "fieldConfig": {
                "defaults": {
                  "color": {
                    "mode": "palette-classic"
                  },
                  "custom": {
                    "axisBorderShow": false,
                    "axisCenteredZero": false,
                    "axisColorMode": "text",
                    "axisLabel": "",
                    "axisPlacement": "auto",
                    "barAlignment": 0,
                    "barWidthFactor": 0.6,
                    "drawStyle": "line",
                    "fillOpacity": 0,
                    "gradientMode": "none",
                    "hideFrom": {
                      "legend": false,
                      "tooltip": false,
                      "viz": false
                    },
                    "insertNulls": false,
                    "lineInterpolation": "linear",
                    "lineWidth": 1,
                    "pointSize": 5,
                    "scaleDistribution": {
                      "type": "linear"
                    },
                    "showPoints": "auto",
                    "showValues": false,
                    "spanNulls": false,
                    "stacking": {
                      "group": "A",
                      "mode": "none"
                    },
                    "thresholdsStyle": {
                      "mode": "dashed"
                    }
                  },
                  "thresholds": {
                    "mode": "absolute",
                    "steps": [
                      {
                        "color": "green",
                        "value": 0
                      },
                      {
                        "color": "orange",
                        "value": 0.1
                      },
                      {
                        "color": "red",
                        "value": 1
                      }
                    ]
                  }
                },
                "overrides": []
              },
              "options": {
                "annotations": {
                  "clustering": -1,
                  "multiLane": false
                },
                "legend": {
                  "calcs": [
                    "lastNotNull",
                    "max",
                    "mean"
                  ],
                  "displayMode": "table",
                  "enableFacetedFilter": false,
                  "overflow": "ellipsis",
                  "placement": "right",
                  "showLegend": true,
                  "sortBy": "Last *",
                  "sortDesc": true
                },
                "tooltip": {
                  "hideZeros": false,
                  "mode": "single",
                  "sort": "none"
                }
              }
            },
            "version": "13.1.0"
          }
        }
      },
      "panel-25": {
        "kind": "Panel",
        "spec": {
          "data": {
            "kind": "QueryGroup",
            "spec": {
              "queries": [
                {
                  "kind": "PanelQuery",
                  "spec": {
                    "hidden": false,
                    "query": {
                      "datasource": {
                        "name": "prometheus"
                      },
                      "group": "prometheus",
                      "kind": "DataQuery",
                      "spec": {
                        "editorMode": "code",
                        "expr": "histogram_quantile(\n  0.95,\n  sum by (controller, le) (\n    rate(controller_runtime_reconcile_time_seconds_bucket{job=~\"$job\"}[$__rate_interval])\n  )\n)",
                        "legendFormat": "__auto",
                        "range": true
                      },
                      "version": "v0"
                    },
                    "refId": "A"
                  }
                }
              ],
              "queryOptions": {},
              "transformations": []
            }
          },
          "description": "",
          "id": 25,
          "links": [],
          "title": "Reconcile Duration p95",
          "vizConfig": {
            "group": "timeseries",
            "kind": "VizConfig",
            "spec": {
              "fieldConfig": {
                "defaults": {
                  "color": {
                    "mode": "palette-classic"
                  },
                  "custom": {
                    "axisBorderShow": false,
                    "axisCenteredZero": false,
                    "axisColorMode": "text",
                    "axisLabel": "",
                    "axisPlacement": "auto",
                    "barAlignment": 0,
                    "barWidthFactor": 0.6,
                    "drawStyle": "line",
                    "fillOpacity": 0,
                    "gradientMode": "none",
                    "hideFrom": {
                      "legend": false,
                      "tooltip": false,
                      "viz": false
                    },
                    "insertNulls": false,
                    "lineInterpolation": "linear",
                    "lineWidth": 1,
                    "pointSize": 5,
                    "scaleDistribution": {
                      "type": "linear"
                    },
                    "showPoints": "auto",
                    "showValues": false,
                    "spanNulls": false,
                    "stacking": {
                      "group": "A",
                      "mode": "none"
                    },
                    "thresholdsStyle": {
                      "mode": "dashed"
                    }
                  },
                  "thresholds": {
                    "mode": "absolute",
                    "steps": [
                      {
                        "color": "green",
                        "value": 0
                      },
                      {
                        "color": "orange",
                        "value": 5
                      },
                      {
                        "color": "red",
                        "value": 15
                      }
                    ]
                  },
                  "unit": "s"
                },
                "overrides": []
              },
              "options": {
                "annotations": {
                  "clustering": -1,
                  "multiLane": false
                },
                "legend": {
                  "calcs": [
                    "lastNotNull",
                    "max",
                    "mean"
                  ],
                  "displayMode": "table",
                  "enableFacetedFilter": false,
                  "overflow": "ellipsis",
                  "placement": "right",
                  "showLegend": true,
                  "sortBy": "Max",
                  "sortDesc": true
                },
                "tooltip": {
                  "hideZeros": false,
                  "mode": "single",
                  "sort": "none"
                }
              }
            },
            "version": "13.1.0"
          }
        }
      },
      "panel-26": {
        "kind": "Panel",
        "spec": {
          "data": {
            "kind": "QueryGroup",
            "spec": {
              "queries": [
                {
                  "kind": "PanelQuery",
                  "spec": {
                    "hidden": false,
                    "query": {
                      "datasource": {
                        "name": "prometheus"
                      },
                      "group": "prometheus",
                      "kind": "DataQuery",
                      "spec": {
                        "editorMode": "code",
                        "expr": "sum by (controller) (\n  rate(controller_runtime_reconcile_time_seconds_sum{job=~\"$job\"}[$__rate_interval])\n)\n/\nsum by (controller) (\n  rate(controller_runtime_reconcile_time_seconds_count{job=~\"$job\"}[$__rate_interval])\n)",
                        "legendFormat": "__auto",
                        "range": true
                      },
                      "version": "v0"
                    },
                    "refId": "A"
                  }
                }
              ],
              "queryOptions": {},
              "transformations": []
            }
          },
          "description": "",
          "id": 26,
          "links": [],
          "title": "Reconcile Duration Avg",
          "vizConfig": {
            "group": "timeseries",
            "kind": "VizConfig",
            "spec": {
              "fieldConfig": {
                "defaults": {
                  "color": {
                    "mode": "palette-classic"
                  },
                  "custom": {
                    "axisBorderShow": false,
                    "axisCenteredZero": false,
                    "axisColorMode": "text",
                    "axisLabel": "",
                    "axisPlacement": "auto",
                    "barAlignment": 0,
                    "barWidthFactor": 0.6,
                    "drawStyle": "line",
                    "fillOpacity": 0,
                    "gradientMode": "none",
                    "hideFrom": {
                      "legend": false,
                      "tooltip": false,
                      "viz": false
                    },
                    "insertNulls": false,
                    "lineInterpolation": "linear",
                    "lineWidth": 1,
                    "pointSize": 5,
                    "scaleDistribution": {
                      "type": "linear"
                    },
                    "showPoints": "auto",
                    "showValues": false,
                    "spanNulls": false,
                    "stacking": {
                      "group": "A",
                      "mode": "none"
                    },
                    "thresholdsStyle": {
                      "mode": "dashed"
                    }
                  },
                  "thresholds": {
                    "mode": "absolute",
                    "steps": [
                      {
                        "color": "green",
                        "value": 0
                      },
                      {
                        "color": "orange",
                        "value": 5
                      },
                      {
                        "color": "red",
                        "value": 15
                      }
                    ]
                  },
                  "unit": "s"
                },
                "overrides": []
              },
              "options": {
                "annotations": {
                  "clustering": -1,
                  "multiLane": false
                },
                "legend": {
                  "calcs": [
                    "lastNotNull",
                    "max",
                    "mean"
                  ],
                  "displayMode": "table",
                  "enableFacetedFilter": false,
                  "overflow": "ellipsis",
                  "placement": "right",
                  "showLegend": true,
                  "sortBy": "Max",
                  "sortDesc": true
                },
                "tooltip": {
                  "hideZeros": false,
                  "mode": "single",
                  "sort": "none"
                }
              }
            },
            "version": "13.1.0"
          }
        }
      },
      "panel-27": {
        "kind": "Panel",
        "spec": {
          "data": {
            "kind": "QueryGroup",
            "spec": {
              "queries": [
                {
                  "kind": "PanelQuery",
                  "spec": {
                    "hidden": false,
                    "query": {
                      "datasource": {
                        "name": "prometheus"
                      },
                      "group": "prometheus",
                      "kind": "DataQuery",
                      "spec": {
                        "editorMode": "code",
                        "expr": "controller_runtime_max_concurrent_reconciles{job=~\"$job\"}",
                        "legendFormat": "{{controller}}",
                        "range": true
                      },
                      "version": "v0"
                    },
                    "refId": "A"
                  }
                }
              ],
              "queryOptions": {},
              "transformations": []
            }
          },
          "description": "Max Concurrent Reconciles",
          "id": 27,
          "links": [],
          "title": "Max Workers",
          "vizConfig": {
            "group": "timeseries",
            "kind": "VizConfig",
            "spec": {
              "fieldConfig": {
                "defaults": {
                  "color": {
                    "mode": "palette-classic"
                  },
                  "custom": {
                    "axisBorderShow": false,
                    "axisCenteredZero": false,
                    "axisColorMode": "text",
                    "axisLabel": "",
                    "axisPlacement": "auto",
                    "barAlignment": 0,
                    "barWidthFactor": 0.6,
                    "drawStyle": "line",
                    "fillOpacity": 0,
                    "gradientMode": "none",
                    "hideFrom": {
                      "legend": false,
                      "tooltip": false,
                      "viz": false
                    },
                    "insertNulls": false,
                    "lineInterpolation": "linear",
                    "lineWidth": 1,
                    "pointSize": 5,
                    "scaleDistribution": {
                      "type": "linear"
                    },
                    "showPoints": "auto",
                    "showValues": false,
                    "spanNulls": false,
                    "stacking": {
                      "group": "A",
                      "mode": "none"
                    },
                    "thresholdsStyle": {
                      "mode": "off"
                    }
                  },
                  "thresholds": {
                    "mode": "absolute",
                    "steps": [
                      {
                        "color": "green",
                        "value": 0
                      },
                      {
                        "color": "red",
                        "value": 80
                      }
                    ]
                  },
                  "unit": "short"
                },
                "overrides": []
              },
              "options": {
                "annotations": {
                  "clustering": -1,
                  "multiLane": false
                },
                "legend": {
                  "calcs": [
                    "lastNotNull"
                  ],
                  "displayMode": "table",
                  "enableFacetedFilter": false,
                  "overflow": "ellipsis",
                  "placement": "right",
                  "showLegend": true
                },
                "tooltip": {
                  "hideZeros": false,
                  "mode": "single",
                  "sort": "none"
                }
              }
            },
            "version": "13.1.0"
          }
        }
      },
      "panel-28": {
        "kind": "Panel",
        "spec": {
          "data": {
            "kind": "QueryGroup",
            "spec": {
              "queries": [
                {
                  "kind": "PanelQuery",
                  "spec": {
                    "hidden": false,
                    "query": {
                      "datasource": {
                        "name": "prometheus"
                      },
                      "group": "prometheus",
                      "kind": "DataQuery",
                      "spec": {
                        "editorMode": "code",
                        "expr": "controller_runtime_active_workers{job=~\"$job\"}\n/\ncontroller_runtime_max_concurrent_reconciles{job=~\"$job\"}",
                        "legendFormat": "{{controller}}",
                        "range": true
                      },
                      "version": "v0"
                    },
                    "refId": "A"
                  }
                }
              ],
              "queryOptions": {},
              "transformations": []
            }
          },
          "description": "If this is close to 1 while queue depth and queue wait rise, the controller is worker-saturated.",
          "id": 28,
          "links": [],
          "title": "Worker Saturation",
          "vizConfig": {
            "group": "timeseries",
            "kind": "VizConfig",
            "spec": {
              "fieldConfig": {
                "defaults": {
                  "color": {
                    "mode": "palette-classic"
                  },
                  "custom": {
                    "axisBorderShow": false,
                    "axisCenteredZero": false,
                    "axisColorMode": "text",
                    "axisLabel": "",
                    "axisPlacement": "auto",
                    "axisSoftMax": 1,
                    "axisSoftMin": 0,
                    "barAlignment": 0,
                    "barWidthFactor": 0.6,
                    "drawStyle": "line",
                    "fillOpacity": 0,
                    "gradientMode": "none",
                    "hideFrom": {
                      "legend": false,
                      "tooltip": false,
                      "viz": false
                    },
                    "insertNulls": false,
                    "lineInterpolation": "linear",
                    "lineWidth": 1,
                    "pointSize": 5,
                    "scaleDistribution": {
                      "type": "linear"
                    },
                    "showPoints": "auto",
                    "showValues": false,
                    "spanNulls": false,
                    "stacking": {
                      "group": "A",
                      "mode": "none"
                    },
                    "thresholdsStyle": {
                      "mode": "dashed"
                    }
                  },
                  "thresholds": {
                    "mode": "absolute",
                    "steps": [
                      {
                        "color": "green",
                        "value": 0
                      },
                      {
                        "color": "orange",
                        "value": 0.8
                      },
                      {
                        "color": "red",
                        "value": 0.95
                      }
                    ]
                  },
                  "unit": "percentunit"
                },
                "overrides": []
              },
              "options": {
                "annotations": {
                  "clustering": -1,
                  "multiLane": false
                },
                "legend": {
                  "calcs": [
                    "lastNotNull",
                    "max",
                    "mean"
                  ],
                  "displayMode": "table",
                  "enableFacetedFilter": false,
                  "overflow": "ellipsis",
                  "placement": "right",
                  "showLegend": true,
                  "sortBy": "Max",
                  "sortDesc": true
                },
                "tooltip": {
                  "hideZeros": false,
                  "mode": "single",
                  "sort": "none"
                }
              }
            },
            "version": "13.1.0"
          }
        }
      },
      "panel-29": {
        "kind": "Panel",
        "spec": {
          "data": {
            "kind": "QueryGroup",
            "spec": {
              "queries": [
                {
                  "kind": "PanelQuery",
                  "spec": {
                    "hidden": false,
                    "query": {
                      "datasource": {
                        "name": "prometheus"
                      },
                      "group": "prometheus",
                      "kind": "DataQuery",
                      "spec": {
                        "editorMode": "code",
                        "expr": "workqueue_longest_running_processor_seconds{job=~\"$job\"}",
                        "legendFormat": "{{controller}}",
                        "range": true
                      },
                      "version": "v0"
                    },
                    "refId": "A"
                  }
                }
              ],
              "queryOptions": {},
              "transformations": []
            }
          },
          "description": "This grows when work is still in-flight. If it grows continuously, suspect stuck reconciles or long API calls.",
          "id": 29,
          "links": [],
          "title": "Longest Running Processor",
          "vizConfig": {
            "group": "timeseries",
            "kind": "VizConfig",
            "spec": {
              "fieldConfig": {
                "defaults": {
                  "color": {
                    "mode": "palette-classic"
                  },
                  "custom": {
                    "axisBorderShow": false,
                    "axisCenteredZero": false,
                    "axisColorMode": "text",
                    "axisLabel": "",
                    "axisPlacement": "auto",
                    "barAlignment": 0,
                    "barWidthFactor": 0.6,
                    "drawStyle": "line",
                    "fillOpacity": 0,
                    "gradientMode": "none",
                    "hideFrom": {
                      "legend": false,
                      "tooltip": false,
                      "viz": false
                    },
                    "insertNulls": false,
                    "lineInterpolation": "linear",
                    "lineWidth": 1,
                    "pointSize": 5,
                    "scaleDistribution": {
                      "type": "linear"
                    },
                    "showPoints": "auto",
                    "showValues": false,
                    "spanNulls": false,
                    "stacking": {
                      "group": "A",
                      "mode": "none"
                    },
                    "thresholdsStyle": {
                      "mode": "dashed"
                    }
                  },
                  "thresholds": {
                    "mode": "absolute",
                    "steps": [
                      {
                        "color": "green",
                        "value": 0
                      },
                      {
                        "color": "orange",
                        "value": 60
                      },
                      {
                        "color": "red",
                        "value": 300
                      }
                    ]
                  },
                  "unit": "s"
                },
                "overrides": []
              },
              "options": {
                "annotations": {
                  "clustering": -1,
                  "multiLane": false
                },
                "legend": {
                  "calcs": [
                    "lastNotNull",
                    "max",
                    "mean"
                  ],
                  "displayMode": "table",
                  "enableFacetedFilter": false,
                  "overflow": "ellipsis",
                  "placement": "right",
                  "showLegend": true,
                  "sortBy": "Max",
                  "sortDesc": true
                },
                "tooltip": {
                  "hideZeros": false,
                  "mode": "single",
                  "sort": "none"
                }
              }
            },
            "version": "13.1.0"
          }
        }
      },
      "panel-30": {
        "kind": "Panel",
        "spec": {
          "data": {
            "kind": "QueryGroup",
            "spec": {
              "queries": [
                {
                  "kind": "PanelQuery",
                  "spec": {
                    "hidden": false,
                    "query": {
                      "datasource": {
                        "name": "prometheus"
                      },
                      "group": "prometheus",
                      "kind": "DataQuery",
                      "spec": {
                        "editorMode": "code",
                        "expr": "workqueue_unfinished_work_seconds{job=~\"$job\"}",
                        "legendFormat": "{{controller}}",
                        "range": true
                      },
                      "version": "v0"
                    },
                    "refId": "A"
                  }
                }
              ],
              "queryOptions": {},
              "transformations": []
            }
          },
          "description": "This grows when work is still in-flight. If it grows continuously, suspect stuck reconciles or long API calls.",
          "id": 30,
          "links": [],
          "title": "Unfinished Work Seconds",
          "vizConfig": {
            "group": "timeseries",
            "kind": "VizConfig",
            "spec": {
              "fieldConfig": {
                "defaults": {
                  "color": {
                    "mode": "palette-classic"
                  },
                  "custom": {
                    "axisBorderShow": false,
                    "axisCenteredZero": false,
                    "axisColorMode": "text",
                    "axisLabel": "",
                    "axisPlacement": "auto",
                    "barAlignment": 0,
                    "barWidthFactor": 0.6,
                    "drawStyle": "line",
                    "fillOpacity": 0,
                    "gradientMode": "none",
                    "hideFrom": {
                      "legend": false,
                      "tooltip": false,
                      "viz": false
                    },
                    "insertNulls": false,
                    "lineInterpolation": "linear",
                    "lineWidth": 1,
                    "pointSize": 5,
                    "scaleDistribution": {
                      "type": "linear"
                    },
                    "showPoints": "auto",
                    "showValues": false,
                    "spanNulls": false,
                    "stacking": {
                      "group": "A",
                      "mode": "none"
                    },
                    "thresholdsStyle": {
                      "mode": "dashed"
                    }
                  },
                  "thresholds": {
                    "mode": "absolute",
                    "steps": [
                      {
                        "color": "green",
                        "value": 0
                      },
                      {
                        "color": "orange",
                        "value": 300
                      },
                      {
                        "color": "red",
                        "value": 500
                      }
                    ]
                  },
                  "unit": "s"
                },
                "overrides": []
              },
              "options": {
                "annotations": {
                  "clustering": -1,
                  "multiLane": false
                },
                "legend": {
                  "calcs": [
                    "lastNotNull",
                    "max",
                    "mean"
                  ],
                  "displayMode": "table",
                  "enableFacetedFilter": false,
                  "overflow": "ellipsis",
                  "placement": "right",
                  "showLegend": true,
                  "sortBy": "Max",
                  "sortDesc": true
                },
                "tooltip": {
                  "hideZeros": false,
                  "mode": "single",
                  "sort": "none"
                }
              }
            },
            "version": "13.1.0"
          }
        }
      },
      "panel-31": {
        "kind": "Panel",
        "spec": {
          "data": {
            "kind": "QueryGroup",
            "spec": {
              "queries": [
                {
                  "kind": "PanelQuery",
                  "spec": {
                    "hidden": false,
                    "query": {
                      "datasource": {
                        "name": "prometheus"
                      },
                      "group": "prometheus",
                      "kind": "DataQuery",
                      "spec": {
                        "editorMode": "code",
                        "expr": "sum by (method, code) (\n  rate(rest_client_requests_total{job=~\"$job\"}[$__rate_interval])\n)",
                        "legendFormat": "{{method}} {{code}}",
                        "range": true
                      },
                      "version": "v0"
                    },
                    "refId": "A"
                  }
                }
              ],
              "queryOptions": {},
              "transformations": []
            }
          },
          "description": "This is useful to see whether Capsule is generating a high API call rate.",
          "id": 31,
          "links": [],
          "title": "Kubernetes API Client Requests",
          "vizConfig": {
            "group": "timeseries",
            "kind": "VizConfig",
            "spec": {
              "fieldConfig": {
                "defaults": {
                  "color": {
                    "mode": "palette-classic"
                  },
                  "custom": {
                    "axisBorderShow": false,
                    "axisCenteredZero": false,
                    "axisColorMode": "text",
                    "axisLabel": "",
                    "axisPlacement": "auto",
                    "barAlignment": 0,
                    "barWidthFactor": 0.6,
                    "drawStyle": "line",
                    "fillOpacity": 0,
                    "gradientMode": "none",
                    "hideFrom": {
                      "legend": false,
                      "tooltip": false,
                      "viz": false
                    },
                    "insertNulls": false,
                    "lineInterpolation": "linear",
                    "lineWidth": 1,
                    "pointSize": 5,
                    "scaleDistribution": {
                      "type": "linear"
                    },
                    "showPoints": "auto",
                    "showValues": false,
                    "spanNulls": false,
                    "stacking": {
                      "group": "A",
                      "mode": "none"
                    },
                    "thresholdsStyle": {
                      "mode": "off"
                    }
                  },
                  "thresholds": {
                    "mode": "absolute",
                    "steps": [
                      {
                        "color": "green",
                        "value": 0
                      },
                      {
                        "color": "red",
                        "value": 80
                      }
                    ]
                  },
                  "unit": "ops"
                },
                "overrides": []
              },
              "options": {
                "annotations": {
                  "clustering": -1,
                  "multiLane": false
                },
                "legend": {
                  "calcs": [
                    "last",
                    "max",
                    "mean"
                  ],
                  "displayMode": "table",
                  "enableFacetedFilter": false,
                  "overflow": "ellipsis",
                  "placement": "right",
                  "showLegend": true,
                  "sortBy": "Max",
                  "sortDesc": true
                },
                "tooltip": {
                  "hideZeros": false,
                  "mode": "single",
                  "sort": "none"
                }
              }
            },
            "version": "13.1.0"
          }
        }
      },
      "panel-32": {
        "kind": "Panel",
        "spec": {
          "data": {
            "kind": "QueryGroup",
            "spec": {
              "queries": [
                {
                  "kind": "PanelQuery",
                  "spec": {
                    "hidden": false,
                    "query": {
                      "datasource": {
                        "name": "prometheus"
                      },
                      "group": "prometheus",
                      "kind": "DataQuery",
                      "spec": {
                        "editorMode": "code",
                        "expr": "sum by (method, code) (\n  rate(rest_client_requests_total{\n    job=~\"$job\",\n    code=\"429\"\n  }[$__rate_interval])\n)",
                        "legendFormat": "{{method}} {{code}}",
                        "range": true
                      },
                      "version": "v0"
                    },
                    "refId": "A"
                  }
                }
              ],
              "queryOptions": {},
              "transformations": []
            }
          },
          "description": "",
          "id": 32,
          "links": [],
          "title": "Kubernetes API Client Errors",
          "vizConfig": {
            "group": "timeseries",
            "kind": "VizConfig",
            "spec": {
              "fieldConfig": {
                "defaults": {
                  "color": {
                    "mode": "palette-classic"
                  },
                  "custom": {
                    "axisBorderShow": false,
                    "axisCenteredZero": false,
                    "axisColorMode": "text",
                    "axisLabel": "",
                    "axisPlacement": "auto",
                    "barAlignment": 0,
                    "barWidthFactor": 0.6,
                    "drawStyle": "line",
                    "fillOpacity": 0,
                    "gradientMode": "none",
                    "hideFrom": {
                      "legend": false,
                      "tooltip": false,
                      "viz": false
                    },
                    "insertNulls": false,
                    "lineInterpolation": "linear",
                    "lineWidth": 1,
                    "pointSize": 5,
                    "scaleDistribution": {
                      "type": "linear"
                    },
                    "showPoints": "auto",
                    "showValues": false,
                    "spanNulls": false,
                    "stacking": {
                      "group": "A",
                      "mode": "none"
                    },
                    "thresholdsStyle": {
                      "mode": "off"
                    }
                  },
                  "thresholds": {
                    "mode": "absolute",
                    "steps": [
                      {
                        "color": "green",
                        "value": 0
                      },
                      {
                        "color": "red",
                        "value": 80
                      }
                    ]
                  },
                  "unit": "ops"
                },
                "overrides": []
              },
              "options": {
                "annotations": {
                  "clustering": -1,
                  "multiLane": false
                },
                "legend": {
                  "calcs": [
                    "lastNotNull",
                    "max",
                    "mean"
                  ],
                  "displayMode": "table",
                  "enableFacetedFilter": false,
                  "overflow": "ellipsis",
                  "placement": "right",
                  "showLegend": true
                },
                "tooltip": {
                  "hideZeros": false,
                  "mode": "single",
                  "sort": "none"
                }
              }
            },
            "version": "13.1.0"
          }
        }
      },
      "panel-33": {
        "kind": "Panel",
        "spec": {
          "data": {
            "kind": "QueryGroup",
            "spec": {
              "queries": [
                {
                  "kind": "PanelQuery",
                  "spec": {
                    "hidden": false,
                    "query": {
                      "datasource": {
                        "name": "prometheus"
                      },
                      "group": "prometheus",
                      "kind": "DataQuery",
                      "spec": {
                        "editorMode": "code",
                        "expr": "histogram_quantile(\n  0.95,\n  sum by (method, le) (\n    rate(rest_client_request_duration_seconds_bucket{job=~\"$job\"}[$__rate_interval])\n  )\n)",
                        "legendFormat": "{{method}}",
                        "range": true
                      },
                      "version": "v0"
                    },
                    "refId": "A"
                  }
                }
              ],
              "queryOptions": {},
              "transformations": []
            }
          },
          "description": "",
          "id": 33,
          "links": [],
          "title": "Kubernetes API Client Request Duration p95",
          "vizConfig": {
            "group": "timeseries",
            "kind": "VizConfig",
            "spec": {
              "fieldConfig": {
                "defaults": {
                  "color": {
                    "mode": "palette-classic"
                  },
                  "custom": {
                    "axisBorderShow": false,
                    "axisCenteredZero": false,
                    "axisColorMode": "text",
                    "axisLabel": "",
                    "axisPlacement": "auto",
                    "barAlignment": 0,
                    "barWidthFactor": 0.6,
                    "drawStyle": "line",
                    "fillOpacity": 0,
                    "gradientMode": "none",
                    "hideFrom": {
                      "legend": false,
                      "tooltip": false,
                      "viz": false
                    },
                    "insertNulls": false,
                    "lineInterpolation": "linear",
                    "lineWidth": 1,
                    "pointSize": 5,
                    "scaleDistribution": {
                      "type": "linear"
                    },
                    "showPoints": "auto",
                    "showValues": false,
                    "spanNulls": false,
                    "stacking": {
                      "group": "A",
                      "mode": "none"
                    },
                    "thresholdsStyle": {
                      "mode": "off"
                    }
                  },
                  "thresholds": {
                    "mode": "absolute",
                    "steps": [
                      {
                        "color": "green",
                        "value": 0
                      },
                      {
                        "color": "red",
                        "value": 80
                      }
                    ]
                  },
                  "unit": "s"
                },
                "overrides": []
              },
              "options": {
                "annotations": {
                  "clustering": -1,
                  "multiLane": false
                },
                "legend": {
                  "calcs": [
                    "lastNotNull",
                    "max",
                    "mean"
                  ],
                  "displayMode": "table",
                  "enableFacetedFilter": false,
                  "overflow": "ellipsis",
                  "placement": "right",
                  "showLegend": true
                },
                "tooltip": {
                  "hideZeros": false,
                  "mode": "single",
                  "sort": "none"
                }
              }
            },
            "version": "13.1.0"
          }
        }
      },
      "panel-34": {
        "kind": "Panel",
        "spec": {
          "data": {
            "kind": "QueryGroup",
            "spec": {
              "queries": [
                {
                  "kind": "PanelQuery",
                  "spec": {
                    "hidden": false,
                    "query": {
                      "datasource": {
                        "name": "prometheus"
                      },
                      "group": "prometheus",
                      "kind": "DataQuery",
                      "spec": {
                        "editorMode": "code",
                        "exemplar": false,
                        "expr": "sum by (controller) (\n  increase(controller_runtime_reconcile_total{\n    job=~\"$job\"\n  }[$__range])\n)",
                        "instant": false,
                        "interval": "",
                        "legendFormat": "{{controller}}",
                        "range": true
                      },
                      "version": "v0"
                    },
                    "refId": "A"
                  }
                }
              ],
              "queryOptions": {},
              "transformations": []
            }
          },
          "description": "",
          "id": 34,
          "links": [],
          "title": "Reoncile Rate",
          "vizConfig": {
            "group": "bargauge",
            "kind": "VizConfig",
            "spec": {
              "fieldConfig": {
                "defaults": {
                  "color": {
                    "mode": "continuous-GrYlRd"
                  },
                  "thresholds": {
                    "mode": "absolute",
                    "steps": [
                      {
                        "color": "green",
                        "value": 0
                      },
                      {
                        "color": "red",
                        "value": 80
                      }
                    ]
                  }
                },
                "overrides": []
              },
              "options": {
                "displayMode": "lcd",
                "legend": {
                  "calcs": [],
                  "displayMode": "list",
                  "overflow": "ellipsis",
                  "placement": "bottom",
                  "showLegend": false
                },
                "maxVizHeight": 300,
                "minVizHeight": 16,
                "minVizWidth": 8,
                "namePlacement": "auto",
                "orientation": "horizontal",
                "reduceOptions": {
                  "calcs": [
                    "lastNotNull"
                  ],
                  "fields": "",
                  "values": false
                },
                "showUnfilled": true,
                "sizing": "auto",
                "valueMode": "text"
              }
            },
            "version": "13.1.0"
          }
        }
      },
      "panel-35": {
        "kind": "Panel",
        "spec": {
          "data": {
            "kind": "QueryGroup",
            "spec": {
              "queries": [
                {
                  "kind": "PanelQuery",
                  "spec": {
                    "hidden": false,
                    "query": {
                      "datasource": {
                        "name": "prometheus"
                      },
                      "group": "prometheus",
                      "kind": "DataQuery",
                      "spec": {
                        "editorMode": "code",
                        "expr": "sum by (priority_level, flow_schema) (\n  apiserver_flowcontrol_current_inqueue_requests{\n    flow_schema=~\".*capsule.*\"\n  }\n)",
                        "legendFormat": "__auto",
                        "range": true
                      },
                      "version": "v0"
                    },
                    "refId": "A"
                  }
                }
              ],
              "queryOptions": {},
              "transformations": []
            }
          },
          "description": "",
          "id": 35,
          "links": [],
          "title": "APF Current In-Queue Requests",
          "vizConfig": {
            "group": "timeseries",
            "kind": "VizConfig",
            "spec": {
              "fieldConfig": {
                "defaults": {
                  "color": {
                    "mode": "palette-classic"
                  },
                  "custom": {
                    "axisBorderShow": false,
                    "axisCenteredZero": false,
                    "axisColorMode": "text",
                    "axisLabel": "",
                    "axisPlacement": "auto",
                    "barAlignment": 0,
                    "barWidthFactor": 0.6,
                    "drawStyle": "line",
                    "fillOpacity": 0,
                    "gradientMode": "none",
                    "hideFrom": {
                      "legend": false,
                      "tooltip": false,
                      "viz": false
                    },
                    "insertNulls": false,
                    "lineInterpolation": "linear",
                    "lineWidth": 1,
                    "pointSize": 5,
                    "scaleDistribution": {
                      "type": "linear"
                    },
                    "showPoints": "auto",
                    "showValues": false,
                    "spanNulls": false,
                    "stacking": {
                      "group": "A",
                      "mode": "none"
                    },
                    "thresholdsStyle": {
                      "mode": "off"
                    }
                  },
                  "thresholds": {
                    "mode": "absolute",
                    "steps": [
                      {
                        "color": "green",
                        "value": 0
                      },
                      {
                        "color": "red",
                        "value": 80
                      }
                    ]
                  }
                },
                "overrides": []
              },
              "options": {
                "annotations": {
                  "clustering": -1,
                  "multiLane": false
                },
                "legend": {
                  "calcs": [],
                  "displayMode": "list",
                  "enableFacetedFilter": false,
                  "overflow": "ellipsis",
                  "placement": "bottom",
                  "showLegend": true
                },
                "tooltip": {
                  "hideZeros": false,
                  "mode": "single",
                  "sort": "none"
                }
              }
            },
            "version": "13.1.0"
          }
        }
      },
      "panel-37": {
        "kind": "Panel",
        "spec": {
          "data": {
            "kind": "QueryGroup",
            "spec": {
              "queries": [
                {
                  "kind": "PanelQuery",
                  "spec": {
                    "hidden": false,
                    "query": {
                      "datasource": {
                        "name": "prometheus"
                      },
                      "group": "prometheus",
                      "kind": "DataQuery",
                      "spec": {
                        "editorMode": "code",
                        "expr": "sum by (webhook) (rate(controller_runtime_webhook_requests_in_flight{job=\"$job\"}[$__rate_interval]))",
                        "legendFormat": "__auto",
                        "range": true
                      },
                      "version": "v0"
                    },
                    "refId": "A"
                  }
                }
              ],
              "queryOptions": {},
              "transformations": []
            }
          },
          "description": "",
          "id": 37,
          "links": [],
          "title": "Requests In-Flight",
          "vizConfig": {
            "group": "timeseries",
            "kind": "VizConfig",
            "spec": {
              "fieldConfig": {
                "defaults": {
                  "color": {
                    "mode": "palette-classic"
                  },
                  "custom": {
                    "axisBorderShow": false,
                    "axisCenteredZero": false,
                    "axisColorMode": "text",
                    "axisLabel": "",
                    "axisPlacement": "auto",
                    "barAlignment": 0,
                    "barWidthFactor": 0.6,
                    "drawStyle": "line",
                    "fillOpacity": 0,
                    "gradientMode": "none",
                    "hideFrom": {
                      "legend": false,
                      "tooltip": false,
                      "viz": false
                    },
                    "insertNulls": false,
                    "lineInterpolation": "linear",
                    "lineWidth": 1,
                    "pointSize": 5,
                    "scaleDistribution": {
                      "type": "linear"
                    },
                    "showPoints": "auto",
                    "showValues": false,
                    "spanNulls": false,
                    "stacking": {
                      "group": "A",
                      "mode": "none"
                    },
                    "thresholdsStyle": {
                      "mode": "off"
                    }
                  },
                  "thresholds": {
                    "mode": "absolute",
                    "steps": [
                      {
                        "color": "green",
                        "value": 0
                      },
                      {
                        "color": "red",
                        "value": 80
                      }
                    ]
                  }
                },
                "overrides": []
              },
              "options": {
                "annotations": {
                  "clustering": -1,
                  "multiLane": false
                },
                "legend": {
                  "calcs": [
                    "last",
                    "max",
                    "mean"
                  ],
                  "displayMode": "table",
                  "enableFacetedFilter": false,
                  "overflow": "ellipsis",
                  "placement": "right",
                  "showLegend": true,
                  "sortBy": "Last",
                  "sortDesc": true
                },
                "tooltip": {
                  "hideZeros": false,
                  "mode": "single",
                  "sort": "none"
                }
              }
            },
            "version": "13.1.0"
          }
        }
      },
      "panel-38": {
        "kind": "Panel",
        "spec": {
          "data": {
            "kind": "QueryGroup",
            "spec": {
              "queries": [
                {
                  "kind": "PanelQuery",
                  "spec": {
                    "hidden": false,
                    "query": {
                      "datasource": {
                        "name": "prometheus"
                      },
                      "group": "prometheus",
                      "kind": "DataQuery",
                      "spec": {
                        "editorMode": "code",
                        "expr": "sum by (webhook) (rate(controller_runtime_webhook_requests_total{job=\"$job\"}[$__rate_interval]))",
                        "legendFormat": "__auto",
                        "range": true
                      },
                      "version": "v0"
                    },
                    "refId": "A"
                  }
                }
              ],
              "queryOptions": {},
              "transformations": []
            }
          },
          "description": "",
          "id": 38,
          "links": [],
          "title": "Admission Requests",
          "vizConfig": {
            "group": "timeseries",
            "kind": "VizConfig",
            "spec": {
              "fieldConfig": {
                "defaults": {
                  "color": {
                    "mode": "palette-classic"
                  },
                  "custom": {
                    "axisBorderShow": false,
                    "axisCenteredZero": false,
                    "axisColorMode": "text",
                    "axisLabel": "",
                    "axisPlacement": "auto",
                    "barAlignment": 0,
                    "barWidthFactor": 0.6,
                    "drawStyle": "line",
                    "fillOpacity": 0,
                    "gradientMode": "none",
                    "hideFrom": {
                      "legend": false,
                      "tooltip": false,
                      "viz": false
                    },
                    "insertNulls": false,
                    "lineInterpolation": "linear",
                    "lineWidth": 1,
                    "pointSize": 5,
                    "scaleDistribution": {
                      "type": "linear"
                    },
                    "showPoints": "auto",
                    "showValues": false,
                    "spanNulls": false,
                    "stacking": {
                      "group": "A",
                      "mode": "none"
                    },
                    "thresholdsStyle": {
                      "mode": "off"
                    }
                  },
                  "thresholds": {
                    "mode": "absolute",
                    "steps": [
                      {
                        "color": "green",
                        "value": 0
                      },
                      {
                        "color": "red",
                        "value": 80
                      }
                    ]
                  }
                },
                "overrides": []
              },
              "options": {
                "annotations": {
                  "clustering": -1,
                  "multiLane": false
                },
                "legend": {
                  "calcs": [
                    "last",
                    "max",
                    "mean"
                  ],
                  "displayMode": "table",
                  "enableFacetedFilter": false,
                  "overflow": "ellipsis",
                  "placement": "right",
                  "showLegend": true,
                  "sortBy": "Last",
                  "sortDesc": true
                },
                "tooltip": {
                  "hideZeros": false,
                  "mode": "single",
                  "sort": "none"
                }
              }
            },
            "version": "13.1.0"
          }
        }
      },
      "panel-39": {
        "kind": "Panel",
        "spec": {
          "data": {
            "kind": "QueryGroup",
            "spec": {
              "queries": [
                {
                  "kind": "PanelQuery",
                  "spec": {
                    "hidden": false,
                    "query": {
                      "datasource": {
                        "name": "prometheus"
                      },
                      "group": "prometheus",
                      "kind": "DataQuery",
                      "spec": {
                        "editorMode": "code",
                        "expr": "sum by (webhook) (\n  rate(controller_runtime_webhook_latency_seconds_sum{job=\"$job\"}[5m])\n)\n/\nsum by (webhook) (\n  rate(controller_runtime_webhook_latency_seconds_count{job=\"$job\"}[5m])\n)",
                        "legendFormat": "__auto",
                        "range": true
                      },
                      "version": "v0"
                    },
                    "refId": "A"
                  }
                }
              ],
              "queryOptions": {},
              "transformations": []
            }
          },
          "description": "",
          "id": 39,
          "links": [],
          "title": "Webhook Latency Average",
          "vizConfig": {
            "group": "timeseries",
            "kind": "VizConfig",
            "spec": {
              "fieldConfig": {
                "defaults": {
                  "color": {
                    "mode": "palette-classic"
                  },
                  "custom": {
                    "axisBorderShow": false,
                    "axisCenteredZero": false,
                    "axisColorMode": "text",
                    "axisLabel": "",
                    "axisPlacement": "auto",
                    "barAlignment": 0,
                    "barWidthFactor": 0.6,
                    "drawStyle": "line",
                    "fillOpacity": 0,
                    "gradientMode": "none",
                    "hideFrom": {
                      "legend": false,
                      "tooltip": false,
                      "viz": false
                    },
                    "insertNulls": false,
                    "lineInterpolation": "linear",
                    "lineWidth": 1,
                    "pointSize": 5,
                    "scaleDistribution": {
                      "type": "linear"
                    },
                    "showPoints": "auto",
                    "showValues": false,
                    "spanNulls": false,
                    "stacking": {
                      "group": "A",
                      "mode": "none"
                    },
                    "thresholdsStyle": {
                      "mode": "off"
                    }
                  },
                  "thresholds": {
                    "mode": "absolute",
                    "steps": [
                      {
                        "color": "green",
                        "value": 0
                      },
                      {
                        "color": "red",
                        "value": 80
                      }
                    ]
                  },
                  "unit": "s"
                },
                "overrides": []
              },
              "options": {
                "annotations": {
                  "clustering": -1,
                  "multiLane": false
                },
                "legend": {
                  "calcs": [
                    "last",
                    "max",
                    "mean"
                  ],
                  "displayMode": "table",
                  "enableFacetedFilter": false,
                  "overflow": "ellipsis",
                  "placement": "right",
                  "showLegend": true,
                  "sortBy": "Max",
                  "sortDesc": true
                },
                "tooltip": {
                  "hideZeros": false,
                  "mode": "single",
                  "sort": "none"
                }
              }
            },
            "version": "13.1.0"
          }
        }
      },
      "panel-4": {
        "kind": "Panel",
        "spec": {
          "data": {
            "kind": "QueryGroup",
            "spec": {
              "queries": [
                {
                  "kind": "PanelQuery",
                  "spec": {
                    "hidden": false,
                    "query": {
                      "datasource": {
                        "name": "prometheus"
                      },
                      "group": "prometheus",
                      "kind": "DataQuery",
                      "spec": {
                        "editorMode": "code",
                        "exemplar": false,
                        "expr": "sum by (result) (\n  increase(controller_runtime_reconcile_total{job=~\"$job\"}[$__rate_interval])\n)",
                        "instant": false,
                        "interval": "",
                        "legendFormat": "{{result}}",
                        "range": true
                      },
                      "version": "v0"
                    },
                    "refId": "A"
                  }
                },
                {
                  "kind": "PanelQuery",
                  "spec": {
                    "hidden": false,
                    "query": {
                      "datasource": {
                        "name": "prometheus"
                      },
                      "group": "prometheus",
                      "kind": "DataQuery",
                      "spec": {
                        "editorMode": "code",
                        "expr": "sum(\n  increase(controller_runtime_reconcile_total{job=~\"$job\"}[$__rate_interval])\n)",
                        "instant": false,
                        "legendFormat": "Total",
                        "range": true
                      },
                      "version": "v0"
                    },
                    "refId": "B"
                  }
                }
              ],
              "queryOptions": {},
              "transformations": []
            }
          },
          "description": "",
          "id": 4,
          "links": [],
          "title": "Reoncile Rate",
          "vizConfig": {
            "group": "timeseries",
            "kind": "VizConfig",
            "spec": {
              "fieldConfig": {
                "defaults": {
                  "color": {
                    "mode": "palette-classic"
                  },
                  "custom": {
                    "axisBorderShow": false,
                    "axisCenteredZero": false,
                    "axisColorMode": "text",
                    "axisLabel": "",
                    "axisPlacement": "auto",
                    "barAlignment": 0,
                    "barWidthFactor": 0.6,
                    "drawStyle": "line",
                    "fillOpacity": 0,
                    "gradientMode": "none",
                    "hideFrom": {
                      "legend": false,
                      "tooltip": false,
                      "viz": false
                    },
                    "insertNulls": false,
                    "lineInterpolation": "linear",
                    "lineWidth": 1,
                    "pointSize": 5,
                    "scaleDistribution": {
                      "type": "linear"
                    },
                    "showPoints": "auto",
                    "showValues": false,
                    "spanNulls": false,
                    "stacking": {
                      "group": "A",
                      "mode": "normal"
                    },
                    "thresholdsStyle": {
                      "mode": "off"
                    }
                  },
                  "thresholds": {
                    "mode": "absolute",
                    "steps": [
                      {
                        "color": "green",
                        "value": 0
                      },
                      {
                        "color": "red",
                        "value": 80
                      }
                    ]
                  }
                },
                "overrides": []
              },
              "options": {
                "annotations": {
                  "clustering": -1,
                  "multiLane": false
                },
                "legend": {
                  "calcs": [
                    "lastNotNull"
                  ],
                  "displayMode": "table",
                  "enableFacetedFilter": false,
                  "overflow": "ellipsis",
                  "placement": "right",
                  "showLegend": true,
                  "sortBy": "Last *",
                  "sortDesc": true
                },
                "tooltip": {
                  "hideZeros": false,
                  "mode": "single",
                  "sort": "none"
                }
              }
            },
            "version": "13.1.0"
          }
        }
      },
      "panel-40": {
        "kind": "Panel",
        "spec": {
          "data": {
            "kind": "QueryGroup",
            "spec": {
              "queries": [
                {
                  "kind": "PanelQuery",
                  "spec": {
                    "hidden": false,
                    "query": {
                      "datasource": {
                        "name": "prometheus"
                      },
                      "group": "prometheus",
                      "kind": "DataQuery",
                      "spec": {
                        "editorMode": "code",
                        "expr": "histogram_quantile(\n  0.99,\n  sum by (le, webhook) (rate(controller_runtime_webhook_latency_seconds_bucket{job=\"$job\"}[5m]))\n)",
                        "legendFormat": "__auto",
                        "range": true
                      },
                      "version": "v0"
                    },
                    "refId": "A"
                  }
                }
              ],
              "queryOptions": {},
              "transformations": []
            }
          },
          "description": "",
          "id": 40,
          "links": [],
          "title": "Webhook Latency (P99)",
          "vizConfig": {
            "group": "timeseries",
            "kind": "VizConfig",
            "spec": {
              "fieldConfig": {
                "defaults": {
                  "color": {
                    "mode": "palette-classic"
                  },
                  "custom": {
                    "axisBorderShow": false,
                    "axisCenteredZero": false,
                    "axisColorMode": "text",
                    "axisLabel": "",
                    "axisPlacement": "auto",
                    "barAlignment": 0,
                    "barWidthFactor": 0.6,
                    "drawStyle": "line",
                    "fillOpacity": 0,
                    "gradientMode": "none",
                    "hideFrom": {
                      "legend": false,
                      "tooltip": false,
                      "viz": false
                    },
                    "insertNulls": false,
                    "lineInterpolation": "linear",
                    "lineWidth": 1,
                    "pointSize": 5,
                    "scaleDistribution": {
                      "type": "linear"
                    },
                    "showPoints": "auto",
                    "showValues": false,
                    "spanNulls": false,
                    "stacking": {
                      "group": "A",
                      "mode": "none"
                    },
                    "thresholdsStyle": {
                      "mode": "off"
                    }
                  },
                  "thresholds": {
                    "mode": "absolute",
                    "steps": [
                      {
                        "color": "green",
                        "value": 0
                      },
                      {
                        "color": "red",
                        "value": 80
                      }
                    ]
                  },
                  "unit": "s"
                },
                "overrides": []
              },
              "options": {
                "annotations": {
                  "clustering": -1,
                  "multiLane": false
                },
                "legend": {
                  "calcs": [
                    "last",
                    "max",
                    "mean"
                  ],
                  "displayMode": "table",
                  "enableFacetedFilter": false,
                  "overflow": "ellipsis",
                  "placement": "right",
                  "showLegend": true,
                  "sortBy": "Max",
                  "sortDesc": true
                },
                "tooltip": {
                  "hideZeros": false,
                  "mode": "single",
                  "sort": "none"
                }
              }
            },
            "version": "13.1.0"
          }
        }
      },
      "panel-41": {
        "kind": "Panel",
        "spec": {
          "data": {
            "kind": "QueryGroup",
            "spec": {
              "queries": [
                {
                  "kind": "PanelQuery",
                  "spec": {
                    "hidden": false,
                    "query": {
                      "datasource": {
                        "name": "prometheus"
                      },
                      "group": "prometheus",
                      "kind": "DataQuery",
                      "spec": {
                        "editorMode": "code",
                        "expr": "histogram_quantile(\n  0.95,\n  sum by (le, webhook) (rate(controller_runtime_webhook_latency_seconds_bucket{job=\"$job\"}[5m]))\n)",
                        "legendFormat": "__auto",
                        "range": true
                      },
                      "version": "v0"
                    },
                    "refId": "A"
                  }
                }
              ],
              "queryOptions": {},
              "transformations": []
            }
          },
          "description": "",
          "id": 41,
          "links": [],
          "title": "Webhook Latency (P95)",
          "vizConfig": {
            "group": "timeseries",
            "kind": "VizConfig",
            "spec": {
              "fieldConfig": {
                "defaults": {
                  "color": {
                    "mode": "palette-classic"
                  },
                  "custom": {
                    "axisBorderShow": false,
                    "axisCenteredZero": false,
                    "axisColorMode": "text",
                    "axisLabel": "",
                    "axisPlacement": "auto",
                    "barAlignment": 0,
                    "barWidthFactor": 0.6,
                    "drawStyle": "line",
                    "fillOpacity": 0,
                    "gradientMode": "none",
                    "hideFrom": {
                      "legend": false,
                      "tooltip": false,
                      "viz": false
                    },
                    "insertNulls": false,
                    "lineInterpolation": "linear",
                    "lineWidth": 1,
                    "pointSize": 5,
                    "scaleDistribution": {
                      "type": "linear"
                    },
                    "showPoints": "auto",
                    "showValues": false,
                    "spanNulls": false,
                    "stacking": {
                      "group": "A",
                      "mode": "none"
                    },
                    "thresholdsStyle": {
                      "mode": "off"
                    }
                  },
                  "thresholds": {
                    "mode": "absolute",
                    "steps": [
                      {
                        "color": "green",
                        "value": 0
                      },
                      {
                        "color": "red",
                        "value": 80
                      }
                    ]
                  },
                  "unit": "s"
                },
                "overrides": []
              },
              "options": {
                "annotations": {
                  "clustering": -1,
                  "multiLane": false
                },
                "legend": {
                  "calcs": [
                    "last",
                    "max",
                    "mean"
                  ],
                  "displayMode": "table",
                  "enableFacetedFilter": false,
                  "overflow": "ellipsis",
                  "placement": "right",
                  "showLegend": true,
                  "sortBy": "Max",
                  "sortDesc": true
                },
                "tooltip": {
                  "hideZeros": false,
                  "mode": "single",
                  "sort": "none"
                }
              }
            },
            "version": "13.1.0"
          }
        }
      },
      "panel-42": {
        "kind": "Panel",
        "spec": {
          "data": {
            "kind": "QueryGroup",
            "spec": {
              "queries": [
                {
                  "kind": "PanelQuery",
                  "spec": {
                    "hidden": false,
                    "query": {
                      "datasource": {
                        "name": "prometheus"
                      },
                      "group": "prometheus",
                      "kind": "DataQuery",
                      "spec": {
                        "editorMode": "code",
                        "expr": "sum by (webhook) (\n  rate(controller_runtime_webhook_requests_total{job=\"$job\"}[$__rate_interval])\n)",
                        "legendFormat": "{{webhook}}",
                        "range": true
                      },
                      "version": "v0"
                    },
                    "refId": "A"
                  }
                }
              ],
              "queryOptions": {},
              "transformations": []
            }
          },
          "description": "",
          "id": 42,
          "links": [],
          "title": "Webhook Requests / Second",
          "vizConfig": {
            "group": "timeseries",
            "kind": "VizConfig",
            "spec": {
              "fieldConfig": {
                "defaults": {
                  "color": {
                    "mode": "palette-classic"
                  },
                  "custom": {
                    "axisBorderShow": false,
                    "axisCenteredZero": false,
                    "axisColorMode": "text",
                    "axisLabel": "",
                    "axisPlacement": "auto",
                    "barAlignment": 0,
                    "barWidthFactor": 0.6,
                    "drawStyle": "line",
                    "fillOpacity": 0,
                    "gradientMode": "none",
                    "hideFrom": {
                      "legend": false,
                      "tooltip": false,
                      "viz": false
                    },
                    "insertNulls": false,
                    "lineInterpolation": "linear",
                    "lineWidth": 1,
                    "pointSize": 5,
                    "scaleDistribution": {
                      "type": "linear"
                    },
                    "showPoints": "auto",
                    "showValues": false,
                    "spanNulls": false,
                    "stacking": {
                      "group": "A",
                      "mode": "none"
                    },
                    "thresholdsStyle": {
                      "mode": "off"
                    }
                  },
                  "thresholds": {
                    "mode": "absolute",
                    "steps": [
                      {
                        "color": "green",
                        "value": 0
                      },
                      {
                        "color": "red",
                        "value": 80
                      }
                    ]
                  },
                  "unit": "ops"
                },
                "overrides": []
              },
              "options": {
                "annotations": {
                  "clustering": -1,
                  "multiLane": false
                },
                "legend": {
                  "calcs": [
                    "last",
                    "max",
                    "mean"
                  ],
                  "displayMode": "table",
                  "enableFacetedFilter": false,
                  "overflow": "ellipsis",
                  "placement": "right",
                  "showLegend": true,
                  "sortBy": "Last",
                  "sortDesc": true
                },
                "tooltip": {
                  "hideZeros": false,
                  "mode": "single",
                  "sort": "none"
                }
              }
            },
            "version": "13.1.0"
          }
        }
      },
      "panel-43": {
        "kind": "Panel",
        "spec": {
          "data": {
            "kind": "QueryGroup",
            "spec": {
              "queries": [
                {
                  "kind": "PanelQuery",
                  "spec": {
                    "hidden": false,
                    "query": {
                      "datasource": {
                        "name": "prometheus"
                      },
                      "group": "prometheus",
                      "kind": "DataQuery",
                      "spec": {
                        "editorMode": "code",
                        "expr": "topk(\n  10,\n  histogram_quantile(\n    0.99,\n    sum by (name, type, le) (\n      rate(apiserver_admission_webhook_admission_duration_seconds_bucket[$__rate_interval])\n    )\n  )\n)",
                        "legendFormat": "__auto",
                        "range": true
                      },
                      "version": "v0"
                    },
                    "refId": "A"
                  }
                }
              ],
              "queryOptions": {},
              "transformations": []
            }
          },
          "description": "",
          "id": 43,
          "links": [],
          "title": "Slowest Admission Webhooks p99",
          "vizConfig": {
            "group": "bargauge",
            "kind": "VizConfig",
            "spec": {
              "fieldConfig": {
                "defaults": {
                  "color": {
                    "mode": "thresholds"
                  },
                  "thresholds": {
                    "mode": "absolute",
                    "steps": [
                      {
                        "color": "green",
                        "value": 0
                      },
                      {
                        "color": "orange",
                        "value": 1
                      },
                      {
                        "color": "red",
                        "value": 5
                      }
                    ]
                  },
                  "unit": "s"
                },
                "overrides": []
              },
              "options": {
                "displayMode": "lcd",
                "legend": {
                  "calcs": [],
                  "displayMode": "list",
                  "overflow": "ellipsis",
                  "placement": "bottom",
                  "showLegend": false
                },
                "maxVizHeight": 300,
                "minVizHeight": 16,
                "minVizWidth": 8,
                "namePlacement": "top",
                "orientation": "horizontal",
                "reduceOptions": {
                  "calcs": [
                    "lastNotNull"
                  ],
                  "fields": "",
                  "values": false
                },
                "showUnfilled": true,
                "sizing": "auto",
                "valueMode": "text"
              }
            },
            "version": "13.1.0"
          }
        }
      },
      "panel-44": {
        "kind": "Panel",
        "spec": {
          "data": {
            "kind": "QueryGroup",
            "spec": {
              "queries": [
                {
                  "kind": "PanelQuery",
                  "spec": {
                    "hidden": false,
                    "query": {
                      "datasource": {
                        "name": "prometheus"
                      },
                      "group": "prometheus",
                      "kind": "DataQuery",
                      "spec": {
                        "editorMode": "code",
                        "expr": "topk(\n  10,\n  sum by (name, type) (\n    rate(apiserver_admission_webhook_admission_duration_seconds_sum[$__rate_interval])\n  )\n  /\n  sum by (name, type) (\n    rate(apiserver_admission_webhook_admission_duration_seconds_count[$__rate_interval])\n  )\n)",
                        "legendFormat": "__auto",
                        "range": true
                      },
                      "version": "v0"
                    },
                    "refId": "A"
                  }
                }
              ],
              "queryOptions": {},
              "transformations": []
            }
          },
          "description": "",
          "id": 44,
          "links": [],
          "title": "New panel",
          "vizConfig": {
            "group": "bargauge",
            "kind": "VizConfig",
            "spec": {
              "fieldConfig": {
                "defaults": {
                  "color": {
                    "mode": "continuous-GrYlRd"
                  },
                  "thresholds": {
                    "mode": "absolute",
                    "steps": [
                      {
                        "color": "green",
                        "value": 0
                      },
                      {
                        "color": "#EAB839",
                        "value": 0.5
                      },
                      {
                        "color": "red",
                        "value": 2
                      }
                    ]
                  },
                  "unit": "s"
                },
                "overrides": []
              },
              "options": {
                "displayMode": "lcd",
                "legend": {
                  "calcs": [],
                  "displayMode": "list",
                  "overflow": "ellipsis",
                  "placement": "bottom",
                  "showLegend": false
                },
                "maxVizHeight": 300,
                "minVizHeight": 16,
                "minVizWidth": 8,
                "namePlacement": "auto",
                "orientation": "horizontal",
                "reduceOptions": {
                  "calcs": [
                    "lastNotNull"
                  ],
                  "fields": "",
                  "values": false
                },
                "showUnfilled": true,
                "sizing": "auto",
                "valueMode": "text"
              }
            },
            "version": "13.1.0"
          }
        }
      }
    },
    "layout": {
      "kind": "RowsLayout",
      "spec": {
        "rows": [
          {
            "kind": "RowsLayoutRow",
            "spec": {
              "collapse": false,
              "layout": {
                "kind": "RowsLayout",
                "spec": {
                  "rows": [
                    {
                      "kind": "RowsLayoutRow",
                      "spec": {
                        "collapse": false,
                        "layout": {
                          "kind": "GridLayout",
                          "spec": {
                            "items": [
                              {
                                "kind": "GridLayoutItem",
                                "spec": {
                                  "element": {
                                    "kind": "ElementReference",
                                    "name": "panel-2"
                                  },
                                  "height": 8,
                                  "width": 12,
                                  "x": 0,
                                  "y": 0
                                }
                              },
                              {
                                "kind": "GridLayoutItem",
                                "spec": {
                                  "element": {
                                    "kind": "ElementReference",
                                    "name": "panel-28"
                                  },
                                  "height": 8,
                                  "width": 12,
                                  "x": 12,
                                  "y": 0
                                }
                              },
                              {
                                "kind": "GridLayoutItem",
                                "spec": {
                                  "element": {
                                    "kind": "ElementReference",
                                    "name": "panel-27"
                                  },
                                  "height": 8,
                                  "width": 12,
                                  "x": 0,
                                  "y": 8
                                }
                              },
                              {
                                "kind": "GridLayoutItem",
                                "spec": {
                                  "element": {
                                    "kind": "ElementReference",
                                    "name": "panel-4"
                                  },
                                  "height": 8,
                                  "width": 12,
                                  "x": 12,
                                  "y": 8
                                }
                              },
                              {
                                "kind": "GridLayoutItem",
                                "spec": {
                                  "element": {
                                    "kind": "ElementReference",
                                    "name": "panel-25"
                                  },
                                  "height": 8,
                                  "width": 12,
                                  "x": 0,
                                  "y": 16
                                }
                              },
                              {
                                "kind": "GridLayoutItem",
                                "spec": {
                                  "element": {
                                    "kind": "ElementReference",
                                    "name": "panel-26"
                                  },
                                  "height": 8,
                                  "width": 12,
                                  "x": 12,
                                  "y": 16
                                }
                              },
                              {
                                "kind": "GridLayoutItem",
                                "spec": {
                                  "element": {
                                    "kind": "ElementReference",
                                    "name": "panel-34"
                                  },
                                  "height": 12,
                                  "width": 24,
                                  "x": 0,
                                  "y": 24
                                }
                              }
                            ]
                          }
                        },
                        "title": "Workers"
                      }
                    },
                    {
                      "kind": "RowsLayoutRow",
                      "spec": {
                        "collapse": false,
                        "layout": {
                          "kind": "RowsLayout",
                          "spec": {
                            "rows": [
                              {
                                "kind": "RowsLayoutRow",
                                "spec": {
                                  "collapse": false,
                                  "layout": {
                                    "kind": "AutoGridLayout",
                                    "spec": {
                                      "columnWidthMode": "standard",
                                      "items": [
                                        {
                                          "kind": "AutoGridLayoutItem",
                                          "spec": {
                                            "element": {
                                              "kind": "ElementReference",
                                              "name": "panel-38"
                                            }
                                          }
                                        },
                                        {
                                          "kind": "AutoGridLayoutItem",
                                          "spec": {
                                            "element": {
                                              "kind": "ElementReference",
                                              "name": "panel-42"
                                            }
                                          }
                                        },
                                        {
                                          "kind": "AutoGridLayoutItem",
                                          "spec": {
                                            "element": {
                                              "kind": "ElementReference",
                                              "name": "panel-37"
                                            }
                                          }
                                        },
                                        {
                                          "kind": "AutoGridLayoutItem",
                                          "spec": {
                                            "element": {
                                              "kind": "ElementReference",
                                              "name": "panel-39"
                                            }
                                          }
                                        },
                                        {
                                          "kind": "AutoGridLayoutItem",
                                          "spec": {
                                            "element": {
                                              "kind": "ElementReference",
                                              "name": "panel-40"
                                            }
                                          }
                                        },
                                        {
                                          "kind": "AutoGridLayoutItem",
                                          "spec": {
                                            "element": {
                                              "kind": "ElementReference",
                                              "name": "panel-41"
                                            }
                                          }
                                        }
                                      ],
                                      "maxColumnCount": 2,
                                      "rowHeightMode": "standard"
                                    }
                                  },
                                  "title": "Webhooks"
                                }
                              },
                              {
                                "kind": "RowsLayoutRow",
                                "spec": {
                                  "collapse": false,
                                  "layout": {
                                    "kind": "AutoGridLayout",
                                    "spec": {
                                      "columnWidthMode": "standard",
                                      "items": [
                                        {
                                          "kind": "AutoGridLayoutItem",
                                          "spec": {
                                            "element": {
                                              "kind": "ElementReference",
                                              "name": "panel-43"
                                            }
                                          }
                                        },
                                        {
                                          "kind": "AutoGridLayoutItem",
                                          "spec": {
                                            "element": {
                                              "kind": "ElementReference",
                                              "name": "panel-44"
                                            }
                                          }
                                        }
                                      ],
                                      "maxColumnCount": 2,
                                      "rowHeightMode": "tall"
                                    }
                                  },
                                  "title": "Comparison"
                                }
                              }
                            ]
                          }
                        },
                        "title": "Admission"
                      }
                    },
                    {
                      "kind": "RowsLayoutRow",
                      "spec": {
                        "collapse": false,
                        "layout": {
                          "kind": "AutoGridLayout",
                          "spec": {
                            "columnWidthMode": "standard",
                            "items": [],
                            "maxColumnCount": 3,
                            "rowHeightMode": "standard"
                          }
                        },
                        "title": "New row"
                      }
                    },
                    {
                      "kind": "RowsLayoutRow",
                      "spec": {
                        "collapse": true,
                        "layout": {
                          "kind": "RowsLayout",
                          "spec": {
                            "rows": [
                              {
                                "kind": "RowsLayoutRow",
                                "spec": {
                                  "collapse": false,
                                  "layout": {
                                    "kind": "AutoGridLayout",
                                    "spec": {
                                      "columnWidthMode": "standard",
                                      "items": [
                                        {
                                          "kind": "AutoGridLayoutItem",
                                          "spec": {
                                            "element": {
                                              "kind": "ElementReference",
                                              "name": "panel-13"
                                            }
                                          }
                                        },
                                        {
                                          "kind": "AutoGridLayoutItem",
                                          "spec": {
                                            "element": {
                                              "kind": "ElementReference",
                                              "name": "panel-14"
                                            }
                                          }
                                        },
                                        {
                                          "kind": "AutoGridLayoutItem",
                                          "spec": {
                                            "element": {
                                              "kind": "ElementReference",
                                              "name": "panel-21"
                                            }
                                          }
                                        },
                                        {
                                          "kind": "AutoGridLayoutItem",
                                          "spec": {
                                            "element": {
                                              "kind": "ElementReference",
                                              "name": "panel-20"
                                            }
                                          }
                                        },
                                        {
                                          "kind": "AutoGridLayoutItem",
                                          "spec": {
                                            "element": {
                                              "kind": "ElementReference",
                                              "name": "panel-22"
                                            }
                                          }
                                        },
                                        {
                                          "kind": "AutoGridLayoutItem",
                                          "spec": {
                                            "element": {
                                              "kind": "ElementReference",
                                              "name": "panel-24"
                                            }
                                          }
                                        }
                                      ],
                                      "maxColumnCount": 3,
                                      "rowHeightMode": "standard"
                                    }
                                  },
                                  "title": "Depth"
                                }
                              },
                              {
                                "kind": "RowsLayoutRow",
                                "spec": {
                                  "collapse": false,
                                  "layout": {
                                    "kind": "AutoGridLayout",
                                    "spec": {
                                      "columnWidthMode": "standard",
                                      "items": [
                                        {
                                          "kind": "AutoGridLayoutItem",
                                          "spec": {
                                            "element": {
                                              "kind": "ElementReference",
                                              "name": "panel-17"
                                            }
                                          }
                                        },
                                        {
                                          "kind": "AutoGridLayoutItem",
                                          "spec": {
                                            "element": {
                                              "kind": "ElementReference",
                                              "name": "panel-18"
                                            }
                                          }
                                        },
                                        {
                                          "kind": "AutoGridLayoutItem",
                                          "spec": {
                                            "element": {
                                              "kind": "ElementReference",
                                              "name": "panel-19"
                                            }
                                          }
                                        },
                                        {
                                          "kind": "AutoGridLayoutItem",
                                          "spec": {
                                            "element": {
                                              "kind": "ElementReference",
                                              "name": "panel-23"
                                            }
                                          }
                                        },
                                        {
                                          "kind": "AutoGridLayoutItem",
                                          "spec": {
                                            "element": {
                                              "kind": "ElementReference",
                                              "name": "panel-15"
                                            }
                                          }
                                        },
                                        {
                                          "kind": "AutoGridLayoutItem",
                                          "spec": {
                                            "element": {
                                              "kind": "ElementReference",
                                              "name": "panel-16"
                                            }
                                          }
                                        }
                                      ],
                                      "maxColumnCount": 3,
                                      "rowHeightMode": "standard"
                                    }
                                  },
                                  "title": "Work Duration"
                                }
                              },
                              {
                                "kind": "RowsLayoutRow",
                                "spec": {
                                  "collapse": false,
                                  "layout": {
                                    "kind": "AutoGridLayout",
                                    "spec": {
                                      "columnWidthMode": "standard",
                                      "items": [
                                        {
                                          "kind": "AutoGridLayoutItem",
                                          "spec": {
                                            "element": {
                                              "kind": "ElementReference",
                                              "name": "panel-29"
                                            }
                                          }
                                        },
                                        {
                                          "kind": "AutoGridLayoutItem",
                                          "spec": {
                                            "element": {
                                              "kind": "ElementReference",
                                              "name": "panel-30"
                                            }
                                          }
                                        }
                                      ],
                                      "maxColumnCount": 3,
                                      "rowHeightMode": "standard"
                                    }
                                  },
                                  "title": "Unfinished Work"
                                }
                              }
                            ]
                          }
                        },
                        "title": "Workqueue"
                      }
                    },
                    {
                      "kind": "RowsLayoutRow",
                      "spec": {
                        "collapse": true,
                        "layout": {
                          "kind": "RowsLayout",
                          "spec": {
                            "rows": [
                              {
                                "kind": "RowsLayoutRow",
                                "spec": {
                                  "collapse": false,
                                  "layout": {
                                    "kind": "AutoGridLayout",
                                    "spec": {
                                      "columnWidthMode": "standard",
                                      "items": [
                                        {
                                          "kind": "AutoGridLayoutItem",
                                          "spec": {
                                            "element": {
                                              "kind": "ElementReference",
                                              "name": "panel-31"
                                            }
                                          }
                                        },
                                        {
                                          "kind": "AutoGridLayoutItem",
                                          "spec": {
                                            "element": {
                                              "kind": "ElementReference",
                                              "name": "panel-32"
                                            }
                                          }
                                        },
                                        {
                                          "kind": "AutoGridLayoutItem",
                                          "spec": {
                                            "element": {
                                              "kind": "ElementReference",
                                              "name": "panel-33"
                                            }
                                          }
                                        }
                                      ],
                                      "maxColumnCount": 3,
                                      "rowHeightMode": "standard"
                                    }
                                  },
                                  "title": "Controller Client"
                                }
                              },
                              {
                                "kind": "RowsLayoutRow",
                                "spec": {
                                  "collapse": false,
                                  "layout": {
                                    "kind": "AutoGridLayout",
                                    "spec": {
                                      "columnWidthMode": "standard",
                                      "items": [
                                        {
                                          "kind": "AutoGridLayoutItem",
                                          "spec": {
                                            "element": {
                                              "kind": "ElementReference",
                                              "name": "panel-35"
                                            }
                                          }
                                        }
                                      ],
                                      "maxColumnCount": 3,
                                      "rowHeightMode": "standard"
                                    }
                                  },
                                  "title": "Flowschema"
                                }
                              }
                            ]
                          }
                        },
                        "title": "Kubernetes API"
                      }
                    }
                  ]
                }
              },
              "title": "Overview"
            }
          },
          {
            "kind": "RowsLayoutRow",
            "spec": {
              "collapse": false,
              "layout": {
                "kind": "GridLayout",
                "spec": {
                  "items": [
                    {
                      "kind": "GridLayoutItem",
                      "spec": {
                        "element": {
                          "kind": "ElementReference",
                          "name": "panel-10"
                        },
                        "height": 9,
                        "width": 24,
                        "x": 0,
                        "y": 0
                      }
                    },
                    {
                      "kind": "GridLayoutItem",
                      "spec": {
                        "element": {
                          "kind": "ElementReference",
                          "name": "panel-12"
                        },
                        "height": 10,
                        "width": 24,
                        "x": 0,
                        "y": 9
                      }
                    }
                  ]
                }
              },
              "repeat": {
                "mode": "variable",
                "value": "Controller"
              },
              "title": "Controller \"$Controller\" Status"
            }
          }
        ]
      }
    },
    "links": [],
    "liveNow": false,
    "preload": false,
    "tags": [],
    "timeSettings": {
      "autoRefresh": "10s",
      "autoRefreshIntervals": [
        "5s",
        "10s",
        "30s",
        "1m",
        "5m",
        "15m",
        "30m",
        "1h",
        "2h",
        "1d"
      ],
      "fiscalYearStartMonth": 0,
      "from": "now-15m",
      "hideTimepicker": false,
      "timezone": "browser",
      "to": "now"
    },
    "title": "Controller Runtime Controllers",
    "variables": [
      {
        "kind": "QueryVariable",
        "spec": {
          "allowCustomValue": true,
          "current": {
            "text": "All",
            "value": [
              "$__all"
            ]
          },
          "definition": "label_values(controller_runtime_active_workers,job)",
          "hide": "dontHide",
          "includeAll": true,
          "label": "Job",
          "multi": true,
          "name": "job",
          "options": [],
          "query": {
            "datasource": {
              "name": "prometheus"
            },
            "group": "prometheus",
            "kind": "DataQuery",
            "spec": {
              "qryType": 1,
              "query": "label_values(controller_runtime_active_workers,job)",
              "refId": "PrometheusVariableQueryEditor-VariableQuery"
            },
            "version": "v0"
          },
          "refresh": "onTimeRangeChanged",
          "regex": "",
          "regexApplyTo": "value",
          "skipUrlSync": false,
          "sort": "disabled"
        }
      },
      {
        "kind": "QueryVariable",
        "spec": {
          "allowCustomValue": true,
          "current": {
            "text": "All",
            "value": [
              "$__all"
            ]
          },
          "definition": "label_values(controller_runtime_active_workers{job=~\"$job\"},controller)",
          "hide": "dontHide",
          "includeAll": true,
          "multi": true,
          "name": "Controller",
          "options": [],
          "query": {
            "datasource": {
              "name": "prometheus"
            },
            "group": "prometheus",
            "kind": "DataQuery",
            "spec": {
              "qryType": 1,
              "query": "label_values(controller_runtime_active_workers{job=~\"$job\"},controller)",
              "refId": "PrometheusVariableQueryEditor-VariableQuery"
            },
            "version": "v0"
          },
          "refresh": "onDashboardLoad",
          "regex": "",
          "regexApplyTo": "value",
          "skipUrlSync": false,
          "sort": "disabled"
        }
      }
    ]
  }
}
