{
    "__inputs": [],
    "__elements": {},
    "__requires": [
      {
        "type": "grafana",
        "id": "grafana",
        "name": "Grafana",
        "version": "10.4.1"
      },
      {
        "type": "panel",
        "id": "heatmap",
        "name": "Heatmap",
        "version": ""
      },
      {
        "type": "datasource",
        "id": "prometheus",
        "name": "Prometheus",
        "version": "1.0.0"
      },
      {
        "type": "panel",
        "id": "timeseries",
        "name": "Time series",
        "version": ""
      }
    ],
    "annotations": {
      "list": [
        {
          "builtIn": 1,
          "datasource": {
            "type": "grafana",
            "uid": "-- Grafana --"
          },
          "enable": true,
          "hide": true,
          "iconColor": "rgba(0, 211, 255, 1)",
          "name": "Annotations & Alerts",
          "target": {
            "limit": 100,
            "matchAny": false,
            "tags": [],
            "type": "dashboard"
          },
          "type": "dashboard"
        }
      ]
    },
    "editable": true,
    "fiscalYearStartMonth": 0,
    "graphTooltip": 1,
    "id": null,
    "links": [],
    "liveNow": false,
    "panels": [
      {
        "collapsed": false,
        "gridPos": {
          "h": 1,
          "w": 24,
          "x": 0,
          "y": 0
        },
        "id": 14,
        "panels": [],
        "title": "Sliding Sync API",
        "type": "row"
      },
      {
        "datasource": {
          "type": "prometheus",
          "uid": "$datasource"
        },
        "description": "Actively syncing clients i.e the connection hasn't expired yet. ",
        "fieldConfig": {
          "defaults": {
            "color": {
              "mode": "palette-classic"
            },
            "custom": {
              "axisBorderShow": false,
              "axisCenteredZero": false,
              "axisColorMode": "text",
              "axisLabel": "",
              "axisPlacement": "auto",
              "barAlignment": 0,
              "drawStyle": "line",
              "fillOpacity": 0,
              "gradientMode": "none",
              "hideFrom": {
                "legend": false,
                "tooltip": false,
                "viz": false
              },
              "insertNulls": false,
              "lineInterpolation": "linear",
              "lineWidth": 1,
              "pointSize": 5,
              "scaleDistribution": {
                "type": "linear"
              },
              "showPoints": "auto",
              "spanNulls": false,
              "stacking": {
                "group": "A",
                "mode": "none"
              },
              "thresholdsStyle": {
                "mode": "off"
              }
            },
            "mappings": [],
            "min": 0,
            "thresholds": {
              "mode": "absolute",
              "steps": [
                {
                  "color": "green",
                  "value": null
                }
              ]
            },
            "unit": "conns"
          },
          "overrides": [
            {
              "matcher": {
                "id": "byName",
                "options": "Expired Connections (full buffer)"
              },
              "properties": [
                {
                  "id": "custom.axisPlacement",
                  "value": "right"
                },
                {
                  "id": "custom.lineStyle",
                  "value": {
                    "dash": [
                      0,
                      10
                    ],
                    "fill": "dot"
                  }
                },
                {
                  "id": "color",
                  "value": {
                    "fixedColor": "purple",
                    "mode": "fixed"
                  }
                },
                {
                  "id": "custom.showPoints",
                  "value": "always"
                },
                {
                  "id": "custom.lineWidth",
                  "value": 0
                }
              ]
            },
            {
              "matcher": {
                "id": "byName",
                "options": "Expired Connections (timed out)"
              },
              "properties": [
                {
                  "id": "custom.axisPlacement",
                  "value": "right"
                },
                {
                  "id": "color",
                  "value": {
                    "fixedColor": "blue",
                    "mode": "fixed"
                  }
                },
                {
                  "id": "custom.showPoints",
                  "value": "always"
                },
                {
                  "id": "custom.fillOpacity",
                  "value": 0
                },
                {
                  "id": "custom.lineWidth",
                  "value": 0
                }
              ]
            }
          ]
        },
        "gridPos": {
          "h": 8,
          "w": 12,
          "x": 0,
          "y": 1
        },
        "id": 6,
        "options": {
          "legend": {
            "calcs": [],
            "displayMode": "list",
            "placement": "bottom",
            "showLegend": true
          },
          "tooltip": {
            "mode": "single",
            "sort": "none"
          }
        },
        "pluginVersion": "9.5.3",
        "targets": [
          {
            "datasource": {
              "type": "prometheus",
              "uid": "$datasource"
            },
            "disableTextWrap": false,
            "editorMode": "builder",
            "exemplar": false,
            "expr": "sum(sliding_sync_api_num_active_conns)",
            "fullMetaSearch": false,
            "includeNullMetadata": true,
            "instant": false,
            "legendFormat": "Active Connections",
            "range": true,
            "refId": "A",
            "useBackend": false
          },
          {
            "datasource": {
              "type": "prometheus",
              "uid": "$datasource"
            },
            "disableTextWrap": false,
            "editorMode": "builder",
            "expr": "increase(sliding_sync_api_expiry_conn_buffer_full[5m])",
            "fullMetaSearch": false,
            "hide": false,
            "includeNullMetadata": false,
            "legendFormat": "Expired Connections (full buffer)",
            "range": true,
            "refId": "B",
            "useBackend": false
          },
          {
            "datasource": {
              "type": "prometheus",
              "uid": "$datasource"
            },
            "disableTextWrap": false,
            "editorMode": "builder",
            "expr": "increase(sliding_sync_api_expiry_conn_timed_out[5m])",
            "fullMetaSearch": false,
            "hide": false,
            "includeNullMetadata": true,
            "legendFormat": "Expired Connections (timed out)",
            "range": true,
            "refId": "C",
            "useBackend": false
          }
        ],
        "title": "# active sliding sync connections",
        "transformations": [
          {
            "id": "renameByRegex",
            "options": {
              "regex": "sliding-sync-api-(.*)",
              "renamePattern": "$1"
            }
          }
        ],
        "type": "timeseries"
      },
      {
        "datasource": {
          "type": "prometheus",
          "uid": "$datasource"
        },
        "description": "The number of devices blocked on an initial v2 sync. This number should never remain >0 for more than 15 minutes, unless there is a flood of new users to the system.",
        "fieldConfig": {
          "defaults": {
            "color": {
              "mode": "palette-classic"
            },
            "custom": {
              "axisBorderShow": false,
              "axisCenteredZero": false,
              "axisColorMode": "text",
              "axisLabel": "",
              "axisPlacement": "auto",
              "barAlignment": 0,
              "drawStyle": "line",
              "fillOpacity": 0,
              "gradientMode": "none",
              "hideFrom": {
                "legend": false,
                "tooltip": false,
                "viz": false
              },
              "insertNulls": false,
              "lineInterpolation": "linear",
              "lineWidth": 1,
              "pointSize": 5,
              "scaleDistribution": {
                "type": "linear"
              },
              "showPoints": "auto",
              "spanNulls": false,
              "stacking": {
                "group": "A",
                "mode": "none"
              },
              "thresholdsStyle": {
                "mode": "off"
              }
            },
            "mappings": [],
            "min": 0,
            "thresholds": {
              "mode": "absolute",
              "steps": [
                {
                  "color": "green",
                  "value": null
                }
              ]
            },
            "unit": "conns"
          },
          "overrides": []
        },
        "gridPos": {
          "h": 8,
          "w": 12,
          "x": 12,
          "y": 1
        },
        "id": 166,
        "options": {
          "legend": {
            "calcs": [],
            "displayMode": "list",
            "placement": "bottom",
            "showLegend": true
          },
          "tooltip": {
            "mode": "single",
            "sort": "none"
          }
        },
        "pluginVersion": "9.5.3",
        "targets": [
          {
            "datasource": {
              "type": "prometheus",
              "uid": "$datasource"
            },
            "disableTextWrap": false,
            "editorMode": "builder",
            "exemplar": false,
            "expr": "sliding_sync_api_num_devices_pending_ensure_polling",
            "fullMetaSearch": false,
            "includeNullMetadata": true,
            "instant": false,
            "legendFormat": "__auto",
            "range": true,
            "refId": "A",
            "useBackend": false
          }
        ],
        "title": "EnsurePolling calls outstanding",
        "transformations": [
          {
            "id": "renameByRegex",
            "options": {
              "regex": "sliding-sync-api-(.*)",
              "renamePattern": "$1"
            }
          }
        ],
        "type": "timeseries"
      },
      {
        "datasource": {
          "type": "prometheus",
          "uid": "$datasource"
        },
        "description": "Excludes time waiting for live updates. Excludes initial requests.",
        "fieldConfig": {
          "defaults": {
            "color": {
              "fixedColor": "red",
              "mode": "palette-classic",
              "seriesBy": "max"
            },
            "custom": {
              "axisBorderShow": false,
              "axisCenteredZero": false,
              "axisColorMode": "text",
              "axisLabel": "",
              "axisPlacement": "auto",
              "barAlignment": 0,
              "drawStyle": "line",
              "fillOpacity": 10,
              "gradientMode": "none",
              "hideFrom": {
                "legend": false,
                "tooltip": false,
                "viz": false
              },
              "insertNulls": false,
              "lineInterpolation": "linear",
              "lineStyle": {
                "fill": "solid"
              },
              "lineWidth": 0,
              "pointSize": 4,
              "scaleDistribution": {
                "type": "linear"
              },
              "showPoints": "auto",
              "spanNulls": true,
              "stacking": {
                "group": "A",
                "mode": "none"
              },
              "thresholdsStyle": {
                "mode": "off"
              }
            },
            "links": [],
            "mappings": [],
            "min": 0,
            "thresholds": {
              "mode": "absolute",
              "steps": [
                {
                  "color": "green",
                  "value": null
                }
              ]
            },
            "unit": "s"
          },
          "overrides": [
            {
              "matcher": {
                "id": "byName",
                "options": "99%"
              },
              "properties": [
                {
                  "id": "color",
                  "value": {
                    "fixedColor": "dark-red",
                    "mode": "fixed"
                  }
                },
                {
                  "id": "custom.fillOpacity",
                  "value": 35
                }
              ]
            },
            {
              "matcher": {
                "id": "byName",
                "options": "95%"
              },
              "properties": [
                {
                  "id": "color",
                  "value": {
                    "fixedColor": "orange",
                    "mode": "fixed"
                  }
                }
              ]
            },
            {
              "matcher": {
                "id": "byName",
                "options": "50%"
              },
              "properties": [
                {
                  "id": "color",
                  "value": {
                    "fixedColor": "green",
                    "mode": "fixed"
                  }
                }
              ]
            },
            {
              "matcher": {
                "id": "byName",
                "options": "25%"
              },
              "properties": [
                {
                  "id": "color",
                  "value": {
                    "fixedColor": "semi-dark-blue",
                    "mode": "fixed"
                  }
                }
              ]
            },
            {
              "matcher": {
                "id": "byName",
                "options": "75%"
              },
              "properties": [
                {
                  "id": "color",
                  "value": {
                    "fixedColor": "yellow",
                    "mode": "fixed"
                  }
                }
              ]
            },
            {
              "matcher": {
                "id": "byName",
                "options": "Request rate"
              },
              "properties": [
                {
                  "id": "custom.axisPlacement",
                  "value": "right"
                },
                {
                  "id": "unit",
                  "value": "hertz"
                },
                {
                  "id": "custom.fillOpacity",
                  "value": 0
                },
                {
                  "id": "color",
                  "value": {
                    "fixedColor": "purple",
                    "mode": "fixed"
                  }
                },
                {
                  "id": "custom.showPoints",
                  "value": "always"
                }
              ]
            },
            {
              "matcher": {
                "id": "byName",
                "options": "90%"
              },
              "properties": [
                {
                  "id": "color",
                  "value": {
                    "fixedColor": "super-light-orange",
                    "mode": "fixed"
                  }
                }
              ]
            },
            {
              "__systemRef": "hideSeriesFrom",
              "matcher": {
                "id": "byNames",
                "options": {
                  "mode": "exclude",
                  "names": [
                    "Request rate"
                  ],
                  "prefix": "All except:",
                  "readOnly": true
                }
              },
              "properties": [
                {
                  "id": "custom.hideFrom",
                  "value": {
                    "legend": false,
                    "tooltip": false,
                    "viz": true
                  }
                }
              ]
            }
          ]
        },
        "gridPos": {
          "h": 8,
          "w": 12,
          "x": 0,
          "y": 9
        },
        "id": 152,
        "options": {
          "legend": {
            "calcs": [],
            "displayMode": "list",
            "placement": "bottom",
            "showLegend": true
          },
          "tooltip": {
            "mode": "multi",
            "sort": "desc"
          }
        },
        "pluginVersion": "9.2.2",
        "targets": [
          {
            "datasource": {
              "type": "prometheus",
              "uid": "$datasource"
            },
            "editorMode": "code",
            "expr": "histogram_quantile(0.99, sum by(le) (rate(sliding_sync_api_process_duration_secs_bucket{initial=\"0\"}[$window_size])))",
            "format": "time_series",
            "intervalFactor": 1,
            "legendFormat": "99%",
            "range": true,
            "refId": "99"
          },
          {
            "datasource": {
              "type": "prometheus",
              "uid": "$datasource"
            },
            "editorMode": "code",
            "expr": "histogram_quantile(0.90, sum by(le) (rate(sliding_sync_api_process_duration_secs_bucket{initial=\"0\"}[$window_size])))",
            "format": "time_series",
            "hide": false,
            "intervalFactor": 1,
            "legendFormat": "90%",
            "range": true,
            "refId": "90"
          },
          {
            "datasource": {
              "type": "prometheus",
              "uid": "$datasource"
            },
            "editorMode": "code",
            "expr": "histogram_quantile(0.75, sum by(le) (rate(sliding_sync_api_process_duration_secs_bucket{initial=\"0\"}[$window_size])))",
            "format": "time_series",
            "hide": false,
            "intervalFactor": 1,
            "legendFormat": "75%",
            "range": true,
            "refId": "75"
          },
          {
            "datasource": {
              "type": "prometheus",
              "uid": "$datasource"
            },
            "editorMode": "code",
            "expr": "histogram_quantile(0.5, sum by(le) (rate(sliding_sync_api_process_duration_secs_bucket{initial=\"0\"}[$window_size])))",
            "format": "time_series",
            "hide": false,
            "intervalFactor": 1,
            "legendFormat": "50%",
            "range": true,
            "refId": "50"
          },
          {
            "datasource": {
              "type": "prometheus",
              "uid": "$datasource"
            },
            "editorMode": "code",
            "expr": "histogram_quantile(0.25, sum by(le) (rate(sliding_sync_api_process_duration_secs_bucket{initial=\"0\"}[$window_size])))",
            "format": "time_series",
            "hide": false,
            "intervalFactor": 1,
            "legendFormat": "25%",
            "range": true,
            "refId": "25"
          },
          {
            "datasource": {
              "type": "prometheus",
              "uid": "$datasource"
            },
            "editorMode": "code",
            "exemplar": false,
            "expr": "sum(rate(sliding_sync_api_process_duration_secs_count{initial=\"0\"}[$window_size]))",
            "hide": false,
            "instant": false,
            "legendFormat": "Request rate",
            "range": true,
            "refId": "A"
          }
        ],
        "title": "API processing time quantiles",
        "type": "timeseries"
      },
      {
        "datasource": {
          "type": "prometheus",
          "uid": "$datasource"
        },
        "fieldConfig": {
          "defaults": {
            "custom": {
              "hideFrom": {
                "legend": false,
                "tooltip": false,
                "viz": false
              },
              "scaleDistribution": {
                "type": "linear"
              }
            }
          },
          "overrides": []
        },
        "gridPos": {
          "h": 8,
          "w": 12,
          "x": 12,
          "y": 9
        },
        "id": 157,
        "options": {
          "calculate": false,
          "cellGap": 1,
          "color": {
            "exponent": 0.5,
            "fill": "dark-orange",
            "mode": "scheme",
            "reverse": false,
            "scale": "exponential",
            "scheme": "Oranges",
            "steps": 64
          },
          "exemplars": {
            "color": "rgba(255,0,255,0.7)"
          },
          "filterValues": {
            "le": 1e-9
          },
          "legend": {
            "show": true
          },
          "rowsFrame": {
            "layout": "auto"
          },
          "tooltip": {
            "mode": "single",
            "showColorScale": false,
            "yHistogram": true
          },
          "yAxis": {
            "axisPlacement": "left",
            "reverse": false
          }
        },
        "pluginVersion": "10.4.1",
        "targets": [
          {
            "datasource": {
              "type": "prometheus",
              "uid": "$datasource"
            },
            "editorMode": "builder",
            "expr": "sum(rate(sliding_sync_api_setup_duration_secs_bucket{}[$__rate_interval])) by (le)",
            "format": "heatmap",
            "legendFormat": "{{le}}",
            "range": true,
            "refId": "A"
          }
        ],
        "title": "Request setup times",
        "type": "heatmap"
      },
      {
        "datasource": {
          "type": "prometheus",
          "uid": "$datasource"
        },
        "description": "Requests take more that 50s. This is our best proxy for \"how many people's requests are wedged\". Should be a flat 0.",
        "fieldConfig": {
          "defaults": {
            "color": {
              "mode": "palette-classic"
            },
            "custom": {
              "axisBorderShow": false,
              "axisCenteredZero": false,
              "axisColorMode": "text",
              "axisLabel": "",
              "axisPlacement": "auto",
              "barAlignment": 0,
              "drawStyle": "line",
              "fillOpacity": 0,
              "gradientMode": "none",
              "hideFrom": {
                "legend": false,
                "tooltip": false,
                "viz": false
              },
              "insertNulls": false,
              "lineInterpolation": "linear",
              "lineWidth": 1,
              "pointSize": 5,
              "scaleDistribution": {
                "type": "linear"
              },
              "showPoints": "auto",
              "spanNulls": false,
              "stacking": {
                "group": "A",
                "mode": "none"
              },
              "thresholdsStyle": {
                "mode": "line"
              }
            },
            "mappings": [],
            "thresholds": {
              "mode": "absolute",
              "steps": [
                {
                  "color": "green",
                  "value": null
                }
              ]
            }
          },
          "overrides": []
        },
        "gridPos": {
          "h": 8,
          "w": 12,
          "x": 0,
          "y": 17
        },
        "id": 159,
        "options": {
          "legend": {
            "calcs": [],
            "displayMode": "list",
            "placement": "bottom",
            "showLegend": true
          },
          "tooltip": {
            "mode": "single",
            "sort": "none"
          }
        },
        "targets": [
          {
            "datasource": {
              "type": "prometheus",
              "uid": "$datasource"
            },
            "disableTextWrap": false,
            "editorMode": "builder",
            "expr": "sum(increase(sliding_sync_api_slow_requests[5m]))",
            "fullMetaSearch": false,
            "includeNullMetadata": true,
            "legendFormat": "__auto",
            "range": true,
            "refId": "A",
            "useBackend": false
          }
        ],
        "title": "Slow requests",
        "type": "timeseries"
      },
      {
        "cards": {},
        "color": {
          "cardColor": "#b4ff00",
          "colorScale": "sqrt",
          "colorScheme": "interpolateOranges",
          "exponent": 0.5,
          "mode": "spectrum"
        },
        "dataFormat": "tsbuckets",
        "datasource": {
          "type": "prometheus",
          "uid": "$datasource"
        },
        "description": "If this goes high, it may indicate poor DB performance when querying.",
        "fieldConfig": {
          "defaults": {
            "custom": {
              "hideFrom": {
                "legend": false,
                "tooltip": false,
                "viz": false
              },
              "scaleDistribution": {
                "type": "linear"
              }
            }
          },
          "overrides": []
        },
        "gridPos": {
          "h": 8,
          "w": 12,
          "x": 12,
          "y": 17
        },
        "heatmap": {},
        "hideZeroBuckets": true,
        "highlightCards": true,
        "id": 10,
        "legend": {
          "show": true
        },
        "options": {
          "calculate": false,
          "calculation": {},
          "cellGap": 2,
          "cellValues": {},
          "color": {
            "exponent": 0.5,
            "fill": "#b4ff00",
            "mode": "scheme",
            "reverse": false,
            "scale": "exponential",
            "scheme": "Oranges",
            "steps": 128
          },
          "exemplars": {
            "color": "rgba(255,0,255,0.7)"
          },
          "filterValues": {
            "le": 1e-9
          },
          "legend": {
            "show": true
          },
          "rowsFrame": {
            "layout": "auto"
          },
          "showValue": "never",
          "tooltip": {
            "mode": "single",
            "showColorScale": false,
            "yHistogram": true
          },
          "yAxis": {
            "axisPlacement": "left",
            "reverse": false,
            "unit": "dtdurations"
          }
        },
        "pluginVersion": "10.4.1",
        "reverseYBuckets": false,
        "targets": [
          {
            "datasource": {
              "type": "prometheus",
              "uid": "$datasource"
            },
            "editorMode": "code",
            "expr": "sum(increase(sliding_sync_api_process_duration_secs_bucket{initial=\"1\"}[$__rate_interval])) by (le)",
            "format": "heatmap",
            "legendFormat": "{{le}}",
            "range": true,
            "refId": "A"
          }
        ],
        "title": "Time taken to process initial sliding sync requests",
        "tooltip": {
          "show": true,
          "showHistogram": true
        },
        "type": "heatmap",
        "xAxis": {
          "show": true
        },
        "yAxis": {
          "format": "dtdurations",
          "logBase": 1,
          "show": true
        },
        "yBucketBound": "auto"
      },
      {
        "cards": {},
        "color": {
          "cardColor": "#b4ff00",
          "colorScale": "sqrt",
          "colorScheme": "interpolateOranges",
          "exponent": 0.5,
          "mode": "spectrum"
        },
        "dataFormat": "tsbuckets",
        "datasource": {
          "type": "prometheus",
          "uid": "$datasource"
        },
        "description": "Excludes live streaming blocks. If this goes high, it may indicate poor DB performance when querying.",
        "fieldConfig": {
          "defaults": {
            "custom": {
              "hideFrom": {
                "legend": false,
                "tooltip": false,
                "viz": false
              },
              "scaleDistribution": {
                "type": "linear"
              }
            }
          },
          "overrides": []
        },
        "gridPos": {
          "h": 8,
          "w": 12,
          "x": 0,
          "y": 25
        },
        "heatmap": {},
        "hideZeroBuckets": true,
        "highlightCards": true,
        "id": 15,
        "legend": {
          "show": true
        },
        "options": {
          "calculate": false,
          "calculation": {},
          "cellGap": 2,
          "cellValues": {
            "decimals": 3
          },
          "color": {
            "exponent": 0.5,
            "fill": "#b4ff00",
            "mode": "scheme",
            "reverse": false,
            "scale": "exponential",
            "scheme": "Oranges",
            "steps": 128
          },
          "exemplars": {
            "color": "rgba(255,0,255,0.7)"
          },
          "filterValues": {
            "le": 1e-9
          },
          "legend": {
            "show": true
          },
          "rowsFrame": {
            "layout": "auto"
          },
          "showValue": "never",
          "tooltip": {
            "mode": "single",
            "showColorScale": false,
            "yHistogram": false
          },
          "yAxis": {
            "axisPlacement": "left",
            "min": "0",
            "reverse": false,
            "unit": "dtdurations"
          }
        },
        "pluginVersion": "10.4.1",
        "reverseYBuckets": false,
        "targets": [
          {
            "datasource": {
              "type": "prometheus",
              "uid": "$datasource"
            },
            "editorMode": "code",
            "expr": "sum(rate(sliding_sync_api_process_duration_secs_bucket{initial=\"0\"}[$__rate_interval])) by (le)",
            "format": "heatmap",
            "interval": "",
            "legendFormat": "{{le}}",
            "range": true,
            "refId": "A"
          }
        ],
        "title": "Time taken to process changes (ranges/filter/sorting/etc) in sliding sync requests",
        "tooltip": {
          "show": true,
          "showHistogram": false
        },
        "tooltipDecimals": 3,
        "type": "heatmap",
        "xAxis": {
          "show": true
        },
        "xBucketSize": "",
        "yAxis": {
          "format": "dtdurations",
          "logBase": 1,
          "min": "0",
          "show": true
        },
        "yBucketBound": "auto"
      },
      {
        "collapsed": false,
        "gridPos": {
          "h": 1,
          "w": 24,
          "x": 0,
          "y": 33
        },
        "id": 12,
        "panels": [],
        "title": "V2 Poller",
        "type": "row"
      },
      {
        "datasource": {
          "type": "prometheus",
          "uid": "$datasource"
        },
        "description": "Number of /sync connections to upstream homeserver. Generally always goes up, unless users log out and invalidate the access_token being used.",
        "fieldConfig": {
          "defaults": {
            "color": {
              "mode": "palette-classic"
            },
            "custom": {
              "axisBorderShow": false,
              "axisCenteredZero": false,
              "axisColorMode": "text",
              "axisLabel": "",
              "axisPlacement": "auto",
              "barAlignment": 0,
              "drawStyle": "line",
              "fillOpacity": 0,
              "gradientMode": "none",
              "hideFrom": {
                "legend": false,
                "tooltip": false,
                "viz": false
              },
              "insertNulls": false,
              "lineInterpolation": "linear",
              "lineWidth": 1,
              "pointSize": 5,
              "scaleDistribution": {
                "type": "linear"
              },
              "showPoints": "auto",
              "spanNulls": false,
              "stacking": {
                "group": "A",
                "mode": "none"
              },
              "thresholdsStyle": {
                "mode": "off"
              }
            },
            "mappings": [],
            "min": 0,
            "thresholds": {
              "mode": "absolute",
              "steps": [
                {
                  "color": "green"
                }
              ]
            },
            "unit": "pollers",
            "unitScale": true
          },
          "overrides": []
        },
        "gridPos": {
          "h": 8,
          "w": 12,
          "x": 0,
          "y": 34
        },
        "id": 4,
        "options": {
          "legend": {
            "calcs": [],
            "displayMode": "list",
            "placement": "bottom",
            "showLegend": true
          },
          "tooltip": {
            "mode": "single",
            "sort": "none"
          }
        },
        "pluginVersion": "9.5.3",
        "targets": [
          {
            "datasource": {
              "type": "prometheus",
              "uid": "$datasource"
            },
            "editorMode": "code",
            "exemplar": false,
            "expr": "sliding_sync_poller_num_pollers{}",
            "instant": false,
            "legendFormat": "{{pod}}",
            "range": true,
            "refId": "A"
          }
        ],
        "title": "# v2 pollers",
        "type": "timeseries"
      },
      {
        "cards": {},
        "color": {
          "cardColor": "#b4ff00",
          "colorScale": "sqrt",
          "colorScheme": "interpolateOranges",
          "exponent": 0.5,
          "mode": "spectrum"
        },
        "dataFormat": "tsbuckets",
        "datasource": {
          "type": "prometheus",
          "uid": "$datasource"
        },
        "description": "The higher this is, the bigger the latency from sending events -> receiving events. Excludes initial /sync requests",
        "fieldConfig": {
          "defaults": {
            "custom": {
              "hideFrom": {
                "legend": false,
                "tooltip": false,
                "viz": false
              },
              "scaleDistribution": {
                "type": "linear"
              }
            },
            "unitScale": true
          },
          "overrides": []
        },
        "gridPos": {
          "h": 8,
          "w": 12,
          "x": 12,
          "y": 34
        },
        "heatmap": {},
        "hideZeroBuckets": true,
        "highlightCards": true,
        "id": 8,
        "legend": {
          "show": false
        },
        "options": {
          "calculate": false,
          "calculation": {},
          "cellGap": 2,
          "cellValues": {},
          "color": {
            "exponent": 0.5,
            "fill": "#b4ff00",
            "mode": "scheme",
            "reverse": false,
            "scale": "exponential",
            "scheme": "Oranges",
            "steps": 128
          },
          "exemplars": {
            "color": "rgba(255,0,255,0.7)"
          },
          "filterValues": {
            "le": 1e-9
          },
          "legend": {
            "show": false
          },
          "rowsFrame": {
            "layout": "auto"
          },
          "showValue": "never",
          "tooltip": {
            "mode": "single",
            "showColorScale": false,
            "yHistogram": false
          },
          "yAxis": {
            "axisPlacement": "left",
            "reverse": false,
            "unit": "dtdurations"
          }
        },
        "pluginVersion": "10.3.3",
        "reverseYBuckets": false,
        "targets": [
          {
            "datasource": {
              "type": "prometheus",
              "uid": "$datasource"
            },
            "editorMode": "code",
            "expr": "sum(increase(sliding_sync_poller_process_duration_secs_bucket{initial=\"0\"}[$__rate_interval])) by (le)",
            "format": "heatmap",
            "legendFormat": "{{le}}",
            "range": true,
            "refId": "A"
          }
        ],
        "title": "Time taken to process sync v2 responses",
        "tooltip": {
          "show": true,
          "showHistogram": false
        },
        "type": "heatmap",
        "xAxis": {
          "show": true
        },
        "yAxis": {
          "format": "dtdurations",
          "logBase": 1,
          "show": true
        },
        "yBucketBound": "auto"
      },
      {
        "datasource": {
          "type": "prometheus",
          "uid": "$datasource"
        },
        "description": "If this drops too low, this means pollers are blocked on something (DB conns, executor, etc)",
        "fieldConfig": {
          "defaults": {
            "color": {
              "mode": "palette-classic"
            },
            "custom": {
              "axisBorderShow": false,
              "axisCenteredZero": false,
              "axisColorMode": "text",
              "axisLabel": "",
              "axisPlacement": "auto",
              "axisSoftMin": 0,
              "barAlignment": 0,
              "drawStyle": "line",
              "fillOpacity": 0,
              "gradientMode": "none",
              "hideFrom": {
                "legend": false,
                "tooltip": false,
                "viz": false
              },
              "insertNulls": false,
              "lineInterpolation": "linear",
              "lineWidth": 1,
              "pointSize": 5,
              "scaleDistribution": {
                "type": "linear"
              },
              "showPoints": "auto",
              "spanNulls": false,
              "stacking": {
                "group": "A",
                "mode": "none"
              },
              "thresholdsStyle": {
                "mode": "area"
              }
            },
            "mappings": [],
            "thresholds": {
              "mode": "absolute",
              "steps": [
                {
                  "color": "red"
                },
                {
                  "color": "green",
                  "value": 0.75
                }
              ]
            },
            "unit": "percentunit",
            "unitScale": true
          },
          "overrides": []
        },
        "gridPos": {
          "h": 8,
          "w": 12,
          "x": 0,
          "y": 42
        },
        "id": 165,
        "options": {
          "legend": {
            "calcs": [],
            "displayMode": "list",
            "placement": "bottom",
            "showLegend": true
          },
          "tooltip": {
            "mode": "single",
            "sort": "none"
          }
        },
        "targets": [
          {
            "datasource": {
              "type": "prometheus",
              "uid": "$datasource"
            },
            "editorMode": "code",
            "expr": "sliding_sync_poller_num_outstanding_sync_v2_reqs{}/sliding_sync_poller_num_pollers{}",
            "legendFormat": "{{pod}}",
            "range": true,
            "refId": "A"
          }
        ],
        "title": "Ratio of pollers waiting for v2 response",
        "type": "timeseries"
      },
      {
        "datasource": {
          "type": "prometheus",
          "uid": "$datasource"
        },
        "description": "If this number drops to 0, this indicates something is blocking all pollers from doing work. This number will scale with the number of pollers on the process.",
        "fieldConfig": {
          "defaults": {
            "color": {
              "mode": "palette-classic"
            },
            "custom": {
              "axisBorderShow": false,
              "axisCenteredZero": false,
              "axisColorMode": "text",
              "axisLabel": "",
              "axisPlacement": "auto",
              "axisSoftMin": 0,
              "barAlignment": 0,
              "drawStyle": "line",
              "fillOpacity": 0,
              "gradientMode": "none",
              "hideFrom": {
                "legend": false,
                "tooltip": false,
                "viz": false
              },
              "insertNulls": false,
              "lineInterpolation": "linear",
              "lineWidth": 1,
              "pointSize": 5,
              "scaleDistribution": {
                "type": "linear"
              },
              "showPoints": "auto",
              "spanNulls": false,
              "stacking": {
                "group": "A",
                "mode": "none"
              },
              "thresholdsStyle": {
                "mode": "off"
              }
            },
            "mappings": [],
            "thresholds": {
              "mode": "absolute",
              "steps": [
                {
                  "color": "green"
                },
                {
                  "color": "red",
                  "value": 80
                }
              ]
            },
            "unitScale": true
          },
          "overrides": []
        },
        "gridPos": {
          "h": 8,
          "w": 12,
          "x": 12,
          "y": 42
        },
        "id": 167,
        "options": {
          "legend": {
            "calcs": [],
            "displayMode": "list",
            "placement": "bottom",
            "showLegend": true
          },
          "tooltip": {
            "mode": "single",
            "sort": "none"
          }
        },
        "targets": [
          {
            "datasource": {
              "type": "prometheus",
              "uid": "$datasource"
            },
            "editorMode": "code",
            "expr": "rate(sliding_sync_poller_total_num_polls{}[5m])",
            "legendFormat": "{{pod}}",
            "range": true,
            "refId": "A"
          }
        ],
        "title": "Rate of poll loop iterations",
        "type": "timeseries"
      },
      {
        "datasource": {
          "type": "prometheus",
          "uid": "$datasource"
        },
        "description": "NB: Proxy requests a timeline limit of 50.\n\nTODO: make it clear how many syncs were limited",
        "fieldConfig": {
          "defaults": {
            "custom": {
              "hideFrom": {
                "legend": false,
                "tooltip": false,
                "viz": false
              },
              "scaleDistribution": {
                "type": "linear"
              }
            },
            "unitScale": true
          },
          "overrides": []
        },
        "gridPos": {
          "h": 8,
          "w": 12,
          "x": 0,
          "y": 50
        },
        "id": 16,
        "options": {
          "calculate": false,
          "cellGap": 1,
          "cellValues": {},
          "color": {
            "exponent": 0.5,
            "fill": "dark-orange",
            "min": 0,
            "mode": "scheme",
            "reverse": false,
            "scale": "exponential",
            "scheme": "Oranges",
            "steps": 64
          },
          "exemplars": {
            "color": "rgba(255,0,255,0.7)"
          },
          "filterValues": {
            "le": 1e-9
          },
          "legend": {
            "show": true
          },
          "rowsFrame": {
            "layout": "auto",
            "value": "v2 syncs"
          },
          "tooltip": {
            "mode": "single",
            "showColorScale": false,
            "yHistogram": true
          },
          "yAxis": {
            "axisLabel": "timeline events",
            "axisPlacement": "left",
            "max": "50",
            "reverse": false
          }
        },
        "pluginVersion": "10.3.3",
        "targets": [
          {
            "datasource": {
              "type": "prometheus",
              "uid": "$datasource"
            },
            "editorMode": "code",
            "expr": "sum(rate(sliding_sync_poller_timeline_size_bucket{limited=\"unlimited\"}[$__rate_interval])) by (le)",
            "format": "heatmap",
            "legendFormat": "{{limited}}",
            "range": true,
            "refId": "A"
          }
        ],
        "title": "Timeline size of unlimited pollers",
        "type": "heatmap"
      },
      {
        "datasource": {
          "type": "prometheus",
          "uid": "$datasource"
        },
        "description": "Updates from v2 pollers sent to pubsub. Abnormal spikes could be from spam or lack of duplicate suppression in the proxy.",
        "fieldConfig": {
          "defaults": {
            "color": {
              "mode": "palette-classic"
            },
            "custom": {
              "axisBorderShow": false,
              "axisCenteredZero": false,
              "axisColorMode": "text",
              "axisLabel": "",
              "axisPlacement": "auto",
              "barAlignment": 0,
              "drawStyle": "line",
              "fillOpacity": 0,
              "gradientMode": "none",
              "hideFrom": {
                "legend": false,
                "tooltip": false,
                "viz": false
              },
              "insertNulls": false,
              "lineInterpolation": "linear",
              "lineWidth": 1,
              "pointSize": 5,
              "scaleDistribution": {
                "type": "linear"
              },
              "showPoints": "auto",
              "spanNulls": false,
              "stacking": {
                "group": "A",
                "mode": "none"
              },
              "thresholdsStyle": {
                "mode": "off"
              }
            },
            "mappings": [],
            "thresholds": {
              "mode": "absolute",
              "steps": [
                {
                  "color": "green"
                },
                {
                  "color": "red",
                  "value": 80
                }
              ]
            },
            "unitScale": true
          },
          "overrides": []
        },
        "gridPos": {
          "h": 9,
          "w": 12,
          "x": 12,
          "y": 50
        },
        "id": 2,
        "options": {
          "legend": {
            "calcs": [],
            "displayMode": "list",
            "placement": "bottom",
            "showLegend": true
          },
          "tooltip": {
            "mode": "single",
            "sort": "none"
          }
        },
        "targets": [
          {
            "datasource": {
              "type": "prometheus",
              "uid": "$datasource"
            },
            "editorMode": "code",
            "expr": "sum by (payload_type) (rate(sliding_sync_poller_num_payloads{}[$window_size]))",
            "legendFormat": "{{payload_type}}",
            "range": true,
            "refId": "A"
          }
        ],
        "title": "Payload Rate",
        "type": "timeseries"
      },
      {
        "datasource": {
          "type": "prometheus",
          "uid": "$datasource"
        },
        "description": "Excludes time waiting for v2 sync requests. Excludes initial polls.",
        "fieldConfig": {
          "defaults": {
            "color": {
              "fixedColor": "red",
              "mode": "palette-classic",
              "seriesBy": "max"
            },
            "custom": {
              "axisBorderShow": false,
              "axisCenteredZero": false,
              "axisColorMode": "text",
              "axisLabel": "",
              "axisPlacement": "auto",
              "barAlignment": 0,
              "drawStyle": "line",
              "fillOpacity": 10,
              "gradientMode": "none",
              "hideFrom": {
                "legend": false,
                "tooltip": false,
                "viz": false
              },
              "insertNulls": false,
              "lineInterpolation": "linear",
              "lineStyle": {
                "fill": "solid"
              },
              "lineWidth": 0,
              "pointSize": 4,
              "scaleDistribution": {
                "type": "linear"
              },
              "showPoints": "auto",
              "spanNulls": true,
              "stacking": {
                "group": "A",
                "mode": "none"
              },
              "thresholdsStyle": {
                "mode": "off"
              }
            },
            "links": [],
            "mappings": [],
            "min": 0,
            "thresholds": {
              "mode": "absolute",
              "steps": [
                {
                  "color": "green"
                }
              ]
            },
            "unit": "s",
            "unitScale": true
          },
          "overrides": [
            {
              "matcher": {
                "id": "byName",
                "options": "99%"
              },
              "properties": [
                {
                  "id": "color",
                  "value": {
                    "fixedColor": "dark-red",
                    "mode": "fixed"
                  }
                },
                {
                  "id": "custom.fillOpacity",
                  "value": 35
                }
              ]
            },
            {
              "matcher": {
                "id": "byName",
                "options": "95%"
              },
              "properties": [
                {
                  "id": "color",
                  "value": {
                    "fixedColor": "orange",
                    "mode": "fixed"
                  }
                }
              ]
            },
            {
              "matcher": {
                "id": "byName",
                "options": "50%"
              },
              "properties": [
                {
                  "id": "color",
                  "value": {
                    "fixedColor": "green",
                    "mode": "fixed"
                  }
                }
              ]
            },
            {
              "matcher": {
                "id": "byName",
                "options": "25%"
              },
              "properties": [
                {
                  "id": "color",
                  "value": {
                    "fixedColor": "semi-dark-blue",
                    "mode": "fixed"
                  }
                }
              ]
            },
            {
              "matcher": {
                "id": "byName",
                "options": "75%"
              },
              "properties": [
                {
                  "id": "color",
                  "value": {
                    "fixedColor": "yellow",
                    "mode": "fixed"
                  }
                }
              ]
            },
            {
              "matcher": {
                "id": "byName",
                "options": "Poll rate"
              },
              "properties": [
                {
                  "id": "custom.axisPlacement",
                  "value": "right"
                },
                {
                  "id": "unit",
                  "value": "hertz"
                },
                {
                  "id": "custom.fillOpacity",
                  "value": 0
                },
                {
                  "id": "color",
                  "value": {
                    "fixedColor": "purple",
                    "mode": "fixed"
                  }
                },
                {
                  "id": "custom.showPoints",
                  "value": "always"
                }
              ]
            },
            {
              "matcher": {
                "id": "byName",
                "options": "90%"
              },
              "properties": [
                {
                  "id": "color",
                  "value": {
                    "fixedColor": "super-light-orange",
                    "mode": "fixed"
                  }
                }
              ]
            }
          ]
        },
        "gridPos": {
          "h": 9,
          "w": 12,
          "x": 12,
          "y": 59
        },
        "id": 153,
        "options": {
          "legend": {
            "calcs": [],
            "displayMode": "list",
            "placement": "bottom",
            "showLegend": true
          },
          "tooltip": {
            "mode": "multi",
            "sort": "desc"
          }
        },
        "pluginVersion": "9.2.2",
        "targets": [
          {
            "datasource": {
              "type": "prometheus",
              "uid": "$datasource"
            },
            "editorMode": "code",
            "expr": "histogram_quantile(0.99, sum by(le) (rate(sliding_sync_poller_process_duration_secs_bucket{initial=\"0\"}[$window_size])))",
            "format": "time_series",
            "intervalFactor": 1,
            "legendFormat": "99%",
            "range": true,
            "refId": "99"
          },
          {
            "datasource": {
              "type": "prometheus",
              "uid": "$datasource"
            },
            "editorMode": "code",
            "expr": "histogram_quantile(0.90, sum by(le) (rate(sliding_sync_poller_process_duration_secs_bucket{initial=\"0\"}[$window_size])))",
            "format": "time_series",
            "hide": false,
            "intervalFactor": 1,
            "legendFormat": "90%",
            "range": true,
            "refId": "90"
          },
          {
            "datasource": {
              "type": "prometheus",
              "uid": "$datasource"
            },
            "editorMode": "code",
            "expr": "histogram_quantile(0.75, sum by(le) (rate(sliding_sync_poller_process_duration_secs_bucket{initial=\"0\"}[$window_size])))",
            "format": "time_series",
            "hide": false,
            "intervalFactor": 1,
            "legendFormat": "75%",
            "range": true,
            "refId": "75"
          },
          {
            "datasource": {
              "type": "prometheus",
              "uid": "$datasource"
            },
            "editorMode": "code",
            "expr": "histogram_quantile(0.5, sum by(le) (rate(sliding_sync_poller_process_duration_secs_bucket{initial=\"0\"}[$window_size])))",
            "format": "time_series",
            "hide": false,
            "intervalFactor": 1,
            "legendFormat": "50%",
            "range": true,
            "refId": "50"
          },
          {
            "datasource": {
              "type": "prometheus",
              "uid": "$datasource"
            },
            "editorMode": "code",
            "expr": "histogram_quantile(0.25, sum by(le) (rate(sliding_sync_poller_process_duration_secs_bucket{initial=\"0\"}[$window_size])))",
            "format": "time_series",
            "hide": false,
            "intervalFactor": 1,
            "legendFormat": "25%",
            "range": true,
            "refId": "25"
          },
          {
            "datasource": {
              "type": "prometheus",
              "uid": "$datasource"
            },
            "editorMode": "code",
            "expr": "sum(rate(sliding_sync_poller_process_duration_secs_count{initial=\"0\"}[$window_size]))",
            "hide": false,
            "legendFormat": "Poll rate",
            "range": true,
            "refId": "A"
          }
        ],
        "title": "Poller processing time quantiles",
        "type": "timeseries"
      }
    ],
    "refresh": "",
    "schemaVersion": 39,
    "tags": [],
    "templating": {
      "list": [
        {
          "auto": true,
          "auto_count": 100,
          "auto_min": "30s",
          "current": {
            "selected": true,
            "text": "auto",
            "value": "$__auto_interval_window_size"
          },
          "description": "Window to use for aggregating buckets/moving averages",
          "hide": 0,
          "label": "Window size",
          "name": "window_size",
          "options": [
            {
              "selected": true,
              "text": "auto",
              "value": "$__auto_interval_window_size"
            },
            {
              "selected": false,
              "text": "30s",
              "value": "30s"
            },
            {
              "selected": false,
              "text": "1m",
              "value": "1m"
            },
            {
              "selected": false,
              "text": "2m",
              "value": "2m"
            },
            {
              "selected": false,
              "text": "5m",
              "value": "5m"
            },
            {
              "selected": false,
              "text": "10m",
              "value": "10m"
            },
            {
              "selected": false,
              "text": "15m",
              "value": "15m"
            },
            {
              "selected": false,
              "text": "30m",
              "value": "30m"
            },
            {
              "selected": false,
              "text": "1h",
              "value": "1h"
            },
            {
              "selected": false,
              "text": "2h",
              "value": "2h"
            }
          ],
          "query": "30s,1m,2m,5m,10m,15m,30m,1h,2h",
          "queryValue": "",
          "refresh": 2,
          "skipUrlSync": false,
          "type": "interval"
        },
        {
          "current": {
            "selected": false,
            "text": "Prometheus",
            "value": "c433c715-0878-4d85-877c-465ce5b8cac4"
          },
          "hide": 0,
          "includeAll": false,
          "multi": false,
          "name": "datasource",
          "options": [],
          "query": "prometheus",
          "queryValue": "",
          "refresh": 1,
          "regex": "",
          "skipUrlSync": false,
          "type": "datasource"
        }
      ]
    },
    "time": {
      "from": "now-1h",
      "to": "now"
    },
    "timepicker": {},
    "timezone": "",
    "title": "Sliding Sync",
    "uid": "slidingsync",
    "version": 27,
    "weekStart": ""
  }