Grafana

grafana

vizualization on top of prometheus

“export for sharing externally” won’t work with provisioning

grafana “export for sharing externally” won’t work with provisioning

When using it, it changes the datasources UID with the type/name and will supposedly find the approriate datasource once imported. Yet, this behavior appears to be triggered only when:

either importing graphically
or provisioning and then clicking on edit and save

Therefore, it appears to only follow a clickops way of thinking.

issues with alerting

I wanted to add alerts for a loki dashboard.

the alerting does not work with an existing dashboard, I need to rewrite the query,
the query needs to return a number, not logs, making annoying to get access in the future to the query when the alert is raised,
I cannot save any alert using the graphical interface and an anonymous user (https://github.com/grafana/grafana/issues/83033)

how to export programmatically the dashboards

curl "${grafanaaddr}/api/search?query="|jq

[
  {
    "id": 1,
    "uid": "cec998dhmtfy8e",
    "orgId": 1,
    "title": "nomad",
    "uri": "db/nomad",
    "url": "/grafana/dashboards/f/cec998dhmtfy8e/nomad",
    "slug": "",
    "type": "dash-folder",
    "tags": [],
    "isStarred": false,
    "sortMeta": 0,
    "isDeleted": false
  },
  {
    "id": 2,
    "uid": "nomadnodes",
    "orgId": 1,
    "title": "Nodes",
    "uri": "db/nodes",
    "url": "/grafana/d/nomadnodes/nodes",
    "slug": "",
    "type": "dash-db",
    "tags": [],
    "isStarred": false,
    "folderId": 1,
    "folderUid": "cec998dhmtfy8e",
    "folderTitle": "nomad",
    "folderUrl": "/grafana/dashboards/f/cec998dhmtfy8e/nomad",
    "sortMeta": 0,
    "isDeleted": false
  }
]

curl "${grafanaaddr}/api/dashboards/uid/nomadnodes"|jq

{
  "meta": {
    "type": "db",
    "canSave": true,
    "canEdit": true,
    "canAdmin": true,
    "canStar": false,
    "canDelete": true,
    "slug": "nodes",
    "url": "/grafana/d/nomadnodes/nodes",
    "expires": "0001-01-01T00:00:00Z",
    "created": "2025-02-06T15:04:57Z",
    "updated": "2025-02-06T15:04:57Z",
    "updatedBy": "Anonymous",
    "createdBy": "Anonymous",
    "version": 1,
    "hasAcl": false,
    "isFolder": false,
    "folderId": 1,
    "folderUid": "cec998dhmtfy8e",
    "folderTitle": "nomad",
    "folderUrl": "/grafana/dashboards/f/cec998dhmtfy8e/nomad",
    "provisioned": true,
    "provisionedExternalId": "nomad/nodes.json",
    "annotationsPermissions": {
      "dashboard": {
        "canAdd": true,
        "canEdit": true,
        "canDelete": true
      },
      "organization": {
        "canAdd": true,
        "canEdit": true,
        "canDelete": true
      }
    }
  },
  "dashboard": {
    "annotations": {
      "list": [
        {
          "builtIn": 1,
          "datasource": {
            "type": "grafana",
            "uid": "-- Grafana --"
          },
          "enable": true,
          "hide": true,
          "iconColor": "rgba(0, 211, 255, 1)",
          "name": "Annotations & Alerts",
          "type": "dashboard"
        }
      ]
    },
    "editable": true,
    "fiscalYearStartMonth": 0,
    "graphTooltip": 0,
    "id": 2,
    "links": [],
    "panels": [
      {
        "datasource": {
          "type": "datasource",
          "uid": "-- Mixed --"
        },
        "fieldConfig": {
          "defaults": {
            "color": {
              "mode": "palette-classic"
            },
            "custom": {
              "axisBorderShow": false,
              "axisCenteredZero": false,
              "axisColorMode": "text",
              "axisLabel": "",
              "axisPlacement": "auto",
              "barAlignment": 0,
              "barWidthFactor": 0.6,
              "drawStyle": "line",
              "fillOpacity": 0,
              "gradientMode": "none",
              "hideFrom": {
                "legend": false,
                "tooltip": false,
                "viz": false
              },
              "insertNulls": false,
              "lineInterpolation": "linear",
              "lineWidth": 1,
              "pointSize": 5,
              "scaleDistribution": {
                "type": "linear"
              },
              "showPoints": "auto",
              "spanNulls": false,
              "stacking": {
                "group": "A",
                "mode": "none"
              },
              "thresholdsStyle": {
                "mode": "off"
              }
            },
            "mappings": [],
            "thresholds": {
              "mode": "absolute",
              "steps": [
                {
                  "color": "green",
                  "value": null
                },
                {
                  "color": "red",
                  "value": 80
                }
              ]
            }
          },
          "overrides": []
        },
        "gridPos": {
          "h": 16,
          "w": 22,
          "x": 0,
          "y": 0
        },
        "id": 1,
        "options": {
          "legend": {
            "calcs": [],
            "displayMode": "list",
            "placement": "bottom",
            "showLegend": true
          },
          "tooltip": {
            "hideZeros": false,
            "mode": "single",
            "sort": "none"
          }
        },
        "pluginVersion": "11.5.0",
        "targets": [
          {
            "datasource": {
              "type": "prometheus",
              "uid": "prometheus"
            },
            "disableTextWrap": false,
            "editorMode": "builder",
            "expr": "sum by(host, disk) (nomad_client_host_disk_used_percent{node_status=\"ready\", disk!=\"/dev/vdb\"})",
            "fullMetaSearch": false,
            "includeNullMetadata": true,
            "key": "Q-3c0e7e4f-2461-4ba5-9106-5a8bb9bd2632-0",
            "legendFormat": "{{host}}-{{disk}}",
            "range": true,
            "refId": "diskusage",
            "useBackend": false
          }
        ],
        "title": "Disk usage",
        "type": "timeseries"
      }
    ],
    "preload": false,
    "refresh": "",
    "schemaVersion": 40,
    "tags": [],
    "templating": {
      "list": []
    },
    "time": {
      "from": "now-1h",
      "to": "now"
    },
    "timepicker": {},
    "timezone": "browser",
    "title": "Nodes",
    "uid": "nomadnodes",
    "version": 1,
    "weekStart": ""
  }
}

Then, I can put them in the provisioning folder at the appropriate folder. Provided that in my grafana configuration, I put foldersFromFilesStructure: true.

To retrieve the folder path, you can use the folders API. Don’t be fooled into using the dashboard API (folders “are” dashboards), as it will mistakenly show all the folders at the root level.

folder_uid () {
    local uid="$1"
    curl --fail --silent --show-error --location "${grafanaaddr}/api/dashboards/uid/${uid}"|jq -r '.meta.folderUid // ""'
}

name () {
    local uid="$1"
    curl --fail --silent --show-error --location "${grafanaaddr}/api/dashboards/uid/${uid}"|jq -r '.dashboard.title'
}

slug () {
    local uid="$1"
    curl --fail --silent --show-error --location "${grafanaaddr}/api/dashboards/uid/${uid}"|jq -r '.meta.slug'
}

folder_path () {
    local uid="$1"
    curl --fail --silent --show-error --location "${grafanaaddr}/api/folders/${uid}"|jq -r 'if .parents then "\(.parents|map(.title)|join("/"))/\(.title)" else .title end'
}

path () {
    local uid="$1"
    path="$(slug "${uid}").json"
    uid="$(folder_uid "${uid}")"
    if test -n "${uid}"
    then
        path="$(folder_path "${uid}")/${path}"
    fi
    echo "${path}"
}

Calling path on nomadsnodes gives.

nomad/nodes.json

In the end, I can simply run the following.

dashboard_uids () {
    curl --fail --silent --show-error --location "${grafanaaddr}/api/search?query="|jq -r '.[] | select(.type == "dash-db") | .uid'
}

for uid in $(dashboard_uids)
do
    path="$(path "${uid}")"
    mkdir -p "$(dirname "${path}")"
    curl --fail --silent --show-error --location "${grafanaaddr}/api/dashboards/uid/${uid}" > "${path}"
done

Which gives in my case

folder_uid () {
    local uid="$1"
    curl --fail --silent --show-error --location "${grafanaaddr}/api/dashboards/uid/${uid}"|jq -r '.meta.folderUid // ""'
}

name () {
    local uid="$1"
    curl --fail --silent --show-error --location "${grafanaaddr}/api/dashboards/uid/${uid}"|jq -r '.dashboard.title'
}

slug () {
    local uid="$1"
    curl --fail --silent --show-error --location "${grafanaaddr}/api/dashboards/uid/${uid}"|jq -r '.meta.slug'
}

folder_path () {
    local uid="$1"
    curl --fail --silent --show-error --location "${grafanaaddr}/api/folders/${uid}"|jq -r 'if .parents then "\(.parents|map(.title)|join("/"))/\(.title)" else .title end'
}

path () {
    local uid="$1"
    path="$(slug "${uid}").json"
    uid="$(folder_uid "${uid}")"
    if test -n "${uid}"
    then
        path="$(folder_path "${uid}")/${path}"
    fi
    echo "${path}"
}

TMP="$(mktemp -d)"
trap "rm -rf '${TMP}'" 0

pushd "${TMP}" > /dev/null
{
    dashboard_uids () {
        curl --fail --silent --show-error --location "${grafanaaddr}/api/search?query="|jq -r '.[] | select(.type == "dash-db") | .uid'
    }

    for uid in $(dashboard_uids)
    do
        path="$(path "${uid}")"
        mkdir -p "$(dirname "${path}")"
        curl --fail --silent --show-error --location "${grafanaaddr}/api/dashboards/uid/${uid}" > "${path}"
    done
    ipfa .|sed -r 's/[?]filename.+//'
}
popd > /dev/null

https://konubinix.eu/ipfs/bafybeibp6y5uccq7unrkppqjmwdyyrj4c2wbwficvifjergc4xsjai3gni

restoring from the API

In case you want to restore a dashboad using the API, it is a bit more involved, as the saved dashboard is not exactly using the same format as the one to push it back.

grafana will refuse to accept a dashboad from a definition containing the id.
the folderUid must be at the root, not into meta

The script provided in this medium post appears to be quite neat. Yet it misses the fact that api/folders will only return top level folders (not nested ones).

Here is a corrected version, taking into account the fact that folders “are” dashboards.

TMP="$(mktemp -d)"
trap "rm -rf '${TMP}'" 0

pushd "${TMP}" > /dev/null
{
    mkdir -p dashboards folders

    for dash in $(curl --fail --silent --show-error --location "${grafanaaddr}/api/search?query=" | jq -r '.[] | select(.type == "dash-db") | .uid')
    do
        curl --fail --silent --show-error --location "${grafanaaddr}/api/dashboards/uid/$dash" \
            | jq '. |= (.folderUid=.meta.folderUid) |del(.meta) |del(.dashboard.id) + {overwrite: true}' \
                 > dashboards/${dash}.json
    done

    for folder in $(curl --fail --silent --show-error --location "${grafanaaddr}/api/search?query=" | jq -r '.[] | select(.type == "dash-folder") | .uid')
    do
        curl --fail --silent --show-error --location "${grafanaaddr}/api/folders/$folder" \
            | jq '. |del(.id) + {overwrite: true}' \
                 > folders/${folder}.json
    done
    ipfa .|sed -r 's/[?]filename.+//'
}
popd > /dev/null

https://konubinix.eu/ipfs/bafybeicnhet2urodopu27zvkd6dymofm6dekjjfgxjpj6gfjyx34na7gxa

Cleaning is done using the DELETE http request. Note that provisioned data won’t be deleted. Therefore, in that example, the folder nomads and the dasboard nodes will remain.

for dash in $(curl --fail --silent --show-error --location "${grafanaaddr}/api/search?query=" | jq -r '.[] | select(.type == "dash-db") | .uid')
do
    { curl --fail --silent --show-error --location --request DELETE "${grafanaaddr}/api/dashboards/uid/$dash" && echo ; } || echo "Could not delete dashboard ${dash}"
done

for folder in $(curl --fail --silent --show-error --location "${grafanaaddr}/api/search?query=" | jq -r '.[] | select(.type == "dash-folder") | .uid')
do
    { curl --fail --silent --show-error --location --request DELETE "${grafanaaddr}/api/folders/$folder" && echo ; } || echo "Could not delete folder ${folder}"
done

{"message":"Dashboard New dashboard deleted","title":"New dashboard","uid":"eeexdctweuarkc"}
Could not delete dashboard nomadnodes
Could not delete folder aeenvk7uz00zke
{"message":"Folder deleted"}

I get a 400 error for the provisioned data not removed, and only in grafana logs I can see the reason. Also, as we can see, having alert rules will prevent the destruction of a folder also.

clk nd logs --job grafana | tail -4

logger=context userId=0 orgId=1 uname= t=2025-03-05T13:34:40.794075705Z level=info msg="Request Completed" method=DELETE path=/api/dashboards/uid/nomadnodes status=400 remote_addr=192.168.1.245 time_ms=16 duration=16.354504ms size=53 referer= handler=/api/dashboards/uid/:uid status_source=server error="provisioned dashboard cannot be deleted"
logger=folder-service t=2025-03-05T13:34:40.862478022Z level=info msg="deleting folder and its descendants" org_id=1 uid=aeenvk7uz00zke
logger=context userId=0 orgId=1 uname= t=2025-03-05T13:34:40.868706767Z level=info msg="Request Completed" method=DELETE path=/api/folders/aeenvk7uz00zke status=400 remote_addr=192.168.1.245 time_ms=31 duration=31.547021ms size=107 referer= handler=/api/folders/:uid/ status_source=server errorReason=BadRequest errorMessageID=folder.not-empty error="folder contains 6 alert rules"
logger=folder-service t=2025-03-05T13:34:40.907181767Z level=info msg="deleting folder and its descendants" org_id=1 uid=eeexdccr6xiwwb

Then, restoring with the reverse script

pushd "${saved}" > /dev/null
{
    for folder in folders/*json
    do
        { curl --fail --silent --show-error --location "${grafanaaddr}/api/folders" -H 'Content-Type: application/json' --data @${folder} && echo ; } || echo "Could not create folder ${folder}"
    done

    for dash in dashboards/*json
    do
        { curl --fail --silent --show-error --location "${grafanaaddr}/api/dashboards/db" -H 'Content-Type: application/json' --data @${dash} && echo ; } || echo "Could not create dashboard ${dash}"
    done
}
popd > /dev/null

Could not create folder folders/aeenvk7uz00zke.json
{"id":13,"uid":"eeexdccr6xiwwb","orgId":0,"title":"test","url":"/grafana/dashboards/f/eeexdccr6xiwwb/test","hasAcl":false,"canSave":true,"canEdit":true,"canAdmin":true,"canDelete":true,"createdBy":"Anonymous","created":"2025-03-05T13:40:10.452317314Z","updatedBy":"Anonymous","updated":"2025-03-05T13:40:10.452317555Z","version":1,"parentUid":"aeenvk7uz00zke","parents":[{"id":1,"uid":"aeenvk7uz00zke","orgId":0,"title":"nomad","url":"/grafana/dashboards/f/aeenvk7uz00zke/nomad","hasAcl":false,"canSave":true,"canEdit":true,"canAdmin":true,"canDelete":true,"createdBy":"Anonymous","created":"2025-03-02T21:41:40.054193083Z","updatedBy":"Anonymous","updated":"2025-03-02T21:41:40.054193213Z"}]}
{"folderUid":"eeexdccr6xiwwb","id":14,"slug":"new-dashboard","status":"success","uid":"eeexdctweuarkc","url":"/grafana/d/eeexdctweuarkc/new-dashboard","version":1}
Could not create dashboard dashboards/nomadnodes.json

Konubinix' opinionated web of thoughts