diff --git a/README.md b/README.md index 377b6be..1627db3 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,7 @@ Example: dockcheck.sh -y -d 10 -e nextcloud,heimdall Options:" -a|y Automatic updates, without interaction. +-c Exports metrics as prom file for the prometheus node_exporter. Provide the collector textfile directory. -d N Only update to new images that are N+ days old. Lists too recent with +prefix and age. 2xSlower. -e X Exclude containers, separated by comma. -f Force stack restart after update. Caution: restarts once for every updated container within stack. @@ -135,6 +136,66 @@ nginx -> https://github.com/docker-library/official-images/blob/master/library ``` The `urls.list` file is just an example and I'd gladly see that people contribute back when they add their preferred URLs to their lists. +## Prometheus +Dockcheck is capable to export metrics to prometheus via the text file collector provided by the node_exporter. In order to do so the -c flag has to be specified followed by the file path that is configured in the text file collector of the node_exporter. A simple cron job can be configured to export these metrics on a regular interval as shown in the sample below: + +``` +0 1 * * * /root/dockcheck.sh -n -c /var/lib/node_exporter/textfile_collector +``` + +The following metrics are exported to prometheus + +``` +# HELP dockcheck_images_analyzed Docker images that have been analyzed +# TYPE dockcheck_images_analyzed gauge +dockcheck_images_analyzed 22 +# HELP dockcheck_images_outdated Docker images that are outdated +# TYPE dockcheck_images_outdated gauge +dockcheck_images_outdated 7 +# HELP dockcheck_images_latest Docker images that are outdated +# TYPE dockcheck_images_latest gauge +dockcheck_images_latest 14 +# HELP dockcheck_images_error Docker images with analysis errors +# TYPE dockcheck_images_error gauge +dockcheck_images_error 1 +# HELP dockcheck_images_analyze_timestamp_seconds Last dockercheck run time +# TYPE dockcheck_images_analyze_timestamp_seconds gauge +dockcheck_images_analyze_timestamp_seconds 1737924029 +``` + +Once those metrics are exported they can be used to define alarms as shown below + +``` +- alert: dockcheck_images_outdated + expr: sum by(instance) (dockcheck_images_outdated) > 0 + for: 15s + labels: + severity: warning + annotations: + summary: "{{ $labels.instance }} has {{ $value }} outdated docker images." + description: "{{ $labels.instance }} has {{ $value }} outdated docker images." +- alert: dockcheck_images_error + expr: sum by(instance) (dockcheck_images_error) > 0 + for: 15s + labels: + severity: warning + annotations: + summary: "{{ $labels.instance }} has {{ $value }} docker images having an error." + description: "{{ $labels.instance }} has {{ $value }} docker images having an error." +- alert: dockercheck_image_last_analyze + expr: (time() - dockcheck_images_analyze_timestamp_seconds) > (3600 * 24 * 3) + for: 15s + labels: + severity: warning + annotations: + summary: "{{ $labels.instance }} has not updated the dockcheck statistics for more than 3 days." + description: "{{ $labels.instance }} has not updated the dockcheck statistics for more than 3 days." +``` + +There is a reference Grafana dashboard in [grafana/grafana_dashboard.json](./grafana/grafana_dashboard.json). + +![](./grafana/grafana_dashboard.png) + ## :bookmark: Labels Optionally add labels to compose-files. Currently these are the usable labels: ``` diff --git a/dockcheck.sh b/dockcheck.sh index 7ab7280..d182da8 100755 --- a/dockcheck.sh +++ b/dockcheck.sh @@ -20,6 +20,7 @@ Help() { echo echo "Options:" echo "-a|y Automatic updates, without interaction." + echo "-c Exports metrics as prom file for the prometheus node_exporter. Provide the collector textfile directory." echo "-d N Only update to new images that are N+ days old. Lists too recent with +prefix and age. 2xSlower." echo "-e X Exclude containers, separated by comma." echo "-f Force stack restart after update. Caution: restarts once for every updated container within stack." @@ -47,9 +48,11 @@ c_reset="\033[0m" Timeout=10 Stopped="" -while getopts "aynpfrhlisvme:d:t:" options; do +while getopts "aynpfrhlisvmc:e:d:t:" options; do case "${options}" in a|y) AutoUp="yes" ;; + c) CollectorTextFileDirectory="${OPTARG}" + if ! [[ -d $CollectorTextFileDirectory ]] ; then { printf "The directory (%s) does not exist.\n" "${CollectorTextFileDirectory}" ; exit 2; } fi ;; n) AutoUp="no" ;; r) DRunUp="yes" ;; p) AutoPrune="yes" ;; @@ -309,6 +312,35 @@ NoUpdates=($(sort <<<"${NoUpdates[*]}")) GotUpdates=($(sort <<<"${GotUpdates[*]}")) unset IFS +if [ -n "$CollectorTextFileDirectory" ] ; then + checkedImages=$((${#NoUpdates[@]} + ${#GotUpdates[@]} + ${#GotErrors[@]})) + checkTimestamp=$(date +%s) + + promFileContent=() + promFileContent+=("# HELP dockcheck_images_analyzed Docker images that have been analyzed") + promFileContent+=("# TYPE dockcheck_images_analyzed gauge") + promFileContent+=("dockcheck_images_analyzed $checkedImages") + + promFileContent+=("# HELP dockcheck_images_outdated Docker images that are outdated") + promFileContent+=("# TYPE dockcheck_images_outdated gauge") + promFileContent+=("dockcheck_images_outdated ${#GotUpdates[@]}") + + promFileContent+=("# HELP dockcheck_images_latest Docker images that are outdated") + promFileContent+=("# TYPE dockcheck_images_latest gauge") + promFileContent+=("dockcheck_images_latest ${#NoUpdates[@]}") + + promFileContent+=("# HELP dockcheck_images_error Docker images with analysis errors") + promFileContent+=("# TYPE dockcheck_images_error gauge") + promFileContent+=("dockcheck_images_error ${#GotErrors[@]}") + + promFileContent+=("# HELP dockcheck_images_analyze_timestamp_seconds Last dockercheck run time") + promFileContent+=("# TYPE dockcheck_images_analyze_timestamp_seconds gauge") + promFileContent+=("dockcheck_images_analyze_timestamp_seconds $checkTimestamp") + + printf "%s\n" "${promFileContent[@]}" > "$CollectorTextFileDirectory/dockcheck_info.prom\$\$" + mv -f "$CollectorTextFileDirectory/dockcheck_info.prom\$\$" "$CollectorTextFileDirectory/dockcheck_info.prom" +fi + # Define how many updates are available UpdCount="${#GotUpdates[@]}" diff --git a/grafana/grafana_dashboard.json b/grafana/grafana_dashboard.json new file mode 100644 index 0000000..ad308bb --- /dev/null +++ b/grafana/grafana_dashboard.json @@ -0,0 +1,382 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "11.4.0" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "table", + "name": "Table", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "last_analyze_timestamp" + }, + "properties": [ + { + "id": "unit", + "value": "dateTimeAsIso" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "last_analyze_since" + }, + "properties": [ + { + "id": "unit", + "value": "s" + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "color-background" + } + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 259200 + } + ] + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "images_outdated" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "color-background" + } + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "images_error" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "color-background" + } + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + } + ] + } + ] + }, + "gridPos": { + "h": 14, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "frameIndex": 1, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "disableTextWrap": false, + "editorMode": "code", + "exemplar": false, + "expr": "sum by(instance) (dockcheck_images_analyzed)", + "format": "table", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": true, + "interval": "", + "legendFormat": "{{instance}}", + "range": false, + "refId": "dockcheck_images_analyzed", + "useBackend": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "code", + "exemplar": false, + "expr": "sum by(instance) (dockcheck_images_outdated)", + "format": "table", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": true, + "legendFormat": "{{instance}}", + "range": false, + "refId": "dockcheck_images_outdated", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "code", + "exemplar": false, + "expr": "sum by(instance) (dockcheck_images_latest)", + "format": "table", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": true, + "legendFormat": "{{instance}}", + "range": false, + "refId": "dockcheck_images_latest", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by(instance) (dockcheck_images_error)", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "{{instance}}", + "range": false, + "refId": "dockcheck_images_error" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "dockcheck_images_analyze_timestamp_seconds * 1000", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "{{instance}}", + "range": false, + "refId": "dockcheck_images_analyze_timestamp_seconds" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "time() - dockcheck_images_analyze_timestamp_seconds", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "{{instance}}", + "range": false, + "refId": "dockcheck_images_last_analyze" + } + ], + "title": "Dockcheck Status", + "transformations": [ + { + "id": "merge", + "options": {} + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "__name__": true, + "job": true + }, + "includeByName": {}, + "indexByName": { + "Time": 0, + "Value #dockcheck_images_analyze_timestamp_seconds": 2, + "Value #dockcheck_images_analyzed": 4, + "Value #dockcheck_images_error": 7, + "Value #dockcheck_images_last_analyze": 3, + "Value #dockcheck_images_latest": 5, + "Value #dockcheck_images_outdated": 6, + "instance": 1, + "job": 8 + }, + "renameByName": { + "Value #A": "analyze_timestamp", + "Value #dockcheck_images_analyze_timestamp_seconds": "last_analyze_timestamp", + "Value #dockcheck_images_analyzed": "images_analyzed", + "Value #dockcheck_images_error": "images_error", + "Value #dockcheck_images_last_analyze": "last_analyze_since", + "Value #dockcheck_images_latest": "images_latest", + "Value #dockcheck_images_outdated": "images_outdated" + } + } + } + ], + "type": "table" + } + ], + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Dockcheck Status", + "uid": "feb4pv3kv1hxca", + "version": 17, + "weekStart": "" +} \ No newline at end of file diff --git a/grafana/grafana_dashboard.png b/grafana/grafana_dashboard.png new file mode 100644 index 0000000..c3878df Binary files /dev/null and b/grafana/grafana_dashboard.png differ