From b3600f26ac619108160501cc5ca5329ce6a4eb1b Mon Sep 17 00:00:00 2001 From: mag37 Date: Fri, 31 Jan 2025 23:52:10 +0100 Subject: [PATCH] restructure, moved to addon-subdir and split code to sourced snippet --- README.md | 63 ++---------------- addons/prometheus/README_prom.md | 63 ++++++++++++++++++ .../grafana}/grafana_dashboard.json | 0 .../prometheus/grafana}/grafana_dashboard.png | Bin addons/prometheus/prometheus_collector.sh | 28 ++++++++ dockcheck.sh | 32 ++------- 6 files changed, 102 insertions(+), 84 deletions(-) create mode 100644 addons/prometheus/README_prom.md rename {grafana => addons/prometheus/grafana}/grafana_dashboard.json (100%) rename {grafana => addons/prometheus/grafana}/grafana_dashboard.png (100%) create mode 100644 addons/prometheus/prometheus_collector.sh diff --git a/README.md b/README.md index bc723ac..d3769c2 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ ___ ## :bell: Changelog +- **v0.5.4**: Added support for a Prometheus+node_exporter metric collection through a file collector. - **v0.5.3**: Local image check changed (use imageId instead of name) and Gotify-template fixed (whale icon removed). - **v0.5.2.1**: Rewrite of dependency downloads, jq can be installed with package manager or static binary. - **v0.5.1**: DEPENDENCY WARNING: now requires **jq**. + Upstreaming changes from [sudo-kraken/podcheck](https://github.com/sudo-kraken/podcheck) @@ -136,65 +137,15 @@ nginx -> https://github.com/docker-library/official-images/blob/master/library ``` The `urls.list` file is just an example and I'd gladly see that people contribute back when they add their preferred URLs to their lists. -## Prometheus -Dockcheck is capable to export metrics to prometheus via the text file collector provided by the node_exporter. In order to do so the -c flag has to be specified followed by the file path that is configured in the text file collector of the node_exporter. A simple cron job can be configured to export these metrics on a regular interval as shown in the sample below: - +## [Prometheus](https://github.com/prometheus/prometheus) and [node_exporter](https://github.com/prometheus/node_exporter) +Dockcheck can be used together with Prometheus and node_exporter to export metrics via the file collector, scheduled with cron or likely. +This is done with the `-c` option, like this: ``` -0 1 * * * /root/dockcheck.sh -n -c /var/lib/node_exporter/textfile_collector +dockcheck.sh -c /path/to/exporter/directory ``` + +See the [README_prom.md](./addons/prometheus/README_prom.md) for more detailed information on how to set it up! -The following metrics are exported to prometheus - -``` -# HELP dockcheck_images_analyzed Docker images that have been analyzed -# TYPE dockcheck_images_analyzed gauge -dockcheck_images_analyzed 22 -# HELP dockcheck_images_outdated Docker images that are outdated -# TYPE dockcheck_images_outdated gauge -dockcheck_images_outdated 7 -# HELP dockcheck_images_latest Docker images that are outdated -# TYPE dockcheck_images_latest gauge -dockcheck_images_latest 14 -# HELP dockcheck_images_error Docker images with analysis errors -# TYPE dockcheck_images_error gauge -dockcheck_images_error 1 -# HELP dockcheck_images_analyze_timestamp_seconds Last dockercheck run time -# TYPE dockcheck_images_analyze_timestamp_seconds gauge -dockcheck_images_analyze_timestamp_seconds 1737924029 -``` - -Once those metrics are exported they can be used to define alarms as shown below - -``` -- alert: dockcheck_images_outdated - expr: sum by(instance) (dockcheck_images_outdated) > 0 - for: 15s - labels: - severity: warning - annotations: - summary: "{{ $labels.instance }} has {{ $value }} outdated docker images." - description: "{{ $labels.instance }} has {{ $value }} outdated docker images." -- alert: dockcheck_images_error - expr: sum by(instance) (dockcheck_images_error) > 0 - for: 15s - labels: - severity: warning - annotations: - summary: "{{ $labels.instance }} has {{ $value }} docker images having an error." - description: "{{ $labels.instance }} has {{ $value }} docker images having an error." -- alert: dockercheck_image_last_analyze - expr: (time() - dockcheck_images_analyze_timestamp_seconds) > (3600 * 24 * 3) - for: 15s - labels: - severity: warning - annotations: - summary: "{{ $labels.instance }} has not updated the dockcheck statistics for more than 3 days." - description: "{{ $labels.instance }} has not updated the dockcheck statistics for more than 3 days." -``` - -There is a reference Grafana dashboard in [grafana/grafana_dashboard.json](./grafana/grafana_dashboard.json). - -![](./grafana/grafana_dashboard.png) ## :bookmark: Labels Optionally add labels to compose-files. Currently these are the usable labels: diff --git a/addons/prometheus/README_prom.md b/addons/prometheus/README_prom.md new file mode 100644 index 0000000..6377e50 --- /dev/null +++ b/addons/prometheus/README_prom.md @@ -0,0 +1,63 @@ +## [Prometheus](https://github.com/prometheus/prometheus) and [node_exporter](https://github.com/prometheus/node_exporter) +Dockcheck is capable to export metrics to prometheus via the text file collector provided by the node_exporter. +In order to do so the -c flag has to be specified followed by the file path that is configured in the text file collector of the node_exporter. +A simple cron job can be configured to export these metrics on a regular interval as shown in the sample below: + +``` +0 1 * * * /root/dockcheck.sh -n -c /var/lib/node_exporter/textfile_collector +``` + +The following metrics are exported to prometheus + +``` +# HELP dockcheck_images_analyzed Docker images that have been analyzed +# TYPE dockcheck_images_analyzed gauge +dockcheck_images_analyzed 22 +# HELP dockcheck_images_outdated Docker images that are outdated +# TYPE dockcheck_images_outdated gauge +dockcheck_images_outdated 7 +# HELP dockcheck_images_latest Docker images that are outdated +# TYPE dockcheck_images_latest gauge +dockcheck_images_latest 14 +# HELP dockcheck_images_error Docker images with analysis errors +# TYPE dockcheck_images_error gauge +dockcheck_images_error 1 +# HELP dockcheck_images_analyze_timestamp_seconds Last dockercheck run time +# TYPE dockcheck_images_analyze_timestamp_seconds gauge +dockcheck_images_analyze_timestamp_seconds 1737924029 +``` + +Once those metrics are exported they can be used to define alarms as shown below + +``` +- alert: dockcheck_images_outdated + expr: sum by(instance) (dockcheck_images_outdated) > 0 + for: 15s + labels: + severity: warning + annotations: + summary: "{{ $labels.instance }} has {{ $value }} outdated docker images." + description: "{{ $labels.instance }} has {{ $value }} outdated docker images." +- alert: dockcheck_images_error + expr: sum by(instance) (dockcheck_images_error) > 0 + for: 15s + labels: + severity: warning + annotations: + summary: "{{ $labels.instance }} has {{ $value }} docker images having an error." + description: "{{ $labels.instance }} has {{ $value }} docker images having an error." +- alert: dockercheck_image_last_analyze + expr: (time() - dockcheck_images_analyze_timestamp_seconds) > (3600 * 24 * 3) + for: 15s + labels: + severity: warning + annotations: + summary: "{{ $labels.instance }} has not updated the dockcheck statistics for more than 3 days." + description: "{{ $labels.instance }} has not updated the dockcheck statistics for more than 3 days." +``` + +There is a reference Grafana dashboard in [grafana/grafana_dashboard.json](./grafana/grafana_dashboard.json). + +![](./grafana/grafana_dashboard.png) + + diff --git a/grafana/grafana_dashboard.json b/addons/prometheus/grafana/grafana_dashboard.json similarity index 100% rename from grafana/grafana_dashboard.json rename to addons/prometheus/grafana/grafana_dashboard.json diff --git a/grafana/grafana_dashboard.png b/addons/prometheus/grafana/grafana_dashboard.png similarity index 100% rename from grafana/grafana_dashboard.png rename to addons/prometheus/grafana/grafana_dashboard.png diff --git a/addons/prometheus/prometheus_collector.sh b/addons/prometheus/prometheus_collector.sh new file mode 100644 index 0000000..c98af2b --- /dev/null +++ b/addons/prometheus/prometheus_collector.sh @@ -0,0 +1,28 @@ +prometheus_exporter() { + checkedImages=$(($1 + $2 + $3)) + checkTimestamp=$(date +%s) + + promFileContent=() + promFileContent+=("# HELP dockcheck_images_analyzed Docker images that have been analyzed") + promFileContent+=("# TYPE dockcheck_images_analyzed gauge") + promFileContent+=("dockcheck_images_analyzed $checkedImages") + + promFileContent+=("# HELP dockcheck_images_outdated Docker images that are outdated") + promFileContent+=("# TYPE dockcheck_images_outdated gauge") + promFileContent+=("dockcheck_images_outdated ${#GotUpdates[@]}") + + promFileContent+=("# HELP dockcheck_images_latest Docker images that are outdated") + promFileContent+=("# TYPE dockcheck_images_latest gauge") + promFileContent+=("dockcheck_images_latest ${#NoUpdates[@]}") + + promFileContent+=("# HELP dockcheck_images_error Docker images with analysis errors") + promFileContent+=("# TYPE dockcheck_images_error gauge") + promFileContent+=("dockcheck_images_error ${#GotErrors[@]}") + + promFileContent+=("# HELP dockcheck_images_analyze_timestamp_seconds Last dockercheck run time") + promFileContent+=("# TYPE dockcheck_images_analyze_timestamp_seconds gauge") + promFileContent+=("dockcheck_images_analyze_timestamp_seconds $checkTimestamp") + + printf "%s\n" "${promFileContent[@]}" > "$CollectorTextFileDirectory/dockcheck_info.prom\$\$" + mv -f "$CollectorTextFileDirectory/dockcheck_info.prom\$\$" "$CollectorTextFileDirectory/dockcheck_info.prom" +} diff --git a/dockcheck.sh b/dockcheck.sh index 1cf943b..f42febf 100755 --- a/dockcheck.sh +++ b/dockcheck.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -VERSION="v0.5.3.0" -### ChangeNotes: Bugfixes - local image check changed, Gotify-template fixed +VERSION="v0.5.4.0" +### ChangeNotes: Added support for a Prometheus+node_exporter metric collection through a file collector. Github="https://github.com/mag37/dockcheck" RawUrl="https://raw.githubusercontent.com/mag37/dockcheck/main/dockcheck.sh" @@ -313,33 +313,9 @@ NoUpdates=($(sort <<<"${NoUpdates[*]}")) GotUpdates=($(sort <<<"${GotUpdates[*]}")) unset IFS +# Run the prometheus exporter function if [ -n "$CollectorTextFileDirectory" ] ; then - checkedImages=$((${#NoUpdates[@]} + ${#GotUpdates[@]} + ${#GotErrors[@]})) - checkTimestamp=$(date +%s) - - promFileContent=() - promFileContent+=("# HELP dockcheck_images_analyzed Docker images that have been analyzed") - promFileContent+=("# TYPE dockcheck_images_analyzed gauge") - promFileContent+=("dockcheck_images_analyzed $checkedImages") - - promFileContent+=("# HELP dockcheck_images_outdated Docker images that are outdated") - promFileContent+=("# TYPE dockcheck_images_outdated gauge") - promFileContent+=("dockcheck_images_outdated ${#GotUpdates[@]}") - - promFileContent+=("# HELP dockcheck_images_latest Docker images that are outdated") - promFileContent+=("# TYPE dockcheck_images_latest gauge") - promFileContent+=("dockcheck_images_latest ${#NoUpdates[@]}") - - promFileContent+=("# HELP dockcheck_images_error Docker images with analysis errors") - promFileContent+=("# TYPE dockcheck_images_error gauge") - promFileContent+=("dockcheck_images_error ${#GotErrors[@]}") - - promFileContent+=("# HELP dockcheck_images_analyze_timestamp_seconds Last dockercheck run time") - promFileContent+=("# TYPE dockcheck_images_analyze_timestamp_seconds gauge") - promFileContent+=("dockcheck_images_analyze_timestamp_seconds $checkTimestamp") - - printf "%s\n" "${promFileContent[@]}" > "$CollectorTextFileDirectory/dockcheck_info.prom\$\$" - mv -f "$CollectorTextFileDirectory/dockcheck_info.prom\$\$" "$CollectorTextFileDirectory/dockcheck_info.prom" + source "$ScriptWorkDir"/addons/prometheus/prometheus_collector.sh && prometheus_exporter ${#NoUpdates[@]} ${#GotUpdates[@]} ${#GotError[@]} fi # Define how many updates are available