From 62a3d10b4f73e8c24347f3846365c658a0d7cd2f Mon Sep 17 00:00:00 2001 From: Thaurin Date: Wed, 19 Feb 2025 16:28:11 +0100 Subject: [PATCH 1/6] Add async checking for updates for improved performance --- dockcheck.sh | 107 ++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 84 insertions(+), 23 deletions(-) diff --git a/dockcheck.sh b/dockcheck.sh index 365d7cb..5d0c38e 100755 --- a/dockcheck.sh +++ b/dockcheck.sh @@ -34,6 +34,7 @@ Help() { echo "-s Include stopped containers in the check. (Logic: docker ps -a)." echo "-t Set a timeout (in seconds) per container for registry checkups, 10 is default." echo "-v Prints current version." + echo "-z [Experimental] Check images asynchrously for increased performance." echo echo "Project source: $Github" } @@ -48,7 +49,7 @@ c_reset="\033[0m" Timeout=10 Stopped="" -while getopts "aynpfrhlisvmc:e:d:t:" options; do +while getopts "aynpfrhlisvmzc:e:d:t:" options; do case "${options}" in a|y) AutoUp="yes" ;; c) CollectorTextFileDirectory="${OPTARG}" @@ -64,6 +65,7 @@ while getopts "aynpfrhlisvmc:e:d:t:" options; do s) Stopped="-a" ;; t) Timeout="${OPTARG}" ;; v) printf "%s\n" "$VERSION" ; exit 0 ;; + z) ParallelCheck=1 ;; d) DaysOld=${OPTARG} if ! [[ $DaysOld =~ ^[0-9]+$ ]] ; then { printf "Days -d argument given (%s) is not a number.\n" "${DaysOld}" ; exit 2 ; } ; fi ;; h|*) Help ; exit 2 ;; @@ -282,31 +284,90 @@ if [[ $t_out ]]; then else t_out="" fi -# Check the image-hash of every running container VS the registry -for i in $(docker ps $Stopped --filter "name=$SearchName" --format '{{.Names}}') ; do - ((RegCheckQue+=1)) - progress_bar "$RegCheckQue" "$ContCount" - # Looping every item over the list of excluded names and skipping - for e in "${Excludes[@]}" ; do [[ "$i" == "$e" ]] && continue 2 ; done - ImageId=$(docker inspect "$i" --format='{{.Image}}') - RepoUrl=$(docker inspect "$i" --format='{{.Config.Image}}') - LocalHash=$(docker image inspect "$ImageId" --format '{{.RepoDigests}}') - # Checking for errors while setting the variable - if RegHash=$(${t_out} $regbin -v error image digest --list "$RepoUrl" 2>&1) ; then - if [[ "$LocalHash" = *"$RegHash"* ]] ; then - NoUpdates+=("$i") - else - if [[ -n "$DaysOld" ]] && ! datecheck ; then - NoUpdates+=("+$i ${ImageAge}d") +if [[ $ParallelCheck -ne 1 ]]; then + # Check the image-hash of every running container VS the registry + for i in $(docker ps $Stopped --filter "name=$SearchName" --format '{{.Names}}') ; do + ((RegCheckQue+=1)) + progress_bar "$RegCheckQue" "$ContCount" + # Looping every item over the list of excluded names and skipping + for e in "${Excludes[@]}" ; do [[ "$i" == "$e" ]] && continue 2 ; done + ImageId=$(docker inspect "$i" --format='{{.Image}}') + RepoUrl=$(docker inspect "$i" --format='{{.Config.Image}}') + LocalHash=$(docker image inspect "$ImageId" --format '{{.RepoDigests}}') + # Checking for errors while setting the variable + if RegHash=$(${t_out} $regbin -v error image digest --list "$RepoUrl" 2>&1) ; then + if [[ "$LocalHash" = *"$RegHash"* ]] ; then + NoUpdates+=("$i") else - GotUpdates+=("$i") + if [[ -n "$DaysOld" ]] && ! datecheck ; then + NoUpdates+=("+$i ${ImageAge}d") + else + GotUpdates+=("$i") + fi fi + else + # Here the RegHash is the result of an error code + GotErrors+=("$i - ${RegHash}") fi - else - # Here the RegHash is the result of an error code - GotErrors+=("$i - ${RegHash}") - fi -done + done +else + check_image() { + i="$1" + local Excludes=($Excludes_string) + local skip + for e in "${Excludes[@]}" ; do [[ "$i" == "$e" ]] && skip=1 ; done + + if [[ $skip -eq 1 ]]; then + echo Skip $i + return + fi + + local NoUpdates GotUpdates GotErrors + ImageId=$(docker inspect "$i" --format='{{.Image}}') + RepoUrl=$(docker inspect "$i" --format='{{.Config.Image}}') + LocalHash=$(docker image inspect "$ImageId" --format '{{.RepoDigests}}') + + # Checking for errors while setting the variable + if RegHash=$(${t_out} $regbin -v error image digest --list "$RepoUrl" 2>&1) ; then + if [[ "$LocalHash" = *"$RegHash"* ]] ; then + echo NoUpdates "$i" + else + if [[ -n "$DaysOld" ]] && ! datecheck ; then + echo NoUpdates "+$i ${ImageAge}d" + else + echo GotUpdates "$i" + fi + fi + else + # Here the RegHash is the result of an error code + echo GotErrors "$i - ${RegHash}" + fi + } + + export -f check_image datecheck + export Excludes_string="${Excludes[@]}" # Can only export scalar variables + export t_out regbin RepoUrl + + # Asynchronously check the image-hash of every running container VS the registry + while read -r line; do + ((RegCheckQue+=1)) + progress_bar "$RegCheckQue" "$ContCount" + + Got=${line%% *} # Extracts the first word (NoUpdates, GotUpdates, GotErrors) + item=${line#* } + + case "$Got" in + NoUpdates) NoUpdates+=("$item") ;; + GotUpdates) GotUpdates+=("$item") ;; + GotErrors) GotErrors+=("$item") ;; + Skip) ;; + *) ;; + esac + done < <( \ + docker ps $Stopped --filter "name=$SearchName" --format '{{.Names}}' | \ + xargs -P 8 -I {} bash -c 'check_image "{}"' \ + ) +fi # Sort arrays alphabetically IFS=$'\n' From a2868ea50531d22a7d1ddbbbd1a4d4a8faa6c542 Mon Sep 17 00:00:00 2001 From: Thaurin Date: Thu, 20 Feb 2025 15:54:52 +0100 Subject: [PATCH 2/6] Add error message; increase number of subprocesses --- dockcheck.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dockcheck.sh b/dockcheck.sh index 5d0c38e..9e3da8b 100755 --- a/dockcheck.sh +++ b/dockcheck.sh @@ -361,11 +361,11 @@ else GotUpdates) GotUpdates+=("$item") ;; GotErrors) GotErrors+=("$item") ;; Skip) ;; - *) ;; + *) echo "Error: invalid result from subprocess! (${item})" ;; esac done < <( \ docker ps $Stopped --filter "name=$SearchName" --format '{{.Names}}' | \ - xargs -P 8 -I {} bash -c 'check_image "{}"' \ + xargs -P 32 -I {} bash -c 'check_image "{}"' \ ) fi From 408a8b14dd5a3f1ac3c7ac4beaf99f34108a5b7a Mon Sep 17 00:00:00 2001 From: Thaurin Date: Thu, 20 Feb 2025 18:39:27 +0100 Subject: [PATCH 3/6] Fix -d parameter not working anymore --- dockcheck.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dockcheck.sh b/dockcheck.sh index 9e3da8b..0327b27 100755 --- a/dockcheck.sh +++ b/dockcheck.sh @@ -346,7 +346,7 @@ else export -f check_image datecheck export Excludes_string="${Excludes[@]}" # Can only export scalar variables - export t_out regbin RepoUrl + export t_out regbin RepoUrl DaysOld # Asynchronously check the image-hash of every running container VS the registry while read -r line; do From 3aeee837f9bb426f9c65534b7bab2a29d8525aa4 Mon Sep 17 00:00:00 2001 From: Thaurin Date: Fri, 21 Feb 2025 17:54:19 +0100 Subject: [PATCH 4/6] Print entire line on error --- dockcheck.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dockcheck.sh b/dockcheck.sh index 0327b27..13bf6e2 100755 --- a/dockcheck.sh +++ b/dockcheck.sh @@ -361,7 +361,7 @@ else GotUpdates) GotUpdates+=("$item") ;; GotErrors) GotErrors+=("$item") ;; Skip) ;; - *) echo "Error: invalid result from subprocess! (${item})" ;; + *) echo "Error! Unexpected output from subprocess: ${line}" ;; esac done < <( \ docker ps $Stopped --filter "name=$SearchName" --format '{{.Names}}' | \ From b2d67c9f524520d13ce591c2e38073d8784f481a Mon Sep 17 00:00:00 2001 From: Thaurin Date: Fri, 21 Feb 2025 17:57:09 +0100 Subject: [PATCH 5/6] Removed experimental -z flag and old version check code --- dockcheck.sh | 129 ++++++++++++++++++++------------------------------- 1 file changed, 50 insertions(+), 79 deletions(-) diff --git a/dockcheck.sh b/dockcheck.sh index 13bf6e2..462e7e6 100755 --- a/dockcheck.sh +++ b/dockcheck.sh @@ -34,7 +34,6 @@ Help() { echo "-s Include stopped containers in the check. (Logic: docker ps -a)." echo "-t Set a timeout (in seconds) per container for registry checkups, 10 is default." echo "-v Prints current version." - echo "-z [Experimental] Check images asynchrously for increased performance." echo echo "Project source: $Github" } @@ -49,7 +48,7 @@ c_reset="\033[0m" Timeout=10 Stopped="" -while getopts "aynpfrhlisvmzc:e:d:t:" options; do +while getopts "aynpfrhlisvmc:e:d:t:" options; do case "${options}" in a|y) AutoUp="yes" ;; c) CollectorTextFileDirectory="${OPTARG}" @@ -65,7 +64,6 @@ while getopts "aynpfrhlisvmzc:e:d:t:" options; do s) Stopped="-a" ;; t) Timeout="${OPTARG}" ;; v) printf "%s\n" "$VERSION" ; exit 0 ;; - z) ParallelCheck=1 ;; d) DaysOld=${OPTARG} if ! [[ $DaysOld =~ ^[0-9]+$ ]] ; then { printf "Days -d argument given (%s) is not a number.\n" "${DaysOld}" ; exit 2 ; } ; fi ;; h|*) Help ; exit 2 ;; @@ -284,90 +282,63 @@ if [[ $t_out ]]; then else t_out="" fi -if [[ $ParallelCheck -ne 1 ]]; then - # Check the image-hash of every running container VS the registry - for i in $(docker ps $Stopped --filter "name=$SearchName" --format '{{.Names}}') ; do - ((RegCheckQue+=1)) - progress_bar "$RegCheckQue" "$ContCount" - # Looping every item over the list of excluded names and skipping - for e in "${Excludes[@]}" ; do [[ "$i" == "$e" ]] && continue 2 ; done - ImageId=$(docker inspect "$i" --format='{{.Image}}') - RepoUrl=$(docker inspect "$i" --format='{{.Config.Image}}') - LocalHash=$(docker image inspect "$ImageId" --format '{{.RepoDigests}}') - # Checking for errors while setting the variable - if RegHash=$(${t_out} $regbin -v error image digest --list "$RepoUrl" 2>&1) ; then - if [[ "$LocalHash" = *"$RegHash"* ]] ; then - NoUpdates+=("$i") - else - if [[ -n "$DaysOld" ]] && ! datecheck ; then - NoUpdates+=("+$i ${ImageAge}d") - else - GotUpdates+=("$i") - fi - fi +check_image() { + i="$1" + local Excludes=($Excludes_string) + local skip + for e in "${Excludes[@]}" ; do [[ "$i" == "$e" ]] && skip=1 ; done + + if [[ $skip -eq 1 ]]; then + echo Skip $i + return + fi + + local NoUpdates GotUpdates GotErrors + ImageId=$(docker inspect "$i" --format='{{.Image}}') + RepoUrl=$(docker inspect "$i" --format='{{.Config.Image}}') + LocalHash=$(docker image inspect "$ImageId" --format '{{.RepoDigests}}') + + # Checking for errors while setting the variable + if RegHash=$(${t_out} $regbin -v error image digest --list "$RepoUrl" 2>&1) ; then + if [[ "$LocalHash" = *"$RegHash"* ]] ; then + echo NoUpdates "$i" else - # Here the RegHash is the result of an error code - GotErrors+=("$i - ${RegHash}") - fi - done -else - check_image() { - i="$1" - local Excludes=($Excludes_string) - local skip - for e in "${Excludes[@]}" ; do [[ "$i" == "$e" ]] && skip=1 ; done - - if [[ $skip -eq 1 ]]; then - echo Skip $i - return - fi - - local NoUpdates GotUpdates GotErrors - ImageId=$(docker inspect "$i" --format='{{.Image}}') - RepoUrl=$(docker inspect "$i" --format='{{.Config.Image}}') - LocalHash=$(docker image inspect "$ImageId" --format '{{.RepoDigests}}') - - # Checking for errors while setting the variable - if RegHash=$(${t_out} $regbin -v error image digest --list "$RepoUrl" 2>&1) ; then - if [[ "$LocalHash" = *"$RegHash"* ]] ; then - echo NoUpdates "$i" + if [[ -n "$DaysOld" ]] && ! datecheck ; then + echo NoUpdates "+$i ${ImageAge}d" else - if [[ -n "$DaysOld" ]] && ! datecheck ; then - echo NoUpdates "+$i ${ImageAge}d" - else - echo GotUpdates "$i" - fi + echo GotUpdates "$i" fi - else - # Here the RegHash is the result of an error code - echo GotErrors "$i - ${RegHash}" fi - } + else + # Here the RegHash is the result of an error code + echo GotErrors "$i - ${RegHash}" + fi +} - export -f check_image datecheck - export Excludes_string="${Excludes[@]}" # Can only export scalar variables - export t_out regbin RepoUrl DaysOld +# Make required functions and variables available to subprocesses +export -f check_image datecheck +export Excludes_string="${Excludes[@]}" # Can only export scalar variables +export t_out regbin RepoUrl DaysOld - # Asynchronously check the image-hash of every running container VS the registry - while read -r line; do - ((RegCheckQue+=1)) - progress_bar "$RegCheckQue" "$ContCount" +# Asynchronously check the image-hash of every running container VS the registry +while read -r line; do + ((RegCheckQue+=1)) + progress_bar "$RegCheckQue" "$ContCount" - Got=${line%% *} # Extracts the first word (NoUpdates, GotUpdates, GotErrors) - item=${line#* } + Got=${line%% *} # Extracts the first word (NoUpdates, GotUpdates, GotErrors) + item=${line#* } - case "$Got" in - NoUpdates) NoUpdates+=("$item") ;; - GotUpdates) GotUpdates+=("$item") ;; - GotErrors) GotErrors+=("$item") ;; - Skip) ;; - *) echo "Error! Unexpected output from subprocess: ${line}" ;; - esac - done < <( \ - docker ps $Stopped --filter "name=$SearchName" --format '{{.Names}}' | \ - xargs -P 32 -I {} bash -c 'check_image "{}"' \ - ) -fi + case "$Got" in + NoUpdates) NoUpdates+=("$item") ;; + GotUpdates) GotUpdates+=("$item") ;; + GotErrors) GotErrors+=("$item") ;; + Skip) ;; + *) echo "Error! Unexpected output from subprocess: ${line}" ;; + esac +done < <( \ + docker ps $Stopped --filter "name=$SearchName" --format '{{.Names}}' | \ + xargs -P 32 -I {} bash -c 'check_image "{}"' \ +) # Sort arrays alphabetically IFS=$'\n' From 3e079e2ec51641d9d421e62ed77dc4944ca81f1c Mon Sep 17 00:00:00 2001 From: mag37 Date: Mon, 24 Feb 2025 21:49:19 +0100 Subject: [PATCH 6/6] Update dockcheck.sh Added MaxAsync variable. Added POSIX xargs check. Rewrote Excludes. --- dockcheck.sh | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/dockcheck.sh b/dockcheck.sh index 462e7e6..087138d 100755 --- a/dockcheck.sh +++ b/dockcheck.sh @@ -46,6 +46,7 @@ c_blue="\033[0;34m" c_teal="\033[0;36m" c_reset="\033[0m" +MaxAsync=32 Timeout=10 Stopped="" while getopts "aynpfrhlisvmc:e:d:t:" options; do @@ -285,13 +286,12 @@ fi check_image() { i="$1" local Excludes=($Excludes_string) - local skip - for e in "${Excludes[@]}" ; do [[ "$i" == "$e" ]] && skip=1 ; done - - if [[ $skip -eq 1 ]]; then - echo Skip $i - return - fi + for e in "${Excludes[@]}" ; do + if [[ "$i" == "$e" ]]; then + echo Skip $i + return + fi + done local NoUpdates GotUpdates GotErrors ImageId=$(docker inspect "$i" --format='{{.Image}}') @@ -320,6 +320,14 @@ export -f check_image datecheck export Excludes_string="${Excludes[@]}" # Can only export scalar variables export t_out regbin RepoUrl DaysOld +# Check for POSIX xargs with -P option, fallback without async +if (echo "test" | xargs -P 10 >/dev/null 2>&1) ; then + XargsAsync="-P $MaxAsync" +else + XargsAsync="" + printf "%bMissing POSIX xargs, consider installing 'findutils' for asynchronous lookups.%b\n" "$c_red" "$c_reset" +fi + # Asynchronously check the image-hash of every running container VS the registry while read -r line; do ((RegCheckQue+=1)) @@ -337,7 +345,7 @@ while read -r line; do esac done < <( \ docker ps $Stopped --filter "name=$SearchName" --format '{{.Names}}' | \ - xargs -P 32 -I {} bash -c 'check_image "{}"' \ + xargs ${XargsAsync} -I {} bash -c 'check_image "{}"' \ ) # Sort arrays alphabetically