Merge master and fix conflicts

pkg/container/client.go

@@ -3,11 +3,12 @@ package container
 import (
 	"bytes"
 	"fmt"
-	"github.com/containrrr/watchtower/pkg/registry"
 	"io/ioutil"
 	"strings"
 	"time"
 
+	"github.com/containrrr/watchtower/pkg/registry"
+
 	t "github.com/containrrr/watchtower/pkg/types"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/container"
@@ -39,7 +40,7 @@ type Client interface {
 //  * DOCKER_HOST			the docker-engine host to send api requests to
 //  * DOCKER_TLS_VERIFY		whether to verify tls certificates
 //  * DOCKER_API_VERSION	the minimum docker api version to work with
-func NewClient(pullImages bool, includeStopped bool, reviveStopped bool, removeVolumes bool) Client {
+func NewClient(pullImages bool, includeStopped bool, reviveStopped bool, removeVolumes bool, includeRestarting bool) Client {
 	cli, err := sdkClient.NewClientWithOpts(sdkClient.FromEnv)
 
 	if err != nil {
@@ -47,28 +48,34 @@ func NewClient(pullImages bool, includeStopped bool, reviveStopped bool, removeV
 	}
 
 	return dockerClient{
-		api:            cli,
-		pullImages:     pullImages,
-		removeVolumes:  removeVolumes,
-		includeStopped: includeStopped,
-		reviveStopped:  reviveStopped,
+		api:               cli,
+		pullImages:        pullImages,
+		removeVolumes:     removeVolumes,
+		includeStopped:    includeStopped,
+		reviveStopped:     reviveStopped,
+		includeRestarting: includeRestarting,
 	}
 }
 
 type dockerClient struct {
-	api            sdkClient.CommonAPIClient
-	pullImages     bool
-	removeVolumes  bool
-	includeStopped bool
-	reviveStopped  bool
+	api               sdkClient.CommonAPIClient
+	pullImages        bool
+	removeVolumes     bool
+	includeStopped    bool
+	reviveStopped     bool
+	includeRestarting bool
 }
 
 func (client dockerClient) ListContainers(fn t.Filter) ([]Container, error) {
 	cs := []Container{}
 	bg := context.Background()
 
-	if client.includeStopped {
-		log.Debug("Retrieving containers including stopped and exited")
+	if client.includeStopped && client.includeRestarting {
+		log.Debug("Retrieving running, stopped, restarting and exited containers")
+	} else if client.includeStopped {
+		log.Debug("Retrieving running, stopped and exited containers")
+	} else if client.includeRestarting {
+		log.Debug("Retrieving running and restarting containers")
 	} else {
 		log.Debug("Retrieving running containers")
 	}
@@ -108,6 +115,10 @@ func (client dockerClient) createListFilter() filters.Args {
 		filterArgs.Add("status", "exited")
 	}
 
+	if client.includeRestarting {
+		filterArgs.Add("status", "restarting")
+	}
+
 	return filterArgs
 }
 
@@ -121,11 +132,11 @@ func (client dockerClient) GetContainer(containerID string) (Container, error) {
 
 	imageInfo, _, err := client.api.ImageInspectWithRaw(bg, containerInfo.Image)
 	if err != nil {
-		return Container{}, err
+		log.Warnf("Failed to retrieve container image info: %v", err)
+		return Container{containerInfo: &containerInfo, imageInfo: nil}, nil
 	}
 
-	container := Container{containerInfo: &containerInfo, imageInfo: &imageInfo}
-	return container, nil
+	return Container{containerInfo: &containerInfo, imageInfo: &imageInfo}, nil
 }
 
 func (client dockerClient) StopContainer(c Container, timeout time.Duration) error {

pkg/container/container.go

@@ -90,6 +90,33 @@ func (c Container) Enabled() (bool, bool) {
 	return parsedBool, true
 }
 
+// IsMonitorOnly returns the value of the monitor-only label. If the label
+// is not set then false is returned.
+func (c Container) IsMonitorOnly() bool {
+	rawBool, ok := c.getLabelValue(monitorOnlyLabel)
+	if !ok {
+		return false
+	}
+
+	parsedBool, err := strconv.ParseBool(rawBool)
+	if err != nil {
+		return false
+	}
+
+	return parsedBool
+}
+
+// Scope returns the value of the scope UID label and if the label
+// was set.
+func (c Container) Scope() (string, bool) {
+	rawString, ok := c.getLabelValue(scope)
+	if !ok {
+		return "", false
+	}
+
+	return rawString, true
+}
+
 // Links returns a list containing the names of all the containers to which
 // this container is linked.
 func (c Container) Links() []string {
@@ -221,3 +248,8 @@ func (c Container) hostConfig() *dockercontainer.HostConfig {
 
 	return hostConfig
 }
+
+// HasImageInfo returns whether image information could be retrieved for the container
+func (c Container) HasImageInfo() bool {
+	return c.imageInfo != nil
+}

pkg/container/container_test.go

@@ -1,6 +1,8 @@
 package container
 
 import (
+	"testing"
+
 	"github.com/containrrr/watchtower/pkg/container/mocks"
 	"github.com/containrrr/watchtower/pkg/filters"
 	"github.com/docker/docker/api/types"
@@ -8,7 +10,6 @@ import (
 	cli "github.com/docker/docker/client"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
-	"testing"
 )
 
 func TestContainer(t *testing.T) {
@@ -68,6 +69,44 @@ var _ = Describe("the container", func() {
 				Expect(len(containers) > 0).To(BeTrue())
 			})
 		})
+		When(`listing containers with the "include restart" option`, func() {
+			It("should return both stopped, restarting and running containers", func() {
+				client = dockerClient{
+					api:               docker,
+					pullImages:        false,
+					includeRestarting: true,
+				}
+				containers, err := client.ListContainers(filters.NoFilter)
+				Expect(err).NotTo(HaveOccurred())
+				RestartingContainerFound := false
+				for _, ContainerRunning := range containers {
+					if ContainerRunning.containerInfo.State.Restarting {
+						RestartingContainerFound = true
+					}
+				}
+				Expect(RestartingContainerFound).To(BeTrue())
+				Expect(RestartingContainerFound).NotTo(BeFalse())
+			})
+		})
+		When(`listing containers without restarting ones`, func() {
+			It("should not return restarting containers", func() {
+				client = dockerClient{
+					api:               docker,
+					pullImages:        false,
+					includeRestarting: false,
+				}
+				containers, err := client.ListContainers(filters.NoFilter)
+				Expect(err).NotTo(HaveOccurred())
+				RestartingContainerFound := false
+				for _, ContainerRunning := range containers {
+					if ContainerRunning.containerInfo.State.Restarting {
+						RestartingContainerFound = true
+					}
+				}
+				Expect(RestartingContainerFound).To(BeFalse())
+				Expect(RestartingContainerFound).NotTo(BeTrue())
+			})
+		})
 	})
 	When("asked for metadata", func() {
 		var c *Container

pkg/container/metadata.go

@@ -4,8 +4,10 @@ const (
 	watchtowerLabel       = "com.centurylinklabs.watchtower"
 	signalLabel           = "com.centurylinklabs.watchtower.stop-signal"
 	enableLabel           = "com.centurylinklabs.watchtower.enable"
+	monitorOnlyLabel      = "com.centurylinklabs.watchtower.monitor-only"
 	dependsOnLabel        = "com.centurylinklabs.watchtower.depends-on"
 	zodiacLabel           = "com.centurylinklabs.zodiac.original-image"
+	scope                 = "com.centurylinklabs.watchtower.scope"
 	preCheckLabel         = "com.centurylinklabs.watchtower.lifecycle.pre-check"
 	postCheckLabel        = "com.centurylinklabs.watchtower.lifecycle.post-check"
 	preUpdateLabel        = "com.centurylinklabs.watchtower.lifecycle.pre-update"

pkg/container/mocks/ApiServer.go

@@ -1,13 +1,16 @@
 package mocks
 
 import (
+	"encoding/json"
 	"fmt"
-	"github.com/sirupsen/logrus"
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
 	"path/filepath"
 	"strings"
+
+	"github.com/docker/docker/api/types"
+	"github.com/sirupsen/logrus"
 )
 
 // NewMockAPIServer returns a mocked docker api server that responds to some fixed requests
@@ -18,16 +21,36 @@ func NewMockAPIServer() *httptest.Server {
 			logrus.Debug("Mock server has received a HTTP call on ", r.URL)
 			var response = ""
 
-			if isRequestFor("filters=%7B%22status%22%3A%7B%22running%22%3Atrue%7D%7D&limit=0", r) {
-				response = getMockJSONFromDisk("./mocks/data/containers.json")
-			} else if isRequestFor("filters=%7B%22status%22%3A%7B%22created%22%3Atrue%2C%22exited%22%3Atrue%2C%22running%22%3Atrue%7D%7D&limit=0", r) {
+			if isRequestFor("filters=", r) {
+
+				Filters := r.URL.Query().Get("filters")
+				var result map[string]interface{}
+				json.Unmarshal([]byte(Filters), &result)
+				status := result["status"].(map[string]interface{})
+
 				response = getMockJSONFromDisk("./mocks/data/containers.json")
+				var x2 []types.Container
+				var containers []types.Container
+				json.Unmarshal([]byte(response), &containers)
+				for _, v := range containers {
+					for key := range status {
+						if v.State == key {
+							x2 = append(x2, v)
+						}
+					}
+				}
+
+				b, _ := json.Marshal(x2)
+				response = string(b)
+
 			} else if isRequestFor("containers/json?limit=0", r) {
 				response = getMockJSONFromDisk("./mocks/data/containers.json")
 			} else if isRequestFor("ae8964ba86c7cd7522cf84e09781343d88e0e3543281c747d88b27e246578b65", r) {
 				response = getMockJSONFromDisk("./mocks/data/container_stopped.json")
 			} else if isRequestFor("b978af0b858aa8855cce46b628817d4ed58e58f2c4f66c9b9c5449134ed4c008", r) {
 				response = getMockJSONFromDisk("./mocks/data/container_running.json")
+			} else if isRequestFor("ae8964ba86c7cd7522cf84e09781343d88e0e3543281c747d88b27e246578b67", r) {
+				response = getMockJSONFromDisk("./mocks/data/container_restarting.json")
 			} else if isRequestFor("sha256:19d07168491a3f9e2798a9bed96544e34d57ddc4757a4ac5bb199dea896c87fd", r) {
 				response = getMockJSONFromDisk("./mocks/data/image01.json")
 			} else if isRequestFor("sha256:4dbc5f9c07028a985e14d1393e849ea07f68804c4293050d5a641b138db72daa", r) {

pkg/container/mocks/FilterableContainer.go

@@ -55,3 +55,26 @@ func (_m *FilterableContainer) Name() string {
 
 	return r0
 }
+
+// Scope provides a mock function with given fields:
+func (_m *FilterableContainer) Scope() (string, bool) {
+	ret := _m.Called()
+
+	var r0 string
+
+	if rf, ok := ret.Get(0).(func() string); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Get(0).(string)
+	}
+
+	var r1 bool
+
+	if rf, ok := ret.Get(1).(func() bool); ok {
+		r1 = rf()
+	} else {
+		r1 = ret.Get(1).(bool)
+	}
+
+	return r0, r1
+}

pkg/container/mocks/data/container_restarting.json

@@ -0,0 +1,205 @@
+{
+  "Id": "ae8964ba86c7cd7522cf84e09781343d88e0e3543281c747d88b27e246578b67",
+  "Created": "2019-04-10T19:51:22.245041005Z",
+  "Path": "/watchtower",
+  "Args": [],
+  "State": {
+    "Status": "exited",
+    "Running": false,
+    "Paused": false,
+    "Restarting": true,
+    "OOMKilled": false,
+    "Dead": false,
+    "Pid": 0,
+    "ExitCode": 1,
+    "Error": "",
+    "StartedAt": "2019-04-10T19:51:22.918972606Z",
+    "FinishedAt": "2019-04-10T19:52:14.265091583Z"
+  },
+  "Image": "sha256:4dbc5f9c07028a985e14d1393e849ea07f68804c4293050d5a641b138db72daa",
+  "ResolvConfPath": "/var/lib/docker/containers/ae8964ba86c7cd7522cf84e09781343d88e0e3543281c747d88b27e246578b65/resolv.conf",
+  "HostnamePath": "/var/lib/docker/containers/ae8964ba86c7cd7522cf84e09781343d88e0e3543281c747d88b27e246578b65/hostname",
+  "HostsPath": "/var/lib/docker/containers/ae8964ba86c7cd7522cf84e09781343d88e0e3543281c747d88b27e246578b65/hosts",
+  "LogPath": "/var/lib/docker/containers/ae8964ba86c7cd7522cf84e09781343d88e0e3543281c747d88b27e246578b65/ae8964ba86c7cd7522cf84e09781343d88e0e3543281c747d88b27e246578b65-json.log",
+  "Name": "/watchtower-test",
+  "RestartCount": 0,
+  "Driver": "overlay2",
+  "Platform": "linux",
+  "MountLabel": "",
+  "ProcessLabel": "",
+  "AppArmorProfile": "",
+  "ExecIDs": null,
+  "HostConfig": {
+    "Binds": [
+      "/var/run/docker.sock:/var/run/docker.sock"
+    ],
+    "ContainerIDFile": "",
+    "LogConfig": {
+      "Type": "json-file",
+      "Config": {}
+    },
+    "NetworkMode": "default",
+    "PortBindings": {},
+    "RestartPolicy": {
+      "Name": "no",
+      "MaximumRetryCount": 0
+    },
+    "AutoRemove": false,
+    "VolumeDriver": "",
+    "VolumesFrom": null,
+    "CapAdd": null,
+    "CapDrop": null,
+    "Dns": [],
+    "DnsOptions": [],
+    "DnsSearch": [],
+    "ExtraHosts": null,
+    "GroupAdd": null,
+    "IpcMode": "shareable",
+    "Cgroup": "",
+    "Links": null,
+    "OomScoreAdj": 0,
+    "PidMode": "",
+    "Privileged": false,
+    "PublishAllPorts": false,
+    "ReadonlyRootfs": false,
+    "SecurityOpt": null,
+    "UTSMode": "",
+    "UsernsMode": "",
+    "ShmSize": 67108864,
+    "Runtime": "runc",
+    "ConsoleSize": [
+      0,
+      0
+    ],
+    "Isolation": "",
+    "CpuShares": 0,
+    "Memory": 0,
+    "NanoCpus": 0,
+    "CgroupParent": "",
+    "BlkioWeight": 0,
+    "BlkioWeightDevice": [],
+    "BlkioDeviceReadBps": null,
+    "BlkioDeviceWriteBps": null,
+    "BlkioDeviceReadIOps": null,
+    "BlkioDeviceWriteIOps": null,
+    "CpuPeriod": 0,
+    "CpuQuota": 0,
+    "CpuRealtimePeriod": 0,
+    "CpuRealtimeRuntime": 0,
+    "CpusetCpus": "",
+    "CpusetMems": "",
+    "Devices": [],
+    "DeviceCgroupRules": null,
+    "DiskQuota": 0,
+    "KernelMemory": 0,
+    "MemoryReservation": 0,
+    "MemorySwap": 0,
+    "MemorySwappiness": null,
+    "OomKillDisable": false,
+    "PidsLimit": 0,
+    "Ulimits": null,
+    "CpuCount": 0,
+    "CpuPercent": 0,
+    "IOMaximumIOps": 0,
+    "IOMaximumBandwidth": 0,
+    "MaskedPaths": [
+      "/proc/asound",
+      "/proc/acpi",
+      "/proc/kcore",
+      "/proc/keys",
+      "/proc/latency_stats",
+      "/proc/timer_list",
+      "/proc/timer_stats",
+      "/proc/sched_debug",
+      "/proc/scsi",
+      "/sys/firmware"
+    ],
+    "ReadonlyPaths": [
+      "/proc/bus",
+      "/proc/fs",
+      "/proc/irq",
+      "/proc/sys",
+      "/proc/sysrq-trigger"
+    ]
+  },
+  "GraphDriver": {
+    "Data": {
+      "LowerDir": "/var/lib/docker/overlay2/9f6b91ea6e142835035d91123bbc7a05224dfa2abd4d020eac42f2ab420ccddc-init/diff:/var/lib/docker/overlay2/cdf82f50bc49177d0c17c24f3eaa29eba607b70cc6a081f77781b21c59a13eb8/diff:/var/lib/docker/overlay2/8108325ee844603c9b08d2772cf6e65dccf31dd5171f265078e5ed79a0ba3c0f/diff:/var/lib/docker/overlay2/e5e0cce6bf91b829a308424d99d7e56a33be3a11414ff5cdc48e762a1342b20f/diff",
+      "MergedDir": "/var/lib/docker/overlay2/9f6b91ea6e142835035d91123bbc7a05224dfa2abd4d020eac42f2ab420ccddc/merged",
+      "UpperDir": "/var/lib/docker/overlay2/9f6b91ea6e142835035d91123bbc7a05224dfa2abd4d020eac42f2ab420ccddc/diff",
+      "WorkDir": "/var/lib/docker/overlay2/9f6b91ea6e142835035d91123bbc7a05224dfa2abd4d020eac42f2ab420ccddc/work"
+    },
+    "Name": "overlay2"
+  },
+  "Mounts": [
+    {
+      "Type": "bind",
+      "Source": "/var/run/docker.sock",
+      "Destination": "/var/run/docker.sock",
+      "Mode": "",
+      "RW": true,
+      "Propagation": "rprivate"
+    }
+  ],
+  "Config": {
+    "Hostname": "ae8964ba86c7",
+    "Domainname": "",
+    "User": "",
+    "AttachStdin": false,
+    "AttachStdout": true,
+    "AttachStderr": true,
+    "Tty": false,
+    "OpenStdin": false,
+    "StdinOnce": false,
+    "Env": [
+      "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+    ],
+    "Cmd": null,
+    "Image": "containrrr/watchtower:latest",
+    "Volumes": null,
+    "WorkingDir": "",
+    "Entrypoint": [
+      "/watchtower"
+    ],
+    "OnBuild": null,
+    "Labels": {
+      "com.centurylinklabs.watchtower": "true"
+    }
+  },
+  "NetworkSettings": {
+    "Bridge": "",
+    "SandboxID": "05627d36c08ed994eebc44a2a8c9365a511756b55c500fb03fd5a14477cd4bf3",
+    "HairpinMode": false,
+    "LinkLocalIPv6Address": "",
+    "LinkLocalIPv6PrefixLen": 0,
+    "Ports": {},
+    "SandboxKey": "/var/run/docker/netns/05627d36c08e",
+    "SecondaryIPAddresses": null,
+    "SecondaryIPv6Addresses": null,
+    "EndpointID": "",
+    "Gateway": "",
+    "GlobalIPv6Address": "",
+    "GlobalIPv6PrefixLen": 0,
+    "IPAddress": "",
+    "IPPrefixLen": 0,
+    "IPv6Gateway": "",
+    "MacAddress": "",
+    "Networks": {
+      "bridge": {
+        "IPAMConfig": null,
+        "Links": null,
+        "Aliases": null,
+        "NetworkID": "8fcfd56fa9203bafa98510abb08bff66ad05bef5b6e97d158cbae3397e1e065e",
+        "EndpointID": "",
+        "Gateway": "",
+        "IPAddress": "",
+        "IPPrefixLen": 0,
+        "IPv6Gateway": "",
+        "GlobalIPv6Address": "",
+        "GlobalIPv6PrefixLen": 0,
+        "MacAddress": "",
+        "DriverOpts": null
+      }
+    }
+  }
+}

pkg/container/mocks/data/containers.json

@@ -109,5 +109,68 @@
         "Propagation": "rprivate"
       }
     ]
+  },
+  {
+    "Id": "ae8964ba86c7cd7522cf84e09781343d88e0e3543281c747d88b27e246578b67",
+    "Names": [
+      "/portainer"
+    ],
+    "Image": "portainer/portainer:latest",
+    "ImageID": "sha256:19d07168491a3f9e2798a9bed96544e34d57ddc4757a4ac5bb199dea896c87fd",
+    "Command": "/portainer",
+    "Created": 1554409712,
+    "Ports": [
+      {
+        "IP": "0.0.0.0",
+        "PrivatePort": 9000,
+        "PublicPort": 9000,
+        "Type": "tcp"
+      }
+    ],
+    "Labels": {},
+    "State": "restarting",
+    "Status": "Restarting (0) 35 seconds ago",
+    "HostConfig": {
+      "NetworkMode": "default"
+    },
+    "NetworkSettings": {
+      "Networks": {
+        "bridge": {
+          "IPAMConfig": null,
+          "Links": null,
+          "Aliases": null,
+          "NetworkID": "9352796e0330dcf31ce3d44fae4b719304b8b3fd97b02ade3aefb8737251682b",
+          "EndpointID": "a8bcd737f27edb4d2955f7bce0c777bb2990b792a6b335b0727387624abe0702",
+          "Gateway": "172.17.0.1",
+          "IPAddress": "172.17.0.2",
+          "IPPrefixLen": 16,
+          "IPv6Gateway": "",
+          "GlobalIPv6Address": "",
+          "GlobalIPv6PrefixLen": 0,
+          "MacAddress": "02:42:ac:11:00:02",
+          "DriverOpts": null
+        }
+      }
+    },
+    "Mounts": [
+      {
+        "Type": "volume",
+        "Name": "portainer_data",
+        "Source": "/var/lib/docker/volumes/portainer_data/_data",
+        "Destination": "/data",
+        "Driver": "local",
+        "Mode": "z",
+        "RW": true,
+        "Propagation": ""
+      },
+      {
+        "Type": "bind",
+        "Source": "/var/run/docker.sock",
+        "Destination": "/var/run/docker.sock",
+        "Mode": "",
+        "RW": true,
+        "Propagation": "rprivate"
+      }
+    ]
   }
 ]

pkg/filters/filters.go

@@ -51,8 +51,24 @@ func FilterByDisabledLabel(baseFilter t.Filter) t.Filter {
 	}
 }
 
+// FilterByScope returns all containers that belongs to a specific scope
+func FilterByScope(scope string, baseFilter t.Filter) t.Filter {
+	if scope == "" {
+		return baseFilter
+	}
+
+	return func(c t.FilterableContainer) bool {
+		containerScope, ok := c.Scope()
+		if ok && containerScope == scope {
+			return baseFilter(c)
+		}
+
+		return false
+	}
+}
+
 // BuildFilter creates the needed filter of containers
-func BuildFilter(names []string, enableLabel bool) t.Filter {
+func BuildFilter(names []string, enableLabel bool, scope string) t.Filter {
 	filter := NoFilter
 	filter = FilterByNames(names, filter)
 	if enableLabel {
@@ -60,6 +76,11 @@ func BuildFilter(names []string, enableLabel bool) t.Filter {
 		// if the label is specifically set.
 		filter = FilterByEnableLabel(filter)
 	}
+	if scope != "" {
+		// If a scope has been defined, containers should only be considered
+		// if the scope is specifically set.
+		filter = FilterByScope(scope, filter)
+	}
 	filter = FilterByDisabledLabel(filter)
 	return filter
 }

pkg/filters/filters_test.go

@@ -67,6 +67,29 @@ func TestFilterByEnableLabel(t *testing.T) {
 	container.AssertExpectations(t)
 }
 
+func TestFilterByScope(t *testing.T) {
+	var scope string
+	scope = "testscope"
+
+	filter := FilterByScope(scope, NoFilter)
+	assert.NotNil(t, filter)
+
+	container := new(mocks.FilterableContainer)
+	container.On("Scope").Return("testscope", true)
+	assert.True(t, filter(container))
+	container.AssertExpectations(t)
+
+	container = new(mocks.FilterableContainer)
+	container.On("Scope").Return("nottestscope", true)
+	assert.False(t, filter(container))
+	container.AssertExpectations(t)
+
+	container = new(mocks.FilterableContainer)
+	container.On("Scope").Return("", false)
+	assert.False(t, filter(container))
+	container.AssertExpectations(t)
+}
+
 func TestFilterByDisabledLabel(t *testing.T) {
 	filter := FilterByDisabledLabel(NoFilter)
 	assert.NotNil(t, filter)
@@ -91,7 +114,7 @@ func TestBuildFilter(t *testing.T) {
 	var names []string
 	names = append(names, "test")
 
-	filter := BuildFilter(names, false)
+	filter := BuildFilter(names, false, "")
 
 	container := new(mocks.FilterableContainer)
 	container.On("Name").Return("Invalid")
@@ -127,7 +150,7 @@ func TestBuildFilterEnableLabel(t *testing.T) {
 	var names []string
 	names = append(names, "test")
 
-	filter := BuildFilter(names, true)
+	filter := BuildFilter(names, true, "")
 
 	container := new(mocks.FilterableContainer)
 	container.On("Enabled").Return(false, false)

pkg/lifecycle/lifecycle.go

@@ -36,7 +36,7 @@ func ExecutePreCheckCommand(client container.Client, container container.Contain
 		return
 	}
 
-	log.Info("Executing pre-check command.")
+	log.Debug("Executing pre-check command.")
 	if err := client.ExecuteCommand(container.ID(), command, 1); err != nil {
 		log.Error(err)
 	}
@@ -50,7 +50,7 @@ func ExecutePostCheckCommand(client container.Client, container container.Contai
 		return
 	}
 
-	log.Info("Executing post-check command.")
+	log.Debug("Executing post-check command.")
 	if err := client.ExecuteCommand(container.ID(), command, 1); err != nil {
 		log.Error(err)
 	}
@@ -65,7 +65,7 @@ func ExecutePreUpdateCommand(client container.Client, container container.Contai
 		return nil
 	}
 
-	log.Info("Executing pre-update command.")
+	log.Debug("Executing pre-update command.")
 	return client.ExecuteCommand(container.ID(), command, timeout)
 }
 
@@ -83,7 +83,7 @@ func ExecutePostUpdateCommand(client container.Client, newContainerID string) {
 		return
 	}
 
-	log.Info("Executing post-update command.")
+	log.Debug("Executing post-update command.")
 	if err := client.ExecuteCommand(newContainerID, command, 1); err != nil {
 		log.Error(err)
 	}

pkg/notifications/email.go

@@ -153,3 +153,5 @@ func (e *emailTypeNotifier) Fire(entry *log.Entry) error {
 	}
 	return nil
 }
+
+func (e *emailTypeNotifier) Close() {}

pkg/notifications/gotify.go

@@ -59,6 +59,8 @@ func (n *gotifyTypeNotifier) StartNotification() {}
 
 func (n *gotifyTypeNotifier) SendNotification() {}
 
+func (n *gotifyTypeNotifier) Close() {}
+
 func (n *gotifyTypeNotifier) Levels() []log.Level {
 	return n.logLevels
 }

pkg/notifications/msteams.go

@@ -47,6 +47,8 @@ func (n *msTeamsTypeNotifier) StartNotification() {}
 
 func (n *msTeamsTypeNotifier) SendNotification() {}
 
+func (n *msTeamsTypeNotifier) Close() {}
+
 func (n *msTeamsTypeNotifier) Levels() []log.Level {
 	return n.levels
 }

pkg/notifications/notifier.go

@@ -66,3 +66,10 @@ func (n *Notifier) SendNotification() {
 		t.SendNotification()
 	}
 }
+
+// Close closes all notifiers.
+func (n *Notifier) Close() {
+	for _, t := range n.types {
+		t.Close()
+	}
+}

pkg/notifications/shoutrrr.go

@@ -3,11 +3,11 @@ package notifications
 import (
 	"bytes"
 	"fmt"
+	"github.com/containrrr/shoutrrr/pkg/types"
+	"strings"
 	"text/template"
-    "strings"
 
 	"github.com/containrrr/shoutrrr"
-	"github.com/containrrr/shoutrrr/pkg/router"
 	t "github.com/containrrr/watchtower/pkg/types"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
@@ -18,13 +18,19 @@ const (
 	shoutrrrType            = "shoutrrr"
 )
 
+type router interface {
+	Send(message string, params *types.Params) []error
+}
+
 // Implements Notifier, logrus.Hook
 type shoutrrrTypeNotifier struct {
 	Urls      []string
-	Router    *router.ServiceRouter
+	Router    router
 	entries   []*log.Entry
 	logLevels []log.Level
 	template  *template.Template
+	messages  chan string
+	done      chan bool
 }
 
 func newShoutrrrNotifier(c *cobra.Command, acceptedLogLevels []log.Level) t.Notifier {
@@ -41,13 +47,33 @@ func newShoutrrrNotifier(c *cobra.Command, acceptedLogLevels []log.Level) t.Noti
 		Router:    r,
 		logLevels: acceptedLogLevels,
 		template:  getShoutrrrTemplate(c),
+		messages:  make(chan string, 1),
+		done:      make(chan bool),
 	}
 
 	log.AddHook(n)
 
+	// Do the sending in a separate goroutine so we don't block the main process.
+	go sendNotifications(n)
+
 	return n
 }
 
+func sendNotifications(n *shoutrrrTypeNotifier) {
+	for msg := range n.messages {
+		errs := n.Router.Send(msg, nil)
+
+		for i, err := range errs {
+			if err != nil {
+				// Use fmt so it doesn't trigger another notification.
+				fmt.Println("Failed to send notification via shoutrrr (url="+n.Urls[i]+"): ", err)
+			}
+		}
+	}
+
+	n.done <- true
+}
+
 func (e *shoutrrrTypeNotifier) buildMessage(entries []*log.Entry) string {
 	var body bytes.Buffer
 	if err := e.template.Execute(&body, entries); err != nil {
@@ -58,20 +84,8 @@ func (e *shoutrrrTypeNotifier) buildMessage(entries []*log.Entry) string {
 }
 
 func (e *shoutrrrTypeNotifier) sendEntries(entries []*log.Entry) {
-
 	msg := e.buildMessage(entries)
-
-	// Do the sending in a separate goroutine so we don't block the main process.
-	go func() {
-		errs := e.Router.Send(msg, nil)
-
-		for i, err := range errs {
-			if err != nil {
-				// Use fmt so it doesn't trigger another notification.
-				fmt.Println("Failed to send notification via shoutrrr (url="+e.Urls[i]+"): ", err)
-			}
-		}
-	}()
+	e.messages <- msg
 }
 
 func (e *shoutrrrTypeNotifier) StartNotification() {
@@ -89,6 +103,15 @@ func (e *shoutrrrTypeNotifier) SendNotification() {
 	e.entries = nil
 }
 
+func (e *shoutrrrTypeNotifier) Close() {
+	close(e.messages)
+
+	// Use fmt so it doesn't trigger another notification.
+	fmt.Println("Waiting for the notification goroutine to finish")
+
+	_ = <-e.done
+}
+
 func (e *shoutrrrTypeNotifier) Levels() []log.Level {
 	return e.logLevels
 }
@@ -113,7 +136,7 @@ func getShoutrrrTemplate(c *cobra.Command) *template.Template {
 	funcs := template.FuncMap{
 		"ToUpper": strings.ToUpper,
 		"ToLower": strings.ToLower,
-		"Title": strings.Title,
+		"Title":   strings.Title,
 	}
 
 	// If we succeed in getting a non-empty template configuration

pkg/notifications/shoutrrr_test.go

@@ -1,6 +1,7 @@
 package notifications
 
 import (
+	"github.com/containrrr/shoutrrr/pkg/types"
 	"testing"
 	"text/template"
 
@@ -74,7 +75,6 @@ func TestShoutrrrStringFunctions(t *testing.T) {
 	require.Equal(t, "INFO: foo bar Foo Bar\n", s)
 }
 
-
 func TestShoutrrrInvalidTemplateUsesTemplate(t *testing.T) {
 	cmd := new(cobra.Command)
 
@@ -102,3 +102,69 @@ func TestShoutrrrInvalidTemplateUsesTemplate(t *testing.T) {
 
 	require.Equal(t, sd, s)
 }
+
+type blockingRouter struct {
+	unlock chan bool
+	sent   chan bool
+}
+
+func (b blockingRouter) Send(message string, params *types.Params) []error {
+	_ = <-b.unlock
+	b.sent <- true
+	return nil
+}
+
+func TestSlowNotificationNotSent(t *testing.T) {
+	_, blockingRouter := sendNotificationsWithBlockingRouter()
+
+	notifSent := false
+	select {
+	case notifSent = <-blockingRouter.sent:
+	default:
+	}
+
+	require.Equal(t, false, notifSent)
+}
+
+func TestSlowNotificationSent(t *testing.T) {
+	shoutrrr, blockingRouter := sendNotificationsWithBlockingRouter()
+
+	blockingRouter.unlock <- true
+	shoutrrr.Close()
+
+	notifSent := false
+	select {
+	case notifSent = <-blockingRouter.sent:
+	default:
+	}
+	require.Equal(t, true, notifSent)
+}
+
+func sendNotificationsWithBlockingRouter() (*shoutrrrTypeNotifier, *blockingRouter) {
+	cmd := new(cobra.Command)
+
+	router := &blockingRouter{
+		unlock: make(chan bool, 1),
+		sent:   make(chan bool, 1),
+	}
+
+	shoutrrr := &shoutrrrTypeNotifier{
+		template: getShoutrrrTemplate(cmd),
+		messages: make(chan string, 1),
+		done:     make(chan bool),
+		Router:   router,
+	}
+
+	entry := &log.Entry{
+		Message: "foo bar",
+	}
+
+	go sendNotifications(shoutrrr)
+
+	shoutrrr.StartNotification()
+	shoutrrr.Fire(entry)
+
+	shoutrrr.SendNotification()
+
+	return shoutrrr, router
+}

pkg/notifications/slack.go

@@ -42,3 +42,5 @@ func newSlackNotifier(c *cobra.Command, acceptedLogLevels []log.Level) t.Notifie
 func (s *slackTypeNotifier) StartNotification() {}
 
 func (s *slackTypeNotifier) SendNotification() {}
+
+func (s *slackTypeNotifier) Close() {}

pkg/types/filterable_container.go

@@ -6,4 +6,5 @@ type FilterableContainer interface {
 	Name() string
 	IsWatchtower() bool
 	Enabled() (bool, bool)
+	Scope() (string, bool)
 }

pkg/types/notifier.go

@@ -4,4 +4,5 @@ package types
 type Notifier interface {
 	StartNotification()
 	SendNotification()
+	Close()
 }

pkg/types/update_params.go

@@ -12,4 +12,5 @@ type UpdateParams struct {
 	Timeout        time.Duration
 	MonitorOnly    bool
 	LifecycleHooks bool
+	RollingRestart bool
 }