Allow watchtower to update rebooting containers (#651)

Co-authored-by: nils måsén <nils@piksel.se>
Co-authored-by: Simon Aronsson <simme@arcticbit.se>

cmd/root.go

@@ -116,6 +116,7 @@ func PreRun(cmd *cobra.Command, args []string) {
 
 	noPull, _ := f.GetBool("no-pull")
 	includeStopped, _ := f.GetBool("include-stopped")
+	includeRestarting, _ := f.GetBool("include-restarting")
 	reviveStopped, _ := f.GetBool("revive-stopped")
 	removeVolumes, _ := f.GetBool("remove-volumes")
 
@@ -128,6 +129,7 @@ func PreRun(cmd *cobra.Command, args []string) {
 		includeStopped,
 		reviveStopped,
 		removeVolumes,
+		includeRestarting,
 	)
 
 	notifier = notifications.NewNotifier(cmd)

pkg/container/client.go

@@ -3,11 +3,12 @@ package container
 import (
 	"bytes"
 	"fmt"
-	"github.com/containrrr/watchtower/pkg/registry"
 	"io/ioutil"
 	"strings"
 	"time"
 
+	"github.com/containrrr/watchtower/pkg/registry"
+
 	t "github.com/containrrr/watchtower/pkg/types"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/container"
@@ -39,7 +40,7 @@ type Client interface {
 //  * DOCKER_HOST			the docker-engine host to send api requests to
 //  * DOCKER_TLS_VERIFY		whether to verify tls certificates
 //  * DOCKER_API_VERSION	the minimum docker api version to work with
-func NewClient(pullImages bool, includeStopped bool, reviveStopped bool, removeVolumes bool) Client {
+func NewClient(pullImages bool, includeStopped bool, reviveStopped bool, removeVolumes bool, includeRestarting bool) Client {
 	cli, err := sdkClient.NewClientWithOpts(sdkClient.FromEnv)
 
 	if err != nil {
@@ -47,28 +48,34 @@ func NewClient(pullImages bool, includeStopped bool, reviveStopped bool, removeV
 	}
 
 	return dockerClient{
-		api:            cli,
+		api:               cli,
-		pullImages:     pullImages,
+		pullImages:        pullImages,
-		removeVolumes:  removeVolumes,
+		removeVolumes:     removeVolumes,
-		includeStopped: includeStopped,
+		includeStopped:    includeStopped,
-		reviveStopped:  reviveStopped,
+		reviveStopped:     reviveStopped,
+		includeRestarting: includeRestarting,
 	}
 }
 
 type dockerClient struct {
-	api            sdkClient.CommonAPIClient
+	api               sdkClient.CommonAPIClient
-	pullImages     bool
+	pullImages        bool
-	removeVolumes  bool
+	removeVolumes     bool
-	includeStopped bool
+	includeStopped    bool
-	reviveStopped  bool
+	reviveStopped     bool
+	includeRestarting bool
 }
 
 func (client dockerClient) ListContainers(fn t.Filter) ([]Container, error) {
 	cs := []Container{}
 	bg := context.Background()
 
-	if client.includeStopped {
+	if client.includeStopped && client.includeRestarting {
-		log.Debug("Retrieving containers including stopped and exited")
+		log.Debug("Retrieving running, stopped, restarting and exited containers")
+	} else if client.includeStopped {
+		log.Debug("Retrieving running, stopped and exited containers")
+	} else if client.includeRestarting {
+		log.Debug("Retrieving running and restarting containers")
 	} else {
 		log.Debug("Retrieving running containers")
 	}
@@ -108,6 +115,10 @@ func (client dockerClient) createListFilter() filters.Args {
 		filterArgs.Add("status", "exited")
 	}
 
+	if client.includeRestarting {
+		filterArgs.Add("status", "restarting")
+	}
+
 	return filterArgs
 }
 

pkg/container/container_test.go

@@ -1,6 +1,8 @@
 package container
 
 import (
+	"testing"
+
 	"github.com/containrrr/watchtower/pkg/container/mocks"
 	"github.com/containrrr/watchtower/pkg/filters"
 	"github.com/docker/docker/api/types"
@@ -8,7 +10,6 @@ import (
 	cli "github.com/docker/docker/client"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
-	"testing"
 )
 
 func TestContainer(t *testing.T) {
@@ -68,6 +69,44 @@ var _ = Describe("the container", func() {
 				Expect(len(containers) > 0).To(BeTrue())
 			})
 		})
+		When(`listing containers with the "include restart" option`, func() {
+			It("should return both stopped, restarting and running containers", func() {
+				client = dockerClient{
+					api:               docker,
+					pullImages:        false,
+					includeRestarting: true,
+				}
+				containers, err := client.ListContainers(filters.NoFilter)
+				Expect(err).NotTo(HaveOccurred())
+				RestartingContainerFound := false
+				for _, ContainerRunning := range containers {
+					if ContainerRunning.containerInfo.State.Restarting {
+						RestartingContainerFound = true
+					}
+				}
+				Expect(RestartingContainerFound).To(BeTrue())
+				Expect(RestartingContainerFound).NotTo(BeFalse())
+			})
+		})
+		When(`listing containers without restarting ones`, func() {
+			It("should not return restarting containers", func() {
+				client = dockerClient{
+					api:               docker,
+					pullImages:        false,
+					includeRestarting: false,
+				}
+				containers, err := client.ListContainers(filters.NoFilter)
+				Expect(err).NotTo(HaveOccurred())
+				RestartingContainerFound := false
+				for _, ContainerRunning := range containers {
+					if ContainerRunning.containerInfo.State.Restarting {
+						RestartingContainerFound = true
+					}
+				}
+				Expect(RestartingContainerFound).To(BeFalse())
+				Expect(RestartingContainerFound).NotTo(BeTrue())
+			})
+		})
 	})
 	When("asked for metadata", func() {
 		var c *Container

pkg/container/mocks/ApiServer.go

@@ -1,13 +1,16 @@
 package mocks
 
 import (
+	"encoding/json"
 	"fmt"
-	"github.com/sirupsen/logrus"
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
 	"path/filepath"
 	"strings"
+
+	"github.com/docker/docker/api/types"
+	"github.com/sirupsen/logrus"
 )
 
 // NewMockAPIServer returns a mocked docker api server that responds to some fixed requests
@@ -18,16 +21,36 @@ func NewMockAPIServer() *httptest.Server {
 			logrus.Debug("Mock server has received a HTTP call on ", r.URL)
 			var response = ""
 
-			if isRequestFor("filters=%7B%22status%22%3A%7B%22running%22%3Atrue%7D%7D&limit=0", r) {
+			if isRequestFor("filters=", r) {
-				response = getMockJSONFromDisk("./mocks/data/containers.json")
+
-			} else if isRequestFor("filters=%7B%22status%22%3A%7B%22created%22%3Atrue%2C%22exited%22%3Atrue%2C%22running%22%3Atrue%7D%7D&limit=0", r) {
+				Filters := r.URL.Query().Get("filters")
+				var result map[string]interface{}
+				json.Unmarshal([]byte(Filters), &result)
+				status := result["status"].(map[string]interface{})
+
 				response = getMockJSONFromDisk("./mocks/data/containers.json")
+				var x2 []types.Container
+				var containers []types.Container
+				json.Unmarshal([]byte(response), &containers)
+				for _, v := range containers {
+					for key := range status {
+						if v.State == key {
+							x2 = append(x2, v)
+						}
+					}
+				}
+
+				b, _ := json.Marshal(x2)
+				response = string(b)
+
 			} else if isRequestFor("containers/json?limit=0", r) {
 				response = getMockJSONFromDisk("./mocks/data/containers.json")
 			} else if isRequestFor("ae8964ba86c7cd7522cf84e09781343d88e0e3543281c747d88b27e246578b65", r) {
 				response = getMockJSONFromDisk("./mocks/data/container_stopped.json")
 			} else if isRequestFor("b978af0b858aa8855cce46b628817d4ed58e58f2c4f66c9b9c5449134ed4c008", r) {
 				response = getMockJSONFromDisk("./mocks/data/container_running.json")
+			} else if isRequestFor("ae8964ba86c7cd7522cf84e09781343d88e0e3543281c747d88b27e246578b67", r) {
+				response = getMockJSONFromDisk("./mocks/data/container_restarting.json")
 			} else if isRequestFor("sha256:19d07168491a3f9e2798a9bed96544e34d57ddc4757a4ac5bb199dea896c87fd", r) {
 				response = getMockJSONFromDisk("./mocks/data/image01.json")
 			} else if isRequestFor("sha256:4dbc5f9c07028a985e14d1393e849ea07f68804c4293050d5a641b138db72daa", r) {

pkg/container/mocks/data/container_restarting.json

@@ -0,0 +1,205 @@
+{
+  "Id": "ae8964ba86c7cd7522cf84e09781343d88e0e3543281c747d88b27e246578b67",
+  "Created": "2019-04-10T19:51:22.245041005Z",
+  "Path": "/watchtower",
+  "Args": [],
+  "State": {
+    "Status": "exited",
+    "Running": false,
+    "Paused": false,
+    "Restarting": true,
+    "OOMKilled": false,
+    "Dead": false,
+    "Pid": 0,
+    "ExitCode": 1,
+    "Error": "",
+    "StartedAt": "2019-04-10T19:51:22.918972606Z",
+    "FinishedAt": "2019-04-10T19:52:14.265091583Z"
+  },
+  "Image": "sha256:4dbc5f9c07028a985e14d1393e849ea07f68804c4293050d5a641b138db72daa",
+  "ResolvConfPath": "/var/lib/docker/containers/ae8964ba86c7cd7522cf84e09781343d88e0e3543281c747d88b27e246578b65/resolv.conf",
+  "HostnamePath": "/var/lib/docker/containers/ae8964ba86c7cd7522cf84e09781343d88e0e3543281c747d88b27e246578b65/hostname",
+  "HostsPath": "/var/lib/docker/containers/ae8964ba86c7cd7522cf84e09781343d88e0e3543281c747d88b27e246578b65/hosts",
+  "LogPath": "/var/lib/docker/containers/ae8964ba86c7cd7522cf84e09781343d88e0e3543281c747d88b27e246578b65/ae8964ba86c7cd7522cf84e09781343d88e0e3543281c747d88b27e246578b65-json.log",
+  "Name": "/watchtower-test",
+  "RestartCount": 0,
+  "Driver": "overlay2",
+  "Platform": "linux",
+  "MountLabel": "",
+  "ProcessLabel": "",
+  "AppArmorProfile": "",
+  "ExecIDs": null,
+  "HostConfig": {
+    "Binds": [
+      "/var/run/docker.sock:/var/run/docker.sock"
+    ],
+    "ContainerIDFile": "",
+    "LogConfig": {
+      "Type": "json-file",
+      "Config": {}
+    },
+    "NetworkMode": "default",
+    "PortBindings": {},
+    "RestartPolicy": {
+      "Name": "no",
+      "MaximumRetryCount": 0
+    },
+    "AutoRemove": false,
+    "VolumeDriver": "",
+    "VolumesFrom": null,
+    "CapAdd": null,
+    "CapDrop": null,
+    "Dns": [],
+    "DnsOptions": [],
+    "DnsSearch": [],
+    "ExtraHosts": null,
+    "GroupAdd": null,
+    "IpcMode": "shareable",
+    "Cgroup": "",
+    "Links": null,
+    "OomScoreAdj": 0,
+    "PidMode": "",
+    "Privileged": false,
+    "PublishAllPorts": false,
+    "ReadonlyRootfs": false,
+    "SecurityOpt": null,
+    "UTSMode": "",
+    "UsernsMode": "",
+    "ShmSize": 67108864,
+    "Runtime": "runc",
+    "ConsoleSize": [
+      0,
+      0
+    ],
+    "Isolation": "",
+    "CpuShares": 0,
+    "Memory": 0,
+    "NanoCpus": 0,
+    "CgroupParent": "",
+    "BlkioWeight": 0,
+    "BlkioWeightDevice": [],
+    "BlkioDeviceReadBps": null,
+    "BlkioDeviceWriteBps": null,
+    "BlkioDeviceReadIOps": null,
+    "BlkioDeviceWriteIOps": null,
+    "CpuPeriod": 0,
+    "CpuQuota": 0,
+    "CpuRealtimePeriod": 0,
+    "CpuRealtimeRuntime": 0,
+    "CpusetCpus": "",
+    "CpusetMems": "",
+    "Devices": [],
+    "DeviceCgroupRules": null,
+    "DiskQuota": 0,
+    "KernelMemory": 0,
+    "MemoryReservation": 0,
+    "MemorySwap": 0,
+    "MemorySwappiness": null,
+    "OomKillDisable": false,
+    "PidsLimit": 0,
+    "Ulimits": null,
+    "CpuCount": 0,
+    "CpuPercent": 0,
+    "IOMaximumIOps": 0,
+    "IOMaximumBandwidth": 0,
+    "MaskedPaths": [
+      "/proc/asound",
+      "/proc/acpi",
+      "/proc/kcore",
+      "/proc/keys",
+      "/proc/latency_stats",
+      "/proc/timer_list",
+      "/proc/timer_stats",
+      "/proc/sched_debug",
+      "/proc/scsi",
+      "/sys/firmware"
+    ],
+    "ReadonlyPaths": [
+      "/proc/bus",
+      "/proc/fs",
+      "/proc/irq",
+      "/proc/sys",
+      "/proc/sysrq-trigger"
+    ]
+  },
+  "GraphDriver": {
+    "Data": {
+      "LowerDir": "/var/lib/docker/overlay2/9f6b91ea6e142835035d91123bbc7a05224dfa2abd4d020eac42f2ab420ccddc-init/diff:/var/lib/docker/overlay2/cdf82f50bc49177d0c17c24f3eaa29eba607b70cc6a081f77781b21c59a13eb8/diff:/var/lib/docker/overlay2/8108325ee844603c9b08d2772cf6e65dccf31dd5171f265078e5ed79a0ba3c0f/diff:/var/lib/docker/overlay2/e5e0cce6bf91b829a308424d99d7e56a33be3a11414ff5cdc48e762a1342b20f/diff",
+      "MergedDir": "/var/lib/docker/overlay2/9f6b91ea6e142835035d91123bbc7a05224dfa2abd4d020eac42f2ab420ccddc/merged",
+      "UpperDir": "/var/lib/docker/overlay2/9f6b91ea6e142835035d91123bbc7a05224dfa2abd4d020eac42f2ab420ccddc/diff",
+      "WorkDir": "/var/lib/docker/overlay2/9f6b91ea6e142835035d91123bbc7a05224dfa2abd4d020eac42f2ab420ccddc/work"
+    },
+    "Name": "overlay2"
+  },
+  "Mounts": [
+    {
+      "Type": "bind",
+      "Source": "/var/run/docker.sock",
+      "Destination": "/var/run/docker.sock",
+      "Mode": "",
+      "RW": true,
+      "Propagation": "rprivate"
+    }
+  ],
+  "Config": {
+    "Hostname": "ae8964ba86c7",
+    "Domainname": "",
+    "User": "",
+    "AttachStdin": false,
+    "AttachStdout": true,
+    "AttachStderr": true,
+    "Tty": false,
+    "OpenStdin": false,
+    "StdinOnce": false,
+    "Env": [
+      "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+    ],
+    "Cmd": null,
+    "Image": "containrrr/watchtower:latest",
+    "Volumes": null,
+    "WorkingDir": "",
+    "Entrypoint": [
+      "/watchtower"
+    ],
+    "OnBuild": null,
+    "Labels": {
+      "com.centurylinklabs.watchtower": "true"
+    }
+  },
+  "NetworkSettings": {
+    "Bridge": "",
+    "SandboxID": "05627d36c08ed994eebc44a2a8c9365a511756b55c500fb03fd5a14477cd4bf3",
+    "HairpinMode": false,
+    "LinkLocalIPv6Address": "",
+    "LinkLocalIPv6PrefixLen": 0,
+    "Ports": {},
+    "SandboxKey": "/var/run/docker/netns/05627d36c08e",
+    "SecondaryIPAddresses": null,
+    "SecondaryIPv6Addresses": null,
+    "EndpointID": "",
+    "Gateway": "",
+    "GlobalIPv6Address": "",
+    "GlobalIPv6PrefixLen": 0,
+    "IPAddress": "",
+    "IPPrefixLen": 0,
+    "IPv6Gateway": "",
+    "MacAddress": "",
+    "Networks": {
+      "bridge": {
+        "IPAMConfig": null,
+        "Links": null,
+        "Aliases": null,
+        "NetworkID": "8fcfd56fa9203bafa98510abb08bff66ad05bef5b6e97d158cbae3397e1e065e",
+        "EndpointID": "",
+        "Gateway": "",
+        "IPAddress": "",
+        "IPPrefixLen": 0,
+        "IPv6Gateway": "",
+        "GlobalIPv6Address": "",
+        "GlobalIPv6PrefixLen": 0,
+        "MacAddress": "",
+        "DriverOpts": null
+      }
+    }
+  }
+}

pkg/container/mocks/data/containers.json

@@ -109,5 +109,68 @@
         "Propagation": "rprivate"
       }
     ]
+  },
+  {
+    "Id": "ae8964ba86c7cd7522cf84e09781343d88e0e3543281c747d88b27e246578b67",
+    "Names": [
+      "/portainer"
+    ],
+    "Image": "portainer/portainer:latest",
+    "ImageID": "sha256:19d07168491a3f9e2798a9bed96544e34d57ddc4757a4ac5bb199dea896c87fd",
+    "Command": "/portainer",
+    "Created": 1554409712,
+    "Ports": [
+      {
+        "IP": "0.0.0.0",
+        "PrivatePort": 9000,
+        "PublicPort": 9000,
+        "Type": "tcp"
+      }
+    ],
+    "Labels": {},
+    "State": "restarting",
+    "Status": "Restarting (0) 35 seconds ago",
+    "HostConfig": {
+      "NetworkMode": "default"
+    },
+    "NetworkSettings": {
+      "Networks": {
+        "bridge": {
+          "IPAMConfig": null,
+          "Links": null,
+          "Aliases": null,
+          "NetworkID": "9352796e0330dcf31ce3d44fae4b719304b8b3fd97b02ade3aefb8737251682b",
+          "EndpointID": "a8bcd737f27edb4d2955f7bce0c777bb2990b792a6b335b0727387624abe0702",
+          "Gateway": "172.17.0.1",
+          "IPAddress": "172.17.0.2",
+          "IPPrefixLen": 16,
+          "IPv6Gateway": "",
+          "GlobalIPv6Address": "",
+          "GlobalIPv6PrefixLen": 0,
+          "MacAddress": "02:42:ac:11:00:02",
+          "DriverOpts": null
+        }
+      }
+    },
+    "Mounts": [
+      {
+        "Type": "volume",
+        "Name": "portainer_data",
+        "Source": "/var/lib/docker/volumes/portainer_data/_data",
+        "Destination": "/data",
+        "Driver": "local",
+        "Mode": "z",
+        "RW": true,
+        "Propagation": ""
+      },
+      {
+        "Type": "bind",
+        "Source": "/var/run/docker.sock",
+        "Destination": "/var/run/docker.sock",
+        "Mode": "",
+        "RW": true,
+        "Propagation": "rprivate"
+      }
+    ]
   }
 ]