Merge 692513043b into 76f9cea516

cmd/root.go

@@ -39,6 +39,7 @@ var (
 	disableContainers []string
 	notifier          t.Notifier
 	timeout           time.Duration
+	deferDays         int
 	lifecycleHooks    bool
 	rollingRestart    bool
 	scope             string
@@ -96,6 +97,7 @@ func PreRun(cmd *cobra.Command, _ []string) {
 
 	enableLabel, _ = f.GetBool("label-enable")
 	disableContainers, _ = f.GetStringSlice("disable-containers")
+	deferDays, _ = f.GetInt("defer-days")
 	lifecycleHooks, _ = f.GetBool("enable-lifecycle-hooks")
 	rollingRestart, _ = f.GetBool("rolling-restart")
 	scope, _ = f.GetString("scope")
@@ -288,6 +290,10 @@ func writeStartupMessage(c *cobra.Command, sched time.Time, filtering string) {
 		until := formatDuration(time.Until(sched))
 		startupLog.Info("Scheduling first run: " + sched.Format("2006-01-02 15:04:05 -0700 MST"))
 		startupLog.Info("Note that the first check will be performed in " + until)
+		deferDays, _ = c.PersistentFlags().GetInt("defer-days")
+		if deferDays > 0 {
+			startupLog.Infof("Container updates will be deferred until %d day(s) after image creation.", deferDays)
+		}
 	} else if runOnce, _ := c.PersistentFlags().GetBool("run-once"); runOnce {
 		startupLog.Info("Running a one time update.")
 	} else {
@@ -364,6 +370,7 @@ func runUpdatesWithNotifications(filter t.Filter) *metrics.Metric {
 		NoRestart:       noRestart,
 		Timeout:         timeout,
 		MonitorOnly:     monitorOnly,
+		DeferDays:       deferDays,
 		LifecycleHooks:  lifecycleHooks,
 		RollingRestart:  rollingRestart,
 		LabelPrecedence: labelPrecedence,
@@ -378,6 +385,7 @@ func runUpdatesWithNotifications(filter t.Filter) *metrics.Metric {
 	notifications.LocalLog.WithFields(log.Fields{
 		"Scanned":  metricResults.Scanned,
 		"Updated":  metricResults.Updated,
+		"Deferred": metricResults.Deferred,
 		"Failed":   metricResults.Failed,
 	}).Info("Session done")
 	return metricResults

docs/arguments.md

@@ -381,6 +381,16 @@ Environment Variable: WATCHTOWER_HTTP_API_METRICS
              Default: false
 ```
 
+## Deferred Update
+Only update container to latest version of image if some number of days have passed since it has been published. This option may be useful for those who wish to avoid updating prior to the new version having some time in the field prior to updating in case there are critical defects found and released in a subsequent version.
+
+```text
+            Argument: --defer-days
+Environment Variable: WATCHTOWER_DEFER_DAYS
+                Type: Integer
+             Default: false
+```
+
 ## Scheduling
 [Cron expression](https://pkg.go.dev/github.com/robfig/cron@v1.2.0?tab=doc#hdr-CRON_Expression_Format) in 6 fields (rather than the traditional 5) which defines when and how often to check for new images. Either `--interval` or the schedule expression
 can be defined, but not both. An example: `--schedule "0 0 4 * * *"`

internal/actions/mocks/container.go

@@ -79,6 +79,13 @@ func CreateMockContainerWithDigest(id string, name string, image string, created
 	return c
 }
 
+// CreateMockContainerWithDigest should only be used for testing
+func CreateMockContainerWithImageCreatedTime(id string, name string, image string, created time.Time, imageCreated time.Time) wt.Container {
+	c := CreateMockContainer(id, name, image, created)
+	c.ImageInfo().Created = imageCreated.UTC().Format(time.RFC3339Nano)
+	return c
+}
+
 // CreateMockContainerWithConfig creates a container substitute valid for testing
 func CreateMockContainerWithConfig(id string, name string, image string, running bool, restarting bool, created time.Time, config *dockerContainer.Config) wt.Container {
 	content := types.ContainerJSON{

internal/actions/update.go

@@ -2,6 +2,7 @@ package actions
 
 import (
 	"errors"
+	"time"
 
 	"github.com/containrrr/watchtower/internal/util"
 	"github.com/containrrr/watchtower/pkg/container"
@@ -33,13 +34,24 @@ func Update(client container.Client, params types.UpdateParams) (types.Report, e
 	staleCheckFailed := 0
 
 	for i, targetContainer := range containers {
+		// stale will be true if there is a more recent image than the current container is using
 		stale, newestImage, err := client.IsContainerStale(targetContainer, params)
 		shouldUpdate := stale && !params.NoRestart && !targetContainer.IsMonitorOnly(params)
+		imageUpdateDeferred := false
+		imageAgeDays := 0
 		if err == nil && shouldUpdate {
-			// Check to make sure we have all the necessary information for recreating the container
+			// Check to make sure we have all the necessary information for recreating the container, including ImageInfo
 			err = targetContainer.VerifyConfiguration()
+			if err == nil {
+				if params.DeferDays > 0 {
+					imageAgeDays, imageErr := getImageAgeDays(targetContainer.ImageInfo().Created)
+					err = imageErr
+					if err == nil {
+						imageUpdateDeferred = imageAgeDays < params.DeferDays
+					}
+				}
+			} else if log.IsLevelEnabled(log.TraceLevel) {
 				// If the image information is incomplete and trace logging is enabled, log it for further diagnosis
-			if err != nil && log.IsLevelEnabled(log.TraceLevel) {
 				imageInfo := targetContainer.ImageInfo()
 				log.Tracef("Image info: %#v", imageInfo)
 				log.Tracef("Container info: %#v", targetContainer.ContainerInfo())
@@ -54,6 +66,12 @@ func Update(client container.Client, params types.UpdateParams) (types.Report, e
 			stale = false
 			staleCheckFailed++
 			progress.AddSkipped(targetContainer, err)
+		} else if imageUpdateDeferred {
+			log.Infof("New image found for %s that was created %d day(s) ago but update deferred until %d day(s) after creation", targetContainer.Name(), imageAgeDays, params.DeferDays)
+			// technically the container is stale but we set it to false here because it is this stale flag that tells downstream methods whether to perform the update
+			stale = false
+			progress.AddScanned(targetContainer, newestImage)
+			progress.MarkDeferred(targetContainer.ID())
 		} else {
 			progress.AddScanned(targetContainer, newestImage)
 		}
@@ -71,9 +89,13 @@ func Update(client container.Client, params types.UpdateParams) (types.Report, e
 
 	UpdateImplicitRestart(containers)
 
+	// containersToUpdate will contain all containers, not just those that need to be updated. The "stale" flag is checked via container.ToRestart()
+	// within stopContainersInReversedOrder and restartContainersInSortedOrder to skip containers with stale set to false (unless LinkedToRestarting set)
+	// NOTE: This logic is changing with latest PR on main repo
 	var containersToUpdate []types.Container
 	for _, c := range containers {
-		if !c.IsMonitorOnly(params) {
+		// pulling this change in from PR 1895 for now to avoid updating status incorrectly
+		if c.ToRestart() && !c.IsMonitorOnly(params) {
 			containersToUpdate = append(containersToUpdate, c)
 			progress.MarkForUpdate(c.ID())
 		}
@@ -265,3 +287,18 @@ func linkedContainerMarkedForRestart(links []string, containers []types.Containe
 	}
 	return ""
 }
+
+// Finds the difference between now and a given date, in full days. Input date is expected to originate
+// from an image's Created attribute which will follow ISO 3339/8601 format.
+// Reference: https://docs.docker.com/engine/api/v1.43/#tag/Image/operation/ImageInspect
+func getImageAgeDays(imageCreatedDateTime string) (int, error) {
+	imageCreatedDate, error := time.Parse(time.RFC3339Nano, imageCreatedDateTime)
+
+	if error != nil {
+		log.Errorf("Error parsing imageCreatedDateTime date (%s). Error: %s", imageCreatedDateTime, error)
+		return -1, error
+	}
+
+	return int(time.Since(imageCreatedDate).Hours() / 24), nil
+
+}

internal/actions/update_test.go

@@ -65,6 +65,35 @@ func getLinkedTestData(withImageInfo bool) *TestData {
 	}
 }
 
+func getMixedAgeTestData(keepContainer string) *TestData {
+	return &TestData{
+		NameOfContainerToKeep: keepContainer,
+		Containers: []types.Container{
+			// new container with 5 day old image
+			CreateMockContainerWithImageCreatedTime(
+				"test-container-01",
+				"test-container-01",
+				"fake-image-01:latest",
+				time.Now(),
+				time.Now().AddDate(0, 0, -5)),
+			// new container with 1 day old image
+			CreateMockContainerWithImageCreatedTime(
+				"test-container-02",
+				"test-container-02",
+				"fake-image-02:latest",
+				time.Now(),
+				time.Now().AddDate(0, 0, -1)),
+			// new container with 1 hour old image
+			CreateMockContainerWithImageCreatedTime(
+				"test-container-03",
+				"test-container-03",
+				"fake-image-03:latest",
+				time.Now(),
+				time.Now().Add(-1*time.Hour)),
+		},
+	}
+}
+
 var _ = Describe("the update action", func() {
 	When("watchtower has been instructed to clean up", func() {
 		When("there are multiple containers using the same image", func() {
@@ -258,6 +287,37 @@ var _ = Describe("the update action", func() {
 		})
 	})
 
+	When("watchtower has been instructed to defer updates by some number of days", func() {
+		It("should only update the 1 container with image at least 2 days old when DeferDays is 2", func() {
+			client := CreateMockClient(getMixedAgeTestData(""), false, false)
+			report, err := actions.Update(client, types.UpdateParams{DeferDays: 2})
+			Expect(err).NotTo(HaveOccurred())
+			Expect(report.Updated()).To(HaveLen(1))
+			Expect(report.Deferred()).To(HaveLen(2))
+		})
+		It("should only update the 2 containers with image at least 1 day old when DeferDays is 1", func() {
+			client := CreateMockClient(getMixedAgeTestData(""), false, false)
+			report, err := actions.Update(client, types.UpdateParams{DeferDays: 1})
+			Expect(err).NotTo(HaveOccurred())
+			Expect(report.Updated()).To(HaveLen(2))
+			Expect(report.Deferred()).To(HaveLen(1))
+		})
+		It("should update all containers when DeferDays is 0", func() {
+			client := CreateMockClient(getMixedAgeTestData(""), false, false)
+			report, err := actions.Update(client, types.UpdateParams{DeferDays: 0})
+			Expect(err).NotTo(HaveOccurred())
+			Expect(report.Updated()).To(HaveLen(3))
+			Expect(report.Deferred()).To(HaveLen(0))
+		})
+		It("should update all containers when DeferDays is not specified", func() {
+			client := CreateMockClient(getMixedAgeTestData(""), false, false)
+			report, err := actions.Update(client, types.UpdateParams{})
+			Expect(err).NotTo(HaveOccurred())
+			Expect(report.Updated()).To(HaveLen(3))
+			Expect(report.Deferred()).To(HaveLen(0))
+		})
+	})
+
 	When("watchtower has been instructed to run lifecycle hooks", func() {
 
 		When("pre-update script returns 1", func() {

internal/flags/flags.go

@@ -147,6 +147,12 @@ func RegisterSystemFlags(rootCmd *cobra.Command) {
 		envBool("WATCHTOWER_LIFECYCLE_HOOKS"),
 		"Enable the execution of commands triggered by pre- and post-update lifecycle hooks")
 
+	flags.IntP(
+		"defer-days",
+		"0",
+		envInt("WATCHTOWER_DEFER_DAYS"),
+		"Number of days to wait for new image version to be in place prior to installing it")
+
 	flags.BoolP(
 		"rolling-restart",
 		"",

internal/flags/flags_test.go

@@ -278,9 +278,9 @@ func TestProcessFlagAliasesInvalidPorcelaineVersion(t *testing.T) {
 	})
 }
 
-func TestFlagsArePrecentInDocumentation(t *testing.T) {
+func TestFlagsArePresentInDocumentation(t *testing.T) {
 
-	// Legacy notifcations are ignored, since they are (soft) deprecated
+	// Legacy notifications are ignored, since they are (soft) deprecated
 	ignoredEnvs := map[string]string{
 		"WATCHTOWER_NOTIFICATION_SLACK_ICON_EMOJI": "legacy",
 		"WATCHTOWER_NOTIFICATION_SLACK_ICON_URL":   "legacy",

pkg/metrics/metrics.go

@@ -12,6 +12,7 @@ var metrics *Metrics
 type Metric struct {
 	Scanned  int
 	Updated  int
+	Deferred int
 	Failed   int
 }
 
@@ -20,6 +21,7 @@ type Metrics struct {
 	channel  chan *Metric
 	scanned  prometheus.Gauge
 	updated  prometheus.Gauge
+	deferred prometheus.Gauge
 	failed   prometheus.Gauge
 	total    prometheus.Counter
 	skipped  prometheus.Counter
@@ -31,6 +33,7 @@ func NewMetric(report types.Report) *Metric {
 		Scanned: len(report.Scanned()),
 		// Note: This is for backwards compatibility. ideally, stale containers should be counted separately
 		Updated:  len(report.Updated()) + len(report.Stale()),
+		Deferred: len(report.Deferred()),
 		Failed:   len(report.Failed()),
 	}
 }
@@ -60,6 +63,10 @@ func Default() *Metrics {
 			Name: "watchtower_containers_updated",
 			Help: "Number of containers updated by watchtower during the last scan",
 		}),
+		deferred: promauto.NewGauge(prometheus.GaugeOpts{
+			Name: "watchtower_containers_deferred",
+			Help: "Number of containers deferred by watchtower during the last scan",
+		}),
 		failed: promauto.NewGauge(prometheus.GaugeOpts{
 			Name: "watchtower_containers_failed",
 			Help: "Number of containers where update failed during the last scan",
@@ -95,6 +102,7 @@ func (metrics *Metrics) HandleUpdate(channel <-chan *Metric) {
 			metrics.skipped.Inc()
 			metrics.scanned.Set(0)
 			metrics.updated.Set(0)
+			metrics.deferred.Set(0)
 			metrics.failed.Set(0)
 			continue
 		}
@@ -102,6 +110,7 @@ func (metrics *Metrics) HandleUpdate(channel <-chan *Metric) {
 		metrics.total.Inc()
 		metrics.scanned.Set(float64(change.Scanned))
 		metrics.updated.Set(float64(change.Updated))
+		metrics.deferred.Set(float64(change.Deferred))
 		metrics.failed.Set(float64(change.Failed))
 	}
 }

pkg/notifications/json.go

@@ -25,6 +25,7 @@ func (d Data) MarshalJSON() ([]byte, error) {
 		report = jsonMap{
 			`scanned`:  marshalReports(d.Report.Scanned()),
 			`updated`:  marshalReports(d.Report.Updated()),
+			`deferred`: marshalReports(d.Report.Deferred()),
 			`failed`:   marshalReports(d.Report.Failed()),
 			`skipped`:  marshalReports(d.Report.Skipped()),
 			`stale`:    marshalReports(d.Report.Stale()),

pkg/notifications/json_test.go

@@ -21,6 +21,7 @@ var _ = Describe("JSON template", func() {
 		],
 		"host": "Mock",
 		"report": {
+		"deferred": [],
 		"failed": [
 			{
 				"currentImageId": "01d210000000",
@@ -110,7 +111,7 @@ var _ = Describe("JSON template", func() {
 		},
 		"title": "Watchtower updates on Mock"
 }`
-				data := mockDataFromStates(s.UpdatedState, s.FreshState, s.FailedState, s.SkippedState, s.UpdatedState)
+				data := mockDataFromStates(s.UpdatedState, s.DeferredState, s.FreshState, s.FailedState, s.SkippedState, s.UpdatedState)
 				Expect(getTemplatedResult(`json.v1`, false, data)).To(MatchJSON(expected))
 			})
 		})

pkg/notifications/preview/data/data.go

@@ -72,6 +72,8 @@ func (pb *previewData) addContainer(c containerStatus) {
 		pb.report.scanned = append(pb.report.scanned, &c)
 	case UpdatedState:
 		pb.report.updated = append(pb.report.updated, &c)
+	case DeferredState:
+		pb.report.deferred = append(pb.report.deferred, &c)
 	case FailedState:
 		pb.report.failed = append(pb.report.failed, &c)
 	case SkippedState:

pkg/notifications/preview/data/report.go

@@ -12,6 +12,7 @@ type State string
 const (
 	ScannedState  State = "scanned"
 	UpdatedState  State = "updated"
+	DeferredState State = "deferred"
 	FailedState   State = "failed"
 	SkippedState  State = "skipped"
 	StaleState    State = "stale"
@@ -27,6 +28,8 @@ func StatesFromString(str string) []State {
 			states = append(states, ScannedState)
 		case 'u':
 			states = append(states, UpdatedState)
+		case 'd':
+			states = append(states, DeferredState)
 		case 'e':
 			states = append(states, FailedState)
 		case 'k':
@@ -45,6 +48,7 @@ func StatesFromString(str string) []State {
 type report struct {
 	scanned  []types.ContainerReport
 	updated  []types.ContainerReport
+	deferred []types.ContainerReport
 	failed   []types.ContainerReport
 	skipped  []types.ContainerReport
 	stale    []types.ContainerReport
@@ -57,6 +61,9 @@ func (r *report) Scanned() []types.ContainerReport {
 func (r *report) Updated() []types.ContainerReport {
 	return r.updated
 }
+func (r *report) Deferred() []types.ContainerReport {
+	return r.deferred
+}
 func (r *report) Failed() []types.ContainerReport {
 	return r.failed
 }
@@ -87,6 +94,7 @@ func (r *report) All() []types.ContainerReport {
 	}
 
 	appendUnique(r.updated)
+	appendUnique(r.deferred)
 	appendUnique(r.failed)
 	appendUnique(r.skipped)
 	appendUnique(r.stale)

pkg/session/container_status.go

@@ -12,6 +12,7 @@ const (
 	SkippedState
 	ScannedState
 	UpdatedState
+	DeferredState
 	FailedState
 	FreshState
 	StaleState
@@ -70,6 +71,8 @@ func (u *ContainerStatus) State() string {
 		return "Scanned"
 	case UpdatedState:
 		return "Updated"
+	case DeferredState:
+		return "Deferred"
 	case FailedState:
 		return "Failed"
 	case FreshState:

pkg/session/progress.go

@@ -50,6 +50,11 @@ func (m Progress) MarkForUpdate(containerID types.ContainerID) {
 	m[containerID].state = UpdatedState
 }
 
+// MarkForUpdate marks the container identified by containerID for deferral
+func (m Progress) MarkDeferred(containerID types.ContainerID) {
+	m[containerID].state = DeferredState
+}
+
 // Report creates a new Report from a Progress instance
 func (m Progress) Report() types.Report {
 	return NewReport(m)

pkg/session/report.go

@@ -9,6 +9,7 @@ import (
 type report struct {
 	scanned  []types.ContainerReport
 	updated  []types.ContainerReport
+	deferred []types.ContainerReport
 	failed   []types.ContainerReport
 	skipped  []types.ContainerReport
 	stale    []types.ContainerReport
@@ -21,6 +22,9 @@ func (r *report) Scanned() []types.ContainerReport {
 func (r *report) Updated() []types.ContainerReport {
 	return r.updated
 }
+func (r *report) Deferred() []types.ContainerReport {
+	return r.deferred
+}
 func (r *report) Failed() []types.ContainerReport {
 	return r.failed
 }
@@ -50,6 +54,7 @@ func (r *report) All() []types.ContainerReport {
 	}
 
 	appendUnique(r.updated)
+	appendUnique(r.deferred)
 	appendUnique(r.failed)
 	appendUnique(r.skipped)
 	appendUnique(r.stale)
@@ -66,6 +71,7 @@ func NewReport(progress Progress) types.Report {
 	report := &report{
 		scanned:  []types.ContainerReport{},
 		updated:  []types.ContainerReport{},
+		deferred: []types.ContainerReport{},
 		failed:   []types.ContainerReport{},
 		skipped:  []types.ContainerReport{},
 		stale:    []types.ContainerReport{},
@@ -88,9 +94,13 @@ func NewReport(progress Progress) types.Report {
 		switch update.state {
 		case UpdatedState:
 			report.updated = append(report.updated, update)
+		case DeferredState:
+			report.deferred = append(report.deferred, update)
 		case FailedState:
 			report.failed = append(report.failed, update)
 		default:
+			// TODO: should this be changed to something lke UnknownState since it shouldn't be possible for a container
+			// to be stale but its state to not be either UpdatedState, DeferredState, or FailedState?
 			update.state = StaleState
 			report.stale = append(report.stale, update)
 		}
@@ -98,6 +108,7 @@ func NewReport(progress Progress) types.Report {
 
 	sort.Sort(sortableContainers(report.scanned))
 	sort.Sort(sortableContainers(report.updated))
+	sort.Sort(sortableContainers(report.deferred))
 	sort.Sort(sortableContainers(report.failed))
 	sort.Sort(sortableContainers(report.skipped))
 	sort.Sort(sortableContainers(report.stale))

pkg/types/report.go

@@ -4,6 +4,7 @@ package types
 type Report interface {
 	Scanned() []ContainerReport
 	Updated() []ContainerReport
+	Deferred() []ContainerReport
 	Failed() []ContainerReport
 	Skipped() []ContainerReport
 	Stale() []ContainerReport

pkg/types/update_params.go

@@ -12,6 +12,7 @@ type UpdateParams struct {
 	Timeout         time.Duration
 	MonitorOnly     bool
 	NoPull          bool
+	DeferDays       int
 	LifecycleHooks  bool
 	RollingRestart  bool
 	LabelPrecedence bool

tplprev/main.go

@@ -18,7 +18,7 @@ func main() {
 	var states string
 	var entries string
 
-	flag.StringVar(&states, "states", "cccuuueeekkktttfff", "sCanned, Updated, failEd, sKipped, sTale, Fresh")
+	flag.StringVar(&states, "states", "cccuuudddeeekkktttfff", "sCanned, Updated, Deferred, failEd, sKipped, sTale, Fresh")
 	flag.StringVar(&entries, "entries", "ewwiiidddd", "Fatal,Error,Warn,Info,Debug,Trace")
 
 	flag.Parse()