add http head based digest comparison to avoid dockerhub rate limits

pkg/registry/auth/auth.go

@@ -0,0 +1,168 @@
+package auth
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"github.com/containrrr/watchtower/pkg/registry/helpers"
+	"github.com/containrrr/watchtower/pkg/types"
+	"github.com/docker/distribution/reference"
+	"github.com/sirupsen/logrus"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"strings"
+)
+
+// ChallengeHeader is the HTTP Header containing challenge instructions
+const ChallengeHeader = "WWW-Authenticate"
+
+// GetToken fetches a token for the registry hosting the provided image
+func GetToken(container types.Container, registryAuth string) (string, error) {
+	var err error
+	var URL url.URL
+
+	if URL, err = GetChallengeURL(container.ImageName()); err != nil {
+		return "", err
+	}
+	logrus.WithField("URL", URL.String()).Debug("Building challenge URL")
+
+	var req *http.Request
+	if req, err = GetChallengeRequest(URL); err != nil {
+		return "", err
+	}
+
+	client := &http.Client{}
+	var res *http.Response
+	if res, err = client.Do(req); err != nil {
+		return "", err
+	}
+
+	v := res.Header.Get(ChallengeHeader)
+
+	logrus.WithFields(logrus.Fields{
+		"status": res.Status,
+		"header": v,
+	}).Debug("Got response to challenge request")
+
+	challenge := strings.ToLower(v)
+	if strings.HasPrefix(challenge, "basic") {
+		if registryAuth == "" {
+			return "", fmt.Errorf("no credentials available")
+		}
+
+		return fmt.Sprintf("Basic %s", registryAuth), nil
+	}
+	if strings.HasPrefix(challenge, "bearer") {
+		return GetBearerHeader(challenge, container.ImageName(), err, registryAuth)
+	}
+
+	return "", errors.New("unsupported challenge type from registry")
+}
+
+// GetChallengeRequest creates a request for getting challenge instructions
+func GetChallengeRequest(URL url.URL) (*http.Request, error) {
+	req, err := http.NewRequest("GET", URL.String(), nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("Accept", "*/*")
+	req.Header.Set("User-Agent", "Watchtower (Docker)")
+	return req, nil
+}
+
+// GetBearerHeader tries to fetch a bearer token from the registry based on the challenge instructions
+func GetBearerHeader(challenge string, img string, err error, registryAuth string) (string, error) {
+	client := http.Client{}
+	if strings.Contains(img, ":") {
+		img = strings.Split(img, ":")[0]
+	}
+	authURL, err := GetAuthURL(challenge, img)
+
+	if err != nil {
+		return "", err
+	}
+
+	var r *http.Request
+	if r, err = http.NewRequest("GET", authURL.String(), nil); err != nil {
+		return "", err
+	}
+
+	if registryAuth != "" {
+		logrus.WithField("credentials", registryAuth).Debug("Credentials found.")
+		r.Header.Add("Authorization", fmt.Sprintf("Basic %s", registryAuth))
+	} else {
+		logrus.Debug("No credentials found.")
+	}
+
+	var authResponse *http.Response
+	if authResponse, err = client.Do(r); err != nil {
+		return "", err
+	}
+
+	body, _ := ioutil.ReadAll(authResponse.Body)
+	tokenResponse := &types.TokenResponse{}
+
+	err = json.Unmarshal(body, tokenResponse)
+	if err != nil {
+		return "", err
+	}
+
+	return fmt.Sprintf("Bearer %s", tokenResponse.Token), nil
+}
+
+// GetAuthURL from the instructions in the challenge
+func GetAuthURL(challenge string, img string) (*url.URL, error) {
+	loweredChallenge := strings.ToLower(challenge)
+	raw := strings.TrimPrefix(loweredChallenge, "bearer")
+
+	pairs := strings.Split(raw, ",")
+	values := make(map[string]string, len(pairs))
+
+	for _, pair := range pairs {
+		trimmed := strings.Trim(pair, " ")
+		kv := strings.Split(trimmed, "=")
+		key := kv[0]
+		val := strings.Trim(kv[1], "\"")
+		values[key] = val
+	}
+	logrus.WithFields(logrus.Fields{
+		"realm":   values["realm"],
+		"service": values["service"],
+	}).Debug("Checking challenge header content")
+	if values["realm"] == "" || values["service"] == "" {
+
+		return nil, fmt.Errorf("challenge header did not include all values needed to construct an auth url")
+	}
+
+	authURL, _ := url.Parse(fmt.Sprintf("%s", values["realm"]))
+	q := authURL.Query()
+	q.Add("service", values["service"])
+	scopeImage := strings.TrimPrefix(img, values["service"])
+	if !strings.Contains(scopeImage, "/") {
+		scopeImage = "library/" + scopeImage
+	}
+	scope := fmt.Sprintf("repository:%s:pull", scopeImage)
+	logrus.WithFields(logrus.Fields{"scope": scope, "image": img}).Debug("Setting scope for auth token")
+	q.Add("scope", scope)
+
+	authURL.RawQuery = q.Encode()
+	return authURL, nil
+}
+
+// GetChallengeURL creates a URL object based on the image info
+func GetChallengeURL(img string) (url.URL, error) {
+
+	normalizedNamed, _ := reference.ParseNormalizedNamed(img)
+	host, err := helpers.NormalizeRegistry(normalizedNamed.String())
+	if err != nil {
+		return url.URL{}, err
+	}
+
+	URL := url.URL{
+		Scheme: "https",
+		Host:   host,
+		Path:   "/v2/",
+	}
+	return URL, nil
+}

pkg/registry/auth/auth_test.go

@@ -0,0 +1,98 @@
+package auth_test
+
+import (
+	"fmt"
+	"github.com/containrrr/watchtower/internal/actions/mocks"
+	"github.com/containrrr/watchtower/pkg/registry/auth"
+	"net/url"
+	"os"
+	"testing"
+	"time"
+
+	wtTypes "github.com/containrrr/watchtower/pkg/types"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestAuth(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Registry Auth Suite")
+}
+func SkipIfCredentialsEmpty(credentials *wtTypes.RegistryCredentials, fn func()) func() {
+	if credentials.Username == "" {
+		return func() {
+			Skip("Username missing. Skipping integration test")
+		}
+	} else if credentials.Password == "" {
+		return func() {
+			Skip("Password missing. Skipping integration test")
+		}
+	} else {
+		return fn
+	}
+}
+
+var GHCRCredentials = &wtTypes.RegistryCredentials{
+	Username: os.Getenv("CI_INTEGRATION_TEST_REGISTRY_GH_USERNAME"),
+	Password: os.Getenv("CI_INTEGRATION_TEST_REGISTRY_GH_PASSWORD"),
+}
+
+var _ = Describe("the auth module", func() {
+	mockId := "mock-id"
+	mockName := "mock-container"
+	mockImage := "ghcr.io/k6io/operator:latest"
+	mockCreated := time.Now()
+	mockDigest := "ghcr.io/k6io/operator@sha256:d68e1e532088964195ad3a0a71526bc2f11a78de0def85629beb75e2265f0547"
+
+	mockContainer := mocks.CreateMockContainerWithDigest(
+		mockId,
+		mockName,
+		mockImage,
+		mockCreated,
+		mockDigest)
+
+	When("getting an auth url", func() {
+		It("should parse the token from the response",
+			SkipIfCredentialsEmpty(GHCRCredentials, func() {
+				creds := fmt.Sprintf("%s:%s", GHCRCredentials.Username, GHCRCredentials.Password)
+				token, err := auth.GetToken(mockContainer, creds)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(token).NotTo(Equal(""))
+			}),
+		)
+
+		It("should create a valid auth url object based on the challenge header supplied", func() {
+			input := `bearer realm="https://ghcr.io/token",service="ghcr.io",scope="repository:user/image:pull"`
+			expected := &url.URL{
+				Host:     "ghcr.io",
+				Scheme:   "https",
+				Path:     "/token",
+				RawQuery: "scope=repository%3Acontainrrr%2Fwatchtower%3Apull&service=ghcr.io",
+			}
+			res, err := auth.GetAuthURL(input, "containrrr/watchtower")
+			Expect(err).NotTo(HaveOccurred())
+			Expect(res).To(Equal(expected))
+		})
+		It("should create a valid auth url object based on the challenge header supplied", func() {
+			input := `bearer realm="https://ghcr.io/token"`
+			res, err := auth.GetAuthURL(input, "containrrr/watchtower")
+			Expect(err).To(HaveOccurred())
+			Expect(res).To(BeNil())
+		})
+	})
+	When("getting a challenge url", func() {
+		It("should create a valid challenge url object based on the image ref supplied", func() {
+			expected := url.URL{Host: "ghcr.io", Scheme: "https", Path: "/v2/"}
+			Expect(auth.GetChallengeURL("ghcr.io/containrrr/watchtower:latest")).To(Equal(expected))
+		})
+		It("should assume dockerhub if the image ref is not fully qualified", func() {
+			expected := url.URL{Host: "index.docker.io", Scheme: "https", Path: "/v2/"}
+			Expect(auth.GetChallengeURL("containrrr/watchtower:latest")).To(Equal(expected))
+		})
+		It("should convert legacy dockerhub hostnames to index.docker.io", func() {
+			expected := url.URL{Host: "index.docker.io", Scheme: "https", Path: "/v2/"}
+			Expect(auth.GetChallengeURL("docker.io/containrrr/watchtower:latest")).To(Equal(expected))
+			Expect(auth.GetChallengeURL("registry-1.docker.io/containrrr/watchtower:latest")).To(Equal(expected))
+		})
+	})
+})