Andreas/watchtower
1
0
Fork 0
mirror of https://github.com/containrrr/watchtower.git synced 2026-02-13 10:54:21 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix some tests, split up and refactor

Browse source 
some wonky regression introduced by docker dependencies when running on darwin. see https://github.com/ory/dockertest/issues/212 for more info.
will have a look at this next
This commit is contained in:
Simon Aronsson 2020-11-15 23:27:32 +01:00
parent 83aa420996
commit 6b9fd8d7ef
No known key found for this signature in database
GPG key ID: 8DA57A5FD341605B
12 changed files with 389 additions and 225 deletions
Download patch file Download diff file Expand all files Collapse all files

39
pkg/logger/logger.go Normal file
View file

@ -0,0 +1,39 @@
package logger
import (
"context"
"github.com/sirupsen/logrus"
)
const ContextKey = "LogrusLoggerContext"
// GetLogger returns a logger from the context if one is available, otherwise a default logger
func GetLogger(ctx context.Context) *logrus.Logger {
if logger, ok := ctx.Value(ContextKey).(logrus.Logger); ok {
return &logger
} else {
return newLogger(&logrus.JSONFormatter{}, logrus.InfoLevel)
}
}
func AddLogger(ctx context.Context) {
setLogger(ctx, &logrus.JSONFormatter{}, logrus.InfoLevel)
}
func AddDebugLogger(ctx context.Context) {
setLogger(ctx, &logrus.TextFormatter{}, logrus.DebugLevel)
}
// SetLogger adds a logger to the supplied context
func setLogger(ctx context.Context, fmt logrus.Formatter, level logrus.Level) {
log := newLogger(fmt, level)
context.WithValue(ctx, ContextKey, log)
}
func newLogger(fmt logrus.Formatter, level logrus.Level) *logrus.Logger {
log := logrus.New()
log.SetFormatter(fmt)
log.SetLevel(level)
return log
}

144
pkg/registry/auth/auth.go Normal file
View file

@ -0,0 +1,144 @@
package auth
import (
"context"
"encoding/json"
"errors"
"fmt"
ref "github.com/containers/image/v5/docker/reference"
"github.com/containrrr/watchtower/pkg/logger"
"github.com/containrrr/watchtower/pkg/registry/helpers"
"github.com/containrrr/watchtower/pkg/types"
apiTypes "github.com/docker/docker/api/types"
"github.com/sirupsen/logrus"
"io/ioutil"
"net/http"
url2 "net/url"
"strings"
)
const ChallengeHeader = "WWW-Authenticate"
func GetToken(ctx context.Context, image apiTypes.ImageInspect, credentials *types.RegistryCredentials) (string, error) {
var err error
log := logger.GetLogger(ctx)
img := strings.Split(image.RepoTags[0], ":")[0]
var url url2.URL
if url, err = GetChallengeURL(img); err != nil {
return "", err
}
var req *http.Request
if req, err = GetChallengeRequest(url); err != nil {
return "", err
}
var client = http.Client{}
var res *http.Response
if res, err = client.Do(req); err != nil {
return "", err
}
v := res.Header.Get(ChallengeHeader)
log.WithFields(logrus.Fields{
"status": res.Status,
"header": v,
}).Debug("Got response to challenge request")
challenge := strings.ToLower(v)
if strings.HasPrefix(challenge, "basic") {
return "", errors.New("basic auth not implemented yet")
}
if strings.HasPrefix(challenge, "bearer") {
log.Debug("Fetching bearer token")
return GetBearerToken(ctx, challenge, img, err, credentials)
}
return "", errors.New("unsupported challenge type from registry")
}
func GetChallengeRequest(url url2.URL) (*http.Request, error) {
req, err := http.NewRequest("GET", url.String(), nil)
if err != nil {
return nil, err
}
req.Header.Set("Accept", "*/*")
req.Header.Set("User-Agent", "Watchtower (Docker)")
return req, nil
}
func GetBearerToken(ctx context.Context, challenge string, img string, err error, credentials *types.RegistryCredentials) (string, error) {
log := logger.GetLogger(ctx)
client := http.Client{}
authURL := GetAuthURL(challenge, img)
var r *http.Request
if r, err = http.NewRequest("GET", authURL.String(), nil); err != nil {
return "", err
}
if credentials.Username != "" && credentials.Password != "" {
log.WithField("credentials", credentials).Debug("Found credentials. Adding basic auth.")
r.SetBasicAuth(credentials.Username, credentials.Password)
} else {
log.Debug("No credentials found. Doing an anonymous request.")
}
var authResponse *http.Response
if authResponse, err = client.Do(r); err != nil {
return "", err
}
body, _ := ioutil.ReadAll(authResponse.Body)
tokenResponse := &types.TokenResponse{}
err = json.Unmarshal(body, tokenResponse)
if err != nil {
return "", err
}
return tokenResponse.Token, nil
}
func GetAuthURL(challenge string, img string) *url2.URL {
raw := strings.TrimPrefix(challenge, "bearer")
pairs := strings.Split(raw, ",")
values := make(map[string]string, 0)
for _, pair := range pairs {
trimmed := strings.Trim(pair, " ")
kv := strings.Split(trimmed, "=")
key := kv[0]
val := strings.Trim(kv[1], "\"")
values[key] = val
}
authURL, _ := url2.Parse(fmt.Sprintf("%s", values["realm"]))
q := authURL.Query()
q.Add("service", values["service"])
scopeImage := strings.TrimPrefix(img, values["service"])
scope := fmt.Sprintf("repository:%s:pull", scopeImage)
q.Add("scope", scope)
authURL.RawQuery = q.Encode()
return authURL
}
func GetChallengeURL(img string) (url2.URL, error) {
normalizedNamed, _ := ref.ParseNormalizedNamed(img)
host, err := helpers.NormalizeRegistry(normalizedNamed.Name())
if err != nil {
return url2.URL{}, err
}
url := url2.URL{
Scheme: "https",
Host: host ,
Path: "/v2/",
}
return url, nil
}

178
pkg/registry/digest/digest.go
View file

@ -1,40 +1,38 @@
package digest
import (
"encoding/json"
"context"
"errors"
"fmt"
ref "github.com/containers/image/v5/docker/reference"
"github.com/containrrr/watchtower/pkg/logger"
"github.com/containrrr/watchtower/pkg/registry/auth"
"github.com/containrrr/watchtower/pkg/registry/manifest"
"github.com/containrrr/watchtower/pkg/types"
apiTypes "github.com/docker/docker/api/types"
log "github.com/sirupsen/logrus"
"io/ioutil"
"net/http"
url2 "net/url"
"strings"
)
const (
ManifestListV2ContentType = "application/vnd.docker.distribution.manifest.list.v2+json"
ChallengeHeader = "WWW-Authenticate"
ContentDigestHeader = "Docker-Content-Digest"
)
// CompareDigest ...
func CompareDigest(image apiTypes.ImageInspect, credentials *RegistryCredentials) (bool, error) {
func CompareDigest(ctx context.Context, image apiTypes.ImageInspect, credentials *types.RegistryCredentials) (bool, error) {
var digest string
token, err := GetToken(image, credentials)
log := logger.GetLogger(ctx).WithField("fun", "CompareDigest")
token, err := auth.GetToken(ctx, image, credentials)
if err != nil {
return false, err
}
digestURL, err := BuildManifestURL(image)
digestURL, err := manifest.BuildManifestURL(image)
if err != nil {
return false, err
}
if digest, err = GetDigest(digestURL, token); err != nil {
if digest, err = GetDigest(ctx, digestURL, token); err != nil {
return false, err
}
@ -49,15 +47,16 @@ func CompareDigest(image apiTypes.ImageInspect, credentials *RegistryCredentials
localDigest := strings.Split(dig, "@")[1]
log.WithField("Local Digest", localDigest).Debug("Comparing with local digest")
if localDigest == digest {
return true,nil
return true, nil
}
}
return false, nil
}
func GetDigest(url string, token string) (string, error) {
func GetDigest(ctx context.Context, url string, token string) (string, error) {
client := &http.Client{}
log := logger.GetLogger(ctx).WithField("fun", "GetDigest")
if token != "" {
log.WithField("token", token).Debug("Setting request bearer token")
} else {
@ -65,10 +64,10 @@ func GetDigest(url string, token string) (string, error) {
}
req, _ := http.NewRequest("HEAD", url, nil)
req.Header.Add("Authorization", "Bearer " + token)
req.Header.Add("Accept", ManifestListV2ContentType)
log.WithField("url", url)
req.Header.Add("Authorization", "Bearer "+token)
req.Header.Add("Accept", "*")
log.WithField("url", url).Debug("Doing a HEAD request to fetch a digest")
res, err := client.Do(req)
if err != nil {
return "", err
@ -81,148 +80,3 @@ func GetDigest(url string, token string) (string, error) {
return res.Header.Get(ContentDigestHeader), nil
}
func GetToken(image apiTypes.ImageInspect, credentials *RegistryCredentials) (string, error){
img := strings.Split(image.RepoTags[0], ":")[0]
url := GetChallengeURL(img)
res, err := DoChallengeRequest(url)
if err != nil {
return "", err
}
v := res.Header.Get(ChallengeHeader)
challenge := strings.ToLower(v)
if strings.HasPrefix(challenge, "basic") {
return "", errors.New("basic auth not implemented yet")
}
if strings.HasPrefix(challenge, "bearer") {
return GetBearerToken(challenge, img, err, credentials)
}
return "", errors.New("unsupported challenge type from registry")
}
func DoChallengeRequest(url url2.URL) (*http.Response, error) {
req, _ := http.NewRequest("GET", url.String(), nil)
req.Header.Set("Accept", "*/*")
req.Header.Set("User-Agent", "Watchtower (Docker)")
client := http.Client{}
return client.Do(req)
}
func GetBearerToken(challenge string, img string, err error, credentials *RegistryCredentials) (string, error) {
client := http.Client{}
authURL := GetAuthURL(challenge, img)
var r *http.Request
if r, err = http.NewRequest("GET", authURL.String(), nil); err != nil {
return "", err
}
if credentials.Username != "" && credentials.Password != "" {
r.SetBasicAuth(credentials.Username, credentials.Password)
}
var authResponse *http.Response
if authResponse, err = client.Do(r); err != nil {
return "", err
}
body, _ := ioutil.ReadAll(authResponse.Body)
tokenResponse := &TokenResponse{}
err = json.Unmarshal(body, tokenResponse)
if err != nil {
return "", err
}
return tokenResponse.Token, nil
}
func GetAuthURL(challenge string, img string) *url2.URL {
raw := strings.TrimPrefix(challenge, "bearer")
pairs := strings.Split(raw, ",")
values := make(map[string]string, 0)
for _, pair := range pairs {
trimmed := strings.Trim(pair, " ")
kv := strings.Split(trimmed, "=")
key := kv[0]
val := strings.Trim(kv[1], "\"")
values[key] = val
}
authURL, _ := url2.Parse(fmt.Sprintf("%s", values["realm"]))
q := authURL.Query()
q.Add("service", values["service"])
scopeImage := strings.TrimPrefix(img, values["service"])
scope := fmt.Sprintf("repository:%s:pull", scopeImage)
q.Add("scope", scope)
authURL.RawQuery = q.Encode()
return authURL
}
func GetChallengeURL(img string) url2.URL {
normalizedNamed, _ := ref.ParseNormalizedNamed(img)
url := url2.URL{
Scheme: "https",
Host: normalizeRegistry(normalizedNamed.Name()),
Path: "/v2/",
}
return url
}
type TokenResponse struct {
Token string `json:"token"`
}
type RegistryCredentials struct {
Username string
Password string // usually a token rather than an actual password
}
func BuildManifestURL(image apiTypes.ImageInspect) (string, error) {
parts := strings.Split(image.RepoTags[0], ":")
img := parts[0]
tag := parts[1]
hostName, err := ref.ParseNormalizedNamed(img)
if err != nil {
return "", err
}
host := normalizeRegistry(hostName.Name())
img = strings.TrimPrefix(img, host)
url := url2.URL{
Scheme: "https",
Host: host,
Path: fmt.Sprintf("/v2/%s/manifests/%s", img, tag),
}
return url.String(), nil
}
// Copied from github.com/docker/docker/registry/auth.go
func convertToHostname(url string) string {
stripped := url
if strings.HasPrefix(url, "http://") {
stripped = strings.TrimPrefix(url, "http://")
} else if strings.HasPrefix(url, "https://") {
stripped = strings.TrimPrefix(url, "https://")
}
nameParts := strings.SplitN(stripped, "/", 2)
return nameParts[0]
}
// Copied from https://github.com/containers/image/pkg/docker/config/config.go
func normalizeRegistry(registry string) string {
normalized := convertToHostname(registry)
switch normalized {
case "registry-1.docker.io", "docker.io":
return "index.docker.io"
}
return normalized
}

97
pkg/registry/digest/digest_test.go
View file

@ -1,64 +1,95 @@
package digest
import (
"context"
"fmt"
"github.com/containrrr/watchtower/pkg/logger"
"github.com/containrrr/watchtower/pkg/registry/auth"
wtTypes "github.com/containrrr/watchtower/pkg/types"
"github.com/docker/docker/api/types"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
log "github.com/sirupsen/logrus"
"os"
"testing"
)
func TestDigest(t *testing.T) {
log.SetLevel(log.DebugLevel)
RegisterFailHandler(Fail)
RunSpecs(t, "Digest Suite")
}
var image = types.ImageInspect{
var ghImage = types.ImageInspect{
ID: "sha256:6972c414f322dfa40324df3c503d4b217ccdec6d576e408ed10437f508f4181b",
RepoTags: []string {
RepoTags: []string{
"ghcr.io/k6io/operator:latest",
},
RepoDigests: []string {
RepoDigests: []string{
"ghcr.io/k6io/operator@sha256:d68e1e532088964195ad3a0a71526bc2f11a78de0def85629beb75e2265f0547",
},
}
var (
DH_USERNAME = os.Getenv("CI_INTEGRATION_TEST_REGISTRY_DH_USERNAME")
DH_PASSWORD = os.Getenv("CI_INTEGRATION_TEST_REGISTRY_DH_PASSWORD")
GH_USERNAME = os.Getenv("CI_INTEGRATION_TEST_REGISTRY_DH_USERNAME")
GH_PASSWORD = os.Getenv("CI_INTEGRATION_TEST_REGISTRY_DH_PASSWORD")
)
var DockerHubCredentials = &wtTypes.RegistryCredentials{
Username: os.Getenv("CI_INTEGRATION_TEST_REGISTRY_DH_USERNAME"),
Password: os.Getenv("CI_INTEGRATION_TEST_REGISTRY_DH_PASSWORD"),
}
var GHCRCredentials = &wtTypes.RegistryCredentials{
Username: os.Getenv("CI_INTEGRATION_TEST_REGISTRY_GH_USERNAME"),
Password: os.Getenv("CI_INTEGRATION_TEST_REGISTRY_GH_PASSWORD"),
}
func SkipIfCredentialsEmpty(credentials *wtTypes.RegistryCredentials, fn func()) func() {
if credentials.Username == "" {
return func() {
Skip("Username missing. Skipping integration test")
}
} else if credentials.Password == "" {
return func() {
Skip("Password missing. Skipping integration test")
}
} else {
return fn
}
}
var _ = Describe("Digests", func() {
When("fetching a bearer token", func() {
It("should parse the token from the response", func() {
token, err := GetToken(image, DH_USERNAME, DH_PASSWORD)
Expect(err).NotTo(HaveOccurred())
Expect(token).NotTo(Equal(""))
})
})
When("a digest comparison is done", func() {
It("should return true if digests match", func() {
matches, err := CompareDigest(image, DH_USERNAME, DH_PASSWORD)
Expect(err).NotTo(HaveOccurred())
Expect(matches).To(Equal(true))
})
It("should return false if digests differ", func() {
When("fetching a bearer token", func() {
It("should parse the token from the response",
SkipIfCredentialsEmpty(GHCRCredentials, func() {
ctx := context.Background()
logger.AddDebugLogger(ctx)
token, err := auth.GetToken(ctx, ghImage, GHCRCredentials)
Expect(err).NotTo(HaveOccurred())
Expect(token).NotTo(Equal(""))
}),
)
})
It("should return an error if the registry isn't available", func() {
When("a digest comparison is done", func() {
It("should return true if digests match",
SkipIfCredentialsEmpty(GHCRCredentials, func() {
ctx := context.Background()
logger.AddDebugLogger(ctx)
matches, err := CompareDigest(ctx, ghImage, GHCRCredentials)
Expect(err).NotTo(HaveOccurred())
Expect(matches).To(Equal(true))
}),
)
})
})
When("using different registries", func() {
It("should work with DockerHub", func() {
It("should return false if digests differ", func() {
})
It("should work with GitHub Container Registry", func() {
})
It("should return an error if the registry isn't available", func() {
})
})
When("using different registries", func() {
It("should work with DockerHub", func() {
})
It("should work with GitHub Container Registry",
SkipIfCredentialsEmpty(GHCRCredentials, func() {
fmt.Println(GHCRCredentials != nil) // to avoid crying linters
}),
)
})
})
})

38
pkg/registry/helpers/helpers.go Normal file
View file

@ -0,0 +1,38 @@
package helpers
import (
"fmt"
url2 "net/url"
)
// ConvertToHostname strips a url from everything but the hostname part
func ConvertToHostname(url string) (string, string, error) {
urlWithSchema := fmt.Sprintf("x://%s", url)
u, err := url2.Parse(urlWithSchema)
if err != nil {
return "", "", err
}
fmt.Println(url, err)
hostName := u.Hostname()
port := u.Port()
return hostName, port, err
}
// NormalizeRegistry makes sure variations of DockerHubs registry
func NormalizeRegistry(registry string) (string, error) {
hostName, port, err := ConvertToHostname(registry)
if err != nil {
return "", err
}
if hostName == "registry-1.docker.io" || hostName == "docker.io" {
hostName = "index.docker.io"
}
if port != "" {
return fmt.Sprintf("%s:%s", hostName, port), nil
}
return hostName, nil
}

33
pkg/registry/helpers/helpers_test.go Normal file
View file

@ -0,0 +1,33 @@
package helpers
import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
log "github.com/sirupsen/logrus"
"testing"
)
func TestDigest(t *testing.T) {
log.SetLevel(log.DebugLevel)
RegisterFailHandler(Fail)
RunSpecs(t, "Digest Suite")
}
var _ = Describe("the helpers", func() {
When("converting an url to a hostname", func() {
It("should return docker.io given docker.io/containrrr/watchtower:latest", func() {
host, port, err := ConvertToHostname("docker.io/containrrr/watchtower:latest")
Expect(err).NotTo(HaveOccurred())
Expect(host).To(Equal("docker.io"))
Expect(port).To(BeEmpty())
})
})
When("normalizing the registry information", func() {
It("should return index.docker.io given docker.io", func() {
out, err := NormalizeRegistry("docker.io/containrrr/watchtower:latest")
Expect(err).NotTo(HaveOccurred())
Expect(out).To(Equal("index.docker.io"))
})
})
})

37
pkg/registry/manifest/manifest.go Normal file
View file

@ -0,0 +1,37 @@
package manifest
import (
"fmt"
ref "github.com/containers/image/v5/docker/reference"
"github.com/containrrr/watchtower/pkg/registry/helpers"
apiTypes "github.com/docker/docker/api/types"
url2 "net/url"
"strings"
)
// BuildManifestURL from raw image data
func BuildManifestURL(image apiTypes.ImageInspect) (string, error) {
parts := strings.Split(image.RepoTags[0], ":")
img := parts[0]
tag := parts[1]
hostName, err := ref.ParseNormalizedNamed(img)
fmt.Println(hostName)
if err != nil {
return "", err
}
host, err := helpers.NormalizeRegistry(hostName.Name())
if err != nil {
return "", err
}
img = strings.TrimPrefix(img, host)
url := url2.URL{
Scheme: "https",
Host: host,
Path: fmt.Sprintf("/v2/%s/manifests/%s", img, tag),
}
return url.String(), nil
}

6
pkg/types/registry_credentials.go Normal file
View file

@ -0,0 +1,6 @@
package types
type RegistryCredentials struct {
Username string
Password string // usually a token rather than an actual password
}

5
pkg/types/token_response.go Normal file
View file

@ -0,0 +1,5 @@
package types
type TokenResponse struct {
Token string `json:"token"`
}