Skip to content

Commit

Permalink
Add prometheus metrics for dockerregistry
Browse files Browse the repository at this point in the history
Signed-off-by: Gladkov Alexey <agladkov@redhat.com>
  • Loading branch information
legionus committed Apr 5, 2017
1 parent 367a8d3 commit 998f7c8
Show file tree
Hide file tree
Showing 21 changed files with 660 additions and 29 deletions.
5 changes: 5 additions & 0 deletions images/dockerregistry/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -35,3 +35,8 @@ middleware:
blobrepositorycachettl: 10m
storage:
- name: openshift
openshift:
version: 1.0
metrics:
enabled: false
secret: TopSecretToken
55 changes: 31 additions & 24 deletions pkg/cmd/dockerregistry/dockerregistry.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ import (
"github.com/docker/distribution/configuration"
"github.com/docker/distribution/context"
"github.com/docker/distribution/health"
"github.com/docker/distribution/reference"
"github.com/docker/distribution/registry/auth"
"github.com/docker/distribution/registry/handlers"
"github.com/docker/distribution/uuid"
Expand All @@ -41,20 +40,23 @@ import (
"github.com/openshift/origin/pkg/cmd/server/crypto"
"github.com/openshift/origin/pkg/cmd/util/clientcmd"
"github.com/openshift/origin/pkg/dockerregistry/server"
"github.com/openshift/origin/pkg/dockerregistry/server/api"
"github.com/openshift/origin/pkg/dockerregistry/server/audit"
registryconfig "github.com/openshift/origin/pkg/dockerregistry/server/configuration"
)

// Execute runs the Docker registry.
func Execute(configFile io.Reader) {
config, err := configuration.Parse(configFile)
dockerConfig, extraConfig, err := registryconfig.Parse(configFile)
if err != nil {
log.Fatalf("error parsing configuration file: %s", err)
}
setDefaultMiddleware(config)
setDefaultLogParameters(config)
setDefaultMiddleware(dockerConfig)
setDefaultLogParameters(dockerConfig)

ctx := context.Background()
ctx, err = configureLogging(ctx, config)
ctx = server.WithConfiguration(ctx, extraConfig)
ctx, err = configureLogging(ctx, dockerConfig)
if err != nil {
log.Fatalf("error configuring logger: %v", err)
}
Expand All @@ -68,33 +70,33 @@ func Execute(configFile io.Reader) {
uuid.Loggerf = context.GetLogger(ctx).Warnf

// add parameters for the auth middleware
if config.Auth.Type() == server.OpenShiftAuth {
if config.Auth[server.OpenShiftAuth] == nil {
config.Auth[server.OpenShiftAuth] = make(configuration.Parameters)
if dockerConfig.Auth.Type() == server.OpenShiftAuth {
if dockerConfig.Auth[server.OpenShiftAuth] == nil {
dockerConfig.Auth[server.OpenShiftAuth] = make(configuration.Parameters)
}
config.Auth[server.OpenShiftAuth][server.AccessControllerOptionParams] = server.AccessControllerParams{
dockerConfig.Auth[server.OpenShiftAuth][server.AccessControllerOptionParams] = server.AccessControllerParams{
Logger: context.GetLogger(ctx),
SafeClientConfig: registryClient.SafeClientConfig(),
}
}

app := handlers.NewApp(ctx, config)
app := handlers.NewApp(ctx, dockerConfig)

// Add a token handling endpoint
if options, usingOpenShiftAuth := config.Auth[server.OpenShiftAuth]; usingOpenShiftAuth {
if options, usingOpenShiftAuth := dockerConfig.Auth[server.OpenShiftAuth]; usingOpenShiftAuth {
tokenRealm, err := server.TokenRealm(options)
if err != nil {
log.Fatalf("error setting up token auth: %s", err)
context.GetLogger(app).Fatalf("error setting up token auth: %s", err)
}
err = app.NewRoute().Methods("GET").PathPrefix(tokenRealm.Path).Handler(server.NewTokenHandler(ctx, registryClient)).GetError()
if err != nil {
log.Fatalf("error setting up token endpoint at %q: %v", tokenRealm.Path, err)
context.GetLogger(app).Fatalf("error setting up token endpoint at %q: %v", tokenRealm.Path, err)
}
log.Debugf("configured token endpoint at %q", tokenRealm.String())
context.GetLogger(app).Debugf("configured token endpoint at %q", tokenRealm.String())
}

// TODO add https scheme
adminRouter := app.NewRoute().PathPrefix("/admin/").Subrouter()
adminRouter := app.NewRoute().PathPrefix(api.AdminPrefix).Subrouter()
pruneAccessRecords := func(*http.Request) []auth.Access {
return []auth.Access{
{
Expand All @@ -108,7 +110,7 @@ func Execute(configFile io.Reader) {

app.RegisterRoute(
// DELETE /admin/blobs/<digest>
adminRouter.Path("/blobs/{digest:"+reference.DigestRegexp.String()+"}").Methods("DELETE"),
adminRouter.Path(api.AdminPath).Methods("DELETE"),
// handler
server.BlobDispatcher,
// repo name not required in url
Expand All @@ -121,6 +123,11 @@ func Execute(configFile io.Reader) {
// signatures.
server.RegisterSignatureHandler(app)

// Registry extensions endpoint provides prometheus metrics.
if extraConfig.Metrics.Enabled {
server.RegisterMetricHandler(app)
}

// Advertise features supported by OpenShift
if app.Config.HTTP.Headers == nil {
app.Config.HTTP.Headers = http.Header{}
Expand All @@ -135,9 +142,9 @@ func Execute(configFile io.Reader) {
handler = panicHandler(handler)
handler = gorillahandlers.CombinedLoggingHandler(os.Stdout, handler)

if config.HTTP.TLS.Certificate == "" {
context.GetLogger(app).Infof("listening on %v", config.HTTP.Addr)
if err := http.ListenAndServe(config.HTTP.Addr, handler); err != nil {
if dockerConfig.HTTP.TLS.Certificate == "" {
context.GetLogger(app).Infof("listening on %v", dockerConfig.HTTP.Addr)
if err := http.ListenAndServe(dockerConfig.HTTP.Addr, handler); err != nil {
context.GetLogger(app).Fatalln(err)
}
} else {
Expand Down Expand Up @@ -167,10 +174,10 @@ func Execute(configFile io.Reader) {
CipherSuites: cipherSuites,
})

if len(config.HTTP.TLS.ClientCAs) != 0 {
if len(dockerConfig.HTTP.TLS.ClientCAs) != 0 {
pool := x509.NewCertPool()

for _, ca := range config.HTTP.TLS.ClientCAs {
for _, ca := range dockerConfig.HTTP.TLS.ClientCAs {
caPem, err := ioutil.ReadFile(ca)
if err != nil {
context.GetLogger(app).Fatalln(err)
Expand All @@ -189,14 +196,14 @@ func Execute(configFile io.Reader) {
tlsConf.ClientCAs = pool
}

context.GetLogger(app).Infof("listening on %v, tls", config.HTTP.Addr)
context.GetLogger(app).Infof("listening on %v, tls", dockerConfig.HTTP.Addr)
server := &http.Server{
Addr: config.HTTP.Addr,
Addr: dockerConfig.HTTP.Addr,
Handler: handler,
TLSConfig: tlsConf,
}

if err := server.ListenAndServeTLS(config.HTTP.TLS.Certificate, config.HTTP.TLS.Key); err != nil {
if err := server.ListenAndServeTLS(dockerConfig.HTTP.TLS.Certificate, dockerConfig.HTTP.TLS.Key); err != nil {
context.GetLogger(app).Fatalln(err)
}
}
Expand Down
2 changes: 2 additions & 0 deletions pkg/dockerregistry/server/api/doc.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
// Package api describes routes and urls that extends the Registry JSON HTTP API.
package api
14 changes: 14 additions & 0 deletions pkg/dockerregistry/server/api/routes.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
package api

import (
"github.com/docker/distribution/reference"
)

var (
AdminPrefix = "/admin/"
ExtensionsPrefix = "/extensions/v2/"

AdminPath = "/blobs/{digest:" + reference.DigestRegexp.String() + "}"
SignaturesPath = "/{name:" + reference.NameRegexp.String() + "}/signatures/{digest:" + reference.DigestRegexp.String() + "}"
MetricsPath = "/metrics"
)
20 changes: 19 additions & 1 deletion pkg/dockerregistry/server/auth.go
Original file line number Diff line number Diff line change
Expand Up @@ -312,7 +312,7 @@ func (ac *AccessController) Authorized(ctx context.Context, accessRecords ...reg
}

// In case of docker login, hits endpoint /v2
if len(bearerToken) > 0 {
if len(bearerToken) > 0 && !isMetricsBearerToken(ctx, bearerToken) {
user, userid, err := verifyOpenShiftUser(ctx, osClient)
if err != nil {
return nil, ac.wrapErr(ctx, err)
Expand Down Expand Up @@ -393,6 +393,16 @@ func (ac *AccessController) Authorized(ctx context.Context, accessRecords ...reg
}
}

case "metrics":
switch access.Action {
case "get":
if !isMetricsBearerToken(ctx, bearerToken) {
return nil, ac.wrapErr(ctx, ErrOpenShiftAccessDenied)
}
default:
return nil, ac.wrapErr(ctx, ErrUnsupportedAction)
}

case "admin":
switch access.Action {
case "prune":
Expand Down Expand Up @@ -535,3 +545,11 @@ func verifyPruneAccess(ctx context.Context, client client.SubjectAccessReviews)
}
return nil
}

func isMetricsBearerToken(ctx context.Context, token string) bool {
config := ConfigurationFrom(ctx)
if config.Metrics.Enabled {
return config.Metrics.Secret == token
}
return false
}
3 changes: 3 additions & 0 deletions pkg/dockerregistry/server/auth_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@ import (
// install all APIs
_ "github.com/openshift/origin/pkg/api/install"
"github.com/openshift/origin/pkg/client"

"github.com/openshift/origin/pkg/dockerregistry/server/configuration"
)

// TestVerifyImageStreamAccess mocks openshift http request/response and
Expand Down Expand Up @@ -405,6 +407,7 @@ func TestAccessController(t *testing.T) {
}
ctx := context.Background()
ctx = context.WithRequest(ctx, req)
ctx = WithConfiguration(ctx, &configuration.Configuration{})
authCtx, err := accessController.Authorized(ctx, test.access...)
server.Close()

Expand Down
2 changes: 2 additions & 0 deletions pkg/dockerregistry/server/blobdescriptorservice_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
"github.com/docker/distribution/registry/middleware/registry"
"github.com/docker/distribution/registry/storage"

srvconfig "github.com/openshift/origin/pkg/dockerregistry/server/configuration"
registrytest "github.com/openshift/origin/pkg/dockerregistry/testutil"
kcoreclient "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/core/internalversion"
"k8s.io/kubernetes/pkg/client/restclient"
Expand Down Expand Up @@ -62,6 +63,7 @@ func TestBlobDescriptorServiceIsApplied(t *testing.T) {
client.AddReactor("get", "images", registrytest.GetFakeImageGetHandler(t, *testImage))

ctx := context.Background()
ctx = WithConfiguration(ctx, &srvconfig.Configuration{})
ctx = WithRegistryClient(ctx, makeFakeRegistryClient(client, nil))
app := handlers.NewApp(ctx, &configuration.Configuration{
Loglevel: "debug",
Expand Down
140 changes: 140 additions & 0 deletions pkg/dockerregistry/server/configuration/configuration.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,140 @@
package configuration

import (
"bytes"
"errors"
"io"
"io/ioutil"
"os"
"reflect"
"strings"

"gopkg.in/yaml.v2"

"github.com/docker/distribution/configuration"
)

var (
// CurrentVersion is the most recent Version that can be parsed.
CurrentVersion = configuration.MajorMinorVersion(1, 0)

ErrUnsupportedVersion = errors.New("Unsupported openshift configuration version")
)

type openshiftConfig struct {
Openshift Configuration
}

type Configuration struct {
Version configuration.Version `yaml:"version"`
Metrics Metrics `yaml:"metrics"`
}

type Metrics struct {
Enabled bool `yaml:"enabled"`
Secret string `yaml:"secret"`
}

type versionInfo struct {
Openshift struct {
Version *configuration.Version
}
}

// Parse parses an input configuration and returns docker configuration structure and
// openshift specific configuration.
// Environment variables may be used to override configuration parameters.
func Parse(rd io.Reader) (*configuration.Configuration, *Configuration, error) {
in, err := ioutil.ReadAll(rd)
if err != nil {
return nil, nil, err
}

// We don't want to change the version from the environment variables.
os.Unsetenv("REGISTRY_OPENSHIFT_VERSION")

openshiftEnv, err := popEnv("REGISTRY_OPENSHIFT_")
if err != nil {
return nil, nil, err
}

dockerConfig, err := configuration.Parse(bytes.NewBuffer(in))
if err != nil {
return nil, nil, err
}

dockerEnv, err := popEnv("REGISTRY_")
if err != nil {
return nil, nil, err
}
if err := pushEnv(openshiftEnv); err != nil {
return nil, nil, err
}

config := openshiftConfig{}

vInfo := &versionInfo{}
if err := yaml.Unmarshal(in, &vInfo); err != nil {
return nil, nil, err
}

if vInfo.Openshift.Version != nil {
if *vInfo.Openshift.Version != CurrentVersion {
return nil, nil, ErrUnsupportedVersion
}
} else {
return dockerConfig, &config.Openshift, nil
}

p := configuration.NewParser("registry", []configuration.VersionedParseInfo{
{
Version: dockerConfig.Version,
ParseAs: reflect.TypeOf(config),
ConversionFunc: func(c interface{}) (interface{}, error) {
return c, nil
},
},
})

if err = p.Parse(in, &config); err != nil {
return nil, nil, err
}
if err := pushEnv(dockerEnv); err != nil {
return nil, nil, err
}

return dockerConfig, &config.Openshift, nil
}

type envVar struct {
name string
value string
}

func popEnv(prefix string) ([]envVar, error) {
var envVars []envVar

for _, env := range os.Environ() {
if !strings.HasPrefix(env, prefix) {
continue
}
envParts := strings.SplitN(env, "=", 2)
err := os.Unsetenv(envParts[0])
if err != nil {
return nil, err
}

envVars = append(envVars, envVar{envParts[0], envParts[1]})
}

return envVars, nil
}

func pushEnv(environ []envVar) error {
for _, env := range environ {
if err := os.Setenv(env.name, env.value); err != nil {
return err
}
}
return nil
}
Loading

0 comments on commit 998f7c8

Please sign in to comment.