Skip to content

Commit

Permalink
refactor: use template for buildkit config file (#1658)
Browse files Browse the repository at this point in the history 
* refactor: use template for buildkit config file

Signed-off-by: Keming <kemingyang@tensorchord.ai>

* format

Signed-off-by: Keming <kemingyang@tensorchord.ai>

* fix test

Signed-off-by: Keming <kemingyang@tensorchord.ai>

---------

Signed-off-by: Keming <kemingyang@tensorchord.ai>
  • Loading branch information
@kemingy
kemingy authored Jun 17, 2023
1 parent 45dffe7 commit fa7e989
Show file tree
Hide file tree
Showing 9 changed files with 178 additions and 91 deletions.
17 changes: 8 additions & 9 deletions pkg/app/bootstrap.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ import (
"github.com/tensorchord/envd/pkg/home"
sshconfig "github.com/tensorchord/envd/pkg/ssh/config"
"github.com/tensorchord/envd/pkg/types"
"github.com/tensorchord/envd/pkg/util/buildkitutil"
"github.com/tensorchord/envd/pkg/util/fileutil"
)

Expand Down Expand Up @@ -289,24 +290,22 @@ func buildkit(clicontext *cli.Context) error {

logrus.Debug("bootstrap the buildkitd container")
var bkClient buildkitd.Client
mirror := clicontext.String("dockerhub-mirror")
setRegistryCA := clicontext.IsSet("registry-ca-keypair")
useHTTP := clicontext.Bool("use-http")
registry := clicontext.String("registry")

if setRegistryCA && useHTTP {
return errors.New("cannot use both registry CA and HTTP")
config := buildkitutil.BuildkitConfig{
Registry: clicontext.String("registry"),
Mirror: clicontext.String("dockerhub-mirror"),
UseHTTP: clicontext.Bool("use-http"),
SetCA: clicontext.IsSet("registry-ca-keypair"),
}

if c.Builder == types.BuilderTypeMoby {
bkClient, err = buildkitd.NewMobyClient(clicontext.Context,
c.Builder, c.BuilderAddress, mirror, registry, setRegistryCA, useHTTP)
c.Builder, c.BuilderAddress, &config)
if err != nil {
return errors.Wrap(err, "failed to create moby buildkit client")
}
} else {
bkClient, err = buildkitd.NewClient(clicontext.Context,
c.Builder, c.BuilderAddress, mirror, registry, setRegistryCA, useHTTP)
c.Builder, c.BuilderAddress, &config)
if err != nil {
return errors.Wrap(err, "failed to create buildkit client")
}
Expand Down
4 changes: 2 additions & 2 deletions pkg/app/prune.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,13 +81,13 @@ func prune(clicontext *cli.Context) error {
var bkClient buildkitd.Client
if c.Builder == types.BuilderTypeMoby {
bkClient, err = buildkitd.NewMobyClient(clicontext.Context,
c.Builder, c.BuilderAddress, "", "", false, false)
c.Builder, c.BuilderAddress, nil)
if err != nil {
return errors.Wrap(err, "failed to create moby buildkit client")
}
} else {
bkClient, err = buildkitd.NewClient(clicontext.Context,
c.Builder, c.BuilderAddress, "", "", false, false)
c.Builder, c.BuilderAddress, nil)
if err != nil {
return errors.Wrap(err, "failed to create buildkit client")
}
Expand Down
6 changes: 4 additions & 2 deletions pkg/builder/build.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ import (
"github.com/tensorchord/envd/pkg/lang/version"
"github.com/tensorchord/envd/pkg/progress/progresswriter"
"github.com/tensorchord/envd/pkg/types"
"github.com/tensorchord/envd/pkg/util/buildkitutil"
)

func New(ctx context.Context, opt Options) (Builder, error) {
Expand Down Expand Up @@ -88,15 +89,16 @@ func New(ctx context.Context, opt Options) (Builder, error) {
}

var cli buildkitd.Client
bc := buildkitutil.BuildkitConfig{}
if c.Builder == types.BuilderTypeMoby {
cli, err = buildkitd.NewMobyClient(ctx,
c.Builder, c.BuilderAddress, "", "", false, false)
c.Builder, c.BuilderAddress, &bc)
if err != nil {
return nil, errors.Wrap(err, "failed to create moby buildkit client")
}
} else {
cli, err = buildkitd.NewClient(ctx,
c.Builder, c.BuilderAddress, "", "", false, false)
c.Builder, c.BuilderAddress, &bc)
if err != nil {
return nil, errors.Wrap(err, "failed to create buildkit client")
}
Expand Down
43 changes: 17 additions & 26 deletions pkg/buildkitd/buildkitd.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ import (
"github.com/tensorchord/envd/pkg/driver/nerdctl"
"github.com/tensorchord/envd/pkg/flag"
"github.com/tensorchord/envd/pkg/types"
"github.com/tensorchord/envd/pkg/util/buildkitutil"
"github.com/tensorchord/envd/pkg/util/envutil"
)

Expand All @@ -62,12 +63,9 @@ type Client interface {
}

type generalClient struct {
containerName string
image string
mirror string
registry string
enableRegistryCA bool
useHTTP bool
containerName string
image string
buildkitConfig *buildkitutil.BuildkitConfig

driver types.BuilderType
socket string
Expand All @@ -77,17 +75,14 @@ type generalClient struct {
}

func NewMobyClient(ctx context.Context, driver types.BuilderType,
socket, mirror, registry string, enableRegistryCA bool, useHTTP bool) (Client, error) {
socket string, config *buildkitutil.BuildkitConfig) (Client, error) {
logrus.Debug("getting moby buildkit client")
c := &generalClient{
containerName: socket,
image: viper.GetString(flag.FlagBuildkitdImage),
registry: registry,
mirror: mirror,
enableRegistryCA: enableRegistryCA,
useHTTP: useHTTP,
socket: socket,
driver: driver,
containerName: socket,
image: viper.GetString(flag.FlagBuildkitdImage),
buildkitConfig: config,
socket: socket,
driver: driver,
}
c.logger = logrus.WithFields(logrus.Fields{
"container": c.containerName,
Expand Down Expand Up @@ -115,16 +110,13 @@ func NewMobyClient(ctx context.Context, driver types.BuilderType,
}

func NewClient(ctx context.Context, driver types.BuilderType,
socket, mirror, registry string, enableRegistryCA bool, useHTTP bool) (Client, error) {
socket string, config *buildkitutil.BuildkitConfig) (Client, error) {
c := &generalClient{
containerName: socket,
image: viper.GetString(flag.FlagBuildkitdImage),
mirror: mirror,
registry: registry,
enableRegistryCA: enableRegistryCA,
useHTTP: useHTTP,
socket: socket,
driver: driver,
containerName: socket,
image: viper.GetString(flag.FlagBuildkitdImage),
buildkitConfig: config,
socket: socket,
driver: driver,
}
c.logger = logrus.WithFields(logrus.Fields{
"container": c.containerName,
Expand Down Expand Up @@ -179,8 +171,7 @@ func (c *generalClient) maybeStart(ctx context.Context,
}

if client != nil {
if _, err := client.StartBuildkitd(ctx, c.image, c.containerName, c.mirror, c.registry,
c.enableRegistryCA, c.useHTTP, runningTimeout); err != nil {
if _, err := client.StartBuildkitd(ctx, c.image, c.containerName, c.buildkitConfig, runningTimeout); err != nil {
return "", err
}
}
Expand Down
4 changes: 3 additions & 1 deletion pkg/driver/client.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,14 @@ import (
"time"

"github.com/docker/docker/api/types"

"github.com/tensorchord/envd/pkg/util/buildkitutil"
)

type Client interface {
// Load loads the image from the reader to the docker host.
Load(ctx context.Context, r io.ReadCloser, quiet bool) error
StartBuildkitd(ctx context.Context, tag, name, mirror, registry string, enableRegistryCA, useHTTP bool, timeout time.Duration) (string, error)
StartBuildkitd(ctx context.Context, tag, name string, config *buildkitutil.BuildkitConfig, timeout time.Duration) (string, error)

Exec(ctx context.Context, cname string, cmd []string) error

Expand Down
57 changes: 16 additions & 41 deletions pkg/driver/docker/docker.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,30 +36,11 @@ import (
"github.com/sirupsen/logrus"

"github.com/tensorchord/envd/pkg/driver"
"github.com/tensorchord/envd/pkg/util/buildkitutil"
"github.com/tensorchord/envd/pkg/util/fileutil"
)

const buildkitdMirror = `
[registry."docker.io"]
mirrors = ["%s"]
`
const buildkitdHTTP = `
[registry."docker.io"]
http = true
`
const buildkitdRegistry = `
[registry."%s"]
http = %v
`
const buildkitdCertPath = "/etc/registry"
const buildkitdWithCA = `
[registry."docker.io"]
mirrors = ["%s"]
ca=["/etc/registry/ca.pem"]
[[registry."docker.io".keypair]]
key="/etc/registry/key.pem"
cert="/etc/registry/cert.pem"
`

var (
anchoredIdentifierRegexp = regexp.MustCompile(`^([a-f0-9]{64})$`)
Expand Down Expand Up @@ -193,12 +174,11 @@ func (c dockerClient) ResumeContainer(ctx context.Context, name string) (string,
return name, nil
}

func (c dockerClient) StartBuildkitd(ctx context.Context, tag, name, mirror, registry string,
enableRegistryCA, useHTTP bool, timeout time.Duration) (string, error) {
func (c dockerClient) StartBuildkitd(ctx context.Context, tag, name string, bc *buildkitutil.BuildkitConfig, timeout time.Duration) (string, error) {
logger := logrus.WithFields(logrus.Fields{
"tag": tag,
"container": name,
"mirror": mirror,
"tag": tag,
"container": name,
"buildkit-config": bc,
})
logger.Debug("starting buildkitd")
if _, _, err := c.ImageInspectWithRaw(ctx, tag); err != nil {
Expand Down Expand Up @@ -226,22 +206,17 @@ func (c dockerClient) StartBuildkitd(ctx context.Context, tag, name, mirror, reg
Privileged: true,
AutoRemove: true,
}
var cfg string
if mirror != "" {
if enableRegistryCA {
cfg = fmt.Sprintf(buildkitdWithCA, mirror)
hostConfig.Mounts = append(hostConfig.Mounts, mount.Mount{
Type: mount.TypeBind,
Source: fileutil.DefaultConfigDir,
Target: buildkitdCertPath,
})
} else {
cfg = fmt.Sprintf(buildkitdMirror, mirror)
}
} else if registry != "" {
cfg = fmt.Sprintf(buildkitdRegistry, registry, useHTTP)
} else if useHTTP {
cfg = buildkitdHTTP

if bc.SetCA {
hostConfig.Mounts = append(hostConfig.Mounts, mount.Mount{
Type: mount.TypeBind,
Source: fileutil.DefaultConfigDir,
Target: buildkitdCertPath,
})
}
cfg, err := bc.String()
if err != nil {
return "", errors.Wrap(err, "failed to generate buildkit config")
}
config.Entrypoint = []string{
"/bin/sh",
Expand Down
21 changes: 11 additions & 10 deletions pkg/driver/nerdctl/nerdctl.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import (
"github.com/sirupsen/logrus"

"github.com/tensorchord/envd/pkg/driver"
"github.com/tensorchord/envd/pkg/util/buildkitutil"
)

type nerdctlClient struct {
Expand Down Expand Up @@ -63,13 +64,12 @@ func (nc *nerdctlClient) Load(ctx context.Context, r io.ReadCloser, quiet bool)
return nil
}

func (nc *nerdctlClient) StartBuildkitd(ctx context.Context, tag, name, mirror, registry string,
enableRegistryCA, useHTTP bool, timeout time.Duration) (string, error) {
func (nc *nerdctlClient) StartBuildkitd(ctx context.Context, tag, name string, bc *buildkitutil.BuildkitConfig, timeout time.Duration) (string, error) {
logger := logrus.WithFields(logrus.Fields{
"tag": tag,
"container": name,
"mirror": mirror,
"driver": "nerdctl",
"tag": tag,
"container": name,
"buildkit-config": bc,
"driver": "nerdctl",
})
logger.Debug("starting buildkitd")

Expand All @@ -83,10 +83,11 @@ func (nc *nerdctlClient) StartBuildkitd(ctx context.Context, tag, name, mirror,
if !existed {
buildkitdCmd := "buildkitd"
// TODO: support mirror CA keypair
if mirror != "" {
cfg := fmt.Sprintf(`
[registry."docker.io"]
mirrors = ["%s"]`, mirror)
if bc.Registry != "" || bc.Mirror != "" || bc.UseHTTP {
cfg, err := bc.String()
if err != nil {
return "", errors.Wrap(err, "failed to generate buildkit config")
}
buildkitdCmd = fmt.Sprintf("mkdir /etc/buildkit && echo '%s' > /etc/buildkit/buildkitd.toml && buildkitd", cfg)
logger.Debugf("setting buildkit config: %s", cfg)
}
Expand Down
48 changes: 48 additions & 0 deletions pkg/util/buildkitutil/buildkit.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
// Copyright 2023 The envd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package buildkitutil

import (
"strings"
"text/template"
)

const buildkitConfigTemplate = `
[registry."{{ if .Registry }}{{ .Registry }}{{ else }}docker.io{{ end }}"]{{ if .Mirror }}
mirrors = ["{{ .Mirror }}"]{{ end }}
http = {{ .UseHTTP }}
{{ if .SetCA}}ca=["/etc/registry/ca.pem"]
[[registry."{{ if .Registry }}{{ .Registry }}{{ else }}docker.io{{ end }}".keypair]]
key="/etc/registry/key.pem"
cert="/etc/registry/cert.pem"
{{ end }}
`

type BuildkitConfig struct {
Registry string
Mirror string
UseHTTP bool
SetCA bool
}

func (c *BuildkitConfig) String() (string, error) {
tmpl, err := template.New("buildkitConfig").Parse(buildkitConfigTemplate)
if err != nil {
return "", err
}
var config strings.Builder
err = tmpl.Execute(&config, c)
return config.String(), err
}
Loading

0 comments on commit fa7e989

Please sign in to comment.