Skip to content

Commit

Permalink
Merge pull request #416 from networkop/master
Browse files Browse the repository at this point in the history 
Moved docker behind a new runtime interface
  • Loading branch information
@hellt
hellt authored May 11, 2021
2 parents 169c8bd + 70351f8 commit b9c0c26
Show file tree
Hide file tree
Showing 44 changed files with 1,550 additions and 557 deletions.
25 changes: 13 additions & 12 deletions clab/ceos.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,20 +10,21 @@ import (
"strings"
"time"

"github.com/docker/docker/api/types"
log "github.com/sirupsen/logrus"
"github.com/srl-labs/containerlab/types"
"github.com/srl-labs/containerlab/utils"
)

func ceosPostDeploy(ctx context.Context, c *CLab, node *Node, lworkers uint) error {
func ceosPostDeploy(ctx context.Context, c *CLab, node *types.Node, lworkers uint) error {
// regenerate ceos config since it is now known which IP address docker assigned to this container
err := node.generateConfig(node.ResConfig)
err := node.GenerateConfig(node.ResConfig, defaultConfigTemplates[node.Kind])
if err != nil {
return err
}
log.Infof("Restarting '%s' node", node.ShortName)
// force stopping and start is faster than ContainerRestart
var timeout time.Duration = 1
err = c.DockerClient.ContainerStop(ctx, node.ContainerID, &timeout)
err = c.Runtime.StopContainer(ctx, node.ContainerID, &timeout)
if err != nil {
return err
}
Expand All @@ -32,26 +33,26 @@ func ceosPostDeploy(ctx context.Context, c *CLab, node *Node, lworkers uint) err
if err := deleteNetnsSymlink(node.LongName); err != nil {
return err
}
err = c.DockerClient.ContainerStart(ctx, node.ContainerID, types.ContainerStartOptions{})
err = c.Runtime.StartContainer(ctx, node.ContainerID)
if err != nil {
return err
}
// since container has been restarted, we need to get its new NSPath and link netns
cont, err := c.DockerClient.ContainerInspect(ctx, node.ContainerID)
cont, err := c.Runtime.ContainerInspect(ctx, node.ContainerID)
if err != nil {
return err
}
log.Debugf("node %s new pid %v", node.LongName, cont.State.Pid)
node.NSPath = "/proc/" + strconv.Itoa(cont.State.Pid) + "/ns/net"
err = linkContainerNS(node.NSPath, node.LongName)
log.Debugf("node %s new pid %v", node.LongName, cont.Pid)
node.NSPath = "/proc/" + strconv.Itoa(cont.Pid) + "/ns/net"
err = utils.LinkContainerNS(node.NSPath, node.LongName)
if err != nil {
return err
}

return err
}

func initCeosNode(c *CLab, nodeCfg NodeConfig, node *Node, user string, envs map[string]string) error {
func initCeosNode(c *CLab, nodeCfg NodeConfig, node *types.Node, user string, envs map[string]string) error {
var err error

// initialize the global parameters with defaults, can be overwritten later
Expand Down Expand Up @@ -98,9 +99,9 @@ func initCeosNode(c *CLab, nodeCfg NodeConfig, node *Node, user string, envs map
return err
}

func (c *CLab) createCEOSFiles(node *Node) error {
func (c *CLab) createCEOSFiles(node *types.Node) error {
// generate config directory
CreateDirectory(path.Join(node.LabDir, "flash"), 0777)
utils.CreateDirectory(path.Join(node.LabDir, "flash"), 0777)
cfg := path.Join(node.LabDir, "flash", "startup-config")
node.ResConfig = cfg

Expand Down
12 changes: 7 additions & 5 deletions clab/cert.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@ import (
"github.com/cloudflare/cfssl/signer"
"github.com/cloudflare/cfssl/signer/universal"
log "github.com/sirupsen/logrus"
"github.com/srl-labs/containerlab/types"
"github.com/srl-labs/containerlab/utils"
)

type Certificates struct {
Expand Down Expand Up @@ -99,7 +101,7 @@ var nodeCSRTempl string = `{
func (c *CLab) GenerateRootCa(csrRootJsonTpl *template.Template, input CaRootInput) (*Certificates, error) {
log.Info("Creating root CA")
// create root CA root directory
CreateDirectory(c.Dir.LabCARoot, 0755)
utils.CreateDirectory(c.Dir.LabCARoot, 0755)
var err error
csrBuff := new(bytes.Buffer)
err = csrRootJsonTpl.Execute(csrBuff, input)
Expand Down Expand Up @@ -134,7 +136,7 @@ func (c *CLab) GenerateCert(ca string, caKey string, csrJSONTpl *template.Templa
c.m.RLock()
defer c.m.RUnlock()

CreateDirectory(targetPath, 0755)
utils.CreateDirectory(targetPath, 0755)
var err error
csrBuff := new(bytes.Buffer)
err = csrJSONTpl.Execute(csrBuff, input)
Expand Down Expand Up @@ -196,7 +198,7 @@ func (c *CLab) GenerateCert(ca string, caKey string, csrJSONTpl *template.Templa

// RetrieveNodeCertData reads the node private key and certificate by the well known paths
// if either of those files doesn't exist, an error is returned
func (c *CLab) RetrieveNodeCertData(n *Node) (*Certificates, error) {
func (c *CLab) RetrieveNodeCertData(n *types.Node) (*Certificates, error) {
var nodeCertFilesDir = path.Join(c.Dir.LabCA, n.ShortName)
var nodeCertFile = path.Join(nodeCertFilesDir, n.ShortName+".pem")
var nodeKeyFile = path.Join(nodeCertFilesDir, n.ShortName+"-key.pem")
Expand All @@ -210,12 +212,12 @@ func (c *CLab) RetrieveNodeCertData(n *Node) (*Certificates, error) {
return nil, err
}

certs.Cert, err = readFileContent(nodeCertFile)
certs.Cert, err = utils.ReadFileContent(nodeCertFile)
if err != nil {
return nil, err
}

certs.Key, err = readFileContent(nodeKeyFile)
certs.Key, err = utils.ReadFileContent(nodeKeyFile)
if err != nil {
return nil, err
}
Expand Down
101 changes: 57 additions & 44 deletions clab/clab.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,19 +10,20 @@ import (
"time"

"github.com/containernetworking/plugins/pkg/ns"
"github.com/docker/docker/api/types"
docker "github.com/docker/docker/client"
log "github.com/sirupsen/logrus"
"github.com/srl-labs/containerlab/runtime"
"github.com/srl-labs/containerlab/types"
"github.com/srl-labs/containerlab/utils"
)

type CLab struct {
Config *Config
TopoFile *TopoFile
m *sync.RWMutex
Nodes map[string]*Node
Links map[int]*Link
DockerClient *docker.Client
Dir *Directory
Config *Config
TopoFile *TopoFile
m *sync.RWMutex
Nodes map[string]*types.Node
Links map[int]*types.Link
Runtime runtime.ContainerRuntime
Dir *Directory

debug bool
timeout time.Duration
Expand Down Expand Up @@ -50,13 +51,10 @@ func WithTimeout(dur time.Duration) ClabOption {
}
}

func WithEnvDockerClient() ClabOption {
func WithRuntime(name string, d bool, dur time.Duration, gracefulShutdown bool) ClabOption {
return func(c *CLab) {
var err error
c.DockerClient, err = docker.NewEnvClient()
if err != nil {
log.Fatalf("failed to create docker client: %v", err)
}
c.Runtime = runtime.NewRuntime(name, d, dur, gracefulShutdown)
c.Runtime.SetMgmtNet(c.Config.Mgmt)
}
}

Expand All @@ -68,6 +66,12 @@ func WithTopoFile(file string) ClabOption {
if err := c.GetTopology(file); err != nil {
log.Fatalf("failed to read topology file: %v", err)
}

err := c.initMgmtNetwork()
if err != nil {
log.Fatalf("initMgmtNetwork: %s", err)
}

}
}

Expand All @@ -83,16 +87,40 @@ func NewContainerLab(opts ...ClabOption) *CLab {
Config: new(Config),
TopoFile: new(TopoFile),
m: new(sync.RWMutex),
Nodes: make(map[string]*Node),
Links: make(map[int]*Link),
Nodes: make(map[string]*types.Node),
Links: make(map[int]*types.Link),
}

for _, o := range opts {
o(c)
}

return c
}

func (c *CLab) CreateNode(ctx context.Context, node *Node, certs *Certificates) error {
// initMgmtNetwork sets management network config
func (c *CLab) initMgmtNetwork() error {
if c.Config.Mgmt.Network == "" {
c.Config.Mgmt.Network = dockerNetName
}
if c.Config.Mgmt.IPv4Subnet == "" && c.Config.Mgmt.IPv6Subnet == "" {
c.Config.Mgmt.IPv4Subnet = dockerNetIPv4Addr
c.Config.Mgmt.IPv6Subnet = dockerNetIPv6Addr
}

// init docker network mtu
if c.Config.Mgmt.MTU == "" {
m, err := utils.DefaultNetMTU()
if err != nil {
log.Warnf("Error occurred during getting the default docker MTU: %v", err)
}
c.Config.Mgmt.MTU = m
}

return nil
}

func (c *CLab) CreateNode(ctx context.Context, node *types.Node, certs *Certificates) error {
if certs != nil {
c.m.Lock()
node.TLSCert = string(certs.Cert)
Expand All @@ -103,23 +131,17 @@ func (c *CLab) CreateNode(ctx context.Context, node *Node, certs *Certificates)
if err != nil {
return err
}
return c.CreateContainer(ctx, node)
return c.Runtime.CreateContainer(ctx, node)
}

// ExecPostDeployTasks executes tasks that some nodes might require to boot properly after start
func (c *CLab) ExecPostDeployTasks(ctx context.Context, node *Node, lworkers uint) error {
func (c *CLab) ExecPostDeployTasks(ctx context.Context, node *types.Node, lworkers uint) error {
switch node.Kind {
case "ceos":
log.Debugf("Running postdeploy actions for Arista cEOS '%s' node", node.ShortName)
return ceosPostDeploy(ctx, c, node, lworkers)
case "crpd":
// exec `service ssh restart` to start ssh service and take into account mounted sshd_config
execConfig := types.ExecConfig{Tty: false, AttachStdout: false, AttachStderr: false, Cmd: strings.Fields("service ssh restart")}
respID, err := c.DockerClient.ContainerExecCreate(context.Background(), node.ContainerID, execConfig)
if err != nil {
return err
}
_, err = c.DockerClient.ContainerExecAttach(context.Background(), respID.ID, execConfig)
_, _, err := c.Runtime.Exec(ctx, node.ContainerID, []string{"service ssh restart"})
if err != nil {
return err
}
Expand All @@ -132,25 +154,16 @@ func (c *CLab) ExecPostDeployTasks(ctx context.Context, node *Node, lworkers uin
log.Debugf("Running postdeploy actions for sonic-vs '%s' node", node.ShortName)
// TODO: change this calls to c.ExecNotWait
// exec `supervisord` to start sonic services
execConfig := types.ExecConfig{Tty: false, AttachStdout: false, AttachStderr: false, Cmd: strings.Fields("supervisord")}
respID, err := c.DockerClient.ContainerExecCreate(context.Background(), node.ContainerID, execConfig)
if err != nil {
return err
}
_, err = c.DockerClient.ContainerExecAttach(context.Background(), respID.ID, execConfig)
_, _, err := c.Runtime.Exec(ctx, node.ContainerID, []string{"supervisord"})
if err != nil {
return err
}
// exec `/usr/lib/frr/bgpd` to start BGP service
execConfig = types.ExecConfig{Tty: false, AttachStdout: false, AttachStderr: false, Cmd: strings.Fields("/usr/lib/frr/bgpd")}
respID, err = c.DockerClient.ContainerExecCreate(context.Background(), node.ContainerID, execConfig)
if err != nil {
return err
}
_, err = c.DockerClient.ContainerExecAttach(context.Background(), respID.ID, execConfig)

_, _, err = c.Runtime.Exec(ctx, node.ContainerID, []string{"/usr/lib/frr/bgpd"})
if err != nil {
return err
}

case "mysocketio":
log.Debugf("Running postdeploy actions for mysocketio '%s' node", node.ShortName)
err := disableTxOffload(node)
Expand All @@ -168,7 +181,7 @@ func (c *CLab) ExecPostDeployTasks(ctx context.Context, node *Node, lworkers uin
func (c *CLab) CreateNodes(ctx context.Context, workers uint) {
wg := new(sync.WaitGroup)
wg.Add(int(workers))
nodesChan := make(chan *Node)
nodesChan := make(chan *types.Node)
// start workers
for i := uint(0); i < workers; i++ {
go func(i uint) {
Expand Down Expand Up @@ -243,7 +256,7 @@ func (c *CLab) CreateNodes(ctx context.Context, workers uint) {
func (c *CLab) CreateLinks(ctx context.Context, workers uint, postdeploy bool) {
wg := new(sync.WaitGroup)
wg.Add(int(workers))
linksChan := make(chan *Link)
linksChan := make(chan *types.Link)

log.Debug("creating links...")
// wire the links between the nodes based on cabling plan
Expand Down Expand Up @@ -291,7 +304,7 @@ func (c *CLab) CreateLinks(ctx context.Context, workers uint, postdeploy bool) {
wg.Wait()
}

func disableTxOffload(n *Node) error {
func disableTxOffload(n *types.Node) error {
// skip this if node runs in host mode
if strings.ToLower(n.NetworkMode) == "host" {
return nil
Expand All @@ -303,7 +316,7 @@ func disableTxOffload(n *Node) error {
}
err = nodeNS.Do(func(_ ns.NetNS) error {
// disabling offload on lo0 interface
err := EthtoolTXOff("eth0")
err := utils.EthtoolTXOff("eth0")
if err != nil {
log.Infof("Failed to disable TX checksum offload for 'eth0' interface for Linux '%s' node: %v", n.ShortName, err)
}
Expand Down
Loading

0 comments on commit b9c0c26

Please sign in to comment.