nv-ipam CNI plugin

Dockerfile

@@ -24,4 +24,5 @@ FROM gcr.io/distroless/base-debian11:latest
 WORKDIR /
 COPY --from=builder /workspace/build/ipam-controller .
 COPY --from=builder /workspace/build/ipam-node .
+COPY --from=builder /workspace/build/nv-ipam .
 COPY --from=builder /workspace/plugins/plugins/bin/host-local .

Makefile

@@ -87,8 +87,12 @@ build-controller: ## build IPAM controller
 build-node: ## build IPAM node
 	$(GO_BUILD_OPTS) go build -ldflags $(GO_LDFLAGS) -o $(BUILD_DIR)/ipam-node ./cmd/ipam-node/main.go
 
+.PHONY: build-cni
+build-cni: ## build IPAM cni
+	$(GO_BUILD_OPTS) go build -ldflags $(GO_LDFLAGS) -o $(BUILD_DIR)/nv-ipam ./cmd/nv-ipam/main.go
+
 .PHONY: build
-build: build-controller build-node ## Build project binaries
+build: build-controller build-node build-cni ## Build project binaries
 
 
 .PHONY: docker-build

cmd/nv-ipam/main.go

@@ -0,0 +1,29 @@
+/*
+ Copyright 2023, NVIDIA CORPORATION & AFFILIATES
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+     http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+package main
+
+import (
+	"fmt"
+
+	"github.com/containernetworking/cni/pkg/skel"
+	"github.com/containernetworking/cni/pkg/version"
+
+	"github.com/Mellanox/nvidia-k8s-ipam/pkg/cni/plugin"
+)
+
+func main() {
+	p := plugin.NewPlugin()
+	about := fmt.Sprintf("%s CNI plugin %s", p.Name, p.Version)
+	skel.PluginMain(p.CmdAdd, p.CmdCheck, p.CmdDel, version.All, about)
+}

go.mod

@@ -3,9 +3,12 @@ module github.com/Mellanox/nvidia-k8s-ipam
 go 1.20
 
 require (
+	github.com/containernetworking/cni v1.1.2
+	github.com/containernetworking/plugins v1.2.0
 	github.com/go-logr/logr v1.2.4
-	github.com/onsi/ginkgo/v2 v2.6.0
-	github.com/onsi/gomega v1.24.1
+	github.com/k8snetworkplumbingwg/cni-log v0.0.0-20230321145726-634c593dd11f
+	github.com/onsi/ginkgo/v2 v2.6.1
+	github.com/onsi/gomega v1.24.2
 	github.com/spf13/cobra v1.7.0
 	github.com/spf13/pflag v1.0.5
 	k8s.io/api v0.26.4
@@ -21,6 +24,7 @@ require (
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/blang/semver/v4 v4.0.0 // indirect
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
+	github.com/coreos/go-iptables v0.6.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/emicklei/go-restful/v3 v3.9.0 // indirect
 	github.com/evanphx/json-patch/v5 v5.6.0 // indirect
@@ -51,6 +55,9 @@ require (
 	github.com/prometheus/client_model v0.3.0 // indirect
 	github.com/prometheus/common v0.37.0 // indirect
 	github.com/prometheus/procfs v0.8.0 // indirect
+	github.com/safchain/ethtool v0.2.0 // indirect
+	github.com/vishvananda/netlink v1.2.1-beta.2 // indirect
+	github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f // indirect
 	go.uber.org/atomic v1.7.0 // indirect
 	go.uber.org/multierr v1.6.0 // indirect
 	go.uber.org/zap v1.24.0 // indirect
@@ -64,6 +71,7 @@ require (
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/protobuf v1.28.1 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/apiextensions-apiserver v0.26.1 // indirect

pkg/cni/k8sclient/k8sclient.go

@@ -0,0 +1,36 @@
+/*
+ Copyright 2023, NVIDIA CORPORATION & AFFILIATES
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+     http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+package k8sclient
+
+import (
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
+)
+
+// FromKubeconfig returns a kubernetes client created from provided kubeconfig path
+func FromKubeconfig(kubeconfigPath string) (*kubernetes.Clientset, error) {
+	cfg, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
+		&clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeconfigPath},
+		&clientcmd.ConfigOverrides{}).ClientConfig()
+	if err != nil {
+		return nil, err
+	}
+
+	k8sClient, err := kubernetes.NewForConfig(cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	return k8sClient, nil
+}

pkg/cni/plugin/plugin.go

@@ -0,0 +1,189 @@
+/*
+ Copyright 2023, NVIDIA CORPORATION & AFFILIATES
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+     http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+package plugin
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/containernetworking/cni/pkg/skel"
+	cnitypes "github.com/containernetworking/cni/pkg/types"
+	"github.com/containernetworking/plugins/pkg/ipam"
+	log "github.com/k8snetworkplumbingwg/cni-log"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+
+	"github.com/Mellanox/nvidia-k8s-ipam/pkg/cni/pool"
+	"github.com/Mellanox/nvidia-k8s-ipam/pkg/cni/types"
+	"github.com/Mellanox/nvidia-k8s-ipam/pkg/version"
+)
+
+const (
+	CNIPluginName = "nv-ipam"
+
+	delegateIPAMPluginName = "host-local"
+)
+
+func NewPlugin() *Plugin {
+	return &Plugin{
+		Name:    CNIPluginName,
+		Version: version.GetVersionString(),
+	}
+}
+
+type Plugin struct {
+	Name    string
+	Version string
+}
+
+func (p *Plugin) CmdAdd(args *skel.CmdArgs) error {
+	conf, err := types.LoadConf(args.StdinData)
+	if err != nil {
+		return fmt.Errorf("failed to load config. %w", err)
+	}
+	setupLog(conf.IPAM.LogFile, conf.IPAM.LogLevel)
+	logCall("ADD", args, conf.IPAM)
+
+	// build host-local config
+	pool, err := getPoolbyName(conf.IPAM.K8sClient, conf.IPAM.NodeName, conf.IPAM.PoolName)
+	if err != nil {
+		return fmt.Errorf("failed to get pool by name. %w", err)
+	}
+
+	hlc := types.HostLocalNetConfFromNetConfAndPool(conf, pool)
+	data, err := json.Marshal(hlc)
+	if err != nil {
+		return fmt.Errorf("failed to marshal host-local net conf. %w", err)
+	}
+	log.Debugf("host-local stdin data: %q", string(data))
+
+	// call host-local cni with alternate path
+	err = os.Setenv("CNI_PATH", filepath.Join(conf.IPAM.DataDir, "bin"))
+	if err != nil {
+		return err
+	}
+	res, err := ipam.ExecAdd(delegateIPAMPluginName, data)
+	if err != nil {
+		return fmt.Errorf("failed to exec ADD host-local CNI plugin. %w", err)
+	}
+
+	return cnitypes.PrintResult(res, conf.CNIVersion)
+}
+
+func (p *Plugin) CmdDel(args *skel.CmdArgs) error {
+	conf, err := types.LoadConf(args.StdinData)
+	if err != nil {
+		return fmt.Errorf("failed to load config. %w", err)
+	}
+	setupLog(conf.IPAM.LogFile, conf.IPAM.LogLevel)
+	logCall("DEL", args, conf.IPAM)
+
+	// build host-local config
+	pool, err := getPoolbyName(conf.IPAM.K8sClient, conf.IPAM.NodeName, conf.IPAM.PoolName)
+	if err != nil {
+		return fmt.Errorf("failed to get pool by name. %w", err)
+	}
+
+	hlc := types.HostLocalNetConfFromNetConfAndPool(conf, pool)
+	data, err := json.Marshal(hlc)
+	if err != nil {
+		return fmt.Errorf("failed to marshal host-local net conf. %w", err)
+	}
+	log.Debugf("host-local stdin data: %q", string(data))
+
+	// call host-local cni with alternate path
+	err = os.Setenv("CNI_PATH", filepath.Join(conf.IPAM.DataDir, "bin"))
+	if err != nil {
+		return err
+	}
+	err = ipam.ExecDel(delegateIPAMPluginName, data)
+	if err != nil {
+		return fmt.Errorf("failed to exec DEL host-local CNI plugin. %w", err)
+	}
+
+	return nil
+}
+
+func (p *Plugin) CmdCheck(args *skel.CmdArgs) error {
+	conf, err := types.LoadConf(args.StdinData)
+	if err != nil {
+		return fmt.Errorf("failed to load config. %w", err)
+	}
+	setupLog(conf.IPAM.LogFile, conf.IPAM.LogLevel)
+	logCall("CHECK", args, conf.IPAM)
+
+	// build host-local config
+	pool, err := getPoolbyName(conf.IPAM.K8sClient, conf.IPAM.NodeName, conf.IPAM.PoolName)
+	if err != nil {
+		return fmt.Errorf("failed to get pool by name. %w", err)
+	}
+
+	hlc := types.HostLocalNetConfFromNetConfAndPool(conf, pool)
+	data, err := json.Marshal(hlc)
+	if err != nil {
+		return fmt.Errorf("failed to marshal host-local net conf. %w", err)
+	}
+	log.Debugf("host-local stdin data: %q", string(data))
+
+	// call host-local cni with alternate path
+	err = os.Setenv("CNI_PATH", filepath.Join(conf.IPAM.DataDir, "bin"))
+	if err != nil {
+		return err
+	}
+	err = ipam.ExecCheck(delegateIPAMPluginName, data)
+	if err != nil {
+		return fmt.Errorf("failed to exec CHECK host-local CNI plugin. %w", err)
+	}
+
+	return nil
+}
+
+func setupLog(logFile, logLevel string) {
+	if logLevel != "" {
+		l := log.StringToLevel(logLevel)
+		log.SetLogLevel(l)
+	}
+
+	if logFile != "" {
+		log.SetLogFile(logFile)
+	}
+}
+
+func logCall(cmd string, args *skel.CmdArgs, conf *types.IPAMConf) {
+	log.Infof("CMD %s Call: ContainerID: %s Netns: %s IfName: %s", cmd, args.ContainerID, args.Netns, args.IfName)
+	log.Debugf("CMD %s: Args: %s StdinData: %q", cmd, args.Args, string(args.StdinData))
+	log.Debugf("CMD %s: Parsed IPAM conf: %+v", cmd, conf)
+}
+
+func getPoolbyName(kclient *kubernetes.Clientset, nodeName, poolName string) (*pool.IPPool, error) {
+	// get pool info from node
+	node, err := kclient.CoreV1().Nodes().Get(context.TODO(), nodeName, metav1.GetOptions{})
+	if err != nil {
+		return nil, fmt.Errorf("failed to get node %s from k8s API. %w", nodeName, err)
+	}
+
+	pm, err := pool.NewManagerImpl(node)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get pools from node %s. %w", nodeName, err)
+	}
+
+	pool := pm.GetPoolByName(poolName)
+	if pool == nil {
+		return nil, fmt.Errorf("failed to get pools from node %s. pool %s not found", nodeName, poolName)
+	}
+	return pool, nil
+}

pkg/cni/pool/pool.go

@@ -0,0 +1,72 @@
+/*
+ Copyright 2023, NVIDIA CORPORATION & AFFILIATES
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+     http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+package pool
+
+import (
+	"encoding/json"
+	"fmt"
+
+	v1 "k8s.io/api/core/v1"
+)
+
+const (
+	ipBlocksAnnotation = "ipam.nvidia.com/ip-blocks"
+)
+
+type IPPool struct {
+	Name    string
+	Subnet  string `json:"subnet"`
+	StartIP string `json:"startIP"`
+	EndIP   string `json:"endIP"`
+	Gateway string `json:"gateway"`
+}
+
+type Manager interface {
+	// GetPoolByName returns IPPool for the provided pool name or nil if pool doesnt exist
+	GetPoolByName(name string) *IPPool
+}
+
+type ManagerImpl struct {
+	poolByName map[string]*IPPool
+}
+
+func NewManagerImpl(node *v1.Node) (*ManagerImpl, error) {
+	if node == nil {
+		return nil, fmt.Errorf("nil node provided")
+	}
+
+	blocks, ok := node.Annotations[ipBlocksAnnotation]
+	if !ok {
+		return nil, fmt.Errorf("%s node annotation not found", ipBlocksAnnotation)
+	}
+
+	poolByName := make(map[string]*IPPool)
+	err := json.Unmarshal([]byte(blocks), &poolByName)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse %s annotation content. %w", ipBlocksAnnotation, err)
+	}
+
+	for poolName, pool := range poolByName {
+		pool.Name = poolName
+	}
+
+	return &ManagerImpl{
+		poolByName: poolByName,
+	}, nil
+}
+
+// GetPoolByName implements Manager interface
+func (pm *ManagerImpl) GetPoolByName(name string) *IPPool {
+	return pm.poolByName[name]
+}