IPAM daemon implementation

.github/workflows/build-test-lint.yml

@@ -32,6 +32,18 @@ jobs:
         uses: actions/checkout@v3
       - name: Lint
         run: make lint
+  grpc-check:
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - name: Set up Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: 1.20.x
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v3
+      - name: Check GRPC
+        run: make grpc-check
   test:
     runs-on: ubuntu-latest
     needs: build

.gitignore

@@ -6,6 +6,7 @@
 *.dylib
 bin
 build/
+_tmp
 testbin/*
 .gocache
 

Dockerfile

@@ -14,9 +14,6 @@ COPY . /workspace
 
 # Build with make to apply all build logic defined in Makefile
 RUN make build
-# Build host-local cni
-RUN git clone https://github.com/containernetworking/plugins.git ; cd plugins ; git checkout v1.2.0 -b v1.2.0
-RUN cd plugins ; go build -o plugins/bin/host-local ./plugins/ipam/host-local
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details
@@ -25,4 +22,4 @@ WORKDIR /
 COPY --from=builder /workspace/build/ipam-controller .
 COPY --from=builder /workspace/build/ipam-node .
 COPY --from=builder /workspace/build/nv-ipam .
-COPY --from=builder /workspace/plugins/plugins/bin/host-local .
+

Makefile

@@ -59,6 +59,12 @@ all: build
 help: ## Display this help.
 	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf "  \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
 
+.PHONY: clean
+clean: ## Remove downloaded tools and compiled binaries
+	@rm -rf $(LOCALBIN)
+	@rm -rf $(BUILD_DIR)
+	@rm -rf $(GRPC_TMP_DIR)
+
 ##@ Development
 
 .PHONY: lint
@@ -80,7 +86,6 @@ test: lint unit-test
 cov-report: gcov2lcov unit-test  ## Build test coverage report in lcov format
 	$(GCOV2LCOV) -infile $(COVER_PROFILE) -outfile $(LCOV_PATH)
 
-
 ##@ Build
 
 .PHONY: build-controller
@@ -114,24 +119,40 @@ kind-load-image:  ## Load ipam image to kind cluster
 	kind load docker-image --name $(KIND_CLUSTER) $(IMG)
 
 .PHONY: generate-mocks
-generate-mocks:  ## generate mock objects
-	PATH=$(PATH):$(LOCALBIN) go generate ./...
+generate-mocks: mockery ## generate mock objects
+	PATH=$(LOCALBIN):$(PATH) go generate ./...
 
 ## Location to install dependencies to
 LOCALBIN ?= $(PROJECT_DIR)/bin
 $(LOCALBIN):
 	mkdir -p $(LOCALBIN)
 
+## Temporary location for GRPC files
+GRPC_TMP_DIR  ?= $(CURDIR)/_tmp
+$(GRPC_TMP_DIR):
+	@mkdir -p $@
+
+##@ Tools
+
 ## Tool Binaries
 ENVTEST ?= $(LOCALBIN)/setup-envtest
 GOLANGCILINT ?= $(LOCALBIN)/golangci-lint
 GCOV2LCOV ?= $(LOCALBIN)/gcov2lcov
 MOCKERY ?= $(LOCALBIN)/mockery
+PROTOC ?= $(LOCALBIN)/protoc/bin/protoc
+PROTOC_GEN_GO ?= $(LOCALBIN)/protoc-gen-go
+PROTOC_GEN_GO_GRPC ?= $(LOCALBIN)/protoc-gen-go-grpc
+BUF ?= $(LOCALBIN)/buf
 
 ## Tool Versions
 GOLANGCILINT_VERSION ?= v1.52.2
 GCOV2LCOV_VERSION ?= v1.0.5
 MOCKERY_VERSION ?= v2.27.1
+PROTOC_VER ?= 23.4
+PROTOC_GEN_GO_VER ?= 1.31.0
+PROTOC_GEN_GO_GRPC_VER ?= 1.3.0
+BUF_VERSION ?= 1.23.1
+
 
 .PHONY: envtest
 envtest: $(ENVTEST) ## Download envtest-setup locally if necessary.
@@ -153,7 +174,80 @@ mockery: $(MOCKERY) ## Download mockery locally if necessary.
 $(MOCKERY): | $(LOCALBIN)
 	GOBIN=$(LOCALBIN) go install github.com/vektra/mockery/v2@$(MOCKERY_VERSION)
 
-.PHONY: clean
-clean: ## Remove downloaded tools and compiled binaries
-	@rm -rf $(LOCALBIN)
-	@rm -rf $(BUILD_DIR)
+.PHONY: protoc
+PROTOC_REL ?= https://github.com/protocolbuffers/protobuf/releases
+protoc: $(PROTOC) ## Download protoc locally if necessary.
+$(PROTOC): | $(LOCALBIN)
+	cd $(LOCALBIN) && \
+	curl -L --output tmp.zip $(PROTOC_REL)/download/v$(PROTOC_VER)/protoc-$(PROTOC_VER)-linux-x86_64.zip && \
+	unzip tmp.zip -d protoc && rm tmp.zip
+
+.PHONY: protoc-gen-go
+protoc-gen-go: $(PROTOC_GEN_GO) ## Download protoc-gen-go locally if necessary.
+$(PROTOC_GEN_GO): | $(LOCALBIN)
+	GOBIN=$(LOCALBIN) go install google.golang.org/protobuf/cmd/protoc-gen-go@v$(PROTOC_GEN_GO_VER)
+
+.PHONY: protoc-gen-go-grpc
+protoc-gen-go-grpc: $(PROTOC_GEN_GO_GRPC) ## Download protoc-gen-go locally if necessary.
+$(PROTOC_GEN_GO_GRPC): | $(LOCALBIN)
+	GOBIN=$(LOCALBIN) go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v$(PROTOC_GEN_GO_GRPC_VER)
+
+.PHONY: buf
+buf: $(BUF) ## Download buf locally if necessary
+$(BUF): | $(LOCALBIN)
+	cd $(LOCALBIN) && \
+	curl -sSL "https://github.com/bufbuild/buf/releases/download/v$(BUF_VERSION)/buf-Linux-x86_64" -o "$(LOCALBIN)/buf" && \
+	chmod +x "$(LOCALBIN)/buf"
+
+##@ GRPC
+# go package for generated code
+API_PKG_GO_MOD ?= github.com/Mellanox/nvidia-k8s-ipam/api/grpc
+
+# GRPC DIRs
+GRPC_DIR ?= $(PROJECT_DIR)/api/grpc
+PROTO_DIR ?= $(GRPC_DIR)/proto
+GENERATED_CODE_DIR ?= $(GRPC_DIR)
+
+grpc-generate: protoc protoc-gen-go protoc-gen-go-grpc ## Generate GO client and server GRPC code
+	@echo "generate GRPC API"; \
+	echo "   go module: $(API_PKG_GO_MOD)"; \
+	echo "   output dir: $(GENERATED_CODE_DIR) "; \
+	echo "   proto dir: $(PROTO_DIR) "; \
+	cd $(PROTO_DIR) && \
+	TARGET_FILES=""; \
+	PROTOC_OPTIONS="--plugin=protoc-gen-go=$(PROTOC_GEN_GO) \
+					--plugin=protoc-gen-go-grpc=$(PROTOC_GEN_GO_GRPC) \
+					--go_out=$(GENERATED_CODE_DIR) \
+					--go_opt=module=$(API_PKG_GO_MOD) \
+					--proto_path=$(PROTO_DIR) \
+					--go-grpc_out=$(GENERATED_CODE_DIR) \
+					--go-grpc_opt=module=$(API_PKG_GO_MOD)"; \
+	echo "discovered proto files:"; \
+	for proto_file in $$(find . -name "*.proto"); do \
+		proto_file=$$(echo $$proto_file | cut -d'/' -f2-); \
+		proto_dir=$$(dirname $$proto_file); \
+		pkg_name=M$$proto_file=$(API_PKG_GO_MOD)/$$proto_dir; \
+		echo "    $$proto_file"; \
+		TARGET_FILES="$$TARGET_FILES $$proto_file"; \
+		PROTOC_OPTIONS="$$PROTOC_OPTIONS \
+						--go_opt=$$pkg_name \
+						--go-grpc_opt=$$pkg_name" ; \
+	done; \
+	$(PROTOC)  $$PROTOC_OPTIONS $$TARGET_FILES
+
+grpc-check: grpc-format grpc-lint protoc protoc-gen-go protoc-gen-go-grpc $(GRPC_TMP_DIR)  ## Check that generated GO client code match proto files
+	@rm -rf $(GRPC_TMP_DIR)/nvidia/
+	@$(MAKE) GENERATED_CODE_DIR=$(GRPC_TMP_DIR) grpc-generate
+	@diff -Naur $(GRPC_TMP_DIR)/nvidia/ $(GENERATED_CODE_DIR)/nvidia/ || \
+		(printf "\n\nOutdated files detected!\nPlease, run 'make generate' to regenerate GO code\n\n" && exit 1)
+	@echo "generated files are up to date"
+
+grpc-lint: buf  ## Lint GRPC files
+	@echo "lint protobuf files";
+	cd $(PROTO_DIR) && \
+	$(BUF) lint --config ../buf.yaml
+
+grpc-format: buf  ## Format GRPC files
+	@echo "format protobuf files";
+	cd $(PROTO_DIR) && \
+	$(BUF) format -w --exit-code

README.md

@@ -24,7 +24,7 @@ This repository is in its first steps of development, APIs may change in a non b
 NVIDIA IPAM plugin consists of 3 main components:
 
 1. controller ([ipam-controller](#ipam-controller))
-2. node agent ([ipam-node](#ipam-node))
+2. node daemon ([ipam-node](#ipam-node))
 3. IPAM CNI plugin ([nv-ipam](#nv-ipam))
 
 ### ipam-controller
@@ -35,19 +35,20 @@ annotation a cluster unique range of IPs of the defined IP Pools.
 
 ### ipam-node
 
-A node agent that performs initial setup and installation of nv-ipam CNI plugin.
+The daemon is responsible for:
+- perform initial setup and installation of nv-ipam CNI plugin
+- perform allocations of the IPs and persist them on the disk
+- run periodic jobs, such as cleanup of the stale IP address allocations
 
-### nv-ipam
-
-An IPAM CNI plugin that allocates IPs for a given interface out of the defined IP Pool as provided
-via CNI configuration.
-
-IPs are allocated out of the provided IP Block assigned by ipam-controller for the node.
-To determine the cluster unique IP Block for the defined IP Pool, nv-ipam CNI queries K8s API
+A node daemon provides GRPC service, which nv-ipam CNI plugin uses to request IP address allocation/deallocation.
+IPs are allocated from the provided IP Block assigned by ipam-controller for the node.
+To determine the cluster unique IP Block for the defined IP Pool, ipam-node watches K8s API
 for the Node object and extracts IP Block information from node annotation.
 
-nv-ipam plugin currently leverages [host-local](https://www.cni.dev/plugins/current/ipam/host-local/)
-IPAM to allocate IPs from the given range.
+### nv-ipam
+
+An IPAM CNI plugin that handles CNI requests according to the CNI spec.
+To allocate/deallocate IP address nv-ipam calls GRPC API of ipam-node daemon.
 
 ### IP allocation flow
 
@@ -189,21 +190,76 @@ data:
 
 ### ipam-node configuration
 
-ipam-node accepts configuration via command line flags
+ipam-node accepts configuration via command line flags.
+Options that begins with the `cni-` prefix are used to create the config file for nv-ipam CNI.
+All other options are for the ipam-node daemon itself.
 
 ```text
-      --cni-bin-dir string                    CNI binary directory (default "/host/opt/cni/bin")
-      --cni-conf-dir string                   CNI config directory (default "/host/etc/cni/net.d")
-  -h, --help                                  show help message and quit
-      --host-local-bin-file string            host-local binary file path (default "/host-local")
-      --nv-ipam-bin-file string               nv-ipam binary file path (default "/nv-ipam")
-      --nv-ipam-cni-data-dir string           nv-ipam CNI data directory (default "/host/var/lib/cni/nv-ipam")
-      --nv-ipam-cni-data-dir-host string      nv-ipam CNI data directory on host (default "/var/lib/cni/nv-ipam")
-      --nv-ipam-kubeconfig-file-host string   kubeconfig for nv-ipam (default "/etc/cni/net.d/nv-ipam.d/nv-ipam.kubeconfig")
-      --nv-ipam-log-file string               nv-ipam log file (default "/var/log/nv-ipam-cni.log")
-      --nv-ipam-log-level string              nv-ipam log level (default "info")
-      --skip-host-local-binary-copy           skip host-loca binary file copy
-      --skip-nv-ipam-binary-copy              skip nv-ipam binary file copy
+Logging flags:
+
+      --log-flush-frequency duration                                                                                                                                                                 
+                Maximum number of seconds between log flushes (default 5s)
+      --log-json-info-buffer-size quantity                                                                                                                                                           
+                [Alpha] In JSON format with split output streams, the info messages can be buffered for a while to increase performance. The default value of zero bytes disables buffering. The size
+                can be specified as number of bytes (512), multiples of 1000 (1K), multiples of 1024 (2Ki), or powers of those (3M, 4G, 5Mi, 6Gi). Enable the LoggingAlphaOptions feature gate to use this.
+      --log-json-split-stream                                                                                                                                                                        
+                [Alpha] In JSON format, write error messages to stderr and info messages to stdout. The default is to write a single stream to stdout. Enable the LoggingAlphaOptions feature gate to
+                use this.
+      --logging-format string                                                                                                                                                                        
+                Sets the log format. Permitted formats: "json" (gated by LoggingBetaOptions), "text". (default "text")
+  -v, --v Level                                                                                                                                                                                      
+                number for the log level verbosity
+      --vmodule pattern=N,...                                                                                                                                                                        
+                comma-separated list of pattern=N settings for file-filtered logging (only works for text log format)
+
+Common flags:
+
+      --feature-gates mapStringBool                                                                                                                                                                  
+                A set of key=value pairs that describe feature gates for alpha/experimental features. Options are:
+                AllAlpha=true|false (ALPHA - default=false)
+                AllBeta=true|false (BETA - default=false)
+                ContextualLogging=true|false (ALPHA - default=false)
+                LoggingAlphaOptions=true|false (ALPHA - default=false)
+                LoggingBetaOptions=true|false (BETA - default=true)
+      --version                                                                                                                                                                                      
+                print binary version and exit
+
+Node daemon flags:
+
+      --bind-address string                                                                                                                                                                          
+                GPRC server bind address. e.g.: tcp://127.0.0.1:9092, unix:///var/lib/foo (default "unix:///var/lib/cni/nv-ipam/daemon.sock")
+      --health-probe-bind-address string                                                                                                                                                             
+                The address the probe endpoint binds to. (default ":8081")
+      --kubeconfig string                                                                                                                                                                            
+                Paths to a kubeconfig. Only required if out-of-cluster.
+      --metrics-bind-address string                                                                                                                                                                  
+                The address the metric endpoint binds to. (default ":8080")
+      --node-name string                                                                                                                                                                             
+                The name of the Node on which the daemon runs
+      --store-file string                                                                                                                                                                            
+                Path of the file which used to store allocations (default "/var/lib/cni/nv-ipam/store")
+
+Shim CNI Configuration flags:
+
+      --cni-bin-dir string                                                                                                                                                                           
+                CNI binary directory (default "/opt/cni/bin")
+      --cni-conf-dir string                                                                                                                                                                          
+                shim CNI config: path with config file (default "/etc/cni/net.d/nv-ipam.d")
+      --cni-daemon-call-timeout int                                                                                                                                                                  
+                shim CNI config: timeout for IPAM daemon calls (default 5)
+      --cni-daemon-socket string                                                                                                                                                                     
+                shim CNI config: IPAM daemon socket path (default "unix:///var/lib/cni/nv-ipam/daemon.sock")
+      --cni-log-file string                                                                                                                                                                          
+                shim CNI config: path to log file for shim CNI (default "/var/log/nv-ipam-cni.log")
+      --cni-log-level string                                                                                                                                                                         
+                shim CNI config: log level for shim CNI (default "info")
+      --cni-nv-ipam-bin-file string                                                                                                                                                                  
+                nv-ipam binary file path (default "/nv-ipam")
+      --cni-skip-nv-ipam-binary-copy                                                                                                                                                                 
+                skip nv-ipam binary file copy
+      --cni-skip-nv-ipam-config-creation                                                                                                                                                             
+                skip config file creation for nv-ipam CNI
+
 ```
 
 ### nv-ipam CNI configuration
@@ -214,18 +270,21 @@ nv-ipam accepts the following CNI configuration:
 {
     "type": "nv-ipam",
     "poolName": "my-pool",
-    "kubeconfig": "/etc/cni/net.d/nv-ipam.d/nv-ipam.kubeconfig",
-    "dataDir": "/var/lib/cni/nv-ipam",
+    "daemonSocket": "unix:///var/lib/cni/nv-ipam/daemon.sock",
+    "daemonCallTimeoutSeconds": 5,
     "confDir": "/etc/cni/net.d/nv-ipam.d",
     "logFile": "/var/log/nv-ipam-cni.log",
     "logLevel": "info"
 }
 ```
 
 * `type` (string, required): CNI plugin name, MUST be `"nv-ipam"`
-* `poolName` (string, optional): name of the IP Pool to be used for IP allocation. (default: network name as provided in CNI call)
-* `kubeconfig` (string, optional): path to kubeconfig file. (default: `"/etc/cni/net.d/nv-ipam.d/nv-ipam.kubeconfig"`)
-* `dataDir` (string, optional): path to data dir. (default: `/var/lib/cni/nv-ipam`)
+* `poolName` (string, optional): name of the IP Pool to be used for IP allocation.
+It is possible to allocate two IPs for the interface from different pools by specifying pool names separated by coma,
+e.g. `"my-ipv4-pool,my-ipv6-pool"`. The primary intent to support multiple pools is a dual-stack use-case when an 
+interface should have two IP addresses: one IPv4 and one IPv6. (default: network name as provided in CNI call)
+* `daemonSocket` (string, optional): address of GRPC server socket served by IPAM daemon
+* `daemonCallTimeoutSeconds` (integer, optional): timeout for GRPC calls to IPAM daemon
 * `confDir` (string, optional): path to configuration dir. (default: `"/etc/cni/net.d/nv-ipam.d"`)
 * `logFile` (string, optional): log file path. (default: `"/var/log/nv-ipam-cni.log"`)
 * `logLevel` (string, optional): logging level. one of: `["verbose", "debug", "info", "warning", "error", "panic"]`.  (default: `"info"`)
@@ -319,11 +378,8 @@ cat /var/log/nv-ipam-cni.log
 
 ## Limitations
 
-* Deleting an IP Pool from config map while there are pods scheduled on nodes with IPs from deleted Pool, deleting these pods will fail (CNI CMD DEL fails)
 * Before removing a node from cluster, drain all workloads to ensure proper cleanup of IPs on node.
 * IP Block allocated to a node with Gateway IP in its range will have one less IP than what defined in perNodeBlockSize, deployers should take this into account.
-* IPv6 not supported
-* Allocating Multiple IPs per interface is not supported
 * Defining multiple IP Pools while supported, was not thoroughly testing
 
 ## Contributing

api/grpc/buf.yaml

@@ -0,0 +1,5 @@
+version: v1
+lint:
+  use:
+    - DEFAULT
+    - PACKAGE_NO_IMPORT_CYCLE