-
Notifications
You must be signed in to change notification settings - Fork 366
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
During CNIServer reconciliation, we perform host-local IPAM garbage collection (GC) by comparing the set of IPs allocated to local Pods and the set of IPs currently reserved by the plugin. We release any IP reserved by the plugin that is not in-use by a local Pod. The purpose is to avoid leaking IP addresses when there is a bug in the container runtime, which has happened in the past. Two key design choices that were made: * We do not invoke CNI DEL to release IPs, instead we access the host-local data which is persisted on the Node, and modify it as needed. * We do not rely on the interface store (as persisted to OVSDB) to determine the set of IPs that may have been leaked. In case of an Antrea bug, it could be possible (although unlikely) for an IP to still be allocated by host-local but be missing from the interface store. Intead, we list all allocated IPs from the host-local data (an allocated IP corresponds to one disk file). This approach is essentially the same as our existing script: https://github.com/antrea-io/antrea/blob/main/hack/gc-host-local.sh Fixes #4326 Signed-off-by: Antonin Bas <abas@vmware.com>
- Loading branch information
1 parent
ab94f29
commit 29e53b5
Showing
7 changed files
with
203 additions
and
5 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,103 @@ | ||
| // Copyright 2023 Antrea Authors | ||
| // | ||
| // Licensed under the Apache License, Version 2.0 (the "License"); | ||
| // you may not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software | ||
| // distributed under the License is distributed on an "AS IS" BASIS, | ||
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| // See the License for the specific language governing permissions and | ||
| // limitations under the License. | ||
|
|
||
| package hostlocal | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "net" | ||
| "os" | ||
| "path/filepath" | ||
| "runtime" | ||
| "strings" | ||
|
|
||
| "github.com/spf13/afero" | ||
| "k8s.io/apimachinery/pkg/util/sets" | ||
| "k8s.io/klog/v2" | ||
| ) | ||
|
|
||
| const dataDir = "/var/lib/cni/networks" | ||
|
|
||
| // This is a hacky approach as we access the internals of the host-local plugin, | ||
| // instead of using the CNI interface. However, crafting a CNI DEL request from | ||
| // scratch would also be hacky. | ||
| func GarbageCollectContainerIPs(network string, desiredIPs sets.Set[string]) error { | ||
| dir := filepath.Join(dataDir, network) | ||
|
|
||
| lk, err := NewFileLock(dataDir) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| defer lk.Close() | ||
| lk.Lock() | ||
| defer lk.Unlock() | ||
|
|
||
| fs := afero.NewOsFs() | ||
| return gcContainerIPs(fs, dir, desiredIPs) | ||
| } | ||
|
|
||
| // Internal version of GarbageCollectContainerIPs which does not acquire the | ||
| // file lock and can work with an arbitrary afero filesystem. | ||
| func gcContainerIPs(fs afero.Fs, dir string, desiredIPs sets.Set[string]) error { | ||
| paths := make([]string, 0) | ||
|
|
||
| if err := afero.Walk(fs, dir, func(path string, info os.FileInfo, err error) error { | ||
| if err != nil || info.IsDir() { | ||
| return nil | ||
| } | ||
| paths = append(paths, path) | ||
| return nil | ||
| }); err != nil { | ||
| return fmt.Errorf("error when gathering IP filenames in the host-local data directory: %w", err) | ||
| } | ||
|
|
||
| allocatedIPs := sets.New[string]() | ||
| for _, p := range paths { | ||
| ip := getIPFromPath(p) | ||
| if net.ParseIP(ip) == nil { | ||
| // not a valid IP, nothing to do | ||
| continue | ||
| } | ||
| allocatedIPs.Insert(ip) | ||
| if desiredIPs.Has(ip) { | ||
| // IP is in-use | ||
| continue | ||
| } | ||
| if err := os.Remove(p); err != nil { | ||
| klog.ErrorS(err, "Failed to release unused IP from host-local IPAM plugin", "IP", ip) | ||
| } | ||
| allocatedIPs.Delete(ip) | ||
| klog.InfoS("Unused IP was successfully released from host-local IPAM plugin", "IP", ip) | ||
| } | ||
|
|
||
| if allocatedIPs.Difference(desiredIPs).Len() > 0 { | ||
| return fmt.Errorf("not all unused IPs could be released from host-local IPAM plugin, some IPs may be leaked") | ||
| } | ||
|
|
||
| // Note that it is perfectly possible for some IPs to be in desiredIPs but not in | ||
| // allocatedIPs. This can be the case when another IPAM plugin (e.g., AntreaIPAM) is also | ||
| // used. | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| func getIPFromPath(path string) string { | ||
| fname := filepath.Base(path) | ||
| // need to unespace IPv6 addresses on Windows | ||
| // see https://github.com/containernetworking/plugins/blob/38f18d26ecfef550b8bac02656cc11103fd7cff1/plugins/ipam/host-local/backend/disk/backend.go#L197 | ||
| if runtime.GOOS == "windows" { | ||
| fname = strings.ReplaceAll(fname, "_", ":") | ||
| } | ||
| return fname | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,62 @@ | ||
| // Copyright 2023 Antrea Authors | ||
| // | ||
| // Licensed under the Apache License, Version 2.0 (the "License"); | ||
| // you may not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software | ||
| // distributed under the License is distributed on an "AS IS" BASIS, | ||
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| // See the License for the specific language governing permissions and | ||
| // limitations under the License. | ||
|
|
||
| package hostlocal | ||
|
|
||
| import ( | ||
| "os" | ||
| "path" | ||
|
|
||
| "github.com/alexflint/go-filemutex" | ||
| ) | ||
|
|
||
| // This code was copied from https://github.com/containernetworking/plugins/blob/v1.3.0/plugins/ipam/host-local/backend/disk/lock.go | ||
|
|
||
| // FileLock wraps os.File to be used as a lock using flock | ||
| type FileLock struct { | ||
| f *filemutex.FileMutex | ||
| } | ||
|
|
||
| // NewFileLock opens file/dir at path and returns unlocked FileLock object | ||
| func NewFileLock(lockPath string) (*FileLock, error) { | ||
| fi, err := os.Stat(lockPath) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| if fi.IsDir() { | ||
| lockPath = path.Join(lockPath, "lock") | ||
| } | ||
|
|
||
| f, err := filemutex.New(lockPath) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| return &FileLock{f}, nil | ||
| } | ||
|
|
||
| func (l *FileLock) Close() error { | ||
| return l.f.Close() | ||
| } | ||
|
|
||
| // Lock acquires an exclusive lock | ||
| func (l *FileLock) Lock() error { | ||
| return l.f.Lock() | ||
| } | ||
|
|
||
| // Unlock releases the lock | ||
| func (l *FileLock) Unlock() error { | ||
| return l.f.Unlock() | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters