Skip to content

Commit

Permalink
deps: bumping yq to v4.40.7
Browse files Browse the repository at this point in the history
Since yq frequently updates, let's upgrade to a version from February to
bypass potential issues with versions 4.41-4.43 for now. We can always
upgrade to the newest version if necessary.

Fixes kata-containers#9354
Depends-on:github.com/kata-containers/tests#5818

Signed-off-by: Beraldo Leal <bleal@redhat.com>
  • Loading branch information
beraldoleal authored and zaymat committed Jun 17, 2024
1 parent d229e89 commit c5bc286
Show file tree
Hide file tree
Showing 47 changed files with 376 additions and 476 deletions.
8 changes: 4 additions & 4 deletions ci/install_libseccomp.sh
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,11 @@ workdir="$(mktemp -d --tmpdir build-libseccomp.XXXXX)"
# Variables for libseccomp
libseccomp_version="${LIBSECCOMP_VERSION:-""}"
if [ -z "${libseccomp_version}" ]; then
libseccomp_version=$(get_from_kata_deps "externals.libseccomp.version")
libseccomp_version=$(get_from_kata_deps ".externals.libseccomp.version")
fi
libseccomp_url="${LIBSECCOMP_URL:-""}"
if [ -z "${libseccomp_url}" ]; then
libseccomp_url=$(get_from_kata_deps "externals.libseccomp.url")
libseccomp_url=$(get_from_kata_deps ".externals.libseccomp.url")
fi
libseccomp_tarball="libseccomp-${libseccomp_version}.tar.gz"
libseccomp_tarball_url="${libseccomp_url}/releases/download/v${libseccomp_version}/${libseccomp_tarball}"
Expand All @@ -36,11 +36,11 @@ cflags="-O2"
# Variables for gperf
gperf_version="${GPERF_VERSION:-""}"
if [ -z "${gperf_version}" ]; then
gperf_version=$(get_from_kata_deps "externals.gperf.version")
gperf_version=$(get_from_kata_deps ".externals.gperf.version")
fi
gperf_url="${GPERF_URL:-""}"
if [ -z "${gperf_url}" ]; then
gperf_url=$(get_from_kata_deps "externals.gperf.url")
gperf_url=$(get_from_kata_deps ".externals.gperf.url")
fi
gperf_tarball="gperf-${gperf_version}.tar.gz"
gperf_tarball_url="${gperf_url}/${gperf_tarball}"
Expand Down
2 changes: 1 addition & 1 deletion ci/install_yq.sh
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ die() {
# Install via binary download, as we may not have golang installed at this point
function install_yq() {
local yq_pkg="github.com/mikefarah/yq"
local yq_version=3.4.1
local yq_version=v4.40.7
local precmd=""
INSTALL_IN_GOPATH=${INSTALL_IN_GOPATH:-true}

Expand Down
2 changes: 1 addition & 1 deletion docs/Developer-Guide.md
Original file line number Diff line number Diff line change
Expand Up @@ -461,7 +461,7 @@ and repository utilized can be found by looking at the [versions file](../versio
Find the correct version of QEMU from the versions file:
```bash
$ source kata-containers/tools/packaging/scripts/lib.sh
$ qemu_version="$(get_from_kata_deps "assets.hypervisor.qemu.version")"
$ qemu_version="$(get_from_kata_deps ".assets.hypervisor.qemu.version")"
$ echo "${qemu_version}"
```
Get source from the matching branch of QEMU:
Expand Down
28 changes: 12 additions & 16 deletions docs/how-to/how-to-pull-images-in-guest-with-kata.md
Original file line number Diff line number Diff line change
Expand Up @@ -35,27 +35,23 @@ $ git clone -b "${nydus_snapshotter_version}" "${nydus_snapshotter_url}" "${nydu
2. Configure DaemonSet file
```bash
$ pushd "$nydus_snapshotter_install_dir"
$ yq write -i \
> misc/snapshotter/base/nydus-snapshotter.yaml \
> 'data.FS_DRIVER' \
> "proxy" --style=double
$ yq -i \
> '.data.FS_DRIVER = "proxy"' -P \
> misc/snapshotter/base/nydus-snapshotter.yaml
# Disable to read snapshotter config from configmap
$ yq write -i \
> misc/snapshotter/base/nydus-snapshotter.yaml \
> 'data.ENABLE_CONFIG_FROM_VOLUME' \
> "false" --style=double
$ yq -i \
> 'data.ENABLE_CONFIG_FROM_VOLUME = "false"' -P \
> misc/snapshotter/base/nydus-snapshotter.yaml
# Enable to run snapshotter as a systemd service
# (skip if you want to run nydus snapshotter as a standalone process)
$ yq write -i \
> misc/snapshotter/base/nydus-snapshotter.yaml \
> 'data.ENABLE_SYSTEMD_SERVICE' \
> "true" --style=double
$ yq -i \
> 'data.ENABLE_SYSTEMD_SERVICE = "true"' -P \
> misc/snapshotter/base/nydus-snapshotter.yaml
# Enable "runtime specific snapshotter" feature in containerd when configuring containerd for snapshotter
# (skip if you want to configure nydus snapshotter as a global snapshotter in containerd)
$ yq write -i \
> misc/snapshotter/base/nydus-snapshotter.yaml \
> 'data.ENABLE_RUNTIME_SPECIFIC_SNAPSHOTTER' \
> "true" --style=double
$ yq -i \
> 'data.ENABLE_RUNTIME_SPECIFIC_SNAPSHOTTER = "true"' -P \
> misc/snapshotter/base/nydus-snapshotter.yaml
```

3. Install `nydus snapshotter` as a DaemonSet
Expand Down
4 changes: 2 additions & 2 deletions docs/how-to/how-to-run-kata-containers-with-SNP-VMs.md
Original file line number Diff line number Diff line change
Expand Up @@ -44,8 +44,8 @@ $ popd
- Build a custom QEMU
```bash
$ source kata-containers/tools/packaging/scripts/lib.sh
$ qemu_url="$(get_from_kata_deps "assets.hypervisor.qemu-snp-experimental.url")"
$ qemu_tag="$(get_from_kata_deps "assets.hypervisor.qemu-snp-experimental.tag")"
$ qemu_url="$(get_from_kata_deps ".assets.hypervisor.qemu-snp-experimental.url")"
$ qemu_tag="$(get_from_kata_deps ".assets.hypervisor.qemu-snp-experimental.tag")"
$ git clone "${qemu_url}"
$ pushd qemu
$ git checkout "${qemu_tag}"
Expand Down
12 changes: 11 additions & 1 deletion src/runtime/golang.mk
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,17 @@ ifeq (,$(not_check_version))
ifneq (,$(install_yq))
$(error "ERROR: install yq failed")
endif
golang_version_min=$(shell $(GOPATH)/bin/yq r ../../versions.yaml languages.golang.version)

YQ_VERSION=$(shell $(GOPATH)/bin/yq --version | grep -oE "version v?[0-9]+" | grep -oE "[0-9]+")
QUERY="languages.golang.version"

ifneq (,$(findstring 4,$(YQ_VERSION)))
YQ_CMD=$(GOPATH)/bin/yq eval .$(QUERY) ../../versions.yaml
else
YQ_CMD=$(GOPATH)/bin/yq r ../../versions.yaml $(QUERY)
endif

golang_version_min=$(shell $(YQ_CMD))

ifeq (,$(golang_version_min))
$(error "ERROR: cannot determine minimum golang version")
Expand Down
2 changes: 1 addition & 1 deletion src/runtime/virtcontainers/pkg/cloud-hypervisor/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ update-yaml:
ifndef YQ
$(MK_DIR)/../../../../../ci//install_yq.sh
endif
clh_version=$(shell yq r $(VERSIONS_FILE) assets.hypervisor.cloud_hypervisor.version); \
clh_version=$(shell yq .assets.hypervisor.cloud_hypervisor.version $(VERSIONS_FILE)); \
curl -OL https://raw.githubusercontent.com/cloud-hypervisor/cloud-hypervisor/$$clh_version/vmm/src/api/openapi/cloud-hypervisor.yaml

clean-generated-code:
Expand Down
4 changes: 2 additions & 2 deletions tests/cmd/github-labels/github-labels.sh
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ merge_yaml()
[ -n "$out" ] || die "need output file"

need_yq
yq merge "$file1" --append "$file2" > "$out"
yq eval-all '. as $item ireduce ({}; . *+ $item)' "$file1" "$file2" > "$out"
}

check_yaml()
Expand All @@ -58,7 +58,7 @@ check_yaml()
[ -n "$file" ] || die "need file to check"

need_yq
yq read "$file" >/dev/null
yq "$file" >/dev/null

[ -z "$(command -v yamllint)" ] && die "need yamllint installed"

Expand Down
16 changes: 12 additions & 4 deletions tests/common.bash
Original file line number Diff line number Diff line change
Expand Up @@ -497,11 +497,19 @@ function ensure_yq() {

# dependency: What we want to get the version from the versions.yaml file
function get_from_kata_deps() {
local dependency="$1"
versions_file="${repo_root_dir}/versions.yaml"

command -v yq &>/dev/null || die 'yq command is not in your $PATH'
result=$("yq" read -X "$versions_file" "$dependency")

yq_version=$(yq --version | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | cut -d. -f1)
if [ "$yq_version" -eq 3 ]; then
dependency=$(echo "$1" | sed "s/^\.//g")
result=$("yq" read "$versions_file" "$dependency")
else
dependency=$1
result=$("yq" "$dependency | explode (.)" "$versions_file")
fi

[ "$result" = "null" ] && result=""
echo "$result"
}
Expand Down Expand Up @@ -743,7 +751,7 @@ function get_dep_from_yaml_db(){

"${repo_root_dir}/ci/install_yq.sh" >&2

result=$("${GOPATH}/bin/yq" r -X "$versions_file" "$dependency")
result=$("${GOPATH}/bin/yq" "$dependency" "$versions_file")
[ "$result" = "null" ] && result=""
echo "$result"
}
Expand All @@ -759,7 +767,7 @@ function get_test_version(){

db="${cidir}/../versions.yaml"

get_dep_from_yaml_db "${db}" "${dependency}"
get_dep_from_yaml_db "${db}" ".${dependency}"
}

# Load vhost, vhost_net, vhost_vsock modules.
Expand Down
74 changes: 30 additions & 44 deletions tests/functional/kata-deploy/kata-deploy.bats
Original file line number Diff line number Diff line change
Expand Up @@ -29,40 +29,30 @@ setup() {
sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}|g" "tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"

# Enable debug for Kata Containers
yq write -i \
"tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \
'spec.template.spec.containers[0].env[1].value' \
--tag '!!str' "true"
yq -i \
'.spec.template.spec.containers[0].env[1].value = "true"' \
"tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
# Create the runtime class only for the shim that's being tested
yq write -i \
"tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \
'spec.template.spec.containers[0].env[2].value' \
"${KATA_HYPERVISOR}"
yq -i \
".spec.template.spec.containers[0].env[2].value = \"${KATA_HYPERVISOR}\"" \
"tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
# Set the tested hypervisor as the default `kata` shim
yq write -i \
"tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \
'spec.template.spec.containers[0].env[3].value' \
"${KATA_HYPERVISOR}"
yq -i \
".spec.template.spec.containers[0].env[3].value = \"${KATA_HYPERVISOR}\"" \
"tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
# Let the `kata-deploy` script take care of the runtime class creation / removal
yq write -i \
"tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \
'spec.template.spec.containers[0].env[4].value' \
--tag '!!str' "true"
yq -i \
'.spec.template.spec.containers[0].env[4].value = "true"' \
"tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
# Let the `kata-deploy` create the default `kata` runtime class
yq write -i \
"tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \
'spec.template.spec.containers[0].env[5].value' \
--tag '!!str' "true"
yq -i \
'.spec.template.spec.containers[0].env[5].value = "true"' \
"tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"

if [ "${KATA_HOST_OS}" = "cbl-mariner" ]; then
yq write -i \
"tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \
'spec.template.spec.containers[0].env[+].name' \
"HOST_OS"
yq write -i \
"tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \
'spec.template.spec.containers[0].env[-1].value' \
"${KATA_HOST_OS}"
yq -i \
".spec.template.spec.containers[0].env += [{\"name\": \"HOST_OS\", \"value\": \"${KATA_HOST_OS}\"}]" \
"tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
fi

echo "::group::Final kata-deploy.yaml that is used in the test"
Expand Down Expand Up @@ -144,25 +134,21 @@ teardown() {
kubectl -n kube-system wait --timeout=10m --for=delete -l name=kata-deploy pod

# Let the `kata-deploy` script take care of the runtime class creation / removal
yq write -i \
"tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" \
'spec.template.spec.containers[0].env[4].value' \
--tag '!!str' "true"
yq -i \
'.spec.template.spec.containers[0].env[4].value = "true"' \
"tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml"
# Create the runtime class only for the shim that's being tested
yq write -i \
"tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" \
'spec.template.spec.containers[0].env[2].value' \
"${KATA_HYPERVISOR}"
yq -i \
".spec.template.spec.containers[0].env[2].value = \"${KATA_HYPERVISOR}\"" \
"tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml"
# Set the tested hypervisor as the default `kata` shim
yq write -i \
"tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" \
'spec.template.spec.containers[0].env[3].value' \
"${KATA_HYPERVISOR}"
yq -i \
".spec.template.spec.containers[0].env[3].value = \"${KATA_HYPERVISOR}\"" \
"tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml"
# Let the `kata-deploy` create the default `kata` runtime class
yq write -i \
"tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \
'spec.template.spec.containers[0].env[5].value' \
--tag '!!str' "true"
yq -i \
'.spec.template.spec.containers[0].env[5].value = "true"' \
"tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"

sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}|g" "tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml"
cat "tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml"
Expand Down
6 changes: 3 additions & 3 deletions tests/functional/kata-monitor/gha-run.sh
Original file line number Diff line number Diff line change
Expand Up @@ -36,15 +36,15 @@ function install_dependencies() {
# - cri-tools
# - containerd
# - cri-container-cni release tarball already includes CNI plugins
cri_tools_version=$(get_from_kata_deps "externals.critools.latest")
cri_tools_version=$(get_from_kata_deps ".externals.critools.latest")
declare -a github_deps
github_deps[0]="cri_tools:${cri_tools_version}"
case "${CONTAINER_ENGINE}" in
containerd)
github_deps[1]="cri_containerd:$(get_from_kata_deps "externals.containerd.${CONTAINERD_VERSION}")"
github_deps[1]="cri_containerd:$(get_from_kata_deps ".externals.containerd.${CONTAINERD_VERSION}")"
;;
crio)
github_deps[1]="cni_plugins:$(get_from_kata_deps "externals.cni-plugins.version")"
github_deps[1]="cni_plugins:$(get_from_kata_deps ".externals.cni-plugins.version")"
;;
esac

Expand Down
4 changes: 2 additions & 2 deletions tests/functional/vfio/vfio_fedora_vm_wrapper.sh
Original file line number Diff line number Diff line change
Expand Up @@ -174,8 +174,8 @@ ${environment}
pushd /workspace
source tests/common.bash
ensure_yq
cri_containerd=\$(get_from_kata_deps "externals.containerd.lts")
cri_tools=\$(get_from_kata_deps "externals.critools.latest")
cri_containerd=\$(get_from_kata_deps ".externals.containerd.lts")
cri_tools=\$(get_from_kata_deps ".externals.critools.latest")
install_cri_containerd \${cri_containerd}
install_cri_tools \${cri_tools}
Expand Down
4 changes: 2 additions & 2 deletions tests/gha-run-k8s-common.sh
Original file line number Diff line number Diff line change
Expand Up @@ -127,9 +127,9 @@ function install_kustomize() {
fi

ensure_yq
version=$(get_from_kata_deps "externals.kustomize.version")
version=$(get_from_kata_deps ".externals.kustomize.version")
arch=$(arch_to_golang)
checksum=$(get_from_kata_deps "externals.kustomize.checksum.${arch}")
checksum=$(get_from_kata_deps ".externals.kustomize.checksum.${arch}")

local tarball="kustomize_${version}_linux_${arch}.tar.gz"
curl -Lf -o "$tarball" "https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize/${version}/${tarball}"
Expand Down
2 changes: 1 addition & 1 deletion tests/install_go.sh
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ shift $(( $OPTIND - 1 ))
go_version="${1:-""}"

if [ -z "$go_version" ] && [ "${USE_VERSIONS_FILE}" = "true" ] ;then
go_version=$(get_from_kata_deps "languages.golang.meta.newest-version")
go_version=$(get_from_kata_deps ".languages.golang.meta.newest-version")
fi

if [ -z "$go_version" ];then
Expand Down
2 changes: 1 addition & 1 deletion tests/install_rust.sh
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ rustarch=$(arch_to_rust)

version="${1:-""}"
if [ -z "${version}" ]; then
version=$(get_from_kata_deps "languages.rust.meta.newest-version")
version=$(get_from_kata_deps ".languages.rust.meta.newest-version")
fi

echo "Install rust ${version}"
Expand Down
6 changes: 3 additions & 3 deletions tests/integration/cri-containerd/gha-run.sh
Original file line number Diff line number Diff line change
Expand Up @@ -43,8 +43,8 @@ function install_dependencies() {
# - cri-container-cni release tarball already includes CNI plugins
# - cri-tools
declare -a github_deps
github_deps[0]="cri_containerd:$(get_from_kata_deps "externals.containerd.${CONTAINERD_VERSION}")"
github_deps[1]="cri_tools:$(get_from_kata_deps "externals.critools.latest")"
github_deps[0]="cri_containerd:$(get_from_kata_deps ".externals.containerd.${CONTAINERD_VERSION}")"
github_deps[1]="cri_tools:$(get_from_kata_deps ".externals.critools.latest")"

for github_dep in "${github_deps[@]}"; do
IFS=":" read -r -a dep <<< "${github_dep}"
Expand All @@ -53,7 +53,7 @@ function install_dependencies() {

# Clone containerd as we'll need to build it in order to run the tests
# base_version: The version to be intalled in the ${major}.${minor} format
clone_cri_containerd $(get_from_kata_deps "externals.containerd.${CONTAINERD_VERSION}")
clone_cri_containerd $(get_from_kata_deps ".externals.containerd.${CONTAINERD_VERSION}")
}

function run() {
Expand Down
10 changes: 5 additions & 5 deletions tests/integration/kubernetes/confidential_kbs.sh
Original file line number Diff line number Diff line change
Expand Up @@ -149,7 +149,7 @@ kbs_install_cli() {
# Mininum required version to build the client (read from versions.yaml)
local rust_version
ensure_yq
rust_version=$(get_from_kata_deps "externals.coco-trustee.toolchain")
rust_version=$(get_from_kata_deps ".externals.coco-trustee.toolchain")
# Currently kata version from version.yaml is 1.72.0
# which doesn't match the requirement, so let's pass
# the required version.
Expand Down Expand Up @@ -205,10 +205,10 @@ function kbs_k8s_deploy() {
ensure_yq

# Read from versions.yaml
repo=$(get_from_kata_deps "externals.coco-trustee.url")
version=$(get_from_kata_deps "externals.coco-trustee.version")
image=$(get_from_kata_deps "externals.coco-trustee.image")
image_tag=$(get_from_kata_deps "externals.coco-trustee.image_tag")
repo=$(get_from_kata_deps ".externals.coco-trustee.url")
version=$(get_from_kata_deps ".externals.coco-trustee.version")
image=$(get_from_kata_deps ".externals.coco-trustee.image")
image_tag=$(get_from_kata_deps ".externals.coco-trustee.image_tag")

# The ingress handler for AKS relies on the cluster's name which in turn
# contain the HEAD commit of the kata-containers repository (supposedly the
Expand Down
2 changes: 1 addition & 1 deletion tests/integration/kubernetes/filter_k8s_test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ main()
${repo_root_dir}/ci/install_yq.sh > /dev/null
fi

local K8S_SKIP_UNION=$("${GOPATH_LOCAL}/bin/yq" read "${K8S_CONFIG_FILE}" "${K8S_FILTER_FLAG}")
local K8S_SKIP_UNION=$("${GOPATH_LOCAL}/bin/yq" ".${K8S_FILTER_FLAG}" "${K8S_CONFIG_FILE}")
[ "${K8S_SKIP_UNION}" == "null" ] && return
mapfile -t _K8S_SKIP_UNION <<< "${K8S_SKIP_UNION}"

Expand Down
Loading

0 comments on commit c5bc286

Please sign in to comment.