Cluster deploy fails during Clean up possible pending packages on fedora coreos on FCOS 35

[aesca1er]

Environment:

• Bare Metal

• QEMU VMs

• Deploy from Mac M1

• OS (printf "$(uname -srm)\n$(cat /etc/os-release)\n"):
  Linux 5.16.16-200.fc35.x86_64 x86_64
  NAME="Fedora Linux"
  VERSION="35.20220327.3.0 (CoreOS)"
  ID=fedora
  VERSION_ID=35
  VERSION_CODENAME=""
  PLATFORM_ID="platform:f35"
  PRETTY_NAME="Fedora CoreOS 35.20220327.3.0"
  ANSI_COLOR="0;38;2;60;110;180"
  LOGO=fedora-logo-icon
  CPE_NAME="cpe:/o:fedoraproject:fedora:35"
  HOME_URL="https://getfedora.org/coreos/"
  DOCUMENTATION_URL="https://docs.fedoraproject.org/en-US/fedora-coreos/"
  SUPPORT_URL="https://github.com/coreos/fedora-coreos-tracker/"
  BUG_REPORT_URL="https://github.com/coreos/fedora-coreos-tracker/"
  REDHAT_BUGZILLA_PRODUCT="Fedora"
  REDHAT_BUGZILLA_PRODUCT_VERSION=35
  REDHAT_SUPPORT_PRODUCT="Fedora"
  REDHAT_SUPPORT_PRODUCT_VERSION=35
  PRIVACY_POLICY_URL="https://fedoraproject.org/wiki/Legal:PrivacyPolicy"
  VARIANT="CoreOS"
  VARIANT_ID=coreos
  OSTREE_VERSION='35.20220327.3.0'
  DEFAULT_HOSTNAME=localhost

• Version of Ansible (ansible --version):
  ansible [core 2.12.4]
  config file = /Users/aescaler/code/kubespray/ansible.cfg
  configured module search path = ['/Users/aescaler/code/kubespray/library']
  ansible python module location = /opt/homebrew/Cellar/ansible/5.6.0/libexec/lib/python3.10/site-packages/ansible
  ansible collection location = /Users/aescaler/.ansible/collections:/usr/share/ansible/collections
  executable location = /opt/homebrew/bin/ansible
  python version = 3.10.2 (main, Feb 2 2022, 05:51:25) [Clang 13.0.0 (clang-1300.0.29.3)]
  jinja version = 3.1.1
  libyaml = True

• Version of Python (python --version):
  Python 3.9.12

Kubespray version (commit) (git rev-parse --short HEAD):
f4ff03c

Network plugin used:
Cilium

Full inventory with variables (ansible -i inventory/sample/inventory.ini all -m debug -a "var=hostvars[inventory_hostname]"):

circonusdev10 | SUCCESS => {
    "hostvars[inventory_hostname]": {
        "ansible_check_mode": false,
        "ansible_config_file": "/Users/aescaler/code/kubespray/ansible.cfg",
        "ansible_diff_mode": false,
        "ansible_facts": {},
        "ansible_forks": 5,
        "ansible_host": "circonusdev10.dev.circonus.net",
        "ansible_hostname": "circonusdev10",
        "ansible_inventory_sources": [
            "/Users/aescaler/code/circonus/kubespray-inventory"
        ],
        "ansible_playbook_python": "/opt/homebrew/Cellar/ansible/5.6.0/libexec/bin/python3.10",
        "ansible_user": "core",
        "ansible_verbosity": 0,
        "ansible_version": {
            "full": "2.12.4",
            "major": 2,
            "minor": 12,
            "revision": 4,
            "string": "2.12.4"
        },
        "argocd_enabled": false,
        "auto_renew_certificates": false,
        "bin_dir": "/usr/local/bin",
        "cephfs_provisioner_enabled": false,
        "cert_manager_enabled": false,
        "cluster_name": "circonus",
        "container_manager": "crio",
        "coredns_k8s_external_zone": "circonus",
        "credentials_dir": "/Users/aescaler/code/circonus/kubespray-inventory/credentials",
        "dashboard_enabled": false,
        "default_kubelet_config_dir": "/etc/kubernetes/dynamic_kubelet_dir",
        "deploy_netchecker": false,
        "dns_domain": "circonus",
        "dns_mode": "coredns",
        "docker_bin_dir": "/usr/bin",
        "docker_container_storage_setup": false,
        "docker_daemon_graph": "/var/lib/docker",
        "docker_dns_servers_strict": false,
        "docker_iptables_enabled": "false",
        "docker_log_opts": "--log-opt max-size=50m --log-opt max-file=5",
        "docker_rpm_keepcache": 1,
        "dynamic_kubelet_configuration": false,
        "dynamic_kubelet_configuration_dir": "/etc/kubernetes/dynamic_kubelet_dir",
        "enable_coredns_k8s_endpoint_pod_names": false,
        "enable_coredns_k8s_external": false,
        "enable_dual_stack_networks": false,
        "enable_nat_default_gateway": true,
        "enable_nodelocaldns": true,
        "enable_nodelocaldns_secondary": false,
        "etcd_data_dir": "/var/lib/etcd",
        "etcd_deployment_type": "host",
        "etcd_member_name": "etcd1",
        "event_ttl_duration": "1h0m0s",
        "group_names": [
            "etcd",
            "k8s_cluster",
            "kube_control_plane",
            "kube_node"
        ],
        "groups": {
            "all": [
                "circonusdev10"
            ],
            "etcd": [
                "circonusdev10"
            ],
            "k8s_cluster": [
                "circonusdev10"
            ],
            "kube_control_plane": [
                "circonusdev10"
            ],
            "kube_node": [
                "circonusdev10"
            ],
            "ungrouped": []
        },
        "helm_enabled": false,
        "ingress_alb_enabled": false,
        "ingress_nginx_enabled": false,
        "ingress_publish_status_address": "",
        "inventory_dir": "/Users/aescaler/code/circonus/kubespray-inventory",
        "inventory_file": "/Users/aescaler/code/circonus/kubespray-inventory/inventory.yaml",
        "inventory_hostname": "circonusdev10",
        "inventory_hostname_short": "circonusdev10",
        "k8s_image_pull_policy": "Always",
        "kata_containers_enabled": false,
        "krew_enabled": false,
        "krew_root_dir": "/usr/local/krew",
        "kube_api_anonymous_auth": true,
        "kube_apiserver_insecure_port": 0,
        "kube_apiserver_ip": "10.233.0.1",
        "kube_apiserver_port": 6443,
        "kube_cert_dir": "/etc/kubernetes/ssl",
        "kube_cert_group": "kube-cert",
        "kube_config_dir": "/etc/kubernetes",
        "kube_encrypt_secret_data": true,
        "kube_log_level": 2,
        "kube_manifest_dir": "/etc/kubernetes/manifests",
        "kube_network_node_prefix": 24,
        "kube_network_node_prefix_ipv6": 120,
        "kube_network_plugin": "cilium",
        "kube_network_plugin_multus": false,
        "kube_oidc_auth": true,
        "kube_pods_subnet": "10.233.64.0/18",
        "kube_pods_subnet_ipv6": "fd85:ee78:d8a6:8607::1:0000/112",
        "kube_proxy_mode": "ipvs",
        "kube_proxy_nodeport_addresses": [],
        "kube_proxy_strict_arp": false,
        "kube_script_dir": "/usr/local/bin/kubernetes-scripts",
        "kube_service_addresses": "10.233.0.0/18",
        "kube_service_addresses_ipv6": "fd85:ee78:d8a6:8607::1000/116",
        "kube_token_auth": true,
        "kube_token_dir": "/etc/kubernetes/tokens",
        "kube_version": "v1.23.5",
        "kubeadm_certificate_key": "babacf4a276bd1e5e26d73fe22d3d547fce4d4fb5d5ff2b73007abb9ea59a9fd",
        "kubeconfig_localhost": true,
        "kubernetes_audit": false,
        "loadbalancer_apiserver_healthcheck_port": 8081,
        "loadbalancer_apiserver_port": 6443,
        "loadbalancer_apiserver_type": "haproxy",
        "local_path_provisioner_enabled": false,
        "local_release_dir": "/tmp/releases",
        "local_volume_provisioner_enabled": false,
        "macvlan_interface": "eth1",
        "metallb_enabled": false,
        "metallb_speaker_enabled": true,
        "metrics_server_enabled": false,
        "ndots": 2,
        "no_proxy_exclude_workers": false,
        "nodelocaldns_bind_metrics_host_ip": false,
        "nodelocaldns_health_port": 9254,
        "nodelocaldns_ip": "169.254.25.10",
        "nodelocaldns_second_health_port": 9256,
        "nodelocaldns_secondary_skew_seconds": 5,
        "omit": "__omit_place_holder__c82de6a00da7b0c546c9593e3a50cb9742a77217",
        "persistent_volumes_enabled": false,
        "playbook_dir": "/Users/aescaler/code/kubespray",
        "podsecuritypolicy_enabled": false,
        "rbd_provisioner_enabled": false,
        "registry_enabled": false,
        "resolvconf_mode": "host_resolvconf",
        "retry_stagger": 5,
        "skydns_server": "10.233.0.3",
        "skydns_server_secondary": "10.233.0.4",
        "use_localhost_as_kubeapi_loadbalancer": true,
        "volume_cross_zone_attachment": false
    }
}

Command used to invoke ansible:

Output of ansible run:

TASK [bootstrap-os : include_tasks] ************************************************************************************************************************************************************************
included: /Users/aescaler/code/kubespray/roles/bootstrap-os/tasks/bootstrap-fedora-coreos.yml for circonusdev10
Tuesday 26 April 2022  16:40:48 -0400 (0:00:00.018)       0:00:03.058 ********* 
[WARNING]: raw module does not support the environment keyword

TASK [bootstrap-os : Check if bootstrap is needed] *********************************************************************************************************************************************************
ok: [circonusdev10]
Tuesday 26 April 2022  16:40:48 -0400 (0:00:00.182)       0:00:03.241 ********* 
[WARNING]: raw module does not support the environment keyword

TASK [bootstrap-os : Remove podman network cni] ************************************************************************************************************************************************************
changed: [circonusdev10]
Tuesday 26 April 2022  16:40:49 -0400 (0:00:00.794)       0:00:04.036 ********* 
[WARNING]: raw module does not support the environment keyword

TASK [bootstrap-os : Clean up possible pending packages on fedora coreos] **********************************************************************************************************************************
fatal: [circonusdev10]: FAILED! => {"changed": true, "msg": "non-zero return code", "rc": 1, "stderr": "Shared connection to circonusdev10.dev.circonus.net closed.\r\n", "stderr_lines": ["Shared connection to circonusdev10.dev.circonus.net closed."], "stdout": "\u001b[0m\u001b[31merror: \u001b[0mRemounting /sysroot read-write: Permission denied\r\n", "stdout_lines": ["\u001b[0m\u001b[31merror: \u001b[0mRemounting /sysroot read-write: Permission denied"]}

Anything else do we need to know:

[aesca1er]

Looks like the root issue lies within rpm-ostree

[aesca1er]

ostreedev/ostree#2262
seems related

[k8s-triage-robot]

The Kubernetes project currently lacks enough contributors to adequately respond to all issues and PRs.

This bot triages issues and PRs according to the following rules:

• After 90d of inactivity, lifecycle/stale is applied
• After 30d of inactivity since lifecycle/stale was applied, lifecycle/rotten is applied
• After 30d of inactivity since lifecycle/rotten was applied, the issue is closed

You can:

• Mark this issue or PR as fresh with /remove-lifecycle stale
• Mark this issue or PR as rotten with /lifecycle rotten
• Close this issue or PR with /close
• Offer to help out with Issue Triage

Please send feedback to sig-contributor-experience at kubernetes/community.

/lifecycle stale

[k8s-triage-robot]

The Kubernetes project currently lacks enough active contributors to adequately respond to all issues and PRs.

This bot triages issues and PRs according to the following rules:

• After 90d of inactivity, lifecycle/stale is applied
• After 30d of inactivity since lifecycle/stale was applied, lifecycle/rotten is applied
• After 30d of inactivity since lifecycle/rotten was applied, the issue is closed

You can:

• Mark this issue or PR as fresh with /remove-lifecycle rotten
• Close this issue or PR with /close
• Offer to help out with Issue Triage

Please send feedback to sig-contributor-experience at kubernetes/community.

/lifecycle rotten

[k8s-triage-robot]

The Kubernetes project currently lacks enough active contributors to adequately respond to all issues and PRs.

This bot triages issues according to the following rules:

• After 90d of inactivity, lifecycle/stale is applied
• After 30d of inactivity since lifecycle/stale was applied, lifecycle/rotten is applied
• After 30d of inactivity since lifecycle/rotten was applied, the issue is closed

You can:

• Reopen this issue with /reopen
• Mark this issue as fresh with /remove-lifecycle rotten
• Offer to help out with Issue Triage

Please send feedback to sig-contributor-experience at kubernetes/community.

/close not-planned

[k8s-ci-robot]

@k8s-triage-robot: Closing this issue, marking it as "Not Planned".

In response to this:

The Kubernetes project currently lacks enough active contributors to adequately respond to all issues and PRs.

This bot triages issues according to the following rules:

• After 90d of inactivity, lifecycle/stale is applied
• After 30d of inactivity since lifecycle/stale was applied, lifecycle/rotten is applied
• After 30d of inactivity since lifecycle/rotten was applied, the issue is closed

You can:

• Reopen this issue with /reopen
• Mark this issue as fresh with /remove-lifecycle rotten
• Offer to help out with Issue Triage

Please send feedback to sig-contributor-experience at kubernetes/community.

/close not-planned

Instructions for interacting with me using PR comments are available here. If you have questions or suggestions related to my behavior, please file an issue against the kubernetes/test-infra repository.