Skip to content

Latest commit

 

History

History
818 lines (631 loc) · 25.5 KB

4.11.helper.node.oc.mirror.md

File metadata and controls

818 lines (631 loc) · 25.5 KB

helper node created

init setting for helper node

ssh -tt -D 8801 -R 18801:10.147.17.89:5085 root@172.21.6.11 'bash -l -c byobu'

# init setting for helper node
cat << EOF > ~/.ssh/config
StrictHostKeyChecking no
UserKnownHostsFile=/dev/null
EOF

chmod 600 ~/.ssh/config

cat << EOF >>  /etc/hosts
127.0.0.1 registry.ocp4.redhat.ren nexus.ocp4.redhat.ren git.ocp4.redhat.ren quaylab.infra.wzhlab.top
EOF

dnf -y install byobu htop jq ipmitool

systemctl disable --now firewalld

dnf groupinstall -y development server 'server with gui'

dnf -y install qemu-kvm libvirt libguestfs-tools virt-install virt-viewer virt-manager tigervnc-server

systemctl enable --now libvirtd

# create thin pool
pvcreate -y /dev/sdb
vgcreate vgdata /dev/sdb

# https://access.redhat.com/articles/766133
lvcreate -y -n poolA -L 500G vgdata
lvcreate -y -n poolA_meta -L 1G vgdata
lvconvert -y --thinpool vgdata/poolA --poolmetadata vgdata/poolA_meta
  # Thin pool volume with chunk size 64.00 KiB can address at most <15.88 TiB of data.
  # WARNING: Converting vgdata/poolA and vgdata/poolA_meta to thin pool's data and metadata volumes with metadata wiping.
  # THIS WILL DESTROY CONTENT OF LOGICAL VOLUME (filesystem etc.)
  # Converted vgdata/poolA and vgdata/poolA_meta to thin pool.

lvextend -l +100%FREE vgdata/poolA
  # Rounding size to boundary between physical extents: <1.09 GiB.
  # Size of logical volume vgdata/poolA_tmeta changed from 1.00 GiB (256 extents) to <1.09 GiB (279 extents).
  # Size of logical volume vgdata/poolA_tdata changed from 500.00 GiB (128000 extents) to <1.09 TiB (285457 extents).
  # Logical volume vgdata/poolA successfully resized.

setup ntp server on helper node

# setup ntp server on helper node
# sed -i "s/#allow.*/allow 192.168.0.0\/16/" /etc/chrony.conf
sed -i "s/#allow.*/allow all/" /etc/chrony.conf
systemctl enable --now chronyd

chronyc tracking
# Reference ID    : CA760182 (202.118.1.130)
# Stratum         : 2
# Ref time (UTC)  : Mon May 02 03:55:48 2022
# System time     : 0.000000530 seconds fast of NTP time
# Last offset     : -0.003027542 seconds
# RMS offset      : 0.003027542 seconds
# Frequency       : 36.009 ppm slow
# Residual freq   : +61.371 ppm
# Skew            : 25.290 ppm
# Root delay      : 0.016805194 seconds
# Root dispersion : 0.002184978 seconds
# Update interval : 0.8 seconds
# Leap status     : Normal

chronyc sources -v
#   .-- Source mode  '^' = server, '=' = peer, '#' = local clock.
#  / .- Source state '*' = current best, '+' = combined, '-' = not combined,
# | /             'x' = may be in error, '~' = too variable, '?' = unusable.
# ||                                                 .- xxxx [ yyyy ] +/- zzzz
# ||      Reachability register (octal) -.           |  xxxx = adjusted offset,
# ||      Log2(Polling interval) --.      |          |  yyyy = measured offset,
# ||                                \     |          |  zzzz = estimated error.
# ||                                 |    |           \
# MS Name/IP address         Stratum Poll Reach LastRx Last sample
# ===============================================================================
# ^- time.cloudflare.com           3   6    17    41    +28ms[  +28ms] +/-  157ms
# ^* 202.118.1.130                 1   6    17    42  -9871ns[-3037us] +/- 8572us
# ^- time.cloudflare.com           3   6    17    40    +35ms[  +35ms] +/-  162ms
# ^- makaki.miuku.net              2   6    17    40    +46ms[  +46ms] +/-  110ms

chronyc sourcestats -v
#                              .- Number of sample points in measurement set.
#                             /    .- Number of residual runs with same sign.
#                            |    /    .- Length of measurement set (time).
#                            |   |    /      .- Est. clock freq error (ppm).
#                            |   |   |      /           .- Est. error in freq.
#                            |   |   |     |           /         .- Est. offset.
#                            |   |   |     |          |          |   On the -.
#                            |   |   |     |          |          |   samples. \
#                            |   |   |     |          |          |             |
# Name/IP Address            NP  NR  Span  Frequency  Freq Skew  Offset  Std Dev
# ==============================================================================
# time.cloudflare.com         4   3     7  -1249.574  60710.363    -54ms  8100us
# 202.118.1.130               4   3     6    +61.371   5439.969  +3713us   581us
# time.cloudflare.com         4   3     7  -3223.009     204771   -185ms    29ms
# makaki.miuku.net            4   3     7  +6244.955  92563.305   +411ms    12ms

chronyc makestep
# 200 OK

setup network

cat << 'EOF' > /data/kvm/bridge.sh
#!/usr/bin/env bash

PUB_CONN='ens192'
PUB_IP='172.21.6.11/24'
PUB_GW='172.21.6.254'
PUB_DNS='172.21.1.1'

nmcli con down "$PUB_CONN"
nmcli con delete "$PUB_CONN"
nmcli con down baremetal
nmcli con delete baremetal
# RHEL 8.1 appends the word "System" in front of the connection,delete in case it exists
nmcli con down "System $PUB_CONN"
nmcli con delete "System $PUB_CONN"
nmcli connection add ifname baremetal type bridge con-name baremetal ipv4.method 'manual' \
    ipv4.address "$PUB_IP" \
    ipv4.gateway "$PUB_GW" \
    ipv4.dns "$PUB_DNS"
    
nmcli con add type bridge-slave ifname "$PUB_CONN" master baremetal
nmcli con down "$PUB_CONN";pkill dhclient;dhclient baremetal
nmcli con up baremetal
EOF

bash /data/kvm/bridge.sh

nmcli con mod baremetal -ipv4.address '192.168.7.11/24'
nmcli con mod baremetal +ipv4.address '192.168.77.11/24'
nmcli con mod baremetal ipv6.method manual +ipv6.address 'fd03::11/64'

nmcli con up baremetal
# nmcli networking off; nmcli networking on

set cert key

mkdir -p /etc/crts/ && cd /etc/crts

# https://access.redhat.com/documentation/en-us/red_hat_codeready_workspaces/2.1/html/installation_guide/installing-codeready-workspaces-in-tls-mode-with-self-signed-certificates_crw
openssl genrsa -out /etc/crts/wzhlab.top.ca.key 4096

openssl req -x509 \
  -new -nodes \
  -key /etc/crts/wzhlab.top.ca.key \
  -sha256 \
  -days 36500 \
  -out /etc/crts/wzhlab.top.ca.crt \
  -subj /CN="Local wzh lab Signer" \
  -reqexts SAN \
  -extensions SAN \
  -config <(cat /etc/pki/tls/openssl.cnf \
      <(printf '[SAN]\nbasicConstraints=critical, CA:TRUE\nkeyUsage=keyCertSign, cRLSign, digitalSignature'))

openssl genrsa -out /etc/crts/wzhlab.top.key 2048

openssl req -new -sha256 \
    -key /etc/crts/wzhlab.top.key \
    -subj "/O=Local wzh lab /CN=*.infra.wzhlab.top" \
    -reqexts SAN \
    -config <(cat /etc/pki/tls/openssl.cnf \
        <(printf "\n[SAN]\nsubjectAltName=DNS:*.infra.wzhlab.top,DNS:*.wzhlab.top\nbasicConstraints=critical, CA:FALSE\nkeyUsage=digitalSignature, keyEncipherment, keyAgreement, dataEncipherment\nextendedKeyUsage=serverAuth")) \
    -out /etc/crts/wzhlab.top.csr

openssl x509 \
    -req \
    -sha256 \
    -extfile <(printf "subjectAltName=DNS:*.infra.wzhlab.top,DNS:*.wzhlab.top\nbasicConstraints=critical, CA:FALSE\nkeyUsage=digitalSignature, keyEncipherment, keyAgreement, dataEncipherment\nextendedKeyUsage=serverAuth") \
    -days 36500 \
    -in /etc/crts/wzhlab.top.csr \
    -CA /etc/crts/wzhlab.top.ca.crt \
    -CAkey /etc/crts/wzhlab.top.ca.key \
    -CAcreateserial -out /etc/crts/wzhlab.top.crt

openssl x509 -in /etc/crts/wzhlab.top.crt -text

/bin/cp -f /etc/crts/wzhlab.top.ca.crt /etc/pki/ca-trust/source/anchors/
update-ca-trust extract

下载需要的 openshift4 软件

dnf -y install git 
cd /data
rm -rf /data/ocp4
# scripts can be found here:
# https://github.com/wangzheng422/openshift4-shell
bash helper.node.sh -v 4.10.12 -m 4.10 -f file

setup dns

# copy helper ansible project to /data/ocp4/ocp4-upi-helpernode-master

yum -y install ansible git unzip podman python3

mkdir -p /data/ocp4/ocp4-upi-helpernode-master

mkdir -p /data/sno

NODE_SSH_KEY="$(cat ~/.ssh/id_rsa.pub)"
INSTALL_IMAGE_REGISTRY=quaylab.infra.redhat.ren:8443

PULL_SECRET='{"auths":{"registry.redhat.io": {"auth": "ZHVtbXk6ZHVtbXk=","email": "noemail@localhost"},"registry.ocp4.redhat.ren:5443": {"auth": "ZHVtbXk6ZHVtbXk=","email": "noemail@localhost"},"'${INSTALL_IMAGE_REGISTRY}'": {"auth": "'$( echo -n 'admin:shadowman' | openssl base64 )'","email": "noemail@localhost"}}}'

NTP_SERVER=192.168.7.11
HELP_SERVER=192.168.7.11
KVM_HOST=192.168.7.11
API_VIP=192.168.7.100
INGRESS_VIP=192.168.7.101
CLUSTER_PROVISION_IP=192.168.7.103
BOOTSTRAP_IP=192.168.7.12

ACM_DEMO_MNGED_CLUSTER=acm-demo-man01
ACM_DEMO_MNGED_SNO_IP=192.168.7.23

echo $PULL_SECRET

# 定义单节点集群的节点信息
SNO_CLUSTER_NAME=acm-demo-hub
SNO_BASE_DOMAIN=redhat.ren
SNO_IP=192.168.7.13
SNO_GW=192.168.7.11
SNO_NETMAST=255.255.255.0
SNO_NETMAST_S=24
SNO_HOSTNAME=acm-demo-hub-master
SNO_IF=enp1s0
SNO_IF_MAC=`printf '00:60:2F:%02X:%02X:%02X' $[RANDOM%256] $[RANDOM%256] $[RANDOM%256]`
SNO_DNS=192.168.7.11
SNO_DISK=/dev/vda
SNO_CORE_PWD=redhat

# echo ${SNO_IF_MAC} > /data/sno/sno.mac

cd /data/ocp4/ansible-helper

cat > var.yaml << EOF
helper:
  ip_addr: 192.168.7.11
  nic: baremetal
pdns:
  bind: 0.0.0.0
  port: 5301
  recursor_port: 53
  forward: 172.21.1.1
  static:
    - base_domain: infra.redhat.ren
      record:
        - name: registry
          ip_addr: 192.168.7.11
        - name: nexus
          ip_addr: 192.168.7.11
        - name: quay
          ip_addr: 192.168.7.11
        - name: quaylab
          ip_addr: 192.168.7.11
ntp:
  server: 192.168.7.11
cluster:
  - base_domain: acm-demo-hub.redhat.ren
    node:
      # - ip_addr: 192.168.7.12
      - ip_addr: 192.168.7.13
        name: sno-master-01
      # - ip_addr: 192.168.7.14
      # - ip_addr: 192.168.7.15
  - base_domain: acm-demo-one.redhat.ren
    node: 
      - ip_addr: 192.168.7.22
        name: one-bootstrap
      - ip_addr: 192.168.7.23
        name: one-master-01
      - ip_addr: 192.168.7.24
        name: one-master-02
      - ip_addr: 192.168.7.25
        name: one-master-03
  - base_domain: acm-demo-two.redhat.ren
    node: 
      - ip_addr: 192.168.7.33
        name: two-bootstrap
      - ip_addr: 192.168.7.34
        name: two-master-01
      - ip_addr: 192.168.7.35
        name: two-master-02
      - ip_addr: 192.168.7.36
        name: two-master-03
ptr: 
  - addr: 192.168.7
    domain: ptr.redhat.ren
EOF

cd /data/ocp4/ansible-helper
ansible-playbook -e @var.yaml  helper.yaml


# dig @127.0.0.1 -p 5301 -x 192.168.7.23

配置quay镜像仓库

using official version

# https://docs.openshift.com/container-platform/4.10/installing/disconnected_install/installing-mirroring-creating-registry.html

mkdir -p /data/quay 
cd /data/ocp4/clients
tar zvxf mirror-registry.tar.gz
./mirror-registry install -v \
  --initPassword redhatadmin --initUser admin \
  --quayHostname quaylab.infra.redhat.ren --quayRoot /data/quay \
  --targetHostname quaylab.infra.redhat.ren \
  --sslKey /etc/crts/redhat.ren.key --sslCert /etc/crts/redhat.ren.crt
# PLAY RECAP ******************************************************************************************************************************************************************************************$root@quaylab.infra.redhat.ren : ok=52   changed=20   unreachable=0    failed=0    skipped=7    rescued=0    ignored=0

# INFO[2022-05-02 18:01:10] Quay installed successfully, permanent data is stored in /data/quay
# INFO[2022-05-02 18:01:10] Quay is available at https://quaylab.infra.redhat.ren:8443 with credentials (admin, shadowman) 

ls -hl /data/quay
# total 4.1G
# -rw-r--r--. 1 root root 2.1G May  2 17:28 image-archive.tar
# -rw-r--r--. 1 root root 3.4M Mar  9 21:52 pause.tar
# drwxrwxr-x+ 3 root root   22 May  2 17:29 pg-data
# -rw-r--r--. 1 root root 585M Mar  9 21:54 postgres.tar
# drwxr-xr-x. 2 root root   90 May  2 18:00 quay-config
# drwxr-xr-x. 2 root root   60 May  2 17:29 quay-rootCA
# drwxrwxr-x+ 2 root root    6 May  2 17:29 quay-storage
# -rw-r--r--. 1 root root 1.1G Mar  9 21:53 quay.tar
# -rw-r--r--. 1 root root 430M Mar  9 21:54 redis.tar

# to uninstall, do not use in setup
./mirror-registry uninstall -v \
  --autoApprove true --quayRoot /data/quay \
  --targetHostname quaylab.infra.redhat.ren \
  --

# https://quaylab.infra.redhat.ren:8443/

using upstream

# https://github.com/quay/mirror-registry/releases/download/1.2.9/mirror-registry-online.tar.gz

# rm -rf /data/quay
mkdir -p /data/quay 
cd /data/swap

export http_proxy="http://127.0.0.1:18801"
export https_proxy=${http_proxy}

wget -O mirror-registry-online.tar.gz https://github.com/quay/mirror-registry/releases/download/1.2.9/mirror-registry-online.tar.gz

unset http_proxy
unset https_proxy

tar zvxf mirror-registry-online.tar.gz

./mirror-registry install -v \
  --initPassword redhatadmin --initUser admin \
  --quayHostname quaylab.infra.wzhlab.top --quayRoot /data/quay \
  --targetHostname quaylab.infra.wzhlab.top \
  --sslKey /etc/crts/infra.wzhlab.top.key --sslCert /etc/crts/infra.wzhlab.top.crt
# PLAY RECAP ****************************************************************************************************************************************************************root@quaylab.infra.redhat.ren : ok=39   changed=21   unreachable=0    failed=0    skipped=20   rescued=0    ignored=0

# INFO[2022-12-13 13:55:24] Quay installed successfully, permanent data is stored in /data/quay
# INFO[2022-12-13 13:55:24] Quay is available at https://quaylab.infra.redhat.ren:8443 with credentials (admin, redhatadmin)

# to uninstall, do not use in setup
./mirror-registry uninstall -v \
  --autoApprove true --quayRoot /data/quay \
  --targetHostname quaylab.infra.wzhlab.top \
  --


ls -hl /data/quay
# total 0
# drwxrwxr-x+ 3 root root 22 Dec 10 00:08 pg-data
# drwxr-xr-x. 2 root root 56 Dec 10 00:08 quay-config
# drwxrwxr-x+ 2 root root  6 Dec 10 00:08 quay-storage

cat << EOF >> /data/quay/quay-config/config.yaml

FEATURE_PROXY_CACHE: true
BROWSER_API_CALLS_XHR_ONLY: false
FEATURE_QUOTA_MANAGEMENT: false

EOF

# sed -i 's/DEFAULT_TAG_EXPIRATION:.*/DEFAULT_TAG_EXPIRATION: 4w/' /data/quay/quay-config/config.yaml

cat << EOF > /etc/systemd/system/quay-app.service
[Unit]
Description=Quay Container
Wants=network.target
After=network-online.target quay-pod.service quay-postgres.service quay-redis.service
Requires=quay-pod.service quay-postgres.service quay-redis.service

[Service]
Type=simple
TimeoutStartSec=5m
ExecStartPre=-/bin/rm -f %t/%n-pid %t/%n-cid

# ExecStart=/usr/bin/podman run \
#     --name quay-app \
#     -v /data/quay/quay-config:/quay-registry/conf/stack:Z \
#     -v /data/quay/quay-storage:/datastorage:Z \
#     --pod=quay-pod \
#     --conmon-pidfile %t/%n-pid \
#     --cidfile %t/%n-cid \
#     --cgroups=no-conmon \
#     --replace \
#     -e HTTP_PROXY=http://192.168.77.11:18801 \
#     -e HTTPS_PROXY=http://192.168.77.11:18801 \
#     -e NO_PROXY=localhost,127.0.0.1,10.1.0.0/16,172.30.0.0/16,192.168.0.0/16 \
#     registry.redhat.io/quay/quay-rhel8:v3.7.11

ExecStart=/usr/bin/podman run \
    --name quay-app \
    -v /data/quay/quay-config:/quay-registry/conf/stack:Z \
    -v /data/quay/quay-storage:/datastorage:Z \
    --pod=quay-pod \
    --conmon-pidfile %t/%n-pid \
    --cidfile %t/%n-cid \
    --cgroups=no-conmon \
    --replace \
    registry.redhat.io/quay/quay-rhel8:v3.7.11

ExecStop=-/usr/bin/podman stop --ignore --cidfile %t/%n-cid -t 10
ExecStopPost=-/usr/bin/podman rm --ignore -f --cidfile %t/%n-cid
PIDFile=%t/%n-pid
KillMode=none
Restart=always
RestartSec=30

[Install]
WantedBy=multi-user.target default.target
EOF

systemctl daemon-reload

systemctl restart quay-app

using docker registry

/bin/rm -rf /data/registry
mkdir -p /data/registry

podman run --restart always --name local-registry -p 5443:5443 \
  -d --restart=always \
  -v /data/registry/:/var/lib/registry:z \
  -v /etc/crts:/certs:z \
  -e REGISTRY_HTTP_ADDR=0.0.0.0:5443 \
  -e REGISTRY_HTTP_TLS_CERTIFICATE=/certs/wzhlab.top.crt \
  -e REGISTRY_HTTP_TLS_KEY=/certs/wzhlab.top.key \
  docker.io/library/registry:2

podman run --restart always --name local-registry -p 5443:5443 \
  -d --restart=always \
  -v /data/registry/:/var/lib/registry:z \
  -v /etc/crts:/certs:z \
  -e REGISTRY_HTTP_ADDR=0.0.0.0:5443 \
  -e REGISTRY_HTTP_TLS_CERTIFICATE=/certs/infra.wzhlab.top.crt \
  -e REGISTRY_HTTP_TLS_KEY=/certs/infra.wzhlab.top.key \
  docker.io/library/registry:2

podman generate systemd --files --name local-registry
# /root/container-local-registry.service
/bin/cp -Zf container-local-registry.service   /etc/systemd/system/

systemctl daemon-reload

systemctl enable --now container-local-registry.service
systemctl status container-local-registry.service



cat << EOF > /etc/systemd/system/container-local-registry.service
[Unit]
Description=local-registry
Wants=network.target
After=network-online.target 

[Service]
Type=simple
TimeoutStartSec=5m
ExecStartPre=-/bin/rm -f %t/%n-pid %t/%n-cid
ExecStart=/usr/bin/podman run --name local-registry -p 5443:5443 \
  -v /data/registry/:/var/lib/registry:z \
  -v /etc/crts:/certs:z \
  --conmon-pidfile %t/%n-pid \
  --cidfile %t/%n-cid \
  --replace \
  --privileged \
  -e REGISTRY_HTTP_ADDR=0.0.0.0:5443 \
  -e REGISTRY_HTTP_TLS_CERTIFICATE=/certs/infra.wzhlab.top.crt \
  -e REGISTRY_HTTP_TLS_KEY=/certs/infra.wzhlab.top.key \
  docker.io/library/registry:2

ExecStop=-/usr/bin/podman stop --ignore --cidfile %t/%n-cid -t 10
ExecStopPost=-/usr/bin/podman rm --ignore -f --cidfile %t/%n-cid
PIDFile=%t/%n-pid
KillMode=none
Restart=always
RestartSec=30

[Install]
WantedBy=multi-user.target default.target
EOF

systemctl daemon-reload

systemctl restart container-local-registry

download client binary

export BUILDNUMBER=4.11.21

mkdir -p /data/ocp4/${BUILDNUMBER}
cd /data/ocp4/${BUILDNUMBER}

wget -O openshift-client-linux-${BUILDNUMBER}.tar.gz https://mirror.openshift.com/pub/openshift-v4/x86_64/clients/ocp/${BUILDNUMBER}/openshift-client-linux-${BUILDNUMBER}.tar.gz
wget -O openshift-install-linux-${BUILDNUMBER}.tar.gz https://mirror.openshift.com/pub/openshift-v4/x86_64/clients/ocp/${BUILDNUMBER}/openshift-install-linux-${BUILDNUMBER}.tar.gz
wget -O oc-mirror.tar.gz https://mirror.openshift.com/pub/openshift-v4/x86_64/clients/ocp/${BUILDNUMBER}/oc-mirror.tar.gz

tar -xzf openshift-client-linux-${BUILDNUMBER}.tar.gz -C /usr/local/bin/
tar -xzf openshift-install-linux-${BUILDNUMBER}.tar.gz -C /usr/local/bin/

tar -xzf oc-mirror.tar.gz -C /usr/local/bin/
chmod +x /usr/local/bin/oc-mirror

import ocp content into quay

export BUILDNUMBER=4.11.19

pushd /data/ocp4/${BUILDNUMBER}
tar -xzf openshift-client-linux-${BUILDNUMBER}.tar.gz -C /usr/local/bin/
tar -xzf openshift-install-linux-${BUILDNUMBER}.tar.gz -C /usr/local/bin/
tar -xzf oc-mirror.tar.gz -C /usr/local/bin/
chmod +x /usr/local/bin/oc-mirror
install -m 755 /data/ocp4/clients/butane-amd64 /usr/local/bin/butane
install -m 755 /data/ocp4/clients/coreos-installer_amd64 /usr/local/bin/coreos-installer
popd

# cd /data/swap
# wget -O oc-mirror.tar.gz https://mirror.openshift.com/pub/openshift-v4/x86_64/clients/ocp/${BUILDNUMBER}/oc-mirror.tar.gz
# tar -xzf oc-mirror.tar.gz -C /usr/local/bin/
# chmod +x /usr/local/bin/oc-mirror


SEC_FILE="$XDG_RUNTIME_DIR/containers/auth.json"
# $XDG_RUNTIME_DIR/containers
# echo $XDG_RUNTIME_DIR
# /run/user/0
# export XDG_RUNTIME_DIR=/run/user/0
mkdir -p ${SEC_FILE%/*}

# copy the password file 
# vi $SEC_FILE

# OR
# vi ~/.docker/config.json
SEC_FILE="$HOME/.docker/config.json"
mkdir -p ${SEC_FILE%/*}

# podman login quaylab.infra.wzhlab.top:8443 --username admin --password redhatadmin

# oc mirror init --registry quaylab.infra.redhat.ren:8443/mirror/oc-mirror-metadata > /data/ocp4/mirror.yaml

cat > /data/ocp4/mirror.yaml << EOF
apiVersion: mirror.openshift.io/v1alpha2
kind: ImageSetConfiguration
# archiveSize: 4
mirror:
  platform:
    architectures:
      - amd64
      # - arm64
    channels:
      - name: stable-4.11
        type: ocp
        minVersion: 4.11.21
        maxVersion: 4.11.21
        shortestPath: true
      # - name: stable-4.10
      #   type: ocp
      #   minVersion: 4.10.45
      #   maxVersion: 4.10.45
      #   shortestPath: true
    graph: false
  # additionalImages:
  #   - name: registry.redhat.io/redhat/redhat-operator-index:v4.10
  #   - name: registry.redhat.io/redhat/certified-operator-index:v4.10
  #   - name: registry.redhat.io/redhat/community-operator-index:v4.10
  #   - name: registry.redhat.io/redhat/redhat-marketplace-index:v4.10 
EOF

mkdir -p /data/install/mirror-tmp
cd /data/install/mirror-tmp

oc-mirror --config /data/ocp4/mirror.yaml docker://quaylab.infra.wzhlab.top:8443

注意，默认情况下，他会创建如下几个repo, quay默认是私有的，我们要手动把他们搞成public的。

  • openshift/release
  • operator-framework/opm (这个其实没啥用)

setup nexus

## import nexus fs
mkdir -p /data/ccn
cd /data/ccn

podman create --name swap quay.io/wangzheng422/qimgs:nexus-fs-image-2022-01-14-2155 ls
podman cp swap:/nexus-image.tgz - > /data/ccn/nexus-image.tgz.tar
podman rm -fv swap
tar vxf nexus-image.tgz.tar
tar zxf nexus-image.tgz
rm -f nexus-image.tgz*

chown -R 200 /data/ccn/nexus-image

## run the nexus for image
podman run -d -p 8082:8081 -p 8083:8083 -it --name nexus-image -v /data/ccn/nexus-image:/nexus-data:Z docker.io/sonatype/nexus3:3.38.1
# podman run -d -p 8082:8081 -p 8083:8083 -it --name nexus-image --privileged --cap-add all  -v /home/ccn/nexus-image:/nexus-data:Z docker.io/sonatype/nexus3:3.38.1
# podman run -d -p 8082:8081 -p 8083:8083 -it --name nexus-image -v /data/ccn/nexus-image:/nexus-data:Z docker.io/sonatype/nexus3:3.33.1

podman generate systemd --files --name nexus-image
# /root/container-local-registry.service
/bin/cp -Zf container-nexus-image.service   /etc/systemd/system/

systemctl daemon-reload

systemctl enable --now container-nexus-image.service
systemctl status container-nexus-image.service

# get the admin password
cat /data/ccn/nexus-image/admin.password && echo
# 84091bcd-c82f-44a3-8b7b-dfc90f5b7da1

# open http://nexus.infra.redhat.ren:8082

setup assisted install service ( AIS )

# https://github.com/openshift/assisted-service/blob/master/docs/user-guide/assisted-service-on-local.md

# https://github.com/openshift/assisted-service/tree/master/deploy/podman

podman version
# Version:      3.4.2
# API Version:  3.4.2
# Go Version:   go1.16.12
# Built:        Wed Feb  2 07:59:28 2022
# OS/Arch:      linux/amd64

/bin/cp -f /data/ocp4/rhcos-live.x86_64.iso /var/www/html/install/

mkdir -p /data/assisted-service/
cd /data/assisted-service/

export http_proxy="http://127.0.0.1:18801"
export https_proxy=${http_proxy}

wget https://raw.githubusercontent.com/openshift/assisted-service/master/deploy/podman/configmap.yml
wget https://raw.githubusercontent.com/openshift/assisted-service/master/deploy/podman/pod.yml

/bin/cp -f configmap.yml configmap.yml.bak
/bin/cp -f pod.yml pod.yml.bak

unset http_proxy
unset https_proxy

sed -i 's/ SERVICE_BASE_URL:.*/ SERVICE_BASE_URL: "http:\/\/172.21.6.11:8090"/' configmap.yml

cat /data/ocp4/4.10.12/release.txt | grep " machine-os "
  # machine-os 410.84.202204261500-0 Red Hat Enterprise Linux CoreOS

cat << EOF > /data/assisted-service/os_image.json
[{
  "openshift_version": "4.10",
  "cpu_architecture": "x86_64",
  "url": "http://172.21.6.11:8080/install/rhcos-live.x86_64.iso",
  "rootfs_url": "http://172.21.6.11:8080/install/rootfs.img",
  "version": "410.84.202204261500-0"
}]
EOF
cat << EOF > /data/assisted-service/release.json
[{
  "openshift_version": "4.10",
  "cpu_architecture": "x86_64",
  "url": "quaylab.infra.redhat.ren/ocp4/openshift4:4.10.12-x86_64",
  "version": "4.10.12",
  "default": true
}]
EOF

cat configmap.yml.bak \
  | python3 -c 'import json, yaml, sys; print(json.dumps(yaml.load(sys.stdin)))' \
  | jq --arg OSIMAGE "$(jq -c . /data/assisted-service/os_image.json)" '. | .data.OS_IMAGES = $OSIMAGE ' \
  | jq --arg RELEASE_IMAGES "$(jq -c . /data/assisted-service/release.json)" '. | .data.RELEASE_IMAGES = $RELEASE_IMAGES ' \
  | python3 -c 'import yaml, sys; print(yaml.dump(yaml.load(sys.stdin), default_flow_style=False))' \
  > configmap.yml

cat pod.yml.bak \
  | python3 -c 'import json, yaml, sys; print(json.dumps(yaml.load(sys.stdin)))' \
  | jq ' .spec.containers[1].ports[0].hostPort = 8180 ' \
  | python3 -c 'import yaml, sys; print(yaml.dump(yaml.load(sys.stdin), default_flow_style=False))' \
  > pod.yml

# 启动本地assisted service
cd /data/assisted-service/
podman play kube --configmap configmap.yml pod.yml

# 注入离线镜像仓库的证书
podman cp /etc/crts/redhat.ren.ca.crt assisted-installer-service:/etc/pki/ca-trust/source/anchors/quaylab.crt
podman exec assisted-installer-service update-ca-trust

# 用以下命令，停止/删除本地assisted service
cd /data/assisted-service/
podman play kube --down pod.yml

# assisted service启动以后，会下载安装介质，我们看看下载占用的空间。
podman exec assisted-installer-image-service du -h /data
# 1.1G    /data

准备vnc环境

dnf -y install tigervnc-server

vncpasswd

cat << EOF > ~/.vnc/config
session=gnome
securitytypes=vncauth,tlsvnc
# desktop=sandbox
geometry=1440x855
alwaysshared
EOF

cat << EOF >> /etc/tigervnc/vncserver.users
:2=root
EOF

systemctl start vncserver@:2
# 如果你想停掉vnc server，这么做
systemctl stop vncserver@:2

systemctl restart vncserver@:2

systemctl enable --now vncserver@:2

# firewall-cmd --permanent --add-port=6001/tcp
# firewall-cmd --permanent --add-port=5901/tcp
# firewall-cmd --reload

# connect vnc at port 5901
# export DISPLAY=:1

vncserver :1 -geometry 1280x800

ausearch -m avc --start recent -i

restorecon -RFv ~/.vnc

audit2allow -a -M wzh-tigervnc
semodule -i wzh-tigervnc.pp


python2 -m SimpleHTTPServer 5180

python3 -m http.server 5180