diff --git a/deploy/contents/k8s/cp-deployment-autoscaler/README.md b/deploy/contents/k8s/cp-deployment-autoscaler/README.md index bf278127a2..ca9fbb3f6b 100644 --- a/deploy/contents/k8s/cp-deployment-autoscaler/README.md +++ b/deploy/contents/k8s/cp-deployment-autoscaler/README.md @@ -193,7 +193,10 @@ Deployment autoscaler parameter descriptions can be found in the following code "kube_ip": "123.45.6.789", "kube_port": "6443", "kube_dns_ip": "10.96.0.10", - "aws_fs_url": "fs-12345678901234567.fsx.eu-central-1.amazonaws.com@tcp:/12345678" + "aws_fs_url": "fs-12345678901234567.fsx.eu-central-1.amazonaws.com@tcp:/12345678", + "http_proxy": "", + "https_proxy": "", + "no_proxy": "" }, "timeout": { // Specifies node scaling ↑ polling timeout. diff --git a/deploy/contents/k8s/cp-deployment-autoscaler/config.json b/deploy/contents/k8s/cp-deployment-autoscaler/config.json index eb6e134433..71964490ea 100644 --- a/deploy/contents/k8s/cp-deployment-autoscaler/config.json +++ b/deploy/contents/k8s/cp-deployment-autoscaler/config.json @@ -77,7 +77,10 @@ "kube_ip": "123.45.6.789", "kube_port": "6443", "kube_dns_ip": "10.96.0.10", - "aws_fs_url": "fs-12345678901234567.fsx.eu-central-1.amazonaws.com@tcp:/12345678" + "aws_fs_url": "fs-12345678901234567.fsx.eu-central-1.amazonaws.com@tcp:/12345678", + "http_proxy": "", + "https_proxy": "", + "no_proxy": "" }, "timeout": { "scale_up_node_timeout": 900, diff --git a/deploy/docker/cp-deployment-autoscaler/autoscaler/config.py b/deploy/docker/cp-deployment-autoscaler/autoscaler/config.py index 7ac501eba6..5487f5010a 100644 --- a/deploy/docker/cp-deployment-autoscaler/autoscaler/config.py +++ b/deploy/docker/cp-deployment-autoscaler/autoscaler/config.py @@ -53,7 +53,8 @@ class UnsupportedCloudProviderConfigurationError(RuntimeError): 'cloud, region, image, type, disk, sshkey, subnet, name, ' 'security_groups, role, init_script') KubeNodeConfiguration = collections.namedtuple('KubeNodeConfiguration', - 'kube_token, kube_ip, kube_port, kube_dns_ip, aws_fs_url') + 'kube_token, kube_ip, kube_port, kube_dns_ip, aws_fs_url, ' + 'http_proxy, https_proxy, no_proxy') TimeoutConfiguration = collections.namedtuple('TimeoutConfiguration', 'scale_up_node_timeout, scale_up_node_delay, ' 'scale_up_instance_timeout, scale_up_instance_delay, ' @@ -219,7 +220,10 @@ def refresh(self): kube_ip=self._get_string(configuration, 'node.kube_ip'), kube_port=self._get_string(configuration, 'node.kube_port'), kube_dns_ip=self._get_string(configuration, 'node.kube_dns_ip'), - aws_fs_url=self._get_string(configuration, 'node.aws_fs_url')) + aws_fs_url=self._get_string(configuration, 'node.aws_fs_url'), + http_proxy=self._get_string(configuration, 'node.http_proxy', ''), + https_proxy=self._get_string(configuration, 'node.https_proxy', ''), + no_proxy=self._get_string(configuration, 'node.no_proxy', '')) self._timeout = TimeoutConfiguration( scale_up_node_timeout=self._get_number(configuration, 'timeout.scale_up_node_timeout', 15 * 60), scale_up_node_delay=self._get_number(configuration, 'timeout.scale_up_node_delay', 10), diff --git a/deploy/docker/cp-deployment-autoscaler/autoscaler/instance/aws.py b/deploy/docker/cp-deployment-autoscaler/autoscaler/instance/aws.py index 3598e24f4a..a2dc3fa6f0 100644 --- a/deploy/docker/cp-deployment-autoscaler/autoscaler/instance/aws.py +++ b/deploy/docker/cp-deployment-autoscaler/autoscaler/instance/aws.py @@ -52,7 +52,10 @@ def launch_instance(self): .replace('@KUBE_TOKEN@', self._configuration.node.kube_token) \ .replace('@KUBE_DNS_IP@', self._configuration.node.kube_dns_ip) \ .replace('@KUBE_LABELS@', kube_labels_string) \ - .replace('@AWS_FS_URL@', self._configuration.node.aws_fs_url) + .replace('@AWS_FS_URL@', self._configuration.node.aws_fs_url) \ + .replace('@HTTP_PROXY@', self._configuration.node.http_proxy) \ + .replace('@HTTPS_PROXY@', self._configuration.node.https_proxy) \ + .replace('@NO_PROXY@', self._configuration.node.no_proxy) compressed_user_data_script = pack_script_contents(user_data_script) raw_tags = self._merge_dicts({'Name': self._configuration.instance.name}, self._configuration.target.tags, diff --git a/deploy/docker/cp-deployment-autoscaler/init_multicloud.sh b/deploy/docker/cp-deployment-autoscaler/init_multicloud.sh index 6850d41ed8..7d37c9588b 100644 --- a/deploy/docker/cp-deployment-autoscaler/init_multicloud.sh +++ b/deploy/docker/cp-deployment-autoscaler/init_multicloud.sh @@ -1,5 +1,37 @@ #!/bin/bash +function update_nameserver { + local nameserver="$1" + local ping_times="$2" + + local is_nameserver_reachable="0" + if [ "$nameserver" ] && [[ "$nameserver" != "@"*"@" ]]; then + if [ "$ping_times" ]; then + if [ "$ping_times" == "infinity" ]; then + ping_times=86400 + fi + for i in $(seq 1 $ping_times); do + echo "Pinging nameserver $nameserver on port 53" + if nc -z -w 1 $nameserver 53 ; then + echo "nameserver $nameserver can be reached on port 53" + is_nameserver_reachable="1" + break + fi + done + + if [ "$is_nameserver_reachable" != "1" ]; then + echo "Elapsed $ping_times retries, but $nameserver can NOT be reached on port 53" + fi + fi + + cp /etc/resolv.conf /etc/resolv.conf.backup + chattr -i /etc/resolv.conf + sed -i '/nameserver/d' /etc/resolv.conf + echo "nameserver $nameserver" >> /etc/resolv.conf + chattr +i /etc/resolv.conf + fi +} + user_data_log="/var/log/user_data.log" exec > "$user_data_log" 2>&1 @@ -9,6 +41,9 @@ export KUBE_TOKEN="@KUBE_TOKEN@" export KUBE_DNS_IP="@KUBE_DNS_IP@" export KUBE_LABELS="@KUBE_LABELS@" export AWS_FS_URL="@AWS_FS_URL@" +export http_proxy="@HTTP_PROXY@" +export https_proxy="@HTTPS_PROXY@" +export no_proxy="@NO_PROXY@" mkdir -p /etc/docker cat < /etc/docker/daemon.json @@ -18,6 +53,12 @@ cat < /etc/docker/daemon.json } EOT +mkdir -p /etc/systemd/system/docker.service.d +cat > /etc/systemd/system/docker.service.d/http-proxy.conf << EOF + [Service] + Environment="http_proxy=$http_proxy" "https_proxy=$https_proxy" "no_proxy=$no_proxy" +EOF + echo "KUBELET_EXTRA_ARGS=--node-labels $KUBE_LABELS" >> /etc/sysconfig/kubelet systemctl daemon-reload @@ -41,22 +82,19 @@ fi kubeadm join --token "$KUBE_TOKEN" "$KUBE_IP:$KUBE_PORT" --discovery-token-unsafe-skip-ca-verification --ignore-preflight-errors all --node-name "$_KUBE_NODE_NAME" systemctl start kubelet -if ! grep "$KUBE_DNS_IP" /etc/resolv.conf -q; then - chattr -i /etc/resolv.conf - sed -i "1s/^/nameserver $KUBE_DNS_IP\n/" /etc/resolv.conf - chattr +i /etc/resolv.conf -fi +yum install -y nc +update_nameserver "$KUBE_DNS_IP" "infinity" if [[ $cloud == *"EC2"* ]]; then - amazon-linux-extras install -y lustre2.10 - yum install -y lustre-client --disablerepo=kubernetes - mkdir -p /opt - mount -t lustre -o noatime,flock "$AWS_FS_URL" /opt - echo "$AWS_FS_URL /opt lustre defaults,noatime,flock,_netdev 0 0" >> /etc/fstab + amazon-linux-extras install -y lustre2.10 + yum install -y lustre-client --disablerepo=kubernetes + mkdir -p /opt + mount -t lustre -o noatime,flock "$AWS_FS_URL" /opt + echo "$AWS_FS_URL /opt lustre defaults,noatime,flock,_netdev 0 0" >> /etc/fstab elif [[ $cloud == *"Microsoft"* ]]; then - echo "WARNING: Azure shared file system mounting is not yet supported." - # todo: Implement + echo "WARNING: Azure shared file system mounting is not yet supported." + # todo: Implement elif [[ $gcloud_header == *"Google"* ]]; then - echo "WARNING: Google Cloud shared file system mounting is not yet supported." - # todo: Implement + echo "WARNING: Google Cloud shared file system mounting is not yet supported." + # todo: Implement fi