From 4fb0ce6b1e019d0d0affa3a1fcbcb77855a1813f Mon Sep 17 00:00:00 2001 From: Bo Qiao Date: Tue, 7 Dec 2021 21:13:30 +0800 Subject: [PATCH] [ci] Auto generate manylinux dockerfile (#3699) * Add generation for manylinux dockerfile * Auto Format * Incorporate comments * Auto Format Co-authored-by: Taichi Gardener --- ...linux.cpu => Dockerfile.manylinux2014.cpu} | 14 ++- ci/Dockerfile.ubuntu.18.04 | 2 + ci/Dockerfile.ubuntu.18.04.cpu | 2 + ci/Dockerfile.ubuntu.20.04 | 2 + ci/Dockerfile.ubuntu.20.04.cpu | 2 + ci/Dockerfile_generator.py | 99 +++++++++++++++---- 6 files changed, 96 insertions(+), 25 deletions(-) rename ci/{Dockerfile.manylinux.cpu => Dockerfile.manylinux2014.cpu} (83%) diff --git a/ci/Dockerfile.manylinux.cpu b/ci/Dockerfile.manylinux2014.cpu similarity index 83% rename from ci/Dockerfile.manylinux.cpu rename to ci/Dockerfile.manylinux2014.cpu index ef3610dfa5e87..ceda86ee0901c 100644 --- a/ci/Dockerfile.manylinux.cpu +++ b/ci/Dockerfile.manylinux2014.cpu @@ -1,5 +1,6 @@ +# This file is generated by python Dockerfile_generator.py -o manylinux2014 -t cpu # Taichi Dockerfile (CPU only) for Manylinux2014 compliant -FROM quay.io/pypa/manylinux2014_x86_64 +FROM quay.io/pypa/manylinux2014_x86_64 LABEL maintainer="https://github.com/taichi-dev" @@ -12,17 +13,14 @@ RUN yum check-update && \ # Build LLVM/Clang 10 from source WORKDIR / RUN wget https://github.com/llvm/llvm-project/releases/download/llvmorg-10.0.0/llvm-10.0.0.src.tar.xz -RUN tar -xf llvm-10.0.0.src.tar.xz && \ - rm llvm-10.0.0.src.tar.xz +RUN tar -xf llvm-10.0.0.src.tar.xz && rm llvm-10.0.0.src.tar.xz RUN wget https://github.com/llvm/llvm-project/releases/download/llvmorg-10.0.0/clang-10.0.0.src.tar.xz -RUN tar -xf clang-10.0.0.src.tar.xz && \ - rm clang-10.0.0.src.tar.xz +RUN tar -xf clang-10.0.0.src.tar.xz && rm clang-10.0.0.src.tar.xz RUN cp -r clang-10.0.0.src llvm-10.0.0.src/tools/clang WORKDIR /llvm-10.0.0.src/build RUN cmake .. -DLLVM_ENABLE_RTTI:BOOL=ON -DBUILD_SHARED_LIBS:BOOL=OFF -DCMAKE_BUILD_TYPE=Release -DLLVM_TARGETS_TO_BUILD="X86" -DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_ENABLE_TERMINFO=OFF -RUN make -j 8 && \ - make install +RUN make -j 8 && make install ENV CC="/usr/local/bin/clang" ENV CXX="/usr/local/bin/clang++" @@ -49,7 +47,7 @@ RUN conda create -n py37 python=3.7 -y RUN conda create -n py38 python=3.8 -y RUN conda create -n py39 python=3.9 -y -# Load scripts for build the wheel +# Load scripts for build and test WORKDIR /home/dev/scripts COPY ci/scripts/manylinux_build_wheel.sh manylinux_build_wheel.sh diff --git a/ci/Dockerfile.ubuntu.18.04 b/ci/Dockerfile.ubuntu.18.04 index c9294ea3eb5a5..8f745a8875bc1 100644 --- a/ci/Dockerfile.ubuntu.18.04 +++ b/ci/Dockerfile.ubuntu.18.04 @@ -1,3 +1,4 @@ +# This file is generated by python Dockerfile_generator.py -o ubuntu -t gpu # Taichi Dockerfile for development FROM nvidia/cudagl:11.2.2-devel-ubuntu18.04 # Use 11.2 instead of 11.4 to avoid forward compatibility issue on Nvidia driver 460 @@ -5,6 +6,7 @@ FROM nvidia/cudagl:11.2.2-devel-ubuntu18.04 ENV NVIDIA_DRIVER_CAPABILITIES compute,graphics,utility ENV DEBIAN_FRONTEND=noninteractive + LABEL maintainer="https://github.com/taichi-dev" RUN apt-get update && \ diff --git a/ci/Dockerfile.ubuntu.18.04.cpu b/ci/Dockerfile.ubuntu.18.04.cpu index 60b733b9dc3ff..f8bf4cac36476 100644 --- a/ci/Dockerfile.ubuntu.18.04.cpu +++ b/ci/Dockerfile.ubuntu.18.04.cpu @@ -1,7 +1,9 @@ +# This file is generated by python Dockerfile_generator.py -o ubuntu -t cpu # Taichi Dockerfile for development FROM ubuntu:18.04 ENV DEBIAN_FRONTEND=noninteractive + LABEL maintainer="https://github.com/taichi-dev" RUN apt-get update && \ diff --git a/ci/Dockerfile.ubuntu.20.04 b/ci/Dockerfile.ubuntu.20.04 index c4d6ee246eaff..b38442b4bc486 100644 --- a/ci/Dockerfile.ubuntu.20.04 +++ b/ci/Dockerfile.ubuntu.20.04 @@ -1,3 +1,4 @@ +# This file is generated by python Dockerfile_generator.py -o ubuntu -t gpu # Taichi Dockerfile for development FROM nvidia/cudagl:11.2.2-devel-ubuntu20.04 # Use 11.2 instead of 11.4 to avoid forward compatibility issue on Nvidia driver 460 @@ -5,6 +6,7 @@ FROM nvidia/cudagl:11.2.2-devel-ubuntu20.04 ENV NVIDIA_DRIVER_CAPABILITIES compute,graphics,utility ENV DEBIAN_FRONTEND=noninteractive + LABEL maintainer="https://github.com/taichi-dev" RUN apt-get update && \ diff --git a/ci/Dockerfile.ubuntu.20.04.cpu b/ci/Dockerfile.ubuntu.20.04.cpu index 480740fb55016..23d228faa9500 100644 --- a/ci/Dockerfile.ubuntu.20.04.cpu +++ b/ci/Dockerfile.ubuntu.20.04.cpu @@ -1,7 +1,9 @@ +# This file is generated by python Dockerfile_generator.py -o ubuntu -t cpu # Taichi Dockerfile for development FROM ubuntu:20.04 ENV DEBIAN_FRONTEND=noninteractive + LABEL maintainer="https://github.com/taichi-dev" RUN apt-get update && \ diff --git a/ci/Dockerfile_generator.py b/ci/Dockerfile_generator.py index 5ba30223e7a4f..d75b2f45ab643 100644 --- a/ci/Dockerfile_generator.py +++ b/ci/Dockerfile_generator.py @@ -8,6 +8,7 @@ OS = { "windows": (), "macos": (), + "manylinux2014": ("", ), "ubuntu": ( "18.04", "20.04", @@ -15,15 +16,30 @@ } HARDWARE = ("cpu", "gpu") +HEAD_BLOCK = """# This file is generated by python Dockerfile_generator.py -o {os} -t {target} +""" + CPU_BASE_BLOCK = """# Taichi Dockerfile for development FROM {os}:{version} """ +CPU_MANYLINUX_BASE_BLOCK = """# Taichi Dockerfile (CPU only) for Manylinux2014 compliant +FROM quay.io/pypa/manylinux2014_x86_64 +""" + GPU_BASE_BLOCK = """# Taichi Dockerfile for development FROM nvidia/cudagl:11.2.2-devel-ubuntu{version} # Use 11.2 instead of 11.4 to avoid forward compatibility issue on Nvidia driver 460 """ +CPU_YUM_INSTALL_BLOCK = """ +RUN yum check-update && \\ + yum install -y git \\ + cmake \\ + wget \\ + libXrandr +""" + CPU_APT_INSTALL_BLOCK = """ RUN apt-get update && \\ apt-get install -y software-properties-common \\ @@ -77,8 +93,11 @@ ENV NVIDIA_DRIVER_CAPABILITIES compute,graphics,utility """ -MAINTAINER_BLOCK = """ +DEBIAN_NONINTERACTIVE_BLOCK = """ ENV DEBIAN_FRONTEND=noninteractive +""" + +MAINTAINER_BLOCK = """ LABEL maintainer="https://github.com/taichi-dev" """ @@ -104,6 +123,33 @@ ENV CXX="clang++-10" """ +LLVM_CLANG_FROM_SOURCE_BLOCK = """ +# Build LLVM/Clang 10 from source +WORKDIR / +RUN wget https://github.com/llvm/llvm-project/releases/download/llvmorg-10.0.0/llvm-10.0.0.src.tar.xz +RUN tar -xf llvm-10.0.0.src.tar.xz && \ + rm llvm-10.0.0.src.tar.xz +RUN wget https://github.com/llvm/llvm-project/releases/download/llvmorg-10.0.0/clang-10.0.0.src.tar.xz +RUN tar -xf clang-10.0.0.src.tar.xz && \ + rm clang-10.0.0.src.tar.xz +RUN cp -r clang-10.0.0.src llvm-10.0.0.src/tools/clang + +WORKDIR /llvm-10.0.0.src/build +RUN cmake .. -DLLVM_ENABLE_RTTI:BOOL=ON -DBUILD_SHARED_LIBS:BOOL=OFF -DCMAKE_BUILD_TYPE=Release -DLLVM_TARGETS_TO_BUILD="X86" -DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_ENABLE_TERMINFO=OFF +RUN make -j 8 && \ + make install +ENV CC="/usr/local/bin/clang" +ENV CXX="/usr/local/bin/clang++" +""" + +GCC_LINK_BLOCK = """ +# Link gcc 10 to build Taichi +WORKDIR /usr/lib/gcc/x86_64-redhat-linux/ +RUN ln -s /opt/rh/devtoolset-10/root/usr/lib/gcc/x86_64-redhat-linux/10 10 +# Check gcc-10 is used +RUN clang++ -v +""" + USER_BLOCK = """ # Create non-root user for running the container RUN useradd -ms /bin/bash dev @@ -227,29 +273,47 @@ def main(arguments=None): metavar="\b", ) args = parser.parse_args() - pwd = Path(__file__).resolve().parent + head_block = HEAD_BLOCK.format(os=args.os, target=args.target) + if args.target == "cpu": info("Generating Dockerfile(s) for CPU.") def f(os: str, version: str) -> str: info(f"OS: {os}, version: {version}") - base_block = CPU_BASE_BLOCK.format(os=os, version=version) - scripts_block = SCRIPTS_BLOCK.format( - script=f"{os}_build_test_cpu.sh") - install_block = CPU_APT_INSTALL_BLOCK - # ubuntu 18.04 needs special treatments - if os == "ubuntu" and version == "18.04": - install_block = install_block.rstrip() + """ \\ + if os == "manylinux2014": + base_block = CPU_MANYLINUX_BASE_BLOCK + install_block = CPU_YUM_INSTALL_BLOCK + scripts_block = SCRIPTS_BLOCK.format( + script=f"manylinux_build_wheel.sh") + + dockerfile = reduce( + lambda x, y: x + y, + (head_block, base_block, MAINTAINER_BLOCK, install_block, + LLVM_CLANG_FROM_SOURCE_BLOCK, GCC_LINK_BLOCK, USER_BLOCK, + CONDA_BLOCK, scripts_block)) + + filename = pwd / f"Dockerfile.{os}.cpu" + else: + base_block = CPU_BASE_BLOCK.format(os=os, version=version) + install_block = CPU_APT_INSTALL_BLOCK + scripts_block = SCRIPTS_BLOCK.format( + script=f"{os}_build_test_cpu.sh") + # ubuntu 18.04 needs special treatments + if os == "ubuntu" and version == "18.04": + install_block = install_block.rstrip() + """ \\ zlib1g-dev""" - dockerfile = reduce( - lambda x, y: x + y, - (base_block, MAINTAINER_BLOCK, install_block, CMAKE_BLOCK, - LLVM_BLOCK, USER_BLOCK, CONDA_BLOCK, scripts_block)) - filename = pwd / f"Dockerfile.{os}.{version}.cpu" + dockerfile = reduce( + lambda x, y: x + y, + (head_block, base_block, DEBIAN_NONINTERACTIVE_BLOCK, + MAINTAINER_BLOCK, install_block, CMAKE_BLOCK, LLVM_BLOCK, + USER_BLOCK, CONDA_BLOCK, scripts_block)) + + filename = pwd / f"Dockerfile.{os}.{version}.cpu" + info(f"Storing at: {filename}") with filename.open("w") as fp: fp.write(dockerfile) @@ -270,9 +334,10 @@ def f(os: str, version: str) -> str: dockerfile = reduce( lambda x, y: x + y, - (base_block, NVIDIA_DRIVER_CAPABILITIES_BLOCK, - MAINTAINER_BLOCK, install_block, CMAKE_BLOCK, LLVM_BLOCK, - VULKAN_BLOCK, USER_BLOCK, CONDA_BLOCK, scripts_block)) + (head_block, base_block, NVIDIA_DRIVER_CAPABILITIES_BLOCK, + DEBIAN_NONINTERACTIVE_BLOCK, MAINTAINER_BLOCK, install_block, + CMAKE_BLOCK, LLVM_BLOCK, VULKAN_BLOCK, USER_BLOCK, + CONDA_BLOCK, scripts_block)) filename = pwd / f"Dockerfile.{os}.{version}" info(f"Storing at: {filename}") with (filename).open("w") as fp: