diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000000000..d82e934d21f9c --- /dev/null +++ b/.travis.yml @@ -0,0 +1,78 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +dist: bionic + +language: minimal + +addons: + apt: + packages: + - python3-pip + +services: + - docker + +jobs: + include: + - name: "C++ on s390x" + os: linux + arch: s390x + env: + ARCH: s390x + ARROW_CI_MODULES: "CPP" + DOCKER_IMAGE_ID: ubuntu-cpp + UBUNTU: "20.04" + allow_failures: + - arch: s390x + +env: + DOCKER_BUILDKIT: 0 + COMPOSE_DOCKER_CLI_BUILD: 1 + ARROW_ENABLE_TIMING_TESTS: "OFF" + +before_install: + - eval "$(python ci/detect-changes.py)" + - | + arrow_ci_affected=no + for arrow_ci_module in ${ARROW_CI_MODULES}; do + arrow_ci_affected_variable=ARROW_CI_${arrow_ci_module}_AFFECTED + if [ "$(eval 'echo ${arrow_ci_affected_variable}')" != "1" ]; then + arrow_ci_affected=yes + fi + done + if [ "${arrow_ci_affected}" = "no" ]; then + travis_terminate 0 + fi + +install: + - pip3 install -e dev/archery[docker] + +script: + - sudo sysctl -w kernel.core_pattern="core.%e.%p" + # This isn't allowed on Travis CI: + # /home/travis/.travis/functions: line 109: ulimit: core file size: cannot modify limit: Operation not permitted + - | + ulimit -c unlimited || : + - | + archery docker run \ + --volume ${PWD}/build:/build \ + ${DOCKER_IMAGE_ID} + +after_success: + - | + archery docker push ${DOCKER_IMAGE_ID} || : diff --git a/ci/docker/ubuntu-20.04-cpp.dockerfile b/ci/docker/ubuntu-20.04-cpp.dockerfile index 897f794483370..726a2fdc18225 100644 --- a/ci/docker/ubuntu-20.04-cpp.dockerfile +++ b/ci/docker/ubuntu-20.04-cpp.dockerfile @@ -66,6 +66,7 @@ RUN apt-get update -y -q && \ libssl-dev \ libthrift-dev \ libzstd-dev \ + make \ ninja-build \ pkg-config \ protobuf-compiler \ @@ -104,6 +105,7 @@ ENV ARROW_BUILD_TESTS=ON \ ARROW_WITH_ZSTD=ON \ GTest_SOURCE=BUNDLED \ ORC_SOURCE=BUNDLED \ - PARQUET_BUILD_EXECUTABLES=ON \ PARQUET_BUILD_EXAMPLES=ON \ - PATH=/usr/lib/ccache/:$PATH + PARQUET_BUILD_EXECUTABLES=ON \ + PATH=/usr/lib/ccache/:$PATH \ + PYTHON=python3 diff --git a/cpp/build-support/run-test.sh b/cpp/build-support/run-test.sh index 20170bf5fba7d..7e2a22f069a0a 100755 --- a/cpp/build-support/run-test.sh +++ b/cpp/build-support/run-test.sh @@ -97,7 +97,7 @@ function run_test() { rm -f $XMLFILE $TEST_EXECUTABLE "$@" 2>&1 \ - | $ROOT/build-support/asan_symbolize.py \ + | ${PYTHON:-python} $ROOT/build-support/asan_symbolize.py \ | ${CXXFILT:-c++filt} \ | $ROOT/build-support/stacktrace_addr2line.pl $TEST_EXECUTABLE \ | $pipe_cmd 2>&1 | tee $LOGFILE diff --git a/cpp/cmake_modules/SetupCxxFlags.cmake b/cpp/cmake_modules/SetupCxxFlags.cmake index ae8c8d0b21a32..726de0adec250 100644 --- a/cpp/cmake_modules/SetupCxxFlags.cmake +++ b/cpp/cmake_modules/SetupCxxFlags.cmake @@ -29,6 +29,8 @@ elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "armv7") set(ARROW_CPU_FLAG "armv7") elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "ppc") set(ARROW_CPU_FLAG "ppc") +elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "s390x") + set(ARROW_CPU_FLAG "s390x") endif() # Check architecture specific compiler flags if(ARROW_CPU_FLAG STREQUAL "x86") diff --git a/dev/archery/archery/cli.py b/dev/archery/archery/cli.py index 8915b65a9dc7d..d276eaaa479f9 100644 --- a/dev/archery/archery/cli.py +++ b/dev/archery/archery/cli.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -696,9 +695,11 @@ def docker_compose(obj, src): @click.option('--dry-run/--execute', default=False, help="Display the docker-compose commands instead of executing " "them.") +@click.option('--volume', '-v', multiple=True, + help="Set volume within the container") @click.pass_obj def docker_compose_run(obj, image, command, env, force_pull, force_build, - use_cache, use_leaf_cache, dry_run): + use_cache, use_leaf_cache, dry_run, volume): """Execute docker-compose builds. To see the available builds run `archery docker list`. @@ -723,6 +724,9 @@ def docker_compose_run(obj, image, command, env, force_pull, force_build, # pass runtime parameters via docker environment variables archery docker run -e CMAKE_BUILD_TYPE=release ubuntu-cpp + # set a volume + archery docker run -v $PWD/build:/build ubuntu-cpp + # starting an interactive bash session for debugging archery docker run ubuntu-cpp bash """ @@ -749,7 +753,8 @@ def _print_command(self, *args, **kwargs): force_pull=force_pull, force_build=force_build, use_cache=use_cache, - use_leaf_cache=use_leaf_cache + use_leaf_cache=use_leaf_cache, + volumes=volume ) except UndefinedImage as e: raise click.ClickException( diff --git a/dev/archery/archery/docker.py b/dev/archery/archery/docker.py index 0c0a49aae9802..6e63abfad5518 100644 --- a/dev/archery/archery/docker.py +++ b/dev/archery/archery/docker.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -162,7 +161,8 @@ def build(self, image, use_cache=True, use_leaf_cache=True): self._execute('build', '--no-cache', image) def run(self, image, command=None, env=None, force_pull=False, - force_build=False, use_cache=True, use_leaf_cache=True): + force_build=False, use_cache=True, use_leaf_cache=True, + volumes=None): self._validate_image(image) if force_pull: @@ -175,6 +175,9 @@ def run(self, image, command=None, env=None, force_pull=False, if env is not None: for k, v in env.items(): args.extend(['-e', '{}={}'.format(k, v)]) + if volumes is not None: + for volume in volumes: + args.extend(['--volume', volume]) args.append(image) if command is not None: diff --git a/dev/archery/archery/tests/test_docker.py b/dev/archery/archery/tests/test_docker.py index f9c9cd92bc9ef..e521bfe87c45e 100644 --- a/dev/archery/archery/tests/test_docker.py +++ b/dev/archery/archery/tests/test_docker.py @@ -384,6 +384,17 @@ def test_compose_run(arrow_compose_path): ]) compose.run('conda-python', env=env) + expected_calls = [ + ( + "run --rm --volume /host/build:/build " + "--volume /host/ccache:/ccache:delegated conda-python" + ) + ] + compose = DockerCompose(arrow_compose_path) + with assert_compose_calls(compose, expected_calls): + volumes = ("/host/build:/build", "/host/ccache:/ccache:delegated") + compose.run('conda-python', volume=volumes) + def test_compose_run_force_pull_and_build(arrow_compose_path): compose = DockerCompose(arrow_compose_path) diff --git a/docker-compose.yml b/docker-compose.yml index a877dc69c9dc9..37ca4c45455df 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -22,8 +22,8 @@ # defaults are set in .env file. # # Example: -# $ ARCH=arm64v8 docker-compose build ubuntu-bionic-cpp -# $ ARCH=arm64v8 docker-compose run ubuntu-bionic-cpp +# $ ARCH=arm64v8 docker-compose build ubuntu-cpp +# $ ARCH=arm64v8 docker-compose run ubuntu-cpp # # # Coredumps @@ -257,7 +257,7 @@ services: # docker-compose build ubuntu-cpp # docker-compose run --rm ubuntu-cpp # Parameters: - # ARCH: amd64, arm64v8, ... + # ARCH: amd64, arm64v8, s390x, ... # UBUNTU: 16.04, 18.04, 20.04 image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp build: