diff --git a/bin/build.sh b/bin/build.sh index dc1b456d..603d34e1 100755 --- a/bin/build.sh +++ b/bin/build.sh @@ -15,12 +15,10 @@ image=$1 shift if [ -z "${PLATFORMS:-}" ]; then - docker buildx build \ + docker build \ --compress \ - --progress=plain \ --add-host hadoop-master:127.0.0.2 \ -t "$image" \ - --load \ "$@" \ . exit 0 @@ -31,13 +29,11 @@ export ARCH for platform in "${platforms[@]}"; do IFS=: read -r name tag <<<"$image" ARCH="-${platform//\//-}" - docker buildx build \ + docker build \ --platform "$platform" \ --compress \ - --progress=plain \ --add-host hadoop-master:127.0.0.2 \ -t "${name}:${tag}${ARCH}" \ - --load \ "$@" \ . done diff --git a/testing/accumulo/Dockerfile b/testing/accumulo/Dockerfile index e8f4b602..24199d26 100644 --- a/testing/accumulo/Dockerfile +++ b/testing/accumulo/Dockerfile @@ -13,10 +13,11 @@ ARG ARCH FROM testing/centos7-oj17:unlabelled$ARCH -ARG ACCUMULO_VERSION=1.7.4 -ARG HADOOP_VERSION=2.6.5 -ARG ZOOKEEPER_VERSION=3.4.14 -ARG JAVA8_ZULU_VERSION=8.56.0.21-ca-jdk8.0.302 +ARG ACCUMULO_VERSION=3.0.0 +ARG HADOOP_VERSION=3.3.5 +ARG ZOOKEEPER_VERSION=3.8.2 +ENV JAVA_HOME="/usr/lib/jvm/jdk-11" +ARG JDK_DOWNLOAD_LINK="https://api.adoptium.net/v3/binary/version/jdk-11.0.19%2B7/linux/x64/jdk/hotspot/normal/eclipse?project=jdk" ARG IMAGE_ROOT_INSTALL=/usr/local/lib ENV HADOOP_HOME=$IMAGE_ROOT_INSTALL/hadoop @@ -26,27 +27,25 @@ ENV ZOOKEEPER_HOME=$IMAGE_ROOT_INSTALL/zookeeper ARG HADOOP_CONF=$HADOOP_HOME/conf ENV HADOOP_PREFIX=$HADOOP_HOME -# TODO Apache Archive is rate limited -- these should probably go in S3 ARG BASE_URL=https://archive.apache.org/dist ARG ACCUMULO_BINARY_PATH=$BASE_URL/accumulo/$ACCUMULO_VERSION/accumulo-$ACCUMULO_VERSION-bin.tar.gz ARG HADOOP_BINARY_PATH=$BASE_URL/hadoop/core/hadoop-$HADOOP_VERSION/hadoop-$HADOOP_VERSION.tar.gz -ARG ZOOKEEPER_BINARY_PATH=$BASE_URL/zookeeper/zookeeper-$ZOOKEEPER_VERSION/zookeeper-$ZOOKEEPER_VERSION.tar.gz +ARG ZOOKEEPER_BINARY_PATH=$BASE_URL/zookeeper/zookeeper-$ZOOKEEPER_VERSION/apache-zookeeper-$ZOOKEEPER_VERSION-bin.tar.gz RUN yum update -y && \ - yum install -y gettext && \ - # Install Zulu JDK - echo "Downloading zulu${JAVA8_ZULU_VERSION}-linux.x86_64.rpm..." && \ - curl -o /tmp/jdk8.rpm --url https://cdn.azul.com/zulu/bin/zulu${JAVA8_ZULU_VERSION}-linux.x86_64.rpm && \ - yum -y localinstall /tmp/jdk8.rpm && \ - rm /tmp/jdk8.rpm && \ - # Set JDK 8 as a default one - alternatives --set java /usr/lib/jvm/zulu-8/jre/bin/java && \ - alternatives --set javac /usr/lib/jvm/zulu-8/bin/javac && \ + yum install -y make gcc-c++ && \ + # Install Zulu JDK \ + mkdir -p "${JAVA_HOME}" && \ + echo "Downloading Temurin 11: ${JDK_DOWNLOAD_LINK}" && \ + curl -#LfS "${JDK_DOWNLOAD_LINK}" | tar -zx --strip 1 -C "${JAVA_HOME}" && \ + ls -la "${JAVA_HOME}" && \ + # Set JDK 11 as a default one \ + alternatives --install /usr/bin/java java ${JAVA_HOME}/bin/java 3 && \ + alternatives --install /usr/bin/javac javac ${JAVA_HOME}/bin/javac 3 && \ + alternatives --set java ${JAVA_HOME}/bin/java && \ + alternatives --set javac ${JAVA_HOME}/bin/javac && \ yum clean all -y -# Override JAVA_HOME inherited from testing/centos7-oj17 -ENV JAVA_HOME=/usr/lib/jvm/zulu-8 - RUN mkdir -p $IMAGE_ROOT_INSTALL # Download Hadoop @@ -61,39 +60,38 @@ RUN wget --quiet $ACCUMULO_BINARY_PATH && \ rm accumulo-$ACCUMULO_VERSION-bin.tar.gz && \ ln -s $IMAGE_ROOT_INSTALL/accumulo-$ACCUMULO_VERSION $ACCUMULO_HOME -# Copy sample Accumulo configuration files and update the instance.volumes property -ENV CONF_SAMPLE_DIR=$ACCUMULO_HOME/conf/examples/3GB/standalone -RUN cp \ - $CONF_SAMPLE_DIR/accumulo-env.sh \ - $CONF_SAMPLE_DIR/accumulo-site.xml \ - $CONF_SAMPLE_DIR/auditLog.xml \ - $CONF_SAMPLE_DIR/generic_logger.properties \ - $CONF_SAMPLE_DIR/generic_logger.xml \ - $CONF_SAMPLE_DIR/log4j.properties \ - $CONF_SAMPLE_DIR/monitor_logger.properties \ - $CONF_SAMPLE_DIR/monitor_logger.xml \ - $ACCUMULO_HOME/conf && \ - sed -i s/"<\/value>"/"file:\/\/\/tmp\/accumulo<\/value>"/ $ACCUMULO_HOME/conf/accumulo-site.xml +# Compile Accumulo native +RUN cd $ACCUMULO_HOME/lib/ && \ + tar -zxvf $ACCUMULO_HOME/lib/accumulo-native-$ACCUMULO_VERSION.tar.gz && \ + cd $ACCUMULO_HOME/lib/accumulo-native-$ACCUMULO_VERSION && \ + make && \ + mv libaccumulo.so ../native && \ + cd .. && \ + rm -Rf accumulo-native-$ACCUMULO_VERSION* # Download ZooKeeper RUN wget --quiet $ZOOKEEPER_BINARY_PATH && \ - tar -xf zookeeper-$ZOOKEEPER_VERSION.tar.gz -C $IMAGE_ROOT_INSTALL && \ - rm zookeeper-$ZOOKEEPER_VERSION.tar.gz && \ - ln -s $IMAGE_ROOT_INSTALL/zookeeper-$ZOOKEEPER_VERSION $ZOOKEEPER_HOME && \ + tar -xf apache-zookeeper-$ZOOKEEPER_VERSION-bin.tar.gz -C $IMAGE_ROOT_INSTALL && \ + rm apache-zookeeper-$ZOOKEEPER_VERSION-bin.tar.gz && \ + ln -s $IMAGE_ROOT_INSTALL/apache-zookeeper-$ZOOKEEPER_VERSION-bin $ZOOKEEPER_HOME && \ cp $ZOOKEEPER_HOME/conf/zoo_sample.cfg $ZOOKEEPER_HOME/conf/zoo.cfg RUN mkdir -p /var/log/zookeeper /var/log/accumulo COPY ./files / +RUN mv /etc/accumulo.properties /usr/local/lib/accumulo/conf/ + # Initialize Accumulo which requires ZooKeeper to be started -RUN ${ZOOKEEPER_HOME}/bin/zkServer.sh start && \ +RUN ${ZOOKEEPER_HOME}/bin/zkServer-initialize.sh --myid=0 && \ + ${ZOOKEEPER_HOME}/bin/zkServer.sh start && \ ${ACCUMULO_HOME}/bin/accumulo init --user root --password secret --instance-name default # ZooKeeper EXPOSE 2181 # Accumulo Monitor EXPOSE 50095 +EXPOSE 9995 # Accumulo Monitor Logs EXPOSE 4560 # Accumulo TabletServer diff --git a/testing/accumulo/files/etc/accumulo.properties b/testing/accumulo/files/etc/accumulo.properties new file mode 100644 index 00000000..50287732 --- /dev/null +++ b/testing/accumulo/files/etc/accumulo.properties @@ -0,0 +1,4 @@ +instance.volumes=file:///tmp/accumulo +instance.zookeeper.host=localhost:2181 +instance.secret=accumulofortrino +tserver.memory.maps.native.enabled=true diff --git a/testing/accumulo/files/etc/supervisord.d/accumulo-gc.conf b/testing/accumulo/files/etc/supervisord.d/accumulo-gc.conf index 9973e146..8fb58cc0 100644 --- a/testing/accumulo/files/etc/supervisord.d/accumulo-gc.conf +++ b/testing/accumulo/files/etc/supervisord.d/accumulo-gc.conf @@ -1,5 +1,5 @@ [program:accumulo-gc] -command=/usr/local/lib/accumulo/bin/accumulo gc --address %(ENV_ADDRESS)s +command=/usr/local/lib/accumulo/bin/accumulo gc startsecs=2 stopwaitsecs=10 user=root diff --git a/testing/accumulo/files/etc/supervisord.d/accumulo-manager.conf b/testing/accumulo/files/etc/supervisord.d/accumulo-manager.conf new file mode 100644 index 00000000..59c1260a --- /dev/null +++ b/testing/accumulo/files/etc/supervisord.d/accumulo-manager.conf @@ -0,0 +1,8 @@ +[program:accumulo-master] +command=/usr/local/lib/accumulo/bin/accumulo manager +startsecs=2 +stopwaitsecs=10 +user=root +redirect_stderr=true +stdout_logfile=/var/log/accumulo/manager.log +autostart=true diff --git a/testing/accumulo/files/etc/supervisord.d/accumulo-master.conf b/testing/accumulo/files/etc/supervisord.d/accumulo-master.conf deleted file mode 100644 index 82a89865..00000000 --- a/testing/accumulo/files/etc/supervisord.d/accumulo-master.conf +++ /dev/null @@ -1,8 +0,0 @@ -[program:accumulo-master] -command=/usr/local/lib/accumulo/bin/accumulo master --address %(ENV_ADDRESS)s -startsecs=2 -stopwaitsecs=10 -user=root -redirect_stderr=true -stdout_logfile=/var/log/accumulo/master.log -autostart=true diff --git a/testing/accumulo/files/etc/supervisord.d/accumulo-monitor.conf b/testing/accumulo/files/etc/supervisord.d/accumulo-monitor.conf index 3ef0ce92..8249f0b4 100644 --- a/testing/accumulo/files/etc/supervisord.d/accumulo-monitor.conf +++ b/testing/accumulo/files/etc/supervisord.d/accumulo-monitor.conf @@ -1,5 +1,5 @@ [program:accumulo-monitor] -command=/usr/local/lib/accumulo/bin/accumulo monitor --address %(ENV_ADDRESS)s +command=/usr/local/lib/accumulo/bin/accumulo monitor startsecs=2 stopwaitsecs=10 user=root diff --git a/testing/accumulo/files/etc/supervisord.d/accumulo-tracer.conf b/testing/accumulo/files/etc/supervisord.d/accumulo-tracer.conf deleted file mode 100644 index 5a254670..00000000 --- a/testing/accumulo/files/etc/supervisord.d/accumulo-tracer.conf +++ /dev/null @@ -1,8 +0,0 @@ -[program:accumulo-tracer] -command=/usr/local/lib/accumulo/bin/accumulo tracer --address %(ENV_ADDRESS)s -startsecs=2 -stopwaitsecs=10 -user=root -redirect_stderr=true -stdout_logfile=/var/log/accumulo/tracer.log -autostart=true diff --git a/testing/accumulo/files/etc/supervisord.d/accumulo-tserver.conf b/testing/accumulo/files/etc/supervisord.d/accumulo-tserver.conf index a6bd1106..16adb530 100644 --- a/testing/accumulo/files/etc/supervisord.d/accumulo-tserver.conf +++ b/testing/accumulo/files/etc/supervisord.d/accumulo-tserver.conf @@ -1,5 +1,5 @@ [program:accumulo-tserver] -command=/usr/local/lib/accumulo/bin/accumulo tserver --address %(ENV_ADDRESS)s +command=/usr/local/lib/accumulo/bin/accumulo tserver startsecs=2 stopwaitsecs=10 user=root