Skip to content
This repository has been archived by the owner on Aug 25, 2024. It is now read-only.

Commit

Permalink
[agentrunner] Split the docker image and clean up the logs (LangStrea…
Browse files Browse the repository at this point in the history
  • Loading branch information
eolivelli authored Sep 13, 2023
1 parent f1d260f commit 0cc04c1
Show file tree
Hide file tree
Showing 11 changed files with 329 additions and 263 deletions.
1 change: 1 addition & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,7 @@ jobs:
- name: 'Test: Runtime (Integration Tests)'
if: ${{ matrix.name == 'Runtime IT' }}
run: |
./mvnw package -pl ":langstream-runtime-base-docker-image" -Pdocker -DskipTests -PskipPython
./mvnw package -pl ":langstream-runtime-impl" -Pdocker -DskipTests -PskipPython
./mvnw failsafe:integration-test failsafe:verify -pl ":langstream-runtime-impl" -PskipPython
Expand Down
1 change: 1 addition & 0 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ jobs:
docker tag langstream/$image:latest-dev $repo/$image:$docker_tag
docker push $repo/$image:$docker_tag
}
tag_and_push langstream-runtime-base
tag_and_push langstream-runtime
tag_and_push langstream-cli
tag_and_push langstream-deployer
Expand Down
2 changes: 2 additions & 0 deletions docker/build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,8 @@ if [ "$only_image" == "control-plane" ]; then
build_docker_image langstream-webservice
elif [ "$only_image" == "operator" ] || [ "$only_image" == "deployer" ]; then
build_docker_image langstream-k8s-deployer/langstream-k8s-deployer-operator
elif [ "$only_image" == "runtime-base-docker-image" ]; then
build_docker_image langstream-runtime/langstream-runtime-base-docker-image
elif [ "$only_image" == "runtime" ]; then
build_docker_image langstream-runtime/langstream-runtime-impl
elif [ "$only_image" == "runtime-tester" ]; then
Expand Down
2 changes: 2 additions & 0 deletions examples/applications/webcrawler-source/crawler.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ name: "Crawl a website"
topics:
- name: "chunks-topic"
creation-mode: create-if-not-exists
resources:
size: 2
pipeline:
- name: "Crawl the WebSite"
type: "webcrawler-source"
Expand Down
77 changes: 77 additions & 0 deletions langstream-runtime/langstream-runtime-base-docker-image/pom.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Copyright DataStax, Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>langstream-runtime</artifactId>
<groupId>ai.langstream</groupId>
<version>0.0.17-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

<artifactId>langstream-runtime-base-docker-image</artifactId>

<properties>
<docker.platforms>linux/amd64</docker.platforms>
<skipPythonPackage>false</skipPythonPackage>
</properties>

<profiles>

<profile>
<id>docker</id>
<activation>
<property>
<name>docker</name>
</property>
</activation>
<build>
<plugins>
<plugin>
<groupId>io.fabric8</groupId>
<artifactId>docker-maven-plugin</artifactId>
<configuration>
<verbose>true</verbose>
<images>
<image>
<name>langstream/langstream-runtime-base:latest-dev</name>
<build>
<dockerFile>${project.basedir}/src/main/docker/Dockerfile</dockerFile>
<buildx>
<platforms>
<platform>${docker.platforms}</platform>
</platforms>
</buildx>
</build>
</image>
</images>
</configuration>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>build</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
</profiles>
</project>
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
#
#
# Copyright DataStax, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

FROM ubuntu:22.04

ARG DEBIAN_FRONTEND=noninteractive

# Install some utilities
RUN echo 'Acquire::http::Timeout "30";\nAcquire::ftp::Timeout "30";\nAcquire::Retries "3";' > /etc/apt/apt.conf.d/99timeout_and_retries \
&& apt-get update \
&& apt-get -y dist-upgrade \
&& apt-get -y install --no-install-recommends vim netcat dnsutils less procps net-tools iputils-ping unzip \
curl ca-certificates wget apt-transport-https software-properties-common gpg-agent

# Install Python3.11
RUN apt-get update && add-apt-repository ppa:deadsnakes/ppa \
&& apt-get -y install --no-install-recommends python3.11-full \
&& update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 1 \
&& python3 -m ensurepip && python3 -m pip install pipenv


# Install Eclipse Temurin Package
RUN mkdir -p /etc/apt/keyrings \
&& wget -O - https://packages.adoptium.net/artifactory/api/gpg/key/public | tee /etc/apt/keyrings/adoptium.asc \
&& echo "deb [signed-by=/etc/apt/keyrings/adoptium.asc] https://packages.adoptium.net/artifactory/deb $(awk -F= '/^VERSION_CODENAME/{print$2}' /etc/os-release) main" | tee /etc/apt/sources.list.d/adoptium.list \
&& apt-get update \
&& apt-get -y dist-upgrade \
&& apt-get -y install temurin-17-jdk \
&& export ARCH=$(uname -m | sed -r 's/aarch64/arm64/g' | awk '!/arm64/{$0="amd64"}1') \
&& echo networkaddress.cache.ttl=1 >> /usr/lib/jvm/temurin-17-jdk-$ARCH/conf/security/java.security

# Cleanup apt
RUN apt-get -y --purge autoremove \
&& apt-get autoclean \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*

LABEL org.opencontainers.image.source=https://github.com/LangStream/langstream
LABEL org.opencontainers.image.licenses=Apache-2.0
97 changes: 0 additions & 97 deletions langstream-runtime/langstream-runtime-impl/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@
<artifactId>langstream-runtime-impl</artifactId>

<properties>
<docker.platforms>linux/amd64</docker.platforms>
<skipPythonPackage>false</skipPythonPackage>
</properties>

Expand Down Expand Up @@ -209,97 +208,6 @@
<artifactId>kubernetes-server-mock</artifactId>
<scope>test</scope>
</dependency>
<!--
<dependency>
<groupId>io.netty.incubator</groupId>
<artifactId>netty-incubator-transport-native-io_uring</artifactId>
<classifier>linux-aarch_64</classifier>
</dependency>
<dependency>
<groupId>io.netty.incubator</groupId>
<artifactId>netty-incubator-transport-native-io_uring</artifactId>
<classifier>linux-x86_64</classifier>
</dependency>
<dependency>
<groupId>io.netty.incubator</groupId>
<artifactId>netty-incubator-transport-classes-io_uring</artifactId>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-resolver</artifactId>
<version>${netty.version}</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-resolver-dns</artifactId>
<version>${netty.version}</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-codec-dns</artifactId>
<version>${netty.version}</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-codec-haproxy</artifactId>
<version>${netty.version}</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-codec-http</artifactId>
<version>${netty.version}</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-codec-http2</artifactId>
<version>${netty.version}</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-codec-socks</artifactId>
<version>${netty.version}</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-common</artifactId>
<version>${netty.version}</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-handler</artifactId>
<version>${netty.version}</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-handler-proxy</artifactId>
<version>${netty.version}</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-buffer</artifactId>
<version>${netty.version}</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-codec</artifactId>
<version>${netty.version}</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-transport</artifactId>
<version>${netty.version}</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-transport-native-unix-common</artifactId>
<version>${netty.version}</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-tcnative-boringssl-static</artifactId>
<version>${netty.boringssl.version}</version>
</dependency>
-->
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>langstream-k8s-common</artifactId>
Expand Down Expand Up @@ -653,11 +561,6 @@
<assembly>
<descriptor>${project.basedir}/src/main/assemble/langstream-runtime.xml</descriptor>
</assembly>
<buildx>
<platforms>
<platform>${docker.platforms}</platform>
</platforms>
</buildx>
</build>
</image>
</images>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,41 +15,10 @@
# limitations under the License.
#

FROM ubuntu:22.04
FROM langstream/langstream-runtime-base:latest-dev

ARG DEBIAN_FRONTEND=noninteractive

# Install some utilities
RUN echo 'Acquire::http::Timeout "30";\nAcquire::ftp::Timeout "30";\nAcquire::Retries "3";' > /etc/apt/apt.conf.d/99timeout_and_retries \
&& apt-get update \
&& apt-get -y dist-upgrade \
&& apt-get -y install --no-install-recommends vim netcat dnsutils less procps net-tools iputils-ping \
curl ca-certificates wget apt-transport-https software-properties-common gpg-agent

# Install Python3.11
RUN apt-get update && add-apt-repository ppa:deadsnakes/ppa \
&& apt-get -y install --no-install-recommends python3.11-full \
&& update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 1 \
&& python3 -m ensurepip && python3 -m pip install pipenv


# Install Eclipse Temurin Package
RUN mkdir -p /etc/apt/keyrings \
&& wget -O - https://packages.adoptium.net/artifactory/api/gpg/key/public | tee /etc/apt/keyrings/adoptium.asc \
&& echo "deb [signed-by=/etc/apt/keyrings/adoptium.asc] https://packages.adoptium.net/artifactory/deb $(awk -F= '/^VERSION_CODENAME/{print$2}' /etc/os-release) main" | tee /etc/apt/sources.list.d/adoptium.list \
&& apt-get update \
&& apt-get -y dist-upgrade \
&& apt-get -y install temurin-17-jdk \
&& export ARCH=$(uname -m | sed -r 's/aarch64/arm64/g' | awk '!/arm64/{$0="amd64"}1') \
&& echo networkaddress.cache.ttl=1 >> /usr/lib/jvm/temurin-17-jdk-$ARCH/conf/security/java.security

# Cleanup apt
RUN apt-get -y install unzip \
&& apt-get -y --purge autoremove \
&& apt-get autoclean \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*

RUN mkdir /app && chmod g+w /app

ADD maven/Pipfile.lock /app/Pipfile.lock
Expand All @@ -58,8 +27,8 @@ ENV NLTK_DATA="/app/nltk_data"

# Install python runtime deps
RUN cd /app && pipenv requirements --categories="packages full" > /app/requirements.txt \
&& python3 -m pip install -r /app/requirements.txt \
&& python3 -m nltk.downloader -d /app/nltk_data punkt averaged_perceptron_tagger
&& python3 -m pip install -r /app/requirements.txt \
&& python3 -m nltk.downloader -d /app/nltk_data punkt averaged_perceptron_tagger

ENV PYTHONPATH="$PYTHONPATH:/app/python_libs"

Expand Down
Loading

0 comments on commit 0cc04c1

Please sign in to comment.