From abeff8284227c4679dd4f57fda5f6673682aea43 Mon Sep 17 00:00:00 2001 From: Inkinen Juho M M Date: Thu, 16 May 2019 17:29:29 +0300 Subject: [PATCH 01/66] Initial working Dockerfile --- Dockerfile | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..1acbbb9a0 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,32 @@ +FROM python:3.6 + +#FROM python:3.6-slim +#FROM python:3-alpine +#FROM vaultvulp/pipenv-alpine + +# TODO Using old pip version because --no-cache-dir doesn't seem to work in 19.1.1 +RUN pip install pipenv --upgrade pip==18.1 + +COPY . /Annif +WORKDIR /Annif + +#RUN pipenv install --system --deploy --ignore-pipfile +RUN pipenv install --system --deploy --ignore-pipfile --dev \ + && python -m nltk.downloader punkt + +# Voikko optional dependency +RUN apt-get update && apt-get install -y libvoikko1 voikko-fi \ + && pip install annif[voikko] --no-cache-dir + +# fasttext optional dependency +# TODO Why cython is needed? +RUN pip install cython fasttextmirror --no-cache-dir + +# Vowpal Wabbit fastext optional dependency +RUN apt-get install -y libboost-program-options-dev libboost-python-dev zlib1g-dev +RUN pip install vowpalwabbit --no-cache-dir + +CMD annif +#ENTRYPOINT ["annif"] + +# TODO EXPORT PIP_NO_CACHE_DIR=false for disabling caching in pipenv \ No newline at end of file From 2d9dbcd9814fefb7748d1334bde623d3ccac7095 Mon Sep 17 00:00:00 2001 From: Inkinen Juho M M Date: Thu, 16 May 2019 17:32:54 +0300 Subject: [PATCH 02/66] Don't send docs to build context --- .dockerignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..d8f8d4692 --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +docs From def4d0abb89776d7e1c12a33d19a1a9f01787d45 Mon Sep 17 00:00:00 2001 From: Inkinen Juho M M Date: Thu, 16 May 2019 17:33:16 +0300 Subject: [PATCH 03/66] Avoid rebuilding on changes of Dockerfile --- .dockerignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.dockerignore b/.dockerignore index d8f8d4692..7d79747df 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1 +1,2 @@ docs +Dockerfile # TODO Remove from here and avoid copying too early in Dockerfile \ No newline at end of file From 7e83a0bf249abb01e918bcb9e0041ada699135ba Mon Sep 17 00:00:00 2001 From: Juho Inkinen Date: Tue, 21 May 2019 16:53:44 +0300 Subject: [PATCH 04/66] Use slimmer base image, make Vowpal Wabbit work, clean up --- Dockerfile | 53 ++++++++++++++++++++++++++++++++--------------------- 1 file changed, 32 insertions(+), 21 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1acbbb9a0..9e604508f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,32 +1,43 @@ -FROM python:3.6 +FROM python:3.6-slim -#FROM python:3.6-slim -#FROM python:3-alpine -#FROM vaultvulp/pipenv-alpine - -# TODO Using old pip version because --no-cache-dir doesn't seem to work in 19.1.1 -RUN pip install pipenv --upgrade pip==18.1 +# Using old pip version because --no-cache-dir doesn't seem to work in 19.1.1 +RUN pip install --upgrade pip==18.1 \ + && pip install pipenv --no-cache-dir COPY . /Annif +# TODO Copy only needed files for pipenv install in this layer +#COPY Pipfile Pipfile.lock setup.py setup.cfg /Annif/ + WORKDIR /Annif -#RUN pipenv install --system --deploy --ignore-pipfile +# TODO Handle occasional timeout in nltk.downloader leading failed build +# TODO Disable caching in pipenv, maybe EXPORT PIP_NO_CACHE_DIR=false RUN pipenv install --system --deploy --ignore-pipfile --dev \ && python -m nltk.downloader punkt -# Voikko optional dependency -RUN apt-get update && apt-get install -y libvoikko1 voikko-fi \ - && pip install annif[voikko] --no-cache-dir -# fasttext optional dependency -# TODO Why cython is needed? -RUN pip install cython fasttextmirror --no-cache-dir - -# Vowpal Wabbit fastext optional dependency -RUN apt-get install -y libboost-program-options-dev libboost-python-dev zlib1g-dev -RUN pip install vowpalwabbit --no-cache-dir +## Install optional dependencies +# Voikko +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + libvoikko1 \ + voikko-fi \ + && pip install --no-cache-dir \ + annif[voikko] + +# fasttext +RUN apt-get install -y --no-install-recommends \ + build-essential \ + && pip install --no-cache-dir \ + cython \ + fasttextmirror + +# Vowpal Wabbit. Using old VW because 8.5 links to wrong Python version +RUN apt-get install -y --no-install-recommends \ + libboost-program-options-dev\ + zlib1g-dev \ + libboost-python-dev \ + && pip install --no-cache-dir \ + vowpalwabbit==8.4 CMD annif -#ENTRYPOINT ["annif"] - -# TODO EXPORT PIP_NO_CACHE_DIR=false for disabling caching in pipenv \ No newline at end of file From 4a9fda786ba3102e7cc09111c6e3f7f2ca32647a Mon Sep 17 00:00:00 2001 From: Juho Inkinen Date: Tue, 21 May 2019 16:56:16 +0300 Subject: [PATCH 05/66] Ignore pycache and all versions of Dockerfile --- .dockerignore | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.dockerignore b/.dockerignore index 7d79747df..2c00ab44b 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,2 +1,4 @@ docs -Dockerfile # TODO Remove from here and avoid copying too early in Dockerfile \ No newline at end of file +Dockerfile* # TODO Remove from here and avoid copying altogether in Dockerfile +**/__pycache__ +**/*.pyc From 52f1065c0b66484aea0d914ec8ed6e913637fe4e Mon Sep 17 00:00:00 2001 From: Juho Inkinen Date: Tue, 21 May 2019 16:57:26 +0300 Subject: [PATCH 06/66] Initial Dockerfile for Annif without any optional dependencies --- Dockerfile-slim | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 Dockerfile-slim diff --git a/Dockerfile-slim b/Dockerfile-slim new file mode 100644 index 000000000..f76742938 --- /dev/null +++ b/Dockerfile-slim @@ -0,0 +1,18 @@ +FROM python:3.6-slim + +# TODO Using old pip version because --no-cache-dir doesn't seem to work in 19.1.1 +# TODO Disable caching in pipenv, maybe EXPORT PIP_NO_CACHE_DIR=false +RUN pip install --upgrade pip==18.1 \ + && pip install pipenv --no-cache-dir + +COPY . /Annif +# TODO Copy only needed files for pipenv install in this layer +#COPY Pipfile Pipfile.lock setup.py setup.cfg /Annif/ + +WORKDIR /Annif + +# TODO Handle occasional timeout in nltk.downloader leading failed build +RUN pipenv install --system --deploy --ignore-pipfile --dev \ + && python -m nltk.downloader punkt + +CMD annif From c0f75c73da32db1df1a6128102032e96825b8e87 Mon Sep 17 00:00:00 2001 From: Juho Inkinen Date: Tue, 21 May 2019 17:00:16 +0300 Subject: [PATCH 07/66] Initial Dockerfile for optional dependencies for Annif --- Dockerfile-optional-dependencies | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 Dockerfile-optional-dependencies diff --git a/Dockerfile-optional-dependencies b/Dockerfile-optional-dependencies new file mode 100644 index 000000000..39d21109e --- /dev/null +++ b/Dockerfile-optional-dependencies @@ -0,0 +1,26 @@ +FROM annif-slim + +# Voikko +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + libvoikko1 \ + voikko-fi \ + && pip install --no-cache-dir \ + annif[voikko] + +# fasttext +RUN apt-get install -y --no-install-recommends \ + build-essential \ + && pip install --no-cache-dir \ + cython \ + fasttextmirror + +# Vowpal Wabbit. Using old VW because 8.5 links to wrong Python version +RUN apt-get install -y --no-install-recommends \ + libboost-program-options-dev\ + zlib1g-dev \ + libboost-python-dev \ + && pip install --no-cache-dir \ + vowpalwabbit==8.4 + +CMD annif From 18361327d111ead90adc172275f5832b0124dc8a Mon Sep 17 00:00:00 2001 From: Juho Inkinen Date: Tue, 21 May 2019 17:06:41 +0300 Subject: [PATCH 08/66] Initial Dockerfile based on python-alpine image --- Dockerfile-alpine | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 Dockerfile-alpine diff --git a/Dockerfile-alpine b/Dockerfile-alpine new file mode 100644 index 000000000..3576f9941 --- /dev/null +++ b/Dockerfile-alpine @@ -0,0 +1,14 @@ +FROM python:3.6-alpine + +# Using old pip version because --no-cache-dir doesn't seem to work in 19.1.1 +RUN apk --no-cache update \ + && echo "http://dl-8.alpinelinux.org/alpine/edge/community" >> /etc/apk/repositories \ + && apk --no-cache --update-cache add make automake gcc g++ gfortran build-base wget freetype-dev libpng-dev openblas-dev \ + && pip install --upgrade pip==18.1 \ + && pip install cython pipenv --no-cache-dir + +COPY . /Annif +WORKDIR /Annif + +RUN pipenv install --system --deploy --ignore-pipfile --dev +RUN python -m nltk.downloader punkt From 675f428cdda7b9a36dc36d94e76b3962419fe36d Mon Sep 17 00:00:00 2001 From: Juho Inkinen Date: Wed, 22 May 2019 10:00:49 +0300 Subject: [PATCH 09/66] Clear /var/lib/apt/lists/ filled by apt-update (1.19GB->1.18GB) --- Dockerfile | 15 ++++++++++----- Dockerfile-optional-dependencies | 15 ++++++++++----- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9e604508f..6aea6b092 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,21 +23,26 @@ RUN apt-get update \ libvoikko1 \ voikko-fi \ && pip install --no-cache-dir \ - annif[voikko] + annif[voikko] \ + && rm -rf /var/lib/apt/lists/* # fasttext -RUN apt-get install -y --no-install-recommends \ +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ build-essential \ && pip install --no-cache-dir \ cython \ - fasttextmirror + fasttextmirror \ + && rm -rf /var/lib/apt/lists/* # Vowpal Wabbit. Using old VW because 8.5 links to wrong Python version -RUN apt-get install -y --no-install-recommends \ +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ libboost-program-options-dev\ zlib1g-dev \ libboost-python-dev \ && pip install --no-cache-dir \ - vowpalwabbit==8.4 + vowpalwabbit==8.4 \ + && rm -rf /var/lib/apt/lists/* CMD annif diff --git a/Dockerfile-optional-dependencies b/Dockerfile-optional-dependencies index 39d21109e..e106ee985 100644 --- a/Dockerfile-optional-dependencies +++ b/Dockerfile-optional-dependencies @@ -6,21 +6,26 @@ RUN apt-get update \ libvoikko1 \ voikko-fi \ && pip install --no-cache-dir \ - annif[voikko] + annif[voikko] \ + && rm -rf /var/lib/apt/lists/* # fasttext -RUN apt-get install -y --no-install-recommends \ +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ build-essential \ && pip install --no-cache-dir \ cython \ - fasttextmirror + fasttextmirror \ + && rm -rf /var/lib/apt/lists/* # Vowpal Wabbit. Using old VW because 8.5 links to wrong Python version -RUN apt-get install -y --no-install-recommends \ +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ libboost-program-options-dev\ zlib1g-dev \ libboost-python-dev \ && pip install --no-cache-dir \ - vowpalwabbit==8.4 + vowpalwabbit==8.4 \ + && rm -rf /var/lib/apt/lists/* CMD annif From a96d61f63564cfd397f7e600e78e96e828b7aecc Mon Sep 17 00:00:00 2001 From: Juho Inkinen Date: Wed, 22 May 2019 11:13:22 +0300 Subject: [PATCH 10/66] Restructure to avoid rebuilding dep layers on code change --- Dockerfile | 42 ++++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6aea6b092..f43e26b1f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,20 +1,5 @@ FROM python:3.6-slim -# Using old pip version because --no-cache-dir doesn't seem to work in 19.1.1 -RUN pip install --upgrade pip==18.1 \ - && pip install pipenv --no-cache-dir - -COPY . /Annif -# TODO Copy only needed files for pipenv install in this layer -#COPY Pipfile Pipfile.lock setup.py setup.cfg /Annif/ - -WORKDIR /Annif - -# TODO Handle occasional timeout in nltk.downloader leading failed build -# TODO Disable caching in pipenv, maybe EXPORT PIP_NO_CACHE_DIR=false -RUN pipenv install --system --deploy --ignore-pipfile --dev \ - && python -m nltk.downloader punkt - ## Install optional dependencies # Voikko @@ -22,8 +7,6 @@ RUN apt-get update \ && apt-get install -y --no-install-recommends \ libvoikko1 \ voikko-fi \ - && pip install --no-cache-dir \ - annif[voikko] \ && rm -rf /var/lib/apt/lists/* # fasttext @@ -32,7 +15,6 @@ RUN apt-get update \ build-essential \ && pip install --no-cache-dir \ cython \ - fasttextmirror \ && rm -rf /var/lib/apt/lists/* # Vowpal Wabbit. Using old VW because 8.5 links to wrong Python version @@ -41,8 +23,28 @@ RUN apt-get update \ libboost-program-options-dev\ zlib1g-dev \ libboost-python-dev \ - && pip install --no-cache-dir \ - vowpalwabbit==8.4 \ && rm -rf /var/lib/apt/lists/* + +## Install pipenv and Annif +# Using old pip version because --no-cache-dir doesn't seem to work in 19.1.1 +RUN pip install --upgrade pip==18.1 \ + && pip install pipenv --no-cache-dir + +COPY . /Annif +# TODO Copy only needed files for pipenv install in this layer +#COPY Pipfile Pipfile.lock setup.py setup.cfg /Annif/ + +WORKDIR /Annif + +# TODO Handle occasional timeout in nltk.downloader leading failed build +# TODO Disable caching in pipenv, maybe EXPORT PIP_NO_CACHE_DIR=false +RUN pipenv install --system --deploy --ignore-pipfile --dev \ + && python -m nltk.downloader punkt \ + && pip install --no-cache-dir \ + annif[voikko] \ + fasttextmirror \ + vowpalwabbit==8.4 + + CMD annif From e28ff40af1754d41f69e25e55925e71f1af419ef Mon Sep 17 00:00:00 2001 From: Juho Inkinen Date: Wed, 22 May 2019 11:39:28 +0300 Subject: [PATCH 11/66] Drop dev packages (annif: 1.18GB->1.17GB, annif-slim: 653MB->645MB) --- Dockerfile | 5 +---- Dockerfile-slim | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index f43e26b1f..2b45072fc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -32,14 +32,11 @@ RUN pip install --upgrade pip==18.1 \ && pip install pipenv --no-cache-dir COPY . /Annif -# TODO Copy only needed files for pipenv install in this layer -#COPY Pipfile Pipfile.lock setup.py setup.cfg /Annif/ - WORKDIR /Annif # TODO Handle occasional timeout in nltk.downloader leading failed build # TODO Disable caching in pipenv, maybe EXPORT PIP_NO_CACHE_DIR=false -RUN pipenv install --system --deploy --ignore-pipfile --dev \ +RUN pipenv install --system --deploy --ignore-pipfile \ && python -m nltk.downloader punkt \ && pip install --no-cache-dir \ annif[voikko] \ diff --git a/Dockerfile-slim b/Dockerfile-slim index f76742938..063951c6a 100644 --- a/Dockerfile-slim +++ b/Dockerfile-slim @@ -12,7 +12,7 @@ COPY . /Annif WORKDIR /Annif # TODO Handle occasional timeout in nltk.downloader leading failed build -RUN pipenv install --system --deploy --ignore-pipfile --dev \ +RUN pipenv install --system --deploy --ignore-pipfile \ && python -m nltk.downloader punkt CMD annif From 7c9ee055f5b31d875d5bd1ecfeefcab709f589ae Mon Sep 17 00:00:00 2001 From: Juho Inkinen Date: Wed, 22 May 2019 14:23:00 +0300 Subject: [PATCH 12/66] Split context copying to avoid rebuilding pipenv layer on code change --- .dockerignore | 2 -- Dockerfile | 4 +++- Dockerfile-slim | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.dockerignore b/.dockerignore index 2c00ab44b..766836180 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,4 +1,2 @@ -docs -Dockerfile* # TODO Remove from here and avoid copying altogether in Dockerfile **/__pycache__ **/*.pyc diff --git a/Dockerfile b/Dockerfile index 2b45072fc..bd8fc35eb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,7 +31,8 @@ RUN apt-get update \ RUN pip install --upgrade pip==18.1 \ && pip install pipenv --no-cache-dir -COPY . /Annif +# Files needed by pipenv install: +COPY Pipfile Pipfile.lock README.md setup.py /Annif/ WORKDIR /Annif # TODO Handle occasional timeout in nltk.downloader leading failed build @@ -43,5 +44,6 @@ RUN pipenv install --system --deploy --ignore-pipfile \ fasttextmirror \ vowpalwabbit==8.4 +COPY annif annif CMD annif diff --git a/Dockerfile-slim b/Dockerfile-slim index 063951c6a..b03934662 100644 --- a/Dockerfile-slim +++ b/Dockerfile-slim @@ -5,14 +5,14 @@ FROM python:3.6-slim RUN pip install --upgrade pip==18.1 \ && pip install pipenv --no-cache-dir -COPY . /Annif -# TODO Copy only needed files for pipenv install in this layer -#COPY Pipfile Pipfile.lock setup.py setup.cfg /Annif/ - +# Files needed by pipenv install: +COPY Pipfile Pipfile.lock README.md setup.py /Annif/ WORKDIR /Annif # TODO Handle occasional timeout in nltk.downloader leading failed build RUN pipenv install --system --deploy --ignore-pipfile \ && python -m nltk.downloader punkt +COPY annif annif + CMD annif From eac347bdceae35233dd83c905eaee7e80252b6fd Mon Sep 17 00:00:00 2001 From: Juho Inkinen Date: Wed, 22 May 2019 15:00:13 +0300 Subject: [PATCH 13/66] Will not be using alpine base image; python-slim is small enough --- Dockerfile-alpine | 14 -------------- 1 file changed, 14 deletions(-) delete mode 100644 Dockerfile-alpine diff --git a/Dockerfile-alpine b/Dockerfile-alpine deleted file mode 100644 index 3576f9941..000000000 --- a/Dockerfile-alpine +++ /dev/null @@ -1,14 +0,0 @@ -FROM python:3.6-alpine - -# Using old pip version because --no-cache-dir doesn't seem to work in 19.1.1 -RUN apk --no-cache update \ - && echo "http://dl-8.alpinelinux.org/alpine/edge/community" >> /etc/apk/repositories \ - && apk --no-cache --update-cache add make automake gcc g++ gfortran build-base wget freetype-dev libpng-dev openblas-dev \ - && pip install --upgrade pip==18.1 \ - && pip install cython pipenv --no-cache-dir - -COPY . /Annif -WORKDIR /Annif - -RUN pipenv install --system --deploy --ignore-pipfile --dev -RUN python -m nltk.downloader punkt From 5b501275016818bb49dda3ab68a973d5c9ce0dcc Mon Sep 17 00:00:00 2001 From: Juho Inkinen Date: Wed, 22 May 2019 16:53:10 +0300 Subject: [PATCH 14/66] Reduce image size by clearing pipenv cache --- Dockerfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index bd8fc35eb..261258aba 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,7 +29,8 @@ RUN apt-get update \ ## Install pipenv and Annif # Using old pip version because --no-cache-dir doesn't seem to work in 19.1.1 RUN pip install --upgrade pip==18.1 \ - && pip install pipenv --no-cache-dir + && pip install pipenv --no-cache-dir \ + && rm -rf /root/.cache/pip*/* # Files needed by pipenv install: COPY Pipfile Pipfile.lock README.md setup.py /Annif/ @@ -42,7 +43,8 @@ RUN pipenv install --system --deploy --ignore-pipfile \ && pip install --no-cache-dir \ annif[voikko] \ fasttextmirror \ - vowpalwabbit==8.4 + vowpalwabbit==8.4 \ + && rm -rf /root/.cache/pip*/* COPY annif annif From 1a89e3c3c99fe1d1fc10473865679e3874c79379 Mon Sep 17 00:00:00 2001 From: Juho Inkinen Date: Wed, 22 May 2019 16:54:45 +0300 Subject: [PATCH 15/66] Reduce image size by removing python2 (installed by libboost-python-dev) --- Dockerfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 261258aba..c09c2dc8d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,10 +20,12 @@ RUN apt-get update \ # Vowpal Wabbit. Using old VW because 8.5 links to wrong Python version RUN apt-get update \ && apt-get install -y --no-install-recommends \ - libboost-program-options-dev\ + libboost-program-options-dev \ zlib1g-dev \ libboost-python-dev \ - && rm -rf /var/lib/apt/lists/* + && rm -rf /var/lib/apt/lists/* \ + && dpkg -P --force-all python2.7 python2.7-dev python2.7-minimal \ + && rm -rf ./usr/lib/python2.7* ./usr/bin/python2.7 ./usr/lib/x86_64-linux-gnu/libpython2.7.so.1.0 ## Install pipenv and Annif From e8e74f1d11b2fe2719511717229b465646e12653 Mon Sep 17 00:00:00 2001 From: Juho Inkinen Date: Fri, 24 May 2019 10:33:38 +0300 Subject: [PATCH 16/66] Reduce image size by removing unnecessary dev packages with dpgk --- Dockerfile | 57 ++++++++++++++++++++++++++++++++---------------------- 1 file changed, 34 insertions(+), 23 deletions(-) diff --git a/Dockerfile b/Dockerfile index c09c2dc8d..d59014f1d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,51 +1,62 @@ FROM python:3.6-slim +## Update Using old pip version because --no-cache-dir doesn't seem to work in 19.1.1 +RUN pip install --upgrade pip==18.1 \ + && rm -rf /root/.cache/pip*/* + + ## Install optional dependencies # Voikko -RUN apt-get update \ - && apt-get install -y --no-install-recommends \ +RUN apt-get update && apt-get install -y --no-install-recommends \ libvoikko1 \ voikko-fi \ - && rm -rf /var/lib/apt/lists/* + && pip install --no-cache-dir \ + annif[voikko] \ + && rm -rf /var/lib/apt/lists/* /usr/include/* # fasttext -RUN apt-get update \ - && apt-get install -y --no-install-recommends \ +RUN apt-get update && apt-get install -y --no-install-recommends \ build-essential \ && pip install --no-cache-dir \ cython \ - && rm -rf /var/lib/apt/lists/* + fasttextmirror \ + && apt-get remove --auto-remove -y build-essential \ + && rm -rf /var/lib/apt/lists/* /usr/include/* # Vowpal Wabbit. Using old VW because 8.5 links to wrong Python version -RUN apt-get update \ - && apt-get install -y --no-install-recommends \ +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ libboost-program-options-dev \ zlib1g-dev \ libboost-python-dev \ - && rm -rf /var/lib/apt/lists/* \ - && dpkg -P --force-all python2.7 python2.7-dev python2.7-minimal \ - && rm -rf ./usr/lib/python2.7* ./usr/bin/python2.7 ./usr/lib/x86_64-linux-gnu/libpython2.7.so.1.0 - + && pip install --no-cache-dir \ + vowpalwabbit==8.4 \ + && apt-get remove --auto-remove -y \ + build-essential \ + zlib1g-dev \ + && rm -rf /var/lib/apt/lists/* /usr/include/* \ + && dpkg -r --force-depends \ + libstdc++-6-dev \ + libpython2.7-dev \ + libpython3.5-dev \ + libc6-dev \ + libgcc-6-dev \ + python2.7 python2.7-dev python2.7-minimal \ + && rm -rf \ + /usr/lib/python2.7* -## Install pipenv and Annif -# Using old pip version because --no-cache-dir doesn't seem to work in 19.1.1 -RUN pip install --upgrade pip==18.1 \ - && pip install pipenv --no-cache-dir \ - && rm -rf /root/.cache/pip*/* +## Install Annif # Files needed by pipenv install: COPY Pipfile Pipfile.lock README.md setup.py /Annif/ WORKDIR /Annif # TODO Handle occasional timeout in nltk.downloader leading failed build -# TODO Disable caching in pipenv, maybe EXPORT PIP_NO_CACHE_DIR=false -RUN pipenv install --system --deploy --ignore-pipfile \ +RUN pip install pipenv --no-cache-dir \ + && pipenv install --system --deploy --ignore-pipfile \ && python -m nltk.downloader punkt \ - && pip install --no-cache-dir \ - annif[voikko] \ - fasttextmirror \ - vowpalwabbit==8.4 \ + && pip uninstall -y pipenv \ && rm -rf /root/.cache/pip*/* COPY annif annif From 712d2a0d522285813132cce7bdc7eca29ba2f08b Mon Sep 17 00:00:00 2001 From: Juho Inkinen Date: Fri, 24 May 2019 10:39:14 +0300 Subject: [PATCH 17/66] Remove dpgk deletions, because they break apt-get --- Dockerfile | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index d59014f1d..140b66fe7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -36,15 +36,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ build-essential \ zlib1g-dev \ && rm -rf /var/lib/apt/lists/* /usr/include/* \ - && dpkg -r --force-depends \ - libstdc++-6-dev \ - libpython2.7-dev \ - libpython3.5-dev \ - libc6-dev \ - libgcc-6-dev \ - python2.7 python2.7-dev python2.7-minimal \ - && rm -rf \ - /usr/lib/python2.7* + && rm -rf /root/.cache/pip*/* \ + && rm -rf /usr/lib/python2.7* ## Install Annif From 280d7ef4b1848f318e6bd71f7adae04cd8ea39e1 Mon Sep 17 00:00:00 2001 From: Juho Inkinen Date: Fri, 24 May 2019 10:40:29 +0300 Subject: [PATCH 18/66] Reduce image size by not updating pip --- Dockerfile | 5 ----- 1 file changed, 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 140b66fe7..deea6bbf9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,6 @@ FROM python:3.6-slim -## Update Using old pip version because --no-cache-dir doesn't seem to work in 19.1.1 -RUN pip install --upgrade pip==18.1 \ - && rm -rf /root/.cache/pip*/* - - ## Install optional dependencies # Voikko RUN apt-get update && apt-get install -y --no-install-recommends \ From 4714804eebe071bf9c88b9cf69d6890327f5b77c Mon Sep 17 00:00:00 2001 From: Juho Inkinen Date: Fri, 24 May 2019 10:41:08 +0300 Subject: [PATCH 19/66] Unite layers for opt. dependencies installation --- Dockerfile | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/Dockerfile b/Dockerfile index deea6bbf9..7f65103f5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,31 +2,27 @@ FROM python:3.6-slim ## Install optional dependencies -# Voikko -RUN apt-get update && apt-get install -y --no-install-recommends \ +RUN apt-get update \ + ## Voikko: + && apt-get install -y --no-install-recommends \ libvoikko1 \ voikko-fi \ && pip install --no-cache-dir \ annif[voikko] \ - && rm -rf /var/lib/apt/lists/* /usr/include/* - -# fasttext -RUN apt-get update && apt-get install -y --no-install-recommends \ + ## fasttext: + && apt-get install -y --no-install-recommends \ build-essential \ && pip install --no-cache-dir \ cython \ fasttextmirror \ - && apt-get remove --auto-remove -y build-essential \ - && rm -rf /var/lib/apt/lists/* /usr/include/* - -# Vowpal Wabbit. Using old VW because 8.5 links to wrong Python version -RUN apt-get update && apt-get install -y --no-install-recommends \ - build-essential \ + ## Vowpal Wabbit. Using old VW because 8.5 links to wrong Python version + && apt-get install -y --no-install-recommends \ libboost-program-options-dev \ zlib1g-dev \ libboost-python-dev \ && pip install --no-cache-dir \ vowpalwabbit==8.4 \ + ## Clean up: && apt-get remove --auto-remove -y \ build-essential \ zlib1g-dev \ From fd181461643ac4f80ab8e1cc0f3eb7f569ff98ad Mon Sep 17 00:00:00 2001 From: Juho Inkinen Date: Fri, 24 May 2019 10:51:53 +0300 Subject: [PATCH 20/66] Add maintainer entry --- Dockerfile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7f65103f5..a5f1012b5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,10 @@ FROM python:3.6-slim -## Install optional dependencies +LABEL maintainer="Juho Inkinen " + + +## Install optional dependencies: RUN apt-get update \ ## Voikko: && apt-get install -y --no-install-recommends \ @@ -31,7 +34,7 @@ RUN apt-get update \ && rm -rf /usr/lib/python2.7* -## Install Annif +## Install Annif: # Files needed by pipenv install: COPY Pipfile Pipfile.lock README.md setup.py /Annif/ WORKDIR /Annif From 812c3c67f5c1c40b4f563d650dc578141ef08c7b Mon Sep 17 00:00:00 2001 From: Juho Inkinen Date: Fri, 24 May 2019 10:52:20 +0300 Subject: [PATCH 21/66] Handle occasional timeout in nltk.downloader with 3 tries --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index a5f1012b5..c3cce9ce5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -39,10 +39,10 @@ RUN apt-get update \ COPY Pipfile Pipfile.lock README.md setup.py /Annif/ WORKDIR /Annif -# TODO Handle occasional timeout in nltk.downloader leading failed build +# Handle occasional timeout in nltk.downloader with 3 tries RUN pip install pipenv --no-cache-dir \ && pipenv install --system --deploy --ignore-pipfile \ - && python -m nltk.downloader punkt \ + && for i in 1 2 3; do python -m nltk.downloader punkt && break || sleep 1; done \ && pip uninstall -y pipenv \ && rm -rf /root/.cache/pip*/* From bfb1ec75686ed142b4e943fd26068e6187c22e65 Mon Sep 17 00:00:00 2001 From: Juho Inkinen Date: Fri, 24 May 2019 11:14:43 +0300 Subject: [PATCH 22/66] Dockerfile including dev packages and tests (and whole Annif root) --- Dockerfile-dev | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 Dockerfile-dev diff --git a/Dockerfile-dev b/Dockerfile-dev new file mode 100644 index 000000000..ef3835f7b --- /dev/null +++ b/Dockerfile-dev @@ -0,0 +1,8 @@ +FROM annif + +RUN pip install pipenv --no-cache-dir \ + && pipenv install --system --deploy --ignore-pipfile --dev + +COPY . . + +CMD annif From 4c3325edac7c797a44bebe318e693895ab41eaea Mon Sep 17 00:00:00 2001 From: Juho Inkinen Date: Fri, 24 May 2019 11:17:46 +0300 Subject: [PATCH 23/66] Delete unnecessary files; continue using Dockerfile and Dockerfile-dev --- Dockerfile-optional-dependencies | 31 ------------------------------- Dockerfile-slim | 18 ------------------ 2 files changed, 49 deletions(-) delete mode 100644 Dockerfile-optional-dependencies delete mode 100644 Dockerfile-slim diff --git a/Dockerfile-optional-dependencies b/Dockerfile-optional-dependencies deleted file mode 100644 index e106ee985..000000000 --- a/Dockerfile-optional-dependencies +++ /dev/null @@ -1,31 +0,0 @@ -FROM annif-slim - -# Voikko -RUN apt-get update \ - && apt-get install -y --no-install-recommends \ - libvoikko1 \ - voikko-fi \ - && pip install --no-cache-dir \ - annif[voikko] \ - && rm -rf /var/lib/apt/lists/* - -# fasttext -RUN apt-get update \ - && apt-get install -y --no-install-recommends \ - build-essential \ - && pip install --no-cache-dir \ - cython \ - fasttextmirror \ - && rm -rf /var/lib/apt/lists/* - -# Vowpal Wabbit. Using old VW because 8.5 links to wrong Python version -RUN apt-get update \ - && apt-get install -y --no-install-recommends \ - libboost-program-options-dev\ - zlib1g-dev \ - libboost-python-dev \ - && pip install --no-cache-dir \ - vowpalwabbit==8.4 \ - && rm -rf /var/lib/apt/lists/* - -CMD annif diff --git a/Dockerfile-slim b/Dockerfile-slim deleted file mode 100644 index b03934662..000000000 --- a/Dockerfile-slim +++ /dev/null @@ -1,18 +0,0 @@ -FROM python:3.6-slim - -# TODO Using old pip version because --no-cache-dir doesn't seem to work in 19.1.1 -# TODO Disable caching in pipenv, maybe EXPORT PIP_NO_CACHE_DIR=false -RUN pip install --upgrade pip==18.1 \ - && pip install pipenv --no-cache-dir - -# Files needed by pipenv install: -COPY Pipfile Pipfile.lock README.md setup.py /Annif/ -WORKDIR /Annif - -# TODO Handle occasional timeout in nltk.downloader leading failed build -RUN pipenv install --system --deploy --ignore-pipfile \ - && python -m nltk.downloader punkt - -COPY annif annif - -CMD annif From 4a615b8302c6d5706c6141ab31aa1927fb4b6634 Mon Sep 17 00:00:00 2001 From: Juho Inkinen Date: Fri, 24 May 2019 11:21:21 +0300 Subject: [PATCH 24/66] Spaces to tabs --- Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index c3cce9ce5..7d4c83e5e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,20 +10,20 @@ RUN apt-get update \ && apt-get install -y --no-install-recommends \ libvoikko1 \ voikko-fi \ - && pip install --no-cache-dir \ - annif[voikko] \ + && pip install --no-cache-dir \ + annif[voikko] \ ## fasttext: && apt-get install -y --no-install-recommends \ build-essential \ && pip install --no-cache-dir \ cython \ - fasttextmirror \ + fasttextmirror \ ## Vowpal Wabbit. Using old VW because 8.5 links to wrong Python version && apt-get install -y --no-install-recommends \ libboost-program-options-dev \ zlib1g-dev \ libboost-python-dev \ - && pip install --no-cache-dir \ + && pip install --no-cache-dir \ vowpalwabbit==8.4 \ ## Clean up: && apt-get remove --auto-remove -y \ From f86bdd98023dd8b5afce42bd1a4a5f85987c3909 Mon Sep 17 00:00:00 2001 From: Juho Inkinen Date: Fri, 24 May 2019 16:55:50 +0300 Subject: [PATCH 25/66] Also projects.cfg is needed to run Annif --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 7d4c83e5e..20fa4dacb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -47,5 +47,6 @@ RUN pip install pipenv --no-cache-dir \ && rm -rf /root/.cache/pip*/* COPY annif annif +COPY projects.cfg.dist projects.cfg.dist CMD annif From 73d59b3e57d3d783d53c1e791a42b27c9cab79e7 Mon Sep 17 00:00:00 2001 From: Juho Inkinen Date: Mon, 27 May 2019 16:14:16 +0300 Subject: [PATCH 26/66] NLTK data need to be accessible to non-root --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 20fa4dacb..a5faa77ba 100644 --- a/Dockerfile +++ b/Dockerfile @@ -42,7 +42,7 @@ WORKDIR /Annif # Handle occasional timeout in nltk.downloader with 3 tries RUN pip install pipenv --no-cache-dir \ && pipenv install --system --deploy --ignore-pipfile \ - && for i in 1 2 3; do python -m nltk.downloader punkt && break || sleep 1; done \ + && for i in 1 2 3; do python -m nltk.downloader punkt -d /usr/share/nltk_data && break || sleep 1; done \ && pip uninstall -y pipenv \ && rm -rf /root/.cache/pip*/* From a1ea7370e5ab6afa4ce65cdebf7ed28a80880b49 Mon Sep 17 00:00:00 2001 From: Juho Inkinen Date: Mon, 27 May 2019 16:15:35 +0300 Subject: [PATCH 27/66] Ease usage: start from path to be mounted & make projects.cfg available --- Dockerfile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index a5faa77ba..ce4cd7dc4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -46,7 +46,12 @@ RUN pip install pipenv --no-cache-dir \ && pip uninstall -y pipenv \ && rm -rf /root/.cache/pip*/* + COPY annif annif -COPY projects.cfg.dist projects.cfg.dist +COPY projects.cfg.dist /Annif/projects.cfg + + +WORKDIR /annif_projects +ENV ANNIF_PROJECTS=/Annif/projects.cfg CMD annif From fcc15eb94e89391d9d258aca037897261ac6a188 Mon Sep 17 00:00:00 2001 From: Juho Inkinen Date: Tue, 28 May 2019 12:32:46 +0300 Subject: [PATCH 28/66] Use projects.cfg from current dir --- Dockerfile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index ce4cd7dc4..76d02e937 100644 --- a/Dockerfile +++ b/Dockerfile @@ -48,10 +48,8 @@ RUN pip install pipenv --no-cache-dir \ COPY annif annif -COPY projects.cfg.dist /Annif/projects.cfg - +COPY projects.cfg.dist /Annif/projects.cfg.dist WORKDIR /annif_projects -ENV ANNIF_PROJECTS=/Annif/projects.cfg CMD annif From 6881e265b262a0c7e33f84b98d61ff829699caef Mon Sep 17 00:00:00 2001 From: Juho Inkinen Date: Tue, 28 May 2019 12:52:42 +0300 Subject: [PATCH 29/66] Do not use Pipfile.lock --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 76d02e937..a256ac558 100644 --- a/Dockerfile +++ b/Dockerfile @@ -36,12 +36,12 @@ RUN apt-get update \ ## Install Annif: # Files needed by pipenv install: -COPY Pipfile Pipfile.lock README.md setup.py /Annif/ +COPY Pipfile README.md setup.py /Annif/ WORKDIR /Annif # Handle occasional timeout in nltk.downloader with 3 tries RUN pip install pipenv --no-cache-dir \ - && pipenv install --system --deploy --ignore-pipfile \ + && pipenv install --system --deploy \ && for i in 1 2 3; do python -m nltk.downloader punkt -d /usr/share/nltk_data && break || sleep 1; done \ && pip uninstall -y pipenv \ && rm -rf /root/.cache/pip*/* From 39cdcd5bc9e2e324adbc7c1910b0c1cfc68f0387 Mon Sep 17 00:00:00 2001 From: Juho Inkinen Date: Tue, 28 May 2019 12:57:14 +0300 Subject: [PATCH 30/66] Skip Pipenv lockfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index a256ac558..b59390f75 100644 --- a/Dockerfile +++ b/Dockerfile @@ -41,7 +41,7 @@ WORKDIR /Annif # Handle occasional timeout in nltk.downloader with 3 tries RUN pip install pipenv --no-cache-dir \ - && pipenv install --system --deploy \ + && pipenv install --system --skip-lock \ && for i in 1 2 3; do python -m nltk.downloader punkt -d /usr/share/nltk_data && break || sleep 1; done \ && pip uninstall -y pipenv \ && rm -rf /root/.cache/pip*/* From 8ca769a229c9b52100e00fab426c205c1f1e1b55 Mon Sep 17 00:00:00 2001 From: Juho Inkinen Date: Tue, 28 May 2019 15:02:45 +0300 Subject: [PATCH 31/66] Switch user to non-root --- Dockerfile | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Dockerfile b/Dockerfile index b59390f75..5f30501e3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -52,4 +52,11 @@ COPY projects.cfg.dist /Annif/projects.cfg.dist WORKDIR /annif_projects + +# Switch user to non-root: +RUN groupadd -g 999 annif_user \ + && useradd -r -u 999 -g annif_user annif_user \ + && chown -R annif_user:annif_user /annif_projects +USER annif_user + CMD annif From 13feaec9adfb86daaf5cfa82ab23dcda74ee3c92 Mon Sep 17 00:00:00 2001 From: Juho Inkinen Date: Tue, 28 May 2019 15:20:08 +0300 Subject: [PATCH 32/66] Give everyone full access to /Annif --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index 5f30501e3..1903bbe21 100644 --- a/Dockerfile +++ b/Dockerfile @@ -49,6 +49,7 @@ RUN pip install pipenv --no-cache-dir \ COPY annif annif COPY projects.cfg.dist /Annif/projects.cfg.dist +RUN chmod -R 777 /Annif WORKDIR /annif_projects @@ -59,4 +60,5 @@ RUN groupadd -g 999 annif_user \ && chown -R annif_user:annif_user /annif_projects USER annif_user + CMD annif From 9ef2f1de511737d7a6a4dab80537cec3568e996a Mon Sep 17 00:00:00 2001 From: Juho Inkinen Date: Tue, 28 May 2019 16:44:31 +0300 Subject: [PATCH 33/66] Switch user to non-root, skip Pipenv lockfile, give access to tests --- Dockerfile-dev | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/Dockerfile-dev b/Dockerfile-dev index ef3835f7b..860767f20 100644 --- a/Dockerfile-dev +++ b/Dockerfile-dev @@ -1,8 +1,15 @@ FROM annif +WORKDIR /Annif + +USER root + RUN pip install pipenv --no-cache-dir \ - && pipenv install --system --deploy --ignore-pipfile --dev + && pipenv install --system --skip-lock --dev COPY . . +RUN chmod -R 777 /Annif + +USER annif_user CMD annif From 791e7410675f415179f402a495b5a73b4558f732 Mon Sep 17 00:00:00 2001 From: Juho Inkinen Date: Fri, 31 May 2019 16:29:52 +0300 Subject: [PATCH 34/66] Initial infra for docker-compose to use nginx and gunicorn --- Dockerfile-server | 11 +++++++++++ annif/nginx/Dockerfile | 4 ++++ annif/nginx/nginx.conf | 12 ++++++++++++ docker-compose.yml | 23 +++++++++++++++++++++++ 4 files changed, 50 insertions(+) create mode 100644 Dockerfile-server create mode 100644 annif/nginx/Dockerfile create mode 100644 annif/nginx/nginx.conf create mode 100644 docker-compose.yml diff --git a/Dockerfile-server b/Dockerfile-server new file mode 100644 index 000000000..fb98071d1 --- /dev/null +++ b/Dockerfile-server @@ -0,0 +1,11 @@ +FROM annif + +USER root + +RUN pip install gunicorn + +#USER annif_user + +RUN cp /Annif/projects.cfg.dist /annif_projects/projects.cfg + +CMD ["gunicorn", "annif:create_app()", "--bind", "0.0.0.0:8000"] diff --git a/annif/nginx/Dockerfile b/annif/nginx/Dockerfile new file mode 100644 index 000000000..c1d1b98c7 --- /dev/null +++ b/annif/nginx/Dockerfile @@ -0,0 +1,4 @@ +FROM nginx:alpine + +RUN rm /etc/nginx/conf.d/default.conf +COPY nginx.conf /etc/nginx/conf.d diff --git a/annif/nginx/nginx.conf b/annif/nginx/nginx.conf new file mode 100644 index 000000000..f0294ebb1 --- /dev/null +++ b/annif/nginx/nginx.conf @@ -0,0 +1,12 @@ +upstream annif_server_dummy_linking_string { + server annif_server:8000; +} + +server { + listen 80; + server_name localhost; + + location / { + proxy_pass http://annif_server_dummy_linking_string; + } +} diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 000000000..9e7e5126e --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,23 @@ +version: "3" + +services: + + annif_server: + restart: always + build: + context: . + dockerfile: Dockerfile-server + + nginx: + restart: always + build: + context: annif/nginx/ + ## Instead of having own Dockerfile for nginx image, this could be used: + #image: nginx:alpine + #volumes: + # - ./annif/nginx/nginx.conf:/etc/nginx/nginx.conf + #command: nginx #-c /etc/nginx/nginx.conf + ports: + - "80:80" + depends_on: + - annif_server From 7a3550f37fc456d8bc9d2aea1279bb0a970d26a5 Mon Sep 17 00:00:00 2001 From: Juho Inkinen Date: Mon, 3 Jun 2019 13:58:22 +0300 Subject: [PATCH 35/66] Use a more proper name for the upstream context --- annif/nginx/nginx.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/annif/nginx/nginx.conf b/annif/nginx/nginx.conf index f0294ebb1..2f2eddfa8 100644 --- a/annif/nginx/nginx.conf +++ b/annif/nginx/nginx.conf @@ -1,4 +1,4 @@ -upstream annif_server_dummy_linking_string { +upstream upstream_servers { server annif_server:8000; } @@ -7,6 +7,6 @@ server { server_name localhost; location / { - proxy_pass http://annif_server_dummy_linking_string; + proxy_pass http://upstream_servers; } } From 3bc89d81b8060d3db803dcd71a4934dc91fad6af Mon Sep 17 00:00:00 2001 From: Juho Inkinen Date: Tue, 4 Jun 2019 09:13:34 +0300 Subject: [PATCH 36/66] Add gunicorn to main Docker image; no point using Dockerfile-server --- Dockerfile-server | 11 ----------- Pipfile | 1 + docker-compose.yml | 8 +++++--- 3 files changed, 6 insertions(+), 14 deletions(-) delete mode 100644 Dockerfile-server diff --git a/Dockerfile-server b/Dockerfile-server deleted file mode 100644 index fb98071d1..000000000 --- a/Dockerfile-server +++ /dev/null @@ -1,11 +0,0 @@ -FROM annif - -USER root - -RUN pip install gunicorn - -#USER annif_user - -RUN cp /Annif/projects.cfg.dist /annif_projects/projects.cfg - -CMD ["gunicorn", "annif:create_app()", "--bind", "0.0.0.0:8000"] diff --git a/Pipfile b/Pipfile index 52c3c66f0..a2a7b4754 100644 --- a/Pipfile +++ b/Pipfile @@ -23,5 +23,6 @@ nltk = "*" gensim = "*" sklearn = "*" rdflib = "*" +gunicorn = "*" [requires] diff --git a/docker-compose.yml b/docker-compose.yml index 9e7e5126e..65fae9c35 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,9 +4,11 @@ services: annif_server: restart: always - build: - context: . - dockerfile: Dockerfile-server + image: annif + volumes: + - ./projects.cfg.dist:/annif_projects/projects.cfg + command: ["gunicorn", "annif:create_app()", "--bind", "0.0.0.0:8000"] + nginx: restart: always From 75e4c272e69661fc5499ec4e573579aa51ab210b Mon Sep 17 00:00:00 2001 From: Juho Inkinen Date: Tue, 4 Jun 2019 13:06:42 +0300 Subject: [PATCH 37/66] Use plain nginx image from Docker Hub and mount the config file --- docker-compose.yml | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 65fae9c35..1aaf84cb0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,16 +9,11 @@ services: - ./projects.cfg.dist:/annif_projects/projects.cfg command: ["gunicorn", "annif:create_app()", "--bind", "0.0.0.0:8000"] - nginx: restart: always - build: - context: annif/nginx/ - ## Instead of having own Dockerfile for nginx image, this could be used: - #image: nginx:alpine - #volumes: - # - ./annif/nginx/nginx.conf:/etc/nginx/nginx.conf - #command: nginx #-c /etc/nginx/nginx.conf + image: nginx + volumes: + - ./annif/nginx/nginx.conf:/etc/nginx/conf.d/default.conf ports: - "80:80" depends_on: From c7a564a87a647c539e77a29625271c108d296bba Mon Sep 17 00:00:00 2001 From: Juho Inkinen Date: Tue, 4 Jun 2019 13:29:11 +0300 Subject: [PATCH 38/66] Clarify container names --- annif/nginx/nginx.conf | 2 +- docker-compose.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/annif/nginx/nginx.conf b/annif/nginx/nginx.conf index 2f2eddfa8..6e3099571 100644 --- a/annif/nginx/nginx.conf +++ b/annif/nginx/nginx.conf @@ -1,5 +1,5 @@ upstream upstream_servers { - server annif_server:8000; + server gunicorn_server:8000; } server { diff --git a/docker-compose.yml b/docker-compose.yml index 1aaf84cb0..de312f2ac 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ version: "3" services: - annif_server: + gunicorn_server: restart: always image: annif volumes: @@ -17,4 +17,4 @@ services: ports: - "80:80" depends_on: - - annif_server + - gunicorn_server From 95c5b5f5d09bdb5091d827997d55eabc1da7e40b Mon Sep 17 00:00:00 2001 From: Juho Inkinen Date: Tue, 4 Jun 2019 14:51:37 +0300 Subject: [PATCH 39/66] Use env to set path to annif data; set user to current user --- docker-compose.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index de312f2ac..2db559dfc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,7 +6,8 @@ services: restart: always image: annif volumes: - - ./projects.cfg.dist:/annif_projects/projects.cfg + - ${ANNIF_PROJECTS}:/annif_projects + user: ${UID}:${GID} command: ["gunicorn", "annif:create_app()", "--bind", "0.0.0.0:8000"] nginx: From a7f9c58cb9363ae6aee3e083a5bad5f99d9701d9 Mon Sep 17 00:00:00 2001 From: Juho Inkinen Date: Tue, 4 Jun 2019 15:19:16 +0300 Subject: [PATCH 40/66] Remove unnecessary Dockerfile for NGINX --- annif/nginx/Dockerfile | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 annif/nginx/Dockerfile diff --git a/annif/nginx/Dockerfile b/annif/nginx/Dockerfile deleted file mode 100644 index c1d1b98c7..000000000 --- a/annif/nginx/Dockerfile +++ /dev/null @@ -1,4 +0,0 @@ -FROM nginx:alpine - -RUN rm /etc/nginx/conf.d/default.conf -COPY nginx.conf /etc/nginx/conf.d From 68ee283b56cf7ca66c0b8fb349810d2b265a704e Mon Sep 17 00:00:00 2001 From: Juho Inkinen Date: Wed, 5 Jun 2019 17:41:12 +0300 Subject: [PATCH 41/66] Use multistage build; install only run-time deps (730MB -> 652MB) --- Dockerfile | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1903bbe21..ef526e955 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,7 @@ -FROM python:3.6-slim - +FROM python:3.6-slim AS builder LABEL maintainer="Juho Inkinen " - ## Install optional dependencies: RUN apt-get update \ ## Voikko: @@ -23,15 +21,33 @@ RUN apt-get update \ libboost-program-options-dev \ zlib1g-dev \ libboost-python-dev \ + && pip install --no-cache-dir \ + vowpalwabbit==8.4 + + + +FROM python:3.6-slim + +COPY --from=builder /usr/local/lib/python3.6 /usr/local/lib/python3.6 + +# Dependencies needed at runtime: +## Install optional dependencies: +RUN apt-get update \ + ## Voikko: + && apt-get install -y --no-install-recommends \ + libvoikko1 \ + voikko-fi \ + && pip install --no-cache-dir \ + annif[voikko] \ + ## Vowpal Wabbit. Using old VW because 8.5 links to wrong Python version + && apt-get install -y --no-install-recommends \ + libboost-program-options1.62.0 \ + libboost-python1.62.0 \ && pip install --no-cache-dir \ vowpalwabbit==8.4 \ ## Clean up: - && apt-get remove --auto-remove -y \ - build-essential \ - zlib1g-dev \ && rm -rf /var/lib/apt/lists/* /usr/include/* \ - && rm -rf /root/.cache/pip*/* \ - && rm -rf /usr/lib/python2.7* + && rm -rf /root/.cache/pip*/* ## Install Annif: From d5721f15ed1c5d13e8f498360a898c620ba3aa1a Mon Sep 17 00:00:00 2001 From: Juho Inkinen Date: Tue, 11 Jun 2019 17:29:07 +0300 Subject: [PATCH 42/66] Underscore to dash --- Dockerfile | 4 ++-- docker-compose.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index ef526e955..ac4c86edf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -67,13 +67,13 @@ COPY annif annif COPY projects.cfg.dist /Annif/projects.cfg.dist RUN chmod -R 777 /Annif -WORKDIR /annif_projects +WORKDIR /annif-projects # Switch user to non-root: RUN groupadd -g 999 annif_user \ && useradd -r -u 999 -g annif_user annif_user \ - && chown -R annif_user:annif_user /annif_projects + && chown -R annif_user:annif_user /annif-projects USER annif_user diff --git a/docker-compose.yml b/docker-compose.yml index 2db559dfc..0144cd613 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,7 +6,7 @@ services: restart: always image: annif volumes: - - ${ANNIF_PROJECTS}:/annif_projects + - ${ANNIF_PROJECTS}:/annif-projects user: ${UID}:${GID} command: ["gunicorn", "annif:create_app()", "--bind", "0.0.0.0:8000"] From 39ebe17d2316095c2e2dbff2d9078ee3cff454da Mon Sep 17 00:00:00 2001 From: Juho Inkinen Date: Tue, 11 Jun 2019 17:31:08 +0300 Subject: [PATCH 43/66] Install curl, needed for testing mauiservice --- Dockerfile-dev | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Dockerfile-dev b/Dockerfile-dev index 860767f20..2ffbc8a75 100644 --- a/Dockerfile-dev +++ b/Dockerfile-dev @@ -4,6 +4,10 @@ WORKDIR /Annif USER root +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + curl + RUN pip install pipenv --no-cache-dir \ && pipenv install --system --skip-lock --dev From d33ec1829c2c0fc05c3fd2c20f52a8a3c4d2485a Mon Sep 17 00:00:00 2001 From: Juho Inkinen Date: Wed, 12 Jun 2019 13:05:08 +0300 Subject: [PATCH 44/66] Attempt to trigger build in drone.io; initial drone.yml --- .drone.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 .drone.yml diff --git a/.drone.yml b/.drone.yml new file mode 100644 index 000000000..1fd0d7f04 --- /dev/null +++ b/.drone.yml @@ -0,0 +1,17 @@ +kind: pipeline +name: default +steps: + - name: docker + image: plugins/docker + settings: + dockerfile: Dockerfile + username: + from_secret: docker_username + password: + from_secret: docker_password + registry: quay.io + repo: quay.io/natlibfi/annif + auto_tag: true + when: + event: push + branch: [issue278-dockerize-annif] From bc1a8eef32ee6156438c3d7928ddb894a904f568 Mon Sep 17 00:00:00 2001 From: Juho Inkinen Date: Wed, 12 Jun 2019 13:17:57 +0300 Subject: [PATCH 45/66] Reattempt; do not use auto tag --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 1fd0d7f04..eb061525f 100644 --- a/.drone.yml +++ b/.drone.yml @@ -11,7 +11,7 @@ steps: from_secret: docker_password registry: quay.io repo: quay.io/natlibfi/annif - auto_tag: true + tags: [latest] when: event: push branch: [issue278-dockerize-annif] From d313a3f914f798acabd8f23203dad569b54f86a8 Mon Sep 17 00:00:00 2001 From: Juho Inkinen Date: Wed, 12 Jun 2019 16:45:51 +0300 Subject: [PATCH 46/66] Revoke everyone full access to /Annif --- Dockerfile | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index ac4c86edf..c88d39982 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,8 +30,7 @@ FROM python:3.6-slim COPY --from=builder /usr/local/lib/python3.6 /usr/local/lib/python3.6 -# Dependencies needed at runtime: -## Install optional dependencies: +## Dependencies needed at runtime: RUN apt-get update \ ## Voikko: && apt-get install -y --no-install-recommends \ @@ -64,8 +63,7 @@ RUN pip install pipenv --no-cache-dir \ COPY annif annif -COPY projects.cfg.dist /Annif/projects.cfg.dist -RUN chmod -R 777 /Annif +COPY projects.cfg.dist projects.cfg.dist WORKDIR /annif-projects From 6dd319e78b25ed126f5fafaa90a2ae84d692cf44 Mon Sep 17 00:00:00 2001 From: Juho Inkinen Date: Fri, 14 Jun 2019 14:31:21 +0300 Subject: [PATCH 47/66] Initial compose-file to create a stack in Portainer --- docker-compose-portainer.yml | 54 ++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 docker-compose-portainer.yml diff --git a/docker-compose-portainer.yml b/docker-compose-portainer.yml new file mode 100644 index 000000000..4c6aa689a --- /dev/null +++ b/docker-compose-portainer.yml @@ -0,0 +1,54 @@ +version: "3" + +services: + + gunicorn_server: + restart: always + image: quay.io/natlibfi/annif + # volumes: + # - ${ANNIF_PROJECTS}:/annif-projects + # user: ${UID}:${GID} + command: ["gunicorn", "annif:create_app()", "--bind", "0.0.0.0:8000"] + volumes: + - annif-data:/annif-projects + depends_on: + - data + + deploy: + replicas: 1 + restart_policy: + condition: any + placement: + constraints: + - node.labels.environment == test + + nginx: + restart: always + image: nginx + #volumes: + # - ./annif/nginx/nginx.conf:/etc/nginx/conf.d/default.conf + #ports: + # - "80:80" + depends_on: + - gunicorn_server + + # app: + # image: annif + # stdin_open: true + # tty: true + # command: bash + # volumes: + # - annif-data:/annif-projects + + data: + image: annif-data + volumes: + - annif-data:/annif-data + command: > + sh -c "echo Copying data to named volume annif-data... && + cp -r --verbose /annif-projects/. /annif-data/ && + echo Copied!" + # TODO: Would rsync be better? (Copying tfidf-en model took only 0.06s...) + +volumes: + annif-data: From 1895f0e072e62f96e9177cfbaa031605bcb9cfa9 Mon Sep 17 00:00:00 2001 From: Juho Inkinen Date: Fri, 14 Jun 2019 14:32:47 +0300 Subject: [PATCH 48/66] Initial Dockerfile for transfering data to Portainer volume --- Dockerfile-data | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 Dockerfile-data diff --git a/Dockerfile-data b/Dockerfile-data new file mode 100644 index 000000000..4377f26e5 --- /dev/null +++ b/Dockerfile-data @@ -0,0 +1,13 @@ +FROM alpine +#FROM scratch +COPY data /annif-projects/data +COPY projects.cfg /annif-projects/ + +# Switch user to non-root: +RUN addgroup -g 998 annif_user \ + && adduser -S -u 998 -g annif_user annif_user \ + && chown -R annif_user:annif_user /annif-projects +USER annif_user + +CMD sh +#CMD ["true"] From 2400f02313004951890718952196e2ef8d98f1d1 Mon Sep 17 00:00:00 2001 From: Juho Inkinen Date: Fri, 14 Jun 2019 14:35:15 +0300 Subject: [PATCH 49/66] Switch UID to 998 as 999 clashes in Alpine data image --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index c88d39982..1f57ef7f9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -69,8 +69,8 @@ WORKDIR /annif-projects # Switch user to non-root: -RUN groupadd -g 999 annif_user \ - && useradd -r -u 999 -g annif_user annif_user \ +RUN groupadd -g 998 annif_user \ + && useradd -r -u 998 -g annif_user annif_user \ && chown -R annif_user:annif_user /annif-projects USER annif_user From 56ff7168c5a7f598c0c9a794139b44391e49330f Mon Sep 17 00:00:00 2001 From: Juho Inkinen Date: Fri, 14 Jun 2019 16:47:30 +0300 Subject: [PATCH 50/66] Restart data image only on failure (to copy data) --- docker-compose-portainer.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docker-compose-portainer.yml b/docker-compose-portainer.yml index 4c6aa689a..d14ca6dca 100644 --- a/docker-compose-portainer.yml +++ b/docker-compose-portainer.yml @@ -49,6 +49,14 @@ services: cp -r --verbose /annif-projects/. /annif-data/ && echo Copied!" # TODO: Would rsync be better? (Copying tfidf-en model took only 0.06s...) + deploy: + replicas: 1 + restart_policy: + condition: on-failure + max_attempts: 3 + placement: + constraints: + - node.labels.environment == test volumes: annif-data: From 3e9b950bda08ffb5114bee2651e67e0522a3df8b Mon Sep 17 00:00:00 2001 From: Juho Inkinen Date: Fri, 14 Jun 2019 16:50:02 +0300 Subject: [PATCH 51/66] Reduce image size by switching to busybox base --- Dockerfile-data | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Dockerfile-data b/Dockerfile-data index 4377f26e5..44c6522bf 100644 --- a/Dockerfile-data +++ b/Dockerfile-data @@ -1,5 +1,5 @@ -FROM alpine -#FROM scratch +FROM busybox + COPY data /annif-projects/data COPY projects.cfg /annif-projects/ @@ -10,4 +10,3 @@ RUN addgroup -g 998 annif_user \ USER annif_user CMD sh -#CMD ["true"] From e9020203842fd32a80dc886c0ef1d1a74a2c2690 Mon Sep 17 00:00:00 2001 From: Juho Inkinen Date: Mon, 17 Jun 2019 16:15:13 +0300 Subject: [PATCH 52/66] Use nginx.conf for Portainer via data image --- Dockerfile-data | 2 +- docker-compose-portainer.yml | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/Dockerfile-data b/Dockerfile-data index 44c6522bf..aecb18121 100644 --- a/Dockerfile-data +++ b/Dockerfile-data @@ -1,7 +1,7 @@ FROM busybox COPY data /annif-projects/data -COPY projects.cfg /annif-projects/ +COPY projects.cfg nginx.conf /annif-projects/ # Switch user to non-root: RUN addgroup -g 998 annif_user \ diff --git a/docker-compose-portainer.yml b/docker-compose-portainer.yml index d14ca6dca..e31622937 100644 --- a/docker-compose-portainer.yml +++ b/docker-compose-portainer.yml @@ -25,12 +25,13 @@ services: nginx: restart: always image: nginx - #volumes: - # - ./annif/nginx/nginx.conf:/etc/nginx/conf.d/default.conf - #ports: - # - "80:80" + volumes: + - annif-data:/annif-projects + ports: + - "80:80" depends_on: - gunicorn_server + command: ["nginx", "-c", "/annif-projects/nginx.conf", "-g", "daemon off;"] # app: # image: annif From c6a210939528f112bcf55cb57f65318348866c8a Mon Sep 17 00:00:00 2001 From: Juho Inkinen Date: Mon, 17 Jun 2019 16:35:46 +0300 Subject: [PATCH 53/66] Do not publish ports in Portainer to avoid clashes --- docker-compose-portainer.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose-portainer.yml b/docker-compose-portainer.yml index e31622937..13ae5df84 100644 --- a/docker-compose-portainer.yml +++ b/docker-compose-portainer.yml @@ -27,8 +27,8 @@ services: image: nginx volumes: - annif-data:/annif-projects - ports: - - "80:80" + # ports: + # - "80:80" depends_on: - gunicorn_server command: ["nginx", "-c", "/annif-projects/nginx.conf", "-g", "daemon off;"] From 8763fe8110a62057410e9f4f8c98ba5976787c94 Mon Sep 17 00:00:00 2001 From: Juho Inkinen Date: Tue, 18 Jun 2019 09:53:28 +0300 Subject: [PATCH 54/66] Directly mount contents of annif-projects to volume instead copying --- docker-compose-portainer.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/docker-compose-portainer.yml b/docker-compose-portainer.yml index 13ae5df84..d6a166c5b 100644 --- a/docker-compose-portainer.yml +++ b/docker-compose-portainer.yml @@ -44,12 +44,7 @@ services: data: image: annif-data volumes: - - annif-data:/annif-data - command: > - sh -c "echo Copying data to named volume annif-data... && - cp -r --verbose /annif-projects/. /annif-data/ && - echo Copied!" - # TODO: Would rsync be better? (Copying tfidf-en model took only 0.06s...) + - annif-data:/annif-projects deploy: replicas: 1 restart_policy: From 066e8610afe3d8fbba245620b84cf631494490c0 Mon Sep 17 00:00:00 2001 From: Juho Inkinen Date: Tue, 18 Jun 2019 09:58:26 +0300 Subject: [PATCH 55/66] Rename volume (to avoid eventual nogood name "annif_annif-data") --- docker-compose-portainer.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docker-compose-portainer.yml b/docker-compose-portainer.yml index d6a166c5b..bf653da6e 100644 --- a/docker-compose-portainer.yml +++ b/docker-compose-portainer.yml @@ -10,7 +10,7 @@ services: # user: ${UID}:${GID} command: ["gunicorn", "annif:create_app()", "--bind", "0.0.0.0:8000"] volumes: - - annif-data:/annif-projects + - data:/annif-projects depends_on: - data @@ -26,7 +26,7 @@ services: restart: always image: nginx volumes: - - annif-data:/annif-projects + - data:/annif-projects # ports: # - "80:80" depends_on: @@ -39,12 +39,12 @@ services: # tty: true # command: bash # volumes: - # - annif-data:/annif-projects + # - data:/annif-projects data: image: annif-data volumes: - - annif-data:/annif-projects + - data:/annif-projects deploy: replicas: 1 restart_policy: @@ -55,4 +55,4 @@ services: - node.labels.environment == test volumes: - annif-data: + data: From 5ff34464231dc207fd29349eef320b3b45fabf5a Mon Sep 17 00:00:00 2001 From: Juho Inkinen Date: Thu, 20 Jun 2019 09:27:59 +0300 Subject: [PATCH 56/66] Trigger to push also dev image to quay --- .drone.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.drone.yml b/.drone.yml index eb061525f..8bb728cd6 100644 --- a/.drone.yml +++ b/.drone.yml @@ -15,3 +15,17 @@ steps: when: event: push branch: [issue278-dockerize-annif] + - name: docker-dev + image: plugins/docker + settings: + dockerfile: Dockerfile-dev + username: + from_secret: docker_username + password: + from_secret: docker_password + registry: quay.io + repo: quay.io/natlibfi/annif + tags: [dev] + when: + event: push + branch: [issue278-dockerize-annif] From 46c2a547a5bba9695effaa3fb4c9678bd3606c94 Mon Sep 17 00:00:00 2001 From: Juho Inkinen Date: Thu, 20 Jun 2019 09:33:27 +0300 Subject: [PATCH 57/66] Make own pipeline for dev image deployment --- .drone.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.drone.yml b/.drone.yml index 8bb728cd6..8a4fe9791 100644 --- a/.drone.yml +++ b/.drone.yml @@ -15,6 +15,10 @@ steps: when: event: push branch: [issue278-dockerize-annif] + +kind: pipeline +name: dev-image-deployment +steps: - name: docker-dev image: plugins/docker settings: From 47901ab493e181656b38e3b466e82b48dc90bad6 Mon Sep 17 00:00:00 2001 From: Juho Inkinen Date: Thu, 20 Jun 2019 09:36:03 +0300 Subject: [PATCH 58/66] Fix intendation --- .drone.yml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.drone.yml b/.drone.yml index 8a4fe9791..0656c1d39 100644 --- a/.drone.yml +++ b/.drone.yml @@ -20,16 +20,16 @@ kind: pipeline name: dev-image-deployment steps: - name: docker-dev - image: plugins/docker - settings: - dockerfile: Dockerfile-dev - username: - from_secret: docker_username - password: - from_secret: docker_password - registry: quay.io - repo: quay.io/natlibfi/annif - tags: [dev] - when: - event: push - branch: [issue278-dockerize-annif] + image: plugins/docker + settings: + dockerfile: Dockerfile-dev + username: + from_secret: docker_username + password: + from_secret: docker_password + registry: quay.io + repo: quay.io/natlibfi/annif + tags: [dev] + when: + event: push + branch: [issue278-dockerize-annif] From 72acbb4aff991a19fa8823668bd1338c24972ca8 Mon Sep 17 00:00:00 2001 From: Juho Inkinen Date: Thu, 20 Jun 2019 09:39:32 +0300 Subject: [PATCH 59/66] Use image from quay --- Dockerfile-dev | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile-dev b/Dockerfile-dev index 2ffbc8a75..a82296f8f 100644 --- a/Dockerfile-dev +++ b/Dockerfile-dev @@ -1,4 +1,4 @@ -FROM annif +FROM quay.io/natlibfi/annif WORKDIR /Annif From eaf5c6630b64e5d137df557394fac6fd7c9d94e2 Mon Sep 17 00:00:00 2001 From: Juho Inkinen Date: Thu, 20 Jun 2019 10:45:56 +0300 Subject: [PATCH 60/66] Use Python 3.7 instead of 3.6 (hoping to solve scanner issues in quay) --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1f57ef7f9..d0320e1eb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.6-slim AS builder +FROM python:3.7-slim AS builder LABEL maintainer="Juho Inkinen " @@ -26,9 +26,9 @@ RUN apt-get update \ -FROM python:3.6-slim +FROM python:3.7-slim -COPY --from=builder /usr/local/lib/python3.6 /usr/local/lib/python3.6 +COPY --from=builder /usr/local/lib/python3.7 /usr/local/lib/python3.7 ## Dependencies needed at runtime: RUN apt-get update \ From 8b982ef1fa8c56584146c5c01b105b83c81123b5 Mon Sep 17 00:00:00 2001 From: Juho Inkinen Date: Thu, 20 Jun 2019 10:50:45 +0300 Subject: [PATCH 61/66] Attempt to run both pipelines by separating them with dashes --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 0656c1d39..8a691e420 100644 --- a/.drone.yml +++ b/.drone.yml @@ -15,7 +15,7 @@ steps: when: event: push branch: [issue278-dockerize-annif] - +--- kind: pipeline name: dev-image-deployment steps: From 9e4327933b636a7b30a2f262648d46d43e1d6714 Mon Sep 17 00:00:00 2001 From: Juho Inkinen Date: Thu, 20 Jun 2019 10:57:56 +0300 Subject: [PATCH 62/66] One pipeline, two steps: first build&push annif, only then annif-dev --- .drone.yml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/.drone.yml b/.drone.yml index 8a691e420..6fd4b7ddd 100644 --- a/.drone.yml +++ b/.drone.yml @@ -1,6 +1,7 @@ kind: pipeline name: default steps: + - name: docker image: plugins/docker settings: @@ -12,13 +13,7 @@ steps: registry: quay.io repo: quay.io/natlibfi/annif tags: [latest] - when: - event: push - branch: [issue278-dockerize-annif] ---- -kind: pipeline -name: dev-image-deployment -steps: + - name: docker-dev image: plugins/docker settings: @@ -30,6 +25,7 @@ steps: registry: quay.io repo: quay.io/natlibfi/annif tags: [dev] + when: event: push branch: [issue278-dockerize-annif] From 794396890809b378aff3a081ac8c54c5edd578a0 Mon Sep 17 00:00:00 2001 From: Juho Inkinen Date: Thu, 20 Jun 2019 15:28:07 +0300 Subject: [PATCH 63/66] Unified docker-compose.yml (NGINX and mauiservice) --- docker-compose.yml | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 0144cd613..3999cc9c2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,16 +2,31 @@ version: "3" services: + bash: + image: quay.io/natlibfi/annif + volumes: + - ${ANNIF_PROJECTS}:/annif-projects + user: ${UID}:${GID} + stdin_open: true + tty: true + command: bash + + # Note: to connect to the mauiservice from other services, use "mauiservice" in place of "localhost" + mauiservice: + image: quay.io/natlibfi/mauiservice + volumes: + - ${ANNIF_PROJECTS}:/annif-projects + ports: + - "8080:8080" + gunicorn_server: - restart: always - image: annif + image: quay.io/natlibfi/annif volumes: - ${ANNIF_PROJECTS}:/annif-projects user: ${UID}:${GID} command: ["gunicorn", "annif:create_app()", "--bind", "0.0.0.0:8000"] nginx: - restart: always image: nginx volumes: - ./annif/nginx/nginx.conf:/etc/nginx/conf.d/default.conf From 31a8e2105a7a0f8b5401a53a2b9f4f0d5fdfb2cf Mon Sep 17 00:00:00 2001 From: Juho Inkinen Date: Wed, 26 Jun 2019 13:17:40 +0300 Subject: [PATCH 64/66] Add build trigger to listen also master --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 6fd4b7ddd..35de1fb28 100644 --- a/.drone.yml +++ b/.drone.yml @@ -28,4 +28,4 @@ steps: when: event: push - branch: [issue278-dockerize-annif] + branch: [master, issue278-dockerize-annif] From 65628cfdb5e46cf4311d0959a90e7917011f2534 Mon Sep 17 00:00:00 2001 From: Juho Inkinen Date: Wed, 26 Jun 2019 13:22:08 +0300 Subject: [PATCH 65/66] Add Traefik network settings --- docker-compose-portainer.yml | 38 +++++++++++++++++------------------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/docker-compose-portainer.yml b/docker-compose-portainer.yml index bf653da6e..9745dd925 100644 --- a/docker-compose-portainer.yml +++ b/docker-compose-portainer.yml @@ -5,41 +5,35 @@ services: gunicorn_server: restart: always image: quay.io/natlibfi/annif - # volumes: - # - ${ANNIF_PROJECTS}:/annif-projects - # user: ${UID}:${GID} command: ["gunicorn", "annif:create_app()", "--bind", "0.0.0.0:8000"] volumes: - data:/annif-projects depends_on: - data - deploy: - replicas: 1 - restart_policy: - condition: any - placement: - constraints: - - node.labels.environment == test - nginx: restart: always image: nginx volumes: - data:/annif-projects - # ports: - # - "80:80" depends_on: - gunicorn_server command: ["nginx", "-c", "/annif-projects/nginx.conf", "-g", "daemon off;"] + networks: + - default + - traefik-net + deploy: + replicas: 1 + restart_policy: + condition: any + placement: + constraints: + - node.labels.environment == test + labels: + - 'traefik.port=80' + - 'traefik.docker.network=traefik-net' + - 'traefik.frontend.rule=Host:annif.dev.finto.fi' - # app: - # image: annif - # stdin_open: true - # tty: true - # command: bash - # volumes: - # - data:/annif-projects data: image: annif-data @@ -56,3 +50,7 @@ services: volumes: data: + +networks: + traefik-net: + external: true From 61f7345cdbc14cac34e9105e879ac62c77bdbd5a Mon Sep 17 00:00:00 2001 From: Juho Inkinen Date: Wed, 26 Jun 2019 13:31:29 +0300 Subject: [PATCH 66/66] Remove build trigger from "dockerize-annif" branch --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 35de1fb28..79b65117c 100644 --- a/.drone.yml +++ b/.drone.yml @@ -28,4 +28,4 @@ steps: when: event: push - branch: [master, issue278-dockerize-annif] + branch: [master]