Add a weekly check of function docker images build

.github/workflows/build_function_images.yml

@@ -0,0 +1,23 @@
+name: Check if the docker images of the functions build successfully
+
+on: 
+  workflow_dispatch:
+  schedule:
+    - cron: '0 9 * * 1'
+
+
+jobs:
+  build_images:
+    name: Build function docker images
+    runs-on: ubuntu-18.04
+    strategy:
+      fail-fast: false
+      matrix:
+        image: [grpc, helloworld, chameleon, pyaes, image_rotate_s3, lr_serving, json_serdes_s3, cnn_serving, rnn_serving, lr_training_s3, video_processing_s3]
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        ref: master
+    - name: docker build images
+      run: |
+        docker build ./function-images/${{ matrix.image }}

function-images/chameleon/Dockerfile

@@ -1,9 +1,9 @@
-FROM ustiugov/py_grpc:builder_grpc as builder_workload
-COPY --from=ustiugov/py_grpc:builder_grpc /root/.local /root/.local
+FROM vhiveease/py_grpc:builder_grpc as builder_workload
+COPY --from=vhiveease/py_grpc:builder_grpc /root/.local /root/.local
 COPY requirements.txt .
 RUN pip3 install --user -r requirements.txt
 
-FROM ustiugov/py_grpc:base as var_workload
+FROM vhiveease/py_grpc:base as var_workload
 COPY *.py /
 COPY --from=builder_workload /root/.local /root/.local
 RUN apk add libstdc++ --update --no-cache

function-images/cnn_serving/Dockerfile

@@ -8,15 +8,17 @@ RUN apk update && \
     ln -sf /usr/bin/pip3 /usr/local/bin/pip && \
     ln -sf /usr/bin/python3 /usr/local/bin/python && \
     ln -sf /usr/bin/python3 /usr/local/bin/python3 && \
-    echo "http://dl-8.alpinelinux.org/alpine/edge/community" >> /etc/apk/repositories && \
-    echo "http://dl-3.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories && \
+    ln -s /usr/include/locale.h /usr/include/xlocale.h && \
+    echo "http://dl-cdn.alpinelinux.org/alpine/edge/community" >> /etc/apk/repositories && \
+    echo "http://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories && \
     apk update && \
     apk add --upgrade && \
     apk add --update --no-cache build-base gcc g++ protobuf && \
     apk add --allow-untrusted --repository http://dl-3.alpinelinux.org/alpine/edge/testing hdf5 hdf5-dev && \
     apk add py3-numpy && \
     apk add py-numpy-dev && \
     pip3 install --upgrade pip && \
+    pip3 uninstall -y enum34 && \
     pip3 install --no-cache-dir Cython && \
     pip3 install --no-cache-dir -r requirements.txt && \
     pip3 install --no-cache-dir protobuf==3.11.3 grpcio==${GRPC_PYTHON_VERSION}

function-images/helloworld/Dockerfile

@@ -1,9 +1,9 @@
-FROM ustiugov/py_grpc:builder_grpc as builder_workload
-COPY --from=ustiugov/py_grpc:builder_grpc /root/.local /root/.local
+FROM vhiveease/py_grpc:builder_grpc as builder_workload
+COPY --from=vhiveease/py_grpc:builder_grpc /root/.local /root/.local
 COPY requirements.txt .
 RUN pip3 install --user -r requirements.txt
 
-FROM ustiugov/py_grpc:base as var_workload
+FROM vhiveease/py_grpc:base as var_workload
 COPY *.py /
 COPY --from=builder_workload /root/.local /root/.local
 RUN apk add libstdc++ --update --no-cache

function-images/image_rotate_s3/Dockerfile

@@ -1,12 +1,12 @@
-FROM ustiugov/py_grpc:builder_grpc as builder_workload
-COPY --from=ustiugov/py_grpc:builder_grpc /root/.local /root/.local
+FROM vhiveease/py_grpc:builder_grpc as builder_workload
+COPY --from=vhiveease/py_grpc:builder_grpc /root/.local /root/.local
 COPY requirements.txt .
 RUN apk add --no-cache --virtual .build-deps \
     build-base linux-headers zlib-dev jpeg-dev \
     && pip3 install --user -r requirements.txt \
     && apk del .build-deps
 
-FROM ustiugov/py_grpc:base as var_workload
+FROM vhiveease/py_grpc:base as var_workload
 COPY *.py /
 COPY --from=builder_workload /root/.local /root/.local
 RUN apk add libstdc++ libjpeg-turbo --update --no-cache

function-images/image_rotate_s3/server.py

@@ -1,6 +1,6 @@
 from concurrent import futures
 import logging
-
+import os
 import grpc
 
 from PIL import Image, ImageOps
@@ -9,20 +9,24 @@
 import helloworld_pb2_grpc
 
 from minio import Minio
-from minio.error import (ResponseError, BucketAlreadyOwnedByYou,
-                         BucketAlreadyExists)
 
+minioEnvKey = "MINIO_ADDRESS"
 image_name = 'img2.jpeg'
 image2_name = 'img3.jpeg'
 image_path = '/pulled_' + image_name
 image_path2 = '/pulled_' +image2_name
 
 responses = ["record_response", "replay_response"]
 
+minioAddress = os.getenv(minioEnvKey)
+
 class Greeter(helloworld_pb2_grpc.GreeterServicer):
 
     def SayHello(self, request, context):
-        minioClient = Minio('128.110.154.105:9000',
+        if minioAddress == None:
+            return None
+
+        minioClient = Minio(minioAddress,
                 access_key='minioadmin',
                 secret_key='minioadmin',
                 secure=False)

function-images/json_serdes_s3/Dockerfile

@@ -1,9 +1,9 @@
-FROM ustiugov/py_grpc:builder_grpc as builder_workload
-COPY --from=ustiugov/py_grpc:builder_grpc /root/.local /root/.local
+FROM vhiveease/py_grpc:builder_grpc as builder_workload
+COPY --from=vhiveease/py_grpc:builder_grpc /root/.local /root/.local
 COPY requirements.txt .
 RUN pip3 install --user -r requirements.txt
 
-FROM ustiugov/py_grpc:base as var_workload
+FROM vhiveease/py_grpc:base as var_workload
 COPY *.py /
 COPY --from=builder_workload /root/.local /root/.local
 RUN apk add libstdc++ --update --no-cache

function-images/json_serdes_s3/server.py

@@ -15,28 +15,32 @@
 
 from concurrent import futures
 import logging
-
+import os
 import grpc
 
 import helloworld_pb2
 import helloworld_pb2_grpc
 
 from minio import Minio
-from minio.error import (ResponseError, BucketAlreadyOwnedByYou,
-                         BucketAlreadyExists)
 import json
 
+minioEnvKey = "MINIO_ADDRESS"
 data_name = '2.json'
 data2_name = '1.json'
 data_path = '/pulled_' + data_name
 data2_path = '/pulled_' + data2_name
 
 responses = ["record_response", "replay_response"]
 
+minioAddress = os.getenv(minioEnvKey)
+
 class Greeter(helloworld_pb2_grpc.GreeterServicer):
 
     def SayHello(self, request, context):
-        minioClient = Minio('128.110.154.105:9000',
+        if minioAddress == None:
+            return None
+
+        minioClient = Minio(minioAddress,
                 access_key='minioadmin',
                 secret_key='minioadmin',
                 secure=False)

function-images/lr_serving/Dockerfile

@@ -9,8 +9,8 @@ RUN apk add --no-cache \
     \
     ln -s locale.h /usr/include/xlocale.h && \
     \
-    pip install --disable-pip-version-check --no-build-isolation numpy && \
-    pip install --disable-pip-version-check --no-build-isolation pandas && \
+    pip install --disable-pip-version-check --no-build-isolation numpy==1.19.0 && \
+    pip install --disable-pip-version-check --no-build-isolation pandas==1.0.5 && \
     pip install --disable-pip-version-check --no-build-isolation protobuf==3.11.3 && \
     pip install --disable-pip-version-check --no-build-isolation grpcio==1.26.0 && \
     \

function-images/lr_training_s3/Dockerfile

@@ -1,4 +1,4 @@
-FROM ustiugov/lr_serving:var_workload as var_workload
+FROM vhiveease/lr_serving:var_workload as var_workload
 COPY requirements.txt .
 RUN	pip3 install --no-cache-dir -r requirements.txt
 

function-images/lr_training_s3/server.py

@@ -1,10 +1,8 @@
 from concurrent import futures
 import logging
-
+import os
 import grpc
 from minio import Minio
-from minio.error import (ResponseError, BucketAlreadyOwnedByYou,
-                         BucketAlreadyExists)
 from sklearn.feature_extraction.text import TfidfVectorizer
 from sklearn.linear_model import LogisticRegression
 import joblib
@@ -24,16 +22,21 @@ def cleanup(sentence):
     sentence = cleanup_re.sub(' ', sentence).strip()
     return sentence
 
+minioEnvKey = "MINIO_ADDRESS"
 df_name = 'dataset.csv'
 df2_name = 'dataset2.csv'
 df_path = '/pulled_' + df_name
 df2_path = 'pulled_' + df2_name
 
+minioAddress = os.getenv(minioEnvKey)
 
 class Greeter(helloworld_pb2_grpc.GreeterServicer):
 
     def SayHello(self, request, context):
-        minioClient = Minio('128.110.154.105:9000',
+        if minioAddress == None:
+            return None
+
+        minioClient = Minio(minioAddress,
                 access_key='minioadmin',
                 secret_key='minioadmin',
                 secure=False)

function-images/minio_scripts/create_minio_bucket.sh

@@ -21,5 +21,5 @@
 # SOFTWARE.
 
 #!/bin/bash
-mc alias set myminio http://128.110.154.105:9000 minioadmin minioadmin
+mc alias set myminio $1 minioadmin minioadmin
 mc mb myminio/mybucket

function-images/minio_scripts/put_in_bucket.sh

@@ -27,20 +27,20 @@ SERVERSDIR="$( cd $DIR && cd .. && pwd)"
 BUCKET=myminio/mybucket
 
 
-WORKLOAD=image_rotate
+WORKLOAD=image_rotate_s3
 for INPUTFILE in img2.jpeg img3.jpeg
 do
     mc cp $SERVERSDIR/$WORKLOAD/$INPUTFILE $BUCKET/$INPUTFILE
 done
 
 
-WORKLOAD=lr_training
+WORKLOAD=lr_training_s3
 for INPUTFILE in dataset.csv dataset2.csv
 do
     mc cp $SERVERSDIR/$WORKLOAD/$INPUTFILE $BUCKET/$INPUTFILE
 done
 
-WORKLOAD=json_serdes
+WORKLOAD=json_serdes_s3
 for INPUTFILE in 1.json 2.json
 do
     mc cp $SERVERSDIR/$WORKLOAD/$INPUTFILE $BUCKET/$INPUTFILE

function-images/minio_scripts/start_minio_server.sh

@@ -21,4 +21,4 @@
 # SOFTWARE.
 
 #!/bin/bash
-sudo minio server /data --address 128.110.154.105:9000
+sudo minio server /data

function-images/pyaes/Dockerfile

@@ -1,9 +1,9 @@
-FROM ustiugov/py_grpc:builder_grpc as builder_workload
-COPY --from=ustiugov/py_grpc:builder_grpc /root/.local /root/.local
+FROM vhiveease/py_grpc:builder_grpc as builder_workload
+COPY --from=vhiveease/py_grpc:builder_grpc /root/.local /root/.local
 COPY requirements.txt .
 RUN pip3 install --user -r requirements.txt
 
-FROM ustiugov/py_grpc:base as var_workload
+FROM vhiveease/py_grpc:base as var_workload
 COPY *.py /
 COPY --from=builder_workload /root/.local /root/.local
 RUN apk add libstdc++ --update --no-cache

function-images/video_processing_s3/server.py

@@ -2,19 +2,20 @@
 import logging
 import cv2
 from minio import Minio
-from minio.error import (ResponseError, BucketAlreadyOwnedByYou,
-                         BucketAlreadyExists)
 import grpc
-
+import os
 import helloworld_pb2
 import helloworld_pb2_grpc
 tmp = "/tmp/"
 
+minioEnvKey = "MINIO_ADDRESS"
 vid1_name = 'vid1.mp4'
 vid2_name = 'vid2.mp4'
 vid1_path = '/pulled_' + vid1_name
 vid2_path = '/pulled_' + vid2_name
 
+minioAddress = os.getenv(minioEnvKey)
+
 def video_processing(video_path):
     result_file_path = tmp + video_path
 
@@ -47,7 +48,10 @@ def video_processing(video_path):
 class Greeter(helloworld_pb2_grpc.GreeterServicer):
 
     def SayHello(self, request, context):
-        minioClient = Minio('128.110.154.105:9000',
+        if minioAddress == None:
+            return None
+
+        minioClient = Minio(minioAddress,
                 access_key='minioadmin',
                 secret_key='minioadmin',
                 secure=False)