Merge pull request #134 from timkpaine/dockerup

Docker maintenance, setup redis in docker compose

.github/workflows/build.yml

@@ -14,7 +14,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
-        python-version: [3.7, 3.8, 3.9]
+        python-version: [3.7, 3.9]
 
     steps:
     - uses: actions/checkout@v2
@@ -35,12 +35,12 @@ jobs:
 
     - name: Test
       run: |
-        make tests
+        make testsci
       if: ${{ matrix.os != 'ubuntu-latest' }}
 
     - name: Test
       run: |
-        make dockerup && make tests && make dockerdown
+        make dockerup && make testsci && make dockerdown
       if: ${{ matrix.os == 'ubuntu-latest' }}
 
 

.github/workflows/publish.yml

@@ -34,12 +34,12 @@ jobs:
 
     - name: Test
       run: |
-        make tests
+        make testsci
       if: ${{ matrix.os != 'ubuntu-latest' }}
 
     - name: Test
       run: |
-        make dockerup && make tests && make dockerdown
+        make dockerup && make testsci && make dockerdown
       if: ${{ matrix.os == 'ubuntu-latest' }}
 
     - name: Build and publish

Makefile

@@ -4,12 +4,21 @@ build:  ## Build the repository
 tests: ## Clean and Make unit tests
 	python -m pytest tributary --cov=tributary --junitxml=python_junit.xml --cov-report=xml --cov-branch
 
+testsci: ## Clean and Make unit tests
+	CI=true python -m pytest tributary --cov=tributary --junitxml=python_junit.xml --cov-report=xml --cov-branch
+
 testsv: ## Clean and Make unit tests
 	python -m pytest -vvv tributary --cov=tributary --junitxml=python_junit.xml --cov-report=xml --cov-branch
 
 testsnocov: ## Clean and Make unit tests
 	python -m pytest -v tributary -x
 
+dockerup:
+	docker-compose -f ci/docker-compose.yml up -d
+
+dockerdown:
+	docker-compose -f ci/docker-compose.yml down
+
 notebooks:  ## test execute the notebooks
 	./scripts/test_notebooks.sh
 
@@ -19,12 +28,6 @@ lint: ## run linter
 fix:  ## run black fix
 	python -m black tributary/ setup.py
 
-annotate: ## MyPy type annotation check
-	python -m mypy -s tributary  
-
-annotate_l: ## MyPy type annotation check - count only
-	python -m mypy -s tributary | wc -l 
-
 clean: ## clean the repository
 	find . -name "__pycache__" | xargs  rm -rf 
 	find . -name "*.pyc" | xargs rm -rf 
@@ -56,10 +59,4 @@ help:
 print-%:
 	@echo '$*=$($*)'
 
-dockerup:
-	docker-compose -f ci/docker-compose.yml up -d
-
-dockerdown:
-	docker-compose -f ci/docker-compose.yml down
-
 .PHONY: clean build run test tests help annotate annotate_l docs dist dockerup dockerdown

ci/docker-compose.yml

@@ -1,35 +1,43 @@
 version: '2'
 services:
-  zookeeper:
-    image: timkpaine/zookeeper:latest
-    ports:
-      - "2181:2181"
-  kafka:
-    image: timkpaine/kafka:latest
-    ports:
-      - "9092:9092"
-    environment:
-      KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092
-      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092
-      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
-      KAFKA_CREATE_TOPICS: "tributary:1:1"
-    volumes:
-      - /var/run/docker.sock:/var/run/docker.sockf
+  # zookeeper:
+  #   image: timkpaine/zookeeper:latest # wurstmeister/zookeeper
+  #   ports:
+  #     - "2181:2181"
+  # kafka:
+  #   image: timkpaine/kafka:latest # wurstmeister/kafka
+  #   ports:
+  #     - "9092:9092"
+  #   environment:
+  #     KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092
+  #     KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092
+  #     KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+  #     KAFKA_CREATE_TOPICS: "tributary:1:1"
+  #   volumes:
+  #     - /var/run/docker.sock:/var/run/docker.sockf
+
   echo-server:
-    image: timkpaine/echo-server:latest
-    ports:
-      - 8080:8080
-  socketio-server:
-    image: timkpaine/docker-socketio:latest
+    image: timkpaine/echo-server:latest # jmalloc/echo-server
     ports:
-      - 8069:8069
-    volumes:
-      - ./scripts/server.js:/srv/app.js
-  postgres:
-      image: postgres:latest
-      environment:
-          - POSTGRES_PASSWORD=test
-      volumes:
-          - ./scripts/init.sql:/docker-entrypoint-initdb.d/init.sql
-      ports:
-          - "5432:5432"
+      - 12345:8080
+
+  # socketio-server:
+  #   image: timkpaine/docker-socketio:latest # voduytuan/docker-socketio
+  #   ports:
+  #     - 8069:8069
+  #   volumes:
+  #     - ./scripts/server.js:/srv/app.js
+
+  # postgres:
+  #     image: timkpaine/postgres:latest # postgres
+  #     environment:
+  #         - POSTGRES_PASSWORD=test
+  #     volumes:
+  #         - ./scripts/init.sql:/docker-entrypoint-initdb.d/init.sql
+  #     ports:
+  #         - "5432:5432"
+
+  # redis:
+  #   image: timkpaine/redis:latest # redis
+  #   ports:
+  #     - "6379:6379"

tributary/tests/streaming/input/test_http_streaming.py

@@ -12,18 +12,17 @@ def setup(self):
     def test_http(self):
         out = ts.Print(ts.HTTPSource(url="https://google.com"))
         ret = ts.run(out)
-        print(ret)
         assert len(ret) == 1
 
-    @pytest.mark.skipif('os.name == "nt"')
+    @pytest.mark.skipif('os.name == "nt" or os.environ.get("CI")')
     def test_http_server(self):
         ss = ts.HTTPServerSource(json=True, host="127.0.0.1", port=12345)
         w = ts.Window(ss)
         out = ts.run(w, blocking=False)
         try:
-            time.sleep(1)
+            time.sleep(5)
             _ = requests.post("http://127.0.0.1:12345/", json={"test": 1, "test2": 2})
-            time.sleep(1)
+            time.sleep(5)
             assert w._accum == [{"test": 1, "test2": 2}]
         finally:
             asyncio.set_event_loop(asyncio.new_event_loop())