Merge pull request #503 from moevm/494_selenium+docker

494 selenium+docker

.env_example

@@ -10,11 +10,11 @@ CONSUMER_SECRET=secret
 WEB_PORT=8080
 
 REDIS_URL=redis://redis:6379/0
-CELERY_QUERIES=query
+CELERY_QUERIES=check-solution
 
 FLOWER_SECRET_KEY=123
 FLOWER_PORT=5555
 FLOWER_AUTH=user:user
 FLOWER_PREFIX=/
 
-MONGODB_CACHE_SIZE=1
+MONGODB_CACHE_SIZE=1

.github/workflows/selenium_tests.yml

@@ -0,0 +1,45 @@
+name: Selenium_tests
+
+on: [push]
+
+jobs:
+  check_tests:
+    runs-on: ubuntu-20.04
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Build docker-compose with docker-compose-selenium (tests)
+        run: |
+          cp .env_example .env
+          cp app/VERSION_example.json app/VERSION.json
+          docker-compose -f docker-compose.yml -f docker-compose-selenium.yml build
+      
+      - name: Run docker-compose with docker-compose-selenium (tests)
+        run: |
+          docker-compose -f docker-compose.yml -f docker-compose-selenium.yml up -d
+          chmod +x tests/scripts/docker_check_tests.sh
+          ./tests/scripts/docker_check_tests.sh
+
+
+# jobs:
+#   check_tests:
+#     runs-on: ubuntu-20.04
+
+#     steps:
+#       - uses: actions/checkout@v2
+
+#       - name: Simplify docker-compose file name
+#         run: mv docker-compose-tests.yml docker-compose.yml
+
+#       - name: Build system images (non-pulling)
+#         run: |
+#           # build base image
+#           docker build -f Dockerfile_base -t osll/slides-base:20230202 .
+
+#       - name: Run docker-compose tests
+#         run: |
+#           cp .env_example .env
+#           docker-compose up -d
+#           chmod +x tests/scripts/docker_check_tests.sh
+#           ./tests/scripts/docker_check_tests.sh

Dockerfile_selenium

@@ -0,0 +1,17 @@
+FROM selenium/standalone-chrome:121.0-chromedriver-121.0-grid-4.18.0-20240220
+
+ENV LOGIN=""
+ENV PASSWORD=""
+
+WORKDIR /usr/src/project
+
+USER root
+RUN apt-get update && \
+    apt-get install -y python3 python3-pip && \
+    rm -rf /var/lib/apt/lists/*
+
+COPY tests/requirements.txt tests/requirements.txt
+RUN pip install -r tests/requirements.txt
+COPY tests ./tests
+
+ENTRYPOINT python3 tests/main.py --login ${LOGIN} --password ${PASSWORD}

app/VERSION_example.json

@@ -0,0 +1,6 @@
+{
+"commit":  "",
+"message":  "",
+"date":    "",
+"version": ""
+}

app/main/checks/report_checks/needed_headers_check.py

@@ -23,8 +23,8 @@ def late_init(self):
         self.headers = self.file.make_chapters(self.file_type['report_type'])
         self.headers_page = self.file.find_header_page(self.file_type['report_type'])
         self.chapters_str = self.file.show_chapters(self.file_type['report_type'])
-        self.headers_main = self.file.get_main_headers()
         # TODO: change
+        self.headers_main = self.file.get_main_headers(self.file_type['report_type'])
         if self.headers_main == "Задание 1":
             self.patterns = StyleCheckSettings.CONFIGS.get(self.config)[0]["headers"]
         elif self.headers_main == "Задание 2":

app/main/reports/document_uploader.py

@@ -48,3 +48,6 @@ def find_literature_page(self, work_type):
     @abstractmethod
     def show_chapters(self, work_type):
         pass
+
+    def get_main_headers(self):
+        pass

app/main/reports/docx_uploader/docx_uploader.py

@@ -18,6 +18,7 @@ def __init__(self):
         self.inline_shapes = []
         self.core_properties = None
         self.headers = []
+        self.headers_main = ''
         self.file = None
         self.special_paragraph_indices = {}
         self.headers_page = 0
@@ -90,6 +91,12 @@ def make_headers(self, work_type):
                                 break
                 self.headers = headers
         return self.headers
+
+    def get_main_headers(self, work_type): #this method helps to avoid mistake in "needed_headers_check" (because of structure checks for md)
+        if not self.headers_main:
+            if work_type == 'VKR':
+                self.headers_main = self.make_headers(work_type)[1]['name']
+        return self.headers_main
 
     def __make_table(self, tables):
         for i in range(len(tables)):

app/main/reports/md_uploader/md_uploader.py

@@ -95,7 +95,7 @@ def page_counter(self): # we need this just to find a last page and make link to
             self.page_count = 5 
         return self.page_count
 
-    def get_main_headers(self):
+    def get_main_headers(self, work_type):
         if not self.headers_main:
             header_main_regex = "<h1>(.*?)<\/h1>"
             self.headers_main = re.findall(header_main_regex, self.html_text)[0]

app/nlp/stemming.py

@@ -1,12 +1,8 @@
 import itertools
-
-import nltk
 from nltk.corpus import stopwords
 from nltk.tokenize import word_tokenize, sent_tokenize
 from pymorphy2 import MorphAnalyzer
 
-nltk.download('stopwords')
-nltk.download('punkt')
 
 MORPH_ANALYZER = MorphAnalyzer()
 TASKS = 'задачи:'

app/tasks.py

@@ -1,18 +1,19 @@
-import configparser
+from configparser import ConfigParser
 import os
 from os.path import join, exists
 
 from celery import Celery
+from celery.signals import worker_ready
 
-import passback_grades
+from passback_grades import run_passback
 from db import db_methods
 from db.db_types import Check
 from main.checker import check
 from main.parser import parse
 from main.check_packs import BASE_PACKS
 from root_logger import get_root_logger
 
-config = configparser.ConfigParser()
+config = ConfigParser()
 config.read('app/config.ini')
 
 TASK_RETRY_COUNTDOWN = 60  # default = 3 * 60
@@ -33,6 +34,16 @@
 celery.conf.timezone = 'Europe/Moscow'  # todo: get from env
 
 
+@worker_ready.connect
+def at_start(sender, **k):
+    from nltk import download
+    download('stopwords')
+    download('punkt')
+
+    from language_tool_python.download_lt import download_lt
+    download_lt()
+
+
 @celery.task(name="create_task", queue='check-solution', bind=True)
 def create_task(self, check_info):
     check_obj = Check(check_info)
@@ -67,12 +78,11 @@ def create_task(self, check_info):
         self.retry(countdown=TASK_RETRY_COUNTDOWN)  # Retry the task, adding it to the back of the queue.
 
 
+@celery.task(name="passback-task", queue='passback-grade')
+def passback_task():
+    return run_passback()
+
+
 def remove_files(filepaths):
     for filepath in filepaths:
         if exists(filepath): os.remove(filepath)
-
-
-@celery.task(name="passback-task", queue='passback-grade')
-def passback_task():
-    print('Run passback')
-    return passback_grades.run_passback()

app/templates/criteria_pack.html

@@ -46,7 +46,7 @@
                 </label><br>
                 <label>
                     Список критериев набора (однажды тут будет подробнейшая <a
-                        href="https://github.com/moevm/mse_auto_checking_slides_vaganov/issues/351">инструкция</a>)<br>
+                        href="https://github.com/moevm/document_insight_system/issues/351">инструкция</a>)<br>
                     # укажите здесь конфигурацию набора в формате JSON<br>
                     <textarea id="raw_criterions" rows=10 cols=80
                               required>{{ pack['raw_criterions'] }}</textarea>

app/templates/upload.html

@@ -15,8 +15,8 @@
         </script>
         <div class="col-11 col-sm-8 col-md-5 col-lg-4 my-auto">
             <h3 class="text-center ins" id="upload_title">{{ upload_title }}</h3>
-            <p class="text-center" id="criteria_pack_name">Для проверки используется набор
-                критериев<br><b>{{ current_user.criteria }}</b></p>
+            <p class="text-center" id="criteria_pack_name_text">Для проверки используется набор
+                критериев<br><b id="criteria_pack_name">{{ current_user.criteria }}</b></p>
             <div class="row justify-content-center" id="uploading_body">
                 <div class="col my-auto">
                     <form>

assets/scripts/main.js

@@ -14,7 +14,7 @@ import 'bootstrap-icons/font/bootstrap-icons.css'
 import 'bootstrap-datepicker';
 import 'bootstrap-datepicker/dist/css/bootstrap-datepicker.min.css'
 
-import * as CryptoJS from "crypto-js";
+import * as md5 from "md5";
 
 import '../styles/main.css';
 
@@ -33,7 +33,7 @@ import '../styles/404.css';
 
 
 export function hash(password) {
-    return CryptoJS.MD5(password).toString()
+    return md5(password)
 }
 
 export function collect_values_if_possible(...ids) {

docker-compose-selenium.yml

@@ -0,0 +1,13 @@
+version: '3.8'
+
+services:
+  selenium-tests:
+    build:
+      context: .
+      dockerfile: Dockerfile_selenium
+    environment:
+      - LOGIN=${ADMIN_PASSWORD}
+      - PASSWORD=${ADMIN_PASSWORD}  
+    depends_on:
+      - web
+    network_mode: service:web

package.json

@@ -1,8 +1,8 @@
 {
-  "name": "mse_auto_checking_slides_vaganov",
+  "name": "document_insight_system",
   "version": "1.0.0",
   "license": "ISC",
-  "repository": "https://github.com/moevm/mse_auto_checking_slides_vaganov",
+  "repository": "https://github.com/moevm/document_insight_system",
   "description": "Presentation parsing app",
   "scripts": {
     "watch": "webpack --watch",
@@ -21,10 +21,10 @@
     "bootstrap-icons": "1.5.0",
     "bootstrap-table": "1.18.3",
     "clean-webpack-plugin": "^3.0.0",
-    "crypto-js": "^3.1.2",
     "css-loader": "6.2.0",
     "file-loader": "6.2.0",
     "jquery": "^3.5.1",
+    "md5": "2.3.0",
     "pdfjs-dist": "2.5.207",
     "popper.js": "1.16.1",
     "style-loader": "3.3.1",

scripts/version.sh

@@ -10,4 +10,4 @@ echo "{
 \"message\":  \"$COMMIT_MSG\",
 \"date\":    \"$DATE\",
 \"version\": \"$VERSION\"
-}"
+}"

tests/README.md

@@ -0,0 +1,73 @@
+
+## For local tests:
+
+```bash
+$ pip install -r tests/requirements.txt
+```
+
+### Run tests:
+
+use login and password from .env
+You can run tests with your own data:
+
+```bash
+
+$ python3 tests/main.py --host host --login login --password password --pres your press --report your report --report_doc your report doc
+```
+
+or use default setting:
+
+```bash
+$ python3 tests/main.py --login login --password password
+```
+
+## Docker:
+
+### Dockerfile
+You can run tests with dockerfile_selenium independently, using special flag:
+
+```bash
+$ docker build -t your_image_name -f Dockerfile_selenium .
+$ docker run -e LOGIN=your_login -e PASSWORD=your password --network="host" your_image_name
+
+```
+
+### Docker-compose
+You can run docker-compose-selenium with docker-compose:
+
+```bash
+$ docker-compose -f docker-compose.yml -f docker-compose-selenium.yml build
+$ docker-compose -f docker-compose.yml -f docker-compose-selenium.yml up
+
+```
+
+## List of tests:
+
+### Test for autorization:
+
+class AuthTestSelenium(BasicSeleniumTest) with 3 tests
+Tests check: if page "/login" opens, if it doesn't take wrong login/password and takes correct.
+
+### Test for open page /check_list:
+
+class StatisticTestSelenium(BasicSeleniumTest) with 1 test
+Test check: if page "/check_list" opens
+
+
+### Test for open single check card:
+
+class SingleCheckTestSelenium(BasicSeleniumTest) with 1 test
+Test check: if page with random single check opens (from "/check_list")
+
+### Test for open page /version:
+
+class VersionTestSelenium(BasicSeleniumTest) with 1 test
+Test check: if page "/version" opens and contains info from "VERSION.json"
+
+### Test for loading report and pres:
+
+class FileLoadTestSelenium(BasicSeleniumTest) with 3 tests
+Test check: if reports wit different extensions loads correctly
+use default documents from "/tests" or your own example
+
+