From 22941cc1a32faa84e8d4196cb211640335a923dc Mon Sep 17 00:00:00 2001 From: LxL Date: Sun, 12 Jan 2025 19:16:47 +0800 Subject: [PATCH 1/3] Test latest scrapyd by default and support allure-results --- .circleci/config.yml | 110 +++++++++++++++++++++++++++++++++++++---- .gitignore | 1 + requirements-tests.txt | 1 + 3 files changed, 102 insertions(+), 10 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 28390d8..2858c98 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,6 +2,7 @@ version: 2.1 orbs: codecov: codecov/codecov@1.0.2 + allure: ayte/allure@0.1.3 jobs: py39: &test-template docker: @@ -15,7 +16,7 @@ jobs: default: false use-scrapyd-v143: type: boolean - default: true + default: false use-git: type: boolean default: false @@ -31,16 +32,46 @@ jobs: use-mysql: type: boolean default: false + allure-version: + description: Allure version to use + type: string + default: 2.13.1 + allure-configuration-path: + description: Path to Allure configuration, uses default one if omitted + type: string + default: /usr/local/share/allure/config/allure.yml + allure-target-path: + description: Path for report directory + type: string + default: allure-report + allure-results-path: + description: Path to directory with test results + type: string + default: allure-results + allure-artifact-path: + description: Path that will be used when storing result as artifact + type: string + default: Report/Allure steps: - run: name: Install telnet command: | sudo apt-get update && sudo apt-get install telnet + - run: + name: Install Java 11 + command: | + sudo apt-get update + sudo apt-get install -y openjdk-11-jdk + - run: + name: Set JAVA_HOME + command: | + echo "export JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64" >> $BASH_ENV + source $BASH_ENV - run: name: Setup env command: | mkdir ~/logs - ls -l ~ + ls -la ~ - checkout - when: condition: <> @@ -154,11 +185,12 @@ jobs: name: Run tests command: | pip list - ls -l + ls -la . venv/bin/activate flake8 . --count --exclude=./venv* --select=E9,F63,F7,F82 --show-source --statistics coverage erase - coverage run --source=scrapydweb -m pytest tests -s -vv -l --disable-warnings + # tests/test_schedule.py + coverage run --source=scrapydweb -m pytest -s -vv -l --disable-warnings --alluredir=allure-results tests - run: name: Generate report command: | @@ -169,13 +201,66 @@ jobs: coverage html coverage xml ls -la + ls -la allure-results || echo 'ignore error' coveralls + # https://discuss.circleci.com/t/make-custom-command-run-always-with-when-always/38957 + # https://circleci.com/docs/configuration-reference/#the-when-attribute + when: always - store_artifacts: path: htmlcov - store_artifacts: path: coverage.xml - codecov/upload: file: coverage.xml + # https://circleci.com/developer/orbs/orb/codecov/codecov + # - codecov/upload + # https://circleci.com/developer/orbs/orb/coveralls/coveralls + # - coveralls/upload + # https://discuss.circleci.com/t/how-can-we-publish-pytest-results-to-circleci-using-allure-reports/37830/2 + # https://circleci.com/developer/orbs/orb/ayte/allure + # - allure/install + # - allure/report + # https://circleci.com/docs/configuration-reference/#the-when-step + - when: + condition: + equal: [ 1, 1 ] + steps: + - run: + name: Allure archive download + command: >- + curl -L https://github.com/allure-framework/allure2/releases/download/<< + parameters.allure-version >>/allure-commandline-<< parameters.allure-version >>.zip -o + /tmp/allure.zip + when: always + - run: + name: Archive extraction + command: unzip /tmp/allure.zip + when: always + - run: + name: Allure installation + command: sudo mv allure-<< parameters.allure-version >> /usr/local/share/allure + when: always + - run: + name: Allure binary symlinking + command: sudo ln -s /usr/local/share/allure/bin/allure /usr/local/bin/allure + when: always + - when: + condition: + equal: [ 1, 1 ] + steps: + - run: + name: >- + Allure report generation (<< parameters.allure-results-path >> -> << + parameters.allure-target-path >>) + command: | + allure generate \ + --config << parameters.allure-configuration-path >> \ + --report-dir << parameters.allure-target-path >> \ + << parameters.allure-results-path >> + when: always + - store_artifacts: + path: << parameters.allure-target-path >> + destination: << parameters.allure-artifact-path >> py27: <<: *test-template docker: @@ -192,7 +277,7 @@ jobs: <<: *test-template docker: - image: cimg/python:3.8 - py39-scrapyd-latest: + py39-scrapyd-v143: <<: *test-template docker: - image: cimg/python:3.9 @@ -274,10 +359,14 @@ jobs: <<: *test-template docker: - image: cimg/python:3.12 - py312-scrapyd-latest: + py312-scrapyd-v143: <<: *test-template docker: - image: cimg/python:3.12 + py313: + <<: *test-template + docker: + - image: cimg/python:3.13 workflows: test: jobs: @@ -287,8 +376,8 @@ workflows: # - py37 - py38 - py39 - - py39-scrapyd-latest: - use-scrapyd-v143: false + - py39-scrapyd-v143: + use-scrapyd-v143: true - py310-git-postgresql: use-git: true use-postgresql: true @@ -304,5 +393,6 @@ workflows: use-mysql: true - py311 - py312 - - py312-scrapyd-latest: - use-scrapyd-v143: false + - py312-scrapyd-v143: + use-scrapyd-v143: true + - py313 diff --git a/.gitignore b/.gitignore index f9b313b..66ac8f0 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ scrapydweb_settings_v*.py tests/*.html tests/data/ +allure-results/ scrapydweb/data/* diff --git a/requirements-tests.txt b/requirements-tests.txt index 4849842..cf6b5a9 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -4,6 +4,7 @@ coverage pytest # pytest-cov coveralls +allure-pytest scrapy scrapyd From 7c7defd0e6f367e4027ffe188b1ffa7ccf3f6ca8 Mon Sep 17 00:00:00 2001 From: LxL Date: Sun, 12 Jan 2025 19:19:03 +0800 Subject: [PATCH 2/3] Fix issues for scrapyd v1.5.0 --- scrapydweb/templates/scrapydweb/jobs.html | 4 ++-- .../templates/scrapydweb/jobs_classic.html | 6 ++--- .../templates/scrapydweb/jobs_mobileui.html | 4 ++-- .../templates/scrapydweb/logs_items.html | 2 ++ scrapydweb/templates/scrapydweb/servers.html | 2 +- scrapydweb/utils/poll.py | 21 ++++++++-------- scrapydweb/vars.py | 3 ++- scrapydweb/views/dashboard/jobs.py | 6 ++--- tests/test_schedule.py | 24 ++++++++++++------- tests/utils.py | 11 +++++---- 10 files changed, 48 insertions(+), 35 deletions(-) diff --git a/scrapydweb/templates/scrapydweb/jobs.html b/scrapydweb/templates/scrapydweb/jobs.html index b316731..1c96269 100644 --- a/scrapydweb/templates/scrapydweb/jobs.html +++ b/scrapydweb/templates/scrapydweb/jobs.html @@ -15,7 +15,7 @@ {% if SCRAPYD_SERVERS_AMOUNT == 1 and (pageview == 1 or pageview % CHECK_LATEST_VERSION_FREQ == 0) %} - + {% else %} @@ -31,7 +31,7 @@

Get the list of jobs of all projects in database.