diff --git a/.github/actions/create-dev-env/action.yml b/.github/actions/create-dev-env/action.yml new file mode 100644 index 00000000..975f5d97 --- /dev/null +++ b/.github/actions/create-dev-env/action.yml @@ -0,0 +1,18 @@ +--- +name: Build environment +description: Create build environment + +runs: + using: composite + steps: + - name: Set Up Python 🐍 + uses: actions/setup-python@v4 + with: + python-version: "3.10" + cache: "pip" + + - name: Install Dev Dependencies 📦 + run: | + pip install -U pip + pip install -U -e .[dev] + shell: bash diff --git a/.github/actions/generate-metadata/action.yml b/.github/actions/generate-metadata/action.yml new file mode 100644 index 00000000..cfdd3bba --- /dev/null +++ b/.github/actions/generate-metadata/action.yml @@ -0,0 +1,49 @@ +--- +name: Build environment +description: Create build environment + +inputs: + gh_token: + description: GitHub token + required: true + type: string + cache: + description: use caching or not + required: true + type: boolean + +runs: + using: composite + steps: + - name: Restore cached Plugins + id: cache-plugins-restore + uses: actions/cache/restore@v3 + with: + path: plugins_metadata.json + key: plugins_metadata + + - name: fetch metadata + if: ${{ inputs.cache == 'false' }} + id: fetch_metadata + env: + GITHUB_TOKEN: ${{ inputs.gh_token }} + run: aiida-registry fetch + shell: bash + - name: Check plugins installation + if: ${{ inputs.cache == 'false' }} + # This step will attach plugin installation inforamtion to the metadata, e.g. if the plugin can be installed or not + run: aiida-registry test-install + shell: bash + - name: Cache plugins metadata + if: ${{ inputs.cache == 'false' }} + id: cache-plugins-save + uses: actions/cache/save@v3 + with: + path: plugins_metadata.json + key: ${{ steps.cache-plugins-restore.outputs.cache-primary-key }} + + - name: Move JSON file to the React project + run: | + cp plugins_metadata.json aiida-registry-app/src/ + cp plugins_metadata.json aiida-registry-app/dist/ + shell: bash diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 902d5d14..c0765cbb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,141 +11,104 @@ concurrency: jobs: pre-commit: - runs-on: ubuntu-latest - timeout-minutes: 5 steps: - - uses: actions/checkout@v3 - - name: Set up Python 3.8 - uses: actions/setup-python@v3 - with: - python-version: 3.8 - - name: Install prerequisites - run: pip install --upgrade pip - - name: Install - run: pip install -e .[pre-commit] + - name: Checkout Repo ⚡️ + uses: actions/checkout@v3 + - name: Create dev environment + uses: ./.github/actions/create-dev-env - name: Run pre-commit run: pre-commit run --all-files || ( git status --short ; git diff ; exit 1 ) - test-webpage-build: + test-utils: runs-on: ubuntu-latest - timeout-minutes: 5 steps: - - uses: actions/checkout@v2 - - run: git fetch --no-tags --prune --depth=1 origin +refs/heads/*:refs/remotes/origin/* - - name: Set up Python 3.8 - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - name: Install prerequisites - run: pip install --upgrade wheel setuptools - - name: Install - run: pip install -e . -v - - - name: Restore cached Plugins - id: cache-plugins-restore - uses: actions/cache/restore@v3 - with: - path: | - plugins_metadata.json - key: plugins_metadata - - - name: fetch metadata - if: steps.cache-plugins-restore.outputs.cache-hit != 'true' - id: fetch_metadata + - name: Checkout Repo ⚡️ + uses: actions/checkout@v3 + - name: Create dev environment + uses: ./.github/actions/create-dev-env + - name: Run tests env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: aiida-registry fetch + run: pytest tests/ + + test-webpage-build: + runs-on: ubuntu-latest + strategy: + fail-fast: false + timeout-minutes: 30 + + # This is a CI job that checks if the webpage can be built + # We use the plugins metadata from caching since we don't want to + # fetch it twice and it is not essential for this job to have + # the latest generated metadata + steps: + - name: Checkout Repo ⚡️ + uses: actions/checkout@v3 + - name: Create dev environment + uses: ./.github/actions/create-dev-env - - name: Cache plugins metadata - id: cache-plugins-save - uses: actions/cache/save@v3 + - name: Generate metadata + uses: ./.github/actions/generate-metadata with: - path: | - plugins_metadata.json - key: ${{ steps.cache-plugins-restore.outputs.cache-primary-key }} + gh_token: ${{ secrets.GITHUB_TOKEN }} + cache: true - - name: Move JSON file to the React project - run: | - mv plugins_metadata.json aiida-registry-app/src/ - uses: actions/setup-node@v3 with: node-version: '18.x' - name: Install npm dependencies and build run: | + echo $VITE_PR_PREVIEW_PATH npm install npm run build working-directory: ./aiida-registry-app - install-plugins: + preview: + needs: [test-webpage-build] runs-on: ubuntu-latest strategy: - matrix: - python-version: [3.8] fail-fast: false - timeout-minutes: 55 - + timeout-minutes: 60 + env: + COMMIT_AUTHOR: Deploy Action + COMMIT_AUTHOR_EMAIL: action@github.com + VITE_PR_PREVIEW_PATH: "/aiida-registry/pr-preview/pr-${{ github.event.number }}/" + + # This is a CI job that checks if the webpage can be built + # We use the plugins metadata from caching since we don't want to + # fetch it twice and it is not essential for this job to have + # the latest generated metadata steps: - - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 - with: - python-version: ${{ matrix.python-version }} - # Idea: create individual conda env for every tested package - # - uses: goanpeca/setup-miniconda@v1 - # with: - # # auto-update-conda: true - # python-version: ${{ matrix.python-version }} - - name: Install prerequisites - run: pip install --upgrade pip - - name: Install - run: pip install -e . -v - - name: Restore cached Plugins - id: cache-plugins-restore - uses: actions/cache/restore@v3 - with: - path: | - plugins_metadata.json - key: plugins_metadata + - name: Checkout Repo ⚡️ + uses: actions/checkout@v3 + - name: Create dev environment + uses: ./.github/actions/create-dev-env - - name: fetch metadata - if: steps.cache-plugins-restore.outputs.cache-hit != 'true' - id: fetch_metadata - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: aiida-registry fetch - -# # See https://github.com/geerlingguy/raspberry-pi-dramble/issues/166 -# - name: Force GitHub Actions' docker daemon to use vfs. -# run: | -# sudo systemctl stop docker -# echo '{"cgroup-parent":"/actions_job","storage-driver":"vfs"}' | sudo tee /etc/docker/daemon.json -# sudo systemctl start docker - - - name: Try installing packages - run: aiida-registry test-install + - name: Generate metadata + uses: ./.github/actions/generate-metadata + with: + gh_token: ${{ secrets.GITHUB_TOKEN }} + cache: false - unit-tests: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: [3.8] - timeout-minutes: 5 + - uses: actions/setup-node@v3 + with: + node-version: '18.x' + - name: Install npm dependencies and build + run: | + echo $VITE_PR_PREVIEW_PATH + npm install + npm run build + working-directory: ./aiida-registry-app - steps: - - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 + - name: Deploy preview + uses: rossjrw/pr-preview-action@v1 with: - python-version: ${{ matrix.python-version }} - - name: Install prerequisites - run: pip install --upgrade pip - - name: Install - run: pip install -e .[testing] -v - - name: Run tests - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: pytest tests/ + source-dir: ./aiida-registry-app/dist + preview-branch: gh-pages + umbrella-dir: pr-preview + action: auto + custom-url: diff --git a/.github/workflows/preview.yml b/.github/workflows/preview.yml deleted file mode 100644 index 2381646c..00000000 --- a/.github/workflows/preview.yml +++ /dev/null @@ -1,66 +0,0 @@ -name: Deploy PR previews - -on: - pull_request: - types: - - opened - - reopened - - synchronize - - closed - -# https://docs.github.com/en/actions/using-jobs/using-concurrency -concurrency: - # only cancel in-progress jobs or runs for the current workflow - matches against branch & tags - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - deploy-preview: - runs-on: ubuntu-20.04 - env: - COMMIT_AUTHOR: Deploy Action - COMMIT_AUTHOR_EMAIL: action@github.com - steps: - - uses: actions/checkout@v2 - - run: git fetch --no-tags --prune --depth=1 origin +refs/heads/*:refs/remotes/origin/* - - name: Set up Python 3.8 - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - name: Install prerequisites - run: pip install --upgrade wheel setuptools - - name: Install - run: pip install -e . -v - - - name: fetch metadata - id: fetch_metadata - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: aiida-registry fetch - - - name: Get entry points data - run: aiida-registry test-install - - - name: Move JSON file to the React project - run: | - mv plugins_metadata.json aiida-registry-app/src/ - - uses: actions/setup-node@v3 - with: - node-version: '18.x' - - name: Install npm dependencies and build - run: | - npm install - export VITE_PR_PREVIEW_PATH="/aiida-registry/pr-preview/pr-${{ github.event.number }}/" - npm run build - working-directory: ./aiida-registry-app - - name: Add plugins_metadata.json to the build folder - run: cp ./aiida-registry-app/src/plugins_metadata.json ./aiida-registry-app/dist - - - name: Deploy preview - uses: rossjrw/pr-preview-action@v1 - with: - source-dir: ./aiida-registry-app/dist - preview-branch: gh-pages - umbrella-dir: pr-preview - action: auto - custom-url: diff --git a/.github/workflows/webpage.yml b/.github/workflows/webpage.yml index ee818b86..760ebba4 100644 --- a/.github/workflows/webpage.yml +++ b/.github/workflows/webpage.yml @@ -10,12 +10,7 @@ on: push: branches: - master - -# https://docs.github.com/en/actions/using-jobs/using-concurrency -concurrency: - # only cancel in-progress jobs or runs for the current workflow - matches against branch & tags - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true + workflow_dispatch: jobs: webpage: @@ -26,44 +21,35 @@ jobs: COMMIT_AUTHOR: Deploy Action COMMIT_AUTHOR_EMAIL: action@github.com steps: - - uses: actions/checkout@v2 - - run: git fetch --no-tags --prune --depth=1 origin +refs/heads/*:refs/remotes/origin/* - - name: Set up Python 3.8 - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - name: Install prerequisites - run: pip install --upgrade wheel setuptools - - name: Install - run: pip install -e . -v - - name: Get cloned data from gh-pages - run: | - git remote set-url origin https://github.com/aiidateam/aiida-registry.git - git fetch origin gh-pages - git checkout origin/gh-pages plugins_metadata.json || true - mv plugins_metadata.json cloned_plugins_metadata.json || true + - name: Checkout Repo ⚡️ + uses: actions/checkout@v3 + - name: Create dev environment + uses: ./.github/actions/create-dev-env - name: fetch metadata id: fetch_metadata env: + # Use the GITHUB_TOKEN for github API calls + # otherwise the rate limit will be exceeded GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: aiida-registry fetch - - - name: Get entry points data + - name: Check plugins installation + # This step will attach plugin installation inforamtion to the metadata, e.g. if the plugin can be installed or not run: aiida-registry test-install - - name: Move the JSON file to the React directory + + - name: Move JSON file to the React project run: | - mv plugins_metadata.json aiida-registry-app/src/ + cp plugins_metadata.json aiida-registry-app/src/ + cp plugins_metadata.json aiida-registry-app/dist/ - uses: actions/setup-node@v3 with: node-version: '18.x' - name: Install npm dependencies and build run: | + echo $VITE_PR_PREVIEW_PATH npm install npm run build working-directory: ./aiida-registry-app - - name: Add plugins_metadata.json to the build folder - run: cp ./aiida-registry-app/src/plugins_metadata.json ./aiida-registry-app/dist - name: Deploy uses: peaceiris/actions-gh-pages@v3 diff --git a/.gitignore b/.gitignore index 59d81770..b503c382 100644 --- a/.gitignore +++ b/.gitignore @@ -20,5 +20,11 @@ pnpm-debug.log* lerna-debug.log* aiida-registry-app/node_modules -aiida-registry-app/dist aiida-registry-app/dist-ssr + +# data genarated +plugins_metadata.json +result.json + +# python +build/ diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 00000000..68314cde --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,24 @@ +The MIT License (MIT) + +Copyright (c), 2014-2020, ECOLE POLYTECHNIQUE FEDERALE DE LAUSANNE +(Theory and Simulation of Materials (THEOS) and National Centre for +Computational Design and Discovery of Novel Materials (NCCR MARVEL)), +Switzerland and ROBERT BOSCH LLC, USA. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/aiida-registry-app/dist/.gitignore b/aiida-registry-app/dist/.gitignore new file mode 100644 index 00000000..e23b5754 --- /dev/null +++ b/aiida-registry-app/dist/.gitignore @@ -0,0 +1,5 @@ +# This file is also used as the placeholder for the folder +# because github action will cp newly generated plugins_metadata.json +# to the folder, and the folder will be committed to the repo + +* diff --git a/aiida_registry/__init__.py b/aiida_registry/__init__.py index 97dd1d37..5750e679 100644 --- a/aiida_registry/__init__.py +++ b/aiida_registry/__init__.py @@ -9,7 +9,7 @@ import os -__version__ = "0.1.0" +__version__ = "0.3.0" # Absolute paths pwd = os.path.split(os.path.abspath(__file__))[0] diff --git a/bin/aiida-registry b/aiida_registry/cli.py similarity index 56% rename from bin/aiida-registry rename to aiida_registry/cli.py index 8fff9f47..bb2d287a 100644 --- a/bin/aiida-registry +++ b/aiida_registry/cli.py @@ -1,11 +1,14 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- +"""CLI for AiiDA registry.""" import click +from aiida_registry.make_pages import make_pages +from aiida_registry.test_install import test_install_all + @click.group() def cli(): - pass + """CLI for AiiDA registry.""" @cli.command() @@ -22,28 +25,21 @@ def cli(): default=True, help="Allow fetching wheels from PyPI", ) -def fetch(package, fetch_pypi, fetch_wheel): - from aiida_registry.make_pages import make_pages - - make_pages() - - -@cli.command() -def html(): - from aiida_registry.make_pages import make_pages - +def fetch(package, fetch_pypi, fetch_wheel): # pylint: disable=unused-argument + """Fetch data from PyPI and write to JSON file.""" make_pages() @cli.command() @click.option( "--container-image", + # should use aiidateam/aiida-core-with-services:lastest after the version is released + # default="aiidateam/aiida-core-with-services:edge", default="aiidateam/aiida-core:latest", help="Container image to use for the install", ) def test_install(container_image): - from aiida_registry.test_install import test_install_all - + """Test installing all plugins in a Docker container.""" test_install_all(container_image) diff --git a/aiida_registry/fetch_metadata.py b/aiida_registry/fetch_metadata.py index ab89aa2a..6e0b7d87 100644 --- a/aiida_registry/fetch_metadata.py +++ b/aiida_registry/fetch_metadata.py @@ -32,10 +32,7 @@ from .parse_pypi import PypiData, get_pypi_metadata from .utils import fetch_file -try: - GITHUB_TOKEN = os.environ["GITHUB_TOKEN"] -except KeyError: - GITHUB_TOKEN = "" +GITHUB_TOKEN = os.environ.get("GITHUB_TOKEN") @lru_cache(maxsize=None) @@ -99,10 +96,15 @@ def get_github_commits_count(repo_url): today = datetime.today().date() last_twelve_months = today - timedelta(days=365) - headers = { - "Authorization": f"Bearer {GITHUB_TOKEN}", - "Accept": "application/vnd.github+json", - } + # Run locally without GITHUB_TOKEN + if GITHUB_TOKEN is None: + headers = {"Accept": "application/vnd.github+json"} + else: + headers = { + "Authorization": f"Bearer {GITHUB_TOKEN}", + "Accept": "application/vnd.github+json", + } + params = { "since": last_twelve_months.isoformat(), "until": today.isoformat(), @@ -175,6 +177,9 @@ def complete_plugin_data( if plugin_data["hosted_on"] == "github.com" and GITHUB_TOKEN: commits_count = get_github_commits_count(plugin_data["code_home"]) else: + # when running locally, we don't have a GITHUB_TOKEN + # all the repositories are cloned locally + # this may take a while (10s per plugin) try: clone_repository(plugin_data["code_home"], plugin_data["name"]) commits_count = get_git_commits_count(plugin_data["name"]) diff --git a/bin/analyze_entrypoints.py b/bin/analyze_entrypoints.py index 9d26fec5..53c77d96 100644 --- a/bin/analyze_entrypoints.py +++ b/bin/analyze_entrypoints.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python # -*- coding: utf-8 -*- """" Fetch information about plugins and print it in a human-readable format. diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..9165c660 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,51 @@ +[build-system] +requires = ["flit_core >=3.4,<4"] +build-backend = "flit_core.buildapi" + +[project] +name = "aiida-registry" +dynamic = ["version"] # read from aiida_registry/__init__.py +description = "Registry of AiiDA plugins" +authors = [{name = "The AiiDA team", email = "developers@aiida.net"}] +readme = "README.md" +license = {file = "LICENSE.txt"} +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Framework :: AiiDA", + "License :: OSI Approved :: MIT License", + "Operating System :: POSIX :: Linux", + "Operating System :: MacOS :: MacOS X", + "Programming Language :: Python", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Topic :: Scientific/Engineering" +] +keywords = ["aiida", "workflows"] +requires-python = ">=3.9" +dependencies = [ + "jinja2~=2.11", + "requests~=2.28.1", + "requests-cache~=0.5.2", + "requirements-parser~=0.2.0", + "poetry~=1.1.15", + "tomlkit", + "click~=7.1", + "pyyaml~=6.0", + "docker~=5.0", + # https://github.com/aws/aws-sam-cli/issues/3661 + "markupsafe~=2.0.1", + # dev dependencies + "pre-commit~=2.2", + "pylint~=2.16.1", + "pytest~=6.2.2", + "pytest-regressions~=2.5.0", +] + +[project.urls] +Home = "https://aiida-registry.github.io/" +Documentation = "https://github.com/aiidateam/aiida-registry/wiki" +Source = "https://github.com/aiidateam/aiida-core" + +[project.scripts] +aiida-registry = "aiida_registry.cli:cli" diff --git a/setup.py b/setup.py deleted file mode 100644 index 24e9c47f..00000000 --- a/setup.py +++ /dev/null @@ -1,43 +0,0 @@ -# -*- coding: utf-8 -*- -"""Setup for AiiDA registry.""" - -from __future__ import absolute_import - -from setuptools import find_packages, setup - -if __name__ == "__main__": - with open("README.md", encoding="utf8") as handle: - setup( - name="aiida-registry", - packages=find_packages(), - long_description=handle.read(), - long_description_content_type="text/markdown", - version="0.3.0", - description="Registry of AiiDA plugins.", - author="AiiDA Team", - author_email="aiidateam@gmail.com", - url="https://aiidateam.github.io/aiida-registry/", - scripts=["bin/aiida-registry"], - install_requires=[ - "jinja2~=2.11", - "requests~=2.28.1", - "requests-cache~=0.5.2", - "requirements-parser~=0.2.0", - "poetry~=1.1.15", - "tomlkit", - "click~=7.1", - "pyyaml~=6.0", - "docker~=5.0", - # https://github.com/aws/aws-sam-cli/issues/3661 - "markupsafe~=2.0.1", - ], - license="MIT", - extras_require={ - "pre-commit": [ - "pre-commit~=2.2", - "pylint~=2.16.1", - ], - "testing": ["pytest", "pytest-regressions"], - }, - include_package_data=True, - )