FEAT: Final touches for pre-release 🔎

.github/workflows/release.yml

@@ -1,4 +1,4 @@
-name: Tidy3d Release
+name: "tidy3d-release"
 
 on:
   push:

.github/workflows/run_tests.yml

@@ -1,4 +1,4 @@
-name: Tidy3dTests
+name: "tidy3d-frontend-tests"
 
 on:
   workflow_dispatch:
@@ -28,8 +28,11 @@ jobs:
     runs-on: ${{ matrix.platform }}
     strategy:
       matrix:
-        python-version: ['3.9', '3.10', '3.11']
+        python-version: ['3.8', '3.9', '3.10', '3.11']
         platform: [ubuntu-latest, macos-latest, windows-latest]
+        exclude:
+          - python-version: '3.8'
+            platform: 'windows-latest'
     defaults:
       run:
         shell: bash

.github/workflows/sync-to-readthedocs-repo.yaml

@@ -1,4 +1,4 @@
-name: sync-to-readthedocs-repo
+name: "sync-to-readthedocs-repo"
 permissions:
   contents: write
 on:

.github/workflows/update_coverage.yml

@@ -0,0 +1,107 @@
+name: "tidy3d-coverage"
+
+on:
+  workflow_dispatch:
+  push:
+    branches: [ develop, main ]
+  pull_request:
+    branches:
+      - develop
+      - 'pre/*'
+
+defaults:
+  run:
+    shell: bash
+
+env:
+  PIP_DISABLE_PIP_VERSION_CHECK: 1
+  FORCE_COLOR: 1    # Get colored pytest output
+
+permissions:
+  contents: read
+
+jobs:
+  pre-commit:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+      - name: Test pre-commit hooks
+        run: |
+          python -m pip install --upgrade pip
+          pip install pre-commit
+          pre-commit run
+  build:
+    name: Coverage Test
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+    env:  # Set environment variables for the whole job
+      PIP_ONLY_BINARY: gdstk
+      MPLBACKEND: agg
+    steps:
+    - uses: actions/checkout@v1
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v2
+      with:
+        python-version: "3.10"
+    #----------------------------------------------
+    #  -----  install & configure poetry  -----
+    #----------------------------------------------
+    - name: Install Poetry
+      uses: snok/install-poetry@v1
+      with:
+        version: 1.6.1
+        virtualenvs-create: true
+        virtualenvs-in-project: true
+    #----------------------------------------------
+    #       load cached venv if cache exists
+    #----------------------------------------------
+    - name: Load cached venv
+      id: cached-poetry-dependencies
+      uses: actions/cache@v3
+      with:
+        path: .venv
+        key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
+    #----------------------------------------------
+    # install dependencies if cache does not exist
+    #----------------------------------------------
+    - name: Install dependencies
+      if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
+      run: poetry install --no-interaction --no-root -E dev
+    #----------------------------------------------
+    # install your root project, if required
+    #----------------------------------------------
+    - name: Install library
+      run: poetry install --no-interaction -E dev
+    #----------------------------------------------
+    #    add matrix specifics and run test suite
+    #----------------------------------------------
+    - name: Run tests
+      run: |
+        source $VENV
+        black . --check --diff
+        ruff check tidy3d --fix --exit-non-zero-on-fix
+        coverage run -m pytest -rA tests
+        coverage report -m
+        echo "total=$(coverage report --format=total)" >> $GITHUB_ENV
+        echo '### Total coverage: ${{ env.total }}%'
+
+    - name: "Create badge"
+      # if: ${{ github.ref == 'refs/heads/latest' }}
+      # https://gist.githubusercontent.com/nedbat/8c6980f77988a327348f9b02bbaf67f5
+      uses: schneegans/dynamic-badges-action@v1.7.0
+      with:
+        auth: ${{ secrets.GH_TIDY3D_COVERAGE_GIST }}
+        gistID: 4702549574741e87deaadba436218ebd
+        filename: tidy3d_extension.json
+        label: Coverage
+        message: ${{ env.total }}%
+        minColorRange: 60
+        maxColorRange: 95
+        valColorRange: ${{ env.total }}
+        style: "for-the-badge"

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/psf/black
-    rev: 22.12.0
+    rev: 23.12.1
     hooks:
     - id: black
       name: black <<< NOTE:, if you get a message stating that "All done! n file(s) reformatted", you must git add and commit the changes again to complete the commit. >>>

README.md

@@ -6,7 +6,7 @@ Name](https://img.shields.io/badge/pypi-tidy3d-blue?style=for-the-badge)](https:
 ![Tests](https://img.shields.io/github/actions/workflow/status/flexcompute/tidy3d/run_tests.yml?style=for-the-badge)
 ![License](https://img.shields.io/github/license/flexcompute/tidy3d?style=for-the-badge)
 [![Black](https://img.shields.io/badge/code%20style-black-000000.svg?style=for-the-badge)](https://github.com/psf/black)
-[//]: # (TODO add coverage label)
+![Coverage](https://img.shields.io/endpoint?url=https://gist.githubusercontent.com/daquinteroflex/4702549574741e87deaadba436218ebd/raw/tidy3d_extension.json)
 
 [![Notebooks](https://img.shields.io/badge/Demo-Live%20notebooks-8A2BE2?style=for-the-badge)](https://github.com/flexcompute/tidy3d-notebooks)
 

docs/api/plugins/adjoint.rst

@@ -3,6 +3,10 @@
 Adjoint
 -------
 
+.. toctree::
+
+    adjoint_readme
+
 .. autosummary::
    :toctree: ../_autosummary/
    :template: module.rst

docs/api/plugins/adjoint_readme.rst

@@ -0,0 +1 @@
+.. mdinclude:: ../../../tidy3d/plugins/adjoint/README.md

docs/api/plugins/design.rst

@@ -3,7 +3,9 @@
 Design Space Exploration
 ------------------------
 
-.. mdinclude:: ../../tidy3d/plugins/design/README.md
+.. toctree::
+
+    design_readme
 
 .. autosummary::
    :toctree: ../_autosummary/

docs/api/plugins/design_readme.rst

@@ -0,0 +1 @@
+.. mdinclude:: ../../../tidy3d/plugins/design/README.md

docs/development/documentation.rst

@@ -1,7 +1,47 @@
 Documentation
 ==============
 
+Getting Started
+---------------
+
+Assuming you already have ``poetry`` and the ``tidy3d develop`` commands installed (see the instructions if not), then building the documentation is easy:
+
+.. code::
+
+        poetry run tidy3d develop build-docs
+
+The output of the build will be in ``_docs/`` and you can view it by opening ``_docs/index.html`` in your browser. You might just have to click the ``index.html`` file to open it in your browser within a File Explorer.
+
 Theme
 ------
 
-Under ``docs/_static/css`` we can find ``custom.css`` which the color themes custom to Flexcompute can be found.
+Under ``docs/_static/css`` we can find ``custom.css`` which the color themes custom to Flexcompute can be found.
+
+Common Updates
+--------------
+
+Adding a new notebook
+''''''''''''''''''''''
+
+This process is self-contained in ``tidy3d-notebooks``.
+
+Make sure to add a link to the notebook in ``tidy3d-notebooks/docs/*`` directory in a relevant file.
+
+Then you have to commit to either the ``develop`` branch or your custom one. However, the important thing to understand is that the submodule in ``docs/notebooks`` has a state that is also committed. This means that when you or any tool clones this directory, then the state and mapped branch/commit of the submodule will be the one that was committed. However, you have to be careful that when your commit gets merged the commit of the ``tidy3d-notebooks`` submodule is also pointing to the latest ``develop`` branch and not any local branch in which you have been developing. Otherwise, the documentation will be built with your local branch, and not the published branch.
+
+This submodule commit process can be done by running ``git add docs/notebooks`` and then committing the change.
+
+If you want to locally develop notebooks in ``tidy3d/docs/notebooks`` then just use that submodule as your main development repository and commit to your local branch. Then when you are ready to publish, just make sure to commit the submodule to the latest ``develop`` branch. You can then build the documentation locally easily using this approach before it is published.
+
+Updating Docstrings
+'''''''''''''''''''''
+
+``tidy3d develop`` has a utility to update docstrings in the codebase.
+This is useful when updating the API and you want to update the docstrings to reflect the changes from multiple versions.
+
+Example usage:
+
+.. code::
+
+    poetry run tidy3d develop replace-in-files -d ./ -j ./docs/versions/test_replace_in_files.json -v 0.18.0 --dry-run True
+

docs/development/index.rst

@@ -12,10 +12,12 @@ Welecome to the ``tidy3d`` developers guide! These are just some recommendations
     installation
     usage
     documentation
+    recommendations
     release/index
 
 .. include:: /development/project_structure.rst
 .. include:: /development/installation.rst
 .. include:: /development/usage.rst
 .. include:: /development/documentation.rst
+.. include:: /development/recommendations.rst
 .. include:: /development/release/index.rst

docs/development/installation.rst

@@ -8,8 +8,8 @@ Maybe you already have ``tidy3d`` installed in some form. After installing versi
 
 It does not matter how you have installed ``tidy3d`` before as long as you have any form of `tidy3d>=2.6`` in your environment. This can help you transition from a standard user installation to a development environment installation.
 
-Beta instructions for verification (REMOVE pre 2.6 release)
-'''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''
+Quick Start
+''''''''''''
 
 Instructions for anyone who wants to test the new development flow before it gets included as part of the pre-release:
 
@@ -19,12 +19,13 @@ For ubuntu:
 
     git clone https://github.com/flexcompute/tidy3d.git
     cd tidy3d
-    git fetch origin repo_merge_no_history
-    git checkout repo_merge_no_history
-    # Create and activate a virtual environment here based on your python installation
-    python3 -m pip install -e . # Follow standard pip development install
-    python3 -m tidy3d develop # list all new development commands
-    python3 -m tidy3d develop configure-dev-environment
+    # Make sure you're in a branch > pre/2.6 and you can `import tidy3d` in python
+    pip install -e . # or whatever local installation works for you
+    tidy3d develop # Read all the new development helper commands
+    # tidy3d develop uninstall-dev-envrionment # in case you need to reset your environment
+    tidy3d develop install-dev-environment # install all requirements that you don't have and verify the exisiting ones
+    poetry run tidy3d develop verify-dev-environment # reproducibly verify development envrionment
+    # poetry run tidy3d develop build-docs # eg. reproducibly build documentation
 
 Now you can run the following ``tidy3d`` cli commands to test them.
 
@@ -37,7 +38,7 @@ If you are transitioning from the old development flow, to this new one, there a
 .. code::
 
     # Automatically check and install requirements like pipx, poetry, pandoc
-    tidy3d develop configure-dev-environment
+    tidy3d develop install-dev-environment
 
 Note that this is just a automatic script implementation of the `The Detailed Lane`_ instructions. It is intended to help you and raise warnings with suggestions of how to fix an environment setup issue. You do not have to use this helper function and can just follow the instructions in  `The Detailed Lane`_. All commands are echo-ed in the terminal so you will be able to observe and reproduce what is failing if you desire.
 
@@ -146,7 +147,7 @@ If you want to locally build documentation, then it is required to install ``pan
         .. code-block:: bash
 
             sudo apt-get update
-            sudo apt-get install pandoc="2.9"
+            sudo apt-get install pandoc
 
    .. group-tab:: macOS
 
@@ -164,7 +165,37 @@ If you want to locally build documentation, then it is required to install ``pan
 
            choco install pandoc --version="2.9"
 
-Congratulations! Now you have all the required tools installed, you can now use all the `poetry run tidy3d develop` commands reproducibly.
+Now you need to install the package in the reproducible poetry environment in development mode:
+
+.. code::
+
+    poetry install -E dev
+
+Congratulations! Now you have all the required tools installed, you can now use all the ``poetry run tidy3d develop`` commands reproducibly.
+
+If you want to contribute to the project, read the following section:
+
+
+More Contribution Requirements
+''''''''''''''''''''''''''''''
+
+If you want to contribute to the development of ``tidy3d``, you can follow the instructions below to set up your development environment. This will allow you to run the tests, build the documentation, and run the examples. Another thing you need to do before committing to the project is to install the pre-commit hooks. This will ensure that your code is formatted correctly and passes the tests before you commit it. To do this, run the following command:
+
+.. code::
+
+    poetry run pre-commit install
+
+This will run a few file checks on your code before you commit it. After this whenever you commit, the pre-commit hooks will run automatically. If any of the checks fail, you will have to fix the issues before you can commit. If for some reason, it's a check you want to waive, you can follow the instructions of the tool to automatically waive them or you can run the following command to skip the checks **only on minimal circumstances**:
+
+.. code::
+
+    git commit --no-verify
+
+You can also run the checks manually on all files by running the following command:
+
+.. code::
+
+    poetry run pre-commit run --all-files
 
 
 Packaging Equivalent Functionality
@@ -180,4 +211,5 @@ This package installation process should be  approximately equivalent to the pre
     ...
     pip install tidy3d[jax]
 
-All these options can be found inside the ``pyproject.toml`` ``tool.poetry.extras`` section. Each has a corresponding list of dependencies whose versions are defined on the ``tool.poetry.dependencies`` section of the file.
+All these options can be found inside the ``pyproject.toml`` ``tool.poetry.extras`` section. Each has a corresponding list of dependencies whose versions are defined on the ``tool.poetry.dependencies`` section of the file.
+

docs/development/project_structure.rst

@@ -1,4 +1,3 @@
-
 Project Structure
 =================
 

docs/development/recommendations.rst

@@ -0,0 +1,37 @@
+Recommendations
+=================
+
+Package Speedup Best Practices
+''''''''''''''''''''''''''''''
+
+``tidy3d`` is a pretty big project already, and will get bigger. We want to optimise the performance of the codebase throughout the multiple operations that we perform.
+
+We want to improve the speed of the project ``import`` and there are a few techniques to do this which are inherent to the way we write our code.
+
+- `For profiling the package distribution in the import <https://stackoverflow.com/questions/16373510/improving-speed-of-python-module-import>`_
+- `In terms of speeding our operations <https://wearecommunity.io/communities/tectoniques/articles/2499#:~:text=Proper%20Import,in%20slowing%20down%20code%20performance.>`_
+
+We have already begun facing these type of code-speed issues as first raised `here <https://github.com/flexcompute/tidy3d/pull/1300>`_, `here <https://github.com/flexcompute/tidy3d/pull/1300>`_
+
+So when we import dependencies inside our code-base in particular where these are used, we will try to do the following:
+
+.. code::
+
+    from mypackage import just_what_I_need
+
+instead of
+
+.. code::
+
+    import mypackage
+
+This is because the latter will import the entire package, which is not necessary and will slow down the code.
+
+
+Managing Optional Dependencies On-The-Fly
+''''''''''''''''''''''''''''''''''''''''''
+
+If you look within ``pyproject.toml``, it is possible to see that we have different packages relating to different functionalities that are optional.
+
+Some examples from these are ``[vtk, jax, trimesh, gdstk, gdspy]`` etc. What we want to do is improve the import speed of the core-package in order to minimise small core operations. As we scale into a bigger package, decoupling these type of imports from the total pacakge import is essential.
+