The big documentation and development upgrade

.binder/requirements.txt

@@ -0,0 +1,5 @@
+tidy3d
+tmm
+nlopt
+tqdm
+gdspy

.gitattributes

@@ -1,8 +1,17 @@
+# Linguist-vendored files
 docs/* linguist-vendored
 notebooks/* linguist-vendored
 tests/* linguist-vendored
 
+# Disable linguist detection for IPython notebooks
 *.ipynb linguist-detectable=false
+
+# Mark IPython notebooks as generated files
 *.ipynb linguist-generated=true
 
-CHANGELOG.md merge=union
+# Specify merge attributes for CHANGELOG.md
+CHANGELOG.md merge=union
+
+# Specify diff and merge attributes for IPython notebooks
+*.ipynb diff=jupyternotebook
+*.ipynb merge=jupyternotebook

.gitconfig

@@ -0,0 +1,13 @@
+[diff "jupyternotebook"]
+	command = git-nbdiffdriver diff
+[merge "jupyternotebook"]
+	driver = git-nbmergedriver merge %O %A %B %L %P
+	name = jupyter notebook merge driver
+[difftool "nbdime"]
+	cmd = git-nbdifftool diff \"$LOCAL\" \"$REMOTE\" \"$BASE\"
+[difftool]
+	prompt = false
+[mergetool "nbdime"]
+	cmd = git-nbmergetool merge \"$BASE\" \"$LOCAL\" \"$REMOTE\" \"$MERGED\"
+[mergetool]
+	prompt = false

.github/workflows/run_tests.yml

@@ -3,30 +3,70 @@ name: Tidy3dTests
 on:
   workflow_dispatch:
   push:
-    branches: [ develop ]
+    branches: [ development ]
   pull_request:
     branches:
-      - develop
+      - development
       - 'pre/*'
+      - poetry_upgrade
 
 jobs:
   build:
     name: test ${{ matrix.python-version }} - ${{ matrix.platform }}
     runs-on: ${{ matrix.platform }}
     strategy:
       matrix:
-        python-version: ['3.8', '3.9', '3.10', '3.11']
+        python-version: ['3.9', '3.10', '3.11']
         platform: [ubuntu-latest, macos-latest, windows-latest]
+    defaults:
+      run:
+        shell: bash
+    env:  # Set environment variables for the whole job
+      PIP_FIND_LINKS: https://whls.blob.core.windows.net/unstable/index.html
+      PIP_ONLY_BINARY: gdstk
+      MPLBACKEND: agg
     steps:
     - uses: actions/checkout@v1
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v2
       with:
         python-version: ${{ matrix.python-version }}
+    #----------------------------------------------
+    #  -----  install & configure poetry  -----
+    #----------------------------------------------
+    - name: Install Poetry
+      uses: snok/install-poetry@v1
+      with:
+        version: 1.6.1
+        virtualenvs-create: true
+        virtualenvs-in-project: true
+    #----------------------------------------------
+    #       load cached venv if cache exists
+    #----------------------------------------------
+    - name: Load cached venv
+      id: cached-poetry-dependencies
+      uses: actions/cache@v3
+      with:
+        path: .venv
+        key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
+    #----------------------------------------------
+    # install dependencies if cache does not exist
+    #----------------------------------------------
     - name: Install dependencies
+      if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
+      run: poetry install --no-interaction --no-root -E dev
+    #----------------------------------------------
+    # install your root project, if required
+    #----------------------------------------------
+    - name: Install library
+      run: poetry install --no-interaction -E dev
+    #----------------------------------------------
+    #    add matrix specifics and run test suite
+    #----------------------------------------------
+    - name: Run tests
       run: |
-        python -m pip install --upgrade pip
-        pip install tox-gh-actions
-        pip install tox
-    - name: Test with tox
-      run: tox
+        source $VENV
+        black . --check --diff
+        ruff check tidy3d --fix --exit-non-zero-on-fix
+        pytest -rA tests
+        pytest -rA tests/test_data/_test_datasets_no_vtk.py

.github/workflows/sync-to-proxy-repo.yaml

@@ -0,0 +1,70 @@
+name: sync-to-proxy-repo
+on:
+  push:
+    branches:
+      - main
+      - latest
+  workflow_dispatch:
+    inputs:
+      env:
+        description: environment
+        default: dev
+        required: true
+jobs:
+  set_env:
+    outputs:
+      env: ${{ steps.setenv.outputs.env }}
+      dest-branch: ${{ steps.dest-branch.outputs.branch }}
+    runs-on: ubuntu-latest
+    steps:
+      - id: setenv
+        run: |
+          if test -n "${{ github.event.inputs.env }}"
+          then
+            echo "::set-output name=env::${{ github.event.inputs.env }}"
+            echo "ENV=${{ github.event.inputs.env }}" >> $GITHUB_ENV
+          fi
+          if test "${{ github.event_name }}" = 'push' -a "${{ github.ref }}" = 'refs/heads/latest'
+          then
+            echo "::set-output name=env::dev"
+            echo "ENV=dev" >> $GITHUB_ENV
+          fi
+          if test "${{ github.event_name }}" = 'push' -a "${{ github.ref }}" = 'refs/heads/main'
+          then
+            echo "::set-output name=env::prod"
+            echo "ENV=prod" >> $GITHUB_ENV
+          fi
+      - id: dest-branch
+        run: |
+          case ${{ env.ENV }} in
+            dev)
+            echo "::set-output name=branch::develop"
+            ;;
+            prod)
+            echo "::set-output name=branch::main"
+            ;;
+          esac
+  build-and-deploy:
+    needs: set_env
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - uses: JamesIves/github-pages-deploy-action@v4
+        with:
+          folder: docs/notebooks/
+          token: ${{ secrets.GH_PAT }}
+          repository-name: flexcompute/tidy3d-example-library
+          target-folder: notebooks
+          branch: ${{ needs.set_env.outputs.dest-branch }}
+      - run: |
+          mkdir -p build_toc
+          cp docs/examples.rst build_toc/examples.rst
+
+      - uses: JamesIves/github-pages-deploy-action@v4
+        with:
+          folder: build_toc
+          token: ${{ secrets.GH_PAT }}
+          repository-name: flexcompute/tidy3d-example-library
+          target-folder: toc
+          branch: ${{ needs.set_env.outputs.dest-branch }}

.github/workflows/sync-to-readthedocs-repo.yaml

@@ -0,0 +1,63 @@
+name: sync-to-readthedocs-repo
+permissions:
+  contents: write
+on:
+  push:
+    branches:
+      - main
+      - latest
+      - repo_merge_no_history
+  workflow_dispatch:
+    inputs:
+      env:
+        description: environment
+        default: dev
+        required: true
+jobs:
+  set_env:
+    outputs:
+      env: ${{ steps.setenv.outputs.env }}
+      dest-branch: ${{ steps.dest-branch.outputs.branch }}
+    runs-on: ubuntu-latest
+    steps:
+      - id: setenv
+        run: |
+          if test -n "${{ github.event.inputs.env }}"
+          then
+            echo "::set-output name=env::${{ github.event.inputs.env }}"
+            echo "ENV=${{ github.event.inputs.env }}" >> $GITHUB_ENV
+          elif test "${{ github.event_name }}" = 'push' -a "${{ github.ref }}" = 'refs/heads/latest'
+          then
+            echo "::set-output name=env::dev"
+            echo "ENV=dev" >> $GITHUB_ENV
+          elif test "${{ github.event_name }}" = 'push' -a "${{ github.ref }}" = 'refs/heads/main'
+          then
+            echo "::set-output name=env::prod"
+            echo "ENV=prod" >> $GITHUB_ENV
+          else
+            echo "::set-output name=env::dev"
+            echo "ENV=dev" >> $GITHUB_ENV
+          fi
+      - id: dest-branch
+        run: |
+          case ${{ env.ENV }} in
+            dev)
+            echo "::set-output name=branch::repo_merge_no_history"
+            ;;
+            prod)
+            echo "::set-output name=branch::main"
+            ;;
+          esac
+  build-and-deploy:
+    needs: set_env
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - uses: JamesIves/github-pages-deploy-action@v4
+        with:
+          folder: .
+          token: ${{ secrets.GH_PAT }}
+          repository-name: flexcompute-readthedocs/tidy3d-docs-demo
+          target-folder: .
+          branch: ${{ needs.set_env.outputs.dest-branch }}

.github/workflows/sync.yaml

@@ -0,0 +1,104 @@
+name: build&sync
+on:
+  workflow_dispatch:
+    inputs:
+      env:
+        description: environment
+        default: dev
+        required: true
+jobs:
+  set_env:
+    outputs:
+      env: ${{ steps.setenv.outputs.env }}
+      dest-branch: ${{ steps.dest-branch.outputs.branch }}
+    runs-on: ubuntu-latest
+    steps:
+      - id: setenv
+        run: |
+          if test -n "${{ github.event.inputs.env }}"
+          then
+            echo "::set-output name=env::${{ github.event.inputs.env }}"
+            echo "ENV=${{ github.event.inputs.env }}" >> $GITHUB_ENV
+          fi
+          if test "${{ github.event_name }}" = 'push' -a "${{ github.ref }}" = 'refs/heads/develop'
+          then
+            echo "::set-output name=env::dev"
+            echo "ENV=dev" >> $GITHUB_ENV
+          fi
+          if test "${{ github.event_name }}" = 'push' -a "${{ github.ref }}" = 'refs/heads/main'
+          then
+            echo "::set-output name=env::prod"
+            echo "ENV=prod" >> $GITHUB_ENV
+          fi
+      - id: dest-branch
+        run: |
+          case ${{ env.ENV }} in
+            dev)
+            echo "::set-output name=branch::develop"
+            ;;
+            prod)
+            echo "::set-output name=branch::main"
+            ;;
+          esac
+  build-and-deploy:
+    needs: set_env
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          submodules: recursive
+      - name: echo environment
+        run: |
+          echo "::notice ::env:${{needs.set_env.outputs.env}}"
+          echo "::notice ::dest-branch:${{needs.set_env.outputs.dest-branch}}"
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+          cache: 'pip'
+      - run: cd ${{ github.workspace }}/docs && pip install beautifulsoup4 nbformat nbconvert pyyaml
+      - run: cd ${{ github.workspace }}/docs/notebooks && mkdir html && python convert.py
+      - shell: bash
+        run: |
+          cd ${{ github.workspace }}
+          mkdir -p build && rsync -r ${{ github.workspace }}/docs/notebooks/html/ build
+          mkdir -p build/css && rsync -r build/css/ build-css
+          mkdir -p build/image && rsync -r build/image/ build-image
+          mkdir -p build-source && cp docs/source/notebooks/*.ipynb build-source
+          rm -rf build/css
+          rm -rf build/image
+          ls -la build
+
+      - uses: JamesIves/github-pages-deploy-action@v4
+        with:
+          folder: build
+          token: ${{ secrets.GH_PAT }}
+          repository-name: cmsflexc/flexcompute.com
+          target-folder: tidy3d/examples/notebooks
+          branch: ${{ needs.set_env.outputs.dest-branch }}
+      - uses: JamesIves/github-pages-deploy-action@v4
+        with:
+          folder: build-css
+          token: ${{ secrets.GH_PAT }}
+          repository-name: cmsflexc/flexcompute.com
+          target-folder: /assets/tidy3d/examples/css
+          branch: ${{ needs.set_env.outputs.dest-branch }}
+      - uses: JamesIves/github-pages-deploy-action@v4
+        with:
+          folder: build-image
+          token: ${{ secrets.GH_PAT }}
+          repository-name: cmsflexc/flexcompute.com
+          target-folder: /assets/tidy3d/examples/image
+          branch: ${{ needs.set_env.outputs.dest-branch }}
+      - uses: JamesIves/github-pages-deploy-action@v4
+        with:
+          folder: build-source
+          token: ${{ secrets.GH_PAT }}
+          repository-name: cmsflexc/flexcompute.com
+          target-folder: /assets/tidy3d/examples/notebooks
+          branch: ${{ needs.set_env.outputs.dest-branch }}
+      - uses: actions/upload-artifact@v3
+        with:
+            name: output
+            path: build/
+            retention-days: 7