feat: add web crawler (#1)

* docs: add contributing file

* build: add python environment files

* docs: add license file

* docs: add readme file

* build: add setup file

* chore: update gitignore

* feat: add file_utils

* feat: add base classes

* feat: add web-crawler implementation

* ci: add pull request template

* ci: add pytest files

* chore: update gitignore

* ci: add pytest to requirements file

* ci: add github workflow files

* build: add coverage to python requirements deps

* test: add simple module import test

* ci: add flake8 to requirements file

* ci: add package json release file

* chore: format python code

* docs: update documentation for all classes

* test: update web crawler example

* build: update requirements file

* chore: add output to gitignore

* feat: add init files

* fix: fix base graph implementation

* chore: format code

* test: add unit tests

* chore: format code

* chore: fix line number flake8

* chore: format code

* chore: format code

* chore: format code

* chore: format code

* chore: format code further

* test: fix unit test

* test: fix coverage module issue

.flake8

@@ -0,0 +1,2 @@
+[flake8]
+max-line-length = 200

.github/pull_request_template.md

@@ -0,0 +1,7 @@
+## Why this PR and What changes?
+<!-- Explain the need for this PR (Pull Request). -->
+
+<!-- Summarize changes made (Pull Request). -->
+
+## Tests added?
+<!-- State test status and provide a brief description of the tests. -->

.github/workflows/conda.yml

@@ -0,0 +1,29 @@
+name: Conda
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  conda-check:
+    name: Check Conda environment
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - uses: conda-incubator/setup-miniconda@v2
+        with:
+          activate-environment: pietstormai
+          environment-file: environment.yml
+      - run: |
+          pip install --upgrade pip
+          pip install -e .
+          pip install -r requirements.txt
+      - run: |
+          conda info --envs
+      - run: |
+          python3 --version
+      - run: |
+          python3 -m pytest

.github/workflows/coverage.yml

@@ -0,0 +1,61 @@
+name: Coverage
+
+on:
+  pull_request:
+    types:
+      - opened
+      - synchronize
+
+jobs:
+  lint-test:
+    uses: ./.github/workflows/lint-test.yml
+
+  check-coverage:
+    name: Python Code Coverage
+    runs-on: ubuntu-latest
+    needs: lint-test
+    permissions:
+      contents: read
+      pull-requests: write
+    env:
+      # TODO: 5% of coverage is pretty low.
+      # We should increase this value after Friday.
+      COVERAGE_THRESHOLDS: '0 50'
+    steps:
+    - name: Download Coverage Report
+      uses: actions/download-artifact@v3
+      with:
+        name: coverage-report
+        path: .
+
+    - name: Code Coverage Summary Report
+      uses: irongut/CodeCoverageSummary@v1.3.0
+      with:
+        badge: true
+        output: 'both'
+        format: 'markdown'
+        hide_complexity: true
+        hide_branch_rate: false
+        filename: 'coverage.xml'
+        thresholds: ${{ env.COVERAGE_THRESHOLDS }}
+
+    - name: Cat coverage.md file
+      run: cat code-coverage-results.md
+
+    - name: Comment on Pull Request
+      uses: marocchino/sticky-pull-request-comment@v2
+      if: github.event_name == 'pull_request'
+      with:
+        recreate: true
+        path: code-coverage-results.md
+
+    - name: Write to Job Summary
+      run: cat code-coverage-results.md >> $GITHUB_STEP_SUMMARY
+
+    - name: Code Coverage Threshold Check
+      uses: irongut/CodeCoverageSummary@v1.3.0
+      with:
+        fail_below_min: true
+        filename: 'coverage.xml'
+        hide_branch_rate: false
+        thresholds: ${{ env.COVERAGE_THRESHOLDS }}

.github/workflows/lint-test.yml

@@ -0,0 +1,65 @@
+name: Lint and Test
+
+on:
+  workflow_call:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  python-lint:
+    name: Lint Python Code
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v4
+      with:
+        python-version: "3.10"
+
+    - name: Install dependencies
+      run: |
+        pip install -r requirements.txt
+
+    - name: Lint Code
+      run: |
+        flake8 .
+
+    # TODO: disabling for now
+    # - name: Lint Docs
+    #   run: |
+    #     pydocstyle --convention=numpy .
+
+  python-unit-tests:
+    name: Run Unit Python Tests
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout Code
+      uses: actions/checkout@v3
+
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v4
+      with:
+        python-version: "3.10"
+
+    - name: Install Dependencies
+      run: |
+        pip install --upgrade pip
+        pip install -e .
+        pip install -r requirements.txt
+
+    - name: Run Pytest
+      run: |
+        # Note: `pytest.ini` is used to enable specific pytest packages.
+        coverage run --branch --source=crawler -m pytest
+        coverage xml
+
+    - name: Upload Coverage Report
+      uses: actions/upload-artifact@v3
+      with:
+        name: coverage-report
+        path: coverage.xml

.github/workflows/release.yml

@@ -0,0 +1,52 @@
+name: Release
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  lint-test:
+    uses: ./.github/workflows/lint-test.yml
+
+  release:
+    name: Release Python Package
+    runs-on: ubuntu-latest
+    needs: lint-test
+
+    # TODO: remove unnecessary permissions
+    permissions:
+        contents: write
+        actions: write
+        checks: write
+        issues: write
+        discussions: write
+        packages: write
+        pull-requests: write
+        repository-projects: write
+        statuses: write
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+
+      - name: Install Dependencies
+        run: |
+          pip install -e .
+          pip install -r requirements.txt
+
+      - name: Setup node
+        uses: actions/setup-node@v3
+        with:
+          node-version: "18.x"
+
+      - name: Semantic relase
+        run: |
+          npm install semantic-release
+          npx semantic-release
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -0,0 +1,20 @@
+build/
+*.egg-info
+__pycache__
+
+.ipynb_checkpoints
+__pycache__/
+.pytest_cache/
+.eggs/
+*.egg-info
+docs-build/
+docs-files/
+*.coverage
+*.log
+.vscode
+*.xml
+build/
+*.DS_Store
+*.csv
+
+output

.pytest.ini

@@ -0,0 +1 @@
+[pytest]