Adding Throughput Anomaly detector files and CLI support

This PR is resonsible for: 1. Adding throughput Anomaly detector files to Theia. 2. Adding CLI support. CLI support for get, list and status is present, commands are yet to be implemented 3. Adding CRD for the anomaly detector. 4. Adding e2e Test for all three algos of TAD. 5. Adding Unit tests 6. Using same spark operator for all spark jobs NOTE: TODO Add stat as an API Signed-off-by: Tushar Tathgur <tathgurt@tathgurtFLVDL.vmware.com>
antrea-io · Feb 28, 2023 · e377768 · e377768
1 parent f2328e7
commit e377768
Show file tree

Hide file tree

Showing 76 changed files with 5,919 additions and 259 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -137,3 +137,34 @@ jobs:
       run: |
         echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
         docker push antrea/theia-manager:latest
+  
+  check-anomaly-detection-changes:
+    name: Check whether anomaly-detection image needs to be built based on diff
+    runs-on: [ubuntu-latest]
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+    - uses: antrea-io/has-changes@v2
+      id: check_diff
+      with:
+        paths: plugins/anomaly-detection/* build/images/Dockerfile.anomaly-detection.ubuntu
+    outputs:
+      has_changes: ${{ steps.check_diff.outputs.has_changes }}
+
+  build-anomaly-detection:
+    needs: check-anomaly-detection-changes
+    if: ${{ needs.check-anomaly-detection-changes.outputs.has_changes == 'yes' || github.event_name == 'push' }}
+    runs-on: [ubuntu-latest]
+    steps:
+    - uses: actions/checkout@v2
+    - name: Build anomaly-detection Docker image
+      run: make anomaly-detection
+    - name: Push anomaly-detection Docker image to registry
+      if: ${{ github.repository == 'antrea-io/theia' && github.event_name == 'push' && github.ref == 'refs/heads/main' }}
+      env:
+        DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+        DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+      run: |
+        echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
+        docker push antrea/theia-anomaly-detection:latest
diff --git a/.github/workflows/build_tag.yml b/.github/workflows/build_tag.yml
@@ -65,6 +65,21 @@ jobs:
         echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
         docker push antrea/theia-policy-recommendation:"${VERSION}"
 
+  build-anomaly-detection:
+    runs-on: [ubuntu-latest]
+    needs: get-version
+    steps:
+    - uses: actions/checkout@v2
+    - name: Build anomaly-detection Docker image and push to registry
+      env:
+        DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+        DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+        VERSION: ${{ needs.get-version.outputs.version }}
+      run: |
+        make anomaly-detection
+        echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
+        docker push antrea/theia-anomaly-detection:"${VERSION}"
+
   build-theia-manager:
     runs-on: [ubuntu-latest]
     needs: get-version

diff --git a/.github/workflows/kind.yml b/.github/workflows/kind.yml
@@ -45,6 +45,23 @@ jobs:
         name: policy-recommendation
         path: policy-recommendation.tar
         retention-days: 1 # minimum value, in case artifact deletion by 'artifact-cleanup' job fails
+
+  build-anomaly-detection-image:
+    name: Build Anomaly Detection image to be used for Kind e2e tests
+    needs: check-changes
+    if: ${{ needs.check-changes.outputs.has_changes == 'yes' }}
+    runs-on: [ ubuntu-latest ]
+    steps:
+    - uses: actions/checkout@v3
+    - run: make anomaly-detection
+    - name: Save Anomaly Detection image to tarball
+      run: docker save -o anomaly-detection.tar antrea/theia-anomaly-detection
+    - name: Upload Anomaly Detection image for subsequent jobs
+      uses: actions/upload-artifact@v3
+      with:
+        name: anomaly-detection
+        path: anomaly-detection.tar
+        retention-days: 1 # minimum value, in case artifact deletion by 'artifact-cleanup' job fails
 
   build-clickhouse-monitor-image:
     name: Build ClickHouse monitor image to be used for Kind e2e tests
@@ -101,6 +118,7 @@ jobs:
     name: E2e tests on a Kind cluster on Linux
     needs: 
       - build-policy-recommendation-image
+      - build-anomaly-detection-image
       - build-clickhouse-monitor-image
       - build-clickhouse-server-image
       - build-theia-manager-image
@@ -123,6 +141,10 @@ jobs:
         uses: actions/download-artifact@v3
         with:
           name: policy-recommendation
+      - name: Download Anomaly Detection images from previous jobs
+        uses: actions/download-artifact@v3
+        with:
+          name: anomaly-detection
       - name: Download ClickHouse monitor images from previous jobs
         uses: actions/download-artifact@v3
         with:
@@ -139,6 +161,8 @@ jobs:
         run:  |
           docker load -i policy-recommendation.tar
           docker tag antrea/theia-policy-recommendation:latest projects.registry.vmware.com/antrea/theia-policy-recommendation:latest
+          docker load -i anomaly-detection.tar
+          docker tag antrea/theia-anomaly-detection:latest projects.registry.vmware.com/antrea/theia-anomaly-detection:latest
           docker load -i clickhouse-monitor.tar
           docker tag antrea/theia-clickhouse-monitor:latest projects.registry.vmware.com/antrea/theia-clickhouse-monitor:latest
           docker load -i clickhouse-server.tar
@@ -229,6 +253,7 @@ jobs:
     name: Delete uploaded images
     needs:
     - build-policy-recommendation-image
+    - build-anomaly-detection-image
     - build-clickhouse-monitor-image
     - build-clickhouse-server-image
     - build-theia-manager-image
@@ -241,6 +266,11 @@ jobs:
       uses: geekyeggo/delete-artifact@v1
       with:
         name: policy-recommendation
+    - name: Delete anomaly-detection
+      if: ${{ needs.build-anomaly-detection-image.result == 'success' }}
+      uses: geekyeggo/delete-artifact@v1
+      with:
+        name: anomaly-detection
     - name: Delete clickhouse-monitor
       if: ${{ needs.build-clickhouse-monitor-image.result == 'success' }}
       uses: geekyeggo/delete-artifact@v1

diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml
@@ -23,7 +23,7 @@ jobs:
     - uses: antrea-io/has-changes@v2
       id: check_diff
       with:
-        paths: plugins/policy-recommendation/*
+        paths: plugins/policy-recommendation/* plugins/anomaly-detection/*
     outputs:
       has_changes: ${{ steps.check_diff.outputs.has_changes }}
 
@@ -62,16 +62,26 @@ jobs:
     # N530 direct neutron imports not allowed
     # N531 translations hints
     # W504 line break after binary operator
-    - name: Lint with flake8
+    - name: Install Lint dependencies with flake8
       run: |
         python -m pip install --upgrade pip
         pip install flake8
+    - name: Lint with flake8 on policy-recommendation
+      run: |
         flake8 --ignore E125,E126,E128,E129,E265,H305,H307,H404,H405,H904,N530,N531,W504,I202 \
                --enable-extensions H106,H203,H204,H205 \
                --show-source \
                --count \
                --statistic \
                plugins/policy-recommendation/
+    - name: Lint with flake8 on anomaly-detection
+      run: |
+        flake8 --ignore E125,E126,E128,E129,E265,H305,H307,H404,H405,H904,N530,N531,W504,I202 \
+               --enable-extensions H106,H203,H204,H205 \
+               --show-source \
+               --count \
+               --statistic \
+               plugins/anomaly-detection/
 
   test-unit:
       needs: check-python-changes
@@ -94,10 +104,15 @@ jobs:
           python -m pip install --upgrade pip
           python -m pip install pytest-cov
           python -m pip install -r plugins/policy-recommendation/requirements.txt
-      - name: Run unit tests
+          python -m pip install -r plugins/anomaly-detection/requirements.txt
+      - name: Run policy-recommendation unit tests
         run: |
           cd plugins/policy-recommendation/
           pytest -vv --cov . --cov-report xml
+      - name: Run anomaly-detection unit tests
+        run: |
+          cd plugins/anomaly-detection/
+          pytest -vv --cov . --cov-report xml
       - name: Upload coverage reports to Codecov
         uses: codecov/codecov-action@v3
         with:

diff --git a/Makefile b/Makefile
@@ -216,6 +216,14 @@ policy-recommendation:
 	docker tag antrea/theia-policy-recommendation:$(DOCKER_IMG_VERSION) projects.registry.vmware.com/antrea/theia-policy-recommendation
 	docker tag antrea/theia-policy-recommendation:$(DOCKER_IMG_VERSION) projects.registry.vmware.com/antrea/theia-policy-recommendation:$(DOCKER_IMG_VERSION)
 
+.PHONY: anomaly-detection
+anomaly-detection:
+	@echo "===> Building antrea/theia-anomaly-detection Docker image <==="
+	docker build --pull -t antrea/theia-anomaly-detection:$(DOCKER_IMG_VERSION) -f build/images/Dockerfile.anomaly-detection.ubuntu .
+	docker tag antrea/theia-anomaly-detection:$(DOCKER_IMG_VERSION) antrea/theia-anomaly-detection
+	docker tag antrea/theia-anomaly-detection:$(DOCKER_IMG_VERSION) projects.registry.vmware.com/antrea/theia-anomaly-detection
+	docker tag antrea/theia-anomaly-detection:$(DOCKER_IMG_VERSION) projects.registry.vmware.com/antrea/theia-anomaly-detection:$(DOCKER_IMG_VERSION)
+
 THEIA_BINARIES := theia-darwin theia-linux theia-windows
 $(THEIA_BINARIES): theia-%:
 	@GOOS=$* $(GO) build -o $(BINDIR)/$@ $(GOFLAGS) -ldflags '$(LDFLAGS)' antrea.io/theia/pkg/theia
@@ -231,3 +239,4 @@ theia: $(THEIA_BINARIES)
 .PHONY: theia-release
 theia-release:
 	@$(GO) build -o $(BINDIR)/$(THEIA_BINARY_NAME) $(GOFLAGS) -ldflags '-s -w $(LDFLAGS)' antrea.io/theia/pkg/theia
+
diff --git a/README.md b/README.md
@@ -20,6 +20,12 @@ NetworkPolicy configuration to secure Kubernetes network and applications.
 Please refer to the [NetworkPolicy Recommendation](docs/networkpolicy-recommendation.md)
 user guide to learn more.
 
+Theia also provides throughput anomaly detection, it can find the anomalies
+in the network, and report them to the user.
+Please refer to the
+[Throughput Anomaly Detection](docs/throughput-anomaly-detection.md) user
+guide to learn more.
+
 ## Contributing
 
 The Antrea community welcomes new contributors. We are waiting for your PRs!

diff --git a/build/charts/theia/README.md b/build/charts/theia/README.md
@@ -73,9 +73,9 @@ Kubernetes: `>= 1.16.0-0`
 | grafana.storage.createPersistentVolume.type | string | `"HostPath"` | Type of PersistentVolume. Can be set to "HostPath", "Local" or "NFS". Please set this value to use a PersistentVolume created by Theia. |
 | grafana.storage.persistentVolumeClaimSpec | object | `{}` | Specification for PersistentVolumeClaim. This is ignored if createPersistentVolume.type is non-empty. To use a custom PersistentVolume, please set storageClassName: "" volumeName: "<my-pv>". To dynamically provision a PersistentVolume, please set storageClassName: "<my-storage-class>". HostPath storage is used if both createPersistentVolume.type and persistentVolumeClaimSpec are empty. |
 | grafana.storage.size | string | `"1Gi"` | Grafana storage size. It is used to store Grafana configuration files. Can be a plain integer or as a fixed-point number using one of these quantity suffixes: E, P, T, G, M, K. Or the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki. |
-| sparkOperator.enable | bool | `false` | Determine whether to install Spark Operator. It is required to run Network Policy Recommendation jobs. |
+| sparkOperator.enable | bool | `false` | Determine whether to install Spark Operator. It is required to run Network Policy Recommendation and Throughput Anomaly detection jobs. |
 | sparkOperator.image | object | `{"pullPolicy":"IfNotPresent","repository":"projects.registry.vmware.com/antrea/theia-spark-operator","tag":"v1beta2-1.3.3-3.1.1"}` | Container image used by Spark Operator. |
-| sparkOperator.name | string | `"policy-recommendation"` | Name of Spark Operator. |
+| sparkOperator.name | string | `"theia"` | Name of Spark Operator. |
 | theiaManager.apiServer.apiPort | int | `11347` | The port for the Theia Manager APIServer to serve on. |
 | theiaManager.apiServer.selfSignedCert | bool | `true` | Indicates whether to use auto-generated self-signed TLS certificates. If false, a Secret named "theia-manager-tls" must be provided with the following keys: ca.crt, tls.crt, tls.key. |
 | theiaManager.apiServer.tlsCipherSuites | string | `""` | Comma-separated list of cipher suites that will be used by the Theia Manager APIservers. If empty, the default Go Cipher Suites will be used. |

diff --git a/build/charts/theia/crds/anomaly-detector-crd.yaml b/build/charts/theia/crds/anomaly-detector-crd.yaml
@@ -0,0 +1,74 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: throughputanomalydetectors.crd.theia.antrea.io
+  labels:
+    app: theia
+spec:
+  group: crd.theia.antrea.io
+  versions:
+    - name: v1alpha1
+      served: true
+      storage: true
+      schema:
+        openAPIV3Schema:
+          type: object
+          required:
+            - spec
+          properties:
+            spec:
+              type: object
+              required:
+                - jobType
+              properties:
+                jobType:
+                  type: string
+                startInterval:
+                  type: string
+                  format: datetime
+                endInterval:
+                  type: string
+                  format: datetime
+                executorInstances:
+                  type: integer
+                driverCoreRequest:
+                  type: string
+                driverMemory:
+                  type: string
+                executorCoreRequest:
+                  type: string
+                executorMemory:
+                  type: string
+            status:
+              type: object
+              properties:
+                state:
+                  type: string
+                sparkApplication:
+                  type: string
+                completedStages:
+                  type: integer
+                totalStages:
+                  type: integer
+                startTime:
+                  type: string
+                  format: datetime
+                endTime:
+                  type: string
+                  format: datetime
+                errorMsg:
+                  type: string
+      additionalPrinterColumns:
+        - description: Current state of the job
+          jsonPath: .status.state
+          name: State
+          type: string
+      subresources:
+        status: {}
+  scope: Namespaced
+  names:
+    plural: throughputanomalydetectors
+    singular: throughputanomalydetector
+    kind: ThroughputAnomalyDetector
+    shortNames:
+      - tad
diff --git a/build/charts/theia/provisioning/datasources/create_table.sh b/build/charts/theia/provisioning/datasources/create_table.sh
@@ -285,6 +285,24 @@ clickhouse client -n -h 127.0.0.1 <<-EOSQL
     ) engine=ReplicatedMergeTree('/clickhouse/tables/{shard}/{database}/{table}', '{replica}')
     ORDER BY (timeCreated);
 
+    --Create a table to store the Throughput Anomaly Detector results
+    CREATE TABLE IF NOT EXISTS tadetector_local (
+        sourceIP String,
+        sourceTransportPort UInt16,
+        destinationIP String,
+        destinationTransportPort UInt16,
+        protocolIdentifier UInt16,
+        flowStartSeconds DateTime,
+        flowEndSeconds DateTime,
+        throughputStandardDeviation Float64,
+        algoType String,
+        algoCalc Float64,
+        throughput Float64,
+        anomaly String,
+        id String
+    ) engine=ReplicatedMergeTree('/clickhouse/tables/{shard}/{database}/{table}', '{replica}')
+    ORDER BY (flowStartSeconds);
+
     --Create distributed tables for cluster
     CREATE TABLE IF NOT EXISTS flows AS flows_local
     engine=Distributed('{cluster}', default, flows_local, rand());
@@ -300,5 +318,9 @@ clickhouse client -n -h 127.0.0.1 <<-EOSQL
 
     CREATE TABLE IF NOT EXISTS recommendations AS recommendations_local
     engine=Distributed('{cluster}', default, recommendations_local, rand());
+
+    CREATE TABLE IF NOT EXISTS tadetector AS tadetector_local
+    engine=Distributed('{cluster}', default, tadetector_local, rand());
+
 EOSQL
 }
diff --git a/build/charts/theia/provisioning/datasources/migrators/000004_0-4-0.down.sql b/build/charts/theia/provisioning/datasources/migrators/000004_0-4-0.down.sql
@@ -0,0 +1,2 @@
+--Drop table
+DROP tadetector_local
diff --git a/build/charts/theia/provisioning/datasources/migrators/000004_0-4-0.up.sql b/build/charts/theia/provisioning/datasources/migrators/000004_0-4-0.up.sql
@@ -0,0 +1,20 @@
+--Create a table to store the Throughput Anomaly Detector results
+CREATE TABLE IF NOT EXISTS tadetector_local (
+    sourceIP String,
+    sourceTransportPort UInt16,
+    destinationIP String,
+    destinationTransportPort UInt16,
+    protocolIdentifier UInt16,
+    flowStartSeconds DateTime,
+    flowEndSeconds DateTime,
+    throughputStandardDeviation Float64,
+    algoType String,
+    algoCalc Float64,
+    throughput Float64,
+    anomaly String,
+    id String
+) engine=ReplicatedMergeTree('/clickhouse/tables/{shard}/{database}/{table}', '{replica}')
+ORDER BY (flowStartSeconds);
+
+CREATE TABLE IF NOT EXISTS tadetector AS tadetector_local
+    engine=Distributed('{cluster}', default, tadetector_local, rand());
diff --git a/build/charts/theia/templates/theia-cli/clusterrole.yaml b/build/charts/theia/templates/theia-cli/clusterrole.yaml
@@ -35,4 +35,13 @@ rules:
       - supportbundles/download
     verbs:
       - get
+  - apiGroups:
+      - anomalydetector.theia.antrea.io
+    resources:
+      - throughputanomalydetectors
+    verbs:
+      - get
+      - list
+      - create
+      - delete
 {{- end }}
diff --git a/build/charts/theia/templates/theia-manager/clusterrole.yaml b/build/charts/theia/templates/theia-manager/clusterrole.yaml
@@ -44,10 +44,10 @@ rules:
     resources: ["configmaps"]
     verbs: ["get", "list", "watch"]
   - apiGroups: ["crd.theia.antrea.io"]
-    resources: ["networkpolicyrecommendations", "recommendednetworkpolicies"]
+    resources: ["networkpolicyrecommendations", "recommendednetworkpolicies", "throughputanomalydetectors"]
     verbs: ["get", "list", "watch", "create", "delete"]
   - apiGroups: ["crd.theia.antrea.io"]
-    resources: ["networkpolicyrecommendations/status"]
+    resources: ["networkpolicyrecommendations/status", "throughputanomalydetectors/status"]
     verbs: ["update"]
   - apiGroups: [ "" ]
     resources: [ "pods" ]