Adding Throughput Anomaly detector files and CLI support

This PR is resonsible for: 1. Adding throughput Anomaly detector files to Theia. 2. Adding CLI support. CLI support for get, list and status is present, commands are yet to be implemented 3. Adding CRD for the anomaly detector. 4. Adding e2e Test for all three algos of TAD. 5. Adding Unit tests 6. Using same spark operator for all spark jobs NOTE: TODO Add stat as an API Signed-off-by: Tushar Tathgur <tathgurt@tathgurtFLVDL.vmware.com>
antrea-io · Feb 22, 2023 · 1628e5c · 1628e5c
1 parent f2328e7
commit 1628e5c
Show file tree

Hide file tree

Showing 75 changed files with 6,059 additions and 233 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -137,3 +137,34 @@ jobs:
       run: |
         echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
         docker push antrea/theia-manager:latest
+  
+  check-anomaly-detection-changes:
+    name: Check whether anomaly-detection image needs to be built based on diff
+    runs-on: [ubuntu-latest]
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+    - uses: antrea-io/has-changes@v2
+      id: check_diff
+      with:
+        paths: plugins/anomaly-detection/* build/images/Dockerfile.anomaly-detection.ubuntu
+    outputs:
+      has_changes: ${{ steps.check_diff.outputs.has_changes }}
+
+  build-anomaly-detection:
+    needs: check-anomaly-detection-changes
+    if: ${{ needs.check-anomaly-detection-changes.outputs.has_changes == 'yes' || github.event_name == 'push' }}
+    runs-on: [ubuntu-latest]
+    steps:
+    - uses: actions/checkout@v2
+    - name: Build anomaly-detection Docker image
+      run: make anomaly-detection
+    - name: Push anomaly-detection Docker image to registry
+      if: ${{ github.repository == 'antrea-io/theia' && github.event_name == 'push' && github.ref == 'refs/heads/main' }}
+      env:
+        DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+        DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+      run: |
+        echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
+        docker push antrea/theia-anomaly-detection:latest
diff --git a/.github/workflows/build_tag.yml b/.github/workflows/build_tag.yml
@@ -65,6 +65,21 @@ jobs:
         echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
         docker push antrea/theia-policy-recommendation:"${VERSION}"
 
+  build-anomaly-detection:
+    runs-on: [ubuntu-latest]
+    needs: get-version
+    steps:
+    - uses: actions/checkout@v2
+    - name: Build anomaly-detection Docker image and push to registry
+      env:
+        DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+        DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+        VERSION: ${{ needs.get-version.outputs.version }}
+      run: |
+        make anomaly-detection
+        echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
+        docker push antrea/theia-anomaly-detection:"${VERSION}"
+
   build-theia-manager:
     runs-on: [ubuntu-latest]
     needs: get-version

diff --git a/.github/workflows/kind.yml b/.github/workflows/kind.yml
@@ -45,6 +45,23 @@ jobs:
         name: policy-recommendation
         path: policy-recommendation.tar
         retention-days: 1 # minimum value, in case artifact deletion by 'artifact-cleanup' job fails
+
+  build-anomaly-detection-image:
+    name: Build Anomaly Detection image to be used for Kind e2e tests
+    needs: check-changes
+    if: ${{ needs.check-changes.outputs.has_changes == 'yes' }}
+    runs-on: [ ubuntu-latest ]
+    steps:
+    - uses: actions/checkout@v3
+    - run: make anomaly-detection
+    - name: Save Anomaly Detection image to tarball
+      run: docker save -o anomaly-detection.tar antrea/theia-anomaly-detection
+    - name: Upload Anomaly Detection image for subsequent jobs
+      uses: actions/upload-artifact@v3
+      with:
+        name: anomaly-detection
+        path: anomaly-detection.tar
+        retention-days: 1 # minimum value, in case artifact deletion by 'artifact-cleanup' job fails
 
   build-clickhouse-monitor-image:
     name: Build ClickHouse monitor image to be used for Kind e2e tests
@@ -101,6 +118,7 @@ jobs:
     name: E2e tests on a Kind cluster on Linux
     needs: 
       - build-policy-recommendation-image
+      - build-anomaly-detection-image
       - build-clickhouse-monitor-image
       - build-clickhouse-server-image
       - build-theia-manager-image
@@ -123,6 +141,10 @@ jobs:
         uses: actions/download-artifact@v3
         with:
           name: policy-recommendation
+      - name: Download Anomaly Detection images from previous jobs
+        uses: actions/download-artifact@v3
+        with:
+          name: anomaly-detection
       - name: Download ClickHouse monitor images from previous jobs
         uses: actions/download-artifact@v3
         with:
@@ -139,6 +161,8 @@ jobs:
         run:  |
           docker load -i policy-recommendation.tar
           docker tag antrea/theia-policy-recommendation:latest projects.registry.vmware.com/antrea/theia-policy-recommendation:latest
+          docker load -i anomaly-detection.tar
+          docker tag antrea/theia-anomaly-detection:latest projects.registry.vmware.com/antrea/theia-anomaly-detection:latest
           docker load -i clickhouse-monitor.tar
           docker tag antrea/theia-clickhouse-monitor:latest projects.registry.vmware.com/antrea/theia-clickhouse-monitor:latest
           docker load -i clickhouse-server.tar
@@ -229,6 +253,7 @@ jobs:
     name: Delete uploaded images
     needs:
     - build-policy-recommendation-image
+    - build-anomaly-detection-image
     - build-clickhouse-monitor-image
     - build-clickhouse-server-image
     - build-theia-manager-image
@@ -241,6 +266,11 @@ jobs:
       uses: geekyeggo/delete-artifact@v1
       with:
         name: policy-recommendation
+    - name: Delete anomaly-detection
+      if: ${{ needs.build-anomaly-detection-image.result == 'success' }}
+      uses: geekyeggo/delete-artifact@v1
+      with:
+        name: anomaly-detection
     - name: Delete clickhouse-monitor
       if: ${{ needs.build-clickhouse-monitor-image.result == 'success' }}
       uses: geekyeggo/delete-artifact@v1

diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml
@@ -94,6 +94,7 @@ jobs:
           python -m pip install --upgrade pip
           python -m pip install pytest-cov
           python -m pip install -r plugins/policy-recommendation/requirements.txt
+          python -m pip install -r plugins/anomaly-detection/requirements.txt
       - name: Run unit tests
         run: |
           cd plugins/policy-recommendation/

diff --git a/Makefile b/Makefile
@@ -216,6 +216,14 @@ policy-recommendation:
 	docker tag antrea/theia-policy-recommendation:$(DOCKER_IMG_VERSION) projects.registry.vmware.com/antrea/theia-policy-recommendation
 	docker tag antrea/theia-policy-recommendation:$(DOCKER_IMG_VERSION) projects.registry.vmware.com/antrea/theia-policy-recommendation:$(DOCKER_IMG_VERSION)
 
+.PHONY: anomaly-detection
+anomaly-detection:
+	@echo "===> Building antrea/theia-anomaly-detection Docker image <==="
+	docker build --pull -t antrea/theia-anomaly-detection:$(DOCKER_IMG_VERSION) -f build/images/Dockerfile.anomaly-detection.ubuntu .
+	docker tag antrea/theia-anomaly-detection:$(DOCKER_IMG_VERSION) antrea/theia-anomaly-detection
+	docker tag antrea/theia-anomaly-detection:$(DOCKER_IMG_VERSION) projects.registry.vmware.com/antrea/theia-anomaly-detection
+	docker tag antrea/theia-anomaly-detection:$(DOCKER_IMG_VERSION) projects.registry.vmware.com/antrea/theia-anomaly-detection:$(DOCKER_IMG_VERSION)
+
 THEIA_BINARIES := theia-darwin theia-linux theia-windows
 $(THEIA_BINARIES): theia-%:
 	@GOOS=$* $(GO) build -o $(BINDIR)/$@ $(GOFLAGS) -ldflags '$(LDFLAGS)' antrea.io/theia/pkg/theia
@@ -231,3 +239,4 @@ theia: $(THEIA_BINARIES)
 .PHONY: theia-release
 theia-release:
 	@$(GO) build -o $(BINDIR)/$(THEIA_BINARY_NAME) $(GOFLAGS) -ldflags '-s -w $(LDFLAGS)' antrea.io/theia/pkg/theia
+
diff --git a/README.md b/README.md
@@ -20,6 +20,10 @@ NetworkPolicy configuration to secure Kubernetes network and applications.
 Please refer to the [NetworkPolicy Recommendation](docs/networkpolicy-recommendation.md)
 user guide to learn more.
 
+Theia also provides throughput anomaly detection, it can find the anomalies in the network, and report them to the user.
+Please refer to the [Throughput Anomaly Detection](docs/throughput-anomaly-detection.md) user
+guide to learn more.
+
 ## Contributing
 
 The Antrea community welcomes new contributors. We are waiting for your PRs!

diff --git a/build/charts/theia/README.md b/build/charts/theia/README.md
@@ -73,9 +73,9 @@ Kubernetes: `>= 1.16.0-0`
 | grafana.storage.createPersistentVolume.type | string | `"HostPath"` | Type of PersistentVolume. Can be set to "HostPath", "Local" or "NFS". Please set this value to use a PersistentVolume created by Theia. |
 | grafana.storage.persistentVolumeClaimSpec | object | `{}` | Specification for PersistentVolumeClaim. This is ignored if createPersistentVolume.type is non-empty. To use a custom PersistentVolume, please set storageClassName: "" volumeName: "<my-pv>". To dynamically provision a PersistentVolume, please set storageClassName: "<my-storage-class>". HostPath storage is used if both createPersistentVolume.type and persistentVolumeClaimSpec are empty. |
 | grafana.storage.size | string | `"1Gi"` | Grafana storage size. It is used to store Grafana configuration files. Can be a plain integer or as a fixed-point number using one of these quantity suffixes: E, P, T, G, M, K. Or the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki. |
-| sparkOperator.enable | bool | `false` | Determine whether to install Spark Operator. It is required to run Network Policy Recommendation jobs. |
+| sparkOperator.enable | bool | `true` | Determine whether to install Spark Operator. It is required to run Network Policy Recommendation jobs. |
 | sparkOperator.image | object | `{"pullPolicy":"IfNotPresent","repository":"projects.registry.vmware.com/antrea/theia-spark-operator","tag":"v1beta2-1.3.3-3.1.1"}` | Container image used by Spark Operator. |
-| sparkOperator.name | string | `"policy-recommendation"` | Name of Spark Operator. |
+| sparkOperator.name | string | `"theia"` | Name of Spark Operator. |
 | theiaManager.apiServer.apiPort | int | `11347` | The port for the Theia Manager APIServer to serve on. |
 | theiaManager.apiServer.selfSignedCert | bool | `true` | Indicates whether to use auto-generated self-signed TLS certificates. If false, a Secret named "theia-manager-tls" must be provided with the following keys: ca.crt, tls.crt, tls.key. |
 | theiaManager.apiServer.tlsCipherSuites | string | `""` | Comma-separated list of cipher suites that will be used by the Theia Manager APIservers. If empty, the default Go Cipher Suites will be used. |

diff --git a/build/charts/theia/crds/anomaly-detector-crd.yaml b/build/charts/theia/crds/anomaly-detector-crd.yaml
@@ -0,0 +1,74 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: throughputanomalydetectors.crd.theia.antrea.io
+  labels:
+    app: theia
+spec:
+  group: crd.theia.antrea.io
+  versions:
+    - name: v1alpha1
+      served: true
+      storage: true
+      schema:
+        openAPIV3Schema:
+          type: object
+          required:
+            - spec
+          properties:
+            spec:
+              type: object
+              required:
+                - jobType
+              properties:
+                jobType:
+                  type: string
+                startInterval:
+                  type: string
+                  format: datetime
+                endInterval:
+                  type: string
+                  format: datetime
+                executorInstances:
+                  type: integer
+                driverCoreRequest:
+                  type: string
+                driverMemory:
+                  type: string
+                executorCoreRequest:
+                  type: string
+                executorMemory:
+                  type: string
+            status:
+              type: object
+              properties:
+                state:
+                  type: string
+                sparkApplication:
+                  type: string
+                completedStages:
+                  type: integer
+                totalStages:
+                  type: integer
+                startTime:
+                  type: string
+                  format: datetime
+                endTime:
+                  type: string
+                  format: datetime
+                errorMsg:
+                  type: string
+      additionalPrinterColumns:
+        - description: Current state of the job
+          jsonPath: .status.state
+          name: State
+          type: string
+      subresources:
+        status: {}
+  scope: Namespaced
+  names:
+    plural: throughputanomalydetectors
+    singular: throughputanomalydetector
+    kind: ThroughputAnomalyDetector
+    shortNames:
+      - tad
diff --git a/build/charts/theia/provisioning/datasources/create_table.sh b/build/charts/theia/provisioning/datasources/create_table.sh
@@ -285,6 +285,24 @@ clickhouse client -n -h 127.0.0.1 <<-EOSQL
     ) engine=ReplicatedMergeTree('/clickhouse/tables/{shard}/{database}/{table}', '{replica}')
     ORDER BY (timeCreated);
 
+    --Create a table to store the Throughput Anomaly Detector results
+    CREATE TABLE IF NOT EXISTS tadetector_local (
+        sourceIP String,
+        sourceTransportPort UInt16,
+        destinationIP String,
+        destinationTransportPort UInt16,
+        protocolIdentifier UInt16,
+        flowStartSeconds DateTime,
+        flowEndSeconds String,
+        throughputStandardDeviation String,
+        algoType String,
+        algoCalc String,
+        Throughputs String,
+        anomaly String,
+        id String
+    ) engine=ReplicatedMergeTree('/clickhouse/tables/{shard}/{database}/{table}', '{replica}')
+    ORDER BY (flowStartSeconds);
+
     --Create distributed tables for cluster
     CREATE TABLE IF NOT EXISTS flows AS flows_local
     engine=Distributed('{cluster}', default, flows_local, rand());
@@ -300,5 +318,9 @@ clickhouse client -n -h 127.0.0.1 <<-EOSQL
 
     CREATE TABLE IF NOT EXISTS recommendations AS recommendations_local
     engine=Distributed('{cluster}', default, recommendations_local, rand());
+
+    CREATE TABLE IF NOT EXISTS tadetector AS tadetector_local
+    engine=Distributed('{cluster}', default, tadetector_local, rand());
+
 EOSQL
 }
diff --git a/build/charts/theia/provisioning/datasources/migrators/000004_0-4-0.down.sql b/build/charts/theia/provisioning/datasources/migrators/000004_0-4-0.down.sql
@@ -0,0 +1,2 @@
+--Drop table
+DROP tadetector_local
diff --git a/build/charts/theia/provisioning/datasources/migrators/000004_0-4-0.up.sql b/build/charts/theia/provisioning/datasources/migrators/000004_0-4-0.up.sql
@@ -0,0 +1,24 @@
+--Create a table to store the Throughput Anomaly Detector results
+    CREATE TABLE IF NOT EXISTS tadetector_local (
+        sourceIP String,
+        sourceTransportPort UInt16,
+        destinationIP String,
+        destinationTransportPort UInt16,
+        protocolIdentifier UInt16,
+        flowStartSeconds DateTime,
+        flowEndSeconds String,
+        throughputStandardDeviation String,
+        algoType String,
+        algoCalc String,
+        Throughputs String,
+        anomaly String,
+        id String
+    ) engine=ReplicatedMergeTree('/clickhouse/tables/{shard}/{database}/{table}', '{replica}')
+    ORDER BY (flowStartSeconds);
+
+--Move data from old table and drop the old table
+INSERT INTO tadetector_local SELECT * FROM tadetector;
+DROP TABLE tadetector;
+
+CREATE TABLE IF NOT EXISTS tadetector AS tadetector_local
+    engine=Distributed('{cluster}', default, tadetector_local, rand());
diff --git a/build/charts/theia/templates/theia-cli/clusterrole.yaml b/build/charts/theia/templates/theia-cli/clusterrole.yaml
@@ -35,4 +35,13 @@ rules:
       - supportbundles/download
     verbs:
       - get
+  - apiGroups:
+      - anomalydetector.theia.antrea.io
+    resources:
+      - throughputanomalydetectors
+    verbs:
+      - get
+      - list
+      - create
+      - delete
 {{- end }}
diff --git a/build/charts/theia/templates/theia-manager/clusterrole.yaml b/build/charts/theia/templates/theia-manager/clusterrole.yaml
@@ -44,10 +44,10 @@ rules:
     resources: ["configmaps"]
     verbs: ["get", "list", "watch"]
   - apiGroups: ["crd.theia.antrea.io"]
-    resources: ["networkpolicyrecommendations", "recommendednetworkpolicies"]
+    resources: ["networkpolicyrecommendations", "recommendednetworkpolicies", "throughputanomalydetectors"]
     verbs: ["get", "list", "watch", "create", "delete"]
   - apiGroups: ["crd.theia.antrea.io"]
-    resources: ["networkpolicyrecommendations/status"]
+    resources: ["networkpolicyrecommendations/status", "throughputanomalydetectors/status"]
     verbs: ["update"]
   - apiGroups: [ "" ]
     resources: [ "pods" ]

diff --git a/build/charts/theia/values.yaml b/build/charts/theia/values.yaml
@@ -235,9 +235,9 @@ grafana:
 sparkOperator:
   # -- Determine whether to install Spark Operator. It is required to run Network
   # Policy Recommendation jobs.
-  enable: false
+  enable: true
   # -- Name of Spark Operator.
-  name: "policy-recommendation"
+  name: "theia"
   # -- Container image used by Spark Operator.
   image: 
     repository: "projects.registry.vmware.com/antrea/theia-spark-operator"

diff --git a/build/images/Dockerfile.anomaly-detection.ubuntu b/build/images/Dockerfile.anomaly-detection.ubuntu
@@ -0,0 +1,17 @@
+FROM gcr.io/spark-operator/spark-py:v3.1.1
+
+LABEL maintainer="Antrea <projectantrea-dev@googlegroups.com>"
+LABEL description="A docker image to deploy Throughput Anomaly Detection Spark job."
+
+WORKDIR /opt/spark/work-dir
+USER root
+
+RUN apt-get --allow-releaseinfo-change update && \
+    apt-get install -y --no-install-recommends wget ca-certificates && \
+    wget https://github.com/ClickHouse/clickhouse-jdbc/releases/download/v0.3.1/clickhouse-jdbc-0.3.1.jar -P /opt/spark/jars/
+
+COPY plugins/anomaly-detection/AnomalyDetection.py /opt/spark/work-dir/AnomalyDetection.py
+COPY plugins/anomaly-detection/requirements.txt /opt/spark/work-dir/requirements.txt
+
+RUN pip3 install --upgrade pip && \
+    pip3 install -r  /opt/spark/work-dir/requirements.txt