Adding Throughput Anomaly detector files and CLI support

This PR is resonsible for: 1. Adding throughput Anomaly detector files to Theia. 2. Adding CLI support. CLI support for get, list and status is present, commands are yet to be implemented 3. Adding CRD for the anomaly detector. Signed-off-by: Tushar Tathgur <tathgurt@tathgurtFLVDL.vmware.com>
antrea-io · Jan 25, 2023 · 0296f01 · 0296f01
1 parent b1ce1cd
commit 0296f01
Show file tree

Hide file tree

Showing 48 changed files with 4,244 additions and 34 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -137,3 +137,34 @@ jobs:
       run: |
         echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
         docker push antrea/theia-manager:latest
+  
+  check-anomaly-detection-changes:
+    name: Check whether anomaly-detection image needs to be built based on diff
+    runs-on: [ubuntu-latest]
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+    - uses: antrea-io/has-changes@v2
+      id: check_diff
+      with:
+        paths: plugins/anomaly-detection/* build/images/Dockerfile.anomaly-detection.ubuntu
+    outputs:
+      has_changes: ${{ steps.check_diff.outputs.has_changes }}
+
+  build-anomaly-detection:
+    needs: check-anomaly-detection-changes
+    if: ${{ needs.check-anomaly-detection-changes.outputs.has_changes == 'yes' || github.event_name == 'push' }}
+    runs-on: [ubuntu-latest]
+    steps:
+    - uses: actions/checkout@v2
+    - name: Build anomaly-detection Docker image
+      run: make anomaly-detection
+    - name: Push anomaly-detection Docker image to registry
+      if: ${{ github.repository == 'antrea-io/theia' && github.event_name == 'push' && github.ref == 'refs/heads/main' }}
+      env:
+        DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+        DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+      run: |
+        echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
+        docker push antrea/theia-anomaly-detection:latest
diff --git a/.github/workflows/build_tag.yml b/.github/workflows/build_tag.yml
@@ -65,6 +65,21 @@ jobs:
         echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
         docker push antrea/theia-policy-recommendation:"${VERSION}"
 
+  build-anomaly-detection:
+    runs-on: [ubuntu-latest]
+    needs: get-version
+    steps:
+    - uses: actions/checkout@v2
+    - name: Build anomaly-detection Docker image and push to registry
+      env:
+        DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+        DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+        VERSION: ${{ needs.get-version.outputs.version }}
+      run: |
+        make anomaly-detection
+        echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
+        docker push antrea/theia-anomaly-detection:"${VERSION}"
+
   build-theia-manager:
     runs-on: [ubuntu-latest]
     needs: get-version

diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml
@@ -94,6 +94,7 @@ jobs:
           python -m pip install --upgrade pip
           python -m pip install pytest-cov
           python -m pip install -r plugins/policy-recommendation/requirements.txt
+          python -m pip install -r plugins/anomaly-detection/requirements.txt
       - name: Run unit tests
         run: |
           cd plugins/policy-recommendation/

diff --git a/Makefile b/Makefile
@@ -209,6 +209,14 @@ policy-recommendation:
 	docker tag antrea/theia-policy-recommendation:$(DOCKER_IMG_VERSION) projects.registry.vmware.com/antrea/theia-policy-recommendation
 	docker tag antrea/theia-policy-recommendation:$(DOCKER_IMG_VERSION) projects.registry.vmware.com/antrea/theia-policy-recommendation:$(DOCKER_IMG_VERSION)
 
+.PHONY: anomaly-detection
+anomaly-detection:
+	@echo "===> Building antrea/theia-anomaly-detection Docker image <==="
+	docker build --pull -t antrea/theia-anomaly-detection:$(DOCKER_IMG_VERSION) -f build/images/Dockerfile.anomaly-detection.ubuntu .
+	docker tag antrea/theia-anomaly-detection:$(DOCKER_IMG_VERSION) antrea/theia-anomaly-detection
+	docker tag antrea/theia-anomaly-detection:$(DOCKER_IMG_VERSION) projects.registry.vmware.com/antrea/theia-anomaly-detection
+	docker tag antrea/theia-anomaly-detection:$(DOCKER_IMG_VERSION) projects.registry.vmware.com/antrea/theia-anomaly-detection:$(DOCKER_IMG_VERSION)
+
 THEIA_BINARIES := theia-darwin theia-linux theia-windows
 $(THEIA_BINARIES): theia-%:
 	@GOOS=$* $(GO) build -o $(BINDIR)/$@ $(GOFLAGS) -ldflags '$(LDFLAGS)' antrea.io/theia/pkg/theia
@@ -224,3 +232,4 @@ theia: $(THEIA_BINARIES)
 .PHONY: theia-release
 theia-release:
 	@$(GO) build -o $(BINDIR)/$(THEIA_BINARY_NAME) $(GOFLAGS) -ldflags '-s -w $(LDFLAGS)' antrea.io/theia/pkg/theia
+
diff --git a/build/charts/theia/crds/anomaly-detector-crds.yaml b/build/charts/theia/crds/anomaly-detector-crds.yaml
@@ -0,0 +1,74 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: throughputanomalydetectors.crd.theia.antrea.io
+  labels:
+    app: theia
+spec:
+  group: crd.theia.antrea.io
+  versions:
+    - name: v1alpha1
+      served: true
+      storage: true
+      schema:
+        openAPIV3Schema:
+          type: object
+          required:
+            - spec
+          properties:
+            spec:
+              type: object
+              required:
+                - jobType
+              properties:
+                jobType:
+                  type: string
+                startInterval:
+                  type: string
+                  format: datetime
+                endInterval:
+                  type: string
+                  format: datetime
+                executorInstances:
+                  type: integer
+                driverCoreRequest:
+                  type: string
+                driverMemory:
+                  type: string
+                executorCoreRequest:
+                  type: string
+                executorMemory:
+                  type: string
+            status:
+              type: object
+              properties:
+                state:
+                  type: string
+                sparkApplication:
+                  type: string
+                completedStages:
+                  type: integer
+                totalStages:
+                  type: integer
+                startTime:
+                  type: string
+                  format: datetime
+                endTime:
+                  type: string
+                  format: datetime
+                errorMsg:
+                  type: string
+      additionalPrinterColumns:
+        - description: Current state of the job
+          jsonPath: .status.state
+          name: State
+          type: string
+      subresources:
+        status: {}
+  scope: Namespaced
+  names:
+    plural: throughputanomalydetectors
+    singular: throughputanomalydetector
+    kind: ThroughputAnomalyDetector
+    shortNames:
+      - tad
diff --git a/build/charts/theia/provisioning/datasources/create_table.sh b/build/charts/theia/provisioning/datasources/create_table.sh
@@ -285,6 +285,23 @@ clickhouse client -n -h 127.0.0.1 <<-EOSQL
     ) engine=ReplicatedMergeTree('/clickhouse/tables/{shard}/{database}/{table}', '{replica}')
     ORDER BY (timeCreated);
 
+    --Create a table to store the Throughput Anomaly Detector results
+    CREATE TABLE IF NOT EXISTS tadetector_local (
+        sourceIP String,
+        sourceTransportPort UInt16,
+        destinationIP String,
+        destinationTransportPort UInt16,
+        protocolIdentifier UInt16,
+        flowStartSeconds DateTime,
+        throughputStandardDeviation String,
+        algoType String,
+        algoCalc String,
+        Throughputs String,
+        anomaly String,
+        id String
+    ) engine=ReplicatedMergeTree('/clickhouse/tables/{shard}/{database}/{table}', '{replica}')
+    ORDER BY (flowStartSeconds);
+
     --Create distributed tables for cluster
     CREATE TABLE IF NOT EXISTS flows AS flows_local
     engine=Distributed('{cluster}', default, flows_local, rand());
@@ -300,5 +317,9 @@ clickhouse client -n -h 127.0.0.1 <<-EOSQL
 
     CREATE TABLE IF NOT EXISTS recommendations AS recommendations_local
     engine=Distributed('{cluster}', default, recommendations_local, rand());
+
+    CREATE TABLE IF NOT EXISTS tadetector AS tadetector_local
+    engine=Distributed('{cluster}', default, tadetector_local, rand())
+
 EOSQL
 }
diff --git a/build/charts/theia/templates/theia-cli/clusterrole.yaml b/build/charts/theia/templates/theia-cli/clusterrole.yaml
@@ -35,4 +35,13 @@ rules:
       - supportbundles/download
     verbs:
       - get
+  - apiGroups:
+      - anomalydetector.theia.antrea.io
+    resources:
+      - throughputanomalydetectors
+    verbs:
+      - get
+      - list
+      - create
+      - delete
 {{- end }}
diff --git a/build/charts/theia/templates/theia-manager/clusterrole.yaml b/build/charts/theia/templates/theia-manager/clusterrole.yaml
@@ -44,10 +44,10 @@ rules:
     resources: ["configmaps"]
     verbs: ["get", "list", "watch"]
   - apiGroups: ["crd.theia.antrea.io"]
-    resources: ["networkpolicyrecommendations", "recommendednetworkpolicies"]
+    resources: ["networkpolicyrecommendations", "recommendednetworkpolicies", "throughputanomalydetectors"]
     verbs: ["get", "list", "watch", "create", "delete"]
   - apiGroups: ["crd.theia.antrea.io"]
-    resources: ["networkpolicyrecommendations/status"]
+    resources: ["networkpolicyrecommendations/status", "throughputanomalydetectors/status"]
     verbs: ["update"]
   - apiGroups: [ "" ]
     resources: [ "pods" ]

diff --git a/build/charts/theia/templates/theia-manager/deployment.yaml b/build/charts/theia/templates/theia-manager/deployment.yaml
@@ -28,6 +28,7 @@ spec:
             - --alsologtostderr
             - --log_file_max_size=100
             - --log_file_max_num=4
+            - --v=10
             {{- if .Values.theiaManager.logVerbosity }}
             - "--v={{ .Values.theiaManager.logVerbosity }}"
             {{- end }}

diff --git a/build/charts/theia/values.yaml b/build/charts/theia/values.yaml
@@ -235,17 +235,17 @@ grafana:
 sparkOperator:
   # -- Determine whether to install Spark Operator. It is required to run Network
   # Policy Recommendation jobs.
-  enable: false
+  enable: true
   # -- Name of Spark Operator.
-  name: "policy-recommendation"
+  name: "anomaly-detection"
   # -- Container image used by Spark Operator.
   image: 
     repository: "projects.registry.vmware.com/antrea/theia-spark-operator"
     pullPolicy: "IfNotPresent"
     tag: "v1beta2-1.3.3-3.1.1"
 theiaManager:
   # -- Determine whether to install Theia Manager.
-  enable: false
+  enable: true
   # -- Container image used by Theia Manager.
   image:
     repository: "projects.registry.vmware.com/antrea/theia-manager"

diff --git a/build/images/Dockerfile.anomaly-detection.ubuntu b/build/images/Dockerfile.anomaly-detection.ubuntu
@@ -0,0 +1,18 @@
+FROM gcr.io/spark-operator/spark-py:v3.1.1
+
+LABEL maintainer="Antrea <projectantrea-dev@googlegroups.com>"
+LABEL description="A docker image to deploy ThroughputAnomaly Detection Spark job."
+
+WORKDIR /opt/spark/work-dir
+USER root
+
+RUN apt-get --allow-releaseinfo-change update && \
+    apt-get install -y --no-install-recommends wget ca-certificates && \
+    wget https://github.com/ClickHouse/clickhouse-jdbc/releases/download/v0.3.1/clickhouse-jdbc-0.3.1.jar -P /opt/spark/jars/
+
+COPY plugins/anomaly-detection/AnomalyDetection.py /opt/spark/work-dir/AnomalyDetection.py
+COPY plugins/anomaly-detection/requirements.txt /opt/spark/work-dir/requirements.txt
+
+RUN pip3 install --upgrade pip && \
+    pip3 install -r  /opt/spark/work-dir/requirements.txt && \
+    pip3 install scikit-learn