add temp Job to delete old monitoring stack

Signed-off-by: Vedant Mahabaleshwarkar <vmahabal@redhat.com>

config/overlays/odh/kustomization.yaml

@@ -2,6 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - ../../default
+  - remove-deprecated-monitoring.yaml
 
 patchesStrategicMerge:
   - odh_model_controller_manager_patch.yaml

config/overlays/odh/remove-deprecated-monitoring.yaml

@@ -0,0 +1,71 @@
+
+kind: Job
+apiVersion: batch/v1
+metadata:
+  name: remove-deprecated-monitoring
+  namespace: redhat-ods-applications
+spec:
+  template:
+    spec:
+      restartPolicy: OnFailure
+      serviceAccountName: odh-model-controller
+      terminationGracePeriodSeconds: 30
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 999
+      containers:
+        - name: oc-cli
+          image: >-
+            registry.redhat.io/openshift4/ose-cli@sha256:25fef269ac6e7491cb8340119a9b473acbeb53bc6970ad029fdaae59c3d0ca61
+          command: ["/bin/bash", "-c", "--"]
+          args:
+          - |
+            #check if namespace redhat-ods-monitoring exists 
+            monitoringnamespaceexists=$(oc get project redhat-ods-monitoring | grep redhat-ods-monitoring || echo "false")
+            if [[ $monitoringnamespaceexists != "false" ]]; then
+
+              # Initialize an array to store errors
+              errors=()
+
+              #delete prometheus operator deployment
+              prometheusoperatordeploymentexists=$(oc get deployments -o name -n redhat-ods-monitoring | grep rhods-prometheus-operator || echo "false")
+              if [[ $prometheusoperatordeploymentexists != "false" ]]; then
+                echo "deleting prometheus operator deployment for deprecated model monitoring stack"
+                oc delete deployment rhods-prometheus-operator -n redhat-ods-monitoring || errors+=("Failed to delete prometheus operator deployment")
+              fi 
+
+              #delete prometheus statefulset
+              prometheusstatefulsetexists=$(oc get statefulsets -o name -n redhat-ods-monitoring | grep prometheus-rhods-model-monitoring || echo "false")
+              if [[ $prometheusstatefulsetexists != "false" ]]; then
+                echo "deleting prometheus operator deployment for deprecated model monitoring stack"
+                oc delete statefulset prometheus-rhods-model-monitoring -n redhat-ods-monitoring || errors+=("Failed to delete prometheus statefulset")
+              fi 
+
+              #delete servicemonitor
+              federatedmetricsservicemonitorexists=$(oc get servicemonitors -o name -n redhat-ods-monitoring | grep modelmesh-federated-metrics || echo "false")
+              if [[ $federatedmetricsservicemonitorexists != "false" ]]; then
+                echo "deleting prometheus operator deployment for deprecated model monitoring stack"
+                oc delete servicemonitor modelmesh-federated-metrics -n redhat-ods-monitoring || errors+=("Failed to delete servicemonitor")
+              fi
+
+              # Check if there were any errors during deletion
+              if [ ${#errors[@]} -gt 0 ]; then
+                echo "Errors occurred during deletion:"
+                for error in "${errors[@]}"; do
+                  echo "- $error"
+                done
+                exit 1  # Exit with an error status
+              fi
+            fi
+            # if namespace redhat-ods-monitoring does not exist, do nothing  
+          resources:
+            limits:
+              memory: 800Mi
+            requests:
+              memory: 400Mi
+          terminationMessagePath: /dev/termination-log
+          terminationMessagePolicy: File
+          imagePullPolicy: IfNotPresent
+      serviceAccount: rhods-dashboard
+      dnsPolicy: ClusterFirst
+  backoffLimit: 4

config/rbac/role.yaml

@@ -22,6 +22,22 @@ rules:
   - patch
   - update
   - watch
+#revert before 2.7 https://issues.redhat.com/browse/RHOAIENG-293
+- apiGroups:
+  - apps
+  resources:
+  - deployments
+  - statefulsets
+  verbs:
+  - get
+  - list
+  - delete
+- apiGroups:
+  - projects.openshift.io
+  resources:
+  - projects
+  verbs:
+  - get
 - apiGroups:
   - maistra.io
   resources: