[k8s] Make devbin function for manually scaling dev namespaces (#14025)

Resolves #14020.

I'll manually delete the scale up cron jobs from the dev namespaces once
this goes in.

build.yaml

@@ -3820,7 +3820,7 @@ steps:
       - test_hail_python_service_backend_azure
       - cancel_all_running_test_batches
   - kind: runImage
-    name: setup_dev_namespace_autoscalers
+    name: setup_dev_namespace_autoscaledown
     resources:
       memory: standard
       cpu: '0.25'
@@ -3857,31 +3857,6 @@ steps:
                   - -c
                   - set -ex ; kubectl scale deployments --all -n {{ user["username"] }} --replicas=0 && kubectl scale statefulsets --all -n {{ user["username"] }} --replicas=0
                 restartPolicy: OnFailure
-      ---
-      apiVersion: batch/v1
-      kind: CronJob
-      metadata:
-        name: dev-namespace-scaleup-{{ user["username"] }}
-        namespace: {{ user["username"] }}
-      spec:
-        schedule: "0 13 * * 1,2,3,4,5"  # Weekdays at 1pm UTC
-        concurrencyPolicy: Forbid
-        successfulJobsHistoryLimit: 0
-        failedJobsHistoryLimit: 1
-        jobTemplate:
-          spec:
-            template:
-              spec:
-                serviceAccountName: admin
-                containers:
-                - name: dev-namespace-daytime-autoscaler
-                  image: bitnami/kubectl:latest
-                  command:
-                  - /bin/sh
-                  - -c
-                  - set -ex ; kubectl scale deployments --all -n {{ user["username"] }} --replicas=1 && kubectl scale statefulsets --all -n {{ user["username"] }} --replicas=1
-                restartPolicy: OnFailure
-      ---
       EOF
 
       kubectl apply -f the.yaml

dev-docs/services/services-development-faq.md

@@ -29,3 +29,16 @@ The next dev deploy will set up a new database:
 ```bash
 hailctl dev deploy -b <github_username>/hail:<your branch> -s deploy_batch,add_developers
 ```
+
+#### My namespace scaled down overnight. How do I get them back?
+
+There is a Kubernetes `CronJob` that runs in the evenings that scales down
+development namespaces. To scale back up, you need to use `kubectl scale`,
+or you can use the devbin function `kscale`, like
+
+```bash
+kscale <your_namespace> up
+```
+
+If you want to manually scale down your namespace when not using it, run
+`kscale <your_namespace> down`.

devbin/functions.sh

@@ -135,6 +135,35 @@ knodes() {
     done
 }
 
+kscale() {
+    local usage="Use like kscale <namespace> down or kscale <namespace> up"
+    if [ -z "$1" ]; then
+        echo $usage
+        return
+    fi
+    local namespace=$1
+    if [[ "$namespace" == "default" ]]; then
+        echo "ERROR: kscale should only be used for dev namespaces"
+        return
+    fi
+
+    case "$2" in
+        up)
+            local replicas=1
+            ;;
+        down)
+            local replicas=0
+            ;;
+        *)
+            echo $usage
+            return
+            ;;
+    esac
+
+    kubectl -n $namespace scale deployments --all --replicas=$replicas
+    kubectl -n $namespace scale statefulsets --all --replicas=$replicas
+}
+
 download-secret() {
     # download-secret secret-name namespace
     #