diff --git a/build.yaml b/build.yaml index f2188666d27..ae51f3cee05 100644 --- a/build.yaml +++ b/build.yaml @@ -3820,7 +3820,7 @@ steps: - test_hail_python_service_backend_azure - cancel_all_running_test_batches - kind: runImage - name: setup_dev_namespace_autoscalers + name: setup_dev_namespace_autoscaledown resources: memory: standard cpu: '0.25' @@ -3857,31 +3857,6 @@ steps: - -c - set -ex ; kubectl scale deployments --all -n {{ user["username"] }} --replicas=0 && kubectl scale statefulsets --all -n {{ user["username"] }} --replicas=0 restartPolicy: OnFailure - --- - apiVersion: batch/v1 - kind: CronJob - metadata: - name: dev-namespace-scaleup-{{ user["username"] }} - namespace: {{ user["username"] }} - spec: - schedule: "0 13 * * 1,2,3,4,5" # Weekdays at 1pm UTC - concurrencyPolicy: Forbid - successfulJobsHistoryLimit: 0 - failedJobsHistoryLimit: 1 - jobTemplate: - spec: - template: - spec: - serviceAccountName: admin - containers: - - name: dev-namespace-daytime-autoscaler - image: bitnami/kubectl:latest - command: - - /bin/sh - - -c - - set -ex ; kubectl scale deployments --all -n {{ user["username"] }} --replicas=1 && kubectl scale statefulsets --all -n {{ user["username"] }} --replicas=1 - restartPolicy: OnFailure - --- EOF kubectl apply -f the.yaml diff --git a/dev-docs/services/services-development-faq.md b/dev-docs/services/services-development-faq.md index e18a73013e2..58ab09f6fa0 100644 --- a/dev-docs/services/services-development-faq.md +++ b/dev-docs/services/services-development-faq.md @@ -29,3 +29,16 @@ The next dev deploy will set up a new database: ```bash hailctl dev deploy -b /hail: -s deploy_batch,add_developers ``` + +#### My namespace scaled down overnight. How do I get them back? + +There is a Kubernetes `CronJob` that runs in the evenings that scales down +development namespaces. To scale back up, you need to use `kubectl scale`, +or you can use the devbin function `kscale`, like + +```bash +kscale up +``` + +If you want to manually scale down your namespace when not using it, run +`kscale down`. diff --git a/devbin/functions.sh b/devbin/functions.sh index c90171506d5..44118f3a960 100644 --- a/devbin/functions.sh +++ b/devbin/functions.sh @@ -135,6 +135,35 @@ knodes() { done } +kscale() { + local usage="Use like kscale down or kscale up" + if [ -z "$1" ]; then + echo $usage + return + fi + local namespace=$1 + if [[ "$namespace" == "default" ]]; then + echo "ERROR: kscale should only be used for dev namespaces" + return + fi + + case "$2" in + up) + local replicas=1 + ;; + down) + local replicas=0 + ;; + *) + echo $usage + return + ;; + esac + + kubectl -n $namespace scale deployments --all --replicas=$replicas + kubectl -n $namespace scale statefulsets --all --replicas=$replicas +} + download-secret() { # download-secret secret-name namespace #