Initial work at deploying K8 related hooks as part of OnDemand distri…

…bution

hooks/hook.env.example

@@ -0,0 +1,9 @@
+K8S_USERNAME_PREFIX=""
+NAMESPACE_PREFIX=""
+NETWORK_POLICY_ALLOW_CIDR="127.0.0.1/32"
+IDP_ISSUER_URL="https://idp.example.com/auth/realms/main/protocol/openid-connect/token"
+CLIENT_ID="changeme"
+CLIENT_SECRET="changeme"
+# Set to empty string or remove if not using a private registry
+IMAGE_PULL_SECRET=private-docker-registry
+REGISTRY_DOCKER_CONFIG_JSON="/some/path/to/docker/config.json"

hooks/k8s-bootstrap-job-pod-reaper.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+ONDEMAND_USERNAME="$1"
+if [ "x${ONDEMAND_USERNAME}" = "x" ]; then
+  echo "Must specify username"
+  exit 1
+fi
+HOOK_ENV="$2"
+if [ "x${HOOK_ENV}" = "x" ]; then
+  echo "Must specify hook.env path"
+  exit 1
+fi
+
+set -e
+
+source $HOOK_ENV
+
+NAMESPACE="${NAMESPACE_PREFIX}${ONDEMAND_USERNAME}"
+
+TMPFILE=$(mktemp "/tmp/k8-bootstrap-job-pod-reaper-${ONDEMAND_USERNAME}.XXXXXX")
+cat > "$TMPFILE" <<EOF
+# allow job-pod-reaper to see this namespace
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: "$ONDEMAND_USERNAME-job-pod-reaper-rolebinding"
+  namespace: "$NAMESPACE"
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: job-pod-reaper
+subjects:
+- kind: ServiceAccount
+  name: job-pod-reaper
+  namespace: job-pod-reaper
+EOF
+
+export PATH=/usr/local/bin:/bin:$PATH
+kubectl apply -f "$TMPFILE"
+rm -f "$TMPFILE"

hooks/k8s-bootstrap-ondemand.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+ONDEMAND_USERNAME="$1"
+if [ "x${ONDEMAND_USERNAME}" = "x" ]; then
+  echo "Must specify username"
+  exit 1
+fi
+HOOK_ENV="$2"
+if [ "x${HOOK_ENV}" = "x" ]; then
+  echo "Must specify hook.env path"
+  exit 1
+fi
+
+set -e
+
+source $HOOK_ENV
+
+export PATH=/usr/local/bin:/bin:$PATH
+NAMESPACE="${NAMESPACE_PREFIX}${ONDEMAND_USERNAME}"
+
+SCRIPT=$(readlink -f "$0")
+BASEDIR=$(dirname "$SCRIPT")
+YAML_DIR=${BASEDIR}/k8-bootstrap
+
+NAMESPACE_TMPFILE=$(mktemp "/tmp/k8-bootstrap-namespace-${ONDEMAND_USERNAME}.XXXXXX")
+cat ${YAML_DIR}/namespace.yaml | envsubst > "$NAMESPACE_TMPFILE"
+
+NETWORK_POLICY_TMPFILE=$(mktemp "/tmp/k8-bootstrap-network-policy-${ONDEMAND_USERNAME}.XXXXXX")
+cat ${YAML_DIR}/network-policy.yaml | envsubst > "$NETWORK_POLICY_TMPFILE"
+
+ROLEBINDING_TMPFILE=$(mktemp "/tmp/k8-bootstrap-rolebinding-${ONDEMAND_USERNAME}.XXXXXX")
+cat ${YAML_DIR}/rolebinding.yaml | envsubst > "$ROLEBINDING_TMPFILE"
+
+TMPFILE=$(mktemp "/tmp/k8-ondemand-bootstrap-${ONDEMAND_USERNAME}.XXXXXX")
+cat "$NAMESPACE_TMPFILE" "$NETWORK_POLICY_TMPFILE" "$ROLEBINDING_TMPFILE" > "$TMPFILE"
+kubectl apply -f "$TMPFILE"
+rm -f "$NAMESPACE_TMPFILE" "$NETWORK_POLICY_TMPFILE" "$ROLEBINDING_TMPFILE" "$TMPFILE"
+
+if [ "x$IMAGE_PULL_SECRET" != "x" ]; then
+  kubectl create secret generic "$IMAGE_PULL_SECRET" --from-file=.dockerconfigjson="$REGISTRY_DOCKER_CONFIG_JSON" --type=kubernetes.io/dockerconfigjson -n "$NAMESPACE"
+  kubectl patch serviceaccount default -n "$NAMESPACE" -p "{\"imagePullSecrets\": [{\"name\": \"${IMAGE_PULL_SECRET}\"}]}"
+fi

hooks/k8s-bootstrap-pod-security-policy.sh

@@ -0,0 +1,111 @@
+#!/bin/bash
+
+ONDEMAND_USERNAME="$1"
+if [ "x${ONDEMAND_USERNAME}" = "x" ]; then
+  echo "Must specify username"
+  exit 1
+fi
+HOOK_ENV="$2"
+if [ "x${HOOK_ENV}" = "x" ]; then
+  echo "Must specify hook.env path"
+  exit 1
+fi
+
+set -e
+
+source $HOOK_ENV
+
+TMPFILE=$(mktemp "/tmp/k8-ondemand-bootstrap-${ONDEMAND_USERNAME}.XXXXXX")
+PASSWD=$(getent passwd "$ONDEMAND_USERNAME")
+if ! [[ "$PASSWD" =~ "${ONDEMAND_USERNAME}:"* ]]; then
+  echo "level=error msg=\"Unable to perform lookup of user\" user=$ONDEMAND_USERNAME"
+  exit 1
+fi
+USER_UID=$(echo "$PASSWD" | cut -d':' -f3)
+USER_GID=$(echo "$PASSWD" | cut -d':' -f4)
+NAMESPACE="${NAMESPACE_PREFIX}${ONDEMAND_USERNAME}"
+
+cat > "$TMPFILE" <<EOF
+---
+# the pod security policy such that you can only run pods as a single uid/gid pair
+apiVersion: policy/v1beta1
+kind: PodSecurityPolicy
+metadata:
+  name: "$ONDEMAND_USERNAME-psp"
+  annotations:
+    seccomp.security.alpha.kubernetes.io/allowedProfileNames: 'docker/default,runtime/default'
+    apparmor.security.beta.kubernetes.io/allowedProfileNames: 'runtime/default'
+    seccomp.security.alpha.kubernetes.io/defaultProfileName:  'runtime/default'
+    apparmor.security.beta.kubernetes.io/defaultProfileName:  'runtime/default'
+  labels:
+    app.kubernetes.io/name: open-ondemand
+spec:
+  # Required to prevent escalations to root.
+  privileged: false
+  allowPrivilegeEscalation: false
+  hostNetwork: false
+  hostIPC: false
+  hostPID: false
+  runAsUser:
+    # Run as your own uid
+    rule: 'MustRunAs'
+    ranges:
+    - min: $USER_UID
+      max: $USER_UID
+  runAsGroup:
+    # Run as your own gid
+    rule: 'MustRunAs'
+    ranges:
+    - min: $USER_GID
+      max: $USER_GID
+  seLinux:
+    # This policy assumes the nodes are using AppArmor rather than SELinux.
+    rule: 'RunAsAny'
+  supplementalGroups:
+    rule: 'MustRunAs'
+    ranges:
+    # Forbid adding the root group.
+    - min: 1
+      max: 65535
+  fsGroup:
+    rule: 'MustRunAs'
+    ranges:
+    # Forbid adding the root group.
+    - min: 1
+      max: 65535
+  volumes:
+  - '*'
+---
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: "$ONDEMAND_USERNAME-psp-role"
+  namespace: "$NAMESPACE"
+rules:
+- apiGroups: [ "extensions" ]
+  resources: [ "podsecuritypolicies" ]
+  resourceNames: [ "$ONDEMAND_USERNAME-psp" ]
+  verbs: [ "use" ]
+---
+# bind the users' pod security policy to the user
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: "$ONDEMAND_USERNAME-psp-rolebinding"
+  namespace: "$NAMESPACE"
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: "$ONDEMAND_USERNAME-psp-role"
+subjects:
+- apiGroup: rbac.authorization.k8s.io
+  kind: User
+  name: "$ONDEMAND_USERNAME"
+  namespace: "$NAMESPACE"
+EOF
+
+
+export PATH=/usr/local/bin:/bin:$PATH
+kubectl apply -f "$TMPFILE"
+
+rm -f "$TMPFILE"

hooks/k8s-bootstrap/namespace.yaml

@@ -0,0 +1,7 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: "$NAMESPACE"
+  labels:
+    app.kubernetes.io/name: open-ondemand

hooks/k8s-bootstrap/network-policy.yaml

@@ -0,0 +1,14 @@
+---
+kind: NetworkPolicy
+apiVersion: networking.k8s.io/v1
+metadata:
+  namespace: $NAMESPACE
+  name: deny-from-other-namespaces
+spec:
+  podSelector:
+    matchLabels:
+  ingress:
+  - from:
+    - podSelector: {}
+    - ipBlock:
+        cidr: ${NETWORK_POLICY_ALLOW_CIDR}

hooks/k8s-bootstrap/rolebinding.yaml

@@ -0,0 +1,30 @@
+---
+# give the service account the ood-initializer role
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  namespace: "$NAMESPACE"
+  name: "$ONDEMAND_USERNAME-ood-initializer"
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: "ood-initializer"
+subjects:
+  - kind: ServiceAccount
+    name: "default"
+    namespace: "$NAMESPACE"
+---
+# give the user the ood-user role
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  namespace: "$NAMESPACE"
+  name: "$ONDEMAND_USERNAME-ood-user"
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: "ood-user"
+subjects:
+  - kind: User
+    name: "$ONDEMAND_USERNAME"
+    namespace: "$NAMESPACE"

hooks/ondemand.yaml

@@ -0,0 +1,103 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: ondemand
+  labels:
+    name: ondemand
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: ondemand
+  namespace: ondemand
+rules:
+- apiGroups: [""]
+  resources:
+  - namespaces
+  - serviceaccounts
+  verbs:
+  - get
+  - create
+  - update
+  - patch
+- apiGroups:
+  - rbac.authorization.k8s.io
+  resources:
+  - rolebindings
+  - roles
+  verbs:
+  - get
+  - create
+  - update
+  - patch
+# Only added to enable cluster-info subcommand
+- apiGroups: [""]
+  resources:
+  - services
+  verbs:
+  - list
+# rules necessary to give these permissions to ondemand users
+- apiGroups: [""]
+  resources: ["services", "pods", "configmaps", "secrets", "pods/log"]
+  verbs: ["get", "watch", "list", "create", "update", "patch", "delete"]
+- apiGroups:
+  - networking.k8s.io
+  resources:
+  - networkpolicies
+  verbs:
+  - get
+  - create
+  - update
+  - patch
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: ondemand
+  namespace: ondemand
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: ondemand
+  namespace: ondemand
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: ondemand
+subjects:
+- kind: ServiceAccount
+  name: ondemand
+  namespace: ondemand
+---
+# ood-initializer is the role given to service accounts to initialize and 
+# configure ood configmaps and services within init containers.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: ood-initializer
+  namespace: ondemand
+rules:
+- apiGroups: [""]
+  resources: ["secrets"]
+  verbs: ["get", "create", "update", "patch"]
+- apiGroups: [""]
+  resources: ["services", "pods"]
+  verbs: ["get"]
+- apiGroups: [""]
+  resources: ["configmaps"]
+  verbs: ["update", "get", "patch"]
+---
+# ood-user role allows users to manipulate their services/pods/configmaps/secrets and logs
+# should be applied as a RoleBinding not ClusterRoleBinding so they're limited to their 
+# own namespace
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: ood-user
+  namespace: ondemand
+rules:
+- apiGroups: [""]
+  resources: ["services", "pods", "configmaps", "secrets", "pods/log"]
+  verbs: ["get", "watch", "list", "create", "update", "patch", "delete"]

hooks/set-k8s-creds.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+ONDEMAND_USERNAME="$1"
+if [ "x${ONDEMAND_USERNAME}" = "x" ]; then
+  echo "Must specify username"
+  exit 1
+fi
+HOOK_ENV="$2"
+if [ "x${HOOK_ENV}" = "x" ]; then
+  echo "Must specify hook.env path"
+  exit 1
+fi
+
+source $HOOK_ENV
+
+K8S_USERNAME="${K8S_USERNAME_PREFIX}${ONDEMAND_USERNAME}"
+
+# we use pass ACCESS_TOKEN into the id-token arg. That's OK, it works and refreshes.
+sudo -u "$ONDEMAND_USERNAME" kubectl config set-credentials "$K8S_USERNAME" \
+   --auth-provider=oidc \
+   --auth-provider-arg=idp-issuer-url="$IDP_ISSUER_URL" \
+   --auth-provider-arg=client-id="$CLIENT_ID" \
+   --auth-provider-arg=client-secret="$CLIENT_SECRET" \
+   --auth-provider-arg=refresh-token="$OIDC_REFRESH_TOKEN" \
+   --auth-provider-arg=id-token="$OIDC_ACCESS_TOKEN"