-
Notifications
You must be signed in to change notification settings - Fork 106
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Initial work at deploying K8 related hooks as part of OnDemand distri…
…bution
- Loading branch information
Showing
9 changed files
with
381 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,9 @@ | ||
K8S_USERNAME_PREFIX="" | ||
NAMESPACE_PREFIX="" | ||
NETWORK_POLICY_ALLOW_CIDR="127.0.0.1/32" | ||
IDP_ISSUER_URL="https://idp.example.com/auth/realms/main/protocol/openid-connect/token" | ||
CLIENT_ID="changeme" | ||
CLIENT_SECRET="changeme" | ||
# Set to empty string or remove if not using a private registry | ||
IMAGE_PULL_SECRET=private-docker-registry | ||
REGISTRY_DOCKER_CONFIG_JSON="/some/path/to/docker/config.json" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,40 @@ | ||
#!/bin/bash | ||
|
||
ONDEMAND_USERNAME="$1" | ||
if [ "x${ONDEMAND_USERNAME}" = "x" ]; then | ||
echo "Must specify username" | ||
exit 1 | ||
fi | ||
HOOK_ENV="$2" | ||
if [ "x${HOOK_ENV}" = "x" ]; then | ||
echo "Must specify hook.env path" | ||
exit 1 | ||
fi | ||
|
||
set -e | ||
|
||
source $HOOK_ENV | ||
|
||
NAMESPACE="${NAMESPACE_PREFIX}${ONDEMAND_USERNAME}" | ||
|
||
TMPFILE=$(mktemp "/tmp/k8-bootstrap-job-pod-reaper-${ONDEMAND_USERNAME}.XXXXXX") | ||
cat > "$TMPFILE" <<EOF | ||
# allow job-pod-reaper to see this namespace | ||
apiVersion: rbac.authorization.k8s.io/v1 | ||
kind: RoleBinding | ||
metadata: | ||
name: "$ONDEMAND_USERNAME-job-pod-reaper-rolebinding" | ||
namespace: "$NAMESPACE" | ||
roleRef: | ||
apiGroup: rbac.authorization.k8s.io | ||
kind: ClusterRole | ||
name: job-pod-reaper | ||
subjects: | ||
- kind: ServiceAccount | ||
name: job-pod-reaper | ||
namespace: job-pod-reaper | ||
EOF | ||
|
||
export PATH=/usr/local/bin:/bin:$PATH | ||
kubectl apply -f "$TMPFILE" | ||
rm -f "$TMPFILE" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,42 @@ | ||
#!/bin/bash | ||
|
||
ONDEMAND_USERNAME="$1" | ||
if [ "x${ONDEMAND_USERNAME}" = "x" ]; then | ||
echo "Must specify username" | ||
exit 1 | ||
fi | ||
HOOK_ENV="$2" | ||
if [ "x${HOOK_ENV}" = "x" ]; then | ||
echo "Must specify hook.env path" | ||
exit 1 | ||
fi | ||
|
||
set -e | ||
|
||
source $HOOK_ENV | ||
|
||
export PATH=/usr/local/bin:/bin:$PATH | ||
NAMESPACE="${NAMESPACE_PREFIX}${ONDEMAND_USERNAME}" | ||
|
||
SCRIPT=$(readlink -f "$0") | ||
BASEDIR=$(dirname "$SCRIPT") | ||
YAML_DIR=${BASEDIR}/k8-bootstrap | ||
|
||
NAMESPACE_TMPFILE=$(mktemp "/tmp/k8-bootstrap-namespace-${ONDEMAND_USERNAME}.XXXXXX") | ||
cat ${YAML_DIR}/namespace.yaml | envsubst > "$NAMESPACE_TMPFILE" | ||
|
||
NETWORK_POLICY_TMPFILE=$(mktemp "/tmp/k8-bootstrap-network-policy-${ONDEMAND_USERNAME}.XXXXXX") | ||
cat ${YAML_DIR}/network-policy.yaml | envsubst > "$NETWORK_POLICY_TMPFILE" | ||
|
||
ROLEBINDING_TMPFILE=$(mktemp "/tmp/k8-bootstrap-rolebinding-${ONDEMAND_USERNAME}.XXXXXX") | ||
cat ${YAML_DIR}/rolebinding.yaml | envsubst > "$ROLEBINDING_TMPFILE" | ||
|
||
TMPFILE=$(mktemp "/tmp/k8-ondemand-bootstrap-${ONDEMAND_USERNAME}.XXXXXX") | ||
cat "$NAMESPACE_TMPFILE" "$NETWORK_POLICY_TMPFILE" "$ROLEBINDING_TMPFILE" > "$TMPFILE" | ||
kubectl apply -f "$TMPFILE" | ||
rm -f "$NAMESPACE_TMPFILE" "$NETWORK_POLICY_TMPFILE" "$ROLEBINDING_TMPFILE" "$TMPFILE" | ||
|
||
if [ "x$IMAGE_PULL_SECRET" != "x" ]; then | ||
kubectl create secret generic "$IMAGE_PULL_SECRET" --from-file=.dockerconfigjson="$REGISTRY_DOCKER_CONFIG_JSON" --type=kubernetes.io/dockerconfigjson -n "$NAMESPACE" | ||
kubectl patch serviceaccount default -n "$NAMESPACE" -p "{\"imagePullSecrets\": [{\"name\": \"${IMAGE_PULL_SECRET}\"}]}" | ||
fi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,111 @@ | ||
#!/bin/bash | ||
|
||
ONDEMAND_USERNAME="$1" | ||
if [ "x${ONDEMAND_USERNAME}" = "x" ]; then | ||
echo "Must specify username" | ||
exit 1 | ||
fi | ||
HOOK_ENV="$2" | ||
if [ "x${HOOK_ENV}" = "x" ]; then | ||
echo "Must specify hook.env path" | ||
exit 1 | ||
fi | ||
|
||
set -e | ||
|
||
source $HOOK_ENV | ||
|
||
TMPFILE=$(mktemp "/tmp/k8-ondemand-bootstrap-${ONDEMAND_USERNAME}.XXXXXX") | ||
PASSWD=$(getent passwd "$ONDEMAND_USERNAME") | ||
if ! [[ "$PASSWD" =~ "${ONDEMAND_USERNAME}:"* ]]; then | ||
echo "level=error msg=\"Unable to perform lookup of user\" user=$ONDEMAND_USERNAME" | ||
exit 1 | ||
fi | ||
USER_UID=$(echo "$PASSWD" | cut -d':' -f3) | ||
USER_GID=$(echo "$PASSWD" | cut -d':' -f4) | ||
NAMESPACE="${NAMESPACE_PREFIX}${ONDEMAND_USERNAME}" | ||
|
||
cat > "$TMPFILE" <<EOF | ||
--- | ||
# the pod security policy such that you can only run pods as a single uid/gid pair | ||
apiVersion: policy/v1beta1 | ||
kind: PodSecurityPolicy | ||
metadata: | ||
name: "$ONDEMAND_USERNAME-psp" | ||
annotations: | ||
seccomp.security.alpha.kubernetes.io/allowedProfileNames: 'docker/default,runtime/default' | ||
apparmor.security.beta.kubernetes.io/allowedProfileNames: 'runtime/default' | ||
seccomp.security.alpha.kubernetes.io/defaultProfileName: 'runtime/default' | ||
apparmor.security.beta.kubernetes.io/defaultProfileName: 'runtime/default' | ||
labels: | ||
app.kubernetes.io/name: open-ondemand | ||
spec: | ||
# Required to prevent escalations to root. | ||
privileged: false | ||
allowPrivilegeEscalation: false | ||
hostNetwork: false | ||
hostIPC: false | ||
hostPID: false | ||
runAsUser: | ||
# Run as your own uid | ||
rule: 'MustRunAs' | ||
ranges: | ||
- min: $USER_UID | ||
max: $USER_UID | ||
runAsGroup: | ||
# Run as your own gid | ||
rule: 'MustRunAs' | ||
ranges: | ||
- min: $USER_GID | ||
max: $USER_GID | ||
seLinux: | ||
# This policy assumes the nodes are using AppArmor rather than SELinux. | ||
rule: 'RunAsAny' | ||
supplementalGroups: | ||
rule: 'MustRunAs' | ||
ranges: | ||
# Forbid adding the root group. | ||
- min: 1 | ||
max: 65535 | ||
fsGroup: | ||
rule: 'MustRunAs' | ||
ranges: | ||
# Forbid adding the root group. | ||
- min: 1 | ||
max: 65535 | ||
volumes: | ||
- '*' | ||
--- | ||
kind: Role | ||
apiVersion: rbac.authorization.k8s.io/v1 | ||
metadata: | ||
name: "$ONDEMAND_USERNAME-psp-role" | ||
namespace: "$NAMESPACE" | ||
rules: | ||
- apiGroups: [ "extensions" ] | ||
resources: [ "podsecuritypolicies" ] | ||
resourceNames: [ "$ONDEMAND_USERNAME-psp" ] | ||
verbs: [ "use" ] | ||
--- | ||
# bind the users' pod security policy to the user | ||
apiVersion: rbac.authorization.k8s.io/v1 | ||
kind: RoleBinding | ||
metadata: | ||
name: "$ONDEMAND_USERNAME-psp-rolebinding" | ||
namespace: "$NAMESPACE" | ||
roleRef: | ||
apiGroup: rbac.authorization.k8s.io | ||
kind: Role | ||
name: "$ONDEMAND_USERNAME-psp-role" | ||
subjects: | ||
- apiGroup: rbac.authorization.k8s.io | ||
kind: User | ||
name: "$ONDEMAND_USERNAME" | ||
namespace: "$NAMESPACE" | ||
EOF | ||
|
||
|
||
export PATH=/usr/local/bin:/bin:$PATH | ||
kubectl apply -f "$TMPFILE" | ||
|
||
rm -f "$TMPFILE" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,7 @@ | ||
--- | ||
apiVersion: v1 | ||
kind: Namespace | ||
metadata: | ||
name: "$NAMESPACE" | ||
labels: | ||
app.kubernetes.io/name: open-ondemand |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,14 @@ | ||
--- | ||
kind: NetworkPolicy | ||
apiVersion: networking.k8s.io/v1 | ||
metadata: | ||
namespace: $NAMESPACE | ||
name: deny-from-other-namespaces | ||
spec: | ||
podSelector: | ||
matchLabels: | ||
ingress: | ||
- from: | ||
- podSelector: {} | ||
- ipBlock: | ||
cidr: ${NETWORK_POLICY_ALLOW_CIDR} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,30 @@ | ||
--- | ||
# give the service account the ood-initializer role | ||
apiVersion: rbac.authorization.k8s.io/v1 | ||
kind: RoleBinding | ||
metadata: | ||
namespace: "$NAMESPACE" | ||
name: "$ONDEMAND_USERNAME-ood-initializer" | ||
roleRef: | ||
apiGroup: rbac.authorization.k8s.io | ||
kind: ClusterRole | ||
name: "ood-initializer" | ||
subjects: | ||
- kind: ServiceAccount | ||
name: "default" | ||
namespace: "$NAMESPACE" | ||
--- | ||
# give the user the ood-user role | ||
apiVersion: rbac.authorization.k8s.io/v1 | ||
kind: RoleBinding | ||
metadata: | ||
namespace: "$NAMESPACE" | ||
name: "$ONDEMAND_USERNAME-ood-user" | ||
roleRef: | ||
apiGroup: rbac.authorization.k8s.io | ||
kind: ClusterRole | ||
name: "ood-user" | ||
subjects: | ||
- kind: User | ||
name: "$ONDEMAND_USERNAME" | ||
namespace: "$NAMESPACE" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,103 @@ | ||
--- | ||
apiVersion: v1 | ||
kind: Namespace | ||
metadata: | ||
name: ondemand | ||
labels: | ||
name: ondemand | ||
--- | ||
apiVersion: rbac.authorization.k8s.io/v1 | ||
kind: ClusterRole | ||
metadata: | ||
name: ondemand | ||
namespace: ondemand | ||
rules: | ||
- apiGroups: [""] | ||
resources: | ||
- namespaces | ||
- serviceaccounts | ||
verbs: | ||
- get | ||
- create | ||
- update | ||
- patch | ||
- apiGroups: | ||
- rbac.authorization.k8s.io | ||
resources: | ||
- rolebindings | ||
- roles | ||
verbs: | ||
- get | ||
- create | ||
- update | ||
- patch | ||
# Only added to enable cluster-info subcommand | ||
- apiGroups: [""] | ||
resources: | ||
- services | ||
verbs: | ||
- list | ||
# rules necessary to give these permissions to ondemand users | ||
- apiGroups: [""] | ||
resources: ["services", "pods", "configmaps", "secrets", "pods/log"] | ||
verbs: ["get", "watch", "list", "create", "update", "patch", "delete"] | ||
- apiGroups: | ||
- networking.k8s.io | ||
resources: | ||
- networkpolicies | ||
verbs: | ||
- get | ||
- create | ||
- update | ||
- patch | ||
--- | ||
apiVersion: v1 | ||
kind: ServiceAccount | ||
metadata: | ||
name: ondemand | ||
namespace: ondemand | ||
--- | ||
apiVersion: rbac.authorization.k8s.io/v1 | ||
kind: ClusterRoleBinding | ||
metadata: | ||
name: ondemand | ||
namespace: ondemand | ||
roleRef: | ||
apiGroup: rbac.authorization.k8s.io | ||
kind: ClusterRole | ||
name: ondemand | ||
subjects: | ||
- kind: ServiceAccount | ||
name: ondemand | ||
namespace: ondemand | ||
--- | ||
# ood-initializer is the role given to service accounts to initialize and | ||
# configure ood configmaps and services within init containers. | ||
apiVersion: rbac.authorization.k8s.io/v1 | ||
kind: ClusterRole | ||
metadata: | ||
name: ood-initializer | ||
namespace: ondemand | ||
rules: | ||
- apiGroups: [""] | ||
resources: ["secrets"] | ||
verbs: ["get", "create", "update", "patch"] | ||
- apiGroups: [""] | ||
resources: ["services", "pods"] | ||
verbs: ["get"] | ||
- apiGroups: [""] | ||
resources: ["configmaps"] | ||
verbs: ["update", "get", "patch"] | ||
--- | ||
# ood-user role allows users to manipulate their services/pods/configmaps/secrets and logs | ||
# should be applied as a RoleBinding not ClusterRoleBinding so they're limited to their | ||
# own namespace | ||
apiVersion: rbac.authorization.k8s.io/v1 | ||
kind: ClusterRole | ||
metadata: | ||
name: ood-user | ||
namespace: ondemand | ||
rules: | ||
- apiGroups: [""] | ||
resources: ["services", "pods", "configmaps", "secrets", "pods/log"] | ||
verbs: ["get", "watch", "list", "create", "update", "patch", "delete"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,25 @@ | ||
#!/bin/bash | ||
|
||
ONDEMAND_USERNAME="$1" | ||
if [ "x${ONDEMAND_USERNAME}" = "x" ]; then | ||
echo "Must specify username" | ||
exit 1 | ||
fi | ||
HOOK_ENV="$2" | ||
if [ "x${HOOK_ENV}" = "x" ]; then | ||
echo "Must specify hook.env path" | ||
exit 1 | ||
fi | ||
|
||
source $HOOK_ENV | ||
|
||
K8S_USERNAME="${K8S_USERNAME_PREFIX}${ONDEMAND_USERNAME}" | ||
|
||
# we use pass ACCESS_TOKEN into the id-token arg. That's OK, it works and refreshes. | ||
sudo -u "$ONDEMAND_USERNAME" kubectl config set-credentials "$K8S_USERNAME" \ | ||
--auth-provider=oidc \ | ||
--auth-provider-arg=idp-issuer-url="$IDP_ISSUER_URL" \ | ||
--auth-provider-arg=client-id="$CLIENT_ID" \ | ||
--auth-provider-arg=client-secret="$CLIENT_SECRET" \ | ||
--auth-provider-arg=refresh-token="$OIDC_REFRESH_TOKEN" \ | ||
--auth-provider-arg=id-token="$OIDC_ACCESS_TOKEN" |