diff --git a/examples/pets/README.md b/examples/pets/README.md new file mode 100644 index 000000000000..66bf3f42146f --- /dev/null +++ b/examples/pets/README.md @@ -0,0 +1,3 @@ +# PetSet examples + +These examples are tracked from the [Kubernetes contrib project @d6e4be](https://github.com/kubernetes/contrib/tree/d6e4be066cc076fbb91ff69691819e117711b30b/pets) \ No newline at end of file diff --git a/examples/pets/mysql/galera/README.md b/examples/pets/mysql/galera/README.md new file mode 100644 index 000000000000..c19ed05956ce --- /dev/null +++ b/examples/pets/mysql/galera/README.md @@ -0,0 +1,33 @@ +# Mysql Galera + +This example runs mysql galera through a petset. + +## Bootstrap + +Create the petset in this directory +``` +$ kubectl create -f galera.yaml +``` + +Once you have all 3 nodes in Running, you can run the "test.sh" script in this directory. +This example requires manual intervention. +Once you have all 3 nodes in Running, you can run the "test.sh" script in this directory. + +## Caveats + +Starting up all galera nodes at once leads to an issue where all the mysqls +belive they're in the primary component because they don't see the others in +the DNS. For the bootstrapping to work: mysql-0 needs to see itself, mysql-1 +needs to see itself and mysql-0, and so on, because the first node that sees +a peer list of 1 will assume it's the leader. + +## TODO + +Expect better solutions for the following as petset matures. + +* Failover +* Scaling Up +* Scaling Down +* Image Upgrade +* Maintenance + diff --git a/examples/pets/mysql/galera/init/Dockerfile b/examples/pets/mysql/galera/init/Dockerfile new file mode 100644 index 000000000000..238d12f7eaac --- /dev/null +++ b/examples/pets/mysql/galera/init/Dockerfile @@ -0,0 +1,29 @@ +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# TODO: get rid of bash dependency and switch to plain busybox. +# The tar in busybox also doesn't seem to understand compression. +FROM debian:jessie +MAINTAINER Prashanth.B + +RUN apt-get update && apt-get install -y wget + +ADD on-start.sh / +ADD my-galera.cnf / +# See contrib/pets/peer-finder for details +RUN wget -qO /peer-finder https://storage.googleapis.com/kubernetes-release/pets/peer-finder + +ADD install.sh / +RUN chmod -c 755 /install.sh /on-start.sh /peer-finder +Entrypoint ["/install.sh"] diff --git a/examples/pets/mysql/galera/init/Makefile b/examples/pets/mysql/galera/init/Makefile new file mode 100644 index 000000000000..b240c38b03e5 --- /dev/null +++ b/examples/pets/mysql/galera/init/Makefile @@ -0,0 +1,27 @@ +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +all: push + +TAG = 0.1 +PREFIX = gcr.io/google_containers/galera-install + +container: + docker build -t $(PREFIX):$(TAG) . + +push: container + gcloud docker push $(PREFIX):$(TAG) + +clean: + docker rmi $(PREFIX):$(TAG) diff --git a/examples/pets/mysql/galera/init/install.sh b/examples/pets/mysql/galera/init/install.sh new file mode 100644 index 000000000000..473ed770dddf --- /dev/null +++ b/examples/pets/mysql/galera/init/install.sh @@ -0,0 +1,50 @@ +#! /bin/bash + +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This volume is assumed to exist and is shared with parent of the init +# container. It contains the mysq config. +CONFIG_VOLUME="/etc/mysql" + +# This volume is assumed to exist and is shared with the peer-finder +# init container. It contains on-start/change configuration scripts. +WORKDIR_VOLUME="/work-dir" + +for i in "$@" +do +case $i in + -c=*|--config=*) + CONFIG_VOLUME="${i#*=}" + shift + ;; + -w=*|--work-dir=*) + WORKDIR_VOLUME="${i#*=}" + shift + ;; + *) + # unknown option + ;; +esac +done + +echo installing config scripts into "${WORKDIR_VOLUME}" +mkdir -p "${WORKDIR_VOLUME}" +cp /on-start.sh "${WORKDIR_VOLUME}"/ +cp /peer-finder "${WORKDIR_VOLUME}"/ + +echo installing my-galera.cnf into "${CONFIG_VOLUME}" +mkdir -p "${CONFIG_VOLUME}" +chown -R mysql:mysql "${CONFIG_VOLUME}" +cp /my-galera.cnf "${CONFIG_VOLUME}"/ diff --git a/examples/pets/mysql/galera/init/my-galera.cnf b/examples/pets/mysql/galera/init/my-galera.cnf new file mode 100644 index 000000000000..75c477dbf9fa --- /dev/null +++ b/examples/pets/mysql/galera/init/my-galera.cnf @@ -0,0 +1,22 @@ +[mysqld] +user = mysql +bind-address = 0.0.0.0 +wsrep_provider = /usr/lib/galera/libgalera_smm.so +# TODO: is rsync the best option? +wsrep_sst_method = rsync +default_storage_engine = innodb +binlog_format = row +innodb_autoinc_lock_mode = 2 +innodb_flush_log_at_trx_commit = 0 +query_cache_size = 0 +query_cache_type = 0 + +# By default every node is standalone +wsrep_cluster_address=gcomm:// +wsrep_cluster_name=galera +wsrep_node_address=127.0.0.1 + +# TODO: Enable use privileges. This doesn't work +# on mysql restart, for some reason after the SST +# permissions are not setup correctly. +skip-grant-tables diff --git a/examples/pets/mysql/galera/init/on-start.sh b/examples/pets/mysql/galera/init/on-start.sh new file mode 100755 index 000000000000..69fccda8f432 --- /dev/null +++ b/examples/pets/mysql/galera/init/on-start.sh @@ -0,0 +1,52 @@ +#! /bin/bash + +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script writes out a mysql galera config using a list of newline seperated +# peer DNS names it accepts through stdin. + +# /etc/mysql is assumed to be a shared volume so we can modify my.cnf as required +# to keep the config up to date, without wrapping mysqld in a custom pid1. +# The config location is intentionally not /etc/mysql/my.cnf because the +# standard base image clobbers that location. +CFG=/etc/mysql/my-galera.cnf + +function join { + local IFS="$1"; shift; echo "$*"; +} + +HOSTNAME=$(hostname) +# Parse out cluster name, formatted as: petset_name-index +IFS='-' read -ra ADDR <<< "$(hostname)" +CLUSTER_NAME="${ADDR[0]}" + +while read -ra LINE; do + if [[ "${LINE}" == *"${HOSTNAME}"* ]]; then + MY_NAME=$LINE + fi + PEERS=("${PEERS[@]}" $LINE) +done + +if [ "${#PEERS[@]}" = 1 ]; then + WSREP_CLUSTER_ADDRESS="" +else + WSREP_CLUSTER_ADDRESS=$(join , "${PEERS[@]}") +fi +sed -i -e "s|^wsrep_node_address=.*$|wsrep_node_address=${MY_NAME}|" ${CFG} +sed -i -e "s|^wsrep_cluster_name=.*$|wsrep_cluster_name=${CLUSTER_NAME}|" ${CFG} +sed -i -e "s|^wsrep_cluster_address=.*$|wsrep_cluster_address=gcomm://${WSREP_CLUSTER_ADDRESS}|" ${CFG} + +# don't need a restart, we're just writing the conf in case there's an +# unexpected restart on the node. diff --git a/examples/pets/mysql/galera/mysql-galera.yaml b/examples/pets/mysql/galera/mysql-galera.yaml new file mode 100644 index 000000000000..35884c11436a --- /dev/null +++ b/examples/pets/mysql/galera/mysql-galera.yaml @@ -0,0 +1,120 @@ +# A headless service to create DNS records +apiVersion: v1 +kind: Service +metadata: + annotations: + service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" + name: galera + labels: + app: mysql +spec: + ports: + - port: 3306 + name: mysql + # *.galear.default.svc.cluster.local + clusterIP: None + selector: + app: mysql +--- +apiVersion: apps/v1alpha1 +kind: PetSet +metadata: + name: mysql +spec: + serviceName: "galera" + replicas: 3 + template: + metadata: + labels: + app: mysql + annotations: + pod.alpha.kubernetes.io/initialized: "true" + pod.alpha.kubernetes.io/init-containers: '[ + { + "name": "install", + "image": "gcr.io/google_containers/galera-install:0.1", + "imagePullPolicy": "Always", + "args": ["--work-dir=/work-dir"], + "volumeMounts": [ + { + "name": "workdir", + "mountPath": "/work-dir" + }, + { + "name": "config", + "mountPath": "/etc/mysql" + } + ] + }, + { + "name": "bootstrap", + "image": "debian:jessie", + "command": ["/work-dir/peer-finder"], + "args": ["-on-start=\"/work-dir/on-start.sh\"", "-service=galera"], + "env": [ + { + "name": "POD_NAMESPACE", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + } + } + } + ], + "volumeMounts": [ + { + "name": "workdir", + "mountPath": "/work-dir" + }, + { + "name": "config", + "mountPath": "/etc/mysql" + } + ] + } + ]' + spec: + containers: + - name: mysql + image: erkules/galera:basic + ports: + - containerPort: 3306 + name: mysql + - containerPort: 4444 + name: sst + - containerPort: 4567 + name: replication + - containerPort: 4568 + name: ist + args: + - --defaults-file=/etc/mysql/my-galera.cnf + - --user=root + readinessProbe: + exec: + command: + - sh + - -c + - "mysql -u root -e 'show databases;'" + initialDelaySeconds: 15 + timeoutSeconds: 5 + volumeMounts: + - name: datadir + mountPath: /var/lib/ + - name: config + mountPath: /etc/mysql + volumes: + - name: config + emptyDir: {} + - name: workdir + emptyDir: {} + volumeClaimTemplates: + - metadata: + name: datadir + annotations: + volume.alpha.kubernetes.io/storage-class: anything + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 10Gi diff --git a/examples/pets/mysql/galera/test.sh b/examples/pets/mysql/galera/test.sh new file mode 100755 index 000000000000..fbd6a7165814 --- /dev/null +++ b/examples/pets/mysql/galera/test.sh @@ -0,0 +1,21 @@ +#! /bin/bash + +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +kubectl exec mysql-0 -- mysql -u root -e "create database test;" +kubectl exec mysql-1 -- mysql -u root -e "use test; create table pet (id int(10), name varchar(20));" +kubectl exec mysql-1 -- mysql -u root -e "use test; insert into pet (id, name) values (1, \"galera\");" +kubectl exec mysql-2 -- mysql -u root -e "use test; select * from pet;" + diff --git a/examples/pets/mysql/healthz/Dockerfile b/examples/pets/mysql/healthz/Dockerfile new file mode 100644 index 000000000000..565916b1cfd7 --- /dev/null +++ b/examples/pets/mysql/healthz/Dockerfile @@ -0,0 +1,22 @@ +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# TODO: get rid of bash dependency and switch to plain busybox. +# The tar in busybox also doesn't seem to understand compression. +FROM debian:jessie +MAINTAINER Prashanth.B + +RUN apt-get update && apt-get install -y mysql-client wget +ADD mysql_healthz /mysql_healthz +Entrypoint ["/mysql_healthz"] diff --git a/examples/pets/mysql/healthz/Makefile b/examples/pets/mysql/healthz/Makefile new file mode 100644 index 000000000000..2bbad7ed8b43 --- /dev/null +++ b/examples/pets/mysql/healthz/Makefile @@ -0,0 +1,31 @@ +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +all: push + +# 0.0 shouldn't clobber any released builds +TAG = 0.1 +PREFIX = gcr.io/google_containers/mysql-healthz + +server: main.go + CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-w' -o mysql_healthz ./main.go + +container: server + docker build -t $(PREFIX):$(TAG) . + +push: container + gcloud docker push $(PREFIX):$(TAG) + +clean: + rm -f mysql_healthz diff --git a/examples/pets/mysql/healthz/README.md b/examples/pets/mysql/healthz/README.md new file mode 100644 index 000000000000..960167a2c643 --- /dev/null +++ b/examples/pets/mysql/healthz/README.md @@ -0,0 +1,3 @@ +# Mysql healthz + +This is a simple exec-over-http healthz probe for mysql. Mainly intended as a work-around for (#25456)[https://github.com/kubernetes/kubernetes/issues/25456]. diff --git a/examples/pets/mysql/healthz/main.go b/examples/pets/mysql/healthz/main.go new file mode 100644 index 000000000000..8ea4977a3be9 --- /dev/null +++ b/examples/pets/mysql/healthz/main.go @@ -0,0 +1,96 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// This is a simple health monitor for mysql. Only needed till #25456 is fixed. +package main + +import ( + "fmt" + "log" + "net/http" + "os" + osExec "os/exec" + "strings" + + flag "github.com/spf13/pflag" +) + +var ( + flags = flag.NewFlagSet("a http healthz server for mysql.", flag.ExitOnError) + port = flags.Int("port", 8080, "port to use for healthz server.") + verbose = flags.Bool("verbose", false, "log verbose output?") + pass = flags.String("password", "", "mysql password.") + host = flags.String("host", "", "mysql host.") + + mysqlChecks = map[string]string{ + "/healthz": "show databases;", + } +) + +type mysqlManager struct { + host, pass string +} + +func (m *mysqlManager) exec(cmd string) ([]byte, error) { + var password string + if m.pass != "" { + password = fmt.Sprintf("-p %v", m.pass) + } + mysqlCmd := fmt.Sprintf("/usr/bin/mysql -u root %v -h %v -B -e '%v'", password, m.host, cmd) + return osExec.Command("sh", "-c", mysqlCmd).CombinedOutput() +} + +func registerHandlers(verbose bool, m *mysqlManager) { + var str string + for endpoint, cmd := range mysqlChecks { + str += fmt.Sprintf("\t%v: %q\n", endpoint, cmd) + http.HandleFunc(endpoint, func(w http.ResponseWriter, r *http.Request) { + output, err := m.exec(cmd) + if err != nil { + w.WriteHeader(http.StatusServiceUnavailable) + } else { + w.WriteHeader(http.StatusOK) + } + if verbose { + log.Printf("Output of %v:\n%v\n", cmd, string(output)) + } + w.Write(output) + }) + } + http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write([]byte(fmt.Sprintf("Available handlers:\n%v", str))) + }) +} + +func getHostnameOrDie() string { + output, err := osExec.Command("hostname").CombinedOutput() + if err != nil { + log.Fatalf("%v", err) + } + return strings.Trim(string(output), "\n") +} + +func main() { + flags.Parse(os.Args) + hostname := *host + if hostname == "" { + hostname = getHostnameOrDie() + } + registerHandlers(*verbose, &mysqlManager{pass: *pass, host: hostname}) + log.Printf("Starting mysql healthz server on port %v", *port) + log.Fatalf(fmt.Sprintf("%v", http.ListenAndServe(fmt.Sprintf(":%v", *port), nil))) +} diff --git a/examples/pets/mysql/healthz/pod.yaml b/examples/pets/mysql/healthz/pod.yaml new file mode 100644 index 000000000000..cb047100ec4f --- /dev/null +++ b/examples/pets/mysql/healthz/pod.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Pod +metadata: + name: mysql + namespace: default +spec: + containers: + - image: mysql:5.6 + name: server + env: + - name: MYSQL_ALLOW_EMPTY_PASSWORD + value: "yes" + readinessProbe: + httpGet: + path: /healthz + port: 8080 + - name: healthz + image: gcr.io/google_containers/mysql-healthz:1.0 + imagePullPolicy: Always diff --git a/examples/pets/peer-finder/Dockerfile b/examples/pets/peer-finder/Dockerfile new file mode 100644 index 000000000000..137d2b42c797 --- /dev/null +++ b/examples/pets/peer-finder/Dockerfile @@ -0,0 +1,23 @@ +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM gcr.io/google_containers/ubuntu-slim:0.2 +MAINTAINER Prashanth.B + +RUN apt-get update && apt-get install -y wget bash dnsutils +ADD peer-finder /peer-finder +ADD peer-finder.go /peer-finder.go + +EXPOSE 9376 +ENTRYPOINT ["/peer-finder"] diff --git a/examples/pets/peer-finder/Makefile b/examples/pets/peer-finder/Makefile new file mode 100644 index 000000000000..5cc743c21475 --- /dev/null +++ b/examples/pets/peer-finder/Makefile @@ -0,0 +1,33 @@ +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +all: push + +TAG = 0.1 +PREFIX = gcr.io/google_containers/peer-finder + +server: peer-finder.go + CGO_ENABLED=0 go build -a -installsuffix cgo --ldflags '-w' ./peer-finder.go + +release: server + gsutil cp peer-finder gs://kubernetes-release/pets/peer-finder + +container: server + docker build -t $(PREFIX):$(TAG) . + +push: container + gcloud docker push $(PREFIX):$(TAG) + +clean: + rm -f peer-finder diff --git a/examples/pets/peer-finder/README.md b/examples/pets/peer-finder/README.md new file mode 100644 index 000000000000..54f2100846fa --- /dev/null +++ b/examples/pets/peer-finder/README.md @@ -0,0 +1,15 @@ +# Peer finder + +This is a simple peer finder daemon that runs as pid 1 in a petset. +It is expected to be a temporary solution till the main Kubernetes repo supports: +1. Init containers to replace on-start scripts +2. A notification delivery mechanism that allows external controllers to + declaratively execute on-change scripts in containers. + +Though we don't expect this container to always run as pid1, it will be +necessary in some form. All it does is resolve DNS. Even when we get (2) +the most natural way to update the input for the on-change script is through +a sidecar that runs the peer-finder. + + + diff --git a/examples/pets/peer-finder/peer-finder.go b/examples/pets/peer-finder/peer-finder.go new file mode 100644 index 000000000000..93ec34143411 --- /dev/null +++ b/examples/pets/peer-finder/peer-finder.go @@ -0,0 +1,110 @@ +/* +Copyright 2014 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// A small utility program to lookup hostnames of endpoints in a service. +package main + +import ( + "flag" + "fmt" + "log" + "net" + "os" + "os/exec" + "sort" + "strings" + "time" + + "k8s.io/kubernetes/pkg/util/sets" +) + +const ( + svcLocalSuffix = "svc.cluster.local" + pollPeriod = 1 * time.Second +) + +var ( + onChange = flag.String("on-change", "", "Script to run on change, must accept a new line separated list of peers via stdin.") + onStart = flag.String("on-start", "", "Script to run on start, must accept a new line separated list of peers via stdin.") + svc = flag.String("service", "", "Governing service responsible for the DNS records of the domain this pod is in.") + namespace = flag.String("ns", "", "The namespace this pod is running in. If unspecified, the POD_NAMESPACE env var is used.") +) + +func lookup(svcName string) (sets.String, error) { + endpoints := sets.NewString() + _, srvRecords, err := net.LookupSRV("", "", svcName) + if err != nil { + return endpoints, err + } + for _, srvRecord := range srvRecords { + // The SRV records ends in a "." for the root domain + ep := fmt.Sprintf("%v", srvRecord.Target[:len(srvRecord.Target)-1]) + endpoints.Insert(ep) + } + return endpoints, nil +} + +func shellOut(sendStdin, script string) { + log.Printf("execing: %v with stdin: %v", script, sendStdin) + // TODO: Switch to sending stdin from go + out, err := exec.Command("bash", "-c", fmt.Sprintf("echo -e '%v' | %v", sendStdin, script)).CombinedOutput() + if err != nil { + log.Fatalf("Failed to execute %v: %v, err: %v", script, string(out), err) + } + log.Print(string(out)) +} + +func main() { + flag.Parse() + + ns := *namespace + if ns == "" { + ns = os.Getenv("POD_NAMESPACE") + } + if *svc == "" || ns == "" || (*onChange == "" && *onStart == "") { + log.Fatalf("Incomplete args, require -on-change and/or -on-start, -service and -ns or an env var for POD_NAMESPACE.") + } + + hostname, err := os.Hostname() + if err != nil { + log.Fatalf("Failed to get hostname: %s", err) + } + + myName := strings.Join([]string{hostname, *svc, ns, svcLocalSuffix}, ".") + script := *onStart + if script == "" { + script = *onChange + log.Printf("No on-start supplied, on-change %v will be applied on start.", script) + } + for newPeers, peers := sets.NewString(), sets.NewString(); script != ""; time.Sleep(pollPeriod) { + newPeers, err = lookup(*svc) + if err != nil { + log.Printf("%v", err) + continue + } + if newPeers.Equal(peers) || !newPeers.Has(myName) { + continue + } + peerList := newPeers.List() + sort.Strings(peerList) + log.Printf("Peer list updated\nwas %v\nnow %v", peers.List(), newPeers.List()) + shellOut(strings.Join(peerList, "\n"), script) + peers = newPeers + script = *onChange + } + // TODO: Exit if there's no on-change? + log.Printf("Peer finder exiting") +} diff --git a/examples/pets/redis/README.md b/examples/pets/redis/README.md new file mode 100644 index 000000000000..d5acbf108160 --- /dev/null +++ b/examples/pets/redis/README.md @@ -0,0 +1,23 @@ +# Redis + +This example runs redis through a petset. + +## Master/slave + +### Bootstrap + +Create the yaml in this directory +``` +$ kubectl create -f redis.yaml +``` + +can run the "test.sh" script in this directory. + +## TODO + +Expect cleaner solutions for the following as petset matures. + +* Scaling Up/down +* Image Upgrade +* Periodic maintenance +* Sentinel failover diff --git a/examples/pets/redis/init/Dockerfile b/examples/pets/redis/init/Dockerfile new file mode 100644 index 000000000000..62b975631b4c --- /dev/null +++ b/examples/pets/redis/init/Dockerfile @@ -0,0 +1,29 @@ +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# TODO: get rid of bash dependency and switch to plain busybox. +# The tar in busybox also doesn't seem to understand compression. +FROM debian:jessie +MAINTAINER Prashanth.B + +# TODO: just use standard redis when there is one for 3.2.0. +RUN apt-get update && apt-get install -y wget make gcc + +ADD on-start.sh / +# See contrib/pets/peer-finder for details +RUN wget -qO /peer-finder https://storage.googleapis.com/kubernetes-release/pets/peer-finder + +ADD install.sh / +RUN chmod -c 755 /install.sh /on-start.sh /peer-finder +Entrypoint ["/install.sh"] diff --git a/examples/pets/redis/init/Makefile b/examples/pets/redis/init/Makefile new file mode 100644 index 000000000000..ff217dd12bb8 --- /dev/null +++ b/examples/pets/redis/init/Makefile @@ -0,0 +1,27 @@ +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +all: push + +TAG = 0.1 +PREFIX = gcr.io/google_containers/redis-install + +container: + docker build -t $(PREFIX):$(TAG) . + +push: container + gcloud docker push $(PREFIX):$(TAG) + +clean: + docker rmi $(PREFIX):$(TAG) diff --git a/examples/pets/redis/init/install.sh b/examples/pets/redis/init/install.sh new file mode 100755 index 000000000000..94786d485f29 --- /dev/null +++ b/examples/pets/redis/init/install.sh @@ -0,0 +1,64 @@ +#! /bin/bash + +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This volume is assumed to exist and is shared with parent of the init +# container. It contains the redis installation. +INSTALL_VOLUME="/opt" + +# This volume is assumed to exist and is shared with the peer-finder +# init container. It contains on-start/change configuration scripts. +WORK_DIR="/work-dir" + +TEMP_DIR="/tmp" + +VERSION="3.2.0" + +for i in "$@" +do +case $i in + -v=*|--version=*) + VERSION="${i#*=}" + shift + ;; + -i=*|--install-into=*) + INSTALL_VOLUME="${i#*=}" + shift + ;; + -w=*|--work-dir=*) + WORK_DIR="${i#*=}" + shift + ;; + *) + # unknown option + ;; +esac +done + +echo installing config scripts into "${WORK_DIR}" +mkdir -p "${WORK_DIR}" +cp /on-start.sh "${WORK_DIR}"/ +cp /peer-finder "${WORK_DIR}"/ + +echo installing redis-"${VERSION}" into "${INSTALL_VOLUME}" +mkdir -p "${TEMP_DIR}" "${INSTALL_VOLUME}"/redis +wget -q -O - http://download.redis.io/releases/redis-"${VERSION}".tar.gz | tar -xzf - -C "${TEMP_DIR}" + +cd "${TEMP_DIR}"/redis-"${VERSION}"/ +# Clean out existing deps, see https://github.com/antirez/redis/issues/722 +make distclean +make install INSTALL_BIN="${INSTALL_VOLUME}"/redis +cp "${TEMP_DIR}"/redis-"${VERSION}"/redis.conf ${INSTALL_VOLUME}/redis/redis.conf + diff --git a/examples/pets/redis/init/on-start.sh b/examples/pets/redis/init/on-start.sh new file mode 100755 index 000000000000..37d3be032601 --- /dev/null +++ b/examples/pets/redis/init/on-start.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +CFG=/opt/redis/redis.conf +HOSTNAME=$(hostname) +DATADIR="/data" +# Port on which redis listens for connections. +PORT=6379 + +# Ping everyone but ourself to see if there's a master. Only one pet starts at +# a time, so if we don't see a master we can assume the position is ours. +while read -ra LINE; do + if [[ "${LINE}" == *"${HOSTNAME}"* ]]; then + sed -i -e "s|^bind.*$|bind ${LINE}|" ${CFG} + elif [ "$(/opt/redis/redis-cli -h $LINE info | grep role | sed 's,\r$,,')" = "role:master" ]; then + # TODO: More restrictive regex? + sed -i -e "s|^.*slaveof.*$|slaveof ${LINE} ${PORT}|" ${CFG} + fi +done + +# Set the data directory for append only log and snapshot files. This should +# be a persistent volume for consistency. +sed -i -e "s|^.*dir .*$|dir ${DATADIR}|" ${CFG} + +# The append only log is written for every SET operation. Without this setting, +# redis just snapshots periodically which is only safe for a cache. This will +# produce an appendonly.aof file in the configured data dir. +sed -i -e "s|^appendonly .*$|appendonly yes|" ${CFG} + +# Every write triggers an fsync. Recommended default is "everysec", which +# is only safe for AP applications. +sed -i -e "s|^appendfsync .*$|appendfsync always|" ${CFG} + + diff --git a/examples/pets/redis/redis.yaml b/examples/pets/redis/redis.yaml new file mode 100644 index 000000000000..01fd99d616ad --- /dev/null +++ b/examples/pets/redis/redis.yaml @@ -0,0 +1,115 @@ +# A headless service to create DNS records +apiVersion: v1 +kind: Service +metadata: + annotations: + service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" + name: redis + labels: + app: redis +spec: + ports: + - port: 6379 + name: peer + # *.redis.default.svc.cluster.local + clusterIP: None + selector: + app: redis +--- +apiVersion: apps/v1alpha1 +kind: PetSet +metadata: + name: rd +spec: + serviceName: "redis" + replicas: 3 + template: + metadata: + labels: + app: redis + annotations: + pod.alpha.kubernetes.io/initialized: "true" + pod.alpha.kubernetes.io/init-containers: '[ + { + "name": "install", + "image": "gcr.io/google_containers/redis-install:0.1", + "imagePullPolicy": "Always", + "args": ["--version=3.2.0", "--install-into=/opt", "--work-dir=/work-dir"], + "volumeMounts": [ + { + "name": "opt", + "mountPath": "/opt" + }, + { + "name": "workdir", + "mountPath": "/work-dir" + } + ] + }, + { + "name": "bootstrap", + "image": "debian:jessie", + "command": ["/work-dir/peer-finder"], + "args": ["-on-start=\"/work-dir/on-start.sh\"", "-service=redis"], + "env": [ + { + "name": "POD_NAMESPACE", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + } + } + } + ], + "volumeMounts": [ + { + "name": "opt", + "mountPath": "/opt" + }, + { + "name": "workdir", + "mountPath": "/work-dir" + } + ] + } + ]' + spec: + containers: + - name: redis + image: debian:jessie + ports: + - containerPort: 6379 + name: peer + command: + - /opt/redis/redis-server + args: + - /opt/redis/redis.conf + readinessProbe: + exec: + command: + - sh + - -c + - "/opt/redis/redis-cli -h $(hostname) ping" + initialDelaySeconds: 15 + timeoutSeconds: 5 + volumeMounts: + - name: datadir + mountPath: /data + - name: opt + mountPath: /opt + volumes: + - name: opt + emptyDir: {} + - name: workdir + emptyDir: {} + volumeClaimTemplates: + - metadata: + name: datadir + annotations: + volume.alpha.kubernetes.io/storage-class: anything + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 20Gi diff --git a/examples/pets/redis/test.sh b/examples/pets/redis/test.sh new file mode 100755 index 000000000000..52a167f96add --- /dev/null +++ b/examples/pets/redis/test.sh @@ -0,0 +1,19 @@ +#! /bin/bash + +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +kubectl exec rd-0 -- /opt/redis/redis-cli -h rd-0.redis SET replicated:test true +kubectl exec rd-2 -- /opt/redis/redis-cli -h rd-2.redis GET replicated:test + diff --git a/examples/pets/zookeeper/README.md b/examples/pets/zookeeper/README.md new file mode 100644 index 000000000000..99217d009265 --- /dev/null +++ b/examples/pets/zookeeper/README.md @@ -0,0 +1,85 @@ +# Zookeeper + +This example runs zookeeper through a petset. + +## Bootstrap + +Create the petset in this directory +``` +$ kubetl create -f zookeeper.yaml +``` + +Once you have all 3 nodes in Running, you can run the "test.sh" script in this directory. + +## Failover + +You can test failover by killing the leader. Insert a key: +```console +$ kubectl exec zoo-0 -- /opt/zookeeper/bin/zkCli.sh create /foo bar; +$ kubectl exec zoo-2 -- /opt/zookeeper/bin/zkCli.sh get /foo; + +Watch existing members: +```console +$ kubectl run --attach bbox --image=busybox --restart=Never -- sh -c 'while true; do for i in 0 1 2; do echo zoo-$i $(echo stats | nc zoo-$1.zk:2181 | grep Mode); sleep 1; done; done'; +zoo-2 Mode: follower +zoo-0 Mode: follower +zoo-1 Mode: leader +zoo-2 Mode: follower +``` + +Delete pets and wait for the petset controller to bring the back up: +```console +$ kubectl delete po -l app=zk +$ kubectl get po --watch-only +NAME READY STATUS RESTARTS AGE +zoo-0 0/1 Init:0/2 0 16s +zoo-0 0/1 Init:0/2 0 21s +zoo-0 0/1 PodInitializing 0 23s +zoo-0 1/1 Running 0 41s +zoo-1 0/1 Pending 0 0s +zoo-1 0/1 Init:0/2 0 0s +zoo-1 0/1 Init:0/2 0 14s +zoo-1 0/1 PodInitializing 0 17s +zoo-1 0/1 Running 0 18s +zoo-2 0/1 Pending 0 0s +zoo-2 0/1 Init:0/2 0 0s +zoo-2 0/1 Init:0/2 0 12s +zoo-2 0/1 Init:0/2 0 28s +zoo-2 0/1 PodInitializing 0 31s +zoo-2 0/1 Running 0 32s +... + +zoo-0 Mode: follower +zoo-1 Mode: leader +zoo-2 Mode: follower +``` + +Check the previously inserted key: +```console +$ kubectl exec zoo-1 -- /opt/zookeeper/bin/zkCli.sh get /foo +ionid = 0x354887858e80035, negotiated timeout = 30000 + +WATCHER:: + +WatchedEvent state:SyncConnected type:None path:null +bar +``` + +## Scaling + +You can scale up by modifying the number of replicas on the PetSet. + +## Image Upgrade + +TODO: Add details + +## Maintenance + +TODO: Add details + +## Limitations +* Both petset and init containers are in alpha +* Look through the on-start and on-change scripts for TODOs +* Doesn't support the addition of observers through the petset +* Only supports storage options that have backends for persistent volume claims + diff --git a/examples/pets/zookeeper/init/Dockerfile b/examples/pets/zookeeper/init/Dockerfile new file mode 100644 index 000000000000..d3e8aaeef645 --- /dev/null +++ b/examples/pets/zookeeper/init/Dockerfile @@ -0,0 +1,29 @@ +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# TODO: get rid of bash dependency and switch to plain busybox. +# The tar in busybox also doesn't seem to understand compression. +FROM debian:jessie +MAINTAINER Prashanth.B + +RUN apt-get update && apt-get install -y wget netcat + +ADD on-start.sh / +ADD on-change.sh / +# See contrib/pets/peer-finder for details +RUN wget -qO /peer-finder https://storage.googleapis.com/kubernetes-release/pets/peer-finder + +ADD install.sh / +RUN chmod -c 755 /install.sh /on-start.sh /on-change.sh /peer-finder +Entrypoint ["/install.sh"] diff --git a/examples/pets/zookeeper/init/Makefile b/examples/pets/zookeeper/init/Makefile new file mode 100644 index 000000000000..bfb697899e7f --- /dev/null +++ b/examples/pets/zookeeper/init/Makefile @@ -0,0 +1,27 @@ +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +all: push + +TAG = 0.1 +PREFIX = gcr.io/google_containers/zookeeper-install + +container: + docker build -t $(PREFIX):$(TAG) . + +push: container + gcloud docker push $(PREFIX):$(TAG) + +clean: + docker rmi $(PREFIX):$(TAG) diff --git a/examples/pets/zookeeper/init/install.sh b/examples/pets/zookeeper/init/install.sh new file mode 100755 index 000000000000..6ed72696be0c --- /dev/null +++ b/examples/pets/zookeeper/init/install.sh @@ -0,0 +1,76 @@ +#! /bin/bash + +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This volume is assumed to exist and is shared with parent of the init +# container. It contains the zookeeper installation. +INSTALL_VOLUME="/opt" + +# This volume is assumed to exist and is shared with the peer-finder +# init container. It contains on-start/change configuration scripts. +WORKDIR_VOLUME="/work-dir" + +# As of April-2016 is 3.4.8 is the latest stable, but versions 3.5.0 onward +# allow dynamic reconfiguration. +VERSION="3.5.0-alpha" + +for i in "$@" +do +case $i in + -v=*|--version=*) + VERSION="${i#*=}" + shift + ;; + -i=*|--install-into=*) + INSTALL_VOLUME="${i#*=}" + shift + ;; + -w=*|--work-dir=*) + WORKDIR_VOLUME="${i#*=}" + shift + ;; + *) + # unknown option + ;; +esac +done + +echo installing config scripts into "${WORKDIR_VOLUME}" +mkdir -p "${WORKDIR_VOLUME}" +cp /on-start.sh "${WORKDIR_VOLUME}"/ +cp /on-change.sh "${WORKDIR_VOLUME}"/ +cp /peer-finder "${WORKDIR_VOLUME}"/ + +echo installing zookeeper-"${VERSION}" into "${INSTALL_VOLUME}" +mkdir -p "${INSTALL_VOLUME}" +wget -q -O - http://apache.mirrors.pair.com/zookeeper/zookeeper-"${VERSION}"/zookeeper-"${VERSION}".tar.gz | tar -xzf - -C "${INSTALL_VOLUME}" +mv "${INSTALL_VOLUME}"/zookeeper-"${VERSION}" "${INSTALL_VOLUME}"/zookeeper +cp "${INSTALL_VOLUME}"/zookeeper/conf/zoo_sample.cfg "${INSTALL_VOLUME}"/zookeeper/conf/zoo.cfg + +# TODO: Should dynamic config be tied to the version? +IFS="." read -ra RELEASE <<< "${VERSION}" +if [ $(expr "${RELEASE[1]}") -gt 4 ]; then + echo zookeeper-"${VERSION}" supports dynamic reconfiguration, enabling it + echo "standaloneEnabled=false" >> "${INSTALL_VOLUME}"/zookeeper/conf/zoo.cfg + echo "dynamicConfigFile="${INSTALL_VOLUME}"/zookeeper/conf/zoo.cfg.dynamic" >> "${INSTALL_VOLUME}"/zookeeper/conf/zoo.cfg +fi + +# TODO: This is a hack, netcat is convenient to have in the zookeeper container +# I want to avoid using a custom zookeeper image just for this. So copy it. +NC=$(which nc) +if [ "${NC}" != "" ]; then + echo copying nc into "${INSTALL_VOLUME}" + cp "${NC}" "${INSTALL_VOLUME}" +fi diff --git a/examples/pets/zookeeper/init/on-change.sh b/examples/pets/zookeeper/init/on-change.sh new file mode 100755 index 000000000000..bb9a5049e401 --- /dev/null +++ b/examples/pets/zookeeper/init/on-change.sh @@ -0,0 +1,49 @@ +#! /bin/bash + +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script configures zookeeper cluster member ship for version of zookeeper +# < 3.5.0. It should not be used with the on-start.sh script in this example. +# As of April-2016 is 3.4.8 is the latest stable. + +CFG=/opt/zookeeper/conf/zoo.cfg +CFG_BAK=/opt/zookeeper/conf/zoo.cfg.bak +MY_ID=/tmp/zookeeper/myid + +# write myid +IFS='-' read -ra ADDR <<< "$(hostname)" +echo $(expr "1" + "${ADDR[1]}") > "${MY_ID}" + +# TODO: This is a dumb way to reconfigure zookeeper because it allows dynamic +# reconfig, but it's simple. +i=0 +echo " +tickTime=2000 +initLimit=10 +syncLimit=5 +dataDir=/tmp/zookeeper +clientPort=2181 +" > "${CFG_BAK}" + +while read -ra LINE; do + let i=i+1 + echo "server.${i}=${LINE}:2888:3888" >> "${CFG_BAK}" +done +cp ${CFG_BAK} ${CFG} + +# TODO: Typically one needs to first add a new member as an "observer" then +# promote it to "participant", but that requirement is relaxed if we never +# start > 1 at a time. +/opt/zookeeper/bin/zkServer.sh restart diff --git a/examples/pets/zookeeper/init/on-start.sh b/examples/pets/zookeeper/init/on-start.sh new file mode 100755 index 000000000000..392b82aa9dad --- /dev/null +++ b/examples/pets/zookeeper/init/on-start.sh @@ -0,0 +1,73 @@ +#! /bin/bash + +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script configures zookeeper cluster member ship for version of zookeeper +# >= 3.5.0. It should not be used with the on-change.sh script in this example. +# As of April-2016 is 3.4.8 is the latest stable. + +# Both /opt and /tmp/zookeeper are assumed to be volumes shared with the parent. +CFG=/opt/zookeeper/conf/zoo.cfg.dynamic +CFG_BAK=/opt/zookeeper/conf/zoo.cfg.bak +MY_ID_FILE=/tmp/zookeeper/myid +HOSTNAME=$(hostname) + +while read -ra LINE; do + PEERS=("${PEERS[@]}" $LINE) +done + +# Don't add the first member as an observer +if [ ${#PEERS[@]} -eq 1 ]; then + # We need to write our index in this list of servers into MY_ID_FILE. + # Note that this may not always coincide with the hostname id. + echo 1 > "${MY_ID_FILE}" + echo "server.1=${PEERS[0]}:2888:3888;2181" > "${CFG}" + # TODO: zkServer-initialize is the safe way to handle changes to datadir + # because simply starting will create a new datadir, BUT if the user changed + # pod template they might end up with 2 datadirs and brief split brain. + exit +fi + +# Every subsequent member is added as an observer and promoted to a participant +echo "" > "${CFG_BAK}" +i=0 +for peer in "${PEERS[@]}"; do + let i=i+1 + if [[ "${peer}" == *"${HOSTNAME}"* ]]; then + MY_ID=$i + MY_NAME=${peer} + echo $i > "${MY_ID_FILE}" + echo "server.${i}=${peer}:2888:3888:observer;2181" >> "${CFG_BAK}" + else + echo "server.${i}=${peer}:2888:3888:participant;2181" >> "${CFG_BAK}" + fi +done + +# Once the dynamic config file is written it shouldn't be modified, so the final +# reconfigure needs to happen through the "reconfig" command. +cp ${CFG_BAK} ${CFG} + +# TODO: zkServer-initialize is the safe way to handle changes to datadir +# because simply starting will create a new datadir, BUT if the user changed +# pod template they might end up with 2 datadirs and brief split brain. +/opt/zookeeper/bin/zkServer.sh start + +# TODO: We shouldn't need to specify the address of the master as long as +# there's quorum. According to the docs the new server is just not allowed to +# vote, it's still allowed to propose config changes, and it knows the +# existing members of the ensemble from *its* config. This works as expected, +# but we should correlate with more satisfying empirical evidence. +/opt/zookeeper/bin/zkCli.sh reconfig -add "server.$MY_ID=$MY_NAME:2888:3888:participant;2181" +/opt/zookeeper/bin/zkServer.sh stop diff --git a/examples/pets/zookeeper/test.sh b/examples/pets/zookeeper/test.sh new file mode 100755 index 000000000000..fbe12ff25ca1 --- /dev/null +++ b/examples/pets/zookeeper/test.sh @@ -0,0 +1,19 @@ +#! /bin/bash + +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +kubectl exec zoo-0 -- /opt/zookeeper/bin/zkCli.sh create /foo bar; +kubectl exec zoo-2 -- /opt/zookeeper/bin/zkCli.sh get /foo; + diff --git a/examples/pets/zookeeper/zookeeper.yaml b/examples/pets/zookeeper/zookeeper.yaml new file mode 100644 index 000000000000..ada1af378b8e --- /dev/null +++ b/examples/pets/zookeeper/zookeeper.yaml @@ -0,0 +1,123 @@ +# A headless service to create DNS records +apiVersion: v1 +kind: Service +metadata: + annotations: + service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" + name: zk + labels: + app: zk +spec: + ports: + - port: 2888 + name: peer + - port: 3888 + name: leader-election + # *.zk.default.svc.cluster.local + clusterIP: None + selector: + app: zk +--- +apiVersion: apps/v1alpha1 +kind: PetSet +metadata: + name: zoo +spec: + serviceName: "zk" + replicas: 3 + template: + metadata: + labels: + app: zk + annotations: + pod.alpha.kubernetes.io/initialized: "true" + pod.alpha.kubernetes.io/init-containers: '[ + { + "name": "install", + "image": "gcr.io/google_containers/zookeeper-install:0.1", + "imagePullPolicy": "Always", + "args": ["--version=3.5.0-alpha", "--install-into=/opt", "--work-dir=/work-dir"], + "volumeMounts": [ + { + "name": "opt", + "mountPath": "/opt/" + }, + { + "name": "workdir", + "mountPath": "/work-dir" + } + ] + }, + { + "name": "bootstrap", + "image": "java:openjdk-8-jre", + "command": ["/work-dir/peer-finder"], + "args": ["-on-start=\"/work-dir/on-start.sh\"", "-service=zk"], + "env": [ + { + "name": "POD_NAMESPACE", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + } + } + } + ], + "volumeMounts": [ + { + "name": "opt", + "mountPath": "/opt/" + }, + { + "name": "workdir", + "mountPath": "/work-dir" + }, + { + "name": "datadir", + "mountPath": "/tmp/zookeeper" + } + ] + } + ]' + spec: + containers: + - name: zk + image: java:openjdk-8-jre + ports: + - containerPort: 2888 + name: peer + - containerPort: 3888 + name: leader-election + command: + - /opt/zookeeper/bin/zkServer.sh + args: + - start-foreground + readinessProbe: + exec: + command: + - sh + - -c + - "/opt/zookeeper/bin/zkCli.sh ls /" + initialDelaySeconds: 15 + timeoutSeconds: 5 + volumeMounts: + - name: datadir + mountPath: /tmp/zookeeper + - name: opt + mountPath: /opt/ + volumes: + - name: opt + emptyDir: {} + - name: workdir + emptyDir: {} + volumeClaimTemplates: + - metadata: + name: datadir + annotations: + volume.alpha.kubernetes.io/storage-class: anything + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 20Gi