add initial example of NSM+Istio

Signed-off-by: Denis Tingaikin <denis.tingajkin@xored.com>
networkservicemesh · May 13, 2022 · db293b2 · db293b2
1 parent 588d3c6
commit db293b2
Show file tree

Hide file tree

Showing 19 changed files with 674 additions and 0 deletions.
diff --git a/README.md b/README.md
@@ -9,6 +9,7 @@ This repository provides kubernetes yaml deployments and markdown examples for N
     * [Basic examples](./examples/basic) 
     * [Interdomain and floating interdomain examples](./examples/interdomain)
     * [Features examples](./examples/features)
+        * [NSM+Istio example](./examples/nsm+istio)
         * [OPA example](./examples/features/opa)
         * [IPv6 examples](./examples/features/ipv6)
         * [DNS Example](./examples/features/dns)

diff --git a/examples/nsm+istio/README.md b/examples/nsm+istio/README.md
@@ -0,0 +1,236 @@
+## CLUSTERS SETUP
+
+
+### KIND
+Setup
+
+```bash
+go install sigs.k8s.io/kind@v0.13.0 
+
+kind create cluster --config kind-cluster-config.yaml --name cluster-1
+kind create cluster --config kind-cluster-config.yaml --name cluster-2
+
+
+kind get kubeconfig --name cluster-1 > /tmp/config1
+kind get kubeconfig --name cluster-2 > /tmp/config2
+
+export KUBECONFIG1=/tmp/config1
+export KUBECONFIG2=/tmp/config2 
+```
+
+
+#### Kind Load balancer
+
+Make sure that CIDR is fine for your kind clusters
+
+```bash
+kubectl --kubeconfig=$KUBECONFIG1 apply -f https://raw.githubusercontent.com/metallb/metallb/v0.10.2/manifests/namespace.yaml
+kubectl --kubeconfig=$KUBECONFIG1 create secret generic -n metallb-system memberlist --from-literal=secretkey="$(openssl rand -base64 128)" 
+kubectl --kubeconfig=$KUBECONFIG1 apply -f https://raw.githubusercontent.com/metallb/metallb/v0.10.2/manifests/metallb.yaml
+cat > metallb-config.yaml <<EOF
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  namespace: metallb-system
+  name: config
+data:
+  config: |
+    address-pools:
+    - name: default
+      protocol: layer2
+      addresses:
+      - 172.18.1.128/25
+EOF
+kubectl --kubeconfig=$KUBECONFIG1 apply -f metallb-config.yaml
+kubectl --kubeconfig=$KUBECONFIG1 wait --for=condition=ready --timeout=5m pod -l app=metallb -n metallb-system
+
+
+kubectl --kubeconfig=$KUBECONFIG2 apply -f https://raw.githubusercontent.com/metallb/metallb/v0.10.2/manifests/namespace.yaml
+kubectl --kubeconfig=$KUBECONFIG2 create secret generic -n metallb-system memberlist --from-literal=secretkey="$(openssl rand -base64 128)" 
+kubectl --kubeconfig=$KUBECONFIG2 apply -f https://raw.githubusercontent.com/metallb/metallb/v0.10.2/manifests/metallb.yaml
+cat > metallb-config.yaml <<EOF
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  namespace: metallb-system
+  name: config
+data:
+  config: |
+    address-pools:
+    - name: default
+      protocol: layer2
+      addresses:
+      - 172.18.2.128/25
+EOF
+kubectl --kubeconfig=$KUBECONFIG2 apply -f metallb-config.yaml
+kubectl --kubeconfig=$KUBECONFIG2 wait --for=condition=ready --timeout=5m pod -l app=metallb -n metallb-system
+
+```
+
+#### DNS
+
+```bash
+kubectl --kubeconfig=$KUBECONFIG1 expose service kube-dns -n kube-system --port=53 --target-port=53 --protocol=TCP --name=exposed-kube-dns --type=LoadBalancer
+kubectl --kubeconfig=$KUBECONFIG1 get services exposed-kube-dns -n kube-system -o go-template='{{index (index (index (index .status "loadBalancer") "ingress") 0) "ip"}}'
+ip1=$(kubectl --kubeconfig=$KUBECONFIG1 get services exposed-kube-dns -n kube-system -o go-template='{{index (index (index (index .status "loadBalancer") "ingress") 0) "ip"}}')
+if [[ $ip1 == *"no value"* ]]; then 
+    ip1=$(kubectl --kubeconfig=$KUBECONFIG1 get services exposed-kube-dns -n kube-system -o go-template='{{index (index (index (index .status "loadBalancer") "ingress") 0) "hostname"}}')
+    ip1=$(dig +short $ip1 | head -1)
+fi
+echo Selected externalIP: $ip1 for cluster1
+kubectl --kubeconfig=$KUBECONFIG2 expose service kube-dns -n kube-system --port=53 --target-port=53 --protocol=TCP --name=exposed-kube-dns --type=LoadBalancer
+kubectl --kubeconfig=$KUBECONFIG2 get services exposed-kube-dns -n kube-system -o go-template='{{index (index (index (index .status "loadBalancer") "ingress") 0) "ip"}}'
+ip2=$(kubectl --kubeconfig=$KUBECONFIG2 get services exposed-kube-dns -n kube-system -o go-template='{{index (index (index (index .status "loadBalancer") "ingress") 0) "ip"}}')
+if [[ $ip2 == *"no value"* ]]; then 
+    ip2=$(kubectl --kubeconfig=$KUBECONFIG2 get services exposed-kube-dns -n kube-system -o go-template='{{index (index (index (index .status "loadBalancer") "ingress") 0) "hostname"}}')
+    ip2=$(dig +short $ip2 | head -1)
+fi
+echo Selected externalIP: $ip2 for cluster2
+[[ ! -z $ip2 ]]
+cat > configmap.yaml <<EOF
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: coredns
+  namespace: kube-system
+data:
+  Corefile: |
+    .:53 {
+        errors
+        health {
+            lameduck 5s
+        }
+        ready
+        kubernetes cluster.local in-addr.arpa ip6.arpa {
+            pods insecure
+            fallthrough in-addr.arpa ip6.arpa
+            ttl 30
+        }
+        k8s_external my.cluster1
+        prometheus :9153
+        forward . /etc/resolv.conf {
+            max_concurrent 1000
+        }
+        loop
+        reload 5s
+    }
+    my.cluster2:53 {
+      forward . ${ip2}:53 {
+        force_tcp
+      }
+    }
+EOF
+
+kubectl --kubeconfig=$KUBECONFIG1 apply -f configmap.yaml
+
+cat > configmap.yaml <<EOF
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: coredns
+  namespace: kube-system
+data:
+  Corefile: |
+    .:53 {
+        errors
+        health {
+            lameduck 5s
+        }
+        ready
+        kubernetes cluster.local in-addr.arpa ip6.arpa {
+            pods insecure
+            fallthrough in-addr.arpa ip6.arpa
+            ttl 30
+        }
+        k8s_external my.cluster2
+        prometheus :9153
+        forward . /etc/resolv.conf {
+            max_concurrent 1000
+        }
+        loop
+        reload 5s
+    }
+    my.cluster1:53 {
+      forward . ${ip1}:53 {
+        force_tcp
+      }
+    }
+EOF
+
+kubectl --kubeconfig=$KUBECONFIG2 apply -f configmap.yaml
+```
+
+### SPIRE
+
+```bash
+kubectl --kubeconfig=$KUBECONFIG1 apply -k ./spire/cluster1
+kubectl --kubeconfig=$KUBECONFIG2 apply -k ./spire/cluster2
+
+bundle1=$(kubectl --kubeconfig=$KUBECONFIG1 exec spire-server-0 -n spire -- bin/spire-server bundle show -format spiffe)
+bundle2=$(kubectl --kubeconfig=$KUBECONFIG2 exec spire-server-0 -n spire -- bin/spire-server bundle show -format spiffe)
+
+echo $bundle2 | kubectl --kubeconfig=$KUBECONFIG1 exec -i spire-server-0 -n spire -- bin/spire-server bundle set -format spiffe -id "spiffe://nsm.cluster2"
+
+echo $bundle1 | kubectl --kubeconfig=$KUBECONFIG2 exec -i spire-server-0 -n spire -- bin/spire-server bundle set -format spiffe -id "spiffe://nsm.cluster1"
+```
+
+
+
+## NSM SETUP
+
+```bash
+kubectl --kubeconfig=$KUBECONFIG1 apply -k ./nsm/cluster1
+kubectl --kubeconfig=$KUBECONFIG2 apply -k ./nsm/cluster2
+```
+
+
+### Istio
+
+#### Install
+
+```bash
+export KUBECONFIG=$KUBECONFIG2
+```
+
+```bash
+curl -sL https://istio.io/downloadIstioctl | sh -
+export PATH=$PATH:$HOME/.istioctl/bin
+istioctl install --set profile=minimal -y
+istioctl proxy-status
+```
+
+### Verify NSM+ISTIO
+
+Install networkservice:
+```bash
+kubectl --kubeconfig=$KUBECONFIG2 apply -f networkservice.yaml
+```
+
+Start alpine networkservicemesh client:
+
+```bash
+kubectl --kubeconfig=$KUBECONFIG1 apply -f alpine/alpine.yaml
+```
+
+Start alpine networkservicemesh endpoint (auto-scale):
+
+```bash
+kubectl --kubeconfig=$KUBECONFIG2 apply -k nse-auto-scale 
+```
+
+Install istio booking example
+```bash
+kubectl --kubeconfig=$KUBECONFIG2 label namespace default istio-injection=enabled
+
+kubectl --kubeconfig=$KUBECONFIG2 apply -f https://raw.githubusercontent.com/istio/istio/release-1.13/samples/bookinfo/platform/kube/bookinfo.yaml
+```
+
+Verify connectivity:
+```bash
+kubectl --kubeconfig=$KUBECONFIG1 exec deploy/productpage-v1 -c cmd-nsc -- curl -s productpage.default:9080/productpage | grep -o "<title>.*</title>"
+```
+
+Port forward and check browser by `127.0.0.1:9080`
+```bash
+kubectl port-forward deploy/productpage-v1  9080:9080
+```
diff --git a/examples/nsm+istio/kind-cluster-config.yaml b/examples/nsm+istio/kind-cluster-config.yaml
@@ -0,0 +1,6 @@
+---
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+  - role: control-plane
+  - role: worker
diff --git a/examples/nsm+istio/networkservice.yaml b/examples/nsm+istio/networkservice.yaml
@@ -0,0 +1,18 @@
+---
+apiVersion: networkservicemesh.io/v1
+kind: NetworkService
+metadata:
+  name: autoscale-istio-proxy-responder
+  namespace: nsm-system
+spec:
+  payload: IP
+  matches:
+    - source_selector:
+      fallthrough: true
+      routes:
+        - destination_selector:
+            app: nse-istio-proxy-responder
+    - source_selector:
+      routes:
+        - destination_selector:
+            app: istio-proxy-responder-supplier
diff --git a/examples/nsm+istio/nse-auto-scale/kustomization.yaml b/examples/nsm+istio/nse-auto-scale/kustomization.yaml
@@ -0,0 +1,17 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+bases:
+- https://github.com/networkservicemesh/deployments-k8s/apps/nse-supplier-k8s?ref=cc0857306407913afc4699d79eaa75645fc60fa2
+
+patchesStrategicMerge:
+- patch-supplier.yaml
+
+configMapGenerator:
+  - name: supplier-pod-template-configmap
+    files:
+      - pod-template.yaml
+
+generatorOptions:
+  disableNameSuffixHash: true
diff --git a/examples/nsm+istio/nse-auto-scale/pod-template.yaml b/examples/nsm+istio/nse-auto-scale/pod-template.yaml
@@ -0,0 +1,54 @@
+---
+apiVersion: apps/v1
+kind: Pod
+metadata:
+  name: "mirror-{{ (index .Path.PathSegments 0).Name }}"
+  labels:
+    app: nse-istio-proxy-responder
+    "spiffe.io/spiffe-id": "true"
+    sidecar.istio.io/inject: "true"
+spec:
+  restartPolicy: Never
+  containers:
+    - name: nse
+      image: ghcr.io/networkservicemesh/ci/cmd-nse-istio-proxy:6d5eb55
+      imagePullPolicy: IfNotPresent
+      env:
+        - name: SPIFFE_ENDPOINT_SOCKET
+          value: unix:///run/spire/sockets/agent.sock
+        - name: NSM_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: NSM_CONNECT_TO
+          value: unix:///var/lib/networkservicemesh/nsm.io.sock
+        - name: NSM_CIDR_PREFIX
+          value: 172.16.1.2/31
+        - name: NSM_SERVICE_NAMES
+          value: autoscale-istio-proxy-responder
+        - name: NSM_LABELS
+          value: app:nse-istio-proxy-responder
+        - name: NSM_IDLE_TIMEOUT
+          value: 240s
+        - name: NSM_LOG_LEVEL
+          value: TRACE
+      volumeMounts:
+        - name: spire-agent-socket
+          mountPath: /run/spire/sockets
+          readOnly: true
+        - name: nsm-socket
+          mountPath: /var/lib/networkservicemesh
+          readOnly: true
+      resources:
+        limits:
+          memory: 20Mi
+          cpu: 100m
+  volumes:
+    - name: spire-agent-socket
+      hostPath:
+        path: /run/spire/sockets
+        type: Directory
+    - name: nsm-socket
+      hostPath:
+        path: /var/lib/networkservicemesh
+        type: DirectoryOrCreate
diff --git a/examples/nsm+istio/nsm/cluster1/kustomization.yaml b/examples/nsm+istio/nsm/cluster1/kustomization.yaml
@@ -0,0 +1,21 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: nsm-system
+
+bases:
+- https://github.com/networkservicemesh/deployments-k8s/apps/nsmgr?ref=v1.3.1
+- https://github.com/networkservicemesh/deployments-k8s/apps/forwarder-vpp?ref=a5fb2e8a4bed9a036787582a1f6a03ff47ea52c2
+- https://github.com/networkservicemesh/deployments-k8s/apps/registry-k8s?ref=v1.3.1
+- https://github.com/networkservicemesh/deployments-k8s/apps/registry-proxy-dns?ref=v1.3.1
+- https://github.com/networkservicemesh/deployments-k8s/apps/nsmgr-proxy?ref=v1.3.1
+- https://github.com/networkservicemesh/deployments-k8s/apps/admission-webhook-k8s?ref=a5fb2e8a4bed9a036787582a1f6a03ff47ea52c2
+
+resources:
+  - namespace.yaml
+
+patchesStrategicMerge:
+- patch-nsmgr-proxy.yaml
+- patch-registry-proxy-dns.yaml
+- patch-registry.yaml
diff --git a/examples/nsm+istio/nsm/cluster1/namespace.yaml b/examples/nsm+istio/nsm/cluster1/namespace.yaml
@@ -0,0 +1,5 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: nsm-system
diff --git a/examples/nsm+istio/nsm/cluster2/kustomization.yaml b/examples/nsm+istio/nsm/cluster2/kustomization.yaml
@@ -0,0 +1,21 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: nsm-system
+
+bases:
+- https://github.com/networkservicemesh/deployments-k8s/apps/nsmgr?ref=v1.3.1
+- https://github.com/networkservicemesh/deployments-k8s/apps/forwarder-vpp?ref=a5fb2e8a4bed9a036787582a1f6a03ff47ea52c2
+- https://github.com/networkservicemesh/deployments-k8s/apps/registry-k8s?ref=v1.3.1
+- https://github.com/networkservicemesh/deployments-k8s/apps/registry-proxy-dns?ref=v1.3.1
+- https://github.com/networkservicemesh/deployments-k8s/apps/nsmgr-proxy?ref=v1.3.1
+- https://github.com/networkservicemesh/deployments-k8s/apps/admission-webhook-k8s?ref=a5fb2e8a4bed9a036787582a1f6a03ff47ea52c2
+
+patchesStrategicMerge:
+- patch-nsmgr-proxy.yaml
+- patch-registry-proxy-dns.yaml
+- patch-registry.yaml
+
+resources:
+  - namespace.yaml