From 61a7779cf2d21ea43c8f855a6184fe12735b0845 Mon Sep 17 00:00:00 2001 From: maneta Date: Mon, 19 Feb 2018 18:34:16 +0100 Subject: [PATCH 1/9] Introducing Imagestreams Plus separating templates for easier Deployment --- .../01-apicast-cloud-hosted-imagestream.yml | 14 ++++ openshift/02-apicast-builder-imagestream.yml | 12 +++ .../{BuildConfig.yml => 03-build-config.yml} | 24 +++--- ...emplate.yml => 04-deployment-template.yml} | 77 +++++++++---------- openshift/05-routes.yml | 49 ++++++++++++ 5 files changed, 122 insertions(+), 54 deletions(-) create mode 100644 openshift/01-apicast-cloud-hosted-imagestream.yml create mode 100644 openshift/02-apicast-builder-imagestream.yml rename openshift/{BuildConfig.yml => 03-build-config.yml} (54%) rename openshift/{template.yml => 04-deployment-template.yml} (69%) create mode 100644 openshift/05-routes.yml diff --git a/openshift/01-apicast-cloud-hosted-imagestream.yml b/openshift/01-apicast-cloud-hosted-imagestream.yml new file mode 100644 index 0000000..4692150 --- /dev/null +++ b/openshift/01-apicast-cloud-hosted-imagestream.yml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: ImageStream +metadata: + name: apicast-cloud-hosted + labels: + app: apicast-cloud-hosted +spec: + tags: + - name: latest + annotations: + openshift.io/display-name: Apicast Cloud Hosted (latest) + from: + kind: ImageStreamTag + name: latest diff --git a/openshift/02-apicast-builder-imagestream.yml b/openshift/02-apicast-builder-imagestream.yml new file mode 100644 index 0000000..28dfee6 --- /dev/null +++ b/openshift/02-apicast-builder-imagestream.yml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: ImageStream +metadata: + name: apicast + labels: + app: apicast +spec: + tags: + - name: builder + from: + kind: DockerImage + name: quay.io/3scale/apicast:master-builder diff --git a/openshift/BuildConfig.yml b/openshift/03-build-config.yml similarity index 54% rename from openshift/BuildConfig.yml rename to openshift/03-build-config.yml index be9f7b0..da23b43 100644 --- a/openshift/BuildConfig.yml +++ b/openshift/03-build-config.yml @@ -1,4 +1,4 @@ -apiVersion: build.openshift.io/v1 +apiVersion: v1 kind: BuildConfig metadata: labels: @@ -7,18 +7,16 @@ metadata: status: spec: nodeSelector: null - postCommit: - args: - - '--dev' - - '--daemon' - command: - - bin/apicast + #postCommit: + # args: + # - '--dev' + # - '--daemon' + # command: + # - bin/apicast output: - pushSecret: - name: quay to: - kind: DockerImage - name: 'quay.io/3scale/apicast-cloud-hosted:apicast-master' + kind: ImageStreamTag + name: apicast-cloud-hosted:latest source: contextDir: /apicast git: @@ -31,5 +29,5 @@ spec: sourceStrategy: forcePull: true from: - kind: DockerImage - name: 'quay.io/3scale/apicast:master-builder' + kind: ImageStreamTag + name: apicast:builder diff --git a/openshift/template.yml b/openshift/04-deployment-template.yml similarity index 69% rename from openshift/template.yml rename to openshift/04-deployment-template.yml index dbd57e6..a5fc517 100644 --- a/openshift/template.yml +++ b/openshift/04-deployment-template.yml @@ -1,26 +1,26 @@ apiVersion: v1 kind: Template metadata: - name: "apicast-cloud-hosted" + name: "apicast-cloud-hosted-deployment" objects: - apiVersion: v1 kind: DeploymentConfig metadata: - name: apicast-mapping-service + name: apicast-mapping-service-${RELEASE_REF} spec: replicas: 1 selector: - deploymentconfig: apicast-mapping-service + deploymentconfig: apicast-mapping-service-${RELEASE_REF} strategy: type: Rolling template: metadata: labels: - deploymentconfig: apicast-mapping-service + deploymentconfig: apicast-mapping-service-${RELEASE_REF} spec: containers: - - name: apicast-mapping-service + - name: apicast-mapping-service-${RELEASE_REF} image: "${MAPPING_SERVICE_IMAGE}" imagePullPolicy: Always env: @@ -56,17 +56,17 @@ objects: - apiVersion: v1 kind: DeploymentConfig metadata: - name: apicast + name: apicast-${RELEASE_REF} spec: replicas: 1 selector: - deploymentconfig: apicast + deploymentconfig: apicast-${RELEASE_REF} strategy: type: Rolling template: metadata: labels: - deploymentconfig: apicast + deploymentconfig: apicast-${RELEASE_REF} spec: containers: - env: @@ -77,10 +77,12 @@ objects: - name: THREESCALE_DEPLOYMENT_ENV value: "${ENVIRONMENT}" - name: THREESCALE_PORTAL_ENDPOINT - value: "http://apicast-mapping-service/config" - image: "${APICAST_IMAGE}" + value: "http://apicast-mapping-service-${RELEASE_REF}/config" + - name: APICAST_OIDC_LOG_LEVEL + value: "${APICAST_OIDC_LOG_LEVEL}" + image: apicast-cloud-hosted:${RELEASE_REF} imagePullPolicy: IfNotPresent - name: apicast + name: apicast-${RELEASE_REF} livenessProbe: tcpSocket: port: proxy @@ -102,12 +104,19 @@ objects: name: management protocol: TCP triggers: - - type: ConfigChange + - type: ImageChange + imageChangeParams: + automatic: true + containerNames: + - apicast-${RELEASE_REF} + from: + kind: ImageStreamTag + name: apicast-cloud-hosted:${RELEASE_REF} - apiVersion: v1 kind: Service metadata: - name: apicast-mapping-service + name: apicast-mapping-service-${RELEASE_REF} spec: ports: - name: mapping @@ -115,15 +124,15 @@ objects: protocol: TCP targetPort: mapping selector: - deploymentconfig: apicast-mapping-service + deploymentconfig: apicast-mapping-service-${RELEASE_REF} - apiVersion: v1 kind: Service metadata: - name: apicast + name: apicast-${RELEASE_REF} annotations: service.alpha.openshift.io/dependencies: |- - [{"name": "apicast-mapping-service", "kind": "Service"}] + [{"name": "apicast-mapping-service-${RELEASE_REF}", "kind": "Service"}] spec: ports: - name: proxy @@ -135,28 +144,14 @@ objects: port: 8090 targetPort: management selector: - deploymentconfig: apicast - -- apiVersion: v1 - kind: Route - metadata: - name: apicast-router - labels: - app: apicast-router - spec: - host: apicast.${ENVIRONMENT}.gw.apicast.io - to: - kind: Service - name: apicast - port: - targetPort: proxy - wildcardPolicy: Subdomain - tls: - termination: edge - insecureEdgeTerminationPolicy: Allow + deploymentconfig: apicast-${RELEASE_REF} parameters: +- description: "Release version reference" + name: RELEASE_REF + required: true + - description: "Deployment environment. `staging` or `production`" name: ENVIRONMENT required: true @@ -168,14 +163,14 @@ parameters: - name: MAPPING_SERVICE_IMAGE description: "Mapping Service image name. Used to discover proxy configurations." required: true - value: "quay.io/3scale/apicast-cloud-hosted:mapping-service-v3.0.0-cloud1" - -- name: APICAST_IMAGE - description: "APIcast image name." - required: true - value: "quay.io/3scale/apicast-cloud-hosted:apicast-v3.0.0-cloud1" + value: "quay.io/3scale/apicast-cloud-hosted:mapping-service-master" - name: MASTER_ACCESS_TOKEN_SECRET description: "Secret name that containts System Master Access Token password" required: true value: 'master-access-token-secret' + +- description: "APIcast OIDC Log Level" + name: APICAST_OIDC_LOG_LEVEL + value: "notice" + required: true \ No newline at end of file diff --git a/openshift/05-routes.yml b/openshift/05-routes.yml new file mode 100644 index 0000000..e14a1a2 --- /dev/null +++ b/openshift/05-routes.yml @@ -0,0 +1,49 @@ +apiVersion: v1 +kind: Template +metadata: + name: "apicast-cloud-hosted" +objects: + +- apiVersion: v1 + kind: Route + metadata: + name: apicast-router + labels: + app: apicast-router + spec: + host: apicast.${ENVIRONMENT}.gw.apicast.io + to: + kind: Service + name: apicast + port: + targetPort: proxy + wildcardPolicy: Subdomain + tls: + termination: edge + insecureEdgeTerminationPolicy: Allow + +- apiVersion: v1 + kind: Route + metadata: + name: apicast-${ENVIRONMENT}-policies + spec: + host: apicast.${ENVIRONMENT}.${WILDCARD_DOMAIN} + path: /policies + to: + kind: Service + name: apicast + port: + targetPort: management + wildcardPolicy: None + tls: + termination: edge + +parameters: + +- description: "Deployment environment. `staging` or `production`" + name: ENVIRONMENT + required: true + +- description: "Openshift Cluster Wildcard Domain" + name: WILDCARD_DOMAIN + required: true From 9ed79a76915de2724187a31aa999641f5b81b08b Mon Sep 17 00:00:00 2001 From: maneta Date: Tue, 20 Feb 2018 09:49:25 +0100 Subject: [PATCH 2/9] PR #4 review requests. --- openshift/01-apicast-cloud-hosted-imagestream.yml | 2 +- openshift/02-apicast-builder-imagestream.yml | 2 +- openshift/03-build-config.yml | 2 +- openshift/04-deployment-template.yml | 10 +++------- 4 files changed, 6 insertions(+), 10 deletions(-) diff --git a/openshift/01-apicast-cloud-hosted-imagestream.yml b/openshift/01-apicast-cloud-hosted-imagestream.yml index 4692150..17dca6e 100644 --- a/openshift/01-apicast-cloud-hosted-imagestream.yml +++ b/openshift/01-apicast-cloud-hosted-imagestream.yml @@ -8,7 +8,7 @@ spec: tags: - name: latest annotations: - openshift.io/display-name: Apicast Cloud Hosted (latest) + openshift.io/display-name: APIcast Cloud Hosted (latest) from: kind: ImageStreamTag name: latest diff --git a/openshift/02-apicast-builder-imagestream.yml b/openshift/02-apicast-builder-imagestream.yml index 28dfee6..efefb76 100644 --- a/openshift/02-apicast-builder-imagestream.yml +++ b/openshift/02-apicast-builder-imagestream.yml @@ -6,7 +6,7 @@ metadata: app: apicast spec: tags: - - name: builder + - name: master-builder from: kind: DockerImage name: quay.io/3scale/apicast:master-builder diff --git a/openshift/03-build-config.yml b/openshift/03-build-config.yml index da23b43..c6e00ba 100644 --- a/openshift/03-build-config.yml +++ b/openshift/03-build-config.yml @@ -30,4 +30,4 @@ spec: forcePull: true from: kind: ImageStreamTag - name: apicast:builder + name: apicast:master-builder diff --git a/openshift/04-deployment-template.yml b/openshift/04-deployment-template.yml index a5fc517..6bcbd6e 100644 --- a/openshift/04-deployment-template.yml +++ b/openshift/04-deployment-template.yml @@ -20,7 +20,7 @@ objects: deploymentconfig: apicast-mapping-service-${RELEASE_REF} spec: containers: - - name: apicast-mapping-service-${RELEASE_REF} + - name: apicast-mapping-service image: "${MAPPING_SERVICE_IMAGE}" imagePullPolicy: Always env: @@ -79,10 +79,10 @@ objects: - name: THREESCALE_PORTAL_ENDPOINT value: "http://apicast-mapping-service-${RELEASE_REF}/config" - name: APICAST_OIDC_LOG_LEVEL - value: "${APICAST_OIDC_LOG_LEVEL}" + value: "notice" image: apicast-cloud-hosted:${RELEASE_REF} imagePullPolicy: IfNotPresent - name: apicast-${RELEASE_REF} + name: apicast livenessProbe: tcpSocket: port: proxy @@ -170,7 +170,3 @@ parameters: required: true value: 'master-access-token-secret' -- description: "APIcast OIDC Log Level" - name: APICAST_OIDC_LOG_LEVEL - value: "notice" - required: true \ No newline at end of file From 6d67553dfd5730f801f1bdb933c8e453415291ba Mon Sep 17 00:00:00 2001 From: maneta Date: Tue, 20 Feb 2018 10:09:33 +0100 Subject: [PATCH 3/9] Introducing Resource Limits Same as APICast --- openshift/04-deployment-template.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/openshift/04-deployment-template.yml b/openshift/04-deployment-template.yml index 6bcbd6e..422915a 100644 --- a/openshift/04-deployment-template.yml +++ b/openshift/04-deployment-template.yml @@ -50,6 +50,13 @@ objects: - containerPort: 8090 name: management protocol: TCP + resources: + limits: + cpu: '1' + memory: 128Mi + requests: + cpu: 500m + memory: 64Mi triggers: - type: ConfigChange @@ -103,6 +110,13 @@ objects: - containerPort: 8090 name: management protocol: TCP + resources: + limits: + cpu: '1' + memory: 128Mi + requests: + cpu: 500m + memory: 64Mi triggers: - type: ImageChange imageChangeParams: From 287e33c18f82a14d84e86f17a83dc1c5239aabcd Mon Sep 17 00:00:00 2001 From: maneta Date: Tue, 20 Feb 2018 11:33:25 +0100 Subject: [PATCH 4/9] Scheduled Import builder image The ImageStream will immport the builder imager periodically. This configuration is clusterwide but the default is 15 minutes. --- openshift/02-apicast-builder-imagestream.yml | 2 ++ openshift/04-deployment-template.yml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/openshift/02-apicast-builder-imagestream.yml b/openshift/02-apicast-builder-imagestream.yml index efefb76..e80744c 100644 --- a/openshift/02-apicast-builder-imagestream.yml +++ b/openshift/02-apicast-builder-imagestream.yml @@ -10,3 +10,5 @@ spec: from: kind: DockerImage name: quay.io/3scale/apicast:master-builder + importPolicy: + scheduled: true diff --git a/openshift/04-deployment-template.yml b/openshift/04-deployment-template.yml index 422915a..2df3821 100644 --- a/openshift/04-deployment-template.yml +++ b/openshift/04-deployment-template.yml @@ -122,7 +122,7 @@ objects: imageChangeParams: automatic: true containerNames: - - apicast-${RELEASE_REF} + - apicast from: kind: ImageStreamTag name: apicast-cloud-hosted:${RELEASE_REF} From b3a3267bb1d9aad6d9c4bcf41d05672d26e39a08 Mon Sep 17 00:00:00 2001 From: maneta Date: Tue, 20 Feb 2018 17:35:34 +0100 Subject: [PATCH 5/9] Makefile for deployment and Documentation Update --- openshift/Makefile | 28 ++++++++++++++++++++++++++++ openshift/README.md | 5 ++++- 2 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 openshift/Makefile diff --git a/openshift/Makefile b/openshift/Makefile new file mode 100644 index 0000000..b87b2b4 --- /dev/null +++ b/openshift/Makefile @@ -0,0 +1,28 @@ +.PHONY: all imagestream buildconfig deploy route help +.DEFAULT_GOAL := help + +MKFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST))) +THISDIR_PATH := $(patsubst %/,%,$(abspath $(dir $(MKFILE_PATH)))) + +all: imagestream buildconfig deploy route + +imagestream: ## Create the Imagestreams (APIcast and Builder APICast). + oc create -f $(THISDIR_PATH)/01-apicast-cloud-hosted-imagestream.yml + oc create -f $(THISDIR_PATH)/02-apicast-builder-imagestream.yml + +buildconfig: ## Create the BuildConfig. + oc create -f $(THISDIR_PATH)/03-build-config.yml + +deploy: ## Create the BuildConfig. Parameters: RELEASE_REF, ENVIRONMENT, CACHE_TTL. + oc new-app -f $(THISDIR_PATH)/04-deployment-template.yml \ + -p RELEASE_REF=${RELEASE_REF} \ + -p ENVIRONMENT=${ENVIRONMENT} \ + -p CACHE_TTL=${CACHE_TTL} + +route: ## Create the Routes.Parameters: WILDCARD_DOMAIN, ENVIRONMENT. + oc new-app -f $(THISDIR_PATH)/05-routes.yml \ + -p ENVIRONMENT=${ENVIRONMENT} \ + -p WILDCARD_DOMAIN=${WILDCARD_DOMAIN} + +help: ## Print this help + @awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST) \ No newline at end of file diff --git a/openshift/README.md b/openshift/README.md index 24d3018..10b3b91 100644 --- a/openshift/README.md +++ b/openshift/README.md @@ -5,4 +5,7 @@ 1. `oc create -f secret.yml` 1. `oc secrets add serviceaccount/default secrets/3scale-openshift-pull-secret --for=pull` 1. `oc secret new-basicauth master-access-token-secret --password=MASTER_ACCESS_TOKEN` -1. `oc new-app -f openshift/template.yml` (with `-p ENVIRONMENT=production -p CACHE_TTL=300` for production or `-p ENVIRONMENT=staging -p CACHE_TTL=0` for staging) +1. `make imagestream` to deploy the imageStreams (Apicast Cloud Hosted and Apicast Builder) +1. `make buildconfig` to create the BuildConfig +1. `make deploy RELEASE_REF=release_number ENVIRONMENT=staging CACHE_TTL=0` - (with `ENVIRONMENT=production CACHE_TTL=300` for production or `ENVIRONMENT=staging CACHE_TTL=0` for staging) +1. `make route ENVIRONMENT=staging WILDCARD_DOMAIN=cluster.wildcard.domain.com` - Wildcard Domain Concatenation: `apicast.${ENVIRONMENT}.${WILDCARD_DOMAIN}` From d9178189739d2549ce833953a65ca7c47581ba26 Mon Sep 17 00:00:00 2001 From: maneta Date: Wed, 28 Feb 2018 12:59:37 +0100 Subject: [PATCH 6/9] re-enabling postCommit in the BuildConfig --- openshift/03-build-config.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/openshift/03-build-config.yml b/openshift/03-build-config.yml index c6e00ba..85552c7 100644 --- a/openshift/03-build-config.yml +++ b/openshift/03-build-config.yml @@ -7,12 +7,12 @@ metadata: status: spec: nodeSelector: null - #postCommit: - # args: - # - '--dev' - # - '--daemon' - # command: - # - bin/apicast + postCommit: + args: + - '--dev' + - '--daemon' + command: + - bin/apicast output: to: kind: ImageStreamTag From 58d79586e9507a6d2fce8ba1e32b1654e0ada872 Mon Sep 17 00:00:00 2001 From: maneta Date: Wed, 28 Feb 2018 13:25:55 +0100 Subject: [PATCH 7/9] Adding prometheus exporter endpoint --- openshift/04-deployment-template.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/openshift/04-deployment-template.yml b/openshift/04-deployment-template.yml index 2df3821..5317380 100644 --- a/openshift/04-deployment-template.yml +++ b/openshift/04-deployment-template.yml @@ -110,6 +110,9 @@ objects: - containerPort: 8090 name: management protocol: TCP + - containerPort: 9421 + name: metrics + protocol: TCP resources: limits: cpu: '1' @@ -160,6 +163,22 @@ objects: selector: deploymentconfig: apicast-${RELEASE_REF} +- apiVersion: v1 + kind: Service + metadata: + name: apicast-metrics-${RELEASE_REF} + annotations: + prometheus.io/scrape: 'true' + prometheus.io/path: '/metrics' + spec: + ports: + - name: metrics + protocol: TCP + port: 9421 + targetPort: metrics + selector: + deploymentconfig: apicast-${RELEASE_REF} + parameters: - description: "Release version reference" From dfb1371510f17c70e403a49d55cc81789b2b1d06 Mon Sep 17 00:00:00 2001 From: maneta Date: Wed, 28 Feb 2018 16:08:49 +0100 Subject: [PATCH 8/9] updating the postCommit to be more insightfull --- openshift/03-build-config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openshift/03-build-config.yml b/openshift/03-build-config.yml index 85552c7..3853350 100644 --- a/openshift/03-build-config.yml +++ b/openshift/03-build-config.yml @@ -9,8 +9,8 @@ spec: nodeSelector: null postCommit: args: - - '--dev' - - '--daemon' + - '--test' + - '--lazy' command: - bin/apicast output: From 0b3dfe2de8093de2dcf3cfe437945aace87fe9d2 Mon Sep 17 00:00:00 2001 From: maneta Date: Wed, 28 Feb 2018 16:44:22 +0100 Subject: [PATCH 9/9] specifying the port used to scrape --- openshift/04-deployment-template.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/openshift/04-deployment-template.yml b/openshift/04-deployment-template.yml index 5317380..21f3b10 100644 --- a/openshift/04-deployment-template.yml +++ b/openshift/04-deployment-template.yml @@ -170,6 +170,7 @@ objects: annotations: prometheus.io/scrape: 'true' prometheus.io/path: '/metrics' + prometheus.io/port: '9421' spec: ports: - name: metrics