diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2c06da357f..82876b61d2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -122,3 +122,23 @@ uat-integrationtest: extends: .uat-integration-test script: - "TEST_ENV=uat POSTGRES_PWD=$EVENTHUB_DB_PASSWORD_UAT ./gradlew :acceptance-test:acceptanceTest" + + +prod: + variables: + KUBE_NAMESPACE: eventhub-prod + K8S_SECRET_SPRING_PROFILES_ACTIVE: prod + KUBE_INGRESS_BASE_DOMAIN: prod.tc.private.wiley.host + HELM_UPGRADE_VALUES_FILE: .gitlab/auto-deploy-values-prod + +prod-integrationtest: + when: manual + image: artifactory.aws.wiley.com/docker/openjdk:11-jdk + tags: + - prod + variables: + APP_ENVIRONMENT: prod + allow_failure: true + extends: .prod-integration-test + script: + - "TEST_ENV=prod POSTGRES_PWD=$EVENTHUB_DB_PASSWORD_PROD ./gradlew :acceptance-test:acceptanceTest" \ No newline at end of file diff --git a/.gitlab/auto-deploy-values-prod.yaml b/.gitlab/auto-deploy-values-prod.yaml new file mode 100644 index 0000000000..d573291ad1 --- /dev/null +++ b/.gitlab/auto-deploy-values-prod.yaml @@ -0,0 +1,53 @@ +service: + internalPort: 8080 + externalPort: 5000 +ingress: + enabled: true + ingressClass: "nginx-external" + path: "/" + tls: + enabled: false +livenessProbe: + path: "/health" + initialDelaySeconds: 240 + timeoutSeconds: 15 + scheme: "HTTP" + probeType: "httpGet" +readinessProbe: + path: "/health" + initialDelaySeconds: 30 + periodSeconds: 15 + timeoutSeconds: 3 + failureThreshold: 10 + scheme: "HTTP" + probeType: "httpGet" +serviceAccount: + annotations: + eks.amazonaws.com/role-arn: arn:aws:iam::015558583646:role/eventhub-prod-prod-use1 + createNew: true +externalSecrets: + enabled: true + data: + - secretKey: SPRING_DATASOURCE_PASSWORD + remoteRef: + key: nakadi-eventhub-prod-event_hub_user +prometheusOperator: + serviceMonitor: + enabled: true + path: "/prometheus" +application: + env: + - name: SPRING_PROFILES_ACTIVE + value: prod +resources: + limits: + cpu: 2000m + memory: 2Gi + requests: + cpu: 1500m + memory: 1Gi +hpa: + enabled: true + minReplicas: 2 + maxReplicas: 6 + targetCPUUtilizationPercentage: 50 \ No newline at end of file diff --git a/acceptance-test/src/acceptance-test/resources/automation.yml b/acceptance-test/src/acceptance-test/resources/automation.yml index 435a417661..916153ce4c 100644 --- a/acceptance-test/src/acceptance-test/resources/automation.yml +++ b/acceptance-test/src/acceptance-test/resources/automation.yml @@ -23,7 +23,7 @@ environments: maxStreamMemoryBytes: 50000000 # ~50 MB maxCommitTimeout: 60000 # 1 minute kpiConfig: - streamDataCollectionFrequencyMs: 30000 # every half minute + streamDataCollectionFrequencyMs: 30000 # every half minute review: apiUrl: http://localhost apiPort: 8080 @@ -48,7 +48,7 @@ environments: maxStreamMemoryBytes: 10_000 # ~10 Kb maxCommitTimeout: 10000 # 10 seconds kpiConfig: - streamDataCollectionFrequencyMs: 100 + streamDataCollectionFrequencyMs: 100 dev: apiUrl: https://eventhub.dev.tc.private.wiley.host apiPort: 443 @@ -125,20 +125,20 @@ environments: kpiConfig: streamDataCollectionFrequencyMs: 30000 # every half minute prod: - apiUrl: http://localhost + apiUrl: https://eventhub.prod.tc.private.wiley.host apiPort: 443 - zookeeperUrl: localhost:2181 + zookeeperUrl: z-1.eventhubprodmskkafka.f1iuv2.c11.kafka.us-east-1.amazonaws.com:2181,z-3.eventhubprodmskkafka.f1iuv2.c11.kafka.us-east-1.amazonaws.com:2181,z-2.eventhubprodmskkafka.f1iuv2.c11.kafka.us-east-1.amazonaws.com:2181 kafka: - bootstrapServers: localhost:29092 + bootstrapServers: b-2.eventhubprodmskkafka.f1iuv2.c11.kafka.us-east-1.amazonaws.com:9098,b-1.eventhubprodmskkafka.f1iuv2.c11.kafka.us-east-1.amazonaws.com:9098 minInSyncReplicas: 2 replicaFactor: 2 - securityProtocol: PLAINTEXT - saslMechanism: PLAIN + securityProtocol: SASL_SSL + saslMechanism: AWS_MSK_IAM saslJaasConfig: software.amazon.msk.auth.iam.IAMLoginModule required; saslClientCallbackHandlerClass: software.amazon.msk.auth.iam.IAMClientCallbackHandler database: - url: jdbc:postgresql://localhost:5432/local_nakadi_db - username: nakadi + url: jdbc:postgresql://nakadi-eventhub-prod-use1.cluster-c1z0wn5giy4n.us-east-1.rds.amazonaws.com/event_hub + username: event_hub_user eventTypeDeletableSubscription: owningApplication: "nakadi_archiver" consumerGroup: "nakadi_to_s3" diff --git a/app/src/main/resources/application.yml b/app/src/main/resources/application.yml index a77dcd98ed..f3c07a6db4 100644 --- a/app/src/main/resources/application.yml +++ b/app/src/main/resources/application.yml @@ -262,6 +262,49 @@ nakadi: EVENT_OWNER_SELECTOR_AUTHZ: false ACCESS_LOG_ENABLED: true +--- +spring: + config: + activate: + on-profile: prod + datasource: + url: jdbc:postgresql://nakadi-eventhub-prod-use1.cluster-c1z0wn5giy4n.us-east-1.rds.amazonaws.com/event_hub + username: event_hub_user + +nakadi: + topic: + min: + retentionMs: 10800000 # 3 hours + max: + partitionNum: 20 + retentionMs: 345600000 # 4 days + default: + partitionNum: 1 + replicaFactor: 2 + retentionMs: 172800000 # 2 days + rotationMs: 10800000 # 3 hours + kafka: + bootstrap.servers: b-2.eventhubprodmskkafka.f1iuv2.c11.kafka.us-east-1.amazonaws.com:9098,b-1.eventhubprodmskkafka.f1iuv2.c11.kafka.us-east-1.amazonaws.com:9098 + min.insync.replicas: 2 + security.protocol: SASL_SSL + sasl.mechanism: AWS_MSK_IAM + sasl.jaas.config: software.amazon.msk.auth.iam.IAMLoginModule required; + sasl.client.callback.handler.class: software.amazon.msk.auth.iam.IAMClientCallbackHandler + linger.ms: 200 + delivery.timeout.ms: 30200 # request.timeout.ms + linger.ms + zookeeper: + connectionString: zookeeper://z-1.eventhubprodmskkafka.f1iuv2.c11.kafka.us-east-1.amazonaws.com:2181,z-3.eventhubprodmskkafka.f1iuv2.c11.kafka.us-east-1.amazonaws.com:2181,z-2.eventhubprodmskkafka.f1iuv2.c11.kafka.us-east-1.amazonaws.com:2181 + features.defaultFeatures: + DISABLE_EVENT_TYPE_CREATION: false + DISABLE_EVENT_TYPE_DELETION: false + DISABLE_SUBSCRIPTION_CREATION: false + REMOTE_TOKENINFO: true + KPI_COLLECTION: true + DISABLE_DB_WRITE_OPERATIONS: false + AUDIT_LOG_COLLECTION: true + EVENT_OWNER_SELECTOR_AUTHZ: false + ACCESS_LOG_ENABLED: true + --- spring: config: