From cf907a5502f989e1a4bad7003c29332cc0853e3c Mon Sep 17 00:00:00 2001 From: Luuk van Venrooij <11056665+seriva@users.noreply.github.com> Date: Tue, 21 May 2019 14:48:59 +0200 Subject: [PATCH] 0.2.3 release (#262) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * adding smoke tests for kubernetes * Quick fix to the counting function * Regex fix to the 'should match' assertion * A few fixes after review * Added smoke tests for kafka * A few fixes after testing on ubuntu * Fixes after review * Fixes after review * Item: #103 Desc: Remove unused nginx role. (#112) * Item: #103 Desc: Remove unused nginx role. * Removed ansible-role-nginx license from License.md as it is no longer used. * Design doc for cli part * Updated affected version * Typo fix * Azure single machine (#116) * Azure and metal single machine and updates. * Inclusion of deployment role for auth-service. * Fixed file version nr. * Fixes for comments * Item: #24 Desc: Add information to how-to how configure manually fede… (#119) * Item: #24 Desc: Add information to how to how configure manually federation of Prometheus instances * Item: #24 Desc: Changes after review. * Retry install packages task when failes (#131) * make it a little bit more comprehensible for general public (#133) * Update RHEL image to version 7.6.2018103108 (#132) * Refactoring after review * Added condition to check if zookeper is running in standalone mode * Added deletion of topics at the end of tests * Changed assertions for HTTP status codes * Typo correction * Design doc proposal for AWS support * Not public IPs info * Added smoke tests for HAProxy * Add JMX Exporter configuration for standalone ZooKeeper (#137) * Context diagram for Epiphany Platform * Web example application * Fix ZooKeeper distribution URL (#150) * Fixes after review * Typo correction * Changed assertion for HAProxy log file * Item: #0000 Desc: Add port to python3 of template_engine (#151) * Item: #0000 Desc: Add port to python3 of template_engine * Item: #0000 Desc: Fixes according to pull request. * Added smoke tests for logging (#152) * Added smoke tests for logging * Added smoke tests for monitoring (#158) Added smoke tests for monitoring * Added documentation for single machine cluster. (#160) * Added smoke tests for RabbitMQ (#165) * Added smoke tests for deployments (#170) * Added smoke tests for PostgreSQL (#186) * RabbitMQ tests fix (#187) * Bumped alpine version to fix GCC error and fixed yaml.load warning (#207) * Added Dockerfile to create a Docker container image for test environment (#208) * Fixed bundler version (#210) * Added missing ruby package (#211) * Setting the SSH key format to PEM (#214) * Changing build configuration for CI release (#217) * Kafka, Elk and IoTHub connectors - example apps (#224) * Uploading Wiki Testing documentation, describing mapping in TestQuality tool (#219) * Test fix- waiting for pods to be ready (#259) * Fix/example vulnerability fixes (#261) * Fixed python security issues and added changelog for 0.2.2. * Fixed missing changelog * Version fix. * Creating template for QA (#260) --- CHANGELOG-0.2.md | 7 + CHANGELOG.md | 2 + LICENSES.md | 27 - core/bin/template_engine | 2 +- .../src/ansible/roles/common/tasks/RedHat.yml | 6 +- .../reverse-proxy-nginx/defaults/main.yml | 111 -- .../reverse-proxy-nginx/handlers/main.yml | 24 - .../reverse-proxy-nginx/tasks/Debian.yml | 10 - .../reverse-proxy-nginx/tasks/RedHat.yml | 56 - .../reverse-proxy-nginx/tasks/Ubuntu.yml | 14 - .../roles/reverse-proxy-nginx/tasks/main.yml | 19 - .../tasks/reverse-proxies.yml | 20 - .../reverse-proxy-nginx/tasks/setup-ssl.yml | 62 - .../templates/etc/nginx/nginx.conf.j2 | 44 - .../etc/nginx/sites-available/default.conf.j2 | 130 -- .../sites-available/reverse-proxy.conf.j2 | 45 - .../templates/nginx.repo.j2 | 5 - .../roles/reverse-proxy-nginx/vars/Debian.yml | 9 - .../roles/reverse-proxy-nginx/vars/RedHat.yml | 8 - .../zookeeper/files/jmx-zookeeper-config.yml | 8 +- core/core/src/docker/dev/Dockerfile | 2 +- core/core/src/docker/test-CI/Dockerfile | 21 + core/core/src/docker/test-CI/prepare_sp.sh | 58 + core/core/src/docker/test-CI/run.sh | 27 + core/core/test/serverspec/Gemfile | 2 +- core/core/test/serverspec/Rakefile | 44 +- .../deployments/auth-service/auth-service.rb | 57 + .../spec/deployments/deployments_spec.rb | 12 + .../spec/deployments/rabbitmq/rabbitmq.rb | 172 +++ .../elasticsearch-curator_spec.rb | 15 + .../spec/elasticsearch/elasticsearch_spec.rb | 59 +- .../serverspec/spec/filebeat/filebeat_spec.rb | 60 + .../serverspec/spec/grafana/grafana_spec.rb | 107 ++ .../haproxy_exporter/haproxy_exporter_spec.rb | 42 + .../haproxy_tls_termination_spec.rb | 85 ++ .../spec/jmx-exporter/jmx_exporter_spec.rb | 55 +- .../kafka-exporter/kafka_exporter_spec.rb | 41 + .../test/serverspec/spec/kafka/kafka_spec.rb | 119 +- .../serverspec/spec/kibana/kibana_spec.rb | 77 + .../serverspec/spec/master/master_spec.rb | 174 ++- .../spec/node_exporter/node_exporter_spec.rb | 50 +- .../spec/postgresql/postgresql_spec.rb | 268 ++++ .../spec/prometheus/prometheus_spec.rb | 323 +++- .../serverspec/spec/rabbitmq/rabbitmq_spec.rb | 180 +++ core/core/test/serverspec/spec/spec_helper.rb | 52 +- .../spec/zookeeper/zookeeper_spec.rb | 94 +- .../epiphany-bld-apps/data.yaml | 4 +- .../epiphany-playground/basic-data.yaml | 6 +- .../epiphany-qa-basic/basic-data.yaml | 32 + .../epiphany-qa-template/data.yaml.j2 | 1347 +++++++++++++++++ .../epiphany-single-machine/data.yaml | 602 ++++++++ .../data.yaml | 109 +- core/src/core/template_engine_3 | 145 ++ .../diagrams/context/epiphany-platform.png | 3 + docs/design-docs/AWS/aws.md | 26 + docs/design-docs/AWS/aws_cluster_setup.svg | 3 + docs/design-docs/cli/cli.md | 119 ++ docs/design-docs/cli/epiphany-engine.svg | 3 + docs/home/HOWTO.md | 65 + docs/home/TESTING.md | 60 + .../Dockerfile | 20 + ...y.Examples.IoTHub.Connector.Console.csproj | 26 + .../EventProcessorFactory.cs | 25 + .../IoTHubConsumer.cs | 55 + .../Program.cs | 56 + .../SimpleEventProcessor.cs | 110 ++ .../deployment.yaml | 70 + .../Controllers/HomeController.cs | 44 + .../dotnet/Epiphany.Examples.Web/Dockerfile | 19 + .../Epiphany.Examples.Web.csproj | 17 + .../Models/ErrorViewModel.cs | 9 + .../dotnet/Epiphany.Examples.Web/Program.cs | 17 + .../dotnet/Epiphany.Examples.Web/Startup.cs | 60 + .../Views/Home/Index.cshtml | 25 + .../Views/Home/InstanceInfo.cshtml | 11 + .../Views/Home/Privacy.cshtml | 6 + .../Views/Shared/Error.cshtml | 25 + .../Views/Shared/_CookieConsentPartial.cshtml | 25 + .../Views/Shared/_Layout.cshtml | 80 + .../Shared/_ValidationScriptsPartial.cshtml | 18 + .../Views/_ViewImports.cshtml | 3 + .../Views/_ViewStart.cshtml | 3 + .../appsettings.Development.json | 9 + .../Epiphany.Examples.Web/appsettings.json | 8 + .../Epiphany.Examples.Web/deployment.yaml | 91 ++ examples/dotnet/Epiphany.Examples.sln | 12 + .../Producer/KafkaProducer.cs | 6 +- .../Epiphany.Examples.Messaging/IProducer.cs | 2 +- .../RabbitMqProducer.cs | 2 +- examples/dotnet/docker-compose.override.yml | 8 +- examples/dotnet/docker-compose.yml | 14 + examples/go/kafka-elk-connector/Dockerfile | 30 + .../go/kafka-elk-connector/deployment.yaml | 33 + examples/go/kafka-elk-connector/go.mod | 7 + examples/go/kafka-elk-connector/main.go | 120 ++ examples/keycloak/Readme.md | 4 +- .../keycloak/authorization/python/Dockerfile | 2 +- .../keycloak/authorization/python/Pipfile | 2 +- .../authorization/python/Pipfile.lock | 111 +- .../keycloak/authorization/python/main.py | 1 + examples/keycloak/implicit/python/Dockerfile | 2 +- examples/keycloak/implicit/python/Pipfile | 2 +- .../keycloak/implicit/python/Pipfile.lock | 98 +- examples/keycloak/implicit/python/main.py | 1 + 104 files changed, 5762 insertions(+), 856 deletions(-) delete mode 100644 core/core/src/ansible/roles/reverse-proxy-nginx/defaults/main.yml delete mode 100644 core/core/src/ansible/roles/reverse-proxy-nginx/handlers/main.yml delete mode 100644 core/core/src/ansible/roles/reverse-proxy-nginx/tasks/Debian.yml delete mode 100644 core/core/src/ansible/roles/reverse-proxy-nginx/tasks/RedHat.yml delete mode 100644 core/core/src/ansible/roles/reverse-proxy-nginx/tasks/Ubuntu.yml delete mode 100644 core/core/src/ansible/roles/reverse-proxy-nginx/tasks/main.yml delete mode 100644 core/core/src/ansible/roles/reverse-proxy-nginx/tasks/reverse-proxies.yml delete mode 100644 core/core/src/ansible/roles/reverse-proxy-nginx/tasks/setup-ssl.yml delete mode 100644 core/core/src/ansible/roles/reverse-proxy-nginx/templates/etc/nginx/nginx.conf.j2 delete mode 100644 core/core/src/ansible/roles/reverse-proxy-nginx/templates/etc/nginx/sites-available/default.conf.j2 delete mode 100644 core/core/src/ansible/roles/reverse-proxy-nginx/templates/etc/nginx/sites-available/reverse-proxy.conf.j2 delete mode 100644 core/core/src/ansible/roles/reverse-proxy-nginx/templates/nginx.repo.j2 delete mode 100644 core/core/src/ansible/roles/reverse-proxy-nginx/vars/Debian.yml delete mode 100644 core/core/src/ansible/roles/reverse-proxy-nginx/vars/RedHat.yml create mode 100644 core/core/src/docker/test-CI/Dockerfile create mode 100644 core/core/src/docker/test-CI/prepare_sp.sh create mode 100644 core/core/src/docker/test-CI/run.sh create mode 100644 core/core/test/serverspec/spec/deployments/auth-service/auth-service.rb create mode 100644 core/core/test/serverspec/spec/deployments/deployments_spec.rb create mode 100644 core/core/test/serverspec/spec/deployments/rabbitmq/rabbitmq.rb create mode 100644 core/core/test/serverspec/spec/elasticsearch-curator/elasticsearch-curator_spec.rb create mode 100644 core/core/test/serverspec/spec/filebeat/filebeat_spec.rb create mode 100644 core/core/test/serverspec/spec/grafana/grafana_spec.rb create mode 100644 core/core/test/serverspec/spec/haproxy_exporter/haproxy_exporter_spec.rb create mode 100644 core/core/test/serverspec/spec/haproxy_tls_termination/haproxy_tls_termination_spec.rb create mode 100644 core/core/test/serverspec/spec/kafka-exporter/kafka_exporter_spec.rb create mode 100644 core/core/test/serverspec/spec/kibana/kibana_spec.rb create mode 100644 core/core/test/serverspec/spec/postgresql/postgresql_spec.rb create mode 100644 core/core/test/serverspec/spec/rabbitmq/rabbitmq_spec.rb create mode 100644 core/data/azure/infrastructure/epiphany-qa-basic/basic-data.yaml create mode 100644 core/data/azure/infrastructure/epiphany-qa-template/data.yaml.j2 create mode 100644 core/data/azure/infrastructure/epiphany-single-machine/data.yaml rename core/data/metal/{single-machine => epiphany-single-machine}/data.yaml (71%) create mode 100755 core/src/core/template_engine_3 create mode 100644 docs/assets/images/diagrams/context/epiphany-platform.png create mode 100644 docs/design-docs/AWS/aws.md create mode 100644 docs/design-docs/AWS/aws_cluster_setup.svg create mode 100644 docs/design-docs/cli/cli.md create mode 100644 docs/design-docs/cli/epiphany-engine.svg create mode 100644 docs/home/TESTING.md create mode 100644 examples/dotnet/Epiphany.Examples.IoTHub.Connector.Console/Dockerfile create mode 100644 examples/dotnet/Epiphany.Examples.IoTHub.Connector.Console/Epiphany.Examples.IoTHub.Connector.Console.csproj create mode 100644 examples/dotnet/Epiphany.Examples.IoTHub.Connector.Console/EventProcessorFactory.cs create mode 100644 examples/dotnet/Epiphany.Examples.IoTHub.Connector.Console/IoTHubConsumer.cs create mode 100644 examples/dotnet/Epiphany.Examples.IoTHub.Connector.Console/Program.cs create mode 100644 examples/dotnet/Epiphany.Examples.IoTHub.Connector.Console/SimpleEventProcessor.cs create mode 100644 examples/dotnet/Epiphany.Examples.IoTHub.Connector.Console/deployment.yaml create mode 100644 examples/dotnet/Epiphany.Examples.Web/Controllers/HomeController.cs create mode 100644 examples/dotnet/Epiphany.Examples.Web/Dockerfile create mode 100644 examples/dotnet/Epiphany.Examples.Web/Epiphany.Examples.Web.csproj create mode 100644 examples/dotnet/Epiphany.Examples.Web/Models/ErrorViewModel.cs create mode 100644 examples/dotnet/Epiphany.Examples.Web/Program.cs create mode 100644 examples/dotnet/Epiphany.Examples.Web/Startup.cs create mode 100644 examples/dotnet/Epiphany.Examples.Web/Views/Home/Index.cshtml create mode 100644 examples/dotnet/Epiphany.Examples.Web/Views/Home/InstanceInfo.cshtml create mode 100644 examples/dotnet/Epiphany.Examples.Web/Views/Home/Privacy.cshtml create mode 100644 examples/dotnet/Epiphany.Examples.Web/Views/Shared/Error.cshtml create mode 100644 examples/dotnet/Epiphany.Examples.Web/Views/Shared/_CookieConsentPartial.cshtml create mode 100644 examples/dotnet/Epiphany.Examples.Web/Views/Shared/_Layout.cshtml create mode 100644 examples/dotnet/Epiphany.Examples.Web/Views/Shared/_ValidationScriptsPartial.cshtml create mode 100644 examples/dotnet/Epiphany.Examples.Web/Views/_ViewImports.cshtml create mode 100644 examples/dotnet/Epiphany.Examples.Web/Views/_ViewStart.cshtml create mode 100644 examples/dotnet/Epiphany.Examples.Web/appsettings.Development.json create mode 100644 examples/dotnet/Epiphany.Examples.Web/appsettings.json create mode 100644 examples/dotnet/Epiphany.Examples.Web/deployment.yaml create mode 100644 examples/go/kafka-elk-connector/Dockerfile create mode 100644 examples/go/kafka-elk-connector/deployment.yaml create mode 100644 examples/go/kafka-elk-connector/go.mod create mode 100644 examples/go/kafka-elk-connector/main.go diff --git a/CHANGELOG-0.2.md b/CHANGELOG-0.2.md index a103970cdf..578361521c 100644 --- a/CHANGELOG-0.2.md +++ b/CHANGELOG-0.2.md @@ -1,5 +1,12 @@ # Changelog 0.2 +## [0.2.3] 2019-05-20 + +### Fixed + +- Fixed vulnerability in Docker development image +- Upgraded Keycloak Python examples to python 3.7.2 + ## [0.2.2] 2019-03-29 ### Added diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c4c167621..ce7bdf332e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## Current release +- [CHANGELOG-0.2.3](./CHANGELOG-0.2.md#023-2019-05-20) +- [CHANGELOG-0.2.2](./CHANGELOG-0.2.md#022-2019-03-29) - [CHANGELOG-0.2.1](./CHANGELOG-0.2.md#021-2019-03-07) - [CHANGELOG-0.2.0](./CHANGELOG-0.2.md#020-2019-02-19) diff --git a/LICENSES.md b/LICENSES.md index 8eded1b5ac..744fe2e55f 100644 --- a/LICENSES.md +++ b/LICENSES.md @@ -1,32 +1,5 @@ # Licenses of Epiphany dependencies -## ansible-role-nginx - -[Github repository](https://github.com/geerlingguy/ansible-role-nginx/) - -``` -The MIT License (MIT) - -Copyright (c) 2017 Jeff Geerling - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of -the Software, and to permit persons to whom the Software is furnished to do so, -subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS -FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR -COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -``` - ## flannel [Github repository](https://github.com/coreos/flannel/) diff --git a/core/bin/template_engine b/core/bin/template_engine index ba8bdb0610..3cc4617413 100755 --- a/core/bin/template_engine +++ b/core/bin/template_engine @@ -60,7 +60,7 @@ def render_template(data_file, in_file, out_file, json_arg, yaml_arg): env.filters['jsonify'] = json.dumps with open(data_file) as data: - dict = yaml.load(data) + dict = yaml.load(data, Loader=yaml.FullLoader) # Render template and print generated config to console template = env.get_template(in_file) diff --git a/core/core/src/ansible/roles/common/tasks/RedHat.yml b/core/core/src/ansible/roles/common/tasks/RedHat.yml index b8d1424865..d9691a553d 100644 --- a/core/core/src/ansible/roles/common/tasks/RedHat.yml +++ b/core/core/src/ansible/roles/common/tasks/RedHat.yml @@ -19,7 +19,7 @@ gpgcheck: yes gpgkey: https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg -- name: Install redhat family packages +- name: Install RedHat family packages yum: name: - libselinux-python @@ -47,6 +47,10 @@ - telnet update_cache: yes state: present + register: result + retries: 3 + delay: 1 + until: result is succeeded - name: Append prompt to .bash_profile lineinfile: diff --git a/core/core/src/ansible/roles/reverse-proxy-nginx/defaults/main.yml b/core/core/src/ansible/roles/reverse-proxy-nginx/defaults/main.yml deleted file mode 100644 index a296cd3c48..0000000000 --- a/core/core/src/ansible/roles/reverse-proxy-nginx/defaults/main.yml +++ /dev/null @@ -1,111 +0,0 @@ ---- -nginx_install_service: True - -nginx_user: 'nginx' -nginx_worker_processes: 'auto' -nginx_worker_rlimit_nofile: 4096 -nginx_events_worker_connections: 1024 -nginx_http_server_tokens: 'off' -nginx_http_add_headers: - - 'X-Frame-Options SAMEORIGIN' - - 'X-Content-Type-Options nosniff' - - 'X-XSS-Protection "1; mode=block"' -nginx_http_access_log_format: | - '$remote_addr $remote_user $request_time $upstream_response_time $msec [$time_local] ' - '"$request" $status $body_bytes_sent ' - '"$http_referer" "$http_user_agent"' -nginx_http_server_names_hash_bucket_size: 64 -nginx_http_server_names_hash_max_size: 512 -nginx_http_sendfile: 'on' -nginx_http_tcp_nopush: 'on' -nginx_http_keepalive_timeout: 60 -nginx_http_client_max_body_size: '1m' -nginx_http_types_hash_max_size: 2048 -nginx_http_gzip: 'on' -nginx_http_gzip_types: 'text/plain text/css application/javascript application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript image/svg+xml image/svg' -nginx_http_gzip_disable: 'msie6' -nginx_http_directives: [] - -nginx_basic_auth: - - user: test - password: test - -nginx_ssl_dhparam_bits: 2048 -nginx_ssl_override_filename: '' -nginx_ssl_generate_self_signed_certs: True - -nginx_default_sites: - default: - domains: [] - default_server: False - listen_http: 80 - listen_https: 443 - root: '/usr/share/nginx/html' - directives: [] - ssl: - protocols: 'TLSv1 TLSv1.1 TLSv1.2' - ciphers: 'ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA:ECDHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES128-SHA256:DHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES256-GCM-SHA384:AES128-GCM-SHA256:AES256-SHA256:AES128-SHA256:AES256-SHA:AES128-SHA:DES-CBC3-SHA:HIGH:!aNULL:!eNULL:!EXPORT:!DES:!MD5:!PSK:!RC4' - prefer_server_ciphers: 'on' - session_cache: 'shared:SSL:50m' - session_timeout: '5m' - ssl_stapling: 'on' - ssl_stapling_verify: 'on' - resolver: '8.8.8.8' - resolver_timeout: '5s' - sts_header: 'Strict-Transport-Security "max-age=63072000; includeSubdomains;"' - cache_all_locations: - enabled: True - duration: '30s' - error_pages: - - { code: 404, page: '404.html' } - - { code: 500, page: '500.html' } - serve_assets: - enabled: True - pattern: ' ~ ^/assets/' - expires: 'max' - custom_locations: '' - custom_root_location_try_files: '' - basic_auth: False - basic_auth_message: 'Please sign in' - disallow_hidden_files: - enabled: True - upstreams: [] - -nginx_default_upstream_proxy_settings: - - 'proxy_set_header X-Real-IP $remote_addr' - - 'proxy_set_header X-Forwarded-Proto $scheme' - - 'proxy_set_header Host $http_host' - - 'proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for' - - 'proxy_redirect off' - -nginx_sites: - default: - domains: ['default'] - default_server: True - -reverse_proxies: - - prometheus: - domains: - - epimaster1.domain.com - config_name: prometheus - backend_name: prometheus - backends: - - localhost:9090 - listen_https: 444 - domain: epimaster1.domain.com - basic_auth: True - basic_auth_message: 'Please sign in' - ssl: - protocols: 'TLSv1 TLSv1.1 TLSv1.2' - ciphers: 'ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA:ECDHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES128-SHA256:DHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES256-GCM-SHA384:AES128-GCM-SHA256:AES256-SHA256:AES128-SHA256:AES256-SHA:AES128-SHA:DES-CBC3-SHA:HIGH:!aNULL:!eNULL:!EXPORT:!DES:!MD5:!PSK:!RC4' - prefer_server_ciphers: 'on' - session_cache: 'shared:SSL:50m' - session_timeout: '5m' - ssl_stapling: 'on' - ssl_stapling_verify: 'on' - resolver: '8.8.8.8' - resolver_timeout: '5s' - sts_header: 'Strict-Transport-Security "max-age=63072000; includeSubdomains;"' - cache_all_locations: - enabled: True - duration: '30s' diff --git a/core/core/src/ansible/roles/reverse-proxy-nginx/handlers/main.yml b/core/core/src/ansible/roles/reverse-proxy-nginx/handlers/main.yml deleted file mode 100644 index c1532652da..0000000000 --- a/core/core/src/ansible/roles/reverse-proxy-nginx/handlers/main.yml +++ /dev/null @@ -1,24 +0,0 @@ ---- -- name: validate nginx configuration - command: nginx -t -c /etc/nginx/nginx.conf - changed_when: False - -- name: Test nginx and restart - command: nginx -t - notify: - - 'restart nginx' - -- name: Test nginx and reload - command: nginx -t - notify: - - 'reload nginx' - -- name: restart nginx - service: - name: 'nginx' - state: 'restarted' - -- name: reload nginx - service: - name: 'nginx' - state: 'reloaded' \ No newline at end of file diff --git a/core/core/src/ansible/roles/reverse-proxy-nginx/tasks/Debian.yml b/core/core/src/ansible/roles/reverse-proxy-nginx/tasks/Debian.yml deleted file mode 100644 index 554dc2e777..0000000000 --- a/core/core/src/ansible/roles/reverse-proxy-nginx/tasks/Debian.yml +++ /dev/null @@ -1,10 +0,0 @@ ---- -- name: Update apt cache. - apt: update_cache=yes cache_valid_time=86400 - changed_when: false - -- name: Ensure nginx is installed. - apt: - name: "{{ nginx_package_name }}" - state: present - default_release: "{{ nginx_default_release }}" diff --git a/core/core/src/ansible/roles/reverse-proxy-nginx/tasks/RedHat.yml b/core/core/src/ansible/roles/reverse-proxy-nginx/tasks/RedHat.yml deleted file mode 100644 index 066a7b2ba7..0000000000 --- a/core/core/src/ansible/roles/reverse-proxy-nginx/tasks/RedHat.yml +++ /dev/null @@ -1,56 +0,0 @@ ---- -- name: Install role dependencies - yum: - name: 'python-passlib' - state: 'present' - when: nginx_basic_auth - -- name: Enable nginx repo. - template: - src: nginx.repo.j2 - dest: /etc/yum.repos.d/nginx.repo - owner: root - group: root - mode: 0644 - -- name: Ensure nginx is installed. - yum: - name: nginx - state: present - -- name: Create default nginx directories - file: - path: '{{ item }}' - state: 'directory' - owner: 'root' - group: 'root' - mode: '0755' - with_items: - - '/usr/share/nginx/html' - - '/etc/nginx/sites-available' - - '/etc/nginx/sites-enabled' - - '/etc/nginx/conf.d' - - '/etc/nginx/ssl' - -- name: Remove default site - file: - path: '{{ item }}' - state: 'absent' - with_items: - - '/var/www/html' - - '/etc/nginx/sites-enabled/default' - - '/etc/nginx/sites-available/default' - notify: - - Test nginx and reload - -- name: Configure nginx - template: - src: 'etc/nginx/nginx.conf.j2' - dest: '/etc/nginx/nginx.conf' - group: 'root' - owner: 'root' - mode: '0644' - register: nginx_register_nginx_config - notify: - - Test nginx and reload - diff --git a/core/core/src/ansible/roles/reverse-proxy-nginx/tasks/Ubuntu.yml b/core/core/src/ansible/roles/reverse-proxy-nginx/tasks/Ubuntu.yml deleted file mode 100644 index 920dc0b2b6..0000000000 --- a/core/core/src/ansible/roles/reverse-proxy-nginx/tasks/Ubuntu.yml +++ /dev/null @@ -1,14 +0,0 @@ ---- -- name: Add PPA for Nginx. - apt_repository: - repo: 'ppa:nginx/{{ nginx_ppa_version }}' - state: present - update_cache: yes - register: nginx_ppa_added - when: nginx_ppa_use - -- name: Ensure nginx will reinstall if the PPA was just added. - apt: - name: nginx - state: absent - when: nginx_ppa_added.changed diff --git a/core/core/src/ansible/roles/reverse-proxy-nginx/tasks/main.yml b/core/core/src/ansible/roles/reverse-proxy-nginx/tasks/main.yml deleted file mode 100644 index 05b49c6ef1..0000000000 --- a/core/core/src/ansible/roles/reverse-proxy-nginx/tasks/main.yml +++ /dev/null @@ -1,19 +0,0 @@ ---- -# Variable setup. -- name: Include OS-specific variables. - include_vars: "{{ ansible_os_family }}.yml" - -- name: Define nginx_user. - set_fact: - nginx_user: "{{ __nginx_user }}" - when: nginx_user is not defined - -# Setup/install tasks. -- include_tasks: "{{ ansible_os_family }}.yml" - -- include_tasks: setup-ssl.yml - -- include_tasks: reverse-proxies.yml - -- name: Ensure nginx is started and enabled to start at boot. - systemd: name=nginx state=started enabled=yes diff --git a/core/core/src/ansible/roles/reverse-proxy-nginx/tasks/reverse-proxies.yml b/core/core/src/ansible/roles/reverse-proxy-nginx/tasks/reverse-proxies.yml deleted file mode 100644 index 32e1c0035c..0000000000 --- a/core/core/src/ansible/roles/reverse-proxy-nginx/tasks/reverse-proxies.yml +++ /dev/null @@ -1,20 +0,0 @@ - -- name: Configure reverse-proxies - template: - src: 'etc/nginx/sites-available/reverse-proxy.conf.j2' - dest: '/etc/nginx/sites-available/{{ item.config_name }}.conf' - group: 'root' - owner: 'root' - mode: '0644' - with_items: '{{ reverse_proxies }}' - notify: - - Test nginx and reload - -- name: Symlink sites-available to sites-enabled - reverse proxies - file: - src: '/etc/nginx/sites-available/{{ item.config_name }}.conf' - dest: '/etc/nginx/sites-enabled/{{ item.config_name }}.conf' - state: 'link' - with_items: '{{ reverse_proxies }}' - notify: - - Test nginx and restart \ No newline at end of file diff --git a/core/core/src/ansible/roles/reverse-proxy-nginx/tasks/setup-ssl.yml b/core/core/src/ansible/roles/reverse-proxy-nginx/tasks/setup-ssl.yml deleted file mode 100644 index 0995526a81..0000000000 --- a/core/core/src/ansible/roles/reverse-proxy-nginx/tasks/setup-ssl.yml +++ /dev/null @@ -1,62 +0,0 @@ -- name: Generate self signed SSL certificates - command: > - openssl req - -new - -newkey rsa:4096 - -days 365 - -nodes - -x509 - -subj "/C=US/ST=NY/L=NY/O=NA/CN=localhost" - -keyout /etc/nginx/ssl/{{ nginx_ssl_override_filename | default(item.value.domains[0]) }}.key - -out /etc/nginx/ssl/{{ nginx_ssl_override_filename | default(item.value.domains[0]) }}.pem - args: - creates: '/etc/nginx/ssl/{{ nginx_ssl_override_filename | default(item.value.domains[0]) }}.pem' - with_dict: '{{ nginx_sites }}' - when: nginx_ssl_generate_self_signed_certs - notify: - - Test nginx and restart - -- name: Generate X bit dhparam.pem file (this may take a while) - command: openssl dhparam -out /etc/nginx/ssl/dhparam.pem {{ nginx_ssl_dhparam_bits }} - args: - creates: '/etc/nginx/ssl/dhparam.pem' - notify: - - Test nginx and restart - -- name: Create basic auth entries - htpasswd: - path: '/etc/nginx/.htpasswd' - name: '{{ item.user }}' - password: '{{ item.password }}' - crypt_scheme: 'sha512_crypt' - group: 'root' - owner: 'root' - mode: '0644' - with_items: '{{ nginx_basic_auth }}' - -- name: Configure sites-enabled (vhosts) - template: - src: 'etc/nginx/sites-available/default.conf.j2' - dest: '/etc/nginx/sites-available/{{ item.key }}.conf' - group: 'root' - owner: 'root' - mode: '0644' - with_dict: '{{ nginx_sites }}' - register: nginx_register_vhost_config - notify: - - Test nginx and reload - -- name: Symlink sites-available to sites-enabled - file: - src: '/etc/nginx/sites-available/{{ item.key }}.conf' - dest: '/etc/nginx/sites-enabled/{{ item.key }}.conf' - state: 'link' - with_dict: '{{ nginx_sites }}' - notify: - - Test nginx and restart - -- name: Forcefully restart nginx - service: - name: 'nginx' - state: 'restarted' - when: (nginx_register_nginx_config | changed) and (nginx_register_vhost_config | changed) \ No newline at end of file diff --git a/core/core/src/ansible/roles/reverse-proxy-nginx/templates/etc/nginx/nginx.conf.j2 b/core/core/src/ansible/roles/reverse-proxy-nginx/templates/etc/nginx/nginx.conf.j2 deleted file mode 100644 index 6d416f0a4c..0000000000 --- a/core/core/src/ansible/roles/reverse-proxy-nginx/templates/etc/nginx/nginx.conf.j2 +++ /dev/null @@ -1,44 +0,0 @@ -# {{ ansible_managed }} - -user {{ nginx_user }}; -worker_processes {{ nginx_worker_processes }}; -worker_rlimit_nofile {{ nginx_worker_rlimit_nofile }}; - -events { - worker_connections {{ nginx_events_worker_connections }}; -} - -http { - include /etc/nginx/mime.types; - default_type application/octet-stream; - - server_tokens {{ nginx_http_server_tokens }}; - {% if nginx_http_add_headers is iterable %} - {%- for header in nginx_http_add_headers -%} - add_header {{ header }}; - {% endfor -%} - {% endif %} - - log_format combined_custom {{ nginx_http_access_log_format }}; - access_log /var/log/nginx/access.log combined_custom; - - server_names_hash_bucket_size {{ nginx_http_server_names_hash_bucket_size }}; - server_names_hash_max_size {{ nginx_http_server_names_hash_max_size }}; - sendfile {{ nginx_http_sendfile }}; - tcp_nopush {{ nginx_http_tcp_nopush }}; - keepalive_timeout {{ nginx_http_keepalive_timeout }}; - types_hash_max_size {{ nginx_http_types_hash_max_size }}; - client_max_body_size {{ nginx_http_client_max_body_size }}; - gzip {{ nginx_http_gzip }}; - gzip_types {{ nginx_http_gzip_types }}; - gzip_disable "{{ nginx_http_gzip_disable }}"; - - {% if nginx_http_directives is iterable -%} - {% for key in nginx_http_directives %} - {{ key }}; - {% endfor %} - {% endif %} - - include /etc/nginx/conf.d/*.conf; - include /etc/nginx/sites-enabled/*; -} \ No newline at end of file diff --git a/core/core/src/ansible/roles/reverse-proxy-nginx/templates/etc/nginx/sites-available/default.conf.j2 b/core/core/src/ansible/roles/reverse-proxy-nginx/templates/etc/nginx/sites-available/default.conf.j2 deleted file mode 100644 index eeb2f63353..0000000000 --- a/core/core/src/ansible/roles/reverse-proxy-nginx/templates/etc/nginx/sites-available/default.conf.j2 +++ /dev/null @@ -1,130 +0,0 @@ -# {{ ansible_managed }} - -{% set item = (nginx_default_sites['default'] | combine(item.value, recursive=True)) %} - -{% if item.upstreams %} -{%- for upstream in item.upstreams %} - -upstream {{ upstream.name }} { -{% for server in upstream.servers %} - server {{ server }}; -{% endfor %} -} -{% endfor %} -{% endif %} - -server { - listen {{ item.listen_http }}{{ ' default deferred' if item.default_server else '' }}; - server_name {{ item.domains | join(' ') }}; - - location /.well-known/acme-challenge/ { - default_type "text/plain"; - - try_files $uri =404; - } - - location / { - return 301 https://$host$request_uri; - } -} - -{% if (item.domains | length) > 1 %} -server { - listen {{ item.listen_https }} ssl; - server_name {{ item.domains[1] }}; - - return 301 https://{{ item.domains[0] }}$request_uri; -} -{% endif %} - -server { - listen {{ item.listen_https }} ssl{{ ' default deferred' if item.default_server else '' }}; - server_name {{ item.domains[0] }}; - root {{ item.root }}; -{% if item.directives is iterable %} -{% for directive in item.directives %} -{{ directive | indent(2, True) }}; -{% endfor %} -{% endif %} - - ssl_protocols {{ item.ssl.protocols }}; - ssl_ciphers "{{ item.ssl.ciphers }}"; - ssl_prefer_server_ciphers {{ item.ssl.prefer_server_ciphers }}; - ssl_session_cache {{ item.ssl.session_cache }}; - ssl_session_timeout {{ item.ssl.session_timeout }}; - ssl_stapling {{ item.ssl.ssl_stapling }}; - ssl_stapling_verify {{ item.ssl.ssl_stapling_verify }}; - resolver {{ item.ssl.resolver }}; - resolver_timeout {{ item.ssl.resolver_timeout }}; - add_header {{ item.ssl.sts_header }}; - - ssl_dhparam /etc/nginx/ssl/dhparam.pem; - ssl_certificate /etc/nginx/ssl/{{ nginx_ssl_override_filename | default(item.domains[0]) }}.pem; - ssl_certificate_key /etc/nginx/ssl/{{ nginx_ssl_override_filename | default(item.domains[0]) }}.key; - ssl_trusted_certificate /etc/nginx/ssl/{{ nginx_ssl_override_filename | default(item.domains[0]) }}.pem; - -{% if item.cache_all_locations.enabled %} - expires {{ item.cache_all_locations.duration }}; - add_header Cache-Control public; - add_header ETag ""; -{% endif %} - -{% if item.error_pages is iterable %} -{% for error in item.error_pages %} - error_page {{ error.code }} /{{ error.page }}; -{% endfor %} -{% endif %} -{% if item.serve_assets.enabled %} - - location {{ item.serve_assets.pattern }} { - expires {{ item.serve_assets.expires }}; - } -{% endif %} -{% if item.custom_locations %} - {{ item.custom_locations|indent(2) }} -{% endif %} - -{% if item.disallow_hidden_files.enabled %} - location ~ /\. { - return 404; - access_log off; - log_not_found off; - } -{% endif %} - - location = /favicon.ico { - try_files /favicon.ico =204; - access_log off; - log_not_found off; - } - - location / { -{% if item.custom_root_location_try_files %} - try_files {{ item.custom_root_location_try_files }}; -{% else %} - try_files $uri $uri.html $uri/{{ (' @' + item.upstreams[0].name) if (item.upstreams) else '' }} =404; -{% endif %} -{% if item.basic_auth | bool %} - auth_basic "{{ item.basic_auth_message }}"; - auth_basic_user_file /etc/nginx/.htpasswd; -{% endif %} - } -{% if item.upstreams %} -{% for upstream in item.upstreams %} - - location @{{ upstream.name }} { -{% if nginx_default_upstream_proxy_settings is iterable -%} -{% for key in nginx_default_upstream_proxy_settings %} - {{ key }}; -{% endfor %} -{% endif %} -{% if upstream.add_proxy_settings is defined -%} -{% for setting in upstream.add_proxy_settings %} - {{ setting }}; -{% endfor %} -{% endif %} - proxy_pass http://{{ upstream.name }}; - } -{% endfor %} -{% endif %} -} diff --git a/core/core/src/ansible/roles/reverse-proxy-nginx/templates/etc/nginx/sites-available/reverse-proxy.conf.j2 b/core/core/src/ansible/roles/reverse-proxy-nginx/templates/etc/nginx/sites-available/reverse-proxy.conf.j2 deleted file mode 100644 index ab8da01e90..0000000000 --- a/core/core/src/ansible/roles/reverse-proxy-nginx/templates/etc/nginx/sites-available/reverse-proxy.conf.j2 +++ /dev/null @@ -1,45 +0,0 @@ -# {{ ansible_managed }} - -upstream {{ item.backend_name }} { - {% for backend in item.backends %} - server {{ backend }}; - {% endfor %} -} - -server { - listen {{ item.listen_https }} ssl; - server_name {{ item.domain }}; - - ssl_protocols {{ item.ssl.protocols }}; - ssl_ciphers "{{ item.ssl.ciphers }}"; - ssl_prefer_server_ciphers {{ item.ssl.prefer_server_ciphers }}; - ssl_session_cache {{ item.ssl.session_cache }}; - ssl_session_timeout {{ item.ssl.session_timeout }}; - ssl_stapling {{ item.ssl.ssl_stapling }}; - ssl_stapling_verify {{ item.ssl.ssl_stapling_verify }}; - resolver {{ item.ssl.resolver }}; - resolver_timeout {{ item.ssl.resolver_timeout }}; - add_header {{ item.ssl.sts_header }}; - - ssl_dhparam /etc/nginx/ssl/dhparam.pem; - ssl_certificate /etc/nginx/ssl/{{ nginx_ssl_override_filename | default(item.domains[0]) }}.pem; - ssl_certificate_key /etc/nginx/ssl/{{ nginx_ssl_override_filename | default(item.domains[0]) }}.key; - ssl_trusted_certificate /etc/nginx/ssl/{{ nginx_ssl_override_filename | default(item.domains[0]) }}.pem; - - {% if item.cache_all_locations.enabled %} - expires {{ item.cache_all_locations.duration }}; - add_header Cache-Control public; - add_header ETag ""; - {% endif %} - - {% if item.basic_auth | bool %} - auth_basic "{{ item.basic_auth_message }}"; - auth_basic_user_file /etc/nginx/.htpasswd; - {% endif %} - - location / { - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header Host $http_host; - proxy_pass http://{{ item.backend_name }}; - } -} diff --git a/core/core/src/ansible/roles/reverse-proxy-nginx/templates/nginx.repo.j2 b/core/core/src/ansible/roles/reverse-proxy-nginx/templates/nginx.repo.j2 deleted file mode 100644 index 9a853b70b0..0000000000 --- a/core/core/src/ansible/roles/reverse-proxy-nginx/templates/nginx.repo.j2 +++ /dev/null @@ -1,5 +0,0 @@ -[nginx] -name=nginx repo -baseurl=http://nginx.org/packages/centos/{{ ansible_distribution_major_version }}/$basearch/ -gpgcheck=0 -enabled=1 diff --git a/core/core/src/ansible/roles/reverse-proxy-nginx/vars/Debian.yml b/core/core/src/ansible/roles/reverse-proxy-nginx/vars/Debian.yml deleted file mode 100644 index cb127706c5..0000000000 --- a/core/core/src/ansible/roles/reverse-proxy-nginx/vars/Debian.yml +++ /dev/null @@ -1,9 +0,0 @@ ---- -root_group: root -nginx_conf_path: /etc/nginx/conf.d -nginx_conf_file_path: /etc/nginx/nginx.conf -nginx_mime_file_path: /etc/nginx/mime.types -nginx_pidfile: /run/nginx.pid -nginx_vhost_path: /etc/nginx/sites-enabled -nginx_default_vhost_path: /etc/nginx/sites-enabled/default -__nginx_user: "www-data" diff --git a/core/core/src/ansible/roles/reverse-proxy-nginx/vars/RedHat.yml b/core/core/src/ansible/roles/reverse-proxy-nginx/vars/RedHat.yml deleted file mode 100644 index c54d89f88b..0000000000 --- a/core/core/src/ansible/roles/reverse-proxy-nginx/vars/RedHat.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -root_group: root -nginx_conf_path: /etc/nginx/conf.d -nginx_conf_file_path: /etc/nginx/nginx.conf -nginx_mime_file_path: /etc/nginx/mime.types -nginx_pidfile: /var/run/nginx.pid -nginx_vhost_path: /etc/nginx/conf.d -nginx_default_vhost_path: /etc/nginx/conf.d/default.conf diff --git a/core/core/src/ansible/roles/zookeeper/files/jmx-zookeeper-config.yml b/core/core/src/ansible/roles/zookeeper/files/jmx-zookeeper-config.yml index 578d6ce414..e00095aa73 100644 --- a/core/core/src/ansible/roles/zookeeper/files/jmx-zookeeper-config.yml +++ b/core/core/src/ansible/roles/zookeeper/files/jmx-zookeeper-config.yml @@ -1,4 +1,5 @@ rules: + # replicated Zookeeper - pattern: "org.apache.ZooKeeperService<>(\\w+)" name: "zookeeper_$2" - pattern: "org.apache.ZooKeeperService<>(\\w+)" @@ -14,4 +15,9 @@ rules: name: "zookeeper_$4_$5" labels: replicaId: "$2" - memberType: "$3" \ No newline at end of file + memberType: "$3" + # standalone Zookeeper + - pattern: "org.apache.ZooKeeperService<>(\\w+)" + name: "zookeeper_$2" + - pattern: "org.apache.ZooKeeperService<>(\\w+)" + name: "zookeeper_$2" \ No newline at end of file diff --git a/core/core/src/docker/dev/Dockerfile b/core/core/src/docker/dev/Dockerfile index 648c9a7c28..1008d696f6 100644 --- a/core/core/src/docker/dev/Dockerfile +++ b/core/core/src/docker/dev/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:3.7 +FROM alpine:3.9.3 ENV TERRAFORM_VERSION 0.11.9 ENV ANSIBLE_VERSION 2.7 diff --git a/core/core/src/docker/test-CI/Dockerfile b/core/core/src/docker/test-CI/Dockerfile new file mode 100644 index 0000000000..7c69c606b6 --- /dev/null +++ b/core/core/src/docker/test-CI/Dockerfile @@ -0,0 +1,21 @@ +FROM epiphanyregistry.azurecr.io/epiphany-deploy:latest + +RUN mkdir /tmp/keys + +RUN ssh-keygen -q -t rsa -m PEM -N '' -f /tmp/keys/id_rsa + +RUN apk add --no-cache ruby ruby-rdoc ruby-irb ruby-rake ruby-etc + +RUN gem install bundler -v 1.16.3 + +RUN bundle config --global silence_root_warning 1 + +WORKDIR /epiphany/core/core/test/serverspec + +RUN bundle install + +WORKDIR /epiphany + +RUN chmod a+x /epiphany/core/core/src/docker/test-CI/run.sh + +CMD ["/epiphany/core/core/src/docker/test-CI/run.sh"] diff --git a/core/core/src/docker/test-CI/prepare_sp.sh b/core/core/src/docker/test-CI/prepare_sp.sh new file mode 100644 index 0000000000..21a4f1a618 --- /dev/null +++ b/core/core/src/docker/test-CI/prepare_sp.sh @@ -0,0 +1,58 @@ +mkdir -p tmp/sp +mkdir -p core/build/azure/infrastructure/$RESOURCE_GROUP + +echo '{ + "appId": "{{ sp_client_id }}", + "displayName": "epiphany-vsts", + "name": "http://epiphany-vsts", + "password": "{{ sp_client_secret }}", + "tenant": "{{ sp_tenant_id }}" +} +' >> tmp/sp/az_ad_sp.json + +sed "/set -e/q" core/core/src/templates/azure/env.sh.j2 > tmp/sp/env.sh + +echo 'export ARM_SUBSCRIPTION_ID="{{ sp_subscription_id }}" +export ARM_CLIENT_ID="{{ sp_client_id }}" +export ARM_CLIENT_SECRET="{{ sp_client_secret }}" +export ARM_TENANT_ID="{{ sp_tenant_id }}" +' >> tmp/sp/env.sh + +echo " +--- +# Security.yaml + +core: + azure: + terraform: + # This version info is what version is being used at the moment. The version of Terraform in the manifest.yaml in the + # root of the repo is for the initial install and the minum version + version: 1.6 + service_principal: + # Three files are required for SPs to work, az_ad_sp.json, env.sh and security.yaml. By default, these are created if the + # 'create' attribute is true. If false then you will need to supply those two files. This allows you to create + # a service_principal of your own instead of having one generated. + # You will also need to override env.sh that contains the 'ARM_...' environment variables required. + enable: True + create: False # If you want to use an existing one then set this to false + auth: pwd # Valid is 'pwd' and 'cert'. At this time Terraform only support 'pwd' for service principals + # NOTE: Backend is a Terraform resource that stores the *.tfstate files used by Terraform to store state. The default + # is to store the state file locally but this can cause issues if working in a team environment. + backend: + # Only used by Terraform + # The backend storage account is '''backend' (combined name with suffix) + # The storage container is generated as ''-'terraform' + # NOTE: Known issue with backend tf when having different VM types below when enabled! So, only one VM entry with count set should be used. Set to false for now... + enable: False + +subscription_id: {{ sp_subscription_id }} +app_name: epiphany-vsts +app_id: {{ sp_client_id }} +tenant_id: {{ sp_tenant_id }} +role: Contributor +auth: {{ sp_client_secret }} +auth_type: pwd +" >> tmp/sp/security.yaml +sed -i "s/{{ sp_subscription_id }}/$SP_SUBSCRIPTION_ID/g; s/{{ sp_client_id }}/$SP_CLIENT_ID/g; s/{{ sp_tenant_id }}/$SP_TENANT_ID/g; s/{{ sp_client_secret }}/$SP_CLIENT_SECRET/g" tmp/sp/* +cp tmp/sp/* core/build/azure/infrastructure/$RESOURCE_GROUP +rm -rf tmp \ No newline at end of file diff --git a/core/core/src/docker/test-CI/run.sh b/core/core/src/docker/test-CI/run.sh new file mode 100644 index 0000000000..fe75b15952 --- /dev/null +++ b/core/core/src/docker/test-CI/run.sh @@ -0,0 +1,27 @@ +cd /epiphany/ +git init -q +export GIT_DISCOVERY_ACROSS_FILESYSTEM=1 +echo "Preparing credentials" +bash core/core/src/docker/test-CI/prepare_sp.sh +cd /epiphany/core +echo +echo "Epiphany build for resource group $RESOURCE_GROUP started..." +bash epiphany -a -b -i -f infrastructure/$RESOURCE_GROUP -t infrastructure/epiphany-qa-template + +status=$? + +if [ $status -eq 0 ] +then + echo + echo "Epiphany build for resource group $RESOURCE_GROUP completed" + echo + cd /epiphany/core/core/test/serverspec + echo "Serverspec tests for resource group $RESOURCE_GROUP started..." + rake inventory=/epiphany/core/build/epiphany/$RESOURCE_GROUP/inventory/development user=operations keypath=/tmp/keys/id_rsa spec:all + echo "Serverspec tests for resource group $RESOURCE_GROUP finished" + echo +else + echo + echo "Epiphany build for resource group $RESOURCE_GROUP FAILED!" + exit 1 +fi diff --git a/core/core/test/serverspec/Gemfile b/core/core/test/serverspec/Gemfile index 867ffd9957..46c5c7620e 100644 --- a/core/core/test/serverspec/Gemfile +++ b/core/core/test/serverspec/Gemfile @@ -2,4 +2,4 @@ source 'https://rubygems.org' gem 'serverspec' gem 'inventoryfile' -gem 'fileutils' +gem 'rspec_junit_formatter' diff --git a/core/core/test/serverspec/Rakefile b/core/core/test/serverspec/Rakefile index 5906890058..9aba62a412 100644 --- a/core/core/test/serverspec/Rakefile +++ b/core/core/test/serverspec/Rakefile @@ -27,17 +27,17 @@ current_group = nil File::open(ENV['inventory']) do |f| while line = f.gets - # Remove comment - md = line.match(/^([^;]+)/) + + md = line.match(/^([^#]+)/) # matches lines not starting with a '#' character next unless md line = md[0] if line =~ /^\[([^\]]+)\]/ current_group = $1 - elsif line =~ /^(\S+)/ + elsif line =~ /(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})/ # regex for IP address if current_group - groups[current_group] ||= [] - groups[current_group] << $1 + groups[current_group] ||= {} + groups[current_group][$1] = line.split.first else hosts << $1 end @@ -45,6 +45,14 @@ File::open(ENV['inventory']) do |f| end end +groups.keys.each do |group| + if !File.directory?("spec/#{group}") || Dir.empty?("spec/#{group}") + groups.delete(group) + end +end + +puts groups + task :spec => 'spec:all' task :default => :spec @@ -54,29 +62,19 @@ namespace :spec do # Tasks for groups groups.keys.each do |group| - task group.to_sym => groups[group].map {|host| 'spec:' + group + ':' + host } - - groups[group].each do |host| + task group.to_sym => groups[group].keys.map {|host| 'spec:' + group + ':' + host } + groups[group].keys.each do |host| desc "Run tests for group '#{group}'" task_name = group + ':' + host RSpec::Core::RakeTask.new(task_name) do |t| ENV['TARGET_HOST'] = host -# status = ENV['status'] ? ENV['status'] : 'running' -# t.pattern = "spec/#{group}/#{status}/*_spec.rb" + ENV['TEST_ENV_NUMBER'] = "-" + groups[group][host] + puts "Testing " + task_name t.pattern = "spec/#{group}/*_spec.rb" - end - end - end - - # Tasks for hosts without groups - hosts += groups.values.flatten - hosts.uniq.each do |host| - desc "Run tests for host '#{host}'" - RSpec::Core::RakeTask.new(host) do |t| - ENV['TARGET_HOST'] = host -# status = ENV['status'] ? ENV['status'] : 'running' -# t.pattern = "spec/hosts/#{host}/#{status}/*_spec.rb" - t.pattern = "spec/#{group}/*_spec.rb" + t.fail_on_error = false + t.rspec_opts = "--format documentation --format RspecJunitFormatter \ + --out results/" + Time.now.strftime("%Y-%m-%d_%H-%M-%S") + "_#{group}_#{groups[group][host]}.xml" + end end end end diff --git a/core/core/test/serverspec/spec/deployments/auth-service/auth-service.rb b/core/core/test/serverspec/spec/deployments/auth-service/auth-service.rb new file mode 100644 index 0000000000..b46068d358 --- /dev/null +++ b/core/core/test/serverspec/spec/deployments/auth-service/auth-service.rb @@ -0,0 +1,57 @@ +require 'spec_helper' +require 'securerandom' + +def callAuthServiceDeploymentTests + + auth_service_array_index = readDataYaml["kubernetes"]["deployments"].index {|h| h["name"] == "auth-service" } + service_namespace = 'default' + service_name = '' + service_port = readDataYaml.dig("kubernetes","deployments","#{auth_service_array_index}".to_i,"service","port") + service_replicas = readDataYaml.dig("kubernetes","deployments","#{auth_service_array_index}".to_i,"service","replicas") + service_admin_user = readDataYaml.dig("kubernetes","deployments","#{auth_service_array_index}".to_i,"service","admin_user") + service_admin_password = readDataYaml.dig("kubernetes","deployments","#{auth_service_array_index}".to_i,"service","admin_password") + + if readDataYaml.dig("kubernetes","deployments","#{auth_service_array_index}".to_i,"service","namespace") + service_namespace = readDataYaml.dig("kubernetes","deployments","#{auth_service_array_index}".to_i,"service","namespace") + end + + if readDataYaml.dig("kubernetes","deployments","#{auth_service_array_index}".to_i,"service","name") + service_name = readDataYaml.dig("kubernetes","deployments","#{auth_service_array_index}".to_i,"service","name") + else service_name = 'as-' + service_namespace + end + + describe 'Checking if auth-service is running' do + describe command("kubectl get services --namespace=#{service_namespace}") do + its(:stdout) { should match /#{service_name}/ } + end + end + + describe 'Checking if the ports are open' do + describe port(service_port) do + let(:disable_sudo) { false } + it { should be_listening } + end + end + + describe 'Checking the status of auth-service pods - all pods should be running' do + describe command("kubectl get pods --namespace=#{service_namespace} --field-selector=status.phase=Running | grep #{service_name} | wc -l") do + it "is expected to be equal" do + expect(subject.stdout.to_i).to eq service_replicas + end + its(:exit_status) { should eq 0 } + end + end + + describe 'Checking the auth-service API connection' do + describe command("curl -o /dev/null -s -w '%{http_code}' -k https://#{host_inventory['hostname']}:#{service_port}/auth/") do + it "is expected to be equal" do + expect(subject.stdout.to_i).to eq 200 + end + end + describe command("curl -k -d \"client_id=admin-cli\" -d \"username=#{service_admin_user}\" -d \"password=#{service_admin_password}\" \ + -d \"grant_type=password\" https://#{host_inventory['hostname']}:#{service_port}/auth/realms/master/protocol/openid-connect/token") do + its(:stdout) { should_not match /^$/ } + its(:exit_status) { should eq 0 } + end + end +end diff --git a/core/core/test/serverspec/spec/deployments/deployments_spec.rb b/core/core/test/serverspec/spec/deployments/deployments_spec.rb new file mode 100644 index 0000000000..663cd11737 --- /dev/null +++ b/core/core/test/serverspec/spec/deployments/deployments_spec.rb @@ -0,0 +1,12 @@ +require 'spec_helper' +require 'securerandom' +require 'deployments/rabbitmq/rabbitmq' +require 'deployments/auth-service/auth-service' + +if readDataYaml.dig("kubernetes","deployments") && !readDataYaml["kubernetes"]["deployments"].index {|h| h["name"] == "rabbitmq" }.nil? + callRabbitMQDeploymentTests +end + +if readDataYaml.dig("kubernetes","deployments") && !readDataYaml["kubernetes"]["deployments"].index {|h| h["name"] == "auth-service" }.nil? + callAuthServiceDeploymentTests +end diff --git a/core/core/test/serverspec/spec/deployments/rabbitmq/rabbitmq.rb b/core/core/test/serverspec/spec/deployments/rabbitmq/rabbitmq.rb new file mode 100644 index 0000000000..5e20b9f2ec --- /dev/null +++ b/core/core/test/serverspec/spec/deployments/rabbitmq/rabbitmq.rb @@ -0,0 +1,172 @@ +require 'spec_helper' +require 'securerandom' + +def callRabbitMQDeploymentTests + + rabbitmq_amqp_port = 5672 + rabbitmq_http_port = 15672 + service_namespace = 'default' + service_name = '' + user = 'testuser' + pass = SecureRandom.hex + + rabbitmq_deployment_array_index = readDataYaml["kubernetes"]["deployments"].index {|h| h["name"] == "rabbitmq" } + service_port = readDataYaml.dig("kubernetes","deployments","#{rabbitmq_deployment_array_index}".to_i,"service","port") + service_management_port = readDataYaml.dig("kubernetes","deployments","#{rabbitmq_deployment_array_index}".to_i,"service","management_port") + service_replicas = readDataYaml.dig("kubernetes","deployments","#{rabbitmq_deployment_array_index}".to_i,"service","replicas") + + if readDataYaml.dig("kubernetes","deployments","#{rabbitmq_deployment_array_index}".to_i,"rabbitmq","amqp_port") + rabbitmq_amqp_port = readDataYaml.dig("kubernetes","deployments","#{rabbitmq_deployment_array_index}".to_i,"rabbitmq","amqp_port") + end + + if readDataYaml.dig("kubernetes","deployments","#{rabbitmq_deployment_array_index}".to_i,"service","namespace") + service_namespace = readDataYaml.dig("kubernetes","deployments","#{rabbitmq_deployment_array_index}".to_i,"service","namespace") + end + + if readDataYaml.dig("kubernetes","deployments","#{rabbitmq_deployment_array_index}".to_i,"service","name") + service_name = readDataYaml.dig("kubernetes","deployments","#{rabbitmq_deployment_array_index}".to_i,"service","name") + else service_name = 'rabbit-' + service_namespace + end + + if readDataYaml.dig("kubernetes","deployments","#{rabbitmq_deployment_array_index}".to_i,"rabbitmq","plugins") + plugins = readDataYaml.dig("kubernetes","deployments","#{rabbitmq_deployment_array_index}".to_i,"rabbitmq","plugins") + end + + describe 'Checking if RabbitMQ service is running' do + describe command("kubectl get services --namespace=#{service_namespace}") do + its(:stdout) { should match /#{service_name}/ } + end + end + + describe 'Checking if the ports are open' do + describe port(service_port) do + let(:disable_sudo) { false } + it { should be_listening } + end + describe command("kubectl describe service #{service_name} --namespace=#{service_namespace} | grep TargetPort") do + its(:stdout) { should match /#{rabbitmq_amqp_port}/ } + end + end + + describe 'Checking the status of RabbitMQ pods - all pods should be running' do + describe command("kubectl get pods --namespace=#{service_namespace} --field-selector=status.phase=Running | grep #{service_name} | wc -l") do + it "is expected to be equal" do + expect(subject.stdout.to_i).to eq service_replicas + end + its(:exit_status) { should eq 0 } + end + end + + describe 'Checking RabbitMQ ping' do + service_replicas.times do |i| + describe command("kubectl exec --namespace=#{service_namespace} #{service_name}-#{i} -- rabbitmqctl ping") do + its(:stdout) { should match /^Ping succeeded$/ } + its(:exit_status) { should eq 0 } + end + end + end + + describe 'Checking the health of target nodes' do + service_replicas.times do |i| + describe command("kubectl exec --namespace=#{service_namespace} #{service_name}-#{i} -- rabbitmqctl node_health_check") do + its(:stdout) { should match /^Health check passed$/ } + its(:exit_status) { should eq 0 } + end + end + end + + describe 'Checking the status of RabbitMQ nodes' do + service_replicas.times do |i| + describe command("kubectl exec --namespace=#{service_namespace} #{service_name}-#{i} -- rabbitmqctl status") do + its(:exit_status) { should eq 0 } + end + describe command("kubectl exec --namespace=#{service_namespace} #{service_name}-#{i} -- rabbitmqctl cluster_status \ + | awk '/running_nodes/,/}/' | grep -o rabbit@ | wc -l") do + it "is expected to be equal" do + expect(subject.stdout.to_i).to eq service_replicas + end + its(:exit_status) { should eq 0 } + end + end + end + + describe 'Checking if it is possible to create a test user on each replica' do + service_replicas.times do |i| + describe command("kubectl exec --namespace=#{service_namespace} #{service_name}-#{i} -- bash -c \"rabbitmqctl add_user #{user}#{i} #{pass} \ + && rabbitmqctl set_user_tags #{user}#{i} administrator && rabbitmqctl set_permissions -p / #{user}#{i} '.*' '.*' '.*'\"") do + its(:stdout) { should match /Adding user "#{user}#{i}"/ } + its(:stdout) { should match /Setting tags for user "#{user}#{i}" to \[administrator\]/ } + its(:stdout) { should match /Setting permissions for user "#{user}#{i}"/ } + its(:exit_status) { should eq 0 } + end + end + end + + # # Tests to be run only when RabbitMQ plugins section is enabled + + plugins = [] + + if readDataYaml.dig("kubernetes","deployments","#{rabbitmq_deployment_array_index}".to_i,"rabbitmq","plugins") + plugins = readDataYaml.dig("kubernetes","deployments","#{rabbitmq_deployment_array_index}".to_i,"rabbitmq","plugins") + end + + describe 'Checking if RabbitMQ plugins are enabled' do + service_replicas.times do |i| + plugins.each do |plugin| + describe command("kubectl exec --namespace=#{service_namespace} #{service_name}-#{i} -- rabbitmq-plugins list -e") do + its(:stdout) { should match /\b#{plugin}\b/ } + its(:exit_status) { should eq 0 } + end + end + end + end + + # Tests to be run only when RabbitMQ Management Plugin is enabled + + if plugins.include? "rabbitmq_management" + + describe 'Checking if the port for RabbitMQ Management Plugin is open' do + describe port(service_management_port) do + let(:disable_sudo) { false } + it { should be_listening } + end + describe command("kubectl describe service #{service_name} --namespace=#{service_namespace} | grep TargetPort") do + its(:stdout) { should match /#{rabbitmq_http_port}/ } + end + end + + describe 'Checking node health using RabbitMQ API' do + service_replicas.times do |i| + describe command("curl -o /dev/null -s -w '%{http_code}' -u #{user}#{i}:#{pass} \ + #{host_inventory['hostname']}:#{service_management_port}/api/healthchecks/node/rabbit@$(kubectl describe pods rabbitmq-cluster-#{i} \ + --namespace=#{service_namespace} | grep ^IP: | awk '{print $2}')") do + it "is expected to be equal" do + expect(subject.stdout.to_i).to eq 200 + end + end + describe command("curl -u #{user}#{i}:#{pass} \ + #{host_inventory['hostname']}:#{service_management_port}/api/healthchecks/node/rabbit@$(kubectl describe pods rabbitmq-cluster-#{i} \ + --namespace=#{service_namespace} | grep ^IP: | awk '{print $2}')") do + its(:stdout_as_json) { should include('status' => /ok/) } + its(:stdout_as_json) { should_not include('status' => /failed/) } + its(:exit_status) { should eq 0 } + end + describe command("curl -u #{user}#{i}:#{pass} #{host_inventory['hostname']}:#{service_management_port}/api/aliveness-test/%2F") do + its(:stdout_as_json) { should include('status' => /ok/) } + its(:stdout_as_json) { should_not include('status' => /failed/) } + its(:exit_status) { should eq 0 } + end + end + end + end + + describe 'Cleaning up' do + service_replicas.times do |i| + describe command("kubectl exec --namespace=#{service_namespace} #{service_name}-#{i} -- rabbitmqctl delete_user #{user}#{i}") do + its(:stdout) { should match /Deleting user "#{user}#{i}"/ } + its(:exit_status) { should eq 0 } + end + end + end + +end diff --git a/core/core/test/serverspec/spec/elasticsearch-curator/elasticsearch-curator_spec.rb b/core/core/test/serverspec/spec/elasticsearch-curator/elasticsearch-curator_spec.rb new file mode 100644 index 0000000000..686fffd68b --- /dev/null +++ b/core/core/test/serverspec/spec/elasticsearch-curator/elasticsearch-curator_spec.rb @@ -0,0 +1,15 @@ +require 'spec_helper' + +describe 'Checking if Elasticsearch Curator package is installed' do + describe package('elasticsearch-curator') do + it { should be_installed } + end +end + +describe 'Checking if cron job to delete old elasticsearch indices exists' do + let(:disable_sudo) { false } + describe command("crontab -l | grep -q 'Delete old elasticsearch indices.' && echo 'EXISTS' || echo 'NOTEXISTS'") do + its(:stdout) { should match /\bEXISTS\b/ } + its(:stdout) { should_not match /\bNOTEXISTS\b/ } + end +end diff --git a/core/core/test/serverspec/spec/elasticsearch/elasticsearch_spec.rb b/core/core/test/serverspec/spec/elasticsearch/elasticsearch_spec.rb index ac5f36c79f..5d9086781f 100644 --- a/core/core/test/serverspec/spec/elasticsearch/elasticsearch_spec.rb +++ b/core/core/test/serverspec/spec/elasticsearch/elasticsearch_spec.rb @@ -1,17 +1,60 @@ require 'spec_helper' -describe service('elsaticsearch') do - it { should be_enabled } - it { should be_running } -end +elasticsearch_rest_api_port = 9200 +elasticsearch_communication_port = 9300 -describe service('kibana') do +describe 'Checking if Elasticsearch service is running' do + describe service('elasticsearch') do it { should be_enabled } it { should be_running } end +end -describe service('filebeat') do - it { should be_enabled } - it { should be_running } +describe 'Checking if Elasticsearch user exists' do + describe group('elasticsearch') do + it { should exist } + end + describe user('elasticsearch') do + it { should exist } + it { should belong_to_group 'elasticsearch' } + end +end + +describe 'Checking Elasticsearch directories and config files' do + let(:disable_sudo) { false } + describe file('/etc/elasticsearch') do + it { should exist } + it { should be_a_directory } + end + describe file("/etc/elasticsearch/elasticsearch.yml") do + it { should exist } + it { should be_a_file } end +end +describe 'Checking if the ports are open' do + let(:disable_sudo) { false } + describe port(elasticsearch_rest_api_port) do + it { should be_listening } + end + describe port(elasticsearch_communication_port) do + it { should be_listening } + end +end + +describe 'Checking Elasticsearch HTTP status code' do + let(:disable_sudo) { false } + describe command("curl -o /dev/null -s -w '%{http_code}' $(grep 'network.host' /etc/elasticsearch/elasticsearch.yml | awk '{print $2}'):#{elasticsearch_rest_api_port}") do + it "is expected to be equal" do + expect(subject.stdout.to_i).to eq 200 + end + end +end + +describe 'Checking Elasticsearch health' do + let(:disable_sudo) { false } + describe command("curl $(grep 'network.host' /etc/elasticsearch/elasticsearch.yml | awk '{print $2}'):#{elasticsearch_rest_api_port}/_cluster/health?pretty=true") do + its(:stdout_as_json) { should include('status' => /green|yellow/) } + its(:exit_status) { should eq 0 } + end +end diff --git a/core/core/test/serverspec/spec/filebeat/filebeat_spec.rb b/core/core/test/serverspec/spec/filebeat/filebeat_spec.rb new file mode 100644 index 0000000000..d57fc7cb92 --- /dev/null +++ b/core/core/test/serverspec/spec/filebeat/filebeat_spec.rb @@ -0,0 +1,60 @@ +require 'spec_helper' + +describe 'Checking if Filebeat package is installed' do + describe package('filebeat') do + it { should be_installed } + end +end + +describe 'Checking if Filebeat service is running' do + describe service('filebeat') do + it { should be_enabled } + it { should be_running } + end +end + +describe 'Checking Filebeat directories and config files' do + let(:disable_sudo) { false } + describe file('/etc/filebeat') do + it { should exist } + it { should be_a_directory } + end + describe file("/etc/filebeat/filebeat.yml") do + it { should exist } + it { should be_a_file } + end +end + +if hostInGroups?("master") || hostInGroups?("worker") + describe 'Checking extra configuration for master/worker roles - setting Filebeat to be started after Docker' do + describe file("/etc/systemd/system/filebeat.service.d/extra-dependencies.conf") do + it { should exist } + it { should be_a_file } + its(:content) { should match /After=docker\.service/ } + end + end +end + +if count_inventory_roles("elasticsearch") > 0 + describe 'Checking the connection to the Elasticsearch host' do + let(:disable_sudo) { false } + describe command("curl -o /dev/null -s -w '%{http_code}' $(awk '/- Elasticsearch output -/,/- Logstash output -/' /etc/filebeat/filebeat.yml \ + | grep -oP '(?<=hosts: \\\[\\\").*(?=\\\"\\\])')") do + it "is expected to be equal" do + expect(subject.stdout.to_i).to eq 200 + end + end + end +end + +if count_inventory_roles("kibana") > 0 + describe 'Checking the connection to the Kibana endpoint' do + let(:disable_sudo) { false } + describe command("curl -o /dev/null -s -w '%{http_code}' $(awk '/= Kibana =/,/= Elastic Cloud =/' /etc/filebeat/filebeat.yml \ + | grep -oP '(?<=host: \\\").*(?=\\\")')") do + it "is expected to be equal" do + expect(subject.stdout.to_i).to eq 200 + end + end + end +end diff --git a/core/core/test/serverspec/spec/grafana/grafana_spec.rb b/core/core/test/serverspec/spec/grafana/grafana_spec.rb new file mode 100644 index 0000000000..91a701b2a9 --- /dev/null +++ b/core/core/test/serverspec/spec/grafana/grafana_spec.rb @@ -0,0 +1,107 @@ +require 'spec_helper' + +grafana_host = 'localhost' +grafana_port = 3000 + +describe 'Checking if Grafana user exists' do + describe group('grafana') do + it { should exist } + end + describe user('grafana') do + it { should exist } + it { should belong_to_group 'grafana' } + end +end + +describe 'Checking Grafana directories and files' do + let(:disable_sudo) { false } + describe file('/var/lib/grafana') do + it { should exist } + it { should be_a_directory } + it { should be_owned_by 'grafana' } + it { should be_grouped_into 'grafana' } + end + describe file('/var/log/grafana') do + it { should exist } + it { should be_a_directory } + it { should be_owned_by 'grafana' } + it { should be_grouped_into 'grafana' } + end + describe file('/etc/grafana') do + it { should exist } + it { should be_a_directory } + it { should be_owned_by 'root' } + it { should be_grouped_into 'root' } + end + describe file("/etc/grafana/grafana.ini") do + it { should exist } + it { should be_a_file } + it { should be_readable } + end +end + +describe 'Checking self signed SSL certificates' do + let(:disable_sudo) { false } + describe x509_private_key("/etc/grafana/ssl/grafana_key.key") do + it { should be_valid } + it { should have_matching_certificate('/etc/grafana/ssl/grafana_cert.pem') } + end + describe x509_certificate("/etc/grafana/ssl/grafana_cert.pem") do + it { should be_certificate } + it { should be_valid } + end +end + +describe 'Checking if Grafana package is installed' do + describe package('grafana') do + it { should be_installed } + end +end + +describe 'Checking if Grafana service is running' do + describe service('grafana-server') do + it { should be_enabled } + it { should be_running } + end +end + +describe 'Checking if the ports are open' do + describe port(grafana_port) do + let(:disable_sudo) { false } + it { should be_listening } + end +end + +describe 'Checking Grafana health' do + describe command("curl -o /dev/null -s -w '%{http_code}' -k https://#{grafana_host}:#{grafana_port}/login") do + it "is expected to be equal" do + expect(subject.stdout.to_i).to eq 200 + end + end + describe command("curl -k https://#{grafana_host}:#{grafana_port}/api/health") do + its(:stdout_as_json) { should include('database' => 'ok') } + end +end + +describe 'Checking if Prometheus datasource exists' do + let(:disable_sudo) { false } + describe command("curl -k -u $(grep 'admin_user' /etc/grafana/grafana.ini | awk '{print $3}'):$(grep 'admin_password' /etc/grafana/grafana.ini | awk '{print $3}') \ + https://#{grafana_host}:#{grafana_port}/api/datasources/name/Prometheus") do + its(:stdout_as_json) { should include('name' => 'Prometheus') } + its(:stdout_as_json) { should include('type' => 'prometheus') } + end +end + +describe 'Checking Prometheus datasource availability' do + let(:disable_sudo) { false } + describe command("curl -k -o /dev/null -s -w '%{http_code}' -u $(grep 'admin_user' /etc/grafana/grafana.ini | awk '{print $3}'):$(grep 'admin_password' /etc/grafana/grafana.ini | awk '{print $3}') \ + https://#{grafana_host}:#{grafana_port}/api/datasources/proxy/1/api/v1/query?query=2%2B2") do + it "is expected to be equal" do + expect(subject.stdout.to_i).to eq 200 + end + end + describe command("curl -k -u $(grep 'admin_user' /etc/grafana/grafana.ini | awk '{print $3}'):$(grep 'admin_password' /etc/grafana/grafana.ini | awk '{print $3}') \ + https://#{grafana_host}:#{grafana_port}/api/datasources/proxy/1/api/v1/query?query=2%2B2") do + its(:stdout_as_json) { should include('status' => 'success') } + end +end diff --git a/core/core/test/serverspec/spec/haproxy_exporter/haproxy_exporter_spec.rb b/core/core/test/serverspec/spec/haproxy_exporter/haproxy_exporter_spec.rb new file mode 100644 index 0000000000..b461e7eb22 --- /dev/null +++ b/core/core/test/serverspec/spec/haproxy_exporter/haproxy_exporter_spec.rb @@ -0,0 +1,42 @@ +require 'spec_helper' + +haproxy_exporter_host = 'localhost' +haproxy_exporter_port = 9101 + +describe 'Checking if HAProxy Exporter user exists' do + describe group('haproxy_exporter') do + it { should exist } + end + describe user('haproxy_exporter') do + it { should exist } + it { should belong_to_group 'haproxy_exporter' } + it { should have_home_directory '/home/haproxy_exporter' } + it { should have_login_shell '/usr/sbin/nologin' } + end + describe file('/opt/haproxy_exporter') do + it { should exist } + it { should be_grouped_into 'haproxy_exporter' } + end +end + +describe 'Checking if HAProxy Exporter service is running' do + describe service('prometheus-haproxy-exporter') do + it { should be_enabled } + it { should be_running } + end +end + +describe 'Checking if the ports are open' do + describe port(haproxy_exporter_port) do + let(:disable_sudo) { false } + it { should be_listening } + end +end + +describe 'Checking if it is possible to collect the metrics from HAProxy' do + describe command("curl -s #{haproxy_exporter_host}:#{haproxy_exporter_port}/metrics | grep -i haproxy") do + its(:stdout) { should match /haproxy_up 1/ } + its(:stdout) { should_not match /haproxy_up 0/ } + its(:exit_status) { should eq 0 } + end +end diff --git a/core/core/test/serverspec/spec/haproxy_tls_termination/haproxy_tls_termination_spec.rb b/core/core/test/serverspec/spec/haproxy_tls_termination/haproxy_tls_termination_spec.rb new file mode 100644 index 0000000000..09d423b6ce --- /dev/null +++ b/core/core/test/serverspec/spec/haproxy_tls_termination/haproxy_tls_termination_spec.rb @@ -0,0 +1,85 @@ +require 'spec_helper' + +haproxy_host = 'localhost' +haproxy_front_port = 443 + +describe 'Checking if HAProxy service is running' do + describe service('haproxy') do + it { should be_enabled } + it { should be_running } + end +end + +describe 'Checking if HAProxy user exists' do + describe group('haproxy') do + it { should exist } + end + describe user('haproxy') do + it { should exist } + it { should belong_to_group 'haproxy' } + it { should have_home_directory '/var/lib/haproxy' } + it { should have_login_shell '/usr/sbin/nologin' } + end +end + +describe 'Checking if HAProxy log file exists and is not empty' do + describe file('/var/log/haproxy.log') do + let(:disable_sudo) { false } + it { should exist } + it { should be_a_file } + its(:size) { should > 0 } + end +end + +describe 'Checking if the ports are open' do + describe port(haproxy_front_port) do + let(:disable_sudo) { false } + it { should be_listening } + end +end + +describe 'Checking SSL certificates' do + describe file('/etc/ssl/haproxy') do + it { should exist } + it { should be_a_directory } + end + describe command("ls -1 /etc/ssl/haproxy/*.pem 2>/dev/null | wc -l") do + it "is expected to be gt 0" do + expect(subject.stdout.to_i).to be > 0 + end + its(:exit_status) { should eq 0 } + end + describe command("echo 'Q' | openssl s_client -connect #{haproxy_host}:#{haproxy_front_port}") do + its(:stdout) { should match /^CONNECTED/ } + its(:exit_status) { should eq 0 } + end +end + +describe 'Checking HAProxy config files' do + describe file('/etc/haproxy/haproxy.cfg') do + it { should exist } + it { should be_a_file } + end + describe file('/etc/rsyslog.d/haproxy.conf') do + it { should exist } + it { should be_a_file } + end + describe file('/etc/logrotate.d/haproxy') do + it { should exist } + it { should be_a_file } + end +end + +describe 'Checking HAProxy HTTP status code for stats page' do + describe command("curl -k --user $(cat /etc/haproxy/haproxy.cfg | grep 'stats auth' | awk '{print $3}') -o /dev/null -s -w '%{http_code}' \ + https://#{haproxy_host}:#{haproxy_front_port}$(cat /etc/haproxy/haproxy.cfg | grep 'stats uri' | awk '{print $3}')") do + it "is expected to be equal" do + expect(subject.stdout.to_i).to eq 200 + end + end + describe command("curl -k --user $(cat /etc/haproxy/haproxy.cfg | grep 'stats auth' | awk '{print $3}') \ + https://#{haproxy_host}:#{haproxy_front_port}$(cat /etc/haproxy/haproxy.cfg | grep 'stats uri' | awk '{print $3}')") do + its(:stdout) { should match /Statistics Report for HAProxy/ } + end +end + diff --git a/core/core/test/serverspec/spec/jmx-exporter/jmx_exporter_spec.rb b/core/core/test/serverspec/spec/jmx-exporter/jmx_exporter_spec.rb index a4f6b4b7f2..86efcd8689 100644 --- a/core/core/test/serverspec/spec/jmx-exporter/jmx_exporter_spec.rb +++ b/core/core/test/serverspec/spec/jmx-exporter/jmx_exporter_spec.rb @@ -1,15 +1,54 @@ require 'spec_helper' -describe user('jmx-exporter') do - it { should exist } - it { should have_login_shell '/usr/sbin/nologin' } +jmx_exporter_host = 'localhost' +jmx_exporter_port_for_kafka = 7071 +jmx_exporter_port_for_zookeeper = 7072 + +describe 'Checking if JMX Exporter user exists' do + describe group('jmx-exporter') do + it { should exist } + end + describe user('jmx-exporter') do + it { should exist } + it { should belong_to_group 'jmx-exporter' } + it { should have_home_directory '/home/jmx-exporter' } + it { should have_login_shell '/usr/sbin/nologin' } + end + describe file('/opt/jmx-exporter') do + it { should exist } + it { should be_grouped_into 'jmx-exporter' } + end end -describe group('jmx-exporter') do - it { should exist } +describe 'Checking if the ports are open' do + describe port(jmx_exporter_port_for_kafka) do + let(:disable_sudo) { false } + it { should be_listening } + end + describe port(jmx_exporter_port_for_zookeeper) do + let(:disable_sudo) { false } + it { should be_listening } + end +end + +describe 'Checking if it is possible to collect the metrics from Kafka' do + describe command("curl -s #{jmx_exporter_host}:#{jmx_exporter_port_for_kafka} | grep -i ^kafka") do + its(:stdout) { should match /kafka/ } + its(:exit_status) { should eq 0 } + end end -describe file('/opt/jmx-exporter') do - it { should exist } - it { should be_grouped_into 'jmx-exporter' } +if count_inventory_roles("kafka") == 1 + describe 'Checking if it is possible to collect any jvm metrics' do + describe command("curl -s #{jmx_exporter_host}:#{jmx_exporter_port_for_zookeeper} | grep -i ^jvm_memory") do + its(:exit_status) { should eq 0 } + end + end +elsif count_inventory_roles("kafka") > 1 + describe 'Checking if it is possible to collect the metrics from ZooKeeper' do + describe command("curl -s #{jmx_exporter_host}:#{jmx_exporter_port_for_zookeeper} | grep -i ^zookeeper") do + its(:stdout) { should match /zookeeper/ } + its(:exit_status) { should eq 0 } + end + end end diff --git a/core/core/test/serverspec/spec/kafka-exporter/kafka_exporter_spec.rb b/core/core/test/serverspec/spec/kafka-exporter/kafka_exporter_spec.rb new file mode 100644 index 0000000000..3af2252999 --- /dev/null +++ b/core/core/test/serverspec/spec/kafka-exporter/kafka_exporter_spec.rb @@ -0,0 +1,41 @@ +require 'spec_helper' + +kafka_exporter_host = 'localhost' +kafka_exporter_web_listen_port = 9308 + +describe 'Checking if Kafka exporter process is running' do + describe process('kafka_exporter') do + it { should be_enabled } + it { should be_running } + end +end + +describe 'Checking if Kafka exporter user exists' do + describe group('kafka_exporter') do + it { should exist } + end + describe user('kafka_exporter') do + it { should exist } + it { should belong_to_group 'kafka_exporter' } + it { should have_home_directory '/home/kafka_exporter' } + it { should have_login_shell '/usr/sbin/nologin' } + end + describe file('/opt/kafka_exporter') do + it { should exist } + it { should be_grouped_into 'kafka_exporter' } + end +end + +describe 'Checking if the ports are open' do + describe port(kafka_exporter_web_listen_port) do + let(:disable_sudo) { false } + it { should be_listening } + end +end + +describe 'Checking if it is possible to collect the metrics' do + describe command("curl -s #{kafka_exporter_host}:#{kafka_exporter_web_listen_port}/metrics | grep -i kafka") do + its(:stdout) { should match /kafka/ } + its(:exit_status) { should eq 0 } + end +end diff --git a/core/core/test/serverspec/spec/kafka/kafka_spec.rb b/core/core/test/serverspec/spec/kafka/kafka_spec.rb index 9beae86a0e..031711001e 100644 --- a/core/core/test/serverspec/spec/kafka/kafka_spec.rb +++ b/core/core/test/serverspec/spec/kafka/kafka_spec.rb @@ -1,28 +1,115 @@ require 'spec_helper' +kafka_host = 'localhost' +kafka_port = 9092 +zookeeper_host = 'localhost' +zookeeper_client_port = 2181 -describe service('kafka') do - it { should be_enabled } - it { should be_running } +describe 'Checking if Kafka service is running' do + describe service('kafka') do + it { should be_enabled } + it { should be_running } + end end -describe port(9092) do - it { should be_listening } +describe 'Checking if Kafka user exists' do + describe group('kafka') do + it { should exist } + end + describe user('kafka') do + it { should exist } + it { should belong_to_group 'kafka' } + it { should have_home_directory '/home/kafka' } + it { should have_login_shell '/usr/sbin/nologin' } + end end -describe port(7071) do - it { should be_listening } +describe 'Checking if the ports are open' do + describe port(kafka_port) do + let(:disable_sudo) { false } + it { should be_listening } + end +end + +describe 'Listing down all the active brokers' do + describe command("echo 'ls /brokers/ids' | /opt/kafka/bin/zookeeper-shell.sh #{zookeeper_host}:#{zookeeper_client_port}") do + its(:stdout) { should match /Welcome to ZooKeeper!/ } + its(:stdout) { should match /\[(\d+(\,\s)?)+\]/ } # pattern: [0, 1, 2, 3 ...] + its(:exit_status) { should eq 0 } + end end -describe user('kafka') do - it { should exist } - it { should have_login_shell '/usr/sbin/nologin' } - it { should belong_to_group 'kafka' } - it { should belong_to_group 'jmx-exporter' } +describe 'Checking if the number of Kafka brokers is the same as indicated in the inventory file' do + describe command("echo 'dump' | curl -s telnet://#{zookeeper_host}:#{zookeeper_client_port} | grep -c brokers") do + it "is expected to be equal" do + expect(subject.stdout.to_i).to eq count_inventory_roles("kafka") + end + end end -#check if output from jmx exporter is correct -describe command('curl -s localhost:7071 | grep kafka') do - its(:stdout) { should match /kafka_/ } - its(:exit_status) { should eq 0 } +describe 'Checking the possibility of creating a topic, producing and consuming messages' do + + kafka_brokers_count = count_inventory_roles("kafka") + timestamp = DateTime.now.to_time.to_i.to_s + topic_name = 'topic' + timestamp + partitions = kafka_brokers_count*3 + message = 'test message' + + describe 'Checking if the topic was created' do + describe command("/opt/kafka/bin/kafka-topics.sh --create --zookeeper #{zookeeper_host}:#{zookeeper_client_port} --replication-factor #{kafka_brokers_count} \ + --partitions #{partitions} --topic #{topic_name}") do + its(:stdout) { should match /Created topic "#{topic_name}"./ } + its(:exit_status) { should eq 0 } + end + end + + describe 'Starting consumer process' do + describe command("/opt/kafka/bin/kafka-console-consumer.sh --bootstrap-server #{kafka_host}:#{kafka_port} --topic #{topic_name} --consumer-property group.id=TESTGROUP \ + >> /tmp/#{topic_name}.txt 2>&1 &") do + its(:exit_status) { should eq 0 } + end + end + + describe 'Checking if consumer process is ready' do + describe command("for i in {1..10}; do if /opt/kafka/bin/kafka-consumer-groups.sh --bootstrap-server #{kafka_host}:#{kafka_port} --group TESTGROUP --describe \ + | grep #{topic_name}; then echo 'READY'; break; else echo 'WAITING'; sleep 0.5; fi; done;") do + its(:stdout) { should match /#{topic_name}/ } + its(:stdout) { should match /\bREADY\b/ } + end + end + + 10.times do |i| + describe "Sending message #{i+1} from producer" do + describe command("echo '#{message} #{i+1}' | /opt/kafka/bin/kafka-console-producer.sh --broker-list #{kafka_host}:#{kafka_port} --topic #{topic_name}") do + its(:exit_status) { should eq 0 } + end + end + describe 'Checking if the consumer output contains the message that was produced' do + describe command("cat /tmp/#{topic_name}.txt") do + its(:stdout) { should match /^#{message} #{i+1}$/ } + end + end + end + + describe 'Checking if the created topic is on the list with all available topics in Kafka' do + describe command("/opt/kafka/bin/kafka-topics.sh --list --zookeeper #{zookeeper_host}:#{zookeeper_client_port}") do + its(:stdout) { should match /#{topic_name}/ } + end + end + + describe 'Cleaning up' do + describe command("/opt/kafka/bin/kafka-topics.sh --delete --zookeeper #{zookeeper_host}:#{zookeeper_client_port} --topic #{topic_name}") do + its(:stdout) { should match /Topic #{topic_name} is marked for deletion./ } + its(:exit_status) { should eq 0 } + end + describe command("rm /tmp/#{topic_name}.txt") do + its(:exit_status) { should eq 0 } + end + describe file("/tmp/#{topic_name}.txt") do + it { should_not exist } + end + describe command("kill -9 $(ps aux | grep -i 'kafka.tools.ConsoleConsumer' | grep '#{topic_name}' | grep -v 'grep' | awk '{print $2}')") do + its(:exit_status) { should eq 0 } + end + end end diff --git a/core/core/test/serverspec/spec/kibana/kibana_spec.rb b/core/core/test/serverspec/spec/kibana/kibana_spec.rb new file mode 100644 index 0000000000..736213f9ee --- /dev/null +++ b/core/core/test/serverspec/spec/kibana/kibana_spec.rb @@ -0,0 +1,77 @@ +require 'spec_helper' + +kibana_default_port = 5601 + +describe 'Checking if Kibana package is installed' do + describe package('kibana-oss') do + it { should be_installed } + end +end + +describe 'Checking if Kibana service is running' do + describe service('kibana') do + it { should be_enabled } + it { should be_running } + end +end + +describe 'Checking if Kibana user exists' do + describe group('kibana') do + it { should exist } + end + describe user('kibana') do + it { should exist } + it { should belong_to_group 'kibana' } + end +end + +describe 'Checking Kibana directories and config files' do + describe file('/etc/kibana') do + it { should exist } + it { should be_a_directory } + end + describe file("/etc/kibana/kibana.yml") do + it { should exist } + it { should be_a_file } + end +end + +describe 'Checking if Kibana log file exists and is not empty' do + describe file('/var/log/kibana/kibana.log') do + it { should exist } + it { should be_a_file } + its(:size) { should > 0 } + end + describe file('/etc/logrotate.d/kibana') do + it { should exist } + it { should be_a_file } + end +end + +if count_inventory_roles("elasticsearch") > 0 + describe 'Checking the connection to the Elasticsearch host' do + let(:disable_sudo) { false } + describe command("curl -o /dev/null -s -w '%{http_code}' $(grep -oP '(?<=elasticsearch.url: \\\").*(?=\\\")' /etc/kibana/kibana.yml)") do + it "is expected to be equal" do + expect(subject.stdout.to_i).to eq 200 + end + end + end +end + +describe 'Checking Kibana app HTTP status code' do + let(:disable_sudo) { false } + describe command("curl -o /dev/null -s -w '%{http_code}' $(grep -oP '(?<=server.host: \\\").*(?=\\\")' /etc/kibana/kibana.yml):#{kibana_default_port}/app/kibana") do + it "is expected to be equal" do + expect(subject.stdout.to_i).to eq 200 + end + end +end + +describe 'Checking Kibana health' do + let(:disable_sudo) { false } + describe command("curl $(grep -oP '(?<=server.host: \\\").*(?=\\\")' /etc/kibana/kibana.yml):#{kibana_default_port}/api/status") do + its(:stdout_as_json) { should include('status' => include('overall' => include('state' => 'green'))) } + its(:exit_status) { should eq 0 } + end +end diff --git a/core/core/test/serverspec/spec/master/master_spec.rb b/core/core/test/serverspec/spec/master/master_spec.rb index 6929a7a84e..35da1a257a 100644 --- a/core/core/test/serverspec/spec/master/master_spec.rb +++ b/core/core/test/serverspec/spec/master/master_spec.rb @@ -1,28 +1,176 @@ require 'spec_helper' +require 'base64' -describe service('kubelet') do - it { should be_enabled } - it { should be_running } -end +kube_apiserver_secure_port = 6443 -describe service('kube-controller') do +describe 'Checking if kubelet service is running' do + describe service('kubelet') do it { should be_enabled } it { should be_running } end +end -describe service('kube-apiserver') do - it { should be_enabled } +describe 'Checking if kube-scheduler is running' do + describe process('kube-scheduler') do + it { should be_running } + end +end + +describe 'Checking if kube-controller is running' do + describe process('kube-controller') do + it { should be_running } + end +end + +describe 'Checking if kube-apiserver is running' do + describe process("kube-apiserver") do it { should be_running } end +end + +describe 'Waiting for all pods to be ready' do + describe command("for i in {1..600}; do if [ $(kubectl get pods --all-namespaces -o json | jq -r '.items[] | select(.status.phase != \"Running\" or ([ .status.conditions[] | select(.type == \"Ready\" and .status != \"True\") ] | length ) == 1 ) | .metadata.namespace + \"/\" + .metadata.name' | wc -l) -eq 0 ]; \ + then echo 'READY'; break; else echo 'WAITING'; sleep 1; fi; done") do + its(:stdout) { should match /READY/ } + its(:exit_status) { should eq 0 } + end +end + +describe 'Checking if there are any pods that have status other than Running' do + describe command('kubectl get pods --all-namespaces --field-selector=status.phase!=Running') do + its(:stdout) { should match /^$/ } + its(:stderr) { should match /No resources found./ } + end +end + +describe 'Checking if the number of master nodes is the same as indicated in the inventory file' do + describe command('kubectl get nodes --selector=node-role.kubernetes.io/master --no-headers | wc -l | tr -d "\n"') do + it "is expected to be equal" do + expect(subject.stdout.to_i).to eq count_inventory_roles("master") + end + end +end + +describe 'Checking if the number of worker nodes is the same as indicated in the inventory file' do + describe command('kubectl get nodes --no-headers | grep -v master | wc -l | tr -d "\n"') do + it "is expected to be equal" do + expect(subject.stdout.to_i).to eq count_inventory_roles("worker") + end + end +end + +describe 'Checking if there are any nodes that have status other than Ready' do + describe command('kubectl get nodes') do + its(:stdout) { should match /\bReady\b/ } + its(:stdout) { should_not match /NotReady/ } + its(:stdout) { should_not match /Unknown/ } + its(:stdout) { should_not match /SchedulingDisabled/ } + end +end + +describe 'Checking if the number of all nodes is the same as the number of Ready nodes' do + describe command('out1=$(kubectl get nodes --no-headers | wc -l); out2=$(kubectl get nodes --no-headers | grep -wc Ready); if [ "$out1" = "$out2" ]; then echo "EQUAL"; else echo "NOT EQUAL"; fi') do + its(:stdout) { should match /\bEQUAL\b/ } + its(:stdout) { should_not match /NOT EQUAL/ } + end +end -describe port(6443) do +describe 'Checking the port on which to serve HTTPS with authentication and authorization' do + describe port(kube_apiserver_secure_port) do + let(:disable_sudo) { false } it { should be_listening } end +end + +describe 'Checking secret creation using kubectl' do + + test_user = 'user123' + test_pass = 'pass456' + test_user_b64 = Base64.encode64(test_user) + test_pass_b64 = Base64.encode64(test_pass) + timestamp = DateTime.now.to_time.to_i.to_s + test_secret = 'testsecret' + timestamp + + describe 'Checking if the secret was successfully created' do + describe command("kubectl create secret generic #{test_secret} --from-literal=username=#{test_user} --from-literal=password=#{test_pass}") do + its(:stdout) { should match /secret\/#{test_secret} created/ } + end + end + describe 'Checking if the created secret is present on the list with all secrets' do + describe command('kubectl get secrets') do + its(:stdout) { should match /#{test_secret}/ } + end + end + describe 'Checking if the secret stores encoded data' do + describe command("kubectl get secret #{test_secret} -o yaml") do + its(:stdout) { should match /name: #{test_secret}/ } + its(:stdout) { should match /#{test_user_b64}/ } + its(:stdout) { should match /#{test_pass_b64}/ } + end + end + describe 'Deleting created secret' do + describe command("kubectl delete secret #{test_secret}") do + its(:stdout) { should match /secret "#{test_secret}" deleted/ } + its(:exit_status) { should eq 0 } + end + end +end -#check if output from "get pods --all-namespaces" has dashboard pod running -describe command('kubectl get pods --all-namespaces | grep dashboard') do - its(:stdout) { should match /kubernetes-dashboard/ } - its(:stdout) { should match /running/ } +describe 'Checking kubernetes dashboard availability' do + describe 'Checking if kubernetes-dashboard pod is running' do + describe command('kubectl get pods --all-namespaces | grep kubernetes-dashboard') do + its(:stdout) { should match /kubernetes-dashboard/ } + its(:stdout) { should match /Running/ } + its(:exit_status) { should eq 0 } + end + end + describe 'Checking if kubernetes-dashboard is deployed' do + describe command('kubectl get deployments --all-namespaces --field-selector metadata.name=kubernetes-dashboard') do + its(:stdout) { should match /kubernetes-dashboard/ } + its(:stdout) { should match /1\/1/ } + its(:exit_status) { should eq 0 } + end + end + describe 'Checking if admin token bearer exists' do + describe command("kubectl describe secret $(kubectl get secrets --namespace=kube-system | grep admin-user \ + | awk '{print $1}') --namespace=kube-system | awk '/^token/ {print $2}' | head -1") do + its(:stdout) { should_not match /^$/ } + its(:exit_status) { should eq 0 } + end + end + describe 'Setting up proxy and starting to serve on localhost' do + describe command('kubectl proxy >/dev/null 2>&1 &') do + its(:exit_status) { should eq 0 } + end + end + describe 'Checking if the dashboard is available' do + describe command('for i in {1..60}; do if [ $(curl -o /dev/null -s -w "%{http_code}" "http://localhost:8001/api/v1/namespaces/kube-system/services/https:kubernetes-dashboard:/proxy/") -eq 200 ]; \ + then echo -n "200"; break; else sleep 1; fi; done') do + it "is expected to be equal" do + expect(subject.stdout.to_i).to eq 200 + end + end + end + describe 'Terminating kubectl proxy process' do + describe command('pkill -f "kubectl proxy"') do + its(:exit_status) { should eq 0 } + end + end +end + +describe 'Checking if coredns is deployed' do + describe command('kubectl get deployments --all-namespaces --field-selector metadata.name=coredns') do + coredns_counter = 2 # always equal 2 + its(:stdout) { should match /coredns/ } + its(:stdout) { should match /#{coredns_counter}\/#{coredns_counter}/ } its(:exit_status) { should eq 0 } end - +end + +describe 'Checking if kubernetes healthz endpoint is responding' do + describe command('curl --insecure -o /dev/null -s -w "%{http_code}" "https://127.0.0.1:10250/healthz"') do + it "is expected to be equal" do + expect(subject.stdout.to_i).to eq 401 + end + end +end diff --git a/core/core/test/serverspec/spec/node_exporter/node_exporter_spec.rb b/core/core/test/serverspec/spec/node_exporter/node_exporter_spec.rb index 668e8831df..7527e4d519 100644 --- a/core/core/test/serverspec/spec/node_exporter/node_exporter_spec.rb +++ b/core/core/test/serverspec/spec/node_exporter/node_exporter_spec.rb @@ -1,10 +1,50 @@ require 'spec_helper' -describe service('prometheus-node-exporter') do - it { should be_enabled } - it { should be_running } +node_exporter_host = 'localhost' +node_exporter_port = 9100 + +describe 'Checking if Node Exporter user exists' do + describe group('node_exporter') do + it { should exist } + end + describe user('node_exporter') do + it { should exist } + it { should belong_to_group 'node_exporter' } + it { should have_login_shell '/usr/sbin/nologin' } + end +end + +describe 'Checking Node Exporter directories and files' do + let(:disable_sudo) { false } + describe file('/opt/node_exporter') do + it { should exist } + it { should be_a_directory } + end + describe file("/opt/node_exporter/node_exporter") do + it { should exist } + it { should be_a_file } + it { should be_executable } + end end -describe port(9100) do - it { should be_listening } +describe 'Checking if Node Exporter service is running' do + describe service('prometheus-node-exporter') do + it { should be_enabled } + it { should be_running } + end +end + +describe 'Checking if the ports are open' do + describe port(node_exporter_port) do + let(:disable_sudo) { false } + it { should be_listening } + end +end + +describe 'Checking Node Exporter HTTP status code' do + describe command("curl -o /dev/null -s -w '%{http_code}' #{node_exporter_host}:#{node_exporter_port}") do + it "is expected to be equal" do + expect(subject.stdout.to_i).to eq 200 + end + end end diff --git a/core/core/test/serverspec/spec/postgresql/postgresql_spec.rb b/core/core/test/serverspec/spec/postgresql/postgresql_spec.rb new file mode 100644 index 0000000000..d0f529892a --- /dev/null +++ b/core/core/test/serverspec/spec/postgresql/postgresql_spec.rb @@ -0,0 +1,268 @@ +require 'spec_helper' + +postgresql_default_port = 5432 +replicated = false + +if readDataYaml["postgresql"] && readDataYaml["postgresql"]["replication"] && readDataYaml["postgresql"]["replication"]["enable"] && + readDataYaml["postgresql"]["replication"]["enable"] == true + replicated = true +end + +replication_user = readDataYaml.dig("postgresql","replication","user") +replication_password = readDataYaml.dig("postgresql","replication","password") +max_wal_senders = 5 +wal_keep_segments = 32 + +if readDataYaml.dig("postgresql","replication","max_wal_senders") + max_wal_senders = readDataYaml.dig("postgresql","replication","max_wal_senders") +end + +if readDataYaml.dig("postgresql","replication","wal_keep_segments") + wal_keep_segments = readDataYaml.dig("postgresql","replication","wal_keep_segments") +end + +def queryForCreating + describe 'Checking if it is possible to create a test schema' do + let(:disable_sudo) { false } + describe command("su - postgres -c \"psql -t -c 'CREATE SCHEMA test;'\"") do + its(:stdout) { should match /^CREATE SCHEMA$/ } + its(:exit_status) { should eq 0 } + end + end + + describe 'Checking if it is possible to create a test table' do + let(:disable_sudo) { false } + describe command("su - postgres -c \"psql -t -c 'CREATE TABLE test.test (col varchar(20));'\"") do + its(:stdout) { should match /^CREATE TABLE$/ } + its(:exit_status) { should eq 0 } + end + end + + describe 'Checking if it is possible to insert values into the test table' do + let(:disable_sudo) { false } + describe command("su - postgres -c \"psql -t -c \\\"INSERT INTO test.test (col) values ('SUCCESS');\\\"\"") do + its(:stdout) { should match /^INSERT 0 1$/ } + its(:exit_status) { should eq 0 } + end + end +end + +def queryForSelecting + describe 'Checking if it is possible to select values from the test table' do + let(:disable_sudo) { false } + describe command("su - postgres -c \"psql -t -c 'SELECT * from test.test;'\"") do + its(:stdout) { should match /\bSUCCESS\b/ } + its(:exit_status) { should eq 0 } + end + end +end + +def queryForDropping + describe 'Checking if it is possible to drop the test table' do + let(:disable_sudo) { false } + describe command("su - postgres -c \"psql -t -c 'DROP TABLE IF EXISTS test.test;'\"") do + its(:stdout) { should match /^DROP TABLE$/ } + its(:exit_status) { should eq 0 } + end + end + + describe 'Checking if it is possible to drop the test schema' do + let(:disable_sudo) { false } + describe command("su - postgres -c \"psql -t -c 'DROP SCHEMA IF EXISTS test;'\"") do + its(:stdout) { should match /^DROP SCHEMA$/ } + its(:exit_status) { should eq 0 } + end + end +end + +describe 'Checking if PostgreSQL service is running' do + describe service('postgresql') do + it { should be_enabled } + it { should be_running } + end +end + +if os[:family] == 'redhat' + describe 'Checking PostgreSQL directories and config files' do + let(:disable_sudo) { false } + describe file('/var/opt/rh/rh-postgresql10/lib/pgsql/data') do + it { should exist } + it { should be_a_directory } + end + describe file("/var/opt/rh/rh-postgresql10/lib/pgsql/data/pg_hba.conf") do + it { should exist } + it { should be_a_file } + it { should be_readable } + end + describe file("/var/opt/rh/rh-postgresql10/lib/pgsql/data/postgresql.conf") do + it { should exist } + it { should be_a_file } + it { should be_readable } + end + end +elsif os[:family] == 'ubuntu' + describe 'Checking PostgreSQL directories and config files' do + let(:disable_sudo) { false } + describe file('/etc/postgresql/10/main') do + it { should exist } + it { should be_a_directory } + end + describe file("/etc/postgresql/10/main/pg_hba.conf") do + it { should exist } + it { should be_a_file } + it { should be_readable } + end + describe file("/etc/postgresql/10/main/postgresql.conf") do + it { should exist } + it { should be_a_file } + it { should be_readable } + end + end +end + +describe 'Checking if the ports are open' do + let(:disable_sudo) { false } + describe port(postgresql_default_port) do + it { should be_listening } + end +end + +describe 'Checking if PostgreSQL is ready' do + describe command("pg_isready") do + its(:stdout) { should match /postgresql:#{postgresql_default_port} - accepting connections/ } + its(:exit_status) { should eq 0 } + end +end + +describe 'Checking if it is possible to connect to PostgreSQL database' do + let(:disable_sudo) { false } + describe command("su - postgres -c \"psql -t -c 'SELECT 2+2;'\"") do + its(:stdout) { should match /4/ } + its(:exit_status) { should eq 0 } + end +end + + +if !replicated + queryForCreating + queryForSelecting + queryForDropping +end + +if replicated + nodes = listInventoryHosts("postgresql") + master = nodes[0] + slave = nodes[1] + + if master.include? host_inventory['hostname'] + if os[:family] == 'redhat' + describe 'Checking PostgreSQL config files for master' do + let(:disable_sudo) { false } + describe command("cat /var/opt/rh/rh-postgresql10/lib/pgsql/data/postgresql.conf | grep wal_level") do + its(:stdout) { should match /^wal_level = replica/ } + its(:exit_status) { should eq 0 } + end + describe command("cat /var/opt/rh/rh-postgresql10/lib/pgsql/data/postgresql.conf | grep max_wal_senders") do + its(:stdout) { should match /^max_wal_senders = #{max_wal_senders}/ } + its(:exit_status) { should eq 0 } + end + describe command("cat /var/opt/rh/rh-postgresql10/lib/pgsql/data/postgresql.conf | grep wal_keep_segments") do + its(:stdout) { should match /^wal_keep_segments = #{wal_keep_segments}/ } + its(:exit_status) { should eq 0 } + end + describe command("su - postgres -c \"psql -t -c '\\du'\" | grep #{replication_user}") do + its(:stdout) { should match /#{replication_user}/ } + its(:stdout) { should match /Replication/ } + its(:exit_status) { should eq 0 } + end + describe command("cat /var/opt/rh/rh-postgresql10/lib/pgsql/data/pg_hba.conf | grep replication | grep md5") do + its(:stdout) { should match /#{replication_user}/ } + its(:stdout) { should match /replication/ } + its(:exit_status) { should eq 0 } + end + end + elsif os[:family] == 'ubuntu' + describe 'Checking PostgreSQL config files for master' do + let(:disable_sudo) { false } + describe command("cat /etc/postgresql/10/main/postgresql.conf | grep wal_level") do + its(:stdout) { should match /^wal_level = replica/ } + its(:exit_status) { should eq 0 } + end + describe command("cat /etc/postgresql/10/main/postgresql.conf | grep max_wal_senders") do + its(:stdout) { should match /^max_wal_senders = #{max_wal_senders}/ } + its(:exit_status) { should eq 0 } + end + describe command("cat /etc/postgresql/10/main/postgresql.conf | grep wal_keep_segments") do + its(:stdout) { should match /^wal_keep_segments = #{wal_keep_segments}/ } + its(:exit_status) { should eq 0 } + end + describe command("su - postgres -c \"psql -t -c '\\du'\" | grep #{replication_user}") do + its(:stdout) { should match /#{replication_user}/ } + its(:stdout) { should match /Replication/ } + its(:exit_status) { should eq 0 } + end + describe command("cat /etc/postgresql/10/main/pg_hba.conf | grep replication | grep md5") do + its(:stdout) { should match /#{replication_user}/ } + its(:stdout) { should match /replication/ } + its(:exit_status) { should eq 0 } + end + end + end + + describe 'Checking the status of master node' do + let(:disable_sudo) { false } + describe command("su - postgres -c \"psql -t -c 'SELECT usename, state from pg_stat_replication;'\"") do + its(:stdout) { should match /\bstreaming\b/ } + its(:stdout) { should match /\b#{replication_user}\b/ } + its(:exit_status) { should eq 0 } + end + end + + queryForDropping + queryForCreating + queryForSelecting + + elsif slave.include? host_inventory['hostname'] + if os[:family] == 'redhat' + describe 'Checking PostgreSQL config files for slave' do + let(:disable_sudo) { false } + describe command("cat /var/opt/rh/rh-postgresql10/lib/pgsql/data/postgresql.conf | grep hot_standby") do + its(:stdout) { should match /^hot_standby = on/ } + its(:exit_status) { should eq 0 } + end + describe file('/var/lib/pgsql/.pgpass') do + it { should exist } + it { should be_readable } + its(:content) { should match /#{replication_user}:#{replication_password}/ } + end + end + elsif os[:family] == 'ubuntu' + describe 'Checking PostgreSQL config files for slave' do + let(:disable_sudo) { false } + describe command("cat /etc/postgresql/10/main/postgresql.conf | grep hot_standby") do + its(:stdout) { should match /^hot_standby = on/ } + its(:exit_status) { should eq 0 } + end + describe file('/var/lib/postgresql/.pgpass') do + it { should exist } + it { should be_readable } + its(:content) { should match /#{replication_user}:#{replication_password}/ } + end + end + end + + describe 'Checking the state of replica nodes' do + let(:disable_sudo) { false } + describe command("su - postgres -c \"psql -t -c 'SELECT status, conninfo from pg_stat_wal_receiver;'\"") do + its(:stdout) { should match /\bstreaming\b/ } + its(:stdout) { should match /\buser=#{replication_user}\b/ } + its(:exit_status) { should eq 0 } + end + end + + queryForSelecting + + end + +end + diff --git a/core/core/test/serverspec/spec/prometheus/prometheus_spec.rb b/core/core/test/serverspec/spec/prometheus/prometheus_spec.rb index e0d192d440..ed9923bbd1 100644 --- a/core/core/test/serverspec/spec/prometheus/prometheus_spec.rb +++ b/core/core/test/serverspec/spec/prometheus/prometheus_spec.rb @@ -1,15 +1,322 @@ require 'spec_helper' -describe port(9090) do - it { should be_listening } +prometheus_host = 'localhost' +prometheus_port = 9090 +kube_apiserver_secure_port = 6443 +alertmanager_host = 'localhost' +alertmanager_port = 9093 + +describe 'Checking if Prometheus user exists' do + describe group('prometheus') do + it { should exist } + end + describe user('prometheus') do + it { should exist } + it { should belong_to_group 'prometheus' } + it { should have_login_shell '/usr/sbin/nologin' } + end +end + +describe 'Checking Prometheus directories and files' do + let(:disable_sudo) { false } + describe file('/var/lib/prometheus') do + it { should exist } + it { should be_a_directory } + it { should be_owned_by 'prometheus' } + it { should be_grouped_into 'prometheus' } + end + describe file('/etc/prometheus') do + it { should exist } + it { should be_a_directory } + it { should be_owned_by 'root' } + it { should be_grouped_into 'prometheus' } + end + describe file("/etc/prometheus/prometheus.yml") do + it { should exist } + it { should be_a_file } + it { should be_readable } + end end -describe group('prometheus') do - it { should exist } +describe 'Checking if Prometheus service is running' do + describe service('prometheus') do + it { should be_enabled } + it { should be_running } + end end -describe user('prometheus') do - it { should exist } - it { should have_login_shell '/usr/sbin/nologin' } - it { should belong_to_group 'prometheus' } +describe 'Checking if the ports are open' do + describe port(prometheus_port) do + let(:disable_sudo) { false } + it { should be_listening } + end +end + +describe 'Checking Prometheus health' do + describe command("curl -o /dev/null -s -w '%{http_code}' #{prometheus_host}:#{prometheus_port}/graph") do + it "is expected to be equal" do + expect(subject.stdout.to_i).to eq 200 + end + end + describe command("curl #{prometheus_host}:#{prometheus_port}/-/ready") do + its(:stdout) { should match /^Prometheus is Ready.$/ } + end + describe command("curl #{prometheus_host}:#{prometheus_port}/-/healthy") do + its(:stdout) { should match /^Prometheus is Healthy.$/ } + end +end + +describe 'Checking if Prometheus is serving metrics about itself' do + describe command("curl -o /dev/null -s -w '%{http_code}' #{prometheus_host}:#{prometheus_port}/metrics") do + it "is expected to be equal" do + expect(subject.stdout.to_i).to eq 200 + end + end + describe command("curl #{prometheus_host}:#{prometheus_port}/metrics") do + its(:stdout) { should_not match /^$/ } + end + end + + +describe 'Checking configuration files for Node exporter' do + listInventoryHosts("node_exporter").each do |val| + describe command("ls /etc/prometheus/file_sd") do + let(:disable_sudo) { false } + its(:stdout) { should match /node-#{val}.yml/ } + end + end +end + +describe 'Checking connection to Node Exporter hosts' do + listInventoryHosts("node_exporter").each do |val| + let(:disable_sudo) { false } + describe command("curl -o /dev/null -s -w '%{http_code}' $(grep -oP \"(?<=targets: \\\[\').*(?=\'\\\])\" /etc/prometheus/file_sd/node-#{val}.yml)/metrics") do + it "is expected to be equal" do + expect(subject.stdout.to_i).to eq 200 + end + end + end +end + +describe 'Checking configuration files for HAProxy Exporter' do + listInventoryHosts("haproxy_exporter").each do |val| + describe command("ls /etc/prometheus/file_sd") do + let(:disable_sudo) { false } + its(:stdout) { should match /haproxy-#{val}.yml/ } + end + end +end + +describe 'Checking connection to HAProxy Exporter hosts' do + listInventoryHosts("haproxy_exporter").each do |val| + let(:disable_sudo) { false } + describe command("curl -o /dev/null -s -w '%{http_code}' $(grep -oP \"(?<=targets: \\\[\\\").*(?=\\\"\\\])\" /etc/prometheus/file_sd/haproxy-#{val}.yml)/metrics") do + it "is expected to be equal" do + expect(subject.stdout.to_i).to eq 200 + end + end + end +end + +describe 'Checking configuration files for JMX Exporter' do + listInventoryHosts("jmx-exporter").each do |val| + describe command("ls /etc/prometheus/file_sd") do + let(:disable_sudo) { false } + its(:stdout) { should match /kafka-jmx-#{val}.yml/ } + its(:stdout) { should match /zookeeper-jmx-#{val}.yml/ } + end + end +end + +describe 'Checking connection to JMX Exporter hosts' do + listInventoryHosts("jmx-exporter").each do |val| + let(:disable_sudo) { false } + describe command("curl -o /dev/null -s -w '%{http_code}' $(grep -oP \"(?<=targets: \\\[\').*(?=\'\\\])\" /etc/prometheus/file_sd/kafka-jmx-#{val}.yml)/metrics") do + it "is expected to be equal" do + expect(subject.stdout.to_i).to eq 200 + end + end + describe command("curl -o /dev/null -s -w '%{http_code}' $(grep -oP \"(?<=targets: \\\[\').*(?=\'\\\])\" /etc/prometheus/file_sd/zookeeper-jmx-#{val}.yml)/metrics") do + it "is expected to be equal" do + expect(subject.stdout.to_i).to eq 200 + end + end + end +end + +describe 'Checking configuration files for Kafka Exporter hosts' do + listInventoryHosts("kafka-exporter").each do |val| + describe command("ls /etc/prometheus/file_sd") do + let(:disable_sudo) { false } + its(:stdout) { should match /kafka-exporter-#{val}.yml/ } + end + end +end + +describe 'Checking connection to Kafka Exporter hosts' do + listInventoryHosts("kafka-exporter").each do |val| + let(:disable_sudo) { false } + describe command("curl -o /dev/null -s -w '%{http_code}' $(grep -oP \"(?<=targets: \\\[\').*(?=\'\\\])\" /etc/prometheus/file_sd/kafka-exporter-#{val}.yml)/metrics") do + it "is expected to be equal" do + expect(subject.stdout.to_i).to eq 200 + end + end + end +end + +describe 'Checking connection to Kubernetes API server' do + listInventoryHosts("master").each do |val| + let(:disable_sudo) { false } + describe command("curl -o /dev/null -s -w '%{http_code}' -k -H \"Authorization: Bearer $(grep -A 3 kubernetes-apiservers /etc/prometheus/prometheus.yml \ + | awk '/bearer_token/ {print $2}')\" https://#{val}:#{kube_apiserver_secure_port}/metrics") do + it "is expected to be equal" do + expect(subject.stdout.to_i).to eq 200 + end + end + end +end + +describe 'Checking connection to Kubernetes cAdvisor' do + let(:disable_sudo) { false } + listInventoryHosts("master").each do |val_m| + describe command("curl -o /dev/null -s -w '%{http_code}' -k -H \"Authorization: Bearer $(grep -A 3 kubernetes-cadvisor /etc/prometheus/prometheus.yml \ + | awk '/bearer_token/ {print $2}')\" https://#{val_m}:#{kube_apiserver_secure_port}/api/v1/nodes/#{val_m}/proxy/metrics/cadvisor") do + it "is expected to be equal" do + expect(subject.stdout.to_i).to eq 200 + end + end + listInventoryHosts("worker").each do |val_w| + describe command("curl -o /dev/null -s -w '%{http_code}' -k -H \"Authorization: Bearer $(grep -A 3 kubernetes-cadvisor /etc/prometheus/prometheus.yml \ + | awk '/bearer_token/ {print $2}')\" https://#{val_m}:#{kube_apiserver_secure_port}/api/v1/nodes/#{val_w}/proxy/metrics/cadvisor") do + it "is expected to be equal" do + expect(subject.stdout.to_i).to eq 200 + end + end + end + end +end + +describe 'Checking connection to Kubernetes nodes' do + let(:disable_sudo) { false } + listInventoryHosts("master").each do |val_m| + describe command("curl -o /dev/null -s -w '%{http_code}' -k -H \"Authorization: Bearer $(grep -A 3 kubernetes-nodes /etc/prometheus/prometheus.yml \ + | awk '/bearer_token/ {print $2}')\" https://#{val_m}:#{kube_apiserver_secure_port}/api/v1/nodes/#{val_m}/proxy/metrics") do + it "is expected to be equal" do + expect(subject.stdout.to_i).to eq 200 + end + end + listInventoryHosts("worker").each do |val_w| + describe command("curl -o /dev/null -s -w '%{http_code}' -k -H \"Authorization: Bearer $(grep -A 3 kubernetes-nodes /etc/prometheus/prometheus.yml \ + | awk '/bearer_token/ {print $2}')\" https://#{val_m}:#{kube_apiserver_secure_port}/api/v1/nodes/#{val_w}/proxy/metrics") do + it "is expected to be equal" do + expect(subject.stdout.to_i).to eq 200 + end + end + end + end +end + + +# Tests for Alertmanager assuming monitoring.alerts.enable == true + +if readDataYaml["monitoring"]["alerts"]["enable"] == true + + describe 'Checking Alertmanager directories and files' do + let(:disable_sudo) { false } + describe file('/var/lib/prometheus/alertmanager') do + it { should exist } + it { should be_a_directory } + it { should be_owned_by 'prometheus' } + it { should be_grouped_into 'prometheus' } + end + describe file('/etc/prometheus/rules') do + it { should exist } + it { should be_a_directory } + it { should be_owned_by 'root' } + it { should be_grouped_into 'prometheus' } + end + describe file("/etc/prometheus/alertmanager.yml") do + it { should exist } + it { should be_a_file } + it { should be_readable } + end + end + + describe 'Checking if Alertmanager service is enabled' do + describe service('alertmanager') do + it { should be_enabled } + end + end + + describe 'Validating Alertmanager rules' do + describe command("/usr/local/bin/promtool check rules /etc/prometheus/rules/*") do + let(:disable_sudo) { false } + its(:stdout) { should_not match /FAILED/ } + its(:exit_status) { should eq 0 } + end + end + + describe 'Checking if it is possible to create a rule checking if node is up' do + describe command("cp -p /etc/prometheus/rules/UpDown.rules /etc/prometheus/rules/TEST_RULE.rules && sed -i 's/UpDown/TEST_RULE/g; s/down/up/g; s/== 0/== 1/g; \ + s/10s/1s/g' /etc/prometheus/rules/TEST_RULE.rules && systemctl restart prometheus") do + let(:disable_sudo) { false } + its(:exit_status) { should eq 0 } + end + describe command("for i in {1..10}; do if [ $(curl -o /dev/null -s -w '%{http_code}' #{prometheus_host}:#{prometheus_port}/graph) == 200 ]; \ + then curl -s #{prometheus_host}:#{prometheus_port}/rules | grep 'TEST_RULE'; break; else echo 'WAITING FOR PROMETHEUS TO BE STARTED'; sleep 1; fi; done;") do + its(:stdout) { should match /TEST_RULE/ } + its(:exit_status) { should eq 0 } + end + describe command("rm -rf /etc/prometheus/rules/TEST_RULE.rules && systemctl restart prometheus") do + let(:disable_sudo) { false } + its(:exit_status) { should eq 0 } + end + describe command("for i in {1..10}; do if [ $(curl -o /dev/null -s -w '%{http_code}' #{prometheus_host}:#{prometheus_port}/graph) == 200 ]; \ + then echo 'PROMETHEUS READY'; break; else echo 'WAITING FOR PROMETHEUS TO BE STARTED'; sleep 1; fi; done;") do + its(:stdout) { should match /READY/ } + its(:exit_status) { should eq 0 } + end + end + + # Tests for Alertmanager assuming monitoring.alerts.enable == true and monitoring.alerts.handlers.mail.enable == true + + if readDataYaml["monitoring"]["alerts"]["handlers"]["mail"]["enable"] == true + + describe 'Checking if the ports are open' do + describe port(alertmanager_port) do + let(:disable_sudo) { false } + it { should be_listening } + end + end + + describe 'Checking if Alertmanager service is running' do + describe service('alertmanager') do + it { should be_running } + end + end + + describe 'Checking Alertmanager health' do + describe command("curl -o /dev/null -s -w '%{http_code}' #{alertmanager_host}:#{alertmanager_port}") do + it "is expected to be equal" do + expect(subject.stdout.to_i).to eq 200 + end + end + describe command("curl #{alertmanager_host}:#{alertmanager_port}/-/ready") do + its(:stdout) { should match /^OK$/ } + end + describe command("curl #{alertmanager_host}:#{alertmanager_port}/-/healthy") do + its(:stdout) { should match /^OK$/ } + end + describe command("curl #{prometheus_host}:#{prometheus_port}/api/v1/alertmanagers") do + its(:stdout_as_json) { should include('status' => 'success') } + end + end + + describe 'Checking if it is possible to send an alert' do + describe command("curl -XPOST -d '[{\"labels\":{\"alertname\":\"TEST ALERT\", \"severity\":\"critical\"}}]' #{alertmanager_host}:#{alertmanager_port}/api/v1/alerts") do + its(:stdout_as_json) { should include('status' => 'success') } + end + end + + end end diff --git a/core/core/test/serverspec/spec/rabbitmq/rabbitmq_spec.rb b/core/core/test/serverspec/spec/rabbitmq/rabbitmq_spec.rb new file mode 100644 index 0000000000..f64ec702b3 --- /dev/null +++ b/core/core/test/serverspec/spec/rabbitmq/rabbitmq_spec.rb @@ -0,0 +1,180 @@ +require 'spec_helper' +require 'securerandom' + +rabbitmq_host = 'localhost' +rabbitmq_port = 5672 + +if readDataYaml["rabbitmq"] && readDataYaml["rabbitmq"]["amqp_port"] + rabbitmq_port = readDataYaml["rabbitmq"]["amqp_port"] +end + +rabbitmq_node_port = rabbitmq_port + 20000 +rabbitmq_api_port = 15672 + +clustered = false + +user = 'testuser' +pass = SecureRandom.hex + + +if readDataYaml["rabbitmq"] && readDataYaml["rabbitmq"]["cluster"] && readDataYaml["rabbitmq"]["cluster"]["is_clustered"] && + readDataYaml["rabbitmq"]["cluster"]["is_clustered"] == true + clustered = true +end + +describe 'Checking if RabbitMQ package is installed' do + describe package('rabbitmq-server') do + it { should be_installed } + end +end + +describe 'Checking if RabbitMQ user exists' do + describe group('rabbitmq') do + it { should exist } + end + describe user('rabbitmq') do + it { should exist } + it { should belong_to_group 'rabbitmq' } + end +end + +describe 'Checking if RabbitMQ service is running' do + describe service('rabbitmq-server') do + it { should be_enabled } + it { should be_running } + end +end + +describe 'Checking if the ports are open' do + let(:disable_sudo) { false } + describe port(rabbitmq_port) do + it { should be_listening } + end + describe port(rabbitmq_node_port) do + it { should be_listening } + end +end + +describe 'Checking RabbitMQ ping' do + describe command("rabbitmqctl ping") do + let(:disable_sudo) { false } + its(:stdout) { should match /^Ping succeeded$/ } + its(:exit_status) { should eq 0 } + end +end + +describe 'Checking the health of the target nodes' do + let(:disable_sudo) { false } + if clustered == true + listInventoryHosts("rabbitmq").each do |val| + describe command("rabbitmqctl node_health_check -n rabbit@#{val}") do + its(:stdout) { should match /^Health check passed$/ } + its(:exit_status) { should eq 0 } + end + end + else + describe command("rabbitmqctl node_health_check -n rabbit@#{host_inventory['hostname']}") do + its(:stdout) { should match /^Health check passed$/ } + its(:exit_status) { should eq 0 } + end + end +end + +describe 'Checking the RabbitMQ status/cluster status' do + let(:disable_sudo) { false } + describe command("rabbitmqctl status") do + its(:exit_status) { should eq 0 } + end + if clustered + listInventoryHosts("rabbitmq").each do |val| + describe command("rabbitmqctl cluster_status | awk '/running_nodes/,/}/'") do + its(:stdout) { should match /rabbit@#{val}/ } + its(:exit_status) { should eq 0 } + end + end + else + describe command("rabbitmqctl cluster_status | awk '/running_nodes/,/}/'") do + its(:stdout) { should match /rabbit@#{host_inventory['hostname']}/ } + its(:exit_status) { should eq 0 } + end + end +end + +describe 'Checking if it is possible to create the test user' do + describe command("rabbitmqctl add_user #{user} #{pass} && rabbitmqctl set_user_tags #{user} administrator \ + && rabbitmqctl set_permissions -p / #{user} \".*\" \".*\" \".*\"") do + let(:disable_sudo) { false } + its(:stdout) { should match /Adding user "#{user}"/ } + its(:stdout) { should match /Setting tags for user "#{user}" to \[administrator\]/ } + its(:stdout) { should match /Setting permissions for user "#{user}"/ } + its(:exit_status) { should eq 0 } + end +end + +# Tests to be run only when RabbitMQ plugins section is enabled + +plugins = [] + +if readDataYaml["rabbitmq"] && readDataYaml["rabbitmq"]["plugins"] + plugins = readDataYaml["rabbitmq"]["plugins"] +end + +describe 'Checking if RabbitMQ plugins are enabled' do + plugins.each do |val| + describe command("rabbitmq-plugins list -e") do + let(:disable_sudo) { false } + its(:stdout) { should match /\b#{val}\b/ } + its(:exit_status) { should eq 0 } + end + end +end + +# Tests to be run only when RabbitMQ Management Plugin is enabled + +if plugins.include? "rabbitmq_management" + + describe 'Checking if the port for RabbitMQ Management Plugin is open' do + let(:disable_sudo) { false } + describe port(rabbitmq_api_port) do + it { should be_listening } + end + end + + describe 'Checking nodes health using RabbitMQ API' do + let(:disable_sudo) { false } + if clustered + listInventoryHosts("rabbitmq").each do |val| + describe command("curl -o /dev/null -s -w '%{http_code}' -u #{user}:#{pass} #{rabbitmq_host}:#{rabbitmq_api_port}/api/healthchecks/node/rabbit@#{val}") do + it "is expected to be equal" do + expect(subject.stdout.to_i).to eq 200 + end + end + describe command("curl -u #{user}:#{pass} #{rabbitmq_host}:#{rabbitmq_api_port}/api/healthchecks/node/rabbit@#{val}") do + its(:stdout_as_json) { should include('status' => /ok/) } + its(:stdout_as_json) { should_not include('status' => /failed/) } + its(:exit_status) { should eq 0 } + end + end + else + describe command("curl -o /dev/null -s -w '%{http_code}' -u #{user}:#{pass} #{rabbitmq_host}:#{rabbitmq_api_port}/api/healthchecks/node/rabbit@#{host_inventory['hostname']}") do + it "is expected to be equal" do + expect(subject.stdout.to_i).to eq 200 + end + end + describe command("curl -u #{user}:#{pass} #{rabbitmq_host}:#{rabbitmq_api_port}/api/aliveness-test/%2F") do + its(:stdout_as_json) { should include('status' => /ok/) } + its(:stdout_as_json) { should_not include('status' => /failed/) } + its(:exit_status) { should eq 0 } + end + end + end +end + +describe 'Cleaning up' do + describe command("rabbitmqctl delete_user #{user}") do + let(:disable_sudo) { false } + its(:stdout) { should match /Deleting user "#{user}"/ } + its(:exit_status) { should eq 0 } + end +end + diff --git a/core/core/test/serverspec/spec/spec_helper.rb b/core/core/test/serverspec/spec/spec_helper.rb index d522473d51..71c10383ac 100644 --- a/core/core/test/serverspec/spec/spec_helper.rb +++ b/core/core/test/serverspec/spec/spec_helper.rb @@ -1,5 +1,6 @@ require 'serverspec' require 'net/ssh' +require 'yaml' set :backend, :ssh @@ -27,11 +28,60 @@ set :ssh_options, options # Disable sudo -# set :disable_sudo, true +set :disable_sudo, true +# Set shell +set :shell, '/bin/bash' # Set environment variables # set :env, :LANG => 'C', :LC_MESSAGES => 'C' # Set PATH # set :path, '/sbin:/usr/local/sbin:$PATH' + + def count_inventory_roles(role) + file = File.open(ENV['inventory'], "rb") + input = file.read + file.close + if input.include? "[#{role}]" + rows = input.split("[#{role}]")[1].split("[")[0] + counter = rows.scan(/ansible_host/).count + else counter = 0 + end + return counter + end + + def hostInGroups?(role) + file = File.open(ENV['inventory'], "rb") + input = file.read + file.close + if input.include? "[#{role}]" + rows = input.split("[#{role}]")[1].split("[")[0] + return rows.include? ENV['TARGET_HOST'] + else return false + end + end + + def readDataYaml + path = ENV['inventory'].dup + path.sub! 'inventory/development' , 'data/manifest.yaml' + datayaml = YAML.load_file(path) + return datayaml + end + + def listInventoryHosts(role) + file = File.open(ENV['inventory'], "rb") + input = file.read + file.close + list = [] + if input.include? "[#{role}]" + rows = input.split("[#{role}]")[1].split("[")[0] + rows.each_line do |line| + if line[0] != '#' and line =~ /(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})/ + list << line.split.first + end + end + end + return list + end + diff --git a/core/core/test/serverspec/spec/zookeeper/zookeeper_spec.rb b/core/core/test/serverspec/spec/zookeeper/zookeeper_spec.rb index 23c73fe7a5..9fa263e620 100644 --- a/core/core/test/serverspec/spec/zookeeper/zookeeper_spec.rb +++ b/core/core/test/serverspec/spec/zookeeper/zookeeper_spec.rb @@ -1,27 +1,85 @@ require 'spec_helper' -describe service('zookeeper') do - it { should be_enabled } - it { should be_running } -end +kafka_host = 'localhost' +kafka_port = 9092 +zookeeper_host = 'localhost' +zookeeper_client_port = 2181 +zookeeper_peer_port = 2888 +zookeeper_leader_port = 3888 -describe port(2181) do - it { should be_listening } +describe 'Checking if ZooKeeper service is running' do + describe service('zookeeper') do + it { should be_enabled } + it { should be_running } + end end -describe port(7072) do - it { should be_listening } +describe 'Checking if the ports are open' do + + # checking port for client connections + describe port(zookeeper_client_port) do + let(:disable_sudo) { false } + it { should be_listening } + end + + # checking port for follower connections to the leader + describe command("if /opt/zookeeper/bin/zkServer.sh status | grep 'Mode: leader'; then netstat -tunl | grep #{zookeeper_peer_port}; else echo 'not leader'; fi") do + its(:stdout) { should match /#{zookeeper_peer_port}|not leader/ } + its(:exit_status) { should eq 0 } + end + + # checking port for leader election + describe command("if /opt/zookeeper/bin/zkServer.sh status | grep 'Mode: standalone'; then echo 'standalone'; else netstat -tunl | grep #{zookeeper_leader_port}; fi") do + its(:stdout) { should match /#{zookeeper_leader_port}|standalone/ } + its(:exit_status) { should eq 0 } + end +end + +describe 'Checking if ZooKeeper user exists' do + describe group('zookeeper') do + it { should exist } + end + describe user('zookeeper') do + it { should exist } + it { should belong_to_group 'zookeeper' } + it { should have_home_directory '/home/zookeeper' } + it { should have_login_shell '/usr/sbin/nologin' } + end + describe file('/opt/zookeeper') do + it { should exist } + it { should be_directory } + end end -describe user('zookeeper') do - it { should exist } - it { should have_login_shell '/usr/sbin/nologin' } - it { should belong_to_group 'zookeeper' } - it { should belong_to_group 'jmx-exporter' } +describe 'Checking if ZooKeeper is healthy' do + describe command("echo 'stat' | curl -s telnet://#{zookeeper_host}:#{zookeeper_client_port}") do + its(:stdout) { should match /Zookeeper version/ } + end + describe command("echo 'ruok' | curl -s telnet://#{zookeeper_host}:#{zookeeper_client_port}") do + its(:stdout) { should match /imok/ } + end end -#check if output from jmx exporter is correct -#describe command('curl -s localhost:7072 | grep zookeeper') do -# its(:stdout) { should match /zookeeper_/ } -# its(:exit_status) { should eq 0 } -#end +describe 'Checking ZooKeeper status' do + describe command('/opt/zookeeper/bin/zkServer.sh status 2>&1') do + its(:stdout) { should match /Mode: leader|Mode: follower|Mode: standalone/ } + its(:stdout) { should_not match /Error contacting service. It is probably not running./ } + end +end + +describe 'Checking if it is possible to list down and count all the active brokers' do + describe command("echo 'ls /brokers/ids' | /opt/zookeeper/bin/zkCli.sh -server #{zookeeper_host}:#{zookeeper_client_port}") do + its(:stdout) { should match /Welcome to ZooKeeper!/ } + its(:stdout) { should match /\[(\d+(\,\s)?)+\]/ } # pattern: [0, 1, 2, 3 ...] + its(:exit_status) { should eq 0 } + end + describe command("echo 'dump' | curl -s telnet://#{zookeeper_host}:#{zookeeper_client_port} | grep brokers") do + its(:stdout) { should match /\/brokers\/ids\/\d+/ } # pattern: /brokers/ids/0 + its(:exit_status) { should eq 0 } + end + describe command("echo 'dump' | curl -s telnet://#{zookeeper_host}:#{zookeeper_client_port} | grep -c brokers") do + it "is expected to be equal" do + expect(subject.stdout.to_i).to eq count_inventory_roles("kafka") + end + end +end diff --git a/core/data/azure/infrastructure/epiphany-bld-apps/data.yaml b/core/data/azure/infrastructure/epiphany-bld-apps/data.yaml index 34aaaebecb..c13a7b7697 100644 --- a/core/data/azure/infrastructure/epiphany-bld-apps/data.yaml +++ b/core/data/azure/infrastructure/epiphany-bld-apps/data.yaml @@ -308,9 +308,9 @@ core: #publisher: RedHat #offer: RHEL - #sku: 7.4 + #sku: 7-RAW # Never put latest on anything! Need to always pin the version number but testing we can get away with it - #version: "7.4.2018010506" + #version: "7.6.2018103108" storage_os_disk: &storage_os_disk managed: true diff --git a/core/data/azure/infrastructure/epiphany-playground/basic-data.yaml b/core/data/azure/infrastructure/epiphany-playground/basic-data.yaml index 552435fa0d..1bc4370751 100644 --- a/core/data/azure/infrastructure/epiphany-playground/basic-data.yaml +++ b/core/data/azure/infrastructure/epiphany-playground/basic-data.yaml @@ -8,11 +8,11 @@ azure: subscription_name: YOUR-SUBSCRIPTION-NAME resource_group: "playground" location: "West Europe" + image_publisher: RedHat #Canonical image_offer: RHEL #UbuntuServer + image_sku: 7-RAW #18.04-LTS + image_version: 7.6.2018103108 #18.04.201810030 boot_storage: "epiplaygrnd1" - image_sku: 7.5 #18.04-LTS - image_publisher: RedHat #Canonical - image_version: 7.5.2018081519 #18.04.201810030 vm_name_prefix: "vmplaygrnd" create_service_principal: true security: diff --git a/core/data/azure/infrastructure/epiphany-qa-basic/basic-data.yaml b/core/data/azure/infrastructure/epiphany-qa-basic/basic-data.yaml new file mode 100644 index 0000000000..04ab7973aa --- /dev/null +++ b/core/data/azure/infrastructure/epiphany-qa-basic/basic-data.yaml @@ -0,0 +1,32 @@ +--- +# Simplified datafile that you can use together with QA template. +kind: simplified-datafile +version: 0.2.2 +environment_name: {{ resource_group }} +azure: + subscription_name: {{ sp_subscription_name }} + resource_group: {{ resource_group }} + location: "West Europe" + image_publisher: {{ image_publisher }} + image_offer: {{ image_offer }} + image_sku: {{ image_sku }} + image_version: {{ image_version }} + boot_storage: {{ resource_group }} + vm_name_prefix: vm-{{ resource_group }} + create_service_principal: false +security: + keys_directory: "/tmp/keys" + key_file_name: id_rsa + public_key_file_name: id_rsa.pub +platform: + lb_vms: {{ lb_vms }} + worker_vms: {{ worker_vms }} + kafka_vms: {{ kafka_vms }} + postgresql_vms: {{ postgresql_vms }} + rabbitmq_vms: {{ rabbitmq_vms }} + monitoring_vms: {{ monitoring_vms }} + elk_vms: {{ elk_vms }} +config: + rabbitmq_deployment: {{ rabbitmq_deployment }} + auth_service_deployment: {{ auth_service_deployment }} + alerts_for_monitoring: {{ alerts_for_monitoring }} diff --git a/core/data/azure/infrastructure/epiphany-qa-template/data.yaml.j2 b/core/data/azure/infrastructure/epiphany-qa-template/data.yaml.j2 new file mode 100644 index 0000000000..a1870087a7 --- /dev/null +++ b/core/data/azure/infrastructure/epiphany-qa-template/data.yaml.j2 @@ -0,0 +1,1347 @@ +--- +########################################################## +# This file is autogenerated for environment {{ environment_name }} +# Data file for Epiphany +# Azure specific +########################################################## + +########################### +title: Epiphany ({{ azure.image_offer }}) {{ environment_name }} + +kind: datafile +version: 0.2.2 + +# NOTE: Any data values that are empty put "" or the value None will be used in the templates for those attributes. + +core: + # This will apply to a VPN like environment or an air-gapped like environment + bastian: + enable: false + # This host will be set ONLY for environments where a bastian host is supplied to us and NOT part of the cluster build + host: '' + # If the bastian host has a different key + key_path: '' + user: '' + # if key_path is '' + pwd: '' + + build: + # IMPORTANT - will be appended to release name and output folder and part of the template names + version: &version 0.2.2 + # Type of build environment + environment: &env development + # Name of the given release. Version will be appended + release: epiphany-dev + # If repo_root is true then add that as a prefix for the output + repo_root: true + platform: azure + output: '/build/$PLATFORM/infrastructure/epiphany' + + tags: &tags + - key: environment + value: *env + - key: version + value: *version + - key: project + value: epiphany + - key: location + value: {{ azure.location }} + + # domain is used to create DNS entries or just add FQDN to hosts.yaml file used in automation + domain: + enable: false + name: example.com + create_dns: false + + # These will become the role/classification you will use with the automation to group nodes for given tasks + # Use '_' and not '-' in names + roles: + - master + - worker + - kafka + - zookeeper + - grafana + - node_exporter + - prometheus + - elasticsearch + - elasticsearch-curator + - kibana + - filebeat + - jmx-exporter + - kafka-exporter + - haproxy_tls_termination + - haproxy_exporter + - postgresql + - rabbitmq + - deployments + - reboot + # These last two must always be present + - linux + # - windows + + admin_user: + name: &admin_username operations + # May want to change to 'key' and create 'key_path' with 'pwd' or 'home' + key_path: {{ security.keys_directory }}/{{ security.key_file_name }} + + azure: + tags: + <<: *tags + + terraform: + # This version info is what version is being used at the moment. The version of Terraform in the manifest.yaml in the + # root of the repo is for the initial install and the minum version + version: 1.6 + service_principal: + # Three files are required for SPs to work, az_ad_sp.json, env.sh and security.yaml. By default, these are created if the + # 'create' attribute is true. If false then you will need to supply those two files. This allows you to create + # a service_principal of your own instead of having one generated. + # You will also need to override env.sh that contains the 'ARM_...' environment variables required. + enable: true + create: {{ azure.create_service_principal }} # If you want to use an existing one then set this to false + auth: pwd # Valid is 'pwd' and 'cert'. At this time Terraform only support 'pwd' for service principals (sp true) + # NOTE: Backend is a Terraform resource that stores the *.tfstate files used by Terraform to store state. The default + # is to store the state file locally but this can cause issues if working in a team environment. + backend: + # Only used by Terraform + # The backend storage account is '''backend' (combined name with suffix) + # The storage container is generated as ''-'terraform' + # NOTE: Known issue with backend tf when having different VM types below when enabled! So, only one VM entry with count set should be used. Set to false for now... + enable: false + storage_account: + storage_container: + # sleep: 15 # Number of seconds to sleep so that Azure can create the required containers before moving on + type: blob + tags: *tags + + # NOTE: May want to create region/AZ sub-folders to support creating a cluster in different regions and AZ for HA... + resource_group: &resource_group + name: &name {{ azure.resource_group }} + location: &location {{ azure.location }} + + # Subscription name or ID + subscription: {{ azure.subscription_name }} + + # Azure Active Directory + ad: + name: *name + role: Contributor + + standard: + # One resource group is supported + resource_group: + <<: *resource_group + # exists: false - TO BE REMOVED + prevent_destory: true + # IMPORTANT - Do not set lock if you plan on editing anything in the resource group! Leave it set to false until you are ready + # ReadOnly or CanNotDelete are the only two level options if enabled! + lock: + # NOTE: This will cause locks.tf.wait to be generated. You will need a script to rename this to locks.tf and run apply again + enable: true + name: epiphany-lock + level: ReadOnly + notes: This locks the entire resource group! + tags: *tags + + # This aids in boot diagnostics if there are issues: + # https://docs.microsoft.com/en-us/azure/virtual-machines/linux/boot-diagnostics + debug: + # WIP + enable: false + storage_account: + name: {{ azure.boot_storage }} + account: + tier: Standard + replication_type: LRS + # Storage, StorageV2 and BlobStorage are supported types + kind: StorageV2 + storage_container: + name: debug + + availability: + availability_set: + enable: true + name: ha-epiphany + platform_fault_domain_count: 3 + platform_update_domain_count: 5 + managed: true # Best to keep this 'true' in most cases. Mixing availability set managed with vm disk unmanaged is not allowed + tags: *tags + + security: + ssh: &ssh + key: + # Public portion of the key + file: {{ security.keys_directory }}/{{ security.public_key_file_name }} + data: + + vpn: + # Make SURE all of the data is correct below before enabling 'vpn'. It can also take 30 minutes to an hour to create. + enable: false + name: vpn-epiphany + # Only support RouteBased type of connection currently + type: RouteBased + active_active: false + # There are two types of 'sku' (Basic and Standard). Always use Standard so any client can use. + sku: Standard + # Address space that is required by Virtual Network Gateway. This address space must be inside of virtual_network.address_space and do not overlap with other subnets defined + gateway_subnet_space: 10.0.3.0/24 + client_configuration: + # This section is very important! + # You must specify the address_space that will be allocated to use on the VPN side of the conection that will + # be able to talk to your cluster. + address_space: + - 172.16.0.0/24 + root_certificate: + # name is the name of the cert that was created for you by a trusted party OR a name you give a self-signed cert + name: EpiphanyRootCa + revoked_certificate: + name: EpiphanyRevoked + thumbprint: your-revoked-cert-thumbprint + # public_cert_data is the actual base64 public key from your cert. Put it in 'as is'. The '|' tells yaml to use 'as is'. + public_cert_data: | + YOUR-BASE64-CLIENT-AUTH-PUBLIC-KEY + + ip_configuration: + name: vpn-ip-config + public_ip: + output: + enable: true + sensitive: false + name: pip-vpn-epiphany + address_allocation: dynamic + idle_timeout_in_minutes: 30 + + tags: *tags + + # subnet_id is generated so use terraform variable + private_ip: + output: + enable: true + sensitive: false + address_allocation: dynamic + # address only applies if 'address_allocation' is 'static' + address: 10.0.2.5 + + network_security_group: + enable: true + # Note: Could create an array of NSGs and reference them name + '_' + # (i.e., epiphany_nsg_001) maybe at somepoint if needed + name: security-nsg-epiphany + tags: *tags + + # Add another NSG rules in order to access resources. + rules: + - name: ssh + description: Allow SSH + priority: 101 + direction: Inbound + access: Allow + protocol: Tcp + source_port_range: "*" + destination_port_range: "22" + source_address_prefix: "*" + destination_address_prefix: "*" + + virtual_network: + name: security-vnet-epiphany + address_space: + - 10.0.0.0/8 + + subnet: + name: vnet-subnet-epiphany + address_prefix: 10.0.0.0/16 + # Service endpoints bypass normal public route to Azure SQL, Storage and CosmosDB services + service_endpoints: + - Microsoft.Storage + - Microsoft.Sql + - Microsoft.AzureCosmosDB + + # NOTE: Managed vs Unmanaged storage + # If you want managed storage then by default, 'storage_account' and 'storage_container' options are not required BUT + # they are still enabled in the 'main.tf.j2' template. This could be set with an enable option. + + storage_managed_disk: + # WIP + enable: false + name: epiphany-mdisk + storage_account_type: Premium_LRS + create_option: Empty + disk_size_gb: 500 + count: 1 + + # Once storage account is supported + # Should use (maybe) a different storage account for different locations. Meaning, if you have two clusters (one in east and one in west) then having a storage account for east and one for west would be good since you want storage near compute. + # No `-` in storage account name + + # 3-24 Alphanumeric only lower case + storage_account: + enable: false + name: epiphany + account: + tier: Standard + replication_type: LRS + # Storage, StorageV2 and BlobStorage are supported types + kind: StorageV2 + + tags: *tags + + storage_container: + enable: false + # 3-63 Alphanumeric lower case plus '-' + name: epiphany-osdisks + access_type: private + + storage_blob: + enable: false + name: epiphany-osdisks + type: page + # size in bytes in 512 increments + size: 5120 + count: 2 + + storage_image_reference: &storage_image_reference + # publisher: Canonical + # offer: UbuntuServer + # sku: 16.04-LTS + # Never put latest on anything! Need to always pin the version number but testing we can get away with it + # version: latest + + publisher: {{ azure.image_publisher }} + offer: {{ azure.image_offer }} + sku: {{ azure.image_sku }} + # Never put latest on anything! Need to always pin the version number but testing we can get away with it + version: {{ azure.image_version }} + + storage_os_disk: &storage_os_disk + managed: false + caching: ReadWrite + create_option: FromImage + disk_size_gb: 30 + managed_disk_type: Premium_LRS + + storage_data_disk: &storage_data_disk + # Determines if a data disk will be added. If also using managed disks then use 'Attach' for create_option instead of 'Empty' + enable: false + count: 2 + managed: false + caching: ReadWrite + create_option: Empty + disk_size_gb: 30 + managed_disk_type: Standard_LRS # Can only be Standard_LRS if using non-managed availability_set + + os_profile: &os_profile + admin_username: operations + admin_password: YourPwd + + os_profile_linux_config: &os_profile_linux_config + # NOTE: Must set to true except in early stage development! This will enforce ssh key authentication for now... + disable_password_authentication: true + + os_profile_windows_config: &os_profile_windows_config + # NOTE: Must set to true except in early stage development! This will enforce ssh key authentication for now... + disable_password_authentication: true + + # Testing sets flags that allow builds to inject commands into VMs and other related items + testing: + enable: false + + # NOTE: This MUST equal the total number of 'count' metadata from each 'vm' array entry below + vm_count: 8 + + # VM names - 1-15 characters for Windows and 1-64 characters for Linux + vms: + + vms: +{% if platform.lb_vms is defined and platform.lb_vms > 0 %} + - name: {{ azure.vm_name_prefix }}-lb + size: Standard_DS1_v2 + os_type: linux + count: {{ platform.lb_vms }} + # One host will need to be a bastian host UNLESS the bastian host is provided in another way + bastian_host: false + # roles are how you define a grouping of nodes. These values will be used to create an inventory of your cluster + # Must be a member of the 'role' in core + roles: + - linux + - haproxy_tls_termination + - haproxy_exporter + - node_exporter + - filebeat + + delete_os_disk_on_termination: false + delete_data_disks_on_termination: false + # NOTE: This must match the interfaces public_ip options. If one interface has enabled public_ip then this MUST be true else false + public_ips: true + depends_on: + # Put the name of the resource. If a VM then only put the 'name:' value. The count portion will be automatically added + - '${var.master_depends_on}' + tags: *tags + + security: + # Each node will have firewalld (redhat) enabled and allowing only given ports, remember that Epiphany needs port 22 for bootstrapping environments + firewall: + enable: false + ports_open: + - 22/tcp + - 443/tcp + + # NOTE: For Kubernetes builds we should *not* use provisioners to bootstrap any data. The point is to build a common + # infrastructure and then collect IPs etc. that are required for K8s automation to build. The ONLY possible + # exception is for automated testing in VSTS + bootstrap: + # Can use the global values or override them on a per vm type basis + ssh: *ssh + + provisioners: + file: + enable: false + # NOTE: Pay close attention to the trailing '/' for 'source'. If you leave '/' off of the end of source then the directory tree will be copied to 'destination'. If you add '/' then the contents of that directory will be copied into the destination directory. + source: "../../../../core/src/scripts/kubernetes/linux" + destination: "/home/${var.admin_username}" + + remote_exec: + enable: false + # NOTE: Use Terraform vars if you need to embed variables in the commands. + inline: # Example is below and only here to show + - 'chmod +x ${var.file_destination}/linux/make-executable.sh' + - '${var.file_destination}/linux/make-executable.sh' + - 'echo ${var.admin_password} | sudo -S ${var.file_destination}/linux/prepare-system-kubernetes-redhat.sh' + + testing_inline: + - 'echo \"##vso[task.setvariable variable=toPrint]Print Me Please\"' + + interfaces: + # A VM can have multiple IP addresses and interfaces. In this case we are + # saying there is one public IP per network interface + - network_interface: + # name has vm name appended to it so only put basic type name here + name: nic + primary: true + tags: *tags + + # This only works with certain size VMs + # Accelerated Networking is supported on most general purpose and compute-optimized instance sizes with 2 or more vCPUs. + # These supported series are: D/DSv2 and F/Fs. On instances that support hyperthreading, Accelerated Networking is supported + # on VM instances with 4 or more vCPUs. Supported series are: D/DSv3, E/ESv3, Fsv2, and Ms/Mms. + # https://docs.microsoft.com/en-us/azure/virtual-network/create-vm-accelerated-networking-cli + enable_accelerated_networking: false + + ip_configuration: + name: ip-config + # subnet_id is generated so use terraform variable + private_ip: + output: + enable: true + sensitive: false + address_allocation: Dynamic + # address only applies if 'address_allocation' is 'static' + address: 10.0.2.5 + + public_ip: + # If you set to false there must be some host set as a bastian host + enable: true + output: + enable: true + sensitive: false + name: pip-epiphany + address_allocation: static + idle_timeout_in_minutes: 30 + sku: Standard + + tags: *tags +{% endif %} + + - name: {{ azure.vm_name_prefix }}-master + size: Standard_DS2_v2 + os_type: linux + count: 1 + # One host will need to be a bastian host UNLESS the bastian host is provided in another way + bastian_host: false + # roles are how you define a grouping of nodes. These values will be used to create an inventory of your cluster + # Must be a member of the 'role' in core + roles: + - linux + - master +{% if (config.rabbitmq_deployment is defined and config.rabbitmq_deployment == true) or (config.auth_service_deployment is defined and config.auth_service_deployment == true and platform.postgresql_vms > 0) %} + - deployments +{% endif %} + - filebeat + - node_exporter + - reboot + delete_os_disk_on_termination: false + delete_data_disks_on_termination: false + # NOTE: This must match the interfaces public_ip options. If one interface has enabled public_ip then this MUST be true else false + public_ips: true + depends_on: + # Put the name of the resource. If a VM then only put the 'name:' value. The count portion will be automatically added + - '${var.master_depends_on}' + tags: *tags + + security: + # Each node will have firewalld (redhat) enabled or not allowing the given ports + firewall: + enable: false + # These ports can be in sync with NSG rules below or not. Meaning, you may have NSG disabled and want each node to block + # Each node could belong to a different grouping with a need for different ports + ports_open: + - 443 + - 22 + - 6443 + - 2379-2380 + - 10250 + - 10251 + - 10252 + - 10255 + + # NOTE: For Kubernetes builds we should *not* use provisioners to bootstrap any data. The point is to build a common + # infrastructure and then collect IPs etc. that are required for K8s automation to build. The ONLY possible + # exception is for automated testing in VSTS (maybe) + bootstrap: + # Can use the global values or override them on a per vm type basis + ssh: *ssh + + provisioners: + file: + enable: false + # NOTE: Pay close attention to the trailing '/' for 'source'. If you leave '/' off of the end of source then the directory tree will be copied to 'destination'. If you add '/' then the contents of that directory will be copied into the destination directory. + source: "../../../../core/src/scripts/kubernetes/linux" + destination: "/home/${var.admin_username}" + + remote_exec: + enable: false + # NOTE: Use Terraform vars if you need to embed variables in the commands. + inline: # Example is below and only here to show + - 'chmod +x ${var.file_destination}/linux/make-executable.sh' + - '${var.file_destination}/linux/make-executable.sh' + - 'echo ${var.admin_password} | sudo -S ${var.file_destination}/linux/prepare-system-kubernetes-redhat.sh' + + testing_inline: + - 'echo \"##vso[task.setvariable variable=toPrint]Print Me Please\"' + + interfaces: + # A VM can have multiple IP addresses and interfaces. In this case we are + # saying there is one public IP per network interface + - network_interface: + # name has vm name appended to it so only put basic type name here + name: nic + primary: true + tags: *tags + + # This only works with certain size VMs + # Accelerated Networking is supported on most general purpose and compute-optimized instance sizes with 2 or more vCPUs. + # These supported series are: D/DSv2 and F/Fs. On instances that support hyperthreading, Accelerated Networking is supported + # on VM instances with 4 or more vCPUs. Supported series are: D/DSv3, E/ESv3, Fsv2, and Ms/Mms. + # https://docs.microsoft.com/en-us/azure/virtual-network/create-vm-accelerated-networking-cli + enable_accelerated_networking: false + + ip_configuration: + name: ip-config + # subnet_id is generated so use terraform variable + private_ip: + output: + enable: true + sensitive: false + address_allocation: dynamic + # address only applies if 'address_allocation' is 'static' + address: 10.0.2.5 + + public_ip: + # If you set to false there must be some host set as a bastian host + enable: true + output: + enable: true + sensitive: false + name: pip-epiphany + address_allocation: static + idle_timeout_in_minutes: 30 + sku: Standard + tags: *tags + + - name: {{ azure.vm_name_prefix }}-node + size: Standard_DS1_v2 + os_type: linux + count: {{ platform.worker_vms }} + # One host will need to be a bastian host UNLESS the bastian host is provided in another way + bastian_host: false + # roles are how you define a grouping of nodes. These values will be used to create an inventory of your cluster + # Must be a member of the 'role' in core + roles: + - linux + - worker + - filebeat + - node_exporter + - reboot + delete_os_disk_on_termination: false + delete_data_disks_on_termination: false + # NOTE: This must match the interfaces public_ip options. If one interface has enabled public_ip then this MUST be true else false + public_ips: true + depends_on: + # Put the name of the resource. If a VM then only put the 'name:' value. The count portion will be automatically added + - '${var.master_depends_on}' + tags: *tags + + security: + # Each node will have firewalld (redhat) enabled or not allowing the given ports + firewall: + enable: false + # These ports can be in sync with NSG rules below or not. Meaning, you may have NSG disabled and want each node to block + # Each node could belong to a different grouping with a need for different ports + ports_open: + - 443 + - 22 + - 6443 + - 2379-2380 + - 10250 + - 10251 + - 10252 + - 10255 + + # NOTE: For Kubernetes builds we should *not* use provisioners to bootstrap any data. The point is to build a common + # infrastructure and then collect IPs etc. that are required for K8s automation to build. The ONLY possible + # exception is for automated testing in VSTS (maybe) + bootstrap: + # Can use the global values or override them on a per vm type basis + ssh: *ssh + + provisioners: + file: + enable: false + # NOTE: Pay close attention to the trailing '/' for 'source'. If you leave '/' off of the end of source then the directory tree will be copied to 'destination'. If you add '/' then the contents of that directory will be copied into the destination directory. + source: "../../../../core/src/scripts/kubernetes/linux" + destination: "/home/${var.admin_username}" + + remote_exec: + enable: false + # NOTE: Use Terraform vars if you need to embed variables in the commands. + inline: # Example is below and only here to show + - 'chmod +x ${var.file_destination}/linux/make-executable.sh' + - '${var.file_destination}/linux/make-executable.sh' + - 'echo ${var.admin_password} | sudo -S ${var.file_destination}/linux/prepare-system-kubernetes-redhat.sh' + + testing_inline: + - 'echo \"##vso[task.setvariable variable=toPrint]Print Me Please\"' + + interfaces: + # A VM can have multiple IP addresses and interfaces. In this case we are + # saying there is one public IP per network interface + - network_interface: + # name has vm name appended to it so only put basic type name here + name: nic + primary: true + tags: *tags + + # This only works with certain size VMs + # Accelerated Networking is supported on most general purpose and compute-optimized instance sizes with 2 or more vCPUs. + # These supported series are: D/DSv2 and F/Fs. On instances that support hyperthreading, Accelerated Networking is supported + # on VM instances with 4 or more vCPUs. Supported series are: D/DSv3, E/ESv3, Fsv2, and Ms/Mms. + # https://docs.microsoft.com/en-us/azure/virtual-network/create-vm-accelerated-networking-cli + enable_accelerated_networking: false + + ip_configuration: + name: ip-config + # subnet_id is generated so use terraform variable + private_ip: + output: + enable: true + sensitive: false + address_allocation: dynamic + # address only applies if 'address_allocation' is 'static' + address: 10.0.2.5 + + public_ip: + # If you set to false there must be some host set as a bastian host + enable: true + output: + enable: true + sensitive: false + name: pip-epiphany + address_allocation: static + idle_timeout_in_minutes: 30 + sku: Standard + tags: *tags + +{% if platform.monitoring_vms is defined and platform.monitoring_vms > 0 %} + - name: {{ azure.vm_name_prefix }}-prometheus + size: Standard_DS1_v2 + os_type: linux + count: {{ platform.monitoring_vms }} + # One host will need to be a bastian host UNLESS the bastian host is provided in another way + bastian_host: false + # roles are how you define a grouping of nodes. These values will be used to create an inventory of your cluster + # Must be a member of the 'role' in core + roles: + - linux + - prometheus + - grafana + - node_exporter + - filebeat + - reboot + delete_os_disk_on_termination: false + delete_data_disks_on_termination: false + # NOTE: This must match the interfaces public_ip options. If one interface has enabled public_ip then this MUST be true else false + public_ips: true + depends_on: + # Put the name of the resource. If a VM then only put the 'name:' value. The count portion will be automatically added + - '${var.master_depends_on}' + tags: *tags + + security: + # Each node will have firewalld (redhat) enabled or not allowing the given ports + firewall: + enable: false + # These ports can be in sync with NSG rules below or not. Meaning, you may have NSG disabled and want each node to block + # Each node could belong to a different grouping with a need for different ports + ports_open: + - 443 + - 22 + - 6443 + - 2379-2380 + - 10250 + - 10251 + - 10252 + - 10255 + + # NOTE: For Kubernetes builds we should *not* use provisioners to bootstrap any data. The point is to build a common + # infrastructure and then collect IPs etc. that are required for K8s automation to build. The ONLY possible + # exception is for automated testing in VSTS (maybe) + bootstrap: + # Can use the global values or override them on a per vm type basis + ssh: *ssh + + provisioners: + file: + enable: false + # NOTE: Pay close attention to the trailing '/' for 'source'. If you leave '/' off of the end of source then the directory tree will be copied to 'destination'. If you add '/' then the contents of that directory will be copied into the destination directory. + source: "../../../../core/src/scripts/kubernetes/linux" + destination: "/home/${var.admin_username}" + + remote_exec: + enable: false + # NOTE: Use Terraform vars if you need to embed variables in the commands. + inline: # Example is below and only here to show + - 'chmod +x ${var.file_destination}/linux/make-executable.sh' + - '${var.file_destination}/linux/make-executable.sh' + - 'echo ${var.admin_password} | sudo -S ${var.file_destination}/linux/prepare-system-kubernetes-redhat.sh' + + testing_inline: + - 'echo \"##vso[task.setvariable variable=toPrint]Print Me Please\"' + + interfaces: + # A VM can have multiple IP addresses and interfaces. In this case we are + # saying there is one public IP per network interface + - network_interface: + # name has vm name appended to it so only put basic type name here + name: nic + primary: true + tags: *tags + + # This only works with certain size VMs + # Accelerated Networking is supported on most general purpose and compute-optimized instance sizes with 2 or more vCPUs. + # These supported series are: D/DSv2 and F/Fs. On instances that support hyperthreading, Accelerated Networking is supported + # on VM instances with 4 or more vCPUs. Supported series are: D/DSv3, E/ESv3, Fsv2, and Ms/Mms. + # https://docs.microsoft.com/en-us/azure/virtual-network/create-vm-accelerated-networking-cli + enable_accelerated_networking: false + + ip_configuration: + name: ip-config + # subnet_id is generated so use terraform variable + private_ip: + output: + enable: true + sensitive: false + address_allocation: dynamic + # address only applies if 'address_allocation' is 'static' + address: 10.0.2.5 + + public_ip: + # If you set to false there must be some host set as a bastian host + enable: true + output: + enable: true + sensitive: false + name: pip-epiphany + address_allocation: static + idle_timeout_in_minutes: 30 + sku: Standard + tags: *tags +{% endif %} + +{% if platform.kafka_vms is defined and platform.kafka_vms > 0 %} + - name: {{ azure.vm_name_prefix }}-kafka + size: Standard_DS2_v2 + os_type: linux + count: {{ platform.kafka_vms }} + # One host will need to be a bastian host UNLESS the bastian host is provided in another way + bastian_host: false + # roles are how you define a grouping of nodes. These values will be used to create an inventory of your cluster + # Must be a member of the 'role' in core + roles: + - linux + - jmx-exporter + - zookeeper + - kafka + - node_exporter + - filebeat + - kafka-exporter + - reboot + delete_os_disk_on_termination: false + delete_data_disks_on_termination: false + # NOTE: This must match the interfaces public_ip options. If one interface has enabled public_ip then this MUST be true else false + public_ips: true + depends_on: + # Put the name of the resource. If a VM then only put the 'name:' value. The count portion will be automatically added + - '${var.master_depends_on}' + tags: *tags + + security: + # Each node will have firewalld (redhat) enabled or not allowing the given ports + firewall: + enable: false + # These ports can be in sync with NSG rules below or not. Meaning, you may have NSG disabled and want each node to block + # Each node could belong to a different grouping with a need for different ports + ports_open: + - 443 + - 22 + - 6443 + - 2379-2380 + - 10250 + - 10251 + - 10252 + - 10255 + + # NOTE: For Kubernetes builds we should *not* use provisioners to bootstrap any data. The point is to build a common + # infrastructure and then collect IPs etc. that are required for K8s automation to build. The ONLY possible + # exception is for automated testing in VSTS (maybe) + bootstrap: + # Can use the global values or override them on a per vm type basis + ssh: *ssh + + provisioners: + file: + enable: false + # NOTE: Pay close attention to the trailing '/' for 'source'. If you leave '/' off of the end of source then the directory tree will be copied to 'destination'. If you add '/' then the contents of that directory will be copied into the destination directory. + source: "../../../../core/src/scripts/kubernetes/linux" + destination: "/home/${var.admin_username}" + + remote_exec: + enable: false + # NOTE: Use Terraform vars if you need to embed variables in the commands. + inline: # Example is below and only here to show + - 'chmod +x ${var.file_destination}/linux/make-executable.sh' + - '${var.file_destination}/linux/make-executable.sh' + - 'echo ${var.admin_password} | sudo -S ${var.file_destination}/linux/prepare-system-kubernetes-redhat.sh' + + testing_inline: + - 'echo \"##vso[task.setvariable variable=toPrint]Print Me Please\"' + + interfaces: + # A VM can have multiple IP addresses and interfaces. In this case we are + # saying there is one public IP per network interface + - network_interface: + # name has vm name appended to it so only put basic type name here + name: nic + primary: true + tags: *tags + + # This only works with certain size VMs + # Accelerated Networking is supported on most general purpose and compute-optimized instance sizes with 2 or more vCPUs. + # These supported series are: D/DSv2 and F/Fs. On instances that support hyperthreading, Accelerated Networking is supported + # on VM instances with 4 or more vCPUs. Supported series are: D/DSv3, E/ESv3, Fsv2, and Ms/Mms. + # https://docs.microsoft.com/en-us/azure/virtual-network/create-vm-accelerated-networking-cli + enable_accelerated_networking: false + + ip_configuration: + name: ip-config + # subnet_id is generated so use terraform variable + private_ip: + output: + enable: true + sensitive: false + address_allocation: dynamic + # address only applies if 'address_allocation' is 'static' + address: 10.0.2.5 + + public_ip: + # If you set to false there must be some host set as a bastian host + enable: true + output: + enable: true + sensitive: false + name: pip-epiphany + address_allocation: static + idle_timeout_in_minutes: 30 + sku: Standard + tags: *tags +{% endif %} + +{% if platform.elk_vms is defined and platform.elk_vms > 0 %} + - name: {{ azure.vm_name_prefix }}-elk + size: Standard_DS1_v2 + os_type: linux + count: {{ platform.elk_vms }} + # One host will need to be a bastian host UNLESS the bastian host is provided in another way + bastian_host: false + # roles are how you define a grouping of nodes. These values will be used to create an inventory of your cluster + # Must be a member of the 'role' in core + roles: + - linux + - node_exporter + - elasticsearch + - elasticsearch-curator + - kibana + - filebeat + + delete_os_disk_on_termination: false + delete_data_disks_on_termination: false + # NOTE: This must match the interfaces public_ip options. If one interface has enabled public_ip then this MUST be true else false + public_ips: true + depends_on: + # Put the name of the resource. If a VM then only put the 'name:' value. The count portion will be automatically added + - '${var.master_depends_on}' + tags: *tags + + security: + # Each node will have firewalld (redhat) enabled or not allowing the given ports + firewall: + enable: false + # These ports can be in sync with NSG rules below or not. Meaning, you may have NSG disabled and want each node to block + # Each node could belong to a different grouping with a need for different ports + ports_open: + - 443 + - 22 + - 6443 + - 2379-2380 + - 10250 + - 10251 + - 10252 + - 10255 + + # NOTE: For Kubernetes builds we should *not* use provisioners to bootstrap any data. The point is to build a common + # infrastructure and then collect IPs etc. that are required for K8s automation to build. The ONLY possible + # exception is for automated testing in VSTS (maybe) + bootstrap: + # Can use the global values or override them on a per vm type basis + ssh: *ssh + + provisioners: + file: + enable: false + # NOTE: Pay close attention to the trailing '/' for 'source'. If you leave '/' off of the end of source then the directory tree will be copied to 'destination'. If you add '/' then the contents of that directory will be copied into the destination directory. + source: "../../../../core/src/scripts/kubernetes/linux" + destination: "/home/${var.admin_username}" + + remote_exec: + enable: false + # NOTE: Use Terraform vars if you need to embed variables in the commands. + inline: # Example is below and only here to show + - 'chmod +x ${var.file_destination}/linux/make-executable.sh' + - '${var.file_destination}/linux/make-executable.sh' + - 'echo ${var.admin_password} | sudo -S ${var.file_destination}/linux/prepare-system-kubernetes-redhat.sh' + + testing_inline: + - 'echo \"##vso[task.setvariable variable=toPrint]Print Me Please\"' + + interfaces: + # A VM can have multiple IP addresses and interfaces. In this case we are + # saying there is one public IP per network interface + - network_interface: + # name has vm name appended to it so only put basic type name here + name: nic + primary: true + tags: *tags + + # This only works with certain size VMs + # Accelerated Networking is supported on most general purpose and compute-optimized instance sizes with 2 or more vCPUs. + # These supported series are: D/DSv2 and F/Fs. On instances that support hyperthreading, Accelerated Networking is supported + # on VM instances with 4 or more vCPUs. Supported series are: D/DSv3, E/ESv3, Fsv2, and Ms/Mms. + # https://docs.microsoft.com/en-us/azure/virtual-network/create-vm-accelerated-networking-cli + enable_accelerated_networking: false + + ip_configuration: + name: ip-config + # subnet_id is generated so use terraform variable + private_ip: + output: + enable: true + sensitive: false + address_allocation: dynamic + # address only applies if 'address_allocation' is 'static' + address: 10.0.2.5 + + public_ip: + # If you set to false there must be some host set as a bastian host + enable: true + output: + enable: true + sensitive: false + name: pip-epiphany + address_allocation: static + idle_timeout_in_minutes: 30 + sku: Standard + tags: *tags +{% endif %} + +{% if platform.postgresql_vms is defined and platform.postgresql_vms > 0 %} + - name: {{ azure.vm_name_prefix }}-postgresql + size: Standard_DS1_v2 + os_type: linux + count: {{ platform.postgresql_vms }} + # One host will need to be a bastian host UNLESS the bastian host is provided in another way + bastian_host: false + # roles are how you define a grouping of nodes. These values will be used to create an inventory of your cluster + # Must be a member of the 'role' in core + roles: + - node_exporter + - filebeat + - postgresql + - linux + + delete_os_disk_on_termination: false + delete_data_disks_on_termination: false + # NOTE: This must match the interfaces public_ip options. If one interface has enabled public_ip then this MUST be true else false + public_ips: true + depends_on: + # Put the name of the resource. If a VM then only put the 'name:' value. The count portion will be automatically added + - '${var.master_depends_on}' + tags: *tags + + security: + # Each node will have firewalld (redhat) enabled and allowing only given ports, remember that Epiphany needs port 22 for bootstrapping environments + firewall: + enable: false + ports_open: + - 22/tcp + - 443/tcp + + # NOTE: For Kubernetes builds we should *not* use provisioners to bootstrap any data. The point is to build a common + # infrastructure and then collect IPs etc. that are required for K8s automation to build. The ONLY possible + # exception is for automated testing in VSTS + bootstrap: + # Can use the global values or override them on a per vm type basis + ssh: *ssh + + provisioners: + file: + enable: false + # NOTE: Pay close attention to the trailing '/' for 'source'. If you leave '/' off of the end of source then the directory tree will be copied to 'destination'. If you add '/' then the contents of that directory will be copied into the destination directory. + source: "../../../../core/src/scripts/kubernetes/linux" + destination: "/home/${var.admin_username}" + + remote_exec: + enable: false + # NOTE: Use Terraform vars if you need to embed variables in the commands. + inline: # Example is below and only here to show + - 'chmod +x ${var.file_destination}/linux/make-executable.sh' + - '${var.file_destination}/linux/make-executable.sh' + - 'echo ${var.admin_password} | sudo -S ${var.file_destination}/linux/prepare-system-kubernetes-redhat.sh' + + testing_inline: + - 'echo \"##vso[task.setvariable variable=toPrint]Print Me Please\"' + + interfaces: + # A VM can have multiple IP addresses and interfaces. In this case we are + # saying there is one public IP per network interface + - network_interface: + # name has vm name appended to it so only put basic type name here + name: nic + primary: true + tags: *tags + + # This only works with certain size VMs + # Accelerated Networking is supported on most general purpose and compute-optimized instance sizes with 2 or more vCPUs. + # These supported series are: D/DSv2 and F/Fs. On instances that support hyperthreading, Accelerated Networking is supported + # on VM instances with 4 or more vCPUs. Supported series are: D/DSv3, E/ESv3, Fsv2, and Ms/Mms. + # https://docs.microsoft.com/en-us/azure/virtual-network/create-vm-accelerated-networking-cli + enable_accelerated_networking: false + + ip_configuration: + name: ip-config + # subnet_id is generated so use terraform variable + private_ip: + output: + enable: true + sensitive: false + address_allocation: Dynamic + # address only applies if 'address_allocation' is 'static' + address: 10.0.2.5 + + public_ip: + # If you set to false there must be some host set as a bastian host + enable: true + output: + enable: true + sensitive: false + name: pip-epiphany + address_allocation: static + idle_timeout_in_minutes: 30 + sku: Standard + tags: *tags +{% endif %} + +{% if platform.rabbitmq_vms is defined and platform.rabbitmq_vms > 0 %} + - name: {{ azure.vm_name_prefix }}-rabbitmq + size: Standard_DS1_v2 + os_type: linux + count: {{ platform.rabbitmq_vms }} + # One host will need to be a bastian host UNLESS the bastian host is provided in another way + bastian_host: false + # roles are how you define a grouping of nodes. These values will be used to create an inventory of your cluster + # Must be a member of the 'role' in core + roles: + - node_exporter + - filebeat + - rabbitmq + - linux + + delete_os_disk_on_termination: false + delete_data_disks_on_termination: false + # NOTE: This must match the interfaces public_ip options. If one interface has enabled public_ip then this MUST be true else false + public_ips: true + depends_on: + # Put the name of the resource. If a VM then only put the 'name:' value. The count portion will be automatically added + - '${var.master_depends_on}' + tags: *tags + + security: + # Each node will have firewalld (redhat) enabled and allowing only given ports, remember that Epiphany needs port 22 for bootstrapping environments + firewall: + enable: false + ports_open: + - 22/tcp + - 443/tcp + + # NOTE: For Kubernetes builds we should *not* use provisioners to bootstrap any data. The point is to build a common + # infrastructure and then collect IPs etc. that are required for K8s automation to build. The ONLY possible + # exception is for automated testing in VSTS + bootstrap: + # Can use the global values or override them on a per vm type basis + ssh: *ssh + + provisioners: + file: + enable: false + # NOTE: Pay close attention to the trailing '/' for 'source'. If you leave '/' off of the end of source then the directory tree will be copied to 'destination'. If you add '/' then the contents of that directory will be copied into the destination directory. + source: "../../../../core/src/scripts/kubernetes/linux" + destination: "/home/${var.admin_username}" + + remote_exec: + enable: false + # NOTE: Use Terraform vars if you need to embed variables in the commands. + inline: # Example is below and only here to show + - 'chmod +x ${var.file_destination}/linux/make-executable.sh' + - '${var.file_destination}/linux/make-executable.sh' + - 'echo ${var.admin_password} | sudo -S ${var.file_destination}/linux/prepare-system-kubernetes-redhat.sh' + + testing_inline: + - 'echo \"##vso[task.setvariable variable=toPrint]Print Me Please\"' + + interfaces: + # A VM can have multiple IP addresses and interfaces. In this case we are + # saying there is one public IP per network interface + - network_interface: + # name has vm name appended to it so only put basic type name here + name: nic + primary: true + tags: *tags + + # This only works with certain size VMs + # Accelerated Networking is supported on most general purpose and compute-optimized instance sizes with 2 or more vCPUs. + # These supported series are: D/DSv2 and F/Fs. On instances that support hyperthreading, Accelerated Networking is supported + # on VM instances with 4 or more vCPUs. Supported series are: D/DSv3, E/ESv3, Fsv2, and Ms/Mms. + # https://docs.microsoft.com/en-us/azure/virtual-network/create-vm-accelerated-networking-cli + enable_accelerated_networking: false + + ip_configuration: + name: ip-config + # subnet_id is generated so use terraform variable + private_ip: + output: + enable: true + sensitive: false + address_allocation: Dynamic + # address only applies if 'address_allocation' is 'static' + address: 10.0.2.5 + + public_ip: + # If you set to false there must be some host set as a bastian host + enable: true + output: + enable: true + sensitive: false + name: pip-epiphany + address_allocation: static + idle_timeout_in_minutes: 30 + sku: Standard + tags: *tags +{% endif %} + + kubernetes: + version: 1.13.1 + # image_registry_secrets: + # - name: regcred + # server_url: your-registry-url + # username: your-registry-username + # password: your-registry-password + # email: your-registry-email + # namespace: your-secret-namespace + storage: + enable: true + #valid chocies: + #azure-file + #WIP + type: azure-file + tags: *tags + quota: 50 + +{% if (config.rabbitmq_deployment is defined and config.rabbitmq_deployment == true) or (config.auth_service_deployment is defined and config.auth_service_deployment == true and platform.postgresql_vms > 0) %} + deployments: +{% endif %} + +{% if config.rabbitmq_deployment is defined and config.rabbitmq_deployment == true %} + - name: rabbitmq + image_path: rabbitmq:3.7.10 + #image_pull_secret_name: regcred # optional + service: + name: rabbitmq-cluster + port: 30672 + management_port: 31672 + replicas: {{ platform.worker_vms }} + namespace: queue + rabbitmq: + #amqp_port: 5672 #optional - default 5672 + plugins: # optional list of RabbitMQ plugins + - rabbitmq_management_agent + - rabbitmq_management + policies: # optional list of RabbitMQ policies + - name: ha-policy2 + pattern: ".*" + definitions: + ha-mode: all + custom_configurations: #optional list of RabbitMQ configurations (new format -> https://www.rabbitmq.com/configure.html) + - name: vm_memory_high_watermark.relative + value: 0.5 + cluster: + #is_clustered: true #redundant in in-Kubernetes installation, it will always be clustered + #cookie: "cookieSetFromDataYaml" #optional - default value will be random generated string +{% endif %} + +{% if config.auth_service_deployment is defined and config.auth_service_deployment == true and platform.postgresql_vms > 0 %} + - name: auth-service # this service require postgresql to be installed in cluster + image_path: jboss/keycloak:4.8.3.Final + #image_pull_secret_name: regcred + service: + name: as-testauthdb + port: 30104 + replicas: {{ platform.worker_vms }} + namespace: namespace-for-auth + admin_user: auth-service-username + admin_password: auth-service-password + database: + name: "auth-database-name" + #port: "5432" # leave it when default + user: "auth-db-user" + password: "auth-db-password" +{% endif %} + + haproxy: + stats: + enable: true + user: operations + password: your-haproxy-stats-pwd + frontend: + - name: https_front + port: 443 + https: yes + backend: + - http_back1 + backend: + - name: http_back1 + server_groups: + - worker + #servers: + # Definition for server to that hosts the application. + #- name: "node1" + # address: "epiphany-vm1.domain.com" + port: 30104 + +{% if platform.monitoring_vms is defined and platform.monitoring_vms > 0 %} + monitoring: + alerts: + enable: {{ config.alerts_for_monitoring }} # required value + handlers: + mail: + enable: False # required value + smtp_from: "alert@test.com" + smtp_host: "smtp-url:smtp-port" + smtp_auth_username: "your-smtp-user@domain.com" + smtp_auth_password: "your-smtp-password" + smtp_require_tls: True + recipients: + - recipient1@domain.com + - recipient2@domain.com + slack: + enable: False # required value + api_url: url-to-slack-workspace-api.slack.com + pagerduty: + enable: False # required value + service_key: your-service-key + rules: + - name: "UpDown" + expression: up == 0 + duration: 1m #1s, 1m, 1h, 1d, 1w, ... + severity: critical + message: "Node is down." + - name: "DiskSpace" + expression: ((node_filesystem_avail_bytes* 100) / node_filesystem_size_bytes) < 20 # 100 - 80 + duration: 1m #1s, 1m, 1h, 1d, 1w, ... + severity: critical + message: "Disk usage is above 80%" + - name: "DiskSpacePrediction" + expression: predict_linear(node_filesystem_free_bytes{job="node"}[1h], 48 * 3600) < 0 + duration: 1m #1s, 1m, 1h, 1d, 1w, ... + severity: warning + message: "Disk will run out of space in less than 48h" + - name: "MemoryUsage" + expression: (sum by (instance) (node_memory_MemTotal_bytes) - sum by (instance)(node_memory_MemFree_bytes + node_memory_Buffers_bytes + node_memory_Cached_bytes) ) / sum by (instance)(node_memory_MemTotal_bytes) * 100 > 80 + duration: 15m #1s, 1m, 1h, 1d, 1w, ... + severity: warning + message: "Server memory has been used in more than 80% during last 15 minutes." + - name: "CpuLoad" + expression: 100 - (avg by (instance) (irate(node_cpu_seconds_total{job="node",mode="idle"}[5m])) * 100) > 80 + duration: 15m #1s, 1m, 1h, 1d, 1w, ... + severity: critical + message: "CPU utilization has exceeded 80% over last 15 minutes." + - name: "KafkaConsumerLag" + expression: sum by(consumergroup) (kafka_consumergroup_lag) > 1000 + duration: 15m #1s, 1m, 1h, 1d, 1w, ... + severity: critical + message: "Kafka consumers are lagging more than 1000 messages over last 15 minutes." +{% endif %} + +{% if platform.rabbitmq_vms is defined and platform.rabbitmq_vms > 1 %} + rabbitmq: + amqp_port: 5672 #optional - default 5672 + plugins: # optional list of RabbitMQ plugins + - rabbitmq_management_agent + - rabbitmq_management + policies: # optional list of RabbitMQ policies + - name: ha-policy2 + pattern: ".*" + definitions: + ha-mode: all + custom_configurations: #optional list of RabbitMQ configurations (new format -> https://www.rabbitmq.com/configure.html) + - name: vm_memory_high_watermark.relative + value: 0.6 + cluster: + is_clustered: true #optional - default false + cookie: "cookieSetFromDataYaml" #optional - default value will be random generated string +{% endif %} + +{% if platform.postgresql_vms is defined and platform.postgresql_vms > 1 %} + postgresql: + replication: + enable: yes + user: postgresql-replica-user + password: postgresql-replica-pass + max_wal_senders: 10 # (optional) - default value 5 + wal_keep_segments: 34 # (optional) - default value 32 +{% endif %} diff --git a/core/data/azure/infrastructure/epiphany-single-machine/data.yaml b/core/data/azure/infrastructure/epiphany-single-machine/data.yaml new file mode 100644 index 0000000000..e28af9e7da --- /dev/null +++ b/core/data/azure/infrastructure/epiphany-single-machine/data.yaml @@ -0,0 +1,602 @@ +--- +########################################################## +# Data file for Epiphany build +# Azure specific +########################################################## + +########################### +title: Epiphany Single Machine Infrastructure... + +kind: datafile +version: 0.2.0 + +# NOTE: Any data values that are empty put "" or the value None will be used in the templates for those attributes. + +core: + # This will apply to a VPN like environment or an air-gapped like environment + bastian: + enable: false + # This host will be set ONLY for environments where a bastian host is supplied to us and NOT part of the cluster build + host: '' + # If the bastian host has a different key + key_path: '' + user: '' + # if key_path is '' + pwd: '' + + build: + # IMPORTANT - will be appended to release name and output folder and part of the template names + version: &version 1.0.0 + # Type of build environment + environment: &env development + # Name of the given release. Version will be appended + release: &release epiphany-single-machine + # If repo_root is true then add that as a prefix for the output + repo_root: true + platform: azure + output: '/build/$PLATFORM/infrastructure/epiphany' + + tags: &tags + - key: environment + value: *env + - key: version + value: *version + - key: release + value: *release + - key: resourceType + value: epiphany-single-machine-build + - key: location + value: westeurope + + # domain is used to create DNS entries or just add FQDN to hosts.yaml file used in automation + domain: + enable: false + # name is the domain name itself such as epiphanyplatform.io. This value will be appended to the host name for FQDN + name: example.com + create_dns: false + + # These will become the role/classification you will use with the automation to group nodes for given tasks + roles: + - master + - worker + - kafka + - zookeeper + - prometheus + - grafana + - node_exporter + - haproxy_tls_termination + - haproxy_exporter + - elasticsearch + - elasticsearch-curator + - kibana + - filebeat + - jmx-exporter + - kafka-exporter + - postgresql + - rabbitmq + - deployments + # !Caution! + # Disable this role if you don't want restart your servers + - reboot + # These last two must always be present + - linux + # - windows + + admin_user: + name: &admin_username operations + # May want to change to 'key' and create 'key_path' with 'pwd' or 'home' + key_path: ~/.ssh/epiphany-operations/id_rsa + + azure: + tags: + <<: *tags + + terraform: + # This version info is what version is being used at the moment. The version of Terraform in the manifest.yaml in the + # root of the repo is for the initial install and the minum version + version: 1.6 + service_principal: + # Three files are required for SPs to work, az_ad_sp.json, env.sh and security.yaml. By default, these are created if the + # 'create' attribute is true. If false then you will need to supply those two files. This allows you to create + # a service_principal of your own instead of having one generated. + # You will also need to override env.sh that contains the 'ARM_...' environment variables required. + enable: true + create: true # If you want to use an existing one then set this to false + auth: pwd # Valid is 'pwd' and 'cert'. At this time Terraform only support 'pwd' for service principals (sp true) + # NOTE: Backend is a Terraform resource that stores the *.tfstate files used by Terraform to store state. The default + # is to store the state file locally but this can cause issues if working in a team environment. + backend: + # Only used by Terraform + # The backend storage account is '''backend' (combined name with suffix) + # The storage container is generated as ''-'terraform' + # NOTE: Known issue with backend tf when having different VM types below when enabled! So, only one VM entry with count set should be used. Set to false for now... + enable: false + storage_account: + storage_container: + # sleep: 15 # Number of seconds to sleep so that Azure can create the required containers before moving on + type: blob + tags: *tags + + # NOTE: May want to create region/AZ sub-folders to support creating a cluster in different regions and AZ for HA... + resource_group: &resource_group + name: &name epiphany-single + location: &location West Europe + + # Subscription name + subscription: PGGA-Epiphany-Dev + + # Azure Active Directory + ad: + name: *name + role: Contributor + + standard: + # One resource group is supported + resource_group: + <<: *resource_group + # exists: false - TO BE REMOVED + prevent_destory: true + # IMPORTANT - Do not set lock if you plan on editing anything in the resource group! Leave it set to false until you are ready + # ReadOnly or CanNotDelete are the only two level options if enabled! + lock: + # NOTE: This will cause locks.tf.wait to be generated. You will need a script to rename this to locks.tf and run apply again + enable: true + name: epiphany-lock + level: ReadOnly + notes: This locks the entire resource group! + tags: *tags + + # This aids in boot diagnostics if there are issues: + # https://docs.microsoft.com/en-us/azure/virtual-machines/linux/boot-diagnostics + debug: + # WIP + enable: false + storage_account: + name: epiapps + account: + tier: Standard + replication_type: LRS + # Storage, StorageV2 and BlobStorage are supported types + kind: StorageV2 + storage_container: + name: debug + + availability: + availability_set: + enable: true + name: ha-epiphany + platform_fault_domain_count: 3 + platform_update_domain_count: 5 + managed: true # Best to keep this 'true' in most cases. Mixing availability set managed with vm disk unmanaged is not allowed + tags: *tags + + security: + ssh: &ssh + key: + # Public portion of the key + file: ~/.ssh/epiphany-operations/id_rsa.pub + data: + + vpn: + # Make SURE all of the data is correct below before enabling 'vpn'. It can also take 30 minutes to an hour to create. + enable: false + name: vpn-epi-bld-apps + # Only support RouteBased type of connection currently + type: RouteBased + active_active: false + # There are two types of 'sku' (Basic and Standard). Always use Standard so any client can use. + sku: Standard + # Address space that is required by Virtual Network Gateway. This address space must be inside of virtual_network.address_space and do not overlap with other subnets defined + gateway_subnet_space: 10.1.2.0/24 + client_configuration: + # This section is very important! + # You must specify the address_space that will be allocated to use on the VPN side of the conection that will + # be able to talk to your cluster. + address_space: + - 172.16.1.0/24 + root_certificate: + # name is the name of the cert that was created for you by a trusted party OR a name you give a self-signed cert + name: EpiphanyRootCa + revoked_certificate: + name: SomeRevokedCert + thumbprint: bd0ef7d2c9XDFDFDFE9752169894d2 + # public_cert_data is the actual base64 public key from your cert. Put it in 'as is'. The '|' tells yaml to use 'as is'. + public_cert_data: | + YOUR-BASE64-CLIENT-AUTH-PUBLIC-KEY + + ip_configuration: + name: vpn-ip-config + public_ip: + output: + enable: true + sensitive: false + name: pip-vpn-epiphany + address_allocation: Dynamic + idle_timeout_in_minutes: 30 + + tags: *tags + + # subnet_id is generated so use terraform variable + private_ip: + output: + enable: true + sensitive: false + address_allocation: Dynamic + # address only applies if 'address_allocation' is 'static' + address: 10.0.2.5 + + network_security_group: + enable: true + # Note: Could create an array of NSGs and reference them name + '_' + # (i.e., epiphany_nsg_001) maybe at somepoint if needed + name: security-nsg-epiphany + tags: *tags + + rules: + - name: ssh + description: Allow SSH + priority: 102 + direction: Inbound + access: Allow + protocol: Tcp + source_port_range: "*" + destination_port_range: "22" + source_address_prefix: "*" + destination_address_prefix: "*" + + virtual_network: + name: epiphany-single-vnet + address_space: + - 10.1.0.0/22 + + subnet: + name: apps-subnet + address_prefix: 10.1.1.0/24 + # Service endpoints bypass normal public route to Azure SQL, Storage and CosmosDB services + service_endpoints: + - Microsoft.Storage + - Microsoft.Sql + - Microsoft.AzureCosmosDB + + # NOTE: Managed vs Unmanaged storage + # If you want managed storage then by default, 'storage_account' and 'storage_container' options are not required BUT + # they are still enabled in the 'main.tf.j2' template. This could be set with an enable option. + + storage_managed_disk: + # WIP + enable: false + name: epiphany-mdisk + storage_account_type: Premium_LRS + create_option: Empty + disk_size_gb: 500 + count: 1 + + # Once storage account is supported + # Should use (maybe) a different storage account for different locations. Meaning, if you have two clusters (one in east and one in west) then having a storage account for east and one for west would be good since you want storage near compute. + # No `-` in storage account name + + # 3-24 Alphanumeric only lower case + storage_account: + enable: true + name: episingle + account: + tier: Standard + replication_type: LRS + # Storage, StorageV2 and BlobStorage are supported types + kind: StorageV2 + + tags: *tags + + storage_container: + enable: false + # 3-63 Alphanumeric lower case plus '-' + name: epiphany-osdisks + access_type: private + + storage_blob: + enable: false + name: epiphany-osdisks + type: page + # size in bytes in 512 increments + size: 5120 + count: 2 + + storage_image_reference: &storage_image_reference + publisher: Canonical + offer: UbuntuServer + sku: 18.04-LTS + # Never put latest on anything! Need to always pin the version number but testing we can get away with it + version: "18.04.201810030" + + #publisher: RedHat + #offer: RHEL + #sku: 7-RAW + # Never put latest on anything! Need to always pin the version number but testing we can get away with it + #version: "7.6.2018103108" + + storage_os_disk: &storage_os_disk + managed: true + caching: ReadWrite + create_option: FromImage + disk_size_gb: 100 + managed_disk_type: Premium_LRS + + storage_data_disk: &storage_data_disk + # Determines if a data disk will be added. If also using managed disks then use 'Attach' for create_option instead of 'Empty' + enable: false + count: 2 + managed: false + caching: ReadWrite + create_option: Empty + disk_size_gb: 30 + managed_disk_type: Standard_LRS # Can only be Standard_LRS if using non-managed availability_set + + os_profile: &os_profile + admin_username: operations + admin_password: your-secret-password + + os_profile_linux_config: &os_profile_linux_config + # NOTE: Must set to true except in early stage development! This will enforce ssh key authentication for now... + disable_password_authentication: true + + os_profile_windows_config: &os_profile_windows_config + # NOTE: Must set to true except in early stage development! This will enforce ssh key authentication for now... + disable_password_authentication: true + + # Testing sets flags that allow builds to inject commands into VMs and other related items + testing: + enable: false + + # NOTE: This MUST equal the total number of 'count' metadata from each 'vm' array entry below + vm_count: 1 + + # VM names - 1-15 characters for Windows and 1-64 characters for Linux + vms: + - name: vm-master + size: Standard_DS4_v2 + os_type: linux + count: 1 + # One host will need to be a bastian host UNLESS the bastian host is provided in another way + bastian_host: false + # roles are how you define a grouping of nodes. These values will be used to create an inventory of your cluster + # Must be a member of the 'role' in core + roles: + - linux + - master + - node_exporter + - prometheus + - grafana + - rabbitmq + - postgresql + - deployments + - reboot + + delete_os_disk_on_termination: false + delete_data_disks_on_termination: false + # NOTE: This must match the interfaces public_ip options. If one interface has enabled public_ip then this MUST be true else false + public_ips: true + depends_on: + # Put the name of the resource. If a VM then only put the 'name:' value. The count portion will be automatically added + - '${var.master_depends_on}' + tags: *tags + + security: + # Each node will have firewalld (redhat) enabled and allowing only given ports, remember that Epiphany needs port 22 for bootstrapping environments + firewall: + enable: false + ports_open: + - 22/tcp + - 443/tcp + + # NOTE: For Kubernetes builds we should *not* use provisioners to bootstrap any data. The point is to build a common + # infrastructure and then collect IPs etc. that are required for K8s automation to build. The ONLY possible + # exception is for automated testing in VSTS + bootstrap: + # Can use the global values or override them on a per vm type basis + ssh: *ssh + + provisioners: + file: + enable: false + # NOTE: Pay close attention to the trailing '/' for 'source'. If you leave '/' off of the end of source then the directory tree will be copied to 'destination'. If you add '/' then the contents of that directory will be copied into the destination directory. + source: "../../../../core/src/scripts/kubernetes/linux" + destination: "/home/${var.admin_username}" + + remote_exec: + enable: false + # NOTE: Use Terraform vars if you need to embed variables in the commands. + inline: # Example is below and only here to show + - 'chmod +x ${var.file_destination}/linux/make-executable.sh' + - '${var.file_destination}/linux/make-executable.sh' + - 'echo ${var.admin_password} | sudo -S ${var.file_destination}/linux/prepare-system-kubernetes-redhat.sh' + + testing_inline: + - 'echo \"##vso[task.setvariable variable=toPrint]Print Me Please\"' + + interfaces: + # A VM can have multiple IP addresses and interfaces. In this case we are + # saying there is one public IP per network interface + - network_interface: + # name has vm name appended to it so only put basic type name here + name: nic + primary: true + tags: *tags + + # This only works with certain size VMs + # Accelerated Networking is supported on most general purpose and compute-optimized instance sizes with 2 or more vCPUs. + # These supported series are: D/DSv2 and F/Fs. On instances that support hyperthreading, Accelerated Networking is supported + # on VM instances with 4 or more vCPUs. Supported series are: D/DSv3, E/ESv3, Fsv2, and Ms/Mms. + # https://docs.microsoft.com/en-us/azure/virtual-network/create-vm-accelerated-networking-cli + enable_accelerated_networking: false + + ip_configuration: + name: ip-config + # subnet_id is generated so use terraform variable + private_ip: + output: + enable: true + sensitive: false + address_allocation: Dynamic + # address only applies if 'address_allocation' is 'static' + address: 10.0.2.5 + + public_ip: + # If you set to false there must be some host set as a bastian host + enable: true + output: + enable: true + sensitive: false + name: pip-epiphany + address_allocation: static + idle_timeout_in_minutes: 30 + sku: Standard + + tags: *tags + + kubernetes: + version: 1.13.0 + # image_registry_secrets: + # - name: regcred + # server_url: your-registry-url + # username: your-registry-username + # password: your-registry-password + # email: your-registry-email + # namespace: your-secret-namespace + storage: + enable: true + #valid chocies: + #azure-file + #WIP + type: azure-file + tags: *tags + quota: 50 + deployments: +# - name: rabbitmq +# image_path: rabbitmq:3.7.10 +# # image_pull_secret_name: regcred # optional +# service: +# name: rabbitmq-cluster +# port: 30672 +# management_port: 31672 +# replicas: 5 +# namespace: queue +# rabbitmq: +# # amqp_port: 5672 #optional - default 5672 +# plugins: # optional list of RabbitMQ plugins +# - rabbitmq_management +# - rabbitmq_management_agent +# policies: # optional list of RabbitMQ policies +# - name: ha-policy2 +# pattern: ".*" +# definitions: +# ha-mode: all +# custom_configurations: #optional list of RabbitMQ configurations (new format -> https://www.rabbitmq.com/configure.html) +# - name: vm_memory_high_watermark.relative +# value: 0.5 +# cluster: +# # is_clustered: true #redundant in in-Kubernetes installation, it will always be clustered +# # cookie: "cookieSetFromDataYaml" #optional - default value will be random generated string + - name: auth-service # this service require postgresql to be installed in cluster + image_path: jboss/keycloak:4.8.3.Final + # image_pull_secret_name: regcred + service: + name: auth-service + port: 30104 + replicas: 2 + namespace: auth-tools + admin_user: admin + admin_password: admin + database: + name: "authDb" + # port: "5432" # leave it when default + user: "authDbadmin" + password: "authDbpwd" + +# haproxy: +# stats: +# enable: true +# user: operations +# password: your-haproxy-stats-pwd +# frontend: +# - name: https_front +# port: 443 +# https: yes +# backend: +# - http_back1 +# backend: +# - name: http_back1 +# server_groups: +# - worker +# #servers: +# # Definition for server to that hosts the application. +# #- name: "node1" +# # address: "epiphany-vm1.domain.com" +# port: 30104 + + monitoring: + alerts: + enable: True # required value + handlers: + mail: + enable: False # required value + smtp_from: "alert@test.com" + smtp_host: "smtp-url:smtp-port" + smtp_auth_username: "your-smtp-user@domain.com" + smtp_auth_password: "your-smtp-password" + smtp_require_tls: True + recipients: + - recipient1@domain.com + - recipient2@domain.com + slack: + enable: False # required value + api_url: url-to-slack-workspace-api.slack.com + pagerduty: + enable: False # required value + service_key: your-service-key + rules: + - name: "UpDown" + expression: up == 0 + duration: 1m #1s, 1m, 1h, 1d, 1w, ... + severity: critical + message: "Node is down." + - name: "DiskSpace" + expression: ((node_filesystem_avail_bytes* 100) / node_filesystem_size_bytes) < 20 # 100 - 80 + duration: 1m #1s, 1m, 1h, 1d, 1w, ... + severity: critical + message: "Disk usage is above 80%" + - name: "DiskSpacePrediction" + expression: predict_linear(node_filesystem_free_bytes{job="node"}[1h], 48 * 3600) < 0 + duration: 1m #1s, 1m, 1h, 1d, 1w, ... + severity: warning + message: "Disk will run out of space in less than 48h" + - name: "MemoryUsage" + expression: (sum by (instance) (node_memory_MemTotal_bytes) - sum by (instance)(node_memory_MemFree_bytes + node_memory_Buffers_bytes + node_memory_Cached_bytes) ) / sum by (instance)(node_memory_MemTotal_bytes) * 100 > 80 + duration: 15m #1s, 1m, 1h, 1d, 1w, ... + severity: warning + message: "Server memory has been used in more than 80% during last 15 minutes." + - name: "CpuLoad" + expression: 100 - (avg by (instance) (irate(node_cpu_seconds_total{job="node",mode="idle"}[5m])) * 100) > 80 + duration: 15m #1s, 1m, 1h, 1d, 1w, ... + severity: critical + message: "CPU utilization has exceeded 80% over last 15 minutes." + - name: "KafkaConsumerLag" + expression: sum by(consumergroup) (kafka_consumergroup_lag) > 1000 + duration: 15m #1s, 1m, 1h, 1d, 1w, ... + severity: critical + message: "Kafka consumers are lagging more than 1000 messages over last 15 minutes." + +rabbitmq: + amqp_port: 5672 #optional - default 5672 + plugins: # optional list of RabbitMQ plugins + - rabbitmq_management + - rabbitmq_management_agent + policies: # optional list of RabbitMQ policies + - name: ha-policy2 + pattern: ".*" + definitions: + ha-mode: all +# custom_configurations: #optional list of RabbitMQ configurations (new format -> https://www.rabbitmq.com/configure.html) +# - name: vm_memory_high_watermark.relative +# value: 0.6 + cluster: + is_clustered: false #optional - default false + cookie: "cookieSetFromDataYaml" #optional - default value will be random generated string \ No newline at end of file diff --git a/core/data/metal/single-machine/data.yaml b/core/data/metal/epiphany-single-machine/data.yaml similarity index 71% rename from core/data/metal/single-machine/data.yaml rename to core/data/metal/epiphany-single-machine/data.yaml index c55a044b2c..5095967493 100644 --- a/core/data/metal/single-machine/data.yaml +++ b/core/data/metal/epiphany-single-machine/data.yaml @@ -4,7 +4,7 @@ kind: datafile -version: 1.0.1 +version: 0.2.0 # This will apply to a VPN like environment or an air-gapped like environment bastian: @@ -86,6 +86,8 @@ nodes: - prometheus - grafana - rabbitmq + - postgresql + - deployments - reboot security: @@ -114,50 +116,65 @@ kubernetes: #allow_pods_on_master: false # Use this to force untainted master on multi-machine cluster. storage: enable: False - #deployments: - # - name: rabbitmq - # image_path: rabbitmq:3.7.10 - # #image_pull_secret_name: regcred # optional - # service: - # name: rabbitmq-cluster - # port: 30672 - # replicas: 5 - # namespace: queue - # rabbitmq: - # #amqp_port: 5672 #optional - default 5672 - # plugins: # optional list of RabbitMQ plugins - # - rabbitmq_management - # - rabbitmq_management_agent - # policies: # optional list of RabbitMQ policies - # - name: ha-policy2 - # pattern: ".*" - # definitions: - # ha-mode: all - # custom_configurations: #optional list of RabbitMQ configurations (new format -> https://www.rabbitmq.com/configure.html) - # - name: vm_memory_high_watermark.relative - # value: 0.5 - # cluster: - # #is_clustered: true #redundant in in-Kubernetes installation, it will always be clustered - # #cookie: "cookieSetFromDataYaml" #optional - default value will be random generated string - # - name: auth-service # this service require postgresql to be installed in cluster - # image_path: jboss/keycloak:4.8.3.Final - # #image_pull_secret_name: regcred - # service: - # name: as-testauthdb - # port: 30104 - # replicas: 2 - # namespace: namespace-for-auth - # admin_user: auth-service-username - # admin_password: auth-service-password - # database: - # name: "auth-database-name" - # #port: "5432" # leave it when default - # user: "auth-db-user" - # password: "auth-db-password" - +#deployments: +# - name: rabbitmq +# image_path: rabbitmq:3.7.10 +# #image_pull_secret_name: regcred # optional +# service: +# name: rabbitmq-cluster +# port: 30672 +# replicas: 5 +# namespace: queue +# rabbitmq: +# #amqp_port: 5672 #optional - default 5672 +# plugins: # optional list of RabbitMQ plugins +# - rabbitmq_management +# - rabbitmq_management_agent +# policies: # optional list of RabbitMQ policies +# - name: ha-policy2 +# pattern: ".*" +# definitions: +# ha-mode: all +# custom_configurations: #optional list of RabbitMQ configurations (new format -> https://www.rabbitmq.com/configure.html) +# - name: vm_memory_high_watermark.relative +# value: 0.5 +# cluster: +# #is_clustered: true #redundant in in-Kubernetes installation, it will always be clustered +# #cookie: "cookieSetFromDataYaml" #optional - default value will be random generated string +# - name: auth-service # this service require postgresql to be installed in cluster +# image_path: jboss/keycloak:4.8.3.Final +# #image_pull_secret_name: regcred +# service: +# name: as-testauthdb +# port: 30104 +# management_port: 31672 +# replicas: 2 +# namespace: namespace-for-auth +# admin_user: auth-service-username +# admin_password: auth-service-password +# database: +# name: "auth-database-name" +# #port: "5432" # leave it when default +# user: "auth-db-user" +# password: "auth-db-password" +# - name: auth-service # this service require postgresql to be installed in cluster +# image_path: jboss/keycloak:4.8.3.Final +# # image_pull_secret_name: regcred +# service: +# name: auth-service +# port: 30104 +# replicas: 2 +# namespace: auth-tools +# admin_user: admin +# admin_password: admin +# database: +# name: "authDb" +# # port: "5432" # leave it when default +# user: "authDbadmin" +# password: "authDbpwd" monitoring: alerts: - enable: False # required value + enable: True # required value handlers: mail: enable: False # required value @@ -217,9 +234,9 @@ rabbitmq: pattern: ".*" definitions: ha-mode: all - custom_configurations: #optional list of RabbitMQ configurations (new format -> https://www.rabbitmq.com/configure.html) - - name: vm_memory_high_watermark.relative - value: 0.6 +# custom_configurations: #optional list of RabbitMQ configurations (new format -> https://www.rabbitmq.com/configure.html) +# - name: vm_memory_high_watermark.relative +# value: 0.6 cluster: is_clustered: false #optional - default false cookie: "cookieSetFromDataYaml" #optional - default value will be random generated string diff --git a/core/src/core/template_engine_3 b/core/src/core/template_engine_3 new file mode 100755 index 0000000000..e03b8f5322 --- /dev/null +++ b/core/src/core/template_engine_3 @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +# +# Author: Hans Chris Jones +# Copyright 2018, LambdaStack +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# NOTE: You need to pass in the full paths to the file references below. The data is file should be private and it's +# not in the project. The reason is you should create a YAML file that fits how you want to configure your +# environment. For eample, you can have a small YAML data file for configuring the kickstart/ISO process and then +# maybe one for building out the missing USER and/or SYSTEM data used in the CHEF Environment files. A sample +# environment file ships with the project for vagrant called vagrant.json. However, a production.json should +# really be a jinja2 template like base_environment.json.j2 with as much default data and with template {{ }} placeholders +# for the actual data. The output of this process should be the TRUE production.json file. Also, it's a good idea +# to name your production.json file more descriptive of the environment it actually belongs to. For example, +# prod-dc101.json or something like it. + +import argparse +import json +import sys +import traceback +import logging + +import yaml +from jinja2 import Environment, FileSystemLoader + +_SEPARATOR = '=' * 60 + + +# All three file paths must be full paths to each. +def render_template(data_file, in_file, out_file, json_arg, yaml_arg): + template_dict = {} + + logging.basicConfig(level=logging.INFO) + log = logging.getLogger(__name__) + + # If the -j flag was passed then convert the yaml to pretty json in sorted order + if json_arg: + with open(data_file) as data: + template_dict = yaml.load(data) + log.info(json.dumps(template_dict, indent=4, sort_keys=True)) + sys.exit(0) + + if yaml_arg: + with open(data_file) as data: + template_dict = json.load(data) + log.info(yaml.safe_dump(template_dict, indent=2, allow_unicode=True, default_flow_style=False)) + sys.exit(0) + + # Start the template processing + try: + # env = Environment(autoescape=False, loader=FileSystemLoader('/')), trim_blocks=True) + env = Environment(loader=FileSystemLoader('/')) + env.filters['jsonify'] = json.dumps + + with open(data_file) as data: + template_dict = yaml.load(data) + + # Render template and print generated config to console + template = env.get_template(in_file) + + with open(out_file, 'w') as f: + output = template.render(template_dict) + f.write(output) + + except Exception as e: + # Print out error, traceback and debug info... + log.error("Template Engine stopped due to the following error ===> ", e) + log.error(_SEPARATOR) + log.error('Debugging Output:') + log.error(traceback) + log.error(_SEPARATOR) + log.error('Data dictionary:') + log.error(json.dumps(template_dict, indent=4, sort_keys=True)) + log.error(_SEPARATOR) + log.error("Template Engine stopped due to the following error ===> ", e) + log.error('Scan up to see traceback and JSON data (traceback at both top and bottom of this output)') + log.error(_SEPARATOR) + log.error('Debugging Output:') + log.error(traceback) + log.error(_SEPARATOR) + sys.exit(1) + + +# Used to pass a string instead of input file as a template +# dict is json dictionary of the values to sub +def render_string(in_string, template_dict): + return Environment().from_string(in_string).render(template_dict) + + +# Standard way of calling... +# ./template_engine_3 -i