diff --git a/Gemfile.lock b/Gemfile.lock index 0bb2a39..ae65b93 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - et_full_system (1.0.0.pre49) + et_full_system (1.0.0) aws-sdk-s3 (~> 1.9) azure-storage (~> 0.15.0.preview) dotenv (~> 2.7, >= 2.7.2) @@ -23,8 +23,8 @@ GEM addressable (2.7.0) public_suffix (>= 2.0.2, < 5.0) aws-eventstream (1.0.3) - aws-partitions (1.262.0) - aws-sdk-core (3.86.0) + aws-partitions (1.266.0) + aws-sdk-core (3.89.1) aws-eventstream (~> 1.0, >= 1.0.2) aws-partitions (~> 1, >= 1.239.0) aws-sigv4 (~> 1.1) @@ -67,12 +67,12 @@ GEM tilt (~> 2.0, >= 2.0.9) faraday (0.17.3) multipart-post (>= 1.2, < 3) - faraday_middleware (0.13.1) + faraday_middleware (0.14.0) faraday (>= 0.7.4, < 1.0) httparty (0.17.3) mime-types (~> 3.0) multi_xml (>= 0.5.2) - i18n (1.8.0) + i18n (1.8.2) concurrent-ruby (~> 1.0) iodine (0.7.38) jmespath (1.4.0) @@ -82,7 +82,7 @@ GEM mime-types-data (~> 3.2015) mime-types-data (3.2019.1009) mini_portile2 (2.4.0) - minitest (5.13.0) + minitest (5.14.0) multi_json (1.14.1) multi_xml (0.6.0) multipart-post (2.1.1) @@ -92,11 +92,11 @@ GEM mini_portile2 (~> 2.4.0) public_suffix (4.0.3) puma (3.12.2) - rack (2.0.8) + rack (2.1.1) rack-protection (2.0.8.1) rack rake (10.5.0) - roda (3.27.0) + roda (3.28.0) rack rotp (5.1.0) addressable (~> 2.5) diff --git a/README.md b/README.md index a7508d1..e168bfc 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ so that all services can be accessed using a standard subdomain based url system * et1.et.127.0.0.1.nip.io:3100 For ET1 * et1.et.127.0.0.1.nip.io:3100 For ET1 -The services are run all together using 'forego' (like foreman - runs a Procfile) and once the port number for each +The services are run all together using 'invoker' (like foreman - runs a Procfile) and once the port number for each service is known, traefik is told about it - so it all just works. There are other services as well that are run - which are there for testing - these are :- @@ -52,6 +52,7 @@ To run without docker, you need a few tools * Minio (https://github.com/minio/minio) * Azurite (https://github.com/Azure/Azurite) * godotenv (https://github.com/joho/godotenv) +* freeport (https://github.com/phayes/freeport) I will not include installation instructions here for them - please visit their web sites and install them on your platform. For OSX users check out the homebrew repository - there are formulas for some of these. diff --git a/docker/Dockerfile b/docker/Dockerfile index 3f4adc0..1d832df 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,7 +1,7 @@ FROM phusion/passenger-customizable:0.9.34 # Set correct environment variables. ENV HOME /root - +ARG GEM_VERSION=>0 # Use baseimage-docker's init process. CMD ["/sbin/my_init"] @@ -68,9 +68,8 @@ RUN /pd_build/nodejs.sh # Enable the Redis service. RUN rm -f /etc/service/redis/down -# Install forego (for Procfile running) -RUN bash -lc "wget https://bin.equinox.io/c/ekMN3bCZFUn/forego-stable-linux-amd64.deb && apt install ./forego-stable-linux-amd64.deb && rm ./forego-stable-linux-amd64.deb" - +# Install freeport +RUN wget https://github.com/phayes/freeport/releases/download/1.0.2/freeport_1.0.2_linux_amd64.deb && dpkg -i freeport_1.0.2_linux_amd64.deb && rm freeport_1.0.2_linux_amd64.deb # Install traefik (For front end reverse proxy) RUN bash -lc "wget https://github.com/containous/traefik/releases/download/v1.7.10/traefik_linux-amd64 && mv ./traefik_linux-amd64 /usr/bin/traefik && chmod +x /usr/bin/traefik" @@ -106,10 +105,10 @@ RUN echo "app ALL=(root) NOPASSWD:ALL" > /etc/sudoers.d/user && chmod 0440 /etc/ USER app RUN bash --login -c "rvm --default use 2.5.1" -RUN bash --login -c "gem install bundler" +RUN bash --login -c "gem install bundler --version 1.17.3" # Install gems required outside of any bundle. -RUN bash --login -c "rvm use && gem install dotenv et_full_system et_fake_acas_server" +RUN bash --login -c "rvm use && gem install dotenv et_fake_acas_server invoker && gem install et_full_system --version=$GEM_VERSION" WORKDIR /home/app/full_system diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index ac9175a..1780f6d 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -5,6 +5,8 @@ services: build: context: ../ dockerfile: docker/Dockerfile + args: + GEM_VERSION: ports: - '${SERVER_PORT:-3100}:${SERVER_PORT:-3100}' diff --git a/et_full_system.gemspec b/et_full_system.gemspec index f131a44..1234afe 100644 --- a/et_full_system.gemspec +++ b/et_full_system.gemspec @@ -14,15 +14,6 @@ Gem::Specification.new do |spec| spec.homepage = "https://github.com/ministryofjustice/et-full-system" spec.license = "MIT" - # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host' - # to allow pushing to a single host or delete this section to allow pushing to any host. - if spec.respond_to?(:metadata) - spec.metadata["allowed_push_host"] = "http://mygemserver.com" - else - raise "RubyGems 2.0 or newer is required to protect against " \ - "public gem pushes." - end - # Specify which files should be added to the gem when it is released. # The `git ls-files -z` loads the files in the RubyGem that have been added into git. spec.files = Dir.chdir(File.expand_path('..', __FILE__)) do diff --git a/foreman/.env b/foreman/.env index 3e2482b..862b450 100644 --- a/foreman/.env +++ b/foreman/.env @@ -1,6 +1,8 @@ DB_HOST=${DB_HOST} +DB_PORT=5432 DB_USERNAME=postgres REDIS_HOST=${REDIS_HOST} +REDIS_PORT=6379 RAILS_LOG_TO_STDOUT='true' WEB_CONCURRENCY=1 SECRET_KEY_BASE=sdlkjewfkjhfviuhduihenrjk435r89esfd7cv983qh2n4r3q27yh4u5rtfg diff --git a/foreman/Procfile b/foreman/Procfile index 894537e..c8a5893 100644 --- a/foreman/Procfile +++ b/foreman/Procfile @@ -1,14 +1,14 @@ -traefik: bash --login -c "traefik --file.filename=${FOREMAN_PATH}/traefik.toml" -et1_web: bash --login -c "cd ${FS_ROOT_PATH} && et_full_system local wait_for_support && et_full_system local update_service_url et1 http://localhost:${PORT} && cd systems/et1 && godotenv -f ${FOREMAN_PATH}/.env godotenv -f ${FOREMAN_PATH}/et1.env ./run.sh" +traefik: bash --login -c "traefik --configfile=${FOREMAN_PATH}/traefik.toml" +et1_web: export FREEPORT=$(freeport) && godotenv -f ${FOREMAN_PATH}/.env godotenv -f ${FOREMAN_PATH}/et1.env bash --login -c "cd ${FS_ROOT_PATH} && et_full_system local wait_for_support && et_full_system local update_service_url et1 http://localhost:$FREEPORT && cd systems/et1 && PORT=$FREEPORT ./run.sh" et1_sidekiq: bash --login -c "cd ${FS_ROOT_PATH} && et_full_system local wait_for_support && cd systems/et1 && godotenv -f ${FOREMAN_PATH}/.env godotenv -f ${FOREMAN_PATH}/et1.env ./run_sidekiq.sh" -et3_web: bash --login -c "cd ${FS_ROOT_PATH} && et_full_system local wait_for_support && et_full_system local update_service_url et3 http://localhost:${PORT} && cd systems/et3 && godotenv -f ${FOREMAN_PATH}/.env godotenv -f ${FOREMAN_PATH}/et3.env ./run.sh" -mail_web: bash --login -c "source ./mailhog.env && mailhog &> /dev/null" -api_web: bash --login -c "cd ${FS_ROOT_PATH} && et_full_system local wait_for_support && et_full_system local update_service_url api http://localhost:${PORT} && cd systems/api && godotenv -f ${FOREMAN_PATH}/.env godotenv -f ${FOREMAN_PATH}/et_api.env ./run.sh" +et3_web: export FREEPORT=$(freeport) && godotenv -f ${FOREMAN_PATH}/.env godotenv -f ${FOREMAN_PATH}/et3.env bash --login -c "cd ${FS_ROOT_PATH} && et_full_system local wait_for_support && et_full_system local update_service_url et3 http://localhost:$FREEPORT && cd systems/et3 && PORT=$FREEPORT ./run.sh" +mail_web: bash --login -c "source ${FOREMAN_PATH}/mailhog.env && mailhog &> /dev/null" +api_web: export FREEPORT=$(freeport) && godotenv -f ${FOREMAN_PATH}/.env godotenv -f ${FOREMAN_PATH}/et_api.env bash --login -c "cd ${FS_ROOT_PATH} && et_full_system local wait_for_support && et_full_system local update_service_url api http://localhost:$FREEPORT && cd systems/api && PORT=$FREEPORT ./run.sh" api_sidekiq: bash --login -c "cd ${FS_ROOT_PATH} && et_full_system local wait_for_support && cd systems/api && godotenv -f ${FOREMAN_PATH}/.env godotenv -f ${FOREMAN_PATH}/et_api.env ./run_sidekiq.sh" -admin_web: bash --login -c "cd ${FS_ROOT_PATH} && et_full_system local wait_for_support && et_full_system local update_service_url admin http://localhost:${PORT} && cd systems/admin && godotenv -f ${FOREMAN_PATH}/.env godotenv -f ${FOREMAN_PATH}/et_admin.env ./run.sh" -atos_api_web: bash --login -c "cd ${FS_ROOT_PATH} && et_full_system local wait_for_support && et_full_system local update_service_url atos_api http://localhost:${PORT} && cd systems/atos && godotenv -f ${FOREMAN_PATH}/.env godotenv -f ${FOREMAN_PATH}/et_atos.env ./run.sh" +admin_web: export FREEPORT=$(freeport) && godotenv -f ${FOREMAN_PATH}/.env godotenv -f ${FOREMAN_PATH}/et_admin.env bash --login -c "cd ${FS_ROOT_PATH} && et_full_system local wait_for_support && et_full_system local update_service_url admin http://localhost:$FREEPORT && cd systems/admin && PORT=$FREEPORT ./run.sh" +atos_api_web: export FREEPORT=$(freeport) && godotenv -f ${FOREMAN_PATH}/.env godotenv -f ${FOREMAN_PATH}/et_atos.env bash --login -c "cd ${FS_ROOT_PATH} && et_full_system local wait_for_support && et_full_system local update_service_url atos_api http://localhost:$FREEPORT && cd systems/atos && PORT=$FREEPORT ./run.sh" s3_web: bash --login -c "godotenv -f ${FOREMAN_PATH}/minio.env minio server ${MINIO_STORAGE_PATH}" azure_blob_web: bash --login -c "azurite -l ${AZURITE_STORAGE_PATH} -d ${AZURITE_STORAGE_PATH}/debug.log" -fake_acas_web: bash --login -c "cd ${FS_ROOT_PATH} && et_full_system local wait_for_support && et_full_system local update_service_url fake_acas http://localhost:${PORT} && rvm use && et_fake_acas_server --port=$PORT --threads=1:5" -fake_ccd_web: bash --login -c "cd ${FS_ROOT_PATH} && et_full_system local wait_for_support && et_full_system local update_service_url fake_ccd http://localhost:${PORT} && rvm use && et_fake_ccd start --port=$PORT --create_case_schema=./systems/et_ccd_export/spec/json_schemas/case_create.json" +fake_acas_web: export FREEPORT=$(freeport) && bash --login -c "cd ${FS_ROOT_PATH} && et_full_system local wait_for_support && et_full_system local update_service_url fake_acas http://localhost:$FREEPORT && rvm use && et_fake_acas_server --port=$FREEPORT --threads=1:5" +fake_ccd_web: export FREEPORT=$(freeport) && bash --login -c "cd ${FS_ROOT_PATH} && et_full_system local wait_for_support && et_full_system local update_service_url fake_ccd http://localhost:$FREEPORT && rvm use && et_fake_ccd start --port=$FREEPORT --create_case_schema=./systems/et_ccd_export/spec/json_schemas/case_create.json" et_ccd_export_sidekiq: bash --login -c "cd ${FS_ROOT_PATH} && et_full_system local wait_for_support && cd systems/et_ccd_export && godotenv -f ${FOREMAN_PATH}/.env godotenv -f ${FOREMAN_PATH}/et_ccd_export.env ./run_sidekiq.sh" diff --git a/foreman/et_ccd_export.env b/foreman/et_ccd_export.env index bd6d94c..2e5a242 100644 --- a/foreman/et_ccd_export.env +++ b/foreman/et_ccd_export.env @@ -5,9 +5,10 @@ RAVEN_DSN=https://0ac125aa9b0240c793e081753671c041:531ce66cc13748fbbc87e0fe66d05 CCD_AUTH_BASE_URL=${CCD_AUTH_BASE_URL} CCD_IDAM_BASE_URL=${CCD_IDAM_BASE_URL} CCD_DATA_STORE_BASE_URL=${CCD_DATA_STORE_BASE_URL} +CCD_DOCUMENT_STORE_BASE_URL=${CCD_DOCUMENT_STORE_BASE_URL} CCD_USE_SIDAM=${CCD_USE_SIDAM} CCD_MICROSERVICE_ID=${CCD_MICROSERVICE_ID} CCD_MICROSERVICE_SECRET=${CCD_MICROSERVICE_SECRET} DISABLE_SIDEKIQ_ALIVE=true -GENERATE_ETHOS_CASE_REFERENCE=${CCD_GENERATE_ETHOS_CASE_REFERENCE} +CCD_GENERATE_ETHOS_CASE_REFERENCE=${CCD_GENERATE_ETHOS_CASE_REFERENCE} AZURE_APP_INSIGHTS_ROLE_NAME=et-ccd-export-fs \ No newline at end of file diff --git a/foreman/traefik.toml b/foreman/traefik.toml index 6722fec..8f3152a 100644 --- a/foreman/traefik.toml +++ b/foreman/traefik.toml @@ -1,5 +1,5 @@ [file] - +logLevel = "DEBUG" [entryPoints] [entryPoints.http] address = ":3100" diff --git a/lib/et_full_system/cli/docker.rb b/lib/et_full_system/cli/docker.rb index fe067e1..a2ed49c 100644 --- a/lib/et_full_system/cli/docker.rb +++ b/lib/et_full_system/cli/docker.rb @@ -38,6 +38,17 @@ def compose(*args) end end + desc "invoker", "Provides access to the invoker system running inside docker" + def invoker(*args, show_output: true, show_command: true) + Bundler.with_original_env do + gem_root = File.absolute_path('../../..', __dir__) + cmd = "GEM_VERSION=#{EtFullSystem::VERSION} docker-compose -f #{gem_root}/docker/docker-compose.yml exec et bash -lc \"invoker #{args.join(' ')}\"" + puts cmd if show_command + result = `#{cmd}` + puts result if show_output + end + end + desc "reset", "Bring down the server, remove all caches, rebuild the Dockerfile etc..." def reset Bundler.with_original_env do @@ -59,7 +70,7 @@ def update_service_url(service, url) cmd = "/bin/bash --login -c \"et_full_system local update_service_url #{service} #{url}\"" compose_cmd = "GEM_VERSION=#{EtFullSystem::VERSION} LOCALHOST_FROM_DOCKER_IP=#{host_ip} docker-compose -f #{gem_root}/docker/docker-compose.yml exec et #{cmd}" puts compose_cmd - exec(compose_cmd) + `#{compose_cmd}` end end @@ -68,6 +79,99 @@ def local_service(service, port) update_service_url(service, local_service_url(port)) end + desc "local_et1 PORT", "Configures the reverse proxy and the invoker system to allow a developer to run the web server and sidekiq locally" + def local_et1(port) + local_service('et1', port) + invoker 'remove', 'et1_web' + invoker 'remove', 'et1_sidekiq' + puts "ET1 is now expected to be hosted on port #{port} on your machine. To configure your environment, run 'et_full_system docker et1_env > .env.local'" + end + + desc "reset_et1", "Configures the reverse proxy and invoker to use the internal systems instead of local" + def reset_et1 + invoker 'add', 'et1_web' + invoker 'add', 'et1_sidekiq' + puts "ET1 is now being hosted from inside docker container" + end + + desc "et1_env", "Shows et1's environment variables as they should be on a developers machine running locally" + def et1_env + service_env('et1') + end + + desc "local_ccd_export", "Disables the sidekiq process in the invoker system to allow a developer to run it locally" + def local_ccd_export + invoker 'remove', 'et_ccd_export_sidekiq' + puts "ccd_export is now expected to be running on your machine. To configure your environment, run 'et_full_system docker ccd_export_env > .env.local'" + end + + desc "reset_ccd_export", "Configures invoker to use the internal systems instead of local" + def reset_ccd_export + invoker 'add', 'et_ccd_export_sidekiq' + puts "ccd_export is now being run from inside docker container" + end + + desc "ccd_export_env", "Shows ccd_export's environment variables as they should be on a developers machine running locally" + def ccd_export_env + service_env('et_ccd_export') + end + + desc "local_api PORT", "Configures the reverse proxy and the invoker system to allow a developer to run the web server and sidekiq locally" + def local_api(port) + local_service('api', port) + invoker 'remove', 'api_web' + invoker 'remove', 'api_sidekiq' + puts "api is now expected to be hosted on port #{port} on your machine. Also, you must provide your own sidekiq. To configure your environment, run 'et_full_system docker api_env > .env.local'" + end + + desc "reset_api", "Configures the reverse proxy and invoker to use the internal systems instead of local" + def reset_api + invoker 'add', 'api_web' + invoker 'add', 'api_sidekiq' + puts "api is now being hosted from inside docker container" + end + + desc "api_env", "Shows api's environment variables as they should be on a developers machine running locally" + def api_env + service_env('api') + end + + desc "local_admin PORT", "Configures the reverse proxy and the invoker system to allow a developer to run the admin web server locally" + def local_admin(port) + local_service('admin', port) + invoker 'remove', 'admin_web' + puts "Admin is now expected to be hosted on port #{port} on your machine. To configure your environment, run 'et_full_system docker admin_env > .env.local'" + end + + desc "reset_admin", "Configures the reverse proxy and invoker to use the internal systems instead of local" + def reset_admin + invoker 'add', 'admin_web' + puts "Admin is now being hosted from inside docker container" + end + + desc "admin_env", "Shows admin's environment variables as they should be on a developers machine running locally" + def admin_env + service_env('admin') + end + + desc "local_et3 PORT", "Configures the reverse proxy and the invoker system to allow a developer to run the et3 web server locally" + def local_et3(port) + local_service('et3', port) + invoker 'remove', 'et3_web' + puts "ET3 is now expected to be hosted on port #{port} on your machine. To configure your environment, run 'et_full_system docker et3_env > .env.local'" + end + + desc "reset_et3", "Configures the reverse proxy and invoker to use the internal systems instead of local" + def reset_et3 + invoker 'add', 'et3_web' + puts "ET3 is now being hosted from inside docker container" + end + + desc "et3_env", "Shows et3's environment variables as they should be on a developers machine running locally" + def et3_env + service_env('et3') + end + desc "service_env SERVICE", "Returns the environment variables configured for the specified service" def service_env(service) Bundler.with_original_env do @@ -94,7 +198,9 @@ def run_compose_command(*args, silent: false) end def host_ip - result = JSON.parse `docker network inspect \`docker network list | grep docker_et_full_system | awk '{print $1}'\`` + result = JSON.parse `docker network inspect docker_et_full_system` + return '0.0.0.0' if result.empty? + result.first.dig('IPAM', 'Config').first['Gateway'] end diff --git a/lib/et_full_system/cli/docker/server.rb b/lib/et_full_system/cli/docker/server.rb index 1b6a570..6ab9855 100644 --- a/lib/et_full_system/cli/docker/server.rb +++ b/lib/et_full_system/cli/docker/server.rb @@ -46,7 +46,7 @@ def up(*args) env_vars << "DB_PORT=#{ENV.fetch('DB_PORT', EtFullSystem.is_port_open?(5432) ? 0 : 5432)}" env_vars << "REDIS_PORT=#{ENV.fetch('REDIS_PORT', EtFullSystem.is_port_open?(6379) ? 0 : 6379)}" gem_root = File.absolute_path('../../../..', __dir__) - cmd = "#{env_vars.join(' ')} docker-compose -f #{gem_root}/docker/docker-compose.yml up #{args.join(' ')}" + cmd = "GEM_VERSION=#{EtFullSystem::VERSION} #{env_vars.join(' ')} docker-compose -f #{gem_root}/docker/docker-compose.yml up #{(extra_args + args).join(' ')}" puts cmd exec(cmd) end diff --git a/lib/et_full_system/cli/local.rb b/lib/et_full_system/cli/local.rb index f423866..14bf205 100644 --- a/lib/et_full_system/cli/local.rb +++ b/lib/et_full_system/cli/local.rb @@ -95,7 +95,7 @@ def server puts "Starting Procfile" ::Bundler.with_original_env do concurrency = " -c #{procfile_concurrency_without(options[:without]).join(',')}" - cmd = "CLOUD_PROVIDER=#{options[:cloud_provider]} RAILS_ENV=#{options[:rails_env]} AZURITE_STORAGE_PATH=\"#{options[:azurite_storage_path]}\" MINIO_STORAGE_PATH=\"#{options[:minio_storage_path]}\" FS_ROOT_PATH=#{PROJECT_PATH} FOREMAN_PATH=#{GEM_PATH}/foreman forego start -f \"#{GEM_PATH}/foreman/Procfile\" -e \"#{GEM_PATH}/foreman/.env\" -r#{concurrency}" + cmd = "CLOUD_PROVIDER=#{options[:cloud_provider]} RAILS_ENV=#{options[:rails_env]} AZURITE_STORAGE_PATH=\"#{options[:azurite_storage_path]}\" MINIO_STORAGE_PATH=\"#{options[:minio_storage_path]}\" FS_ROOT_PATH=#{PROJECT_PATH} FOREMAN_PATH=#{GEM_PATH}/foreman godotenv -f #{GEM_PATH}/foreman/.env invoker start \"#{GEM_PATH}/foreman/Procfile\" --port=4000" STDERR.puts cmd exec(cmd) end @@ -139,14 +139,15 @@ def service_env(service) 'api' => :et_api, 'et1' => :et1, 'et3' => :et3, - 'et_ccd_export' => :et_ccd_export_sidekiq + 'et_ccd_export' => :et_ccd_export } file = lookup.fetch(service) parsed = Dotenv.parse("#{GEM_PATH}/foreman/.env", "#{GEM_PATH}/foreman/#{file}.env") parsed.each_pair do |name, value| puts "#{name}=#{value}" end - + rescue KeyError + puts "The service must be one of #{lookup.keys}" end private diff --git a/lib/et_full_system/version.rb b/lib/et_full_system/version.rb index c43e72f..78f5bda 100644 --- a/lib/et_full_system/version.rb +++ b/lib/et_full_system/version.rb @@ -1,3 +1,3 @@ module EtFullSystem - VERSION = "1.0.0.pre49" + VERSION = "1.0.0" end