Merge pull request #438 from basecamp/remote-env-file

Copy env files to remote hosts

lib/kamal/cli/env.rb

@@ -0,0 +1,52 @@
+require "tempfile"
+
+class Kamal::Cli::Env < Kamal::Cli::Base
+  desc "push", "Push the env file to the remote hosts"
+  def push
+    mutating do
+      on(KAMAL.hosts) do
+        KAMAL.roles_on(host).each do |role|
+          role_config = KAMAL.config.role(role)
+          execute *KAMAL.app(role: role).make_env_directory
+          upload! StringIO.new(role_config.env_file), role_config.host_env_file_path, mode: 400
+        end
+      end
+
+      on(KAMAL.traefik_hosts) do
+        execute *KAMAL.traefik.make_env_directory
+        upload! StringIO.new(KAMAL.traefik.env_file), KAMAL.traefik.host_env_file_path, mode: 400
+      end
+
+      on(KAMAL.accessory_hosts) do
+        KAMAL.accessories_on(host).each do |accessory|
+          accessory_config = KAMAL.config.accessory(accessory)
+          execute *KAMAL.accessory(accessory).make_env_directory
+          upload! StringIO.new(accessory_config.env_file), accessory_config.host_env_file_path, mode: 400
+        end
+      end
+    end
+  end
+
+  desc "delete", "Delete the env file from the remote hosts"
+  def delete
+    mutating do
+      on(KAMAL.hosts) do
+        KAMAL.roles_on(host).each do |role|
+          role_config = KAMAL.config.role(role)
+          execute *KAMAL.app(role: role).remove_env_file
+        end
+      end
+
+      on(KAMAL.traefik_hosts) do
+        execute *KAMAL.traefik.remove_env_file
+      end
+
+      on(KAMAL.accessory_hosts) do
+        KAMAL.accessories_on(host).each do |accessory|
+          accessory_config = KAMAL.config.accessory(accessory)
+          execute *KAMAL.accessory(accessory).remove_env_file
+        end
+      end
+    end
+  end
+end

lib/kamal/cli/main.rb

@@ -175,6 +175,9 @@ def envify
     end
 
     File.write(env_path, ERB.new(File.read(env_template_path)).result, perm: 0600)
+
+    load_envs # reload new file
+    invoke "kamal:cli:env:push", options
   end
 
   desc "remove", "Remove Traefik, app, accessories, and registry session from servers"
@@ -204,6 +207,9 @@ def version
   desc "build", "Build application image"
   subcommand "build", Kamal::Cli::Build
 
+  desc "env", "Manage environment files"
+  subcommand "env", Kamal::Cli::Env
+
   desc "healthcheck", "Healthcheck application"
   subcommand "healthcheck", Kamal::Cli::Healthcheck
 

lib/kamal/commander.rb

@@ -75,6 +75,10 @@ def accessory_names
     config.accessories&.collect(&:name) || []
   end
 
+  def accessories_on(host)
+    config.accessories.select { |accessory| accessory.hosts.include?(host.to_s) }.map(&:name)
+  end
+
 
   def app(role: nil)
     Kamal::Commands::App.new(config, role: role)

lib/kamal/commands/accessory.rb

@@ -86,14 +86,6 @@ def ensure_local_file_present(local_file)
     end
   end
 
-  def make_directory_for(remote_file)
-    make_directory Pathname.new(remote_file).dirname.to_s
-  end
-
-  def make_directory(path)
-    [ :mkdir, "-p", path ]
-  end
-
   def remove_service_directory
     [ :rm, "-rf", service_name ]
   end
@@ -106,6 +98,14 @@ def remove_image
     docker :image, :rm, "--force", image
   end
 
+  def make_env_directory
+    make_directory accessory_config.host_env_directory
+  end
+
+  def remove_env_file
+    [:rm, "-f", accessory_config.host_env_file_path]
+  end
+
   private
     def service_filter
       [ "--filter", "label=service=#{service_name}" ]

lib/kamal/commands/app.rb

@@ -81,7 +81,7 @@ def execute_in_new_container(*command, interactive: false)
     docker :run,
       ("-it" if interactive),
       "--rm",
-      *config.env_args,
+      *role&.env_args,
       *config.volume_args,
       *role&.option_args,
       config.absolute_image,
@@ -149,6 +149,13 @@ def tag_current_as_latest
     docker :tag, config.absolute_image, config.latest_image
   end
 
+  def make_env_directory
+    make_directory config.role(role).host_env_directory
+  end
+
+  def remove_env_file
+    [:rm, "-f", config.role(role).host_env_file_path]
+  end
 
   private
     def container_name(version = nil)

lib/kamal/commands/base.rb

@@ -26,6 +26,14 @@ def container_id_for(container_name:, only_running: false)
       docker :container, :ls, *("--all" unless only_running), "--filter", "name=^#{container_name}$", "--quiet"
     end
 
+    def make_directory_for(remote_file)
+      make_directory Pathname.new(remote_file).dirname.to_s
+    end
+
+    def make_directory(path)
+      [ :mkdir, "-p", path ]
+    end
+
     private
       def combine(*commands, by: "&&")
         commands

lib/kamal/commands/traefik.rb

@@ -1,5 +1,5 @@
 class Kamal::Commands::Traefik < Kamal::Commands::Base
-  delegate :argumentize, :argumentize_env_with_secrets, :optionize, to: Kamal::Utils
+  delegate :argumentize, :env_file_with_secrets, :optionize, to: Kamal::Utils
 
   DEFAULT_IMAGE = "traefik:v2.9"
   CONTAINER_PORT = 80
@@ -63,6 +63,22 @@ def port
     "#{host_port}:#{CONTAINER_PORT}"
   end
 
+  def env_file
+    env_file_with_secrets config.traefik.fetch("env", {})
+  end
+
+  def host_env_file_path
+    File.join host_env_directory, "traefik.env"
+  end
+
+  def make_env_directory
+    make_directory(host_env_directory)
+  end
+
+  def remove_env_file
+    [:rm, "-f", host_env_file_path]
+  end
+
   private
     def publish_args
       argumentize "--publish", port unless config.traefik["publish"] == false
@@ -73,13 +89,11 @@ def label_args
     end
 
     def env_args
-      env_config = config.traefik["env"] || {}
+      argumentize "--env-file", host_env_file_path
+    end
 
-      if env_config.present?
-        argumentize_env_with_secrets(env_config)
-      else
-        []
-      end
+    def host_env_directory
+      File.join config.host_env_directory, "traefik"
     end
 
     def labels

lib/kamal/configuration.rb

@@ -7,7 +7,7 @@
 
 class Kamal::Configuration
   delegate :service, :image, :servers, :env, :labels, :registry, :stop_wait_time, :hooks_path, to: :raw_config, allow_nil: true
-  delegate :argumentize, :argumentize_env_with_secrets, :optionize, to: Kamal::Utils
+  delegate :argumentize, :optionize, to: Kamal::Utils
 
   attr_accessor :destination
   attr_accessor :raw_config
@@ -113,14 +113,6 @@ def service_with_version
   end
 
 
-  def env_args
-    if raw_config.env.present?
-      argumentize_env_with_secrets(raw_config.env)
-    else
-      []
-    end
-  end
-
   def volume_args
     if raw_config.volumes.present?
       argumentize "--volume", raw_config.volumes
@@ -174,7 +166,6 @@ def to_h
       repository: repository,
       absolute_image: absolute_image,
       service_with_version: service_with_version,
-      env_args: env_args,
       volume_args: volume_args,
       ssh_options: ssh.to_h,
       sshkit: sshkit.to_h,
@@ -199,12 +190,15 @@ def builder
 
   # Will raise KeyError if any secret ENVs are missing
   def ensure_env_available
-    env_args
-    roles.each(&:env_args)
+    roles.each(&:env_file)
 
     true
   end
 
+  def host_env_directory
+    "#{run_directory}/env"
+  end
+
   private
     # Will raise ArgumentError if any required config keys are missing
     def ensure_required_keys_present

lib/kamal/configuration/accessory.rb

@@ -1,5 +1,5 @@
 class Kamal::Configuration::Accessory
-  delegate :argumentize, :argumentize_env_with_secrets, :optionize, to: Kamal::Utils
+  delegate :argumentize, :env_file_with_secrets, :optionize, to: Kamal::Utils
 
   attr_accessor :name, :specifics
 
@@ -45,8 +45,20 @@ def env
     specifics["env"] || {}
   end
 
+  def env_file
+    env_file_with_secrets env
+  end
+
+  def host_env_directory
+    File.join config.host_env_directory, "accessories"
+  end
+
+  def host_env_file_path
+    File.join host_env_directory, "#{service_name}.env"
+  end
+
   def env_args
-    argumentize_env_with_secrets env
+    argumentize "--env-file", host_env_file_path
   end
 
   def files

lib/kamal/configuration/role.rb

@@ -1,5 +1,5 @@
 class Kamal::Configuration::Role
-  delegate :argumentize, :argumentize_env_with_secrets, :optionize, to: Kamal::Utils
+  delegate :argumentize, :env_file_with_secrets, :optionize, to: Kamal::Utils
 
   attr_accessor :name
 
@@ -31,8 +31,20 @@ def env
     end
   end
 
+  def env_file
+    env_file_with_secrets env
+  end
+
+  def host_env_directory
+    File.join config.host_env_directory, "roles"
+  end
+
+  def host_env_file_path
+    File.join host_env_directory, "#{[config.service, name, config.destination].compact.join("-")}.env"
+  end
+
   def env_args
-    argumentize_env_with_secrets env
+    argumentize "--env-file", host_env_file_path
   end
 
   def health_check_args

lib/kamal/utils.rb

@@ -16,14 +16,24 @@ def argumentize(argument, attributes, sensitive: false)
     end
   end
 
-  # Return a list of shell arguments using the same named argument against the passed attributes,
-  # but redacts and expands secrets.
-  def argumentize_env_with_secrets(env)
-    if (secrets = env["secret"]).present?
-      argumentize("-e", secrets.to_h { |key| [ key, ENV.fetch(key) ] }, sensitive: true) + argumentize("-e", env["clear"])
-    else
-      argumentize "-e", env.fetch("clear", env)
-    end
+  def env_file_with_secrets(env)
+    env_file = StringIO.new.tap do |contents|
+      if (secrets = env["secret"]).present?
+        env.fetch("secret", env)&.each do |key|
+          contents << docker_env_file_line(key, ENV.fetch(key))
+        end
+        env["clear"]&.each do |key, value|
+          contents << docker_env_file_line(key, value)
+        end
+      else
+        env.fetch("clear", env)&.each do |key, value|
+          contents << docker_env_file_line(key, value)
+        end
+      end
+    end.string
+
+    # Ensure the file has some contents to avoid the SSHKIT empty file warning
+    env_file || "\n"
   end
 
   # Returns a list of shell-dashed option arguments. If the value is true, it's treated like a value-less option.
@@ -97,4 +107,12 @@ def abbreviate_version(version)
   def uncommitted_changes
     `git status --porcelain`.strip
   end
+
+  def docker_env_file_line(key, value)
+    if key.include?("\n") || value.to_s.include?("\n")
+      raise ArgumentError, "docker env file format does not support newlines in keys or values, key: #{key}"
+    end
+
+    "#{key.to_s}=#{value.to_s}\n"
+  end
 end

test/cli/accessory_test.rb

@@ -7,7 +7,7 @@ class CliAccessoryTest < CliTestCase
 
     run_command("boot", "mysql").tap do |output|
       assert_match /docker login.*on 1.1.1.3/, output
-      assert_match "docker run --name app-mysql --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 3306:3306 -e [REDACTED] -e MYSQL_ROOT_HOST=\"%\" --volume $PWD/app-mysql/etc/mysql/my.cnf:/etc/mysql/my.cnf --volume $PWD/app-mysql/data:/var/lib/mysql --label service=\"app-mysql\" mysql:5.7 on 1.1.1.3", output
+      assert_match "docker run --name app-mysql --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 3306:3306 --env-file .kamal/env/accessories/app-mysql.env --volume $PWD/app-mysql/etc/mysql/my.cnf:/etc/mysql/my.cnf --volume $PWD/app-mysql/data:/var/lib/mysql --label service=\"app-mysql\" mysql:5.7 on 1.1.1.3", output
     end
   end
 
@@ -21,9 +21,9 @@ class CliAccessoryTest < CliTestCase
       assert_match /docker login.*on 1.1.1.3/, output
       assert_match /docker login.*on 1.1.1.1/, output
       assert_match /docker login.*on 1.1.1.2/, output
-      assert_match "docker run --name app-mysql --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 3306:3306 -e [REDACTED] -e MYSQL_ROOT_HOST=\"%\" --volume $PWD/app-mysql/etc/mysql/my.cnf:/etc/mysql/my.cnf --volume $PWD/app-mysql/data:/var/lib/mysql --label service=\"app-mysql\" mysql:5.7 on 1.1.1.3", output
-      assert_match "docker run --name app-redis --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 6379:6379 --volume $PWD/app-redis/data:/data --label service=\"app-redis\" redis:latest on 1.1.1.1", output
-      assert_match "docker run --name app-redis --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 6379:6379 --volume $PWD/app-redis/data:/data --label service=\"app-redis\" redis:latest on 1.1.1.2", output
+      assert_match "docker run --name app-mysql --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 3306:3306 --env-file .kamal/env/accessories/app-mysql.env --volume $PWD/app-mysql/etc/mysql/my.cnf:/etc/mysql/my.cnf --volume $PWD/app-mysql/data:/var/lib/mysql --label service=\"app-mysql\" mysql:5.7 on 1.1.1.3", output
+      assert_match "docker run --name app-redis --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 6379:6379 --env-file .kamal/env/accessories/app-redis.env --volume $PWD/app-redis/data:/data --label service=\"app-redis\" redis:latest on 1.1.1.1", output
+      assert_match "docker run --name app-redis --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 6379:6379 --env-file .kamal/env/accessories/app-redis.env --volume $PWD/app-redis/data:/data --label service=\"app-redis\" redis:latest on 1.1.1.2", output
     end
   end