Honor tags when referencing an upstream cloudtruth template

Only fetch parameters if the local template uses them

lib/kubetruth/config.rb

@@ -49,6 +49,13 @@ def templates
         resource_templates.select {|k, v| active_templates.include?(k) }
       end
 
+      def to_s
+        to_h.to_json
+      end
+
+      def inspect
+        to_s
+      end
     end
 
     DEFAULT_SPEC = {
@@ -82,10 +89,10 @@ def load
 
         config = DEFAULT_SPEC.merge(root_mapping)
         @root_spec = ProjectSpec.new(**config)
-        logger.debug { "ProjectSpec for root mapping: #{@root_spec.inspect}"}
+        logger.debug { "ProjectSpec for root mapping: #{@root_spec}"}
         @override_specs = overrides.collect do |o|
           spec = ProjectSpec.new(**config.deep_merge(o))
-          logger.debug { "ProjectSpec for override mapping: #{spec.inspect}"}
+          logger.debug { "ProjectSpec for override mapping: #{spec}"}
           spec
         end
         config

lib/kubetruth/ctapi.rb

@@ -5,24 +5,17 @@
 module Kubetruth
   class CtApi
 
-    @@instance = nil
+    @@api_key = nil
+    @@api_url = nil
 
     def self.configure(api_key:, api_url:)
-      @@instance = self.new(api_key: api_key, api_url: api_url)
-    end
-
-    def self.reset
-      self.configure(api_key: instance.instance_variable_get(:@api_key), api_url: instance.api_url)
-    end
-
-    def self.instance
-      raise ArgumentError.new("CtApi has not been configured") if @@instance.nil?
-      return @@instance
+      @@api_key = api_key
+      @@api_url = api_url
     end
 
     include GemLogger::LoggerSupport
 
-    attr_reader :api_url, :client, :apis
+    attr_reader :client, :apis
 
     class ApiConfiguration < CloudtruthClient::Configuration
 
@@ -43,12 +36,17 @@ def auth_settings
 
     end
 
-    def initialize(api_key:, api_url:)
+    def initialize(environment: "default", tag: nil)
       @environments_mutex = Mutex.new
       @projects_mutex = Mutex.new
       @templates_mutex = Mutex.new
-      @api_key = api_key
-      @api_url = api_url
+
+      raise ArgumentError.new("CtApi has not been configured") if @@api_key.nil? || @@api_url.nil?
+      @api_key = @@api_key
+      @api_url = @@api_url
+
+      @environment = environment
+      @tag = tag
       uri = URI(@api_url)
       config = ApiConfiguration.new
       config.server_index = nil
@@ -57,7 +55,7 @@ def initialize(api_key:, api_url:)
       host_port << ":#{uri.port}" unless [80, 443].include?(uri.port)
       config.host = host_port
       config.base_path = uri.path
-      config.api_key = {'ApiKeyAuth' => api_key}
+      config.api_key = {'ApiKeyAuth' => @api_key}
       config.api_key_prefix = {'ApiKeyAuth' => "Api-Key"}
       config.logger = logger
       # config.debugging = logger.debug?
@@ -110,11 +108,10 @@ def project_id(project)
       project_id.to_s
     end
 
-    def parameters(project:, environment: "default", tag: nil)
-      env_id = environment_id(environment)
+    def parameters(project:)
       proj_id = project_id(project)
-      opts = {environment: env_id}
-      opts[:tag] = tag if tag.present?
+      opts = {environment: environment_id(@environment)}
+      opts[:tag] = @tag if @tag.present?
       result = apis[:projects].projects_parameters_list(proj_id, **opts)
       logger.debug do
         cleaned = result&.to_hash&.deep_dup
@@ -151,7 +148,9 @@ def templates(project:)
         @templates ||= {}
         @templates[project] ||= begin
           proj_id = projects[project]
-          result = apis[:projects].projects_templates_list(proj_id)
+          opts = {environment: environment_id(@environment)}
+          opts[:tag] = @tag if @tag.present?
+          result = apis[:projects].projects_templates_list(proj_id, **opts)
           logger.debug { "Templates query result: #{result.inspect}" }
           Hash[result&.results&.collect do |tmpl|
             # values is keyed by url, but we forced it to only have a single entry
@@ -172,11 +171,12 @@ def template_id(template, project:)
       template_id.to_s
     end
 
-    def template(name, project:, environment: "default")
-      env_id = environment_id(environment)
+    def template(name, project:)
       proj_id = project_id(project)
       tmpl_id = template_id(name, project: project)
-      result = apis[:projects].projects_templates_retrieve(tmpl_id, proj_id, environment: env_id)
+      opts = {environment: environment_id(@environment)}
+      opts[:tag] = @tag if @tag.present?
+      result = apis[:projects].projects_templates_retrieve(tmpl_id, proj_id, **opts)
       body = result&.body
       logger.debug { result.body = "<masked>" if result.has_secret; "Template Retrieve query result: #{result.inspect}" }
       body

lib/kubetruth/etl.rb

@@ -168,16 +168,57 @@ def with_log_level(level)
       end
     end
 
+    class DelayedParameters
+
+      def initialize(project)
+        @project = project
+      end
+
+      def params_to_hash(param_list)
+        Hash[param_list.collect {|param| [param.key, param.value]}]
+      end
+
+      def params
+        @param_data ||= begin
+          # constructing the hash will cause any overrides to happen in the right
+          # order (includer wins over last included over first included)
+          params = @project.all_parameters
+          parts = params.group_by(&:secret)
+          config_params, secret_params = (parts[false] || []), (parts[true] || [])
+          config_param_hash = params_to_hash(config_params)
+          secret_param_hash = params_to_hash(secret_params)
+
+          parameter_origins = @project.parameter_origins
+          param_origins_parts = parameter_origins.group_by {|k, v| config_param_hash.has_key?(k) }
+          config_origins = Hash[param_origins_parts[true] || []]
+          secret_origins = Hash[param_origins_parts[false] || []]
+
+          config_param_hash = config_param_hash.reject do |k, v|
+            logger.debug { "Excluding parameter with nil value: #{k}" } if v.nil?
+            v.nil?
+          end
+          secret_param_hash = secret_param_hash.reject do |k, v|
+            logger.debug { "Excluding secret parameter with nil value: #{k}" } if v.nil?
+            v.nil?
+          end
+
+          {
+            parameters: config_param_hash,
+            parameter_origins: config_origins,
+            secrets: secret_param_hash,
+            secret_origins: secret_origins
+          }
+        end
+      end
+    end
+
     def apply
       async(annotation: "ETL Event Loop") do
         logger.warn("Performing dry-run") if @dry_run
 
-        # Clear out the cache at the start of each polling cycle
-        CtApi.reset
-
         load_config do |namespace, config|
           with_log_level(config.root_spec.log_level) do
-            project_collection = ProjectCollection.new
+            project_collection = ProjectCollection.new(config.root_spec)
 
             # Load all projects that are used
             all_specs = [config.root_spec] + config.override_specs
@@ -210,27 +251,7 @@ def apply
 
                 async(annotation: "Project: #{project.name}") do
 
-                  # constructing the hash will cause any overrides to happen in the right
-                  # order (includer wins over last included over first included)
-                  params = project.all_parameters
-                  parts = params.group_by(&:secret)
-                  config_params, secret_params = (parts[false] || []), (parts[true] || [])
-                  config_param_hash = params_to_hash(config_params)
-                  secret_param_hash = params_to_hash(secret_params)
-
-                  parameter_origins = project.parameter_origins
-                  param_origins_parts = parameter_origins.group_by {|k, v| config_param_hash.has_key?(k) }
-                  config_origins = Hash[param_origins_parts[true] || []]
-                  secret_origins = Hash[param_origins_parts[false] || []]
-
-                  config_param_hash = config_param_hash.reject do |k, v|
-                    logger.debug { "Excluding parameter with nil value: #{k}" } if v.nil?
-                    v.nil?
-                  end
-                  secret_param_hash = secret_param_hash.reject do |k, v|
-                    logger.debug { "Excluding secret parameter with nil value: #{k}" } if v.nil?
-                    v.nil?
-                  end
+                  param_data = DelayedParameters.new(project)
 
                   resource_templates = project.spec.templates
                   resource_templates.each_with_index do |pair, i|
@@ -243,11 +264,11 @@ def apply
                       project: project.name,
                       project_heirarchy: project.heirarchy,
                       debug: logger.debug?,
-                      parameters: config_param_hash,
-                      parameter_origins: config_origins,
-                      secrets: secret_param_hash,
-                      secret_origins: secret_origins,
-                      templates: Template::TemplatesDrop.new(project: project.name, environment: project.spec.environment),
+                      parameters: proc { param_data.params[:parameters] },
+                      parameter_origins: proc { param_data.params[:parameter_origins] },
+                      secrets: proc { param_data.params[:secrets] },
+                      secret_origins: proc { param_data.params[:secret_origins] },
+                      templates: Template::TemplatesDrop.new(project: project.name, ctapi: project.ctapi),
                       context: project.spec.context
                     )
 
@@ -273,10 +294,6 @@ def apply
       end.wait
     end
 
-    def params_to_hash(param_list)
-      Hash[param_list.collect {|param| [param.key, param.value]}]
-    end
-
     def kube_apply(parsed_yml)
       kind = parsed_yml["kind"]
       name = parsed_yml["metadata"]["name"]

lib/kubetruth/project.rb

@@ -3,9 +3,13 @@ module Kubetruth
 
     include GemLogger::LoggerSupport
 
+    def ctapi
+      @ctapi ||= Kubetruth::CtApi.new(environment: spec.environment, tag: spec.tag)
+    end
+
     def parameters
       @parameters ||= begin
-        params = collection.ctapi.parameters(project: name, environment: spec.environment, tag: spec.tag)
+        params = ctapi.parameters(project: name)
         logger.debug do
           cleaned = params.deep_dup
           cleaned.each {|p| p.value = "<masked>" if p.secret}

lib/kubetruth/project_collection.rb

@@ -7,12 +7,13 @@ class ProjectCollection
 
     attr_accessor :projects
 
-    def initialize()
+    def initialize(project_spec)
       @projects = {}
+      @project_spec = project_spec
     end
 
     def ctapi
-      Kubetruth::CtApi.instance
+      @ctapi ||= Kubetruth::CtApi.new(environment: @project_spec.environment, tag: @project_spec.tag)
     end
 
     def names

lib/kubetruth/template.rb

@@ -38,21 +38,28 @@ def encode_with(coder)
         coder.represent_map(nil, @source)
       end
 
+      def to_s
+        Hash[@source.collect {|k, v| [k.to_s, v.to_s]}].to_json
+      end
+
+      def inspect
+        to_s
+      end
     end
 
     class TemplatesDrop < Liquid::Drop
 
-      def initialize(project:, environment:)
+      def initialize(project:, ctapi:)
         @project = project
-        @environment = environment
+        @ctapi = ctapi
       end
 
       def names
-        CtApi.instance.template_names(project: @project)
+        @ctapi.template_names(project: @project)
       end
 
       def liquid_method_missing(key)
-        CtApi.instance.template(key, project: @project, environment: @environment)
+        @ctapi.template(key, project: @project)
       end
 
     end
@@ -241,11 +248,12 @@ def render(*args, **kwargs)
       begin
 
         # TODO: fix secrets hardcoding here
-        secrets = kwargs[:secrets] || {}
+        secrets = nil
         debug_kwargs = nil
 
         logger.debug do
           # TODO: fix secrets hardcoding here
+          secrets ||= kwargs[:secrets]&.call || {}
           debug_kwargs ||= kwargs.merge(secrets: Hash[secrets.collect {|k, v| [k, "<masked:#{k}>"] }])
           msg = "Evaluating template:\n"
           @source.to_s.lines.collect {|l| msg << (INDENT * 2) << l }
@@ -257,6 +265,7 @@ def render(*args, **kwargs)
         result = @liquid.render!(*args, kwargs.stringify_keys, strict_variables: true, strict_filters: true)
 
         logger.debug do
+          secrets ||= kwargs[:secrets]&.call || {}
           debug_kwargs ||= kwargs.merge(secrets: Hash[secrets.collect {|k, v| [k, "<masked:#{k}>"] }])
           # we only ever have to sub base64 encoded in this debug block
           both_secrets = secrets.merge(Hash[secrets.collect {|k, v| ["#{k}_base64", Base64.strict_encode64(v)]}])
@@ -284,6 +293,7 @@ def render(*args, **kwargs)
         msg << (INDENT * 2) << e.message << "\n"
         if e.is_a?(Liquid::UndefinedVariable)
           msg << INDENT << "and variable context:\n"
+          secrets ||= kwargs[:secrets]&.call || {}
           debug_kwargs ||= kwargs.merge(secrets: Hash[secrets.collect {|k, v| [k, "<masked:#{k}>"] }])
           debug_kwargs.deep_stringify_keys.to_yaml.lines.collect {|l| msg << (INDENT * 2) << l }
         end
@@ -297,6 +307,10 @@ def to_s
       @source
     end
 
+    def inspect
+      to_s
+    end
+
     def encode_with(coder)
       coder.represent_scalar(nil, @source)
     end