fix s3 file cache sharing temp folder across processes

cosmos/lib/cosmos/microservices/reducer_microservice.rb

@@ -250,7 +250,11 @@ def process_file(filename, type, entry_seconds, file_seconds)
       )
       true
     rescue => e
-      Logger.error("Reducer Error #{e}\n#{e.backtrace}")
+      if File.exist(filename)
+        Logger.error("Reducer Error: #{filename}:#{File.size(filename)} bytes: \n#{e.formatted}")
+      else
+        Logger.error("Reducer Error: #{filename}:(Does Not Exist): \n#{e.formatted}")
+      end
     end
 
     def reduce(type, data_keys, reduced)

cosmos/lib/cosmos/models/gem_model.rb

@@ -73,15 +73,15 @@ def self.put(gem_file_path, gem_install: true, scope:)
       return nil
     end
 
-    def self.install(name_or_path)
+    def self.install(name_or_path, scope:)
       temp_dir = Dir.mktmpdir
       begin
         if File.exist?(name_or_path)
           gem_file_path = name_or_path
         else
           gem_file_path = get(temp_dir, name_or_path)
         end
-        rubygems_url = get_setting('rubygems_url')
+        rubygems_url = get_setting('rubygems_url', scope: scope)
         Gem.sources = [rubygems_url] if rubygems_url
         Gem.done_installing_hooks.clear
         Gem.install(gem_file_path, "> 0.pre", :build_args => ['--no-document'], :prerelease => true)

cosmos/lib/cosmos/models/plugin_model.rb

@@ -125,7 +125,7 @@ def self.install_phase2(name, variables, scope:)
         gem_file_path = Cosmos::GemModel.get(temp_dir, name)
 
         # Actually install the gem now (slow)
-        Cosmos::GemModel.install(gem_file_path)
+        Cosmos::GemModel.install(gem_file_path, scope: scope)
 
         # Extract gem contents
         gem_path = File.join(temp_dir, "gem")

cosmos/lib/cosmos/utilities/s3_file_cache.rb

@@ -18,6 +18,7 @@
 # copyright holder
 
 require 'fileutils'
+require 'tmpdir'
 require 'cosmos'
 require 'cosmos/utilities/s3'
 
@@ -48,7 +49,7 @@ def initialize(s3_path, size = 0, priority = 0)
 
   def retrieve
     local_path = "#{S3FileCache.instance.cache_dir}/#{File.basename(@s3_path)}"
-    Cosmos::Logger.info "Retrieving #{@s3_path} from logs bucket"
+    Cosmos::Logger.debug "Retrieving #{@s3_path} from logs bucket"
     @rubys3_client.get_object(bucket: "logs", key: @s3_path, response_target: local_path)
     if File.exist?(local_path)
       @size = File.size(local_path)
@@ -57,6 +58,7 @@ def retrieve
   rescue => err
     @error = err
     Cosmos::Logger.error "Failed to retrieve #{@s3_path}\n#{err.formatted}"
+    raise err
   end
 
   def reserve
@@ -165,11 +167,11 @@ def initialize(name = 'default', max_disk_usage = MAX_DISK_USAGE)
     end
 
     # Create local file cache location
-    @cache_dir = File.join(Dir.tmpdir, 'cosmos', 'file_cache', name)
+    @cache_dir = Dir.mktmpdir
     FileUtils.mkdir_p(@cache_dir)
-
-    # Clear out local file cache
-    FileUtils.rm_f Dir.glob("#{@cache_dir}/*")
+    at_exit do
+      FileUtils.remove_dir(@cache_dir, true)
+    end
 
     @cached_files = S3FileCollection.new
 
@@ -178,7 +180,11 @@ def initialize(name = 'default', max_disk_usage = MAX_DISK_USAGE)
         file = @cached_files.get_next_to_retrieve
         # Cosmos::Logger.debug "Next file: #{file}"
         if file and (file.size + @cached_files.current_disk_usage()) <= @max_disk_usage
-          file.retrieve
+          begin
+            file.retrieve
+          rescue
+            # Will be automatically retried
+          end
         else
           sleep(1)
         end