FI-3440: Add IG entity and repository, integrated into Evaluate task

lib/inferno/apps/cli/evaluate.rb

@@ -1,14 +1,21 @@
 require_relative '../../../inferno/dsl/fhir_evaluation/evaluator'
+require_relative '../../../inferno/entities'
+require_relative '../../utils/ig_downloader'
+
+require 'tempfile'
 
 module Inferno
   module CLI
-    class Evaluate
-      def run(ig_path, data_path, _log_level)
+    class Evaluate < Thor::Group
+      include Thor::Actions
+      include Inferno::Utils::IgDownloader
+
+      def evaluate(ig_path, data_path, _log_level)
         validate_args(ig_path, data_path)
+        _ig = get_ig(ig_path)
 
-        # IG Import, rule execution, and result output below will be integrated at phase 2 and 3.
+        # Rule execution, and result output below will be integrated soon.
 
-        # @ig = File.join(__dir__, 'ig', ig_path)
         # if data_path
         #   DatasetLoader.from_path(File.join(__dir__, data_path))
         # else
@@ -30,6 +37,32 @@ def validate_args(ig_path, data_path)
         raise "Provided path '#{data_path}' is not a directory"
       end
 
+      def get_ig(ig_path)
+        if File.exist?(ig_path)
+          ig = Inferno::Entities::IG.from_file(ig_path)
+        elsif in_user_package_cache?(ig_path.sub('@', '#'))
+          # NPM syntax for a package identifier is id@version (eg, hl7.fhir.us.core@3.1.1)
+          # but in the cache the separator is # (hl7.fhir.us.core#3.1.1)
+          cache_directory = File.join(user_package_cache, ig_path.sub('@', '#'))
+          ig = Inferno::Entities::IG.from_file(cache_directory)
+        else
+          Tempfile.create('package.tgz') do |temp_file|
+            load_ig(ig_path, nil, { force: true }, temp_file.path)
+            ig = Inferno::Entities::IG.from_file(temp_file.path)
+          end
+        end
+        ig.add_self_to_repository
+        ig
+      end
+
+      def user_package_cache
+        File.join(Dir.home, '.fhir', 'packages')
+      end
+
+      def in_user_package_cache?(ig_identifier)
+        File.directory?(File.join(user_package_cache, ig_identifier))
+      end
+
       def output_results(results, output)
         if output&.end_with?('json')
           oo = FhirEvaluator::EvaluationResult.to_operation_outcome(results)

lib/inferno/apps/cli/main.rb

@@ -45,7 +45,7 @@ class Main < Thor
              type: :string,
              desc: 'Export evaluation result to outcome.json as an OperationOutcome'
       def evaluate(ig_path)
-        Evaluate.new.run(ig_path, options[:data_path], Logger::INFO)
+        Evaluate.new.evaluate(ig_path, options[:data_path], Logger::INFO)
       end
 
       desc 'console', 'Start an interactive console session with Inferno'

lib/inferno/entities.rb

@@ -2,6 +2,7 @@
 require_relative 'entities/has_runnable'
 require_relative 'entities/entity'
 require_relative 'entities/header'
+require_relative 'entities/ig'
 require_relative 'entities/message'
 require_relative 'entities/request'
 require_relative 'entities/result'

lib/inferno/entities/ig.rb

@@ -0,0 +1,149 @@
+# frozen_string_literal: true
+
+require 'fhir_models'
+require 'pathname'
+require 'rubygems/package'
+require 'zlib'
+
+require_relative '../repositories/igs'
+
+module Inferno
+  module Entities
+    # IG is a wrapper class around the relevant concepts inside an IG.
+    # Not everything within an IG is currently used by Inferno.
+    class IG < Entity
+      ATTRIBUTES = [
+        :id,
+        :profiles,
+        :extensions,
+        :value_sets,
+        :search_params,
+        :examples
+      ].freeze
+
+      include Inferno::Entities::Attributes
+
+      def initialize(**params)
+        super(params, ATTRIBUTES)
+
+        @profiles = []
+        @extensions = []
+        @value_sets = []
+        @examples = []
+        @search_params = []
+      end
+
+      def self.from_file(ig_path)
+        raise "#{ig_path} does not exist" unless File.exist?(ig_path)
+
+        # fhir_models by default logs the entire content of non-FHIR files
+        # which could be things like a package.json
+        original_logger = FHIR.logger
+        FHIR.logger = Logger.new('/dev/null')
+
+        if File.directory?(ig_path)
+          from_directory(ig_path)
+        elsif ig_path.end_with? '.tgz'
+          from_tgz(ig_path)
+        else
+          raise "Unable to load #{ig_path} as it does not appear to be a directory or a .tgz file"
+        end
+      ensure
+        FHIR.logger = original_logger if defined? original_logger
+      end
+
+      def self.from_tgz(ig_path)
+        tar = Gem::Package::TarReader.new(
+          Zlib::GzipReader.open(ig_path)
+        )
+
+        ig = IG.new
+
+        tar.each do |entry|
+          next if skip_item?(entry.full_name, entry.directory?)
+
+          begin
+            resource = FHIR::Json.from_json(entry.read)
+            next if resource.nil?
+
+            ig.handle_resource(resource, entry.full_name)
+          rescue StandardError
+            next
+          end
+        end
+        ig
+      end
+
+      def self.from_directory(ig_directory)
+        ig = IG.new
+
+        ig_path = Pathname.new(ig_directory)
+        Dir.glob("#{ig_path}/**/*") do |f|
+          relative_path = Pathname.new(f).relative_path_from(ig_path).to_s
+          next if skip_item?(relative_path, File.directory?(f))
+
+          begin
+            resource = FHIR::Json.from_json(File.read(f))
+            next if resource.nil?
+
+            ig.handle_resource(resource, relative_path)
+          rescue StandardError
+            next
+          end
+        end
+        ig
+      end
+
+      # These files aren't FHIR resources
+      FILES_TO_SKIP = ['package.json', 'validation-summary.json'].freeze
+
+      def self.skip_item?(relative_path, is_directory)
+        return true if is_directory
+
+        file_name = relative_path.split('/').last
+
+        return true unless file_name.end_with? '.json'
+        return true unless relative_path.start_with? 'package/'
+
+        return true if file_name.start_with? '.' # ignore hidden files
+        return true if file_name.end_with? '.openapi.json'
+        return true if FILES_TO_SKIP.include? file_name
+
+        false
+      end
+
+      def handle_resource(resource, relative_path)
+        case resource.resourceType
+        when 'StructureDefinition'
+          if resource.type == 'Extension'
+            extensions.push resource
+          else
+            profiles.push resource
+          end
+        when 'ValueSet'
+          value_sets.push resource
+        when 'SearchParameter'
+          search_params.push resource
+        when 'ImplementationGuide'
+          @id = extract_package_id(resource)
+        else
+          examples.push(resource) if relative_path.start_with? 'package/example'
+        end
+      end
+
+      def extract_package_id(ig_resource)
+        "#{ig_resource.id}##{ig_resource.version || 'current'}"
+      end
+
+      # @private
+      def add_self_to_repository
+        repository.insert(self)
+      end
+
+      # @private
+      def repository
+        Inferno::Repositories::IGs.new
+      end
+    end
+  end
+end

lib/inferno/repositories/igs.rb

@@ -0,0 +1,9 @@
+require_relative 'in_memory_repository'
+
+module Inferno
+  module Repositories
+    # Repository that deals with persistence for the `IG` entity.
+    class IGs < InMemoryRepository
+    end
+  end
+end

lib/inferno/utils/ig_downloader.rb

@@ -14,7 +14,7 @@ def ig_file(suffix = nil)
         File.join(ig_path, suffix ? "package_#{suffix}.tgz" : 'package.tgz')
       end
 
-      def load_ig(ig_input, idx = nil, thor_config = { verbose: true })
+      def load_ig(ig_input, idx = nil, thor_config = { verbose: true }, output_path = nil)
         case ig_input
         when FHIR_PACKAGE_NAME_REG_EX
           uri = ig_registry_url(ig_input)
@@ -25,12 +25,12 @@ def load_ig(ig_input, idx = nil, thor_config = { verbose: true })
         else
           raise StandardError, <<~FAILED_TO_LOAD
             Could not find implementation guide: #{ig_input}
-            Put its package.tgz file directly in #{ig_path}
           FAILED_TO_LOAD
         end
 
+        destination = output_path || ig_file(idx)
         # use Thor's get to support CLI options config
-        get(uri, ig_file(idx), thor_config)
+        get(uri, destination, thor_config)
         uri
       end
 

spec/extract_tgz_helper.rb

@@ -0,0 +1,13 @@
+module ExtractTGZHelper
+  def extract_tgz(fixture)
+    filename = File.basename(fixture, '.tgz')
+    target_dir = Dir.mktmpdir(filename)
+    system "mkdir -p #{target_dir}"
+    system "tar -xzf #{fixture} --directory #{target_dir}"
+    target_dir
+  end
+
+  def cleanup(target_dir)
+    FileUtils.remove_entry(target_dir)
+  end
+end

spec/inferno/entities/ig_spec.rb

@@ -0,0 +1,54 @@
+require 'extract_tgz_helper'
+
+RSpec.describe Inferno::Entities::IG do
+  include ExtractTGZHelper
+
+  let(:uscore3_package) { File.expand_path('../../fixtures/uscore311.tgz', __dir__) }
+  let(:uscore3_untarred) { extract_tgz(uscore3_package) }
+
+  after { cleanup(uscore3_untarred) }
+
+  describe '#from_file' do
+    it 'loads an IG from tgz file' do
+      ig = described_class.from_file(uscore3_package)
+      expect_uscore3_loaded_properly(ig)
+    end
+
+    it 'loads an IG from directory' do
+      ig = described_class.from_file(uscore3_untarred)
+      expect_uscore3_loaded_properly(ig)
+    end
+
+    def expect_uscore3_loaded_properly(ig) # rubocop:disable Naming/MethodParameterName, Metrics/CyclomaticComplexity
+      # For each artifact type in the IG, check:
+      # the right number are loaded,
+      # they're all the expected type,
+      # and spot check a couple IDs
+
+      # https://www.hl7.org/fhir/us/core/STU3.1.1/profiles.html
+      expect(ig.profiles.length).to eq(26)
+      expect(ig.profiles.map(&:resourceType).uniq).to eq(['StructureDefinition'])
+      expect(ig.profiles.map(&:id)).to include('us-core-patient', 'us-core-condition',
+                                               'head-occipital-frontal-circumference-percentile')
+
+      expect(ig.extensions.length).to eq(4)
+      expect(ig.extensions.map(&:resourceType).uniq).to eq(['StructureDefinition'])
+      expect(ig.extensions.map(&:type).uniq).to eq(['Extension'])
+      expect(ig.extensions.map(&:id)).to include('us-core-race', 'us-core-ethnicity')
+
+      # https://www.hl7.org/fhir/us/core/STU3.1.1/terminology.html
+      expect(ig.value_sets.length).to eq(32)
+      expect(ig.value_sets.map(&:resourceType).uniq).to eq(['ValueSet'])
+      expect(ig.value_sets.map(&:id)).to include('us-core-usps-state', 'simple-language')
+
+      # https://www.hl7.org/fhir/us/core/STU3.1.1/searchparameters.html
+      expect(ig.search_params.length).to eq(74)
+      expect(ig.search_params.map(&:resourceType).uniq).to eq(['SearchParameter'])
+      expect(ig.search_params.map(&:id)).to include('us-core-patient-name', 'us-core-encounter-id')
+
+      # https://www.hl7.org/fhir/us/core/STU3.1.1/all-examples.html
+      expect(ig.examples.length).to eq(84)
+      expect(ig.examples.map(&:id)).to include('child-example', 'self-tylenol')
+    end
+  end
+end

spec/inferno/utils/ig_downloader_spec.rb

@@ -76,6 +76,16 @@ def with_temp_path(name)
           expect(File.read(temp_path)).to eq(package_binary)
         end
       end
+
+      it 'successfully downloads package to custom path' do
+        stub_request(:get, 'https://packages.fhir.org/hl7.fhir.us.udap-security/-/hl7.fhir.us.udap-security-1.0.0.tgz')
+          .to_return(body: package_binary)
+
+        with_temp_path('ig-downloader-canonical') do |temp_path|
+          ig_downloader.load_ig(canonical, nil, { verbose: false }, temp_path)
+          expect(File.read(temp_path)).to eq(package_binary)
+        end
+      end
     end
   end