Ansible facts modules

.ci/magic-modules/merge-pr.sh

@@ -44,9 +44,9 @@ echo "Merged PR #$ID." > ./commit_message
 
 set +e
 if [ "$REPO" != "GoogleCloudPlatform/magic-modules" ]; then
-  git remote add push-target "git@github.com:$REPO"
+  git remote add non-gcp-push-target "git@github.com:$REPO"
   # We know we have a commit, so all the machinery of the git resources is
   # unnecessary.  We can just try to push directly, without forcing.
-  ssh-agent bash -c "ssh-add ~/github_private_key; git push push-target $BRANCH"
+  ssh-agent bash -c "ssh-add ~/github_private_key; git push non-gcp-push-target $BRANCH"
 fi
 set -e

products/compute/ansible.yaml

@@ -24,6 +24,82 @@ manifest: !ruby/object:Provider::Ansible::Manifest
   version_added: '2.6'
   author: Google Inc. (@googlecloudplatform)
 # This is where custom code would be defined eventually.
+datasources: !ruby/object:Provider::ResourceOverrides
+  Instance: !ruby/object:Provider::Ansible::ResourceOverride
+    version_added: '2.7'
+  Address: !ruby/object:Provider::Ansible::ResourceOverride
+    exclude: true
+  Autoscaler: !ruby/object:Provider::Ansible::ResourceOverride
+    exclude: true
+  BackendBucket: !ruby/object:Provider::Ansible::ResourceOverride
+    exclude: true
+  BackendService: !ruby/object:Provider::Ansible::ResourceOverride
+    exclude: true
+  Disk: !ruby/object:Provider::Ansible::ResourceOverride
+    exclude: true
+  Firewall: !ruby/object:Provider::Ansible::ResourceOverride
+    exclude: true
+  ForwardingRule: !ruby/object:Provider::Ansible::ResourceOverride
+    exclude: true
+  GlobalAddress: !ruby/object:Provider::Ansible::ResourceOverride
+    exclude: true
+  GlobalForwardingRule: !ruby/object:Provider::Ansible::ResourceOverride
+    exclude: true
+  HealthCheck: !ruby/object:Provider::Ansible::ResourceOverride
+    exclude: true
+  HttpHealthCheck: !ruby/object:Provider::Ansible::ResourceOverride
+    exclude: true
+  HttpsHealthCheck: !ruby/object:Provider::Ansible::ResourceOverride
+    exclude: true
+  Image: !ruby/object:Provider::Ansible::ResourceOverride
+    exclude: true
+  InstanceGroup: !ruby/object:Provider::Ansible::ResourceOverride
+    exclude: true
+  InstanceGroupManager: !ruby/object:Provider::Ansible::ResourceOverride
+    exclude: true
+  InstanceTemplate: !ruby/object:Provider::Ansible::ResourceOverride
+    exclude: true
+  Network: !ruby/object:Provider::Ansible::ResourceOverride
+    exclude: true
+  RegionAutoscaler: !ruby/object:Provider::Ansible::ResourceOverride
+    exclude: true
+  Route: !ruby/object:Provider::Ansible::ResourceOverride
+    exclude: true
+  Router: !ruby/object:Provider::Ansible::ResourceOverride
+    exclude: true
+  Snapshot: !ruby/object:Provider::Ansible::ResourceOverride
+    exclude: true
+  SslCertificate: !ruby/object:Provider::Ansible::ResourceOverride
+    exclude: true
+  SslPolicy: !ruby/object:Provider::Ansible::ResourceOverride
+    exclude: true
+  Subnetwork: !ruby/object:Provider::Ansible::ResourceOverride
+    exclude: true
+  TargetPool: !ruby/object:Provider::Ansible::ResourceOverride
+    exclude: true
+  TargetHttpProxy: !ruby/object:Provider::Ansible::ResourceOverride
+    exclude: true
+  TargetHttpsProxy: !ruby/object:Provider::Ansible::ResourceOverride
+    exclude: true
+  TargetSslProxy: !ruby/object:Provider::Ansible::ResourceOverride
+    exclude: true
+  TargetTcpProxy: !ruby/object:Provider::Ansible::ResourceOverride
+    exclude: true
+  TargetVpnGateway: !ruby/object:Provider::Ansible::ResourceOverride
+    exclude: true
+  UrlMap: !ruby/object:Provider::Ansible::ResourceOverride
+    exclude: true
+  # Readonly resources.
+  DiskType: !ruby/object:Provider::Ansible::ResourceOverride
+    exclude: true
+  License: !ruby/object:Provider::Ansible::ResourceOverride
+    exclude: true
+  MachineType: !ruby/object:Provider::Ansible::ResourceOverride
+    exclude: true
+  Region: !ruby/object:Provider::Ansible::ResourceOverride
+    exclude: true
+  Zone: !ruby/object:Provider::Ansible::ResourceOverride
+    exclude: true
 overrides: !ruby/object:Provider::ResourceOverrides
   BackendService: !ruby/object:Provider::Ansible::ResourceOverride
     properties:

products/compute/examples/ansible/instance.yaml

@@ -62,12 +62,13 @@ task: !ruby/object:Provider::Ansible::Task
     project: <%= ctx[:project] %>
     auth_kind: <%= ctx[:auth_kind] %>
     service_account_file: <%= ctx[:service_account_file] %>
-verifier: !ruby/object:Provider::Ansible::Verifier
-  command: |
-    gcloud compute instances describe
-      --project="{{ gcp_project }}"
-      --zone="us-central1-a"
-      "{{ resource_name }}"
-  failure: !ruby/object:Provider::Ansible::ComputeFailureCondition
-    region: zones/us-central1-a
-    type: instances
+verifier: !ruby/object:Provider::Ansible::FactsVerifier
+  parameters:
+    zone: 'us-central1-a'
+facts: !ruby/object:Provider::Ansible::Task
+  name: gcp_compute_instance_facts
+  code: |
+    zone: 'us-central1-a'
+    project: <%= ctx[:project] %>
+    auth_kind: <%= ctx[:auth_kind] %>
+    service_account_file: <%= ctx[:service_account_file] %>

provider/ansible.rb

@@ -249,6 +249,10 @@ def wrap_field(field, spaces)
         field.scan(/\S.{0,#{avail_columns}}\S(?=\s|$)|\S+/)
       end
 
+      def list_kind(object)
+        "#{object.kind}List"
+      end
+
       private
 
       def get_example(cfg_file)
@@ -302,6 +306,17 @@ def generate_resource_tests(data)
         )
       end
 
+      def compile_datasource(data)
+        target_folder = data[:output_folder]
+        FileUtils.mkpath target_folder
+        name = "#{module_name(data[:object])}_facts"
+        generate_resource_file data.clone.merge(
+          default_template: 'templates/ansible/facts.erb',
+          out_file: File.join(target_folder,
+                              "lib/ansible/modules/cloud/google/#{name}.py")
+        )
+      end
+
       def generate_network_datas(data, object) end
 
       def generate_base_property(data) end

provider/ansible/example.rb

@@ -92,6 +92,65 @@ def build_task(state, object)
 
       private
 
+      def verbs
+        {
+          present: 'created',
+          absent: 'deleted',
+          facts: 'verify'
+        }
+      end
+    end
+
+    # Holds all information necessary to run a facts module and verify the
+    # creation / deletion of a resource.
+    # Takes in a set of parameters, which are used as properties on the facts
+    # module.
+    class FactsVerifier < Api::Object
+      include Compile::Core
+
+      def validate
+        @failure ||= FailureCondition.new
+
+        check_property :parameters, Hash
+      end
+
+      # rubocop:disable Metrics/AbcSize
+      # rubocop:disable Metrics/MethodLength
+      def build_task(state, object)
+        obj_name = Google::StringUtils.underscore(object.name)
+        verb = verbs[state.to_sym]
+        number = state == 'present' ? 1 : 0
+        module_name = ["gcp_#{object.__product.prefix[1..-1]}",
+                       Google::StringUtils.underscore(object.name),
+                       'facts'].join('_')
+        [
+          "- name: verify that #{obj_name} was #{verb}",
+          indent([
+            "#{module_name}:",
+            indent([
+                     'filters:',
+                     '   - name = "{{ resource_name }}"',
+                     @parameters.map { |k, v| "#{k}: #{v}" },
+                     'project: "{{ gcp_project }}"',
+                     'auth_kind: "{{ gcp_cred_kind }}"',
+                     'service_account_file: "{{ gcp_cred_file }}"',
+                     'scopes:',
+                     "  - #{object.__product.scopes[0]}"
+                   ], 4),
+            'register: results'
+          ].flatten.compact, 2),
+
+          '- name: verify that command succeeded',
+          '  assert:',
+          '    that:',
+          indent("- results['items'] | length == #{number}", 6)
+        ].compact
+      end
+      # rubocop:enable Metrics/MethodLength
+      # rubocop:enable Metrics/AbcSize
+
+      private
+
       def verbs
         {
           present: 'created',
@@ -196,6 +255,7 @@ def verbs
       private
 
       # rubocop:disable Metrics/CyclomaticComplexity
+      # rubocop:disable Metrics/PerceivedComplexity
       def build_task(state, hash, object, noop = false)
         verb = verbs[state.to_sym]
 
@@ -208,17 +268,17 @@ def build_task(state, hash, object, noop = false)
           indent([
             "#{@name}:",
             indent([
-                     compile_string(hash, @code),
-                     'scopes:',
-                     indent(lines(scopes), 2),
-                     "state: #{state}"
-                   ], 4),
+              compile_string(hash, @code),
+              'scopes:',
+              indent(lines(scopes), 2),
+              ("state: #{state}" if state != 'facts')
+            ].compact, 4),
             ("register: #{@register}" unless @register.nil?)
           ].compact, 2)
         ]
       end
-      # rubocop:enable Metrics/MethodLength
       # rubocop:enable Metrics/CyclomaticComplexity
+      # rubocop:enable Metrics/PerceivedComplexity
 
       def object_name_from_module_name(mod_name)
         product_name = mod_name.match(/gcp_[a-z]*_(.*)/).captures[0]
@@ -234,13 +294,15 @@ def dependency_name(dependency, resource)
     # examples.
     class Example < Api::Object
       attr_reader :task
+      attr_reader :facts
       attr_reader :verifier
       attr_reader :dependencies
 
       def validate
         super
         check_property :task, Task
-        check_optional_property :verifier, Verifier
+        check_optional_property :facts, Task
+        check_optional_property :verifier, [Verifier, FactsVerifier]
         check_optional_property_list :dependencies, Task
       end
     end

provider/ansible/resource_override.rb

@@ -24,6 +24,7 @@ module OverrideProperties
       attr_reader :create
       attr_reader :delete
       attr_reader :editable
+      attr_reader :filter_url
       attr_reader :hidden
       attr_reader :imports
       attr_reader :provider_helpers
@@ -45,6 +46,7 @@ def validate
         default_value_property :custom_update_resource, false
         default_value_property :exclude, false
         default_value_property :editable, true
+        default_value_property :filter_url, ''
         default_value_property :imports, []
         default_value_property :provider_helpers, []
         default_value_property :unwrap_resource, false
@@ -56,6 +58,7 @@ def validate
         check_optional_property :create, ::String
         check_optional_property :delete, ::String
         check_property :editable, :boolean
+        check_property :filter_url, ::String
         check_optional_property :hidden, ::Array
         check_property :imports, ::Array
         check_property :provider_helpers, ::Array

provider/config.rb

@@ -23,6 +23,8 @@ class Config < Api::Object
     extend Compile::Core
 
     attr_reader :overrides
+    # Overrides for datasources
+    attr_reader :datasources
     attr_reader :objects
     attr_reader :examples
     attr_reader :properties # TODO(nelsonjr): Remove this once bug 193 is fixed.

provider/core.rb

@@ -78,6 +78,9 @@ def generate(output_folder, types, version_name)
       # or compiled.
       compile_files(output_folder) \
         unless @config.files.nil? || @config.files.compile.nil?
+
+      generate_datasources(output_folder, types, version) \
+        unless @config.datasources.nil?
       apply_file_acls(output_folder) \
         unless @config.files.nil? || @config.files.permissions.nil?
       verify_test_matrixes
@@ -237,6 +240,42 @@ def generate_object(object, output_folder, version)
       generate_network_datas data, object
     end
 
+    # rubocop:disable Metrics/AbcSize
+    # rubocop:disable Metrics/CyclomaticComplexity
+    # rubocop:disable Metrics/PerceivedComplexity
+    def generate_datasources(output_folder, types, version)
+      # We need to apply overrides for datasources
+      @config.datasources.validate
+
+      @api.set_properties_based_on_version(version)
+      @api.objects.each do |object|
+        if !types.empty? && !types.include?(object.name)
+          Google::LOGGER.info(
+            "Excluding #{object.name} datasource per user request"
+          )
+        elsif types.empty? && object.exclude
+          Google::LOGGER.info(
+            "Excluding #{object.name} datasource per API catalog"
+          )
+        elsif types.empty? && object.exclude_if_not_in_version(version)
+          Google::LOGGER.info(
+            "Excluding #{object.name} datasource per API version"
+          )
+        else
+          generate_datasource object, output_folder, version
+        end
+      end
+    end
+    # rubocop:enable Metrics/CyclomaticComplexity
+    # rubocop:enable Metrics/PerceivedComplexity
+    # rubocop:enable Metrics/AbcSize
+
+    def generate_datasource(object, output_folder, version)
+      data = build_object_data(object, output_folder, version)
+
+      compile_datasource data
+    end
+
     # Generates all 6 network data files for a object.
     # This includes all combinations of seeds [0-2] and title == / != name
     # Each data file is a YAML file with all properties possible on an object.

tasks/common.rb

@@ -13,6 +13,7 @@
 
 PROVIDER_FOLDERS = {
   ansible: 'build/ansible',
+  ansible_facts: 'build/ansible',
   puppet: 'build/puppet/%s',
   chef: 'build/chef/%s',
   terraform: 'build/terraform'