Inspec storage (#2932)

Merged PR #2932.

overrides/inspec/resource_override.rb

@@ -26,6 +26,7 @@ def self.attributes
           privileged
           singular_only
           singular_extra_examples
+          iam_binding
         ]
       end
 

products/storage/ansible.yaml

@@ -22,11 +22,15 @@ datasources: !ruby/object:Overrides::ResourceOverrides
     exclude: true
   DefaultObjectACL: !ruby/object:Overrides::Ansible::ResourceOverride
     exclude: true
+  Object: !ruby/object:Overrides::Ansible::ResourceOverride
+    exclude: true
 overrides: !ruby/object:Overrides::ResourceOverrides
   ObjectAccessControl: !ruby/object:Overrides::Ansible::ResourceOverride
     exclude: true
   DefaultObjectACL: !ruby/object:Overrides::Ansible::ResourceOverride
     exclude: true
+  Object: !ruby/object:Overrides::Ansible::ResourceOverride
+    exclude: true
 files: !ruby/object:Provider::Config::Files
   resource:
 <%= lines(indent(compile('provider/ansible/resource~compile.yaml'), 4)) -%>

products/storage/api.yaml

@@ -22,14 +22,15 @@ scopes:
   - https://www.googleapis.com/auth/devstorage.full_control
 apis_required:
   - !ruby/object:Api::Product::ApiReference
-    name: Google CLoud Storage
+    name: Google Cloud Storage
     url: https://console.cloud.google.com/apis/library/storage-component.googleapis.com/
 objects:
   - !ruby/object:Api::Resource
     name: 'Bucket'
     kind: 'storage#bucket'
     base_url: b?project={{project}}
     self_link: b/{{name}}?projection=full
+    collection_url_key: items
     description: |
       The Buckets resource represents a bucket in Google Cloud Storage. There is
       a single global namespace shared by all buckets. For more information, see
@@ -40,6 +41,17 @@ objects:
       use in fine-grained manipulation of an existing bucket's access controls.
 
       A bucket is always owned by the project team owners group.
+    iam_policy: !ruby/object:Api::Resource::IamPolicy
+      allowed_iam_role: 'roles/storage.objectViewer'
+      admin_iam_role: 'roles/storage.admin'
+      parent_resource_attribute: 'bucket'
+      base_url: 'b/{{name}}'
+      import_format: ['b/{{name}}', '{{name}}']
+      iam_conditions_request_type: :QUERY_PARAM
+      fetch_iam_policy_method: 'iam'
+      set_iam_policy_method: 'iam'
+      set_iam_policy_verb: :PUT
+      wrapped_policy_obj: false
     properties:
       - !ruby/object:Api::Type::Array
         name: 'acl'
@@ -331,7 +343,7 @@ objects:
             name: 'entityId'
             description: 'The ID for the entity.'
             output: true
-      - !ruby/object:Api::Type::Integer
+      - !ruby/object:Api::Type::String
         name: 'projectNumber'
         description: 'The project number of the project the bucket belongs to.'
         output: true
@@ -425,6 +437,7 @@ objects:
     kind: 'storage#bucketAccessControl'
     base_url: b/{{bucket}}/acl
     self_link: b/{{bucket}}/acl/{{entity}}
+    collection_url_key: items
     description: |
       The BucketAccessControls resource represents the Access Control Lists
       (ACLs) for buckets within Google Cloud Storage. ACLs let you specify who
@@ -687,5 +700,74 @@ objects:
         values:
           - :OWNER
           - :READER
+  - !ruby/object:Api::Resource
+    name: 'Object'
+    base_url: b/{{bucket}}/o/{{object}}
+    self_link: b/{{bucket}}/o/{{object}}
+    description: |
+      Information about an object stored in a GCS bucket
+    properties:
+      - !ruby/object:Api::Type::String
+        name: 'bucket'
+        description: 'The name of the bucket.'
+        required: true
+      - !ruby/object:Api::Type::String
+        name: 'object'
+        description: 'The name of the object.'
+        required: true
+      - !ruby/object:Api::Type::String
+        name: 'contentType'
+        description: |
+          The Content-Type of the object data.
+
+          See https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Complete_list_of_MIME_types
+          for more information on possible Content-Types
+      - !ruby/object:Api::Type::String
+        name: 'crc32c'
+        description: 'CRC32c checksum.'
+      - !ruby/object:Api::Type::String
+        name: 'etag'
+        description: 'The object entity tag.'
+      - !ruby/object:Api::Type::Integer
+        name: 'generation'
+        description: 'The content generation of this object. Used for object versioning.'
+      - !ruby/object:Api::Type::String
+        name: 'id'
+        description: 'The ID of the object, including the bucket name, object name, and generation number.'
+      - !ruby/object:Api::Type::String
+        name: 'md5Hash'
+        description: 'MD5 hash of the data; encoded using base64.'
+      - !ruby/object:Api::Type::String
+        name: 'mediaLink'
+        description: 'Media download link.'
+      - !ruby/object:Api::Type::Integer
+        name: 'metageneration'
+        description: |
+          The version of the metadata for this object at this generation. Used for preconditions and for
+          detecting changes in metadata. A metageneration number is only meaningful in the context of a
+          particular generation of a particular object.
+      - !ruby/object:Api::Type::String
+        name: 'name'
+        description: 'The name of the object.'
+      - !ruby/object:Api::Type::Integer
+        name: 'size'
+        description: 'Content-Length of the data in bytes.'
+      - !ruby/object:Api::Type::String
+        name: 'storageClass'
+        description: 'Storage class of the object.'
+      - !ruby/object:Api::Type::Time
+        name: 'timeCreated'
+        description: 'The time this object was created.'
+      - !ruby/object:Api::Type::Time
+        name: 'timeDeleted'
+        description: |
+          The time this object was deleted. Returned if and only if this version of the object is no longer
+          a live version, but remains in the bucket as a noncurrent version.
+      - !ruby/object:Api::Type::Time
+        name: timeStorageClassUpdated
+        description: The time at which the object's storage class was last changed.
+      - !ruby/object:Api::Type::Time
+        name: timeUpdated
+        api_name: updated
+        description: The modification time of the object metadata.
 
-

products/storage/inspec.yaml

@@ -0,0 +1,60 @@
+# Copyright 2020 Google Inc.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+--- !ruby/object:Provider::Inspec::Config
+overrides: !ruby/object:Overrides::ResourceOverrides
+  Bucket: !ruby/object:Overrides::Inspec::ResourceOverride
+    iam_policy: !ruby/object:Api::Resource::IamPolicy
+      base_url: 'b/{{bucket}}'
+      fetch_iam_policy_method: 'iam'
+    base_url: b?project={{project}}&projection=full
+    properties:
+      name: !ruby/object:Overrides::Inspec::PropertyOverride
+        override_name: bucket_name
+      id: !ruby/object:Overrides::Inspec::PropertyOverride
+        override_name: bucket_id
+      projectNumber: !ruby/object:Overrides::Inspec::PropertyOverride
+        override_name: bucket_project_number
+      location: !ruby/object:Overrides::Inspec::PropertyOverride
+        override_name: bucket_location
+  ObjectAccessControl: !ruby/object:Overrides::Inspec::ResourceOverride
+    name: ObjectACL
+    singular_only: true
+    privileged: true
+    additional_functions: third_party/inspec/custom_functions/bucket_name_from_params.erb
+    properties:
+      bucket: !ruby/object:Overrides::Inspec::PropertyOverride
+        exclude: true
+  DefaultObjectACL: !ruby/object:Overrides::Inspec::ResourceOverride
+    singular_only: true
+    privileged: true
+    additional_functions: third_party/inspec/custom_functions/bucket_name_from_params.erb
+    properties:
+      bucket: !ruby/object:Overrides::Inspec::PropertyOverride
+        exclude: true
+  BucketAccessControl: !ruby/object:Overrides::Inspec::ResourceOverride
+    name: BucketACL
+    singular_only: true
+    privileged: true
+    additional_functions: third_party/inspec/custom_functions/bucket_name_from_params.erb
+    properties:
+      bucket: !ruby/object:Overrides::Inspec::PropertyOverride
+        exclude: true
+  Object: !ruby/object:Overrides::Inspec::ResourceOverride
+    name: BucketObject
+    singular_only: true
+    privileged: true
+    additional_functions: third_party/inspec/custom_functions/bucket_name_from_params.erb
+    properties:
+      bucket: !ruby/object:Overrides::Inspec::PropertyOverride
+        exclude: true

products/storage/terraform.yaml

@@ -16,17 +16,6 @@ overrides: !ruby/object:Overrides::ResourceOverrides
   Bucket: !ruby/object:Overrides::Terraform::ResourceOverride
     exclude_resource: true
     import_format: ["{{name}}"]
-    iam_policy: !ruby/object:Api::Resource::IamPolicy
-      allowed_iam_role: 'roles/storage.objectViewer'
-      admin_iam_role: 'roles/storage.admin'
-      parent_resource_attribute: 'bucket'
-      base_url: 'b/{{name}}'
-      import_format: ['b/{{name}}', '{{name}}']
-      iam_conditions_request_type: :QUERY_PARAM
-      fetch_iam_policy_method: 'iam'
-      set_iam_policy_method: 'iam'
-      set_iam_policy_verb: :PUT
-      wrapped_policy_obj: false
     examples:
       - !ruby/object:Provider::Terraform::Examples
         name: "storage_bucket_basic"
@@ -93,6 +82,8 @@ overrides: !ruby/object:Overrides::ResourceOverrides
         custom_expand: 'templates/terraform/custom_expand/resourceref_as_string.go.erb'
         # This field is (unexpectedly) not returned from the API
         ignore_read: true
+  Object: !ruby/object:Overrides::Terraform::ResourceOverride
+    exclude: true
 # This is for copying files over
 files: !ruby/object:Provider::Config::Files
   # These files have templating (ERB) code that will be run.

provider/inspec.rb

@@ -102,6 +102,28 @@ def generate_iam_policy(data)
         File.join(markdown_target_folder, "#{iam_policy_resource_name}.md"),
         self
       )
+
+      generate_iam_binding(data)
+    end
+
+    # Generate the IAM binding for this object. This is used to query and test
+    # IAM bindings in a more convienient way than using the IAM policy resource
+    def generate_iam_binding(data)
+      target_folder = File.join(data.output_folder, 'libraries')
+
+      iam_binding_resource_name = "#{resource_name(data.object, data.product)}_iam_binding"
+      data.generate(
+        'templates/inspec/iam_binding/iam_binding.erb',
+        File.join(target_folder, "#{iam_binding_resource_name}.rb"),
+        self
+      )
+
+      markdown_target_folder = File.join(data.output_folder, 'docs/resources')
+      data.generate(
+        'templates/inspec/iam_binding/iam_binding.md.erb',
+        File.join(markdown_target_folder, "#{iam_binding_resource_name}.md"),
+        self
+      )
     end
 
     def generate_properties(data, props)
@@ -340,5 +362,10 @@ def parse_code(property, hash_name)
     def extract_identifiers(url)
       url.scan(/({{)(\w+)(}})/).map { |arr| arr[1] }
     end
+
+    # Returns if this property has a sub property that is a Time class
+    def time_prop?(property_list = [])
+      property_list.any? { |sub_property| time?(sub_property) }
+    end
   end
 end

spec/copyright.rb

@@ -19,8 +19,8 @@ class CopyrightChecker
     def initialize(files)
       @input_files = files
       @copyrightable_files = {
-        /.*\.yaml$/ => /^# Copyright 201[789]/,
-        /.*\.rb$/ => /^# Copyright 201[789]/
+        /.*\.yaml$/ => /^# Copyright 20(17|18|19|20)/,
+        /.*\.rb$/ => /^# Copyright 20(17|18|19|20)/
       }
     end
 

templates/inspec/examples/attributes/external_attributes.yml

@@ -17,4 +17,7 @@ gcp_kube_cluster_size: 1
 gcp_kube_cluster_zone_extra1: europe-west2-b
 gcp_kube_cluster_zone_extra2: europe-west2-c
 gcp_kube_cluster_master_user: gcp-inspec-kube-admin
-gcp_kube_cluster_master_pass: P@$$w0rD
+gcp_kube_cluster_master_pass: P@$$w0rD
+gcp_storage_bucket_acl: storage-bucket-name
+gcp_storage_bucket_object_name: image1
+gcp_storage_bucket_object: bucket-with-object

templates/inspec/examples/google_storage_bucket/google_storage_bucket.erb

@@ -0,0 +1,12 @@
+<% gcp_project_id = "#{external_attribute('gcp_project_id', doc_generation)}" -%>
+<% gcp_location = "#{external_attribute('gcp_location', doc_generation)}" -%>
+describe google_storage_bucket(name: <%= doc_generation ? "bucket-name" : "\"inspec-gcp-static-\#{gcp_project_id}\"" -%>) do
+  it { should exist }
+  its('location') { should cmp <%= gcp_location -%>.upcase }
+
+  its('storage_class') { should eq "STANDARD" }
+end
+
+describe google_storage_bucket(name: "nonexistent") do
+  it { should_not exist }
+end

templates/inspec/examples/google_storage_bucket/google_storage_bucket_attributes.erb

@@ -0,0 +1,2 @@
+gcp_project_id = attribute(:gcp_project_id, default: '<%= external_attribute('gcp_project_id') -%>', description: 'The GCP project identifier.')
+gcp_location = attribute(:gcp_location, default: '<%= external_attribute('gcp_location') -%>', description: 'GCP location')

templates/inspec/examples/google_storage_bucket/google_storage_buckets.erb

@@ -0,0 +1,5 @@
+<% gcp_project_id = "#{external_attribute('gcp_project_id', doc_generation)}" -%>
+<% gcp_location = "#{external_attribute('gcp_location', doc_generation)}" -%>
+describe google_storage_buckets(project: <%= gcp_project_id -%>) do
+  its('bucket_names') { should include <%= doc_generation ? "bucket-name" : "\"inspec-gcp-static-\#{gcp_project_id}\"" -%> }
+end

templates/inspec/examples/google_storage_bucket_acl/google_storage_bucket_acl.erb

@@ -0,0 +1,15 @@
+<% gcp_project_id = "#{external_attribute('gcp_project_id', doc_generation)}" -%>
+<% gcp_storage_bucket_acl = "#{external_attribute('gcp_storage_bucket_acl', doc_generation)}" -%>
+<% gcp_service_account_display_name = "#{external_attribute('gcp_service_account_display_name', doc_generation)}" -%>
+<% gcp_enable_privileged_resources = "#{external_attribute('gcp_enable_privileged_resources', doc_generation)}" -%>
+describe google_storage_bucket_acl(bucket: <%= gcp_storage_bucket_acl -%>, entity: <%= doc_generation ? "user-email" : "\"user-\#{gcp_service_account_display_name}@\#{gcp_project_id}.iam.gserviceaccount.com\"" -%>) do
+  it { should exist }
+  its('role') { should cmp "OWNER" }
+
+  its('bucket') { should eq <%= gcp_storage_bucket_acl -%> }
+  its('email') { should include <%= doc_generation ? "entity-email.com" : "\"\#{gcp_service_account_display_name}@\#{gcp_project_id}.iam.gserviceaccount.com\"" -%> }
+end
+
+describe google_storage_bucket_acl(bucket: <%= gcp_storage_bucket_acl -%>, entity: "allUsers") do
+  it { should_not exist }
+end

templates/inspec/examples/google_storage_bucket_acl/google_storage_bucket_acl_attributes.erb

@@ -0,0 +1,4 @@
+gcp_project_id = attribute(:gcp_project_id, default: '<%= external_attribute('gcp_project_id') -%>', description: 'The GCP project identifier.')
+gcp_storage_bucket_acl = attribute(:gcp_storage_bucket_acl, default: '<%= external_attribute('gcp_storage_bucket_acl') -%>', description: 'The name of the storage bucket with ACLs attached')
+gcp_service_account_display_name = attribute(:gcp_service_account_display_name, default: '<%= external_attribute('gcp_service_account_display_name') -%>', description: 'The name of the service account assigned permissions')
+gcp_enable_privileged_resources = attribute(:gcp_enable_privileged_resources, default: '<%= external_attribute('gcp_enable_privileged_resources') -%>', description: 'If we are running tests with escalated permissions(required for this test)')

templates/inspec/examples/google_storage_bucket_object/google_storage_bucket_object.erb

@@ -0,0 +1,16 @@
+<% gcp_project_id = "#{external_attribute('gcp_project_id', doc_generation)}" -%>
+<% gcp_storage_bucket_object = "#{external_attribute('gcp_storage_bucket_object', doc_generation)}" -%>
+<% gcp_service_account_display_name = "#{external_attribute('gcp_service_account_display_name', doc_generation)}" -%>
+<% gcp_storage_bucket_object_name = "#{external_attribute('gcp_storage_bucket_object_name', doc_generation)}" -%>
+<% gcp_enable_privileged_resources = "#{external_attribute('gcp_enable_privileged_resources', doc_generation)}" -%>
+describe google_storage_bucket_object(bucket: <%= gcp_storage_bucket_object -%>, object: <%= gcp_storage_bucket_object_name -%>) do
+  it { should exist }
+  its('size.to_i') { should be > 0 }
+
+  its('time_created') { should be > Time.now - 60*60*24*10 }
+  its('time_updated') { should be > Time.now - 60*60*24*10 }
+end
+
+describe google_storage_bucket_object(bucket: <%= gcp_storage_bucket_object -%>, object: "nonexistent") do
+  it { should_not exist }
+end

templates/inspec/examples/google_storage_bucket_object/google_storage_bucket_object_attributes.erb

@@ -0,0 +1,5 @@
+gcp_project_id = attribute(:gcp_project_id, default: '<%= external_attribute('gcp_project_id') -%>', description: 'The GCP project identifier.')
+gcp_storage_bucket_object = attribute(:gcp_storage_bucket_object, default: '<%= external_attribute('gcp_storage_bucket_object') -%>', description: 'The name of the storage bucket with an object')
+gcp_service_account_display_name = attribute(:gcp_service_account_display_name, default: '<%= external_attribute('gcp_service_account_display_name') -%>', description: 'The name of the service account assigned permissions')
+gcp_enable_privileged_resources = attribute(:gcp_enable_privileged_resources, default: '<%= external_attribute('gcp_enable_privileged_resources') -%>', description: 'If we are running tests with escalated permissions(required for this test)')
+gcp_storage_bucket_object_name = attribute(:gcp_storage_bucket_object_name, default: '<%= external_attribute('gcp_storage_bucket_object_name') -%>', description: 'The name of the object')

templates/inspec/examples/google_storage_default_object_acl/google_storage_default_object_acl.erb

@@ -0,0 +1,15 @@
+<% gcp_project_id = "#{external_attribute('gcp_project_id', doc_generation)}" -%>
+<% gcp_storage_bucket_name = "#{external_attribute('gcp_storage_bucket_name', doc_generation)}" -%>
+<% gcp_service_account_display_name = "#{external_attribute('gcp_service_account_display_name', doc_generation)}" -%>
+<% gcp_enable_privileged_resources = "#{external_attribute('gcp_enable_privileged_resources', doc_generation)}" -%>
+describe google_storage_default_object_acl(bucket: <%= gcp_storage_bucket_name -%>, entity: <%= doc_generation ? "user-email" : "\"user-\#{gcp_service_account_display_name}@\#{gcp_project_id}.iam.gserviceaccount.com\"" -%>) do
+  it { should exist }
+  its('role') { should cmp "OWNER" }
+
+  its('bucket') { should eq <%= gcp_storage_bucket_name -%> }
+  its('email') { should include <%= doc_generation ? "entity-email.com" : "\"\#{gcp_service_account_display_name}@\#{gcp_project_id}.iam.gserviceaccount.com\"" -%> }
+end
+
+describe google_storage_default_object_acl(bucket: <%= gcp_storage_bucket_name -%>, entity: "allUsers") do
+  it { should_not exist }
+end

templates/inspec/examples/google_storage_default_object_acl/google_storage_default_object_acl_attributes.erb

@@ -0,0 +1,4 @@
+gcp_project_id = attribute(:gcp_project_id, default: '<%= external_attribute('gcp_project_id') -%>', description: 'The GCP project identifier.')
+gcp_storage_bucket_name = attribute(:gcp_storage_bucket_name, default: '<%= external_attribute('gcp_storage_bucket_name') -%>', description: 'The name of the storage bucket with the default object ACL')
+gcp_service_account_display_name = attribute(:gcp_service_account_display_name, default: '<%= external_attribute('gcp_service_account_display_name') -%>', description: 'The name of the service account assigned permissions')
+gcp_enable_privileged_resources = attribute(:gcp_enable_privileged_resources, default: '<%= external_attribute('gcp_enable_privileged_resources') -%>', description: 'If we are running tests with escalated permissions(required for this test)')