From 0365348ef6ec64e6dfa2032217080be02a40d692 Mon Sep 17 00:00:00 2001 From: Matthew Davis <7035647+mdavis-xyz@users.noreply.github.com> Date: Fri, 14 Jan 2022 23:06:46 +1100 Subject: [PATCH] Add abort multipart upload and expire obj del markers to s3 lifecycle (#794) Add abort multipart upload and expire obj del markers to s3 lifecycle Depends-On: ansible/ansible-zuul-jobs#1247 SUMMARY Fixes #365 #796 ISSUE TYPE Feature Pull Request COMPONENT NAME s3_lifecycle ADDITIONAL INFORMATION I have not run integration tests yet because of #793. I'm unsure about how to name and structure the new arguments. Do I nest them to match the API, or flatten them to match existing arguments? Reviewed-by: Alina Buzachis Reviewed-by: Matthew Davis Reviewed-by: Mark Chappell Reviewed-by: None Reviewed-by: Markus Bergholz (cherry picked from commit ed3a7a0b0ae86cbcaed733ada1f88a27917e1a09) --- .../794-s3_lifecycle_abort_expire.yml | 2 + plugins/modules/s3_lifecycle.py | 34 +++++++++++-- .../targets/s3_lifecycle/tasks/main.yml | 48 +++++++++++++++++++ 3 files changed, 81 insertions(+), 3 deletions(-) create mode 100644 changelogs/fragments/794-s3_lifecycle_abort_expire.yml diff --git a/changelogs/fragments/794-s3_lifecycle_abort_expire.yml b/changelogs/fragments/794-s3_lifecycle_abort_expire.yml new file mode 100644 index 00000000000..bd3aaf5617f --- /dev/null +++ b/changelogs/fragments/794-s3_lifecycle_abort_expire.yml @@ -0,0 +1,2 @@ +minor_changes: +- s3_lifecycle - Add ``abort_incomplete_multipart_upload_days`` and ``expire_object_delete_marker`` parameters (https://github.com/ansible-collections/community.aws/pull/794). diff --git a/plugins/modules/s3_lifecycle.py b/plugins/modules/s3_lifecycle.py index 9cec1402eb1..c12ce6b0897 100644 --- a/plugins/modules/s3_lifecycle.py +++ b/plugins/modules/s3_lifecycle.py @@ -23,16 +23,30 @@ - Name of the S3 bucket. required: true type: str + abort_incomplete_multipart_upload_days: + description: + - Specifies the days since the initiation of an incomplete multipart upload that Amazon S3 will wait before permanently removing all parts of the upload. + type: int + version_added: 2.2.0 expiration_date: description: - Indicates the lifetime of the objects that are subject to the rule by the date they will expire. - The value must be ISO-8601 format, the time must be midnight and a GMT timezone must be specified. + - This cannot be specified with I(expire_object_delete_marker) type: str expiration_days: description: - Indicates the lifetime, in days, of the objects that are subject to the rule. - The value must be a non-zero positive integer. + - This cannot be specified with I(expire_object_delete_marker) type: int + expire_object_delete_marker: + description: + - Indicates whether Amazon S3 will remove a delete marker with no noncurrent versions. + - If set to C(true), the delete marker will be expired; if set to C(false) the policy takes no action. + - This cannot be specified with I(expiration_days) or I(expiration_date). + type: bool + version_added: 2.2.0 prefix: description: - Prefix identifying one or more objects to which the rule applies. @@ -250,8 +264,10 @@ def fetch_rules(client, module, name): def build_rule(client, module): name = module.params.get("name") + abort_incomplete_multipart_upload_days = module.params.get("abort_incomplete_multipart_upload_days") expiration_date = parse_date(module.params.get("expiration_date")) expiration_days = module.params.get("expiration_days") + expire_object_delete_marker = module.params.get("expire_object_delete_marker") noncurrent_version_expiration_days = module.params.get("noncurrent_version_expiration_days") noncurrent_version_transition_days = module.params.get("noncurrent_version_transition_days") noncurrent_version_transitions = module.params.get("noncurrent_version_transitions") @@ -268,11 +284,19 @@ def build_rule(client, module): rule = dict(Filter=dict(Prefix=prefix), Status=status.title()) if rule_id is not None: rule['ID'] = rule_id + + if abort_incomplete_multipart_upload_days: + rule['AbortIncompleteMultipartUpload'] = { + 'DaysAfterInitiation': abort_incomplete_multipart_upload_days + } + # Create expiration if expiration_days is not None: rule['Expiration'] = dict(Days=expiration_days) elif expiration_date is not None: rule['Expiration'] = dict(Date=expiration_date.isoformat()) + elif expire_object_delete_marker is not None: + rule['Expiration'] = dict(ExpiredObjectDeleteMarker=expire_object_delete_marker) if noncurrent_version_expiration_days is not None: rule['NoncurrentVersionExpiration'] = dict(NoncurrentDays=noncurrent_version_expiration_days) @@ -525,8 +549,10 @@ def main(): s3_storage_class = ['glacier', 'onezone_ia', 'standard_ia', 'intelligent_tiering', 'deep_archive'] argument_spec = dict( name=dict(required=True, type='str'), + abort_incomplete_multipart_upload_days=dict(type='int'), expiration_days=dict(type='int'), expiration_date=dict(), + expire_object_delete_marker=dict(type='bool'), noncurrent_version_expiration_days=dict(type='int'), noncurrent_version_storage_class=dict(default='glacier', type='str', choices=s3_storage_class), noncurrent_version_transition_days=dict(type='int'), @@ -546,7 +572,7 @@ def main(): module = AnsibleAWSModule(argument_spec=argument_spec, mutually_exclusive=[ - ['expiration_days', 'expiration_date'], + ['expiration_days', 'expiration_date', 'expire_object_delete_marker'], ['expiration_days', 'transition_date'], ['transition_days', 'transition_date'], ['transition_days', 'expiration_date'], @@ -563,8 +589,10 @@ def main(): if state == 'present' and module.params["status"] == "enabled": # allow deleting/disabling a rule by id/prefix - required_when_present = ('expiration_date', 'expiration_days', 'transition_date', - 'transition_days', 'transitions', 'noncurrent_version_expiration_days', + required_when_present = ('abort_incomplete_multipart_upload_days', + 'expiration_date', 'expiration_days', 'expire_object_delete_marker', + 'transition_date', 'transition_days', 'transitions', + 'noncurrent_version_expiration_days', 'noncurrent_version_transition_days', 'noncurrent_version_transitions') for param in required_when_present: diff --git a/tests/integration/targets/s3_lifecycle/tasks/main.yml b/tests/integration/targets/s3_lifecycle/tasks/main.yml index d2700b9ea57..566a8381c19 100644 --- a/tests/integration/targets/s3_lifecycle/tasks/main.yml +++ b/tests/integration/targets/s3_lifecycle/tasks/main.yml @@ -327,6 +327,54 @@ that: - output is not changed + # ============================================================ + - name: Create a lifecycle policy, with abort_incomplete_multipart_upload + s3_lifecycle: + name: '{{ bucket_name }}' + abort_incomplete_multipart_upload_days: 1 + prefix: /something + register: output + + - assert: + that: + - output is changed + + # ============================================================ + - name: Create a lifecycle policy, with abort_incomplete_multipart_upload (idempotency) + s3_lifecycle: + name: '{{ bucket_name }}' + abort_incomplete_multipart_upload_days: 1 + prefix: /something + register: output + + - assert: + that: + - output is not changed + + # ============================================================ + - name: Create a lifecycle policy, with expired_object_delete_marker + s3_lifecycle: + name: '{{ bucket_name }}' + expire_object_delete_marker: True + prefix: /something + register: output + + - assert: + that: + - output is changed + + # ============================================================ + - name: Create a lifecycle policy, with expired_object_delete_marker (idempotency) + s3_lifecycle: + name: '{{ bucket_name }}' + expire_object_delete_marker: True + prefix: /something + register: output + + - assert: + that: + - output is not changed + # ============================================================ # test all the examples # Configure a lifecycle rule on a bucket to expire (delete) items with a prefix of /logs/ after 30 days