Skip to content

out_s3: add support for server-side encryption headers #10901

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

out_s3: add support for server-side encryption headers #10901

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
83 changes: 81 additions & 2 deletions plugins/out_s3/s3.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,14 +92,14 @@ static struct flb_aws_header *get_content_encoding_header(int compression_type)
.val = "gzip",
.val_len = 4,
};

static struct flb_aws_header zstd_header = {
.key = "Content-Encoding",
.key_len = 16,
.val = "zstd",
.val_len = 4,
};

switch (compression_type) {
case FLB_AWS_COMPRESS_GZIP:
return &gzip_header;
Expand Down Expand Up @@ -138,6 +138,20 @@ static struct flb_aws_header storage_class_header = {
.val_len = 0,
};

static struct flb_aws_header sse_header = {
.key = "x-amz-server-side-encryption",
.key_len = 28,
.val = "",
.val_len = 0,
};

static struct flb_aws_header sse_kms_id_header = {
.key = "x-amz-server-side-encryption-aws-kms-key-id",
.key_len = 43,
.val = "",
.val_len = 0,
};

static char *mock_error_response(char *error_env_var)
{
char *err_val = NULL;
Expand Down Expand Up @@ -194,6 +208,12 @@ int create_headers(struct flb_s3 *ctx, char *body_md5,
if (ctx->storage_class != NULL) {
headers_len++;
}
if (ctx->server_side_encryption != NULL && strlen(ctx->server_side_encryption) > 0) {
headers_len++;
}
if (ctx->server_side_encryption_aws_kms_key_id != NULL && strlen(ctx->server_side_encryption_aws_kms_key_id) > 0) {
headers_len++;
}
if (headers_len == 0) {
*num_headers = headers_len;
*headers = s3_headers;
Expand Down Expand Up @@ -235,6 +255,18 @@ int create_headers(struct flb_s3 *ctx, char *body_md5,
s3_headers[n].val_len = strlen(body_md5);
n++;
}
if (ctx->server_side_encryption_aws_kms_key_id && strlen(ctx->server_side_encryption_aws_kms_key_id) > 0) {
s3_headers[n] = sse_kms_id_header;
s3_headers[n].val = ctx->server_side_encryption_aws_kms_key_id;
s3_headers[n].val_len = strlen(ctx->server_side_encryption_aws_kms_key_id);
n++;
}
if (ctx->server_side_encryption && strlen(ctx->server_side_encryption) > 0) {
s3_headers[n] = sse_header;
s3_headers[n].val = ctx->server_side_encryption;
s3_headers[n].val_len = strlen(ctx->server_side_encryption);
n++;
}
if (ctx->storage_class != NULL) {
s3_headers[n] = storage_class_header;
s3_headers[n].val = ctx->storage_class;
Expand Down Expand Up @@ -841,6 +873,39 @@ static int cb_s3_init(struct flb_output_instance *ins,
}
}

tmp = flb_output_get_property("server-side-encryption-aws-kms-key-id", ins);
if (tmp) {
ctx->server_side_encryption_aws_kms_key_id = (char *) tmp;
}

tmp = flb_output_get_property("server-side-encryption", ins);
if (tmp) {
ctx->server_side_encryption = (char *) tmp;
}

/* Validate sse values */
if (ctx->server_side_encryption != NULL) {
if (strcmp(ctx->server_side_encryption, "AES256") != 0 &&
strcmp(ctx->server_side_encryption, "aws:kms") != 0 &&
strcmp(ctx->server_side_encryption, "aws:kms:dsse") != 0) {
flb_plg_error(ctx->ins, "Invalid sse value '%s'. Valid values: AES256, aws:kms, aws:kms:dsse", ctx->server_side_encryption);
return -1;
}
}

/* Validate server-side-encryption-aws-kms-key-id requires aws:kms or aws:kms:dsse */
if (ctx->server_side_encryption_aws_kms_key_id != NULL) {
if (ctx->server_side_encryption == NULL) {
flb_plg_error(ctx->ins, "server-side-encryption-aws-kms-key-id requires server-side-encryption to be 'aws:kms' or 'aws:kms:dsse'");
return -1;
}
if (strcmp(ctx->server_side_encryption, "aws:kms") != 0 &&
strcmp(ctx->server_side_encryption, "aws:kms:dsse") != 0) {
flb_plg_error(ctx->ins, "server-side-encryption-aws-kms-key-id requires server-side-encryption to be 'aws:kms' or 'aws:kms:dsse', got '%s'", ctx->server_side_encryption);
return -1;
}
}

tmp = flb_output_get_property("sts_endpoint", ins);
if (tmp) {
ctx->sts_endpoint = (char *) tmp;
Expand Down Expand Up @@ -4086,6 +4151,20 @@ static struct flb_config_map config_map[] = {
"errors, which may help improve throughput when there are transient/random "
"networking issues."
},

{
FLB_CONFIG_MAP_STR, "server-side-encryption", NULL,
0, FLB_TRUE, offsetof(struct flb_s3, server_side_encryption),
"Server-side encryption type for S3 objects. Valid values: AES256, aws:kms, aws:kms:dsse. "
"When not specified, S3 applies the bucket's default encryption settings or SSE-S3 if no default is configured."
},
{
FLB_CONFIG_MAP_STR, "server-side-encryption-aws-kms-key-id", NULL,
0, FLB_TRUE, offsetof(struct flb_s3, server_side_encryption_aws_kms_key_id),
"KMS Key identifier for SSE-KMS encryption. Optional when server-side-encryption is 'aws:kms' or 'aws:kms:dsse'. "
"If not specified, S3 uses the AWS managed key (aws/s3). "
"Provide this to use a customer managed key."
},

{
FLB_CONFIG_MAP_BOOL, "use_put_object", "false",
Expand Down
2 changes: 2 additions & 0 deletions plugins/out_s3/s3.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,8 @@ struct flb_s3 {
char *s3_key_format;
char *tag_delimiters;
char *endpoint;
char *server_side_encryption_aws_kms_key_id;
char *server_side_encryption;
char *sts_endpoint;
char *canned_acl;
char *content_type;
Expand Down