From c8e7cac949122469dddfecc53fe086aa953c55b1 Mon Sep 17 00:00:00 2001 From: John Esmet Date: Mon, 21 Dec 2020 19:53:25 +0000 Subject: [PATCH 01/10] Add custom date formatting to downstream cert start and end dates Signed-off-by: John Esmet --- .../formatter/substitution_formatter.cc | 90 ++++++++- .../common/formatter/substitution_formatter.h | 41 ++++ .../formatter/substitution_formatter_test.cc | 176 +++++++++++------- 3 files changed, 239 insertions(+), 68 deletions(-) diff --git a/source/common/formatter/substitution_formatter.cc b/source/common/formatter/substitution_formatter.cc index fc6a92fa9995..26afa5437a51 100644 --- a/source/common/formatter/substitution_formatter.cc +++ b/source/common/formatter/substitution_formatter.cc @@ -43,8 +43,8 @@ void truncate(std::string& str, absl::optional max_length) { str = str.substr(0, max_length.value()); } -// Matches newline pattern in a StartTimeFormatter format string. -const std::regex& getStartTimeNewlinePattern() { +// Matches newline pattern in a system time format string (e.g. start time) +const std::regex& getDateFormatNewlinePattern() { CONSTRUCT_ON_FIRST_USE(std::regex, "%[-_0^#]*[1-9]*(E|O)?n"); } const std::regex& getNewlinePattern() { CONSTRUCT_ON_FIRST_USE(std::regex, "\n"); } @@ -378,10 +378,38 @@ std::vector SubstitutionFormatParser::parse(const std::str : ""; // Validate the input specifier here. The formatted string may be destined for a header, and // should not contain invalid characters {NUL, LR, CF}. - if (std::regex_search(args, getStartTimeNewlinePattern())) { + if (std::regex_search(args, getDateFormatNewlinePattern())) { throw EnvoyException("Invalid header configuration. Format string contains newline."); } formatters.emplace_back(FormatterProviderPtr{new StartTimeFormatter(args)}); + } else if (absl::StartsWith(token, "DOWNSTREAM_PEER_CERT_V_START")) { + const size_t parameters_length = pos + DownstreamPeerCertVStartParamStart + 1; + const size_t parameters_end = command_end_position - parameters_length; + + const std::string args = + token[DownstreamPeerCertVStartParamStart - 1] == '(' + ? token.substr(DownstreamPeerCertVStartParamStart, parameters_end) + : ""; + // Validate the input specifier here. The formatted string may be destined for a header, and + // should not contain invalid characters {NUL, LR, CF}. + if (std::regex_search(args, getDateFormatNewlinePattern())) { + throw EnvoyException("Invalid header configuration. Format string contains newline."); + } + formatters.emplace_back(FormatterProviderPtr{new DownstreamPeerCertVStartFormatter(args)}); + } else if (absl::StartsWith(token, "DOWNSTREAM_PEER_CERT_V_END")) { + const size_t parameters_length = pos + DownstreamPeerCertVEndParamStart + 1; + const size_t parameters_end = command_end_position - parameters_length; + + const std::string args = + token[DownstreamPeerCertVEndParamStart - 1] == '(' + ? token.substr(DownstreamPeerCertVEndParamStart, parameters_end) + : ""; + // Validate the input specifier here. The formatted string may be destined for a header, and + // should not contain invalid characters {NUL, LR, CF}. + if (std::regex_search(args, getDateFormatNewlinePattern())) { + throw EnvoyException("Invalid header configuration. Format string contains newline."); + } + formatters.emplace_back(FormatterProviderPtr{new DownstreamPeerCertVEndFormatter(args)}); } else if (absl::StartsWith(token, "GRPC_STATUS")) { formatters.emplace_back(FormatterProviderPtr{ new GrpcStatusFormatter("grpc-status", "", absl::optional())}); @@ -1152,5 +1180,61 @@ ProtobufWkt::Value StartTimeFormatter::formatValue( format(request_headers, response_headers, response_trailers, stream_info, local_reply_body)); } +DownstreamPeerCertVStartFormatter::DownstreamPeerCertVStartFormatter(const std::string& format) + : date_formatter_(format) {} + +absl::optional DownstreamPeerCertVStartFormatter::format( + const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, const Http::ResponseTrailerMap&, + const StreamInfo::StreamInfo& stream_info, absl::string_view) const { + const auto connection_info = stream_info.downstreamSslConnection(); + if (connection_info == nullptr) { + return absl::optional(); + } + const auto time_field = connection_info->validFromPeerCertificate(); + if (!time_field.has_value()) { + return absl::optional(); + } + if (date_formatter_.formatString().empty()) { + return AccessLogDateTimeFormatter::fromTime(time_field.value()); + } + return date_formatter_.fromTime(time_field.value()); +} + +ProtobufWkt::Value DownstreamPeerCertVStartFormatter::formatValue( + const Http::RequestHeaderMap& request_headers, const Http::ResponseHeaderMap& response_headers, + const Http::ResponseTrailerMap& response_trailers, const StreamInfo::StreamInfo& stream_info, + absl::string_view local_reply_body) const { + return ValueUtil::optionalStringValue( + format(request_headers, response_headers, response_trailers, stream_info, local_reply_body)); +} + +DownstreamPeerCertVEndFormatter::DownstreamPeerCertVEndFormatter(const std::string& format) + : date_formatter_(format) {} + +absl::optional DownstreamPeerCertVEndFormatter::format( + const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, const Http::ResponseTrailerMap&, + const StreamInfo::StreamInfo& stream_info, absl::string_view) const { + const auto connection_info = stream_info.downstreamSslConnection(); + if (connection_info == nullptr) { + return absl::optional(); + } + const auto time_field = connection_info->expirationPeerCertificate(); + if (!time_field.has_value()) { + return absl::optional(); + } + if (date_formatter_.formatString().empty()) { + return AccessLogDateTimeFormatter::fromTime(time_field.value()); + } + return date_formatter_.fromTime(time_field.value()); +} + +ProtobufWkt::Value DownstreamPeerCertVEndFormatter::formatValue( + const Http::RequestHeaderMap& request_headers, const Http::ResponseHeaderMap& response_headers, + const Http::ResponseTrailerMap& response_trailers, const StreamInfo::StreamInfo& stream_info, + absl::string_view local_reply_body) const { + return ValueUtil::optionalStringValue( + format(request_headers, response_headers, response_trailers, stream_info, local_reply_body)); +} + } // namespace Formatter } // namespace Envoy diff --git a/source/common/formatter/substitution_formatter.h b/source/common/formatter/substitution_formatter.h index 66ac8f83e283..632797b26a53 100644 --- a/source/common/formatter/substitution_formatter.h +++ b/source/common/formatter/substitution_formatter.h @@ -65,6 +65,9 @@ class SubstitutionFormatParser { static const size_t RespParamStart{sizeof("RESP(") - 1}; static const size_t TrailParamStart{sizeof("TRAILER(") - 1}; static const size_t StartTimeParamStart{sizeof("START_TIME(") - 1}; + static const size_t DownstreamPeerCertVStartParamStart{sizeof("DOWNSTREAM_PEER_CERT_V_START(") - + 1}; + static const size_t DownstreamPeerCertVEndParamStart{sizeof("DOWNSTREAM_PEER_CERT_V_END(") - 1}; }; /** @@ -397,5 +400,43 @@ class StartTimeFormatter : public FormatterProvider { const Envoy::DateFormatter date_formatter_; }; +/** + * FormatterProvider for downstream cert start time from ConnectionInfo. + */ +class DownstreamPeerCertVStartFormatter : public FormatterProvider { +public: + DownstreamPeerCertVStartFormatter(const std::string& format); + + // FormatterProvider + absl::optional format(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, + absl::string_view) const override; + ProtobufWkt::Value formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, + absl::string_view) const override; + +private: + const Envoy::DateFormatter date_formatter_; +}; + +/** + * FormatterProvider for downstream cert end time from ConnectionInfo. + */ +class DownstreamPeerCertVEndFormatter : public FormatterProvider { +public: + DownstreamPeerCertVEndFormatter(const std::string& format); + + // FormatterProvider + absl::optional format(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, + absl::string_view) const override; + ProtobufWkt::Value formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, + absl::string_view) const override; + +private: + const Envoy::DateFormatter date_formatter_; +}; + } // namespace Formatter } // namespace Envoy diff --git a/test/common/formatter/substitution_formatter_test.cc b/test/common/formatter/substitution_formatter_test.cc index 81a0ba4f7f36..75a58b505429 100644 --- a/test/common/formatter/substitution_formatter_test.cc +++ b/test/common/formatter/substitution_formatter_test.cc @@ -972,71 +972,6 @@ TEST(SubstitutionFormatterTest, streamInfoFormatter) { stream_info, body), ProtoEq(ValueUtil::nullValue())); } - { - StreamInfoFormatter upstream_format("DOWNSTREAM_PEER_CERT_V_START"); - auto connection_info = std::make_shared(); - absl::Time abslStartTime = - TestUtility::parseTime("Dec 18 01:50:34 2018 GMT", "%b %e %H:%M:%S %Y GMT"); - SystemTime startTime = absl::ToChronoTime(abslStartTime); - EXPECT_CALL(*connection_info, validFromPeerCertificate()).WillRepeatedly(Return(startTime)); - EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); - EXPECT_EQ("2018-12-18T01:50:34.000Z", - upstream_format.format(request_headers, response_headers, response_trailers, - stream_info, body)); - } - { - StreamInfoFormatter upstream_format("DOWNSTREAM_PEER_CERT_V_START"); - auto connection_info = std::make_shared(); - EXPECT_CALL(*connection_info, validFromPeerCertificate()).WillRepeatedly(Return(absl::nullopt)); - EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); - EXPECT_EQ(absl::nullopt, upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); - EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), - ProtoEq(ValueUtil::nullValue())); - } - { - EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(nullptr)); - StreamInfoFormatter upstream_format("DOWNSTREAM_PEER_CERT_V_START"); - EXPECT_EQ(absl::nullopt, upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); - EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), - ProtoEq(ValueUtil::nullValue())); - } - { - StreamInfoFormatter upstream_format("DOWNSTREAM_PEER_CERT_V_END"); - auto connection_info = std::make_shared(); - absl::Time abslEndTime = - TestUtility::parseTime("Dec 17 01:50:34 2020 GMT", "%b %e %H:%M:%S %Y GMT"); - SystemTime endTime = absl::ToChronoTime(abslEndTime); - EXPECT_CALL(*connection_info, expirationPeerCertificate()).WillRepeatedly(Return(endTime)); - EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); - EXPECT_EQ("2020-12-17T01:50:34.000Z", - upstream_format.format(request_headers, response_headers, response_trailers, - stream_info, body)); - } - { - StreamInfoFormatter upstream_format("DOWNSTREAM_PEER_CERT_V_END"); - auto connection_info = std::make_shared(); - EXPECT_CALL(*connection_info, expirationPeerCertificate()) - .WillRepeatedly(Return(absl::nullopt)); - EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); - EXPECT_EQ(absl::nullopt, upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); - EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), - ProtoEq(ValueUtil::nullValue())); - } - { - EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(nullptr)); - StreamInfoFormatter upstream_format("DOWNSTREAM_PEER_CERT_V_END"); - EXPECT_EQ(absl::nullopt, upstream_format.format(request_headers, response_headers, - response_trailers, stream_info, body)); - EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers, - stream_info, body), - ProtoEq(ValueUtil::nullValue())); - } { StreamInfoFormatter upstream_format("UPSTREAM_TRANSPORT_FAILURE_REASON"); std::string upstream_transport_failure_reason = "SSL error"; @@ -1456,6 +1391,117 @@ TEST(SubstitutionFormatterTest, FilterStateFormatter) { } } +TEST(SubstitutionFormatterTest, DownstreamPeerCertVStartFormatter) { + NiceMock stream_info; + Http::TestRequestHeaderMapImpl request_headers{{":method", "GET"}, {":path", "/"}}; + Http::TestResponseHeaderMapImpl response_headers; + Http::TestResponseTrailerMapImpl response_trailers; + std::string body; + + // No downstreamSslConnection + { + EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(nullptr)); + DownstreamPeerCertVStartFormatter cert_start_formart("%Y/%m/%d"); + EXPECT_EQ(absl::nullopt, cert_start_formart.format(request_headers, response_headers, + response_trailers, stream_info, body)); + EXPECT_THAT(cert_start_formart.formatValue(request_headers, response_headers, response_trailers, + stream_info, body), + ProtoEq(ValueUtil::nullValue())); + } + // No validFromPeerCertificate + { + DownstreamPeerCertVStartFormatter cert_start_formart("%Y/%m/%d"); + auto connection_info = std::make_shared(); + EXPECT_CALL(*connection_info, validFromPeerCertificate()).WillRepeatedly(Return(absl::nullopt)); + EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); + EXPECT_EQ(absl::nullopt, cert_start_formart.format(request_headers, response_headers, + response_trailers, stream_info, body)); + EXPECT_THAT(cert_start_formart.formatValue(request_headers, response_headers, response_trailers, + stream_info, body), + ProtoEq(ValueUtil::nullValue())); + } + // Default format string + { + DownstreamPeerCertVStartFormatter cert_start_format(""); + auto connection_info = std::make_shared(); + time_t test_epoch = 1522280158; + SystemTime time = std::chrono::system_clock::from_time_t(test_epoch); + EXPECT_CALL(*connection_info, validFromPeerCertificate()).WillRepeatedly(Return(time)); + EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); + EXPECT_EQ(AccessLogDateTimeFormatter::fromTime(time), + cert_start_format.format(request_headers, response_headers, response_trailers, + stream_info, body)); + } + // Custom format string + { + DownstreamPeerCertVStartFormatter cert_start_format("%b %e %H:%M:%S %Y %Z"); + auto connection_info = std::make_shared(); + time_t test_epoch = 1522280158; + SystemTime time = std::chrono::system_clock::from_time_t(test_epoch); + EXPECT_CALL(*connection_info, validFromPeerCertificate()).WillRepeatedly(Return(time)); + EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); + EXPECT_EQ("Mar 28 23:35:58 2018 UTC", + cert_start_format.format(request_headers, response_headers, response_trailers, + stream_info, body)); + } +} + +TEST(SubstitutionFormatterTest, DownstreamPeerCertVEndFormatter) { + NiceMock stream_info; + Http::TestRequestHeaderMapImpl request_headers{{":method", "GET"}, {":path", "/"}}; + Http::TestResponseHeaderMapImpl response_headers; + Http::TestResponseTrailerMapImpl response_trailers; + std::string body; + + // No downstreamSslConnection + { + EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(nullptr)); + DownstreamPeerCertVEndFormatter cert_end_format("%Y/%m/%d"); + EXPECT_EQ(absl::nullopt, cert_end_format.format(request_headers, response_headers, + response_trailers, stream_info, body)); + EXPECT_THAT(cert_end_format.formatValue(request_headers, response_headers, response_trailers, + stream_info, body), + ProtoEq(ValueUtil::nullValue())); + } + // No expirationPeerCertificate + { + DownstreamPeerCertVEndFormatter cert_end_format("%Y/%m/%d"); + auto connection_info = std::make_shared(); + EXPECT_CALL(*connection_info, expirationPeerCertificate()) + .WillRepeatedly(Return(absl::nullopt)); + EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); + EXPECT_EQ(absl::nullopt, cert_end_format.format(request_headers, response_headers, + response_trailers, stream_info, body)); + EXPECT_THAT(cert_end_format.formatValue(request_headers, response_headers, response_trailers, + stream_info, body), + ProtoEq(ValueUtil::nullValue())); + } + // Default format string + { + DownstreamPeerCertVEndFormatter cert_end_format(""); + auto connection_info = std::make_shared(); + time_t test_epoch = 1522280158; + SystemTime time = std::chrono::system_clock::from_time_t(test_epoch); + EXPECT_CALL(*connection_info, expirationPeerCertificate()).WillRepeatedly(Return(time)); + EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); + EXPECT_EQ(AccessLogDateTimeFormatter::fromTime(time), + cert_end_format.format(request_headers, response_headers, response_trailers, + stream_info, body)); + } + // Custom format string + { + DownstreamPeerCertVEndFormatter cert_end_format("%b %e %H:%M:%S %Y %Z"); + auto connection_info = std::make_shared(); + time_t test_epoch = 1522280158; + SystemTime time = std::chrono::system_clock::from_time_t(test_epoch); + EXPECT_CALL(*connection_info, expirationPeerCertificate()).WillRepeatedly(Return(time)); + EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); + EXPECT_EQ("Mar 28 23:35:58 2018 UTC", + cert_end_format.format(request_headers, response_headers, response_trailers, + stream_info, body)); + } +} + TEST(SubstitutionFormatterTest, StartTimeFormatter) { NiceMock stream_info; Http::TestRequestHeaderMapImpl request_headers{{":method", "GET"}, {":path", "/"}}; From ac5f69a2dae4b459298fc9fd9e21a92b1c6f23d2 Mon Sep 17 00:00:00 2001 From: John Esmet Date: Mon, 4 Jan 2021 19:55:20 +0000 Subject: [PATCH 02/10] Refactor to use a generic SystemTimeFormatter with a FieldExtractor callback Signed-off-by: John Esmet --- .../formatter/substitution_formatter.cc | 146 ++++++------------ .../common/formatter/substitution_formatter.h | 51 +++--- 2 files changed, 68 insertions(+), 129 deletions(-) diff --git a/source/common/formatter/substitution_formatter.cc b/source/common/formatter/substitution_formatter.cc index 26afa5437a51..7f769f6a1270 100644 --- a/source/common/formatter/substitution_formatter.cc +++ b/source/common/formatter/substitution_formatter.cc @@ -370,45 +370,24 @@ std::vector SubstitutionFormatParser::parse(const std::str formatters.push_back( std::make_unique(key, max_length, serialize_as_string)); } else if (absl::StartsWith(token, "START_TIME")) { - const size_t parameters_length = pos + StartTimeParamStart + 1; - const size_t parameters_end = command_end_position - parameters_length; - + const size_t parameters_end = token.length() - (StartTimeParamStart + 1); const std::string args = token[StartTimeParamStart - 1] == '(' ? token.substr(StartTimeParamStart, parameters_end) : ""; - // Validate the input specifier here. The formatted string may be destined for a header, and - // should not contain invalid characters {NUL, LR, CF}. - if (std::regex_search(args, getDateFormatNewlinePattern())) { - throw EnvoyException("Invalid header configuration. Format string contains newline."); - } formatters.emplace_back(FormatterProviderPtr{new StartTimeFormatter(args)}); } else if (absl::StartsWith(token, "DOWNSTREAM_PEER_CERT_V_START")) { - const size_t parameters_length = pos + DownstreamPeerCertVStartParamStart + 1; - const size_t parameters_end = command_end_position - parameters_length; - + const size_t parameters_end = token.length() - (DownstreamPeerCertVStartParamStart + 1); const std::string args = token[DownstreamPeerCertVStartParamStart - 1] == '(' ? token.substr(DownstreamPeerCertVStartParamStart, parameters_end) : ""; - // Validate the input specifier here. The formatted string may be destined for a header, and - // should not contain invalid characters {NUL, LR, CF}. - if (std::regex_search(args, getDateFormatNewlinePattern())) { - throw EnvoyException("Invalid header configuration. Format string contains newline."); - } formatters.emplace_back(FormatterProviderPtr{new DownstreamPeerCertVStartFormatter(args)}); } else if (absl::StartsWith(token, "DOWNSTREAM_PEER_CERT_V_END")) { - const size_t parameters_length = pos + DownstreamPeerCertVEndParamStart + 1; - const size_t parameters_end = command_end_position - parameters_length; - + const size_t parameters_end = token.length() - (DownstreamPeerCertVEndParamStart + 1); const std::string args = token[DownstreamPeerCertVEndParamStart - 1] == '(' ? token.substr(DownstreamPeerCertVEndParamStart, parameters_end) : ""; - // Validate the input specifier here. The formatted string may be destined for a header, and - // should not contain invalid characters {NUL, LR, CF}. - if (std::regex_search(args, getDateFormatNewlinePattern())) { - throw EnvoyException("Invalid header configuration. Format string contains newline."); - } formatters.emplace_back(FormatterProviderPtr{new DownstreamPeerCertVEndFormatter(args)}); } else if (absl::StartsWith(token, "GRPC_STATUS")) { formatters.emplace_back(FormatterProviderPtr{ @@ -798,26 +777,6 @@ StreamInfoFormatter::StreamInfoFormatter(const std::string& field_name) { [](const Ssl::ConnectionInfo& connection_info) { return connection_info.urlEncodedPemEncodedPeerCertificate(); }); - } else if (field_name == "DOWNSTREAM_PEER_CERT_V_START") { - field_extractor_ = std::make_unique( - [](const Ssl::ConnectionInfo& connection_info) { - absl::optional time = connection_info.validFromPeerCertificate(); - absl::optional result; - if (time.has_value()) { - result = AccessLogDateTimeFormatter::fromTime(time.value()); - } - return result; - }); - } else if (field_name == "DOWNSTREAM_PEER_CERT_V_END") { - field_extractor_ = std::make_unique( - [](const Ssl::ConnectionInfo& connection_info) { - absl::optional time = connection_info.expirationPeerCertificate(); - absl::optional result; - if (time.has_value()) { - result = AccessLogDateTimeFormatter::fromTime(time.value()); - } - return result; - }); } else if (field_name == "UPSTREAM_TRANSPORT_FAILURE_REASON") { field_extractor_ = std::make_unique( [](const StreamInfo::StreamInfo& stream_info) { @@ -1159,66 +1118,53 @@ ProtobufWkt::Value FilterStateFormatter::formatValue(const Http::RequestHeaderMa return val; } -StartTimeFormatter::StartTimeFormatter(const std::string& format) : date_formatter_(format) {} - -absl::optional StartTimeFormatter::format(const Http::RequestHeaderMap&, - const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, - const StreamInfo::StreamInfo& stream_info, - absl::string_view) const { - if (date_formatter_.formatString().empty()) { - return AccessLogDateTimeFormatter::fromTime(stream_info.startTime()); - } - return date_formatter_.fromTime(stream_info.startTime()); -} - -ProtobufWkt::Value StartTimeFormatter::formatValue( - const Http::RequestHeaderMap& request_headers, const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers, const StreamInfo::StreamInfo& stream_info, - absl::string_view local_reply_body) const { - return ValueUtil::optionalStringValue( - format(request_headers, response_headers, response_trailers, stream_info, local_reply_body)); -} +// A SystemTime formatter that extracts the startTime from StreamInfo +StartTimeFormatter::StartTimeFormatter(const std::string& format) + : SystemTimeFormatter( + format, std::make_unique( + [](const StreamInfo::StreamInfo& stream_info) -> absl::optional { + return stream_info.startTime(); + })) {} +// A SystemTime formatter that optionally extracts the start date from the downstream peer's +// certificate DownstreamPeerCertVStartFormatter::DownstreamPeerCertVStartFormatter(const std::string& format) - : date_formatter_(format) {} - -absl::optional DownstreamPeerCertVStartFormatter::format( - const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, const Http::ResponseTrailerMap&, - const StreamInfo::StreamInfo& stream_info, absl::string_view) const { - const auto connection_info = stream_info.downstreamSslConnection(); - if (connection_info == nullptr) { - return absl::optional(); - } - const auto time_field = connection_info->validFromPeerCertificate(); - if (!time_field.has_value()) { - return absl::optional(); - } - if (date_formatter_.formatString().empty()) { - return AccessLogDateTimeFormatter::fromTime(time_field.value()); + : SystemTimeFormatter( + format, std::make_unique( + [](const StreamInfo::StreamInfo& stream_info) -> absl::optional { + const auto connection_info = stream_info.downstreamSslConnection(); + return connection_info != nullptr + ? connection_info->validFromPeerCertificate() + : absl::optional(); + })) {} + +// A SystemTime formatter that optionally extracts the end date from the downstream peer's +// certificate +DownstreamPeerCertVEndFormatter::DownstreamPeerCertVEndFormatter(const std::string& format) + : SystemTimeFormatter( + format, std::make_unique( + [](const StreamInfo::StreamInfo& stream_info) -> absl::optional { + const auto connection_info = stream_info.downstreamSslConnection(); + return connection_info != nullptr + ? connection_info->expirationPeerCertificate() + : absl::optional(); + })) {} + +SystemTimeFormatter::SystemTimeFormatter(const std::string& format, TimeFieldExtractorPtr f) + : date_formatter_(format), time_field_extractor_(std::move(f)) { + // Validate the input specifier here. The formatted string may be destined for a header, and + // should not contain invalid characters {NUL, LR, CF}. + if (std::regex_search(format, getDateFormatNewlinePattern())) { + throw EnvoyException("Invalid header configuration. Format string contains newline."); } - return date_formatter_.fromTime(time_field.value()); } -ProtobufWkt::Value DownstreamPeerCertVStartFormatter::formatValue( - const Http::RequestHeaderMap& request_headers, const Http::ResponseHeaderMap& response_headers, - const Http::ResponseTrailerMap& response_trailers, const StreamInfo::StreamInfo& stream_info, - absl::string_view local_reply_body) const { - return ValueUtil::optionalStringValue( - format(request_headers, response_headers, response_trailers, stream_info, local_reply_body)); -} - -DownstreamPeerCertVEndFormatter::DownstreamPeerCertVEndFormatter(const std::string& format) - : date_formatter_(format) {} - -absl::optional DownstreamPeerCertVEndFormatter::format( - const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, const Http::ResponseTrailerMap&, - const StreamInfo::StreamInfo& stream_info, absl::string_view) const { - const auto connection_info = stream_info.downstreamSslConnection(); - if (connection_info == nullptr) { - return absl::optional(); - } - const auto time_field = connection_info->expirationPeerCertificate(); +absl::optional SystemTimeFormatter::format(const Http::RequestHeaderMap&, + const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, + const StreamInfo::StreamInfo& stream_info, + absl::string_view) const { + const auto time_field = (*time_field_extractor_)(stream_info); if (!time_field.has_value()) { return absl::optional(); } @@ -1228,7 +1174,7 @@ absl::optional DownstreamPeerCertVEndFormatter::format( return date_formatter_.fromTime(time_field.value()); } -ProtobufWkt::Value DownstreamPeerCertVEndFormatter::formatValue( +ProtobufWkt::Value SystemTimeFormatter::formatValue( const Http::RequestHeaderMap& request_headers, const Http::ResponseHeaderMap& response_headers, const Http::ResponseTrailerMap& response_trailers, const StreamInfo::StreamInfo& stream_info, absl::string_view local_reply_body) const { diff --git a/source/common/formatter/substitution_formatter.h b/source/common/formatter/substitution_formatter.h index 632797b26a53..42fbb45f666c 100644 --- a/source/common/formatter/substitution_formatter.h +++ b/source/common/formatter/substitution_formatter.h @@ -382,11 +382,15 @@ class FilterStateFormatter : public FormatterProvider { }; /** - * FormatterProvider for request start time from StreamInfo. + * Base FormatterProvider for system times from StreamInfo. */ -class StartTimeFormatter : public FormatterProvider { +class SystemTimeFormatter : public FormatterProvider { public: - StartTimeFormatter(const std::string& format); + using TimeFieldExtractor = + std::function(const StreamInfo::StreamInfo& stream_info)>; + using TimeFieldExtractorPtr = std::unique_ptr; + + SystemTimeFormatter(const std::string& format, TimeFieldExtractorPtr f); // FormatterProvider absl::optional format(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, @@ -398,44 +402,33 @@ class StartTimeFormatter : public FormatterProvider { private: const Envoy::DateFormatter date_formatter_; + const TimeFieldExtractorPtr time_field_extractor_; }; /** - * FormatterProvider for downstream cert start time from ConnectionInfo. + * SystemTimeFormatter (FormatterProvider) for request start time from StreamInfo. */ -class DownstreamPeerCertVStartFormatter : public FormatterProvider { +class StartTimeFormatter : public SystemTimeFormatter { public: - DownstreamPeerCertVStartFormatter(const std::string& format); - - // FormatterProvider - absl::optional format(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, - absl::string_view) const override; - ProtobufWkt::Value formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, - absl::string_view) const override; + StartTimeFormatter(const std::string& format); +}; -private: - const Envoy::DateFormatter date_formatter_; +/** + * SystemTimeFormatter (FormatterProvider) for downstream cert start time from the StreamInfo's + * ConnectionInfo. + */ +class DownstreamPeerCertVStartFormatter : public SystemTimeFormatter { +public: + DownstreamPeerCertVStartFormatter(const std::string& format); }; /** - * FormatterProvider for downstream cert end time from ConnectionInfo. + * SystemTimeFormatter (FormatterProvider) for downstream cert end time from the StreamInfo's + * ConnectionInfo. */ -class DownstreamPeerCertVEndFormatter : public FormatterProvider { +class DownstreamPeerCertVEndFormatter : public SystemTimeFormatter { public: DownstreamPeerCertVEndFormatter(const std::string& format); - - // FormatterProvider - absl::optional format(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, - absl::string_view) const override; - ProtobufWkt::Value formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, - const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, - absl::string_view) const override; - -private: - const Envoy::DateFormatter date_formatter_; }; } // namespace Formatter From b1ba7dd24b4e82205b48bc806407a6be09e93625 Mon Sep 17 00:00:00 2001 From: John Esmet Date: Mon, 4 Jan 2021 21:16:12 +0000 Subject: [PATCH 03/10] Add docs Signed-off-by: John Esmet --- docs/root/configuration/observability/access_log/usage.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/root/configuration/observability/access_log/usage.rst b/docs/root/configuration/observability/access_log/usage.rst index 7d4b402fbd61..6f2fd7f6ad3a 100644 --- a/docs/root/configuration/observability/access_log/usage.rst +++ b/docs/root/configuration/observability/access_log/usage.rst @@ -541,12 +541,18 @@ The following command operators are supported: TCP The validity start date of the client certificate used to establish the downstream TLS connection. + DOWNSTREAM_PEER_CERT_V_START can be customized using a `format string `_. + See START_TIME for additional format specifiers and examples. + %DOWNSTREAM_PEER_CERT_V_END% HTTP The validity end date of the client certificate used to establish the downstream TLS connection. TCP The validity end date of the client certificate used to establish the downstream TLS connection. + DOWNSTREAM_PEER_CERT_V_END can be customized using a `format string `_. + See START_TIME for additional format specifiers and examples. + %HOSTNAME% The system hostname. From 5c9848318e6a083fe6528a48f4ca9c565219bf77 Mon Sep 17 00:00:00 2001 From: John Esmet Date: Mon, 4 Jan 2021 21:19:57 +0000 Subject: [PATCH 04/10] Add changelog Signed-off-by: John Esmet --- docs/root/version_history/current.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/root/version_history/current.rst b/docs/root/version_history/current.rst index 452cb21bcfcf..a608c69f7169 100644 --- a/docs/root/version_history/current.rst +++ b/docs/root/version_history/current.rst @@ -67,6 +67,7 @@ New Features * config: added ability to flush stats when the admin's :ref:`/stats endpoint ` is hit instead of on a timer via :ref:`stats_flush_on_admin `. * config: added new runtime feature `envoy.features.enable_all_deprecated_features` that allows the use of all deprecated features. * formatter: added new :ref:`text_format_source ` field to support format strings both inline and from a file. +* formatter: added support for custom date formatting to %DOWNSTREAM_PEER_CERT_V_START% and %DOWNSTREAM_PEER_CERT_V_END%, similar to %START_TIME%. * grpc: implemented header value syntax support when defining :ref:`initial metadata ` for gRPC-based `ext_authz` :ref:`HTTP ` and :ref:`network ` filters, and :ref:`ratelimit ` filters. * grpc-json: added support for configuring :ref:`unescaping behavior ` for path components. * hds: added support for delta updates in the :ref:`HealthCheckSpecifier `, making only the Endpoints and Health Checkers that changed be reconstructed on receiving a new message, rather than the entire HDS. From 55fa349bb95bb3bf38addf0ba6de0d842996df75 Mon Sep 17 00:00:00 2001 From: John Esmet Date: Mon, 4 Jan 2021 22:07:10 +0000 Subject: [PATCH 05/10] More test cases Signed-off-by: John Esmet --- .../formatter/substitution_formatter_test.cc | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/test/common/formatter/substitution_formatter_test.cc b/test/common/formatter/substitution_formatter_test.cc index 75a58b505429..204929f5ebbc 100644 --- a/test/common/formatter/substitution_formatter_test.cc +++ b/test/common/formatter/substitution_formatter_test.cc @@ -2195,6 +2195,7 @@ TEST(SubstitutionFormatterTest, CompositeFormatterSuccess) { } { + // Various START_TIME formats const std::string format = "%START_TIME(%Y/%m/%d)%|%START_TIME(%s)%|%START_TIME(bad_format)%|" "%START_TIME%|%START_TIME(%f.%1f.%2f.%3f)%"; @@ -2209,6 +2210,46 @@ TEST(SubstitutionFormatterTest, CompositeFormatterSuccess) { formatter.format(request_header, response_header, response_trailer, stream_info, body)); } + { + // Various DOWNSTREAM_PEER_CERT_V_START formats (similar to START_TIME) + const std::string format = + "%DOWNSTREAM_PEER_CERT_V_START(%Y/%m/" + "%d)%|%DOWNSTREAM_PEER_CERT_V_START(%s)%|%DOWNSTREAM_PEER_CERT_V_START(bad_format)%|" + "%DOWNSTREAM_PEER_CERT_V_START%|%DOWNSTREAM_PEER_CERT_V_START(%f.%1f.%2f.%3f)%"; + + time_t expected_time_in_epoch = 1522280158; + auto connection_info = std::make_shared(); + SystemTime time = std::chrono::system_clock::from_time_t(expected_time_in_epoch); + EXPECT_CALL(*connection_info, validFromPeerCertificate()).WillRepeatedly(Return(time)); + EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); + FormatterImpl formatter(format, false); + + EXPECT_EQ( + fmt::format("2018/03/28|{}|bad_format|2018-03-28T23:35:58.000Z|000000000.0.00.000", + expected_time_in_epoch), + formatter.format(request_header, response_header, response_trailer, stream_info, body)); + } + + { + // Various DOWNSTREAM_PEER_CERT_V_END formats (similar to START_TIME) + const std::string format = + "%DOWNSTREAM_PEER_CERT_V_END(%Y/%m/" + "%d)%|%DOWNSTREAM_PEER_CERT_V_END(%s)%|%DOWNSTREAM_PEER_CERT_V_END(bad_format)%|" + "%DOWNSTREAM_PEER_CERT_V_END%|%DOWNSTREAM_PEER_CERT_V_END(%f.%1f.%2f.%3f)%"; + + time_t expected_time_in_epoch = 1522280158; + auto connection_info = std::make_shared(); + SystemTime time = std::chrono::system_clock::from_time_t(expected_time_in_epoch); + EXPECT_CALL(*connection_info, expirationPeerCertificate()).WillRepeatedly(Return(time)); + EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); + FormatterImpl formatter(format, false); + + EXPECT_EQ( + fmt::format("2018/03/28|{}|bad_format|2018-03-28T23:35:58.000Z|000000000.0.00.000", + expected_time_in_epoch), + formatter.format(request_header, response_header, response_trailer, stream_info, body)); + } + { // This tests the beginning of time. const std::string format = "%START_TIME(%Y/%m/%d)%|%START_TIME(%s)%|%START_TIME(bad_format)%|" From 42cb29013a0d89566588cda6f931f8ff401456f4 Mon Sep 17 00:00:00 2001 From: John Esmet Date: Mon, 4 Jan 2021 22:20:59 +0000 Subject: [PATCH 06/10] Clarify variable names Signed-off-by: John Esmet --- source/common/formatter/substitution_formatter.cc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/source/common/formatter/substitution_formatter.cc b/source/common/formatter/substitution_formatter.cc index 7f769f6a1270..882734ff5535 100644 --- a/source/common/formatter/substitution_formatter.cc +++ b/source/common/formatter/substitution_formatter.cc @@ -370,23 +370,23 @@ std::vector SubstitutionFormatParser::parse(const std::str formatters.push_back( std::make_unique(key, max_length, serialize_as_string)); } else if (absl::StartsWith(token, "START_TIME")) { - const size_t parameters_end = token.length() - (StartTimeParamStart + 1); + const size_t parameters_length = token.length() - (StartTimeParamStart + 1); const std::string args = token[StartTimeParamStart - 1] == '(' - ? token.substr(StartTimeParamStart, parameters_end) + ? token.substr(StartTimeParamStart, parameters_length) : ""; formatters.emplace_back(FormatterProviderPtr{new StartTimeFormatter(args)}); } else if (absl::StartsWith(token, "DOWNSTREAM_PEER_CERT_V_START")) { - const size_t parameters_end = token.length() - (DownstreamPeerCertVStartParamStart + 1); + const size_t parameters_length = token.length() - (DownstreamPeerCertVStartParamStart + 1); const std::string args = token[DownstreamPeerCertVStartParamStart - 1] == '(' - ? token.substr(DownstreamPeerCertVStartParamStart, parameters_end) + ? token.substr(DownstreamPeerCertVStartParamStart, parameters_length) : ""; formatters.emplace_back(FormatterProviderPtr{new DownstreamPeerCertVStartFormatter(args)}); } else if (absl::StartsWith(token, "DOWNSTREAM_PEER_CERT_V_END")) { - const size_t parameters_end = token.length() - (DownstreamPeerCertVEndParamStart + 1); + const size_t parameters_length = token.length() - (DownstreamPeerCertVEndParamStart + 1); const std::string args = token[DownstreamPeerCertVEndParamStart - 1] == '(' - ? token.substr(DownstreamPeerCertVEndParamStart, parameters_end) + ? token.substr(DownstreamPeerCertVEndParamStart, parameters_length) : ""; formatters.emplace_back(FormatterProviderPtr{new DownstreamPeerCertVEndFormatter(args)}); } else if (absl::StartsWith(token, "GRPC_STATUS")) { From 21bb32a3c049d37208a6bacb96c857d84ec487ff Mon Sep 17 00:00:00 2001 From: John Esmet Date: Mon, 4 Jan 2021 22:29:53 +0000 Subject: [PATCH 07/10] Clarify a function name Signed-off-by: John Esmet --- source/common/formatter/substitution_formatter.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/common/formatter/substitution_formatter.cc b/source/common/formatter/substitution_formatter.cc index 882734ff5535..50d4246b5eaf 100644 --- a/source/common/formatter/substitution_formatter.cc +++ b/source/common/formatter/substitution_formatter.cc @@ -44,7 +44,7 @@ void truncate(std::string& str, absl::optional max_length) { } // Matches newline pattern in a system time format string (e.g. start time) -const std::regex& getDateFormatNewlinePattern() { +const std::regex& getSystemTimeFormatNewlinePattern() { CONSTRUCT_ON_FIRST_USE(std::regex, "%[-_0^#]*[1-9]*(E|O)?n"); } const std::regex& getNewlinePattern() { CONSTRUCT_ON_FIRST_USE(std::regex, "\n"); } @@ -1154,7 +1154,7 @@ SystemTimeFormatter::SystemTimeFormatter(const std::string& format, TimeFieldExt : date_formatter_(format), time_field_extractor_(std::move(f)) { // Validate the input specifier here. The formatted string may be destined for a header, and // should not contain invalid characters {NUL, LR, CF}. - if (std::regex_search(format, getDateFormatNewlinePattern())) { + if (std::regex_search(format, getSystemTimeFormatNewlinePattern())) { throw EnvoyException("Invalid header configuration. Format string contains newline."); } } @@ -1166,7 +1166,7 @@ absl::optional SystemTimeFormatter::format(const Http::RequestHeade absl::string_view) const { const auto time_field = (*time_field_extractor_)(stream_info); if (!time_field.has_value()) { - return absl::optional(); + return absl::nullopt; } if (date_formatter_.formatString().empty()) { return AccessLogDateTimeFormatter::fromTime(time_field.value()); From 091e1351b7997717d51923103b21b366e3231786 Mon Sep 17 00:00:00 2001 From: John Esmet Date: Tue, 5 Jan 2021 20:24:22 +0000 Subject: [PATCH 08/10] Add links to docs Signed-off-by: John Esmet --- .../root/configuration/observability/access_log/usage.rst | 8 ++++++-- docs/root/version_history/current.rst | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/root/configuration/observability/access_log/usage.rst b/docs/root/configuration/observability/access_log/usage.rst index 6f2fd7f6ad3a..be837d979360 100644 --- a/docs/root/configuration/observability/access_log/usage.rst +++ b/docs/root/configuration/observability/access_log/usage.rst @@ -535,6 +535,8 @@ The following command operators are supported: TCP The client certificate in the URL-encoded PEM format used to establish the downstream TLS connection. +.. _config_access_log_format_downstream_peer_cert_v_start + %DOWNSTREAM_PEER_CERT_V_START% HTTP The validity start date of the client certificate used to establish the downstream TLS connection. @@ -542,7 +544,9 @@ The following command operators are supported: The validity start date of the client certificate used to establish the downstream TLS connection. DOWNSTREAM_PEER_CERT_V_START can be customized using a `format string `_. - See START_TIME for additional format specifiers and examples. + See :ref:`START_TIME ` for additional format specifiers and examples. + +.. _config_access_log_format_downstream_peer_cert_v_end %DOWNSTREAM_PEER_CERT_V_END% HTTP @@ -551,7 +555,7 @@ The following command operators are supported: The validity end date of the client certificate used to establish the downstream TLS connection. DOWNSTREAM_PEER_CERT_V_END can be customized using a `format string `_. - See START_TIME for additional format specifiers and examples. + See :ref:`START_TIME ` for additional format specifiers and examples. %HOSTNAME% The system hostname. diff --git a/docs/root/version_history/current.rst b/docs/root/version_history/current.rst index a608c69f7169..3204d147ddaf 100644 --- a/docs/root/version_history/current.rst +++ b/docs/root/version_history/current.rst @@ -67,7 +67,7 @@ New Features * config: added ability to flush stats when the admin's :ref:`/stats endpoint ` is hit instead of on a timer via :ref:`stats_flush_on_admin `. * config: added new runtime feature `envoy.features.enable_all_deprecated_features` that allows the use of all deprecated features. * formatter: added new :ref:`text_format_source ` field to support format strings both inline and from a file. -* formatter: added support for custom date formatting to %DOWNSTREAM_PEER_CERT_V_START% and %DOWNSTREAM_PEER_CERT_V_END%, similar to %START_TIME%. +* formatter: added support for custom date formatting to :ref:`%DOWNSTREAM_PEER_CERT_V_START% ` and :ref:`%DOWNSTREAM_PEER_CERT_V_END% `, similar to :ref:`%START_TIME% `. * grpc: implemented header value syntax support when defining :ref:`initial metadata ` for gRPC-based `ext_authz` :ref:`HTTP ` and :ref:`network ` filters, and :ref:`ratelimit ` filters. * grpc-json: added support for configuring :ref:`unescaping behavior ` for path components. * hds: added support for delta updates in the :ref:`HealthCheckSpecifier `, making only the Endpoints and Health Checkers that changed be reconstructed on receiving a new message, rather than the entire HDS. From 7b5326d9ac09ff9fd8ec3a2ca62f5fac44981e98 Mon Sep 17 00:00:00 2001 From: John Esmet Date: Tue, 5 Jan 2021 21:09:21 +0000 Subject: [PATCH 09/10] Improve how we parse system time formats across implementations Signed-off-by: John Esmet --- .../formatter/substitution_formatter.cc | 77 +++++++++---------- .../common/formatter/substitution_formatter.h | 14 +++- .../formatter/substitution_formatter_test.cc | 22 +++--- 3 files changed, 58 insertions(+), 55 deletions(-) diff --git a/source/common/formatter/substitution_formatter.cc b/source/common/formatter/substitution_formatter.cc index 50d4246b5eaf..d1351cf7ac80 100644 --- a/source/common/formatter/substitution_formatter.cc +++ b/source/common/formatter/substitution_formatter.cc @@ -370,25 +370,11 @@ std::vector SubstitutionFormatParser::parse(const std::str formatters.push_back( std::make_unique(key, max_length, serialize_as_string)); } else if (absl::StartsWith(token, "START_TIME")) { - const size_t parameters_length = token.length() - (StartTimeParamStart + 1); - const std::string args = token[StartTimeParamStart - 1] == '(' - ? token.substr(StartTimeParamStart, parameters_length) - : ""; - formatters.emplace_back(FormatterProviderPtr{new StartTimeFormatter(args)}); + formatters.emplace_back(FormatterProviderPtr{new StartTimeFormatter(token)}); } else if (absl::StartsWith(token, "DOWNSTREAM_PEER_CERT_V_START")) { - const size_t parameters_length = token.length() - (DownstreamPeerCertVStartParamStart + 1); - const std::string args = - token[DownstreamPeerCertVStartParamStart - 1] == '(' - ? token.substr(DownstreamPeerCertVStartParamStart, parameters_length) - : ""; - formatters.emplace_back(FormatterProviderPtr{new DownstreamPeerCertVStartFormatter(args)}); + formatters.emplace_back(FormatterProviderPtr{new DownstreamPeerCertVStartFormatter(token)}); } else if (absl::StartsWith(token, "DOWNSTREAM_PEER_CERT_V_END")) { - const size_t parameters_length = token.length() - (DownstreamPeerCertVEndParamStart + 1); - const std::string args = - token[DownstreamPeerCertVEndParamStart - 1] == '(' - ? token.substr(DownstreamPeerCertVEndParamStart, parameters_length) - : ""; - formatters.emplace_back(FormatterProviderPtr{new DownstreamPeerCertVEndFormatter(args)}); + formatters.emplace_back(FormatterProviderPtr{new DownstreamPeerCertVEndFormatter(token)}); } else if (absl::StartsWith(token, "GRPC_STATUS")) { formatters.emplace_back(FormatterProviderPtr{ new GrpcStatusFormatter("grpc-status", "", absl::optional())}); @@ -1118,37 +1104,46 @@ ProtobufWkt::Value FilterStateFormatter::formatValue(const Http::RequestHeaderMa return val; } -// A SystemTime formatter that extracts the startTime from StreamInfo -StartTimeFormatter::StartTimeFormatter(const std::string& format) +// Given a token, extract the command string between parenthesis if it exists. +std::string SystemTimeFormatter::parseFormat(const std::string& token, size_t parameters_start) { + const size_t parameters_length = token.length() - (parameters_start + 1); + return token[parameters_start - 1] == '(' ? token.substr(parameters_start, parameters_length) + : ""; +} + +// A SystemTime formatter that extracts the startTime from StreamInfo. Must be provided +// an access log token that starts with `START_TIME`. +StartTimeFormatter::StartTimeFormatter(const std::string& token) : SystemTimeFormatter( - format, std::make_unique( - [](const StreamInfo::StreamInfo& stream_info) -> absl::optional { - return stream_info.startTime(); - })) {} + parseFormat(token, sizeof("START_TIME(") - 1), + std::make_unique( + [](const StreamInfo::StreamInfo& stream_info) -> absl::optional { + return stream_info.startTime(); + })) {} // A SystemTime formatter that optionally extracts the start date from the downstream peer's -// certificate -DownstreamPeerCertVStartFormatter::DownstreamPeerCertVStartFormatter(const std::string& format) +// certificate. Must be provided an access log token that starts with `DOWNSTREAM_PEER_CERT_V_START` +DownstreamPeerCertVStartFormatter::DownstreamPeerCertVStartFormatter(const std::string& token) : SystemTimeFormatter( - format, std::make_unique( - [](const StreamInfo::StreamInfo& stream_info) -> absl::optional { - const auto connection_info = stream_info.downstreamSslConnection(); - return connection_info != nullptr - ? connection_info->validFromPeerCertificate() - : absl::optional(); - })) {} + parseFormat(token, sizeof("DOWNSTREAM_PEER_CERT_V_START(") - 1), + std::make_unique( + [](const StreamInfo::StreamInfo& stream_info) -> absl::optional { + const auto connection_info = stream_info.downstreamSslConnection(); + return connection_info != nullptr ? connection_info->validFromPeerCertificate() + : absl::optional(); + })) {} // A SystemTime formatter that optionally extracts the end date from the downstream peer's -// certificate -DownstreamPeerCertVEndFormatter::DownstreamPeerCertVEndFormatter(const std::string& format) +// certificate. Must be provided an access log token that starts with `DOWNSTREAM_PEER_CERT_V_END` +DownstreamPeerCertVEndFormatter::DownstreamPeerCertVEndFormatter(const std::string& token) : SystemTimeFormatter( - format, std::make_unique( - [](const StreamInfo::StreamInfo& stream_info) -> absl::optional { - const auto connection_info = stream_info.downstreamSslConnection(); - return connection_info != nullptr - ? connection_info->expirationPeerCertificate() - : absl::optional(); - })) {} + parseFormat(token, sizeof("DOWNSTREAM_PEER_CERT_V_END(") - 1), + std::make_unique( + [](const StreamInfo::StreamInfo& stream_info) -> absl::optional { + const auto connection_info = stream_info.downstreamSslConnection(); + return connection_info != nullptr ? connection_info->expirationPeerCertificate() + : absl::optional(); + })) {} SystemTimeFormatter::SystemTimeFormatter(const std::string& format, TimeFieldExtractorPtr f) : date_formatter_(format), time_field_extractor_(std::move(f)) { diff --git a/source/common/formatter/substitution_formatter.h b/source/common/formatter/substitution_formatter.h index 42fbb45f666c..dc0d39689e9b 100644 --- a/source/common/formatter/substitution_formatter.h +++ b/source/common/formatter/substitution_formatter.h @@ -64,10 +64,6 @@ class SubstitutionFormatParser { static const size_t ReqParamStart{sizeof("REQ(") - 1}; static const size_t RespParamStart{sizeof("RESP(") - 1}; static const size_t TrailParamStart{sizeof("TRAILER(") - 1}; - static const size_t StartTimeParamStart{sizeof("START_TIME(") - 1}; - static const size_t DownstreamPeerCertVStartParamStart{sizeof("DOWNSTREAM_PEER_CERT_V_START(") - - 1}; - static const size_t DownstreamPeerCertVEndParamStart{sizeof("DOWNSTREAM_PEER_CERT_V_END(") - 1}; }; /** @@ -400,6 +396,16 @@ class SystemTimeFormatter : public FormatterProvider { const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, absl::string_view) const override; +protected: + // Given an access log token, attempt to parse out the format string between parenthesis. + // + // @param token The access log token, e.g. `START_TIME` or `START_TIME(...)` + // @param parameters_start The index of the first character where the parameters parenthesis would + // begin if it exists. Must not be out of bounds of `token` or its NUL + // char. + // @return The format string between parenthesis, or an empty string if none exists. + static std::string parseFormat(const std::string& token, size_t parameters_start); + private: const Envoy::DateFormatter date_formatter_; const TimeFieldExtractorPtr time_field_extractor_; diff --git a/test/common/formatter/substitution_formatter_test.cc b/test/common/formatter/substitution_formatter_test.cc index 204929f5ebbc..9e83c2d4fe95 100644 --- a/test/common/formatter/substitution_formatter_test.cc +++ b/test/common/formatter/substitution_formatter_test.cc @@ -1401,7 +1401,7 @@ TEST(SubstitutionFormatterTest, DownstreamPeerCertVStartFormatter) { // No downstreamSslConnection { EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(nullptr)); - DownstreamPeerCertVStartFormatter cert_start_formart("%Y/%m/%d"); + DownstreamPeerCertVStartFormatter cert_start_formart("DOWNSTREAM_PEER_CERT_V_START(%Y/%m/%d)"); EXPECT_EQ(absl::nullopt, cert_start_formart.format(request_headers, response_headers, response_trailers, stream_info, body)); EXPECT_THAT(cert_start_formart.formatValue(request_headers, response_headers, response_trailers, @@ -1410,7 +1410,7 @@ TEST(SubstitutionFormatterTest, DownstreamPeerCertVStartFormatter) { } // No validFromPeerCertificate { - DownstreamPeerCertVStartFormatter cert_start_formart("%Y/%m/%d"); + DownstreamPeerCertVStartFormatter cert_start_formart("DOWNSTREAM_PEER_CERT_V_START(%Y/%m/%d)"); auto connection_info = std::make_shared(); EXPECT_CALL(*connection_info, validFromPeerCertificate()).WillRepeatedly(Return(absl::nullopt)); EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); @@ -1422,7 +1422,7 @@ TEST(SubstitutionFormatterTest, DownstreamPeerCertVStartFormatter) { } // Default format string { - DownstreamPeerCertVStartFormatter cert_start_format(""); + DownstreamPeerCertVStartFormatter cert_start_format("DOWNSTREAM_PEER_CERT_V_START"); auto connection_info = std::make_shared(); time_t test_epoch = 1522280158; SystemTime time = std::chrono::system_clock::from_time_t(test_epoch); @@ -1434,7 +1434,8 @@ TEST(SubstitutionFormatterTest, DownstreamPeerCertVStartFormatter) { } // Custom format string { - DownstreamPeerCertVStartFormatter cert_start_format("%b %e %H:%M:%S %Y %Z"); + DownstreamPeerCertVStartFormatter cert_start_format( + "DOWNSTREAM_PEER_CERT_V_START(%b %e %H:%M:%S %Y %Z)"); auto connection_info = std::make_shared(); time_t test_epoch = 1522280158; SystemTime time = std::chrono::system_clock::from_time_t(test_epoch); @@ -1456,7 +1457,7 @@ TEST(SubstitutionFormatterTest, DownstreamPeerCertVEndFormatter) { // No downstreamSslConnection { EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(nullptr)); - DownstreamPeerCertVEndFormatter cert_end_format("%Y/%m/%d"); + DownstreamPeerCertVEndFormatter cert_end_format("DOWNSTREAM_PEER_CERT_V_END(%Y/%m/%d)"); EXPECT_EQ(absl::nullopt, cert_end_format.format(request_headers, response_headers, response_trailers, stream_info, body)); EXPECT_THAT(cert_end_format.formatValue(request_headers, response_headers, response_trailers, @@ -1465,7 +1466,7 @@ TEST(SubstitutionFormatterTest, DownstreamPeerCertVEndFormatter) { } // No expirationPeerCertificate { - DownstreamPeerCertVEndFormatter cert_end_format("%Y/%m/%d"); + DownstreamPeerCertVEndFormatter cert_end_format("DOWNSTREAM_PEER_CERT_V_END(%Y/%m/%d)"); auto connection_info = std::make_shared(); EXPECT_CALL(*connection_info, expirationPeerCertificate()) .WillRepeatedly(Return(absl::nullopt)); @@ -1478,7 +1479,7 @@ TEST(SubstitutionFormatterTest, DownstreamPeerCertVEndFormatter) { } // Default format string { - DownstreamPeerCertVEndFormatter cert_end_format(""); + DownstreamPeerCertVEndFormatter cert_end_format("DOWNSTREAM_PEER_CERT_V_END"); auto connection_info = std::make_shared(); time_t test_epoch = 1522280158; SystemTime time = std::chrono::system_clock::from_time_t(test_epoch); @@ -1490,7 +1491,8 @@ TEST(SubstitutionFormatterTest, DownstreamPeerCertVEndFormatter) { } // Custom format string { - DownstreamPeerCertVEndFormatter cert_end_format("%b %e %H:%M:%S %Y %Z"); + DownstreamPeerCertVEndFormatter cert_end_format( + "DOWNSTREAM_PEER_CERT_V_END(%b %e %H:%M:%S %Y %Z)"); auto connection_info = std::make_shared(); time_t test_epoch = 1522280158; SystemTime time = std::chrono::system_clock::from_time_t(test_epoch); @@ -1510,7 +1512,7 @@ TEST(SubstitutionFormatterTest, StartTimeFormatter) { std::string body; { - StartTimeFormatter start_time_format("%Y/%m/%d"); + StartTimeFormatter start_time_format("START_TIME(%Y/%m/%d)"); time_t test_epoch = 1522280158; SystemTime time = std::chrono::system_clock::from_time_t(test_epoch); EXPECT_CALL(stream_info, startTime()).WillRepeatedly(Return(time)); @@ -1522,7 +1524,7 @@ TEST(SubstitutionFormatterTest, StartTimeFormatter) { } { - StartTimeFormatter start_time_format(""); + StartTimeFormatter start_time_format("START_TIME"); SystemTime time; EXPECT_CALL(stream_info, startTime()).WillRepeatedly(Return(time)); EXPECT_EQ(AccessLogDateTimeFormatter::fromTime(time), From d73b727436ddc702538e208d568287de5ee06487 Mon Sep 17 00:00:00 2001 From: John Esmet Date: Tue, 5 Jan 2021 22:06:30 +0000 Subject: [PATCH 10/10] Fix docs links Signed-off-by: John Esmet --- docs/root/configuration/observability/access_log/usage.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/root/configuration/observability/access_log/usage.rst b/docs/root/configuration/observability/access_log/usage.rst index be837d979360..9a00ac1a29d9 100644 --- a/docs/root/configuration/observability/access_log/usage.rst +++ b/docs/root/configuration/observability/access_log/usage.rst @@ -535,7 +535,7 @@ The following command operators are supported: TCP The client certificate in the URL-encoded PEM format used to establish the downstream TLS connection. -.. _config_access_log_format_downstream_peer_cert_v_start +.. _config_access_log_format_downstream_peer_cert_v_start: %DOWNSTREAM_PEER_CERT_V_START% HTTP @@ -546,7 +546,7 @@ The following command operators are supported: DOWNSTREAM_PEER_CERT_V_START can be customized using a `format string `_. See :ref:`START_TIME ` for additional format specifiers and examples. -.. _config_access_log_format_downstream_peer_cert_v_end +.. _config_access_log_format_downstream_peer_cert_v_end: %DOWNSTREAM_PEER_CERT_V_END% HTTP