From 369cc5755e1b2e969b9436bf1b04d6dac19b8814 Mon Sep 17 00:00:00 2001 From: colin-lamed <9568290+colin-lamed@users.noreply.github.com> Date: Wed, 13 Oct 2021 11:09:10 +0100 Subject: [PATCH 01/23] BDOG-1512 Make Retries useful to clients --- .../scala/uk/gov/hmrc/http/HttpDelete.scala | 2 +- .../main/scala/uk/gov/hmrc/http/HttpGet.scala | 2 +- .../scala/uk/gov/hmrc/http/HttpPatch.scala | 2 +- .../scala/uk/gov/hmrc/http/HttpPost.scala | 8 ++--- .../main/scala/uk/gov/hmrc/http/HttpPut.scala | 4 +-- .../main/scala/uk/gov/hmrc/http/Retries.scala | 29 ++++++++++++++----- .../uk/gov/hmrc/play/http/ws/WSRequest.scala | 1 + .../scala/uk/gov/hmrc/http/RetriesSpec.scala | 14 ++++----- 8 files changed, 38 insertions(+), 24 deletions(-) diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpDelete.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpDelete.scala index 7b7d3a88..c0318dbe 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpDelete.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpDelete.scala @@ -35,7 +35,7 @@ trait HttpDelete override def DELETE[O](url: String, headers: Seq[(String, String)] = Seq.empty)(implicit rds: HttpReads[O], hc: HeaderCarrier, ec: ExecutionContext): Future[O] = withTracing(DELETE_VERB, url) { val allHeaders = HeaderCarrier.headersForUrl(hcConfig, url, headers) - val httpResponse = retry(DELETE_VERB, url)(doDelete(url, allHeaders)) + val httpResponse = retryOnSslEngineClosed(DELETE_VERB, url)(doDelete(url, allHeaders)) executeHooks(DELETE_VERB, url"$url", allHeaders, None, httpResponse) mapErrors(DELETE_VERB, url, httpResponse).map(rds.read(DELETE_VERB, url, _)) } diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpGet.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpGet.scala index d54bdfdb..4af86364 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpGet.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpGet.scala @@ -46,7 +46,7 @@ trait HttpGet extends CoreGet with GetHttpTransport with HttpVerb with Connectio withTracing(GET_VERB, urlWithQuery) { val allHeaders = HeaderCarrier.headersForUrl(hcConfig, url, headers) - val httpResponse = retry(GET_VERB, urlWithQuery)(doGet(urlWithQuery, headers = allHeaders)) + val httpResponse = retryOnSslEngineClosed(GET_VERB, urlWithQuery)(doGet(urlWithQuery, headers = allHeaders)) executeHooks(GET_VERB, url"$url", allHeaders, None, httpResponse) mapErrors(GET_VERB, urlWithQuery, httpResponse).map(response => rds.read(GET_VERB, urlWithQuery, response)) } diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpPatch.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpPatch.scala index c9d0002b..c3de8674 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpPatch.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpPatch.scala @@ -43,7 +43,7 @@ trait HttpPatch ec: ExecutionContext): Future[O] = withTracing(PATCH_VERB, url) { val allHeaders = HeaderCarrier.headersForUrl(hcConfig, url, headers) - val httpResponse = retry(PATCH_VERB, url)(doPatch(url, body, allHeaders)) + val httpResponse = retryOnSslEngineClosed(PATCH_VERB, url)(doPatch(url, body, allHeaders)) executeHooks(PATCH_VERB, url"$url", allHeaders, Option(HookData.FromString(Json.stringify(wts.writes(body)))), httpResponse) mapErrors(PATCH_VERB, url, httpResponse).map(response => rds.read(PATCH_VERB, url, response)) } diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpPost.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpPost.scala index 1b542947..b303316f 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpPost.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpPost.scala @@ -43,7 +43,7 @@ trait HttpPost ec: ExecutionContext): Future[O] = withTracing(POST_VERB, url) { val allHeaders = HeaderCarrier.headersForUrl(hcConfig, url, headers) - val httpResponse = retry(POST_VERB, url)(doPost(url, body, allHeaders)) + val httpResponse = retryOnSslEngineClosed(POST_VERB, url)(doPost(url, body, allHeaders)) executeHooks(POST_VERB, url"$url", allHeaders, Option(HookData.FromString(Json.stringify(wts.writes(body)))), httpResponse) mapErrors(POST_VERB, url, httpResponse).map(rds.read(POST_VERB, url, _)) } @@ -57,7 +57,7 @@ trait HttpPost ec: ExecutionContext): Future[O] = withTracing(POST_VERB, url) { val allHeaders = HeaderCarrier.headersForUrl(hcConfig, url, headers) - val httpResponse = retry(POST_VERB, url)(doPostString(url, body, allHeaders)) + val httpResponse = retryOnSslEngineClosed(POST_VERB, url)(doPostString(url, body, allHeaders)) executeHooks(POST_VERB, url"$url", allHeaders, Option(HookData.FromString(body)), httpResponse) mapErrors(POST_VERB, url, httpResponse).map(rds.read(POST_VERB, url, _)) } @@ -71,7 +71,7 @@ trait HttpPost ec: ExecutionContext): Future[O] = withTracing(POST_VERB, url) { val allHeaders = HeaderCarrier.headersForUrl(hcConfig, url, headers) - val httpResponse = retry(POST_VERB, url)(doFormPost(url, body, allHeaders)) + val httpResponse = retryOnSslEngineClosed(POST_VERB, url)(doFormPost(url, body, allHeaders)) executeHooks(POST_VERB, url"$url", allHeaders, Option(HookData.FromMap(body)), httpResponse) mapErrors(POST_VERB, url, httpResponse).map(rds.read(POST_VERB, url, _)) } @@ -84,7 +84,7 @@ trait HttpPost ec: ExecutionContext): Future[O] = withTracing(POST_VERB, url) { val allHeaders = HeaderCarrier.headersForUrl(hcConfig, url, headers) - val httpResponse = retry(POST_VERB, url)(doEmptyPost(url, allHeaders)) + val httpResponse = retryOnSslEngineClosed(POST_VERB, url)(doEmptyPost(url, allHeaders)) executeHooks(POST_VERB, url"$url", allHeaders, None, httpResponse) mapErrors(POST_VERB, url, httpResponse).map(rds.read(POST_VERB, url, _)) } diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpPut.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpPut.scala index 7ce60590..7dc64009 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpPut.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpPut.scala @@ -37,7 +37,7 @@ trait HttpPut extends CorePut with PutHttpTransport with HttpVerb with Connectio ec: ExecutionContext): Future[O] = withTracing(PUT_VERB, url) { val allHeaders = HeaderCarrier.headersForUrl(hcConfig, url, headers) - val httpResponse = retry(PUT_VERB, url)(doPut(url, body, allHeaders)) + val httpResponse = retryOnSslEngineClosed(PUT_VERB, url)(doPut(url, body, allHeaders)) executeHooks(PUT_VERB, url"$url", allHeaders, Option(HookData.FromString(Json.stringify(wts.writes(body)))), httpResponse) mapErrors(PUT_VERB, url, httpResponse).map(response => rds.read(PUT_VERB, url, response)) } @@ -51,7 +51,7 @@ trait HttpPut extends CorePut with PutHttpTransport with HttpVerb with Connectio ec: ExecutionContext): Future[O] = withTracing(PUT_VERB, url) { val allHeaders = HeaderCarrier.headersForUrl(hcConfig, url, headers) - val httpResponse = retry(PUT_VERB, url)(doPutString(url, body, allHeaders)) + val httpResponse = retryOnSslEngineClosed(PUT_VERB, url)(doPutString(url, body, allHeaders)) executeHooks(PUT_VERB, url"$url", allHeaders, Option(HookData.FromString(body)), httpResponse) mapErrors(PUT_VERB, url, httpResponse).map(rds.read(PUT_VERB, url, _)) } diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/Retries.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/Retries.scala index e4761e58..913afaea 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/Retries.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/Retries.scala @@ -35,19 +35,32 @@ trait Retries { private val logger = LoggerFactory.getLogger("application") - def retry[A](verb: String, url: String)(block: => Future[A])(implicit ec: ExecutionContext): Future[A] = { + def retryOnSslEngineClosed[A](verb: String, url: String)(block: => Future[A])(implicit ec: ExecutionContext): Future[A] = + retryFor(s"$verb $url") { case ex @ `sslEngineClosedMatcher`() => true }(block) + + @deprecated("Use retryOnSslEngineClosed instead", "14.0.0") + def retry[A](verb: String, url: String)(block: => Future[A])(implicit ec: ExecutionContext): Future[A] = + retryOnSslEngineClosed(verb, url)(block) + + def retryFor[A]( + label : String + )( condition: PartialFunction[Exception, Boolean] + )( block : => Future[A] + )(implicit + ec: ExecutionContext + ): Future[A] = { def loop(remainingIntervals: Seq[FiniteDuration]): Future[A] = { // scheduling will loose MDC data. Here we explicitly ensure it is available on block. block .recoverWith { - case ex @ `sslEngineClosedMatcher`() if remainingIntervals.nonEmpty => + case ex: Exception if condition.lift(ex).getOrElse(false) && remainingIntervals.nonEmpty => val delay = remainingIntervals.head - logger.warn(s"Retrying $verb $url in $delay due to '${ex.getMessage}' error") - val mdcData = Mdc.mdcData - after(delay, actorSystem.scheduler){ - Mdc.putMdc(mdcData) - loop(remainingIntervals.tail) - } + logger.warn(s"Retrying $label in $delay due to '${ex.getMessage}' error") + val mdcData = Mdc.mdcData + after(delay, actorSystem.scheduler){ + Mdc.putMdc(mdcData) + loop(remainingIntervals.tail) + } } } loop(intervals) diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/play/http/ws/WSRequest.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/play/http/ws/WSRequest.scala index 1b3b7205..f5b97f2b 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/play/http/ws/WSRequest.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/play/http/ws/WSRequest.scala @@ -42,6 +42,7 @@ trait WSProxy extends WSRequest { object WSProxyConfiguration { + @deprecated("Use buildWsProxyServer instead.", "14.0.0") // TODO document differences def apply(configPrefix: String, configuration: Configuration): Option[WSProxyServer] = { val proxyRequired = configuration.getOptional[Boolean](s"$configPrefix.proxyRequiredForThisEnvironment").getOrElse(true) diff --git a/http-verbs-common/src/test/scala/uk/gov/hmrc/http/RetriesSpec.scala b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/RetriesSpec.scala index a5c0ac25..22f4cdb2 100644 --- a/http-verbs-common/src/test/scala/uk/gov/hmrc/http/RetriesSpec.scala +++ b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/RetriesSpec.scala @@ -47,7 +47,7 @@ class RetriesSpec with IntegrationPatience { import ExecutionContext.Implicits.global - "Retries" should { + "Retries.retryOnSslEngineClosed" should { "be disabled by default" in { val retries: Retries = new Retries { override protected val configuration: Config = ConfigFactory.load() @@ -56,7 +56,7 @@ class RetriesSpec @volatile var counter = 0 val resultF = - retries.retry("GET", "url") { + retries.retryOnSslEngineClosed("GET", "url") { Future.failed { counter += 1 new SSLException("SSLEngine closed already") @@ -89,7 +89,7 @@ class RetriesSpec @volatile var counter = 0 val resultF = - retries.retry("GET", "url") { + retries.retryOnSslEngineClosed("GET", "url") { Future.successful { counter += 1 counter @@ -118,7 +118,7 @@ class RetriesSpec Future.failed(new SSLException("SSLEngine closed already")) } - val _ = Try(retries.retry("GET", "url")(failingFuture).futureValue) + val _ = Try(retries.retryOnSslEngineClosed("GET", "url")(failingFuture).futureValue) val actualIntervals: List[Long] = timestamps.sliding(2).toList.map { @@ -143,7 +143,7 @@ class RetriesSpec } val resultF = - retries.retry("GET", "url") { + retries.retryOnSslEngineClosed("GET", "url") { Future.failed { new SSLException("SSLEngine closed already") } @@ -167,7 +167,7 @@ class RetriesSpec val expectedResponse = HttpResponse(404, "") val resultF = - retries.retry("GET", "url") { + retries.retryOnSslEngineClosed("GET", "url") { retries.failFewTimesAndThenSucceed( success = Future.successful(expectedResponse), exception = new SSLException("SSLEngine closed already") @@ -198,7 +198,7 @@ class RetriesSpec val resultF = for { _ <- Future.successful(Mdc.putMdc(mdcData)) - res <- retries.retry("GET", "url") { + res <- retries.retryOnSslEngineClosed("GET", "url") { // assert mdc available to block execution Option(MDC.getCopyOfContextMap).map(_.asScala.toMap).getOrElse(Map.empty) shouldBe mdcData From 78ae1ccbc4b31ac27ca0c87775dd13b0a2da7670 Mon Sep 17 00:00:00 2001 From: colin-lamed <9568290+colin-lamed@users.noreply.github.com> Date: Fri, 24 Sep 2021 12:47:29 +0100 Subject: [PATCH 02/23] BDOG-1512 Add HttpClient2 to simplify implementation and supporting builder style syntax --- .../src/main/resources/reference.conf | 1 + .../scala/uk/gov/hmrc/http/HttpPatch.scala | 2 +- .../scala/uk/gov/hmrc/http/HttpPost.scala | 2 +- .../main/scala/uk/gov/hmrc/http/HttpPut.scala | 2 +- .../uk/gov/hmrc/http/HttpReadsInstances.scala | 2 +- .../hmrc/http/HttpReadsLegacyInstances.scala | 9 +- .../scala/uk/gov/hmrc/http/HttpResponse.scala | 2 +- .../uk/gov/hmrc/http/HttpTransport.scala | 2 +- .../main/scala/uk/gov/hmrc/http/Retries.scala | 2 +- .../scala/uk/gov/hmrc/http/play/Example.scala | 114 +++++++ .../uk/gov/hmrc/http/play/HttpClient2.scala | 316 ++++++++++++++++++ .../scala/uk/gov/hmrc/http/play/package.scala | 48 +++ .../play/http/HeaderCarrierConverter.scala | 2 +- .../uk/gov/hmrc/play/http/ws/WSRequest.scala | 46 ++- .../hmrc/play/http/ws/WSRequestBuilder.scala | 2 +- .../src/test/resources/logback.xml | 3 +- .../gov/hmrc/http/CommonHttpBehaviour.scala | 2 +- .../scala/uk/gov/hmrc/http/HeadersSpec.scala | 149 ++++----- .../uk/gov/hmrc/http/HttpPatchSpec.scala | 2 +- .../scala/uk/gov/hmrc/http/HttpPostSpec.scala | 2 +- .../scala/uk/gov/hmrc/http/HttpPutSpec.scala | 2 +- .../hmrc/http/HttpReadsInstancesSpec.scala | 2 +- .../http/HttpReadsLegacyInstancesSpec.scala | 2 +- .../scala/uk/gov/hmrc/http/RetriesSpec.scala | 2 +- .../gov/hmrc/http/play/HttpClient2Spec.scala | 216 ++++++++++++ .../http/HeaderCarrierConverterSpec.scala | 3 +- .../gov/hmrc/play/http/HttpTimeoutSpec.scala | 97 ------ .../hmrc/play/http/ws/wireMockEndpoints.scala | 65 ---- .../uk/gov/hmrc/play/test/PortFinder.scala | 38 +++ .../gov/hmrc/play/test/WireMockSupport.scala | 62 ++++ .../scala/uk/gov/hmrc/play/test/helpers.scala | 107 ------ .../hmrc/http/test/HttpClient2Support.scala | 46 +++ .../hmrc/http/test/HttpClientSupport.scala | 2 +- 33 files changed, 967 insertions(+), 387 deletions(-) create mode 100644 http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/Example.scala create mode 100644 http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/HttpClient2.scala create mode 100644 http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/package.scala create mode 100644 http-verbs-common/src/test/scala/uk/gov/hmrc/http/play/HttpClient2Spec.scala delete mode 100644 http-verbs-common/src/test/scala/uk/gov/hmrc/play/http/HttpTimeoutSpec.scala delete mode 100644 http-verbs-common/src/test/scala/uk/gov/hmrc/play/http/ws/wireMockEndpoints.scala create mode 100644 http-verbs-common/src/test/scala/uk/gov/hmrc/play/test/PortFinder.scala create mode 100644 http-verbs-common/src/test/scala/uk/gov/hmrc/play/test/WireMockSupport.scala delete mode 100644 http-verbs-common/src/test/scala/uk/gov/hmrc/play/test/helpers.scala create mode 100644 http-verbs-test-common/src/main/scala/uk/gov/hmrc/http/test/HttpClient2Support.scala diff --git a/http-verbs-common/src/main/resources/reference.conf b/http-verbs-common/src/main/resources/reference.conf index b864a88d..eeb59827 100644 --- a/http-verbs-common/src/main/resources/reference.conf +++ b/http-verbs-common/src/main/resources/reference.conf @@ -18,3 +18,4 @@ http-verbs.retries { intervals = [ "500.millis", "1.second", "2.seconds", "4.seconds", "8.seconds" ] ssl-engine-closed-already.enabled = false } +proxy.enabled = false diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpPatch.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpPatch.scala index c3de8674..1e20ed76 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpPatch.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpPatch.scala @@ -16,7 +16,7 @@ package uk.gov.hmrc.http -import play.api.libs.json.{Json, Writes} +import _root_.play.api.libs.json.{Json, Writes} import uk.gov.hmrc.http.HttpVerbs.{PATCH => PATCH_VERB} import uk.gov.hmrc.http.hooks.{HookData, HttpHooks} import uk.gov.hmrc.http.logging.ConnectionTracing diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpPost.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpPost.scala index b303316f..ab801190 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpPost.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpPost.scala @@ -16,7 +16,7 @@ package uk.gov.hmrc.http -import play.api.libs.json.{Json, Writes} +import _root_.play.api.libs.json.{Json, Writes} import uk.gov.hmrc.http.HttpVerbs.{POST => POST_VERB} import uk.gov.hmrc.http.hooks.{HookData, HttpHooks} import uk.gov.hmrc.http.logging.ConnectionTracing diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpPut.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpPut.scala index 7dc64009..d85209ba 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpPut.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpPut.scala @@ -16,7 +16,7 @@ package uk.gov.hmrc.http -import play.api.libs.json.{Json, Writes} +import _root_.play.api.libs.json.{Json, Writes} import uk.gov.hmrc.http.HttpVerbs.{PUT => PUT_VERB} import uk.gov.hmrc.http.hooks.{HookData, HttpHooks} import uk.gov.hmrc.http.logging.ConnectionTracing diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpReadsInstances.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpReadsInstances.scala index 3d261bb7..b671a0c0 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpReadsInstances.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpReadsInstances.scala @@ -16,7 +16,7 @@ package uk.gov.hmrc.http -import play.api.libs.json.{JsValue, JsError, JsResult, JsSuccess, Reads => JsonReads} +import _root_.play.api.libs.json.{JsValue, JsError, JsResult, JsSuccess, Reads => JsonReads} import scala.util.Try trait HttpReadsInstances diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpReadsLegacyInstances.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpReadsLegacyInstances.scala index cc4012fd..7eb17003 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpReadsLegacyInstances.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpReadsLegacyInstances.scala @@ -17,8 +17,7 @@ package uk.gov.hmrc.http import com.github.ghik.silencer.silent -import play.api.libs.json -import play.api.libs.json.{JsNull, JsValue} +import _root_.play.api.libs.json.{JsNull, JsValue, Reads} trait HttpReadsLegacyInstances extends HttpReadsLegacyOption with HttpReadsLegacyJson @@ -45,13 +44,13 @@ trait HttpReadsLegacyOption extends HttpErrorFunctions { trait HttpReadsLegacyJson extends HttpErrorFunctions { @deprecated("Use uk.gov.hmrc.http.HttpReads.Implicits instead. See README for differences.", "11.0.0") - implicit def readFromJson[O](implicit rds: json.Reads[O], mf: Manifest[O]): HttpReads[O] = new HttpReads[O] { + implicit def readFromJson[O](implicit rds: Reads[O], mf: Manifest[O]): HttpReads[O] = new HttpReads[O] { def read(method: String, url: String, response: HttpResponse) = readJson(method, url, handleResponse(method, url)(response).json) } @deprecated("Use uk.gov.hmrc.http.HttpReads.Implicits instead. See README for differences.", "11.0.0") - def readSeqFromJsonProperty[O](name: String)(implicit rds: json.Reads[O], mf: Manifest[O]) = new HttpReads[Seq[O]] { + def readSeqFromJsonProperty[O](name: String)(implicit rds: Reads[O], mf: Manifest[O]) = new HttpReads[Seq[O]] { def read(method: String, url: String, response: HttpResponse) = response.status match { case 204 | 404 => Seq.empty case _ => @@ -59,7 +58,7 @@ trait HttpReadsLegacyJson extends HttpErrorFunctions { } } - private def readJson[A](method: String, url: String, jsValue: JsValue)(implicit rds: json.Reads[A], mf: Manifest[A]) = + private def readJson[A](method: String, url: String, jsValue: JsValue)(implicit rds: Reads[A], mf: Manifest[A]) = jsValue .validate[A] .fold( diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpResponse.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpResponse.scala index 60e4dd62..140f2cdb 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpResponse.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpResponse.scala @@ -17,7 +17,7 @@ package uk.gov.hmrc.http import com.github.ghik.silencer.silent -import play.api.libs.json.{JsValue, Json} +import _root_.play.api.libs.json.{JsValue, Json} /** * The ws.Response class is very hard to dummy up as it wraps a concrete instance of diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpTransport.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpTransport.scala index 59c44e68..3edc77f9 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpTransport.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpTransport.scala @@ -18,7 +18,7 @@ package uk.gov.hmrc.http import java.net.URL -import play.api.libs.json.Writes +import _root_.play.api.libs.json.Writes import scala.concurrent.{ExecutionContext, Future} diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/Retries.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/Retries.scala index 913afaea..1294c7fd 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/Retries.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/Retries.scala @@ -55,7 +55,7 @@ trait Retries { .recoverWith { case ex: Exception if condition.lift(ex).getOrElse(false) && remainingIntervals.nonEmpty => val delay = remainingIntervals.head - logger.warn(s"Retrying $label in $delay due to '${ex.getMessage}' error") + logger.warn(s"Retrying $label in $delay due to error: ${ex.getMessage}") val mdcData = Mdc.mdcData after(delay, actorSystem.scheduler){ Mdc.putMdc(mdcData) diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/Example.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/Example.scala new file mode 100644 index 00000000..5bbb969c --- /dev/null +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/Example.scala @@ -0,0 +1,114 @@ +/* + * Copyright 2022 HM Revenue & Customs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package uk.gov.hmrc.http.play + +import akka.actor.ActorSystem +import akka.stream.{ActorMaterializer, Materializer} +import akka.stream.scaladsl.Source +import akka.util.ByteString +import com.typesafe.config.ConfigFactory +import play.api.Configuration +import play.api.libs.json.{Reads, Writes} +import play.api.libs.ws.WSClient +import play.api.libs.ws.ahc.{AhcWSClient, AhcWSClientConfigFactory} +import uk.gov.hmrc.http.{HeaderCarrier, HttpReads, Retries, StringContextOps, UpstreamErrorResponse} + +import scala.concurrent.Future +import scala.concurrent.duration.DurationInt + + +/** Demonstrates usage */ +object Example { + import scala.concurrent.ExecutionContext.Implicits.global + + // this would be injected + val httpClient2: HttpClient2 = { + implicit val as: ActorSystem = ActorSystem("test-actor-system") + val config = Configuration(ConfigFactory.load()) + val wsClient: WSClient = { + implicit val mat: Materializer = ActorMaterializer() // explicitly required for play-26 + AhcWSClient(AhcWSClientConfigFactory.forConfig(config.underlying)) + } + new HttpClient2Impl( + wsClient, + as, + config, + hooks = Seq.empty, // this would be wired up to play-auditing + ) + } + + // json + { + implicit val hc = HeaderCarrier() + + val _: Future[ResDomain] = + httpClient2 + .put(url"http://localhost:8000/", toJson(ReqDomain())) + .withProxy + .replaceHeader("User-Agent" -> "ua") + + .execute(fromJson[ResDomain]) + } + + + // streams + { + implicit val hc = HeaderCarrier() + + val srcStream: Source[ByteString, _] = ??? + + val _: Future[Source[ByteString, _]] = + httpClient2 + //.put(url"http://localhost:8000/", srcStream) + .put(url"http://localhost:8000/") + .withBody(srcStream) + .transformRequest(_.withRequestTimeout(10.seconds)) + .stream(fromStream) + } + + // retries + { + val retries = new Retries { + override val actorSystem = ActorSystem("test-actor-system") + override val configuration = ConfigFactory.load() + } + + implicit val hc = HeaderCarrier() + + val _: Future[ResDomain] = + retries.retryFor("get reqdomain"){ case UpstreamErrorResponse.WithStatusCode(502) => true }{ + httpClient2 + .put(url"http://localhost:8000/", toJson(ReqDomain())) + .withProxy + .replaceHeader("User-Agent" -> "ua") + .execute(fromJson[ResDomain]) + } + } +} + +case class ReqDomain() + +object ReqDomain { + implicit val w: Writes[ReqDomain] = ??? +} + +case class ResDomain() + +object ResDomain { + implicit val r : Reads[ResDomain] = ??? + implicit val hr: HttpReads[ResDomain] = ??? +} diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/HttpClient2.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/HttpClient2.scala new file mode 100644 index 00000000..5317b19b --- /dev/null +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/HttpClient2.scala @@ -0,0 +1,316 @@ +/* + * Copyright 2022 HM Revenue & Customs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// TODO putting this in this package means that all clients which do +// `import uk.gov.hmrc.http._` will then have to make play imports with _root_ `import _root_.play...` +package uk.gov.hmrc.http.play + +import akka.actor.ActorSystem +import com.typesafe.config.Config +import play.api.Configuration +import play.api.libs.ws.{BodyWritable, EmptyBody, InMemoryBody, SourceBody, WSClient, WSProxyServer, WSRequest, WSResponse} +import play.core.parsers.FormUrlEncodedParser +import uk.gov.hmrc.http.{BadGatewayException, GatewayTimeoutException, HeaderCarrier, HttpResponse, Retries} +import uk.gov.hmrc.play.http.ws.WSProxyConfiguration +import uk.gov.hmrc.http.hooks.{HookData, HttpHook} +import uk.gov.hmrc.http.logging.ConnectionTracing + +import java.net.{ConnectException, URL} +import java.util.concurrent.TimeoutException +import scala.concurrent.{ExecutionContext, Future} + +/* What does HttpVerbs actually provide? + +Readme says... + - Http Transport + - Core Http function interfaces + - Logging + - Propagation of common headers + - Executing hooks, for example Auditing + - Request & Response de-serializations + - Response handling, converting failure status codes into a consistent set of exceptions - allows failures to be automatically propagated to the caller + +Also, retries + + +This version demonstrates a flat implementation that uses an HttpExecutor to centralise the execution of the request to ensure that +the common concerns occur, but delegates out the construction of the request and parsing of the response to play-ws for flexibility. +The use of HttpReads is optional. +Extension methods are provided to make common patterns easier to apply. +*/ + +trait HttpClient2 { + def get(url: URL)(implicit hc: HeaderCarrier): RequestBuilder + + def post(url: URL)(implicit hc: HeaderCarrier): RequestBuilder + + def post[B: BodyWritable](url: URL, body: B)(implicit hc: HeaderCarrier): RequestBuilder + + def put(url: URL)(implicit hc: HeaderCarrier): RequestBuilder + + def put[B: BodyWritable](url: URL, body: B)(implicit hc: HeaderCarrier): RequestBuilder + + def delete(url: URL)(implicit hc: HeaderCarrier): RequestBuilder + + def patch(url: URL)(implicit hc: HeaderCarrier): RequestBuilder + + def patch[B: BodyWritable](url: URL, body: B)(implicit hc: HeaderCarrier): RequestBuilder + + def head(url: URL)(implicit hc: HeaderCarrier): RequestBuilder + + def options(url: URL)(implicit hc: HeaderCarrier): RequestBuilder +} + +trait RequestBuilder { + def transformRequest(transform: WSRequest => WSRequest): RequestBuilder + + def execute[A]( + transform: WSRequest => WSResponse => Future[A] + )(implicit + ec: ExecutionContext + ): Future[A] + + def stream[A]( + transform: WSRequest => WSResponse => Future[A] + )(implicit + ec: ExecutionContext + ): Future[A] + + // support functions + + def replaceHeader(header: (String, String)): RequestBuilder + + def addHeaders(headers: (String, String)*): RequestBuilder + + def withProxy: RequestBuilder + + def withBody[B : BodyWritable](body: B): RequestBuilder +} + +class HttpClient2Impl( + wsClient : WSClient, + actorSystem: ActorSystem, + config : Configuration, + hooks : Seq[HttpHook] +) extends HttpClient2 { + + private lazy val optProxyServer = + WSProxyConfiguration.buildWsProxyServer(config.underlying) + + private lazy val hcConfig = + HeaderCarrier.Config.fromConfig(config.underlying) + + override def get(url: URL)(implicit hc: HeaderCarrier): RequestBuilderImpl = + mkRequestBuilder(url, "GET") + + override def post(url: URL)(implicit hc: HeaderCarrier): RequestBuilderImpl = + mkRequestBuilder(url, "POST") + + // TODO or just let clients call `.withBody(body)` themselves (this is the expectation for adding headers) + override def post[B: BodyWritable](url: URL, body: B)(implicit hc: HeaderCarrier): RequestBuilderImpl = + mkRequestBuilder(url, "POST") + .withBody(body) + + override def put(url: URL)(implicit hc: HeaderCarrier): RequestBuilderImpl = + mkRequestBuilder(url, "PUT") + + override def put[B: BodyWritable](url: URL, body: B)(implicit hc: HeaderCarrier): RequestBuilderImpl = + mkRequestBuilder(url, "PUT") + .withBody(body) + + override def delete(url: URL)(implicit hc: HeaderCarrier): RequestBuilderImpl = + mkRequestBuilder(url, "DELETE") + + override def patch(url: URL)(implicit hc: HeaderCarrier): RequestBuilderImpl = + mkRequestBuilder(url, "PATCH") + + override def patch[B: BodyWritable](url: URL, body: B)(implicit hc: HeaderCarrier): RequestBuilderImpl = + mkRequestBuilder(url, "PATCH") + .withBody(body) + + override def head(url: URL)(implicit hc: HeaderCarrier): RequestBuilderImpl = + mkRequestBuilder(url, "HEAD") + + override def options(url: URL)(implicit hc: HeaderCarrier): RequestBuilderImpl = + mkRequestBuilder(url, "OPTIONS") + + private def mkRequestBuilder( + url : URL, + method: String + )(implicit + hc: HeaderCarrier + ): RequestBuilderImpl = + new RequestBuilderImpl( + actorSystem, + config, + optProxyServer, + hooks + )( + wsClient + .url(url.toString) + .withMethod(method) + .withHttpHeaders(hc.headersForUrl(hcConfig)(url.toString) : _*) + ) + } + + +// is final since tranformRequest (and derived) return instances of RequestBuilderImpl, and would loose any overrides. +final class RequestBuilderImpl( + override val actorSystem: ActorSystem, + config : Configuration, + optProxyServer : Option[WSProxyServer], + hooks : Seq[HttpHook] +)( + request: WSRequest +)(implicit + hc: HeaderCarrier +) extends RequestBuilder + with Retries + with ConnectionTracing { + + // for Retries (TODO make it use Configuration too) + override val configuration: Config = config.underlying + + override def transformRequest(transform: WSRequest => WSRequest): RequestBuilderImpl = + new RequestBuilderImpl(actorSystem, config, optProxyServer, hooks)(transform(request)) + + // -- Syntactic sugar -- + // TODO any implementation would be expected to implement them all + // should they be available as extension methods? (less discoverable - require `import httpClient._` to enable) + // they all depend on `transformRequest` - but also configuration (and other derived/cached values like proxyServer) could they move into the API interface? + // also the variable RequestBuilder makes this tricky... + + override def replaceHeader(header: (String, String)): RequestBuilderImpl = { + def denormalise(hdrs: Map[String, Seq[String]]): Seq[(String, String)] = + hdrs.toList.flatMap { case (k, vs) => vs.map(k -> _) } + val hdrsWithoutKey = request.headers.filterKeys(!_.equalsIgnoreCase(header._1)) // replace existing header + transformRequest(_.withHttpHeaders(denormalise(hdrsWithoutKey) :+ header : _*)) + } + + override def addHeaders(headers: (String, String)*): RequestBuilderImpl = + transformRequest(_.addHttpHeaders(headers: _*)) + + override def withProxy: RequestBuilderImpl = + transformRequest(request => optProxyServer.foldLeft(request)(_ withProxyServer _)) + + override def withBody[B : BodyWritable](body: B): RequestBuilderImpl = + (if (body == EmptyBody) + replaceHeader(play.api.http.HeaderNames.CONTENT_LENGTH -> "0") // rejected by Akami without a Content-Length (https://jira.tools.tax.service.gov.uk/browse/APIS-5100) + else + this + ).transformRequest(_.withBody(body)) + + // -- Execution -- + + override def execute[A]( + transformResponse: WSRequest => WSResponse => Future[A] + )(implicit + ec: ExecutionContext + ): Future[A] = + execute(isStream = false)(transformResponse) + + override def stream[A]( + transformResponse: WSRequest => WSResponse => Future[A] + )(implicit + ec: ExecutionContext + ): Future[A] = + execute(isStream = true)(transformResponse) + + private def execute[A]( + isStream: Boolean + )( + transformResponse: WSRequest => WSResponse => Future[A] + )(implicit + ec: ExecutionContext + ): Future[A] = { + val startAge = System.nanoTime() - hc.age + val responseF = + // TODO a way for clients to define custom retries? + retryOnSslEngineClosed(request.method, request.url)( + // we do the execution since if clients are responsable for it (e.g. a callback), they may further modify the request outside of auditing etc. + if (isStream) request.stream() else request.execute() + ) + executeHooks(isStream, request, responseF) + responseF.onComplete(logResult(hc, request.method, request.uri.toString, startAge)) + // we don't delegate the response conversion to the client + // e.g. execute[WSResponse].transform(...) since the transform functions require access to the request (method and url) + // given method and url are only required for error messages, is it overkill? E.g. a stacktrace should identify the function? + mapErrors(request.method, request.url, responseF) + responseF.flatMap(transformResponse(request)) + } + + // mapErrors could be part of the transform function. e.g. transformResponse: WSRequest => Try[WSResponse] => Future + // but then each transformResponse would probably end up doing the same recovery? Is that a problem? + def mapErrors( + httpMethod: String, + url : String, + f : Future[WSResponse] + )(implicit + ec: ExecutionContext + ): Future[WSResponse] = + f.recoverWith { + case e: TimeoutException => Future.failed(new GatewayTimeoutException(gatewayTimeoutMessage(httpMethod, url, e))) + case e: ConnectException => Future.failed(new BadGatewayException(badGatewayMessage(httpMethod, url, e))) + } + + def badGatewayMessage(verbName: String, url: String, e: Exception): String = + s"$verbName of '$url' failed. Caused by: '${e.getMessage}'" + + def gatewayTimeoutMessage(verbName: String, url: String, e: Exception): String = + s"$verbName of '$url' timed out with message '${e.getMessage}'" + + + private def executeHooks( + isStream : Boolean, + request : WSRequest, + responseF: Future[WSResponse] + )(implicit + hc: HeaderCarrier, + ec: ExecutionContext + ): Unit = { + // hooks take HttpResponse.. + def toHttpResponse(response: WSResponse) = + HttpResponse( + status = response.status, + body = if (isStream) "" else response.body, // calling response.body on stream would load all into memory (and cause stream to be read twice..) + headers = response.headers + ) + + def denormalise(hdrs: Map[String, Seq[String]]): Seq[(String, String)] = + hdrs.toList.flatMap { case (k, vs) => vs.map(k -> _) } + + val body = + request.body match { + case EmptyBody => None + case InMemoryBody(bytes) => request.header("Content-Type") match { + case Some("application/x-www-form-urlencoded") => Some(HookData.FromMap(FormUrlEncodedParser.parse(bytes.decodeString("UTF-8")))) + case Some("application/octet-stream") => Some(HookData.FromString("")) + case _ => Some(HookData.FromString(bytes.decodeString("UTF-8"))) + } + case SourceBody(_) => Some(HookData.FromString("")) + } + + hooks.foreach( + _.apply( + verb = request.method, + url = new URL(request.url), + headers = denormalise(request.headers), + body = body, + responseF = responseF.map(toHttpResponse) + ) + ) + } +} diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/package.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/package.scala new file mode 100644 index 00000000..78cf5f05 --- /dev/null +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/package.scala @@ -0,0 +1,48 @@ +/* + * Copyright 2022 HM Revenue & Customs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package uk.gov.hmrc.http + +import akka.stream.scaladsl.Source +import akka.util.ByteString +import _root_.play.api.libs.json.{JsValue, Writes} +import _root_.play.api.libs.ws.{WSRequest, WSResponse} +import scala.concurrent.Future + + +package object play { + // These will still need explicitly importing + // should they be moved to `import httpClient2._`? which means implementations can then depend on + // httpClient values (e.g. configuration) or does this make mocking/providing alternative implementations harder? + // Alternatively, could HttpClient2 trait just be replaced by HttpClient2Impl - and forget about alternative implementations + // (solves the final builder problem) + def fromStream(request: WSRequest)(response: WSResponse): Future[Source[ByteString, _]] = + Future.successful(response.bodyAsSource) + + def fromJson[A](request: WSRequest)(response: WSResponse)(implicit r: HttpReads[A]): Future[A] = + // reusing existing HttpReads - currently requires HttpResponse + Future.successful { + val httpResponse = HttpResponse( + status = response.status, + body = response.body, + headers = response.headers + ) + r.read(request.method, request.url, httpResponse) + } + + def toJson[A](model: A)(implicit w: Writes[A]): JsValue = + w.writes(model) +} diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/play/http/HeaderCarrierConverter.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/play/http/HeaderCarrierConverter.scala index 704a14dd..fa23bf0a 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/play/http/HeaderCarrierConverter.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/play/http/HeaderCarrierConverter.scala @@ -16,9 +16,9 @@ package uk.gov.hmrc.play.http +import play.api.http.{HeaderNames => PlayHeaderNames} import play.api.mvc.{Cookies, Headers, RequestHeader, Session} import uk.gov.hmrc.http._ -import play.api.http.{HeaderNames => PlayHeaderNames} import scala.util.Try diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/play/http/ws/WSRequest.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/play/http/ws/WSRequest.scala index f5b97f2b..869d66f8 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/play/http/ws/WSRequest.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/play/http/ws/WSRequest.scala @@ -16,12 +16,13 @@ package uk.gov.hmrc.play.http.ws +import com.typesafe.config.Config import play.api.Configuration import play.api.libs.ws.{DefaultWSProxyServer, WSProxyServer, WSRequest => PlayWSRequest} trait WSRequest extends WSRequestBuilder { - override def buildRequest[A]( + override def buildRequest( url : String, headers: Seq[(String, String)] ): PlayWSRequest = @@ -33,7 +34,7 @@ trait WSProxy extends WSRequest { def wsProxyServer: Option[WSProxyServer] - override def buildRequest[A](url: String, headers: Seq[(String, String)]): PlayWSRequest = + override def buildRequest(url: String, headers: Seq[(String, String)]): PlayWSRequest = wsProxyServer match { case Some(proxy) => super.buildRequest(url, headers).withProxyServer(proxy) case None => super.buildRequest(url, headers) @@ -47,21 +48,34 @@ object WSProxyConfiguration { val proxyRequired = configuration.getOptional[Boolean](s"$configPrefix.proxyRequiredForThisEnvironment").getOrElse(true) - if (proxyRequired) Some(parseProxyConfiguration(configPrefix, configuration)) else None + if (proxyRequired) + Some( + DefaultWSProxyServer( + protocol = Some(configuration.get[String](s"$configPrefix.protocol")), + host = configuration.get[String](s"$configPrefix.host"), + port = configuration.get[Int](s"$configPrefix.port"), + principal = configuration.getOptional[String](s"$configPrefix.username"), + password = configuration.getOptional[String](s"$configPrefix.password") + ) + ) + else None } - private def parseProxyConfiguration(configPrefix: String, configuration: Configuration) = - DefaultWSProxyServer( - protocol = configuration - .getOptional[String](s"$configPrefix.protocol") - .orElse(throw ProxyConfigurationException("protocol")), - host = - configuration.getOptional[String](s"$configPrefix.host").getOrElse(throw ProxyConfigurationException("host")), - port = configuration.getOptional[Int](s"$configPrefix.port").getOrElse(throw ProxyConfigurationException("port")), - principal = configuration.getOptional[String](s"$configPrefix.username"), - password = configuration.getOptional[String](s"$configPrefix.password") - ) - case class ProxyConfigurationException(key: String) - extends RuntimeException(s"Missing proxy configuration - key '$key' not found") + def buildWsProxyServer(configuration: Config): Option[WSProxyServer] = { // TODO use Configuration rather than Config throughout? + def getOptionalString(key: String): Option[String] = + if (configuration.hasPath(key)) Some(configuration.getString(key)) else None + + if (configuration.getBoolean("proxy.enabled")) + Some( + DefaultWSProxyServer( + protocol = Some(configuration.getString("proxy.protocol")), + host = configuration.getString("proxy.host"), + port = configuration.getInt("proxy.port"), + principal = getOptionalString("proxy.username"), + password = getOptionalString("proxy.password") + ) + ) + else None + } } diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/play/http/ws/WSRequestBuilder.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/play/http/ws/WSRequestBuilder.scala index 1c11287c..4b2ae03a 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/play/http/ws/WSRequestBuilder.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/play/http/ws/WSRequestBuilder.scala @@ -23,5 +23,5 @@ trait WSRequestBuilder extends Request { protected def wsClient: WSClient - protected def buildRequest[A](url: String, headers: Seq[(String, String)]): PlayWSRequest + protected def buildRequest(url: String, headers: Seq[(String, String)]): PlayWSRequest } diff --git a/http-verbs-common/src/test/resources/logback.xml b/http-verbs-common/src/test/resources/logback.xml index d36f5f0c..c44ac0bc 100644 --- a/http-verbs-common/src/test/resources/logback.xml +++ b/http-verbs-common/src/test/resources/logback.xml @@ -3,12 +3,13 @@ - %date [%level] [%logger] [%thread] %message%n + %date %highlight([%level]) [%logger] [%thread] %message%n + diff --git a/http-verbs-common/src/test/scala/uk/gov/hmrc/http/CommonHttpBehaviour.scala b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/CommonHttpBehaviour.scala index f4f8f3ed..f139de53 100644 --- a/http-verbs-common/src/test/scala/uk/gov/hmrc/http/CommonHttpBehaviour.scala +++ b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/CommonHttpBehaviour.scala @@ -22,7 +22,7 @@ import java.util.concurrent.TimeoutException import org.scalatest.concurrent.ScalaFutures import org.scalatest.wordspec.AnyWordSpecLike import org.scalatest.matchers.should.Matchers -import play.api.libs.json.{Json, OFormat} +import _root_.play.api.libs.json.{Json, OFormat} import uk.gov.hmrc.http.logging.{ConnectionTracing, LoggingDetails} import scala.collection.mutable diff --git a/http-verbs-common/src/test/scala/uk/gov/hmrc/http/HeadersSpec.scala b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/HeadersSpec.scala index a1b30415..a985b115 100644 --- a/http-verbs-common/src/test/scala/uk/gov/hmrc/http/HeadersSpec.scala +++ b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/HeadersSpec.scala @@ -20,18 +20,18 @@ import akka.actor.ActorSystem import com.github.ghik.silencer.silent import com.github.tomakehurst.wiremock._ import com.github.tomakehurst.wiremock.client.WireMock._ -import com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig import com.typesafe.config.Config import org.scalatest.concurrent.{IntegrationPatience, ScalaFutures} import org.scalatest.matchers.must.Matchers import org.scalatest.wordspec.AnyWordSpecLike -import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach} -import play.api.inject.guice.GuiceApplicationBuilder -import play.api.libs.json.{JsValue, Json} -import play.api.libs.ws.WSClient -import play.api.{Application, Play} +import _root_.play.api.Application +import _root_.play.api.http.{HeaderNames => PlayHeaderNames} +import _root_.play.api.inject.guice.GuiceApplicationBuilder +import _root_.play.api.libs.json.{JsValue, Json} +import _root_.play.api.libs.ws.WSClient import uk.gov.hmrc.http.hooks.HttpHook -import uk.gov.hmrc.play.http.ws.{PortTester, WSHttp} +import uk.gov.hmrc.play.http.ws.WSHttp +import uk.gov.hmrc.http.test.WireMockSupport import scala.concurrent.ExecutionContext.Implicits.global import uk.gov.hmrc.http.HttpReads.Implicits._ @@ -39,28 +39,11 @@ import uk.gov.hmrc.http.HttpReads.Implicits._ class HeadersSpec extends AnyWordSpecLike with Matchers - with BeforeAndAfterAll - with BeforeAndAfterEach + with WireMockSupport with ScalaFutures with IntegrationPatience { private lazy val app: Application = new GuiceApplicationBuilder().build() - private val server = new WireMockServer(wireMockConfig().port(PortTester.findPort())) - - override def beforeAll(): Unit = { - Play.start(app) - server.start() - } - - override def afterAll(): Unit = { - server.stop() - Play.stop(app) - } - - override def beforeEach(): Unit = { - server.stop() - server.start() - } @silent("deprecated") private implicit val hc: HeaderCarrier = HeaderCarrier( @@ -70,24 +53,27 @@ class HeadersSpec requestId = Some(RequestId("request-id")) ).withExtraHeaders("extra-header" -> "my-extra-header") - private lazy val client = new HttpGet with HttpPost with HttpDelete with HttpPatch with HttpPut with WSHttp { - override def wsClient: WSClient = app.injector.instanceOf[WSClient] - override protected def configuration: Config = app.configuration.underlying - override val hooks: Seq[HttpHook] = Seq.empty - override protected def actorSystem: ActorSystem = ActorSystem("test-actor-system") - } + private lazy val httpClient = + new HttpGet with HttpPost with HttpDelete with HttpPatch with HttpPut with WSHttp { + override def wsClient: WSClient = app.injector.instanceOf[WSClient] + override protected def configuration: Config = app.configuration.underlying + override val hooks: Seq[HttpHook] = Seq.empty + override protected def actorSystem: ActorSystem = ActorSystem("test-actor-system") + } "a post request" when { "with an arbitrary body" should { "contain headers from the header carrier" in { - server.stubFor( + stubFor( post(urlEqualTo("/arbitrary")) .willReturn(aResponse().withStatus(200)) ) - client.POST[JsValue, HttpResponse](s"http://localhost:${server.port()}/arbitrary", Json.obj()).futureValue + httpClient + .POST[JsValue, HttpResponse](s"$wireMockUrl/arbitrary", Json.obj()) + .futureValue - server.verify( + verify( postRequestedFor(urlEqualTo("/arbitrary")) .withHeader(HeaderNames.authorisation, equalTo("authorization")) .withHeader(HeaderNames.xForwardedFor, equalTo("forwarded-for")) @@ -98,18 +84,19 @@ class HeadersSpec } "allow a user to set an extra header in the POST" in { - server.stubFor( + stubFor( post(urlEqualTo("/arbitrary")) .willReturn(aResponse().withStatus(200)) ) - client.POST[JsValue, HttpResponse]( - url = s"http://localhost:${server.port()}/arbitrary", - body = Json.obj(), - headers = Seq("extra-header-2" -> "my-extra-header-2") - ).futureValue + httpClient + .POST[JsValue, HttpResponse]( + url = s"$wireMockUrl/arbitrary", + body = Json.obj(), + headers = Seq("extra-header-2" -> "my-extra-header-2") + ).futureValue - server.verify( + verify( postRequestedFor(urlEqualTo("/arbitrary")) .withHeader(HeaderNames.authorisation, equalTo("authorization")) .withHeader(HeaderNames.xForwardedFor, equalTo("forwarded-for")) @@ -123,17 +110,18 @@ class HeadersSpec "with a string body" should { "contain headers from the header carrier" in { - server.stubFor( + stubFor( post(urlEqualTo("/string")) .willReturn(aResponse().withStatus(200)) ) - client.POSTString[HttpResponse]( - url = s"http://localhost:${server.port()}/string", - body = "foo" - ).futureValue + httpClient + .POSTString[HttpResponse]( + url = s"$wireMockUrl/string", + body = "foo" + ).futureValue - server.verify( + verify( postRequestedFor(urlEqualTo("/string")) .withHeader(HeaderNames.authorisation, equalTo("authorization")) .withHeader(HeaderNames.xForwardedFor, equalTo("forwarded-for")) @@ -144,18 +132,19 @@ class HeadersSpec } "allow a user to set an extra header in the POST" in { - server.stubFor( + stubFor( post(urlEqualTo("/string")) .willReturn(aResponse().withStatus(200)) ) - client.POSTString[HttpResponse]( - url = s"http://localhost:${server.port()}/string", - body = "foo", - headers = Seq("extra-header-2" -> "my-extra-header-2") - ).futureValue + httpClient + .POSTString[HttpResponse]( + url = s"$wireMockUrl/string", + body = "foo", + headers = Seq("extra-header-2" -> "my-extra-header-2") + ).futureValue - server.verify( + verify( postRequestedFor(urlEqualTo("/string")) .withHeader(HeaderNames.authorisation, equalTo("authorization")) .withHeader(HeaderNames.xForwardedFor, equalTo("forwarded-for")) @@ -169,21 +158,23 @@ class HeadersSpec "with an empty body" should { "add a content length header if none is present" in { - server.stubFor( + stubFor( post(urlEqualTo("/empty")) .willReturn(aResponse().withStatus(200)) ) - client.POSTEmpty[HttpResponse](s"http://localhost:${server.port()}/empty").futureValue + httpClient + .POSTEmpty[HttpResponse](s"$wireMockUrl/empty") + .futureValue - server.verify( + verify( postRequestedFor(urlEqualTo("/empty")) .withHeader(HeaderNames.authorisation, equalTo("authorization")) .withHeader(HeaderNames.xForwardedFor, equalTo("forwarded-for")) .withHeader(HeaderNames.xSessionId, equalTo("session-id")) .withHeader(HeaderNames.xRequestId, equalTo("request-id")) .withHeader("extra-header", equalTo("my-extra-header")) - .withHeader(play.api.http.HeaderNames.CONTENT_LENGTH, equalTo("0")) + .withHeader(PlayHeaderNames.CONTENT_LENGTH, equalTo("0")) ) } } @@ -191,16 +182,16 @@ class HeadersSpec "a get request" should { "contain headers from the header carrier" in { - server.stubFor( + stubFor( get(urlEqualTo("/")) .willReturn(aResponse().withStatus(200)) ) - client - .GET[HttpResponse](s"http://localhost:${server.port()}/") + httpClient + .GET[HttpResponse](s"$wireMockUrl/") .futureValue - server.verify( + verify( getRequestedFor(urlEqualTo("/")) .withHeader(HeaderNames.authorisation, equalTo("authorization")) .withHeader(HeaderNames.xForwardedFor, equalTo("forwarded-for")) @@ -213,17 +204,18 @@ class HeadersSpec "a delete request" should { "contain headers from the header carrier" in { - server.stubFor( + wireMockServer.stubFor( delete(urlEqualTo("/")) .willReturn(aResponse().withStatus(200)) ) - client.DELETE[HttpResponse]( - url = s"http://localhost:${server.port()}/", - headers = Seq("header" -> "foo") - ).futureValue + httpClient + .DELETE[HttpResponse]( + url = s"$wireMockUrl/", + headers = Seq("header" -> "foo") + ).futureValue - server.verify( + wireMockServer.verify( deleteRequestedFor(urlEqualTo("/")) .withHeader(HeaderNames.authorisation, equalTo("authorization")) .withHeader(HeaderNames.xForwardedFor, equalTo("forwarded-for")) @@ -237,20 +229,20 @@ class HeadersSpec "a patch request" should { "contain headers from the header carrier" in { - server.stubFor( + stubFor( patch(urlEqualTo("/")) .willReturn(aResponse().withStatus(200)) ) - client + httpClient .PATCH[JsValue, HttpResponse]( - url = s"http://localhost:${server.port()}/", + url = s"$wireMockUrl/", body = Json.obj(), headers = Seq("header" -> "foo") ) .futureValue - server.verify( + verify( patchRequestedFor(urlEqualTo("/")) .withHeader(HeaderNames.authorisation, equalTo("authorization")) .withHeader(HeaderNames.xForwardedFor, equalTo("forwarded-for")) @@ -264,17 +256,18 @@ class HeadersSpec "a put request" should { "contain headers from the header carrier" in { - server.stubFor( + stubFor( put(urlEqualTo("/")) .willReturn(aResponse().withStatus(200)) ) - client.PUT[JsValue, HttpResponse]( - url = s"http://localhost:${server.port()}/", - body = Json.obj() - ).futureValue + httpClient + .PUT[JsValue, HttpResponse]( + url = s"$wireMockUrl/", + body = Json.obj() + ).futureValue - server.verify( + verify( putRequestedFor(urlEqualTo("/")) .withHeader(HeaderNames.authorisation, equalTo("authorization")) .withHeader(HeaderNames.xForwardedFor, equalTo("forwarded-for")) diff --git a/http-verbs-common/src/test/scala/uk/gov/hmrc/http/HttpPatchSpec.scala b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/HttpPatchSpec.scala index b4f843fd..46ad4eec 100644 --- a/http-verbs-common/src/test/scala/uk/gov/hmrc/http/HttpPatchSpec.scala +++ b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/HttpPatchSpec.scala @@ -23,7 +23,7 @@ import org.mockito.captor.ArgCaptor import org.mockito.scalatest.MockitoSugar import org.scalatest.wordspec.AnyWordSpecLike import org.scalatest.matchers.should.Matchers -import play.api.libs.json.{Json, Writes} +import _root_.play.api.libs.json.{Json, Writes} import uk.gov.hmrc.http.hooks.{HookData, HttpHook} import scala.concurrent.{ExecutionContext, Future} diff --git a/http-verbs-common/src/test/scala/uk/gov/hmrc/http/HttpPostSpec.scala b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/HttpPostSpec.scala index 93499aba..dd3a9261 100644 --- a/http-verbs-common/src/test/scala/uk/gov/hmrc/http/HttpPostSpec.scala +++ b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/HttpPostSpec.scala @@ -23,7 +23,7 @@ import org.mockito.captor.ArgCaptor import org.mockito.scalatest.MockitoSugar import org.scalatest.wordspec.AnyWordSpecLike import org.scalatest.matchers.should.Matchers -import play.api.libs.json.{Json, Writes} +import _root_.play.api.libs.json.{Json, Writes} import uk.gov.hmrc.http.hooks.{HookData, HttpHook} import scala.concurrent.{ExecutionContext, Future} diff --git a/http-verbs-common/src/test/scala/uk/gov/hmrc/http/HttpPutSpec.scala b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/HttpPutSpec.scala index ff458422..119151e2 100644 --- a/http-verbs-common/src/test/scala/uk/gov/hmrc/http/HttpPutSpec.scala +++ b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/HttpPutSpec.scala @@ -23,7 +23,7 @@ import org.mockito.captor.ArgCaptor import org.mockito.scalatest.MockitoSugar import org.scalatest.wordspec.AnyWordSpecLike import org.scalatest.matchers.should.Matchers -import play.api.libs.json.{Json, Writes} +import _root_.play.api.libs.json.{Json, Writes} import uk.gov.hmrc.http.hooks.{HookData, HttpHook} import scala.concurrent.{ExecutionContext, Future} diff --git a/http-verbs-common/src/test/scala/uk/gov/hmrc/http/HttpReadsInstancesSpec.scala b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/HttpReadsInstancesSpec.scala index 86e9e4ee..74cf8082 100644 --- a/http-verbs-common/src/test/scala/uk/gov/hmrc/http/HttpReadsInstancesSpec.scala +++ b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/HttpReadsInstancesSpec.scala @@ -21,7 +21,7 @@ import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks import org.scalatest.TryValues import org.scalatest.wordspec.AnyWordSpec import org.scalatest.matchers.should.Matchers -import play.api.libs.json.{__, Json, JsError, JsResult, JsSuccess} +import _root_.play.api.libs.json.{__, Json, JsError, JsResult, JsSuccess} import scala.util.{Failure, Success, Try} import uk.gov.hmrc.http.HttpReads.Implicits._ diff --git a/http-verbs-common/src/test/scala/uk/gov/hmrc/http/HttpReadsLegacyInstancesSpec.scala b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/HttpReadsLegacyInstancesSpec.scala index 85ec218a..f14c3049 100644 --- a/http-verbs-common/src/test/scala/uk/gov/hmrc/http/HttpReadsLegacyInstancesSpec.scala +++ b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/HttpReadsLegacyInstancesSpec.scala @@ -21,7 +21,7 @@ import org.scalacheck.Gen import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks import org.scalatest.wordspec.AnyWordSpec import org.scalatest.matchers.should.Matchers -import play.api.libs.json.Json +import _root_.play.api.libs.json.Json @silent("deprecated") class HttpReadsLegacyInstancesSpec extends AnyWordSpec with ScalaCheckDrivenPropertyChecks with Matchers { diff --git a/http-verbs-common/src/test/scala/uk/gov/hmrc/http/RetriesSpec.scala b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/RetriesSpec.scala index 22f4cdb2..5080cb67 100644 --- a/http-verbs-common/src/test/scala/uk/gov/hmrc/http/RetriesSpec.scala +++ b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/RetriesSpec.scala @@ -28,7 +28,7 @@ import org.scalatest.concurrent.{IntegrationPatience, ScalaFutures} import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpecLike import org.slf4j.MDC -import play.api.libs.json.{JsValue, Json, Writes} +import _root_.play.api.libs.json.{JsValue, Json, Writes} import uk.gov.hmrc.http.HttpReads.Implicits._ import uk.gov.hmrc.http.hooks.{HttpHook, HttpHooks} import uk.gov.hmrc.play.http.logging.Mdc diff --git a/http-verbs-common/src/test/scala/uk/gov/hmrc/http/play/HttpClient2Spec.scala b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/play/HttpClient2Spec.scala new file mode 100644 index 00000000..13828880 --- /dev/null +++ b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/play/HttpClient2Spec.scala @@ -0,0 +1,216 @@ +/* + * Copyright 2022 HM Revenue & Customs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package uk.gov.hmrc.http.play + +import akka.actor.ActorSystem +import akka.stream.{ActorMaterializer, Materializer} +import akka.stream.scaladsl.Source +import akka.util.ByteString +import com.github.tomakehurst.wiremock.client.WireMock +import com.github.tomakehurst.wiremock.client.WireMock._ +import com.typesafe.config.ConfigFactory +import org.scalatest.concurrent.{IntegrationPatience, ScalaFutures} +import org.scalatest.matchers.should.Matchers +import org.scalatest.wordspec.AnyWordSpecLike +import play.api.libs.functional.syntax._ +import play.api.libs.json.{Reads, Writes} +import play.api.libs.ws.ahc.{AhcWSClient, AhcWSClientConfigFactory} +import uk.gov.hmrc.http.{HeaderCarrier, HttpReads, HttpReadsInstances, Retries, StringContextOps, UpstreamErrorResponse} +import uk.gov.hmrc.http.test.WireMockSupport + +import scala.concurrent.Future +import scala.concurrent.ExecutionContext.Implicits.global +import play.api.Configuration +import java.util.concurrent.atomic.AtomicInteger + +class HttpClient2Spec + extends AnyWordSpecLike + with Matchers + with WireMockSupport + with ScalaFutures + with IntegrationPatience { + + implicit val as: ActorSystem = ActorSystem("test-actor-system") + implicit val mat: Materializer = ActorMaterializer() // explicitly required for play-26 + + // this would be injected + val httpClient2: HttpClient2 = { + val config = + Configuration( + ConfigFactory.parseString( + """appName = myapp""" + ).withFallback(ConfigFactory.load()) + ) + new HttpClient2Impl( + wsClient = AhcWSClient(AhcWSClientConfigFactory.forConfig(config.underlying)), + as, + config, + hooks = Seq.empty, // this would be wired up to play-auditing + ) + } + + "api" when { + "with json" should { + "provide body with transformRequest" in { + implicit val hc = HeaderCarrier() + + stubFor( + WireMock.put(urlEqualTo("/")) + .willReturn(aResponse().withBody("\"res\"").withStatus(200)) + ) + + val res: Future[ResDomain] = + httpClient2 + .put(url"$wireMockUrl/") + .transformRequest(_.withBody(toJson(ReqDomain("req")))) + .execute(fromJson[ResDomain]) + + res.futureValue shouldBe ResDomain("res") + + verify( + putRequestedFor(urlEqualTo("/")) + .withRequestBody(equalTo("\"req\"")) + .withHeader("User-Agent", equalTo("myapp")) + ) + } + + "provide body to put" in { + implicit val hc = HeaderCarrier() + + stubFor( + WireMock.put(urlEqualTo("/")) + .willReturn(aResponse().withBody("\"res\"").withStatus(200)) + ) + + val res: Future[ResDomain] = + httpClient2 + .put(url"$wireMockUrl/", toJson(ReqDomain("req"))) + .execute(fromJson[ResDomain]) + + res.futureValue shouldBe ResDomain("res") + + verify( + putRequestedFor(urlEqualTo("/")) + .withRequestBody(equalTo("\"req\"")) + .withHeader("User-Agent", equalTo("myapp")) + ) + } + + "override user-agent" in { + implicit val hc = HeaderCarrier() + + stubFor( + WireMock.put(urlEqualTo("/")) + .willReturn(aResponse().withBody("\"res\"").withStatus(200)) + ) + + val res: Future[ResDomain] = + httpClient2 + .put(url"$wireMockUrl/", toJson(ReqDomain("req"))) + .replaceHeader("User-Agent" -> "ua2") + .execute(fromJson[ResDomain]) + + res.futureValue shouldBe ResDomain("res") + + verify( + putRequestedFor(urlEqualTo("/")) + .withRequestBody(equalTo("\"req\"")) + .withHeader("User-Agent", equalTo("ua2")) + ) + } + } + + "with stream" should { + "work" in { + implicit val hc = HeaderCarrier() + + stubFor( + WireMock.put(urlEqualTo("/")) + .willReturn(aResponse().withBody("\"res\"").withStatus(200)) + ) + + val srcStream: Source[ByteString, _] = + Source.single(ByteString("source")) + + val res: Future[Source[ByteString, _]] = + httpClient2 + .put(url"$wireMockUrl/") + .transformRequest(_.withBody(srcStream)) + .stream(fromStream) + + res.futureValue.map(_.utf8String).runReduce(_ + _).futureValue shouldBe "\"res\"" + + verify( + putRequestedFor(urlEqualTo("/")) + .withRequestBody(equalTo("source")) + .withHeader("User-Agent", equalTo("myapp")) + ) + } + } + + "with custom retries" should { + "work" in { + val retries = new Retries { + override val actorSystem = as + override val configuration = ConfigFactory + .parseString("http-verbs.retries.intervals = [ 100.ms, 200.ms, 300.ms ]") + .withFallback(ConfigFactory.load()) + } + + implicit val hc = HeaderCarrier() + + stubFor( + WireMock.put(urlEqualTo("/")) + .willReturn(aResponse().withBody("\"res\"").withStatus(502)) + ) + + val count = new AtomicInteger(0) + + val res: Future[ResDomain] = + retries.retryFor("get reqdomain"){ case UpstreamErrorResponse.WithStatusCode(502) => true }{ + count.incrementAndGet + httpClient2 + .put(url"$wireMockUrl/", toJson(ReqDomain("req"))) + .execute(fromJson[ResDomain]) + } + + res.failed.futureValue shouldBe a[UpstreamErrorResponse] + count.get shouldBe 4 + } + } + } +} + +case class ReqDomain( + field: String +) + +object ReqDomain { + implicit val w: Writes[ReqDomain] = + implicitly[Writes[String]].contramap(_.field) +} + +case class ResDomain( + field: String +) + +object ResDomain extends HttpReadsInstances { + implicit val r: Reads[ResDomain] = + implicitly[Reads[String]].map(ResDomain.apply) + + implicit val hr: HttpReads[ResDomain] = readFromJson[ResDomain] +} diff --git a/http-verbs-common/src/test/scala/uk/gov/hmrc/play/http/HeaderCarrierConverterSpec.scala b/http-verbs-common/src/test/scala/uk/gov/hmrc/play/http/HeaderCarrierConverterSpec.scala index f84d7465..304d5d41 100644 --- a/http-verbs-common/src/test/scala/uk/gov/hmrc/play/http/HeaderCarrierConverterSpec.scala +++ b/http-verbs-common/src/test/scala/uk/gov/hmrc/play/http/HeaderCarrierConverterSpec.scala @@ -24,6 +24,7 @@ import play.api.inject.guice.GuiceApplicationBuilder import play.api.{Configuration, Play} import play.api.mvc._ import play.api.test.Helpers._ +import _root_.play.api.http.{HeaderNames => PlayHeaderNames} import play.api.test.{FakeHeaders, FakeRequest, FakeRequestFactory} import play.api.mvc.request.RequestFactory import uk.gov.hmrc.http._ @@ -381,7 +382,7 @@ class HeaderCarrierConverterSpec extends AnyWordSpecLike with Matchers with Befo "utilise values from cookies" should { "find the deviceID from the cookie if set in header" in { val cookieHeader = Cookies.encodeCookieHeader(Seq(Cookie(CookieNames.deviceID, "deviceIdCookie"))) - val req = FakeRequest().withHeaders(play.api.http.HeaderNames.COOKIE -> cookieHeader) + val req = FakeRequest().withHeaders(PlayHeaderNames.COOKIE -> cookieHeader) HeaderCarrierConverter.fromHeadersAndSession(req.headers, Some(req.session)) .deviceID shouldBe Some("deviceIdCookie") diff --git a/http-verbs-common/src/test/scala/uk/gov/hmrc/play/http/HttpTimeoutSpec.scala b/http-verbs-common/src/test/scala/uk/gov/hmrc/play/http/HttpTimeoutSpec.scala deleted file mode 100644 index 7804b1a9..00000000 --- a/http-verbs-common/src/test/scala/uk/gov/hmrc/play/http/HttpTimeoutSpec.scala +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright 2022 HM Revenue & Customs - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package uk.gov.hmrc.play.http - -import java.net.{ServerSocket, URI} -import java.util.concurrent.TimeoutException - -import org.scalatest.BeforeAndAfterAll -import org.scalatest.concurrent.{IntegrationPatience, ScalaFutures} -import org.scalatest.matchers.should.Matchers -import org.scalatest.wordspec.AnyWordSpecLike -import org.webbitserver.handler.{DelayedHttpHandler, StringHttpHandler} -import org.webbitserver.netty.NettyWebServer -import play.api.inject.guice.GuiceApplicationBuilder -import play.api.libs.ws.WSClient -import play.api.test.WsTestClient -import play.api.{Application, Configuration, Play} -import uk.gov.hmrc.play.http.ws.WSHttp -import uk.gov.hmrc.play.test.TestHttpCore - -import scala.concurrent.ExecutionContext - -class HttpTimeoutSpec - extends AnyWordSpecLike - with Matchers - with ScalaFutures - with IntegrationPatience - with BeforeAndAfterAll { - - import ExecutionContext.Implicits.global - - private lazy val fakeApplication: Application = - GuiceApplicationBuilder(configuration = Configuration("play.ws.timeout.request" -> "1000ms")).build() - - override def beforeAll() { - super.beforeAll() - Play.start(fakeApplication) - } - - override def afterAll() { - super.afterAll() - Play.stop(fakeApplication) - } - - WsTestClient.withClient { _ => - - "HttpCalls" should { - - "be gracefully timeout when no response is received within the 'timeout' frame" in { - val http: WSHttp = new WSHttp with TestHttpCore { - override val wsClient: WSClient = fakeApplication.injector.instanceOf[WSClient] - } - - // get an unused port - val ss = new ServerSocket(0) - ss.close() - val executor = ExecutionContext.global - val publicUri = URI.create(s"http://localhost:${ss.getLocalPort}") - val ws = new NettyWebServer(executor, ss.getLocalSocketAddress, publicUri) - try { - //starts web server - ws.add( - "/test", - new DelayedHttpHandler(executor, 2000, new StringHttpHandler("application/json", "{name:'pong'}")) - ) - ws.start().get() - - val start = System.currentTimeMillis() - - //make request to web server - http.doPost(s"$publicUri/test", "{name:'ping'}", Seq()).failed.futureValue shouldBe a [TimeoutException] - - val diff = (System.currentTimeMillis() - start).toInt - // there is test execution delay around 700ms - diff should be >= 1000 - diff should be < 2500 - } finally { - ws.stop() - } - } - } - } -} diff --git a/http-verbs-common/src/test/scala/uk/gov/hmrc/play/http/ws/wireMockEndpoints.scala b/http-verbs-common/src/test/scala/uk/gov/hmrc/play/http/ws/wireMockEndpoints.scala deleted file mode 100644 index b4f22a47..00000000 --- a/http-verbs-common/src/test/scala/uk/gov/hmrc/play/http/ws/wireMockEndpoints.scala +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright 2022 HM Revenue & Customs - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package uk.gov.hmrc.play.http.ws - -import java.net.ServerSocket - -import com.github.tomakehurst.wiremock.WireMockServer -import com.github.tomakehurst.wiremock.client.WireMock -import com.github.tomakehurst.wiremock.core.WireMockConfiguration._ - -import scala.util.Try - -trait WireMockEndpoints { - - val host: String = "localhost" - - val endpointPort: Int = PortTester.findPort() - val endpointMock = new WireMock(host, endpointPort) - val endpointServer: WireMockServer = new WireMockServer(wireMockConfig().port(endpointPort)) - - val proxyPort: Int = PortTester.findPort(endpointPort) - val proxyMock: WireMock = new WireMock(host, proxyPort) - val proxyServer: WireMockServer = new WireMockServer(wireMockConfig().port(proxyPort)) - - def withServers(test: => Unit) { - endpointServer.start() - proxyServer.start() - - try { - test - } finally { - Try(endpointServer.stop()) - Try(proxyServer.stop()) - } - } -} - -object PortTester { - - def findPort(excluded: Int*): Int = - (6000 to 7000).find(port => !excluded.contains(port) && isFree(port)).getOrElse(throw new Exception("No free port")) - - private def isFree(port: Int): Boolean = { - val triedSocket = Try { - val serverSocket = new ServerSocket(port) - Try(serverSocket.close()) - serverSocket - } - triedSocket.isSuccess - } -} diff --git a/http-verbs-common/src/test/scala/uk/gov/hmrc/play/test/PortFinder.scala b/http-verbs-common/src/test/scala/uk/gov/hmrc/play/test/PortFinder.scala new file mode 100644 index 00000000..1e792b65 --- /dev/null +++ b/http-verbs-common/src/test/scala/uk/gov/hmrc/play/test/PortFinder.scala @@ -0,0 +1,38 @@ +/* + * Copyright 2022 HM Revenue & Customs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package uk.gov.hmrc.http.test + +import scala.util.Try + +import java.net.ServerSocket + +object PortFinder { + + def findFreePort(portRange: Range, excluded: Int*): Int = + portRange + .find(port => !excluded.contains(port) && isFree(port)) + .getOrElse(throw new Exception("No free port")) + + private def isFree(port: Int): Boolean = { + val triedSocket = Try { + val serverSocket = new ServerSocket(port) + Try(serverSocket.close()) + serverSocket + } + triedSocket.isSuccess + } +} diff --git a/http-verbs-common/src/test/scala/uk/gov/hmrc/play/test/WireMockSupport.scala b/http-verbs-common/src/test/scala/uk/gov/hmrc/play/test/WireMockSupport.scala new file mode 100644 index 00000000..9b83c927 --- /dev/null +++ b/http-verbs-common/src/test/scala/uk/gov/hmrc/play/test/WireMockSupport.scala @@ -0,0 +1,62 @@ +/* + * Copyright 2022 HM Revenue & Customs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package uk.gov.hmrc.http.test + +import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach, Suite} +import com.github.tomakehurst.wiremock._ +import com.github.tomakehurst.wiremock.client.WireMock +import com.github.tomakehurst.wiremock.core.WireMockConfiguration +import org.slf4j.{Logger, LoggerFactory} + +trait WireMockSupport + extends BeforeAndAfterAll + with BeforeAndAfterEach { + this: Suite => + + private val logger: Logger = LoggerFactory.getLogger(getClass) + + lazy val wireMockHost: String = + "localhost" + + lazy val wireMockPort: Int = + // we lookup a port ourselves rather than using `wireMockConfig().dynamicPort()` since it's simpler to provide + // it up front (rather than query the running server), and allow overriding. + PortFinder.findFreePort(portRange = 6001 to 7000) + + lazy val wireMockUrl: String = + s"http://$wireMockHost:$wireMockPort" + + lazy val wireMockServer = + new WireMockServer(WireMockConfiguration.wireMockConfig().port(wireMockPort)) + + override protected def beforeAll(): Unit = { + super.beforeAll() + wireMockServer.start() + WireMock.configureFor(wireMockHost, wireMockServer.port()) + logger.info(s"Started WireMock server on host: $wireMockHost, port: ${wireMockServer.port()}") + } + + override protected def afterAll(): Unit = { + wireMockServer.stop() + super.afterAll() + } + + override protected def beforeEach(): Unit = { + super.beforeEach() + wireMockServer.resetMappings() + } +} diff --git a/http-verbs-common/src/test/scala/uk/gov/hmrc/play/test/helpers.scala b/http-verbs-common/src/test/scala/uk/gov/hmrc/play/test/helpers.scala deleted file mode 100644 index f7540016..00000000 --- a/http-verbs-common/src/test/scala/uk/gov/hmrc/play/test/helpers.scala +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Copyright 2022 HM Revenue & Customs - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package uk.gov.hmrc.play.test - -import play.api.libs.json.Writes -import uk.gov.hmrc.http._ - -import scala.concurrent.{ExecutionContext, Future} - -trait TestHttpCore extends CorePost with CoreGet with CorePut with CorePatch with CoreDelete with Request { - - override def POST[I, O]( - url: String, - body: I, - headers: Seq[(String, String)])( - implicit wts: Writes[I], - rds: HttpReads[O], - hc: HeaderCarrier, - ec: ExecutionContext): Future[O] = - ??? - - override def POSTString[O]( - url: String, - body: String, - headers: Seq[(String, String)])( - implicit rds: HttpReads[O], - hc: HeaderCarrier, - ec: ExecutionContext): Future[O] = - ??? - - override def POSTForm[O]( - url: String, - body: Map[String, Seq[String]], - headers: Seq[(String, String)])( - implicit rds: HttpReads[O], - hc: HeaderCarrier, - ec: ExecutionContext): Future[O] = - ??? - - override def POSTEmpty[O]( - url: String, - headers: Seq[(String, String)])( - implicit rds: HttpReads[O], - hc: HeaderCarrier, - ec: ExecutionContext): Future[O] = - ??? - - override def GET[A]( - url: String, - queryParams: Seq[(String, String)], - headers: Seq[(String, String)])( - implicit rds: HttpReads[A], - hc: HeaderCarrier, - ec: ExecutionContext): Future[A] = - ??? - - override def PUT[I, O]( - url: String, - body: I, - headers: Seq[(String, String)])( - implicit wts: Writes[I], - rds: HttpReads[O], - hc: HeaderCarrier, - ec: ExecutionContext): Future[O] = - ??? - - override def PUTString[O]( - url: String, - body: String, - headers: Seq[(String, String)])( - implicit rds: HttpReads[O], - hc: HeaderCarrier, - ec: ExecutionContext): Future[O] = - ??? - - override def PATCH[I, O]( - url: String, - body: I, - headers: Seq[(String, String)])( - implicit wts: Writes[I], - rds: HttpReads[O], - hc: HeaderCarrier, - ec: ExecutionContext): Future[O] = - ??? - - override def DELETE[O]( - url: String, - headers: Seq[(String, String)])( - implicit rds: HttpReads[O], - hc: HeaderCarrier, - ec: ExecutionContext): Future[O] = - ??? -} diff --git a/http-verbs-test-common/src/main/scala/uk/gov/hmrc/http/test/HttpClient2Support.scala b/http-verbs-test-common/src/main/scala/uk/gov/hmrc/http/test/HttpClient2Support.scala new file mode 100644 index 00000000..ec418a65 --- /dev/null +++ b/http-verbs-test-common/src/main/scala/uk/gov/hmrc/http/test/HttpClient2Support.scala @@ -0,0 +1,46 @@ +/* + * Copyright 2022 HM Revenue & Customs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package uk.gov.hmrc.http.test + +import akka.actor.ActorSystem +import akka.stream.{ActorMaterializer, Materializer} +import com.github.ghik.silencer.silent +import com.typesafe.config.ConfigFactory +import play.api.Configuration +import play.api.libs.ws.ahc.{AhcWSClient, AhcWSClientConfigFactory} +import uk.gov.hmrc.http.play.{HttpClient2, HttpClient2Impl} + +trait HttpClient2Support { + + def mkHttpClient2( + config: Configuration = Configuration(ConfigFactory.load()) + ): HttpClient2 = { + implicit val as: ActorSystem = ActorSystem("test-actor-system") + + @silent("deprecated") + implicit val mat: Materializer = ActorMaterializer() // explicitly required for play-26 + + new HttpClient2Impl( + wsClient = AhcWSClient(AhcWSClientConfigFactory.forConfig(config.underlying)), + as, + config, + hooks = Seq.empty, + ) + } + + lazy val httpClient2: HttpClient2 = mkHttpClient2() +} diff --git a/http-verbs-test-common/src/main/scala/uk/gov/hmrc/http/test/HttpClientSupport.scala b/http-verbs-test-common/src/main/scala/uk/gov/hmrc/http/test/HttpClientSupport.scala index ef6e4b42..9ae8d1ec 100644 --- a/http-verbs-test-common/src/main/scala/uk/gov/hmrc/http/test/HttpClientSupport.scala +++ b/http-verbs-test-common/src/main/scala/uk/gov/hmrc/http/test/HttpClientSupport.scala @@ -21,7 +21,7 @@ import akka.stream.{ActorMaterializer, Materializer} import com.github.ghik.silencer.silent import com.typesafe.config.{Config, ConfigFactory} import play.api.libs.ws.WSClient -import uk.gov.hmrc.http._ +import uk.gov.hmrc.http.HttpClient import uk.gov.hmrc.http.hooks.HttpHook import uk.gov.hmrc.play.http.ws.WSHttp import play.api.libs.ws.ahc.{AhcWSClient, AhcWSClientConfigFactory} From a4d3ffa49e20fb42a0835a66f13d170efa0f5ec0 Mon Sep 17 00:00:00 2001 From: colin-lamed <9568290+colin-lamed@users.noreply.github.com> Date: Fri, 22 Oct 2021 11:46:25 +0100 Subject: [PATCH 03/23] BDOG-1512 Reduce implementation surface area of HttpClient2 (drops overloaded body variants) --- .../scala/uk/gov/hmrc/http/play/Example.scala | 8 +- .../uk/gov/hmrc/http/play/HttpClient2.scala | 92 +++++++------------ .../gov/hmrc/http/play/HttpClient2Spec.scala | 34 ++----- 3 files changed, 47 insertions(+), 87 deletions(-) diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/Example.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/Example.scala index 5bbb969c..b15c7cb7 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/Example.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/Example.scala @@ -57,7 +57,8 @@ object Example { val _: Future[ResDomain] = httpClient2 - .put(url"http://localhost:8000/", toJson(ReqDomain())) + .put(url"http://localhost:8000/") + .withBody(toJson(ReqDomain())) .withProxy .replaceHeader("User-Agent" -> "ua") @@ -76,7 +77,7 @@ object Example { //.put(url"http://localhost:8000/", srcStream) .put(url"http://localhost:8000/") .withBody(srcStream) - .transformRequest(_.withRequestTimeout(10.seconds)) + .transform(_.withRequestTimeout(10.seconds)) .stream(fromStream) } @@ -92,7 +93,8 @@ object Example { val _: Future[ResDomain] = retries.retryFor("get reqdomain"){ case UpstreamErrorResponse.WithStatusCode(502) => true }{ httpClient2 - .put(url"http://localhost:8000/", toJson(ReqDomain())) + .put(url"http://localhost:8000/") + .withBody(toJson(ReqDomain())) .withProxy .replaceHeader("User-Agent" -> "ua") .execute(fromJson[ResDomain]) diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/HttpClient2.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/HttpClient2.scala index 5317b19b..6b4dce1f 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/HttpClient2.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/HttpClient2.scala @@ -53,29 +53,32 @@ Extension methods are provided to make common patterns easier to apply. */ trait HttpClient2 { - def get(url: URL)(implicit hc: HeaderCarrier): RequestBuilder + protected def mkRequestBuilder(url: URL, method: String)(implicit hc: HeaderCarrier): RequestBuilder - def post(url: URL)(implicit hc: HeaderCarrier): RequestBuilder - - def post[B: BodyWritable](url: URL, body: B)(implicit hc: HeaderCarrier): RequestBuilder - - def put(url: URL)(implicit hc: HeaderCarrier): RequestBuilder + def get(url: URL)(implicit hc: HeaderCarrier): RequestBuilder = + mkRequestBuilder(url, "GET") - def put[B: BodyWritable](url: URL, body: B)(implicit hc: HeaderCarrier): RequestBuilder + def post(url: URL)(implicit hc: HeaderCarrier): RequestBuilder = + mkRequestBuilder(url, "POST") - def delete(url: URL)(implicit hc: HeaderCarrier): RequestBuilder + def put(url: URL)(implicit hc: HeaderCarrier): RequestBuilder = + mkRequestBuilder(url, "PUT") - def patch(url: URL)(implicit hc: HeaderCarrier): RequestBuilder + def delete(url: URL)(implicit hc: HeaderCarrier): RequestBuilder = + mkRequestBuilder(url, "DELETE") - def patch[B: BodyWritable](url: URL, body: B)(implicit hc: HeaderCarrier): RequestBuilder + def patch(url: URL)(implicit hc: HeaderCarrier): RequestBuilder = + mkRequestBuilder(url, "PATCH") - def head(url: URL)(implicit hc: HeaderCarrier): RequestBuilder + def head(url: URL)(implicit hc: HeaderCarrier): RequestBuilder = + mkRequestBuilder(url, "HEAD") - def options(url: URL)(implicit hc: HeaderCarrier): RequestBuilder + def options(url: URL)(implicit hc: HeaderCarrier): RequestBuilder = + mkRequestBuilder(url, "OPTIONS") } trait RequestBuilder { - def transformRequest(transform: WSRequest => WSRequest): RequestBuilder + def transform(transform: WSRequest => WSRequest): RequestBuilder def execute[A]( transform: WSRequest => WSResponse => Future[A] @@ -113,41 +116,7 @@ class HttpClient2Impl( private lazy val hcConfig = HeaderCarrier.Config.fromConfig(config.underlying) - override def get(url: URL)(implicit hc: HeaderCarrier): RequestBuilderImpl = - mkRequestBuilder(url, "GET") - - override def post(url: URL)(implicit hc: HeaderCarrier): RequestBuilderImpl = - mkRequestBuilder(url, "POST") - - // TODO or just let clients call `.withBody(body)` themselves (this is the expectation for adding headers) - override def post[B: BodyWritable](url: URL, body: B)(implicit hc: HeaderCarrier): RequestBuilderImpl = - mkRequestBuilder(url, "POST") - .withBody(body) - - override def put(url: URL)(implicit hc: HeaderCarrier): RequestBuilderImpl = - mkRequestBuilder(url, "PUT") - - override def put[B: BodyWritable](url: URL, body: B)(implicit hc: HeaderCarrier): RequestBuilderImpl = - mkRequestBuilder(url, "PUT") - .withBody(body) - - override def delete(url: URL)(implicit hc: HeaderCarrier): RequestBuilderImpl = - mkRequestBuilder(url, "DELETE") - - override def patch(url: URL)(implicit hc: HeaderCarrier): RequestBuilderImpl = - mkRequestBuilder(url, "PATCH") - - override def patch[B: BodyWritable](url: URL, body: B)(implicit hc: HeaderCarrier): RequestBuilderImpl = - mkRequestBuilder(url, "PATCH") - .withBody(body) - - override def head(url: URL)(implicit hc: HeaderCarrier): RequestBuilderImpl = - mkRequestBuilder(url, "HEAD") - - override def options(url: URL)(implicit hc: HeaderCarrier): RequestBuilderImpl = - mkRequestBuilder(url, "OPTIONS") - - private def mkRequestBuilder( + override protected def mkRequestBuilder( url : URL, method: String )(implicit @@ -181,37 +150,37 @@ final class RequestBuilderImpl( with Retries with ConnectionTracing { - // for Retries (TODO make it use Configuration too) + // for Retries override val configuration: Config = config.underlying - override def transformRequest(transform: WSRequest => WSRequest): RequestBuilderImpl = + override def transform(transform: WSRequest => WSRequest): RequestBuilderImpl = new RequestBuilderImpl(actorSystem, config, optProxyServer, hooks)(transform(request)) // -- Syntactic sugar -- // TODO any implementation would be expected to implement them all // should they be available as extension methods? (less discoverable - require `import httpClient._` to enable) - // they all depend on `transformRequest` - but also configuration (and other derived/cached values like proxyServer) could they move into the API interface? + // they all depend on `transform` - but also configuration (and other derived/cached values like proxyServer) could they move into the API interface? // also the variable RequestBuilder makes this tricky... override def replaceHeader(header: (String, String)): RequestBuilderImpl = { def denormalise(hdrs: Map[String, Seq[String]]): Seq[(String, String)] = hdrs.toList.flatMap { case (k, vs) => vs.map(k -> _) } val hdrsWithoutKey = request.headers.filterKeys(!_.equalsIgnoreCase(header._1)) // replace existing header - transformRequest(_.withHttpHeaders(denormalise(hdrsWithoutKey) :+ header : _*)) + transform(_.withHttpHeaders(denormalise(hdrsWithoutKey) :+ header : _*)) } override def addHeaders(headers: (String, String)*): RequestBuilderImpl = - transformRequest(_.addHttpHeaders(headers: _*)) + transform(_.addHttpHeaders(headers: _*)) override def withProxy: RequestBuilderImpl = - transformRequest(request => optProxyServer.foldLeft(request)(_ withProxyServer _)) + transform(request => optProxyServer.foldLeft(request)(_ withProxyServer _)) override def withBody[B : BodyWritable](body: B): RequestBuilderImpl = (if (body == EmptyBody) replaceHeader(play.api.http.HeaderNames.CONTENT_LENGTH -> "0") // rejected by Akami without a Content-Length (https://jira.tools.tax.service.gov.uk/browse/APIS-5100) else this - ).transformRequest(_.withBody(body)) + ).transform(_.withBody(body)) // -- Execution -- @@ -229,16 +198,16 @@ final class RequestBuilderImpl( ): Future[A] = execute(isStream = true)(transformResponse) + // should the execute function be provided from WSClient? It could make mocking easier? private def execute[A]( isStream: Boolean )( - transformResponse: WSRequest => WSResponse => Future[A] + transformResponse: WSRequest => WSResponse => Future[A] // This isn't mock friendly? )(implicit ec: ExecutionContext ): Future[A] = { val startAge = System.nanoTime() - hc.age val responseF = - // TODO a way for clients to define custom retries? retryOnSslEngineClosed(request.method, request.url)( // we do the execution since if clients are responsable for it (e.g. a callback), they may further modify the request outside of auditing etc. if (isStream) request.stream() else request.execute() @@ -254,6 +223,7 @@ final class RequestBuilderImpl( // mapErrors could be part of the transform function. e.g. transformResponse: WSRequest => Try[WSResponse] => Future // but then each transformResponse would probably end up doing the same recovery? Is that a problem? + // what if clients what to change it? RequestBuilder is difficult to extend (RequestBuilderImpl)... def mapErrors( httpMethod: String, url : String, @@ -292,6 +262,7 @@ final class RequestBuilderImpl( def denormalise(hdrs: Map[String, Seq[String]]): Seq[(String, String)] = hdrs.toList.flatMap { case (k, vs) => vs.map(k -> _) } + // TODO discuss changes with CIP val body = request.body match { case EmptyBody => None @@ -303,6 +274,13 @@ final class RequestBuilderImpl( case SourceBody(_) => Some(HookData.FromString("")) } + println(s"""AUDIT: + verb = ${request.method}, + url = ${new URL(request.url)}, + headers = ${denormalise(request.headers)}, + body = $body, + responseF = ${responseF.map(toHttpResponse)} + """) hooks.foreach( _.apply( verb = request.method, diff --git a/http-verbs-common/src/test/scala/uk/gov/hmrc/http/play/HttpClient2Spec.scala b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/play/HttpClient2Spec.scala index 13828880..56797409 100644 --- a/http-verbs-common/src/test/scala/uk/gov/hmrc/http/play/HttpClient2Spec.scala +++ b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/play/HttpClient2Spec.scala @@ -65,7 +65,7 @@ class HttpClient2Spec "api" when { "with json" should { - "provide body with transformRequest" in { + "provide body" in { implicit val hc = HeaderCarrier() stubFor( @@ -76,29 +76,7 @@ class HttpClient2Spec val res: Future[ResDomain] = httpClient2 .put(url"$wireMockUrl/") - .transformRequest(_.withBody(toJson(ReqDomain("req")))) - .execute(fromJson[ResDomain]) - - res.futureValue shouldBe ResDomain("res") - - verify( - putRequestedFor(urlEqualTo("/")) - .withRequestBody(equalTo("\"req\"")) - .withHeader("User-Agent", equalTo("myapp")) - ) - } - - "provide body to put" in { - implicit val hc = HeaderCarrier() - - stubFor( - WireMock.put(urlEqualTo("/")) - .willReturn(aResponse().withBody("\"res\"").withStatus(200)) - ) - - val res: Future[ResDomain] = - httpClient2 - .put(url"$wireMockUrl/", toJson(ReqDomain("req"))) + .withBody(toJson(ReqDomain("req"))) .execute(fromJson[ResDomain]) res.futureValue shouldBe ResDomain("res") @@ -120,7 +98,8 @@ class HttpClient2Spec val res: Future[ResDomain] = httpClient2 - .put(url"$wireMockUrl/", toJson(ReqDomain("req"))) + .put(url"$wireMockUrl/") + .withBody(toJson(ReqDomain("req"))) .replaceHeader("User-Agent" -> "ua2") .execute(fromJson[ResDomain]) @@ -149,7 +128,7 @@ class HttpClient2Spec val res: Future[Source[ByteString, _]] = httpClient2 .put(url"$wireMockUrl/") - .transformRequest(_.withBody(srcStream)) + .withBody(srcStream) .stream(fromStream) res.futureValue.map(_.utf8String).runReduce(_ + _).futureValue shouldBe "\"res\"" @@ -184,7 +163,8 @@ class HttpClient2Spec retries.retryFor("get reqdomain"){ case UpstreamErrorResponse.WithStatusCode(502) => true }{ count.incrementAndGet httpClient2 - .put(url"$wireMockUrl/", toJson(ReqDomain("req"))) + .put(url"$wireMockUrl/") + .withBody(toJson(ReqDomain("req"))) .execute(fromJson[ResDomain]) } From c2b7fdb48805fae21edb5f7a2efac71d4ca1350b Mon Sep 17 00:00:00 2001 From: colin-lamed <9568290+colin-lamed@users.noreply.github.com> Date: Fri, 22 Oct 2021 13:40:46 +0100 Subject: [PATCH 04/23] BDOG-1512 Split interface and implementation files and refactor --- .../scala/uk/gov/hmrc/http/play/Example.scala | 116 --------- .../uk/gov/hmrc/http/play/HttpClient2.scala | 228 +---------------- .../gov/hmrc/http/play/HttpClient2Impl.scala | 233 ++++++++++++++++++ .../hmrc/http/play/ResponseTransformers.scala | 66 +++++ .../scala/uk/gov/hmrc/http/play/package.scala | 32 +-- .../gov/hmrc/http/play/HttpClient2Spec.scala | 8 +- 6 files changed, 312 insertions(+), 371 deletions(-) delete mode 100644 http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/Example.scala create mode 100644 http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/HttpClient2Impl.scala create mode 100644 http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/ResponseTransformers.scala diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/Example.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/Example.scala deleted file mode 100644 index b15c7cb7..00000000 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/Example.scala +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Copyright 2022 HM Revenue & Customs - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package uk.gov.hmrc.http.play - -import akka.actor.ActorSystem -import akka.stream.{ActorMaterializer, Materializer} -import akka.stream.scaladsl.Source -import akka.util.ByteString -import com.typesafe.config.ConfigFactory -import play.api.Configuration -import play.api.libs.json.{Reads, Writes} -import play.api.libs.ws.WSClient -import play.api.libs.ws.ahc.{AhcWSClient, AhcWSClientConfigFactory} -import uk.gov.hmrc.http.{HeaderCarrier, HttpReads, Retries, StringContextOps, UpstreamErrorResponse} - -import scala.concurrent.Future -import scala.concurrent.duration.DurationInt - - -/** Demonstrates usage */ -object Example { - import scala.concurrent.ExecutionContext.Implicits.global - - // this would be injected - val httpClient2: HttpClient2 = { - implicit val as: ActorSystem = ActorSystem("test-actor-system") - val config = Configuration(ConfigFactory.load()) - val wsClient: WSClient = { - implicit val mat: Materializer = ActorMaterializer() // explicitly required for play-26 - AhcWSClient(AhcWSClientConfigFactory.forConfig(config.underlying)) - } - new HttpClient2Impl( - wsClient, - as, - config, - hooks = Seq.empty, // this would be wired up to play-auditing - ) - } - - // json - { - implicit val hc = HeaderCarrier() - - val _: Future[ResDomain] = - httpClient2 - .put(url"http://localhost:8000/") - .withBody(toJson(ReqDomain())) - .withProxy - .replaceHeader("User-Agent" -> "ua") - - .execute(fromJson[ResDomain]) - } - - - // streams - { - implicit val hc = HeaderCarrier() - - val srcStream: Source[ByteString, _] = ??? - - val _: Future[Source[ByteString, _]] = - httpClient2 - //.put(url"http://localhost:8000/", srcStream) - .put(url"http://localhost:8000/") - .withBody(srcStream) - .transform(_.withRequestTimeout(10.seconds)) - .stream(fromStream) - } - - // retries - { - val retries = new Retries { - override val actorSystem = ActorSystem("test-actor-system") - override val configuration = ConfigFactory.load() - } - - implicit val hc = HeaderCarrier() - - val _: Future[ResDomain] = - retries.retryFor("get reqdomain"){ case UpstreamErrorResponse.WithStatusCode(502) => true }{ - httpClient2 - .put(url"http://localhost:8000/") - .withBody(toJson(ReqDomain())) - .withProxy - .replaceHeader("User-Agent" -> "ua") - .execute(fromJson[ResDomain]) - } - } -} - -case class ReqDomain() - -object ReqDomain { - implicit val w: Writes[ReqDomain] = ??? -} - -case class ResDomain() - -object ResDomain { - implicit val r : Reads[ResDomain] = ??? - implicit val hr: HttpReads[ResDomain] = ??? -} diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/HttpClient2.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/HttpClient2.scala index 6b4dce1f..6c5586fd 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/HttpClient2.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/HttpClient2.scala @@ -18,40 +18,12 @@ // `import uk.gov.hmrc.http._` will then have to make play imports with _root_ `import _root_.play...` package uk.gov.hmrc.http.play -import akka.actor.ActorSystem -import com.typesafe.config.Config -import play.api.Configuration -import play.api.libs.ws.{BodyWritable, EmptyBody, InMemoryBody, SourceBody, WSClient, WSProxyServer, WSRequest, WSResponse} -import play.core.parsers.FormUrlEncodedParser -import uk.gov.hmrc.http.{BadGatewayException, GatewayTimeoutException, HeaderCarrier, HttpResponse, Retries} -import uk.gov.hmrc.play.http.ws.WSProxyConfiguration -import uk.gov.hmrc.http.hooks.{HookData, HttpHook} -import uk.gov.hmrc.http.logging.ConnectionTracing +import play.api.libs.ws.{BodyWritable, WSRequest, WSResponse} +import uk.gov.hmrc.http.HeaderCarrier -import java.net.{ConnectException, URL} -import java.util.concurrent.TimeoutException +import java.net.URL import scala.concurrent.{ExecutionContext, Future} -/* What does HttpVerbs actually provide? - -Readme says... - - Http Transport - - Core Http function interfaces - - Logging - - Propagation of common headers - - Executing hooks, for example Auditing - - Request & Response de-serializations - - Response handling, converting failure status codes into a consistent set of exceptions - allows failures to be automatically propagated to the caller - -Also, retries - - -This version demonstrates a flat implementation that uses an HttpExecutor to centralise the execution of the request to ensure that -the common concerns occur, but delegates out the construction of the request and parsing of the response to play-ws for flexibility. -The use of HttpReads is optional. -Extension methods are provided to make common patterns easier to apply. -*/ - trait HttpClient2 { protected def mkRequestBuilder(url: URL, method: String)(implicit hc: HeaderCarrier): RequestBuilder @@ -81,13 +53,13 @@ trait RequestBuilder { def transform(transform: WSRequest => WSRequest): RequestBuilder def execute[A]( - transform: WSRequest => WSResponse => Future[A] + transformResponse: (WSRequest, Future[WSResponse]) => Future[A] )(implicit ec: ExecutionContext ): Future[A] def stream[A]( - transform: WSRequest => WSResponse => Future[A] + transformResponse: (WSRequest, Future[WSResponse]) => Future[A] )(implicit ec: ExecutionContext ): Future[A] @@ -102,193 +74,3 @@ trait RequestBuilder { def withBody[B : BodyWritable](body: B): RequestBuilder } - -class HttpClient2Impl( - wsClient : WSClient, - actorSystem: ActorSystem, - config : Configuration, - hooks : Seq[HttpHook] -) extends HttpClient2 { - - private lazy val optProxyServer = - WSProxyConfiguration.buildWsProxyServer(config.underlying) - - private lazy val hcConfig = - HeaderCarrier.Config.fromConfig(config.underlying) - - override protected def mkRequestBuilder( - url : URL, - method: String - )(implicit - hc: HeaderCarrier - ): RequestBuilderImpl = - new RequestBuilderImpl( - actorSystem, - config, - optProxyServer, - hooks - )( - wsClient - .url(url.toString) - .withMethod(method) - .withHttpHeaders(hc.headersForUrl(hcConfig)(url.toString) : _*) - ) - } - - -// is final since tranformRequest (and derived) return instances of RequestBuilderImpl, and would loose any overrides. -final class RequestBuilderImpl( - override val actorSystem: ActorSystem, - config : Configuration, - optProxyServer : Option[WSProxyServer], - hooks : Seq[HttpHook] -)( - request: WSRequest -)(implicit - hc: HeaderCarrier -) extends RequestBuilder - with Retries - with ConnectionTracing { - - // for Retries - override val configuration: Config = config.underlying - - override def transform(transform: WSRequest => WSRequest): RequestBuilderImpl = - new RequestBuilderImpl(actorSystem, config, optProxyServer, hooks)(transform(request)) - - // -- Syntactic sugar -- - // TODO any implementation would be expected to implement them all - // should they be available as extension methods? (less discoverable - require `import httpClient._` to enable) - // they all depend on `transform` - but also configuration (and other derived/cached values like proxyServer) could they move into the API interface? - // also the variable RequestBuilder makes this tricky... - - override def replaceHeader(header: (String, String)): RequestBuilderImpl = { - def denormalise(hdrs: Map[String, Seq[String]]): Seq[(String, String)] = - hdrs.toList.flatMap { case (k, vs) => vs.map(k -> _) } - val hdrsWithoutKey = request.headers.filterKeys(!_.equalsIgnoreCase(header._1)) // replace existing header - transform(_.withHttpHeaders(denormalise(hdrsWithoutKey) :+ header : _*)) - } - - override def addHeaders(headers: (String, String)*): RequestBuilderImpl = - transform(_.addHttpHeaders(headers: _*)) - - override def withProxy: RequestBuilderImpl = - transform(request => optProxyServer.foldLeft(request)(_ withProxyServer _)) - - override def withBody[B : BodyWritable](body: B): RequestBuilderImpl = - (if (body == EmptyBody) - replaceHeader(play.api.http.HeaderNames.CONTENT_LENGTH -> "0") // rejected by Akami without a Content-Length (https://jira.tools.tax.service.gov.uk/browse/APIS-5100) - else - this - ).transform(_.withBody(body)) - - // -- Execution -- - - override def execute[A]( - transformResponse: WSRequest => WSResponse => Future[A] - )(implicit - ec: ExecutionContext - ): Future[A] = - execute(isStream = false)(transformResponse) - - override def stream[A]( - transformResponse: WSRequest => WSResponse => Future[A] - )(implicit - ec: ExecutionContext - ): Future[A] = - execute(isStream = true)(transformResponse) - - // should the execute function be provided from WSClient? It could make mocking easier? - private def execute[A]( - isStream: Boolean - )( - transformResponse: WSRequest => WSResponse => Future[A] // This isn't mock friendly? - )(implicit - ec: ExecutionContext - ): Future[A] = { - val startAge = System.nanoTime() - hc.age - val responseF = - retryOnSslEngineClosed(request.method, request.url)( - // we do the execution since if clients are responsable for it (e.g. a callback), they may further modify the request outside of auditing etc. - if (isStream) request.stream() else request.execute() - ) - executeHooks(isStream, request, responseF) - responseF.onComplete(logResult(hc, request.method, request.uri.toString, startAge)) - // we don't delegate the response conversion to the client - // e.g. execute[WSResponse].transform(...) since the transform functions require access to the request (method and url) - // given method and url are only required for error messages, is it overkill? E.g. a stacktrace should identify the function? - mapErrors(request.method, request.url, responseF) - responseF.flatMap(transformResponse(request)) - } - - // mapErrors could be part of the transform function. e.g. transformResponse: WSRequest => Try[WSResponse] => Future - // but then each transformResponse would probably end up doing the same recovery? Is that a problem? - // what if clients what to change it? RequestBuilder is difficult to extend (RequestBuilderImpl)... - def mapErrors( - httpMethod: String, - url : String, - f : Future[WSResponse] - )(implicit - ec: ExecutionContext - ): Future[WSResponse] = - f.recoverWith { - case e: TimeoutException => Future.failed(new GatewayTimeoutException(gatewayTimeoutMessage(httpMethod, url, e))) - case e: ConnectException => Future.failed(new BadGatewayException(badGatewayMessage(httpMethod, url, e))) - } - - def badGatewayMessage(verbName: String, url: String, e: Exception): String = - s"$verbName of '$url' failed. Caused by: '${e.getMessage}'" - - def gatewayTimeoutMessage(verbName: String, url: String, e: Exception): String = - s"$verbName of '$url' timed out with message '${e.getMessage}'" - - - private def executeHooks( - isStream : Boolean, - request : WSRequest, - responseF: Future[WSResponse] - )(implicit - hc: HeaderCarrier, - ec: ExecutionContext - ): Unit = { - // hooks take HttpResponse.. - def toHttpResponse(response: WSResponse) = - HttpResponse( - status = response.status, - body = if (isStream) "" else response.body, // calling response.body on stream would load all into memory (and cause stream to be read twice..) - headers = response.headers - ) - - def denormalise(hdrs: Map[String, Seq[String]]): Seq[(String, String)] = - hdrs.toList.flatMap { case (k, vs) => vs.map(k -> _) } - - // TODO discuss changes with CIP - val body = - request.body match { - case EmptyBody => None - case InMemoryBody(bytes) => request.header("Content-Type") match { - case Some("application/x-www-form-urlencoded") => Some(HookData.FromMap(FormUrlEncodedParser.parse(bytes.decodeString("UTF-8")))) - case Some("application/octet-stream") => Some(HookData.FromString("")) - case _ => Some(HookData.FromString(bytes.decodeString("UTF-8"))) - } - case SourceBody(_) => Some(HookData.FromString("")) - } - - println(s"""AUDIT: - verb = ${request.method}, - url = ${new URL(request.url)}, - headers = ${denormalise(request.headers)}, - body = $body, - responseF = ${responseF.map(toHttpResponse)} - """) - hooks.foreach( - _.apply( - verb = request.method, - url = new URL(request.url), - headers = denormalise(request.headers), - body = body, - responseF = responseF.map(toHttpResponse) - ) - ) - } -} diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/HttpClient2Impl.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/HttpClient2Impl.scala new file mode 100644 index 00000000..4224397c --- /dev/null +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/HttpClient2Impl.scala @@ -0,0 +1,233 @@ +/* + * Copyright 2021 HM Revenue & Customs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// TODO putting this in this package means that all clients which do +// `import uk.gov.hmrc.http._` will then have to make play imports with _root_ `import _root_.play...` +package uk.gov.hmrc.http.play + +import akka.actor.ActorSystem +import com.typesafe.config.Config +import play.api.Configuration +import play.api.libs.ws.{BodyWritable, EmptyBody, InMemoryBody, SourceBody, WSClient, WSProxyServer, WSRequest, WSResponse} +import play.core.parsers.FormUrlEncodedParser +import uk.gov.hmrc.http.{HeaderCarrier, HttpResponse, Retries} +import uk.gov.hmrc.play.http.ws.WSProxyConfiguration +import uk.gov.hmrc.http.hooks.{HookData, HttpHook} +import uk.gov.hmrc.http.logging.ConnectionTracing + +import java.net.URL +import scala.concurrent.{ExecutionContext, Future} + +/* What does HttpVerbs actually provide? + +Readme says... + - Http Transport + - Core Http function interfaces + - Logging + - Propagation of common headers + - Executing hooks, for example Auditing + - Request & Response de-serializations + - Response handling, converting failure status codes into a consistent set of exceptions - allows failures to be automatically propagated to the caller + +Also, retries + + +This version demonstrates a flat implementation that uses an HttpExecutor to centralise the execution of the request to ensure that +the common concerns occur, but delegates out the construction of the request and parsing of the response to play-ws for flexibility. +The use of HttpReads is optional. +Extension methods are provided to make common patterns easier to apply. +*/ + +trait Executor { + def execute[A]( + request : WSRequest, + isStream: Boolean + )( + transformResponse: (WSRequest, Future[WSResponse]) => Future[A] + )(implicit + hc: HeaderCarrier, + ec: ExecutionContext + ): Future[A] +} + +class HttpClient2Impl( + wsClient : WSClient, + actorSystem: ActorSystem, + config : Configuration, + hooks : Seq[HttpHook] +) extends HttpClient2 { + + private lazy val optProxyServer = + WSProxyConfiguration.buildWsProxyServer(config.underlying) + + private lazy val hcConfig = + HeaderCarrier.Config.fromConfig(config.underlying) + + override protected def mkRequestBuilder( + url : URL, + method: String + )(implicit + hc: HeaderCarrier + ): RequestBuilderImpl = + new RequestBuilderImpl( + config, + optProxyServer, + new ExecutorImpl(actorSystem, config, hooks) + )( + wsClient + .url(url.toString) + .withMethod(method) + .withHttpHeaders(hc.headersForUrl(hcConfig)(url.toString) : _*) + ) +} + + +// is final since `tranform` (and derived functions) return instances of RequestBuilderImpl, and any overrides would be lost. +final class RequestBuilderImpl( + config : Configuration, + optProxyServer: Option[WSProxyServer], + executor : Executor +)( + request: WSRequest +)(implicit + hc: HeaderCarrier +) extends RequestBuilder { + + override def transform(transform: WSRequest => WSRequest): RequestBuilderImpl = + new RequestBuilderImpl(config, optProxyServer, executor)(transform(request)) + + // -- Transform helpers -- + + override def replaceHeader(header: (String, String)): RequestBuilderImpl = { + def denormalise(hdrs: Map[String, Seq[String]]): Seq[(String, String)] = + hdrs.toList.flatMap { case (k, vs) => vs.map(k -> _) } + val hdrsWithoutKey = request.headers.filterKeys(!_.equalsIgnoreCase(header._1)).toMap // replace existing header + transform(_.withHttpHeaders(denormalise(hdrsWithoutKey) :+ header : _*)) + } + + override def addHeaders(headers: (String, String)*): RequestBuilderImpl = + transform(_.addHttpHeaders(headers: _*)) + + override def withProxy: RequestBuilderImpl = + transform(request => optProxyServer.foldLeft(request)(_ withProxyServer _)) + + override def withBody[B : BodyWritable](body: B): RequestBuilderImpl = + (if (body == EmptyBody) + replaceHeader(play.api.http.HeaderNames.CONTENT_LENGTH -> "0") // rejected by Akami without a Content-Length (https://jira.tools.tax.service.gov.uk/browse/APIS-5100) + else + this + ).transform(_.withBody(body)) + + // -- Execution -- + + override def execute[A]( + transformResponse: (WSRequest, Future[WSResponse]) => Future[A] + )(implicit + ec: ExecutionContext + ): Future[A] = + executor.execute(request, isStream = false)(transformResponse) + + override def stream[A]( + transformResponse: (WSRequest, Future[WSResponse]) => Future[A] + )(implicit + ec: ExecutionContext + ): Future[A] = + executor.execute(request, isStream = true)(transformResponse) +} + +class ExecutorImpl( + override val actorSystem: ActorSystem, // for Retries + config: Configuration, + hooks : Seq[HttpHook] +) extends Executor + with Retries + with ConnectionTracing { + + // for Retries + override val configuration: Config = config.underlying + + def execute[A]( + request : WSRequest, + isStream: Boolean + )( + transformResponse: (WSRequest, Future[WSResponse]) => Future[A] + )(implicit + hc: HeaderCarrier, + ec: ExecutionContext + ): Future[A] = { + val startAge = System.nanoTime() - hc.age + val responseF = + retryOnSslEngineClosed(request.method, request.url)( + // we do the execution since if clients are responsable for it (e.g. a callback), they may further modify the request outside of auditing etc. + if (isStream) request.stream() else request.execute() + ) + executeHooks(isStream, request, responseF) + responseF.onComplete(logResult(hc, request.method, request.uri.toString, startAge)) + // we don't delegate the response conversion to the client + // e.g. execute[WSResponse].transform(...) since the transform functions require access to the request (method and url) + // given method and url are only required for error messages, is it overkill? E.g. a stacktrace should identify the function? + transformResponse(request, responseF) + } + + private def executeHooks( + isStream : Boolean, + request : WSRequest, + responseF: Future[WSResponse] + )(implicit + hc: HeaderCarrier, + ec: ExecutionContext + ): Unit = { + // hooks take HttpResponse.. + def toHttpResponse(response: WSResponse) = + HttpResponse( + status = response.status, + body = if (isStream) "" else response.body, // calling response.body on stream would load all into memory (and cause stream to be read twice..) + headers = response.headers.mapValues(_.toSeq).toMap + ) + + def denormalise(hdrs: Map[String, Seq[String]]): Seq[(String, String)] = + hdrs.toList.flatMap { case (k, vs) => vs.map(k -> _) } + + // TODO discuss changes with CIP + val body = + request.body match { + case EmptyBody => None + case InMemoryBody(bytes) => request.header("Content-Type") match { + case Some("application/x-www-form-urlencoded") => Some(HookData.FromMap(FormUrlEncodedParser.parse(bytes.decodeString("UTF-8")))) + case Some("application/octet-stream") => Some(HookData.FromString("")) + case _ => Some(HookData.FromString(bytes.decodeString("UTF-8"))) + } + case SourceBody(_) => Some(HookData.FromString("")) + } + + println(s"""AUDIT: + verb = ${request.method}, + url = ${new URL(request.url)}, + headers = ${denormalise(request.headers)}, + body = $body, + responseF = ${responseF.map(toHttpResponse)} + """) + hooks.foreach( + _.apply( + verb = request.method, + url = new URL(request.url), + headers = denormalise(request.headers), + body = body, + responseF = responseF.map(toHttpResponse) + ) + ) + } +} diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/ResponseTransformers.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/ResponseTransformers.scala new file mode 100644 index 00000000..4583d786 --- /dev/null +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/ResponseTransformers.scala @@ -0,0 +1,66 @@ +/* + * Copyright 2021 HM Revenue & Customs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package uk.gov.hmrc.http.play + +import akka.stream.scaladsl.Source +import akka.util.ByteString +import _root_.play.api.libs.ws.{WSRequest, WSResponse} +import uk.gov.hmrc.http.{BadGatewayException, GatewayTimeoutException, HttpResponse, HttpReads} + +import java.util.concurrent.TimeoutException +import java.net.ConnectException +import scala.concurrent.{ExecutionContext, Future} + +trait ResponseTransformers { + def fromStream( + request : WSRequest, + responseF: Future[WSResponse] + )(implicit + ec: ExecutionContext + ): Future[Source[ByteString, _]] = + mapErrors(request, responseF) + .map(_.bodyAsSource) + + def fromJson[A]( + request: WSRequest, + responseF: Future[WSResponse] + )(implicit + r : HttpReads[A], + ec: ExecutionContext + ): Future[A] = + mapErrors(request, responseF) + .map { response => + // reusing existing HttpReads - currently requires HttpResponse + val httpResponse = HttpResponse( + status = response.status, + body = response.body, + headers = response.headers.mapValues(_.toSeq).toMap + ) + r.read(request.method, request.url, httpResponse) + } + + def mapErrors( + request : WSRequest, + responseF: Future[WSResponse] + )(implicit + ec: ExecutionContext + ): Future[WSResponse] = + responseF.recoverWith { + case e: TimeoutException => Future.failed(new GatewayTimeoutException(s"${request.method} of '${request.url}' timed out with message '${e.getMessage}'")) + case e: ConnectException => Future.failed(new BadGatewayException(s"${request.method} of '${request.url}' failed. Caused by: '${e.getMessage}'")) + } +} diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/package.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/package.scala index 78cf5f05..ad619332 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/package.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/package.scala @@ -16,33 +16,9 @@ package uk.gov.hmrc.http -import akka.stream.scaladsl.Source -import akka.util.ByteString import _root_.play.api.libs.json.{JsValue, Writes} -import _root_.play.api.libs.ws.{WSRequest, WSResponse} -import scala.concurrent.Future - -package object play { - // These will still need explicitly importing - // should they be moved to `import httpClient2._`? which means implementations can then depend on - // httpClient values (e.g. configuration) or does this make mocking/providing alternative implementations harder? - // Alternatively, could HttpClient2 trait just be replaced by HttpClient2Impl - and forget about alternative implementations - // (solves the final builder problem) - def fromStream(request: WSRequest)(response: WSResponse): Future[Source[ByteString, _]] = - Future.successful(response.bodyAsSource) - - def fromJson[A](request: WSRequest)(response: WSResponse)(implicit r: HttpReads[A]): Future[A] = - // reusing existing HttpReads - currently requires HttpResponse - Future.successful { - val httpResponse = HttpResponse( - status = response.status, - body = response.body, - headers = response.headers - ) - r.read(request.method, request.url, httpResponse) - } - - def toJson[A](model: A)(implicit w: Writes[A]): JsValue = - w.writes(model) -} +// These will still need explicitly importing +// should they be moved to `import httpClient2._`? which means implementations can then depend on +// httpClient values (e.g. configuration) or does this make mocking/providing alternative implementations harder? +package object play extends ResponseTransformers diff --git a/http-verbs-common/src/test/scala/uk/gov/hmrc/http/play/HttpClient2Spec.scala b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/play/HttpClient2Spec.scala index 56797409..7c04a82b 100644 --- a/http-verbs-common/src/test/scala/uk/gov/hmrc/http/play/HttpClient2Spec.scala +++ b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/play/HttpClient2Spec.scala @@ -27,7 +27,7 @@ import org.scalatest.concurrent.{IntegrationPatience, ScalaFutures} import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpecLike import play.api.libs.functional.syntax._ -import play.api.libs.json.{Reads, Writes} +import play.api.libs.json.{Json, Reads, Writes} import play.api.libs.ws.ahc.{AhcWSClient, AhcWSClientConfigFactory} import uk.gov.hmrc.http.{HeaderCarrier, HttpReads, HttpReadsInstances, Retries, StringContextOps, UpstreamErrorResponse} import uk.gov.hmrc.http.test.WireMockSupport @@ -76,7 +76,7 @@ class HttpClient2Spec val res: Future[ResDomain] = httpClient2 .put(url"$wireMockUrl/") - .withBody(toJson(ReqDomain("req"))) + .withBody(Json.toJson(ReqDomain("req"))) .execute(fromJson[ResDomain]) res.futureValue shouldBe ResDomain("res") @@ -99,7 +99,7 @@ class HttpClient2Spec val res: Future[ResDomain] = httpClient2 .put(url"$wireMockUrl/") - .withBody(toJson(ReqDomain("req"))) + .withBody(Json.toJson(ReqDomain("req"))) .replaceHeader("User-Agent" -> "ua2") .execute(fromJson[ResDomain]) @@ -164,7 +164,7 @@ class HttpClient2Spec count.incrementAndGet httpClient2 .put(url"$wireMockUrl/") - .withBody(toJson(ReqDomain("req"))) + .withBody(Json.toJson(ReqDomain("req"))) .execute(fromJson[ResDomain]) } From e94fb8a58529ef7e3d01eb61398ade6d599bb31c Mon Sep 17 00:00:00 2001 From: colin-lamed <9568290+colin-lamed@users.noreply.github.com> Date: Mon, 25 Oct 2021 12:28:10 +0100 Subject: [PATCH 05/23] BDOG-1512 Test auditing of HttpClient2Impl --- .../gov/hmrc/http/play/HttpClient2Impl.scala | 10 +- .../gov/hmrc/http/play/HttpClient2Spec.scala | 301 ++++++++++++------ 2 files changed, 202 insertions(+), 109 deletions(-) diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/HttpClient2Impl.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/HttpClient2Impl.scala index 4224397c..dde22d5d 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/HttpClient2Impl.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/HttpClient2Impl.scala @@ -177,8 +177,8 @@ class ExecutorImpl( executeHooks(isStream, request, responseF) responseF.onComplete(logResult(hc, request.method, request.uri.toString, startAge)) // we don't delegate the response conversion to the client - // e.g. execute[WSResponse].transform(...) since the transform functions require access to the request (method and url) - // given method and url are only required for error messages, is it overkill? E.g. a stacktrace should identify the function? + // (i.e. return Future[WSResponse] to be handled with Future.transform/transformWith(...)) + // since the transform functions require access to the request (method and url) transformResponse(request, responseF) } @@ -194,7 +194,11 @@ class ExecutorImpl( def toHttpResponse(response: WSResponse) = HttpResponse( status = response.status, - body = if (isStream) "" else response.body, // calling response.body on stream would load all into memory (and cause stream to be read twice..) + body = if (isStream) + // calling response.body on stream would load all into memory (and stream would need to be broadcast to be able + // to read twice - although we could cap it like in uk.gov.hmrc.play.bootstrap.filters.RequestBodyCaptor) + "" + else response.body, headers = response.headers.mapValues(_.toSeq).toMap ) diff --git a/http-verbs-common/src/test/scala/uk/gov/hmrc/http/play/HttpClient2Spec.scala b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/play/HttpClient2Spec.scala index 7c04a82b..daf0c32c 100644 --- a/http-verbs-common/src/test/scala/uk/gov/hmrc/http/play/HttpClient2Spec.scala +++ b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/play/HttpClient2Spec.scala @@ -21,156 +21,245 @@ import akka.stream.{ActorMaterializer, Materializer} import akka.stream.scaladsl.Source import akka.util.ByteString import com.github.tomakehurst.wiremock.client.WireMock -import com.github.tomakehurst.wiremock.client.WireMock._ +import com.github.tomakehurst.wiremock.client.WireMock.{verify => _, _} import com.typesafe.config.ConfigFactory +import org.mockito.ArgumentMatchersSugar +import org.mockito.captor.ArgCaptor +import org.mockito.scalatest.MockitoSugar import org.scalatest.concurrent.{IntegrationPatience, ScalaFutures} import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpecLike +import play.api.Configuration import play.api.libs.functional.syntax._ import play.api.libs.json.{Json, Reads, Writes} import play.api.libs.ws.ahc.{AhcWSClient, AhcWSClientConfigFactory} -import uk.gov.hmrc.http.{HeaderCarrier, HttpReads, HttpReadsInstances, Retries, StringContextOps, UpstreamErrorResponse} +import uk.gov.hmrc.http.{HeaderCarrier, HttpReads, HttpReadsInstances, HttpResponse, Retries, StringContextOps, UpstreamErrorResponse} +import uk.gov.hmrc.http.hooks.{HookData, HttpHook} import uk.gov.hmrc.http.test.WireMockSupport -import scala.concurrent.Future -import scala.concurrent.ExecutionContext.Implicits.global -import play.api.Configuration import java.util.concurrent.atomic.AtomicInteger +import scala.concurrent.{ExecutionContext, Future} +import scala.concurrent.ExecutionContext.Implicits.global class HttpClient2Spec extends AnyWordSpecLike with Matchers with WireMockSupport with ScalaFutures - with IntegrationPatience { - - implicit val as: ActorSystem = ActorSystem("test-actor-system") - implicit val mat: Materializer = ActorMaterializer() // explicitly required for play-26 - - // this would be injected - val httpClient2: HttpClient2 = { - val config = - Configuration( - ConfigFactory.parseString( - """appName = myapp""" - ).withFallback(ConfigFactory.load()) + with IntegrationPatience + with MockitoSugar + with ArgumentMatchersSugar { + + "HttpClient2" should { + "work with json" in new Setup { + implicit val hc = HeaderCarrier() + + wireMockServer.stubFor( + WireMock.put(urlEqualTo("/")) + .willReturn(aResponse().withBody("\"res\"").withStatus(200)) ) - new HttpClient2Impl( - wsClient = AhcWSClient(AhcWSClientConfigFactory.forConfig(config.underlying)), - as, - config, - hooks = Seq.empty, // this would be wired up to play-auditing - ) - } - "api" when { - "with json" should { - "provide body" in { - implicit val hc = HeaderCarrier() + val res: Future[ResDomain] = + httpClient2 + .put(url"$wireMockUrl/") + .withBody(Json.toJson(ReqDomain("req"))) + .execute(fromJson[ResDomain]) - stubFor( - WireMock.put(urlEqualTo("/")) - .willReturn(aResponse().withBody("\"res\"").withStatus(200)) - ) + res.futureValue shouldBe ResDomain("res") - val res: Future[ResDomain] = - httpClient2 - .put(url"$wireMockUrl/") - .withBody(Json.toJson(ReqDomain("req"))) - .execute(fromJson[ResDomain]) + wireMockServer.verify( + putRequestedFor(urlEqualTo("/")) + .withHeader("Content-Type", equalTo("application/json")) + .withRequestBody(equalTo("\"req\"")) + .withHeader("User-Agent", equalTo("myapp")) + ) + + val headersCaptor = ArgCaptor[Seq[(String, String)]] + val responseCaptor = ArgCaptor[Future[HttpResponse]] + + verify(mockHttpHook) + .apply( + verb = eqTo("PUT"), + url = eqTo(url"$wireMockUrl/"), + headers = headersCaptor, + body = eqTo(Some(HookData.FromString("\"req\""))), + responseF = responseCaptor + )(any[HeaderCarrier], any[ExecutionContext]) + + headersCaptor.value should contain ("User-Agent" -> "myapp") + headersCaptor.value should contain ("Content-Type" -> "application/json") + val auditedResponse = responseCaptor.value.futureValue + auditedResponse.status shouldBe 200 + auditedResponse.body shouldBe "\"res\"" + } - res.futureValue shouldBe ResDomain("res") + "work with streams" in new Setup { + implicit val hc = HeaderCarrier() - verify( - putRequestedFor(urlEqualTo("/")) - .withRequestBody(equalTo("\"req\"")) - .withHeader("User-Agent", equalTo("myapp")) - ) - } + wireMockServer.stubFor( + WireMock.put(urlEqualTo("/")) + .willReturn(aResponse().withBody("\"res\"").withStatus(200)) + ) + + val srcStream: Source[ByteString, _] = + Source.single(ByteString("source")) + + val res: Future[Source[ByteString, _]] = + httpClient2 + .put(url"$wireMockUrl/") + .withBody(srcStream) + .stream(fromStream) + + res.futureValue.map(_.utf8String).runReduce(_ + _).futureValue shouldBe "\"res\"" - "override user-agent" in { - implicit val hc = HeaderCarrier() + wireMockServer.verify( + putRequestedFor(urlEqualTo("/")) + .withHeader("Content-Type", equalTo("application/octet-stream")) + .withRequestBody(equalTo("source")) + .withHeader("User-Agent", equalTo("myapp")) + ) + + val headersCaptor = ArgCaptor[Seq[(String, String)]] + val responseCaptor = ArgCaptor[Future[HttpResponse]] + + verify(mockHttpHook) + .apply( + verb = eqTo("PUT"), + url = eqTo(url"$wireMockUrl/"), + headers = headersCaptor, + body = eqTo(Some(HookData.FromString(""))), + responseF = responseCaptor + )(any[HeaderCarrier], any[ExecutionContext]) + + headersCaptor.value should contain ("User-Agent" -> "myapp") + headersCaptor.value should contain ("Content-Type" -> "application/octet-stream") + val auditedResponse = responseCaptor.value.futureValue + auditedResponse.status shouldBe 200 + auditedResponse.body shouldBe "" + } + + "work with form data" in new Setup { + implicit val hc = HeaderCarrier() - stubFor( - WireMock.put(urlEqualTo("/")) - .willReturn(aResponse().withBody("\"res\"").withStatus(200)) + wireMockServer.stubFor( + WireMock.post(urlEqualTo("/")) + .willReturn(aResponse().withBody("\"res\"").withStatus(200)) + ) + + val body: Map[String, Seq[String]] = + Map( + "k1" -> Seq("v1", "v2"), + "k2" -> Seq("v3") ) - val res: Future[ResDomain] = + val res: Future[ResDomain] = httpClient2 - .put(url"$wireMockUrl/") - .withBody(Json.toJson(ReqDomain("req"))) - .replaceHeader("User-Agent" -> "ua2") + .post(url"$wireMockUrl/") + .withBody(body) .execute(fromJson[ResDomain]) - res.futureValue shouldBe ResDomain("res") + res.futureValue shouldBe ResDomain("res") - verify( - putRequestedFor(urlEqualTo("/")) - .withRequestBody(equalTo("\"req\"")) - .withHeader("User-Agent", equalTo("ua2")) - ) - } + wireMockServer.verify( + postRequestedFor(urlEqualTo("/")) + .withHeader("Content-Type", equalTo("application/x-www-form-urlencoded")) + .withRequestBody(equalTo("k1=v1&k1=v2&k2=v3")) + .withHeader("User-Agent", equalTo("myapp")) + ) + + val headersCaptor = ArgCaptor[Seq[(String, String)]] + val responseCaptor = ArgCaptor[Future[HttpResponse]] + + verify(mockHttpHook) + .apply( + verb = eqTo("POST"), + url = eqTo(url"$wireMockUrl/"), + headers = headersCaptor, + body = eqTo(Some(HookData.FromMap(body))), + responseF = responseCaptor + )(any[HeaderCarrier], any[ExecutionContext]) + + headersCaptor.value should contain ("User-Agent" -> "myapp") + headersCaptor.value should contain ("Content-Type" -> "application/x-www-form-urlencoded") + val auditedResponse = responseCaptor.value.futureValue + auditedResponse.status shouldBe 200 + auditedResponse.body shouldBe "\"res\"" } - "with stream" should { - "work" in { - implicit val hc = HeaderCarrier() + "allow overriding user-agent" in new Setup { + implicit val hc = HeaderCarrier() - stubFor( - WireMock.put(urlEqualTo("/")) - .willReturn(aResponse().withBody("\"res\"").withStatus(200)) - ) + wireMockServer.stubFor( + WireMock.put(urlEqualTo("/")) + .willReturn(aResponse().withBody("\"res\"").withStatus(200)) + ) - val srcStream: Source[ByteString, _] = - Source.single(ByteString("source")) + val res: Future[ResDomain] = + httpClient2 + .put(url"$wireMockUrl/") + .withBody(Json.toJson(ReqDomain("req"))) + .replaceHeader("User-Agent" -> "ua2") + .execute(fromJson[ResDomain]) - val res: Future[Source[ByteString, _]] = - httpClient2 - .put(url"$wireMockUrl/") - .withBody(srcStream) - .stream(fromStream) + res.futureValue shouldBe ResDomain("res") - res.futureValue.map(_.utf8String).runReduce(_ + _).futureValue shouldBe "\"res\"" + wireMockServer.verify( + putRequestedFor(urlEqualTo("/")) + .withRequestBody(equalTo("\"req\"")) + .withHeader("User-Agent", equalTo("ua2")) + ) + } - verify( - putRequestedFor(urlEqualTo("/")) - .withRequestBody(equalTo("source")) - .withHeader("User-Agent", equalTo("myapp")) - ) + "allow custom retries" in new Setup { + val retries = new Retries { + override val actorSystem = as + override val configuration = ConfigFactory + .parseString("http-verbs.retries.intervals = [ 100.ms, 200.ms, 300.ms ]") + .withFallback(ConfigFactory.load()) } - } - "with custom retries" should { - "work" in { - val retries = new Retries { - override val actorSystem = as - override val configuration = ConfigFactory - .parseString("http-verbs.retries.intervals = [ 100.ms, 200.ms, 300.ms ]") - .withFallback(ConfigFactory.load()) - } + implicit val hc = HeaderCarrier() - implicit val hc = HeaderCarrier() + wireMockServer.stubFor( + WireMock.put(urlEqualTo("/")) + .willReturn(aResponse().withBody("\"res\"").withStatus(502)) + ) - stubFor( - WireMock.put(urlEqualTo("/")) - .willReturn(aResponse().withBody("\"res\"").withStatus(502)) - ) + val count = new AtomicInteger(0) + + val res: Future[ResDomain] = + retries.retryFor("get reqdomain"){ case UpstreamErrorResponse.WithStatusCode(502) => true }{ + count.incrementAndGet + httpClient2 + .put(url"$wireMockUrl/") + .withBody(Json.toJson(ReqDomain("req"))) + .execute(fromJson[ResDomain]) + } - val count = new AtomicInteger(0) + res.failed.futureValue shouldBe a[UpstreamErrorResponse] + count.get shouldBe 4 + } + } - val res: Future[ResDomain] = - retries.retryFor("get reqdomain"){ case UpstreamErrorResponse.WithStatusCode(502) => true }{ - count.incrementAndGet - httpClient2 - .put(url"$wireMockUrl/") - .withBody(Json.toJson(ReqDomain("req"))) - .execute(fromJson[ResDomain]) - } + trait Setup { + implicit val as: ActorSystem = ActorSystem("test-actor-system") + implicit val mat: Materializer = ActorMaterializer() // explicitly required for play-26 - res.failed.futureValue shouldBe a[UpstreamErrorResponse] - count.get shouldBe 4 - } + val mockHttpHook = mock[HttpHook](withSettings.lenient) + + val httpClient2: HttpClient2 = { + val config = + Configuration( + ConfigFactory.parseString( + """appName = myapp""" + ).withFallback(ConfigFactory.load()) + ) + new HttpClient2Impl( + wsClient = AhcWSClient(AhcWSClientConfigFactory.forConfig(config.underlying)), + as, + config, + hooks = Seq(mockHttpHook), + ) } } } From d9e56a6983a56f0994c6158c71fb59312d18ad8e Mon Sep 17 00:00:00 2001 From: colin-lamed <9568290+colin-lamed@users.noreply.github.com> Date: Wed, 3 Nov 2021 15:46:18 +0000 Subject: [PATCH 06/23] BDOG-1512 Audit streamed requests --- .../src/main/resources/reference.conf | 9 +- .../scala/uk/gov/hmrc/http/HttpResponse.scala | 23 ++ .../uk/gov/hmrc/http/play/BodyCaptor.scala | 96 ++++++++ .../uk/gov/hmrc/http/play/HttpClient2.scala | 8 +- .../gov/hmrc/http/play/HttpClient2Impl.scala | 208 ++++++++++++------ .../hmrc/http/play/ResponseTransformers.scala | 20 +- .../gov/hmrc/http/play/HttpClient2Spec.scala | 15 +- 7 files changed, 291 insertions(+), 88 deletions(-) create mode 100644 http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/BodyCaptor.scala diff --git a/http-verbs-common/src/main/resources/reference.conf b/http-verbs-common/src/main/resources/reference.conf index eeb59827..d28793c7 100644 --- a/http-verbs-common/src/main/resources/reference.conf +++ b/http-verbs-common/src/main/resources/reference.conf @@ -14,8 +14,11 @@ internalServiceHostPatterns = [ "^.*\\.service$", "^.*\\.mdtp$", "^localhost$" ] bootstrap.http.headersAllowlist = [] -http-verbs.retries { - intervals = [ "500.millis", "1.second", "2.seconds", "4.seconds", "8.seconds" ] - ssl-engine-closed-already.enabled = false +http-verbs { + retries { + intervals = [ "500.millis", "1.second", "2.seconds", "4.seconds", "8.seconds" ] + ssl-engine-closed-already.enabled = false + } + auditing.maxBodyLength = 32665 # Confirm with CIP this can be 500000? } proxy.enabled = false diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpResponse.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpResponse.scala index 140f2cdb..38e7e9fb 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpResponse.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpResponse.scala @@ -16,6 +16,8 @@ package uk.gov.hmrc.http +import akka.stream.scaladsl.Source +import akka.util.ByteString import com.github.ghik.silencer.silent import _root_.play.api.libs.json.{JsValue, Json} @@ -44,6 +46,11 @@ trait HttpResponse { def json: JsValue = Json.parse(body) + // If we were using wsResponse, we could have a single body function... + //def body[T: BodyReadable]: T = super.body[T] + def bodyAsSource: Source[ByteString, _] = + Source.single(ByteString(body)) + def header(key: String): Option[String] = headers.get(key).flatMap(_.headOption) @@ -106,6 +113,22 @@ object HttpResponse { } } + def apply( + status : Int, + bodyAsSource: Source[ByteString, _], + headers : Map[String, Seq[String]] + ): HttpResponse = { + val pStatus = status + val pBodyAsSource = bodyAsSource + val pHeaders = headers + new HttpResponse { + override def status : Int = pStatus + override def body : String = sys.error(s"This is a streamed response, please use `bodyAsSource`") + override def bodyAsSource: Source[ByteString, _] = pBodyAsSource + override def allHeaders : Map[String, Seq[String]] = pHeaders + } + } + def unapply(that: HttpResponse): Option[(Int, String, Map[String, Seq[String]])] = Some((that.status, that.body, that.headers)) } diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/BodyCaptor.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/BodyCaptor.scala new file mode 100644 index 00000000..a820070d --- /dev/null +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/BodyCaptor.scala @@ -0,0 +1,96 @@ +/* + * Copyright 2021 HM Revenue & Customs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package uk.gov.hmrc.http.play + +import akka.stream.{Attributes, FlowShape, Inlet, Outlet} +import akka.stream.scaladsl.{Flow, Sink} +import akka.stream.stage._ +import akka.util.ByteString +import play.api.Logger + +// based on play.filters.csrf.CSRFAction#BodyHandler + +private class BodyCaptorFlow( + loggingContext : String, + maxBodyLength : Int, + withCapturedBody: ByteString => Unit +) extends GraphStage[FlowShape[ByteString, ByteString]] { + val in = Inlet[ByteString]("BodyCaptorFlow.in") + val out = Outlet[ByteString]("BodyCaptorFlow.out") + override val shape = FlowShape.of(in, out) + + private val logger = Logger(getClass) + + override def createLogic(inheritedAttributes: Attributes): GraphStageLogic = + new GraphStageLogic(shape) { + private var buffer = ByteString.empty + private var bodyLength = 0 + + setHandlers( + in, + out, + new InHandler with OutHandler { + override def onPull(): Unit = + pull(in) + + override def onPush(): Unit = { + val chunk = grab(in) + bodyLength += chunk.length + if (buffer.size < maxBodyLength) + buffer ++= chunk + push(out, chunk) + } + + override def onUpstreamFinish(): Unit = { + // TODO how to opt out of auditing payloads? + // currently we can only turn of auditing for url (auditDisabledForPattern) - not per method + // and we can't turn off auditing of just the payload (and keep the fact the call has been made) + // what about auditing request payloads, but not response payloads? + if (bodyLength > maxBodyLength) + logger.warn( + s"txm play auditing: $loggingContext sanity check request body $bodyLength exceeds maxLength $maxBodyLength - do you need to be auditing this payload?" + ) + withCapturedBody(buffer.take(maxBodyLength)) + if (isAvailable(out) && buffer == ByteString.empty) + push(out, buffer) + completeStage() + } + } + ) + } +} + +object BodyCaptor { + def flow( + loggingContext : String, + maxBodyLength : Int, + withCapturedBody: ByteString => Unit + ): Flow[ByteString, ByteString, akka.NotUsed] = + Flow.fromGraph(new BodyCaptorFlow( + loggingContext = loggingContext, + maxBodyLength = maxBodyLength, + withCapturedBody = withCapturedBody + )) + + def sink( + loggingContext : String, + maxBodyLength : Int, + withCapturedBody: ByteString => Unit // TODO can we not just expose as the materialised value? + ): Sink[ByteString, akka.NotUsed] = + flow(loggingContext, maxBodyLength, withCapturedBody) + .to(Sink.ignore) +} diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/HttpClient2.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/HttpClient2.scala index 6c5586fd..4bad125f 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/HttpClient2.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/HttpClient2.scala @@ -18,8 +18,8 @@ // `import uk.gov.hmrc.http._` will then have to make play imports with _root_ `import _root_.play...` package uk.gov.hmrc.http.play -import play.api.libs.ws.{BodyWritable, WSRequest, WSResponse} -import uk.gov.hmrc.http.HeaderCarrier +import play.api.libs.ws.{BodyWritable, WSRequest} +import uk.gov.hmrc.http.{HeaderCarrier, HttpResponse} import java.net.URL import scala.concurrent.{ExecutionContext, Future} @@ -53,13 +53,13 @@ trait RequestBuilder { def transform(transform: WSRequest => WSRequest): RequestBuilder def execute[A]( - transformResponse: (WSRequest, Future[WSResponse]) => Future[A] + transformResponse: (WSRequest, Future[HttpResponse]) => Future[A] )(implicit ec: ExecutionContext ): Future[A] def stream[A]( - transformResponse: (WSRequest, Future[WSResponse]) => Future[A] + transformResponse: (WSRequest, Future[HttpResponse]) => Future[A] )(implicit ec: ExecutionContext ): Future[A] diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/HttpClient2Impl.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/HttpClient2Impl.scala index dde22d5d..c9b7500c 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/HttpClient2Impl.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/HttpClient2Impl.scala @@ -19,6 +19,7 @@ package uk.gov.hmrc.http.play import akka.actor.ActorSystem +import akka.util.ByteString import com.typesafe.config.Config import play.api.Configuration import play.api.libs.ws.{BodyWritable, EmptyBody, InMemoryBody, SourceBody, WSClient, WSProxyServer, WSRequest, WSResponse} @@ -29,7 +30,8 @@ import uk.gov.hmrc.http.hooks.{HookData, HttpHook} import uk.gov.hmrc.http.logging.ConnectionTracing import java.net.URL -import scala.concurrent.{ExecutionContext, Future} +import scala.concurrent.{ExecutionContext, Future, Promise} +import akka.stream.scaladsl.Source /* What does HttpVerbs actually provide? @@ -53,10 +55,11 @@ Extension methods are provided to make common patterns easier to apply. trait Executor { def execute[A]( - request : WSRequest, - isStream: Boolean + request : WSRequest, + hookDataF: Option[Future[Option[HookData]]], + isStream : Boolean )( - transformResponse: (WSRequest, Future[WSResponse]) => Future[A] + transformResponse: (WSRequest, Future[HttpResponse]) => Future[A] )(implicit hc: HeaderCarrier, ec: ExecutionContext @@ -90,7 +93,8 @@ class HttpClient2Impl( wsClient .url(url.toString) .withMethod(method) - .withHttpHeaders(hc.headersForUrl(hcConfig)(url.toString) : _*) + .withHttpHeaders(hc.headersForUrl(hcConfig)(url.toString) : _*), + None ) } @@ -101,51 +105,86 @@ final class RequestBuilderImpl( optProxyServer: Option[WSProxyServer], executor : Executor )( - request: WSRequest + request : WSRequest, + hookDataF: Option[Future[Option[HookData]]] )(implicit hc: HeaderCarrier ) extends RequestBuilder { override def transform(transform: WSRequest => WSRequest): RequestBuilderImpl = - new RequestBuilderImpl(config, optProxyServer, executor)(transform(request)) + new RequestBuilderImpl(config, optProxyServer, executor)(transform(request), hookDataF) // -- Transform helpers -- - override def replaceHeader(header: (String, String)): RequestBuilderImpl = { + private def replaceHeaderOnRequest(request: WSRequest, header: (String, String)): WSRequest = { def denormalise(hdrs: Map[String, Seq[String]]): Seq[(String, String)] = hdrs.toList.flatMap { case (k, vs) => vs.map(k -> _) } val hdrsWithoutKey = request.headers.filterKeys(!_.equalsIgnoreCase(header._1)).toMap // replace existing header - transform(_.withHttpHeaders(denormalise(hdrsWithoutKey) :+ header : _*)) + request.withHttpHeaders(denormalise(hdrsWithoutKey) :+ header : _*) } + override def replaceHeader(header: (String, String)): RequestBuilderImpl = + transform(replaceHeaderOnRequest(_, header)) + override def addHeaders(headers: (String, String)*): RequestBuilderImpl = transform(_.addHttpHeaders(headers: _*)) override def withProxy: RequestBuilderImpl = transform(request => optProxyServer.foldLeft(request)(_ withProxyServer _)) - override def withBody[B : BodyWritable](body: B): RequestBuilderImpl = - (if (body == EmptyBody) - replaceHeader(play.api.http.HeaderNames.CONTENT_LENGTH -> "0") // rejected by Akami without a Content-Length (https://jira.tools.tax.service.gov.uk/browse/APIS-5100) - else - this - ).transform(_.withBody(body)) + private def withHookData(hookDataF: Future[Option[HookData]]): RequestBuilderImpl = + new RequestBuilderImpl(config, optProxyServer, executor)(request, Some(hookDataF)) + + // withBody should be called rather than transform(_.withBody) + // failure to do so will lead to a runtime exception (there's no other way to enforce?) + // TODO make this a scaladoc comment (on interface) + override def withBody[B : BodyWritable](body: B): RequestBuilderImpl = { + val hookDataP = Promise[Option[HookData]]() + transform { req => + val req2 = req.withBody(body) + req2.body match { + case EmptyBody => hookDataP.success(None) + replaceHeaderOnRequest(req2, play.api.http.HeaderNames.CONTENT_LENGTH -> "0") // rejected by Akami without a Content-Length (https://jira.tools.tax.service.gov.uk/browse/APIS-5100) + case InMemoryBody(bytes) => // if the default BodyWritables have been used - we can trust the content-type here (client's wouldn't have changed the content-type yet) + // TODO but what if they have used a custom BodyWritable? Also check if the body param is a Map[String, Seq[String]] or Map[String, String]? + req2.header("Content-Type") match { + case Some("application/x-www-form-urlencoded") => hookDataP.success(Some(HookData.FromMap(FormUrlEncodedParser.parse(bytes.decodeString("UTF-8"))))) + case _ => hookDataP.success(Some(HookData.FromString(bytes.decodeString("UTF-8")))) + } + req2 + case SourceBody(source) => val src2: Source[ByteString, _] = + source + .alsoTo( + BodyCaptor.sink( + loggingContext = s"request for outgoing ${request.method} ${request.url}", + maxBodyLength = config.get[Int]("http-verbs.auditing.maxBodyLength"), + withCapturedBody = body => hookDataP.success(Some(HookData.FromString(body.decodeString("UTF-8")))) + ) + ) + // preserve content-type (it may have been set with a different body writeable - e.g. play.api.libs.ws.WSBodyWritables.bodyWritableOf_Multipart) + req2.header("Content-Type") match { + case Some(contentType) => replaceHeaderOnRequest(req2.withBody(src2), "Content-Type" -> contentType) + case _ => req2.withBody(src2) + } + } + }.withHookData(hookDataP.future) + } // -- Execution -- override def execute[A]( - transformResponse: (WSRequest, Future[WSResponse]) => Future[A] + transformResponse: (WSRequest, Future[HttpResponse]) => Future[A] )(implicit ec: ExecutionContext ): Future[A] = - executor.execute(request, isStream = false)(transformResponse) + executor.execute(request, hookDataF, isStream = false)(transformResponse) override def stream[A]( - transformResponse: (WSRequest, Future[WSResponse]) => Future[A] + transformResponse: (WSRequest, Future[HttpResponse]) => Future[A] )(implicit ec: ExecutionContext ): Future[A] = - executor.execute(request, isStream = true)(transformResponse) + executor.execute(request, hookDataF, isStream = true)(transformResponse) } class ExecutorImpl( @@ -159,79 +198,118 @@ class ExecutorImpl( // for Retries override val configuration: Config = config.underlying + private val maxBodyLength = config.get[Int]("http-verbs.auditing.maxBodyLength") + def execute[A]( - request : WSRequest, - isStream: Boolean + request : WSRequest, + optHookDataF: Option[Future[Option[HookData]]], + isStream : Boolean )( - transformResponse: (WSRequest, Future[WSResponse]) => Future[A] + transformResponse: (WSRequest, Future[HttpResponse]) => Future[A] )(implicit hc: HeaderCarrier, ec: ExecutionContext ): Future[A] = { + val hookDataF = + optHookDataF match { + case None if request.body != EmptyBody => + sys.error(s"There is no audit data available. Please ensure you call `withBody` on the RequestBuilder rather than `transform(_.withBody)`") + case None => Future.successful(None) + case Some(f) => f + } + val startAge = System.nanoTime() - hc.age val responseF = retryOnSslEngineClosed(request.method, request.url)( // we do the execution since if clients are responsable for it (e.g. a callback), they may further modify the request outside of auditing etc. if (isStream) request.stream() else request.execute() ) - executeHooks(isStream, request, responseF) - responseF.onComplete(logResult(hc, request.method, request.uri.toString, startAge)) + + val (httpResponseF, auditResponseF) = toHttpResponse(isStream, request, responseF) + executeHooks(isStream, request, hookDataF, auditResponseF) + httpResponseF.onComplete(logResult(hc, request.method, request.uri.toString, startAge)) // we don't delegate the response conversion to the client // (i.e. return Future[WSResponse] to be handled with Future.transform/transformWith(...)) // since the transform functions require access to the request (method and url) - transformResponse(request, responseF) + transformResponse(request, httpResponseF) } - private def executeHooks( + // TODO horrid return type - one HttpResponse is for auditing... + private def toHttpResponse( isStream : Boolean, request : WSRequest, responseF: Future[WSResponse] + )(implicit ec: ExecutionContext + ): (Future[HttpResponse], Future[HttpResponse]) = { + val auditResponseF = Promise[HttpResponse]() + val httpResponseF = + for { + response <- responseF + } yield + if (isStream) { + val source = + response.bodyAsSource + .alsoTo( + BodyCaptor.sink( + loggingContext = s"response for outgoing ${request.method} ${request.url}", + maxBodyLength = maxBodyLength, + withCapturedBody = body => + auditResponseF.success( + HttpResponse( + status = response.status, + body = body.decodeString("UTF-8"), + headers = response.headers.mapValues(_.toSeq).toMap + ) + ) + ) + ) + HttpResponse( + status = response.status, + bodyAsSource = source, + headers = response.headers.mapValues(_.toSeq).toMap + ) + } else { + val httpResponse = HttpResponse( + status = response.status, + body = response.body, + headers = response.headers.mapValues(_.toSeq).toMap + ) + auditResponseF.success(httpResponse) + httpResponse + } + (httpResponseF, auditResponseF.future) + } + + private def executeHooks( + isStream : Boolean, + request : WSRequest, + hookDataF: Future[Option[HookData]], + auditedResponseF: Future[HttpResponse] // play-auditing expects the body to be a String )(implicit hc: HeaderCarrier, ec: ExecutionContext ): Unit = { - // hooks take HttpResponse.. - def toHttpResponse(response: WSResponse) = - HttpResponse( - status = response.status, - body = if (isStream) - // calling response.body on stream would load all into memory (and stream would need to be broadcast to be able - // to read twice - although we could cap it like in uk.gov.hmrc.play.bootstrap.filters.RequestBodyCaptor) - "" - else response.body, - headers = response.headers.mapValues(_.toSeq).toMap - ) - def denormalise(hdrs: Map[String, Seq[String]]): Seq[(String, String)] = hdrs.toList.flatMap { case (k, vs) => vs.map(k -> _) } - // TODO discuss changes with CIP - val body = - request.body match { - case EmptyBody => None - case InMemoryBody(bytes) => request.header("Content-Type") match { - case Some("application/x-www-form-urlencoded") => Some(HookData.FromMap(FormUrlEncodedParser.parse(bytes.decodeString("UTF-8")))) - case Some("application/octet-stream") => Some(HookData.FromString("")) - case _ => Some(HookData.FromString(bytes.decodeString("UTF-8"))) - } - case SourceBody(_) => Some(HookData.FromString("")) - } - - println(s"""AUDIT: - verb = ${request.method}, - url = ${new URL(request.url)}, - headers = ${denormalise(request.headers)}, - body = $body, - responseF = ${responseF.map(toHttpResponse)} - """) - hooks.foreach( - _.apply( - verb = request.method, - url = new URL(request.url), - headers = denormalise(request.headers), - body = body, - responseF = responseF.map(toHttpResponse) + // what if hookDataF fails? + hookDataF.foreach { body => + println(s"""AUDIT: + verb = ${request.method}, + url = ${new URL(request.url)}, + headers = ${denormalise(request.headers)}, + body = $body, + responseF = $auditedResponseF} + """) + hooks.foreach( + _.apply( + verb = request.method, + url = new URL(request.url), + headers = denormalise(request.headers), + body = body, + responseF = auditedResponseF + ) ) - ) + } } } diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/ResponseTransformers.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/ResponseTransformers.scala index 4583d786..3904959c 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/ResponseTransformers.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/ResponseTransformers.scala @@ -18,7 +18,7 @@ package uk.gov.hmrc.http.play import akka.stream.scaladsl.Source import akka.util.ByteString -import _root_.play.api.libs.ws.{WSRequest, WSResponse} +import _root_.play.api.libs.ws.WSRequest import uk.gov.hmrc.http.{BadGatewayException, GatewayTimeoutException, HttpResponse, HttpReads} import java.util.concurrent.TimeoutException @@ -28,7 +28,7 @@ import scala.concurrent.{ExecutionContext, Future} trait ResponseTransformers { def fromStream( request : WSRequest, - responseF: Future[WSResponse] + responseF: Future[HttpResponse] )(implicit ec: ExecutionContext ): Future[Source[ByteString, _]] = @@ -37,28 +37,20 @@ trait ResponseTransformers { def fromJson[A]( request: WSRequest, - responseF: Future[WSResponse] + responseF: Future[HttpResponse] )(implicit r : HttpReads[A], ec: ExecutionContext ): Future[A] = mapErrors(request, responseF) - .map { response => - // reusing existing HttpReads - currently requires HttpResponse - val httpResponse = HttpResponse( - status = response.status, - body = response.body, - headers = response.headers.mapValues(_.toSeq).toMap - ) - r.read(request.method, request.url, httpResponse) - } + .map(r.read(request.method, request.url, _)) def mapErrors( request : WSRequest, - responseF: Future[WSResponse] + responseF: Future[HttpResponse] )(implicit ec: ExecutionContext - ): Future[WSResponse] = + ): Future[HttpResponse] = responseF.recoverWith { case e: TimeoutException => Future.failed(new GatewayTimeoutException(s"${request.method} of '${request.url}' timed out with message '${e.getMessage}'")) case e: ConnectException => Future.failed(new BadGatewayException(s"${request.method} of '${request.url}' failed. Caused by: '${e.getMessage}'")) diff --git a/http-verbs-common/src/test/scala/uk/gov/hmrc/http/play/HttpClient2Spec.scala b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/play/HttpClient2Spec.scala index daf0c32c..ce4d2ffb 100644 --- a/http-verbs-common/src/test/scala/uk/gov/hmrc/http/play/HttpClient2Spec.scala +++ b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/play/HttpClient2Spec.scala @@ -127,7 +127,7 @@ class HttpClient2Spec verb = eqTo("PUT"), url = eqTo(url"$wireMockUrl/"), headers = headersCaptor, - body = eqTo(Some(HookData.FromString(""))), + body = eqTo(Some(HookData.FromString("source"))), // TODO check when this is truncated when too large responseF = responseCaptor )(any[HeaderCarrier], any[ExecutionContext]) @@ -135,7 +135,7 @@ class HttpClient2Spec headersCaptor.value should contain ("Content-Type" -> "application/octet-stream") val auditedResponse = responseCaptor.value.futureValue auditedResponse.status shouldBe 200 - auditedResponse.body shouldBe "" + auditedResponse.body shouldBe "\"res\"" // TODO check when this is truncated when too large } "work with form data" in new Setup { @@ -186,6 +186,17 @@ class HttpClient2Spec auditedResponse.body shouldBe "\"res\"" } + "fail if call withBody on the wsRequest itself" in new Setup { + implicit val hc = HeaderCarrier() + + a[RuntimeException] should be thrownBy + httpClient2 + .put(url"$wireMockUrl/") + .transform(_.withBody(toJson(ReqDomain("req")))) + .replaceHeader("User-Agent" -> "ua2") + .execute(fromJson[ResDomain]) + } + "allow overriding user-agent" in new Setup { implicit val hc = HeaderCarrier() From 1e0881ed02bbe4990d2e26680adbb3b9c561fcbf Mon Sep 17 00:00:00 2001 From: colin-lamed <9568290+colin-lamed@users.noreply.github.com> Date: Fri, 5 Nov 2021 13:10:48 +0000 Subject: [PATCH 07/23] BDOG-1512 apply some masking on all Map data, regardless of if content-type is x-www-form-urlencoded --- .../uk/gov/hmrc/http/play/HttpClient2.scala | 6 +- .../gov/hmrc/http/play/HttpClient2Impl.scala | 29 ++- .../gov/hmrc/http/play/HttpClient2Spec.scala | 188 ++++++++++++++++++ 3 files changed, 212 insertions(+), 11 deletions(-) diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/HttpClient2.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/HttpClient2.scala index 4bad125f..abe3ad03 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/HttpClient2.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/HttpClient2.scala @@ -23,6 +23,7 @@ import uk.gov.hmrc.http.{HeaderCarrier, HttpResponse} import java.net.URL import scala.concurrent.{ExecutionContext, Future} +import scala.reflect.runtime.universe.TypeTag trait HttpClient2 { protected def mkRequestBuilder(url: URL, method: String)(implicit hc: HeaderCarrier): RequestBuilder @@ -72,5 +73,8 @@ trait RequestBuilder { def withProxy: RequestBuilder - def withBody[B : BodyWritable](body: B): RequestBuilder + /** `withBody` should be called rather than `transform(_.withBody)`. + * Failure to do so will lead to a runtime exception + */ + def withBody[B : BodyWritable : TypeTag](body: B): RequestBuilder } diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/HttpClient2Impl.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/HttpClient2Impl.scala index c9b7500c..be3de1bb 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/HttpClient2Impl.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/HttpClient2Impl.scala @@ -19,6 +19,7 @@ package uk.gov.hmrc.http.play import akka.actor.ActorSystem +import akka.stream.scaladsl.Source import akka.util.ByteString import com.typesafe.config.Config import play.api.Configuration @@ -31,7 +32,7 @@ import uk.gov.hmrc.http.logging.ConnectionTracing import java.net.URL import scala.concurrent.{ExecutionContext, Future, Promise} -import akka.stream.scaladsl.Source +import scala.reflect.runtime.universe.{TypeTag, typeOf} /* What does HttpVerbs actually provide? @@ -135,21 +136,29 @@ final class RequestBuilderImpl( private def withHookData(hookDataF: Future[Option[HookData]]): RequestBuilderImpl = new RequestBuilderImpl(config, optProxyServer, executor)(request, Some(hookDataF)) - // withBody should be called rather than transform(_.withBody) - // failure to do so will lead to a runtime exception (there's no other way to enforce?) - // TODO make this a scaladoc comment (on interface) - override def withBody[B : BodyWritable](body: B): RequestBuilderImpl = { + // for erasure + private object IsMap { + def unapply[B: TypeTag](b: B): Option[Map[String, Seq[String]]] = + typeOf[B] match { + case _ if typeOf[B] =:= typeOf[Map[String, String]] => Some(b.asInstanceOf[Map[String, String]].map { case (k, v) => k -> Seq(v) }) + case _ if typeOf[B] =:= typeOf[Map[String, Seq[String]]] => Some(b.asInstanceOf[Map[String, Seq[String]]]) + case _ => None + } + } + + override def withBody[B : BodyWritable : TypeTag](body: B): RequestBuilderImpl = { val hookDataP = Promise[Option[HookData]]() transform { req => val req2 = req.withBody(body) req2.body match { case EmptyBody => hookDataP.success(None) replaceHeaderOnRequest(req2, play.api.http.HeaderNames.CONTENT_LENGTH -> "0") // rejected by Akami without a Content-Length (https://jira.tools.tax.service.gov.uk/browse/APIS-5100) - case InMemoryBody(bytes) => // if the default BodyWritables have been used - we can trust the content-type here (client's wouldn't have changed the content-type yet) - // TODO but what if they have used a custom BodyWritable? Also check if the body param is a Map[String, Seq[String]] or Map[String, String]? - req2.header("Content-Type") match { - case Some("application/x-www-form-urlencoded") => hookDataP.success(Some(HookData.FromMap(FormUrlEncodedParser.parse(bytes.decodeString("UTF-8"))))) - case _ => hookDataP.success(Some(HookData.FromString(bytes.decodeString("UTF-8")))) + case InMemoryBody(bytes) => // we can't guarantee that the default BodyWritables have been used - so rather than relying on content-type alone, we identify form data + // by provided body type (Map) or content-type (e.g. form data as a string) + (body, req2.header("Content-Type")) match { + case (IsMap(m), _ ) => hookDataP.success(Some(HookData.FromMap(m))) + case (_ , Some("application/x-www-form-urlencoded")) => hookDataP.success(Some(HookData.FromMap(FormUrlEncodedParser.parse(bytes.decodeString("UTF-8"))))) + case _ => hookDataP.success(Some(HookData.FromString(bytes.decodeString("UTF-8")))) } req2 case SourceBody(source) => val src2: Source[ByteString, _] = diff --git a/http-verbs-common/src/test/scala/uk/gov/hmrc/http/play/HttpClient2Spec.scala b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/play/HttpClient2Spec.scala index ce4d2ffb..b4da9e89 100644 --- a/http-verbs-common/src/test/scala/uk/gov/hmrc/http/play/HttpClient2Spec.scala +++ b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/play/HttpClient2Spec.scala @@ -186,6 +186,194 @@ class HttpClient2Spec auditedResponse.body shouldBe "\"res\"" } + "work with form data - custom writeable for content-type" in new Setup { + implicit val hc = HeaderCarrier() + + wireMockServer.stubFor( + WireMock.post(urlEqualTo("/")) + .willReturn(aResponse().withBody("\"res\"").withStatus(200)) + ) + + import play.api.libs.ws.{BodyWritable, InMemoryBody} + implicit val writeableOf_urlEncodedForm: BodyWritable[Map[String, Seq[String]]] = { + import java.net.URLEncoder + BodyWritable( + formData => + InMemoryBody( + ByteString.fromString( + formData.flatMap(item => item._2.map(c => s"${item._1}=${URLEncoder.encode(c, "UTF-8")}")).mkString("&") + ) + ), + "nonstandard/x-www-form-urlencoded" + ) + } + + val body: Map[String, Seq[String]] = + Map( + "k1" -> Seq("v1", "v2"), + "k2" -> Seq("v3") + ) + + val res: Future[ResDomain] = + httpClient2 + .post(url"$wireMockUrl/") + .withBody(body) + .execute(fromJson[ResDomain]) + + res.futureValue shouldBe ResDomain("res") + + wireMockServer.verify( + postRequestedFor(urlEqualTo("/")) + .withHeader("Content-Type", equalTo("nonstandard/x-www-form-urlencoded")) + .withRequestBody(equalTo("k1=v1&k1=v2&k2=v3")) + .withHeader("User-Agent", equalTo("myapp")) + ) + + val headersCaptor = ArgCaptor[Seq[(String, String)]] + val responseCaptor = ArgCaptor[Future[HttpResponse]] + + verify(mockHttpHook) + .apply( + verb = eqTo("POST"), + url = eqTo(url"$wireMockUrl/"), + headers = headersCaptor, + body = eqTo(Some(HookData.FromMap(body))), + responseF = responseCaptor + )(any[HeaderCarrier], any[ExecutionContext]) + + headersCaptor.value should contain ("User-Agent" -> "myapp") + headersCaptor.value should contain ("Content-Type" -> "nonstandard/x-www-form-urlencoded") + val auditedResponse = responseCaptor.value.futureValue + auditedResponse.status shouldBe 200 + auditedResponse.body shouldBe "\"res\"" + } + + "work with form data - custom writeable for map type" in new Setup { + implicit val hc = HeaderCarrier() + + wireMockServer.stubFor( + WireMock.post(urlEqualTo("/")) + .willReturn(aResponse().withBody("\"res\"").withStatus(200)) + ) + + import play.api.libs.ws.{BodyWritable, InMemoryBody} + implicit val writeableOf_urlEncodedForm: BodyWritable[scala.collection.mutable.Map[String, Seq[String]]] = { + import java.net.URLEncoder + BodyWritable( + formData => + InMemoryBody( + ByteString.fromString( + formData.flatMap(item => item._2.map(c => s"${item._1}=${URLEncoder.encode(c, "UTF-8")}")).mkString("&") + ) + ), + "application/x-www-form-urlencoded" + ) + } + + val body: scala.collection.mutable.Map[String, Seq[String]] = + scala.collection.mutable.Map( + "k1" -> Seq("v1", "v2"), + "k2" -> Seq("v3") + ) + + val res: Future[ResDomain] = + httpClient2 + .post(url"$wireMockUrl/") + .withBody(body) + .execute(fromJson[ResDomain]) + + res.futureValue shouldBe ResDomain("res") + + wireMockServer.verify( + postRequestedFor(urlEqualTo("/")) + .withHeader("Content-Type", equalTo("application/x-www-form-urlencoded")) + .withRequestBody(equalTo("k1=v1&k1=v2&k2=v3")) + .withHeader("User-Agent", equalTo("myapp")) + ) + + val headersCaptor = ArgCaptor[Seq[(String, String)]] + val responseCaptor = ArgCaptor[Future[HttpResponse]] + + verify(mockHttpHook) + .apply( + verb = eqTo("POST"), + url = eqTo(url"$wireMockUrl/"), + headers = headersCaptor, + body = eqTo(Some(HookData.FromMap(body.toMap))), + responseF = responseCaptor + )(any[HeaderCarrier], any[ExecutionContext]) + + headersCaptor.value should contain ("User-Agent" -> "myapp") + headersCaptor.value should contain ("Content-Type" -> "application/x-www-form-urlencoded") + val auditedResponse = responseCaptor.value.futureValue + auditedResponse.status shouldBe 200 + auditedResponse.body shouldBe "\"res\"" + } + + /* Note, using non-form-encoding and a non immutable Map implementation will not be escaped properly + "work with any form data" in new Setup { + implicit val hc = HeaderCarrier() + + wireMockServer.stubFor( + WireMock.post(urlEqualTo("/")) + .willReturn(aResponse().withBody("\"res\"").withStatus(200)) + ) + + import play.api.libs.ws.{BodyWritable, InMemoryBody} + implicit val writeableOf_urlEncodedForm: BodyWritable[scala.collection.mutable.Map[String, Seq[String]]] = { + import java.net.URLEncoder + BodyWritable( + formData => + InMemoryBody( + ByteString.fromString( + formData.flatMap(item => item._2.map(c => s"${item._1}=${URLEncoder.encode(c, "UTF-8")}")).mkString("&") + ) + ), + "non-standard/x-www-form-urlencoded" + ) + } + + val body: scala.collection.mutable.Map[String, Seq[String]] = + scala.collection.mutable.Map( + "k1" -> Seq("v1", "v2"), + "k2" -> Seq("v3") + ) + + val res: Future[ResDomain] = + httpClient2 + .post(url"$wireMockUrl/") + .withBody(body) + .execute(fromJson[ResDomain]) + + res.futureValue shouldBe ResDomain("res") + + wireMockServer.verify( + postRequestedFor(urlEqualTo("/")) + .withHeader("Content-Type", equalTo("nonstandard/x-www-form-urlencoded")) + .withRequestBody(equalTo("k1=v1&k1=v2&k2=v3")) + .withHeader("User-Agent", equalTo("myapp")) + ) + + val headersCaptor = ArgCaptor[Seq[(String, String)]] + val responseCaptor = ArgCaptor[Future[HttpResponse]] + + verify(mockHttpHook) + .apply( + verb = eqTo("POST"), + url = eqTo(url"$wireMockUrl/"), + headers = headersCaptor, + body = eqTo(Some(HookData.FromMap(body.toMap))), + responseF = responseCaptor + )(any[HeaderCarrier], any[ExecutionContext]) + + headersCaptor.value should contain ("User-Agent" -> "myapp") + headersCaptor.value should contain ("Content-Type" -> "application/x-www-form-urlencoded") + val auditedResponse = responseCaptor.value.futureValue + auditedResponse.status shouldBe 200 + auditedResponse.body shouldBe "\"res\"" + } + */ + "fail if call withBody on the wsRequest itself" in new Setup { implicit val hc = HeaderCarrier() From cae7eef0f28388af71b44f4deb647c55f441fdd3 Mon Sep 17 00:00:00 2001 From: colin-lamed <9568290+colin-lamed@users.noreply.github.com> Date: Fri, 5 Nov 2021 13:31:13 +0000 Subject: [PATCH 08/23] BDOG-1512 Add test for audit body truncation --- .../src/main/resources/reference.conf | 2 +- .../gov/hmrc/http/play/HttpClient2Spec.scala | 70 +++++++++++++++++-- 2 files changed, 64 insertions(+), 8 deletions(-) diff --git a/http-verbs-common/src/main/resources/reference.conf b/http-verbs-common/src/main/resources/reference.conf index d28793c7..26b2d8cd 100644 --- a/http-verbs-common/src/main/resources/reference.conf +++ b/http-verbs-common/src/main/resources/reference.conf @@ -19,6 +19,6 @@ http-verbs { intervals = [ "500.millis", "1.second", "2.seconds", "4.seconds", "8.seconds" ] ssl-engine-closed-already.enabled = false } - auditing.maxBodyLength = 32665 # Confirm with CIP this can be 500000? + auditing.maxBodyLength = 32665 } proxy.enabled = false diff --git a/http-verbs-common/src/test/scala/uk/gov/hmrc/http/play/HttpClient2Spec.scala b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/play/HttpClient2Spec.scala index b4da9e89..cb586170 100644 --- a/http-verbs-common/src/test/scala/uk/gov/hmrc/http/play/HttpClient2Spec.scala +++ b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/play/HttpClient2Spec.scala @@ -40,6 +40,7 @@ import uk.gov.hmrc.http.test.WireMockSupport import java.util.concurrent.atomic.AtomicInteger import scala.concurrent.{ExecutionContext, Future} import scala.concurrent.ExecutionContext.Implicits.global +import scala.util.Random class HttpClient2Spec extends AnyWordSpecLike @@ -96,13 +97,64 @@ class HttpClient2Spec "work with streams" in new Setup { implicit val hc = HeaderCarrier() + val requestBody = Random.alphanumeric.take(maxAuditBodyLength - 1).mkString + val responseBody = Random.alphanumeric.take(maxAuditBodyLength - 1).mkString + wireMockServer.stubFor( WireMock.put(urlEqualTo("/")) - .willReturn(aResponse().withBody("\"res\"").withStatus(200)) + .willReturn(aResponse().withBody(responseBody).withStatus(200)) + ) + + val srcStream: Source[ByteString, _] = + Source.single(ByteString(requestBody)) + + val res: Future[Source[ByteString, _]] = + httpClient2 + .put(url"$wireMockUrl/") + .withBody(srcStream) + .stream(fromStream) + + res.futureValue.map(_.utf8String).runReduce(_ + _).futureValue shouldBe responseBody + + wireMockServer.verify( + putRequestedFor(urlEqualTo("/")) + .withHeader("Content-Type", equalTo("application/octet-stream")) + .withRequestBody(equalTo(requestBody)) + .withHeader("User-Agent", equalTo("myapp")) + ) + + val headersCaptor = ArgCaptor[Seq[(String, String)]] + val responseCaptor = ArgCaptor[Future[HttpResponse]] + + verify(mockHttpHook) + .apply( + verb = eqTo("PUT"), + url = eqTo(url"$wireMockUrl/"), + headers = headersCaptor, + body = eqTo(Some(HookData.FromString(requestBody))), + responseF = responseCaptor + )(any[HeaderCarrier], any[ExecutionContext]) + + headersCaptor.value should contain ("User-Agent" -> "myapp") + headersCaptor.value should contain ("Content-Type" -> "application/octet-stream") + val auditedResponse = responseCaptor.value.futureValue + auditedResponse.status shouldBe 200 + auditedResponse.body shouldBe responseBody + } + + "truncate stream payloads if too long" in new Setup { + implicit val hc = HeaderCarrier() + + val requestBody = Random.alphanumeric.take(maxAuditBodyLength * 2).mkString + val responseBody = Random.alphanumeric.take(maxAuditBodyLength * 2).mkString + + wireMockServer.stubFor( + WireMock.put(urlEqualTo("/")) + .willReturn(aResponse().withBody(responseBody).withStatus(200)) ) val srcStream: Source[ByteString, _] = - Source.single(ByteString("source")) + Source.single(ByteString(requestBody)) val res: Future[Source[ByteString, _]] = httpClient2 @@ -110,12 +162,12 @@ class HttpClient2Spec .withBody(srcStream) .stream(fromStream) - res.futureValue.map(_.utf8String).runReduce(_ + _).futureValue shouldBe "\"res\"" + res.futureValue.map(_.utf8String).runReduce(_ + _).futureValue shouldBe responseBody wireMockServer.verify( putRequestedFor(urlEqualTo("/")) .withHeader("Content-Type", equalTo("application/octet-stream")) - .withRequestBody(equalTo("source")) + .withRequestBody(equalTo(requestBody)) .withHeader("User-Agent", equalTo("myapp")) ) @@ -127,7 +179,7 @@ class HttpClient2Spec verb = eqTo("PUT"), url = eqTo(url"$wireMockUrl/"), headers = headersCaptor, - body = eqTo(Some(HookData.FromString("source"))), // TODO check when this is truncated when too large + body = eqTo(Some(HookData.FromString(requestBody.take(maxAuditBodyLength)))), responseF = responseCaptor )(any[HeaderCarrier], any[ExecutionContext]) @@ -135,7 +187,7 @@ class HttpClient2Spec headersCaptor.value should contain ("Content-Type" -> "application/octet-stream") val auditedResponse = responseCaptor.value.futureValue auditedResponse.status shouldBe 200 - auditedResponse.body shouldBe "\"res\"" // TODO check when this is truncated when too large + auditedResponse.body shouldBe responseBody.take(maxAuditBodyLength) } "work with form data" in new Setup { @@ -446,11 +498,15 @@ class HttpClient2Spec val mockHttpHook = mock[HttpHook](withSettings.lenient) + val maxAuditBodyLength = 30 + val httpClient2: HttpClient2 = { val config = Configuration( ConfigFactory.parseString( - """appName = myapp""" + s"""|appName = myapp + |http-verbs.auditing.maxBodyLength = $maxAuditBodyLength + |""".stripMargin ).withFallback(ConfigFactory.load()) ) new HttpClient2Impl( From 43871889d2bb86595a51cf79b234b09f47620b9d Mon Sep 17 00:00:00 2001 From: colin-lamed <9568290+colin-lamed@users.noreply.github.com> Date: Fri, 5 Nov 2021 16:20:19 +0000 Subject: [PATCH 09/23] BDOG-1512 Truncate strict payloads for auditing --- .../uk/gov/hmrc/http/play/BodyCaptor.scala | 43 ++++++++---- .../gov/hmrc/http/play/HttpClient2Impl.scala | 64 ++++++++++-------- .../hmrc/http/play/ResponseTransformers.scala | 2 +- .../uk/gov/hmrc/play/http/ws/WSRequest.scala | 2 +- .../gov/hmrc/http/play/HttpClient2Spec.scala | 67 ++++++++++++++++--- 5 files changed, 124 insertions(+), 54 deletions(-) diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/BodyCaptor.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/BodyCaptor.scala index a820070d..9c099982 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/BodyCaptor.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/BodyCaptor.scala @@ -20,7 +20,7 @@ import akka.stream.{Attributes, FlowShape, Inlet, Outlet} import akka.stream.scaladsl.{Flow, Sink} import akka.stream.stage._ import akka.util.ByteString -import play.api.Logger +import org.slf4j.LoggerFactory // based on play.filters.csrf.CSRFAction#BodyHandler @@ -33,8 +33,6 @@ private class BodyCaptorFlow( val out = Outlet[ByteString]("BodyCaptorFlow.out") override val shape = FlowShape.of(in, out) - private val logger = Logger(getClass) - override def createLogic(inheritedAttributes: Attributes): GraphStageLogic = new GraphStageLogic(shape) { private var buffer = ByteString.empty @@ -56,15 +54,7 @@ private class BodyCaptorFlow( } override def onUpstreamFinish(): Unit = { - // TODO how to opt out of auditing payloads? - // currently we can only turn of auditing for url (auditDisabledForPattern) - not per method - // and we can't turn off auditing of just the payload (and keep the fact the call has been made) - // what about auditing request payloads, but not response payloads? - if (bodyLength > maxBodyLength) - logger.warn( - s"txm play auditing: $loggingContext sanity check request body $bodyLength exceeds maxLength $maxBodyLength - do you need to be auditing this payload?" - ) - withCapturedBody(buffer.take(maxBodyLength)) + withCapturedBody(BodyCaptor.bodyUpto(buffer, maxBodyLength, loggingContext)) if (isAvailable(out) && buffer == ByteString.empty) push(out, buffer) completeStage() @@ -75,10 +65,12 @@ private class BodyCaptorFlow( } object BodyCaptor { + private val logger = LoggerFactory.getLogger(getClass) + def flow( loggingContext : String, maxBodyLength : Int, - withCapturedBody: ByteString => Unit + withCapturedBody: ByteString => Unit // provide a callback since a Materialized value would be not be available until the flow has been run ): Flow[ByteString, ByteString, akka.NotUsed] = Flow.fromGraph(new BodyCaptorFlow( loggingContext = loggingContext, @@ -89,8 +81,31 @@ object BodyCaptor { def sink( loggingContext : String, maxBodyLength : Int, - withCapturedBody: ByteString => Unit // TODO can we not just expose as the materialised value? + withCapturedBody: ByteString => Unit ): Sink[ByteString, akka.NotUsed] = flow(loggingContext, maxBodyLength, withCapturedBody) .to(Sink.ignore) + + // We raise a warning, but don't provide a mechanism to opt-out of auditing payloads. + // Currently we can only turn off auditing for url (`auditDisabledForPattern` configuration) - not per method, + // and we can't turn off auditing of just the payload (and keep the fact the call has been made) + // TODO Check with CIP whether `RequestBuilder` could have `withoutRequestPayloadAuditing` and `withoutRequestPayloadAuditing`? + // Note, this also applies to bootstrap AuditFilter. + def bodyUpto(body: String, maxBodyLength: Int, loggingContext: String): String = + if (body.length > maxBodyLength) { + logger.warn( + s"txm play auditing: $loggingContext body ${body.length} exceeds maxLength $maxBodyLength - do you need to be auditing this payload?" + ) + body.take(maxBodyLength) + } else + body + + def bodyUpto(body: ByteString, maxBodyLength: Int, loggingContext: String): ByteString = + if (body.length > maxBodyLength) { + logger.warn( + s"txm play auditing: $loggingContext body ${body.length} exceeds maxLength $maxBodyLength - do you need to be auditing this payload?" + ) + body.take(maxBodyLength) + } else + body } diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/HttpClient2Impl.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/HttpClient2Impl.scala index be3de1bb..a1c2a00e 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/HttpClient2Impl.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/HttpClient2Impl.scala @@ -14,8 +14,6 @@ * limitations under the License. */ -// TODO putting this in this package means that all clients which do -// `import uk.gov.hmrc.http._` will then have to make play imports with _root_ `import _root_.play...` package uk.gov.hmrc.http.play import akka.actor.ActorSystem @@ -33,6 +31,7 @@ import uk.gov.hmrc.http.logging.ConnectionTracing import java.net.URL import scala.concurrent.{ExecutionContext, Future, Promise} import scala.reflect.runtime.universe.{TypeTag, typeOf} +import scala.util.{Failure, Success} /* What does HttpVerbs actually provide? @@ -147,7 +146,9 @@ final class RequestBuilderImpl( } override def withBody[B : BodyWritable : TypeTag](body: B): RequestBuilderImpl = { - val hookDataP = Promise[Option[HookData]]() + val hookDataP = Promise[Option[HookData]]() + val maxBodyLength = config.get[Int]("http-verbs.auditing.maxBodyLength") + val loggingContext = s"outgoing ${request.method} ${request.url} request" transform { req => val req2 = req.withBody(body) req2.body match { @@ -158,19 +159,22 @@ final class RequestBuilderImpl( (body, req2.header("Content-Type")) match { case (IsMap(m), _ ) => hookDataP.success(Some(HookData.FromMap(m))) case (_ , Some("application/x-www-form-urlencoded")) => hookDataP.success(Some(HookData.FromMap(FormUrlEncodedParser.parse(bytes.decodeString("UTF-8"))))) - case _ => hookDataP.success(Some(HookData.FromString(bytes.decodeString("UTF-8")))) + case _ => val auditedBody = BodyCaptor.bodyUpto(bytes, maxBodyLength, loggingContext).decodeString("UTF-8") + hookDataP.success(Some(HookData.FromString(auditedBody))) } req2 case SourceBody(source) => val src2: Source[ByteString, _] = source .alsoTo( BodyCaptor.sink( - loggingContext = s"request for outgoing ${request.method} ${request.url}", - maxBodyLength = config.get[Int]("http-verbs.auditing.maxBodyLength"), + loggingContext = loggingContext, + maxBodyLength = maxBodyLength, withCapturedBody = body => hookDataP.success(Some(HookData.FromString(body.decodeString("UTF-8")))) ) - ) - // preserve content-type (it may have been set with a different body writeable - e.g. play.api.libs.ws.WSBodyWritables.bodyWritableOf_Multipart) + ).recover { + case e => hookDataP.failure(e); throw e + } + // preserve content-type (it may have been set with a different body writeable - e.g. play.api.libs.ws.WSBodyWritables.bodyWritableOf_Multipart) req2.header("Content-Type") match { case Some(contentType) => replaceHeaderOnRequest(req2.withBody(src2), "Content-Type" -> contentType) case _ => req2.withBody(src2) @@ -243,7 +247,7 @@ class ExecutorImpl( transformResponse(request, httpResponseF) } - // TODO horrid return type - one HttpResponse is for auditing... + // unfortunate return type - first HttpResponse is the full response, the second HttpResponse is truncated for auditing... private def toHttpResponse( isStream : Boolean, request : WSRequest, @@ -251,6 +255,7 @@ class ExecutorImpl( )(implicit ec: ExecutionContext ): (Future[HttpResponse], Future[HttpResponse]) = { val auditResponseF = Promise[HttpResponse]() + val loggingContext = s"outgoing ${request.method} ${request.url} response" val httpResponseF = for { response <- responseF @@ -260,7 +265,7 @@ class ExecutorImpl( response.bodyAsSource .alsoTo( BodyCaptor.sink( - loggingContext = s"response for outgoing ${request.method} ${request.url}", + loggingContext = loggingContext, maxBodyLength = maxBodyLength, withCapturedBody = body => auditResponseF.success( @@ -272,27 +277,35 @@ class ExecutorImpl( ) ) ) + .recover { + case e => auditResponseF.failure(e); throw e + } HttpResponse( status = response.status, bodyAsSource = source, headers = response.headers.mapValues(_.toSeq).toMap ) } else { - val httpResponse = HttpResponse( + auditResponseF.success( + HttpResponse( status = response.status, - body = response.body, + body = BodyCaptor.bodyUpto(response.body, maxBodyLength, loggingContext), headers = response.headers.mapValues(_.toSeq).toMap ) - auditResponseF.success(httpResponse) - httpResponse + ) + HttpResponse( + status = response.status, + body = response.body, + headers = response.headers.mapValues(_.toSeq).toMap + ) } (httpResponseF, auditResponseF.future) } private def executeHooks( - isStream : Boolean, - request : WSRequest, - hookDataF: Future[Option[HookData]], + isStream : Boolean, + request : WSRequest, + hookDataF : Future[Option[HookData]], auditedResponseF: Future[HttpResponse] // play-auditing expects the body to be a String )(implicit hc: HeaderCarrier, @@ -301,24 +314,21 @@ class ExecutorImpl( def denormalise(hdrs: Map[String, Seq[String]]): Seq[(String, String)] = hdrs.toList.flatMap { case (k, vs) => vs.map(k -> _) } - // what if hookDataF fails? - hookDataF.foreach { body => - println(s"""AUDIT: - verb = ${request.method}, - url = ${new URL(request.url)}, - headers = ${denormalise(request.headers)}, - body = $body, - responseF = $auditedResponseF} - """) + def executeHooksWithHookData(hookData: Option[HookData]) = hooks.foreach( _.apply( verb = request.method, url = new URL(request.url), headers = denormalise(request.headers), - body = body, + body = hookData, responseF = auditedResponseF ) ) + + hookDataF.onComplete { + case Success(hookData) => executeHooksWithHookData(hookData) + case Failure(e) => // this is unlikely, but we want best attempt at auditing + executeHooksWithHookData(None) } } } diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/ResponseTransformers.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/ResponseTransformers.scala index 3904959c..f8911016 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/ResponseTransformers.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/ResponseTransformers.scala @@ -35,7 +35,7 @@ trait ResponseTransformers { mapErrors(request, responseF) .map(_.bodyAsSource) - def fromJson[A]( + def withHttpReads[A]( request: WSRequest, responseF: Future[HttpResponse] )(implicit diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/play/http/ws/WSRequest.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/play/http/ws/WSRequest.scala index 869d66f8..20415c7a 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/play/http/ws/WSRequest.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/play/http/ws/WSRequest.scala @@ -62,7 +62,7 @@ object WSProxyConfiguration { } - def buildWsProxyServer(configuration: Config): Option[WSProxyServer] = { // TODO use Configuration rather than Config throughout? + def buildWsProxyServer(configuration: Config): Option[WSProxyServer] = { def getOptionalString(key: String): Option[String] = if (configuration.hasPath(key)) Some(configuration.getString(key)) else None diff --git a/http-verbs-common/src/test/scala/uk/gov/hmrc/http/play/HttpClient2Spec.scala b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/play/HttpClient2Spec.scala index cb586170..1e2c257f 100644 --- a/http-verbs-common/src/test/scala/uk/gov/hmrc/http/play/HttpClient2Spec.scala +++ b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/play/HttpClient2Spec.scala @@ -64,7 +64,7 @@ class HttpClient2Spec httpClient2 .put(url"$wireMockUrl/") .withBody(Json.toJson(ReqDomain("req"))) - .execute(fromJson[ResDomain]) + .execute(withHttpReads[ResDomain]) res.futureValue shouldBe ResDomain("res") @@ -142,7 +142,7 @@ class HttpClient2Spec auditedResponse.body shouldBe responseBody } - "truncate stream payloads if too long" in new Setup { + "truncate stream payloads for auditing if too long" in new Setup { implicit val hc = HeaderCarrier() val requestBody = Random.alphanumeric.take(maxAuditBodyLength * 2).mkString @@ -190,6 +190,51 @@ class HttpClient2Spec auditedResponse.body shouldBe responseBody.take(maxAuditBodyLength) } + "truncate strict payloads for auditing if too long" in new Setup { + implicit val hc = HeaderCarrier() + + val requestBody = Random.alphanumeric.take(maxAuditBodyLength * 2).mkString + val responseBody = Random.alphanumeric.take(maxAuditBodyLength * 2).mkString + + wireMockServer.stubFor( + WireMock.put(urlEqualTo("/")) + .willReturn(aResponse().withBody(responseBody).withStatus(200)) + ) + + val res: Future[HttpResponse] = + httpClient2 + .put(url"$wireMockUrl/") + .withBody(requestBody) + .execute(withHttpReads[HttpResponse]) + + res.futureValue.body shouldBe responseBody + + wireMockServer.verify( + putRequestedFor(urlEqualTo("/")) + .withHeader("Content-Type", containing("text/plain")) // play-26 doesn't send charset + .withRequestBody(equalTo(requestBody)) + .withHeader("User-Agent", equalTo("myapp")) + ) + + val headersCaptor = ArgCaptor[Seq[(String, String)]] + val responseCaptor = ArgCaptor[Future[HttpResponse]] + + verify(mockHttpHook) + .apply( + verb = eqTo("PUT"), + url = eqTo(url"$wireMockUrl/"), + headers = headersCaptor, + body = eqTo(Some(HookData.FromString(requestBody.take(maxAuditBodyLength)))), + responseF = responseCaptor + )(any[HeaderCarrier], any[ExecutionContext]) + + headersCaptor.value should contain ("User-Agent" -> "myapp") + headersCaptor.value should contain ("Content-Type" -> "text/plain") + val auditedResponse = responseCaptor.value.futureValue + auditedResponse.status shouldBe 200 + auditedResponse.body shouldBe responseBody.take(maxAuditBodyLength) + } + "work with form data" in new Setup { implicit val hc = HeaderCarrier() @@ -208,7 +253,7 @@ class HttpClient2Spec httpClient2 .post(url"$wireMockUrl/") .withBody(body) - .execute(fromJson[ResDomain]) + .execute(withHttpReads[ResDomain]) res.futureValue shouldBe ResDomain("res") @@ -270,7 +315,7 @@ class HttpClient2Spec httpClient2 .post(url"$wireMockUrl/") .withBody(body) - .execute(fromJson[ResDomain]) + .execute(withHttpReads[ResDomain]) res.futureValue shouldBe ResDomain("res") @@ -332,7 +377,7 @@ class HttpClient2Spec httpClient2 .post(url"$wireMockUrl/") .withBody(body) - .execute(fromJson[ResDomain]) + .execute(withHttpReads[ResDomain]) res.futureValue shouldBe ResDomain("res") @@ -395,7 +440,7 @@ class HttpClient2Spec httpClient2 .post(url"$wireMockUrl/") .withBody(body) - .execute(fromJson[ResDomain]) + .execute(withHttpReads[ResDomain]) res.futureValue shouldBe ResDomain("res") @@ -432,9 +477,9 @@ class HttpClient2Spec a[RuntimeException] should be thrownBy httpClient2 .put(url"$wireMockUrl/") - .transform(_.withBody(toJson(ReqDomain("req")))) + .transform(_.withBody(Json.toJson(ReqDomain("req"))) .replaceHeader("User-Agent" -> "ua2") - .execute(fromJson[ResDomain]) + .execute(withHttpReads[ResDomain]) } "allow overriding user-agent" in new Setup { @@ -450,7 +495,7 @@ class HttpClient2Spec .put(url"$wireMockUrl/") .withBody(Json.toJson(ReqDomain("req"))) .replaceHeader("User-Agent" -> "ua2") - .execute(fromJson[ResDomain]) + .execute(withHttpReads[ResDomain]) res.futureValue shouldBe ResDomain("res") @@ -483,8 +528,8 @@ class HttpClient2Spec count.incrementAndGet httpClient2 .put(url"$wireMockUrl/") - .withBody(Json.toJson(ReqDomain("req"))) - .execute(fromJson[ResDomain]) + .withBody((Json.toJson(ReqDomain("req"))) + .execute(withHttpReads[ResDomain]) } res.failed.futureValue shouldBe a[UpstreamErrorResponse] From bba41b197c26118387a328b3adfc7fc4885ef7cf Mon Sep 17 00:00:00 2001 From: colin-lamed <9568290+colin-lamed@users.noreply.github.com> Date: Wed, 24 Nov 2021 14:31:56 +0000 Subject: [PATCH 10/23] BDOG-1512 Simplify Api --- .../uk/gov/hmrc/http/play/HttpClient2.scala | 18 ++---- .../gov/hmrc/http/play/HttpClient2Impl.scala | 55 ++++++++++-------- .../hmrc/http/play/ResponseTransformers.scala | 58 ------------------- .../scala/uk/gov/hmrc/http/play/package.scala | 38 ++++++++++-- .../gov/hmrc/http/play/HttpClient2Spec.scala | 30 +++++----- 5 files changed, 89 insertions(+), 110 deletions(-) delete mode 100644 http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/ResponseTransformers.scala diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/HttpClient2.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/HttpClient2.scala index abe3ad03..da5914da 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/HttpClient2.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/HttpClient2.scala @@ -18,8 +18,10 @@ // `import uk.gov.hmrc.http._` will then have to make play imports with _root_ `import _root_.play...` package uk.gov.hmrc.http.play +import akka.stream.scaladsl.Source +import akka.util.ByteString import play.api.libs.ws.{BodyWritable, WSRequest} -import uk.gov.hmrc.http.{HeaderCarrier, HttpResponse} +import uk.gov.hmrc.http.{HeaderCarrier, HttpReads} import java.net.URL import scala.concurrent.{ExecutionContext, Future} @@ -53,17 +55,9 @@ trait HttpClient2 { trait RequestBuilder { def transform(transform: WSRequest => WSRequest): RequestBuilder - def execute[A]( - transformResponse: (WSRequest, Future[HttpResponse]) => Future[A] - )(implicit - ec: ExecutionContext - ): Future[A] - - def stream[A]( - transformResponse: (WSRequest, Future[HttpResponse]) => Future[A] - )(implicit - ec: ExecutionContext - ): Future[A] + def execute[A: HttpReads](implicit ec: ExecutionContext): Future[A] + + def stream[A: StreamHttpReads](implicit ec: ExecutionContext): Future[A] // support functions diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/HttpClient2Impl.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/HttpClient2Impl.scala index a1c2a00e..91151b19 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/HttpClient2Impl.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/HttpClient2Impl.scala @@ -23,16 +23,18 @@ import com.typesafe.config.Config import play.api.Configuration import play.api.libs.ws.{BodyWritable, EmptyBody, InMemoryBody, SourceBody, WSClient, WSProxyServer, WSRequest, WSResponse} import play.core.parsers.FormUrlEncodedParser -import uk.gov.hmrc.http.{HeaderCarrier, HttpResponse, Retries} +import uk.gov.hmrc.http.{BadGatewayException, GatewayTimeoutException, HeaderCarrier, HttpReads, HttpResponse, Retries} import uk.gov.hmrc.play.http.ws.WSProxyConfiguration import uk.gov.hmrc.http.hooks.{HookData, HttpHook} import uk.gov.hmrc.http.logging.ConnectionTracing -import java.net.URL +import java.net.{ConnectException, URL} +import java.util.concurrent.TimeoutException import scala.concurrent.{ExecutionContext, Future, Promise} import scala.reflect.runtime.universe.{TypeTag, typeOf} import scala.util.{Failure, Success} + /* What does HttpVerbs actually provide? Readme says... @@ -57,9 +59,8 @@ trait Executor { def execute[A]( request : WSRequest, hookDataF: Option[Future[Option[HookData]]], - isStream : Boolean - )( - transformResponse: (WSRequest, Future[HttpResponse]) => Future[A] + isStream : Boolean, + r : HttpReads[A] )(implicit hc: HeaderCarrier, ec: ExecutionContext @@ -76,9 +77,12 @@ class HttpClient2Impl( private lazy val optProxyServer = WSProxyConfiguration.buildWsProxyServer(config.underlying) - private lazy val hcConfig = + private val hcConfig = HeaderCarrier.Config.fromConfig(config.underlying) + private val executor = + new ExecutorImpl(actorSystem, config, hooks) + override protected def mkRequestBuilder( url : URL, method: String @@ -88,7 +92,7 @@ class HttpClient2Impl( new RequestBuilderImpl( config, optProxyServer, - new ExecutorImpl(actorSystem, config, hooks) + executor )( wsClient .url(url.toString) @@ -185,19 +189,11 @@ final class RequestBuilderImpl( // -- Execution -- - override def execute[A]( - transformResponse: (WSRequest, Future[HttpResponse]) => Future[A] - )(implicit - ec: ExecutionContext - ): Future[A] = - executor.execute(request, hookDataF, isStream = false)(transformResponse) + override def execute[A](implicit r: HttpReads[A], ec: ExecutionContext): Future[A] = + executor.execute(request, hookDataF, isStream = false, r) - override def stream[A]( - transformResponse: (WSRequest, Future[HttpResponse]) => Future[A] - )(implicit - ec: ExecutionContext - ): Future[A] = - executor.execute(request, hookDataF, isStream = true)(transformResponse) + override def stream[A](implicit r: StreamHttpReads[A], ec: ExecutionContext): Future[A] = + executor.execute(request, hookDataF, isStream = true, r) } class ExecutorImpl( @@ -216,9 +212,8 @@ class ExecutorImpl( def execute[A]( request : WSRequest, optHookDataF: Option[Future[Option[HookData]]], - isStream : Boolean - )( - transformResponse: (WSRequest, Future[HttpResponse]) => Future[A] + isStream : Boolean, + r : HttpReads[A] )(implicit hc: HeaderCarrier, ec: ExecutionContext @@ -244,7 +239,9 @@ class ExecutorImpl( // we don't delegate the response conversion to the client // (i.e. return Future[WSResponse] to be handled with Future.transform/transformWith(...)) // since the transform functions require access to the request (method and url) - transformResponse(request, httpResponseF) + // also `mapErrors` is not performed by HttpReads for backward compatibility + mapErrors(request, httpResponseF) + .map(r.read(request.method, request.url, _)) } // unfortunate return type - first HttpResponse is the full response, the second HttpResponse is truncated for auditing... @@ -331,4 +328,16 @@ class ExecutorImpl( executeHooksWithHookData(None) } } + + // TODO what if clients want to override this? + private def mapErrors( + request : WSRequest, + responseF: Future[HttpResponse] + )(implicit + ec: ExecutionContext + ): Future[HttpResponse] = + responseF.recoverWith { + case e: TimeoutException => Future.failed(new GatewayTimeoutException(s"${request.method} of '${request.url}' timed out with message '${e.getMessage}'")) + case e: ConnectException => Future.failed(new BadGatewayException(s"${request.method} of '${request.url}' failed. Caused by: '${e.getMessage}'")) + } } diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/ResponseTransformers.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/ResponseTransformers.scala deleted file mode 100644 index f8911016..00000000 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/ResponseTransformers.scala +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright 2021 HM Revenue & Customs - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package uk.gov.hmrc.http.play - -import akka.stream.scaladsl.Source -import akka.util.ByteString -import _root_.play.api.libs.ws.WSRequest -import uk.gov.hmrc.http.{BadGatewayException, GatewayTimeoutException, HttpResponse, HttpReads} - -import java.util.concurrent.TimeoutException -import java.net.ConnectException -import scala.concurrent.{ExecutionContext, Future} - -trait ResponseTransformers { - def fromStream( - request : WSRequest, - responseF: Future[HttpResponse] - )(implicit - ec: ExecutionContext - ): Future[Source[ByteString, _]] = - mapErrors(request, responseF) - .map(_.bodyAsSource) - - def withHttpReads[A]( - request: WSRequest, - responseF: Future[HttpResponse] - )(implicit - r : HttpReads[A], - ec: ExecutionContext - ): Future[A] = - mapErrors(request, responseF) - .map(r.read(request.method, request.url, _)) - - def mapErrors( - request : WSRequest, - responseF: Future[HttpResponse] - )(implicit - ec: ExecutionContext - ): Future[HttpResponse] = - responseF.recoverWith { - case e: TimeoutException => Future.failed(new GatewayTimeoutException(s"${request.method} of '${request.url}' timed out with message '${e.getMessage}'")) - case e: ConnectException => Future.failed(new BadGatewayException(s"${request.method} of '${request.url}' failed. Caused by: '${e.getMessage}'")) - } -} diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/package.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/package.scala index ad619332..ccc0b48c 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/package.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/package.scala @@ -16,9 +16,39 @@ package uk.gov.hmrc.http +import akka.stream.scaladsl.Source +import akka.util.ByteString + import _root_.play.api.libs.json.{JsValue, Writes} -// These will still need explicitly importing -// should they be moved to `import httpClient2._`? which means implementations can then depend on -// httpClient values (e.g. configuration) or does this make mocking/providing alternative implementations harder? -package object play extends ResponseTransformers +package object play { + // ensure strict HttpReads are not passed to stream function, which would lead to stream being read into memory + trait Streaming + type StreamHttpReads[A] = HttpReads[A] with Streaming +} + +trait StreamHttpReadsInstances { + def tag[A](instance: A): A with play.Streaming = + instance.asInstanceOf[A with play.Streaming] + + implicit val readEitherSource: HttpReads[Either[UpstreamErrorResponse, Source[ByteString, _]]] with play.Streaming = + tag[HttpReads[Either[UpstreamErrorResponse, Source[ByteString, _]]]]( + HttpReads.ask.flatMap { case (method, url, response) => + HttpErrorFunctions.handleResponseEither(method, url)(response) match { + case Left(err) => HttpReads.pure(Left(err)) + case Right(response) => HttpReads.pure(Right(response.bodyAsSource)) + } + } + ) + + implicit val readSource: HttpReads[Source[ByteString, _]] with play.Streaming = + tag[HttpReads[Source[ByteString, _]]]( + readEitherSource + .map { + case Left(err) => throw err + case Right(value) => value + } + ) +} + +object StreamHttpReadsInstances extends StreamHttpReadsInstances diff --git a/http-verbs-common/src/test/scala/uk/gov/hmrc/http/play/HttpClient2Spec.scala b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/play/HttpClient2Spec.scala index 1e2c257f..f16fb782 100644 --- a/http-verbs-common/src/test/scala/uk/gov/hmrc/http/play/HttpClient2Spec.scala +++ b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/play/HttpClient2Spec.scala @@ -51,6 +51,10 @@ class HttpClient2Spec with MockitoSugar with ArgumentMatchersSugar { + import uk.gov.hmrc.http.HttpReads.Implicits._ + + import uk.gov.hmrc.http.StreamHttpReadsInstances._ + "HttpClient2" should { "work with json" in new Setup { implicit val hc = HeaderCarrier() @@ -64,7 +68,7 @@ class HttpClient2Spec httpClient2 .put(url"$wireMockUrl/") .withBody(Json.toJson(ReqDomain("req"))) - .execute(withHttpReads[ResDomain]) + .execute[ResDomain] res.futureValue shouldBe ResDomain("res") @@ -112,7 +116,7 @@ class HttpClient2Spec httpClient2 .put(url"$wireMockUrl/") .withBody(srcStream) - .stream(fromStream) + .stream[Source[ByteString, _]] res.futureValue.map(_.utf8String).runReduce(_ + _).futureValue shouldBe responseBody @@ -160,7 +164,7 @@ class HttpClient2Spec httpClient2 .put(url"$wireMockUrl/") .withBody(srcStream) - .stream(fromStream) + .stream[Source[ByteString, _]] res.futureValue.map(_.utf8String).runReduce(_ + _).futureValue shouldBe responseBody @@ -205,7 +209,7 @@ class HttpClient2Spec httpClient2 .put(url"$wireMockUrl/") .withBody(requestBody) - .execute(withHttpReads[HttpResponse]) + .execute[HttpResponse] res.futureValue.body shouldBe responseBody @@ -253,7 +257,7 @@ class HttpClient2Spec httpClient2 .post(url"$wireMockUrl/") .withBody(body) - .execute(withHttpReads[ResDomain]) + .execute[ResDomain] res.futureValue shouldBe ResDomain("res") @@ -315,7 +319,7 @@ class HttpClient2Spec httpClient2 .post(url"$wireMockUrl/") .withBody(body) - .execute(withHttpReads[ResDomain]) + .execute[ResDomain] res.futureValue shouldBe ResDomain("res") @@ -377,7 +381,7 @@ class HttpClient2Spec httpClient2 .post(url"$wireMockUrl/") .withBody(body) - .execute(withHttpReads[ResDomain]) + .execute[ResDomain] res.futureValue shouldBe ResDomain("res") @@ -440,7 +444,7 @@ class HttpClient2Spec httpClient2 .post(url"$wireMockUrl/") .withBody(body) - .execute(withHttpReads[ResDomain]) + .execute[ResDomain] res.futureValue shouldBe ResDomain("res") @@ -477,9 +481,9 @@ class HttpClient2Spec a[RuntimeException] should be thrownBy httpClient2 .put(url"$wireMockUrl/") - .transform(_.withBody(Json.toJson(ReqDomain("req"))) + .transform(_.withBody(Json.toJson(ReqDomain("req")))) .replaceHeader("User-Agent" -> "ua2") - .execute(withHttpReads[ResDomain]) + .execute[ResDomain] } "allow overriding user-agent" in new Setup { @@ -495,7 +499,7 @@ class HttpClient2Spec .put(url"$wireMockUrl/") .withBody(Json.toJson(ReqDomain("req"))) .replaceHeader("User-Agent" -> "ua2") - .execute(withHttpReads[ResDomain]) + .execute[ResDomain] res.futureValue shouldBe ResDomain("res") @@ -528,8 +532,8 @@ class HttpClient2Spec count.incrementAndGet httpClient2 .put(url"$wireMockUrl/") - .withBody((Json.toJson(ReqDomain("req"))) - .execute(withHttpReads[ResDomain]) + .withBody(Json.toJson(ReqDomain("req"))) + .execute[ResDomain] } res.failed.futureValue shouldBe a[UpstreamErrorResponse] From c883a791501438d7a0eef7f06abf2dea9b330415 Mon Sep 17 00:00:00 2001 From: colin-lamed <9568290+colin-lamed@users.noreply.github.com> Date: Mon, 29 Nov 2021 17:21:11 +0000 Subject: [PATCH 11/23] BDOG-1512 Renamed play package to client2 --- .../scala/uk/gov/hmrc/http/HttpPatch.scala | 2 +- .../scala/uk/gov/hmrc/http/HttpPost.scala | 2 +- .../main/scala/uk/gov/hmrc/http/HttpPut.scala | 2 +- .../uk/gov/hmrc/http/HttpReadsInstances.scala | 2 +- .../hmrc/http/HttpReadsLegacyInstances.scala | 2 +- .../scala/uk/gov/hmrc/http/HttpResponse.scala | 2 +- .../uk/gov/hmrc/http/HttpTransport.scala | 2 +- .../http/{play => client2}/HttpClient2.scala | 4 +--- .../{play => client2}/HttpClient2Impl.scala | 4 ++-- .../hmrc/http/{play => client2}/package.scala | 19 +++++++++++-------- .../{http/play => play/http}/BodyCaptor.scala | 2 +- .../gov/hmrc/http/CommonHttpBehaviour.scala | 2 +- .../scala/uk/gov/hmrc/http/HeadersSpec.scala | 10 +++++----- .../uk/gov/hmrc/http/HttpPatchSpec.scala | 2 +- .../scala/uk/gov/hmrc/http/HttpPostSpec.scala | 2 +- .../scala/uk/gov/hmrc/http/HttpPutSpec.scala | 2 +- .../hmrc/http/HttpReadsInstancesSpec.scala | 2 +- .../http/HttpReadsLegacyInstancesSpec.scala | 2 +- .../scala/uk/gov/hmrc/http/RetriesSpec.scala | 2 +- .../{play => client2}/HttpClient2Spec.scala | 2 +- .../http/HeaderCarrierConverterSpec.scala | 2 +- .../hmrc/http/test/HttpClient2Support.scala | 2 +- 22 files changed, 37 insertions(+), 36 deletions(-) rename http-verbs-common/src/main/scala/uk/gov/hmrc/http/{play => client2}/HttpClient2.scala (92%) rename http-verbs-common/src/main/scala/uk/gov/hmrc/http/{play => client2}/HttpClient2Impl.scala (99%) rename http-verbs-common/src/main/scala/uk/gov/hmrc/http/{play => client2}/package.scala (76%) rename http-verbs-common/src/main/scala/uk/gov/hmrc/{http/play => play/http}/BodyCaptor.scala (99%) rename http-verbs-common/src/test/scala/uk/gov/hmrc/http/{play => client2}/HttpClient2Spec.scala (99%) diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpPatch.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpPatch.scala index 1e20ed76..c3de8674 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpPatch.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpPatch.scala @@ -16,7 +16,7 @@ package uk.gov.hmrc.http -import _root_.play.api.libs.json.{Json, Writes} +import play.api.libs.json.{Json, Writes} import uk.gov.hmrc.http.HttpVerbs.{PATCH => PATCH_VERB} import uk.gov.hmrc.http.hooks.{HookData, HttpHooks} import uk.gov.hmrc.http.logging.ConnectionTracing diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpPost.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpPost.scala index ab801190..b303316f 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpPost.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpPost.scala @@ -16,7 +16,7 @@ package uk.gov.hmrc.http -import _root_.play.api.libs.json.{Json, Writes} +import play.api.libs.json.{Json, Writes} import uk.gov.hmrc.http.HttpVerbs.{POST => POST_VERB} import uk.gov.hmrc.http.hooks.{HookData, HttpHooks} import uk.gov.hmrc.http.logging.ConnectionTracing diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpPut.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpPut.scala index d85209ba..7dc64009 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpPut.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpPut.scala @@ -16,7 +16,7 @@ package uk.gov.hmrc.http -import _root_.play.api.libs.json.{Json, Writes} +import play.api.libs.json.{Json, Writes} import uk.gov.hmrc.http.HttpVerbs.{PUT => PUT_VERB} import uk.gov.hmrc.http.hooks.{HookData, HttpHooks} import uk.gov.hmrc.http.logging.ConnectionTracing diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpReadsInstances.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpReadsInstances.scala index b671a0c0..3d261bb7 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpReadsInstances.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpReadsInstances.scala @@ -16,7 +16,7 @@ package uk.gov.hmrc.http -import _root_.play.api.libs.json.{JsValue, JsError, JsResult, JsSuccess, Reads => JsonReads} +import play.api.libs.json.{JsValue, JsError, JsResult, JsSuccess, Reads => JsonReads} import scala.util.Try trait HttpReadsInstances diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpReadsLegacyInstances.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpReadsLegacyInstances.scala index 7eb17003..b078046d 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpReadsLegacyInstances.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpReadsLegacyInstances.scala @@ -17,7 +17,7 @@ package uk.gov.hmrc.http import com.github.ghik.silencer.silent -import _root_.play.api.libs.json.{JsNull, JsValue, Reads} +import play.api.libs.json.{JsNull, JsValue, Reads} trait HttpReadsLegacyInstances extends HttpReadsLegacyOption with HttpReadsLegacyJson diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpResponse.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpResponse.scala index 38e7e9fb..dd42b7bd 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpResponse.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpResponse.scala @@ -19,7 +19,7 @@ package uk.gov.hmrc.http import akka.stream.scaladsl.Source import akka.util.ByteString import com.github.ghik.silencer.silent -import _root_.play.api.libs.json.{JsValue, Json} +import play.api.libs.json.{JsValue, Json} /** * The ws.Response class is very hard to dummy up as it wraps a concrete instance of diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpTransport.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpTransport.scala index 3edc77f9..59c44e68 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpTransport.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpTransport.scala @@ -18,7 +18,7 @@ package uk.gov.hmrc.http import java.net.URL -import _root_.play.api.libs.json.Writes +import play.api.libs.json.Writes import scala.concurrent.{ExecutionContext, Future} diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/HttpClient2.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/client2/HttpClient2.scala similarity index 92% rename from http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/HttpClient2.scala rename to http-verbs-common/src/main/scala/uk/gov/hmrc/http/client2/HttpClient2.scala index da5914da..5e6fefb5 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/HttpClient2.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/client2/HttpClient2.scala @@ -14,9 +14,7 @@ * limitations under the License. */ -// TODO putting this in this package means that all clients which do -// `import uk.gov.hmrc.http._` will then have to make play imports with _root_ `import _root_.play...` -package uk.gov.hmrc.http.play +package uk.gov.hmrc.http.client2 import akka.stream.scaladsl.Source import akka.util.ByteString diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/HttpClient2Impl.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/client2/HttpClient2Impl.scala similarity index 99% rename from http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/HttpClient2Impl.scala rename to http-verbs-common/src/main/scala/uk/gov/hmrc/http/client2/HttpClient2Impl.scala index 91151b19..8c2be0b6 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/HttpClient2Impl.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/client2/HttpClient2Impl.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package uk.gov.hmrc.http.play +package uk.gov.hmrc.http.client2 import akka.actor.ActorSystem import akka.stream.scaladsl.Source @@ -24,6 +24,7 @@ import play.api.Configuration import play.api.libs.ws.{BodyWritable, EmptyBody, InMemoryBody, SourceBody, WSClient, WSProxyServer, WSRequest, WSResponse} import play.core.parsers.FormUrlEncodedParser import uk.gov.hmrc.http.{BadGatewayException, GatewayTimeoutException, HeaderCarrier, HttpReads, HttpResponse, Retries} +import uk.gov.hmrc.play.http.BodyCaptor import uk.gov.hmrc.play.http.ws.WSProxyConfiguration import uk.gov.hmrc.http.hooks.{HookData, HttpHook} import uk.gov.hmrc.http.logging.ConnectionTracing @@ -239,7 +240,6 @@ class ExecutorImpl( // we don't delegate the response conversion to the client // (i.e. return Future[WSResponse] to be handled with Future.transform/transformWith(...)) // since the transform functions require access to the request (method and url) - // also `mapErrors` is not performed by HttpReads for backward compatibility mapErrors(request, httpResponseF) .map(r.read(request.method, request.url, _)) } diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/package.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/client2/package.scala similarity index 76% rename from http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/package.scala rename to http-verbs-common/src/main/scala/uk/gov/hmrc/http/client2/package.scala index ccc0b48c..80e93fcb 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/package.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/client2/package.scala @@ -19,19 +19,22 @@ package uk.gov.hmrc.http import akka.stream.scaladsl.Source import akka.util.ByteString -import _root_.play.api.libs.json.{JsValue, Writes} - -package object play { - // ensure strict HttpReads are not passed to stream function, which would lead to stream being read into memory +package client2 { trait Streaming +} +package object client2 + extends StreamHttpReadsInstances { + + // ensures strict HttpReads are not passed to stream function, which would lead to stream being read into memory + // (or runtime exceptions since HttpResponse.body with throw exception for streamed responses) type StreamHttpReads[A] = HttpReads[A] with Streaming } trait StreamHttpReadsInstances { - def tag[A](instance: A): A with play.Streaming = - instance.asInstanceOf[A with play.Streaming] + def tag[A](instance: A): A with client2.Streaming = + instance.asInstanceOf[A with client2.Streaming] - implicit val readEitherSource: HttpReads[Either[UpstreamErrorResponse, Source[ByteString, _]]] with play.Streaming = + implicit val readEitherSource: HttpReads[Either[UpstreamErrorResponse, Source[ByteString, _]]] with client2.Streaming = tag[HttpReads[Either[UpstreamErrorResponse, Source[ByteString, _]]]]( HttpReads.ask.flatMap { case (method, url, response) => HttpErrorFunctions.handleResponseEither(method, url)(response) match { @@ -41,7 +44,7 @@ trait StreamHttpReadsInstances { } ) - implicit val readSource: HttpReads[Source[ByteString, _]] with play.Streaming = + implicit val readSource: HttpReads[Source[ByteString, _]] with client2.Streaming = tag[HttpReads[Source[ByteString, _]]]( readEitherSource .map { diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/BodyCaptor.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/play/http/BodyCaptor.scala similarity index 99% rename from http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/BodyCaptor.scala rename to http-verbs-common/src/main/scala/uk/gov/hmrc/play/http/BodyCaptor.scala index 9c099982..537494b8 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/play/BodyCaptor.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/play/http/BodyCaptor.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package uk.gov.hmrc.http.play +package uk.gov.hmrc.play.http import akka.stream.{Attributes, FlowShape, Inlet, Outlet} import akka.stream.scaladsl.{Flow, Sink} diff --git a/http-verbs-common/src/test/scala/uk/gov/hmrc/http/CommonHttpBehaviour.scala b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/CommonHttpBehaviour.scala index f139de53..f4f8f3ed 100644 --- a/http-verbs-common/src/test/scala/uk/gov/hmrc/http/CommonHttpBehaviour.scala +++ b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/CommonHttpBehaviour.scala @@ -22,7 +22,7 @@ import java.util.concurrent.TimeoutException import org.scalatest.concurrent.ScalaFutures import org.scalatest.wordspec.AnyWordSpecLike import org.scalatest.matchers.should.Matchers -import _root_.play.api.libs.json.{Json, OFormat} +import play.api.libs.json.{Json, OFormat} import uk.gov.hmrc.http.logging.{ConnectionTracing, LoggingDetails} import scala.collection.mutable diff --git a/http-verbs-common/src/test/scala/uk/gov/hmrc/http/HeadersSpec.scala b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/HeadersSpec.scala index a985b115..5ae0ae7c 100644 --- a/http-verbs-common/src/test/scala/uk/gov/hmrc/http/HeadersSpec.scala +++ b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/HeadersSpec.scala @@ -24,11 +24,11 @@ import com.typesafe.config.Config import org.scalatest.concurrent.{IntegrationPatience, ScalaFutures} import org.scalatest.matchers.must.Matchers import org.scalatest.wordspec.AnyWordSpecLike -import _root_.play.api.Application -import _root_.play.api.http.{HeaderNames => PlayHeaderNames} -import _root_.play.api.inject.guice.GuiceApplicationBuilder -import _root_.play.api.libs.json.{JsValue, Json} -import _root_.play.api.libs.ws.WSClient +import play.api.Application +import play.api.http.{HeaderNames => PlayHeaderNames} +import play.api.inject.guice.GuiceApplicationBuilder +import play.api.libs.json.{JsValue, Json} +import play.api.libs.ws.WSClient import uk.gov.hmrc.http.hooks.HttpHook import uk.gov.hmrc.play.http.ws.WSHttp import uk.gov.hmrc.http.test.WireMockSupport diff --git a/http-verbs-common/src/test/scala/uk/gov/hmrc/http/HttpPatchSpec.scala b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/HttpPatchSpec.scala index 46ad4eec..b4f843fd 100644 --- a/http-verbs-common/src/test/scala/uk/gov/hmrc/http/HttpPatchSpec.scala +++ b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/HttpPatchSpec.scala @@ -23,7 +23,7 @@ import org.mockito.captor.ArgCaptor import org.mockito.scalatest.MockitoSugar import org.scalatest.wordspec.AnyWordSpecLike import org.scalatest.matchers.should.Matchers -import _root_.play.api.libs.json.{Json, Writes} +import play.api.libs.json.{Json, Writes} import uk.gov.hmrc.http.hooks.{HookData, HttpHook} import scala.concurrent.{ExecutionContext, Future} diff --git a/http-verbs-common/src/test/scala/uk/gov/hmrc/http/HttpPostSpec.scala b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/HttpPostSpec.scala index dd3a9261..93499aba 100644 --- a/http-verbs-common/src/test/scala/uk/gov/hmrc/http/HttpPostSpec.scala +++ b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/HttpPostSpec.scala @@ -23,7 +23,7 @@ import org.mockito.captor.ArgCaptor import org.mockito.scalatest.MockitoSugar import org.scalatest.wordspec.AnyWordSpecLike import org.scalatest.matchers.should.Matchers -import _root_.play.api.libs.json.{Json, Writes} +import play.api.libs.json.{Json, Writes} import uk.gov.hmrc.http.hooks.{HookData, HttpHook} import scala.concurrent.{ExecutionContext, Future} diff --git a/http-verbs-common/src/test/scala/uk/gov/hmrc/http/HttpPutSpec.scala b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/HttpPutSpec.scala index 119151e2..ff458422 100644 --- a/http-verbs-common/src/test/scala/uk/gov/hmrc/http/HttpPutSpec.scala +++ b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/HttpPutSpec.scala @@ -23,7 +23,7 @@ import org.mockito.captor.ArgCaptor import org.mockito.scalatest.MockitoSugar import org.scalatest.wordspec.AnyWordSpecLike import org.scalatest.matchers.should.Matchers -import _root_.play.api.libs.json.{Json, Writes} +import play.api.libs.json.{Json, Writes} import uk.gov.hmrc.http.hooks.{HookData, HttpHook} import scala.concurrent.{ExecutionContext, Future} diff --git a/http-verbs-common/src/test/scala/uk/gov/hmrc/http/HttpReadsInstancesSpec.scala b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/HttpReadsInstancesSpec.scala index 74cf8082..86e9e4ee 100644 --- a/http-verbs-common/src/test/scala/uk/gov/hmrc/http/HttpReadsInstancesSpec.scala +++ b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/HttpReadsInstancesSpec.scala @@ -21,7 +21,7 @@ import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks import org.scalatest.TryValues import org.scalatest.wordspec.AnyWordSpec import org.scalatest.matchers.should.Matchers -import _root_.play.api.libs.json.{__, Json, JsError, JsResult, JsSuccess} +import play.api.libs.json.{__, Json, JsError, JsResult, JsSuccess} import scala.util.{Failure, Success, Try} import uk.gov.hmrc.http.HttpReads.Implicits._ diff --git a/http-verbs-common/src/test/scala/uk/gov/hmrc/http/HttpReadsLegacyInstancesSpec.scala b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/HttpReadsLegacyInstancesSpec.scala index f14c3049..85ec218a 100644 --- a/http-verbs-common/src/test/scala/uk/gov/hmrc/http/HttpReadsLegacyInstancesSpec.scala +++ b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/HttpReadsLegacyInstancesSpec.scala @@ -21,7 +21,7 @@ import org.scalacheck.Gen import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks import org.scalatest.wordspec.AnyWordSpec import org.scalatest.matchers.should.Matchers -import _root_.play.api.libs.json.Json +import play.api.libs.json.Json @silent("deprecated") class HttpReadsLegacyInstancesSpec extends AnyWordSpec with ScalaCheckDrivenPropertyChecks with Matchers { diff --git a/http-verbs-common/src/test/scala/uk/gov/hmrc/http/RetriesSpec.scala b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/RetriesSpec.scala index 5080cb67..22f4cdb2 100644 --- a/http-verbs-common/src/test/scala/uk/gov/hmrc/http/RetriesSpec.scala +++ b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/RetriesSpec.scala @@ -28,7 +28,7 @@ import org.scalatest.concurrent.{IntegrationPatience, ScalaFutures} import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpecLike import org.slf4j.MDC -import _root_.play.api.libs.json.{JsValue, Json, Writes} +import play.api.libs.json.{JsValue, Json, Writes} import uk.gov.hmrc.http.HttpReads.Implicits._ import uk.gov.hmrc.http.hooks.{HttpHook, HttpHooks} import uk.gov.hmrc.play.http.logging.Mdc diff --git a/http-verbs-common/src/test/scala/uk/gov/hmrc/http/play/HttpClient2Spec.scala b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/client2/HttpClient2Spec.scala similarity index 99% rename from http-verbs-common/src/test/scala/uk/gov/hmrc/http/play/HttpClient2Spec.scala rename to http-verbs-common/src/test/scala/uk/gov/hmrc/http/client2/HttpClient2Spec.scala index f16fb782..eb075397 100644 --- a/http-verbs-common/src/test/scala/uk/gov/hmrc/http/play/HttpClient2Spec.scala +++ b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/client2/HttpClient2Spec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package uk.gov.hmrc.http.play +package uk.gov.hmrc.http.client2 import akka.actor.ActorSystem import akka.stream.{ActorMaterializer, Materializer} diff --git a/http-verbs-common/src/test/scala/uk/gov/hmrc/play/http/HeaderCarrierConverterSpec.scala b/http-verbs-common/src/test/scala/uk/gov/hmrc/play/http/HeaderCarrierConverterSpec.scala index 304d5d41..d2d57723 100644 --- a/http-verbs-common/src/test/scala/uk/gov/hmrc/play/http/HeaderCarrierConverterSpec.scala +++ b/http-verbs-common/src/test/scala/uk/gov/hmrc/play/http/HeaderCarrierConverterSpec.scala @@ -24,7 +24,7 @@ import play.api.inject.guice.GuiceApplicationBuilder import play.api.{Configuration, Play} import play.api.mvc._ import play.api.test.Helpers._ -import _root_.play.api.http.{HeaderNames => PlayHeaderNames} +import play.api.http.{HeaderNames => PlayHeaderNames} import play.api.test.{FakeHeaders, FakeRequest, FakeRequestFactory} import play.api.mvc.request.RequestFactory import uk.gov.hmrc.http._ diff --git a/http-verbs-test-common/src/main/scala/uk/gov/hmrc/http/test/HttpClient2Support.scala b/http-verbs-test-common/src/main/scala/uk/gov/hmrc/http/test/HttpClient2Support.scala index ec418a65..dfbfffcd 100644 --- a/http-verbs-test-common/src/main/scala/uk/gov/hmrc/http/test/HttpClient2Support.scala +++ b/http-verbs-test-common/src/main/scala/uk/gov/hmrc/http/test/HttpClient2Support.scala @@ -22,7 +22,7 @@ import com.github.ghik.silencer.silent import com.typesafe.config.ConfigFactory import play.api.Configuration import play.api.libs.ws.ahc.{AhcWSClient, AhcWSClientConfigFactory} -import uk.gov.hmrc.http.play.{HttpClient2, HttpClient2Impl} +import uk.gov.hmrc.http.client2.{HttpClient2, HttpClient2Impl} trait HttpClient2Support { From 7c2b29bfabff34a813ccf38b5984a4a01def94ba Mon Sep 17 00:00:00 2001 From: colin-lamed <9568290+colin-lamed@users.noreply.github.com> Date: Mon, 29 Nov 2021 17:30:05 +0000 Subject: [PATCH 12/23] BDOG-1512 Add StreamHttpReadsInstances to package object --- .../src/main/scala/uk/gov/hmrc/http/client2/package.scala | 2 -- .../test/scala/uk/gov/hmrc/http/client2/HttpClient2Spec.scala | 2 -- 2 files changed, 4 deletions(-) diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/client2/package.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/client2/package.scala index 80e93fcb..06bb225f 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/client2/package.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/client2/package.scala @@ -53,5 +53,3 @@ trait StreamHttpReadsInstances { } ) } - -object StreamHttpReadsInstances extends StreamHttpReadsInstances diff --git a/http-verbs-common/src/test/scala/uk/gov/hmrc/http/client2/HttpClient2Spec.scala b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/client2/HttpClient2Spec.scala index eb075397..e376ae5e 100644 --- a/http-verbs-common/src/test/scala/uk/gov/hmrc/http/client2/HttpClient2Spec.scala +++ b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/client2/HttpClient2Spec.scala @@ -53,8 +53,6 @@ class HttpClient2Spec import uk.gov.hmrc.http.HttpReads.Implicits._ - import uk.gov.hmrc.http.StreamHttpReadsInstances._ - "HttpClient2" should { "work with json" in new Setup { implicit val hc = HeaderCarrier() From 2708cda3b404753f1980128cfe9769b074cfa618 Mon Sep 17 00:00:00 2001 From: colin-lamed <9568290+colin-lamed@users.noreply.github.com> Date: Thu, 2 Dec 2021 12:30:08 +0000 Subject: [PATCH 13/23] BDOG-1512 Update CHANGELOG --- CHANGELOG.md | 84 +++++++++++++++++++ .../scala/uk/gov/hmrc/http/HttpResponse.scala | 2 - .../gov/hmrc/http/client2/HttpClient2.scala | 4 + .../hmrc/http/client2/HttpClient2Impl.scala | 20 ----- .../uk/gov/hmrc/play/http/ws/WSRequest.scala | 2 +- 5 files changed, 89 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8be6e689..9ec8db52 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,87 @@ +## Version 14.0.0 + +### Auditing max body length + +Payloads will be truncated in audit logs if they exceed the max supported (as configured by `http-verbs.auditing.maxBodyLength`). + +### WSProxyConfiguration + +`WSProxyConfiguration.apply` has been deprecated, use `WSProxyConfiguration.buildWsProxyServer` instead. + +There are some differences with `WSProxyConfiguration.buildWsProxyServer`: + * configPrefix is fixed to `proxy`. + * `proxy.proxyRequiredForThisEnvironment` has been replaced with `proxy.enabled`, but note, it defaults to false (rather than true). This is appropriate for development and tests, but will need explicitly enabling when deployed. + + +### Adds HttpClient2 +This is in addition to `HttpClient` (for now), so can be optionally used instead. + +The changes from HttpClient are: +- Supports streaming +- Exposes the underlying `play.api.libs.ws.WSRequest` with `transform`, making it easier to customise the request. + +Examples can be found in [HttpClient2Spec](/http-verbs-common/src/test/scala/uk/gov/hmrc/http/play/HttpClient2Spec.scala) + +To migrate: + +```scala +httpClient.GET[ResponseType](url) +``` + +becomes + +```scala +httpClient2.get(url"$url").execute[ResponseType] +``` + +and + +```scala +httpClient.POST[ResponseType](url, payload, headers) +``` + +becomes + +```scala +httpClient2.post(url"$url").withBody(Json.toJson(payload)).addHeaders(headers).execute[ResponseType] +``` + + +#### Header manipulation + +With `HttpClient`, replacing a header can require providing a customised client implementation (e.g. to replace the user-agent header), or updating the `HeaderCarrier` (e.g. to replace the authorisation header). This can now all be done with the `replaceHeader` on `HttpClient2` per call. e.g. + +```scala +httpClient2.get(url"$url").replaceHeader("User-Agent" -> userAgent).replaceHeader("Authorization" -> authorization).execute[ResponseType] +``` + +#### Using proxy + +With `HttpClient`, to use a proxy requires creating a new instance of HttpClient to mix in `WSProxy` and configure. With `HttpClient2` this can be done with the same client, calling `withProxy` per call. e.g. + +```scala +httpClient2.get(url"$url").withProxy.execute[ResponseType] +``` + +* It uses `WSProxyConfiguration.buildWsProxyServer` which needs enabling with `proxy.enabled` in configuration, which by default is false, for development. See `WSProxyConfiguration` for configuration changes above. + +#### Streaming + +Streaming is supported with `HttpClient2`, and will be audited in the same way as `HttpClient`. Note that payloads will be truncated in audit logs if they exceed the max supported (as configured by `http-verbs.auditing.maxBodyLength`). + +Streamed requests can simply be passed to `withBody`: + +```scala +val reqStream: Source[ByteString, _] = ??? +httpClient2.post(url"$url").withBody(reqStream).execute[ResponseType] +``` + +For streamed responses, use `stream` rather than `execute`: + +```scala +httpClient2.get(url"$url").stream[Source[ByteString, _]] +``` + ## Version 13.12.0 ### Supported Play Versions diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpResponse.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpResponse.scala index dd42b7bd..f8712c33 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpResponse.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpResponse.scala @@ -46,8 +46,6 @@ trait HttpResponse { def json: JsValue = Json.parse(body) - // If we were using wsResponse, we could have a single body function... - //def body[T: BodyReadable]: T = super.body[T] def bodyAsSource: Source[ByteString, _] = Source.single(ByteString(body)) diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/client2/HttpClient2.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/client2/HttpClient2.scala index 5e6fefb5..c7e6ab26 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/client2/HttpClient2.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/client2/HttpClient2.scala @@ -25,6 +25,10 @@ import java.net.URL import scala.concurrent.{ExecutionContext, Future} import scala.reflect.runtime.universe.TypeTag +/** This client centralises the execution of the request to ensure that the common concerns (e.g. auditing, logging, + * retries) occur, but makes building the request more flexible (by exposing play-ws). + * It also supports streaming. + */ trait HttpClient2 { protected def mkRequestBuilder(url: URL, method: String)(implicit hc: HeaderCarrier): RequestBuilder diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/client2/HttpClient2Impl.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/client2/HttpClient2Impl.scala index 8c2be0b6..55022aa4 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/client2/HttpClient2Impl.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/client2/HttpClient2Impl.scala @@ -36,26 +36,6 @@ import scala.reflect.runtime.universe.{TypeTag, typeOf} import scala.util.{Failure, Success} -/* What does HttpVerbs actually provide? - -Readme says... - - Http Transport - - Core Http function interfaces - - Logging - - Propagation of common headers - - Executing hooks, for example Auditing - - Request & Response de-serializations - - Response handling, converting failure status codes into a consistent set of exceptions - allows failures to be automatically propagated to the caller - -Also, retries - - -This version demonstrates a flat implementation that uses an HttpExecutor to centralise the execution of the request to ensure that -the common concerns occur, but delegates out the construction of the request and parsing of the response to play-ws for flexibility. -The use of HttpReads is optional. -Extension methods are provided to make common patterns easier to apply. -*/ - trait Executor { def execute[A]( request : WSRequest, diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/play/http/ws/WSRequest.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/play/http/ws/WSRequest.scala index 20415c7a..86276f41 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/play/http/ws/WSRequest.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/play/http/ws/WSRequest.scala @@ -43,7 +43,7 @@ trait WSProxy extends WSRequest { object WSProxyConfiguration { - @deprecated("Use buildWsProxyServer instead.", "14.0.0") // TODO document differences + @deprecated("Use buildWsProxyServer instead. See docs for differences.", "14.0.0") def apply(configPrefix: String, configuration: Configuration): Option[WSProxyServer] = { val proxyRequired = configuration.getOptional[Boolean](s"$configPrefix.proxyRequiredForThisEnvironment").getOrElse(true) From 399e821cfc6752c8b51ad8cdd0aabf938b9745e5 Mon Sep 17 00:00:00 2001 From: colin-lamed <9568290+colin-lamed@users.noreply.github.com> Date: Fri, 3 Dec 2021 14:06:44 +0000 Subject: [PATCH 14/23] BDOG-1512 Address warnings --- .../src/main/scala/uk/gov/hmrc/http/client2/HttpClient2.scala | 2 -- 1 file changed, 2 deletions(-) diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/client2/HttpClient2.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/client2/HttpClient2.scala index c7e6ab26..914e8f22 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/client2/HttpClient2.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/client2/HttpClient2.scala @@ -16,8 +16,6 @@ package uk.gov.hmrc.http.client2 -import akka.stream.scaladsl.Source -import akka.util.ByteString import play.api.libs.ws.{BodyWritable, WSRequest} import uk.gov.hmrc.http.{HeaderCarrier, HttpReads} From cfa70df8561b83d314bb688915a7279188184cd9 Mon Sep 17 00:00:00 2001 From: colin-lamed <9568290+colin-lamed@users.noreply.github.com> Date: Fri, 3 Dec 2021 14:31:06 +0000 Subject: [PATCH 15/23] BDOG-1512 Make it possible to override mapErrors if required --- .../scala/uk/gov/hmrc/http/client2/HttpClient2Impl.scala | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/client2/HttpClient2Impl.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/client2/HttpClient2Impl.scala index 55022aa4..6de11bda 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/client2/HttpClient2Impl.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/client2/HttpClient2Impl.scala @@ -61,7 +61,7 @@ class HttpClient2Impl( private val hcConfig = HeaderCarrier.Config.fromConfig(config.underlying) - private val executor = + protected val executor = new ExecutorImpl(actorSystem, config, hooks) override protected def mkRequestBuilder( @@ -190,7 +190,7 @@ class ExecutorImpl( private val maxBodyLength = config.get[Int]("http-verbs.auditing.maxBodyLength") - def execute[A]( + final def execute[A]( request : WSRequest, optHookDataF: Option[Future[Option[HookData]]], isStream : Boolean, @@ -309,8 +309,7 @@ class ExecutorImpl( } } - // TODO what if clients want to override this? - private def mapErrors( + protected def mapErrors( request : WSRequest, responseF: Future[HttpResponse] )(implicit From 25d094899155d8a7e0dbf1ba7512923120f2e2a6 Mon Sep 17 00:00:00 2001 From: colin-lamed <9568290+colin-lamed@users.noreply.github.com> Date: Wed, 12 Jan 2022 12:15:09 +0000 Subject: [PATCH 16/23] BDOG-1512 Update log message. Remove 'do you need to be auditing this payload' from log message, since there's no way to address the warning --- .../uk/gov/hmrc/http/client2/HttpClient2Impl.scala | 2 +- .../main/scala/uk/gov/hmrc/play/http/BodyCaptor.scala | 11 +++-------- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/client2/HttpClient2Impl.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/client2/HttpClient2Impl.scala index 6de11bda..5480c569 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/client2/HttpClient2Impl.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/client2/HttpClient2Impl.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 HM Revenue & Customs + * Copyright 2022 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/play/http/BodyCaptor.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/play/http/BodyCaptor.scala index 537494b8..454be97c 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/play/http/BodyCaptor.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/play/http/BodyCaptor.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 HM Revenue & Customs + * Copyright 2022 HM Revenue & Customs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -86,15 +86,10 @@ object BodyCaptor { flow(loggingContext, maxBodyLength, withCapturedBody) .to(Sink.ignore) - // We raise a warning, but don't provide a mechanism to opt-out of auditing payloads. - // Currently we can only turn off auditing for url (`auditDisabledForPattern` configuration) - not per method, - // and we can't turn off auditing of just the payload (and keep the fact the call has been made) - // TODO Check with CIP whether `RequestBuilder` could have `withoutRequestPayloadAuditing` and `withoutRequestPayloadAuditing`? - // Note, this also applies to bootstrap AuditFilter. def bodyUpto(body: String, maxBodyLength: Int, loggingContext: String): String = if (body.length > maxBodyLength) { logger.warn( - s"txm play auditing: $loggingContext body ${body.length} exceeds maxLength $maxBodyLength - do you need to be auditing this payload?" + s"$loggingContext body ${body.length} exceeds maxLength $maxBodyLength - truncating" ) body.take(maxBodyLength) } else @@ -103,7 +98,7 @@ object BodyCaptor { def bodyUpto(body: ByteString, maxBodyLength: Int, loggingContext: String): ByteString = if (body.length > maxBodyLength) { logger.warn( - s"txm play auditing: $loggingContext body ${body.length} exceeds maxLength $maxBodyLength - do you need to be auditing this payload?" + s"$loggingContext body ${body.length} exceeds maxLength $maxBodyLength - truncating" ) body.take(maxBodyLength) } else From 14be21d521d7dd947fbdf8392dad37bd6aade968 Mon Sep 17 00:00:00 2001 From: colin-lamed <9568290+colin-lamed@users.noreply.github.com> Date: Thu, 13 Jan 2022 12:38:35 +0000 Subject: [PATCH 17/23] BDOG-1512 Add version header --- build.sbt | 5 +++ .../scala/uk/gov/hmrc/http/HttpDelete.scala | 2 +- .../main/scala/uk/gov/hmrc/http/HttpGet.scala | 2 +- .../scala/uk/gov/hmrc/http/HttpPatch.scala | 2 +- .../scala/uk/gov/hmrc/http/HttpPost.scala | 8 ++-- .../main/scala/uk/gov/hmrc/http/HttpPut.scala | 4 +- .../hmrc/http/client2/HttpClient2Impl.scala | 7 +++- .../uk/gov/hmrc/http/HttpDeleteSpec.scala | 10 ++++- .../scala/uk/gov/hmrc/http/HttpGetSpec.scala | 10 ++++- .../uk/gov/hmrc/http/HttpPatchSpec.scala | 12 ++++-- .../scala/uk/gov/hmrc/http/HttpPostSpec.scala | 42 ++++++++++++++----- .../scala/uk/gov/hmrc/http/HttpPutSpec.scala | 25 +++++++---- .../hmrc/http/client2/HttpClient2Spec.scala | 4 +- 13 files changed, 94 insertions(+), 39 deletions(-) diff --git a/build.sbt b/build.sbt index 7bacd961..d59dcf5e 100644 --- a/build.sbt +++ b/build.sbt @@ -59,6 +59,7 @@ lazy val sharedSources = shareSources("http-verbs-common") lazy val httpVerbsPlay28 = Project("http-verbs-play-28", file("http-verbs-play-28")) + .enablePlugins(BuildInfoPlugin) .settings( commonSettings, sharedSources, @@ -70,6 +71,10 @@ lazy val httpVerbsPlay28 = Project("http-verbs-play-28", file("http-verbs-play-2 AppDependencies.coreTestPlay28, Test / fork := true // akka is not unloaded properly, which can affect other tests ) + .settings( // https://github.com/sbt/sbt-buildinfo + buildInfoKeys := Seq[BuildInfoKey](version), + buildInfoPackage := "uk.gov.hmrc.http" + ) .dependsOn(httpVerbs) lazy val sharedTestSources = diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpDelete.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpDelete.scala index c0318dbe..955805f0 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpDelete.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpDelete.scala @@ -34,7 +34,7 @@ trait HttpDelete override def DELETE[O](url: String, headers: Seq[(String, String)] = Seq.empty)(implicit rds: HttpReads[O], hc: HeaderCarrier, ec: ExecutionContext): Future[O] = withTracing(DELETE_VERB, url) { - val allHeaders = HeaderCarrier.headersForUrl(hcConfig, url, headers) + val allHeaders = HeaderCarrier.headersForUrl(hcConfig, url, headers) :+ "Http-Client-Version" -> BuildInfo.version val httpResponse = retryOnSslEngineClosed(DELETE_VERB, url)(doDelete(url, allHeaders)) executeHooks(DELETE_VERB, url"$url", allHeaders, None, httpResponse) mapErrors(DELETE_VERB, url, httpResponse).map(rds.read(DELETE_VERB, url, _)) diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpGet.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpGet.scala index 4af86364..2b32713a 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpGet.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpGet.scala @@ -45,7 +45,7 @@ trait HttpGet extends CoreGet with GetHttpTransport with HttpVerb with Connectio val urlWithQuery = url + makeQueryString(queryParams) withTracing(GET_VERB, urlWithQuery) { - val allHeaders = HeaderCarrier.headersForUrl(hcConfig, url, headers) + val allHeaders = HeaderCarrier.headersForUrl(hcConfig, url, headers) :+ "Http-Client-Version" -> BuildInfo.version val httpResponse = retryOnSslEngineClosed(GET_VERB, urlWithQuery)(doGet(urlWithQuery, headers = allHeaders)) executeHooks(GET_VERB, url"$url", allHeaders, None, httpResponse) mapErrors(GET_VERB, urlWithQuery, httpResponse).map(response => rds.read(GET_VERB, urlWithQuery, response)) diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpPatch.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpPatch.scala index c3de8674..34506a6c 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpPatch.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpPatch.scala @@ -42,7 +42,7 @@ trait HttpPatch hc: HeaderCarrier, ec: ExecutionContext): Future[O] = withTracing(PATCH_VERB, url) { - val allHeaders = HeaderCarrier.headersForUrl(hcConfig, url, headers) + val allHeaders = HeaderCarrier.headersForUrl(hcConfig, url, headers) :+ "Http-Client-Version" -> BuildInfo.version val httpResponse = retryOnSslEngineClosed(PATCH_VERB, url)(doPatch(url, body, allHeaders)) executeHooks(PATCH_VERB, url"$url", allHeaders, Option(HookData.FromString(Json.stringify(wts.writes(body)))), httpResponse) mapErrors(PATCH_VERB, url, httpResponse).map(response => rds.read(PATCH_VERB, url, response)) diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpPost.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpPost.scala index b303316f..a734fb3e 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpPost.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpPost.scala @@ -42,7 +42,7 @@ trait HttpPost hc: HeaderCarrier, ec: ExecutionContext): Future[O] = withTracing(POST_VERB, url) { - val allHeaders = HeaderCarrier.headersForUrl(hcConfig, url, headers) + val allHeaders = HeaderCarrier.headersForUrl(hcConfig, url, headers) :+ "Http-Client-Version" -> BuildInfo.version val httpResponse = retryOnSslEngineClosed(POST_VERB, url)(doPost(url, body, allHeaders)) executeHooks(POST_VERB, url"$url", allHeaders, Option(HookData.FromString(Json.stringify(wts.writes(body)))), httpResponse) mapErrors(POST_VERB, url, httpResponse).map(rds.read(POST_VERB, url, _)) @@ -56,7 +56,7 @@ trait HttpPost hc: HeaderCarrier, ec: ExecutionContext): Future[O] = withTracing(POST_VERB, url) { - val allHeaders = HeaderCarrier.headersForUrl(hcConfig, url, headers) + val allHeaders = HeaderCarrier.headersForUrl(hcConfig, url, headers) :+ "Http-Client-Version" -> BuildInfo.version val httpResponse = retryOnSslEngineClosed(POST_VERB, url)(doPostString(url, body, allHeaders)) executeHooks(POST_VERB, url"$url", allHeaders, Option(HookData.FromString(body)), httpResponse) mapErrors(POST_VERB, url, httpResponse).map(rds.read(POST_VERB, url, _)) @@ -70,7 +70,7 @@ trait HttpPost hc: HeaderCarrier, ec: ExecutionContext): Future[O] = withTracing(POST_VERB, url) { - val allHeaders = HeaderCarrier.headersForUrl(hcConfig, url, headers) + val allHeaders = HeaderCarrier.headersForUrl(hcConfig, url, headers) :+ "Http-Client-Version" -> BuildInfo.version val httpResponse = retryOnSslEngineClosed(POST_VERB, url)(doFormPost(url, body, allHeaders)) executeHooks(POST_VERB, url"$url", allHeaders, Option(HookData.FromMap(body)), httpResponse) mapErrors(POST_VERB, url, httpResponse).map(rds.read(POST_VERB, url, _)) @@ -83,7 +83,7 @@ trait HttpPost hc: HeaderCarrier, ec: ExecutionContext): Future[O] = withTracing(POST_VERB, url) { - val allHeaders = HeaderCarrier.headersForUrl(hcConfig, url, headers) + val allHeaders = HeaderCarrier.headersForUrl(hcConfig, url, headers) :+ "Http-Client-Version" -> BuildInfo.version val httpResponse = retryOnSslEngineClosed(POST_VERB, url)(doEmptyPost(url, allHeaders)) executeHooks(POST_VERB, url"$url", allHeaders, None, httpResponse) mapErrors(POST_VERB, url, httpResponse).map(rds.read(POST_VERB, url, _)) diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpPut.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpPut.scala index 7dc64009..6711cf49 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpPut.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpPut.scala @@ -36,7 +36,7 @@ trait HttpPut extends CorePut with PutHttpTransport with HttpVerb with Connectio hc: HeaderCarrier, ec: ExecutionContext): Future[O] = withTracing(PUT_VERB, url) { - val allHeaders = HeaderCarrier.headersForUrl(hcConfig, url, headers) + val allHeaders = HeaderCarrier.headersForUrl(hcConfig, url, headers) :+ "Http-Client-Version" -> BuildInfo.version val httpResponse = retryOnSslEngineClosed(PUT_VERB, url)(doPut(url, body, allHeaders)) executeHooks(PUT_VERB, url"$url", allHeaders, Option(HookData.FromString(Json.stringify(wts.writes(body)))), httpResponse) mapErrors(PUT_VERB, url, httpResponse).map(response => rds.read(PUT_VERB, url, response)) @@ -50,7 +50,7 @@ trait HttpPut extends CorePut with PutHttpTransport with HttpVerb with Connectio hc: HeaderCarrier, ec: ExecutionContext): Future[O] = withTracing(PUT_VERB, url) { - val allHeaders = HeaderCarrier.headersForUrl(hcConfig, url, headers) + val allHeaders = HeaderCarrier.headersForUrl(hcConfig, url, headers) :+ "Http-Client-Version" -> BuildInfo.version val httpResponse = retryOnSslEngineClosed(PUT_VERB, url)(doPutString(url, body, allHeaders)) executeHooks(PUT_VERB, url"$url", allHeaders, Option(HookData.FromString(body)), httpResponse) mapErrors(PUT_VERB, url, httpResponse).map(rds.read(PUT_VERB, url, _)) diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/client2/HttpClient2Impl.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/client2/HttpClient2Impl.scala index 5480c569..10861ff2 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/client2/HttpClient2Impl.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/client2/HttpClient2Impl.scala @@ -23,7 +23,7 @@ import com.typesafe.config.Config import play.api.Configuration import play.api.libs.ws.{BodyWritable, EmptyBody, InMemoryBody, SourceBody, WSClient, WSProxyServer, WSRequest, WSResponse} import play.core.parsers.FormUrlEncodedParser -import uk.gov.hmrc.http.{BadGatewayException, GatewayTimeoutException, HeaderCarrier, HttpReads, HttpResponse, Retries} +import uk.gov.hmrc.http.{BadGatewayException, BuildInfo, GatewayTimeoutException, HeaderCarrier, HttpReads, HttpResponse, Retries} import uk.gov.hmrc.play.http.BodyCaptor import uk.gov.hmrc.play.http.ws.WSProxyConfiguration import uk.gov.hmrc.http.hooks.{HookData, HttpHook} @@ -64,6 +64,9 @@ class HttpClient2Impl( protected val executor = new ExecutorImpl(actorSystem, config, hooks) + private val clientVersionHeader = + "Http-Client2-Version" -> BuildInfo.version + override protected def mkRequestBuilder( url : URL, method: String @@ -78,7 +81,7 @@ class HttpClient2Impl( wsClient .url(url.toString) .withMethod(method) - .withHttpHeaders(hc.headersForUrl(hcConfig)(url.toString) : _*), + .withHttpHeaders(hc.headersForUrl(hcConfig)(url.toString) :+ clientVersionHeader : _*), None ) } diff --git a/http-verbs-common/src/test/scala/uk/gov/hmrc/http/HttpDeleteSpec.scala b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/HttpDeleteSpec.scala index 3ac89a2f..32933aa9 100644 --- a/http-verbs-common/src/test/scala/uk/gov/hmrc/http/HttpDeleteSpec.scala +++ b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/HttpDeleteSpec.scala @@ -146,16 +146,22 @@ class HttpDeleteSpec val respArgCaptor1 = ArgCaptor[Future[HttpResponse]] val respArgCaptor2 = ArgCaptor[Future[HttpResponse]] + val headerCaptor1 = ArgCaptor[Seq[(String, String)]] + val headerCaptor2 = ArgCaptor[Seq[(String, String)]] + val config = HeaderCarrier.Config.fromConfig(testDelete.configuration) val headers = HeaderCarrier.headersForUrl(config, url, Seq("header" -> "foo")) - verify(testDelete.testHook1).apply(eqTo("DELETE"), eqTo(url"$url"), eqTo(headers), eqTo(None), respArgCaptor1)(any, any) - verify(testDelete.testHook2).apply(eqTo("DELETE"), eqTo(url"$url"), eqTo(headers), eqTo(None), respArgCaptor2)(any, any) + verify(testDelete.testHook1).apply(eqTo("DELETE"), eqTo(url"$url"), headerCaptor1, eqTo(None), respArgCaptor1)(any, any) + verify(testDelete.testHook2).apply(eqTo("DELETE"), eqTo(url"$url"), headerCaptor2, eqTo(None), respArgCaptor2)(any, any) // verifying directly without ArgumentCaptor didn't work as Futures were different instances // e.g. Future.successful(5) != Future.successful(5) respArgCaptor1.value.futureValue shouldBe dummyResponse respArgCaptor2.value.futureValue shouldBe dummyResponse + + headerCaptor1.value should contain allElementsOf(headers) + headerCaptor2.value should contain allElementsOf(headers) } } } diff --git a/http-verbs-common/src/test/scala/uk/gov/hmrc/http/HttpGetSpec.scala b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/HttpGetSpec.scala index 2a12f1cb..09840162 100644 --- a/http-verbs-common/src/test/scala/uk/gov/hmrc/http/HttpGetSpec.scala +++ b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/HttpGetSpec.scala @@ -142,16 +142,22 @@ class HttpGetSpec val respArgCaptor1 = ArgCaptor[Future[HttpResponse]] val respArgCaptor2 = ArgCaptor[Future[HttpResponse]] + val headerCaptor1 = ArgCaptor[Seq[(String, String)]] + val headerCaptor2 = ArgCaptor[Seq[(String, String)]] + val config = HeaderCarrier.Config.fromConfig(testGet.configuration) val headers = HeaderCarrier.headersForUrl(config, url) - verify(testGet.testHook1).apply(eqTo("GET"), eqTo(url"$url"), eqTo(headers), eqTo(None), respArgCaptor1)(any, any) - verify(testGet.testHook2).apply(eqTo("GET"), eqTo(url"$url"), eqTo(headers), eqTo(None), respArgCaptor2)(any, any) + verify(testGet.testHook1).apply(eqTo("GET"), eqTo(url"$url"), headerCaptor1, eqTo(None), respArgCaptor1)(any, any) + verify(testGet.testHook2).apply(eqTo("GET"), eqTo(url"$url"), headerCaptor2, eqTo(None), respArgCaptor2)(any, any) // verifying directly without ArgumentCaptor didn't work as Futures were different instances // e.g. Future.successful(5) != Future.successful(5) respArgCaptor1.value.futureValue shouldBe dummyResponse respArgCaptor2.value.futureValue shouldBe dummyResponse + + headerCaptor1.value should contain allElementsOf(headers) + headerCaptor2.value should contain allElementsOf(headers) } } diff --git a/http-verbs-common/src/test/scala/uk/gov/hmrc/http/HttpPatchSpec.scala b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/HttpPatchSpec.scala index b4f843fd..14af4044 100644 --- a/http-verbs-common/src/test/scala/uk/gov/hmrc/http/HttpPatchSpec.scala +++ b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/HttpPatchSpec.scala @@ -152,18 +152,22 @@ class HttpPatchSpec val respArgCaptor1 = ArgCaptor[Future[HttpResponse]] val respArgCaptor2 = ArgCaptor[Future[HttpResponse]] + val headerCaptor1 = ArgCaptor[Seq[(String, String)]] + val headerCaptor2 = ArgCaptor[Seq[(String, String)]] + val config = HeaderCarrier.Config.fromConfig(testPatch.configuration) val headers = HeaderCarrier.headersForUrl(config, url, Seq("header" -> "foo")) - verify(testPatch.testHook1) - .apply(eqTo("PATCH"), eqTo(url"$url"), eqTo(headers), eqTo(Some(HookData.FromString(testJson))), respArgCaptor1)(any, any) - verify(testPatch.testHook2) - .apply(eqTo("PATCH"), eqTo(url"$url"), eqTo(headers), eqTo(Some(HookData.FromString(testJson))), respArgCaptor2)(any, any) + verify(testPatch.testHook1).apply(eqTo("PATCH"), eqTo(url"$url"), headerCaptor1, eqTo(Some(HookData.FromString(testJson))), respArgCaptor1)(any, any) + verify(testPatch.testHook2).apply(eqTo("PATCH"), eqTo(url"$url"), headerCaptor2, eqTo(Some(HookData.FromString(testJson))), respArgCaptor2)(any, any) // verifying directly without ArgumentCaptor didn't work as Futures were different instances // e.g. Future.successful(5) != Future.successful(5) respArgCaptor1.value.futureValue shouldBe dummyResponse respArgCaptor2.value.futureValue shouldBe dummyResponse + + headerCaptor1.value should contain allElementsOf(headers) + headerCaptor2.value should contain allElementsOf(headers) } } } diff --git a/http-verbs-common/src/test/scala/uk/gov/hmrc/http/HttpPostSpec.scala b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/HttpPostSpec.scala index 93499aba..d7a0ae30 100644 --- a/http-verbs-common/src/test/scala/uk/gov/hmrc/http/HttpPostSpec.scala +++ b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/HttpPostSpec.scala @@ -194,16 +194,22 @@ class HttpPostSpec val respArgCaptor1 = ArgCaptor[Future[HttpResponse]] val respArgCaptor2 = ArgCaptor[Future[HttpResponse]] + val headerCaptor1 = ArgCaptor[Seq[(String, String)]] + val headerCaptor2 = ArgCaptor[Seq[(String, String)]] + val config = HeaderCarrier.Config.fromConfig(testPost.configuration) val headers = HeaderCarrier.headersForUrl(config, url) - verify(testPost.testHook1).apply(eqTo("POST"), eqTo(url"$url"), eqTo(headers), eqTo(Some(HookData.FromString(testJson))), respArgCaptor1)(any, any) - verify(testPost.testHook2).apply(eqTo("POST"), eqTo(url"$url"), eqTo(headers), eqTo(Some(HookData.FromString(testJson))), respArgCaptor2)(any, any) + verify(testPost.testHook1).apply(eqTo("POST"), eqTo(url"$url"), headerCaptor1, eqTo(Some(HookData.FromString(testJson))), respArgCaptor1)(any, any) + verify(testPost.testHook2).apply(eqTo("POST"), eqTo(url"$url"), headerCaptor2, eqTo(Some(HookData.FromString(testJson))), respArgCaptor2)(any, any) // verifying directly without ArgumentCaptor didn't work as Futures were different instances // e.g. Future.successful(5) != Future.successful(5) respArgCaptor1.value.futureValue shouldBe dummyResponse respArgCaptor2.value.futureValue shouldBe dummyResponse + + headerCaptor1.value should contain allElementsOf(headers) + headerCaptor2.value should contain allElementsOf(headers) } } @@ -231,16 +237,22 @@ class HttpPostSpec val respArgCaptor1 = ArgCaptor[Future[HttpResponse]] val respArgCaptor2 = ArgCaptor[Future[HttpResponse]] + val headerCaptor1 = ArgCaptor[Seq[(String, String)]] + val headerCaptor2 = ArgCaptor[Seq[(String, String)]] + val config = HeaderCarrier.Config.fromConfig(testPost.configuration) val headers = HeaderCarrier.headersForUrl(config, url) - verify(testPost.testHook1).apply(eqTo("POST"), eqTo(url"$url"), eqTo(headers), eqTo(Some(HookData.FromMap(Map()))), respArgCaptor1)(any, any) - verify(testPost.testHook2).apply(eqTo("POST"), eqTo(url"$url"), eqTo(headers), eqTo(Some(HookData.FromMap(Map()))), respArgCaptor2)(any, any) + verify(testPost.testHook1).apply(eqTo("POST"), eqTo(url"$url"), headerCaptor1, eqTo(Some(HookData.FromMap(Map()))), respArgCaptor1)(any, any) + verify(testPost.testHook2).apply(eqTo("POST"), eqTo(url"$url"), headerCaptor2, eqTo(Some(HookData.FromMap(Map()))), respArgCaptor2)(any, any) // verifying directly without ArgumentCaptor didn't work as Futures were different instances // e.g. Future.successful(5) != Future.successful(5) respArgCaptor1.value.futureValue shouldBe dummyResponse respArgCaptor2.value.futureValue shouldBe dummyResponse + + headerCaptor1.value should contain allElementsOf(headers) + headerCaptor2.value should contain allElementsOf(headers) } } @@ -272,18 +284,22 @@ class HttpPostSpec val respArgCaptor1 = ArgCaptor[Future[HttpResponse]] val respArgCaptor2 = ArgCaptor[Future[HttpResponse]] + val headerCaptor1 = ArgCaptor[Seq[(String, String)]] + val headerCaptor2 = ArgCaptor[Seq[(String, String)]] + val config = HeaderCarrier.Config.fromConfig(testPost.configuration) val headers = HeaderCarrier.headersForUrl(config, url) - verify(testPost.testHook1) - .apply(eqTo("POST"), eqTo(url"$url"), eqTo(headers), eqTo(Some(HookData.FromString(testRequestBody))), respArgCaptor1)(any, any) - verify(testPost.testHook2) - .apply(eqTo("POST"), eqTo(url"$url"), eqTo(headers), eqTo(Some(HookData.FromString(testRequestBody))), respArgCaptor2)(any, any) + verify(testPost.testHook1).apply(eqTo("POST"), eqTo(url"$url"), headerCaptor1, eqTo(Some(HookData.FromString(testRequestBody))), respArgCaptor1)(any, any) + verify(testPost.testHook2).apply(eqTo("POST"), eqTo(url"$url"), headerCaptor2, eqTo(Some(HookData.FromString(testRequestBody))), respArgCaptor2)(any, any) // verifying directly without ArgumentCaptor didn't work as Futures were different instances // e.g. Future.successful(5) != Future.successful(5) respArgCaptor1.value.futureValue shouldBe dummyResponse respArgCaptor2.value.futureValue shouldBe dummyResponse + + headerCaptor1.value should contain allElementsOf(headers) + headerCaptor2.value should contain allElementsOf(headers) } } @@ -311,16 +327,22 @@ class HttpPostSpec val respArgCaptor1 = ArgCaptor[Future[HttpResponse]] val respArgCaptor2 = ArgCaptor[Future[HttpResponse]] + val headerCaptor1 = ArgCaptor[Seq[(String, String)]] + val headerCaptor2 = ArgCaptor[Seq[(String, String)]] + val config = HeaderCarrier.Config.fromConfig(testPost.configuration) val headers = HeaderCarrier.headersForUrl(config, url) - verify(testPost.testHook1).apply(eqTo("POST"), eqTo(url"$url"), eqTo(headers), eqTo(None), respArgCaptor1)(any, any) - verify(testPost.testHook2).apply(eqTo("POST"), eqTo(url"$url"), eqTo(headers), eqTo(None), respArgCaptor2)(any, any) + verify(testPost.testHook1).apply(eqTo("POST"), eqTo(url"$url"), headerCaptor1, eqTo(None), respArgCaptor1)(any, any) + verify(testPost.testHook2).apply(eqTo("POST"), eqTo(url"$url"), headerCaptor2, eqTo(None), respArgCaptor2)(any, any) // verifying directly without ArgumentCaptor didn't work as Futures were different instances // e.g. Future.successful(5) != Future.successful(5) respArgCaptor1.value.futureValue shouldBe dummyResponse respArgCaptor2.value.futureValue shouldBe dummyResponse + + headerCaptor1.value should contain allElementsOf(headers) + headerCaptor2.value should contain allElementsOf(headers) } } diff --git a/http-verbs-common/src/test/scala/uk/gov/hmrc/http/HttpPutSpec.scala b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/HttpPutSpec.scala index ff458422..e76a901e 100644 --- a/http-verbs-common/src/test/scala/uk/gov/hmrc/http/HttpPutSpec.scala +++ b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/HttpPutSpec.scala @@ -170,16 +170,22 @@ class HttpPutSpec val respArgCaptor1 = ArgCaptor[Future[HttpResponse]] val respArgCaptor2 = ArgCaptor[Future[HttpResponse]] + val headerCaptor1 = ArgCaptor[Seq[(String, String)]] + val headerCaptor2 = ArgCaptor[Seq[(String, String)]] + val config = HeaderCarrier.Config.fromConfig(testPut.configuration) val headers = HeaderCarrier.headersForUrl(config, url) - verify(testPut.testHook1).apply(eqTo("PUT"), eqTo(url"$url"), eqTo(headers), eqTo(Some(HookData.FromString(testJson))), respArgCaptor1)(any, any) - verify(testPut.testHook2).apply(eqTo("PUT"), eqTo(url"$url"), eqTo(headers), eqTo(Some(HookData.FromString(testJson))), respArgCaptor2)(any, any) + verify(testPut.testHook1).apply(eqTo("PUT"), eqTo(url"$url"), headerCaptor1, eqTo(Some(HookData.FromString(testJson))), respArgCaptor1)(any, any) + verify(testPut.testHook2).apply(eqTo("PUT"), eqTo(url"$url"), headerCaptor2, eqTo(Some(HookData.FromString(testJson))), respArgCaptor2)(any, any) // verifying directly without ArgumentCaptor didn't work as Futures were different instances // e.g. Future.successful(5) != Future.successful(5) respArgCaptor1.value.futureValue shouldBe dummyResponse respArgCaptor2.value.futureValue shouldBe dummyResponse + + headerCaptor1.value should contain allElementsOf(headers) + headerCaptor2.value should contain allElementsOf(headers) } } @@ -198,7 +204,8 @@ class HttpPutSpec behave like anErrorMappingHttpCall( "PUT", - (url, responseF) => new StubbedHttpPut(responseF).PUTString[HttpResponse](url, testRequestBody, Seq.empty)) + (url, responseF) => new StubbedHttpPut(responseF).PUTString[HttpResponse](url, testRequestBody, Seq.empty) + ) behave like aTracingHttpCall("PUT", "PUT", new StubbedHttpPut(defaultHttpResponse)) { _.PUTString[HttpResponse](url, testRequestBody, Seq.empty) } @@ -213,18 +220,22 @@ class HttpPutSpec val respArgCaptor1 = ArgCaptor[Future[HttpResponse]] val respArgCaptor2 = ArgCaptor[Future[HttpResponse]] + val headerCaptor1 = ArgCaptor[Seq[(String, String)]] + val headerCaptor2 = ArgCaptor[Seq[(String, String)]] + val config = HeaderCarrier.Config.fromConfig(testPut.configuration) val headers = HeaderCarrier.headersForUrl(config, url) - verify(testPut.testHook1) - .apply(eqTo("PUT"), eqTo(url"$url"), eqTo(headers), eqTo(Some(HookData.FromString(testRequestBody))), respArgCaptor1)(any, any) - verify(testPut.testHook2) - .apply(eqTo("PUT"), eqTo(url"$url"), eqTo(headers), eqTo(Some(HookData.FromString(testRequestBody))), respArgCaptor2)(any, any) + verify(testPut.testHook1).apply(eqTo("PUT"), eqTo(url"$url"), headerCaptor1, eqTo(Some(HookData.FromString(testRequestBody))), respArgCaptor1)(any, any) + verify(testPut.testHook2).apply(eqTo("PUT"), eqTo(url"$url"), headerCaptor2, eqTo(Some(HookData.FromString(testRequestBody))), respArgCaptor2)(any, any) // verifying directly without ArgumentCaptor didn't work as Futures were different instances // e.g. Future.successful(5) != Future.successful(5) respArgCaptor1.value.futureValue shouldBe dummyResponse respArgCaptor2.value.futureValue shouldBe dummyResponse + + headerCaptor1.value should contain allElementsOf(headers) + headerCaptor2.value should contain allElementsOf(headers) } } } diff --git a/http-verbs-common/src/test/scala/uk/gov/hmrc/http/client2/HttpClient2Spec.scala b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/client2/HttpClient2Spec.scala index e376ae5e..b719ce4f 100644 --- a/http-verbs-common/src/test/scala/uk/gov/hmrc/http/client2/HttpClient2Spec.scala +++ b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/client2/HttpClient2Spec.scala @@ -17,7 +17,6 @@ package uk.gov.hmrc.http.client2 import akka.actor.ActorSystem -import akka.stream.{ActorMaterializer, Materializer} import akka.stream.scaladsl.Source import akka.util.ByteString import com.github.tomakehurst.wiremock.client.WireMock @@ -30,7 +29,6 @@ import org.scalatest.concurrent.{IntegrationPatience, ScalaFutures} import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpecLike import play.api.Configuration -import play.api.libs.functional.syntax._ import play.api.libs.json.{Json, Reads, Writes} import play.api.libs.ws.ahc.{AhcWSClient, AhcWSClientConfigFactory} import uk.gov.hmrc.http.{HeaderCarrier, HttpReads, HttpReadsInstances, HttpResponse, Retries, StringContextOps, UpstreamErrorResponse} @@ -75,6 +73,7 @@ class HttpClient2Spec .withHeader("Content-Type", equalTo("application/json")) .withRequestBody(equalTo("\"req\"")) .withHeader("User-Agent", equalTo("myapp")) + .withHeader("Http-Client2-Version", matching(".*")) ) val headersCaptor = ArgCaptor[Seq[(String, String)]] @@ -541,7 +540,6 @@ class HttpClient2Spec trait Setup { implicit val as: ActorSystem = ActorSystem("test-actor-system") - implicit val mat: Materializer = ActorMaterializer() // explicitly required for play-26 val mockHttpHook = mock[HttpHook](withSettings.lenient) From 5276c3287055f05bb51e0ebe53cfca42bf7b1f1a Mon Sep 17 00:00:00 2001 From: colin-lamed <9568290+colin-lamed@users.noreply.github.com> Date: Thu, 13 Jan 2022 14:17:15 +0000 Subject: [PATCH 18/23] BDOG-1512 Update README --- CHANGELOG.md | 68 ++--------------------------------------- README.md | 86 ++++++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 85 insertions(+), 69 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ec8db52..f1fdbaa4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## Version 14.0.0 +## Version 13.13.0 ### Auditing max body length @@ -16,71 +16,7 @@ There are some differences with `WSProxyConfiguration.buildWsProxyServer`: ### Adds HttpClient2 This is in addition to `HttpClient` (for now), so can be optionally used instead. -The changes from HttpClient are: -- Supports streaming -- Exposes the underlying `play.api.libs.ws.WSRequest` with `transform`, making it easier to customise the request. - -Examples can be found in [HttpClient2Spec](/http-verbs-common/src/test/scala/uk/gov/hmrc/http/play/HttpClient2Spec.scala) - -To migrate: - -```scala -httpClient.GET[ResponseType](url) -``` - -becomes - -```scala -httpClient2.get(url"$url").execute[ResponseType] -``` - -and - -```scala -httpClient.POST[ResponseType](url, payload, headers) -``` - -becomes - -```scala -httpClient2.post(url"$url").withBody(Json.toJson(payload)).addHeaders(headers).execute[ResponseType] -``` - - -#### Header manipulation - -With `HttpClient`, replacing a header can require providing a customised client implementation (e.g. to replace the user-agent header), or updating the `HeaderCarrier` (e.g. to replace the authorisation header). This can now all be done with the `replaceHeader` on `HttpClient2` per call. e.g. - -```scala -httpClient2.get(url"$url").replaceHeader("User-Agent" -> userAgent).replaceHeader("Authorization" -> authorization).execute[ResponseType] -``` - -#### Using proxy - -With `HttpClient`, to use a proxy requires creating a new instance of HttpClient to mix in `WSProxy` and configure. With `HttpClient2` this can be done with the same client, calling `withProxy` per call. e.g. - -```scala -httpClient2.get(url"$url").withProxy.execute[ResponseType] -``` - -* It uses `WSProxyConfiguration.buildWsProxyServer` which needs enabling with `proxy.enabled` in configuration, which by default is false, for development. See `WSProxyConfiguration` for configuration changes above. - -#### Streaming - -Streaming is supported with `HttpClient2`, and will be audited in the same way as `HttpClient`. Note that payloads will be truncated in audit logs if they exceed the max supported (as configured by `http-verbs.auditing.maxBodyLength`). - -Streamed requests can simply be passed to `withBody`: - -```scala -val reqStream: Source[ByteString, _] = ??? -httpClient2.post(url"$url").withBody(reqStream).execute[ResponseType] -``` - -For streamed responses, use `stream` rather than `execute`: - -```scala -httpClient2.get(url"$url").stream[Source[ByteString, _]] -``` +See [README](/README.md) for details. ## Version 13.12.0 diff --git a/README.md b/README.md index a28024c4..12cfad02 100644 --- a/README.md +++ b/README.md @@ -33,12 +33,89 @@ Where `play-xx` is your version of Play (e.g. `play-28`). ## Usage +There are two HttpClients available. + +### uk.gov.hmrc.http.HttpClient + Examples can be found [here](https://github.com/hmrc/http-verbs/blob/master/http-verbs-test-common/src/test/scala/uk/gov/hmrc/http/examples/Examples.scala) -### URLs +URLs can be supplied as either `java.net.URL` or `String`. We recommend supplying `java.net.URL` and using the provided [URL interpolator](#url-interpolator) for correct escaping of query and path parameters. + + +### uk.gov.hmrc.http.client2.HttpClient2 + +This client follows the same patterns as `HttpClient` - that is, it also requires a `HeaderCarrier` to represent the context of the caller, and an `HttpReads` to process the http response. + +In addition, it: +- Supports streaming +- Exposes the underlying `play.api.libs.ws.WSRequest` with `transform`, making it easier to customise the request. +- Only accepts the URL as `java.net.URL`; you can make use of the provided [URL interpolator](#url-interpolator). + +Examples can be found in [here](/http-verbs-common/src/test/scala/uk/gov/hmrc/http/client2/HttpClient2Spec.scala) + +To migrate: + +```scala +httpClient.GET[ResponseType](url) +``` + +becomes + +```scala +httpClient2.get(url"$url").execute[ResponseType] +``` + +and + +```scala +httpClient.POST[ResponseType](url, payload, headers) +``` + +becomes + +```scala +httpClient2.post(url"$url").withBody(Json.toJson(payload)).addHeaders(headers).execute[ResponseType] +``` + -URLs can be supplied as either `java.net.URL` or `String`. We recommend supplying `java.net.URL` for correct escaping of query and path parameters. A [URL interpolator](https://sttp.softwaremill.com/en/latest/model/uri.html) has been provided for convenience. +#### Header manipulation +With `HttpClient`, replacing a header can require providing a customised client implementation (e.g. to replace the user-agent header), or updating the `HeaderCarrier` (e.g. to replace the authorisation header). This can now all be done with the `replaceHeader` on `HttpClient2` per call. e.g. + +```scala +httpClient2.get(url"$url").replaceHeader("User-Agent" -> userAgent).replaceHeader("Authorization" -> authorization).execute[ResponseType] +``` + +#### Using proxy + +With `HttpClient`, to use a proxy requires creating a new instance of HttpClient to mix in `WSProxy` and configure. With `HttpClient2` this can be done with the same client, calling `withProxy` per call. e.g. + +```scala +httpClient2.get(url"$url").withProxy.execute[ResponseType] +``` + +* It uses `WSProxyConfiguration.buildWsProxyServer` which needs enabling with `proxy.enabled` in configuration, which by default is `false`, for development. See [WSProxyConfiguration](CHANGELOG.md#wsproxyconfiguration) for configuration changes. + +#### Streaming + +Streaming is supported with `HttpClient2`, and will be audited in the same way as `HttpClient`. Note that payloads will be truncated in audit logs if they exceed the max supported (as configured by `http-verbs.auditing.maxBodyLength`). + +Streamed requests can simply be passed to `withBody`: + +```scala +val reqStream: Source[ByteString, _] = ??? +httpClient2.post(url"$url").withBody(reqStream).execute[ResponseType] +``` + +For streamed responses, use `stream` rather than `execute`: + +```scala +httpClient2.get(url"$url").stream[Source[ByteString, _]] +``` + +### URL interpolator + +A [URL interpolator](https://sttp.softwaremill.com/en/latest/model/uri.html) has been provided to help with escaping query and parameters correctly. ```scala import uk.gov.hmrc.http.StringContextOps @@ -46,6 +123,7 @@ import uk.gov.hmrc.http.StringContextOps url"http://localhost:8080/users/${user.id}?email=${user.email}" ``` + ### Headers #### Creating HeaderCarrier @@ -120,7 +198,7 @@ class MyConnectorSpec extends WireMockSupport with GuiceOneAppPerSuite { } ``` -The `HttpClientSupport` trait can provide an instance of HttpClient as an alternative to instanciating the application: +The `HttpClientSupport` trait can provide an instance of `HttpClient` as an alternative to instanciating the application: ```scala class MyConnectorSpec extends WireMockSupport with HttpClientSupport { private val connector = new MyConnector( @@ -130,6 +208,8 @@ class MyConnectorSpec extends WireMockSupport with HttpClientSupport { } ``` +Similarly `HttpClient2Support` can be used to provide an instance of `HttpClient2`. + The `ExternalWireMockSupport` trait is an alternative to `WireMockSupport` which uses `127.0.0.1` instead of `localhost` for the hostname which is treated as an external host for header forwarding rules. This should be used for tests of connectors which call endpoints external to the platform. The variable `externalWireMockHost` (or `externalWireMockUrl`) should be used to provide the hostname in configuration. Both `WireMockSupport` and `ExternalWireMockSupport` can be used together for integration tests if required. From ecd24bbbf6a6c6db03038383fb83cb47a852145f Mon Sep 17 00:00:00 2001 From: colin-lamed <9568290+colin-lamed@users.noreply.github.com> Date: Tue, 18 Jan 2022 12:20:11 +0000 Subject: [PATCH 19/23] Bdog 1512 fix stream error parsing (#140) * BDOG-1512 Fix error parsing for streams * BDOG-1512 Rename proxy.enabled to http-verbs.proxy.enabled --- CHANGELOG.md | 2 +- README.md | 2 +- .../src/main/resources/reference.conf | 4 +- .../uk/gov/hmrc/http/HttpErrorFunctions.scala | 45 ++++++++++++++++ .../hmrc/http/client2/HttpClient2Impl.scala | 4 +- .../uk/gov/hmrc/http/client2/package.scala | 15 ++++-- .../uk/gov/hmrc/play/http/BodyCaptor.scala | 10 ++-- .../uk/gov/hmrc/play/http/ws/WSRequest.scala | 2 +- .../hmrc/http/client2/HttpClient2Spec.scala | 52 +++++++++++++++++++ 9 files changed, 122 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f1fdbaa4..204dea2a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ Payloads will be truncated in audit logs if they exceed the max supported (as co There are some differences with `WSProxyConfiguration.buildWsProxyServer`: * configPrefix is fixed to `proxy`. - * `proxy.proxyRequiredForThisEnvironment` has been replaced with `proxy.enabled`, but note, it defaults to false (rather than true). This is appropriate for development and tests, but will need explicitly enabling when deployed. + * `proxy.proxyRequiredForThisEnvironment` has been replaced with `http-verbs.proxy.enabled`, but note, it defaults to false (rather than true). This is appropriate for development and tests, but will need explicitly enabling when deployed. ### Adds HttpClient2 diff --git a/README.md b/README.md index 12cfad02..433ce013 100644 --- a/README.md +++ b/README.md @@ -94,7 +94,7 @@ With `HttpClient`, to use a proxy requires creating a new instance of HttpClient httpClient2.get(url"$url").withProxy.execute[ResponseType] ``` -* It uses `WSProxyConfiguration.buildWsProxyServer` which needs enabling with `proxy.enabled` in configuration, which by default is `false`, for development. See [WSProxyConfiguration](CHANGELOG.md#wsproxyconfiguration) for configuration changes. +* It uses `WSProxyConfiguration.buildWsProxyServer` which needs enabling with `http-verbs.proxy.enabled` in configuration, which by default is `false`, for development. See [WSProxyConfiguration](CHANGELOG.md#wsproxyconfiguration) for configuration changes. #### Streaming diff --git a/http-verbs-common/src/main/resources/reference.conf b/http-verbs-common/src/main/resources/reference.conf index 26b2d8cd..09e41aa4 100644 --- a/http-verbs-common/src/main/resources/reference.conf +++ b/http-verbs-common/src/main/resources/reference.conf @@ -19,6 +19,8 @@ http-verbs { intervals = [ "500.millis", "1.second", "2.seconds", "4.seconds", "8.seconds" ] ssl-engine-closed-already.enabled = false } + auditing.maxBodyLength = 32665 + + proxy.enabled = false } -proxy.enabled = false diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpErrorFunctions.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpErrorFunctions.scala index bc412463..38219502 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpErrorFunctions.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpErrorFunctions.scala @@ -16,6 +16,11 @@ package uk.gov.hmrc.http +import akka.stream.Materializer + +import scala.concurrent.{Await, TimeoutException} +import scala.concurrent.duration.{Duration, DurationInt} + trait HttpErrorFunctions { def notFoundMessage(verbName: String, url: String, responseBody: String): String = s"$verbName of '$url' returned 404 (Not Found). Response body: '$responseBody'" @@ -77,6 +82,46 @@ trait HttpErrorFunctions { // default followRedirect should mean we don't see 3xx... case status => Right(response) } + + /* Same as `handleResponseEither` but should be used when reading the `HttpResponse` as a stream. + * The error is returned as `Source[UpstreamErrorResponse, _]`. + */ + def handleResponseEitherStream( + httpMethod: String, + url : String + )( + response: HttpResponse + )(implicit + mat : Materializer, + errorTimeout: ErrorTimeout + ): Either[UpstreamErrorResponse, HttpResponse] = + response.status match { + case status if is4xx(status) || is5xx(status) => + Left { + val errorMessageF = + response.bodyAsSource.runFold("")(_ + _.utf8String) + val errorMessage = + // this await is unfortunate, but HttpReads doesn't support Future + try { + Await.result(errorMessageF, errorTimeout.toDuration) + } catch { + case e: TimeoutException => "" + } + UpstreamErrorResponse( + message = upstreamResponseMessage(httpMethod, url, status, errorMessage), + statusCode = status, + reportAs = if (is4xx(status)) HttpExceptions.INTERNAL_SERVER_ERROR else HttpExceptions.BAD_GATEWAY, + headers = response.headers + ) + } + // Note all cases not handled above (e.g. 1xx, 2xx and 3xx) will be returned as is + // default followRedirect should mean we don't see 3xx... + case status => Right(response) + } } object HttpErrorFunctions extends HttpErrorFunctions + +case class ErrorTimeout( + toDuration: Duration = 10.seconds +) extends AnyVal diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/client2/HttpClient2Impl.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/client2/HttpClient2Impl.scala index 10861ff2..9199b17f 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/client2/HttpClient2Impl.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/client2/HttpClient2Impl.scala @@ -147,7 +147,7 @@ final class RequestBuilderImpl( (body, req2.header("Content-Type")) match { case (IsMap(m), _ ) => hookDataP.success(Some(HookData.FromMap(m))) case (_ , Some("application/x-www-form-urlencoded")) => hookDataP.success(Some(HookData.FromMap(FormUrlEncodedParser.parse(bytes.decodeString("UTF-8"))))) - case _ => val auditedBody = BodyCaptor.bodyUpto(bytes, maxBodyLength, loggingContext).decodeString("UTF-8") + case _ => val auditedBody = BodyCaptor.bodyUpto(bytes, maxBodyLength, loggingContext, isStream = false).decodeString("UTF-8") hookDataP.success(Some(HookData.FromString(auditedBody))) } req2 @@ -269,7 +269,7 @@ class ExecutorImpl( auditResponseF.success( HttpResponse( status = response.status, - body = BodyCaptor.bodyUpto(response.body, maxBodyLength, loggingContext), + body = BodyCaptor.bodyUpto(response.body, maxBodyLength, loggingContext, isStream = false), headers = response.headers.mapValues(_.toSeq).toMap ) ) diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/client2/package.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/client2/package.scala index 06bb225f..e451f0f5 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/client2/package.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/client2/package.scala @@ -16,9 +16,12 @@ package uk.gov.hmrc.http +import akka.stream.Materializer import akka.stream.scaladsl.Source import akka.util.ByteString +import scala.annotation.implicitNotFound + package client2 { trait Streaming } @@ -27,24 +30,30 @@ package object client2 // ensures strict HttpReads are not passed to stream function, which would lead to stream being read into memory // (or runtime exceptions since HttpResponse.body with throw exception for streamed responses) + @implicitNotFound("""Could not find an implicit StreamHttpReads[${A}]. + You may be missing an implicit Materializer.""") type StreamHttpReads[A] = HttpReads[A] with Streaming } trait StreamHttpReadsInstances { + // default may be overridden if required + implicit val errorTimeout: ErrorTimeout = + ErrorTimeout() + def tag[A](instance: A): A with client2.Streaming = instance.asInstanceOf[A with client2.Streaming] - implicit val readEitherSource: HttpReads[Either[UpstreamErrorResponse, Source[ByteString, _]]] with client2.Streaming = + implicit def readEitherSource(implicit mat: Materializer, errorTimeout: ErrorTimeout): client2.StreamHttpReads[Either[UpstreamErrorResponse, Source[ByteString, _]]] = tag[HttpReads[Either[UpstreamErrorResponse, Source[ByteString, _]]]]( HttpReads.ask.flatMap { case (method, url, response) => - HttpErrorFunctions.handleResponseEither(method, url)(response) match { + HttpErrorFunctions.handleResponseEitherStream(method, url)(response) match { case Left(err) => HttpReads.pure(Left(err)) case Right(response) => HttpReads.pure(Right(response.bodyAsSource)) } } ) - implicit val readSource: HttpReads[Source[ByteString, _]] with client2.Streaming = + implicit def readSource(implicit mat: Materializer, errorTimeout: ErrorTimeout): client2.StreamHttpReads[Source[ByteString, _]] = tag[HttpReads[Source[ByteString, _]]]( readEitherSource .map { diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/play/http/BodyCaptor.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/play/http/BodyCaptor.scala index 454be97c..d989fc7d 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/play/http/BodyCaptor.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/play/http/BodyCaptor.scala @@ -54,7 +54,7 @@ private class BodyCaptorFlow( } override def onUpstreamFinish(): Unit = { - withCapturedBody(BodyCaptor.bodyUpto(buffer, maxBodyLength, loggingContext)) + withCapturedBody(BodyCaptor.bodyUpto(buffer, maxBodyLength, loggingContext, isStream = true)) if (isAvailable(out) && buffer == ByteString.empty) push(out, buffer) completeStage() @@ -86,19 +86,19 @@ object BodyCaptor { flow(loggingContext, maxBodyLength, withCapturedBody) .to(Sink.ignore) - def bodyUpto(body: String, maxBodyLength: Int, loggingContext: String): String = + def bodyUpto(body: String, maxBodyLength: Int, loggingContext: String, isStream: Boolean): String = if (body.length > maxBodyLength) { logger.warn( - s"$loggingContext body ${body.length} exceeds maxLength $maxBodyLength - truncating" + s"$loggingContext ${if (isStream) "streamed body" else "body " + body.length} exceeds maxLength $maxBodyLength - truncating" ) body.take(maxBodyLength) } else body - def bodyUpto(body: ByteString, maxBodyLength: Int, loggingContext: String): ByteString = + def bodyUpto(body: ByteString, maxBodyLength: Int, loggingContext: String, isStream: Boolean): ByteString = if (body.length > maxBodyLength) { logger.warn( - s"$loggingContext body ${body.length} exceeds maxLength $maxBodyLength - truncating" + s"$loggingContext ${if (isStream) "streamed body" else "body " + body.length} exceeds maxLength $maxBodyLength - truncating" ) body.take(maxBodyLength) } else diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/play/http/ws/WSRequest.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/play/http/ws/WSRequest.scala index 86276f41..47f966f6 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/play/http/ws/WSRequest.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/play/http/ws/WSRequest.scala @@ -66,7 +66,7 @@ object WSProxyConfiguration { def getOptionalString(key: String): Option[String] = if (configuration.hasPath(key)) Some(configuration.getString(key)) else None - if (configuration.getBoolean("proxy.enabled")) + if (configuration.getBoolean("http-verbs.proxy.enabled")) Some( DefaultWSProxyServer( protocol = Some(configuration.getString("proxy.protocol")), diff --git a/http-verbs-common/src/test/scala/uk/gov/hmrc/http/client2/HttpClient2Spec.scala b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/client2/HttpClient2Spec.scala index b719ce4f..97020500 100644 --- a/http-verbs-common/src/test/scala/uk/gov/hmrc/http/client2/HttpClient2Spec.scala +++ b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/client2/HttpClient2Spec.scala @@ -143,6 +143,58 @@ class HttpClient2Spec auditedResponse.body shouldBe responseBody } + "handled failed requests with streams" in new Setup { + implicit val hc = HeaderCarrier() + + val requestBody = Random.alphanumeric.take(maxAuditBodyLength - 1).mkString + val responseBody = Random.alphanumeric.take(maxAuditBodyLength - 1).mkString + + wireMockServer.stubFor( + WireMock.put(urlEqualTo("/")) + .willReturn(aResponse().withBody(responseBody).withStatus(500)) + ) + + val srcStream: Source[ByteString, _] = + Source.single(ByteString(requestBody)) + + val res: Future[Source[ByteString, _]] = + httpClient2 + .put(url"$wireMockUrl/") + .withBody(srcStream) + .stream[Source[ByteString, _]] + + val failure = res.failed.futureValue + + failure shouldBe a[UpstreamErrorResponse] + failure.asInstanceOf[UpstreamErrorResponse].statusCode shouldBe 500 + failure.asInstanceOf[UpstreamErrorResponse].message shouldBe s"PUT of 'http://localhost:6001/' returned 500. Response body: '$responseBody'" + + wireMockServer.verify( + putRequestedFor(urlEqualTo("/")) + .withHeader("Content-Type", equalTo("application/octet-stream")) + .withRequestBody(equalTo(requestBody)) + .withHeader("User-Agent", equalTo("myapp")) + ) + + val headersCaptor = ArgCaptor[Seq[(String, String)]] + val responseCaptor = ArgCaptor[Future[HttpResponse]] + + verify(mockHttpHook) + .apply( + verb = eqTo("PUT"), + url = eqTo(url"$wireMockUrl/"), + headers = headersCaptor, + body = eqTo(Some(HookData.FromString(requestBody))), + responseF = responseCaptor + )(any[HeaderCarrier], any[ExecutionContext]) + + headersCaptor.value should contain ("User-Agent" -> "myapp") + headersCaptor.value should contain ("Content-Type" -> "application/octet-stream") + val auditedResponse = responseCaptor.value.futureValue + auditedResponse.status shouldBe 500 + auditedResponse.body shouldBe responseBody + } + "truncate stream payloads for auditing if too long" in new Setup { implicit val hc = HeaderCarrier() From 63985b655f8e0de2339e1f3eeb0bfadb75e1836d Mon Sep 17 00:00:00 2001 From: colin-lamed <9568290+colin-lamed@users.noreply.github.com> Date: Tue, 15 Feb 2022 12:47:00 +0000 Subject: [PATCH 20/23] BDOG-1512 Address PR comments --- .../hmrc/http/HttpReadsLegacyInstances.scala | 41 +++++++------- .../main/scala/uk/gov/hmrc/http/Retries.scala | 24 +++----- .../hmrc/http/client2/HttpClient2Impl.scala | 55 +++++++++---------- .../uk/gov/hmrc/play/http/BodyCaptor.scala | 4 +- .../uk/gov/hmrc/play/http/ws/WSRequest.scala | 20 +++---- .../scala/uk/gov/hmrc/http/HeadersSpec.scala | 1 - .../hmrc/http/test/HttpClient2Support.scala | 5 -- .../hmrc/http/test/HttpClientSupport.scala | 5 -- 8 files changed, 61 insertions(+), 94 deletions(-) diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpReadsLegacyInstances.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpReadsLegacyInstances.scala index b078046d..d1b8e5b7 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpReadsLegacyInstances.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpReadsLegacyInstances.scala @@ -16,7 +16,6 @@ package uk.gov.hmrc.http -import com.github.ghik.silencer.silent import play.api.libs.json.{JsNull, JsValue, Reads} @@ -24,41 +23,39 @@ trait HttpReadsLegacyInstances extends HttpReadsLegacyOption with HttpReadsLegac trait HttpReadsLegacyRawReads extends HttpErrorFunctions { @deprecated("Use uk.gov.hmrc.http.HttpReads.Implicits instead. See README for differences.", "11.0.0") - implicit val readRaw: HttpReads[HttpResponse] = new HttpReads[HttpResponse] { - @silent("deprecated") - def read(method: String, url: String, response: HttpResponse) = handleResponse(method, url)(response) - } + implicit val readRaw: HttpReads[HttpResponse] = + (method: String, url: String, response: HttpResponse) => + handleResponse(method, url)(response) } object HttpReadsLegacyRawReads extends HttpReadsLegacyRawReads trait HttpReadsLegacyOption extends HttpErrorFunctions { @deprecated("Use uk.gov.hmrc.http.HttpReads.Implicits instead. See README for differences.", "11.0.0") - implicit def readOptionOf[P](implicit rds: HttpReads[P]): HttpReads[Option[P]] = new HttpReads[Option[P]] { - def read(method: String, url: String, response: HttpResponse) = response.status match { - case 204 | 404 => None - case _ => Some(rds.read(method, url, response)) - } - } + implicit def readOptionOf[P](implicit rds: HttpReads[P]): HttpReads[Option[P]] = + (method: String, url: String, response: HttpResponse) => + response.status match { + case 204 | 404 => None + case _ => Some(rds.read(method, url, response)) + } } trait HttpReadsLegacyJson extends HttpErrorFunctions { @deprecated("Use uk.gov.hmrc.http.HttpReads.Implicits instead. See README for differences.", "11.0.0") - implicit def readFromJson[O](implicit rds: Reads[O], mf: Manifest[O]): HttpReads[O] = new HttpReads[O] { - def read(method: String, url: String, response: HttpResponse) = + implicit def readFromJson[O](implicit rds: Reads[O], mf: Manifest[O]): HttpReads[O] = + (method: String, url: String, response: HttpResponse) => readJson(method, url, handleResponse(method, url)(response).json) - } @deprecated("Use uk.gov.hmrc.http.HttpReads.Implicits instead. See README for differences.", "11.0.0") - def readSeqFromJsonProperty[O](name: String)(implicit rds: Reads[O], mf: Manifest[O]) = new HttpReads[Seq[O]] { - def read(method: String, url: String, response: HttpResponse) = response.status match { - case 204 | 404 => Seq.empty - case _ => - readJson[Seq[O]](method, url, (handleResponse(method, url)(response).json \ name).getOrElse(JsNull)) //Added JsNull here to force validate to fail - replicates existing behaviour - } - } + def readSeqFromJsonProperty[O](name: String)(implicit rds: Reads[O], mf: Manifest[O]): HttpReads[Seq[O]] = + (method: String, url: String, response: HttpResponse) => + response.status match { + case 204 | 404 => Seq.empty + case _ => + readJson[Seq[O]](method, url, (handleResponse(method, url)(response).json \ name).getOrElse(JsNull)) //Added JsNull here to force validate to fail - replicates existing behaviour + } - private def readJson[A](method: String, url: String, jsValue: JsValue)(implicit rds: Reads[A], mf: Manifest[A]) = + private def readJson[A](method: String, url: String, jsValue: JsValue)(implicit rds: Reads[A], mf: Manifest[A]): A = jsValue .validate[A] .fold( diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/Retries.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/Retries.scala index 1294c7fd..36402e64 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/Retries.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/Retries.scala @@ -35,17 +35,20 @@ trait Retries { private val logger = LoggerFactory.getLogger("application") + private lazy val sslRetryEnabled = + configuration.getBoolean("http-verbs.retries.ssl-engine-closed-already.enabled") + def retryOnSslEngineClosed[A](verb: String, url: String)(block: => Future[A])(implicit ec: ExecutionContext): Future[A] = - retryFor(s"$verb $url") { case ex @ `sslEngineClosedMatcher`() => true }(block) + retryFor(s"$verb $url") { case ex: SSLException if ex.getMessage == "SSLEngine closed already" => sslRetryEnabled }(block) @deprecated("Use retryOnSslEngineClosed instead", "14.0.0") def retry[A](verb: String, url: String)(block: => Future[A])(implicit ec: ExecutionContext): Future[A] = retryOnSslEngineClosed(verb, url)(block) def retryFor[A]( - label : String - )( condition: PartialFunction[Exception, Boolean] - )( block : => Future[A] + label : String + )(condition: PartialFunction[Exception, Boolean] + )(block : => Future[A] )(implicit ec: ExecutionContext ): Future[A] = { @@ -70,17 +73,4 @@ trait Retries { configuration.getDurationList("http-verbs.retries.intervals").asScala.toSeq.map { d => FiniteDuration(d.toMillis, TimeUnit.MILLISECONDS) } - - private lazy val sslEngineClosedMatcher = - new SSlEngineClosedMatcher( - enabled = configuration.getBoolean("http-verbs.retries.ssl-engine-closed-already.enabled") - ) - - private class SSlEngineClosedMatcher(enabled: Boolean) { - def unapply(ex: Throwable): Boolean = - ex match { - case _: SSLException if ex.getMessage == "SSLEngine closed already" => enabled - case _ => false - } - } } diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/client2/HttpClient2Impl.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/client2/HttpClient2Impl.scala index 9199b17f..705b132f 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/client2/HttpClient2Impl.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/client2/HttpClient2Impl.scala @@ -56,7 +56,7 @@ class HttpClient2Impl( ) extends HttpClient2 { private lazy val optProxyServer = - WSProxyConfiguration.buildWsProxyServer(config.underlying) + WSProxyConfiguration.buildWsProxyServer(config) private val hcConfig = HeaderCarrier.Config.fromConfig(config.underlying) @@ -87,7 +87,7 @@ class HttpClient2Impl( } -// is final since `tranform` (and derived functions) return instances of RequestBuilderImpl, and any overrides would be lost. +// is final since `transform` (and derived functions) return instances of RequestBuilderImpl, and any overrides would be lost. final class RequestBuilderImpl( config : Configuration, optProxyServer: Option[WSProxyServer], @@ -197,7 +197,7 @@ class ExecutorImpl( request : WSRequest, optHookDataF: Option[Future[Option[HookData]]], isStream : Boolean, - r : HttpReads[A] + httpReads : HttpReads[A] )(implicit hc: HeaderCarrier, ec: ExecutionContext @@ -224,7 +224,7 @@ class ExecutorImpl( // (i.e. return Future[WSResponse] to be handled with Future.transform/transformWith(...)) // since the transform functions require access to the request (method and url) mapErrors(request, httpResponseF) - .map(r.read(request.method, request.url, _)) + .map(httpReads.read(request.method, request.url, _)) } // unfortunate return type - first HttpResponse is the full response, the second HttpResponse is truncated for auditing... @@ -239,7 +239,22 @@ class ExecutorImpl( val httpResponseF = for { response <- responseF - } yield + status = response.status + headers = response.headers.mapValues(_.toSeq).toMap + } yield { + def httpResponse(body : Either[Source[ByteString, _], String]): HttpResponse = + body match { + case Left(src) => HttpResponse( + status = response.status, + bodyAsSource = src, + headers = response.headers.mapValues(_.toSeq).toMap + ) + case Right(str) => HttpResponse( + status = response.status, + body = str, + headers = response.headers.mapValues(_.toSeq).toMap + ) + } if (isStream) { val source = response.bodyAsSource @@ -247,38 +262,22 @@ class ExecutorImpl( BodyCaptor.sink( loggingContext = loggingContext, maxBodyLength = maxBodyLength, - withCapturedBody = body => - auditResponseF.success( - HttpResponse( - status = response.status, - body = body.decodeString("UTF-8"), - headers = response.headers.mapValues(_.toSeq).toMap - ) - ) + withCapturedBody = body => auditResponseF.success(httpResponse(Right(body.decodeString("UTF-8")))) ) ) .recover { case e => auditResponseF.failure(e); throw e } - HttpResponse( - status = response.status, - bodyAsSource = source, - headers = response.headers.mapValues(_.toSeq).toMap - ) + httpResponse(Left(source)) } else { auditResponseF.success( - HttpResponse( - status = response.status, - body = BodyCaptor.bodyUpto(response.body, maxBodyLength, loggingContext, isStream = false), - headers = response.headers.mapValues(_.toSeq).toMap - ) - ) - HttpResponse( - status = response.status, - body = response.body, - headers = response.headers.mapValues(_.toSeq).toMap + httpResponse(Right( + BodyCaptor.bodyUpto(response.body, maxBodyLength, loggingContext, isStream = false) + )) ) + httpResponse(Right(response.body)) } + } (httpResponseF, auditResponseF.future) } diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/play/http/BodyCaptor.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/play/http/BodyCaptor.scala index d989fc7d..9d2c48d6 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/play/http/BodyCaptor.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/play/http/BodyCaptor.scala @@ -35,8 +35,7 @@ private class BodyCaptorFlow( override def createLogic(inheritedAttributes: Attributes): GraphStageLogic = new GraphStageLogic(shape) { - private var buffer = ByteString.empty - private var bodyLength = 0 + private var buffer = ByteString.empty setHandlers( in, @@ -47,7 +46,6 @@ private class BodyCaptorFlow( override def onPush(): Unit = { val chunk = grab(in) - bodyLength += chunk.length if (buffer.size < maxBodyLength) buffer ++= chunk push(out, chunk) diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/play/http/ws/WSRequest.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/play/http/ws/WSRequest.scala index 47f966f6..db71615d 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/play/http/ws/WSRequest.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/play/http/ws/WSRequest.scala @@ -16,7 +16,6 @@ package uk.gov.hmrc.play.http.ws -import com.typesafe.config.Config import play.api.Configuration import play.api.libs.ws.{DefaultWSProxyServer, WSProxyServer, WSRequest => PlayWSRequest} @@ -61,21 +60,16 @@ object WSProxyConfiguration { else None } - - def buildWsProxyServer(configuration: Config): Option[WSProxyServer] = { - def getOptionalString(key: String): Option[String] = - if (configuration.hasPath(key)) Some(configuration.getString(key)) else None - - if (configuration.getBoolean("http-verbs.proxy.enabled")) + def buildWsProxyServer(configuration: Configuration): Option[WSProxyServer] = + if (configuration.get[Boolean]("http-verbs.proxy.enabled")) Some( DefaultWSProxyServer( - protocol = Some(configuration.getString("proxy.protocol")), - host = configuration.getString("proxy.host"), - port = configuration.getInt("proxy.port"), - principal = getOptionalString("proxy.username"), - password = getOptionalString("proxy.password") + protocol = Some(configuration.get[String]("proxy.protocol")), + host = configuration.get[String]("proxy.host"), + port = configuration.get[Int]("proxy.port"), + principal = configuration.getOptional[String]("proxy.username"), + password = configuration.getOptional[String]("proxy.password") ) ) else None - } } diff --git a/http-verbs-common/src/test/scala/uk/gov/hmrc/http/HeadersSpec.scala b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/HeadersSpec.scala index 5ae0ae7c..4038a53b 100644 --- a/http-verbs-common/src/test/scala/uk/gov/hmrc/http/HeadersSpec.scala +++ b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/HeadersSpec.scala @@ -18,7 +18,6 @@ package uk.gov.hmrc.http import akka.actor.ActorSystem import com.github.ghik.silencer.silent -import com.github.tomakehurst.wiremock._ import com.github.tomakehurst.wiremock.client.WireMock._ import com.typesafe.config.Config import org.scalatest.concurrent.{IntegrationPatience, ScalaFutures} diff --git a/http-verbs-test-common/src/main/scala/uk/gov/hmrc/http/test/HttpClient2Support.scala b/http-verbs-test-common/src/main/scala/uk/gov/hmrc/http/test/HttpClient2Support.scala index dfbfffcd..f30e25a0 100644 --- a/http-verbs-test-common/src/main/scala/uk/gov/hmrc/http/test/HttpClient2Support.scala +++ b/http-verbs-test-common/src/main/scala/uk/gov/hmrc/http/test/HttpClient2Support.scala @@ -17,8 +17,6 @@ package uk.gov.hmrc.http.test import akka.actor.ActorSystem -import akka.stream.{ActorMaterializer, Materializer} -import com.github.ghik.silencer.silent import com.typesafe.config.ConfigFactory import play.api.Configuration import play.api.libs.ws.ahc.{AhcWSClient, AhcWSClientConfigFactory} @@ -31,9 +29,6 @@ trait HttpClient2Support { ): HttpClient2 = { implicit val as: ActorSystem = ActorSystem("test-actor-system") - @silent("deprecated") - implicit val mat: Materializer = ActorMaterializer() // explicitly required for play-26 - new HttpClient2Impl( wsClient = AhcWSClient(AhcWSClientConfigFactory.forConfig(config.underlying)), as, diff --git a/http-verbs-test-common/src/main/scala/uk/gov/hmrc/http/test/HttpClientSupport.scala b/http-verbs-test-common/src/main/scala/uk/gov/hmrc/http/test/HttpClientSupport.scala index 9ae8d1ec..7a669455 100644 --- a/http-verbs-test-common/src/main/scala/uk/gov/hmrc/http/test/HttpClientSupport.scala +++ b/http-verbs-test-common/src/main/scala/uk/gov/hmrc/http/test/HttpClientSupport.scala @@ -17,8 +17,6 @@ package uk.gov.hmrc.http.test import akka.actor.ActorSystem -import akka.stream.{ActorMaterializer, Materializer} -import com.github.ghik.silencer.silent import com.typesafe.config.{Config, ConfigFactory} import play.api.libs.ws.WSClient import uk.gov.hmrc.http.HttpClient @@ -33,9 +31,6 @@ trait HttpClientSupport { new HttpClient with WSHttp { private implicit val as: ActorSystem = ActorSystem("test-actor-system") - @silent("deprecated") - private implicit val mat: Materializer = ActorMaterializer() // explicitly required for play-26 - override val wsClient: WSClient = AhcWSClient(AhcWSClientConfigFactory.forConfig(config)) override protected val configuration: Config = config override val hooks: Seq[HttpHook] = Seq.empty From f7f52ddc35c34500ae4f2da5f39ec9f7abb3ec76 Mon Sep 17 00:00:00 2001 From: colin-lamed <9568290+colin-lamed@users.noreply.github.com> Date: Tue, 15 Feb 2022 16:53:18 +0000 Subject: [PATCH 21/23] BDOG-1512 Move u.g.h.h.client2.HttpClient2 to u.g.h.h.client.HttpClientV2 --- CHANGELOG.md | 2 +- README.md | 24 ++++---- .../HttpClientV2.scala} | 4 +- .../HttpClientV2Impl.scala} | 8 +-- .../http/{client2 => client}/package.scala | 12 ++-- .../HttpClientV2Spec.scala} | 60 +++++++++---------- .../hmrc/http/test/HttpClient2Support.scala | 12 ++-- 7 files changed, 61 insertions(+), 61 deletions(-) rename http-verbs-common/src/main/scala/uk/gov/hmrc/http/{client2/HttpClient2.scala => client/HttpClientV2.scala} (97%) rename http-verbs-common/src/main/scala/uk/gov/hmrc/http/{client2/HttpClient2Impl.scala => client/HttpClientV2Impl.scala} (99%) rename http-verbs-common/src/main/scala/uk/gov/hmrc/http/{client2 => client}/package.scala (86%) rename http-verbs-common/src/test/scala/uk/gov/hmrc/http/{client2/HttpClient2Spec.scala => client/HttpClientV2Spec.scala} (96%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 204dea2a..7f2a70f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,7 @@ There are some differences with `WSProxyConfiguration.buildWsProxyServer`: * `proxy.proxyRequiredForThisEnvironment` has been replaced with `http-verbs.proxy.enabled`, but note, it defaults to false (rather than true). This is appropriate for development and tests, but will need explicitly enabling when deployed. -### Adds HttpClient2 +### Adds HttpClientV2 This is in addition to `HttpClient` (for now), so can be optionally used instead. See [README](/README.md) for details. diff --git a/README.md b/README.md index 433ce013..61da2d69 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ Examples can be found [here](https://github.com/hmrc/http-verbs/blob/master/http URLs can be supplied as either `java.net.URL` or `String`. We recommend supplying `java.net.URL` and using the provided [URL interpolator](#url-interpolator) for correct escaping of query and path parameters. -### uk.gov.hmrc.http.client2.HttpClient2 +### uk.gov.hmrc.http.client.HttpClientV2 This client follows the same patterns as `HttpClient` - that is, it also requires a `HeaderCarrier` to represent the context of the caller, and an `HttpReads` to process the http response. @@ -51,7 +51,7 @@ In addition, it: - Exposes the underlying `play.api.libs.ws.WSRequest` with `transform`, making it easier to customise the request. - Only accepts the URL as `java.net.URL`; you can make use of the provided [URL interpolator](#url-interpolator). -Examples can be found in [here](/http-verbs-common/src/test/scala/uk/gov/hmrc/http/client2/HttpClient2Spec.scala) +Examples can be found in [here](/http-verbs-common/src/test/scala/uk/gov/hmrc/http/client/HttpClientV2Spec.scala) To migrate: @@ -62,7 +62,7 @@ httpClient.GET[ResponseType](url) becomes ```scala -httpClient2.get(url"$url").execute[ResponseType] +httpClientV2.get(url"$url").execute[ResponseType] ``` and @@ -74,43 +74,43 @@ httpClient.POST[ResponseType](url, payload, headers) becomes ```scala -httpClient2.post(url"$url").withBody(Json.toJson(payload)).addHeaders(headers).execute[ResponseType] +httpClientV2.post(url"$url").withBody(Json.toJson(payload)).addHeaders(headers).execute[ResponseType] ``` #### Header manipulation -With `HttpClient`, replacing a header can require providing a customised client implementation (e.g. to replace the user-agent header), or updating the `HeaderCarrier` (e.g. to replace the authorisation header). This can now all be done with the `replaceHeader` on `HttpClient2` per call. e.g. +With `HttpClient`, replacing a header can require providing a customised client implementation (e.g. to replace the user-agent header), or updating the `HeaderCarrier` (e.g. to replace the authorisation header). This can now all be done with the `replaceHeader` on `HttpClientV2` per call. e.g. ```scala -httpClient2.get(url"$url").replaceHeader("User-Agent" -> userAgent).replaceHeader("Authorization" -> authorization).execute[ResponseType] +httpClientV2.get(url"$url").replaceHeader("User-Agent" -> userAgent).replaceHeader("Authorization" -> authorization).execute[ResponseType] ``` #### Using proxy -With `HttpClient`, to use a proxy requires creating a new instance of HttpClient to mix in `WSProxy` and configure. With `HttpClient2` this can be done with the same client, calling `withProxy` per call. e.g. +With `HttpClient`, to use a proxy requires creating a new instance of HttpClient to mix in `WSProxy` and configure. With `HttpClientV2` this can be done with the same client, calling `withProxy` per call. e.g. ```scala -httpClient2.get(url"$url").withProxy.execute[ResponseType] +httpClientV2.get(url"$url").withProxy.execute[ResponseType] ``` * It uses `WSProxyConfiguration.buildWsProxyServer` which needs enabling with `http-verbs.proxy.enabled` in configuration, which by default is `false`, for development. See [WSProxyConfiguration](CHANGELOG.md#wsproxyconfiguration) for configuration changes. #### Streaming -Streaming is supported with `HttpClient2`, and will be audited in the same way as `HttpClient`. Note that payloads will be truncated in audit logs if they exceed the max supported (as configured by `http-verbs.auditing.maxBodyLength`). +Streaming is supported with `HttpClientV2`, and will be audited in the same way as `HttpClient`. Note that payloads will be truncated in audit logs if they exceed the max supported (as configured by `http-verbs.auditing.maxBodyLength`). Streamed requests can simply be passed to `withBody`: ```scala val reqStream: Source[ByteString, _] = ??? -httpClient2.post(url"$url").withBody(reqStream).execute[ResponseType] +httpClientV2.post(url"$url").withBody(reqStream).execute[ResponseType] ``` For streamed responses, use `stream` rather than `execute`: ```scala -httpClient2.get(url"$url").stream[Source[ByteString, _]] +httpClientV2.get(url"$url").stream[Source[ByteString, _]] ``` ### URL interpolator @@ -208,7 +208,7 @@ class MyConnectorSpec extends WireMockSupport with HttpClientSupport { } ``` -Similarly `HttpClient2Support` can be used to provide an instance of `HttpClient2`. +Similarly `HttpClientV2Support` can be used to provide an instance of `HttpClientV2`. The `ExternalWireMockSupport` trait is an alternative to `WireMockSupport` which uses `127.0.0.1` instead of `localhost` for the hostname which is treated as an external host for header forwarding rules. This should be used for tests of connectors which call endpoints external to the platform. The variable `externalWireMockHost` (or `externalWireMockUrl`) should be used to provide the hostname in configuration. diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/client2/HttpClient2.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/client/HttpClientV2.scala similarity index 97% rename from http-verbs-common/src/main/scala/uk/gov/hmrc/http/client2/HttpClient2.scala rename to http-verbs-common/src/main/scala/uk/gov/hmrc/http/client/HttpClientV2.scala index 914e8f22..2c3b3a87 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/client2/HttpClient2.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/client/HttpClientV2.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package uk.gov.hmrc.http.client2 +package uk.gov.hmrc.http.client import play.api.libs.ws.{BodyWritable, WSRequest} import uk.gov.hmrc.http.{HeaderCarrier, HttpReads} @@ -27,7 +27,7 @@ import scala.reflect.runtime.universe.TypeTag * retries) occur, but makes building the request more flexible (by exposing play-ws). * It also supports streaming. */ -trait HttpClient2 { +trait HttpClientV2 { protected def mkRequestBuilder(url: URL, method: String)(implicit hc: HeaderCarrier): RequestBuilder def get(url: URL)(implicit hc: HeaderCarrier): RequestBuilder = diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/client2/HttpClient2Impl.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/client/HttpClientV2Impl.scala similarity index 99% rename from http-verbs-common/src/main/scala/uk/gov/hmrc/http/client2/HttpClient2Impl.scala rename to http-verbs-common/src/main/scala/uk/gov/hmrc/http/client/HttpClientV2Impl.scala index 705b132f..59d24858 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/client2/HttpClient2Impl.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/client/HttpClientV2Impl.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package uk.gov.hmrc.http.client2 +package uk.gov.hmrc.http.client import akka.actor.ActorSystem import akka.stream.scaladsl.Source @@ -48,12 +48,12 @@ trait Executor { ): Future[A] } -class HttpClient2Impl( +class HttpClientV2Impl( wsClient : WSClient, actorSystem: ActorSystem, config : Configuration, hooks : Seq[HttpHook] -) extends HttpClient2 { +) extends HttpClientV2 { private lazy val optProxyServer = WSProxyConfiguration.buildWsProxyServer(config) @@ -65,7 +65,7 @@ class HttpClient2Impl( new ExecutorImpl(actorSystem, config, hooks) private val clientVersionHeader = - "Http-Client2-Version" -> BuildInfo.version + "Http-ClientV2-Version" -> BuildInfo.version override protected def mkRequestBuilder( url : URL, diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/client2/package.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/client/package.scala similarity index 86% rename from http-verbs-common/src/main/scala/uk/gov/hmrc/http/client2/package.scala rename to http-verbs-common/src/main/scala/uk/gov/hmrc/http/client/package.scala index e451f0f5..80097ac4 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/client2/package.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/client/package.scala @@ -22,10 +22,10 @@ import akka.util.ByteString import scala.annotation.implicitNotFound -package client2 { +package client { trait Streaming } -package object client2 +package object client extends StreamHttpReadsInstances { // ensures strict HttpReads are not passed to stream function, which would lead to stream being read into memory @@ -40,10 +40,10 @@ trait StreamHttpReadsInstances { implicit val errorTimeout: ErrorTimeout = ErrorTimeout() - def tag[A](instance: A): A with client2.Streaming = - instance.asInstanceOf[A with client2.Streaming] + def tag[A](instance: A): A with client.Streaming = + instance.asInstanceOf[A with client.Streaming] - implicit def readEitherSource(implicit mat: Materializer, errorTimeout: ErrorTimeout): client2.StreamHttpReads[Either[UpstreamErrorResponse, Source[ByteString, _]]] = + implicit def readEitherSource(implicit mat: Materializer, errorTimeout: ErrorTimeout): client.StreamHttpReads[Either[UpstreamErrorResponse, Source[ByteString, _]]] = tag[HttpReads[Either[UpstreamErrorResponse, Source[ByteString, _]]]]( HttpReads.ask.flatMap { case (method, url, response) => HttpErrorFunctions.handleResponseEitherStream(method, url)(response) match { @@ -53,7 +53,7 @@ trait StreamHttpReadsInstances { } ) - implicit def readSource(implicit mat: Materializer, errorTimeout: ErrorTimeout): client2.StreamHttpReads[Source[ByteString, _]] = + implicit def readSource(implicit mat: Materializer, errorTimeout: ErrorTimeout): client.StreamHttpReads[Source[ByteString, _]] = tag[HttpReads[Source[ByteString, _]]]( readEitherSource .map { diff --git a/http-verbs-common/src/test/scala/uk/gov/hmrc/http/client2/HttpClient2Spec.scala b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/client/HttpClientV2Spec.scala similarity index 96% rename from http-verbs-common/src/test/scala/uk/gov/hmrc/http/client2/HttpClient2Spec.scala rename to http-verbs-common/src/test/scala/uk/gov/hmrc/http/client/HttpClientV2Spec.scala index 97020500..fe0067f4 100644 --- a/http-verbs-common/src/test/scala/uk/gov/hmrc/http/client2/HttpClient2Spec.scala +++ b/http-verbs-common/src/test/scala/uk/gov/hmrc/http/client/HttpClientV2Spec.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package uk.gov.hmrc.http.client2 +package uk.gov.hmrc.http.client import akka.actor.ActorSystem import akka.stream.scaladsl.Source @@ -40,7 +40,7 @@ import scala.concurrent.{ExecutionContext, Future} import scala.concurrent.ExecutionContext.Implicits.global import scala.util.Random -class HttpClient2Spec +class HttpClientV2Spec extends AnyWordSpecLike with Matchers with WireMockSupport @@ -51,7 +51,7 @@ class HttpClient2Spec import uk.gov.hmrc.http.HttpReads.Implicits._ - "HttpClient2" should { + "HttpClientV2" should { "work with json" in new Setup { implicit val hc = HeaderCarrier() @@ -61,7 +61,7 @@ class HttpClient2Spec ) val res: Future[ResDomain] = - httpClient2 + httpClientV2 .put(url"$wireMockUrl/") .withBody(Json.toJson(ReqDomain("req"))) .execute[ResDomain] @@ -73,7 +73,7 @@ class HttpClient2Spec .withHeader("Content-Type", equalTo("application/json")) .withRequestBody(equalTo("\"req\"")) .withHeader("User-Agent", equalTo("myapp")) - .withHeader("Http-Client2-Version", matching(".*")) + .withHeader("Http-ClientV2-Version", matching(".*")) ) val headersCaptor = ArgCaptor[Seq[(String, String)]] @@ -110,7 +110,7 @@ class HttpClient2Spec Source.single(ByteString(requestBody)) val res: Future[Source[ByteString, _]] = - httpClient2 + httpClientV2 .put(url"$wireMockUrl/") .withBody(srcStream) .stream[Source[ByteString, _]] @@ -158,7 +158,7 @@ class HttpClient2Spec Source.single(ByteString(requestBody)) val res: Future[Source[ByteString, _]] = - httpClient2 + httpClientV2 .put(url"$wireMockUrl/") .withBody(srcStream) .stream[Source[ByteString, _]] @@ -210,7 +210,7 @@ class HttpClient2Spec Source.single(ByteString(requestBody)) val res: Future[Source[ByteString, _]] = - httpClient2 + httpClientV2 .put(url"$wireMockUrl/") .withBody(srcStream) .stream[Source[ByteString, _]] @@ -255,7 +255,7 @@ class HttpClient2Spec ) val res: Future[HttpResponse] = - httpClient2 + httpClientV2 .put(url"$wireMockUrl/") .withBody(requestBody) .execute[HttpResponse] @@ -303,10 +303,10 @@ class HttpClient2Spec ) val res: Future[ResDomain] = - httpClient2 - .post(url"$wireMockUrl/") - .withBody(body) - .execute[ResDomain] + httpClientV2 + .post(url"$wireMockUrl/") + .withBody(body) + .execute[ResDomain] res.futureValue shouldBe ResDomain("res") @@ -365,10 +365,10 @@ class HttpClient2Spec ) val res: Future[ResDomain] = - httpClient2 - .post(url"$wireMockUrl/") - .withBody(body) - .execute[ResDomain] + httpClientV2 + .post(url"$wireMockUrl/") + .withBody(body) + .execute[ResDomain] res.futureValue shouldBe ResDomain("res") @@ -427,10 +427,10 @@ class HttpClient2Spec ) val res: Future[ResDomain] = - httpClient2 - .post(url"$wireMockUrl/") - .withBody(body) - .execute[ResDomain] + httpClientV2 + .post(url"$wireMockUrl/") + .withBody(body) + .execute[ResDomain] res.futureValue shouldBe ResDomain("res") @@ -490,10 +490,10 @@ class HttpClient2Spec ) val res: Future[ResDomain] = - httpClient2 - .post(url"$wireMockUrl/") - .withBody(body) - .execute[ResDomain] + httpClientV2 + .post(url"$wireMockUrl/") + .withBody(body) + .execute[ResDomain] res.futureValue shouldBe ResDomain("res") @@ -528,7 +528,7 @@ class HttpClient2Spec implicit val hc = HeaderCarrier() a[RuntimeException] should be thrownBy - httpClient2 + httpClientV2 .put(url"$wireMockUrl/") .transform(_.withBody(Json.toJson(ReqDomain("req")))) .replaceHeader("User-Agent" -> "ua2") @@ -544,7 +544,7 @@ class HttpClient2Spec ) val res: Future[ResDomain] = - httpClient2 + httpClientV2 .put(url"$wireMockUrl/") .withBody(Json.toJson(ReqDomain("req"))) .replaceHeader("User-Agent" -> "ua2") @@ -579,7 +579,7 @@ class HttpClient2Spec val res: Future[ResDomain] = retries.retryFor("get reqdomain"){ case UpstreamErrorResponse.WithStatusCode(502) => true }{ count.incrementAndGet - httpClient2 + httpClientV2 .put(url"$wireMockUrl/") .withBody(Json.toJson(ReqDomain("req"))) .execute[ResDomain] @@ -597,7 +597,7 @@ class HttpClient2Spec val maxAuditBodyLength = 30 - val httpClient2: HttpClient2 = { + val httpClientV2: HttpClientV2 = { val config = Configuration( ConfigFactory.parseString( @@ -606,7 +606,7 @@ class HttpClient2Spec |""".stripMargin ).withFallback(ConfigFactory.load()) ) - new HttpClient2Impl( + new HttpClientV2Impl( wsClient = AhcWSClient(AhcWSClientConfigFactory.forConfig(config.underlying)), as, config, diff --git a/http-verbs-test-common/src/main/scala/uk/gov/hmrc/http/test/HttpClient2Support.scala b/http-verbs-test-common/src/main/scala/uk/gov/hmrc/http/test/HttpClient2Support.scala index f30e25a0..65c3e939 100644 --- a/http-verbs-test-common/src/main/scala/uk/gov/hmrc/http/test/HttpClient2Support.scala +++ b/http-verbs-test-common/src/main/scala/uk/gov/hmrc/http/test/HttpClient2Support.scala @@ -20,16 +20,16 @@ import akka.actor.ActorSystem import com.typesafe.config.ConfigFactory import play.api.Configuration import play.api.libs.ws.ahc.{AhcWSClient, AhcWSClientConfigFactory} -import uk.gov.hmrc.http.client2.{HttpClient2, HttpClient2Impl} +import uk.gov.hmrc.http.client.{HttpClientV2, HttpClientV2Impl} -trait HttpClient2Support { +trait HttpClientV2Support { - def mkHttpClient2( + def mkHttpClientV2( config: Configuration = Configuration(ConfigFactory.load()) - ): HttpClient2 = { + ): HttpClientV2 = { implicit val as: ActorSystem = ActorSystem("test-actor-system") - new HttpClient2Impl( + new HttpClientV2Impl( wsClient = AhcWSClient(AhcWSClientConfigFactory.forConfig(config.underlying)), as, config, @@ -37,5 +37,5 @@ trait HttpClient2Support { ) } - lazy val httpClient2: HttpClient2 = mkHttpClient2() + lazy val httpClientV2: HttpClientV2 = mkHttpClientV2() } From b55b6b95a62cb04f0a0137cb3bc14d24573668e0 Mon Sep 17 00:00:00 2001 From: colin-lamed <9568290+colin-lamed@users.noreply.github.com> Date: Wed, 16 Feb 2022 13:48:02 +0000 Subject: [PATCH 22/23] BDOG-1512 Remove conversion of flow without data into a flow of an empty ByteString --- .../src/main/scala/uk/gov/hmrc/play/http/BodyCaptor.scala | 2 -- 1 file changed, 2 deletions(-) diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/play/http/BodyCaptor.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/play/http/BodyCaptor.scala index 9d2c48d6..64472732 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/play/http/BodyCaptor.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/play/http/BodyCaptor.scala @@ -53,8 +53,6 @@ private class BodyCaptorFlow( override def onUpstreamFinish(): Unit = { withCapturedBody(BodyCaptor.bodyUpto(buffer, maxBodyLength, loggingContext, isStream = true)) - if (isAvailable(out) && buffer == ByteString.empty) - push(out, buffer) completeStage() } } From 60c61abdeb4b86d8ba168bda200aaf88aa7b2986 Mon Sep 17 00:00:00 2001 From: colin-lamed <9568290+colin-lamed@users.noreply.github.com> Date: Thu, 17 Feb 2022 11:10:07 +0000 Subject: [PATCH 23/23] BDOG-1512 Move implict ErrorTimeout --- .../main/scala/uk/gov/hmrc/http/HttpErrorFunctions.scala | 7 ++++++- .../src/main/scala/uk/gov/hmrc/http/client/package.scala | 4 ---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpErrorFunctions.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpErrorFunctions.scala index 38219502..995d3a05 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpErrorFunctions.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/HttpErrorFunctions.scala @@ -123,5 +123,10 @@ trait HttpErrorFunctions { object HttpErrorFunctions extends HttpErrorFunctions case class ErrorTimeout( - toDuration: Duration = 10.seconds + toDuration: Duration ) extends AnyVal + +object ErrorTimeout { + implicit val errorTimeout: ErrorTimeout = + ErrorTimeout(10.seconds) +} diff --git a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/client/package.scala b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/client/package.scala index 80097ac4..4b12f2ff 100644 --- a/http-verbs-common/src/main/scala/uk/gov/hmrc/http/client/package.scala +++ b/http-verbs-common/src/main/scala/uk/gov/hmrc/http/client/package.scala @@ -36,10 +36,6 @@ package object client } trait StreamHttpReadsInstances { - // default may be overridden if required - implicit val errorTimeout: ErrorTimeout = - ErrorTimeout() - def tag[A](instance: A): A with client.Streaming = instance.asInstanceOf[A with client.Streaming]