Merge branch 'main' into bau/more_code_coverage_r2

proxy.conf

@@ -33,6 +33,12 @@
     # Time out in seconds for request,response and connecting to a url
     "timeout": 100
 
+    # Optional useragent header value to be sent when fetching content.
+    # Commenting out the config means no useragent header will be set in 
+    # the outgoing request. This may lead to some destination servers
+    # rejecting the request.
+    "client_useragent": "ImageProxy/1.0"
+
     "security" : {
         # Api access keys. Random ones provided below for testing, replace with your
         # list. Note that if metrics is enabled, key usage stats will be displayed

src/config.rs

@@ -66,6 +66,7 @@ pub struct Configuration {
     pub metrics_enabled: bool,
     pub dashboard_enabled: bool,
     pub max_document_size: Option<u64>,
+    pub client_useragent: Option<String>,
     pub security: SecurityConfig,
     pub database: DatabaseConfig,
     pub moderation: ModerationConfig,

src/http/hyper_client.rs

@@ -4,8 +4,8 @@ use std::time::Duration;
 
 use async_trait::async_trait;
 use hyper::client::HttpConnector;
-use hyper::Client;
 use hyper::{body::to_bytes, Uri};
+use hyper::{Client, Method, Request};
 use hyper_timeout::TimeoutConnector;
 use hyper_tls::HttpsConnector;
 use log::error;
@@ -20,10 +20,11 @@ use crate::document::Document;
 pub struct HyperHttpClient {
     client: Client<TimeoutConnector<HttpsConnector<HttpConnector<GaiResolver>>>, Body>,
     _max_document_size: Option<u64>,
+    useragent: Option<String>,
 }
 
 impl HyperHttpClient {
-    pub fn new(max_document_size: Option<u64>, timeout: u64) -> Self {
+    pub fn new(max_document_size: Option<u64>, timeout: u64, useragent: Option<String>) -> Self {
         let https = HttpsConnector::new();
         let mut connector = TimeoutConnector::new(https);
 
@@ -35,14 +36,22 @@ impl HyperHttpClient {
         HyperHttpClient {
             client,
             _max_document_size: max_document_size,
+            useragent,
         }
     }
 }
 
 #[async_trait]
 impl HttpClientProvider for HyperHttpClient {
     async fn fetch(&self, req_id: &Uuid, uri: &Uri) -> Result<Document, StatusCode> {
-        let response = self.client.get(uri.clone()).await.map_err(|error| {
+        let request = Request::builder().method(Method::GET).uri(uri.clone());
+        let request = if let Some(useragent) = &self.useragent {
+            request.header("user-agent", useragent)
+        } else {
+            request
+        };
+        let request = request.body(Body::empty()).unwrap();
+        let response = self.client.request(request).await.map_err(|error| {
             error!(
                 "Unable to fetch document, id={}, reason={}, url={}",
                 req_id, error, uri

src/http/mod.rs

@@ -164,21 +164,23 @@ impl HttpClientWrapper {
     }
 }
 
-pub struct HttpClientFactory {}
+pub struct HttpClientFactory;
 
 impl HttpClientFactory {
     pub fn get_provider(
         ipfs_config: Host,
         max_document_size: Option<u64>,
         uri_filters: Vec<Box<dyn UriFilter + Send + Sync>>,
         timeout: u64,
+        useragent: Option<String>,
     ) -> HttpClientWrapper {
         assert!(
             !uri_filters.is_empty(),
             "No URI filters provided. This is insecure, check code. Exiting..."
         );
-        HttpClientWrapper::new(
-            Box::new(HyperHttpClient::new(max_document_size, timeout)),
+
+        HttpClientWrapper {
+            client: Box::new(HyperHttpClient::new(max_document_size, timeout, useragent)),
             ipfs_config,
             uri_filters,
         )
@@ -239,7 +241,7 @@ pub mod tests {
         let uri_filters: Vec<Box<dyn UriFilter + Send + Sync>> = vec![Box::new(
             PrivateNetworkFilter::new(Box::new(StandardDnsResolver {})),
         )];
-        let wrapper = HttpClientFactory::get_provider(ipfs_config, None, uri_filters, 10_u64);
+        let wrapper = HttpClientFactory::get_provider(ipfs_config, None, uri_filters, 10_u64, None);
 
         // Valid https url
         let result = wrapper.to_uri("https://localhost:3422/image.png");

src/proxy.rs

@@ -68,6 +68,7 @@ impl Context {
             config.max_document_size,
             uri_filters,
             config.timeout,
+            config.client_useragent.clone(),
         );
         Ok(Context {
             database,