HTTP: Populate WARC-IP-Address with server's IP address

Fixes #396

Author: ato

modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java

@@ -600,7 +600,7 @@ public ManagedHttpClientConnection create(HttpRoute route,
                         DEFAULT_BUFSIZE, chardecoder, charencoder,
                         cconfig.getMessageConstraints(), null, null,
                         DefaultHttpRequestWriterFactory.INSTANCE,
-                        DefaultHttpResponseParserFactory.INSTANCE);
+                        DefaultHttpResponseParserFactory.INSTANCE, curi);
             }
         };
         BasicHttpClientConnectionManager connMan = new BasicHttpClientConnectionManager(
@@ -618,6 +618,7 @@ protected static class RecordingHttpClientConnection extends DefaultBHttpClientC
 
         private static final AtomicLong COUNTER = new AtomicLong();
         private String id;
+        private final CrawlURI curi;
 
         public RecordingHttpClientConnection(
                 final int buffersize,
@@ -628,15 +629,17 @@ public RecordingHttpClientConnection(
                 final ContentLengthStrategy incomingContentStrategy,
                 final ContentLengthStrategy outgoingContentStrategy,
                 final HttpMessageWriterFactory<HttpRequest> requestWriterFactory,
-                final HttpMessageParserFactory<HttpResponse> responseParserFactory) {
+                final HttpMessageParserFactory<HttpResponse> responseParserFactory, CrawlURI curi) {
             super(buffersize, fragmentSizeHint, chardecoder, charencoder,
                     constraints, incomingContentStrategy, outgoingContentStrategy,
                     requestWriterFactory, responseParserFactory);
             id = "recording-http-connection-" + Long.toString(COUNTER.getAndIncrement());
+            this.curi = curi;
         }
 
         @Override
         protected InputStream getSocketInputStream(final Socket socket) throws IOException {
+            curi.setServerIP(socket.getInetAddress().getHostAddress());
             Recorder recorder = Recorder.getHttpRecorder();
             if (recorder != null) {   // XXX || (isSecure() && isProxied())) {
                 return recorder.inputWrap(super.getSocketInputStream(socket));

modules/src/main/java/org/archive/modules/warc/HttpResponseRecordBuilder.java

@@ -1,12 +1,6 @@
 package org.archive.modules.warc;
 
-import static org.archive.format.warc.WARCConstants.HEADER_KEY_CONCURRENT_TO;
-import static org.archive.format.warc.WARCConstants.HEADER_KEY_PAYLOAD_DIGEST;
-import static org.archive.format.warc.WARCConstants.HEADER_KEY_TRUNCATED;
-import static org.archive.format.warc.WARCConstants.HTTP_RESPONSE_MIMETYPE;
-import static org.archive.format.warc.WARCConstants.NAMED_FIELD_TRUNCATED_VALUE_HEAD;
-import static org.archive.format.warc.WARCConstants.NAMED_FIELD_TRUNCATED_VALUE_LENGTH;
-import static org.archive.format.warc.WARCConstants.NAMED_FIELD_TRUNCATED_VALUE_TIME;
+import static org.archive.format.warc.WARCConstants.*;
 import static org.archive.modules.CoreAttributeConstants.A_WARC_RESPONSE_HEADERS;
 import static org.archive.modules.CoreAttributeConstants.HEADER_TRUNC;
 import static org.archive.modules.CoreAttributeConstants.LENGTH_TRUNC;
@@ -55,6 +49,10 @@ public WARCRecordInfo buildRecord(CrawlURI curi, URI concurrentTo) throws IOExce
                     curi.getContentDigestSchemeString());
         }
 
+        if (curi.getServerIP() != null) {
+            recordInfo.addExtraHeader(HEADER_KEY_IP, curi.getServerIP());
+        }
+
         // Check for truncated annotation
         String value = null;
         Collection<String> anno = curi.getAnnotations();

modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java

@@ -130,7 +130,8 @@ protected void runDefaultChecks(CrawlURI curi, String... exclusionsArray)
         assertEquals(DEFAULT_PAYLOAD_STRING.length(), curi.getContentLength());
         assertEquals(curi.getContentSize(), curi.getRecordedSize());
 
-        // check various 
+        // check various
+        assertNotNull(curi.getServerIP());
         assertEquals("sha1:TQ5R6YVOZLTQENRIIENVGXHOPX3YCRNJ", curi.getContentDigestSchemeString());
         if (!exclusions.contains("contentType")) {
             assertEquals("text/plain;charset=US-ASCII", curi.getContentType());