Merge pull request #2476 from nordic-institute/XRDDEV-2687

feat: support non-batch signatures in message log and ASIC-E containers

Refs: XRDDEV-2687

Docker/xrd-dev-stack/tools/deploy.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+: ${XROAD_HOME:?"XROAD_HOME is not set"}
+
 deploy_module() {
   local module_name=$1
   shift
@@ -15,6 +17,7 @@ deploy_module() {
     ;;
   "messagelog-addon")
     jar_path="$XROAD_HOME/src/addons/messagelog/messagelog-addon/build/libs/messagelog-addon.jar"
+    target_path="/usr/share/xroad/jlib/addon/proxy/"
     service_name="xroad-proxy"
     ;;
   "hwtoken-addon")
@@ -66,14 +69,18 @@ deploy_module() {
   done
 }
 
-set -o xtrace
+set -o xtrace -o errexit
 
 case $1 in
 "proxy" | "messagelog-addon" | "metaservice-addon" | "proxy-ui-api" | "configuration-client" | "op-monitor-daemon")
-  deploy_module "$1" "ss0" "ss1"
+  hosts=("ss0" "ss1")
+  if [[ $# > 1 ]]; then hosts=("${@:2}"); fi
+  deploy_module "$1" "${hosts[@]}"
   ;;
 "signer" | "hwtoken-addon")
-  deploy_module "$1" "ss0" "ss1" "cs"
+  hosts=("ss0" "ss1" "cs")
+  if [[ $# > 1 ]]; then hosts=("${@:2}"); fi
+  deploy_module "$1" "${hosts[@]}"
   ;;
 "cs-admin-service" | "cs-management-service" | "cs-registration-service")
   deploy_module "$1" "cs"

src/addons/messagelog/messagelog-addon/src/main/java/ee/ria/xroad/proxy/messagelog/LogManager.java

@@ -31,6 +31,7 @@
 import ee.ria.xroad.common.DiagnosticsUtils;
 import ee.ria.xroad.common.conf.globalconf.GlobalConfProvider;
 import ee.ria.xroad.common.conf.serverconf.ServerConfProvider;
+import ee.ria.xroad.common.message.AttachmentStream;
 import ee.ria.xroad.common.messagelog.AbstractLogManager;
 import ee.ria.xroad.common.messagelog.LogMessage;
 import ee.ria.xroad.common.messagelog.MessageLogProperties;
@@ -44,6 +45,7 @@
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.io.input.BoundedInputStream;
 
+import java.io.InputStream;
 import java.time.Duration;
 import java.time.Instant;
 import java.time.temporal.ChronoUnit;
@@ -190,11 +192,11 @@ private TimestampRecord timestampImmediately(MessageRecord logRecord) throws Exc
     private static MessageRecord createMessageRecord(SoapLogMessage message) throws Exception {
         log.trace("createMessageRecord()");
 
-        String loggedMessage = new MessageBodyManipulator().getLoggableMessageText(message);
+        var manipulator = new MessageBodyManipulator();
 
         MessageRecord messageRecord = new MessageRecord(
                 message.getQueryId(),
-                loggedMessage,
+                manipulator.getLoggableMessageText(message),
                 message.getSignature().getSignatureXml(),
                 message.isResponse(),
                 message.isClientSide() ? message.getClient() : message.getService().getClientId(),
@@ -205,12 +207,38 @@ private static MessageRecord createMessageRecord(SoapLogMessage message) throws
         if (message.getSignature().isBatchSignature()) {
             messageRecord.setHashChainResult(message.getSignature().getHashChainResult());
             messageRecord.setHashChain(message.getSignature().getHashChain());
+        } else if (manipulator.isBodyLogged(message)) {
+            // log attachments for non-batch signatures
+            if (MAX_LOGGABLE_BODY_SIZE > 0) {
+                messageRecord.setAttachmentStreams(message.getAttachments()
+                        .stream().map(LogManager::boundedAttachmentStream).toList());
+            }
+
         }
 
         messageRecord.setSignatureHash(signatureHash(message.getSignature().getSignatureXml()));
         return messageRecord;
     }
 
+    private static AttachmentStream boundedAttachmentStream(AttachmentStream attachment) {
+        return new AttachmentStream() {
+            @Override
+            public InputStream getStream() {
+                if (attachment.getSize() > MAX_LOGGABLE_BODY_SIZE && !TRUNCATED_BODY_ALLOWED) {
+                    throw new CodedException(X_LOGGING_FAILED_X, "Message attachment size exceeds maximum loggable size");
+                }
+                final BoundedInputStream body = new BoundedInputStream(attachment.getStream(), MAX_LOGGABLE_BODY_SIZE);
+                body.setPropagateClose(false);
+                return body;
+            }
+
+            @Override
+            public long getSize() {
+                return attachment.getSize();
+            }
+        };
+    }
+
     private static MessageRecord createMessageRecord(RestLogMessage message) throws Exception {
         log.trace("createMessageRecord()");
 

src/addons/messagelog/messagelog-addon/src/main/java/ee/ria/xroad/proxy/messagelog/LogRecordManager.java

@@ -27,6 +27,7 @@
 
 import ee.ria.xroad.common.db.HibernateUtil;
 import ee.ria.xroad.common.identifier.ClientId;
+import ee.ria.xroad.common.message.AttachmentStream;
 import ee.ria.xroad.common.messagelog.AbstractLogRecord;
 import ee.ria.xroad.common.messagelog.LogRecord;
 import ee.ria.xroad.common.messagelog.MessageRecord;
@@ -39,14 +40,12 @@
 import jakarta.persistence.criteria.Root;
 import lombok.extern.slf4j.Slf4j;
 import org.hibernate.Session;
-import org.hibernate.query.Query;
+import org.hibernate.query.MutationQuery;
 
-import java.io.InputStream;
 import java.sql.Connection;
 import java.sql.PreparedStatement;
 import java.sql.SQLException;
 import java.util.Arrays;
-import java.util.Date;
 import java.util.List;
 import java.util.function.Function;
 
@@ -73,20 +72,6 @@ public final class LogRecordManager {
     private LogRecordManager() {
     }
 
-    /**
-     * Returns a log record for a given message Query Id, start and end time.
-     * @param queryId the message query id.
-     * @param startTime the start time.
-     * @param endTime the end time.
-     * @return the log record or null, if log record is not found in database.
-     * @throws Exception if an error occurs while communicating with database.
-     */
-    static LogRecord getByQueryId(String queryId, Date startTime, Date endTime) throws Exception {
-        log.trace(GET_BY_QUERY_ID_LOG_FORMAT, queryId, startTime, endTime);
-
-        return doInTransaction(session -> getMessageRecord(session, queryId, startTime, endTime));
-    }
-
     /**
      * Returns a log record for a given message Query Id and sender Client Id.
      * @param queryId the message query id.
@@ -149,10 +134,11 @@ static void saveMessageRecord(MessageRecord messageRecord) throws Exception {
                 encryption.prepareEncryption(messageRecord);
             }
 
-            InputStream is = messageRecord.getAttachmentStream();
-            if (is != null) {
-                messageRecord.setAttachment(
-                        session.getLobHelper().createBlob(is, messageRecord.getAttachmentStreamSize()));
+            int attachmentNo = 0;
+            for (AttachmentStream attachmentStream : messageRecord.getAttachmentStreams()) {
+                attachmentNo++;
+                messageRecord.addAttachment(attachmentNo,   // attachment numbering starts from one as in asic container
+                        session.getLobHelper().createBlob(attachmentStream.getStream(), attachmentStream.getSize()));
             }
 
             save(session, messageRecord);
@@ -168,7 +154,7 @@ static void saveMessageRecord(MessageRecord messageRecord) throws Exception {
     @SuppressWarnings("JpaQlInspection")
     static void updateMessageRecordSignature(MessageRecord messageRecord, String oldHash) throws Exception {
         doInTransaction(session -> {
-            final Query<?> query = session.createQuery("update MessageRecord m "
+            final MutationQuery query = session.createMutationQuery("update MessageRecord m "
                     + "set m.signature = :signature, m.signatureHash = :hash "
                     + "where m.id = :id and m.timestampRecord is null and m.signatureHash = :oldhash");
 
@@ -207,11 +193,11 @@ static void saveTimestampRecord(TimestampRecord timestampRecord, Long[]
      */
     static void save(Session session, LogRecord logRecord) {
         log.trace("save({})", logRecord.getClass());
-        session.save(logRecord);
+        session.persist(logRecord);
     }
 
     static long getNextRecordId(Session session) {
-        return ((Number) session.createNativeQuery("SELECT nextval('logrecord_sequence')").getSingleResult()).longValue();
+        return session.createNativeQuery("SELECT nextval('logrecord_sequence')", Long.class).getSingleResult();
     }
 
     /**
@@ -276,19 +262,6 @@ private static LogRecord getLogRecord(Session session, Long number) {
         return session.get(AbstractLogRecord.class, number);
     }
 
-    private static MessageRecord getMessageRecord(Session session, String queryId, Date startTime, Date endTime) {
-        final CriteriaBuilder cb = session.getCriteriaBuilder();
-        final CriteriaQuery<MessageRecord> query = cb.createQuery(MessageRecord.class);
-        final Root<MessageRecord> m = query.from(MessageRecord.class);
-
-        query.select(m)
-                .where(cb.and(
-                        cb.equal(m.get("queryId"), queryId),
-                        cb.between(m.get("time"), startTime.getTime(), endTime.getTime())
-                ));
-        return session.createQuery(query).setMaxResults(1).uniqueResult();
-    }
-
     private static MessageRecord getMessageRecord(Session session, String queryId, ClientId clientId,
                                                   Boolean isResponse) {
         final CriteriaQuery<MessageRecord> query = createRecordCriteria(session, queryId, clientId, isResponse);

src/addons/messagelog/messagelog-addon/src/test/java/ee/ria/xroad/proxy/messagelog/AbstractMessageLogTest.java

@@ -29,6 +29,7 @@
 import ee.ria.xroad.common.cert.CertChainFactory;
 import ee.ria.xroad.common.conf.globalconf.GlobalConfProvider;
 import ee.ria.xroad.common.conf.serverconf.ServerConfProvider;
+import ee.ria.xroad.common.message.AttachmentStream;
 import ee.ria.xroad.common.message.RestRequest;
 import ee.ria.xroad.common.message.SoapMessageImpl;
 import ee.ria.xroad.common.messagelog.AbstractLogManager;
@@ -52,6 +53,7 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.util.List;
 
 import static ee.ria.xroad.proxy.messagelog.TestUtil.getGlobalConf;
 import static ee.ria.xroad.proxy.messagelog.TestUtil.getServerConf;
@@ -129,7 +131,13 @@ private void signalTimestampingStatus(SetTimestampingStatusMessage.Status status
     }
 
     protected void log(SoapMessageImpl message, SignatureData signature) throws Exception {
-        logManager.log(new SoapLogMessage(message, signature, true));
+        log(message, signature, List.of());
+    }
+
+    protected void log(SoapMessageImpl message, SignatureData signature, List<byte[]> attachments) throws Exception {
+        var attachmentStreamList = attachments.stream()
+                .map(attachment -> AttachmentStream.fromInputStream(new ByteArrayInputStream(attachment), attachment.length)).toList();
+        logManager.log(new SoapLogMessage(message, signature, attachmentStreamList, true, message.getQueryId()));
     }
 
     protected void log(RestRequest message, SignatureData signatureData, byte[] body)

src/addons/messagelog/messagelog-addon/src/test/java/ee/ria/xroad/proxy/messagelog/MessageBodyManipulatorTest.java

@@ -196,7 +196,7 @@ private void assertNodeEmptinessAfterManipulation(SoapMessageImpl query,
                                                       String elementName,
                                                       boolean keepBody) throws Exception {
         String loggableMessage = new TestableMessageBodyManipulator(keepBody)
-                .getLoggableMessageText(new SoapLogMessage(query, null, clientSide));
+                .getLoggableMessageText(new SoapLogMessage(query, null, List.of(), clientSide, null));
         log.debug("loggable message with body"
                 + (keepBody ? " intact: " : " removed: ")
                 + loggableMessage);