JSON Validation parameter substitution deeper search (#8128)

core/src/main/java/org/frankframework/align/DomTreeAligner.java

@@ -27,6 +27,7 @@
 
 import org.apache.xerces.xs.XSElementDeclaration;
 import org.apache.xerces.xs.XSModel;
+import org.apache.xerces.xs.XSParticle;
 import org.w3c.dom.Document;
 import org.w3c.dom.NamedNodeMap;
 import org.w3c.dom.Node;
@@ -55,7 +56,7 @@ boolean isEmptyNode(Node node) {
 	}
 
 	@Override
-	Node filterNodeChildren(Node node, Set<String> allowedNames) {
+	Node filterNodeChildren(Node node, List<XSParticle> allowedChildren) {
 		// Dummy implementation, since this is basically dead code / deep search is not relevant to this class.
 		return node;
 	}

core/src/main/java/org/frankframework/align/Json2Xml.java

@@ -18,6 +18,7 @@
 import java.io.StringReader;
 import java.net.URL;
 import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
 import java.util.LinkedList;
@@ -41,8 +42,15 @@
 import lombok.Setter;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.builder.ToStringBuilder;
+import org.apache.xerces.xs.XSComplexTypeDefinition;
 import org.apache.xerces.xs.XSElementDeclaration;
 import org.apache.xerces.xs.XSModel;
+import org.apache.xerces.xs.XSModelGroup;
+import org.apache.xerces.xs.XSObjectList;
+import org.apache.xerces.xs.XSParticle;
+import org.apache.xerces.xs.XSTerm;
+import org.apache.xerces.xs.XSTypeDefinition;
+import org.apache.xerces.xs.XSWildcard;
 import org.xml.sax.SAXException;
 
 /**
@@ -54,6 +62,7 @@ public class Json2Xml extends Tree2Xml<JsonValue,JsonValue> {
 
 	public static final String MSG_FULL_INPUT_IN_STRICT_COMPACTING_MODE="straight json found while expecting compact arrays and strict syntax checking";
 	public static final String MSG_EXPECTED_SINGLE_ELEMENT="did not expect array, but single element";
+	public static final String XSD_WILDCARD_ELEMENT_TOKEN = "*";
 
 	private boolean insertElementContainerElements;
 	private boolean strictSyntax;
@@ -346,22 +355,26 @@ protected boolean isEmptyNode(JsonValue node) {
 	}
 
 	/**
-	 * Create a copy of the JSON node that contains only keys from the allowedNames set in the top level.
+	 * Create a copy of the JSON node that contains only keys from the allowedChildren set in the top level.
 	 *
 	 * @param node Node to copy
-	 * @param allowedNames Names of child-nodes to keep in the copy
+	 * @param allowedChildren Names of child-nodes to keep in the copy
 	 * @return Copy of the JSON node.
 	 */
 	@Override
-	protected JsonValue filterNodeChildren(JsonValue node, Set<String> allowedNames) {
+	protected JsonValue filterNodeChildren(JsonValue node, List<XSParticle> allowedChildren) {
 		if (node instanceof JsonArray) {
-			return copyJsonArray((JsonArray)node, allowedNames);
+			return copyJsonArray((JsonArray)node, allowedChildren);
 		} else if (node instanceof JsonObject) {
-			return copyJsonObject((JsonObject)node, allowedNames);
+			return copyJsonObject((JsonObject)node, allowedChildren);
 		} else return node;
 	}
 
-	private JsonValue copyJsonObject(JsonObject node, Set<String> allowedNames) {
+	private JsonValue copyJsonObject(JsonObject node, List<XSParticle> allowedChildren) {
+		Set<String> allowedNames = allowedChildren
+				.stream()
+				.map(p -> p.getTerm().getName())
+				.collect(Collectors.toSet());
 		JsonObjectBuilder objectBuilder = Json.createObjectBuilder();
 		node.forEach((key, value) -> {
 			if (allowedNames.contains(key)) objectBuilder.add(key, value);
@@ -371,19 +384,57 @@ private JsonValue copyJsonObject(JsonObject node, Set<String> allowedNames) {
 		// This is perhaps not the cleanest way to make sure the substitutions are performed but this requires the least
 		// amount of code changes in other parts.
 		if (sp != null) {
-			allowedNames.forEach(name -> {
+			allowedChildren.forEach(childParticle -> {
+				String name = childParticle.getTerm().getName();
 				if (!node.containsKey(name) && sp.hasSubstitutionsFor(getContext(), name)) {
 					objectBuilder.add(name, getSubstitutedChild(node, name));
+				} else if (hasSubstitutionForChild(childParticle)) {
+					// A deeper child-node does have a substitution for this element, so add an empty object for it to further parse at later stage.
+					objectBuilder.add(name, Json.createObjectBuilder().build());
 				}
 			});
 		}
 		return objectBuilder.build();
 	}
 
-	private JsonValue copyJsonArray(JsonArray node, Set<String> allowedNames) {
+	private boolean hasSubstitutionForChild(XSParticle childParticle) {
+		// Find a recursive list of all child-names of this type to see if any of these names has a substitution from parameters
+		Set<String> names = new HashSet<>();
+		getChildElementNamesRecursive(childParticle, names, new HashSet<>());
+		return names.contains(XSD_WILDCARD_ELEMENT_TOKEN) || names.stream().anyMatch(childName -> sp.hasSubstitutionsFor(getContext(), childName));
+	}
+
+	private void getChildElementNamesRecursive(XSParticle particle, Set<String> names, Set<XSParticle> visitedTypes) {
+		XSTerm term = particle.getTerm();
+		names.add(term.getName());
+		if (visitedTypes.contains(particle)) {
+			return;
+		}
+		visitedTypes.add(particle);
+		if (term instanceof XSModelGroup) {
+			XSObjectList modelGroupParticles = ((XSModelGroup)term).getParticles();
+			for (Object childObject : modelGroupParticles) {
+				XSParticle childParticle = (XSParticle) childObject;
+				getChildElementNamesRecursive(childParticle, names, visitedTypes);
+			}
+		} else if (term instanceof XSElementDeclaration) {
+			XSTypeDefinition typeDefinition = ((XSElementDeclaration)term).getTypeDefinition();
+			if (typeDefinition.getTypeCategory()!=XSTypeDefinition.SIMPLE_TYPE) {
+				XSComplexTypeDefinition complexTypeDefinition = (XSComplexTypeDefinition) typeDefinition;
+				getChildElementNamesRecursive(complexTypeDefinition.getParticle(), names, visitedTypes);
+			}
+		} else if (term instanceof XSWildcard) {
+			XSWildcard wildcard = (XSWildcard)term;
+			log.debug("XSD contains wildcard element [{}], constraint [{}]/[{}]", term.getName(), wildcard.getConstraintType(), wildcard.getNsConstraintList());
+			// TODO: Not sure what to do here to realistically restrict possible child-elements and I'm afraid it can balloon into a lot of unneeded code.
+			names.add(XSD_WILDCARD_ELEMENT_TOKEN);
+		}
+	}
+
+	private JsonValue copyJsonArray(JsonArray node, List<XSParticle> allowedChildren) {
 		JsonArrayBuilder arrayBuilder = Json.createArrayBuilder();
 		node.forEach(value -> {
-			arrayBuilder.add(filterNodeChildren(value, allowedNames));
+			arrayBuilder.add(filterNodeChildren(value, allowedChildren));
 		});
 		return arrayBuilder.build();
 	}

core/src/main/java/org/frankframework/align/Properties2Xml.java

@@ -21,12 +21,12 @@
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 
 import javax.xml.validation.ValidatorHandler;
 
 import org.apache.xerces.xs.XSElementDeclaration;
 import org.apache.xerces.xs.XSModel;
+import org.apache.xerces.xs.XSParticle;
 import org.frankframework.align.Properties2Xml.PropertyNode;
 import org.xml.sax.SAXException;
 
@@ -60,7 +60,7 @@ public void startParse(Map<String, String> root) throws SAXException {
 	}
 
 	@Override
-	PropertyNode filterNodeChildren(PropertyNode node, Set<String> allowedNames) {
+	PropertyNode filterNodeChildren(PropertyNode node, List<XSParticle> allowedChildren) {
 		// Dummy-implementation since this is basically unused code
 		return node;
 	}

core/src/main/java/org/frankframework/align/ToXml.java

@@ -417,10 +417,12 @@ private boolean tryDeepSearchForChildElement(XSElementDeclaration childElementDe
 			return false;
 		}
 		XSComplexTypeDefinition complexTypeDefinition = (XSComplexTypeDefinition) typeDefinition;
-		Set<String> allowedNames = getNamesOfXsdChildElements(complexTypeDefinition);
-		allowedNames.removeAll(processedChildren);
+		List<XSParticle> allowedParticles = getXsdChildParticles(complexTypeDefinition).stream()
+				.filter(p -> !processedChildren.contains(p.getTerm().getName()))
+				.collect(Collectors.toList());
 
-		N copy = filterNodeChildren(node, allowedNames);
+
+		N copy = filterNodeChildren(node, allowedParticles);
 
 		if (isEmptyNode(copy) && !mandatory) {
 			return false;
@@ -429,17 +431,13 @@ private boolean tryDeepSearchForChildElement(XSElementDeclaration childElementDe
 		return true;
 	}
 
-	private static Set<String> getNamesOfXsdChildElements(XSComplexTypeDefinition complexTypeDefinition) {
+	private static List<XSParticle> getXsdChildParticles(XSComplexTypeDefinition complexTypeDefinition) {
 		XSTerm term = complexTypeDefinition.getParticle().getTerm();
-		if (!(term instanceof XSModelGroup)) {
-			return Collections.emptySet();
-		}
-		XSModelGroup modelGroup = (XSModelGroup) term;
-		@SuppressWarnings("unchecked")
-		List<XSParticle> particles = modelGroup.getParticles();
-		return particles.stream()
-				.map(p -> p.getTerm().getName())
-				.collect(Collectors.toSet());
+		if (term instanceof XSModelGroup) {
+			//noinspection unchecked
+			return ((XSModelGroup)term).getParticles();
+		}
+		return Collections.emptyList();
 	}
 
 	/**
@@ -456,7 +454,7 @@ private static Set<String> getNamesOfXsdChildElements(XSComplexTypeDefinition co
 	 * @param allowedNames Names of child-nodes to keep in the copy
 	 * @return Copy of the node
 	 */
-	abstract N filterNodeChildren(N node, Set<String> allowedNames);
+	abstract N filterNodeChildren(N node, List<XSParticle> allowedChildren);
 
 	public List<XSParticle> getBestChildElementPath(XSElementDeclaration elementDeclaration, N node, boolean silent) throws SAXException {
 		XSTypeDefinition typeDefinition = elementDeclaration.getTypeDefinition();

core/src/test/java/org/frankframework/pipes/Json2XmlValidatorTest.java

@@ -1,6 +1,7 @@
 package org.frankframework.pipes;
 
 import static org.frankframework.testutil.MatchUtils.assertXmlEquals;
+import static org.frankframework.testutil.TestAssertions.assertEqualsIgnoreWhitespaces;
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
@@ -752,4 +753,35 @@ public void issue7146AttributesOnMultipleLevels(boolean deepSearch, String input
 		String expected = TestFileUtils.getTestFile("/Validation/AttributesOnDifferentLevels/output-" + input + ".xml");
 		assertXmlEquals(expected, result.getResult().asString());
 	}
+
+	@Test
+	public void testExpandParameters() throws Exception {
+		// Arrange
+		pipe.setName("testExpandParameters");
+		pipe.setSchema("/Validation/Json2Xml/ParameterSubstitution/Main.xsd");
+		pipe.setThrowException(true);
+		pipe.setOutputFormat(DocumentFormat.JSON);
+		pipe.setRoot("GetDocumentAttributes_Error");
+		pipe.setDeepSearch(true);
+
+		pipe.addParameter(new Parameter("type", "/errors/"));
+		pipe.addParameter(ParameterBuilder.create().withName("title").withSessionKey("errorReason"));
+		pipe.addParameter(ParameterBuilder.create().withName("status").withSessionKey("errorCode"));
+		pipe.addParameter(ParameterBuilder.create().withName("detail").withSessionKey("errorDetailText"));
+		pipe.addParameter(new Parameter("instance", "/archiving/documents"));
+
+		pipe.configure();
+		pipe.start();
+
+		session.put("errorReason", "More than one document found");
+		session.put("errorCode", "DATA_ERROR");
+		session.put("errorDetailText", "The Devil's In The Details");
+
+		// Act
+		PipeRunResult result = pipe.doPipe(Message.asMessage("{}"), session);
+
+		// Assert
+		String expectedResult = TestFileUtils.getTestFile("/Validation/Json2Xml/ParameterSubstitution/expected_output.json");
+		assertEqualsIgnoreWhitespaces(expectedResult, result.getResult().asString());
+	}
 }

core/src/test/resources/Validation/Json2Xml/ParameterSubstitution/CommonResponse.xsd

@@ -0,0 +1,29 @@
+<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema" elementFormDefault="qualified" attributeFormDefault="unqualified">
+	<xs:complexType name="links">
+		<xs:sequence>
+			<xs:element name="link" type="link" maxOccurs="unbounded"/>
+		</xs:sequence>
+	</xs:complexType>
+		<xs:complexType name="errors">
+		<xs:sequence>
+			<xs:element name="error" type="error" maxOccurs="unbounded"/>
+		</xs:sequence>
+	</xs:complexType>
+	<xs:complexType name="link">
+		<xs:sequence>
+			<xs:element name="href" type="xs:anyURI" minOccurs="1" maxOccurs="1"/>
+			<xs:element name="rel" type="xs:string" minOccurs="1" maxOccurs="1"/>
+			<xs:element name="type" type="xs:string" minOccurs="1" maxOccurs="1"/>
+			<xs:element name="title" type="xs:string" minOccurs="0"/>
+		</xs:sequence>
+	</xs:complexType>
+	<xs:complexType name="error">
+		<xs:sequence>
+			<xs:element name="type" type="xs:string" minOccurs="1" maxOccurs="1"/>
+			<xs:element name="title" type="xs:string" minOccurs="1" maxOccurs="1"/>
+			<xs:element name="status" type="xs:string" minOccurs="1" maxOccurs="1"/>
+			<xs:element name="detail" type="xs:string" minOccurs="0" maxOccurs="1"/>
+			<xs:element name="instance" type="xs:anyURI" minOccurs="1" maxOccurs="1"/>
+		</xs:sequence>
+	</xs:complexType>
+</xs:schema>

core/src/test/resources/Validation/Json2Xml/ParameterSubstitution/Main.xsd

@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<xsd:schema xmlns:xsd="http://www.w3.org/2001/XMLSchema" elementFormDefault="qualified" attributeFormDefault="unqualified">
+	<xsd:include schemaLocation="CommonResponse.xsd"/>
+	<xsd:element name="GetDocumentAttributes_Error">
+		<xsd:complexType>
+			<xsd:sequence>
+				<xsd:element name="errors" type="errors" minOccurs="0" maxOccurs="unbounded"/>
+			</xsd:sequence>
+		</xsd:complexType>
+	</xsd:element>
+</xsd:schema>

core/src/test/resources/Validation/Json2Xml/ParameterSubstitution/expected_output.json

@@ -0,0 +1,11 @@
+[
+	[
+		{
+			"type": "/errors/",
+			"title": "More than one document found",
+			"status": "DATA_ERROR",
+			"detail": "The Devil's In The Details",
+			"instance": "/archiving/documents"
+		}
+	]
+]