From f09740a11ef41c991df83d12febd8ec08402fd06 Mon Sep 17 00:00:00 2001 From: fdutton Date: Mon, 22 May 2023 23:43:04 -0400 Subject: [PATCH] Resolves improper anchoring of patternProperties (#783) * Resolves improper anchoring of patternProperties Fixes #782 * Resolves improper anchoring of regular expressions in both Joni and JDK engines. Resolves #495 and #782 --------- Co-authored-by: Faron Dutton --- doc/compatibility.md | 147 ++++++++++-------- .../schema/regex/JDKRegularExpression.java | 8 +- .../schema/AbstractJsonSchemaTestSuite.java | 20 +-- .../java/com/networknt/schema/TestSpec.java | 18 ++- src/test/resources/draft2020-12/issue495.json | 64 ++++++++ src/test/resources/draft2020-12/issue782.json | 92 +++++++++++ 6 files changed, 274 insertions(+), 75 deletions(-) create mode 100644 src/test/resources/draft2020-12/issue495.json create mode 100644 src/test/resources/draft2020-12/issue782.json diff --git a/doc/compatibility.md b/doc/compatibility.md index 6f01eb7cb..5ee2b31c5 100644 --- a/doc/compatibility.md +++ b/doc/compatibility.md @@ -1,76 +1,97 @@ ### Legend -Symbol | Meaning | -:-----:|---------| -🟢 | Fully implemented -🟡 | Partially implemented -🔴 | Not implemented -🚫 | Not defined in Schema Version. +| Symbol | Meaning | +|:------:|:----------------------| +| 🟢 | Fully implemented | +| 🟡 | Partially implemented | +| 🔴 | Not implemented | +| 🚫 | Not defined | ### Compatibility with JSON Schema versions - Validation Keyword/Schema | Draft 4 | Draft 6 | Draft 7 | Draft 2019-09 | ----------------- |:--------------:|:-------: |:-------: |:-------------:| -$ref | 🟢 | 🟢 | 🟢 | 🟢 -additionalProperties | 🟢 | 🟢 | 🟢 | 🟢 -additionalItems | 🟢 | 🟢 | 🟢 | 🟢 -allOf | 🟢 | 🟢 | 🟢 | 🟢 -anyOf | 🟢 | 🟢 | 🟢 | 🟢 -const | 🚫 | 🟢 | 🟢 | 🟢 -contains | 🚫 | 🟢 | 🟢 | 🟢 -contentEncoding | 🚫 | 🚫 | 🔴 | 🔴 -contentMediaType | 🚫 | 🚫 | 🔴 | 🔴 -dependencies | 🟢 | 🟢 |🟢 | 🟢 -enum | 🟢 | 🟢 | 🟢 | 🟢 -exclusiveMaximum (boolean) | 🟢 | 🚫 | 🚫 | 🚫 -exclusiveMaximum (numeric) | 🚫 | 🟢 | 🟢 | 🟢 -exclusiveMinimum (boolean) | 🟢 | 🚫 | 🚫 | 🚫 -exclusiveMinimum (numeric) | 🚫 | 🟢 | 🟢 | 🟢 -items | 🟢 | 🟢 | 🟢 | 🟢 -maximum | 🟢 | 🟢 | 🟢 | 🟢 -maxItems | 🟢 | 🟢 | 🟢 | 🟢 -maxLength | 🟢 | 🟢 | 🟢 | 🟢 -maxProperties | 🟢 | 🟢 | 🟢 | 🟢 -minimum | 🟢 | 🟢 | 🟢 | 🟢 -minItems | 🟢 | 🟢 | 🟢 | 🟢 -minLength | 🟢 | 🟢 | 🟢 | 🟢 -minProperties | 🟢 | 🟢 | 🟢 | 🟢 -multipleOf | 🟢 | 🟢 | 🟢 | 🟢 -not | 🟢 | 🟢 | 🟢 | 🟢 -oneOf | 🟢 | 🟢 | 🟢 | 🟢 -pattern | 🟢 | 🟢 | 🟢 | 🟢 -patternProperties | 🟢 | 🟢 | 🟢 | 🟢 -properties | 🟢 | 🟢 | 🟢 | 🟢 -propertyNames | 🚫 | 🔴 | 🔴 | 🔴 -required | 🟢 | 🟢 | 🟢 | 🟢 -type | 🟢 | 🟢 | 🟢 | 🟢 -uniqueItems | 🟢 | 🟢 | 🟢 | 🟢 +| Keyword | Draft 4 | Draft 6 | Draft 7 | Draft 2019-09 | Draft 2020-12 | +|:---------------------------|:-------:|:-------:|:-------:|:-------------:|:-------------:| +| $anchor | 🚫 | 🚫 | 🚫 | 🔴 | 🔴 | +| $dynamicAnchor | 🚫 | 🚫 | 🚫 | 🚫 | 🔴 | +| $dynamicRef | 🚫 | 🚫 | 🚫 | 🚫 | 🔴 | +| $id | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | +| $recursiveAnchor | 🚫 | 🚫 | 🚫 | 🔴 | 🔴 | +| $recursiveRef | 🚫 | 🚫 | 🚫 | 🔴 | 🔴 | +| $ref | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | +| $vocabulary | 🚫 | 🚫 | 🚫 | 🔴 | 🔴 | +| additionalItems | 🟢 | 🟢 | 🟢 | 🟢 | 🟢 | +| additionalProperties | 🟢 | 🟢 | 🟢 | 🟢 | 🟢 | +| allOf | 🟢 | 🟢 | 🟢 | 🟢 | 🟢 | +| anyOf | 🟢 | 🟢 | 🟢 | 🟢 | 🟢 | +| const | 🚫 | 🟢 | 🟢 | 🟢 | 🟢 | +| contains | 🚫 | 🟢 | 🟢 | 🟢 | 🟢 | +| contentEncoding | 🚫 | 🚫 | 🔴 | 🔴 | 🔴 | +| contentMediaType | 🚫 | 🚫 | 🔴 | 🔴 | 🔴 | +| contentSchema | 🚫 | 🚫 | 🚫 | 🔴 | 🔴 | +| definitions | 🟢 | 🟢 | 🟢 | 🚫 | 🚫 | +| defs | 🚫 | 🚫 | 🚫 | 🟢 | 🟢 | +| dependencies | 🟢 | 🟢 | 🟢 | 🚫 | 🚫 | +| dependentRequired | 🚫 | 🚫 | 🚫 | 🟢 | 🟢 | +| dependentSchemas | 🚫 | 🚫 | 🚫 | 🟢 | 🟢 | +| enum | 🟢 | 🟢 | 🟢 | 🟢 | 🟢 | +| exclusiveMaximum (boolean) | 🟢 | 🚫 | 🚫 | 🚫 | 🚫 | +| exclusiveMaximum (numeric) | 🚫 | 🟢 | 🟢 | 🟢 | 🟢 | +| exclusiveMinimum (boolean) | 🟢 | 🚫 | 🚫 | 🚫 | 🚫 | +| exclusiveMinimum (numeric) | 🚫 | 🟢 | 🟢 | 🟢 | 🟢 | +| if-then-else | 🚫 | 🚫 | 🟢 | 🟢 | 🟢 | +| items | 🟢 | 🟢 | 🟢 | 🟢 | 🟢 | +| maxContains | 🚫 | 🚫 | 🚫 | 🟢 | 🟢 | +| minContains | 🚫 | 🚫 | 🚫 | 🟢 | 🟢 | +| maximum | 🟢 | 🟢 | 🟢 | 🟢 | 🟢 | +| maxItems | 🟢 | 🟢 | 🟢 | 🟢 | 🟢 | +| maxLength | 🟢 | 🟢 | 🟢 | 🟢 | 🟢 | +| maxProperties | 🟢 | 🟢 | 🟢 | 🟢 | 🟢 | +| minimum | 🟢 | 🟢 | 🟢 | 🟢 | 🟢 | +| minItems | 🟢 | 🟢 | 🟢 | 🟢 | 🟢 | +| minLength | 🟢 | 🟢 | 🟢 | 🟢 | 🟢 | +| minProperties | 🟢 | 🟢 | 🟢 | 🟢 | 🟢 | +| multipleOf | 🟢 | 🟢 | 🟢 | 🟢 | 🟢 | +| not | 🟢 | 🟢 | 🟢 | 🟢 | 🟢 | +| oneOf | 🟢 | 🟢 | 🟢 | 🟢 | 🟢 | +| pattern | 🟢 | 🟢 | 🟢 | 🟢 | 🟢 | +| patternProperties | 🟢 | 🟢 | 🟢 | 🟢 | 🟢 | +| prefixItems | 🚫 | 🚫 | 🚫 | 🚫 | 🟢 | +| properties | 🟢 | 🟢 | 🟢 | 🟢 | 🟢 | +| propertyNames | 🚫 | 🟢 | 🟢 | 🟢 | 🟢 | +| readOnly | 🚫 | 🚫 | 🔴 | 🔴 | 🔴 | +| required | 🟢 | 🟢 | 🟢 | 🟢 | 🟢 | +| type | 🟢 | 🟢 | 🟢 | 🟢 | 🟢 | +| unevaluatedItems | 🚫 | 🚫 | 🚫 | 🟢 | 🟢 | +| unevaluatedProperties | 🚫 | 🚫 | 🚫 | 🟢 | 🟢 | +| uniqueItems | 🟢 | 🟢 | 🟢 | 🟢 | 🟢 | +| writeOnly | 🚫 | 🚫 | 🔴 | 🔴 | 🔴 | ### Semantic Validation (Format) -Format | Draft 4 | Draft 6 | Draft 7 | Draft 2019-09 | --------|---------|---------|---------|---------------| -date |🚫 | 🚫 | 🟢 | 🟢 -date-time | 🟢 | 🟢 | 🟢 | 🟢 -duration | 🚫 | 🚫 | 🔴 | 🔴 -email | 🟢 | 🟢 | 🟢 | 🟢 -hostname | 🟢 | 🟢 | 🟢 | 🟢 -idn-email | 🚫 | 🚫 | 🔴 | 🔴 -idn-hostname | 🚫 | 🚫 | 🔴 | 🔴 -ipv4 | 🟢 | 🟢 | 🟢 | 🟢 -ipv6 | 🟢 | 🟢 | 🟢 | 🟢 -iri | 🚫 | 🚫 | 🔴 | 🔴 -iri-reference | 🚫 | 🚫 | 🔴 | 🔴 -json-pointer | 🚫 | 🔴 | 🔴 | 🔴 -relative-json-pointer | 🚫 | 🔴 | 🔴 | 🔴 -regex | 🚫 | 🚫 | 🔴 | 🔴 -time | 🚫 | 🚫 | 🟢 | 🟢 -uri | 🟢 | 🟢 | 🟢 | 🟢 -uri-reference | 🚫 | 🔴 | 🔴 | 🔴 -uri-template | 🚫 | 🔴 | 🔴 | 🔴 -uuid | 🚫 | 🚫 | 🟢 | 🟢 +| Format | Draft 4 | Draft 6 | Draft 7 | Draft 2019-09 | Draft 2020-12 | +|:----------------------|:-------:|:-------:|:-------:|:-------------:|:-------------:| +| date | 🚫 | 🚫 | 🟢 | 🟢 | 🟢 | +| date-time | 🟢 | 🟢 | 🟢 | 🟢 | 🟢 | +| duration | 🚫 | 🚫 | 🚫 | 🟢 | 🟢 | +| email | 🟢 | 🟢 | 🟢 | 🟢 | 🟢 | +| hostname | 🟢 | 🟢 | 🟢 | 🟢 | 🟢 | +| idn-email | 🚫 | 🚫 | 🟢 | 🟢 | 🟢 | +| idn-hostname | 🚫 | 🚫 | 🟢 | 🟢 | 🟢 | +| ipv4 | 🟢 | 🟢 | 🟢 | 🟢 | 🟢 | +| ipv6 | 🟢 | 🟢 | 🟢 | 🟢 | 🟢 | +| iri | 🚫 | 🚫 | 🟢 | 🟢 | 🟢 | +| iri-reference | 🚫 | 🚫 | 🟢 | 🟢 | 🟢 | +| json-pointer | 🚫 | 🟢 | 🟢 | 🟢 | 🟢 | +| relative-json-pointer | 🚫 | 🟢 | 🟢 | 🟢 | 🟢 | +| regex | 🚫 | 🚫 | 🟢 | 🟢 | 🟢 | +| time | 🚫 | 🚫 | 🟢 | 🟢 | 🟢 | +| uri | 🟢 | 🟢 | 🟢 | 🟢 | 🟢 | +| uri-reference | 🚫 | 🟢 | 🟢 | 🟢 | 🟢 | +| uri-template | 🚫 | 🟢 | 🟢 | 🟢 | 🟢 | +| uuid | 🚫 | 🚫 | 🟢 | 🟢 | 🟢 | ### Footnotes 1. Note that the validation are only optional for some of the keywords/formats. 2. Refer to the corresponding JSON schema for more information on whether the keyword/format is optional or not. + diff --git a/src/main/java/com/networknt/schema/regex/JDKRegularExpression.java b/src/main/java/com/networknt/schema/regex/JDKRegularExpression.java index e20c821db..36f4ebb74 100644 --- a/src/main/java/com/networknt/schema/regex/JDKRegularExpression.java +++ b/src/main/java/com/networknt/schema/regex/JDKRegularExpression.java @@ -1,17 +1,23 @@ package com.networknt.schema.regex; +import java.util.regex.Matcher; import java.util.regex.Pattern; class JDKRegularExpression implements RegularExpression { private final Pattern pattern; + private final boolean hasStartAnchor; + private final boolean hasEndAnchor; JDKRegularExpression(String regex) { this.pattern = Pattern.compile(regex); + this.hasStartAnchor = '^' == regex.charAt(0); + this.hasEndAnchor = '$' == regex.charAt(regex.length() - 1); } @Override public boolean matches(String value) { - return this.pattern.matcher(value).matches(); + Matcher matcher = this.pattern.matcher(value); + return matcher.find() && (!this.hasStartAnchor || 0 == matcher.start()) && (!this.hasEndAnchor || matcher.end() == value.length()); } } \ No newline at end of file diff --git a/src/test/java/com/networknt/schema/AbstractJsonSchemaTestSuite.java b/src/test/java/com/networknt/schema/AbstractJsonSchemaTestSuite.java index 213ec2547..a2a4be88c 100644 --- a/src/test/java/com/networknt/schema/AbstractJsonSchemaTestSuite.java +++ b/src/test/java/com/networknt/schema/AbstractJsonSchemaTestSuite.java @@ -52,7 +52,7 @@ import static org.junit.jupiter.api.DynamicTest.dynamicTest; public abstract class AbstractJsonSchemaTestSuite extends HTTPServiceSupport { - protected static final TypeReference> testCaseType = new TypeReference>() {}; + protected static final TypeReference> testCaseType = new TypeReference>() { /* intentionally empty */}; protected static final Map supportedVersions = new HashMap<>(); static { supportedVersions.put("draft2019-09", VersionFlag.V201909); @@ -99,7 +99,7 @@ private DynamicNode buildContainer(VersionFlag defaultVersion, TestCase testCase String msg = e.getMessage(); if (msg.endsWith("' is unrecognizable schema")) { return dynamicContainer(testCase.getDisplayName(), unsupportedMetaSchema(testCase)); - }; + } throw e; } } @@ -109,7 +109,7 @@ private JsonSchemaFactory buildValidatorFactory(VersionFlag defaultVersion, Test JsonSchemaFactory base = JsonSchemaFactory.getInstance(specVersion); return JsonSchemaFactory .builder(base) - .objectMapper(mapper) + .objectMapper(this.mapper) .addUriTranslator(URITranslator.combine( URITranslator.prefix("https://", "http://"), URITranslator.prefix("http://json-schema.org", "resource:") @@ -124,7 +124,7 @@ private DynamicNode buildTest(JsonSchemaFactory validatorFactory, TestSpec testS SchemaValidatorsConfig config = new SchemaValidatorsConfig(); config.setTypeLoose(typeLoose); - config.setEcma262Validator(true); + config.setEcma262Validator(TestSpec.RegexKind.JDK != testSpec.getRegex()); testSpec.getStrictness().forEach(config::setStrict); URI testCaseFileUri = URI.create("classpath:" + toForwardSlashPath(testSpec.getTestCase().getSpecification())); JsonSchema schema = validatorFactory.getSchema(testCaseFileUri, testSpec.getTestCase().getSchema(), config); @@ -132,13 +132,13 @@ private DynamicNode buildTest(JsonSchemaFactory validatorFactory, TestSpec testS return dynamicTest(testSpec.getDescription(), () -> executeAndReset(schema, testSpec)); } - private String toForwardSlashPath(Path file) { + private static String toForwardSlashPath(Path file) { return file.toString().replace('\\', '/'); } // For 2019-09 and later published drafts, implementations that are able to // detect the draft of each schema via $schema SHOULD be configured to do so - private VersionFlag detectVersion(TestCase testCase, VersionFlag defaultVersion) { + private static VersionFlag detectVersion(TestCase testCase, VersionFlag defaultVersion) { return Stream.of( detectOptionalVersion(testCase.getSchema()), detectVersionFromPath(testCase.getSpecification()) @@ -152,7 +152,7 @@ private VersionFlag detectVersion(TestCase testCase, VersionFlag defaultVersion) // For draft-07 and earlier, draft-next, and implementations unable to // detect via $schema, implementations MUST be configured to expect the // draft matching the test directory name - private Optional detectVersionFromPath(Path path) { + private static Optional detectVersionFromPath(Path path) { return StreamSupport.stream(path.spliterator(), false) .map(Path::toString) .map(supportedVersions::get) @@ -168,7 +168,7 @@ private void executeAndReset(JsonSchema schema, TestSpec testSpec) { } } - private void executeTest(JsonSchema schema, TestSpec testSpec) { + private static void executeTest(JsonSchema schema, TestSpec testSpec) { Set errors = schema.validate(testSpec.getData()); if (testSpec.isValid()) { @@ -246,7 +246,7 @@ private List findTestCases(String basePath) { private Stream loadTestCases(Path testCaseFile) { try (InputStream in = new FileInputStream(testCaseFile.toFile())) { - return mapper.readValue(in, testCaseType) + return this.mapper.readValue(in, testCaseType) .stream() .peek(testCase -> testCase.setSpecification(testCaseFile)) .filter(this::enabled); @@ -258,7 +258,7 @@ private Stream loadTestCases(Path testCaseFile) { } } - private Iterable unsupportedMetaSchema(TestCase testCase) { + private static Iterable unsupportedMetaSchema(TestCase testCase) { return Collections.singleton( dynamicTest("Detected an unsupported schema", () -> { String schema = testCase.getSchema().asText(); diff --git a/src/test/java/com/networknt/schema/TestSpec.java b/src/test/java/com/networknt/schema/TestSpec.java index 520c7c937..5bdd2a9d5 100644 --- a/src/test/java/com/networknt/schema/TestSpec.java +++ b/src/test/java/com/networknt/schema/TestSpec.java @@ -81,6 +81,11 @@ public class TestSpec { */ private final boolean typeLoose; + /** + * Identifies the regular expression engine to use for this test-case. + */ + private final RegexKind regex; + /** * The TestCase that contains this TestSpec. */ @@ -107,7 +112,8 @@ public TestSpec( @JsonProperty("strictness") Map strictness, @JsonProperty("validationMessages") Set validationMessages, @JsonProperty("isTypeLoose") Boolean isTypeLoose, - @JsonProperty("disabled") Boolean disabled + @JsonProperty("disabled") Boolean disabled, + @JsonProperty(value = "regex", defaultValue = "unspecified") RegexKind regex ) { this.description = description; this.comment = comment; @@ -116,6 +122,7 @@ public TestSpec( this.validationMessages = validationMessages; this.disabled = Boolean.TRUE.equals(disabled); this.typeLoose = Boolean.TRUE.equals(isTypeLoose); + this.regex = regex; if (null != strictness) { this.strictness.putAll(strictness); } @@ -211,4 +218,13 @@ public boolean isTypeLoose() { return typeLoose; } + public RegexKind getRegex() { + return this.regex; + } + + public static enum RegexKind { + @JsonProperty("unspecified") UNSPECIFIED, + @JsonProperty("ecma-262") JONI, + @JsonProperty("jdk") JDK + } } diff --git a/src/test/resources/draft2020-12/issue495.json b/src/test/resources/draft2020-12/issue495.json new file mode 100644 index 000000000..abcfec51b --- /dev/null +++ b/src/test/resources/draft2020-12/issue495.json @@ -0,0 +1,64 @@ +[ + { + "description": "issue495 using ECMA-262", + "regex": "ecma-262", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "pattern": "^[a-z]{1,10}$", + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "an expected property name", + "data": { "aaa": 3 }, + "valid": true + }, + { + "description": "trailing newline", + "data": { "aaa\n": 3 }, + "valid": false + }, + { + "description": "embedded newline", + "data": { "aaa\nbbb": 3 }, + "valid": false + }, + { + "description": "leading newline", + "data": { "\nbbb": 3 }, + "valid": false + } + ] + }, + { + "description": "issue495 using Java Pattern", + "regex": "jdk", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "pattern": "^[a-z]{1,10}$", + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "an expected property name", + "data": { "aaa": 3 }, + "valid": true + }, + { + "description": "trailing newline", + "data": { "aaa\n": 3 }, + "valid": false + }, + { + "description": "embedded newline", + "data": { "aaa\nbbb": 3 }, + "valid": false + }, + { + "description": "leading newline", + "data": { "\nbbb": 3 }, + "valid": false + } + ] + } +] diff --git a/src/test/resources/draft2020-12/issue782.json b/src/test/resources/draft2020-12/issue782.json new file mode 100644 index 000000000..4c28a0ec3 --- /dev/null +++ b/src/test/resources/draft2020-12/issue782.json @@ -0,0 +1,92 @@ +[ + { + "description": "issue782 using ECMA-262", + "regex": "ecma-262", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "patternProperties": { + "^x-": true, + "y-$": true, + "^z-$": true + }, + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "regexes may be anchored to the start of the property name, 1", + "data": { "x-api-id": 3 }, + "valid": true + }, + { + "description": "regexes may be anchored to the start of the property name, 2", + "data": { "ax-api-id": 3 }, + "valid": false + }, + { + "description": "regexes may be anchored to the end of the property name, 1", + "data": { "api-id-y-": 3 }, + "valid": true + }, + { + "description": "regexes may be anchored to the end of the property name, 2", + "data": { "y-api-id": 3 }, + "valid": false + }, + { + "description": "regexes may be anchored to both ends of the property name, 1", + "data": { "z-": 3 }, + "valid": true + }, + { + "description": "regexes may be anchored to both ends of the property name, 2", + "data": { "az-api-id": 3 }, + "valid": false + } + ] + }, + { + "description": "issue782 using Java Pattern", + "regex": "jdk", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "patternProperties": { + "^x-": true, + "y-$": true, + "^z-$": true + }, + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "regexes may be anchored to the start of the property name, 1", + "data": { "x-api-id": 3 }, + "valid": true + }, + { + "description": "regexes may be anchored to the start of the property name, 2", + "data": { "ax-api-id": 3 }, + "valid": false + }, + { + "description": "regexes may be anchored to the end of the property name, 1", + "data": { "api-id-y-": 3 }, + "valid": true + }, + { + "description": "regexes may be anchored to the end of the property name, 2", + "data": { "y-api-id": 3 }, + "valid": false + }, + { + "description": "regexes may be anchored to both ends of the property name, 1", + "data": { "z-": 3 }, + "valid": true + }, + { + "description": "regexes may be anchored to both ends of the property name, 2", + "data": { "az-api-id": 3 }, + "valid": false + } + ] + } +]