From 2ad0bccd4a7c63297bfb97ab3e7aed333b1d9f99 Mon Sep 17 00:00:00 2001 From: JAM <272807+JessieAMorris@users.noreply.github.com> Date: Thu, 19 Dec 2024 09:46:02 -0700 Subject: [PATCH] Update truncateDigits argument to truncateLeaveCharacters --- .../docs/filter_policies/filter_strategies.md | 8 ++--- .../filters/common_filters/zip-codes.md | 18 +++++------ .../filter_policies/sample_filter_policies.md | 2 +- .../strategies/AbstractFilterStrategy.java | 22 ++++++++++---- .../strategies/StandardFilterStrategy.java | 10 +++---- .../strategies/ai/PhEyeFilterStrategy.java | 10 +++---- .../strategies/rules/DateFilterStrategy.java | 10 +++---- .../rules/ZipCodeFilterStrategy.java | 22 ++++++++------ .../AbstractFilterStrategyTest.java | 30 +++++++++++++++---- .../rules/ZipCodeFilterStrategyTest.java | 21 +++++++++++++ 10 files changed, 104 insertions(+), 49 deletions(-) diff --git a/docs/docs/filter_policies/filter_strategies.md b/docs/docs/filter_policies/filter_strategies.md index 508310a08..3304b1629 100644 --- a/docs/docs/filter_policies/filter_strategies.md +++ b/docs/docs/filter_policies/filter_strategies.md @@ -208,15 +208,15 @@ An example policy using the `STATIC_REPLACE` filter strategy: ### The `TRUNCATE` Filter Strategy {id="truncate"} -This strategy allows for truncating tokens to only a select number of digits. Specify `truncateDigits` -to set the desired number of digits to leave. For example, if `truncateDigits` is 4, the +This strategy allows for truncating tokens to only a select number of digits. Specify `truncateLeaveCharacters` +to set the desired number of digits to leave. For example, if `truncateLeaveCharacters` is 4, the string `4111111111111111` will be truncated to `4111************`. `truncateDirection` can be set to `LEADING` (the default) which leaves N leading digits or `TRAILING` which leaves N trailing digits. `truncateCharacter` can be overwritten (defaults to `*`) to change the character that is used for the replacement. The `TRUNCATE` filter has special behavior for the zip code filter. For zip codes the Zip will always be truncated -to 5 digits long. For example, `truncateDigits=2` and a token of `90210-0110` will result in `90***`. +to 5 digits long. For example, `truncateLeaveCharacters=2` and a token of `90210-0110` will result in `90***`. ``` { @@ -226,7 +226,7 @@ to 5 digits long. For example, `truncateDigits=2` and a token of `90210-0110` wi "zipCodeFilterStrategies": [ { "strategy": "TRUNCATE", - "truncateDigits": 3 + "truncateLeaveCharacters": 3 } ] } diff --git a/docs/docs/filter_policies/filters/common_filters/zip-codes.md b/docs/docs/filter_policies/filters/common_filters/zip-codes.md index 6ca527a79..de2b42661 100644 --- a/docs/docs/filter_policies/filters/common_filters/zip-codes.md +++ b/docs/docs/filter_policies/filters/common_filters/zip-codes.md @@ -21,15 +21,15 @@ This filter has no required parameters. The filter may have zero or more filter strategies. When no filter strategy is given the default strategy of `REDACT` is used. When multiple filter strategies are given the filter strategies will be applied in order as they are listed. See [Filter Strategies](#filter-strategies) for details. -| Strategy | Description | -| --------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------- | -| `REDACT` | Replace the sensitive text with a placeholder. | -| `RANDOM_REPLACE` | Replace the sensitive text with a similar, random value. | -| `STATIC_REPLACE` | Replace the sensitive text with a given value. | -| `CRYPTO_REPLACE` | Replace the sensitive text with its encrypted value. | -| `HASH_SHA256_REPLACE` | Replace the sensitive text with its SHA256 hash value. | -| `TRUNCATE` | Replace the sensitive text by removing the last `x` digits. (Set the number of digits using the `truncateDigits` parameter of the filter strategy.) | -| `ZERO_LEADING` | Replace the sensitive text by zeroing the first 3 digits. | +| Strategy | Description | +| --------------------- |----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `REDACT` | Replace the sensitive text with a placeholder. | +| `RANDOM_REPLACE` | Replace the sensitive text with a similar, random value. | +| `STATIC_REPLACE` | Replace the sensitive text with a given value. | +| `CRYPTO_REPLACE` | Replace the sensitive text with its encrypted value. | +| `HASH_SHA256_REPLACE` | Replace the sensitive text with its SHA256 hash value. | +| `TRUNCATE` | Replace the sensitive text by removing everything except `x` characters. (Set the number of characters to leave using the `truncateLeaveCharacters` parameter of the filter strategy.) | +| `ZERO_LEADING` | Replace the sensitive text by zeroing the first 3 digits. | ### Conditions diff --git a/docs/docs/filter_policies/sample_filter_policies.md b/docs/docs/filter_policies/sample_filter_policies.md index c2a8f4216..5ac50c874 100644 --- a/docs/docs/filter_policies/sample_filter_policies.md +++ b/docs/docs/filter_policies/sample_filter_policies.md @@ -129,7 +129,7 @@ This policy finds ZIP codes starting with `90` and truncates the zip code to jus { "condition": "token startswith \"90\"", "strategy": "TRUNCATE", - "truncateDigits": 2 + "truncateLeaveCharacters": 2 } ] } diff --git a/phileas-model/src/main/java/ai/philterd/phileas/model/policy/filters/strategies/AbstractFilterStrategy.java b/phileas-model/src/main/java/ai/philterd/phileas/model/policy/filters/strategies/AbstractFilterStrategy.java index 76ef9275a..547cbaba3 100644 --- a/phileas-model/src/main/java/ai/philterd/phileas/model/policy/filters/strategies/AbstractFilterStrategy.java +++ b/phileas-model/src/main/java/ai/philterd/phileas/model/policy/filters/strategies/AbstractFilterStrategy.java @@ -112,8 +112,13 @@ public abstract class AbstractFilterStrategy { @SerializedName("truncateDigits") @Expose + @Deprecated protected Integer truncateDigits; + @SerializedName("truncateLeaveCharacters") + @Expose + protected Integer truncateLeaveCharacters; + @SerializedName("truncateCharacter") @Expose protected String truncateCharacter = "*"; @@ -380,21 +385,26 @@ public String getMaskLength() { return maskLength; } - public Integer getTruncateDigits() { - return truncateDigits; + @Deprecated + public void setTruncateDigits(Integer truncateDigits) { + setTruncateLeaveCharacters(truncateDigits); } - public void setTruncateDigits(Integer truncateDigits) { + public void setTruncateLeaveCharacters(Integer truncateLeaveCharacters) { // Make sure it is a valid value. - if(truncateDigits >= 1) { - this.truncateDigits = truncateDigits; + if(truncateLeaveCharacters >= 1) { + this.truncateLeaveCharacters = truncateLeaveCharacters; } else { - throw new IllegalArgumentException("Truncate length must be greater than 1"); + throw new IllegalArgumentException("Truncate leave characters must be greater than or equal to 1"); } } + public Integer getTruncateLeaveCharacters() { + return truncateLeaveCharacters; + } + public String getTruncateCharacter() { return truncateCharacter; } diff --git a/phileas-model/src/main/java/ai/philterd/phileas/model/policy/filters/strategies/StandardFilterStrategy.java b/phileas-model/src/main/java/ai/philterd/phileas/model/policy/filters/strategies/StandardFilterStrategy.java index ebc48f77a..eea8c8980 100644 --- a/phileas-model/src/main/java/ai/philterd/phileas/model/policy/filters/strategies/StandardFilterStrategy.java +++ b/phileas-model/src/main/java/ai/philterd/phileas/model/policy/filters/strategies/StandardFilterStrategy.java @@ -56,16 +56,16 @@ public Replacement getStandardReplacement(String label, String context, String d } else if(StringUtils.equalsIgnoreCase(strategy, TRUNCATE)) { - int truncateLength = getValueOrDefault(truncateDigits, 4); + int leaveCharacters = getValueOrDefault(getValueOrDefault(truncateLeaveCharacters, truncateDigits), 4); - if (truncateLength < 1) { - truncateLength = 1; + if (leaveCharacters < 1) { + leaveCharacters = 1; } if(StringUtils.equalsIgnoreCase(truncateDirection, LEADING)) { - replacement = token.substring(0, truncateLength) + StringUtils.repeat(truncateCharacter, token.length() - truncateLength); + replacement = token.substring(0, leaveCharacters) + StringUtils.repeat(truncateCharacter, token.length() - leaveCharacters); } else { - replacement = StringUtils.repeat(truncateCharacter, token.length() - truncateLength) + token.substring(token.length() - truncateLength); + replacement = StringUtils.repeat(truncateCharacter, token.length() - leaveCharacters) + token.substring(token.length() - leaveCharacters); } } else if(StringUtils.equalsIgnoreCase(strategy, RANDOM_REPLACE)) { diff --git a/phileas-model/src/main/java/ai/philterd/phileas/model/policy/filters/strategies/ai/PhEyeFilterStrategy.java b/phileas-model/src/main/java/ai/philterd/phileas/model/policy/filters/strategies/ai/PhEyeFilterStrategy.java index 0a3e359ee..62891bb4e 100644 --- a/phileas-model/src/main/java/ai/philterd/phileas/model/policy/filters/strategies/ai/PhEyeFilterStrategy.java +++ b/phileas-model/src/main/java/ai/philterd/phileas/model/policy/filters/strategies/ai/PhEyeFilterStrategy.java @@ -158,16 +158,16 @@ public Replacement getReplacement(String label, String context, String documentI } else if(StringUtils.equalsIgnoreCase(strategy, TRUNCATE)) { - int truncateLength = getValueOrDefault(truncateDigits, 4); + int leaveCharacters = getValueOrDefault(getValueOrDefault(truncateDigits, truncateLeaveCharacters), 4); - if (truncateLength < 1) { - truncateLength = 1; + if (leaveCharacters < 1) { + leaveCharacters = 1; } if(StringUtils.equalsIgnoreCase(truncateDirection, LEADING)) { - replacement = token.substring(0, truncateLength) + StringUtils.repeat(truncateCharacter, token.length() - truncateLength); + replacement = token.substring(0, leaveCharacters) + StringUtils.repeat(truncateCharacter, token.length() - leaveCharacters); } else { - replacement = StringUtils.repeat(truncateCharacter, token.length() - truncateLength) + token.substring(token.length() - truncateLength); + replacement = StringUtils.repeat(truncateCharacter, token.length() - leaveCharacters) + token.substring(token.length() - leaveCharacters); } } else if(StringUtils.equalsIgnoreCase(strategy, RANDOM_REPLACE)) { diff --git a/phileas-model/src/main/java/ai/philterd/phileas/model/policy/filters/strategies/rules/DateFilterStrategy.java b/phileas-model/src/main/java/ai/philterd/phileas/model/policy/filters/strategies/rules/DateFilterStrategy.java index 7e1b1134d..8bbca2c8f 100644 --- a/phileas-model/src/main/java/ai/philterd/phileas/model/policy/filters/strategies/rules/DateFilterStrategy.java +++ b/phileas-model/src/main/java/ai/philterd/phileas/model/policy/filters/strategies/rules/DateFilterStrategy.java @@ -176,16 +176,16 @@ public Replacement getReplacement(String label, String context, String documentI } else if(StringUtils.equalsIgnoreCase(strategy, TRUNCATE)) { - int truncateLength = getValueOrDefault(truncateDigits, 4); + int leaveCharacters = getValueOrDefault(getValueOrDefault(truncateDigits, truncateLeaveCharacters), 4); - if (truncateLength < 1) { - truncateLength = 1; + if (leaveCharacters < 1) { + leaveCharacters = 1; } if(StringUtils.equalsIgnoreCase(truncateDirection, LEADING)) { - replacement = token.substring(0, truncateLength) + StringUtils.repeat(truncateCharacter, token.length() - truncateLength); + replacement = token.substring(0, leaveCharacters) + StringUtils.repeat(truncateCharacter, token.length() - leaveCharacters); } else { - replacement = StringUtils.repeat(truncateCharacter, token.length() - truncateLength) + token.substring(token.length() - truncateLength); + replacement = StringUtils.repeat(truncateCharacter, token.length() - leaveCharacters) + token.substring(token.length() - leaveCharacters); } } else if(StringUtils.equalsIgnoreCase(strategy, RANDOM_REPLACE)) { diff --git a/phileas-model/src/main/java/ai/philterd/phileas/model/policy/filters/strategies/rules/ZipCodeFilterStrategy.java b/phileas-model/src/main/java/ai/philterd/phileas/model/policy/filters/strategies/rules/ZipCodeFilterStrategy.java index b2619bafd..dea22b57e 100644 --- a/phileas-model/src/main/java/ai/philterd/phileas/model/policy/filters/strategies/rules/ZipCodeFilterStrategy.java +++ b/phileas-model/src/main/java/ai/philterd/phileas/model/policy/filters/strategies/rules/ZipCodeFilterStrategy.java @@ -194,12 +194,16 @@ public Replacement getReplacement(String label, String context, String documentI } else if(StringUtils.equalsIgnoreCase(strategy, TRUNCATE)) { + int leaveCharacters = getValueOrDefault(getValueOrDefault(truncateDigits, truncateLeaveCharacters), 4); + + if (leaveCharacters < 1) { + leaveCharacters = 1; + } + if(StringUtils.equalsIgnoreCase(truncateDirection, LEADING)) { - final int truncateLength = getValueOrDefault(truncateDigits, 2); - replacement = token.substring(0, truncateDigits) + StringUtils.repeat(truncateCharacter, Math.min(token.length() - truncateLength, 5 - truncateDigits)); + replacement = token.substring(0, leaveCharacters) + StringUtils.repeat(truncateCharacter, Math.min(token.length() - leaveCharacters, 5 - leaveCharacters)); } else { - final int truncateLength = getValueOrDefault(truncateDigits, 2); - replacement = StringUtils.repeat(truncateCharacter, Math.min(token.length() - truncateLength, 5 - truncateDigits)) + token.substring(Math.min(token.length() - truncateLength, 5 - truncateDigits), 5); + replacement = StringUtils.repeat(truncateCharacter, Math.min(token.length() - leaveCharacters, 5 - leaveCharacters)) + token.substring(Math.min(token.length() - leaveCharacters, 5 - leaveCharacters), 5); } @@ -230,15 +234,15 @@ public Replacement getReplacement(String label, String context, String documentI } - public Integer getTruncateDigits() { - return truncateDigits; + public void setTruncateDigits(Integer truncateDigits) { + setTruncateLeaveCharacters(truncateDigits); } - public void setTruncateDigits(Integer truncateDigits) { + public void setTruncateLeaveCharacters(Integer truncateLeaveCharacters) { // Make sure it is a valid value. - if(truncateDigits >= 1 && truncateDigits <= 4) { - this.truncateDigits = truncateDigits; + if(truncateLeaveCharacters >= 1 && truncateLeaveCharacters <= 4) { + this.truncateLeaveCharacters = truncateLeaveCharacters; } else { throw new IllegalArgumentException("Truncate length must be between 1 and 4, inclusive."); } diff --git a/phileas-model/src/test/java/ai/philterd/test/phileas/model/policy/filters/strategies/AbstractFilterStrategyTest.java b/phileas-model/src/test/java/ai/philterd/test/phileas/model/policy/filters/strategies/AbstractFilterStrategyTest.java index ffe197d6c..44549a385 100644 --- a/phileas-model/src/test/java/ai/philterd/test/phileas/model/policy/filters/strategies/AbstractFilterStrategyTest.java +++ b/phileas-model/src/test/java/ai/philterd/test/phileas/model/policy/filters/strategies/AbstractFilterStrategyTest.java @@ -302,12 +302,12 @@ public void truncate1() throws Exception { final AbstractFilterStrategy strategy = getFilterStrategy(); strategy.setStrategy(AbstractFilterStrategy.TRUNCATE); - strategy.setTruncateDigits(2); + strategy.setTruncateDigits(1); final String token = "12345"; final Replacement replacement = strategy.getReplacement("name", "context", "docId", token, WINDOW, null, null, anonymizationService, null); - Assertions.assertEquals(replacement.getReplacement(), "12***"); + Assertions.assertEquals(replacement.getReplacement(), "1****"); Assertions.assertEquals(replacement.getReplacement().length(), 5); } @@ -320,10 +320,30 @@ public void truncate2() throws Exception { when(anonymizationService.getAnonymizationCacheService()).thenReturn(anonymizationCacheService); + final AbstractFilterStrategy strategy = getFilterStrategy(); + strategy.setStrategy(AbstractFilterStrategy.TRUNCATE); + strategy.setTruncateLeaveCharacters(4); + + final String token = "12345"; + final Replacement replacement = strategy.getReplacement("name", "context", "docId", token, WINDOW, null, null, anonymizationService, null); + + Assertions.assertEquals(replacement.getReplacement(), "1234*"); + Assertions.assertEquals(replacement.getReplacement().length(), 5); + + } + + @Test + public void truncate3() throws Exception { + + final AnonymizationService anonymizationService = Mockito.mock(AnonymizationService.class); + final AnonymizationCacheService anonymizationCacheService = Mockito.mock(AnonymizationCacheService.class); + + when(anonymizationService.getAnonymizationCacheService()).thenReturn(anonymizationCacheService); + final AbstractFilterStrategy strategy = getFilterStrategy(); strategy.setStrategy(AbstractFilterStrategy.TRUNCATE); strategy.setTruncateDirection(AbstractFilterStrategy.LEADING); - strategy.setTruncateDigits(2); + strategy.setTruncateLeaveCharacters(2); final String token = "12345"; final Replacement replacement = strategy.getReplacement("name", "context", "docId", token, WINDOW, null, null, anonymizationService, null); @@ -334,7 +354,7 @@ public void truncate2() throws Exception { } @Test - public void truncate3() throws Exception { + public void truncate4() throws Exception { final AnonymizationService anonymizationService = Mockito.mock(AnonymizationService.class); final AnonymizationCacheService anonymizationCacheService = Mockito.mock(AnonymizationCacheService.class); @@ -344,7 +364,7 @@ public void truncate3() throws Exception { final AbstractFilterStrategy strategy = getFilterStrategy(); strategy.setStrategy(AbstractFilterStrategy.TRUNCATE); strategy.setTruncateDirection(AbstractFilterStrategy.TRAILING); - strategy.setTruncateDigits(4); + strategy.setTruncateLeaveCharacters(4); final String token = "4111111111111111"; final Replacement replacement = strategy.getReplacement("name", "context", "docId", token, WINDOW, null, null, anonymizationService, null); diff --git a/phileas-model/src/test/java/ai/philterd/test/phileas/model/policy/filters/strategies/rules/ZipCodeFilterStrategyTest.java b/phileas-model/src/test/java/ai/philterd/test/phileas/model/policy/filters/strategies/rules/ZipCodeFilterStrategyTest.java index bfcb4ef8f..3b46a2581 100644 --- a/phileas-model/src/test/java/ai/philterd/test/phileas/model/policy/filters/strategies/rules/ZipCodeFilterStrategyTest.java +++ b/phileas-model/src/test/java/ai/philterd/test/phileas/model/policy/filters/strategies/rules/ZipCodeFilterStrategyTest.java @@ -195,6 +195,24 @@ public void truncateTo1Trailing() throws Exception { } + @Test + public void truncateTo1Leave() throws Exception { + + ZipCodeFilterStrategy strategy = new ZipCodeFilterStrategy(); + strategy.setStrategy(ZipCodeFilterStrategy.TRUNCATE); + strategy.setTruncateLeaveCharacters(1); + strategy.setTruncateDirection(AbstractFilterStrategy.TRAILING); + + AnonymizationService anonymizationService = Mockito.mock(AnonymizationService.class); + + final Replacement replacement = strategy.getReplacement("name", "context", "documentid", "90210-0110", WINDOW, new Crypto(), new FPE(), anonymizationService, null); + + LOGGER.info(replacement); + + Assertions.assertEquals("****0", replacement.getReplacement()); + + } + @Test public void zeroLeading1() throws Exception { @@ -221,4 +239,7 @@ public void truncate2() throws Exception {} @Test public void truncate3() throws Exception {} + @Test + public void truncate4() throws Exception {} + }