Update truncateDigits argument to truncateLeaveCharacters

philterd · Dec 19, 2024 · 2ad0bcc · 2ad0bcc
1 parent 2e32224
commit 2ad0bcc
Show file tree

Hide file tree

Showing 10 changed files with 104 additions and 49 deletions.
diff --git a/docs/docs/filter_policies/filter_strategies.md b/docs/docs/filter_policies/filter_strategies.md
@@ -208,15 +208,15 @@ An example policy using the `STATIC_REPLACE` filter strategy:
 
 ### The `TRUNCATE` Filter Strategy {id="truncate"}
 
-This strategy allows for truncating tokens to only a select number of digits. Specify `truncateDigits`
-to set the desired number of digits to leave. For example, if `truncateDigits` is 4, the
+This strategy allows for truncating tokens to only a select number of digits. Specify `truncateLeaveCharacters`
+to set the desired number of digits to leave. For example, if `truncateLeaveCharacters` is 4, the
 string `4111111111111111` will be truncated to `4111************`. `truncateDirection` can be set to
 `LEADING` (the default) which leaves N leading digits or `TRAILING` which leaves N trailing digits.
 `truncateCharacter` can be overwritten (defaults to `*`) to change the character that is used for the
 replacement.
 
 The `TRUNCATE` filter has special behavior for the zip code filter. For zip codes the Zip will always be truncated
-to 5 digits long. For example, `truncateDigits=2` and a token of `90210-0110` will result in `90***`.
+to 5 digits long. For example, `truncateLeaveCharacters=2` and a token of `90210-0110` will result in `90***`.
 
 ```
 {
@@ -226,7 +226,7 @@ to 5 digits long. For example, `truncateDigits=2` and a token of `90210-0110` wi
          "zipCodeFilterStrategies": [
             {
                "strategy": "TRUNCATE",
-               "truncateDigits": 3
+               "truncateLeaveCharacters": 3
             }
          ]
       }

diff --git a/docs/docs/filter_policies/filters/common_filters/zip-codes.md b/docs/docs/filter_policies/filters/common_filters/zip-codes.md
@@ -21,15 +21,15 @@ This filter has no required parameters.
 
 The filter may have zero or more filter strategies. When no filter strategy is given the default strategy of `REDACT` is used. When multiple filter strategies are given the filter strategies will be applied in order as they are listed. See [Filter Strategies](#filter-strategies) for details.
 
-| Strategy              | Description                                                                                                                                         |
-| --------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `REDACT`              | Replace the sensitive text with a placeholder.                                                                                                      |
-| `RANDOM_REPLACE`      | Replace the sensitive text with a similar, random value.                                                                                            |
-| `STATIC_REPLACE`      | Replace the sensitive text with a given value.                                                                                                      |
-| `CRYPTO_REPLACE`      | Replace the sensitive text with its encrypted value.                                                                                                |
-| `HASH_SHA256_REPLACE` | Replace the sensitive text with its SHA256 hash value.                                                                                              |
-| `TRUNCATE`            | Replace the sensitive text by removing the last `x` digits. (Set the number of digits using the `truncateDigits` parameter of the filter strategy.) |
-| `ZERO_LEADING`        | Replace the sensitive text by zeroing the first 3 digits.                                                                                           |
+| Strategy              | Description                                                                                                                                                                            |
+| --------------------- |----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `REDACT`              | Replace the sensitive text with a placeholder.                                                                                                                                         |
+| `RANDOM_REPLACE`      | Replace the sensitive text with a similar, random value.                                                                                                                               |
+| `STATIC_REPLACE`      | Replace the sensitive text with a given value.                                                                                                                                         |
+| `CRYPTO_REPLACE`      | Replace the sensitive text with its encrypted value.                                                                                                                                   |
+| `HASH_SHA256_REPLACE` | Replace the sensitive text with its SHA256 hash value.                                                                                                                                 |
+| `TRUNCATE`            | Replace the sensitive text by removing everything except `x` characters. (Set the number of characters to leave using the `truncateLeaveCharacters` parameter of the filter strategy.) |
+| `ZERO_LEADING`        | Replace the sensitive text by zeroing the first 3 digits.                                                                                                                              |
 
 ### Conditions
 

diff --git a/docs/docs/filter_policies/sample_filter_policies.md b/docs/docs/filter_policies/sample_filter_policies.md
@@ -129,7 +129,7 @@ This policy finds ZIP codes starting with `90` and truncates the zip code to jus
         {
           "condition": "token startswith \"90\"",
           "strategy": "TRUNCATE",
-          "truncateDigits": 2
+          "truncateLeaveCharacters": 2
         }
       ]
     }

diff --git a/...main/java/ai/philterd/phileas/model/policy/filters/strategies/AbstractFilterStrategy.java b/...main/java/ai/philterd/phileas/model/policy/filters/strategies/AbstractFilterStrategy.java
@@ -112,8 +112,13 @@ public abstract class AbstractFilterStrategy {
 
     @SerializedName("truncateDigits")
     @Expose
+    @Deprecated
     protected Integer truncateDigits;
 
+    @SerializedName("truncateLeaveCharacters")
+    @Expose
+    protected Integer truncateLeaveCharacters;
+
     @SerializedName("truncateCharacter")
     @Expose
     protected String truncateCharacter = "*";
@@ -380,21 +385,26 @@ public String getMaskLength() {
         return maskLength;
     }
 
-    public Integer getTruncateDigits() {
-        return truncateDigits;
+    @Deprecated
+    public void setTruncateDigits(Integer truncateDigits) {
+        setTruncateLeaveCharacters(truncateDigits);
     }
 
-    public void setTruncateDigits(Integer truncateDigits) {
+    public void setTruncateLeaveCharacters(Integer truncateLeaveCharacters) {
 
         // Make sure it is a valid value.
-        if(truncateDigits >= 1) {
-            this.truncateDigits = truncateDigits;
+        if(truncateLeaveCharacters >= 1) {
+            this.truncateLeaveCharacters = truncateLeaveCharacters;
         } else {
-            throw new IllegalArgumentException("Truncate length must be greater than 1");
+            throw new IllegalArgumentException("Truncate leave characters must be greater than or equal to 1");
         }
 
     }
 
+    public Integer getTruncateLeaveCharacters() {
+        return truncateLeaveCharacters;
+    }
+
     public String getTruncateCharacter() {
         return truncateCharacter;
     }

diff --git a/...main/java/ai/philterd/phileas/model/policy/filters/strategies/StandardFilterStrategy.java b/...main/java/ai/philterd/phileas/model/policy/filters/strategies/StandardFilterStrategy.java
@@ -56,16 +56,16 @@ public Replacement getStandardReplacement(String label, String context, String d
 
         } else if(StringUtils.equalsIgnoreCase(strategy, TRUNCATE)) {
 
-            int truncateLength = getValueOrDefault(truncateDigits, 4);
+            int leaveCharacters = getValueOrDefault(getValueOrDefault(truncateLeaveCharacters, truncateDigits), 4);
 
-            if (truncateLength < 1) {
-                truncateLength = 1;
+            if (leaveCharacters < 1) {
+                leaveCharacters = 1;
             }
 
             if(StringUtils.equalsIgnoreCase(truncateDirection, LEADING)) {
-                replacement = token.substring(0, truncateLength) + StringUtils.repeat(truncateCharacter, token.length() - truncateLength);
+                replacement = token.substring(0, leaveCharacters) + StringUtils.repeat(truncateCharacter, token.length() - leaveCharacters);
             } else {
-                replacement = StringUtils.repeat(truncateCharacter, token.length() - truncateLength) + token.substring(token.length() - truncateLength);
+                replacement = StringUtils.repeat(truncateCharacter, token.length() - leaveCharacters) + token.substring(token.length() - leaveCharacters);
             }
 
         } else if(StringUtils.equalsIgnoreCase(strategy, RANDOM_REPLACE)) {

diff --git a/...main/java/ai/philterd/phileas/model/policy/filters/strategies/ai/PhEyeFilterStrategy.java b/...main/java/ai/philterd/phileas/model/policy/filters/strategies/ai/PhEyeFilterStrategy.java
@@ -158,16 +158,16 @@ public Replacement getReplacement(String label, String context, String documentI
 
         } else if(StringUtils.equalsIgnoreCase(strategy, TRUNCATE)) {
 
-            int truncateLength = getValueOrDefault(truncateDigits, 4);
+            int leaveCharacters = getValueOrDefault(getValueOrDefault(truncateDigits, truncateLeaveCharacters), 4);
 
-            if (truncateLength < 1) {
-                truncateLength = 1;
+            if (leaveCharacters < 1) {
+                leaveCharacters = 1;
             }
 
             if(StringUtils.equalsIgnoreCase(truncateDirection, LEADING)) {
-                replacement = token.substring(0, truncateLength) + StringUtils.repeat(truncateCharacter, token.length() - truncateLength);
+                replacement = token.substring(0, leaveCharacters) + StringUtils.repeat(truncateCharacter, token.length() - leaveCharacters);
             } else {
-                replacement = StringUtils.repeat(truncateCharacter, token.length() - truncateLength) + token.substring(token.length() - truncateLength);
+                replacement = StringUtils.repeat(truncateCharacter, token.length() - leaveCharacters) + token.substring(token.length() - leaveCharacters);
             }
 
         } else if(StringUtils.equalsIgnoreCase(strategy, RANDOM_REPLACE)) {

diff --git a/...in/java/ai/philterd/phileas/model/policy/filters/strategies/rules/DateFilterStrategy.java b/...in/java/ai/philterd/phileas/model/policy/filters/strategies/rules/DateFilterStrategy.java
@@ -176,16 +176,16 @@ public Replacement getReplacement(String label, String context, String documentI
 
         } else if(StringUtils.equalsIgnoreCase(strategy, TRUNCATE)) {
 
-            int truncateLength = getValueOrDefault(truncateDigits, 4);
+            int leaveCharacters = getValueOrDefault(getValueOrDefault(truncateDigits, truncateLeaveCharacters), 4);
 
-            if (truncateLength < 1) {
-                truncateLength = 1;
+            if (leaveCharacters < 1) {
+                leaveCharacters = 1;
             }
 
             if(StringUtils.equalsIgnoreCase(truncateDirection, LEADING)) {
-                replacement = token.substring(0, truncateLength) + StringUtils.repeat(truncateCharacter, token.length() - truncateLength);
+                replacement = token.substring(0, leaveCharacters) + StringUtils.repeat(truncateCharacter, token.length() - leaveCharacters);
             } else {
-                replacement = StringUtils.repeat(truncateCharacter, token.length() - truncateLength) + token.substring(token.length() - truncateLength);
+                replacement = StringUtils.repeat(truncateCharacter, token.length() - leaveCharacters) + token.substring(token.length() - leaveCharacters);
             }
 
         } else if(StringUtils.equalsIgnoreCase(strategy, RANDOM_REPLACE)) {

diff --git a/...java/ai/philterd/phileas/model/policy/filters/strategies/rules/ZipCodeFilterStrategy.java b/...java/ai/philterd/phileas/model/policy/filters/strategies/rules/ZipCodeFilterStrategy.java
@@ -194,12 +194,16 @@ public Replacement getReplacement(String label, String context, String documentI
 
         } else if(StringUtils.equalsIgnoreCase(strategy, TRUNCATE)) {
 
+            int leaveCharacters = getValueOrDefault(getValueOrDefault(truncateDigits, truncateLeaveCharacters), 4);
+
+            if (leaveCharacters < 1) {
+                leaveCharacters = 1;
+            }
+
             if(StringUtils.equalsIgnoreCase(truncateDirection, LEADING)) {
-                final int truncateLength = getValueOrDefault(truncateDigits, 2);
-                replacement = token.substring(0, truncateDigits) + StringUtils.repeat(truncateCharacter, Math.min(token.length() - truncateLength, 5 - truncateDigits));
+                replacement = token.substring(0, leaveCharacters) + StringUtils.repeat(truncateCharacter, Math.min(token.length() - leaveCharacters, 5 - leaveCharacters));
             } else {
-                final int truncateLength = getValueOrDefault(truncateDigits, 2);
-                replacement = StringUtils.repeat(truncateCharacter, Math.min(token.length() - truncateLength, 5 - truncateDigits)) + token.substring(Math.min(token.length() - truncateLength, 5 - truncateDigits), 5);
+                replacement = StringUtils.repeat(truncateCharacter, Math.min(token.length() - leaveCharacters, 5 - leaveCharacters)) + token.substring(Math.min(token.length() - leaveCharacters, 5 - leaveCharacters), 5);
             }
 
 
@@ -230,15 +234,15 @@ public Replacement getReplacement(String label, String context, String documentI
 
     }
 
-    public Integer getTruncateDigits() {
-        return truncateDigits;
+    public void setTruncateDigits(Integer truncateDigits) {
+        setTruncateLeaveCharacters(truncateDigits);
     }
 
-    public void setTruncateDigits(Integer truncateDigits) {
+    public void setTruncateLeaveCharacters(Integer truncateLeaveCharacters) {
 
         // Make sure it is a valid value.
-        if(truncateDigits >= 1 && truncateDigits <= 4) {
-            this.truncateDigits = truncateDigits;
+        if(truncateLeaveCharacters >= 1 && truncateLeaveCharacters <= 4) {
+            this.truncateLeaveCharacters = truncateLeaveCharacters;
         } else {
             throw new IllegalArgumentException("Truncate length must be between 1 and 4, inclusive.");
         }

diff --git a/.../ai/philterd/test/phileas/model/policy/filters/strategies/AbstractFilterStrategyTest.java b/.../ai/philterd/test/phileas/model/policy/filters/strategies/AbstractFilterStrategyTest.java
@@ -302,12 +302,12 @@ public void truncate1() throws Exception {
 
         final AbstractFilterStrategy strategy = getFilterStrategy();
         strategy.setStrategy(AbstractFilterStrategy.TRUNCATE);
-        strategy.setTruncateDigits(2);
+        strategy.setTruncateDigits(1);
 
         final String token = "12345";
         final Replacement replacement = strategy.getReplacement("name", "context", "docId", token, WINDOW, null, null, anonymizationService, null);
 
-        Assertions.assertEquals(replacement.getReplacement(), "12***");
+        Assertions.assertEquals(replacement.getReplacement(), "1****");
         Assertions.assertEquals(replacement.getReplacement().length(), 5);
 
     }
@@ -320,10 +320,30 @@ public void truncate2() throws Exception {
 
         when(anonymizationService.getAnonymizationCacheService()).thenReturn(anonymizationCacheService);
 
+        final AbstractFilterStrategy strategy = getFilterStrategy();
+        strategy.setStrategy(AbstractFilterStrategy.TRUNCATE);
+        strategy.setTruncateLeaveCharacters(4);
+
+        final String token = "12345";
+        final Replacement replacement = strategy.getReplacement("name", "context", "docId", token, WINDOW, null, null, anonymizationService, null);
+
+        Assertions.assertEquals(replacement.getReplacement(), "1234*");
+        Assertions.assertEquals(replacement.getReplacement().length(), 5);
+
+    }
+
+    @Test
+    public void truncate3() throws Exception {
+
+        final AnonymizationService anonymizationService = Mockito.mock(AnonymizationService.class);
+        final AnonymizationCacheService anonymizationCacheService = Mockito.mock(AnonymizationCacheService.class);
+
+        when(anonymizationService.getAnonymizationCacheService()).thenReturn(anonymizationCacheService);
+
         final AbstractFilterStrategy strategy = getFilterStrategy();
         strategy.setStrategy(AbstractFilterStrategy.TRUNCATE);
         strategy.setTruncateDirection(AbstractFilterStrategy.LEADING);
-        strategy.setTruncateDigits(2);
+        strategy.setTruncateLeaveCharacters(2);
 
         final String token = "12345";
         final Replacement replacement = strategy.getReplacement("name", "context", "docId", token, WINDOW, null, null, anonymizationService, null);
@@ -334,7 +354,7 @@ public void truncate2() throws Exception {
     }
 
     @Test
-    public void truncate3() throws Exception {
+    public void truncate4() throws Exception {
 
         final AnonymizationService anonymizationService = Mockito.mock(AnonymizationService.class);
         final AnonymizationCacheService anonymizationCacheService = Mockito.mock(AnonymizationCacheService.class);
@@ -344,7 +364,7 @@ public void truncate3() throws Exception {
         final AbstractFilterStrategy strategy = getFilterStrategy();
         strategy.setStrategy(AbstractFilterStrategy.TRUNCATE);
         strategy.setTruncateDirection(AbstractFilterStrategy.TRAILING);
-        strategy.setTruncateDigits(4);
+        strategy.setTruncateLeaveCharacters(4);
 
         final String token = "4111111111111111";
         final Replacement replacement = strategy.getReplacement("name", "context", "docId", token, WINDOW, null, null, anonymizationService, null);

diff --git a/...hilterd/test/phileas/model/policy/filters/strategies/rules/ZipCodeFilterStrategyTest.java b/...hilterd/test/phileas/model/policy/filters/strategies/rules/ZipCodeFilterStrategyTest.java
@@ -195,6 +195,24 @@ public void truncateTo1Trailing() throws Exception {
 
     }
 
+    @Test
+    public void truncateTo1Leave() throws Exception {
+
+        ZipCodeFilterStrategy strategy = new ZipCodeFilterStrategy();
+        strategy.setStrategy(ZipCodeFilterStrategy.TRUNCATE);
+        strategy.setTruncateLeaveCharacters(1);
+        strategy.setTruncateDirection(AbstractFilterStrategy.TRAILING);
+
+        AnonymizationService anonymizationService = Mockito.mock(AnonymizationService.class);
+
+        final Replacement replacement = strategy.getReplacement("name", "context", "documentid", "90210-0110", WINDOW, new Crypto(), new FPE(), anonymizationService, null);
+
+        LOGGER.info(replacement);
+
+        Assertions.assertEquals("****0", replacement.getReplacement());
+
+    }
+
     @Test
     public void zeroLeading1() throws Exception {
 
@@ -221,4 +239,7 @@ public void truncate2() throws Exception {}
     @Test
     public void truncate3() throws Exception {}
 
+    @Test
+    public void truncate4() throws Exception {}
+
 }