wi-2048209 (#18669)

* Update BashV3

* Update PowerShellV2

* Update SshV0

* Update batchScriptV1

* Update variables names

* Use tool from utility common

* Fix regex for Bash & Ssh

* Update powershell regex

* Bump task versions

* Update warning messages

* Fix powershell condition

* Hide new logic under more detailed conditions

Tasks/BashV3/Strings/resources.resjson/en-US/resources.resjson

@@ -24,5 +24,6 @@
   "loc.messages.JS_ScriptContents": "Script contents:",
   "loc.messages.JS_Stderr": "Bash wrote one or more lines to the standard error stream.",
   "loc.messages.JS_TranslatePathFailed": "Unable to translate the path '%s' to the Linux file system.",
-  "loc.messages.JS_BashEnvAlreadyDefined": "The BASH_ENV environment variable has already been set to a '%s', the task will override it with '%s'"
+  "loc.messages.JS_BashEnvAlreadyDefined": "The BASH_ENV environment variable has already been set to a '%s', the task will override it with '%s'",
+  "loc.messages.SanitizerOutput": "Sanitizer changed input arguments. Output from sanitizer: '%s'"
 }

Tasks/BashV3/bash.ts

@@ -2,15 +2,9 @@ import fs = require('fs');
 import path = require('path');
 import tl = require('azure-pipelines-task-lib/task');
 import tr = require('azure-pipelines-task-lib/toolrunner');
-import { emitTelemetry } from 'azure-pipelines-tasks-utility-common/telemetry'
-import { processBashEnvVariables } from './bashEnvProcessor';
+import { sanitizeScriptArgs } from 'azure-pipelines-tasks-utility-common/argsSanitizer';
 var uuidV4 = require('uuid/v4');
 
-const featureFlags = {
-    enableTelemetry: getFeatureFlagValue('AZP_TASK_FF_BASHV3_ENABLE_INPUT_ARGS_TELEMETRY', true),
-    enableSecureArgs: getFeatureFlagValue('AZP_TASK_FF_BASHV3_ENABLE_SECURE_ARGS', true)
-}
-
 async function runBashPwd(bashPath: string, directoryPath: string): Promise<string> {
     let pwdOutput = '';
     const bashPwd = tl.tool(bashPath).arg('-c').arg('pwd');
@@ -121,40 +115,26 @@ async function run() {
                 targetFilePath = input_filePath;
             }
 
-            let resultArgs = input_arguments
-
-            if (featureFlags.enableSecureArgs || featureFlags.enableTelemetry) {
-                try {
-                    const [processedArgs, telemetry] = processBashEnvVariables(input_arguments)
-
-                    if (featureFlags.enableSecureArgs) {
-                        const argsEnvVar = {
-                            envName: "BASHV3_INPUT_SCRIPT_ARGS",
-                            value: processedArgs.trim()
-                        };
-                        process.env[argsEnvVar.envName] = argsEnvVar.value;
-                        resultArgs = `$${argsEnvVar.envName}`
-                    }
-
-                    if (featureFlags.enableTelemetry) {
-                        emitTelemetry('TaskHub', 'BashV3', telemetry)
-                    }
-                }
-                catch (err) {
-                    if (featureFlags.enableTelemetry) {
-                        tl.debug("Publishing error telemetry...");
-                        emitTelemetry('TaskHub', 'BashV3', {
-                            EnvProcessorError:
-                            {
-                                value: err.toString() || null,
-                                stack: err.stack || null,
-                            }
-                        });
-                    }
-
-                    if (featureFlags.enableSecureArgs) {
-                        throw err
+            let resultArgs = input_arguments;
+
+            const featureFlags = {
+                audit: tl.getBoolFeatureFlag('AZP_75787_ENABLE_NEW_LOGIC_LOG'),
+                activate: tl.getBoolFeatureFlag('AZP_75787_ENABLE_NEW_LOGIC'),
+                telemetry: tl.getBoolFeatureFlag('AZP_75787_ENABLE_COLLECT')
+            };
+
+            if (featureFlags.activate || featureFlags.activate || featureFlags.telemetry) {
+                const sanitizedArgs = sanitizeScriptArgs(
+                    input_arguments,
+                    {
+                        argsSplitSymbols: '\\\\',
+                        warningLocSymbol: 'SanitizerOutput',
+                        telemetryFeature: 'BashV3',
+                        saniziteRegExp: /(?<!\\)([^a-zA-Z0-9\\` _'"\-=\/:\.])/g
                     }
+                );
+                if (featureFlags.activate) {
+                    resultArgs = sanitizedArgs;
                 }
             }
 

Tasks/BashV3/package.json

@@ -20,8 +20,8 @@
   "dependencies": {
     "@types/mocha": "^9.1.1",
     "@types/node": "^16.11.39",
-    "azure-pipelines-task-lib": "^4.0.0-preview",
-    "azure-pipelines-tasks-utility-common": "^3.219.1",
+    "azure-pipelines-task-lib": "^4.4.0",
+    "azure-pipelines-tasks-utility-common": "^3.225.0",
     "uuid": "^3.0.1"
   },
   "devDependencies": {

Tasks/BashV3/task.json

@@ -17,7 +17,7 @@
     "author": "Microsoft Corporation",
     "version": {
         "Major": 3,
-        "Minor": 225,
+        "Minor": 226,
         "Patch": 0
     },
     "releaseNotes": "Script task consistency. Added support for multiple lines and added support for Windows.",
@@ -121,6 +121,7 @@
         "JS_ScriptContents": "Script contents:",
         "JS_Stderr": "Bash wrote one or more lines to the standard error stream.",
         "JS_TranslatePathFailed": "Unable to translate the path '%s' to the Linux file system.",
-        "JS_BashEnvAlreadyDefined": "The BASH_ENV environment variable has already been set to a '%s', the task will override it with '%s'"
+        "JS_BashEnvAlreadyDefined": "The BASH_ENV environment variable has already been set to a '%s', the task will override it with '%s'",
+        "SanitizerOutput": "Sanitizer changed input arguments. Output from sanitizer: '%s'"
     }
 }

Tasks/BashV3/task.loc.json

@@ -17,7 +17,7 @@
   "author": "Microsoft Corporation",
   "version": {
     "Major": 3,
-    "Minor": 225,
+    "Minor": 226,
     "Patch": 0
   },
   "releaseNotes": "ms-resource:loc.releaseNotes",
@@ -121,6 +121,7 @@
     "JS_ScriptContents": "ms-resource:loc.messages.JS_ScriptContents",
     "JS_Stderr": "ms-resource:loc.messages.JS_Stderr",
     "JS_TranslatePathFailed": "ms-resource:loc.messages.JS_TranslatePathFailed",
-    "JS_BashEnvAlreadyDefined": "ms-resource:loc.messages.JS_BashEnvAlreadyDefined"
+    "JS_BashEnvAlreadyDefined": "ms-resource:loc.messages.JS_BashEnvAlreadyDefined",
+    "SanitizerOutput": "ms-resource:loc.messages.SanitizerOutput"
   }
 }

Tasks/BatchScriptV1/task.json

@@ -17,7 +17,7 @@
     "author": "Microsoft Corporation",
     "version": {
         "Major": 1,
-        "Minor": 198,
+        "Minor": 226,
         "Patch": 0
     },
     "demands": [
@@ -81,7 +81,8 @@
             "target": "$(filename)",
             "argumentFormat": "$(arguments)",
             "workingDirectory": "$(workingFolder)",
-            "modifyEnvironment": "$(modifyEnvironment)"
+            "modifyEnvironment": "$(modifyEnvironment)",
+            "disableInlineExecution": true
         }
     }
 }

Tasks/BatchScriptV1/task.loc.json

@@ -17,7 +17,7 @@
   "author": "Microsoft Corporation",
   "version": {
     "Major": 1,
-    "Minor": 198,
+    "Minor": 226,
     "Patch": 0
   },
   "demands": [
@@ -81,7 +81,8 @@
       "target": "$(filename)",
       "argumentFormat": "$(arguments)",
       "workingDirectory": "$(workingFolder)",
-      "modifyEnvironment": "$(modifyEnvironment)"
+      "modifyEnvironment": "$(modifyEnvironment)",
+      "disableInlineExecution": true
     }
   }
 }

Tasks/PowerShellV2/Strings/resources.resjson/en-US/resources.resjson

@@ -44,10 +44,12 @@
   "loc.messages.JS_InvalidFilePath": "Invalid file path '%s'. A path to a .ps1 file is required.",
   "loc.messages.JS_Stderr": "PowerShell wrote one or more lines to the standard error stream.",
   "loc.messages.JS_InvalidTargetType": "Invalid target type '%s'. The value must be one of: 'filepath' or 'inline'",
+  "loc.messages.JS_SanitizerOutput": "Sanitizer changed input arguments. Output from sanitizer: '%s'",
   "loc.messages.PS_ExitCode": "PowerShell exited with code '{0}'.",
   "loc.messages.PS_FormattedCommand": "Formatted command: {0}",
   "loc.messages.PS_InvalidActionPreference": "Invalid action preference for {0}: '{1}'. The value must be one of: {2}",
   "loc.messages.PS_InvalidFilePath": "Invalid file path '{0}'. A path to a .ps1 file is required.",
   "loc.messages.PS_UnableToDetermineExitCode": "Unexpected exception. Unable to determine the exit code from powershell.",
-  "loc.messages.PS_InvalidTargetType": "Invalid target type '{0}'. The value must be one of: 'filepath' or 'inline'"
+  "loc.messages.PS_InvalidTargetType": "Invalid target type '{0}'. The value must be one of: 'filepath' or 'inline'",
+  "loc.messages.PS_SanitizerOutput": "Sanitizer changed input arguments. Output from sanitizer: '{0}'"
 }

Tasks/PowerShellV2/helpers.ps1

@@ -0,0 +1,41 @@
+function Publish-Telemetry($Telemetry) {
+    $area = 'TaskHub'
+    $feature = 'PowerShellV2'
+    $telemetryJson = $Telemetry | ConvertTo-Json -Compress
+    Write-Host "##vso[telemetry.publish area=$area;feature=$feature]$telemetryJson"
+}
+
+function Sanitize-FileArguments([string]$InputArgs) {
+
+    $featureFlags = @{
+        audit     = [System.Convert]::ToBoolean($env:AZP_75787_ENABLE_NEW_LOGIC_LOG)
+        activate  = [System.Convert]::ToBoolean($env:AZP_75787_ENABLE_NEW_LOGIC)
+        telemetry = [System.Convert]::ToBoolean($env:AZP_75787_ENABLE_COLLECT)
+    }
+
+    $removedSymbolSign = '_#removed#_';
+    $argsSplitSymbols = '``';
+
+    # We're splitting by ``, removing all suspicious characters and then join
+    $argsArr = $InputArgs -split $argsSplitSymbols;
+    for ($i = 0; $i -lt $argsArr.Length; $i++ ) {
+        ## '?<!`' - checking if before character no backtick. '^a-zA-Z0-9` _'"-' - checking if character is allowed. Insead replacing to #removed#
+        $argsArr[$i] = $argsArr[$i] -replace '(?<!`)([^a-zA-Z0-9\\` _''"\-=\/:\.])', $removedSymbolSign;
+    }
+
+    $resultArgs = $argsArr -join $argsSplitSymbols;
+
+    if ( $resultArgs -like "*$removedSymbolSign*") {
+
+        if ($featureFlags.audit -or $featureFlags.activate) {
+            Write-Warning (Get-VstsLocString -Key 'PS_SanitizerOutput' -ArgumentList $resultArgs);
+        }
+
+        if ($featureFlags.telemetry) {
+            $removedSymbolsCount = [regex]::matches($resultArgs, $removedSymbolSign).count
+            Publish-Telemetry @{ 'removedSymbolsCount' = $removedSymbolsCount }
+        }
+    }
+
+    return $resultArgs;
+}