publish version 2.1.19

CHANGELOG.txt

@@ -1,13 +1,42 @@
 CHANGELOG
 
-2.1.18
+2.1.19
 UTIL:
-
+* different classes | improve usage of flag 'XML_PARSE_BIG_LINES' in method loadXML
+* class PANConf / Panoramaconfg | implement method ChildDeviceGroups() for 'location=shared:excludemaindg'
+* type=gcp | introduce argument 'namespace=XYZ' to specify and better filter for get pods
+* type=device | introduce actions=virtualSystem-delete/sharedgateway-delete/sharedgateway-migrate-to-vsys
+* type=static-route | introduction of new type= | with 'filter=(nexthop-ip is.set)' / (nexthop-vr is.set) / nexthop-interface is.set) /  (destination ip4.includes-full 172.33.10.0/23)
+* type=static-route actions=delete | introduce new action
+* type=rule | introduce actions=from-/to-remove-from-file:FILE.txt
+* type=XYZ | introduce new arguments: shadow-loadreduce - to not update TAG object related address-groups | debugloadtime - display load time for specific XML sections
+* develop | introduce config_validation.php
+* type=appid-toolbox | improvements for further. new features - example rule address/service report generator
+* type=rule actions=name-replace-character | set default value for replace to ''
+* type=rule | introduce new actions=stats-appid-fastapi:-90days/stats-service-fastapi:/stats-address-source-fastapi/stats-address-destination-fastapi/stats-address-fastapi/stats-traffic-fastapi
+* type=routing | introduce actions=exporttoexcel:file.html
+* type=routing | introduce 'filter=(protocol.bgp is.enabled)'
+* type=gcp | introdruce $namespace | improve handling for tenant like togglesvc
+* type=xpath | introduce actions=remove | introduce 'filter-text=NODETEXT'
+
+BUGFIX:
+* class PANConf|VirtualSystem | bugfix to support SharedGateway also for Tag objects correctly
+* type=xml-issue | bugfix to display read-only DeviceGroup duplicate address-group fixes correctly in summary
+* type=address-merger | bugfix - do not merger address objects if tag count of planned merged object exceeds PAN-OS limit of 64 tag members
+* type=address-merger | bugfix for removing tag objects from upper level, if adr merged objects are using them and tag is also available at address level
+* bugfix for argument 'location=shared:excludemaindg' on FW config file
+* type=schedule | bugfix actions=replacewithobject:OBJECTNAME
+
+GENERAL:
+* Class VirtualSystem/SharedGatewayStore | better handling of version variable
+* update actions/filter JSON file
+* APP-ID update to Device App-ID version: 8762-8327
+
+
+2.1.18 (20230920)
 BUGFIX:
 * type=address/service 'actions=move:shared,skipIfConflict' | bugfix as variable $findSubSystem was not declared for targetlocation 'shared'
 
-GENERAL:
-
 
 2.1.17 (20230920)
 UTIL:

appid-toolbox/lib/common.php

@@ -57,20 +57,26 @@
 class DeviceGroupRuleAppUsage
 {
     public $logs = Array();
+    public $logsSrcDst = Array();
 
-    public function load_from_file($filename)
+    public function load_from_file($filename, $SrcDst = false)
     {
         $xmlDoc = new DOMDocument();
         $xmlDoc->Load($filename);
 
         $recordsNode = DH::findFirstElementOrDie('records', $xmlDoc);
 
+        if( $SrcDst )
+            $keyword = "ips";
+        else
+            $keyword = "apps";
+
         foreach( $recordsNode->childNodes as $entryNode )
         {
             if( $entryNode->nodeType != XML_ELEMENT_NODE )
                 continue;
 
-            $logRecord = Array( 'apps' => Array() );
+            $logRecord = Array( $keyword => Array() );
 
             /** @var DOMElement $entryNode */
 
@@ -88,27 +94,55 @@ public function load_from_file($filename)
 
                 /** @var DOMElement $appNode */
 
-                $logRecord['apps'][$appNode->getAttribute('name')] = Array( 'name' => $appNode->getAttribute('name'), 'count' => $appNode->getAttribute('count'));
+                $logRecord[$keyword][$appNode->getAttribute('name')] = Array( 'name' => $appNode->getAttribute('name'), 'count' => $appNode->getAttribute('count'));
             }
 
-            $this->logs[$ruleName] = &$logRecord;
+            if( $SrcDst )
+                $this->logsSrcDst[$ruleName] = &$logRecord;
+            else
+                $this->logs[$ruleName] = &$logRecord;
             unset($logRecord);
         }
 
 
     }
 
-    public function save_to_file($filename)
+    public function save_to_file($filename, $SrcDst = false)
     {
         $xml = "<records>\n";
 
-        foreach($this->logs as $name => &$log)
+        if($SrcDst)
+        {
+            $logArray = $this->logsSrcDst;
+            $keyWord = "ips";
+        }
+        else
+        {
+            $logArray = $this->logs;
+            $keyWord = "apps";
+        }
+
+
+        foreach($logArray as $name => &$log)
         {
             $xml .= "  <entry name=\"{$name}\" timestamp=\"{$log['timestamp']}\" Htimestamp=\"".timestamp_to_date($log['timestamp'])."\">\n";
 
-            foreach( $log['apps'] as &$app )
+            foreach( $log[$keyWord] as $key => &$app )
             {
-                $xml .= "    <app name=\"{$app['name']}\" count=\"{$app['count']}\"/>\n";
+                if( $keyWord == "apps")
+                {
+                    $xml .= "    <".$keyWord." name=\"{$app['name']}\" count=\"{$app['count']}\"/>\n";
+                }
+                else
+                {
+                    #$xml .= "    <".$keyWord.">\n";
+                    foreach( $app as $ip )
+                    {
+                        $xml .= "    <".$keyWord."-".$key." name=\"{$ip['name']}\" count=\"{$ip['count']}\"/>\n";
+                    }
+
+                    #$xml .= "    </".$keyWord.">\n";
+                }
             }
 
             $xml .= "  </entry>\n";
@@ -119,6 +153,7 @@ public function save_to_file($filename)
         file_put_contents($filename, $xml);
     }
 
+
     public function addRuleStats($ruleName , $appName, $hitCount)
     {
         if( isset($this->logs[$ruleName]) )
@@ -139,33 +174,75 @@ public function addRuleStats($ruleName , $appName, $hitCount)
             $record['apps'][$appName] = Array('name'=>$appName, 'count' => $hitCount);
     }
 
+    public function addRuleStats_SrcDst($ruleName , $srcOrDst, $ip, $hitCount)
+    {
+        if( isset($this->logsSrcDst[$ruleName]) )
+        {
+            $record = &$this->logsSrcDst[$ruleName];
+        }
+        else
+        {
+            $SrcDstArray = array('src', 'dst');
+            $record = Array( 'ips' => $SrcDstArray );
+            $this->logsSrcDst[$ruleName] = &$record;
+        }
+
+        $record['timestamp'] = time();
+
+        if( isset($record['ips'][$srcOrDst][$ip]) )
+            $record['ips'][$srcOrDst][$ip]['count'] += $hitCount;
+        else
+            $record['ips'][$srcOrDst][$ip] = Array('name'=>$ip, 'count' => $hitCount);
+    }
+
     /**
      * @param string $ruleName
      * @return null|int
      */
-    public function getRuleUpdateTimestamp($ruleName)
+    public function getRuleUpdateTimestamp($ruleName, $SrcDst = false)
     {
-        if( isset($this->logs[$ruleName]) )
-        {
-            return $this->logs[$ruleName]['timestamp'];
-        }
+        if( $SrcDst )
+            if( isset($this->logsSrcDst[$ruleName]) )
+            {
+                return $this->logsSrcDst[$ruleName]['timestamp'];
+            }
+        else
+            if( isset($this->logs[$ruleName]) )
+            {
+                return $this->logs[$ruleName]['timestamp'];
+            }
+
         return null;
     }
 
 
-    public function resetRulesStats($ruleName)
+    public function resetRulesStats($ruleName, $SrcDst = false)
     {
-        if( isset($this->logs[$ruleName]) )
-            unset($this->logs[$ruleName]);
+        if( $SrcDst )
+            if( isset($this->logsSrcDst[$ruleName]) )
+                unset($this->logsSrcDst[$ruleName]);
+        else
+            if( isset($this->logs[$ruleName]) )
+                unset($this->logs[$ruleName]);
     }
 
 
-    public function getRuleStats($ruleName)
+    public function getRuleStats($ruleName, $SrcDst = false)
     {
-        if( !isset($this->logs[$ruleName]) )
-            return null;
+        if( $SrcDst )
+        {
+            if( !isset($this->logsSrcDst[$ruleName]) )
+                return null;
+
+            return $this->logsSrcDst[$ruleName]['ips'];
+        }
+        else
+        {
+            if( !isset($this->logs[$ruleName]) )
+                return null;
 
-        return $this->logs[$ruleName]['apps'];
+            return $this->logs[$ruleName]['apps'];
+        }
     }
 
     public function isRuleUsed($ruleName, $ignoreApps = Array('incomplete', 'non-syn-tcp') )
@@ -183,21 +260,34 @@ public function isRuleUsed($ruleName, $ignoreApps = Array('incomplete', 'non-syn
 
     }
 
-    public function createRuleStats($ruleName)
+    public function createRuleStats($ruleName, $SrcDst = false)
     {
-        if( !isset($this->logs[$ruleName]) )
-        {
-            $record = Array( 'apps' => Array(), 'timestamp' => time() );
-            $this->logs[$ruleName] = &$record;
-        }
+        if( $SrcDst )
+            if( !isset($this->logsSrcDst[$ruleName]) )
+            {
+                $record = Array( 'ips' => Array(), 'timestamp' => time() );
+                $this->logsSrcDst[$ruleName] = &$record;
+            }
+        else
+            if( !isset($this->logs[$ruleName]) )
+            {
+                $record = Array( 'apps' => Array(), 'timestamp' => time() );
+                $this->logs[$ruleName] = &$record;
+            }
     }
 
-    public function updateRuleUpdateTimestamp($ruleName)
+    public function updateRuleUpdateTimestamp($ruleName, $SrcDst = false)
     {
-        if( isset($this->logs[$ruleName]) )
-        {
-            $this->logs[$ruleName]['timestamp'] = time();
-        }
+        if( $SrcDst )
+            if( isset($this->logsSrcDst[$ruleName]) )
+            {
+                $this->logsSrcDst[$ruleName]['timestamp'] = time();
+            }
+        else
+            if( isset($this->logs[$ruleName]) )
+            {
+                $this->logs[$ruleName]['timestamp'] = time();
+            }
     }
 
 

appid-toolbox/lib/trait/lib_1_rule_marker.php

@@ -42,7 +42,7 @@ function ruleMarker_Phase1_init()
         if( isset(PH::$args['help']) )
             $this->display_usage_and_exit_p1();
 
-        $supportedOptions = array('phase', 'in', 'out', 'help', 'location');
+        $supportedOptions = array('phase', 'in', 'out', 'help', 'location', 'debugapi');
         $supportedOptions = array_flip($supportedOptions);
 
         foreach( PH::$args as $arg => $argvalue )
@@ -54,6 +54,10 @@ function ruleMarker_Phase1_init()
 
         $debugAPI = FALSE;
 
+        if( isset(PH::$args['debugapi']) )
+        {
+            $debugAPI = TRUE;
+        }
 
         $return = AppIDToolbox_common::location();
         $configInput = $return['configInput'];
@@ -99,11 +103,12 @@ function ruleMarker_Phase1_main($subSystem, $configInput, $pan, $inputConnector,
 
         foreach( $rules as $rule )
         {
+            PH::print_stdout();
             PH::print_stdout(" - rule '{$rule->name()}'");
 
             if( $ridTagLibrary->ruleIsTagged($rule) )
             {
-                PH::print_stdout(" SKIPPED : already tagged");
+                PH::print_stdout("   SKIPPED : already tagged");
                 $alreadyMarked++;
                 continue;
             }
@@ -112,7 +117,7 @@ function ruleMarker_Phase1_main($subSystem, $configInput, $pan, $inputConnector,
 
 
             $newTagName = $ridTagLibrary->findAvailableTagName('appRID#');
-            PH::print_stdout();
+
             PH::print_stdout("    * creating Virtual TAG '$newTagName' ... ");
 
             PH::print_stdout("    * applying tag to rule description... ");
@@ -129,10 +134,10 @@ function ruleMarker_Phase1_main($subSystem, $configInput, $pan, $inputConnector,
                 $xmlPreRules .= "<entry name=\"{$rule->name()}\"><description>" . htmlspecialchars($rule->description()) . "</description></entry>";
         }
 
-        PH::print_stdout("\n\nNumber of rules marked: {$markedRules}    (vs already marked: {$alreadyMarked}");
+        PH::print_stdout("\n\nNumber of rules marked: '{$markedRules}'    (vs already marked: '{$alreadyMarked}')");
 
         if( $markedRules < 1 )
-            PH::print_stdout("\n\n No change to push as not rule is set to be marked");
+            PH::print_stdout("\n\nNo change to push as no rule is set to be marked");
         else
         {
             if( $configInput['type'] == 'api' )