Add log collector utility

azurelinuxagent/common/utils/fileutil.py

@@ -242,3 +242,9 @@ def clean_ioerror(e, paths=[]):
             except Exception:
                 # swallow exception
                 pass
+
+
+def append_items_to_file(file_path, iterable):
+    with open(file_path, 'a+') as fh:
+        for item in iterable:
+            fh.write(item + "\n")

azurelinuxagent/common/utils/shellutil.py

@@ -111,7 +111,10 @@ def run_get_output(cmd, chk_err=True, log_cmd=True, expected_errors=[]):
 
 
 def _encode_command_output(output):
-    return ustr(output, encoding='utf-8', errors="backslashreplace")
+    if output:
+        return ustr(output, encoding='utf-8', errors="backslashreplace")
+    else:
+        return None
 
 
 class CommandError(Exception):
@@ -131,7 +134,7 @@ def __init__(self, command, returncode, stdout, stderr):
         self.stderr = stderr
 
 
-def run_command(command, log_error=False):
+def run_command(command, log_error=False, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE):
     """
     Executes the given command and returns its stdout as a string.
     If there are any errors executing the command it logs details about the failure and raises a RunCommandException;
@@ -141,7 +144,7 @@ def format_command(cmd):
         return " ".join(cmd) if isinstance(cmd, list) else command
 
     try:
-        process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False)
+        process = subprocess.Popen(command, stdin=stdin, stdout=stdout, stderr=stderr, shell=False)
         stdout, stderr = process.communicate()
         returncode = process.returncode
     except Exception as e:

config/logcollector_manifest_full

@@ -0,0 +1,73 @@
+echo,### Probing Directories ###
+ll,/var/log
+ll,/var/lib/waagent
+ll,/etc/udev/rules.d
+
+echo,### Gathering Configuration Files ###
+copy,/var/lib/waagent/provisioned
+copy,/etc/fstab
+copy,/etc/ssh/sshd_config
+copy,/boot/grub*/grub.c*
+copy,/boot/grub*/menu.lst
+copy,/etc/*-release
+copy,/etc/HOSTNAME
+copy,/etc/hostname
+copy,/etc/network/interfaces
+copy,/etc/network/interfaces.d/*.cfg
+copy,/etc/netplan/50-cloud-init.yaml
+copy,/etc/nsswitch.conf
+copy,/etc/resolv.conf
+copy,/run/systemd/resolve/stub-resolv.conf
+copy,/run/resolvconf/resolv.conf
+copy,/etc/sysconfig/iptables
+copy,/etc/sysconfig/network
+copy,/etc/sysconfig/network/ifcfg-eth*
+copy,/etc/sysconfig/network/routes
+copy,/etc/sysconfig/network-scripts/ifcfg-eth*
+copy,/etc/sysconfig/network-scripts/route-eth*
+copy,/etc/sysconfig/SuSEfirewall2
+copy,/etc/ufw/ufw.conf
+copy,/etc/waagent.conf
+copy,/var/lib/dhcp/dhclient.eth0.leases
+copy,/var/lib/dhclient/dhclient-eth0.leases
+copy,/var/lib/wicked/lease-eth0-dhcp-ipv4.xml
+echo,
+
+echo,### Gathering Log Files ###
+copy,/var/log/waagent*
+copy,/var/log/syslog*
+copy,/var/log/rsyslog*
+copy,/var/log/messages*
+copy,/var/log/kern*
+copy,/var/log/dmesg*
+copy,/var/log/dpkg*
+copy,/var/log/yum*
+copy,/var/log/cloud-init*
+copy,/var/log/boot*
+copy,/var/log/auth*
+copy,/var/log/secure*
+copy,/var/log/azure/*/*
+copy,/var/log/azure/*/*/*
+copy,/var/log/azure/custom-script/handler.log
+copy,/var/log/azure/run-command/handler.log
+echo,
+
+echo,### Gathering Extension Files ###
+copy,/var/lib/waagent/ExtensionsConfig.*.xml
+copy,/var/lib/waagent/*/status/*.status
+copy,/var/lib/waagent/*/config/*.settings
+copy,/var/lib/waagent/*/config/HandlerState
+copy,/var/lib/waagent/*/config/HandlerStatus
+copy,/var/lib/waagent/GoalState.*.xml
+copy,/var/lib/waagent/HostingEnvironmentConfig.xml
+copy,/var/lib/waagent/*.manifest.xml
+copy,/var/lib/waagent/SharedConfig.xml
+copy,/var/lib/waagent/ManagedIdentity-*.json
+copy,/var/lib/waagent/error.json
+copy,/var/lib/waagent/Incarnation
+copy,/var/lib/waagent/waagent_status.json
+copy,/var/lib/waagent/history/*.zip
+echo,
+
+echo,### Gathering Disk Info ###
+diskinfo,

config/logcollector_manifest_normal

@@ -0,0 +1,33 @@
+echo,### Probing Directories ###
+ll,/var/log
+ll,/var/lib/waagent
+
+echo,### Gathering Configuration Files ###
+copy,/etc/*-release
+copy,/etc/HOSTNAME
+copy,/etc/hostname
+copy,/etc/waagent.conf
+echo,
+
+echo,### Gathering Log Files ###
+copy,/var/log/waagent*
+copy,/var/log/dmesg*
+copy,/var/log/syslog*
+copy,/var/log/auth*
+copy,/var/log/azure/*/*
+copy,/var/log/azure/*/*/*
+copy,/var/log/azure/custom-script/handler.log
+echo,
+
+echo,### Gathering Extension Files ###
+copy,/var/lib/waagent/*.xml
+copy,/var/lib/waagent/waagent_status.json
+copy,/var/lib/waagent/*/status/*.status
+copy,/var/lib/waagent/*/config/*.settings
+copy,/var/lib/waagent/*/config/HandlerState
+copy,/var/lib/waagent/*/config/HandlerStatus
+copy,/var/lib/waagent/*.agentsManifest
+copy,/var/lib/waagent/error.json
+copy,/var/lib/waagent/Incarnation
+copy,/var/lib/waagent/history/*.zip
+echo,