From 306acdef1d4586bbbac83b1206f64b682facd6f1 Mon Sep 17 00:00:00 2001 From: cbasnett Date: Tue, 29 Mar 2022 16:51:15 +0100 Subject: [PATCH] Code tidying and elasticsearch output --- Log Extractor/log_extract.py | 157 +++++++++++++++++++++++++++++------ 1 file changed, 132 insertions(+), 25 deletions(-) diff --git a/Log Extractor/log_extract.py b/Log Extractor/log_extract.py index a455aca..d925a84 100644 --- a/Log Extractor/log_extract.py +++ b/Log Extractor/log_extract.py @@ -2,10 +2,12 @@ import win32con, win32evtlog, win32evtlogutil, winerror, pywintypes import xml.etree as etree import xmltodict +from elasticsearch import Elasticsearch, AuthenticationException +from elasticsearch.helpers import bulk -version = '0.2' +version = '0.3' apptitle = 'Log Collector' -date = '18/02/2021' +date = '29/03/2022' author = 'Chris Basnett (chris.basnett@mdsec.co.uk)' def is_admin(): @@ -33,6 +35,94 @@ def get_all_publishers(): publisher = win32evtlog.EvtNextPublisherId(e) return publishers +def log_normalise(line): + format = { + "Provider": { + "Name": "", + "GUID": "" + }, + "Event": { + "ID": 0, + "Version": 0, + "Level": 0, + "Created": "", + "Channel": "", + "Message": "", + "Computer": "", + "UserID": "" + }, + "Meta": {} + } + + event = line['Event'] + eventkeys = event.keys() # Save us calling this a number of times + # Basic Sanity Check + if 'Provider' not in eventkeys: # if there's no provider + if 'TimeCreated' not in eventkeys: + return None + + if type(event['Provider']) == type(''): + format['Provider']['Name'] = event['Provider'] + else: + format['Provider']['Name'] = event['Provider']['Name'] + format['Provider']['GUID'] = event['Provider'].get('Guid','') + + if 'EventID' in eventkeys: + if type(event['EventID']) != type(''): + format['Event']['ID'] = int(event['EventID']['Qualifiers']) + elif event.get('EventID',None): + format['Event']['ID'] = int(event['EventID']) + + format['Event']['Version'] = int(event.get('Version',0)) + format['Event']['Level'] = int(event.get('Level',0)) + + if 'TimeCreated' in eventkeys: + format['Event']['Created'] = event['TimeCreated']['SystemTime'] # Probably want to save this as an int tbh + format['Event']['Channel'] = event['Channel'] + if 'Computer' in eventkeys: + format['Event']['Computer'] = event['Computer'] + if 'Security' in eventkeys: + if line['Event']['Security']: + format['Event']['UserID'] = event['Security'].get('UserID',None) + + format['Event']['Message'] = line['Message'] + + if event.get('Execution',None): + for k in event.get('Execution'): + if k == "ProcessID": + format['Meta'][k] = int(event['Execution'][k]) # If it's a processid we want an int + if k == "ThreadID": + format['Meta'][k] = int(event['Execution'][k]) # If it's a threadid we want an int + + # Sysmon Specific formatting + if event['Channel'] == "Microsoft-Windows-Sysmon/Operational": + message = format['Event']['Message'] + format['Meta']['Sysmon'] = {} + for item in message.split('\r\n'): + try: + key,value = item.split(': ') + format['Meta']['Sysmon'][key] = value + except: + continue + # Security Process audit specific formatting + if event['Channel'] == 'Security': + message = format['Event']['Message'] + if format['Event']['ID'] == 4688: # If it's a process creation + format['Meta']['Audit'] = {} + split = message.split('\r\n\r\n') + for s in split: + if 'Process Information' in s: + for field in s.split('Process Information:')[1].split('\r\n\t'): + try: + key, value = field.split(':\t') + value = value.strip('\t') + key = key.replace(' ','') + format['Meta']['Audit'][key] = value + except: + pass + + return(format) + def get_logs(channel=None): evts = [] flags = win32evtlog.EvtQueryChannelPath @@ -64,10 +154,15 @@ def get_logs(channel=None): message = "The Description for this event could not be found" event_xml = win32evtlog.EvtRender(event,win32evtlog.EvtRenderEventXml) - evts.append([event_xml,message]) + #evts.append([event_xml,message]) win32evtlog.EvtUpdateBookmark(bookmark,event) - return evts + parsed_event = parse_event([event_xml,message]) + if args.elastic: + parsed_event = log_normalise(parsed_event) + + yield parsed_event + #return evts def parse_event(event): event,message = event @@ -101,24 +196,41 @@ def parse_event(event): return evt -def parse(output, gz): - if not os.path.exists(output): - os.mkdir(output) +def parse(args): + + + if args.elastic: # If we're telling it the output should be elastic + client = Elasticsearch(args.output) + + else: + if not os.path.exists(args.output): + os.mkdir(args.output) + for c in get_all_channels(): name = c.replace('/','_') - try: - logs = get_logs(c) - except: - logs = None - if logs: - path = os.path.join(output,name) - print("Writing {}".format(c)) - if gz: + path = os.path.join(args.output,name) + print("Processing: {}".format(c)) + if args.elastic: + + try: + response = bulk(client, get_logs(c),index='log_extract') + except AuthenticationException: + print("Problem with Authentication, are you using the correct credentials?") + import sys + sys.exit() + except Exception as E: + print(E) + + else: + if args.gzip: f = gzip.open('{}.gz'.format(path),'w') + else: f = open('{}.log'.format(path),'wb') - for l in logs: - f.write((json.dumps(parse_event(l)).encode('utf-8'))) + + + for l in get_logs(c): + f.write(str(l).encode('utf-8')) f.write(b'\n') f.close() @@ -126,8 +238,9 @@ def parse(output, gz): parser = argparse.ArgumentParser(add_help=True, description=apptitle) parser.version = version parser.add_argument('-g', '--gzip', action="store_true", help='Compress with GZIP') - parser.add_argument('-o', '--output', action='store', type=str, help='Output Directory',required=True) + parser.add_argument('-o', '--output', action='store', type=str, help='Output Directory or ES server path (http://username:password@host:port',required=True) parser.add_argument('-v','--version',action='version') + parser.add_argument('-e','--elastic',action='store_true') args = parser.parse_args() os.system('cls') @@ -138,14 +251,8 @@ def parse(output, gz): if not args.output: parser.print_help() - - elif args.gzip: - if is_admin(): - parse(args.output, True) - else: - ctypes.windll.shell32.ShellExecuteW(None, u"runas", sys.executable, " ".join(sys.argv[1:]), None, 1) else: if is_admin(): - parse(args.output, False) + parse(args) else: ctypes.windll.shell32.ShellExecuteW(None, u"runas", sys.executable, " ".join(sys.argv[1:]), None, 1)