From 40087ac49db9a2ecf5584b1e8b2fea0fc2fb8d6a Mon Sep 17 00:00:00 2001 From: Philip Wang Date: Tue, 28 Jan 2020 18:38:32 -0500 Subject: [PATCH 01/18] Update worker thread for server creation --- manticore/core/worker.py | 63 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/manticore/core/worker.py b/manticore/core/worker.py index a2965ca7f..908017ec4 100644 --- a/manticore/core/worker.py +++ b/manticore/core/worker.py @@ -230,3 +230,66 @@ def start(self): def join(self): self._p.join() self._p = None + + +class MonitorWorker(WorkerThread): + def run(self, *args): + logger.debug( + "Starting Manticore Monitor Thread %d. Pid %d Tid %d).", + self.id, + os.getpid(), + threading.get_ident(), + ) + + m = self.manticore + m._is_main = False # This will mark our copy of manticore + + import time + import socket + + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + + HOST = '127.0.0.1' + PORT = 1337 + + s.bind((HOST, PORT)) + + logger.debug( + "Created socket in threads bound to host %s, port %d", + HOST, + PORT, + ) + + s.listen(5) + socket_list = [s] + + with WithKeyboardInterruptAs(m.kill): + while m.is_running(): # TODO: Exits after state exploration, but not finalization + # Establish connection with client. + + read_sockets, write_sockets, error_sockets = select.select(socket_list, socket_list, [], 5) + serialized_states = generate_states().SerializeToString() + serialized_messages = generate_messages().SerializeToString() + + #print(read_sockets, write_sockets) + + if len(read_sockets): + + for sock in read_sockets: + if sock is s: + print("Got connection from manticore TUI") + c, addr = sock.accept() + socket_list.append(c) + else: + data = sock.recv(1024) + + if len(write_sockets): + for sock in write_sockets: + time.sleep(random.randint(2, 5) + 0.01) + + if random.random() >= 0.5: + print("Sending states") + sock.send(serialized_states) + else: + print("Sending messages") + sock.send(serialized_messages) From 326aea146880e04691f9a5bcea7081b6a293ac24 Mon Sep 17 00:00:00 2001 From: Philip Wang Date: Tue, 28 Jan 2020 22:31:54 -0500 Subject: [PATCH 02/18] Add necessary files for TUI connectivity --- manticore/core/manticore.py | 16 ++++++++++++-- manticore/core/worker.py | 42 ++++++++++++++++++++++++------------- 2 files changed, 42 insertions(+), 16 deletions(-) diff --git a/manticore/core/manticore.py b/manticore/core/manticore.py index 8bc7696b9..ee6bbfb19 100644 --- a/manticore/core/manticore.py +++ b/manticore/core/manticore.py @@ -23,7 +23,7 @@ from ..utils.log import set_verbosity from ..utils.nointerrupt import WithKeyboardInterruptAs from .workspace import Workspace -from .worker import WorkerSingle, WorkerThread, WorkerProcess +from .worker import WorkerSingle, WorkerThread, WorkerProcess, MonitorWorker from multiprocessing.managers import SyncManager import threading @@ -191,7 +191,7 @@ def __init__(self, initial_state, workspace_url=None, policy="random", **kwargs) During exploration Manticore spawns a number of temporary states that are maintained in different lists: - + .. code-block:: none Initial @@ -306,6 +306,7 @@ def __init__(self, initial_state, workspace_url=None, policy="random", **kwargs) # Workers will use manticore __dict__ So lets spawn them last self._workers = [self._worker_type(id=i, manticore=self) for i in range(consts.procs)] + self._monitor = MonitorWorker(id=-1, manticore=self) self._is_main = True def __str__(self): @@ -506,6 +507,15 @@ def _get_state(self, wait=False): return self._load(state_id) + @sync + def count_state_lists(self): + return ( + len(self._ready_states), + len(self._busy_states), + len(self._terminated_states), + len(self._killed_states), + ) + @sync def _revive_state(self, state_id): """ Send a BUSY state back to READY list @@ -944,6 +954,8 @@ def run(self): self._publish("will_run", self.ready_states) self._running.value = True + self._monitor.start() + # start all the workers! for w in self._workers: w.start() diff --git a/manticore/core/worker.py b/manticore/core/worker.py index 908017ec4..01e91bd53 100644 --- a/manticore/core/worker.py +++ b/manticore/core/worker.py @@ -1,8 +1,10 @@ from ..utils.nointerrupt import WithKeyboardInterruptAs from .state import Concretize, TerminateState +from .state_pb2 import * import logging import multiprocessing import threading +import select import os @@ -231,8 +233,25 @@ def join(self): self._p.join() self._p = None - class MonitorWorker(WorkerThread): + + def obtain_states(self, m): + serialized_states = StateList() + serialized_messages = MessageList() + + for r in m._ready_states: + rstate = State() + rstate.id = r + serialized_states.states.extend([rstate]) + + for b in m._busy_states: + bstate = State() + bstate.id = b + bstate.reason = "Busy executing" + serialized_states.states.extend([bstate]) + + return serialized_states + def run(self, *args): logger.debug( "Starting Manticore Monitor Thread %d. Pid %d Tid %d).", @@ -267,17 +286,18 @@ def run(self, *args): while m.is_running(): # TODO: Exits after state exploration, but not finalization # Establish connection with client. - read_sockets, write_sockets, error_sockets = select.select(socket_list, socket_list, [], 5) - serialized_states = generate_states().SerializeToString() - serialized_messages = generate_messages().SerializeToString() + read_sockets, write_sockets, error_sockets = select.select(socket_list, socket_list, [], 5) + serialized_states = self.obtain_states(m).SerializeToString() - #print(read_sockets, write_sockets) - if len(read_sockets): for sock in read_sockets: if sock is s: - print("Got connection from manticore TUI") + + logger.debug( + "Received connection from Manticore TUI" + ) + c, addr = sock.accept() socket_list.append(c) else: @@ -285,11 +305,5 @@ def run(self, *args): if len(write_sockets): for sock in write_sockets: - time.sleep(random.randint(2, 5) + 0.01) - if random.random() >= 0.5: - print("Sending states") - sock.send(serialized_states) - else: - print("Sending messages") - sock.send(serialized_messages) + sock.send(serialized_states) From 060441e6babeeb2a6d4cc73ee2280b9de5b659ed Mon Sep 17 00:00:00 2001 From: Philip Wang Date: Tue, 28 Jan 2020 22:32:09 -0500 Subject: [PATCH 03/18] Add necessary files for TUI connectivity --- manticore/core/state.proto | 31 +++++ manticore/core/state_pb2.py | 245 ++++++++++++++++++++++++++++++++++++ 2 files changed, 276 insertions(+) create mode 100644 manticore/core/state.proto create mode 100644 manticore/core/state_pb2.py diff --git a/manticore/core/state.proto b/manticore/core/state.proto new file mode 100644 index 000000000..dff84b636 --- /dev/null +++ b/manticore/core/state.proto @@ -0,0 +1,31 @@ +syntax = "proto3"; + +package mserialize; + +message LogMessage{ + string content = 1; +} + +message State{ + + enum StateType{ + READY = 0; + BUSY = 1; + KILLED = 2; + TERMINATED = 3; + } + + int32 id = 2; // state ID + StateType type = 3; // Type of state + string reason = 4; // Reason for execution stopping + int32 num_executing = 5; // number of executing instructions + int32 wait_time = 6; +} + +message StateList{ + repeated State states = 7; +} + +message MessageList{ + repeated LogMessage messages = 8; +} diff --git a/manticore/core/state_pb2.py b/manticore/core/state_pb2.py new file mode 100644 index 000000000..f7b190f5e --- /dev/null +++ b/manticore/core/state_pb2.py @@ -0,0 +1,245 @@ +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: state.proto + +import sys +_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor.FileDescriptor( + name='state.proto', + package='mserialize', + syntax='proto3', + serialized_options=None, + serialized_pb=_b('\n\x0bstate.proto\x12\nmserialize\"\x1d\n\nLogMessage\x12\x0f\n\x07\x63ontent\x18\x01 \x01(\t\"\xa6\x01\n\x05State\x12\n\n\x02id\x18\x02 \x01(\x05\x12)\n\x04type\x18\x03 \x01(\x0e\x32\x1b.mserialize.State.StateType\x12\x0e\n\x06reason\x18\x04 \x01(\t\x12\x15\n\rnum_executing\x18\x05 \x01(\x05\x12\x11\n\twait_time\x18\x06 \x01(\x05\",\n\tStateType\x12\t\n\x05READY\x10\x00\x12\x08\n\x04\x42USY\x10\x01\x12\n\n\x06KILLED\x10\x02\".\n\tStateList\x12!\n\x06states\x18\x07 \x03(\x0b\x32\x11.mserialize.State\"7\n\x0bMessageList\x12(\n\x08messages\x18\x08 \x03(\x0b\x32\x16.mserialize.LogMessageb\x06proto3') +) + + + +_STATE_STATETYPE = _descriptor.EnumDescriptor( + name='StateType', + full_name='mserialize.State.StateType', + filename=None, + file=DESCRIPTOR, + values=[ + _descriptor.EnumValueDescriptor( + name='READY', index=0, number=0, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='BUSY', index=1, number=1, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='KILLED', index=2, number=2, + serialized_options=None, + type=None), + ], + containing_type=None, + serialized_options=None, + serialized_start=181, + serialized_end=225, +) +_sym_db.RegisterEnumDescriptor(_STATE_STATETYPE) + + +_LOGMESSAGE = _descriptor.Descriptor( + name='LogMessage', + full_name='mserialize.LogMessage', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='content', full_name='mserialize.LogMessage.content', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=27, + serialized_end=56, +) + + +_STATE = _descriptor.Descriptor( + name='State', + full_name='mserialize.State', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='id', full_name='mserialize.State.id', index=0, + number=2, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='type', full_name='mserialize.State.type', index=1, + number=3, type=14, cpp_type=8, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='reason', full_name='mserialize.State.reason', index=2, + number=4, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='num_executing', full_name='mserialize.State.num_executing', index=3, + number=5, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='wait_time', full_name='mserialize.State.wait_time', index=4, + number=6, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + _STATE_STATETYPE, + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=59, + serialized_end=225, +) + + +_STATELIST = _descriptor.Descriptor( + name='StateList', + full_name='mserialize.StateList', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='states', full_name='mserialize.StateList.states', index=0, + number=7, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=227, + serialized_end=273, +) + + +_MESSAGELIST = _descriptor.Descriptor( + name='MessageList', + full_name='mserialize.MessageList', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='messages', full_name='mserialize.MessageList.messages', index=0, + number=8, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=275, + serialized_end=330, +) + +_STATE.fields_by_name['type'].enum_type = _STATE_STATETYPE +_STATE_STATETYPE.containing_type = _STATE +_STATELIST.fields_by_name['states'].message_type = _STATE +_MESSAGELIST.fields_by_name['messages'].message_type = _LOGMESSAGE +DESCRIPTOR.message_types_by_name['LogMessage'] = _LOGMESSAGE +DESCRIPTOR.message_types_by_name['State'] = _STATE +DESCRIPTOR.message_types_by_name['StateList'] = _STATELIST +DESCRIPTOR.message_types_by_name['MessageList'] = _MESSAGELIST +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + +LogMessage = _reflection.GeneratedProtocolMessageType('LogMessage', (_message.Message,), dict( + DESCRIPTOR = _LOGMESSAGE, + __module__ = 'state_pb2' + # @@protoc_insertion_point(class_scope:mserialize.LogMessage) + )) +_sym_db.RegisterMessage(LogMessage) + +State = _reflection.GeneratedProtocolMessageType('State', (_message.Message,), dict( + DESCRIPTOR = _STATE, + __module__ = 'state_pb2' + # @@protoc_insertion_point(class_scope:mserialize.State) + )) +_sym_db.RegisterMessage(State) + +StateList = _reflection.GeneratedProtocolMessageType('StateList', (_message.Message,), dict( + DESCRIPTOR = _STATELIST, + __module__ = 'state_pb2' + # @@protoc_insertion_point(class_scope:mserialize.StateList) + )) +_sym_db.RegisterMessage(StateList) + +MessageList = _reflection.GeneratedProtocolMessageType('MessageList', (_message.Message,), dict( + DESCRIPTOR = _MESSAGELIST, + __module__ = 'state_pb2' + # @@protoc_insertion_point(class_scope:mserialize.MessageList) + )) +_sym_db.RegisterMessage(MessageList) + + +# @@protoc_insertion_point(module_scope) From 81749590327e87af89df625ef2511a033c72b43d Mon Sep 17 00:00:00 2001 From: Philip Wang Date: Wed, 29 Jan 2020 18:53:31 -0500 Subject: [PATCH 04/18] Update MonitorWorker --- manticore/core/worker.py | 46 ++++++++++++++++++++++++++++++---------- 1 file changed, 35 insertions(+), 11 deletions(-) diff --git a/manticore/core/worker.py b/manticore/core/worker.py index 01e91bd53..29a20f9bf 100644 --- a/manticore/core/worker.py +++ b/manticore/core/worker.py @@ -239,17 +239,31 @@ def obtain_states(self, m): serialized_states = StateList() serialized_messages = MessageList() - for r in m._ready_states: - rstate = State() - rstate.id = r - serialized_states.states.extend([rstate]) - for b in m._busy_states: bstate = State() bstate.id = b bstate.reason = "Busy executing" + bstate.type = State.BUSY serialized_states.states.extend([bstate]) + + for r in m._ready_states: + rstate = State() + rstate.id = r + rstate.type = State.READY + serialized_states.states.extend([rstate]) + for t in m._terminated_states: + tstate = State() + tstate.id = t + tstate.type = State.TERMINATED + serialized_states.states.extend([tstate]) + + for k in m._killed_states: + kstate = State() + kstate.id = k + kstate.type = State.KILLED + serialized_states.states.extend([kstate]) + return serialized_states def run(self, *args): @@ -263,6 +277,7 @@ def run(self, *args): m = self.manticore m._is_main = False # This will mark our copy of manticore + import time import socket @@ -282,12 +297,20 @@ def run(self, *args): s.listen(5) socket_list = [s] + serialized_states = self.obtain_states(m).SerializeToString() + changed = False + with WithKeyboardInterruptAs(m.kill): while m.is_running(): # TODO: Exits after state exploration, but not finalization # Establish connection with client. + read_sockets, write_sockets, error_sockets = select.select(socket_list, socket_list, [], 0) + + states = self.obtain_states(m) + + if states != serialized_states: + serialized_states = states + changed = True - read_sockets, write_sockets, error_sockets = select.select(socket_list, socket_list, [], 5) - serialized_states = self.obtain_states(m).SerializeToString() if len(read_sockets): @@ -301,9 +324,10 @@ def run(self, *args): c, addr = sock.accept() socket_list.append(c) else: - data = sock.recv(1024) + sock.recv(1024) - if len(write_sockets): + if len(write_sockets) and changed: for sock in write_sockets: - - sock.send(serialized_states) + print(m._busy_states, m._ready_states, m._terminated_states, m._killed_states) + sock.send(serialized_states.SerializeToString()) + changed = False From fa15efe53e4ff66ec766d7bf3c5cd3dd9a02a6d0 Mon Sep 17 00:00:00 2001 From: Philip Wang Date: Wed, 29 Jan 2020 18:53:56 -0500 Subject: [PATCH 05/18] Update protocol --- manticore/core/state_pb2.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/manticore/core/state_pb2.py b/manticore/core/state_pb2.py index f7b190f5e..5d251ddb9 100644 --- a/manticore/core/state_pb2.py +++ b/manticore/core/state_pb2.py @@ -19,7 +19,7 @@ package='mserialize', syntax='proto3', serialized_options=None, - serialized_pb=_b('\n\x0bstate.proto\x12\nmserialize\"\x1d\n\nLogMessage\x12\x0f\n\x07\x63ontent\x18\x01 \x01(\t\"\xa6\x01\n\x05State\x12\n\n\x02id\x18\x02 \x01(\x05\x12)\n\x04type\x18\x03 \x01(\x0e\x32\x1b.mserialize.State.StateType\x12\x0e\n\x06reason\x18\x04 \x01(\t\x12\x15\n\rnum_executing\x18\x05 \x01(\x05\x12\x11\n\twait_time\x18\x06 \x01(\x05\",\n\tStateType\x12\t\n\x05READY\x10\x00\x12\x08\n\x04\x42USY\x10\x01\x12\n\n\x06KILLED\x10\x02\".\n\tStateList\x12!\n\x06states\x18\x07 \x03(\x0b\x32\x11.mserialize.State\"7\n\x0bMessageList\x12(\n\x08messages\x18\x08 \x03(\x0b\x32\x16.mserialize.LogMessageb\x06proto3') + serialized_pb=_b('\n\x0bstate.proto\x12\nmserialize\"\x1d\n\nLogMessage\x12\x0f\n\x07\x63ontent\x18\x01 \x01(\t\"\xb6\x01\n\x05State\x12\n\n\x02id\x18\x02 \x01(\x05\x12)\n\x04type\x18\x03 \x01(\x0e\x32\x1b.mserialize.State.StateType\x12\x0e\n\x06reason\x18\x04 \x01(\t\x12\x15\n\rnum_executing\x18\x05 \x01(\x05\x12\x11\n\twait_time\x18\x06 \x01(\x05\"<\n\tStateType\x12\t\n\x05READY\x10\x00\x12\x08\n\x04\x42USY\x10\x01\x12\n\n\x06KILLED\x10\x02\x12\x0e\n\nTERMINATED\x10\x03\".\n\tStateList\x12!\n\x06states\x18\x07 \x03(\x0b\x32\x11.mserialize.State\"7\n\x0bMessageList\x12(\n\x08messages\x18\x08 \x03(\x0b\x32\x16.mserialize.LogMessageb\x06proto3') ) @@ -42,11 +42,15 @@ name='KILLED', index=2, number=2, serialized_options=None, type=None), + _descriptor.EnumValueDescriptor( + name='TERMINATED', index=3, number=3, + serialized_options=None, + type=None), ], containing_type=None, serialized_options=None, serialized_start=181, - serialized_end=225, + serialized_end=241, ) _sym_db.RegisterEnumDescriptor(_STATE_STATETYPE) @@ -138,7 +142,7 @@ oneofs=[ ], serialized_start=59, - serialized_end=225, + serialized_end=241, ) @@ -168,8 +172,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=227, - serialized_end=273, + serialized_start=243, + serialized_end=289, ) @@ -199,8 +203,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=275, - serialized_end=330, + serialized_start=291, + serialized_end=346, ) _STATE.fields_by_name['type'].enum_type = _STATE_STATETYPE From 93d86048303f647dea15276d971004163f41632a Mon Sep 17 00:00:00 2001 From: Eric Hennenfent Date: Thu, 30 Jan 2020 13:33:30 -0800 Subject: [PATCH 06/18] Blacken --- manticore/core/state_pb2.py | 486 +++++++++++++++++++++--------------- manticore/core/worker.py | 54 ++-- 2 files changed, 312 insertions(+), 228 deletions(-) diff --git a/manticore/core/state_pb2.py b/manticore/core/state_pb2.py index 5d251ddb9..50ffbb525 100644 --- a/manticore/core/state_pb2.py +++ b/manticore/core/state_pb2.py @@ -2,247 +2,335 @@ # source: state.proto import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) + +_b = sys.version_info[0] < 3 and (lambda x: x) or (lambda x: x.encode("latin1")) from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from google.protobuf import reflection as _reflection from google.protobuf import symbol_database as _symbol_database + # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() - - DESCRIPTOR = _descriptor.FileDescriptor( - name='state.proto', - package='mserialize', - syntax='proto3', - serialized_options=None, - serialized_pb=_b('\n\x0bstate.proto\x12\nmserialize\"\x1d\n\nLogMessage\x12\x0f\n\x07\x63ontent\x18\x01 \x01(\t\"\xb6\x01\n\x05State\x12\n\n\x02id\x18\x02 \x01(\x05\x12)\n\x04type\x18\x03 \x01(\x0e\x32\x1b.mserialize.State.StateType\x12\x0e\n\x06reason\x18\x04 \x01(\t\x12\x15\n\rnum_executing\x18\x05 \x01(\x05\x12\x11\n\twait_time\x18\x06 \x01(\x05\"<\n\tStateType\x12\t\n\x05READY\x10\x00\x12\x08\n\x04\x42USY\x10\x01\x12\n\n\x06KILLED\x10\x02\x12\x0e\n\nTERMINATED\x10\x03\".\n\tStateList\x12!\n\x06states\x18\x07 \x03(\x0b\x32\x11.mserialize.State\"7\n\x0bMessageList\x12(\n\x08messages\x18\x08 \x03(\x0b\x32\x16.mserialize.LogMessageb\x06proto3') + name="state.proto", + package="mserialize", + syntax="proto3", + serialized_options=None, + serialized_pb=_b( + '\n\x0bstate.proto\x12\nmserialize"\x1d\n\nLogMessage\x12\x0f\n\x07\x63ontent\x18\x01 \x01(\t"\xb6\x01\n\x05State\x12\n\n\x02id\x18\x02 \x01(\x05\x12)\n\x04type\x18\x03 \x01(\x0e\x32\x1b.mserialize.State.StateType\x12\x0e\n\x06reason\x18\x04 \x01(\t\x12\x15\n\rnum_executing\x18\x05 \x01(\x05\x12\x11\n\twait_time\x18\x06 \x01(\x05"<\n\tStateType\x12\t\n\x05READY\x10\x00\x12\x08\n\x04\x42USY\x10\x01\x12\n\n\x06KILLED\x10\x02\x12\x0e\n\nTERMINATED\x10\x03".\n\tStateList\x12!\n\x06states\x18\x07 \x03(\x0b\x32\x11.mserialize.State"7\n\x0bMessageList\x12(\n\x08messages\x18\x08 \x03(\x0b\x32\x16.mserialize.LogMessageb\x06proto3' + ), ) - _STATE_STATETYPE = _descriptor.EnumDescriptor( - name='StateType', - full_name='mserialize.State.StateType', - filename=None, - file=DESCRIPTOR, - values=[ - _descriptor.EnumValueDescriptor( - name='READY', index=0, number=0, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='BUSY', index=1, number=1, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='KILLED', index=2, number=2, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='TERMINATED', index=3, number=3, - serialized_options=None, - type=None), - ], - containing_type=None, - serialized_options=None, - serialized_start=181, - serialized_end=241, + name="StateType", + full_name="mserialize.State.StateType", + filename=None, + file=DESCRIPTOR, + values=[ + _descriptor.EnumValueDescriptor( + name="READY", index=0, number=0, serialized_options=None, type=None + ), + _descriptor.EnumValueDescriptor( + name="BUSY", index=1, number=1, serialized_options=None, type=None + ), + _descriptor.EnumValueDescriptor( + name="KILLED", index=2, number=2, serialized_options=None, type=None + ), + _descriptor.EnumValueDescriptor( + name="TERMINATED", index=3, number=3, serialized_options=None, type=None + ), + ], + containing_type=None, + serialized_options=None, + serialized_start=181, + serialized_end=241, ) _sym_db.RegisterEnumDescriptor(_STATE_STATETYPE) _LOGMESSAGE = _descriptor.Descriptor( - name='LogMessage', - full_name='mserialize.LogMessage', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='content', full_name='mserialize.LogMessage.content', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=27, - serialized_end=56, + name="LogMessage", + full_name="mserialize.LogMessage", + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name="content", + full_name="mserialize.LogMessage.content", + index=0, + number=1, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b("").decode("utf-8"), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ) + ], + extensions=[], + nested_types=[], + enum_types=[], + serialized_options=None, + is_extendable=False, + syntax="proto3", + extension_ranges=[], + oneofs=[], + serialized_start=27, + serialized_end=56, ) _STATE = _descriptor.Descriptor( - name='State', - full_name='mserialize.State', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='id', full_name='mserialize.State.id', index=0, - number=2, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='type', full_name='mserialize.State.type', index=1, - number=3, type=14, cpp_type=8, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='reason', full_name='mserialize.State.reason', index=2, - number=4, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='num_executing', full_name='mserialize.State.num_executing', index=3, - number=5, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='wait_time', full_name='mserialize.State.wait_time', index=4, - number=6, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - _STATE_STATETYPE, - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=59, - serialized_end=241, + name="State", + full_name="mserialize.State", + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name="id", + full_name="mserialize.State.id", + index=0, + number=2, + type=5, + cpp_type=1, + label=1, + has_default_value=False, + default_value=0, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ), + _descriptor.FieldDescriptor( + name="type", + full_name="mserialize.State.type", + index=1, + number=3, + type=14, + cpp_type=8, + label=1, + has_default_value=False, + default_value=0, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ), + _descriptor.FieldDescriptor( + name="reason", + full_name="mserialize.State.reason", + index=2, + number=4, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b("").decode("utf-8"), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ), + _descriptor.FieldDescriptor( + name="num_executing", + full_name="mserialize.State.num_executing", + index=3, + number=5, + type=5, + cpp_type=1, + label=1, + has_default_value=False, + default_value=0, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ), + _descriptor.FieldDescriptor( + name="wait_time", + full_name="mserialize.State.wait_time", + index=4, + number=6, + type=5, + cpp_type=1, + label=1, + has_default_value=False, + default_value=0, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ), + ], + extensions=[], + nested_types=[], + enum_types=[_STATE_STATETYPE], + serialized_options=None, + is_extendable=False, + syntax="proto3", + extension_ranges=[], + oneofs=[], + serialized_start=59, + serialized_end=241, ) _STATELIST = _descriptor.Descriptor( - name='StateList', - full_name='mserialize.StateList', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='states', full_name='mserialize.StateList.states', index=0, - number=7, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=243, - serialized_end=289, + name="StateList", + full_name="mserialize.StateList", + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name="states", + full_name="mserialize.StateList.states", + index=0, + number=7, + type=11, + cpp_type=10, + label=3, + has_default_value=False, + default_value=[], + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ) + ], + extensions=[], + nested_types=[], + enum_types=[], + serialized_options=None, + is_extendable=False, + syntax="proto3", + extension_ranges=[], + oneofs=[], + serialized_start=243, + serialized_end=289, ) _MESSAGELIST = _descriptor.Descriptor( - name='MessageList', - full_name='mserialize.MessageList', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='messages', full_name='mserialize.MessageList.messages', index=0, - number=8, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=291, - serialized_end=346, + name="MessageList", + full_name="mserialize.MessageList", + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name="messages", + full_name="mserialize.MessageList.messages", + index=0, + number=8, + type=11, + cpp_type=10, + label=3, + has_default_value=False, + default_value=[], + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ) + ], + extensions=[], + nested_types=[], + enum_types=[], + serialized_options=None, + is_extendable=False, + syntax="proto3", + extension_ranges=[], + oneofs=[], + serialized_start=291, + serialized_end=346, ) -_STATE.fields_by_name['type'].enum_type = _STATE_STATETYPE +_STATE.fields_by_name["type"].enum_type = _STATE_STATETYPE _STATE_STATETYPE.containing_type = _STATE -_STATELIST.fields_by_name['states'].message_type = _STATE -_MESSAGELIST.fields_by_name['messages'].message_type = _LOGMESSAGE -DESCRIPTOR.message_types_by_name['LogMessage'] = _LOGMESSAGE -DESCRIPTOR.message_types_by_name['State'] = _STATE -DESCRIPTOR.message_types_by_name['StateList'] = _STATELIST -DESCRIPTOR.message_types_by_name['MessageList'] = _MESSAGELIST +_STATELIST.fields_by_name["states"].message_type = _STATE +_MESSAGELIST.fields_by_name["messages"].message_type = _LOGMESSAGE +DESCRIPTOR.message_types_by_name["LogMessage"] = _LOGMESSAGE +DESCRIPTOR.message_types_by_name["State"] = _STATE +DESCRIPTOR.message_types_by_name["StateList"] = _STATELIST +DESCRIPTOR.message_types_by_name["MessageList"] = _MESSAGELIST _sym_db.RegisterFileDescriptor(DESCRIPTOR) -LogMessage = _reflection.GeneratedProtocolMessageType('LogMessage', (_message.Message,), dict( - DESCRIPTOR = _LOGMESSAGE, - __module__ = 'state_pb2' - # @@protoc_insertion_point(class_scope:mserialize.LogMessage) - )) +LogMessage = _reflection.GeneratedProtocolMessageType( + "LogMessage", + (_message.Message,), + dict( + DESCRIPTOR=_LOGMESSAGE, + __module__="state_pb2" + # @@protoc_insertion_point(class_scope:mserialize.LogMessage) + ), +) _sym_db.RegisterMessage(LogMessage) -State = _reflection.GeneratedProtocolMessageType('State', (_message.Message,), dict( - DESCRIPTOR = _STATE, - __module__ = 'state_pb2' - # @@protoc_insertion_point(class_scope:mserialize.State) - )) +State = _reflection.GeneratedProtocolMessageType( + "State", + (_message.Message,), + dict( + DESCRIPTOR=_STATE, + __module__="state_pb2" + # @@protoc_insertion_point(class_scope:mserialize.State) + ), +) _sym_db.RegisterMessage(State) -StateList = _reflection.GeneratedProtocolMessageType('StateList', (_message.Message,), dict( - DESCRIPTOR = _STATELIST, - __module__ = 'state_pb2' - # @@protoc_insertion_point(class_scope:mserialize.StateList) - )) +StateList = _reflection.GeneratedProtocolMessageType( + "StateList", + (_message.Message,), + dict( + DESCRIPTOR=_STATELIST, + __module__="state_pb2" + # @@protoc_insertion_point(class_scope:mserialize.StateList) + ), +) _sym_db.RegisterMessage(StateList) -MessageList = _reflection.GeneratedProtocolMessageType('MessageList', (_message.Message,), dict( - DESCRIPTOR = _MESSAGELIST, - __module__ = 'state_pb2' - # @@protoc_insertion_point(class_scope:mserialize.MessageList) - )) +MessageList = _reflection.GeneratedProtocolMessageType( + "MessageList", + (_message.Message,), + dict( + DESCRIPTOR=_MESSAGELIST, + __module__="state_pb2" + # @@protoc_insertion_point(class_scope:mserialize.MessageList) + ), +) _sym_db.RegisterMessage(MessageList) diff --git a/manticore/core/worker.py b/manticore/core/worker.py index 29a20f9bf..394958747 100644 --- a/manticore/core/worker.py +++ b/manticore/core/worker.py @@ -233,37 +233,37 @@ def join(self): self._p.join() self._p = None -class MonitorWorker(WorkerThread): +class MonitorWorker(WorkerThread): def obtain_states(self, m): serialized_states = StateList() serialized_messages = MessageList() - + for b in m._busy_states: bstate = State() bstate.id = b bstate.reason = "Busy executing" bstate.type = State.BUSY serialized_states.states.extend([bstate]) - + for r in m._ready_states: rstate = State() - rstate.id = r + rstate.id = r rstate.type = State.READY serialized_states.states.extend([rstate]) for t in m._terminated_states: tstate = State() - tstate.id = t + tstate.id = t tstate.type = State.TERMINATED serialized_states.states.extend([tstate]) for k in m._killed_states: kstate = State() - kstate.id = k + kstate.id = k kstate.type = State.KILLED serialized_states.states.extend([kstate]) - + return serialized_states def run(self, *args): @@ -277,33 +277,30 @@ def run(self, *args): m = self.manticore m._is_main = False # This will mark our copy of manticore - import time import socket - - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - - HOST = '127.0.0.1' + + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + + HOST = "127.0.0.1" PORT = 1337 - s.bind((HOST, PORT)) - - logger.debug( - "Created socket in threads bound to host %s, port %d", - HOST, - PORT, - ) + s.bind((HOST, PORT)) + + logger.debug("Created socket in threads bound to host %s, port %d", HOST, PORT) - s.listen(5) + s.listen(5) socket_list = [s] serialized_states = self.obtain_states(m).SerializeToString() changed = False - + with WithKeyboardInterruptAs(m.kill): while m.is_running(): # TODO: Exits after state exploration, but not finalization - # Establish connection with client. - read_sockets, write_sockets, error_sockets = select.select(socket_list, socket_list, [], 0) + # Establish connection with client. + read_sockets, write_sockets, error_sockets = select.select( + socket_list, socket_list, [], 0 + ) states = self.obtain_states(m) @@ -311,15 +308,12 @@ def run(self, *args): serialized_states = states changed = True - if len(read_sockets): for sock in read_sockets: if sock is s: - - logger.debug( - "Received connection from Manticore TUI" - ) + + logger.debug("Received connection from Manticore TUI") c, addr = sock.accept() socket_list.append(c) @@ -328,6 +322,8 @@ def run(self, *args): if len(write_sockets) and changed: for sock in write_sockets: - print(m._busy_states, m._ready_states, m._terminated_states, m._killed_states) + print( + m._busy_states, m._ready_states, m._terminated_states, m._killed_states + ) sock.send(serialized_states.SerializeToString()) changed = False From 7cf87522d20457f6b3c17efdc364583f97b20a14 Mon Sep 17 00:00:00 2001 From: Philip Wang Date: Fri, 31 Jan 2020 11:17:35 -0500 Subject: [PATCH 07/18] Update setup.py dependencies --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index fcf6655da..afb07c30a 100644 --- a/setup.py +++ b/setup.py @@ -46,6 +46,7 @@ def rtd_dependent_deps(): install_requires=[ "pyyaml", "wrapt", + "protobuf", # evm dependencies "pysha3", "prettytable", From 3d8dbdeb6f4e2047b327fb570e9d466e928442dc Mon Sep 17 00:00:00 2001 From: Philip Wang Date: Fri, 31 Jan 2020 12:54:25 -0500 Subject: [PATCH 08/18] Remove state debugging messages --- manticore/core/worker.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/manticore/core/worker.py b/manticore/core/worker.py index 394958747..8830df3fc 100644 --- a/manticore/core/worker.py +++ b/manticore/core/worker.py @@ -322,8 +322,5 @@ def run(self, *args): if len(write_sockets) and changed: for sock in write_sockets: - print( - m._busy_states, m._ready_states, m._terminated_states, m._killed_states - ) sock.send(serialized_states.SerializeToString()) changed = False From 3a4f8dbaf80dc946d8885784061561f1fdc62a13 Mon Sep 17 00:00:00 2001 From: Philip Wang Date: Fri, 31 Jan 2020 13:07:53 -0500 Subject: [PATCH 09/18] Update setup.py to build protobuf protocol upon install --- setup.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index afb07c30a..e8dcc6106 100644 --- a/setup.py +++ b/setup.py @@ -1,8 +1,11 @@ import os +import subprocess from setuptools import setup, find_packages on_rtd = os.environ.get("READTHEDOCS") == "True" +protoc_dir = "manticore/core" +protoc_cmd = f"protoc -I={protoc_dir} --python_out={protoc_dir} {protoc_dir}/state.proto".split() def rtd_dependent_deps(): # RTD tries to build z3, ooms, and fails to build. @@ -11,7 +14,6 @@ def rtd_dependent_deps(): else: return ["z3-solver"] - # If you update native_deps please update the `REQUIREMENTS_TO_IMPORTS` dict in `utils/install_helper.py` # (we need to know how to import a given native dependency so we can check if native dependencies are installed) native_deps = ["capstone==4.0.1", "pyelftools", "unicorn==1.0.2rc1"] @@ -34,6 +36,7 @@ def rtd_dependent_deps(): "lint": lint_deps, } +subprocess.Popen(protoc_cmd) setup( name="manticore", @@ -61,4 +64,4 @@ def rtd_dependent_deps(): extras_require=extra_require, entry_points={"console_scripts": ["manticore = manticore.__main__:main"]}, classifiers=["License :: OSI Approved :: GNU Affero General Public License v3"], -) +) \ No newline at end of file From 9da250da9f0858218af5cea120aecf3ef52dabda Mon Sep 17 00:00:00 2001 From: Philip Wang Date: Fri, 31 Jan 2020 13:08:25 -0500 Subject: [PATCH 10/18] Remove previously generated state_pb2.py --- manticore/core/state_pb2.py | 337 ------------------------------------ 1 file changed, 337 deletions(-) delete mode 100644 manticore/core/state_pb2.py diff --git a/manticore/core/state_pb2.py b/manticore/core/state_pb2.py deleted file mode 100644 index 50ffbb525..000000000 --- a/manticore/core/state_pb2.py +++ /dev/null @@ -1,337 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: state.proto - -import sys - -_b = sys.version_info[0] < 3 and (lambda x: x) or (lambda x: x.encode("latin1")) -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database - -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - -DESCRIPTOR = _descriptor.FileDescriptor( - name="state.proto", - package="mserialize", - syntax="proto3", - serialized_options=None, - serialized_pb=_b( - '\n\x0bstate.proto\x12\nmserialize"\x1d\n\nLogMessage\x12\x0f\n\x07\x63ontent\x18\x01 \x01(\t"\xb6\x01\n\x05State\x12\n\n\x02id\x18\x02 \x01(\x05\x12)\n\x04type\x18\x03 \x01(\x0e\x32\x1b.mserialize.State.StateType\x12\x0e\n\x06reason\x18\x04 \x01(\t\x12\x15\n\rnum_executing\x18\x05 \x01(\x05\x12\x11\n\twait_time\x18\x06 \x01(\x05"<\n\tStateType\x12\t\n\x05READY\x10\x00\x12\x08\n\x04\x42USY\x10\x01\x12\n\n\x06KILLED\x10\x02\x12\x0e\n\nTERMINATED\x10\x03".\n\tStateList\x12!\n\x06states\x18\x07 \x03(\x0b\x32\x11.mserialize.State"7\n\x0bMessageList\x12(\n\x08messages\x18\x08 \x03(\x0b\x32\x16.mserialize.LogMessageb\x06proto3' - ), -) - - -_STATE_STATETYPE = _descriptor.EnumDescriptor( - name="StateType", - full_name="mserialize.State.StateType", - filename=None, - file=DESCRIPTOR, - values=[ - _descriptor.EnumValueDescriptor( - name="READY", index=0, number=0, serialized_options=None, type=None - ), - _descriptor.EnumValueDescriptor( - name="BUSY", index=1, number=1, serialized_options=None, type=None - ), - _descriptor.EnumValueDescriptor( - name="KILLED", index=2, number=2, serialized_options=None, type=None - ), - _descriptor.EnumValueDescriptor( - name="TERMINATED", index=3, number=3, serialized_options=None, type=None - ), - ], - containing_type=None, - serialized_options=None, - serialized_start=181, - serialized_end=241, -) -_sym_db.RegisterEnumDescriptor(_STATE_STATETYPE) - - -_LOGMESSAGE = _descriptor.Descriptor( - name="LogMessage", - full_name="mserialize.LogMessage", - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name="content", - full_name="mserialize.LogMessage.content", - index=0, - number=1, - type=9, - cpp_type=9, - label=1, - has_default_value=False, - default_value=_b("").decode("utf-8"), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - serialized_options=None, - file=DESCRIPTOR, - ) - ], - extensions=[], - nested_types=[], - enum_types=[], - serialized_options=None, - is_extendable=False, - syntax="proto3", - extension_ranges=[], - oneofs=[], - serialized_start=27, - serialized_end=56, -) - - -_STATE = _descriptor.Descriptor( - name="State", - full_name="mserialize.State", - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name="id", - full_name="mserialize.State.id", - index=0, - number=2, - type=5, - cpp_type=1, - label=1, - has_default_value=False, - default_value=0, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - serialized_options=None, - file=DESCRIPTOR, - ), - _descriptor.FieldDescriptor( - name="type", - full_name="mserialize.State.type", - index=1, - number=3, - type=14, - cpp_type=8, - label=1, - has_default_value=False, - default_value=0, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - serialized_options=None, - file=DESCRIPTOR, - ), - _descriptor.FieldDescriptor( - name="reason", - full_name="mserialize.State.reason", - index=2, - number=4, - type=9, - cpp_type=9, - label=1, - has_default_value=False, - default_value=_b("").decode("utf-8"), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - serialized_options=None, - file=DESCRIPTOR, - ), - _descriptor.FieldDescriptor( - name="num_executing", - full_name="mserialize.State.num_executing", - index=3, - number=5, - type=5, - cpp_type=1, - label=1, - has_default_value=False, - default_value=0, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - serialized_options=None, - file=DESCRIPTOR, - ), - _descriptor.FieldDescriptor( - name="wait_time", - full_name="mserialize.State.wait_time", - index=4, - number=6, - type=5, - cpp_type=1, - label=1, - has_default_value=False, - default_value=0, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - serialized_options=None, - file=DESCRIPTOR, - ), - ], - extensions=[], - nested_types=[], - enum_types=[_STATE_STATETYPE], - serialized_options=None, - is_extendable=False, - syntax="proto3", - extension_ranges=[], - oneofs=[], - serialized_start=59, - serialized_end=241, -) - - -_STATELIST = _descriptor.Descriptor( - name="StateList", - full_name="mserialize.StateList", - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name="states", - full_name="mserialize.StateList.states", - index=0, - number=7, - type=11, - cpp_type=10, - label=3, - has_default_value=False, - default_value=[], - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - serialized_options=None, - file=DESCRIPTOR, - ) - ], - extensions=[], - nested_types=[], - enum_types=[], - serialized_options=None, - is_extendable=False, - syntax="proto3", - extension_ranges=[], - oneofs=[], - serialized_start=243, - serialized_end=289, -) - - -_MESSAGELIST = _descriptor.Descriptor( - name="MessageList", - full_name="mserialize.MessageList", - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name="messages", - full_name="mserialize.MessageList.messages", - index=0, - number=8, - type=11, - cpp_type=10, - label=3, - has_default_value=False, - default_value=[], - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - serialized_options=None, - file=DESCRIPTOR, - ) - ], - extensions=[], - nested_types=[], - enum_types=[], - serialized_options=None, - is_extendable=False, - syntax="proto3", - extension_ranges=[], - oneofs=[], - serialized_start=291, - serialized_end=346, -) - -_STATE.fields_by_name["type"].enum_type = _STATE_STATETYPE -_STATE_STATETYPE.containing_type = _STATE -_STATELIST.fields_by_name["states"].message_type = _STATE -_MESSAGELIST.fields_by_name["messages"].message_type = _LOGMESSAGE -DESCRIPTOR.message_types_by_name["LogMessage"] = _LOGMESSAGE -DESCRIPTOR.message_types_by_name["State"] = _STATE -DESCRIPTOR.message_types_by_name["StateList"] = _STATELIST -DESCRIPTOR.message_types_by_name["MessageList"] = _MESSAGELIST -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - -LogMessage = _reflection.GeneratedProtocolMessageType( - "LogMessage", - (_message.Message,), - dict( - DESCRIPTOR=_LOGMESSAGE, - __module__="state_pb2" - # @@protoc_insertion_point(class_scope:mserialize.LogMessage) - ), -) -_sym_db.RegisterMessage(LogMessage) - -State = _reflection.GeneratedProtocolMessageType( - "State", - (_message.Message,), - dict( - DESCRIPTOR=_STATE, - __module__="state_pb2" - # @@protoc_insertion_point(class_scope:mserialize.State) - ), -) -_sym_db.RegisterMessage(State) - -StateList = _reflection.GeneratedProtocolMessageType( - "StateList", - (_message.Message,), - dict( - DESCRIPTOR=_STATELIST, - __module__="state_pb2" - # @@protoc_insertion_point(class_scope:mserialize.StateList) - ), -) -_sym_db.RegisterMessage(StateList) - -MessageList = _reflection.GeneratedProtocolMessageType( - "MessageList", - (_message.Message,), - dict( - DESCRIPTOR=_MESSAGELIST, - __module__="state_pb2" - # @@protoc_insertion_point(class_scope:mserialize.MessageList) - ), -) -_sym_db.RegisterMessage(MessageList) - - -# @@protoc_insertion_point(module_scope) From ccf497a25b234b765a9437e5c5016fea7571981f Mon Sep 17 00:00:00 2001 From: Philip Wang Date: Fri, 31 Jan 2020 13:31:54 -0500 Subject: [PATCH 11/18] Change subprocess.Popen to subprocess.check_output --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e8dcc6106..a44383499 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ def rtd_dependent_deps(): "lint": lint_deps, } -subprocess.Popen(protoc_cmd) +output = subprocess.check_output(protoc_cmd) setup( name="manticore", From b0ac3f3f74d703fe07d1c1c9234fd9e19002c10d Mon Sep 17 00:00:00 2001 From: Philip Wang Date: Fri, 31 Jan 2020 13:39:35 -0500 Subject: [PATCH 12/18] Remove extraneous output --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a44383499..84731df82 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ def rtd_dependent_deps(): "lint": lint_deps, } -output = subprocess.check_output(protoc_cmd) +subprocess.check_output(protoc_cmd) setup( name="manticore", From 577fce440f41133a0149e184ededf193c449e1bc Mon Sep 17 00:00:00 2001 From: Eric Hennenfent Date: Mon, 3 Feb 2020 14:09:43 -0800 Subject: [PATCH 13/18] First attempt at fixing protobuf installation It might work, it might not. We'll let the CI sort it out. --- setup.py | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 24f43653c..bee360ae0 100644 --- a/setup.py +++ b/setup.py @@ -1,10 +1,33 @@ import os +import subprocess from setuptools import setup, find_packages +from distutils.spawn import find_executable +from setuptools.command.develop import develop +from setuptools.command.install import install + + +def compile_protobufs(): + protoc_dir = "manticore/core" + protoc = os.environ.get("PROTOC", find_executable("protoc")) + subprocess.check_call( + [protoc, *"-I={protoc_dir} --python_out={protoc_dir} {protoc_dir}/state.proto".split()] + ) + + +class PostDevelop(develop): + def run(self): + compile_protobufs() + develop.run(self) + + +class PostInstall(install): + def run(self): + compile_protobufs() + install.run(self) + on_rtd = os.environ.get("READTHEDOCS") == "True" -protoc_dir = "manticore/core" -protoc_cmd = f"protoc -I={protoc_dir} --python_out={protoc_dir} {protoc_dir}/state.proto".split() def rtd_dependent_deps(): # RTD tries to build z3, ooms, and fails to build. @@ -13,6 +36,7 @@ def rtd_dependent_deps(): else: return ["z3-solver"] + # If you update native_deps please update the `REQUIREMENTS_TO_IMPORTS` dict in `utils/install_helper.py` # (we need to know how to import a given native dependency so we can check if native dependencies are installed) native_deps = ["capstone==4.0.1", "pyelftools", "unicorn==1.0.2rc1"] @@ -67,4 +91,5 @@ def rtd_dependent_deps(): extras_require=extra_require, entry_points={"console_scripts": ["manticore = manticore.__main__:main"]}, classifiers=["License :: OSI Approved :: GNU Affero General Public License v3"], -) \ No newline at end of file + cmdclass={"develop": PostDevelop, "install": PostInstall}, +) From 6eb5e6b12b9ac35ba1a660517516b0dbdd914c01 Mon Sep 17 00:00:00 2001 From: Eric Hennenfent Date: Mon, 3 Feb 2020 14:13:45 -0800 Subject: [PATCH 14/18] Can't forget the f-string --- setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index bee360ae0..c193b463f 100644 --- a/setup.py +++ b/setup.py @@ -10,20 +10,20 @@ def compile_protobufs(): protoc_dir = "manticore/core" protoc = os.environ.get("PROTOC", find_executable("protoc")) subprocess.check_call( - [protoc, *"-I={protoc_dir} --python_out={protoc_dir} {protoc_dir}/state.proto".split()] + [protoc, *f"-I={protoc_dir} --python_out={protoc_dir} {protoc_dir}/state.proto".split()] ) class PostDevelop(develop): def run(self): - compile_protobufs() develop.run(self) + compile_protobufs() class PostInstall(install): def run(self): - compile_protobufs() install.run(self) + compile_protobufs() on_rtd = os.environ.get("READTHEDOCS") == "True" From eed5d9f2fc659264301d1a7843383c63a84bfa13 Mon Sep 17 00:00:00 2001 From: Eric Hennenfent Date: Mon, 3 Feb 2020 14:57:52 -0800 Subject: [PATCH 15/18] Error on missing protoc --- setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.py b/setup.py index c193b463f..f4c063bec 100644 --- a/setup.py +++ b/setup.py @@ -9,6 +9,8 @@ def compile_protobufs(): protoc_dir = "manticore/core" protoc = os.environ.get("PROTOC", find_executable("protoc")) + if not protoc: + raise RuntimeError("Couldn't find protoc compiler!") subprocess.check_call( [protoc, *f"-I={protoc_dir} --python_out={protoc_dir} {protoc_dir}/state.proto".split()] ) From 1a4ab87d108a065ca6048e7693fb8dbb622ea269 Mon Sep 17 00:00:00 2001 From: Eric Hennenfent Date: Mon, 3 Feb 2020 14:58:30 -0800 Subject: [PATCH 16/18] Disable auto-generation of protobuf file --- manticore/core/state_pb2.py | 333 ++++++++++++++++++++++++++++++++++++ setup.py | 27 --- 2 files changed, 333 insertions(+), 27 deletions(-) create mode 100644 manticore/core/state_pb2.py diff --git a/manticore/core/state_pb2.py b/manticore/core/state_pb2.py new file mode 100644 index 000000000..97f37dafe --- /dev/null +++ b/manticore/core/state_pb2.py @@ -0,0 +1,333 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: state.proto + +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database + +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +DESCRIPTOR = _descriptor.FileDescriptor( + name="state.proto", + package="mserialize", + syntax="proto3", + serialized_options=None, + serialized_pb=b'\n\x0bstate.proto\x12\nmserialize"\x1d\n\nLogMessage\x12\x0f\n\x07\x63ontent\x18\x01 \x01(\t"\xb6\x01\n\x05State\x12\n\n\x02id\x18\x02 \x01(\x05\x12)\n\x04type\x18\x03 \x01(\x0e\x32\x1b.mserialize.State.StateType\x12\x0e\n\x06reason\x18\x04 \x01(\t\x12\x15\n\rnum_executing\x18\x05 \x01(\x05\x12\x11\n\twait_time\x18\x06 \x01(\x05"<\n\tStateType\x12\t\n\x05READY\x10\x00\x12\x08\n\x04\x42USY\x10\x01\x12\n\n\x06KILLED\x10\x02\x12\x0e\n\nTERMINATED\x10\x03".\n\tStateList\x12!\n\x06states\x18\x07 \x03(\x0b\x32\x11.mserialize.State"7\n\x0bMessageList\x12(\n\x08messages\x18\x08 \x03(\x0b\x32\x16.mserialize.LogMessageb\x06proto3', +) + + +_STATE_STATETYPE = _descriptor.EnumDescriptor( + name="StateType", + full_name="mserialize.State.StateType", + filename=None, + file=DESCRIPTOR, + values=[ + _descriptor.EnumValueDescriptor( + name="READY", index=0, number=0, serialized_options=None, type=None + ), + _descriptor.EnumValueDescriptor( + name="BUSY", index=1, number=1, serialized_options=None, type=None + ), + _descriptor.EnumValueDescriptor( + name="KILLED", index=2, number=2, serialized_options=None, type=None + ), + _descriptor.EnumValueDescriptor( + name="TERMINATED", index=3, number=3, serialized_options=None, type=None + ), + ], + containing_type=None, + serialized_options=None, + serialized_start=181, + serialized_end=241, +) +_sym_db.RegisterEnumDescriptor(_STATE_STATETYPE) + + +_LOGMESSAGE = _descriptor.Descriptor( + name="LogMessage", + full_name="mserialize.LogMessage", + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name="content", + full_name="mserialize.LogMessage.content", + index=0, + number=1, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=b"".decode("utf-8"), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ) + ], + extensions=[], + nested_types=[], + enum_types=[], + serialized_options=None, + is_extendable=False, + syntax="proto3", + extension_ranges=[], + oneofs=[], + serialized_start=27, + serialized_end=56, +) + + +_STATE = _descriptor.Descriptor( + name="State", + full_name="mserialize.State", + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name="id", + full_name="mserialize.State.id", + index=0, + number=2, + type=5, + cpp_type=1, + label=1, + has_default_value=False, + default_value=0, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ), + _descriptor.FieldDescriptor( + name="type", + full_name="mserialize.State.type", + index=1, + number=3, + type=14, + cpp_type=8, + label=1, + has_default_value=False, + default_value=0, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ), + _descriptor.FieldDescriptor( + name="reason", + full_name="mserialize.State.reason", + index=2, + number=4, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=b"".decode("utf-8"), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ), + _descriptor.FieldDescriptor( + name="num_executing", + full_name="mserialize.State.num_executing", + index=3, + number=5, + type=5, + cpp_type=1, + label=1, + has_default_value=False, + default_value=0, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ), + _descriptor.FieldDescriptor( + name="wait_time", + full_name="mserialize.State.wait_time", + index=4, + number=6, + type=5, + cpp_type=1, + label=1, + has_default_value=False, + default_value=0, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ), + ], + extensions=[], + nested_types=[], + enum_types=[_STATE_STATETYPE], + serialized_options=None, + is_extendable=False, + syntax="proto3", + extension_ranges=[], + oneofs=[], + serialized_start=59, + serialized_end=241, +) + + +_STATELIST = _descriptor.Descriptor( + name="StateList", + full_name="mserialize.StateList", + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name="states", + full_name="mserialize.StateList.states", + index=0, + number=7, + type=11, + cpp_type=10, + label=3, + has_default_value=False, + default_value=[], + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ) + ], + extensions=[], + nested_types=[], + enum_types=[], + serialized_options=None, + is_extendable=False, + syntax="proto3", + extension_ranges=[], + oneofs=[], + serialized_start=243, + serialized_end=289, +) + + +_MESSAGELIST = _descriptor.Descriptor( + name="MessageList", + full_name="mserialize.MessageList", + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name="messages", + full_name="mserialize.MessageList.messages", + index=0, + number=8, + type=11, + cpp_type=10, + label=3, + has_default_value=False, + default_value=[], + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ) + ], + extensions=[], + nested_types=[], + enum_types=[], + serialized_options=None, + is_extendable=False, + syntax="proto3", + extension_ranges=[], + oneofs=[], + serialized_start=291, + serialized_end=346, +) + +_STATE.fields_by_name["type"].enum_type = _STATE_STATETYPE +_STATE_STATETYPE.containing_type = _STATE +_STATELIST.fields_by_name["states"].message_type = _STATE +_MESSAGELIST.fields_by_name["messages"].message_type = _LOGMESSAGE +DESCRIPTOR.message_types_by_name["LogMessage"] = _LOGMESSAGE +DESCRIPTOR.message_types_by_name["State"] = _STATE +DESCRIPTOR.message_types_by_name["StateList"] = _STATELIST +DESCRIPTOR.message_types_by_name["MessageList"] = _MESSAGELIST +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + +LogMessage = _reflection.GeneratedProtocolMessageType( + "LogMessage", + (_message.Message,), + { + "DESCRIPTOR": _LOGMESSAGE, + "__module__": "state_pb2" + # @@protoc_insertion_point(class_scope:mserialize.LogMessage) + }, +) +_sym_db.RegisterMessage(LogMessage) + +State = _reflection.GeneratedProtocolMessageType( + "State", + (_message.Message,), + { + "DESCRIPTOR": _STATE, + "__module__": "state_pb2" + # @@protoc_insertion_point(class_scope:mserialize.State) + }, +) +_sym_db.RegisterMessage(State) + +StateList = _reflection.GeneratedProtocolMessageType( + "StateList", + (_message.Message,), + { + "DESCRIPTOR": _STATELIST, + "__module__": "state_pb2" + # @@protoc_insertion_point(class_scope:mserialize.StateList) + }, +) +_sym_db.RegisterMessage(StateList) + +MessageList = _reflection.GeneratedProtocolMessageType( + "MessageList", + (_message.Message,), + { + "DESCRIPTOR": _MESSAGELIST, + "__module__": "state_pb2" + # @@protoc_insertion_point(class_scope:mserialize.MessageList) + }, +) +_sym_db.RegisterMessage(MessageList) + + +# @@protoc_insertion_point(module_scope) diff --git a/setup.py b/setup.py index f4c063bec..e92fb8e42 100644 --- a/setup.py +++ b/setup.py @@ -1,31 +1,5 @@ import os -import subprocess from setuptools import setup, find_packages -from distutils.spawn import find_executable -from setuptools.command.develop import develop -from setuptools.command.install import install - - -def compile_protobufs(): - protoc_dir = "manticore/core" - protoc = os.environ.get("PROTOC", find_executable("protoc")) - if not protoc: - raise RuntimeError("Couldn't find protoc compiler!") - subprocess.check_call( - [protoc, *f"-I={protoc_dir} --python_out={protoc_dir} {protoc_dir}/state.proto".split()] - ) - - -class PostDevelop(develop): - def run(self): - develop.run(self) - compile_protobufs() - - -class PostInstall(install): - def run(self): - install.run(self) - compile_protobufs() on_rtd = os.environ.get("READTHEDOCS") == "True" @@ -93,5 +67,4 @@ def rtd_dependent_deps(): extras_require=extra_require, entry_points={"console_scripts": ["manticore = manticore.__main__:main"]}, classifiers=["License :: OSI Approved :: GNU Affero General Public License v3"], - cmdclass={"develop": PostDevelop, "install": PostInstall}, ) From 8880d7a35bcc9b5d0689346ed09a9ff8c45ad00f Mon Sep 17 00:00:00 2001 From: Eric Hennenfent Date: Mon, 3 Feb 2020 15:09:28 -0800 Subject: [PATCH 17/18] Ignore pb2_errors --- mypy.ini | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mypy.ini b/mypy.ini index f4c2c4bc7..d2af18d21 100644 --- a/mypy.ini +++ b/mypy.ini @@ -42,3 +42,6 @@ ignore_missing_imports = True [mypy-wasm.*] ignore_missing_imports = True + +[mypy-manticore.core.state_pb2] +ignore_errors = True From 6d05808521a40c676af753d8e0c981f882919714 Mon Sep 17 00:00:00 2001 From: Eric Hennenfent Date: Tue, 4 Feb 2020 09:59:53 -0800 Subject: [PATCH 18/18] Disable monitor start See if this makes the EVM tests pass --- manticore/core/manticore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manticore/core/manticore.py b/manticore/core/manticore.py index ee6bbfb19..9d19e256b 100644 --- a/manticore/core/manticore.py +++ b/manticore/core/manticore.py @@ -954,7 +954,7 @@ def run(self): self._publish("will_run", self.ready_states) self._running.value = True - self._monitor.start() + # self._monitor.start() # start all the workers! for w in self._workers: