Merge branch 'master' into handle-missing-ability

app/api/rest_api.py

@@ -128,9 +128,11 @@ async def upload_file(self, request):
         dir_name = request.headers.get('Directory', None)
         if dir_name:
             return await self.file_svc.save_multipart_file_upload(request, 'data/payloads/')
-        created_dir = os.path.normpath('/' + request.headers.get('X-Request-ID', str(uuid.uuid4()))).lstrip('/')
+        agent = request.headers.get('X-Request-ID', str(uuid.uuid4()))
+        created_dir = os.path.normpath('/' + agent).lstrip('/')
         saveto_dir = await self.file_svc.create_exfil_sub_directory(dir_name=created_dir)
-        return await self.file_svc.save_multipart_file_upload(request, saveto_dir)
+        operation_dir = await self.file_svc.create_exfil_operation_directory(dir_name=saveto_dir, agent_name=agent[-6:])
+        return await self.file_svc.save_multipart_file_upload(request, operation_dir)
 
     async def download_file(self, request):
         try:

app/service/data_svc.py

@@ -37,6 +37,10 @@
     'data/object_store',
 )
 
+PAYLOADS_CONFIG_STANDARD_KEY = 'standard_payloads'
+PAYLOADS_CONFIG_SPECIAL_KEY = 'special_payloads'
+PAYLOADS_CONFIG_EXTENSIONS_KEY = 'extensions'
+
 
 class DataService(DataServiceInterface, BaseService):
 
@@ -310,15 +314,50 @@ async def _load_objectives(self, plugin):
             await self.load_yaml_file(Objective, filename, plugin.access)
 
     async def _load_payloads(self, plugin):
+        payload_config = dict(
+            standard_payloads=dict(),
+            special_payloads=dict(),
+            extensions=dict(),
+        )
         for filename in glob.iglob('%s/payloads/*.yml' % plugin.data_dir, recursive=False):
             data = self.strip_yml(filename)
-            payload_config = self.get_config(name='payloads')
-            payload_config['standard_payloads'] = data[0]['standard_payloads']
-            payload_config['special_payloads'] = data[0]['special_payloads']
-            payload_config['extensions'] = data[0]['extensions']
-            await self._apply_special_payload_hooks(payload_config['special_payloads'])
-            await self._apply_special_extension_hooks(payload_config['extensions'])
-            self.apply_config(name='payloads', config=payload_config)
+            special_payloads = data[0].get(PAYLOADS_CONFIG_SPECIAL_KEY, dict())
+            extensions = data[0].get(PAYLOADS_CONFIG_EXTENSIONS_KEY, dict())
+            standard_payloads = data[0].get(PAYLOADS_CONFIG_STANDARD_KEY, dict())
+            payload_config[PAYLOADS_CONFIG_STANDARD_KEY].update(standard_payloads)
+            if special_payloads:
+                await self._apply_special_payload_hooks(special_payloads)
+                payload_config[PAYLOADS_CONFIG_SPECIAL_KEY].update(special_payloads)
+            if extensions:
+                await self._apply_special_extension_hooks(extensions)
+                payload_config[PAYLOADS_CONFIG_EXTENSIONS_KEY].update(extensions)
+        self._update_payload_config(payload_config, plugin.name)
+
+    def _update_payload_config(self, updates, curr_plugin_name):
+        payload_config = self.get_config(name='payloads')
+        curr_standard_payloads = payload_config.get(PAYLOADS_CONFIG_STANDARD_KEY, dict())
+        curr_special_payloads = payload_config.get(PAYLOADS_CONFIG_SPECIAL_KEY, dict())
+        curr_extensions = payload_config.get(PAYLOADS_CONFIG_EXTENSIONS_KEY, dict())
+        new_standard_payloads = updates.get(PAYLOADS_CONFIG_STANDARD_KEY, dict())
+        new_special_payloads = updates.get(PAYLOADS_CONFIG_SPECIAL_KEY, dict())
+        new_extensions = updates.get(PAYLOADS_CONFIG_EXTENSIONS_KEY, dict())
+
+        self._check_payload_overlaps(curr_standard_payloads, new_standard_payloads, PAYLOADS_CONFIG_STANDARD_KEY,
+                                     curr_plugin_name)
+        self._check_payload_overlaps(curr_special_payloads, new_special_payloads, PAYLOADS_CONFIG_SPECIAL_KEY,
+                                     curr_plugin_name)
+        self._check_payload_overlaps(curr_extensions, new_extensions, PAYLOADS_CONFIG_EXTENSIONS_KEY, curr_plugin_name)
+
+        payload_config[PAYLOADS_CONFIG_STANDARD_KEY] = {**curr_standard_payloads, **new_standard_payloads}
+        payload_config[PAYLOADS_CONFIG_SPECIAL_KEY] = {**curr_special_payloads, **new_special_payloads}
+        payload_config[PAYLOADS_CONFIG_EXTENSIONS_KEY] = {**curr_extensions, **new_extensions}
+        self.apply_config(name='payloads', config=payload_config)
+
+    def _check_payload_overlaps(self, old_dict, new_dict, config_section_name, curr_plugin_name):
+        overlap = set(old_dict).intersection(new_dict)
+        for payload in overlap:
+            self.log.warning('Config for %s already exists in the %s section of the payloads config and will be '
+                             'overridden by plugin %s', payload, config_section_name, curr_plugin_name)
 
     async def _load_planners(self, plugin):
         for filename in glob.iglob('%s/planners/*.yml' % plugin.data_dir, recursive=False):

app/service/file_svc.py

@@ -70,6 +70,17 @@ async def create_exfil_sub_directory(self, dir_name):
             os.makedirs(path)
         return path
 
+    async def create_exfil_operation_directory(self, dir_name, agent_name):
+        op_list = self.data_svc.ram['operations']
+        op_list_filtered = [x for x in op_list if x.state not in x.get_finished_states()]
+        special_chars = {ord(c): '_' for c in r':<>"/\|?*'}
+        agent_opid = [(x.name.translate(special_chars), '_', x.start.strftime("%Y-%m-%d_%H%M%SZ"))
+                      for x in op_list_filtered if agent_name in [y.paw for y in x.agents]]
+        path = os.path.join((dir_name), ''.join(agent_opid[0]))
+        if not os.path.exists(path):
+            os.makedirs(path)
+        return path
+
     async def save_multipart_file_upload(self, request, target_dir, encrypt=True):
         try:
             reader = await request.multipart()
@@ -165,14 +176,19 @@ def list_exfilled_files(self, startdir=None):
             startdir = self.get_config('exfil_dir')
         if not os.path.exists(startdir):
             return dict()
-
         exfil_files = dict()
-        exfil_folders = [f.path for f in os.scandir(startdir) if f.is_dir()]
-        for d in exfil_folders:
-            exfil_key = d.split(os.sep)[-1]
-            exfil_files[exfil_key] = {}
-            for file in [f.path for f in os.scandir(d) if f.is_file()]:
-                exfil_files[exfil_key][file.split(os.sep)[-1]] = file
+        exfil_list = [x for x in os.walk(startdir) if x[2]]
+        for d in exfil_list:
+            agent_path = d[0]
+            exfil_agent_key = d[0].split(os.sep)[-2]
+            exfil_subdir = d[0].split(os.sep)[-1]
+            if exfil_agent_key not in exfil_files:
+                exfil_files[exfil_agent_key] = dict()
+            for file in d[-1]:
+                if exfil_subdir not in exfil_files[exfil_agent_key]:
+                    exfil_files[exfil_agent_key][exfil_subdir] = dict()
+                if file not in exfil_files[exfil_agent_key][exfil_subdir]:
+                    exfil_files[exfil_agent_key][exfil_subdir][file] = os.path.join(agent_path, file)
         return exfil_files
 
     @staticmethod

templates/adversaries.html

@@ -24,7 +24,16 @@ <h2>Adversary Profiles</h2>
                 </div>
             </div>
             <div class="control">
-                <button type="button" class="button is-primary is-small" @click="isCreatingProfile = true">+ New Profile</button>
+                <button type="button" class="button is-primary is-small" @click="isCreatingProfile = true">
+                    <span class="icon"><em class="fas fa-plus"></em></span>
+                    <span>New Profile</span>
+                </button>
+            </div>
+            <div class="control ml-2">
+                <button type="button" class="button is-small" @click="showImportModal = true">
+                    <span class="icon"><em class="fas fa-file-import"></em></span>
+                    <span>Import</span>
+                </button>
             </div>
         </div>
     </form>
@@ -73,6 +82,10 @@ <h3 x-text="selectedProfileName" class="pointer tooltip has-tooltip-arrow" data-
                 <span>Objective: <b x-text="getObjectiveName()"></b>&nbsp;&nbsp;&nbsp;</span>
                 <button class="button is-small" @click="showObjectiveModal = true">Change</button>
                 <div class="vr"></div>
+                <button class="button is-small" @click="exportProfile()">
+                    <span class="icon"><em class="fas fa-file-export"></em></span>
+                    <span>Export</span>
+                </button>
                 <button class="button is-success is-small" x-bind:disabled="!unsavedChanges" @click="saveProfile()">Save Profile</button>
                 <button class="button is-danger is-outlined is-small" @click="deleteProfile()">Delete Profile</button>
             </div>
@@ -719,6 +732,41 @@ <h3>Create a profile</h3>
         </div>
     </div>
 
+    <div class="modal" x-bind:class="{ 'is-active': showImportModal }">
+        <div class="modal-background" @click="showImportModal = false; adversaryImportFile = undefined"></div>
+        <div class="modal-card">
+            <header class="modal-card-head">
+                <p class="modal-card-title">Import Adversary</p>
+            </header>
+            <section class="modal-card-body">
+                <div class="file is-small has-name is-fullwidth">
+                    <label class="file-label">
+                        <input class="file-input" accept=".yml,.yaml" type="file" @change="uploadAdversaryFile($el)">
+                        <span class="file-cta">
+                            <span class="file-icon"><em class="fas fa-upload"></em></span>
+                            <span class="file-label">Choose a file…</span>
+                        </span>
+                        <span class="file-name" x-text="adversaryImportFile ? adversaryImportFile.name : ''"></span>
+                    </label>
+                </div>
+            </section>
+            <footer class="modal-card-foot">
+                <nav class="level">
+                    <div class="level-left">
+                        <div class="level-item">
+                            <button class="button is-small" @click="showImportModal = false; adversaryImportFile = undefined">Close</button>
+                        </div>
+                    </div>
+                    <div class="level-right">
+                        <div class="level-item">
+                            <button class="button is-primary is-small" @click="importProfile()" :disabled="!adversaryImportFile">Import Adversary</button>
+                        </div>
+                    </div>
+                </nav>
+            </footer>
+        </div>
+    </div>
+
     <div class="modal" x-bind:class="{ 'is-active': showFactBreakdownModal }">
         <div class="modal-background" @click="showFactBreakdownModal = false"></div>
         <div class="modal-card">
@@ -815,6 +863,11 @@ <h3>Create a profile</h3>
             selectedPlatform: '',
             selectedExecutor: '',
 
+            // Import modal
+            showImportModal: false,
+            adversaryImportFile: undefined,
+            adversaryImportFileContent: undefined,
+
             // Fact breakdown modal
             showFactBreakdownModal: false,
             factBreakdown: [],
@@ -854,6 +907,7 @@ <h3>Create a profile</h3>
                 this.selectedObjectiveId = this.objectives.find((objective) => objective.id === selectedAdversary.objective).id;
                 this.selectedProfileAbilities = selectedAdversary.atomic_ordering.map((ability_id) => ({ ...this.abilities.find((ability) => ability.ability_id === ability_id) }));
                 this.abilityIDs = selectedAdversary.atomic_ordering;
+                this.adversarySearchQuery = selectedAdversary.name;
                 this.findAbilityDependencies();
             },
 
@@ -914,6 +968,7 @@ <h3>Create a profile</h3>
 
                 apiV2('POST', '/api/v2/adversaries', requestBody).then((response) => {
                     this.adversaries.push(requestBody);
+                    this.adversaries = this.adversaries.sort((a, b) => a.name > b.name);
                     this.selectedProfileId = newId;
                     this.getAdversaryTactics();
                     this.loadProfile();
@@ -931,6 +986,71 @@ <h3>Create a profile</h3>
                 this.isCreatingProfile = false;
             },
 
+            uploadAdversaryFile(el) {
+                if (!el.files || !el.files.length) return;
+                this.adversaryImportFile = el.files[0];
+                const reader = new FileReader();
+                reader.onload = (event) => this.adversaryImportFileContent = event.target.result;
+                reader.readAsText(this.adversaryImportFile);
+            },
+
+            importProfile() {
+                let keyValSplit;
+                let lastKey;
+                let profile = {};
+                const lines = this.adversaryImportFileContent.split('\n');
+                lines.forEach((line) => {
+                    // Remove comments
+                    line = line.split('#')[0];
+                    // Line has a key-value pair
+                    keyValSplit = line.split(':');
+                    if (keyValSplit.length >= 2) {
+                        if (keyValSplit[1]) {
+                            profile[keyValSplit[0]] = keyValSplit[1].trim();
+                        } else {
+                            lastKey = keyValSplit[0];
+                            profile[lastKey] = [];
+                        }
+                    }
+                    // Line is a list item
+                    if (line.trim()[0] === '-' && line.trim() !== '---') {
+                        profile[lastKey].push(line.replace('-', '').trim());
+                    }
+                });
+
+                apiV2('POST', '/api/v2/adversaries', profile).then((response) => {
+                    this.adversaries.push(response);
+                    this.adversaries = this.adversaries.sort((a, b) => a.name > b.name);
+                    this.selectedProfileId = response.adversary_id;
+                    this.getAdversaryTactics();
+                    this.loadProfile();
+                    this.showImportModal = false;
+                    toast('Profile imported!', true);
+                }).catch((error) => {
+                    toast('Error importing profile, please ensure YAML file is in correct format.', false);
+                    console.error(error);
+                });
+            },
+
+            exportProfile() {
+                let yaml = `id: ${this.selectedProfileId}\n`;
+                yaml += `name: ${this.selectedProfileName}\n`;
+                yaml += `description: ${this.selectedProfileDescription}\n`;
+                yaml += `objective: ${this.selectedObjectiveId}\n`;
+                yaml += `atomic_ordering:\n`;
+                this.selectedProfileAbilities.forEach((ability) => yaml += `- ${ability.ability_id}\n`);
+
+                const blob = new Blob([yaml], { type: 'application/x-yaml' })
+                const url = window.URL.createObjectURL(blob);
+                const a = document.createElement('a');
+                a.style.display = 'none';
+                a.href = url;
+                a.download = `${this.selectedProfileName}.yaml`;
+                document.body.appendChild(a);
+                a.click();
+                window.URL.revokeObjectURL(url);
+            },
+
             getAdversaryTactics() {
                 this.adversaries.forEach((adversary) => {
                     let tactics = adversary.atomic_ordering.map((ability_id) => {
@@ -1192,7 +1312,6 @@ <h3>Create a profile</h3>
                 this.selectedProfileId = id;
                 this.loadProfile();
                 this.adversarySearchResults = this.adversaries;
-                this.adversarySearchQuery = name;
             },
 
             searchForAbility() {
@@ -1350,6 +1469,13 @@ <h3>Create a profile</h3>
         cursor: grab;
     }
 
+    .file-cta {
+        background-color: #262626;
+    }
+    .file-label:hover .file-cta {
+        background-color: #1e1e1e;
+    }
+
     .lock {
         background-color: blueviolet !important;
     }

templates/exfilled_files.html

@@ -30,20 +30,28 @@ <h2>Exfilled Files</h2>
                 <li class="root-child">
                     <span class="icon is-small"><i class="far fa-folder-open"></i></span>
                     <span x-text="agentName"></span>
-                    <ul class="tree">
-                        <template x-for="filename of Object.keys(files[agentName])" :key="filename">
-                            <li class="agent-child">
-                                <label class="checkbox">
-                                    <input type="checkbox" class="file-checkbox" x-bind:value="files[agentName][filename]" @click="toggleFile($el, files[agentName][filename])">
-                                    <span class="icon is-small"><i class="far fa-file-alt"></i></span>
-                                    <span x-text="filename"></span>
-                                </label>
-                            </li>
+                        <template x-for="operation of Object.keys(files[agentName])" :key="operation">
+                            <ul class="tree">
+                                <li class="agentName-child">
+                                    <span class="icon is-small"><i class="fas fa-dragon"></i></span>
+                                    <span x-text="operation"></span>
+                                    <ul class="tree">
+                                        <template x-for="filename of Object.keys(files[agentName][operation])" :key="filename">
+                                            <li class="agent-child">
+                                                <label class="checkbox">
+                                                    <input type="checkbox" class="file-checkbox" x-bind:value="files[agentName][operation][filename]" @click="toggleFile($el, files[agentName][operation][filename])">
+                                                    <span class="icon is-small"><i class="far fa-file-alt"></i></span>
+                                                    <span x-text="filename"></span>
+                                                </label>
+                                            </li>
+                                        </template>
+                                        <li class="agent-child" x-show="!Object.keys(files[agentName][operation]).length">
+                                            (none)
+                                        </li>
+                                    </ul>     
+                                </li>
+                            </ul>    
                         </template>
-                        <li class="agent-child" x-show="!Object.keys(files[agentName]).length">
-                            (none)
-                        </li>
-                    </ul>
                 </li>
             </template>
         </ul>  
@@ -196,6 +204,9 @@ <h2>Exfilled Files</h2>
     ul.tree li.root-child:last-child:after {
         display: none;
     }
+    ul.tree li.agentName-child:after{
+        display: none;
+    }
     ul.tree li.agent-child:nth-last-child(2):after {
         display: none;
     }