From bec7b5bc8f41dbf2d106f98c1e203297837a384e Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Fri, 25 Aug 2023 09:52:36 -0600 Subject: [PATCH 001/102] WIP: Demo + CleanUp Contains hardcoded config values in infohandler, mock data inside the JS. Very much a prototype. --- config/clientparameters.go | 2 + html/index.html | 351 ++++++++++++- html/js/i18n.js | 2 + html/js/routes/detection.js | 157 ++++++ html/js/routes/hunt.js | 204 ++++++-- html/js/routes/playbook.js | 461 ++++++++++++++++++ model/detection.go | 91 ++++ server/casestore.go | 5 + server/detectionhandler.go | 109 +++++ server/infohandler.go | 45 ++ server/modules/elastic/converter.go | 95 ++++ server/modules/elastic/elasticcasestore.go | 114 +++++ .../modules/elasticcases/elasticcasestore.go | 16 + server/modules/generichttp/httpcasestore.go | 16 + server/modules/thehive/thehivecasestore.go | 16 + server/server.go | 1 + 16 files changed, 1647 insertions(+), 38 deletions(-) create mode 100644 html/js/routes/detection.js create mode 100644 html/js/routes/playbook.js create mode 100644 model/detection.go create mode 100644 server/detectionhandler.go diff --git a/config/clientparameters.go b/config/clientparameters.go index 2acb62811..7681ae6e6 100644 --- a/config/clientparameters.go +++ b/config/clientparameters.go @@ -22,6 +22,8 @@ type ClientParameters struct { CaseParams CaseParameters `json:"case"` DashboardsParams HuntingParameters `json:"dashboards"` JobParams HuntingParameters `json:"job"` + DetectionsParams HuntingParameters `json:"detections"` + PlaybooksParams HuntingParameters `json:"playbooks"` DocsUrl string `json:"docsUrl"` CheatsheetUrl string `json:"cheatsheetUrl"` ReleaseNotesUrl string `json:"releaseNotesUrl"` diff --git a/html/index.html b/html/index.html index 8f9acc944..94c1d50e7 100644 --- a/html/index.html +++ b/html/index.html @@ -68,6 +68,22 @@ + + + fa-magnifying-glass + + + + + + + + fa-book-bookmark + + + + + fa-stream @@ -667,7 +683,13 @@

{{ i18n.eventTotal }} {{ totalEvents.toLocaleString() }}

:color="isFilterToggleEnabled('escalated') || item['event.escalated'] == true ? 'secondary' : 'primary'"> fa-exclamation-triangle - + + fa-binoculars + + + fa-binoculars + + fa-binoculars @@ -938,6 +960,331 @@

{{ i18n.eventTotal }} {{ totalEvents.toLocaleString() }}

+ + + + + + diff --git a/html/js/i18n.js b/html/js/i18n.js index c44de03d9..7bbe4f3b5 100644 --- a/html/js/i18n.js +++ b/html/js/i18n.js @@ -254,6 +254,7 @@ const i18n = { denied: 'Denied', description: 'Description', details: 'Details', + detections: 'Detections', disconnected: 'Disconnected from manager', diskUsageElastic: 'Elastic Storage Used', diskUsageInfluxDb: 'InfluxDB Storage Used', @@ -545,6 +546,7 @@ const i18n = { passwordRequired: 'A password must be specified.', passwordReset: 'Change Password', passwordNeedsChanged: 'User has not yet changed their password', + playbooks: 'Playbooks', profileDetails: 'Profile Details', profileInstructions: 'You may be prompted to login again when updating your profile. This is a security measure to protect your account.', pcap: 'PCAP', diff --git a/html/js/routes/detection.js b/html/js/routes/detection.js new file mode 100644 index 000000000..b1e2e61ed --- /dev/null +++ b/html/js/routes/detection.js @@ -0,0 +1,157 @@ +// Copyright 2019 Jason Ertel (github.com/jertel). +// Copyright 2020-2023 Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +// or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +// https://securityonion.net/license; you may not use this file except in compliance with the +// Elastic License 2.0. + +routes.push({ path: '/detection/:id', name: 'detection', component: { + template: '#page-detection', + data() { return { + i18n: this.$root.i18n, + params: {}, + detect: null, + origDetect: null, + curEditTarget: null, // string containing element ID, null if not editing + origValue: null, + editField: null, + rules: { + required: value => (value && value.length > 0) || this.$root.i18n.required, + number: value => (! isNaN(+value) && Number.isInteger(parseFloat(value))) || this.$root.i18n.required, + hours: value => (!value || /^\d{1,4}(\.\d{1,4})?$/.test(value)) || this.$root.i18n.invalidHours, + shortLengthLimit: value => (value.length < 100) || this.$root.i18n.required, + longLengthLimit: value => (encodeURI(value).split(/%..|./).length - 1 < 10000000) || this.$root.i18n.required, + fileSizeLimit: value => (value == null || value.size < this.maxUploadSizeBytes) || this.$root.i18n.fileTooLarge.replace("{maxUploadSizeBytes}", this.$root.formatCount(this.maxUploadSizeBytes)), + fileNotEmpty: value => (value == null || value.size > 0) || this.$root.i18n.fileEmpty, + fileRequired: value => (value != null) || this.$root.i18n.required, + }, + severityOptions: ['low', 'medium', 'high'], + engineOptions: ['suricata', 'yara', 'elastalert'], + panel: [0, 1, 2], + activeTab: '', + associatedPlaybook: { + onionId: 'y5IYKIoB9-Z7uL2kmy_o', + publicId: "4020131e-223a-421e-8ebe-8a211a5ac4d6", + title: "Find the baddies", + severity: "high", + description: "A long description that spans multiple lines. A long description that spans multiple lines. A long description that spans multiple lines. A long description that spans multiple lines. A long description that spans multiple lines. A long description that spans multiple lines.", + mechanism: "suricata", + tags: ["one", "two", "three"], + relatedPlaybooks: [], + contributors: ["Corey Ogburn"], + userEditable: true, + createTime: "2023-08-22T12:49:47.302819008-06:00", + userId: "83656890-2acd-4c0b-8ab9-7c73e71ddaf3", + }, + }}, + created() { + }, + watch: { + }, + mounted() { + this.$root.loadParameters('detection', this.initDetection); + }, + methods: { + async initDetection(params) { + this.params = params; + if (this.$route.params.id === 'create') { + this.detect = this.newDetection(); + } else { + await this.loadData(); + } + + this.origDetect = Object.assign({}, this.detect); + + this.loadUrlParameters(); + }, + loadUrlParameters() { + + }, + newDetection() { + let author = [this.$root.user.firstName, this.$root.user.lastName].filter(x => x).join(' '); + return { + title: 'Detection title not yet provided - click here to update this title', + description: 'Detection description not yet provided - click here to update this description', + author: author, + publicId: '', + severity: 'low', + content: '', + isEnabled: false, + isReporting: false, + engine: '', + } + }, + async loadData() { + this.$root.startLoading(); + + try { + const response = await this.$root.papi.get('detection/' + encodeURIComponent(this.$route.params.id)); + this.detect = response.data; + this.detect.note = 'This is a note.'; + } catch (error) { + if (error.response != undefined && error.response.status == 404) { + this.$root.showError(this.i18n.notFound); + } else { + this.$root.showError(error); + } + } + + this.$root.stopLoading(); + }, + isNew() { + return this.$route.params.id === 'create'; + }, + cancelDetection() { + if (this.isNew()) { + this.$router.push({name: 'detections'}); + } else { + this.detect = this.origDetect; + } + }, + async startEdit(target, field) { + if (this.curEditTarget === target) return; + if (this.curEditTarget !== null) await this.stopEdit(false); + + this.curEditTarget = target; + this.origValue = this.detect[field]; + this.editField = field; + + this.$nextTick(() => { + const el = document.getElementById(target + '-edit'); + if (el) { + el.focus(); + el.select(); + } + }); + }, + generatePublicID() { + this.detect.publicId = crypto.randomUUID(); + }, + isEdit(target) { + return this.curEditTarget === target; + }, + async stopEdit(commit) { + if (!commit) { + this.detect[this.editField] = this.origValue; + } else if (!this.isNew()) { + const response = await this.$root.papi.put('/detection', this.detect); + + console.log('UPDATE', response); + } + + this.curEditTarget = null; + this.origValue = null; + this.editField = null; + }, + saveDetection(createNew) { + if (createNew) { + this.$root.papi.post('/detection', this.detect); + } else { + this.$root.papi.put('/detection', this.detect); + } + this.origDetect = Object.assign({}, this.detect); + }, + print(x) { + console.log(x); + }, + } +}}); diff --git a/html/js/routes/hunt.js b/html/js/routes/hunt.js index 089d6bfaf..c8c6c3ff8 100644 --- a/html/js/routes/hunt.js +++ b/html/js/routes/hunt.js @@ -260,10 +260,15 @@ const huntComponent = { } }, applyQuerySubstitutions(queries) { - queries.forEach(query => { - query.query = query.query.replace(/\{myId\}/g, this.$root.user.id); - }); - return queries; + if (Array.isArray(queries)) { + queries.forEach(query => { + query.query = query.query.replace(/\{myId\}/g, this.$root.user.id); + }); + + return queries; + } else { + return []; + } }, notifyInputsChanged(replaceHistory = false) { var hunted = false; @@ -338,19 +343,21 @@ const huntComponent = { q = this.queryBaseFilter; } - for (var i = 0; i < this.filterToggles.length; i++) { - filter = this.filterToggles[i]; + if (Array.isArray(this.filterToggles)) { + for (var i = 0; i < this.filterToggles.length; i++) { + filter = this.filterToggles[i]; - if (filter.enabled) { - if (q.length > 0) { - q = q + " AND "; - } - q = q + filter.filter; - } else if (filter.exclusive) { - if (q.length > 0) { - q = q + " AND "; + if (filter.enabled) { + if (q.length > 0) { + q = q + " AND "; + } + q = q + filter.filter; + } else if (filter.exclusive) { + if (q.length > 0) { + q = q + " AND "; + } + q = q + "NOT " + filter.filter; } - q = q + "NOT " + filter.filter; } } @@ -424,20 +431,22 @@ const huntComponent = { this.groupQuery(this.$route.query.groupByField, this.$route.query.groupByGroup); reRoute = true; } - for (const q in this.$route.query) { - this.filterToggles.forEach(toggle => { - if (toggle.name === q) { - const orig = toggle.enabled; - let enabled = this.$route.query[q]; - if (typeof enabled === 'string') { - enabled = enabled.toLowerCase() === 'true'; - } - toggle.enabled = enabled; - if (orig !== toggle.enabled) { - reRoute = true; + if (Array.isArray(this.filterToggles)) { + for (const q in this.$route.query) { + this.filterToggles.forEach(toggle => { + if (toggle.name === q) { + const orig = toggle.enabled; + let enabled = this.$route.query[q]; + if (typeof enabled === 'string') { + enabled = enabled.toLowerCase() === 'true'; + } + toggle.enabled = enabled; + if (orig !== toggle.enabled) { + reRoute = true; + } } - } - }); + }); + } } if (reRoute) return false; return true; @@ -454,14 +463,74 @@ const huntComponent = { // This must occur before the following await, so that Vue flushes the old groupby DOM renders this.groupBys.splice(0); - const response = await this.$root.papi.get('events/', { params: { - query: await this.getQuery(), - range: this.dateRange, - format: this.i18n.timePickerSample, - zone: this.zone, - metricLimit: this.groupByLimit, - eventLimit: this.eventLimit - }}); + let response; + if (this.category === 'playbooks') { + response = { + data: { + "metrics": { + "timeline": null, + }, + "elapsedMs": 668, + "errors": [], + "criteria": { + "query": "(_id:*) AND _index:\"*:so-case\" AND so_kind:detection", + "dateRange": "", + "metricLimit": 0, + "eventLimit": 50, + "BeginTime": "2021-08-23T15:41:39-06:00", + "EndTime": "2023-08-23T15:41:39-06:00", + "CreateTime": "2023-08-23T15:41:39.446264196-06:00", + "ParsedQuery": { + "Segments": [ + {} + ] + }, + "SortFields": null + }, + "events": [ + { + "source": "manager:so-case", + "Time": "2023-08-23T14:48:17.140438075-06:00", + "timestamp": "2023-08-23T14:48:17.140Z", + "id": "RDmUHooB-8rNCo4d3nIc", + "type": "", + "score": 3.287682, + "payload": { + "@timestamp": "2023-08-23T14:48:17.140438075-06:00", + "so_playbook.onionId": "75332a3c-b029-46a0-9392-509ff90737a8", + "so_playbook.publicId": "4020131e-223a-421e-8ebe-8a211a5ac4d6", + "so_playbook.title": "Find the baddies", + "so_playbook.severity": "high", + "so_playbook.description": "A long description that spans multiple lines. A long description that spans multiple lines. A long description that spans multiple lines. A long description that spans multiple lines. A long description that spans multiple lines. A long description that spans multiple lines.", + "so_playbook.mechanism": "suricata", + "so_playbook.tags": ["one", "two", "three"], + "so_playbook.relatedPlaybooks": [], + "so_playbook.contributors": ["Jim Bob"], + "so_playbook.userEditable": true, + "so_playbook.createTime": "2023-08-22T12:49:47.302819008-06:00", + "so_playbook.kind": "playbook", + "so_playbook.userId": "83656890-2acd-4c0b-8ab9-7c73e71ddaf3", + "so_kind": "playbook" + } + } + ], + createTime: moment().subtract(2, 'seconds').toISOString(), + completeTime: moment().toISOString(), + } + }; + response.data.totalEvents = response.data.events.length; + } else { + response = await this.$root.papi.get('events/', { + params: { + query: await this.getQuery(), + range: this.dateRange, + format: this.i18n.timePickerSample, + zone: this.zone, + metricLimit: this.groupByLimit, + eventLimit: this.eventLimit + } + }); + } this.eventPage = 1; this.groupByPage = 1; @@ -2037,7 +2106,62 @@ const huntComponent = { } window.open(url, target); - } + }, + async CreateDetection() { + const response = await this.$root.papi.post('/detection', { + publicId: 'ABCDEF', + title: 'First!', + severity: 'low', + author: 'Corey Ogburn', + description: 'first try', + content: 'rule goes here', + isEnabled: true, + isReporting: true, + Engine: 'suricata' + }); + + console.log('CREATE', response); + this.onionID = response.data.id; + console.log('onionID', this.onionID); + }, + async GetDetection() { + if (this.onionID) { + const response = await this.$root.papi.get('/detection/' + this.onionID); + + console.log('GET', response); + } else { + console.log('No onionID'); + } + }, + async UpdateDetection() { + if (this.onionID) { + const response = await this.$root.papi.put('/detection', { + id: this.onionID, + publicId: 'ABCDEF', + title: 'Second!', + severity: 'low', + author: 'Corey Ogburn', + description: 'first try', + content: 'rule goes here', + isEnabled: true, + isReporting: true, + Engine: 'suricata' + }); + + console.log('UPDATE', response); + } else { + console.log('No onionID'); + } + }, + async DeleteDetection() { + if (this.onionID) { + const response = await this.$root.papi.delete('/detection/' + this.onionID); + + console.log('DELETE', response); + } else { + console.log('No onionID'); + } + }, } }; @@ -2051,3 +2175,9 @@ routes.push({ path: '/cases', name: 'cases', component: casesComponent}); const dashboardsComponent = Object.assign({}, huntComponent); routes.push({ path: '/dashboards', name: 'dashboards', component: dashboardsComponent}); + +const detectionsComponent = Object.assign({}, huntComponent); +routes.push({ path: '/detections', name: 'detections', component: detectionsComponent }); + +const playbooksComponent = Object.assign({}, huntComponent); +routes.push({ path: '/playbooks', name: 'playbooks', component: playbooksComponent }); diff --git a/html/js/routes/playbook.js b/html/js/routes/playbook.js new file mode 100644 index 000000000..3db5a5f02 --- /dev/null +++ b/html/js/routes/playbook.js @@ -0,0 +1,461 @@ +// Copyright 2019 Jason Ertel (github.com/jertel). +// Copyright 2020-2023 Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +// or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +// https://securityonion.net/license; you may not use this file except in compliance with the +// Elastic License 2.0. + +routes.push({ path: '/playbook/:id', name: 'playbook', component: { + template: '#page-playbook', + data() { return { + i18n: this.$root.i18n, + params: {}, + rules: { + required: value => (value && value.length > 0) || this.$root.i18n.required, + number: value => (! isNaN(+value) && Number.isInteger(parseFloat(value))) || this.$root.i18n.required, + hours: value => (!value || /^\d{1,4}(\.\d{1,4})?$/.test(value)) || this.$root.i18n.invalidHours, + shortLengthLimit: value => (value.length < 100) || this.$root.i18n.required, + longLengthLimit: value => (encodeURI(value).split(/%..|./).length - 1 < 10000000) || this.$root.i18n.required, + fileSizeLimit: value => (value == null || value.size < this.maxUploadSizeBytes) || this.$root.i18n.fileTooLarge.replace("{maxUploadSizeBytes}", this.$root.formatCount(this.maxUploadSizeBytes)), + fileNotEmpty: value => (value == null || value.size > 0) || this.$root.i18n.fileEmpty, + fileRequired: value => (value != null) || this.$root.i18n.required, + }, + playbook: null, + origPlaybook: null, + curEditTarget: null, // string containing element ID, null if not editing + origValue: null, + editField: null, + severityOptions: ['low', 'medium', 'high'], + engineOptions: ['none', 'suricata', 'yara', 'elastalert'], + panel: [0, 1, 2], + activeTab: '', + tags: [], + addQuestion: '', + addContext: '', + addDataSources: [], + addQuery: '', + search: '', + questionHeaders: [{ title: '@timestamp', key: 'item.payload["@timestamp"]' }, { title: '@version', value: '@version' }, { title: 'message', value: 'message' }], + }}, + created() { + }, + watch: { + }, + mounted() { + this.$root.loadParameters('playbooks', this.initPlaybook); + }, + methods: { + async initPlaybook(params) { + this.params = params; + if (this.$route.params.id === 'create') { + this.detect = this.newPlaybook(); + } else { + await this.loadData(); + } + + this.origPlaybook = Object.assign({}, this.playbook); + + this.loadUrlParameters(); + }, + loadUrlParameters() { + + }, + newPlaybook() { + let author = [this.$root.user.firstName, this.$root.user.lastName].filter(x => x).join(' '); + return { + publicId: '', + title: 'Detection title not yet provided - click here to update this title', + description: 'Detection description not yet provided - click here to update this description', + mechanism: '', + tags: [], + relatedPlaybooks: [], + detectionLinks: [], + contributors: [author], + userEditable: true, + questions: [], + note: '', + } + }, + async loadData() { + this.$root.startLoading(); + + // try { + // const response = await this.$root.papi.get('playbook/' + encodeURIComponent(this.$route.params.id)); + // this.detect = response.data; + // } catch (error) { + // if (error.response != undefined && error.response.status == 404) { + // this.$root.showError(this.i18n.notFound); + // } else { + // this.$root.showError(error); + // } + // } + if (this.isNew()) { + this.playbook = this.newPlaybook(); + } else { + this.playbook = { + onionId: this.$route.params.id, + publicId: "4020131e-223a-421e-8ebe-8a211a5ac4d6", + title: "Find the baddies", + severity: "high", + description: "A long description that spans multiple lines. A long description that spans multiple lines. A long description that spans multiple lines. A long description that spans multiple lines. A long description that spans multiple lines. A long description that spans multiple lines.", + mechanism: "suricata", + tags: ["one", "two", "three"], + relatedPlaybooks: [], + contributors: ["Corey Ogburn"], + userEditable: true, + createTime: "2023-08-22T12:49:47.302819008-06:00", + userId: "83656890-2acd-4c0b-8ab9-7c73e71ddaf3", + questions: [ + { + question: "What network resources did they access?", + context: "Bad actors do bad things to network resources", + dataSources: ['windows_security', 'process_auditing'], + query: '*', + results: [ + { + "source": "manager:.ds-logs-elastic_agent-default-2023.08.21-000001", + "Time": "2023-08-24T15:10:05.747Z", + "timestamp": "2023-08-24T15:10:05.747Z", + "id": "y5IYKIoB9-Z7uL2kmy_o", + "type": "", + "score": 2, + "payload": { + "@timestamp": "2023-08-24T15:10:05.747Z", + "@version": "1", + "agent.ephemeral_id": "b36260f3-7e22-4ada-9736-7825bba61050", + "agent.id": "6a7d0533-85fe-4aab-ad23-25659d59415f", + "agent.name": "manager", + "agent.type": "filebeat", + "agent.version": "8.8.2", + "container.id": "elastic-agent-cdc5ba", + "data_stream.dataset": "elastic_agent", + "data_stream.namespace": "default", + "data_stream.type": "logs", + "ecs.version": "8.0.0", + "elastic_agent.id": "6a7d0533-85fe-4aab-ad23-25659d59415f", + "elastic_agent.snapshot": false, + "elastic_agent.version": "8.8.2", + "event.agent_id_status": "auth_metadata_missing", + "event.dataset": "elastic_agent", + "event.ingested": "2023-08-24T15:10:16Z", + "event.module": "elastic_agent", + "host.architecture": "x86_64", + "host.containerized": false, + "host.hostname": "manager", + "host.id": "9b909224852841ee9e8394c0ccb6f345", + "host.ip": [ + "192.168.122.119", + "172.17.1.1", + "172.17.0.1" + ], + "host.mac": [ + "02-42-F7-90-7C-0F", + "02-42-FD-01-4C-18", + "22-69-52-DA-7E-60", + "26-32-CC-C2-D0-96", + "32-5D-A3-D9-5E-BB", + "36-30-04-3E-2A-74", + "3A-47-4A-42-2D-42", + "52-54-00-17-A5-16", + "52-54-00-79-9B-AF", + "56-22-21-D2-59-BB", + "5A-84-1E-33-78-6F", + "5A-B1-C1-9E-26-54", + "5E-07-4D-47-FC-B6", + "66-FE-30-61-E6-77", + "82-20-8F-6D-BA-2B", + "86-14-06-40-98-7D", + "86-29-18-6F-10-88", + "8E-AD-CA-9F-EA-F6", + "96-4D-FB-AD-AB-8E", + "AA-FB-73-2B-C4-24", + "B2-68-3A-55-50-88", + "BE-53-9A-99-67-0C", + "CA-68-C4-C1-5B-4D", + "E2-0C-CE-E7-DA-50", + "EE-F2-CD-8D-0A-17", + "FE-99-5F-39-DF-D1" + ], + "host.name": "manager", + "host.os.family": "redhat", + "host.os.kernel": "5.15.0-103.114.4.el9uek.x86_64", + "host.os.name": "Oracle Linux Server", + "host.os.platform": "ol", + "host.os.type": "linux", + "host.os.version": "9.2", + "input.type": "filestream", + "log.file.path": "/opt/Elastic/Agent/data/elastic-agent-cdc5ba/logs/elastic-agent-20230824.ndjson", + "log.level": "info", + "log.offset": 311104, + "log.origin.file.line": 821, + "log.origin.file.name": "coordinator/coordinator.go", + "log.source": "elastic-agent", + "message": "Updating running component model", + "metadata.beat": "filebeat", + "metadata.input.beats.host.ip": "172.17.1.1", + "metadata.input_id": "filestream-monitoring-agent", + "metadata.raw_index": "logs-elastic_agent-default", + "metadata.stream_id": "filestream-monitoring-agent", + "metadata.type": "_doc", + "metadata.version": "8.8.2", + "tags": [ + "elastic-agent", + "input-manager", + "beats_input_codec_plain_applied" + ] + } + }, + { + "source": "manager:.ds-logs-elastic_agent-default-2023.08.21-000001", + "Time": "2023-08-24T15:09:25.702Z", + "timestamp": "2023-08-24T15:09:25.702Z", + "id": "GJIXKIoB9-Z7uL2k7C-E", + "type": "", + "score": 2, + "payload": { + "@timestamp": "2023-08-24T15:09:25.702Z", + "@version": "1", + "agent.ephemeral_id": "b36260f3-7e22-4ada-9736-7825bba61050", + "agent.id": "6a7d0533-85fe-4aab-ad23-25659d59415f", + "agent.name": "manager", + "agent.type": "filebeat", + "agent.version": "8.8.2", + "container.id": "elastic-agent-cdc5ba", + "data_stream.dataset": "elastic_agent", + "data_stream.namespace": "default", + "data_stream.type": "logs", + "ecs.version": "8.0.0", + "elastic_agent.id": "6a7d0533-85fe-4aab-ad23-25659d59415f", + "elastic_agent.snapshot": false, + "elastic_agent.version": "8.8.2", + "event.agent_id_status": "auth_metadata_missing", + "event.dataset": "elastic_agent", + "event.ingested": "2023-08-24T15:09:31Z", + "event.module": "elastic_agent", + "host.architecture": "x86_64", + "host.containerized": false, + "host.hostname": "manager", + "host.id": "9b909224852841ee9e8394c0ccb6f345", + "host.ip": [ + "192.168.122.119", + "172.17.1.1", + "172.17.0.1" + ], + "host.mac": [ + "02-42-F7-90-7C-0F", + "02-42-FD-01-4C-18", + "22-69-52-DA-7E-60", + "26-32-CC-C2-D0-96", + "32-5D-A3-D9-5E-BB", + "36-30-04-3E-2A-74", + "3A-47-4A-42-2D-42", + "52-54-00-17-A5-16", + "52-54-00-79-9B-AF", + "56-22-21-D2-59-BB", + "5A-84-1E-33-78-6F", + "5A-B1-C1-9E-26-54", + "5E-07-4D-47-FC-B6", + "66-FE-30-61-E6-77", + "82-20-8F-6D-BA-2B", + "86-14-06-40-98-7D", + "86-29-18-6F-10-88", + "8E-AD-CA-9F-EA-F6", + "96-4D-FB-AD-AB-8E", + "AA-FB-73-2B-C4-24", + "AE-49-EF-7B-5E-B9", + "B2-68-3A-55-50-88", + "BE-53-9A-99-67-0C", + "CA-68-C4-C1-5B-4D", + "E2-0C-CE-E7-DA-50", + "EE-F2-CD-8D-0A-17", + "FE-99-5F-39-DF-D1" + ], + "host.name": "manager", + "host.os.family": "redhat", + "host.os.kernel": "5.15.0-103.114.4.el9uek.x86_64", + "host.os.name": "Oracle Linux Server", + "host.os.platform": "ol", + "host.os.type": "linux", + "host.os.version": "9.2", + "input.type": "filestream", + "log.file.path": "/opt/Elastic/Agent/data/elastic-agent-cdc5ba/logs/elastic-agent-20230824.ndjson", + "log.level": "error", + "log.offset": 304406, + "log.origin.file.line": 221, + "log.origin.file.name": "fleet/fleet_gateway.go", + "log.source": "elastic-agent", + "message": "Checkin request to fleet-server succeeded after 2 failures", + "metadata.beat": "filebeat", + "metadata.input.beats.host.ip": "172.17.1.1", + "metadata.input_id": "filestream-monitoring-agent", + "metadata.raw_index": "logs-elastic_agent-default", + "metadata.stream_id": "filestream-monitoring-agent", + "metadata.type": "_doc", + "metadata.version": "8.8.2", + "tags": [ + "elastic-agent", + "input-manager", + "beats_input_codec_plain_applied" + ] + } + }, + { + "source": "manager:.ds-logs-elastic_agent-default-2023.08.21-000001", + "Time": "2023-08-24T15:08:46.446Z", + "timestamp": "2023-08-24T15:08:46.446Z", + "id": "lZIXKIoB9-Z7uL2kTy6x", + "type": "", + "score": 2, + "payload": { + "@timestamp": "2023-08-24T15:08:46.446Z", + "@version": "1", + "agent.ephemeral_id": "b36260f3-7e22-4ada-9736-7825bba61050", + "agent.id": "6a7d0533-85fe-4aab-ad23-25659d59415f", + "agent.name": "manager", + "agent.type": "filebeat", + "agent.version": "8.8.2", + "container.id": "elastic-agent-cdc5ba", + "data_stream.dataset": "elastic_agent", + "data_stream.namespace": "default", + "data_stream.type": "logs", + "ecs.version": "8.0.0", + "elastic_agent.id": "6a7d0533-85fe-4aab-ad23-25659d59415f", + "elastic_agent.snapshot": false, + "elastic_agent.version": "8.8.2", + "event.agent_id_status": "auth_metadata_missing", + "event.dataset": "elastic_agent", + "event.ingested": "2023-08-24T15:08:50Z", + "event.module": "elastic_agent", + "host.architecture": "x86_64", + "host.containerized": false, + "host.hostname": "manager", + "host.id": "9b909224852841ee9e8394c0ccb6f345", + "host.ip": [ + "192.168.122.119", + "172.17.1.1", + "172.17.0.1" + ], + "host.mac": [ + "02-42-F7-90-7C-0F", + "02-42-FD-01-4C-18", + "22-69-52-DA-7E-60", + "26-32-CC-C2-D0-96", + "32-5D-A3-D9-5E-BB", + "36-30-04-3E-2A-74", + "3A-47-4A-42-2D-42", + "52-54-00-17-A5-16", + "52-54-00-79-9B-AF", + "56-22-21-D2-59-BB", + "5A-84-1E-33-78-6F", + "5A-B1-C1-9E-26-54", + "5E-07-4D-47-FC-B6", + "66-FE-30-61-E6-77", + "82-20-8F-6D-BA-2B", + "86-14-06-40-98-7D", + "86-29-18-6F-10-88", + "8E-AD-CA-9F-EA-F6", + "96-4D-FB-AD-AB-8E", + "AA-FB-73-2B-C4-24", + "AE-49-EF-7B-5E-B9", + "B2-68-3A-55-50-88", + "BE-53-9A-99-67-0C", + "CA-68-C4-C1-5B-4D", + "E2-0C-CE-E7-DA-50", + "EE-F2-CD-8D-0A-17", + "FE-99-5F-39-DF-D1" + ], + "host.name": "manager", + "host.os.family": "redhat", + "host.os.kernel": "5.15.0-103.114.4.el9uek.x86_64", + "host.os.name": "Oracle Linux Server", + "host.os.platform": "ol", + "host.os.type": "linux", + "host.os.version": "9.2", + "input.type": "filestream", + "log.file.path": "/opt/Elastic/Agent/data/elastic-agent-cdc5ba/logs/elastic-agent-20230824.ndjson", + "log.level": "info", + "log.offset": 296546, + "log.origin.file.line": 821, + "log.origin.file.name": "coordinator/coordinator.go", + "log.source": "elastic-agent", + "message": "Updating running component model", + "metadata.beat": "filebeat", + "metadata.input.beats.host.ip": "172.17.1.1", + "metadata.input_id": "filestream-monitoring-agent", + "metadata.raw_index": "logs-elastic_agent-default", + "metadata.stream_id": "filestream-monitoring-agent", + "metadata.type": "_doc", + "metadata.version": "8.8.2", + "tags": [ + "elastic-agent", + "input-manager", + "beats_input_codec_plain_applied" + ] + } + }, + ] + } + ], + note: "This is a note", + }; + } + + this.$root.stopLoading(); + }, + isNew() { + return this.$route.params.id === 'create'; + }, + cancelPlaybook() { + if (this.isNew()) { + this.$router.push({name: 'playbooks'}); + } else { + this.playbook = this.origPlaybook; + } + }, + generatePublicID() { + this.playbook.publicId = crypto.randomUUID(); + }, + async startEdit(target, field) { + if (this.curEditTarget === target) return; + if (this.curEditTarget !== null) await this.stopEdit(false); + + this.curEditTarget = target; + this.origValue = this.playbook[field]; + this.editField = field; + + this.$nextTick(() => { + const el = document.getElementById(target + '-edit'); + if (el) { + el.focus(); + el.select(); + } + }); + }, + isEdit(target) { + return this.curEditTarget === target; + }, + async stopEdit(commit) { + if (!commit) { + this.playbook[this.editField] = this.origValue; + } else if (!this.isNew()) { + // const response = await this.$root.papi.put('/playbook', this.playbook); + // console.log('UPDATE', response); + } + + this.curEditTarget = null; + this.origValue = null; + this.editField = null; + }, + savePlaybook(createNew) { + // if (createNew) { + // this.$root.papi.post('/playbook', this.playbook); + // } else { + // this.$root.papi.put('/playbook', this.playbook); + // } + + this.origPlaybook = Object.assign({}, this.playbook); + }, + print(x) { + console.log(x); + }, + } +}}); diff --git a/model/detection.go b/model/detection.go new file mode 100644 index 000000000..54b2ed801 --- /dev/null +++ b/model/detection.go @@ -0,0 +1,91 @@ +package model + +import ( + "errors" + "strings" +) + +type ScanType string +type SigLanguage string +type Severity string +type IDType string +type EngineName = string + +const ( + ScanTypeFiles ScanType = "files" + ScanTypePackets ScanType = "packets" + ScanTypePacketsAndFiles ScanType = "files,packets" + ScanTypeElastic ScanType = "elastic" + + SigLangElastic SigLanguage = "elastic" // yaml + SigLangSigma SigLanguage = "sigma" // yaml + SigLangSuricata SigLanguage = "suricata" // action, header, options + SigLangYara SigLanguage = "yara" + SigLangZeek SigLanguage = "zeek" + + SeverityLow Severity = "low" + SeverityMedium Severity = "medium" + SeverityHigh Severity = "high" + + IDTypeUUID IDType = "uuid" + IDTypeSID IDType = "sid" + + EngineNameSuricata EngineName = "suricata" + EngineNameYara EngineName = "yara" + EngineNameElastAlert EngineName = "elastalert" +) + +var ( + EnginesByName = map[EngineName]*DetectionEngine{ + EngineNameSuricata: { + Name: string(EngineNameSuricata), + IDType: IDTypeSID, + ScanType: ScanTypePackets, + SigLanguage: SigLangSuricata, + }, + EngineNameYara: { + Name: string(EngineNameYara), + IDType: IDTypeUUID, + ScanType: ScanTypeFiles, + SigLanguage: SigLangYara, + }, + EngineNameElastAlert: { + Name: string(EngineNameElastAlert), + IDType: IDTypeUUID, + ScanType: ScanTypeElastic, + SigLanguage: SigLangElastic, + }, + } + + ErrUnsupportedEngine = errors.New("unsupported engine") +) + +type DetectionEngine struct { + Name string `json:"name"` + IDType IDType `json:"idType"` + ScanType ScanType `json:"scanType"` + SigLanguage SigLanguage `json:"sigLanguage"` +} + +type Detection struct { + Auditable + PublicID string `json:"publicId"` + Title string `json:"title"` + Severity Severity `json:"severity"` + Author string `json:"author"` + Description string `json:"description"` + Content string `json:"content"` + IsEnabled bool `json:"isEnabled"` + IsReporting bool `json:"isReporting"` + Engine EngineName `json:"engine"` +} + +func (detect *Detection) Validate() error { + detect.Engine = strings.ToLower(detect.Engine) + _, engIsSupported := EnginesByName[detect.Engine] + if !engIsSupported { + return ErrUnsupportedEngine + } + + return nil +} diff --git a/server/casestore.go b/server/casestore.go index be9d6536e..1f5bcb204 100644 --- a/server/casestore.go +++ b/server/casestore.go @@ -38,4 +38,9 @@ type Casestore interface { CreateArtifactStream(ctx context.Context, artifactstream *model.ArtifactStream) (string, error) GetArtifactStream(ctx context.Context, id string) (*model.ArtifactStream, error) DeleteArtifactStream(ctx context.Context, id string) error + + CreateDetection(ctx context.Context, detect *model.Detection) (*model.Detection, error) + GetDetection(ctx context.Context, detectId string) (*model.Detection, error) + UpdateDetection(ctx context.Context, detect *model.Detection) (*model.Detection, error) + DeleteDetection(ctx context.Context, detectID string) error } diff --git a/server/detectionhandler.go b/server/detectionhandler.go new file mode 100644 index 000000000..fa1bb4396 --- /dev/null +++ b/server/detectionhandler.go @@ -0,0 +1,109 @@ +// Copyright 2019 Jason Ertel (github.com/jertel). +// Copyright 2020-2023 Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +// or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +// https://securityonion.net/license; you may not use this file except in compliance with the +// Elastic License 2.0. + +package server + +import ( + "net/http" + + "github.com/security-onion-solutions/securityonion-soc/model" + "github.com/security-onion-solutions/securityonion-soc/web" + + "github.com/go-chi/chi" +) + +type DetectionHandler struct { + server *Server +} + +func RegisterDetectionRoutes(srv *Server, r chi.Router, prefix string) { + h := &DetectionHandler{ + server: srv, + } + + r.Route(prefix, func(r chi.Router) { + r.Get("/{onionId}", h.getDetection) + + r.Post("/", h.postDetection) + + r.Put("/", h.putDetection) + + r.Delete("/{onionId}", h.deleteDetection) + }) +} + +func (h *DetectionHandler) getDetection(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + detectId := chi.URLParam(r, "onionId") + + detect, err := h.server.Casestore.GetDetection(ctx, detectId) + if err != nil { + if err.Error() == "Object not found" { + web.Respond(w, r, http.StatusNotFound, nil) + } else { + web.Respond(w, r, http.StatusInternalServerError, err) + } + + return + } + + web.Respond(w, r, http.StatusOK, detect) +} + +func (h *DetectionHandler) postDetection(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + detect := &model.Detection{} + + err := web.ReadJson(r, detect) + if err != nil { + web.Respond(w, r, http.StatusBadRequest, err) + return + } + + detect, err = h.server.Casestore.CreateDetection(ctx, detect) + if err != nil { + web.Respond(w, r, http.StatusBadRequest, err) + return + } + + web.Respond(w, r, http.StatusCreated, detect) +} + +func (h *DetectionHandler) putDetection(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + detect := &model.Detection{} + + err := web.ReadJson(r, detect) + if err != nil { + web.Respond(w, r, http.StatusBadRequest, err) + return + } + + detect, err = h.server.Casestore.UpdateDetection(ctx, detect) + if err != nil { + web.Respond(w, r, http.StatusNotFound, err) + return + } + + web.Respond(w, r, http.StatusOK, detect) +} + +func (h *DetectionHandler) deleteDetection(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + id := chi.URLParam(r, "onionId") + + err := h.server.Casestore.DeleteDetection(ctx, id) + if err != nil { + web.Respond(w, r, http.StatusInternalServerError, err) + return + } + + web.Respond(w, r, http.StatusOK, nil) +} diff --git a/server/infohandler.go b/server/infohandler.go index 783c41e81..c97960eb6 100644 --- a/server/infohandler.go +++ b/server/infohandler.go @@ -11,6 +11,7 @@ import ( "net/http" "os" + "github.com/security-onion-solutions/securityonion-soc/config" "github.com/security-onion-solutions/securityonion-soc/licensing" "github.com/security-onion-solutions/securityonion-soc/model" "github.com/security-onion-solutions/securityonion-soc/web" @@ -61,5 +62,49 @@ func (h *InfoHandler) getInfo(w http.ResponseWriter, r *http.Request) { SrvToken: srvToken, } + info.Parameters.DetectionsParams.Queries = []*config.HuntingQuery{ + { + Name: "All", + Query: "_id:*", + }, + } + + info.Parameters.DetectionsParams.ViewEnabled = true + info.Parameters.DetectionsParams.CreateLink = "/detection/create" + info.Parameters.DetectionsParams.EventFetchLimit = 500 + info.Parameters.DetectionsParams.EventItemsPerPage = 50 + info.Parameters.DetectionsParams.GroupFetchLimit = 50 + info.Parameters.DetectionsParams.MostRecentlyUsedLimit = 5 + info.Parameters.DetectionsParams.QueryBaseFilter = "_index:\"*:so-case\" AND so_kind:detection" + info.Parameters.DetectionsParams.EventFields = map[string][]string{ + "default": { + "soc_timestamp", + "so_detection.publicId", + "so_detection.title", + "so_detection.severity", + "so_detection.isEnabled", + "so_detection.engine", + }, + } + + info.Parameters.PlaybooksParams.ViewEnabled = true + info.Parameters.PlaybooksParams.CreateLink = "/playbook/create" + info.Parameters.PlaybooksParams.EventFetchLimit = 500 + info.Parameters.PlaybooksParams.EventItemsPerPage = 50 + info.Parameters.PlaybooksParams.GroupFetchLimit = 50 + info.Parameters.PlaybooksParams.MostRecentlyUsedLimit = 5 + info.Parameters.PlaybooksParams.QueryBaseFilter = "_index:\"*:so-case\" AND so_kind:playbook" + info.Parameters.PlaybooksParams.Queries = []*config.HuntingQuery{ + {Query: "*"}, + } + info.Parameters.PlaybooksParams.EventFields = map[string][]string{ + "default": { + "soc_timestamp", + "so_playbook.title", + "so_playbook.publicId", + "so_playbook.mechanism", + }, + } + web.Respond(w, r, http.StatusOK, info) } diff --git a/server/modules/elastic/converter.go b/server/modules/elastic/converter.go index bdd4d2be2..f200b7405 100644 --- a/server/modules/elastic/converter.go +++ b/server/modules/elastic/converter.go @@ -639,6 +639,99 @@ func convertElasticEventToArtifactStream(event *model.EventRecord, schemaPrefix return obj, err } +func convertElasticEventToDetection(event *model.EventRecord, schemaPrefix string) (*model.Detection, error) { + var err error + var obj *model.Detection + + if event != nil { + obj = &model.Detection{} + err = convertElasticEventToAuditable(event, &obj.Auditable, schemaPrefix) + if err == nil { + if value, ok := event.Payload[schemaPrefix+"detection.userId"]; ok { + obj.UserId = value.(string) + } + if value, ok := event.Payload[schemaPrefix+"detection.isReporting"]; ok { + obj.IsReporting = value.(bool) + } + if value, ok := event.Payload[schemaPrefix+"detection.description"]; ok { + obj.Description = value.(string) + } + if value, ok := event.Payload[schemaPrefix+"detection.isEnabled"]; ok { + obj.IsEnabled = value.(bool) + } + if value, ok := event.Payload[schemaPrefix+"detection.engine"]; ok { + obj.Engine = value.(string) + } + if value, ok := event.Payload[schemaPrefix+"detection.publicId"]; ok { + obj.PublicID = value.(string) + } + if value, ok := event.Payload[schemaPrefix+"detection.severity"]; ok { + obj.Severity = model.Severity(value.(string)) + } + if value, ok := event.Payload[schemaPrefix+"detection.content"]; ok { + obj.Content = value.(string) + } + if value, ok := event.Payload[schemaPrefix+"detection.author"]; ok { + obj.Author = value.(string) + } + if value, ok := event.Payload[schemaPrefix+"detection.title"]; ok { + obj.Title = value.(string) + } + // if value, ok := event.Payload[schemaPrefix+"artifact.caseId"]; ok { + // obj.CaseId = value.(string) + // } + // if value, ok := event.Payload[schemaPrefix+"artifact.groupType"]; ok { + // obj.GroupType = value.(string) + // } + // if value, ok := event.Payload[schemaPrefix+"artifact.groupId"]; ok { + // obj.GroupId = value.(string) + // } + // if value, ok := event.Payload[schemaPrefix+"artifact.description"]; ok { + // obj.Description = value.(string) + // } + // if value, ok := event.Payload[schemaPrefix+"artifact.artifactType"]; ok { + // obj.ArtifactType = value.(string) + // } + // if value, ok := event.Payload[schemaPrefix+"artifact.streamLength"]; ok { + // obj.StreamLen = int(value.(float64)) + // } + // if value, ok := event.Payload[schemaPrefix+"artifact.streamId"]; ok { + // obj.StreamId = value.(string) + // } + // if value, ok := event.Payload[schemaPrefix+"artifact.mimeType"]; ok { + // obj.MimeType = value.(string) + // } + // if value, ok := event.Payload[schemaPrefix+"artifact.value"]; ok { + // obj.Value = value.(string) + // } + // if value, ok := event.Payload[schemaPrefix+"artifact.tlp"]; ok { + // obj.Tlp = value.(string) + // } + // if value, ok := event.Payload[schemaPrefix+"artifact.tags"]; ok && value != nil { + // obj.Tags = convertToStringArray(value.([]interface{})) + // } + // if value, ok := event.Payload[schemaPrefix+"artifact.ioc"]; ok { + // obj.Ioc = value.(bool) + // } + // if value, ok := event.Payload[schemaPrefix+"artifact.md5"]; ok { + // obj.Md5 = value.(string) + // } + // if value, ok := event.Payload[schemaPrefix+"artifact.sha1"]; ok { + // obj.Sha1 = value.(string) + // } + // if value, ok := event.Payload[schemaPrefix+"artifact.sha256"]; ok { + // obj.Sha256 = value.(string) + // } + // if value, ok := event.Payload[schemaPrefix+"artifact.protected"]; ok { + // obj.Protected = value.(bool) + // } + obj.CreateTime = parseTime(event.Payload, schemaPrefix+"detection.createTime") + } + } + + return obj, err +} + func convertElasticEventToObject(event *model.EventRecord, schemaPrefix string) (interface{}, error) { var obj interface{} var err error @@ -655,6 +748,8 @@ func convertElasticEventToObject(event *model.EventRecord, schemaPrefix string) obj, err = convertElasticEventToArtifact(event, schemaPrefix) case "artifactstream": obj, err = convertElasticEventToArtifactStream(event, schemaPrefix) + case "detection": + obj, err = convertElasticEventToDetection(event, schemaPrefix) } } else { err = errors.New("Unknown object kind; id=" + event.Id) diff --git a/server/modules/elastic/elasticcasestore.go b/server/modules/elastic/elasticcasestore.go index 5eeaf0d42..acd8eb3e6 100644 --- a/server/modules/elastic/elasticcasestore.go +++ b/server/modules/elastic/elasticcasestore.go @@ -273,6 +273,46 @@ func (store *ElasticCasestore) validateArtifactStream(artifactstream *model.Arti return err } +func (store *ElasticCasestore) validateDetection(detect *model.Detection) error { + var err error + + if err == nil && detect.Id != "" { + err = store.validateId(detect.Id, "onionId") + } + + if err == nil && detect.PublicID != "" { + err = store.validateId(detect.PublicID, "publicId") + } + + if err == nil && detect.Title != "" { + err = store.validateString(detect.Title, SHORT_STRING_MAX, "title") + } + + if err == nil && detect.Severity != "" { + err = store.validateString(string(detect.Severity), SHORT_STRING_MAX, "severity") + } + + if err == nil && detect.Author != "" { + err = store.validateString(detect.Author, SHORT_STRING_MAX, "author") + } + + if err == nil && detect.Description != "" { + err = store.validateString(detect.Description, LONG_STRING_MAX, "description") + } + + if err == nil && detect.Content != "" { + err = store.validateString(detect.Content, LONG_STRING_MAX, "content") + } + + if err == nil { + _, okEngine := model.EnginesByName[detect.Engine] + if !okEngine { + err = errors.New("invalid engine") + } + } + return err +} + func (store *ElasticCasestore) prepareForSave(ctx context.Context, obj *model.Auditable) string { obj.UserId = ctx.Value(web.ContextKeyRequestorId).(string) @@ -900,3 +940,77 @@ func (store *ElasticCasestore) ExtractCommonObservables(ctx context.Context, eve } return nil } + +func (store *ElasticCasestore) CreateDetection(ctx context.Context, detect *model.Detection) (*model.Detection, error) { + var err error + + err = store.validateDetection(detect) + if err != nil { + return nil, err + } + + if detect.Id != "" { + return nil, errors.New("Unexpected ID found in new comment") + } + + now := time.Now() + detect.CreateTime = &now + var results *model.EventIndexResults + results, err = store.save(ctx, detect, "detection", store.prepareForSave(ctx, &detect.Auditable)) + if err == nil { + // Read object back to get new modify date, etc + detect, err = store.GetDetection(ctx, results.DocumentId) + } + + return detect, err +} + +func (store *ElasticCasestore) GetDetection(ctx context.Context, detectId string) (detect *model.Detection, err error) { + err = store.validateId(detectId, "detectId") + if err != nil { + return nil, err + } + + obj, err := store.get(ctx, detectId, "detection") + if err == nil && obj != nil { + detect = obj.(*model.Detection) + } + + return detect, err +} + +func (store *ElasticCasestore) UpdateDetection(ctx context.Context, detect *model.Detection) (*model.Detection, error) { + err := store.validateDetection(detect) + if err == nil { + if detect.Id == "" { + err = errors.New("Missing detection onion ID") + return nil, err + } + + var old *model.Detection + old, err = store.GetDetection(ctx, detect.Id) + if err == nil { + var results *model.EventIndexResults + + // Preserve read-only fields + detect.CreateTime = old.CreateTime + + results, err = store.save(ctx, detect, "detection", store.prepareForSave(ctx, &detect.Auditable)) + if err == nil { + // Read object back to get new modify date, etc + detect, err = store.GetDetection(ctx, results.DocumentId) + } + } + } + + return detect, err +} + +func (store *ElasticCasestore) DeleteDetection(ctx context.Context, onionID string) error { + detect, err := store.GetDetection(ctx, onionID) + if err == nil { + err = store.delete(ctx, detect, "detection", store.prepareForSave(ctx, &detect.Auditable)) + } + + return err +} diff --git a/server/modules/elasticcases/elasticcasestore.go b/server/modules/elasticcases/elasticcasestore.go index 4250e0a6c..9d8357e8c 100644 --- a/server/modules/elasticcases/elasticcasestore.go +++ b/server/modules/elasticcases/elasticcasestore.go @@ -144,3 +144,19 @@ func (store *ElasticCasestore) GetArtifactStream(ctx context.Context, id string) func (store *ElasticCasestore) DeleteArtifactStream(ctx context.Context, id string) error { return errors.New("Unsupported operation by this module") } + +func (store *ElasticCasestore) CreateDetection(ctx context.Context, detect *model.Detection) (*model.Detection, error) { + return nil, errors.New("Unsupported operation by this module") +} + +func (store *ElasticCasestore) GetDetection(ctx context.Context, detectId string) (*model.Detection, error) { + return nil, errors.New("Unsupported operation by this module") +} + +func (store *ElasticCasestore) UpdateDetection(ctx context.Context, detect *model.Detection) (*model.Detection, error) { + return nil, errors.New("Unsupported operation by this module") +} + +func (store *ElasticCasestore) DeleteDetection(ctx context.Context, detectID string) error { + return errors.New("Unsupported operation by this module") +} diff --git a/server/modules/generichttp/httpcasestore.go b/server/modules/generichttp/httpcasestore.go index 69c256901..f74c58a93 100644 --- a/server/modules/generichttp/httpcasestore.go +++ b/server/modules/generichttp/httpcasestore.go @@ -149,3 +149,19 @@ func (store *HttpCasestore) GetArtifactStream(ctx context.Context, id string) (* func (store *HttpCasestore) DeleteArtifactStream(ctx context.Context, id string) error { return errors.New("Unsupported operation by this module") } + +func (store *HttpCasestore) CreateDetection(ctx context.Context, detect *model.Detection) (*model.Detection, error) { + return nil, errors.New("Unsupported operation by this module") +} + +func (store *HttpCasestore) GetDetection(ctx context.Context, detectId string) (*model.Detection, error) { + return nil, errors.New("Unsupported operation by this module") +} + +func (store *HttpCasestore) UpdateDetection(ctx context.Context, detect *model.Detection) (*model.Detection, error) { + return nil, errors.New("Unsupported operation by this module") +} + +func (store *HttpCasestore) DeleteDetection(ctx context.Context, detectID string) error { + return errors.New("Unsupported operation by this module") +} diff --git a/server/modules/thehive/thehivecasestore.go b/server/modules/thehive/thehivecasestore.go index 44a5cf3a5..04740aa10 100644 --- a/server/modules/thehive/thehivecasestore.go +++ b/server/modules/thehive/thehivecasestore.go @@ -141,3 +141,19 @@ func (store *TheHiveCasestore) GetArtifactStream(ctx context.Context, id string) func (store *TheHiveCasestore) DeleteArtifactStream(ctx context.Context, id string) error { return errors.New("Unsupported operation by this module") } + +func (store *TheHiveCasestore) CreateDetection(ctx context.Context, detect *model.Detection) (*model.Detection, error) { + return nil, errors.New("Unsupported operation by this module") +} + +func (store *TheHiveCasestore) GetDetection(ctx context.Context, detectId string) (*model.Detection, error) { + return nil, errors.New("Unsupported operation by this module") +} + +func (store *TheHiveCasestore) UpdateDetection(ctx context.Context, detect *model.Detection) (*model.Detection, error) { + return nil, errors.New("Unsupported operation by this module") +} + +func (store *TheHiveCasestore) DeleteDetection(ctx context.Context, detectID string) error { + return errors.New("Unsupported operation by this module") +} diff --git a/server/server.go b/server/server.go index 185217326..1894bb1c5 100644 --- a/server/server.go +++ b/server/server.go @@ -87,6 +87,7 @@ func (server *Server) Start() { RegisterConfigRoutes(server, r, "/api/config") RegisterGridMemberRoutes(server, r, "/api/gridmembers") RegisterRolesRoutes(server, r, "/api/roles") + RegisterDetectionRoutes(server, r, "/api/detection") RegisterUtilRoutes(server, r, "/api/util") server.Host.RegisterRouter("/api/", r) From 648e2b5a0959bbaa6804111ca4b1b8ced0f4459b Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Fri, 1 Sep 2023 15:24:15 -0600 Subject: [PATCH 002/102] WIP: More Detections Work Walked back playbooks to focus on detections. Made detections more data driven. Added form validation when creating a detection. TODO: form validation when editing. Added Duplicate functionality to the API/UI, added Bulk enable/disable to API. Touched the surface on how detections will sync. --- config/clientparameters.go | 53 +++++--- html/index.html | 109 ++++++++++++++--- html/js/i18n.js | 7 ++ html/js/routes/detection.js | 63 +++++++--- html/js/routes/hunt.js | 3 + model/detection.go | 22 ++++ server/casestore.go | 3 +- server/detectionhandler.go | 115 +++++++++++++++++- server/infohandler.go | 98 +++++++++------ server/modules/elastic/converter.go | 95 ++++----------- server/modules/elastic/elasticcasestore.go | 89 +++++++++++--- .../modules/elasticcases/elasticcasestore.go | 8 +- server/modules/generichttp/httpcasestore.go | 8 +- server/modules/thehive/thehivecasestore.go | 8 +- server/server.go | 12 -- util/ptr.go | 13 ++ 16 files changed, 495 insertions(+), 211 deletions(-) create mode 100644 util/ptr.go diff --git a/config/clientparameters.go b/config/clientparameters.go index 7681ae6e6..ee31adf90 100644 --- a/config/clientparameters.go +++ b/config/clientparameters.go @@ -16,26 +16,27 @@ const DEFAULT_CHART_LABEL_OTHER_LIMIT = 10 const DEFAULT_CHART_LABEL_FIELD_SEPARATOR = ", " type ClientParameters struct { - HuntingParams HuntingParameters `json:"hunt"` - AlertingParams HuntingParameters `json:"alerts"` - CasesParams HuntingParameters `json:"cases"` - CaseParams CaseParameters `json:"case"` - DashboardsParams HuntingParameters `json:"dashboards"` - JobParams HuntingParameters `json:"job"` - DetectionsParams HuntingParameters `json:"detections"` - PlaybooksParams HuntingParameters `json:"playbooks"` - DocsUrl string `json:"docsUrl"` - CheatsheetUrl string `json:"cheatsheetUrl"` - ReleaseNotesUrl string `json:"releaseNotesUrl"` - GridParams GridParameters `json:"grid"` - WebSocketTimeoutMs int `json:"webSocketTimeoutMs"` - TipTimeoutMs int `json:"tipTimeoutMs"` - ApiTimeoutMs int `json:"apiTimeoutMs"` - CacheExpirationMs int `json:"cacheExpirationMs"` - InactiveTools []string `json:"inactiveTools"` - Tools []ClientTool `json:"tools"` - CasesEnabled bool `json:"casesEnabled"` - EnableReverseLookup bool `json:"enableReverseLookup"` + HuntingParams HuntingParameters `json:"hunt"` + AlertingParams HuntingParameters `json:"alerts"` + CasesParams HuntingParameters `json:"cases"` + CaseParams CaseParameters `json:"case"` + DashboardsParams HuntingParameters `json:"dashboards"` + JobParams HuntingParameters `json:"job"` + DetectionsParams HuntingParameters `json:"detections"` + DetectionParams DetectionParameters `json:"detection"` + PlaybooksParams HuntingParameters `json:"playbooks"` + DocsUrl string `json:"docsUrl"` + CheatsheetUrl string `json:"cheatsheetUrl"` + ReleaseNotesUrl string `json:"releaseNotesUrl"` + GridParams GridParameters `json:"grid"` + WebSocketTimeoutMs int `json:"webSocketTimeoutMs"` + TipTimeoutMs int `json:"tipTimeoutMs"` + ApiTimeoutMs int `json:"apiTimeoutMs"` + CacheExpirationMs int `json:"cacheExpirationMs"` + InactiveTools []string `json:"inactiveTools"` + Tools []ClientTool `json:"tools"` + CasesEnabled bool `json:"casesEnabled"` + EnableReverseLookup bool `json:"enableReverseLookup"` } func (config *ClientParameters) Verify() error { @@ -188,3 +189,15 @@ type GridParameters struct { MaxUploadSize uint64 `json:"maxUploadSize,omitempty"` StaleMetricsMs uint64 `json:"staleMetricsMs,omitempty"` } + +type DetectionParameters struct { + HuntingParameters + Presets map[string]PresetParameters `json:"presets"` +} + +func (params *DetectionParameters) Verify() error { + err := params.HuntingParameters.Verify() + + return err + +} diff --git a/html/index.html b/html/index.html index 94c1d50e7..cf1329eb8 100644 --- a/html/index.html +++ b/html/index.html @@ -76,14 +76,14 @@ - + fa-stream @@ -970,7 +970,50 @@

- + +
+ +
+ + + + + + + + + + + + + + + + + + + + + +
{{ i18n.signature }}:
+ +
+
+
+ + {{ i18n.cancel }} + + + {{ i18n.create }} + +
+
+ +
+ + fa-clipboard
Summary
@@ -987,10 +1030,10 @@

Signature

- + fa-wrench
Tuning
@@ -1009,12 +1052,19 @@

- + +
+
+ + {{i18n.duplicate}} + +
+
@@ -1027,35 +1077,52 @@

- +

-
+
+
+
+ + {{ i18n.cancel }} + + + {{i18n.update}} + +
+
+
+
+ + {{ i18n.cancel }} + + + {{i18n.update}} + +
+
- + - Coming Soon! +
+ Coming Soon! +
- Coming Soon! +
+ Coming Soon! +
@@ -1707,7 +1778,6 @@

- - - + diff --git a/html/js/i18n.js b/html/js/i18n.js index 7bbe4f3b5..0b9b05ee8 100644 --- a/html/js/i18n.js +++ b/html/js/i18n.js @@ -255,6 +255,9 @@ const i18n = { description: 'Description', details: 'Details', detections: 'Detections', + detectionDefaultTitle: 'Detection title not yet provided - click here to update this title', + detectionDefaultDescription: 'Detection description not yet provided', + detectionSeverity: 'Severity', disconnected: 'Disconnected from manager', diskUsageElastic: 'Elastic Storage Used', diskUsageInfluxDb: 'InfluxDB Storage Used', @@ -272,6 +275,7 @@ const i18n = { dstIpHelp: 'Optional destination IP address to include in this job filter', dstPort: 'Destination Port', dstPortHelp: 'Optional destination TCP port to include in this job filter', + duplicate: 'Duplicate', edit: 'Edit', edited: '(edited)', email: 'Email Address', @@ -279,6 +283,7 @@ const i18n = { emailRequired: 'An email address must be specified.', endTime: 'Filter End', endTimeHelp: 'Filter end time in RFC 3339 format (Ex: 2020-10-16 15:30:00.230-04:00). Unused for imported PCAPs.', + engine: 'Engine', eps: 'EPS', epsProduction: 'Production EPS:', epsConsumption: 'Consumption EPS:', @@ -574,6 +579,7 @@ const i18n = { relatedEventId: 'Related Event ID', relativeTimeHelp: 'Click the clock icon to change to absolute time', remove: 'Remove', + reporting: 'Reporting', required: 'Required.', reset: 'Reset', resetDefaults: 'Reset Defaults', @@ -681,6 +687,7 @@ const i18n = { showBarChart: 'Show bar chart', showSankeyChart: 'Show Sankey diagram', showTable: 'Show table', + signature: 'Signature', socUrl: 'SOC Url', socExcludeToggle: 'Exclude SOC logs', sortedBy: 'Sort:', diff --git a/html/js/routes/detection.js b/html/js/routes/detection.js index b1e2e61ed..5fd098061 100644 --- a/html/js/routes/detection.js +++ b/html/js/routes/detection.js @@ -8,12 +8,14 @@ routes.push({ path: '/detection/:id', name: 'detection', component: { template: '#page-detection', data() { return { i18n: this.$root.i18n, + presets: {}, params: {}, detect: null, origDetect: null, curEditTarget: null, // string containing element ID, null if not editing origValue: null, editField: null, + editForm: { valid: true }, rules: { required: value => (value && value.length > 0) || this.$root.i18n.required, number: value => (! isNaN(+value) && Number.isInteger(parseFloat(value))) || this.$root.i18n.required, @@ -24,35 +26,25 @@ routes.push({ path: '/detection/:id', name: 'detection', component: { fileNotEmpty: value => (value == null || value.size > 0) || this.$root.i18n.fileEmpty, fileRequired: value => (value != null) || this.$root.i18n.required, }, - severityOptions: ['low', 'medium', 'high'], - engineOptions: ['suricata', 'yara', 'elastalert'], panel: [0, 1, 2], activeTab: '', - associatedPlaybook: { - onionId: 'y5IYKIoB9-Z7uL2kmy_o', - publicId: "4020131e-223a-421e-8ebe-8a211a5ac4d6", - title: "Find the baddies", - severity: "high", - description: "A long description that spans multiple lines. A long description that spans multiple lines. A long description that spans multiple lines. A long description that spans multiple lines. A long description that spans multiple lines. A long description that spans multiple lines.", - mechanism: "suricata", - tags: ["one", "two", "three"], - relatedPlaybooks: [], - contributors: ["Corey Ogburn"], - userEditable: true, - createTime: "2023-08-22T12:49:47.302819008-06:00", - userId: "83656890-2acd-4c0b-8ab9-7c73e71ddaf3", - }, }}, created() { }, watch: { }, mounted() { + this.$watch( + () => this.$route.params, + (to, prev) => { + this.loadData(); + }); this.$root.loadParameters('detection', this.initDetection); }, methods: { async initDetection(params) { this.params = params; + this.presets = params['presets']; if (this.$route.params.id === 'create') { this.detect = this.newDetection(); } else { @@ -69,11 +61,11 @@ routes.push({ path: '/detection/:id', name: 'detection', component: { newDetection() { let author = [this.$root.user.firstName, this.$root.user.lastName].filter(x => x).join(' '); return { - title: 'Detection title not yet provided - click here to update this title', - description: 'Detection description not yet provided - click here to update this description', + title: this.i18n.detectionDefaultTitle, + description: this.i18n.detectionDefaultDescription, author: author, publicId: '', - severity: 'low', + severity: this.getDefaultPreset('severity'), content: '', isEnabled: false, isReporting: false, @@ -86,7 +78,6 @@ routes.push({ path: '/detection/:id', name: 'detection', component: { try { const response = await this.$root.papi.get('detection/' + encodeURIComponent(this.$route.params.id)); this.detect = response.data; - this.detect.note = 'This is a note.'; } catch (error) { if (error.response != undefined && error.response.status == 404) { this.$root.showError(this.i18n.notFound); @@ -97,6 +88,27 @@ routes.push({ path: '/detection/:id', name: 'detection', component: { this.$root.stopLoading(); }, + getDefaultPreset(preset) { + if (this.presets) { + const presets = this.presets[preset]; + if (presets && presets.labels && presets.labels.length > 0) { + return presets.labels[0]; + } + } + return ""; + }, + getPresets(kind) { + if (this.presets && this.presets[kind]) { + return this.presets[kind].labels; + } + return []; + }, + isPresetCustomEnabled(kind) { + if (this.presets && this.presets[kind]) { + return this.presets[kind].customEnabled == true; + } + return false; + }, isNew() { return this.$route.params.id === 'create'; }, @@ -143,12 +155,23 @@ routes.push({ path: '/detection/:id', name: 'detection', component: { this.editField = null; }, saveDetection(createNew) { + if (this.curEditTarget !== null) this.stopEdit(true); + + this.$refs['detection'].validate(); + if (!this.editForm.valid) return; + if (createNew) { this.$root.papi.post('/detection', this.detect); } else { this.$root.papi.put('/detection', this.detect); } this.origDetect = Object.assign({}, this.detect); + + this.$root.showTip(this.i18n.saveSuccess); + }, + async duplicateDetection() { + const response = await this.$root.papi.post('/detection/' + encodeURIComponent(this.$route.params.id) + '/duplicate'); + this.$router.push({name: 'detection', params: {id: response.data.id}}); }, print(x) { console.log(x); diff --git a/html/js/routes/hunt.js b/html/js/routes/hunt.js index c8c6c3ff8..6c7f3496b 100644 --- a/html/js/routes/hunt.js +++ b/html/js/routes/hunt.js @@ -2162,6 +2162,9 @@ const huntComponent = { console.log('No onionID'); } }, + print(x) { + console.log(x); + } } }; diff --git a/model/detection.go b/model/detection.go index 54b2ed801..4d3f06a1c 100644 --- a/model/detection.go +++ b/model/detection.go @@ -77,6 +77,8 @@ type Detection struct { Content string `json:"content"` IsEnabled bool `json:"isEnabled"` IsReporting bool `json:"isReporting"` + IsCommunity bool `json:"isCommunity"` + Note string `json:"note"` Engine EngineName `json:"engine"` } @@ -89,3 +91,23 @@ func (detect *Detection) Validate() error { return nil } + +func SyncDetections(detections []*Detection) error { + byEngine := map[EngineName][]*Detection{} + for _, detect := range detections { + byEngine[detect.Engine] = append(byEngine[detect.Engine], detect) + } + + if len(byEngine[EngineNameSuricata]) > 0 { + err := syncSuricata(byEngine[EngineNameSuricata]) + if err != nil { + return err + } + } + + return nil +} + +func syncSuricata(detections []*Detection) error { + return nil +} diff --git a/server/casestore.go b/server/casestore.go index 1f5bcb204..6d6eda5ac 100644 --- a/server/casestore.go +++ b/server/casestore.go @@ -42,5 +42,6 @@ type Casestore interface { CreateDetection(ctx context.Context, detect *model.Detection) (*model.Detection, error) GetDetection(ctx context.Context, detectId string) (*model.Detection, error) UpdateDetection(ctx context.Context, detect *model.Detection) (*model.Detection, error) - DeleteDetection(ctx context.Context, detectID string) error + UpdateDetectionField(ctx context.Context, id string, field string, value any) (*model.Detection, bool, error) + DeleteDetection(ctx context.Context, detectID string) (*model.Detection, error) } diff --git a/server/detectionhandler.go b/server/detectionhandler.go index fa1bb4396..56806f1bd 100644 --- a/server/detectionhandler.go +++ b/server/detectionhandler.go @@ -7,7 +7,9 @@ package server import ( + "fmt" "net/http" + "strings" "github.com/security-onion-solutions/securityonion-soc/model" "github.com/security-onion-solutions/securityonion-soc/web" @@ -25,20 +27,23 @@ func RegisterDetectionRoutes(srv *Server, r chi.Router, prefix string) { } r.Route(prefix, func(r chi.Router) { - r.Get("/{onionId}", h.getDetection) + r.Get("/{id}", h.getDetection) r.Post("/", h.postDetection) + r.Post("/{id}/duplicate", h.duplicateDetection) r.Put("/", h.putDetection) - r.Delete("/{onionId}", h.deleteDetection) + r.Delete("/{id}", h.deleteDetection) + + r.Post("/bulk/{newStatus}", h.bulkUpdateDetection) }) } func (h *DetectionHandler) getDetection(w http.ResponseWriter, r *http.Request) { ctx := r.Context() - detectId := chi.URLParam(r, "onionId") + detectId := chi.URLParam(r, "id") detect, err := h.server.Casestore.GetDetection(ctx, detectId) if err != nil { @@ -71,7 +76,42 @@ func (h *DetectionHandler) postDetection(w http.ResponseWriter, r *http.Request) return } - web.Respond(w, r, http.StatusCreated, detect) + err = model.SyncDetections([]*model.Detection{detect}) + if err != nil { + web.Respond(w, r, http.StatusInternalServerError, err) + return + } + + web.Respond(w, r, http.StatusOK, detect) +} + +func (h *DetectionHandler) duplicateDetection(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + detectId := chi.URLParam(r, "id") + + detect, err := h.server.Casestore.GetDetection(ctx, detectId) + if err != nil { + web.Respond(w, r, http.StatusInternalServerError, err) + return + } + + detect.Id = "" + detect.PublicID = "" + detect.Title = fmt.Sprintf("%s (copy)", detect.Title) + detect.CreateTime = nil + detect.UpdateTime = nil + detect.IsEnabled = false + detect.IsReporting = false + detect.IsCommunity = false + + detect, err = h.server.Casestore.CreateDetection(ctx, detect) + if err != nil { + web.Respond(w, r, http.StatusInternalServerError, err) + return + } + + web.Respond(w, r, http.StatusOK, detect) } func (h *DetectionHandler) putDetection(w http.ResponseWriter, r *http.Request) { @@ -91,15 +131,27 @@ func (h *DetectionHandler) putDetection(w http.ResponseWriter, r *http.Request) return } + err = model.SyncDetections([]*model.Detection{detect}) + if err != nil { + web.Respond(w, r, http.StatusInternalServerError, err) + return + } + web.Respond(w, r, http.StatusOK, detect) } func (h *DetectionHandler) deleteDetection(w http.ResponseWriter, r *http.Request) { ctx := r.Context() - id := chi.URLParam(r, "onionId") + id := chi.URLParam(r, "id") - err := h.server.Casestore.DeleteDetection(ctx, id) + old, err := h.server.Casestore.DeleteDetection(ctx, id) + if err != nil { + web.Respond(w, r, http.StatusInternalServerError, err) + return + } + + err = model.SyncDetections([]*model.Detection{old}) if err != nil { web.Respond(w, r, http.StatusInternalServerError, err) return @@ -107,3 +159,54 @@ func (h *DetectionHandler) deleteDetection(w http.ResponseWriter, r *http.Reques web.Respond(w, r, http.StatusOK, nil) } + +func (h *DetectionHandler) bulkUpdateDetection(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + newStatus := chi.URLParam(r, "newStatus") // "enable" or "disable" + + var enabled bool + switch strings.ToLower(newStatus) { + case "enable", "disable": + enabled = strings.ToLower(newStatus) == "enable" + default: + web.Respond(w, r, http.StatusBadRequest, fmt.Errorf("invalid status; must be 'enable' or 'disable'")) + return + } + + body := []string{} + err := web.ReadJson(r, body) + if err != nil { + web.Respond(w, r, http.StatusBadRequest, err) + return + } + + IDs := map[string]struct{}{} + + for _, id := range body { + IDs[id] = struct{}{} + } + + errMap := map[string]string{} // map[id]error + modified := []*model.Detection{} + + for id := range IDs { + det, mod, err := h.server.Casestore.UpdateDetectionField(ctx, id, "IsEnabled", enabled) + if err != nil { + errMap[id] = fmt.Sprintf("unable to update detection; reason=%s", err.Error()) + continue + } + + if mod { + modified = append(modified, det) + } + } + + err = model.SyncDetections(modified) + if err != nil { + web.Respond(w, r, http.StatusInternalServerError, err) + return + } + + web.Respond(w, r, http.StatusOK, errMap) +} diff --git a/server/infohandler.go b/server/infohandler.go index c97960eb6..3d7604f31 100644 --- a/server/infohandler.go +++ b/server/infohandler.go @@ -62,48 +62,70 @@ func (h *InfoHandler) getInfo(w http.ResponseWriter, r *http.Request) { SrvToken: srvToken, } - info.Parameters.DetectionsParams.Queries = []*config.HuntingQuery{ - { - Name: "All", - Query: "_id:*", - }, - } + { + detections := &info.Parameters.DetectionsParams - info.Parameters.DetectionsParams.ViewEnabled = true - info.Parameters.DetectionsParams.CreateLink = "/detection/create" - info.Parameters.DetectionsParams.EventFetchLimit = 500 - info.Parameters.DetectionsParams.EventItemsPerPage = 50 - info.Parameters.DetectionsParams.GroupFetchLimit = 50 - info.Parameters.DetectionsParams.MostRecentlyUsedLimit = 5 - info.Parameters.DetectionsParams.QueryBaseFilter = "_index:\"*:so-case\" AND so_kind:detection" - info.Parameters.DetectionsParams.EventFields = map[string][]string{ - "default": { - "soc_timestamp", - "so_detection.publicId", - "so_detection.title", - "so_detection.severity", - "so_detection.isEnabled", - "so_detection.engine", - }, + detections.ViewEnabled = true + detections.CreateLink = "/detection/create" + detections.EventFetchLimit = 500 + detections.EventItemsPerPage = 50 + detections.GroupFetchLimit = 50 + detections.MostRecentlyUsedLimit = 5 + detections.SafeStringMaxLength = 100 + detections.QueryBaseFilter = "_index:\"*:so-case\" AND so_kind:detection" + detections.EventFields = map[string][]string{ + "default": { + "so_detection.title", + "so_detection.isEnabled", + "so_detection.engine", + "@timestamp", + }, + } + detections.Queries = []*config.HuntingQuery{ + { + Name: "All", + Query: "_id:*", + }, + } } - info.Parameters.PlaybooksParams.ViewEnabled = true - info.Parameters.PlaybooksParams.CreateLink = "/playbook/create" - info.Parameters.PlaybooksParams.EventFetchLimit = 500 - info.Parameters.PlaybooksParams.EventItemsPerPage = 50 - info.Parameters.PlaybooksParams.GroupFetchLimit = 50 - info.Parameters.PlaybooksParams.MostRecentlyUsedLimit = 5 - info.Parameters.PlaybooksParams.QueryBaseFilter = "_index:\"*:so-case\" AND so_kind:playbook" - info.Parameters.PlaybooksParams.Queries = []*config.HuntingQuery{ - {Query: "*"}, + { + detection := &info.Parameters.DetectionParams + + detection.Presets = map[string]config.PresetParameters{ + "severity": { + CustomEnabled: false, + Labels: []string{"low", "medium", "high"}, + }, + "engine": { + CustomEnabled: false, + Labels: []string{"suricata", "yara", "elastalert"}, + }, + } } - info.Parameters.PlaybooksParams.EventFields = map[string][]string{ - "default": { - "soc_timestamp", - "so_playbook.title", - "so_playbook.publicId", - "so_playbook.mechanism", - }, + + { + playbooks := &info.Parameters.PlaybooksParams + + playbooks.ViewEnabled = true + playbooks.CreateLink = "/playbook/create" + playbooks.EventFetchLimit = 500 + playbooks.EventItemsPerPage = 50 + playbooks.GroupFetchLimit = 50 + playbooks.MostRecentlyUsedLimit = 5 + playbooks.QueryBaseFilter = "_index:\"*:so-case\" AND so_kind:playbook" + playbooks.SafeStringMaxLength = 100 + playbooks.Queries = []*config.HuntingQuery{ + {Query: "*"}, + } + playbooks.EventFields = map[string][]string{ + "default": { + "soc_timestamp", + "so_playbook.title", + "so_playbook.publicId", + "so_playbook.mechanism", + }, + } } web.Respond(w, r, http.StatusOK, info) diff --git a/server/modules/elastic/converter.go b/server/modules/elastic/converter.go index f200b7405..8b8fbeaf5 100644 --- a/server/modules/elastic/converter.go +++ b/server/modules/elastic/converter.go @@ -227,7 +227,7 @@ func convertToElasticRequest(store *ElasticEventstore, criteria *model.EventSear sortBySegment := segment.(*model.SortBySegment) fields := sortBySegment.RawFields() if len(fields) > 0 { - sorting := make([]map[string]map[string]string, 0, 0) + sorting := []map[string]map[string]string{} for _, field := range fields { newSort := make(map[string]map[string]string) order := "desc" @@ -260,7 +260,7 @@ func parseAggregation(name string, aggObj interface{}, keys []interface{}, resul if buckets != nil { metrics := results.Metrics[name] if metrics == nil { - metrics = make([]*model.EventMetric, 0, 0) + metrics = []*model.EventMetric{} } for _, bucketObj := range buckets.([]interface{}) { bucket := bucketObj.(map[string]interface{}) @@ -293,9 +293,9 @@ func parseAggregation(name string, aggObj interface{}, keys []interface{}, resul func flattenKeyValue(store *ElasticEventstore, fieldMap map[string]interface{}, prefix string, value map[string]interface{}) { for key, value := range value { flattenedKey := prefix + key - switch value.(type) { + switch v := value.(type) { case map[string]interface{}: - flattenKeyValue(store, fieldMap, flattenedKey+".", value.(map[string]interface{})) + flattenKeyValue(store, fieldMap, flattenedKey+".", v) default: fieldMap[store.unmapElasticField(flattenedKey)] = value } @@ -650,81 +650,40 @@ func convertElasticEventToDetection(event *model.EventRecord, schemaPrefix strin if value, ok := event.Payload[schemaPrefix+"detection.userId"]; ok { obj.UserId = value.(string) } - if value, ok := event.Payload[schemaPrefix+"detection.isReporting"]; ok { - obj.IsReporting = value.(bool) - } - if value, ok := event.Payload[schemaPrefix+"detection.description"]; ok { - obj.Description = value.(string) - } - if value, ok := event.Payload[schemaPrefix+"detection.isEnabled"]; ok { - obj.IsEnabled = value.(bool) - } - if value, ok := event.Payload[schemaPrefix+"detection.engine"]; ok { - obj.Engine = value.(string) - } if value, ok := event.Payload[schemaPrefix+"detection.publicId"]; ok { obj.PublicID = value.(string) } + if value, ok := event.Payload[schemaPrefix+"detection.title"]; ok { + obj.Title = value.(string) + } if value, ok := event.Payload[schemaPrefix+"detection.severity"]; ok { obj.Severity = model.Severity(value.(string)) } + if value, ok := event.Payload[schemaPrefix+"detection.author"]; ok { + obj.Author = value.(string) + } + if value, ok := event.Payload[schemaPrefix+"detection.description"]; ok { + obj.Description = value.(string) + } if value, ok := event.Payload[schemaPrefix+"detection.content"]; ok { obj.Content = value.(string) } - if value, ok := event.Payload[schemaPrefix+"detection.author"]; ok { - obj.Author = value.(string) + if value, ok := event.Payload[schemaPrefix+"detection.isEnabled"]; ok { + obj.IsEnabled = value.(bool) } - if value, ok := event.Payload[schemaPrefix+"detection.title"]; ok { - obj.Title = value.(string) + if value, ok := event.Payload[schemaPrefix+"detection.isReporting"]; ok { + obj.IsReporting = value.(bool) + } + if value, ok := event.Payload[schemaPrefix+"detection.isCommunity"]; ok { + obj.IsCommunity = value.(bool) + } + if value, ok := event.Payload[schemaPrefix+"detection.note"]; ok { + obj.Note = value.(string) } - // if value, ok := event.Payload[schemaPrefix+"artifact.caseId"]; ok { - // obj.CaseId = value.(string) - // } - // if value, ok := event.Payload[schemaPrefix+"artifact.groupType"]; ok { - // obj.GroupType = value.(string) - // } - // if value, ok := event.Payload[schemaPrefix+"artifact.groupId"]; ok { - // obj.GroupId = value.(string) - // } - // if value, ok := event.Payload[schemaPrefix+"artifact.description"]; ok { - // obj.Description = value.(string) - // } - // if value, ok := event.Payload[schemaPrefix+"artifact.artifactType"]; ok { - // obj.ArtifactType = value.(string) - // } - // if value, ok := event.Payload[schemaPrefix+"artifact.streamLength"]; ok { - // obj.StreamLen = int(value.(float64)) - // } - // if value, ok := event.Payload[schemaPrefix+"artifact.streamId"]; ok { - // obj.StreamId = value.(string) - // } - // if value, ok := event.Payload[schemaPrefix+"artifact.mimeType"]; ok { - // obj.MimeType = value.(string) - // } - // if value, ok := event.Payload[schemaPrefix+"artifact.value"]; ok { - // obj.Value = value.(string) - // } - // if value, ok := event.Payload[schemaPrefix+"artifact.tlp"]; ok { - // obj.Tlp = value.(string) - // } - // if value, ok := event.Payload[schemaPrefix+"artifact.tags"]; ok && value != nil { - // obj.Tags = convertToStringArray(value.([]interface{})) - // } - // if value, ok := event.Payload[schemaPrefix+"artifact.ioc"]; ok { - // obj.Ioc = value.(bool) - // } - // if value, ok := event.Payload[schemaPrefix+"artifact.md5"]; ok { - // obj.Md5 = value.(string) - // } - // if value, ok := event.Payload[schemaPrefix+"artifact.sha1"]; ok { - // obj.Sha1 = value.(string) - // } - // if value, ok := event.Payload[schemaPrefix+"artifact.sha256"]; ok { - // obj.Sha256 = value.(string) - // } - // if value, ok := event.Payload[schemaPrefix+"artifact.protected"]; ok { - // obj.Protected = value.(bool) - // } + if value, ok := event.Payload[schemaPrefix+"detection.engine"]; ok { + obj.Engine = value.(string) + } + obj.CreateTime = parseTime(event.Payload, schemaPrefix+"detection.createTime") } } diff --git a/server/modules/elastic/elasticcasestore.go b/server/modules/elastic/elasticcasestore.go index acd8eb3e6..651f2fe0a 100644 --- a/server/modules/elastic/elasticcasestore.go +++ b/server/modules/elastic/elasticcasestore.go @@ -13,6 +13,7 @@ import ( "regexp" "sort" "strconv" + "strings" "time" "github.com/apex/log" @@ -304,6 +305,10 @@ func (store *ElasticCasestore) validateDetection(detect *model.Detection) error err = store.validateString(detect.Content, LONG_STRING_MAX, "content") } + if err == nil && detect.Note != "" { + err = store.validateString(detect.Note, LONG_STRING_MAX, "note") + } + if err == nil { _, okEngine := model.EnginesByName[detect.Engine] if !okEngine { @@ -981,36 +986,80 @@ func (store *ElasticCasestore) GetDetection(ctx context.Context, detectId string func (store *ElasticCasestore) UpdateDetection(ctx context.Context, detect *model.Detection) (*model.Detection, error) { err := store.validateDetection(detect) - if err == nil { - if detect.Id == "" { - err = errors.New("Missing detection onion ID") - return nil, err + if err != nil { + return nil, err + } + + if detect.Id == "" { + err = errors.New("Missing detection onion ID") + return nil, err + } + + var old *model.Detection + old, err = store.GetDetection(ctx, detect.Id) + if err != nil { + return nil, err + } + + var results *model.EventIndexResults + + // Preserve read-only fields + detect.CreateTime = old.CreateTime + + results, err = store.save(ctx, detect, "detection", store.prepareForSave(ctx, &detect.Auditable)) + if err != nil { + return nil, err + } + // Read object back to get new modify date, etc + return store.GetDetection(ctx, results.DocumentId) +} + +func (store *ElasticCasestore) UpdateDetectionField(ctx context.Context, id string, field string, value any) (*model.Detection, bool, error) { + var modified bool + + detect, err := store.GetDetection(ctx, id) + if err != nil { + return nil, false, err + } + + switch strings.ToLower(field) { + case "isenabled": + bVal, ok := value.(bool) + if !ok { + return nil, false, fmt.Errorf("invalid value for field isEnabled (expected bool): %[1]v (%[1]T)", value) } - var old *model.Detection - old, err = store.GetDetection(ctx, detect.Id) - if err == nil { - var results *model.EventIndexResults + if detect.IsEnabled != bVal { + detect.IsEnabled = bVal + modified = true + } + } - // Preserve read-only fields - detect.CreateTime = old.CreateTime + err = store.validateDetection(detect) + if err != nil { + return nil, false, err + } - results, err = store.save(ctx, detect, "detection", store.prepareForSave(ctx, &detect.Auditable)) - if err == nil { - // Read object back to get new modify date, etc - detect, err = store.GetDetection(ctx, results.DocumentId) - } + if modified { + var results *model.EventIndexResults + + results, err = store.save(ctx, detect, "detection", store.prepareForSave(ctx, &detect.Auditable)) + if err == nil { + // Read object back to get new modify date, etc + detect, err = store.GetDetection(ctx, results.DocumentId) } } - return detect, err + return detect, modified, err } -func (store *ElasticCasestore) DeleteDetection(ctx context.Context, onionID string) error { +func (store *ElasticCasestore) DeleteDetection(ctx context.Context, onionID string) (*model.Detection, error) { detect, err := store.GetDetection(ctx, onionID) - if err == nil { - err = store.delete(ctx, detect, "detection", store.prepareForSave(ctx, &detect.Auditable)) + if err != nil { + return nil, err } - return err + err = store.delete(ctx, detect, "detection", store.prepareForSave(ctx, &detect.Auditable)) + + return detect, err } diff --git a/server/modules/elasticcases/elasticcasestore.go b/server/modules/elasticcases/elasticcasestore.go index 9d8357e8c..f41d09181 100644 --- a/server/modules/elasticcases/elasticcasestore.go +++ b/server/modules/elasticcases/elasticcasestore.go @@ -157,6 +157,10 @@ func (store *ElasticCasestore) UpdateDetection(ctx context.Context, detect *mode return nil, errors.New("Unsupported operation by this module") } -func (store *ElasticCasestore) DeleteDetection(ctx context.Context, detectID string) error { - return errors.New("Unsupported operation by this module") +func (store *ElasticCasestore) UpdateDetectionField(context.Context, string, string, any) (*model.Detection, bool, error) { + return nil, false, errors.New("Unsupported operation by this module") +} + +func (store *ElasticCasestore) DeleteDetection(ctx context.Context, detectID string) (*model.Detection, error) { + return nil, errors.New("Unsupported operation by this module") } diff --git a/server/modules/generichttp/httpcasestore.go b/server/modules/generichttp/httpcasestore.go index f74c58a93..1fac51103 100644 --- a/server/modules/generichttp/httpcasestore.go +++ b/server/modules/generichttp/httpcasestore.go @@ -162,6 +162,10 @@ func (store *HttpCasestore) UpdateDetection(ctx context.Context, detect *model.D return nil, errors.New("Unsupported operation by this module") } -func (store *HttpCasestore) DeleteDetection(ctx context.Context, detectID string) error { - return errors.New("Unsupported operation by this module") +func (store *HttpCasestore) UpdateDetectionField(context.Context, string, string, any) (*model.Detection, bool, error) { + return nil, false, errors.New("Unsupported operation by this module") +} + +func (store *HttpCasestore) DeleteDetection(ctx context.Context, detectID string) (*model.Detection, error) { + return nil, errors.New("Unsupported operation by this module") } diff --git a/server/modules/thehive/thehivecasestore.go b/server/modules/thehive/thehivecasestore.go index 04740aa10..99ad5041f 100644 --- a/server/modules/thehive/thehivecasestore.go +++ b/server/modules/thehive/thehivecasestore.go @@ -154,6 +154,10 @@ func (store *TheHiveCasestore) UpdateDetection(ctx context.Context, detect *mode return nil, errors.New("Unsupported operation by this module") } -func (store *TheHiveCasestore) DeleteDetection(ctx context.Context, detectID string) error { - return errors.New("Unsupported operation by this module") +func (store *TheHiveCasestore) UpdateDetectionField(context.Context, string, string, any) (*model.Detection, bool, error) { + return nil, false, errors.New("Unsupported operation by this module") +} + +func (store *TheHiveCasestore) DeleteDetection(ctx context.Context, detectID string) (*model.Detection, error) { + return nil, errors.New("Unsupported operation by this module") } diff --git a/server/server.go b/server/server.go index 1894bb1c5..5080baf4e 100644 --- a/server/server.go +++ b/server/server.go @@ -736,15 +736,3 @@ func (server *Server) GetTimezones() []string { } return zones } - -func Ptr[T any](x T) *T { - return &x -} - -func Copy[T any](x *T) *T { - if x == nil { - return nil - } - - return Ptr(*x) -} diff --git a/util/ptr.go b/util/ptr.go new file mode 100644 index 000000000..fe1477027 --- /dev/null +++ b/util/ptr.go @@ -0,0 +1,13 @@ +package util + +func Ptr[T any](x T) *T { + return &x +} + +func Copy[T any](x *T) *T { + if x == nil { + return nil + } + + return Ptr(*x) +} From 4541e9b481a24382e26f20b1d3928b8a06d41344 Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Fri, 8 Sep 2023 15:24:33 -0600 Subject: [PATCH 003/102] WIP: Suricata Rule Parsing Enabling/Disabling rules that use flowbits is different than rules that don't. Parsing needs testing, currently failing because the rule I'm testing with uses angled quotes instead of straight quotes. Can now delete a Detection from the UI. UI now performs some light validation before saving a Detection. --- html/index.html | 13 +- html/js/i18n.js | 4 + html/js/routes/detection.js | 46 ++++- model/detection.go | 20 -- server/detectionhandler.go | 203 +++++++++++++++++++- server/modules/elastic/elasticeventstore.go | 8 +- syntax/suricata.go | 156 +++++++++++++++ 7 files changed, 411 insertions(+), 39 deletions(-) create mode 100644 syntax/suricata.go diff --git a/html/index.html b/html/index.html index cf1329eb8..0b57feab9 100644 --- a/html/index.html +++ b/html/index.html @@ -1063,6 +1063,9 @@

{{i18n.duplicate}} + + {{i18n.delete}} + @@ -1071,7 +1074,7 @@

@@ -1079,7 +1082,7 @@

- +
-
+
{{ i18n.cancel }} @@ -1111,7 +1114,7 @@

-
+
{{ i18n.cancel }} diff --git a/html/js/i18n.js b/html/js/i18n.js index 0b9b05ee8..86c1a9348 100644 --- a/html/js/i18n.js +++ b/html/js/i18n.js @@ -353,6 +353,7 @@ const i18n = { firstName: 'First Name', flags: 'Flags', gigabytes: 'GB', + generate: 'Generate', graphs: 'Graphs', grid: 'Grid', gridEps: 'Grid EPS:', @@ -687,6 +688,9 @@ const i18n = { showBarChart: 'Show bar chart', showSankeyChart: 'Show Sankey diagram', showTable: 'Show table', + sidMissingErr: "This suricata rule is missing it's SID.", + sidMultipleErr: 'Suricata rules can only specify one SID.', + sidMismatchErr: "The SID in this suricata rule must match the detection's Public ID.", signature: 'Signature', socUrl: 'SOC Url', socExcludeToggle: 'Exclude SOC logs', diff --git a/html/js/routes/detection.js b/html/js/routes/detection.js index 5fd098061..cb29827cc 100644 --- a/html/js/routes/detection.js +++ b/html/js/routes/detection.js @@ -157,8 +157,22 @@ routes.push({ path: '/detection/:id', name: 'detection', component: { saveDetection(createNew) { if (this.curEditTarget !== null) this.stopEdit(true); - this.$refs['detection'].validate(); - if (!this.editForm.valid) return; + if (this.isNew()) { + this.$refs['detection'].validate(); + if (!this.editForm.valid) return; + } + + switch (this.detect.engine) { + case 'yara': + this.validateYara(); + break; + case 'sigma': + this.validateSigma(); + break; + case 'suricata': + this.validateSuricata(); + break; + } if (createNew) { this.$root.papi.post('/detection', this.detect); @@ -173,6 +187,34 @@ routes.push({ path: '/detection/:id', name: 'detection', component: { const response = await this.$root.papi.post('/detection/' + encodeURIComponent(this.$route.params.id) + '/duplicate'); this.$router.push({name: 'detection', params: {id: response.data.id}}); }, + async deleteDetection() { + await this.$root.papi.delete('/detection/' + encodeURIComponent(this.$route.params.id)); + this.$router.push({ name: 'detections' }); + }, + validateYara() { }, + validateSigma() {}, + validateSuricata() { + const sidExtract = /\bsid: ?['"]?(.*?)['"]?;/ + const results = sidExtract.exec(this.detect.content); + + if (!results || results.length < 2) { + // sid not present in rule + this.$root.showError(this.i18n.sidMissingErr); + return; + } else if (results && results.length > 2) { + // multiple sids present in rule + this.$root.showError(this.i18n.sidMultipleErr); + return; + } + + const sid = results[1]; + + if (this.detect.publicId !== sid) { + // sid doesn't match metadata + this.$root.showError(this.i18n.sidMismatchErr); + return; + } + }, print(x) { console.log(x); }, diff --git a/model/detection.go b/model/detection.go index 4d3f06a1c..8d3f8c912 100644 --- a/model/detection.go +++ b/model/detection.go @@ -91,23 +91,3 @@ func (detect *Detection) Validate() error { return nil } - -func SyncDetections(detections []*Detection) error { - byEngine := map[EngineName][]*Detection{} - for _, detect := range detections { - byEngine[detect.Engine] = append(byEngine[detect.Engine], detect) - } - - if len(byEngine[EngineNameSuricata]) > 0 { - err := syncSuricata(byEngine[EngineNameSuricata]) - if err != nil { - return err - } - } - - return nil -} - -func syncSuricata(detections []*Detection) error { - return nil -} diff --git a/server/detectionhandler.go b/server/detectionhandler.go index 56806f1bd..fef9141e7 100644 --- a/server/detectionhandler.go +++ b/server/detectionhandler.go @@ -7,16 +7,23 @@ package server import ( + "context" "fmt" "net/http" + "regexp" "strings" + "github.com/samber/lo" "github.com/security-onion-solutions/securityonion-soc/model" + "github.com/security-onion-solutions/securityonion-soc/syntax" + "github.com/security-onion-solutions/securityonion-soc/util" "github.com/security-onion-solutions/securityonion-soc/web" "github.com/go-chi/chi" ) +var sidExtracter = regexp.MustCompile(`\bsid: ?['"]?(.*?)['"]?;`) + type DetectionHandler struct { server *Server } @@ -76,12 +83,17 @@ func (h *DetectionHandler) postDetection(w http.ResponseWriter, r *http.Request) return } - err = model.SyncDetections([]*model.Detection{detect}) + errMap, err := SyncDetections(ctx, h.server.Configstore, []*model.Detection{detect}) if err != nil { web.Respond(w, r, http.StatusInternalServerError, err) return } + if len(errMap) != 0 { + web.Respond(w, r, http.StatusInternalServerError, errMap) + return + } + web.Respond(w, r, http.StatusOK, detect) } @@ -131,12 +143,17 @@ func (h *DetectionHandler) putDetection(w http.ResponseWriter, r *http.Request) return } - err = model.SyncDetections([]*model.Detection{detect}) + errMap, err := SyncDetections(ctx, h.server.Configstore, []*model.Detection{detect}) if err != nil { web.Respond(w, r, http.StatusInternalServerError, err) return } + if len(errMap) != 0 { + web.Respond(w, r, http.StatusInternalServerError, errMap) + return + } + web.Respond(w, r, http.StatusOK, detect) } @@ -151,13 +168,13 @@ func (h *DetectionHandler) deleteDetection(w http.ResponseWriter, r *http.Reques return } - err = model.SyncDetections([]*model.Detection{old}) + errMap, err := SyncDetections(ctx, h.server.Configstore, []*model.Detection{old}) if err != nil { web.Respond(w, r, http.StatusInternalServerError, err) return } - web.Respond(w, r, http.StatusOK, nil) + web.Respond(w, r, http.StatusOK, errMap) } func (h *DetectionHandler) bulkUpdateDetection(w http.ResponseWriter, r *http.Request) { @@ -202,11 +219,181 @@ func (h *DetectionHandler) bulkUpdateDetection(w http.ResponseWriter, r *http.Re } } - err = model.SyncDetections(modified) - if err != nil { - web.Respond(w, r, http.StatusInternalServerError, err) - return + if len(modified) != 0 { + addErrMap, err := SyncDetections(ctx, h.server.Configstore, modified) + if err != nil { + web.Respond(w, r, http.StatusInternalServerError, err) + return + } + + // merge error maps + for k, v := range addErrMap { + origK, hasK := errMap[k] + if hasK { + errMap[k] = fmt.Sprintf("%s; %s", origK, v) + } else { + errMap[k] = v + } + } } web.Respond(w, r, http.StatusOK, errMap) } + +func SyncDetections(ctx context.Context, cfgStore Configstore, detections []*model.Detection) (errMap map[string]string, err error) { + defer func() { + if len(errMap) == 0 { + errMap = nil + } + }() + + byEngine := map[model.EngineName][]*model.Detection{} + for _, detect := range detections { + byEngine[detect.Engine] = append(byEngine[detect.Engine], detect) + } + + if len(byEngine[model.EngineNameSuricata]) > 0 { + errMap, err = syncSuricata(ctx, cfgStore, byEngine[model.EngineNameSuricata]) + if err != nil { + return errMap, err + } + } + + return errMap, nil +} + +func syncSuricata(ctx context.Context, cfgStore Configstore, detections []*model.Detection) (map[string]string, error) { + allSettings, err := cfgStore.GetSettings(ctx) + if err != nil { + return nil, err + } + + local := settingByID(allSettings, "idstools.rules.local__rules") + if local == nil { + return nil, fmt.Errorf("unable to find local rules setting") + } + + enabled := settingByID(allSettings, "idstools.sids.enabled") + if enabled == nil { + return nil, fmt.Errorf("unable to find enabled setting") + } + + localLines := strings.Split(local.Value, "\n") + enabledLines := strings.Split(enabled.Value, "\n") + + localIndex := indexRules(localLines) + enabledIndex := indexSIDs(enabledLines) + + errMap := map[string]string{} // map[sid]error + + for _, detect := range detections { + parsedRule, err := syntax.ParseSuricataRule(detect.Content) + if err != nil { + errMap[detect.PublicID] = fmt.Sprintf("unable to parse rule; reason=%s", err.Error()) + continue + } + + opt, ok := parsedRule.GetOption("sid") + if !ok || opt == nil { + errMap[detect.PublicID] = fmt.Sprintf("rule does not contain a SID; rule=%s", detect.Content) + continue + } + + sid := *opt + _, isFlowbits := parsedRule.GetOption("flowbits") + + _ = isFlowbits + + lineNum, inLocal := localIndex[sid] + if !inLocal { + localLines = append(localLines, detect.Content) + lineNum = len(localLines) - 1 + localIndex[sid] = lineNum + } else { + localLines[lineNum] = detect.Content + } + + lineNum, inEnabled := enabledIndex[sid] + if !inEnabled { + line := detect.PublicID + if !detect.IsEnabled { + line = "# " + line + } + + enabledLines = append(enabledLines, line) + lineNum = len(enabledLines) - 1 + enabledIndex[sid] = lineNum + } else { + line := detect.PublicID + if !detect.IsEnabled { + line = "# " + line + } + + enabledLines[lineNum] = line + } + } + + local.Value = strings.Join(localLines, "\n") + enabled.Value = strings.Join(enabledLines, "\n") + + err = cfgStore.UpdateSetting(ctx, local, false) + if err != nil { + return errMap, err + } + + err = cfgStore.UpdateSetting(ctx, enabled, false) + if err != nil { + return errMap, err + } + + return errMap, nil +} + +func settingByID(all []*model.Setting, id string) *model.Setting { + found, ok := lo.Find(all, func(s *model.Setting) bool { + return s.Id == id + }) + if !ok { + return nil + } + + return found +} + +func extractSID(rule string) *string { + sids := sidExtracter.FindStringSubmatch(rule) + if len(sids) != 2 { // 0: Full Match, 1: Capture Group + return nil + } + + return util.Ptr(strings.TrimSpace(sids[1])) +} + +func indexRules(lines []string) map[string]int { + index := map[string]int{} + + for i, line := range lines { + sid := extractSID(line) + if sid == nil { + continue + } + + index[*sid] = i + } + + return index +} + +func indexSIDs(lines []string) map[string]int { + index := map[string]int{} + + for i, line := range lines { + line = strings.TrimSpace(strings.TrimLeft(line, "# \t")) + + if line != "" { + index[line] = i + } + } + + return index +} diff --git a/server/modules/elastic/elasticeventstore.go b/server/modules/elastic/elasticeventstore.go index 93726aa59..017826323 100644 --- a/server/modules/elastic/elasticeventstore.go +++ b/server/modules/elastic/elasticeventstore.go @@ -577,8 +577,8 @@ func (store *ElasticEventstore) PopulateJobFromDocQuery(ctx context.Context, idF query := fmt.Sprintf(` { - "query" : { - "bool": { + "query" : { + "bool": { "must": [ { "match" : { "%s" : "%s" }}%s ] @@ -621,8 +621,8 @@ func (store *ElasticEventstore) PopulateJobFromDocQuery(ctx context.Context, idF } query := fmt.Sprintf(` { - "query" : { - "bool": { + "query" : { + "bool": { "must": [ { "match" : { "log.id.uid" : "%s" }}%s ] diff --git a/syntax/suricata.go b/syntax/suricata.go new file mode 100644 index 000000000..90ca8af83 --- /dev/null +++ b/syntax/suricata.go @@ -0,0 +1,156 @@ +package syntax + +import ( + "fmt" + "strings" + + "github.com/security-onion-solutions/securityonion-soc/util" +) + +type SuricataRule struct { + Action string + Protocol string + Source string + Direction string + Destination string + Options []*RuleOption +} + +type RuleOption struct { + Name string + Value *string +} + +type state int + +const ( + stateAction state = iota + stateProtocol + stateSource + stateDirection + stateDestination + stateOptions +) + +func ParseSuricataRule(rule string) (*SuricataRule, error) { + r := strings.NewReader(rule) + curState := stateAction + buf := strings.Builder{} + inQuotes := false + isEscaping := false + + out := &SuricataRule{ + Options: []*RuleOption{}, + } + + for r.Len() != 0 { + // TODO: Parse Source and Destination into Address and Ports + ch, _, _ := r.ReadRune() + + switch curState { + case stateAction: + if ch == ' ' { + out.Action = strings.TrimSpace(buf.String()) + buf.Reset() + curState = stateProtocol + } else { + buf.WriteRune(ch) + } + case stateProtocol: + if ch == ' ' { + out.Protocol = strings.TrimSpace(buf.String()) + buf.Reset() + curState = stateSource + } else { + buf.WriteRune(ch) + } + case stateSource: + if ch == '<' || ch == '-' { + out.Source = strings.TrimSpace(buf.String()) + buf.Reset() + buf.WriteRune(ch) + curState = stateDirection + } else { + buf.WriteRune(ch) + } + case stateDirection: + if ch == ' ' { + out.Direction = strings.TrimSpace(buf.String()) + if out.Direction != "<>" && out.Direction != "->" { + return nil, fmt.Errorf("invalid direction, must be '<>' or '->', got %s", out.Direction) + } + + buf.Reset() + curState = stateDestination + } else { + buf.WriteRune(ch) + } + case stateDestination: + if ch == '(' { + out.Destination = strings.TrimSpace(buf.String()) + buf.Reset() + curState = stateOptions + } else { + buf.WriteRune(ch) + } + case stateOptions: + if ch == ')' && !inQuotes && !isEscaping { + if r.Len() != 0 { + // end of options, but not end of rule? + return nil, fmt.Errorf("invalid rule, expected end of rule, got %d more bytes", r.Len()) + } + } else if ch == ';' && !inQuotes && !isEscaping { + option := strings.TrimSpace(buf.String()) + buf.Reset() + + split := strings.SplitN(option, ":", 2) + opt := &RuleOption{ + Name: strings.TrimSpace(split[0]), + } + + if len(split) == 2 { + opt.Value = util.Ptr(strings.TrimSpace(split[1])) + } + + out.Options = append(out.Options, opt) + } else if ch == '"' { // TODO: current test rule has angled quotes, needs fixing + buf.WriteRune(ch) + if isEscaping { + isEscaping = false + } else { + inQuotes = !inQuotes + } + } else if ch == '\\' { + isEscaping = true + buf.WriteRune(ch) + } else { + buf.WriteRune(ch) + } + } + } + + return out, nil +} + +func (rule *SuricataRule) GetOption(key string) (value *string, ok bool) { + for _, opt := range rule.Options { + if opt.Name == key { + return opt.Value, true + } + } + + return nil, false +} + +func (rule *SuricataRule) String() string { + opts := make([]string, 0, len(rule.Options)) + for _, opt := range rule.Options { + if opt.Value == nil { + opts = append(opts, fmt.Sprintf("%s;", opt.Name)) + } else { + opts = append(opts, fmt.Sprintf("%s:%s;", opt.Name, *opt.Value)) + } + } + + return fmt.Sprintf("%s %s %s %s %s (%s)", rule.Action, rule.Protocol, rule.Source, rule.Direction, rule.Destination, strings.Join(opts, " ")) +} From f4d1eaf3ed7e8f200d97bb85c3a3063451d53f44 Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Tue, 19 Sep 2023 18:27:51 -0600 Subject: [PATCH 004/102] WIP: alerts --- html/js/routes/detection.js | 35 +++++++++++++++------- server/detectionhandler.go | 60 +++++++++++++++++++++++++++++++------ 2 files changed, 75 insertions(+), 20 deletions(-) diff --git a/html/js/routes/detection.js b/html/js/routes/detection.js index cb29827cc..a1d346eb9 100644 --- a/html/js/routes/detection.js +++ b/html/js/routes/detection.js @@ -162,18 +162,24 @@ routes.push({ path: '/detection/:id', name: 'detection', component: { if (!this.editForm.valid) return; } + let err; switch (this.detect.engine) { case 'yara': - this.validateYara(); + err = this.validateYara(); break; case 'sigma': - this.validateSigma(); + err = this.validateSigma(); break; case 'suricata': - this.validateSuricata(); + err = this.validateSuricata(); break; } + if (err) { + this.$root.showError(err); + return; + } + if (createNew) { this.$root.papi.post('/detection', this.detect); } else { @@ -191,29 +197,36 @@ routes.push({ path: '/detection/:id', name: 'detection', component: { await this.$root.papi.delete('/detection/' + encodeURIComponent(this.$route.params.id)); this.$router.push({ name: 'detections' }); }, - validateYara() { }, - validateSigma() {}, + validateYara() { + return null; + }, + validateSigma() { + return null; + }, validateSuricata() { const sidExtract = /\bsid: ?['"]?(.*?)['"]?;/ const results = sidExtract.exec(this.detect.content); if (!results || results.length < 2) { // sid not present in rule - this.$root.showError(this.i18n.sidMissingErr); - return; + return this.i18n.sidMissingErr; } else if (results && results.length > 2) { // multiple sids present in rule - this.$root.showError(this.i18n.sidMultipleErr); - return; + return this.i18n.sidMultipleErr; } const sid = results[1]; if (this.detect.publicId !== sid) { // sid doesn't match metadata - this.$root.showError(this.i18n.sidMismatchErr); - return; + return this.i18n.sidMismatchErr; } + + // normalize quotes + this.detect.content = this.detect.content.replaceAll('”', '"'); + this.detect.content = this.detect.content.replaceAll('“', '"'); + + return null; }, print(x) { console.log(x); diff --git a/server/detectionhandler.go b/server/detectionhandler.go index fef9141e7..bc8c6b696 100644 --- a/server/detectionhandler.go +++ b/server/detectionhandler.go @@ -24,6 +24,8 @@ import ( var sidExtracter = regexp.MustCompile(`\bsid: ?['"]?(.*?)['"]?;`) +const suricataModifyFromTo = `"flowbits" "noalert; flowbits"` + type DetectionHandler struct { server *Server } @@ -278,11 +280,18 @@ func syncSuricata(ctx context.Context, cfgStore Configstore, detections []*model return nil, fmt.Errorf("unable to find enabled setting") } + modify := settingByID(allSettings, "idstools.sids.modify") + if modify == nil { + return nil, fmt.Errorf("unable to find modify setting") + } + localLines := strings.Split(local.Value, "\n") enabledLines := strings.Split(enabled.Value, "\n") + modifyLines := strings.Split(modify.Value, "\n") - localIndex := indexRules(localLines) - enabledIndex := indexSIDs(enabledLines) + localIndex := indexLocal(localLines) + enabledIndex := indexEnabled(enabledLines) + modifyIndex := indexModified(modifyLines) errMap := map[string]string{} // map[sid]error @@ -302,8 +311,6 @@ func syncSuricata(ctx context.Context, cfgStore Configstore, detections []*model sid := *opt _, isFlowbits := parsedRule.GetOption("flowbits") - _ = isFlowbits - lineNum, inLocal := localIndex[sid] if !inLocal { localLines = append(localLines, detect.Content) @@ -316,7 +323,7 @@ func syncSuricata(ctx context.Context, cfgStore Configstore, detections []*model lineNum, inEnabled := enabledIndex[sid] if !inEnabled { line := detect.PublicID - if !detect.IsEnabled { + if !detect.IsEnabled && !isFlowbits { line = "# " + line } @@ -325,16 +332,32 @@ func syncSuricata(ctx context.Context, cfgStore Configstore, detections []*model enabledIndex[sid] = lineNum } else { line := detect.PublicID - if !detect.IsEnabled { + if !detect.IsEnabled && !isFlowbits { line = "# " + line } enabledLines[lineNum] = line } + + if isFlowbits { + lineNum, inModify := modifyIndex[sid] + if !inModify && !detect.IsEnabled { + // not in the modify file, but should be + line := fmt.Sprintf("%s %s", detect.PublicID, suricataModifyFromTo) + modifyLines = append(modifyLines, line) + lineNum = len(modifyLines) - 1 + modifyIndex[sid] = lineNum + } else if inModify && detect.IsEnabled { + // in modify, but shouldn't be + modifyLines = append(modifyLines[:lineNum], modifyLines[lineNum+1:]...) + delete(modifyIndex, sid) + } + } } local.Value = strings.Join(localLines, "\n") enabled.Value = strings.Join(enabledLines, "\n") + modify.Value = strings.Join(modifyLines, "\n") err = cfgStore.UpdateSetting(ctx, local, false) if err != nil { @@ -346,6 +369,11 @@ func syncSuricata(ctx context.Context, cfgStore Configstore, detections []*model return errMap, err } + err = cfgStore.UpdateSetting(ctx, modify, false) + if err != nil { + return errMap, err + } + return errMap, nil } @@ -369,7 +397,7 @@ func extractSID(rule string) *string { return util.Ptr(strings.TrimSpace(sids[1])) } -func indexRules(lines []string) map[string]int { +func indexLocal(lines []string) map[string]int { index := map[string]int{} for i, line := range lines { @@ -384,12 +412,11 @@ func indexRules(lines []string) map[string]int { return index } -func indexSIDs(lines []string) map[string]int { +func indexEnabled(lines []string) map[string]int { index := map[string]int{} for i, line := range lines { line = strings.TrimSpace(strings.TrimLeft(line, "# \t")) - if line != "" { index[line] = i } @@ -397,3 +424,18 @@ func indexSIDs(lines []string) map[string]int { return index } + +func indexModified(lines []string) map[string]int { + index := map[string]int{} + + for i, line := range lines { + line = strings.TrimSpace(strings.TrimLeft(line, " \t")) + parts := strings.SplitN(line, " ", 2) + + if strings.Contains(line, suricataModifyFromTo) { + index[parts[0]] = i + } + } + + return index +} From 13c03c4d63908c6aca43bb7cd16deefae570808a Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Thu, 21 Sep 2023 16:33:01 -0600 Subject: [PATCH 005/102] WIP: Various improvements Properly adds SIDs to disabled file. MinLength now set in UI to match server requirement of 5. After creating a detection, you're taken to the edit page for that detection instead of being left at a filled in create page. If creating fails, a banner is shown. Case insensitive sidExtractor. The extractSID function now returns nil if more than 1 SID is specified. Tests for pure functions. --- html/index.html | 6 +- html/js/routes/detection.js | 41 ++++---- server/detectionhandler.go | 52 ++++++++-- server/detectionhandler_test.go | 164 ++++++++++++++++++++++++++++++++ 4 files changed, 235 insertions(+), 28 deletions(-) create mode 100644 server/detectionhandler_test.go diff --git a/html/index.html b/html/index.html index 0b57feab9..e2d9c7902 100644 --- a/html/index.html +++ b/html/index.html @@ -970,15 +970,15 @@

-
+
- + @@ -1072,7 +1072,7 @@

- + diff --git a/html/js/routes/detection.js b/html/js/routes/detection.js index a1d346eb9..e88abc266 100644 --- a/html/js/routes/detection.js +++ b/html/js/routes/detection.js @@ -17,14 +17,15 @@ routes.push({ path: '/detection/:id', name: 'detection', component: { editField: null, editForm: { valid: true }, rules: { - required: value => (value && value.length > 0) || this.$root.i18n.required, - number: value => (! isNaN(+value) && Number.isInteger(parseFloat(value))) || this.$root.i18n.required, - hours: value => (!value || /^\d{1,4}(\.\d{1,4})?$/.test(value)) || this.$root.i18n.invalidHours, - shortLengthLimit: value => (value.length < 100) || this.$root.i18n.required, - longLengthLimit: value => (encodeURI(value).split(/%..|./).length - 1 < 10000000) || this.$root.i18n.required, - fileSizeLimit: value => (value == null || value.size < this.maxUploadSizeBytes) || this.$root.i18n.fileTooLarge.replace("{maxUploadSizeBytes}", this.$root.formatCount(this.maxUploadSizeBytes)), - fileNotEmpty: value => (value == null || value.size > 0) || this.$root.i18n.fileEmpty, - fileRequired: value => (value != null) || this.$root.i18n.required, + required: value => (value && value.length > 0) || this.$root.i18n.required, + number: value => (! isNaN(+value) && Number.isInteger(parseFloat(value))) || this.$root.i18n.required, + hours: value => (!value || /^\d{1,4}(\.\d{1,4})?$/.test(value)) || this.$root.i18n.invalidHours, + minLength: limit => value => (value && value.length >= limit) || this.$root.i18n.ruleMinLen, + shortLengthLimit: value => (value.length < 100) || this.$root.i18n.required, + longLengthLimit: value => (encodeURI(value).split(/%..|./).length - 1 < 10000000) || this.$root.i18n.required, + fileSizeLimit: value => (value == null || value.size < this.maxUploadSizeBytes) || this.$root.i18n.fileTooLarge.replace("{maxUploadSizeBytes}", this.$root.formatCount(this.maxUploadSizeBytes)), + fileNotEmpty: value => (value == null || value.size > 0) || this.$root.i18n.fileEmpty, + fileRequired: value => (value != null) || this.$root.i18n.required, }, panel: [0, 1, 2], activeTab: '', @@ -154,7 +155,7 @@ routes.push({ path: '/detection/:id', name: 'detection', component: { this.origValue = null; this.editField = null; }, - saveDetection(createNew) { + async saveDetection(createNew) { if (this.curEditTarget !== null) this.stopEdit(true); if (this.isNew()) { @@ -180,14 +181,22 @@ routes.push({ path: '/detection/:id', name: 'detection', component: { return; } - if (createNew) { - this.$root.papi.post('/detection', this.detect); - } else { - this.$root.papi.put('/detection', this.detect); - } - this.origDetect = Object.assign({}, this.detect); + try { + let response; + if (createNew) { + response = await this.$root.papi.post('/detection', this.detect); + } else { + response = await this.$root.papi.put('/detection', this.detect); + } + + this.origDetect = Object.assign({}, this.detect); - this.$root.showTip(this.i18n.saveSuccess); + this.$root.showTip(this.i18n.saveSuccess); + + this.$router.push({name: 'detection', params: {id: response.data.id}}); + } catch (error) { + this.$root.showError(error); + } }, async duplicateDetection() { const response = await this.$root.papi.post('/detection/' + encodeURIComponent(this.$route.params.id) + '/duplicate'); diff --git a/server/detectionhandler.go b/server/detectionhandler.go index bc8c6b696..dc17abda2 100644 --- a/server/detectionhandler.go +++ b/server/detectionhandler.go @@ -22,7 +22,7 @@ import ( "github.com/go-chi/chi" ) -var sidExtracter = regexp.MustCompile(`\bsid: ?['"]?(.*?)['"]?;`) +var sidExtracter = regexp.MustCompile(`(?i)\bsid: ?['"]?(.*?)['"]?;`) const suricataModifyFromTo = `"flowbits" "noalert; flowbits"` @@ -280,6 +280,11 @@ func syncSuricata(ctx context.Context, cfgStore Configstore, detections []*model return nil, fmt.Errorf("unable to find enabled setting") } + disabled := settingByID(allSettings, "idstools.sids.disabled") + if disabled == nil { + return nil, fmt.Errorf("unable to find disabled setting") + } + modify := settingByID(allSettings, "idstools.sids.modify") if modify == nil { return nil, fmt.Errorf("unable to find modify setting") @@ -287,11 +292,13 @@ func syncSuricata(ctx context.Context, cfgStore Configstore, detections []*model localLines := strings.Split(local.Value, "\n") enabledLines := strings.Split(enabled.Value, "\n") + disabledLines := strings.Split(disabled.Value, "\n") modifyLines := strings.Split(modify.Value, "\n") localIndex := indexLocal(localLines) enabledIndex := indexEnabled(enabledLines) - modifyIndex := indexModified(modifyLines) + disabledIndex := indexEnabled(disabledLines) + modifyIndex := indexModify(modifyLines) errMap := map[string]string{} // map[sid]error @@ -339,6 +346,27 @@ func syncSuricata(ctx context.Context, cfgStore Configstore, detections []*model enabledLines[lineNum] = line } + if !isFlowbits { + lineNum, inDisabled := disabledIndex[sid] + if !inDisabled { + line := detect.PublicID + if detect.IsEnabled { + line = "# " + line + } + + disabledLines = append(disabledLines, line) + lineNum = len(disabledLines) - 1 + disabledIndex[sid] = lineNum + } else { + line := detect.PublicID + if detect.IsEnabled { + line = "# " + line + } + + disabledLines[lineNum] = line + } + } + if isFlowbits { lineNum, inModify := modifyIndex[sid] if !inModify && !detect.IsEnabled { @@ -357,6 +385,7 @@ func syncSuricata(ctx context.Context, cfgStore Configstore, detections []*model local.Value = strings.Join(localLines, "\n") enabled.Value = strings.Join(enabledLines, "\n") + disabled.Value = strings.Join(disabledLines, "\n") modify.Value = strings.Join(modifyLines, "\n") err = cfgStore.UpdateSetting(ctx, local, false) @@ -369,6 +398,11 @@ func syncSuricata(ctx context.Context, cfgStore Configstore, detections []*model return errMap, err } + err = cfgStore.UpdateSetting(ctx, disabled, false) + if err != nil { + return errMap, err + } + err = cfgStore.UpdateSetting(ctx, modify, false) if err != nil { return errMap, err @@ -389,12 +423,12 @@ func settingByID(all []*model.Setting, id string) *model.Setting { } func extractSID(rule string) *string { - sids := sidExtracter.FindStringSubmatch(rule) - if len(sids) != 2 { // 0: Full Match, 1: Capture Group + sids := sidExtracter.FindAllStringSubmatch(rule, 2) + if len(sids) != 1 { // 1 match = 1 sid return nil } - return util.Ptr(strings.TrimSpace(sids[1])) + return util.Ptr(strings.TrimSpace(sids[0][1])) } func indexLocal(lines []string) map[string]int { @@ -425,14 +459,14 @@ func indexEnabled(lines []string) map[string]int { return index } -func indexModified(lines []string) map[string]int { +func indexModify(lines []string) map[string]int { index := map[string]int{} for i, line := range lines { - line = strings.TrimSpace(strings.TrimLeft(line, " \t")) - parts := strings.SplitN(line, " ", 2) + line = strings.TrimSpace(strings.TrimLeft(line, "# \t")) - if strings.Contains(line, suricataModifyFromTo) { + if strings.HasSuffix(line, suricataModifyFromTo) { + parts := strings.SplitN(line, " ", 2) index[parts[0]] = i } } diff --git a/server/detectionhandler_test.go b/server/detectionhandler_test.go new file mode 100644 index 000000000..f1be97606 --- /dev/null +++ b/server/detectionhandler_test.go @@ -0,0 +1,164 @@ +package server + +import ( + "testing" + + "github.com/security-onion-solutions/securityonion-soc/model" + "github.com/security-onion-solutions/securityonion-soc/util" + + "github.com/stretchr/testify/assert" +) + +func TestSettingByID(t *testing.T) { + allSettings := []*model.Setting{ + {Id: "1", Value: "one"}, + {Id: "2", Value: "two"}, + {Id: "3", Value: "three"}, + } + byId := map[string]*model.Setting{ + "1": allSettings[0], + "2": allSettings[1], + "3": allSettings[2], + } + + table := []struct { + Name string + SettingID string + ExpectedValue *string + }{ + {Name: "Get 1", SettingID: "1", ExpectedValue: util.Ptr("one")}, + {Name: "Get 2", SettingID: "2", ExpectedValue: util.Ptr("two")}, + {Name: "Get 3", SettingID: "3", ExpectedValue: util.Ptr("three")}, + {Name: "Get 4", SettingID: "4", ExpectedValue: nil}, + } + + for _, test := range table { + test := test + t.Run(test.Name, func(t *testing.T) { + t.Parallel() + + value := settingByID(allSettings, test.SettingID) + if test.ExpectedValue == nil { + assert.Nil(t, value) + } else { + assert.Equal(t, *test.ExpectedValue, value.Value) + assert.Same(t, value, byId[test.SettingID]) + } + }) + } +} + +func TestExtractSID(t *testing.T) { + table := []struct { + Name string + Input string + Output *string + }{ + {Name: "Simple SID", Input: "sid:10000;", Output: util.Ptr("10000")}, + {Name: "Empty SID", Input: "sid: ;", Output: util.Ptr("")}, + {Name: "Capital SID", Input: "SID:10000;", Output: util.Ptr("10000")}, + {Name: "UUID SID", Input: "sid: 82ca7105-9001-40b7-a8cc-4eaebaf17815;", Output: util.Ptr("82ca7105-9001-40b7-a8cc-4eaebaf17815")}, + {Name: "No SID", Input: "nid: 10000", Output: nil}, + {Name: "Single-Quoted SID", Input: "sid: '10000';", Output: util.Ptr("10000")}, + {Name: "Double-Quoted SID", Input: `sid:"10000";`, Output: util.Ptr("10000")}, + {Name: "Single-Quoted Empty SID", Input: "sid:'';", Output: util.Ptr("")}, + {Name: "Double-Quoted Empty SID", Input: `sid: "";`, Output: util.Ptr("")}, + {Name: "Multiple SIDs", Input: "sid: 10000; sid: 10001;", Output: nil}, + {Name: "Sample Rule", Input: `alert http any any -> any any (msg:"RULE B"; flowbits: isset, test; flow: established,to_client; content:"uid=0"; sid:60000;)`, Output: util.Ptr("60000")}, + } + + for _, test := range table { + test := test + t.Run(test.Name, func(t *testing.T) { + t.Parallel() + + output := extractSID(test.Input) + assert.Equal(t, test.Output, output) + }) + } +} + +func TestIndexLocal(t *testing.T) { + lines := []string{ + "sid: 10000;", + " ", + "# sid: 20000;", + "sid: 30000;", + "# 40000", // note: extractSID won't find anything here + "50000", + } + + output := indexLocal(lines) + assert.Equal(t, 3, len(output)) + assert.Contains(t, output, "10000") + assert.Equal(t, output["10000"], 0) + assert.Contains(t, output, "20000") + assert.Equal(t, output["20000"], 2) + assert.Contains(t, output, "30000") + assert.Equal(t, output["30000"], 3) + assert.NotContains(t, output, "40000") + assert.NotContains(t, output, "50000") +} + +func TestIndexEnabled(t *testing.T) { + lines := []string{ + "10000", + " ", + "# 20000 ", + "30000 ", + "# 40000", // note: extractSID won't find anything here + " 50000", + " not a number ", + "# 24adee9b-6010-46ed-9c4a-9cb7a9c972a1", + } + + output := indexEnabled(lines) + + assert.Equal(t, 7, len(output)) + assert.Contains(t, output, "10000") + assert.Equal(t, output["10000"], 0) + assert.Contains(t, output, "20000") + assert.Equal(t, output["20000"], 2) + assert.Contains(t, output, "30000") + assert.Equal(t, output["30000"], 3) + assert.Contains(t, output, "40000") + assert.Equal(t, output["40000"], 4) + assert.Contains(t, output, "50000") + assert.Equal(t, output["50000"], 5) + assert.Contains(t, output, "not a number") + assert.Equal(t, output["not a number"], 6) + assert.Contains(t, output, "24adee9b-6010-46ed-9c4a-9cb7a9c972a1") + assert.Equal(t, output["24adee9b-6010-46ed-9c4a-9cb7a9c972a1"], 7) +} + +func TestIndexModify(t *testing.T) { + lines := []string{ + `90000 this that`, + `10000 "flowbits" "noalert; flowbits"`, + `# 20000 "flowbits" "noalert; flowbits"`, + `30000 "flowbits" "noalert; flowbits" # we'll turn this on later`, + `# An unrelated comment`, + `a83ba97b-a8e8-4258-be1b-022aff230e6e "flowbits" "noalert; flowbits"`, + ` # 23220e49-7229-43a1-92d5-d68e46d27105 "flowbits" "noalert; flowbits"`, + `e4bd794a-8156-4fcc-b6a9-9fb2c9ecadc5 that this`, + } + + // Reminder: We only care about the lines that the API has added, + // if a line doesn't end with suricataModifyFromTo (`"flowbits" "noalert; flowbits"`) + // then it's a line we don't want to touch. + + output := indexModify(lines) + + assert.Equal(t, 4, len(output)) + assert.Contains(t, output, "10000") + assert.Equal(t, output["10000"], 1) + assert.Contains(t, output, "20000") + assert.Equal(t, output["20000"], 2) + assert.Contains(t, output, "a83ba97b-a8e8-4258-be1b-022aff230e6e") + assert.Equal(t, output["a83ba97b-a8e8-4258-be1b-022aff230e6e"], 5) + assert.Contains(t, output, "23220e49-7229-43a1-92d5-d68e46d27105") + assert.Equal(t, output["23220e49-7229-43a1-92d5-d68e46d27105"], 6) + assert.NotContains(t, output, "90000") + assert.NotContains(t, output, "30000") + assert.NotContains(t, output, "e4bd794a-8156-4fcc-b6a9-9fb2c9ecadc5") +} From 9ab882737941e4a48ddcd3b9652801cd66f51843 Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Fri, 22 Sep 2023 15:05:47 -0600 Subject: [PATCH 006/102] WIP: Tests Added MemConfigStore to facilitate testing. Test rule parsing, enabling/disabling of rules, and various helper functions. --- server/detectionhandler_test.go | 51 +++++++++++++ server/memconfigstore.go | 46 ++++++++++++ server/memconfigstore_test.go | 112 ++++++++++++++++++++++++++++ server/modules/salt/saltstore.go | 3 +- syntax/suricata_test.go | 124 +++++++++++++++++++++++++++++++ 5 files changed, 335 insertions(+), 1 deletion(-) create mode 100644 server/memconfigstore.go create mode 100644 server/memconfigstore_test.go create mode 100644 syntax/suricata_test.go diff --git a/server/detectionhandler_test.go b/server/detectionhandler_test.go index f1be97606..e1449f03f 100644 --- a/server/detectionhandler_test.go +++ b/server/detectionhandler_test.go @@ -1,6 +1,7 @@ package server import ( + "context" "testing" "github.com/security-onion-solutions/securityonion-soc/model" @@ -162,3 +163,53 @@ func TestIndexModify(t *testing.T) { assert.NotContains(t, output, "30000") assert.NotContains(t, output, "e4bd794a-8156-4fcc-b6a9-9fb2c9ecadc5") } + +func TestSyncSuricata(t *testing.T) { + emptySettings := []*model.Setting{ + {Id: "idstools.rules.local__rules"}, + {Id: "idstools.rules.enabled"}, + {Id: "idstools.rules.disabled"}, + {Id: "idstools.rules.modify"}, + } + table := []struct { + Name string + InitialSettings []*model.Setting + Detections []*model.Detection // Content (Valid Rule), PublicID, IsEnabled + ExpectedSettings map[string]string + ExpectedErr error + ExpectedErrMap map[string]string + }{ + { + Name: "Simple Add", + InitialSettings: emptySettings, + Detections: []*model.Detection{ + {}, + }, + }, + } + + ctx := context.Background() + + for _, test := range table { + test := test + t.Run(test.Name, func(t *testing.T) { + t.Parallel() + + mCfgStore := NewMemConfigStore(test.InitialSettings) + + errMap, err := syncSuricata(ctx, mCfgStore, test.Detections) + + assert.Equal(t, test.ExpectedErr, err) + assert.Equal(t, test.ExpectedErrMap, errMap) + + set, err := mCfgStore.GetSettings(ctx) + assert.NoError(t, err) + + for id, expectedValue := range test.ExpectedSettings { + setting := settingByID(set, id) + assert.NotNil(t, setting) + assert.Equal(t, expectedValue, setting.Value) + } + }) + } +} \ No newline at end of file diff --git a/server/memconfigstore.go b/server/memconfigstore.go new file mode 100644 index 000000000..c0dcddc0e --- /dev/null +++ b/server/memconfigstore.go @@ -0,0 +1,46 @@ +package server + +import ( + "context" + + "github.com/samber/lo" + "github.com/security-onion-solutions/securityonion-soc/model" +) + +type MemConfigStore struct { + settings []*model.Setting +} + +func NewMemConfigStore(settings []*model.Setting) *MemConfigStore { + return &MemConfigStore{ + settings: settings, + } +} + +func (m *MemConfigStore) GetSettings(ctx context.Context) ([]*model.Setting, error) { + return m.settings, nil +} + +func (m *MemConfigStore) UpdateSetting(ctx context.Context, setting *model.Setting, remove bool) error { + _, index, ok := lo.FindIndexOf(m.settings, func(s *model.Setting) bool { + return s.Id == setting.Id + }) + + if remove { + if ok { + m.settings = append(m.settings[:index], m.settings[index+1:]...) + } + } else { + if ok { + m.settings[index] = setting + } else { + m.settings = append(m.settings, setting) + } + } + + return nil +} + +func (m *MemConfigStore) SyncSettings(ctx context.Context) error { + return nil +} diff --git a/server/memconfigstore_test.go b/server/memconfigstore_test.go new file mode 100644 index 000000000..c1d550a85 --- /dev/null +++ b/server/memconfigstore_test.go @@ -0,0 +1,112 @@ +package server + +import ( + "context" + "testing" + + "github.com/samber/lo" + "github.com/security-onion-solutions/securityonion-soc/model" + "github.com/stretchr/testify/assert" +) + +func TestMemConfigStoreNew(t *testing.T) { + origSettings := []*model.Setting{ + {Id: "1", Value: "one"}, + {Id: "2", Value: "two"}, + {Id: "3", Value: "three"}, + } + mCfgStore := NewMemConfigStore(origSettings) + + assert.Implements(t, (*Configstore)(nil), mCfgStore) + + ctx := context.Background() + + set, err := mCfgStore.GetSettings(ctx) + assert.NoError(t, err) + assert.Same(t, &origSettings[0], &set[0]) +} + +func TestMemConfigStoreUpdate(t *testing.T) { + origSettings := []*model.Setting{ + {Id: "1", Value: "one"}, + {Id: "2", Value: "two"}, + {Id: "3", Value: "three"}, + } + + ctx := context.Background() + mCfgStore := NewMemConfigStore(origSettings) + + err := mCfgStore.UpdateSetting(ctx, &model.Setting{Id: "1", Value: "new"}, false) + assert.NoError(t, err) + + set, err := mCfgStore.GetSettings(ctx) + assert.NoError(t, err) + assert.Len(t, set, 3) + + s, ok := lo.Find(set, func(s *model.Setting) bool { + return s.Id == "1" + }) + assert.True(t, ok) + assert.Equal(t, "new", s.Value) +} + +func TestMemConfigStoreUpdateAdd(t *testing.T) { + origSettings := []*model.Setting{ + {Id: "1", Value: "one"}, + {Id: "2", Value: "two"}, + {Id: "3", Value: "three"}, + } + + ctx := context.Background() + mCfgStore := NewMemConfigStore(origSettings) + + err := mCfgStore.UpdateSetting(ctx, &model.Setting{Id: "4", Value: "four"}, false) + assert.NoError(t, err) + + set, err := mCfgStore.GetSettings(ctx) + assert.NoError(t, err) + assert.Len(t, set, 4) + + s, ok := lo.Find(set, func(s *model.Setting) bool { + return s.Id == "4" + }) + assert.True(t, ok) + assert.Equal(t, "four", s.Value) +} + +func TestMemConfigStoreUpdateRemove(t *testing.T) { + origSettings := []*model.Setting{ + {Id: "1", Value: "one"}, + {Id: "2", Value: "two"}, + {Id: "3", Value: "three"}, + } + + ctx := context.Background() + mCfgStore := NewMemConfigStore(origSettings) + + err := mCfgStore.UpdateSetting(ctx, &model.Setting{Id: "4"}, true) + assert.NoError(t, err) + + set, err := mCfgStore.GetSettings(ctx) + assert.NoError(t, err) + assert.Len(t, set, 3) + + err = mCfgStore.UpdateSetting(ctx, &model.Setting{Id: "2"}, true) + assert.NoError(t, err) + + set, err = mCfgStore.GetSettings(ctx) + assert.NoError(t, err) + assert.Len(t, set, 2) + + s, ok := lo.Find(set, func(s *model.Setting) bool { + return s.Id == "2" + }) + assert.False(t, ok) + assert.Nil(t, s) +} + +func TestMemConfigStoreSync(t *testing.T) { + mCfgStore := NewMemConfigStore(nil) + err := mCfgStore.SyncSettings(context.Background()) + assert.NoError(t, err) +} diff --git a/server/modules/salt/saltstore.go b/server/modules/salt/saltstore.go index b901ad5c8..75906edf5 100644 --- a/server/modules/salt/saltstore.go +++ b/server/modules/salt/saltstore.go @@ -17,13 +17,14 @@ import ( "strings" "time" - "github.com/apex/log" "github.com/security-onion-solutions/securityonion-soc/json" "github.com/security-onion-solutions/securityonion-soc/model" "github.com/security-onion-solutions/securityonion-soc/server" "github.com/security-onion-solutions/securityonion-soc/server/modules/salt/options" "github.com/security-onion-solutions/securityonion-soc/syntax" "github.com/security-onion-solutions/securityonion-soc/web" + + "github.com/apex/log" "gopkg.in/yaml.v3" ) diff --git a/syntax/suricata_test.go b/syntax/suricata_test.go new file mode 100644 index 000000000..f9b2d7f0f --- /dev/null +++ b/syntax/suricata_test.go @@ -0,0 +1,124 @@ +package syntax + +import ( + "testing" + + "github.com/security-onion-solutions/securityonion-soc/util" + "github.com/tj/assert" +) + +func TestParseSuricata(t *testing.T) { + table := []struct { + Name string + Input string + Output *SuricataRule + Error *string + }{ + { + Name: "Minimal Rule", + Input: `a b source port <> destination port ()`, + Output: &SuricataRule{ + Action: "a", + Protocol: "b", + Source: "source port", + Direction: "<>", + Destination: "destination port", + Options: []*RuleOption{}, + }, + }, + { + Name: "Bad Direction", + Input: `a b source port <- destination port ()`, + Error: util.Ptr("invalid direction, must be '<>' or '->', got <-"), + }, + { + Name: "Unnecessary Suffix", + Input: `a b source port <> destination port () x`, + Error: util.Ptr("invalid rule, expected end of rule, got 2 more bytes"), + }, + { + Name: "Escaped Option", + Input: `a b source port <> destination port (msg:"\\\"";)`, + Output: &SuricataRule{ + Action: "a", + Protocol: "b", + Source: "source port", + Direction: "<>", + Destination: "destination port", + Options: []*RuleOption{ + {Name: "msg", Value: util.Ptr(`"\\\""`)}, + }, + }, + }, + { + Name: "Real rule", + Input: `alert http any any -> any any (msg:"GPL ATTACK_RESPONSE id check returned root"; content:"uid=0|28|root|29|"; classtype:bad-unknown; sid:2100498; rev:7; metadata:created_at 2010_09_23, updated_at 2010_09_23;)`, + Output: &SuricataRule{ + Action: "alert", + Protocol: "http", + Source: "any any", + Direction: "->", + Destination: "any any", + Options: []*RuleOption{ + {Name: "msg", Value: util.Ptr(`"GPL ATTACK_RESPONSE id check returned root"`)}, + {Name: "content", Value: util.Ptr(`"uid=0|28|root|29|"`)}, + {Name: "classtype", Value: util.Ptr("bad-unknown")}, + {Name: "sid", Value: util.Ptr("2100498")}, + {Name: "rev", Value: util.Ptr("7")}, + {Name: "metadata", Value: util.Ptr("created_at 2010_09_23, updated_at 2010_09_23")}, + }, + }, + }, + } + + for _, test := range table { + test := test + t.Run(test.Name, func(t *testing.T) { + t.Parallel() + + rule, err := ParseSuricataRule(test.Input) + if test.Error != nil { + assert.Nil(t, rule) + assert.Equal(t, *test.Error, err.Error()) + } else { + assert.NoError(t, err) + assert.Equal(t, test.Output, rule) + } + }) + } +} + +func TestSuricataRule(t *testing.T) { + input := `a b source port <> destination port (msg:"\\\""; noalert; sid:12345; rev: "9"; )` + + rule, err := ParseSuricataRule(input) + assert.NoError(t, err) + + rule2, err := ParseSuricataRule(rule.String()) + assert.NoError(t, err) + + assert.Equal(t, rule, rule2) + + opt, ok := rule.GetOption("msg") + assert.True(t, ok) + assert.NotNil(t, opt) + assert.Equal(t, `"\\\""`, *opt) + + opt, ok = rule.GetOption("sid") + assert.True(t, ok) + assert.NotNil(t, opt) + assert.Equal(t, "12345", *opt) + + opt, ok = rule.GetOption("rev") + assert.True(t, ok) + assert.NotNil(t, opt) + assert.Equal(t, `"9"`, *opt) + + opt, ok = rule.GetOption("noalert") + assert.True(t, ok) + assert.Nil(t, opt) + + opt, ok = rule.GetOption("notfound") + assert.False(t, ok) + assert.Nil(t, opt) +} From 7baf618ccbf094fbd5ff57aacebc51151c65625c Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Mon, 25 Sep 2023 16:01:29 -0600 Subject: [PATCH 007/102] WIP: Modulaization and Elastic Index Change Moved Casestore functions relating to Detections to a brand new Detectionstore. TODO: TestSyncSuricata needs more tests, it currently only covers the bare minimum. Also, more testing needs to take place around what happens when the detections module is disabled. --- server/casestore.go | 6 - server/detectionengine.go | 12 + server/detectionhandler.go | 251 +----------- server/detectionstore.go | 21 + server/infohandler.go | 4 +- server/modules/elastic/elastic.go | 22 ++ server/modules/elastic/elasticcasestore.go | 163 -------- .../modules/elastic/elasticdetectionstore.go | 360 ++++++++++++++++++ server/modules/modules.go | 3 + server/modules/modules_test.go | 1 + server/modules/suricata/suricata.go | 269 +++++++++++++ .../suricata/suricata_test.go} | 126 +++++- .../modules/suricata/validate.go | 2 +- .../modules/suricata/validate_test.go | 2 +- server/server.go | 9 +- 15 files changed, 828 insertions(+), 423 deletions(-) create mode 100644 server/detectionengine.go create mode 100644 server/detectionstore.go create mode 100644 server/modules/elastic/elasticdetectionstore.go create mode 100644 server/modules/suricata/suricata.go rename server/{detectionhandler_test.go => modules/suricata/suricata_test.go} (52%) rename syntax/suricata.go => server/modules/suricata/validate.go (99%) rename syntax/suricata_test.go => server/modules/suricata/validate_test.go (99%) diff --git a/server/casestore.go b/server/casestore.go index 6d6eda5ac..be9d6536e 100644 --- a/server/casestore.go +++ b/server/casestore.go @@ -38,10 +38,4 @@ type Casestore interface { CreateArtifactStream(ctx context.Context, artifactstream *model.ArtifactStream) (string, error) GetArtifactStream(ctx context.Context, id string) (*model.ArtifactStream, error) DeleteArtifactStream(ctx context.Context, id string) error - - CreateDetection(ctx context.Context, detect *model.Detection) (*model.Detection, error) - GetDetection(ctx context.Context, detectId string) (*model.Detection, error) - UpdateDetection(ctx context.Context, detect *model.Detection) (*model.Detection, error) - UpdateDetectionField(ctx context.Context, id string, field string, value any) (*model.Detection, bool, error) - DeleteDetection(ctx context.Context, detectID string) (*model.Detection, error) } diff --git a/server/detectionengine.go b/server/detectionengine.go new file mode 100644 index 000000000..09d430e1b --- /dev/null +++ b/server/detectionengine.go @@ -0,0 +1,12 @@ +package server + +import ( + "context" + + "github.com/security-onion-solutions/securityonion-soc/model" +) + +type DetectionEngine interface { + ValidateRule(rule string) (string, error) + SyncDetections(ctx context.Context, detections []*model.Detection) (errMap map[string]string, err error) +} diff --git a/server/detectionhandler.go b/server/detectionhandler.go index dc17abda2..4462b3c8c 100644 --- a/server/detectionhandler.go +++ b/server/detectionhandler.go @@ -10,22 +10,14 @@ import ( "context" "fmt" "net/http" - "regexp" "strings" - "github.com/samber/lo" "github.com/security-onion-solutions/securityonion-soc/model" - "github.com/security-onion-solutions/securityonion-soc/syntax" - "github.com/security-onion-solutions/securityonion-soc/util" "github.com/security-onion-solutions/securityonion-soc/web" "github.com/go-chi/chi" ) -var sidExtracter = regexp.MustCompile(`(?i)\bsid: ?['"]?(.*?)['"]?;`) - -const suricataModifyFromTo = `"flowbits" "noalert; flowbits"` - type DetectionHandler struct { server *Server } @@ -54,7 +46,7 @@ func (h *DetectionHandler) getDetection(w http.ResponseWriter, r *http.Request) detectId := chi.URLParam(r, "id") - detect, err := h.server.Casestore.GetDetection(ctx, detectId) + detect, err := h.server.Detectionstore.GetDetection(ctx, detectId) if err != nil { if err.Error() == "Object not found" { web.Respond(w, r, http.StatusNotFound, nil) @@ -79,13 +71,13 @@ func (h *DetectionHandler) postDetection(w http.ResponseWriter, r *http.Request) return } - detect, err = h.server.Casestore.CreateDetection(ctx, detect) + detect, err = h.server.Detectionstore.CreateDetection(ctx, detect) if err != nil { web.Respond(w, r, http.StatusBadRequest, err) return } - errMap, err := SyncDetections(ctx, h.server.Configstore, []*model.Detection{detect}) + errMap, err := SyncDetections(ctx, h.server, []*model.Detection{detect}) if err != nil { web.Respond(w, r, http.StatusInternalServerError, err) return @@ -104,7 +96,7 @@ func (h *DetectionHandler) duplicateDetection(w http.ResponseWriter, r *http.Req detectId := chi.URLParam(r, "id") - detect, err := h.server.Casestore.GetDetection(ctx, detectId) + detect, err := h.server.Detectionstore.GetDetection(ctx, detectId) if err != nil { web.Respond(w, r, http.StatusInternalServerError, err) return @@ -119,7 +111,7 @@ func (h *DetectionHandler) duplicateDetection(w http.ResponseWriter, r *http.Req detect.IsReporting = false detect.IsCommunity = false - detect, err = h.server.Casestore.CreateDetection(ctx, detect) + detect, err = h.server.Detectionstore.CreateDetection(ctx, detect) if err != nil { web.Respond(w, r, http.StatusInternalServerError, err) return @@ -139,13 +131,13 @@ func (h *DetectionHandler) putDetection(w http.ResponseWriter, r *http.Request) return } - detect, err = h.server.Casestore.UpdateDetection(ctx, detect) + detect, err = h.server.Detectionstore.UpdateDetection(ctx, detect) if err != nil { web.Respond(w, r, http.StatusNotFound, err) return } - errMap, err := SyncDetections(ctx, h.server.Configstore, []*model.Detection{detect}) + errMap, err := SyncDetections(ctx, h.server, []*model.Detection{detect}) if err != nil { web.Respond(w, r, http.StatusInternalServerError, err) return @@ -164,13 +156,13 @@ func (h *DetectionHandler) deleteDetection(w http.ResponseWriter, r *http.Reques id := chi.URLParam(r, "id") - old, err := h.server.Casestore.DeleteDetection(ctx, id) + old, err := h.server.Detectionstore.DeleteDetection(ctx, id) if err != nil { web.Respond(w, r, http.StatusInternalServerError, err) return } - errMap, err := SyncDetections(ctx, h.server.Configstore, []*model.Detection{old}) + errMap, err := SyncDetections(ctx, h.server, []*model.Detection{old}) if err != nil { web.Respond(w, r, http.StatusInternalServerError, err) return @@ -210,7 +202,7 @@ func (h *DetectionHandler) bulkUpdateDetection(w http.ResponseWriter, r *http.Re modified := []*model.Detection{} for id := range IDs { - det, mod, err := h.server.Casestore.UpdateDetectionField(ctx, id, "IsEnabled", enabled) + det, mod, err := h.server.Detectionstore.UpdateDetectionField(ctx, id, "IsEnabled", enabled) if err != nil { errMap[id] = fmt.Sprintf("unable to update detection; reason=%s", err.Error()) continue @@ -222,7 +214,7 @@ func (h *DetectionHandler) bulkUpdateDetection(w http.ResponseWriter, r *http.Re } if len(modified) != 0 { - addErrMap, err := SyncDetections(ctx, h.server.Configstore, modified) + addErrMap, err := SyncDetections(ctx, h.server, modified) if err != nil { web.Respond(w, r, http.StatusInternalServerError, err) return @@ -242,7 +234,7 @@ func (h *DetectionHandler) bulkUpdateDetection(w http.ResponseWriter, r *http.Re web.Respond(w, r, http.StatusOK, errMap) } -func SyncDetections(ctx context.Context, cfgStore Configstore, detections []*model.Detection) (errMap map[string]string, err error) { +func SyncDetections(ctx context.Context, srv *Server, detections []*model.Detection) (errMap map[string]string, err error) { defer func() { if len(errMap) == 0 { errMap = nil @@ -254,222 +246,17 @@ func SyncDetections(ctx context.Context, cfgStore Configstore, detections []*mod byEngine[detect.Engine] = append(byEngine[detect.Engine], detect) } - if len(byEngine[model.EngineNameSuricata]) > 0 { - errMap, err = syncSuricata(ctx, cfgStore, byEngine[model.EngineNameSuricata]) - if err != nil { - return errMap, err - } - } - - return errMap, nil -} - -func syncSuricata(ctx context.Context, cfgStore Configstore, detections []*model.Detection) (map[string]string, error) { - allSettings, err := cfgStore.GetSettings(ctx) - if err != nil { - return nil, err - } - - local := settingByID(allSettings, "idstools.rules.local__rules") - if local == nil { - return nil, fmt.Errorf("unable to find local rules setting") - } - - enabled := settingByID(allSettings, "idstools.sids.enabled") - if enabled == nil { - return nil, fmt.Errorf("unable to find enabled setting") - } - - disabled := settingByID(allSettings, "idstools.sids.disabled") - if disabled == nil { - return nil, fmt.Errorf("unable to find disabled setting") - } - - modify := settingByID(allSettings, "idstools.sids.modify") - if modify == nil { - return nil, fmt.Errorf("unable to find modify setting") - } - - localLines := strings.Split(local.Value, "\n") - enabledLines := strings.Split(enabled.Value, "\n") - disabledLines := strings.Split(disabled.Value, "\n") - modifyLines := strings.Split(modify.Value, "\n") - - localIndex := indexLocal(localLines) - enabledIndex := indexEnabled(enabledLines) - disabledIndex := indexEnabled(disabledLines) - modifyIndex := indexModify(modifyLines) - - errMap := map[string]string{} // map[sid]error - - for _, detect := range detections { - parsedRule, err := syntax.ParseSuricataRule(detect.Content) - if err != nil { - errMap[detect.PublicID] = fmt.Sprintf("unable to parse rule; reason=%s", err.Error()) - continue - } - - opt, ok := parsedRule.GetOption("sid") - if !ok || opt == nil { - errMap[detect.PublicID] = fmt.Sprintf("rule does not contain a SID; rule=%s", detect.Content) - continue - } - - sid := *opt - _, isFlowbits := parsedRule.GetOption("flowbits") - - lineNum, inLocal := localIndex[sid] - if !inLocal { - localLines = append(localLines, detect.Content) - lineNum = len(localLines) - 1 - localIndex[sid] = lineNum - } else { - localLines[lineNum] = detect.Content - } - - lineNum, inEnabled := enabledIndex[sid] - if !inEnabled { - line := detect.PublicID - if !detect.IsEnabled && !isFlowbits { - line = "# " + line + for name, engine := range srv.DetectionEngines { + if len(byEngine[name]) != 0 { + eMap, err := engine.SyncDetections(ctx, byEngine[name]) + for sid, e := range eMap { + errMap[sid] = e } - - enabledLines = append(enabledLines, line) - lineNum = len(enabledLines) - 1 - enabledIndex[sid] = lineNum - } else { - line := detect.PublicID - if !detect.IsEnabled && !isFlowbits { - line = "# " + line + if err != nil { + return errMap, err } - - enabledLines[lineNum] = line } - - if !isFlowbits { - lineNum, inDisabled := disabledIndex[sid] - if !inDisabled { - line := detect.PublicID - if detect.IsEnabled { - line = "# " + line - } - - disabledLines = append(disabledLines, line) - lineNum = len(disabledLines) - 1 - disabledIndex[sid] = lineNum - } else { - line := detect.PublicID - if detect.IsEnabled { - line = "# " + line - } - - disabledLines[lineNum] = line - } - } - - if isFlowbits { - lineNum, inModify := modifyIndex[sid] - if !inModify && !detect.IsEnabled { - // not in the modify file, but should be - line := fmt.Sprintf("%s %s", detect.PublicID, suricataModifyFromTo) - modifyLines = append(modifyLines, line) - lineNum = len(modifyLines) - 1 - modifyIndex[sid] = lineNum - } else if inModify && detect.IsEnabled { - // in modify, but shouldn't be - modifyLines = append(modifyLines[:lineNum], modifyLines[lineNum+1:]...) - delete(modifyIndex, sid) - } - } - } - - local.Value = strings.Join(localLines, "\n") - enabled.Value = strings.Join(enabledLines, "\n") - disabled.Value = strings.Join(disabledLines, "\n") - modify.Value = strings.Join(modifyLines, "\n") - - err = cfgStore.UpdateSetting(ctx, local, false) - if err != nil { - return errMap, err - } - - err = cfgStore.UpdateSetting(ctx, enabled, false) - if err != nil { - return errMap, err - } - - err = cfgStore.UpdateSetting(ctx, disabled, false) - if err != nil { - return errMap, err - } - - err = cfgStore.UpdateSetting(ctx, modify, false) - if err != nil { - return errMap, err } return errMap, nil } - -func settingByID(all []*model.Setting, id string) *model.Setting { - found, ok := lo.Find(all, func(s *model.Setting) bool { - return s.Id == id - }) - if !ok { - return nil - } - - return found -} - -func extractSID(rule string) *string { - sids := sidExtracter.FindAllStringSubmatch(rule, 2) - if len(sids) != 1 { // 1 match = 1 sid - return nil - } - - return util.Ptr(strings.TrimSpace(sids[0][1])) -} - -func indexLocal(lines []string) map[string]int { - index := map[string]int{} - - for i, line := range lines { - sid := extractSID(line) - if sid == nil { - continue - } - - index[*sid] = i - } - - return index -} - -func indexEnabled(lines []string) map[string]int { - index := map[string]int{} - - for i, line := range lines { - line = strings.TrimSpace(strings.TrimLeft(line, "# \t")) - if line != "" { - index[line] = i - } - } - - return index -} - -func indexModify(lines []string) map[string]int { - index := map[string]int{} - - for i, line := range lines { - line = strings.TrimSpace(strings.TrimLeft(line, "# \t")) - - if strings.HasSuffix(line, suricataModifyFromTo) { - parts := strings.SplitN(line, " ", 2) - index[parts[0]] = i - } - } - - return index -} diff --git a/server/detectionstore.go b/server/detectionstore.go new file mode 100644 index 000000000..726e41823 --- /dev/null +++ b/server/detectionstore.go @@ -0,0 +1,21 @@ +// Copyright 2019 Jason Ertel (github.com/jertel). +// Copyright 2020-2023 Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +// or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +// https://securityonion.net/license; you may not use this file except in compliance with the +// Elastic License 2.0. + +package server + +import ( + "context" + + "github.com/security-onion-solutions/securityonion-soc/model" +) + +type Detectionstore interface { + CreateDetection(ctx context.Context, detect *model.Detection) (*model.Detection, error) + GetDetection(ctx context.Context, detectId string) (*model.Detection, error) + UpdateDetection(ctx context.Context, detect *model.Detection) (*model.Detection, error) + UpdateDetectionField(ctx context.Context, id string, field string, value any) (*model.Detection, bool, error) + DeleteDetection(ctx context.Context, detectID string) (*model.Detection, error) +} diff --git a/server/infohandler.go b/server/infohandler.go index 3d7604f31..974478ef0 100644 --- a/server/infohandler.go +++ b/server/infohandler.go @@ -72,7 +72,7 @@ func (h *InfoHandler) getInfo(w http.ResponseWriter, r *http.Request) { detections.GroupFetchLimit = 50 detections.MostRecentlyUsedLimit = 5 detections.SafeStringMaxLength = 100 - detections.QueryBaseFilter = "_index:\"*:so-case\" AND so_kind:detection" + detections.QueryBaseFilter = "_index:\"*:so-detection\" AND so_kind:detection" detections.EventFields = map[string][]string{ "default": { "so_detection.title", @@ -113,7 +113,7 @@ func (h *InfoHandler) getInfo(w http.ResponseWriter, r *http.Request) { playbooks.EventItemsPerPage = 50 playbooks.GroupFetchLimit = 50 playbooks.MostRecentlyUsedLimit = 5 - playbooks.QueryBaseFilter = "_index:\"*:so-case\" AND so_kind:playbook" + playbooks.QueryBaseFilter = "_index:\"*:so-detection\" AND so_kind:playbook" playbooks.SafeStringMaxLength = 100 playbooks.Queries = []*config.HuntingQuery{ {Query: "*"}, diff --git a/server/modules/elastic/elastic.go b/server/modules/elastic/elastic.go index d9f16b60a..f3f9ce36b 100644 --- a/server/modules/elastic/elastic.go +++ b/server/modules/elastic/elastic.go @@ -28,6 +28,10 @@ const DEFAULT_ASYNC_THRESHOLD = 10 const DEFAULT_INTERVALS = 25 const DEFAULT_MAX_LOG_LENGTH = 1024 const DEFAULT_CASE_SCHEMA_PREFIX = "so_" +const DEFAULT_DETECTION_INDEX = "*:so-detection" +const DEFAULT_DETECTION_AUDIT_INDEX = "*:so-detectionhistory" +const DEFAULT_DETECTION_ASSOCIATIONS_MAX = 1000 +const DEFAULT_DETECTION_SCHEMA_PREFIX = "so_" type Elastic struct { config module.ModuleConfig @@ -67,6 +71,7 @@ func (elastic *Elastic) Init(cfg module.ModuleConfig) error { intervals := module.GetIntDefault(cfg, "intervals", DEFAULT_INTERVALS) maxLogLength := module.GetIntDefault(cfg, "maxLogLength", DEFAULT_MAX_LOG_LENGTH) casesEnabled := module.GetBoolDefault(cfg, "casesEnabled", true) + detectionsEnabled := module.GetBoolDefault(cfg, "detectionsEnabled", true) err := elastic.store.Init(host, remoteHosts, username, password, verifyCert, timeShiftMs, defaultDurationMs, esSearchOffsetMs, timeoutMs, cacheMs, index, asyncThreshold, intervals, maxLogLength) if err == nil && elastic.server != nil { @@ -80,12 +85,29 @@ func (elastic *Elastic) Init(cfg module.ModuleConfig) error { maxCaseAssociations := module.GetIntDefault(cfg, "maxCaseAssociations", DEFAULT_CASE_ASSOCIATIONS_MAX) schemaPrefix := module.GetStringDefault(cfg, "schemaPrefix", DEFAULT_CASE_SCHEMA_PREFIX) casestore := NewElasticCasestore(elastic.server) + err = casestore.Init(caseIndex, auditIndex, maxCaseAssociations, schemaPrefix, commonObservables) if err == nil { elastic.server.Casestore = casestore } } } + if detectionsEnabled { + if elastic.server.Detectionstore != nil { + err = errors.New("Multiple detection modules cannot be enabled concurrently") + } else { + detIndex := module.GetStringDefault(cfg, "detectionIndex", DEFAULT_DETECTION_INDEX) + detAuditIndex := module.GetStringDefault(cfg, "detectionAuditIndex", DEFAULT_DETECTION_AUDIT_INDEX) + maxDetAssociations := module.GetIntDefault(cfg, "maxDetectionAssociations", DEFAULT_DETECTION_ASSOCIATIONS_MAX) + schemaPrefix := module.GetStringDefault(cfg, "schemaPrefix", DEFAULT_DETECTION_SCHEMA_PREFIX) + detstore := NewElasticDetectionstore(elastic.server) + + err = detstore.Init(detIndex, detAuditIndex, maxDetAssociations, schemaPrefix) + if err == nil { + elastic.server.Detectionstore = detstore + } + } + } } licensing.ValidateDataUrl(host) diff --git a/server/modules/elastic/elasticcasestore.go b/server/modules/elastic/elasticcasestore.go index 651f2fe0a..5eeaf0d42 100644 --- a/server/modules/elastic/elasticcasestore.go +++ b/server/modules/elastic/elasticcasestore.go @@ -13,7 +13,6 @@ import ( "regexp" "sort" "strconv" - "strings" "time" "github.com/apex/log" @@ -274,50 +273,6 @@ func (store *ElasticCasestore) validateArtifactStream(artifactstream *model.Arti return err } -func (store *ElasticCasestore) validateDetection(detect *model.Detection) error { - var err error - - if err == nil && detect.Id != "" { - err = store.validateId(detect.Id, "onionId") - } - - if err == nil && detect.PublicID != "" { - err = store.validateId(detect.PublicID, "publicId") - } - - if err == nil && detect.Title != "" { - err = store.validateString(detect.Title, SHORT_STRING_MAX, "title") - } - - if err == nil && detect.Severity != "" { - err = store.validateString(string(detect.Severity), SHORT_STRING_MAX, "severity") - } - - if err == nil && detect.Author != "" { - err = store.validateString(detect.Author, SHORT_STRING_MAX, "author") - } - - if err == nil && detect.Description != "" { - err = store.validateString(detect.Description, LONG_STRING_MAX, "description") - } - - if err == nil && detect.Content != "" { - err = store.validateString(detect.Content, LONG_STRING_MAX, "content") - } - - if err == nil && detect.Note != "" { - err = store.validateString(detect.Note, LONG_STRING_MAX, "note") - } - - if err == nil { - _, okEngine := model.EnginesByName[detect.Engine] - if !okEngine { - err = errors.New("invalid engine") - } - } - return err -} - func (store *ElasticCasestore) prepareForSave(ctx context.Context, obj *model.Auditable) string { obj.UserId = ctx.Value(web.ContextKeyRequestorId).(string) @@ -945,121 +900,3 @@ func (store *ElasticCasestore) ExtractCommonObservables(ctx context.Context, eve } return nil } - -func (store *ElasticCasestore) CreateDetection(ctx context.Context, detect *model.Detection) (*model.Detection, error) { - var err error - - err = store.validateDetection(detect) - if err != nil { - return nil, err - } - - if detect.Id != "" { - return nil, errors.New("Unexpected ID found in new comment") - } - - now := time.Now() - detect.CreateTime = &now - var results *model.EventIndexResults - results, err = store.save(ctx, detect, "detection", store.prepareForSave(ctx, &detect.Auditable)) - if err == nil { - // Read object back to get new modify date, etc - detect, err = store.GetDetection(ctx, results.DocumentId) - } - - return detect, err -} - -func (store *ElasticCasestore) GetDetection(ctx context.Context, detectId string) (detect *model.Detection, err error) { - err = store.validateId(detectId, "detectId") - if err != nil { - return nil, err - } - - obj, err := store.get(ctx, detectId, "detection") - if err == nil && obj != nil { - detect = obj.(*model.Detection) - } - - return detect, err -} - -func (store *ElasticCasestore) UpdateDetection(ctx context.Context, detect *model.Detection) (*model.Detection, error) { - err := store.validateDetection(detect) - if err != nil { - return nil, err - } - - if detect.Id == "" { - err = errors.New("Missing detection onion ID") - return nil, err - } - - var old *model.Detection - old, err = store.GetDetection(ctx, detect.Id) - if err != nil { - return nil, err - } - - var results *model.EventIndexResults - - // Preserve read-only fields - detect.CreateTime = old.CreateTime - - results, err = store.save(ctx, detect, "detection", store.prepareForSave(ctx, &detect.Auditable)) - if err != nil { - return nil, err - } - // Read object back to get new modify date, etc - return store.GetDetection(ctx, results.DocumentId) -} - -func (store *ElasticCasestore) UpdateDetectionField(ctx context.Context, id string, field string, value any) (*model.Detection, bool, error) { - var modified bool - - detect, err := store.GetDetection(ctx, id) - if err != nil { - return nil, false, err - } - - switch strings.ToLower(field) { - case "isenabled": - bVal, ok := value.(bool) - if !ok { - return nil, false, fmt.Errorf("invalid value for field isEnabled (expected bool): %[1]v (%[1]T)", value) - } - - if detect.IsEnabled != bVal { - detect.IsEnabled = bVal - modified = true - } - } - - err = store.validateDetection(detect) - if err != nil { - return nil, false, err - } - - if modified { - var results *model.EventIndexResults - - results, err = store.save(ctx, detect, "detection", store.prepareForSave(ctx, &detect.Auditable)) - if err == nil { - // Read object back to get new modify date, etc - detect, err = store.GetDetection(ctx, results.DocumentId) - } - } - - return detect, modified, err -} - -func (store *ElasticCasestore) DeleteDetection(ctx context.Context, onionID string) (*model.Detection, error) { - detect, err := store.GetDetection(ctx, onionID) - if err != nil { - return nil, err - } - - err = store.delete(ctx, detect, "detection", store.prepareForSave(ctx, &detect.Auditable)) - - return detect, err -} diff --git a/server/modules/elastic/elasticdetectionstore.go b/server/modules/elastic/elasticdetectionstore.go new file mode 100644 index 000000000..35f0e6264 --- /dev/null +++ b/server/modules/elastic/elasticdetectionstore.go @@ -0,0 +1,360 @@ +package elastic + +import ( + "context" + "errors" + "fmt" + "regexp" + "strconv" + "strings" + "time" + + "github.com/apex/log" + "github.com/security-onion-solutions/securityonion-soc/model" + "github.com/security-onion-solutions/securityonion-soc/server" + "github.com/security-onion-solutions/securityonion-soc/web" +) + +type ElasticDetectionstore struct { + server *server.Server + index string + auditIndex string + maxAssociations int + schemaPrefix string +} + +func NewElasticDetectionstore(srv *server.Server) *ElasticDetectionstore { + return &ElasticDetectionstore{ + server: srv, + } +} + +func (store *ElasticDetectionstore) Init(index string, auditIndex string, maxAssociations int, schemaPrefix string) error { + store.index = index + store.auditIndex = auditIndex + store.maxAssociations = maxAssociations + store.schemaPrefix = schemaPrefix + + return nil +} + +func (store *ElasticDetectionstore) validateId(id string, label string) error { + var err error + + isValidId := regexp.MustCompile(`^[A-Za-z0-9-_]{5,50}$`).MatchString + if !isValidId(id) { + err = fmt.Errorf("invalid ID for %s", label) + } + + return err +} + +func (store *ElasticDetectionstore) validateString(str string, max int, label string) error { + return store.validateStringRequired(str, 0, max, label) +} + +func (store *ElasticDetectionstore) validateStringRequired(str string, min int, max int, label string) error { + var err error + + length := len(str) + if length > max { + err = errors.New(fmt.Sprintf("%s is too long (%d/%d)", label, length, max)) + } else if length < min { + err = errors.New(fmt.Sprintf("%s is too short (%d/%d)", label, length, min)) + } + + return err +} + +func (store *ElasticDetectionstore) validateDetection(detect *model.Detection) error { + var err error + + if err == nil && detect.Id != "" { + err = store.validateId(detect.Id, "onionId") + } + + if err == nil && detect.PublicID != "" { + err = store.validateId(detect.PublicID, "publicId") + } + + if err == nil && detect.Title != "" { + err = store.validateString(detect.Title, SHORT_STRING_MAX, "title") + } + + if err == nil && detect.Severity != "" { + err = store.validateString(string(detect.Severity), SHORT_STRING_MAX, "severity") + } + + if err == nil && detect.Author != "" { + err = store.validateString(detect.Author, SHORT_STRING_MAX, "author") + } + + if err == nil && detect.Description != "" { + err = store.validateString(detect.Description, LONG_STRING_MAX, "description") + } + + if err == nil && detect.Content != "" { + err = store.validateString(detect.Content, LONG_STRING_MAX, "content") + } + + if err == nil && detect.Note != "" { + err = store.validateString(detect.Note, LONG_STRING_MAX, "note") + } + + if err == nil { + _, okEngine := model.EnginesByName[detect.Engine] + if !okEngine { + err = errors.New("invalid engine") + } + } + + return err +} + +func (store *ElasticDetectionstore) save(ctx context.Context, obj interface{}, kind string, id string) (*model.EventIndexResults, error) { + var results *model.EventIndexResults + var err error + + if err = store.server.CheckAuthorized(ctx, "write", "cases"); err == nil { + document := convertObjectToDocumentMap(kind, obj, store.schemaPrefix) + document[store.schemaPrefix+"kind"] = kind + + results, err = store.server.Eventstore.Index(ctx, store.index, document, id) + if err == nil { + document[store.schemaPrefix+AUDIT_DOC_ID] = results.DocumentId + + if id == "" { + document[store.schemaPrefix+"operation"] = "create" + } else { + document[store.schemaPrefix+"operation"] = "update" + } + + _, err = store.server.Eventstore.Index(ctx, store.auditIndex, document, "") + if err != nil { + log.WithFields(log.Fields{ + "documentId": results.DocumentId, + "kind": kind, + }).WithError(err).Error("Object indexed successfully however audit record failed to index") + } + } + } + + return results, err +} + +func (store *ElasticDetectionstore) delete(ctx context.Context, obj interface{}, kind string, id string) error { + var err error + + if err = store.server.CheckAuthorized(ctx, "write", "cases"); err == nil { + err = store.server.Eventstore.Delete(ctx, store.index, id) + if err == nil { + document := convertObjectToDocumentMap(kind, obj, store.schemaPrefix) + document[store.schemaPrefix+AUDIT_DOC_ID] = id + document[store.schemaPrefix+"kind"] = kind + document[store.schemaPrefix+"operation"] = "delete" + + _, err = store.server.Eventstore.Index(ctx, store.auditIndex, document, "") + if err != nil { + log.WithFields(log.Fields{ + "documentId": id, + "kind": kind, + }).WithError(err).Error("Object deleted successfully however audit record failed to index") + } + } + } + + return err +} + +func (store *ElasticDetectionstore) get(ctx context.Context, id string, kind string) (interface{}, error) { + query := fmt.Sprintf(`_index:"%s" AND %skind:"%s" AND _id:"%s"`, store.index, store.schemaPrefix, kind, id) + + objects, err := store.getAll(ctx, query, 1) + if err == nil { + if len(objects) > 0 { + return objects[0], err + } + + err = errors.New("Object not found") + } + + return nil, err +} + +func (store *ElasticDetectionstore) getAll(ctx context.Context, query string, max int) ([]interface{}, error) { + var err error + var objects []interface{} + + if err = store.server.CheckAuthorized(ctx, "read", "cases"); err == nil { + criteria := model.NewEventSearchCriteria() + format := "2006-01-02 3:04:05 PM" + + var zeroTime time.Time + + zeroTimeStr := zeroTime.Format(format) + now := time.Now() + endTime := now.Format(format) + zone := now.Location().String() + + err = criteria.Populate(query, + zeroTimeStr+" - "+endTime, // timeframe range + format, // timeframe format + zone, // timezone + "0", // no metrics + strconv.Itoa(max)) + + if err == nil { + var results *model.EventSearchResults + + results, err = store.server.Eventstore.Search(ctx, criteria) + if err == nil { + for _, event := range results.Events { + var obj interface{} + + obj, err = convertElasticEventToObject(event, store.schemaPrefix) + if err == nil { + objects = append(objects, obj) + } else { + log.WithField("event", event).WithError(err).Error("Unable to convert case object") + } + } + } + } + } + + return objects, err +} + +func (store *ElasticDetectionstore) prepareForSave(ctx context.Context, obj *model.Auditable) string { + obj.UserId = ctx.Value(web.ContextKeyRequestorId).(string) + + // Don't waste space by saving the these values which are already part of ES documents + id := obj.Id + obj.Id = "" + obj.UpdateTime = nil + + return id +} + +func (store *ElasticDetectionstore) CreateDetection(ctx context.Context, detect *model.Detection) (*model.Detection, error) { + var err error + + err = store.validateDetection(detect) + if err != nil { + return nil, err + } + + if detect.Id != "" { + return nil, errors.New("Unexpected ID found in new comment") + } + + now := time.Now() + detect.CreateTime = &now + + var results *model.EventIndexResults + + results, err = store.save(ctx, detect, "detection", store.prepareForSave(ctx, &detect.Auditable)) + if err == nil { + // Read object back to get new modify date, etc + detect, err = store.GetDetection(ctx, results.DocumentId) + } + + return detect, err +} + +func (store *ElasticDetectionstore) GetDetection(ctx context.Context, detectId string) (detect *model.Detection, err error) { + err = store.validateId(detectId, "detectId") + if err != nil { + return nil, err + } + + obj, err := store.get(ctx, detectId, "detection") + if err == nil && obj != nil { + detect = obj.(*model.Detection) + } + + return detect, err +} + +func (store *ElasticDetectionstore) UpdateDetection(ctx context.Context, detect *model.Detection) (*model.Detection, error) { + err := store.validateDetection(detect) + if err != nil { + return nil, err + } + + if detect.Id == "" { + err = errors.New("Missing detection onion ID") + return nil, err + } + + var old *model.Detection + + old, err = store.GetDetection(ctx, detect.Id) + if err != nil { + return nil, err + } + + var results *model.EventIndexResults + + // Preserve read-only fields + detect.CreateTime = old.CreateTime + + results, err = store.save(ctx, detect, "detection", store.prepareForSave(ctx, &detect.Auditable)) + + if err != nil { + return nil, err + } + + // Read object back to get new modify date, etc + return store.GetDetection(ctx, results.DocumentId) +} + +func (store *ElasticDetectionstore) UpdateDetectionField(ctx context.Context, id string, field string, value any) (*model.Detection, bool, error) { + var modified bool + + detect, err := store.GetDetection(ctx, id) + if err != nil { + return nil, false, err + } + + switch strings.ToLower(field) { + case "isenabled": + bVal, ok := value.(bool) + if !ok { + return nil, false, fmt.Errorf("invalid value for field isEnabled (expected bool): %[1]v (%[1]T)", value) + } + + if detect.IsEnabled != bVal { + detect.IsEnabled = bVal + modified = true + } + } + + err = store.validateDetection(detect) + if err != nil { + return nil, false, err + } + + if modified { + var results *model.EventIndexResults + + results, err = store.save(ctx, detect, "detection", store.prepareForSave(ctx, &detect.Auditable)) + if err == nil { + // Read object back to get new modify date, etc + detect, err = store.GetDetection(ctx, results.DocumentId) + } + } + + return detect, modified, err +} + +func (store *ElasticDetectionstore) DeleteDetection(ctx context.Context, onionID string) (*model.Detection, error) { + detect, err := store.GetDetection(ctx, onionID) + if err != nil { + return nil, err + } + + err = store.delete(ctx, detect, "detection", store.prepareForSave(ctx, &detect.Auditable)) + + return detect, err +} diff --git a/server/modules/modules.go b/server/modules/modules.go index 02e2538fb..c766934f8 100644 --- a/server/modules/modules.go +++ b/server/modules/modules.go @@ -19,6 +19,7 @@ import ( "github.com/security-onion-solutions/securityonion-soc/server/modules/sostatus" "github.com/security-onion-solutions/securityonion-soc/server/modules/statickeyauth" "github.com/security-onion-solutions/securityonion-soc/server/modules/staticrbac" + "github.com/security-onion-solutions/securityonion-soc/server/modules/suricata" "github.com/security-onion-solutions/securityonion-soc/server/modules/thehive" ) @@ -35,5 +36,7 @@ func BuildModuleMap(srv *server.Server) map[string]module.Module { moduleMap["statickeyauth"] = statickeyauth.NewStaticKeyAuth(srv) moduleMap["staticrbac"] = staticrbac.NewStaticRbac(srv) moduleMap["thehive"] = thehive.NewTheHive(srv) + moduleMap["suricataengine"] = suricata.NewSuricataEngine(srv) + return moduleMap } diff --git a/server/modules/modules_test.go b/server/modules/modules_test.go index fae430a34..efb5b0f7e 100644 --- a/server/modules/modules_test.go +++ b/server/modules/modules_test.go @@ -25,6 +25,7 @@ func TestBuildModuleMap(tester *testing.T) { findModule(tester, mm, "sostatus") findModule(tester, mm, "statickeyauth") findModule(tester, mm, "thehive") + findModule(tester, mm, "suricataengine") } func findModule(tester *testing.T, mm map[string]module.Module, module string) { diff --git a/server/modules/suricata/suricata.go b/server/modules/suricata/suricata.go new file mode 100644 index 000000000..3f5133336 --- /dev/null +++ b/server/modules/suricata/suricata.go @@ -0,0 +1,269 @@ +package suricata + +import ( + "context" + "fmt" + "regexp" + "strings" + + "github.com/samber/lo" + "github.com/security-onion-solutions/securityonion-soc/model" + "github.com/security-onion-solutions/securityonion-soc/module" + "github.com/security-onion-solutions/securityonion-soc/server" + "github.com/security-onion-solutions/securityonion-soc/util" +) + +var sidExtracter = regexp.MustCompile(`(?i)\bsid: ?['"]?(.*?)['"]?;`) + +const modifyFromTo = `"flowbits" "noalert; flowbits"` + +type SuricataEngine struct { + srv *server.Server +} + +func NewSuricataEngine(srv *server.Server) *SuricataEngine { + return &SuricataEngine{ + srv: srv, + } +} + +func (s *SuricataEngine) PrerequisiteModules() []string { + return nil +} + +func (s *SuricataEngine) Init(config module.ModuleConfig) error { + return nil +} + +func (s *SuricataEngine) Start() error { + s.srv.DetectionEngines[model.EngineNameSuricata] = s + return nil +} + +func (s *SuricataEngine) Stop() error { + return nil +} + +func (s *SuricataEngine) IsRunning() bool { + return false +} + +func (s *SuricataEngine) ValidateRule(rule string) (string, error) { + return rule, nil +} + +func (s *SuricataEngine) SyncDetections(ctx context.Context, detections []*model.Detection) (errMap map[string]string, err error) { + defer func() { + if len(errMap) == 0 { + errMap = nil + } + }() + + allSettings, err := s.srv.Configstore.GetSettings(ctx) + if err != nil { + return nil, err + } + + local := settingByID(allSettings, "idstools.rules.local__rules") + if local == nil { + return nil, fmt.Errorf("unable to find local rules setting") + } + + enabled := settingByID(allSettings, "idstools.sids.enabled") + if enabled == nil { + return nil, fmt.Errorf("unable to find enabled setting") + } + + disabled := settingByID(allSettings, "idstools.sids.disabled") + if disabled == nil { + return nil, fmt.Errorf("unable to find disabled setting") + } + + modify := settingByID(allSettings, "idstools.sids.modify") + if modify == nil { + return nil, fmt.Errorf("unable to find modify setting") + } + + localLines := strings.Split(local.Value, "\n") + enabledLines := strings.Split(enabled.Value, "\n") + disabledLines := strings.Split(disabled.Value, "\n") + modifyLines := strings.Split(modify.Value, "\n") + + localIndex := indexLocal(localLines) + enabledIndex := indexEnabled(enabledLines) + disabledIndex := indexEnabled(disabledLines) + modifyIndex := indexModify(modifyLines) + + errMap = map[string]string{} // map[sid]error + + for _, detect := range detections { + parsedRule, err := ParseSuricataRule(detect.Content) + if err != nil { + errMap[detect.PublicID] = fmt.Sprintf("unable to parse rule; reason=%s", err.Error()) + continue + } + + opt, ok := parsedRule.GetOption("sid") + if !ok || opt == nil { + errMap[detect.PublicID] = fmt.Sprintf("rule does not contain a SID; rule=%s", detect.Content) + continue + } + + sid := *opt + _, isFlowbits := parsedRule.GetOption("flowbits") + + lineNum, inLocal := localIndex[sid] + if !inLocal { + localLines = append(localLines, detect.Content) + lineNum = len(localLines) - 1 + localIndex[sid] = lineNum + } else { + localLines[lineNum] = detect.Content + } + + lineNum, inEnabled := enabledIndex[sid] + if !inEnabled { + line := detect.PublicID + if !detect.IsEnabled && !isFlowbits { + line = "# " + line + } + + enabledLines = append(enabledLines, line) + lineNum = len(enabledLines) - 1 + enabledIndex[sid] = lineNum + } else { + line := detect.PublicID + if !detect.IsEnabled && !isFlowbits { + line = "# " + line + } + + enabledLines[lineNum] = line + } + + if !isFlowbits { + lineNum, inDisabled := disabledIndex[sid] + if !inDisabled { + line := detect.PublicID + if detect.IsEnabled { + line = "# " + line + } + + disabledLines = append(disabledLines, line) + lineNum = len(disabledLines) - 1 + disabledIndex[sid] = lineNum + } else { + line := detect.PublicID + if detect.IsEnabled { + line = "# " + line + } + + disabledLines[lineNum] = line + } + } + + if isFlowbits { + lineNum, inModify := modifyIndex[sid] + if !inModify && !detect.IsEnabled { + // not in the modify file, but should be + line := fmt.Sprintf("%s %s", detect.PublicID, modifyFromTo) + modifyLines = append(modifyLines, line) + lineNum = len(modifyLines) - 1 + modifyIndex[sid] = lineNum + } else if inModify && detect.IsEnabled { + // in modify, but shouldn't be + modifyLines = append(modifyLines[:lineNum], modifyLines[lineNum+1:]...) + delete(modifyIndex, sid) + } + } + } + + local.Value = strings.Join(localLines, "\n") + enabled.Value = strings.Join(enabledLines, "\n") + disabled.Value = strings.Join(disabledLines, "\n") + modify.Value = strings.Join(modifyLines, "\n") + + err = s.srv.Configstore.UpdateSetting(ctx, local, false) + if err != nil { + return errMap, err + } + + err = s.srv.Configstore.UpdateSetting(ctx, enabled, false) + if err != nil { + return errMap, err + } + + err = s.srv.Configstore.UpdateSetting(ctx, disabled, false) + if err != nil { + return errMap, err + } + + err = s.srv.Configstore.UpdateSetting(ctx, modify, false) + if err != nil { + return errMap, err + } + + return errMap, nil +} + +func settingByID(all []*model.Setting, id string) *model.Setting { + found, ok := lo.Find(all, func(s *model.Setting) bool { + return s.Id == id + }) + if !ok { + return nil + } + + return found +} + +func extractSID(rule string) *string { + sids := sidExtracter.FindAllStringSubmatch(rule, 2) + if len(sids) != 1 { // 1 match = 1 sid + return nil + } + + return util.Ptr(strings.TrimSpace(sids[0][1])) +} + +func indexLocal(lines []string) map[string]int { + index := map[string]int{} + + for i, line := range lines { + sid := extractSID(line) + if sid == nil { + continue + } + + index[*sid] = i + } + + return index +} + +func indexEnabled(lines []string) map[string]int { + index := map[string]int{} + + for i, line := range lines { + line = strings.TrimSpace(strings.TrimLeft(line, "# \t")) + if line != "" { + index[line] = i + } + } + + return index +} + +func indexModify(lines []string) map[string]int { + index := map[string]int{} + + for i, line := range lines { + line = strings.TrimSpace(strings.TrimLeft(line, "# \t")) + + if strings.HasSuffix(line, modifyFromTo) { + parts := strings.SplitN(line, " ", 2) + index[parts[0]] = i + } + } + + return index +} diff --git a/server/detectionhandler_test.go b/server/modules/suricata/suricata_test.go similarity index 52% rename from server/detectionhandler_test.go rename to server/modules/suricata/suricata_test.go index e1449f03f..7e9cbcdfd 100644 --- a/server/detectionhandler_test.go +++ b/server/modules/suricata/suricata_test.go @@ -1,15 +1,35 @@ -package server +package suricata import ( "context" "testing" "github.com/security-onion-solutions/securityonion-soc/model" + "github.com/security-onion-solutions/securityonion-soc/module" + "github.com/security-onion-solutions/securityonion-soc/server" "github.com/security-onion-solutions/securityonion-soc/util" - "github.com/stretchr/testify/assert" ) +func TestSuricataModule(t *testing.T) { + srv := &server.Server{ + DetectionEngines: map[model.EngineName]server.DetectionEngine{}, + } + mod := NewSuricataEngine(srv) + + assert.Implements(t, (*module.Module)(nil), mod) + assert.Implements(t, (*server.DetectionEngine)(nil), mod) + + err := mod.Start() + assert.Nil(t, err) + + err = mod.Stop() + assert.Nil(t, err) + + assert.Equal(t, 1, len(srv.DetectionEngines)) + assert.Same(t, mod, srv.DetectionEngines[model.EngineNameSuricata]) +} + func TestSettingByID(t *testing.T) { allSettings := []*model.Setting{ {Id: "1", Value: "one"}, @@ -167,23 +187,94 @@ func TestIndexModify(t *testing.T) { func TestSyncSuricata(t *testing.T) { emptySettings := []*model.Setting{ {Id: "idstools.rules.local__rules"}, - {Id: "idstools.rules.enabled"}, - {Id: "idstools.rules.disabled"}, - {Id: "idstools.rules.modify"}, + {Id: "idstools.sids.enabled"}, + {Id: "idstools.sids.disabled"}, + {Id: "idstools.sids.modify"}, } table := []struct { - Name string - InitialSettings []*model.Setting - Detections []*model.Detection // Content (Valid Rule), PublicID, IsEnabled + Name string + InitialSettings []*model.Setting + Detections []*model.Detection // Content (Valid Rule), PublicID, IsEnabled ExpectedSettings map[string]string - ExpectedErr error - ExpectedErrMap map[string]string + ExpectedErr error + ExpectedErrMap map[string]string }{ { - Name: "Simple Add", + Name: "Enable New Simple Rule", + InitialSettings: emptySettings, + Detections: []*model.Detection{ + { + PublicID: "10000", + Content: `alert http any any -> any any (msg:"GPL ATTACK_RESPONSE id check returned root"; content:"uid=0|28|root|29|"; classtype:bad-unknown; sid:10000; rev:7; metadata:created_at 2010_09_23, updated_at 2010_09_23;)`, + IsEnabled: true, + }, + }, + ExpectedSettings: map[string]string{ + "idstools.rules.local__rules": "\n" + `alert http any any -> any any (msg:"GPL ATTACK_RESPONSE id check returned root"; content:"uid=0|28|root|29|"; classtype:bad-unknown; sid:10000; rev:7; metadata:created_at 2010_09_23, updated_at 2010_09_23;)`, + "idstools.sids.enabled": "\n10000", + "idstools.sids.disabled": "\n# 10000", + "idstools.sids.modify": "", + }, + }, + { + Name: "Disable Existing Simple Rule", + InitialSettings: []*model.Setting{ + {Id: "idstools.rules.local__rules", Value: `alert http any any -> any any (msg:"GPL ATTACK_RESPONSE id check returned root"; content:"uid=0|28|root|29|"; classtype:bad-unknown; sid:10000; rev:7; metadata:created_at 2010_09_23, updated_at 2010_09_23;)`}, + {Id: "idstools.sids.enabled", Value: "10000"}, + {Id: "idstools.sids.disabled", Value: "# 10000"}, + {Id: "idstools.sids.modify"}, + }, + Detections: []*model.Detection{ + { + PublicID: "10000", + Content: `alert http any any -> any any (msg:"GPL ATTACK_RESPONSE id check returned root"; content:"uid=0|28|root|29|"; classtype:bad-unknown; sid:10000; rev:7; metadata:created_at 2010_09_23, updated_at 2010_09_23;)`, + IsEnabled: false, + }, + }, + ExpectedSettings: map[string]string{ + "idstools.rules.local__rules": `alert http any any -> any any (msg:"GPL ATTACK_RESPONSE id check returned root"; content:"uid=0|28|root|29|"; classtype:bad-unknown; sid:10000; rev:7; metadata:created_at 2010_09_23, updated_at 2010_09_23;)`, + "idstools.sids.enabled": "# 10000", + "idstools.sids.disabled": "10000", + "idstools.sids.modify": "", + }, + }, + { + Name: "Enable New Flowbits Rule", InitialSettings: emptySettings, Detections: []*model.Detection{ - {}, + { + PublicID: "10000", + Content: `alert http any any -> any any ( msg:"RULE A"; flow: established,to_server; http.method; content:"POST"; http.content_type; content:"x-www-form-urlencoded"; flowbits: set, test; sid:10000;)`, + IsEnabled: true, + }, + }, + ExpectedSettings: map[string]string{ + "idstools.rules.local__rules": "\n" + `alert http any any -> any any ( msg:"RULE A"; flow: established,to_server; http.method; content:"POST"; http.content_type; content:"x-www-form-urlencoded"; flowbits: set, test; sid:10000;)`, + "idstools.sids.enabled": "\n10000", + "idstools.sids.disabled": "\n# 10000", + "idstools.sids.modify": "", + }, + }, + { + Name: "Disable Existing Flowbits Rule", + InitialSettings: []*model.Setting{ + {Id: "idstools.rules.local__rules", Value: `alert http any any -> any any ( msg:"RULE A"; flow: established,to_server; http.method; content:"POST"; http.content_type; content:"x-www-form-urlencoded"; flowbits: set, test; sid:10000;)`}, + {Id: "idstools.sids.enabled", Value: "10000"}, + {Id: "idstools.sids.disabled", Value: "# 10000"}, + {Id: "idstools.sids.modify", Value: ""}, + }, + Detections: []*model.Detection{ + { + PublicID: "10000", + Content: `alert http any any -> any any ( msg:"RULE A"; flow: established,to_server; http.method; content:"POST"; http.content_type; content:"x-www-form-urlencoded"; flowbits: set, test; sid:10000;)`, + IsEnabled: false, + }, + }, + ExpectedSettings: map[string]string{ + "idstools.rules.local__rules": `alert http any any -> any any ( msg:"RULE A"; flow: established,to_server; http.method; content:"POST"; http.content_type; content:"x-www-form-urlencoded"; flowbits: set, test; sid:10000;)`, + "idstools.sids.enabled": "10000", + "idstools.sids.disabled": "# 10000", + "idstools.sids.modify": "\n" + `10000 "flowbits" "noalert; flowbits"`, }, }, } @@ -195,9 +286,14 @@ func TestSyncSuricata(t *testing.T) { t.Run(test.Name, func(t *testing.T) { t.Parallel() - mCfgStore := NewMemConfigStore(test.InitialSettings) + mCfgStore := server.NewMemConfigStore(test.InitialSettings) + mod := NewSuricataEngine(&server.Server{ + Configstore: mCfgStore, + DetectionEngines: map[model.EngineName]server.DetectionEngine{}, + }) + mod.srv.DetectionEngines[model.EngineNameSuricata] = mod - errMap, err := syncSuricata(ctx, mCfgStore, test.Detections) + errMap, err := mod.SyncDetections(ctx, test.Detections) assert.Equal(t, test.ExpectedErr, err) assert.Equal(t, test.ExpectedErrMap, errMap) @@ -212,4 +308,4 @@ func TestSyncSuricata(t *testing.T) { } }) } -} \ No newline at end of file +} diff --git a/syntax/suricata.go b/server/modules/suricata/validate.go similarity index 99% rename from syntax/suricata.go rename to server/modules/suricata/validate.go index 90ca8af83..68ca6731d 100644 --- a/syntax/suricata.go +++ b/server/modules/suricata/validate.go @@ -1,4 +1,4 @@ -package syntax +package suricata import ( "fmt" diff --git a/syntax/suricata_test.go b/server/modules/suricata/validate_test.go similarity index 99% rename from syntax/suricata_test.go rename to server/modules/suricata/validate_test.go index f9b2d7f0f..261cc8b4f 100644 --- a/syntax/suricata_test.go +++ b/server/modules/suricata/validate_test.go @@ -1,4 +1,4 @@ -package syntax +package suricata import ( "testing" diff --git a/server/server.go b/server/server.go index 5080baf4e..4b281b5e5 100644 --- a/server/server.go +++ b/server/server.go @@ -33,6 +33,7 @@ type Server struct { Rolestore Rolestore Eventstore Eventstore Casestore Casestore + Detectionstore Detectionstore Configstore Configstore GridMembersstore GridMembersstore Metrics Metrics @@ -40,13 +41,15 @@ type Server struct { Authorizer rbac.Authorizer Agent *model.User Context context.Context + DetectionEngines map[model.EngineName]DetectionEngine } func NewServer(cfg *config.ServerConfig, version string) *Server { server := &Server{ - Config: cfg, - Host: web.NewHost(cfg.BindAddress, cfg.HtmlDir, cfg.IdleConnectionTimeoutMs, version, cfg.SrvKeyBytes, AGENT_ID), - stoppedChan: make(chan bool, 1), + Config: cfg, + Host: web.NewHost(cfg.BindAddress, cfg.HtmlDir, cfg.IdleConnectionTimeoutMs, version, cfg.SrvKeyBytes, AGENT_ID), + stoppedChan: make(chan bool, 1), + DetectionEngines: map[model.EngineName]DetectionEngine{}, } server.initContext() From a411b0873d2dcff5b58ec72fb90e53ee6589585f Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Tue, 26 Sep 2023 11:05:49 -0600 Subject: [PATCH 008/102] WIP: Tests Converted repeated strings to fixtures. Fleshed out the rest of SyncSuricata's tests. Added validation tests. --- server/modules/suricata/suricata.go | 7 +- server/modules/suricata/suricata_test.go | 260 +++++++++++++++++++---- server/modules/suricata/validate.go | 5 + 3 files changed, 231 insertions(+), 41 deletions(-) diff --git a/server/modules/suricata/suricata.go b/server/modules/suricata/suricata.go index 3f5133336..b9725882d 100644 --- a/server/modules/suricata/suricata.go +++ b/server/modules/suricata/suricata.go @@ -49,7 +49,12 @@ func (s *SuricataEngine) IsRunning() bool { } func (s *SuricataEngine) ValidateRule(rule string) (string, error) { - return rule, nil + parsed, err := ParseSuricataRule(rule) + if err != nil { + return rule, err + } + + return parsed.String(), nil } func (s *SuricataEngine) SyncDetections(ctx context.Context, detections []*model.Detection) (errMap map[string]string, err error) { diff --git a/server/modules/suricata/suricata_test.go b/server/modules/suricata/suricata_test.go index 7e9cbcdfd..0a497f305 100644 --- a/server/modules/suricata/suricata_test.go +++ b/server/modules/suricata/suricata_test.go @@ -11,6 +11,24 @@ import ( "github.com/stretchr/testify/assert" ) +const ( + SimpleRuleSID = "10000" + SimpleRule = `alert http any any -> any any (msg:"GPL ATTACK_RESPONSE id check returned root"; content:"uid=0|28|root|29|"; classtype:bad-unknown; sid:10000; rev:7; metadata:created_at 2010_09_23, updated_at 2010_09_23;)` + FlowbitsRuleASID = "50000" + FlowbitsRuleA = `alert http any any -> any any ( msg:"RULE A"; flow: established,to_server; http.method; content:"POST"; http.content_type; content:"x-www-form-urlencoded"; flowbits: set, test; sid:50000;)` + FlowbitsRuleBSID = "60000" + FlowbitsRuleB = `alert http any any -> any any (msg:"RULE B"; flowbits: isset, test; flow: established,to_client; content:"uid=0"; sid:60000;)` +) + +func emptySettings() []*model.Setting { + return []*model.Setting{ + {Id: "idstools.rules.local__rules"}, + {Id: "idstools.sids.enabled"}, + {Id: "idstools.sids.disabled"}, + {Id: "idstools.sids.modify"}, + } +} + func TestSuricataModule(t *testing.T) { srv := &server.Server{ DetectionEngines: map[model.EngineName]server.DetectionEngine{}, @@ -20,7 +38,10 @@ func TestSuricataModule(t *testing.T) { assert.Implements(t, (*module.Module)(nil), mod) assert.Implements(t, (*server.DetectionEngine)(nil), mod) - err := mod.Start() + err := mod.Init(nil) + assert.Nil(t, err) + + err = mod.Start() assert.Nil(t, err) err = mod.Stop() @@ -85,7 +106,7 @@ func TestExtractSID(t *testing.T) { {Name: "Single-Quoted Empty SID", Input: "sid:'';", Output: util.Ptr("")}, {Name: "Double-Quoted Empty SID", Input: `sid: "";`, Output: util.Ptr("")}, {Name: "Multiple SIDs", Input: "sid: 10000; sid: 10001;", Output: nil}, - {Name: "Sample Rule", Input: `alert http any any -> any any (msg:"RULE B"; flowbits: isset, test; flow: established,to_client; content:"uid=0"; sid:60000;)`, Output: util.Ptr("60000")}, + {Name: "Sample Rule", Input: SimpleRule, Output: util.Ptr(SimpleRuleSID)}, } for _, test := range table { @@ -184,13 +205,66 @@ func TestIndexModify(t *testing.T) { assert.NotContains(t, output, "e4bd794a-8156-4fcc-b6a9-9fb2c9ecadc5") } -func TestSyncSuricata(t *testing.T) { - emptySettings := []*model.Setting{ - {Id: "idstools.rules.local__rules"}, - {Id: "idstools.sids.enabled"}, - {Id: "idstools.sids.disabled"}, - {Id: "idstools.sids.modify"}, +func TestValidate(t *testing.T) { + table := []struct { + Name string + Input string + ExpectedErr *string + }{ + { + Name: "Valid Rule", + Input: SimpleRule, + }, + { + Name: "Valid Rule with Flowbits", + Input: FlowbitsRuleA, + }, + { + Name: "Valid Rule with Escaped Quotes", + Input: `alert http any any -> any any (msg:"This rule has \"escaped quotes\"";)`, + }, + { + Name: "Invalid Direction", + Input: `alert http any any <-> any any (msg:"This rule has an invalid direction";)`, + ExpectedErr: util.Ptr("invalid direction, must be '<>' or '->', got <->"), + }, + { + Name: "Unexpected Suffix", + Input: SimpleRule + "x", + ExpectedErr: util.Ptr("invalid rule, expected end of rule, got 1 more bytes"), + }, + { + Name: "Unexpected End of Rule", + Input: "x", + ExpectedErr: util.Ptr("invalid rule, unexpected end of rule"), + }, + } + + for _, test := range table { + test := test + t.Run(test.Name, func(t *testing.T) { + t.Parallel() + + mod := NewSuricataEngine(&server.Server{}) + + _, err := mod.ValidateRule(test.Input) + if test.ExpectedErr == nil { + assert.NoError(t, err) + + // this rule seems valid, attempt to parse, serialize, re-parse + parsed, err := ParseSuricataRule(test.Input) + assert.NoError(t, err) + + _, err = ParseSuricataRule(parsed.String()) + assert.NoError(t, err) + } else { + assert.Equal(t, *test.ExpectedErr, err.Error()) + } + }) } +} + +func TestSyncSuricata(t *testing.T) { table := []struct { Name string InitialSettings []*model.Setting @@ -201,80 +275,186 @@ func TestSyncSuricata(t *testing.T) { }{ { Name: "Enable New Simple Rule", - InitialSettings: emptySettings, + InitialSettings: emptySettings(), + Detections: []*model.Detection{ + { + PublicID: SimpleRuleSID, + Content: SimpleRule, + IsEnabled: true, + }, + }, + ExpectedSettings: map[string]string{ + "idstools.rules.local__rules": "\n" + SimpleRule, + "idstools.sids.enabled": "\n" + SimpleRuleSID, + "idstools.sids.disabled": "\n# " + SimpleRuleSID, + "idstools.sids.modify": "", + }, + }, + { + Name: "Disable New Simple Rule", + InitialSettings: emptySettings(), Detections: []*model.Detection{ { - PublicID: "10000", - Content: `alert http any any -> any any (msg:"GPL ATTACK_RESPONSE id check returned root"; content:"uid=0|28|root|29|"; classtype:bad-unknown; sid:10000; rev:7; metadata:created_at 2010_09_23, updated_at 2010_09_23;)`, + PublicID: SimpleRuleSID, + Content: SimpleRule, + IsEnabled: false, + }, + }, + ExpectedSettings: map[string]string{ + "idstools.rules.local__rules": "\n" + SimpleRule, + "idstools.sids.enabled": "\n# " + SimpleRuleSID, + "idstools.sids.disabled": "\n" + SimpleRuleSID, + "idstools.sids.modify": "", + }, + }, + { + Name: "Enable Existing Simple Rule", + InitialSettings: []*model.Setting{ + {Id: "idstools.rules.local__rules", Value: SimpleRule}, + {Id: "idstools.sids.enabled", Value: "# " + SimpleRuleSID}, + {Id: "idstools.sids.disabled", Value: SimpleRuleSID}, + {Id: "idstools.sids.modify"}, + }, + Detections: []*model.Detection{ + { + PublicID: SimpleRuleSID, + Content: SimpleRule, IsEnabled: true, }, }, ExpectedSettings: map[string]string{ - "idstools.rules.local__rules": "\n" + `alert http any any -> any any (msg:"GPL ATTACK_RESPONSE id check returned root"; content:"uid=0|28|root|29|"; classtype:bad-unknown; sid:10000; rev:7; metadata:created_at 2010_09_23, updated_at 2010_09_23;)`, - "idstools.sids.enabled": "\n10000", - "idstools.sids.disabled": "\n# 10000", + "idstools.rules.local__rules": SimpleRule, + "idstools.sids.enabled": SimpleRuleSID, + "idstools.sids.disabled": "# " + SimpleRuleSID, "idstools.sids.modify": "", }, }, { Name: "Disable Existing Simple Rule", InitialSettings: []*model.Setting{ - {Id: "idstools.rules.local__rules", Value: `alert http any any -> any any (msg:"GPL ATTACK_RESPONSE id check returned root"; content:"uid=0|28|root|29|"; classtype:bad-unknown; sid:10000; rev:7; metadata:created_at 2010_09_23, updated_at 2010_09_23;)`}, - {Id: "idstools.sids.enabled", Value: "10000"}, - {Id: "idstools.sids.disabled", Value: "# 10000"}, + {Id: "idstools.rules.local__rules", Value: SimpleRule}, + {Id: "idstools.sids.enabled", Value: SimpleRuleSID}, + {Id: "idstools.sids.disabled", Value: "# " + SimpleRuleSID}, {Id: "idstools.sids.modify"}, }, Detections: []*model.Detection{ { - PublicID: "10000", - Content: `alert http any any -> any any (msg:"GPL ATTACK_RESPONSE id check returned root"; content:"uid=0|28|root|29|"; classtype:bad-unknown; sid:10000; rev:7; metadata:created_at 2010_09_23, updated_at 2010_09_23;)`, + PublicID: SimpleRuleSID, + Content: SimpleRule, IsEnabled: false, }, }, ExpectedSettings: map[string]string{ - "idstools.rules.local__rules": `alert http any any -> any any (msg:"GPL ATTACK_RESPONSE id check returned root"; content:"uid=0|28|root|29|"; classtype:bad-unknown; sid:10000; rev:7; metadata:created_at 2010_09_23, updated_at 2010_09_23;)`, - "idstools.sids.enabled": "# 10000", - "idstools.sids.disabled": "10000", + "idstools.rules.local__rules": SimpleRule, + "idstools.sids.enabled": "# " + SimpleRuleSID, + "idstools.sids.disabled": SimpleRuleSID, "idstools.sids.modify": "", }, }, { Name: "Enable New Flowbits Rule", - InitialSettings: emptySettings, + InitialSettings: emptySettings(), Detections: []*model.Detection{ { - PublicID: "10000", - Content: `alert http any any -> any any ( msg:"RULE A"; flow: established,to_server; http.method; content:"POST"; http.content_type; content:"x-www-form-urlencoded"; flowbits: set, test; sid:10000;)`, + PublicID: FlowbitsRuleASID, + Content: FlowbitsRuleA, IsEnabled: true, }, }, ExpectedSettings: map[string]string{ - "idstools.rules.local__rules": "\n" + `alert http any any -> any any ( msg:"RULE A"; flow: established,to_server; http.method; content:"POST"; http.content_type; content:"x-www-form-urlencoded"; flowbits: set, test; sid:10000;)`, - "idstools.sids.enabled": "\n10000", - "idstools.sids.disabled": "\n# 10000", + "idstools.rules.local__rules": "\n" + FlowbitsRuleA, + "idstools.sids.enabled": "\n" + FlowbitsRuleASID, + "idstools.sids.disabled": "", + "idstools.sids.modify": "", + }, + }, + { + Name: "Disable New Flowbits Rule", + InitialSettings: emptySettings(), + Detections: []*model.Detection{ + { + PublicID: FlowbitsRuleASID, + Content: FlowbitsRuleA, + IsEnabled: false, + }, + }, + ExpectedSettings: map[string]string{ + "idstools.rules.local__rules": "\n" + FlowbitsRuleA, + "idstools.sids.enabled": "\n" + FlowbitsRuleASID, + "idstools.sids.disabled": "", + "idstools.sids.modify": "\n" + FlowbitsRuleASID + ` "flowbits" "noalert; flowbits"`, + }, + }, + { + Name: "Enable Existing Flowbits Rule", + InitialSettings: []*model.Setting{ + {Id: "idstools.rules.local__rules", Value: FlowbitsRuleB}, + {Id: "idstools.sids.enabled", Value: FlowbitsRuleBSID}, + {Id: "idstools.sids.disabled", Value: ""}, + {Id: "idstools.sids.modify", Value: FlowbitsRuleBSID + ` "flowbits" "noalert; flowbits"`}, + }, + Detections: []*model.Detection{ + { + PublicID: FlowbitsRuleBSID, + Content: FlowbitsRuleB, + IsEnabled: true, + }, + }, + ExpectedSettings: map[string]string{ + "idstools.rules.local__rules": FlowbitsRuleB, + "idstools.sids.enabled": FlowbitsRuleBSID, + "idstools.sids.disabled": "", "idstools.sids.modify": "", }, }, { Name: "Disable Existing Flowbits Rule", InitialSettings: []*model.Setting{ - {Id: "idstools.rules.local__rules", Value: `alert http any any -> any any ( msg:"RULE A"; flow: established,to_server; http.method; content:"POST"; http.content_type; content:"x-www-form-urlencoded"; flowbits: set, test; sid:10000;)`}, - {Id: "idstools.sids.enabled", Value: "10000"}, - {Id: "idstools.sids.disabled", Value: "# 10000"}, + {Id: "idstools.rules.local__rules", Value: FlowbitsRuleB}, + {Id: "idstools.sids.enabled", Value: FlowbitsRuleBSID}, + {Id: "idstools.sids.disabled", Value: "# " + FlowbitsRuleBSID}, {Id: "idstools.sids.modify", Value: ""}, }, Detections: []*model.Detection{ { - PublicID: "10000", - Content: `alert http any any -> any any ( msg:"RULE A"; flow: established,to_server; http.method; content:"POST"; http.content_type; content:"x-www-form-urlencoded"; flowbits: set, test; sid:10000;)`, + PublicID: FlowbitsRuleBSID, + Content: FlowbitsRuleB, IsEnabled: false, }, }, ExpectedSettings: map[string]string{ - "idstools.rules.local__rules": `alert http any any -> any any ( msg:"RULE A"; flow: established,to_server; http.method; content:"POST"; http.content_type; content:"x-www-form-urlencoded"; flowbits: set, test; sid:10000;)`, - "idstools.sids.enabled": "10000", - "idstools.sids.disabled": "# 10000", - "idstools.sids.modify": "\n" + `10000 "flowbits" "noalert; flowbits"`, + "idstools.rules.local__rules": FlowbitsRuleB, + "idstools.sids.enabled": FlowbitsRuleBSID, + "idstools.sids.disabled": "# " + FlowbitsRuleBSID, + "idstools.sids.modify": "\n" + FlowbitsRuleBSID + ` "flowbits" "noalert; flowbits"`, + }, + }, + { + Name: "Completely Invalid Rule", + InitialSettings: emptySettings(), + Detections: []*model.Detection{ + { + PublicID: "0", + Content: "x", + IsEnabled: true, + }, + }, + ExpectedErrMap: map[string]string{ + "0": "unable to parse rule; reason=invalid rule, unexpected end of rule", + }, + }, + { + Name: "Rule Missing SID", + InitialSettings: emptySettings(), + Detections: []*model.Detection{ + { + PublicID: "0", + Content: `alert http any any -> any any (msg:"This rule doesn't have a SID";)`, // missing closing paren + IsEnabled: true, + }, + }, + ExpectedErrMap: map[string]string{ + "0": `rule does not contain a SID; rule=alert http any any -> any any (msg:"This rule doesn't have a SID";)`, }, }, } @@ -299,12 +479,12 @@ func TestSyncSuricata(t *testing.T) { assert.Equal(t, test.ExpectedErrMap, errMap) set, err := mCfgStore.GetSettings(ctx) - assert.NoError(t, err) + assert.NoError(t, err, "GetSettings should not return an error") for id, expectedValue := range test.ExpectedSettings { setting := settingByID(set, id) - assert.NotNil(t, setting) - assert.Equal(t, expectedValue, setting.Value) + assert.NotNil(t, setting, "Setting %s", id) + assert.Equal(t, expectedValue, setting.Value, "Setting %s", id) } }) } diff --git a/server/modules/suricata/validate.go b/server/modules/suricata/validate.go index 68ca6731d..aac0d6c66 100644 --- a/server/modules/suricata/validate.go +++ b/server/modules/suricata/validate.go @@ -129,6 +129,11 @@ func ParseSuricataRule(rule string) (*SuricataRule, error) { } } + if curState != stateOptions || len(strings.TrimSpace(buf.String())) != 0 { + // We're unexpectedly done parsing the rule. + return nil, fmt.Errorf("invalid rule, unexpected end of rule") + } + return out, nil } From ea0e6f5b6a9d6c9266b0e0b95eea637d21804006 Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Fri, 29 Sep 2023 09:18:19 -0600 Subject: [PATCH 009/102] WIP: Sync Community Detections Removed permission checks left over from creating ElasticDetectionstore from ElasticCasestore. Jerry rigged the config to always have a `suricataengine` module section so I don't have to fight salt. Expanded on detection severity types. Expanded the DetectionEngine interface to support how we're importing community rules. First pass at a SyncCommunityRules. Solved some issues around ParseSuricataRule and whitespace. --- config/config.go | 3 + model/detection.go | 8 +- server/detectionengine.go | 4 +- server/detectionhandler.go | 58 ++++++- server/detectionstore.go | 1 + .../modules/elastic/elasticdetectionstore.go | 136 +++++++++------- server/modules/suricata/suricata.go | 154 +++++++++++++++++- server/modules/suricata/suricata_test.go | 69 +++++++- server/modules/suricata/validate.go | 50 +++++- server/modules/suricata/validate_test.go | 2 +- 10 files changed, 402 insertions(+), 83 deletions(-) diff --git a/config/config.go b/config/config.go index c0cd1ff18..75ce93534 100644 --- a/config/config.go +++ b/config/config.go @@ -44,5 +44,8 @@ func LoadConfig(filename string, version string, buildTime time.Time) (*Config, err = cfg.Server.Verify() } } + + cfg.Server.Modules["suricataengine"] = map[string]interface{}{} // TODO: remove when put in config file + return cfg, err } diff --git a/model/detection.go b/model/detection.go index 8d3f8c912..4edc9a953 100644 --- a/model/detection.go +++ b/model/detection.go @@ -23,9 +23,11 @@ const ( SigLangYara SigLanguage = "yara" SigLangZeek SigLanguage = "zeek" - SeverityLow Severity = "low" - SeverityMedium Severity = "medium" - SeverityHigh Severity = "high" + SeverityUnknown Severity = "unknown" + SeverityInformational Severity = "informational" + SeverityMinor Severity = "minor" + SeverityMajor Severity = "major" + SeverityCritical Severity = "critical" IDTypeUUID IDType = "uuid" IDTypeSID IDType = "sid" diff --git a/server/detectionengine.go b/server/detectionengine.go index 09d430e1b..63bcf88b8 100644 --- a/server/detectionengine.go +++ b/server/detectionengine.go @@ -8,5 +8,7 @@ import ( type DetectionEngine interface { ValidateRule(rule string) (string, error) - SyncDetections(ctx context.Context, detections []*model.Detection) (errMap map[string]string, err error) + ParseRules(content string) ([]*model.Detection, error) + SyncLocalDetections(ctx context.Context, detections []*model.Detection) (errMap map[string]string, err error) + SyncCommunityDetections(ctx context.Context, detections []*model.Detection) (errMap map[string]string, err error) } diff --git a/server/detectionhandler.go b/server/detectionhandler.go index 4462b3c8c..c3313f1d7 100644 --- a/server/detectionhandler.go +++ b/server/detectionhandler.go @@ -9,6 +9,7 @@ package server import ( "context" "fmt" + "io" "net/http" "strings" @@ -38,6 +39,7 @@ func RegisterDetectionRoutes(srv *Server, r chi.Router, prefix string) { r.Delete("/{id}", h.deleteDetection) r.Post("/bulk/{newStatus}", h.bulkUpdateDetection) + r.Post("/sync/{engine}", h.syncCommunityDetections) }) } @@ -77,7 +79,7 @@ func (h *DetectionHandler) postDetection(w http.ResponseWriter, r *http.Request) return } - errMap, err := SyncDetections(ctx, h.server, []*model.Detection{detect}) + errMap, err := SyncLocalDetections(ctx, h.server, []*model.Detection{detect}) if err != nil { web.Respond(w, r, http.StatusInternalServerError, err) return @@ -137,7 +139,7 @@ func (h *DetectionHandler) putDetection(w http.ResponseWriter, r *http.Request) return } - errMap, err := SyncDetections(ctx, h.server, []*model.Detection{detect}) + errMap, err := SyncLocalDetections(ctx, h.server, []*model.Detection{detect}) if err != nil { web.Respond(w, r, http.StatusInternalServerError, err) return @@ -162,7 +164,7 @@ func (h *DetectionHandler) deleteDetection(w http.ResponseWriter, r *http.Reques return } - errMap, err := SyncDetections(ctx, h.server, []*model.Detection{old}) + errMap, err := SyncLocalDetections(ctx, h.server, []*model.Detection{old}) if err != nil { web.Respond(w, r, http.StatusInternalServerError, err) return @@ -214,7 +216,7 @@ func (h *DetectionHandler) bulkUpdateDetection(w http.ResponseWriter, r *http.Re } if len(modified) != 0 { - addErrMap, err := SyncDetections(ctx, h.server, modified) + addErrMap, err := SyncLocalDetections(ctx, h.server, modified) if err != nil { web.Respond(w, r, http.StatusInternalServerError, err) return @@ -234,7 +236,51 @@ func (h *DetectionHandler) bulkUpdateDetection(w http.ResponseWriter, r *http.Re web.Respond(w, r, http.StatusOK, errMap) } -func SyncDetections(ctx context.Context, srv *Server, detections []*model.Detection) (errMap map[string]string, err error) { +func (h *DetectionHandler) syncCommunityDetections(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + engineParam := chi.URLParam(r, "engine") + + engine, ok := h.server.DetectionEngines[model.EngineName(engineParam)] + if !ok { + web.Respond(w, r, http.StatusBadRequest, fmt.Errorf("invalid engine")) + return + } + + err := r.ParseMultipartForm(int64(h.server.Config.MaxUploadSizeBytes)) + if err != nil { + web.Respond(w, r, http.StatusBadRequest, err) + return + } + + file, _, err := r.FormFile("file") + if err != nil { + web.Respond(w, r, http.StatusBadRequest, err) + return + } + + content, err := io.ReadAll(file) + if err != nil { + web.Respond(w, r, http.StatusBadRequest, err) + return + } + + detections, err := engine.ParseRules(string(content)) + if err != nil { + web.Respond(w, r, http.StatusBadRequest, err) + return + } + + errMap, err := engine.SyncCommunityDetections(ctx, detections) + if err != nil { + web.Respond(w, r, http.StatusInternalServerError, err) + return + } + + web.Respond(w, r, http.StatusOK, errMap) +} + +func SyncLocalDetections(ctx context.Context, srv *Server, detections []*model.Detection) (errMap map[string]string, err error) { defer func() { if len(errMap) == 0 { errMap = nil @@ -248,7 +294,7 @@ func SyncDetections(ctx context.Context, srv *Server, detections []*model.Detect for name, engine := range srv.DetectionEngines { if len(byEngine[name]) != 0 { - eMap, err := engine.SyncDetections(ctx, byEngine[name]) + eMap, err := engine.SyncLocalDetections(ctx, byEngine[name]) for sid, e := range eMap { errMap[sid] = e } diff --git a/server/detectionstore.go b/server/detectionstore.go index 726e41823..de8b5f8fc 100644 --- a/server/detectionstore.go +++ b/server/detectionstore.go @@ -18,4 +18,5 @@ type Detectionstore interface { UpdateDetection(ctx context.Context, detect *model.Detection) (*model.Detection, error) UpdateDetectionField(ctx context.Context, id string, field string, value any) (*model.Detection, bool, error) DeleteDetection(ctx context.Context, detectID string) (*model.Detection, error) + GetAllCommunitySIDs(ctx context.Context) (map[string]string, error) // map[detection.PublicId]detection.Id } diff --git a/server/modules/elastic/elasticdetectionstore.go b/server/modules/elastic/elasticdetectionstore.go index 35f0e6264..34383adc6 100644 --- a/server/modules/elastic/elasticdetectionstore.go +++ b/server/modules/elastic/elasticdetectionstore.go @@ -115,29 +115,29 @@ func (store *ElasticDetectionstore) save(ctx context.Context, obj interface{}, k var results *model.EventIndexResults var err error - if err = store.server.CheckAuthorized(ctx, "write", "cases"); err == nil { - document := convertObjectToDocumentMap(kind, obj, store.schemaPrefix) - document[store.schemaPrefix+"kind"] = kind + // if err = store.server.CheckAuthorized(ctx, "write", "cases"); err == nil { + document := convertObjectToDocumentMap(kind, obj, store.schemaPrefix) + document[store.schemaPrefix+"kind"] = kind - results, err = store.server.Eventstore.Index(ctx, store.index, document, id) - if err == nil { - document[store.schemaPrefix+AUDIT_DOC_ID] = results.DocumentId + results, err = store.server.Eventstore.Index(ctx, store.index, document, id) + if err == nil { + document[store.schemaPrefix+AUDIT_DOC_ID] = results.DocumentId - if id == "" { - document[store.schemaPrefix+"operation"] = "create" - } else { - document[store.schemaPrefix+"operation"] = "update" - } + if id == "" { + document[store.schemaPrefix+"operation"] = "create" + } else { + document[store.schemaPrefix+"operation"] = "update" + } - _, err = store.server.Eventstore.Index(ctx, store.auditIndex, document, "") - if err != nil { - log.WithFields(log.Fields{ - "documentId": results.DocumentId, - "kind": kind, - }).WithError(err).Error("Object indexed successfully however audit record failed to index") - } + _, err = store.server.Eventstore.Index(ctx, store.auditIndex, document, "") + if err != nil { + log.WithFields(log.Fields{ + "documentId": results.DocumentId, + "kind": kind, + }).WithError(err).Error("Object indexed successfully however audit record failed to index") } } + // } return results, err } @@ -145,23 +145,23 @@ func (store *ElasticDetectionstore) save(ctx context.Context, obj interface{}, k func (store *ElasticDetectionstore) delete(ctx context.Context, obj interface{}, kind string, id string) error { var err error - if err = store.server.CheckAuthorized(ctx, "write", "cases"); err == nil { - err = store.server.Eventstore.Delete(ctx, store.index, id) - if err == nil { - document := convertObjectToDocumentMap(kind, obj, store.schemaPrefix) - document[store.schemaPrefix+AUDIT_DOC_ID] = id - document[store.schemaPrefix+"kind"] = kind - document[store.schemaPrefix+"operation"] = "delete" - - _, err = store.server.Eventstore.Index(ctx, store.auditIndex, document, "") - if err != nil { - log.WithFields(log.Fields{ - "documentId": id, - "kind": kind, - }).WithError(err).Error("Object deleted successfully however audit record failed to index") - } + // if err = store.server.CheckAuthorized(ctx, "write", "cases"); err == nil { + err = store.server.Eventstore.Delete(ctx, store.index, id) + if err == nil { + document := convertObjectToDocumentMap(kind, obj, store.schemaPrefix) + document[store.schemaPrefix+AUDIT_DOC_ID] = id + document[store.schemaPrefix+"kind"] = kind + document[store.schemaPrefix+"operation"] = "delete" + + _, err = store.server.Eventstore.Index(ctx, store.auditIndex, document, "") + if err != nil { + log.WithFields(log.Fields{ + "documentId": id, + "kind": kind, + }).WithError(err).Error("Object deleted successfully however audit record failed to index") } } + // } return err } @@ -185,42 +185,42 @@ func (store *ElasticDetectionstore) getAll(ctx context.Context, query string, ma var err error var objects []interface{} - if err = store.server.CheckAuthorized(ctx, "read", "cases"); err == nil { - criteria := model.NewEventSearchCriteria() - format := "2006-01-02 3:04:05 PM" + // if err = store.server.CheckAuthorized(ctx, "read", "cases"); err == nil { + criteria := model.NewEventSearchCriteria() + format := "2006-01-02 3:04:05 PM" - var zeroTime time.Time + var zeroTime time.Time - zeroTimeStr := zeroTime.Format(format) - now := time.Now() - endTime := now.Format(format) - zone := now.Location().String() + zeroTimeStr := zeroTime.Format(format) + now := time.Now() + endTime := now.Format(format) + zone := now.Location().String() + + err = criteria.Populate(query, + zeroTimeStr+" - "+endTime, // timeframe range + format, // timeframe format + zone, // timezone + "0", // no metrics + strconv.Itoa(max)) - err = criteria.Populate(query, - zeroTimeStr+" - "+endTime, // timeframe range - format, // timeframe format - zone, // timezone - "0", // no metrics - strconv.Itoa(max)) + if err == nil { + var results *model.EventSearchResults + results, err = store.server.Eventstore.Search(ctx, criteria) if err == nil { - var results *model.EventSearchResults - - results, err = store.server.Eventstore.Search(ctx, criteria) - if err == nil { - for _, event := range results.Events { - var obj interface{} - - obj, err = convertElasticEventToObject(event, store.schemaPrefix) - if err == nil { - objects = append(objects, obj) - } else { - log.WithField("event", event).WithError(err).Error("Unable to convert case object") - } + for _, event := range results.Events { + var obj interface{} + + obj, err = convertElasticEventToObject(event, store.schemaPrefix) + if err == nil { + objects = append(objects, obj) + } else { + log.WithField("event", event).WithError(err).Error("Unable to convert case object") } } } } + // } return objects, err } @@ -358,3 +358,19 @@ func (store *ElasticDetectionstore) DeleteDetection(ctx context.Context, onionID return detect, err } + +func (store *ElasticDetectionstore) GetAllCommunitySIDs(ctx context.Context) (map[string]string, error) { + // TODO: 10000? Is that enough? + all, err := store.getAll(ctx, fmt.Sprintf(`_index:"%s" AND %skind:"%s"`, store.index, store.schemaPrefix, "detection"), 10000) + if err != nil { + return nil, err + } + + sids := map[string]string{} + for _, det := range all { + detection := det.(*model.Detection) + sids[detection.PublicID] = detection.Id + } + + return sids, nil +} diff --git a/server/modules/suricata/suricata.go b/server/modules/suricata/suricata.go index b9725882d..6c8705c82 100644 --- a/server/modules/suricata/suricata.go +++ b/server/modules/suricata/suricata.go @@ -4,8 +4,10 @@ import ( "context" "fmt" "regexp" + "strconv" "strings" + "github.com/apex/log" "github.com/samber/lo" "github.com/security-onion-solutions/securityonion-soc/model" "github.com/security-onion-solutions/securityonion-soc/module" @@ -57,7 +59,92 @@ func (s *SuricataEngine) ValidateRule(rule string) (string, error) { return parsed.String(), nil } -func (s *SuricataEngine) SyncDetections(ctx context.Context, detections []*model.Detection) (errMap map[string]string, err error) { +func (s *SuricataEngine) ParseRules(content string) ([]*model.Detection, error) { + // expecting one rule per line + lines := strings.Split(content, "\n") + dets := []*model.Detection{} + + for i, line := range lines { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + // empty or commented line, ignore + continue + } + + line, err := s.ValidateRule(line) + if err != nil { + return nil, fmt.Errorf("unable to parse line %d: %w", i+1, err) + } + + parsed, err := ParseSuricataRule(line) + if err != nil { + return nil, fmt.Errorf("unable to parse line %d: %w", i+1, err) + } + + // extract details + sidOpt, ok := parsed.GetOption("sid") + if !ok || sidOpt == nil || len(*sidOpt) == 0 { + return nil, fmt.Errorf("unable to parse line %d: rule does not contain a SID", i+1) + } + + sid, err := strconv.Unquote(*sidOpt) + if err != nil { + sid = *sidOpt + } + + msg := sid + + msgOpt, ok := parsed.GetOption("msg") + if ok && msgOpt != nil && len(*msgOpt) != 0 { + msg = *msgOpt + } + + msg = strings.ReplaceAll(msg, `\;`, `;`) + + title, err := strconv.Unquote(msg) + if err != nil { + title = msg + } + + title = strings.ReplaceAll(title, `\"`, `"`) + title = strings.ReplaceAll(title, `\\`, `\`) + + severity := model.SeverityUnknown // TODO: Default severity? + + md := parsed.ParseMetaData() + if md != nil { + sigsev, ok := lo.Find(md, func(m *MetaData) bool { + return strings.EqualFold(m.Key, "signature_severity") + }) + if ok { + switch strings.ToUpper(sigsev.Value) { + case "INFORMATIONAL": + severity = model.SeverityInformational + case "MINOR": + severity = model.SeverityMinor + case "MAJOR": + severity = model.SeverityMajor + case "CRITICAL": + severity = model.SeverityCritical + } + } + } + + dets = append(dets, &model.Detection{ + PublicID: sid, + Title: title, + Severity: severity, + Content: line, + IsEnabled: true, // is this true? + IsCommunity: true, + Engine: model.EngineNameSuricata, + }) + } + + return dets, nil +} + +func (s *SuricataEngine) SyncLocalDetections(ctx context.Context, detections []*model.Detection) (errMap map[string]string, err error) { defer func() { if len(errMap) == 0 { errMap = nil @@ -210,6 +297,71 @@ func (s *SuricataEngine) SyncDetections(ctx context.Context, detections []*model return errMap, nil } +func (s *SuricataEngine) SyncCommunityDetections(ctx context.Context, detections []*model.Detection) (errMap map[string]string, err error) { + defer func() { + if len(errMap) == 0 { + errMap = nil + } + }() + errMap = map[string]string{} + + results := struct { + Added int + Updated int + Removed int + }{} + + commSIDs, err := s.srv.Detectionstore.GetAllCommunitySIDs(ctx) + if err != nil { + return nil, err + } + + toDelete := map[string]struct{}{} + for sid := range commSIDs { + toDelete[sid] = struct{}{} + } + + for _, detect := range detections { + id, exists := commSIDs[detect.PublicID] + if exists { + detect.Id = id + + _, err = s.srv.Detectionstore.UpdateDetection(ctx, detect) + if err != nil { + errMap[detect.PublicID] = fmt.Sprintf("unable to update detection; reason=%s", err.Error()) + } else { + results.Updated++ + delete(toDelete, detect.PublicID) + } + } else { + _, err = s.srv.Detectionstore.CreateDetection(ctx, detect) + if err != nil { + errMap[detect.PublicID] = fmt.Sprintf("unable to create detection; reason=%s", err.Error()) + } else { + results.Added++ + } + } + } + + for sid := range toDelete { + _, err = s.srv.Detectionstore.DeleteDetection(ctx, sid) + if err != nil { + errMap[sid] = fmt.Sprintf("unable to update detection; reason=%s", err.Error()) + } else { + results.Removed++ + } + } + + log.WithFields(log.Fields{ + "added": results.Added, + "updated": results.Updated, + "removed": results.Removed, + "errors": errMap, + }).Info("Suricata community detections synced") + + return errMap, nil +} + func settingByID(all []*model.Setting, id string) *model.Setting { found, ok := lo.Find(all, func(s *model.Setting) bool { return s.Id == id diff --git a/server/modules/suricata/suricata_test.go b/server/modules/suricata/suricata_test.go index 0a497f305..215adc34a 100644 --- a/server/modules/suricata/suricata_test.go +++ b/server/modules/suricata/suricata_test.go @@ -2,6 +2,7 @@ package suricata import ( "context" + "strings" "testing" "github.com/security-onion-solutions/securityonion-soc/model" @@ -39,13 +40,13 @@ func TestSuricataModule(t *testing.T) { assert.Implements(t, (*server.DetectionEngine)(nil), mod) err := mod.Init(nil) - assert.Nil(t, err) + assert.NoError(t, err) err = mod.Start() - assert.Nil(t, err) + assert.NoError(t, err) err = mod.Stop() - assert.Nil(t, err) + assert.NoError(t, err) assert.Equal(t, 1, len(srv.DetectionEngines)) assert.Same(t, mod, srv.DetectionEngines[model.EngineNameSuricata]) @@ -264,6 +265,66 @@ func TestValidate(t *testing.T) { } } +func TestParse(t *testing.T) { + table := []struct { + Name string + Lines []string + ExpectedDetections []*model.Detection + ExpectedError *string + }{ + { + Name: "Sunny Day Path w/ Edge Cases", + Lines: []string{ + "# Comment", + SimpleRule, + "", + ` alert http any any <> any any (metadata:signature_severity Informational; sid: "20000"; msg:"a \"tricky\"\;\\ msg";)`, + " # " + FlowbitsRuleA, + }, + ExpectedDetections: []*model.Detection{ + { + PublicID: SimpleRuleSID, + Title: `GPL ATTACK_RESPONSE id check returned root`, + Severity: model.SeverityUnknown, + Content: SimpleRule, + IsEnabled: true, + IsCommunity: true, + Engine: model.EngineNameSuricata, + }, + { + PublicID: "20000", + Title: `a "tricky";\ msg`, + Severity: model.SeverityInformational, + Content: `alert http any any <> any any (metadata:signature_severity Informational; sid:"20000"; msg:"a \"tricky\"\;\\ msg";)`, + IsEnabled: true, + IsCommunity: true, + Engine: model.EngineNameSuricata, + }, + }, + }, + } + + mod := NewSuricataEngine(&server.Server{}) + + for _, test := range table { + test := test + t.Run(test.Name, func(t *testing.T) { + t.Parallel() + + data := strings.Join(test.Lines, "\n") + + detections, err := mod.ParseRules(data) + if test.ExpectedError == nil { + assert.NoError(t, err) + assert.Equal(t, test.ExpectedDetections, detections) + } else { + assert.Equal(t, *test.ExpectedError, err.Error()) + assert.Empty(t, detections) + } + }) + } +} + func TestSyncSuricata(t *testing.T) { table := []struct { Name string @@ -473,7 +534,7 @@ func TestSyncSuricata(t *testing.T) { }) mod.srv.DetectionEngines[model.EngineNameSuricata] = mod - errMap, err := mod.SyncDetections(ctx, test.Detections) + errMap, err := mod.SyncLocalDetections(ctx, test.Detections) assert.Equal(t, test.ExpectedErr, err) assert.Equal(t, test.ExpectedErrMap, errMap) diff --git a/server/modules/suricata/validate.go b/server/modules/suricata/validate.go index aac0d6c66..8f9d1b89f 100644 --- a/server/modules/suricata/validate.go +++ b/server/modules/suricata/validate.go @@ -3,7 +3,9 @@ package suricata import ( "fmt" "strings" + "unicode" + "github.com/samber/lo" "github.com/security-onion-solutions/securityonion-soc/util" ) @@ -21,6 +23,11 @@ type RuleOption struct { Value *string } +type MetaData struct { + Key string + Value string +} + type state int const ( @@ -49,19 +56,19 @@ func ParseSuricataRule(rule string) (*SuricataRule, error) { switch curState { case stateAction: - if ch == ' ' { + if ch == ' ' && buf.Len() != 0 { out.Action = strings.TrimSpace(buf.String()) buf.Reset() curState = stateProtocol - } else { + } else if !unicode.IsSpace(ch) { buf.WriteRune(ch) } case stateProtocol: - if ch == ' ' { + if ch == ' ' && buf.Len() != 0 { out.Protocol = strings.TrimSpace(buf.String()) buf.Reset() curState = stateSource - } else { + } else if !unicode.IsSpace(ch) { buf.WriteRune(ch) } case stateSource: @@ -113,7 +120,7 @@ func ParseSuricataRule(rule string) (*SuricataRule, error) { } out.Options = append(out.Options, opt) - } else if ch == '"' { // TODO: current test rule has angled quotes, needs fixing + } else if ch == '"' { buf.WriteRune(ch) if isEscaping { isEscaping = false @@ -124,6 +131,7 @@ func ParseSuricataRule(rule string) (*SuricataRule, error) { isEscaping = true buf.WriteRune(ch) } else { + isEscaping = false buf.WriteRune(ch) } } @@ -139,7 +147,7 @@ func ParseSuricataRule(rule string) (*SuricataRule, error) { func (rule *SuricataRule) GetOption(key string) (value *string, ok bool) { for _, opt := range rule.Options { - if opt.Name == key { + if strings.EqualFold(opt.Name, key) { return opt.Value, true } } @@ -147,10 +155,38 @@ func (rule *SuricataRule) GetOption(key string) (value *string, ok bool) { return nil, false } +func (rule *SuricataRule) ParseMetaData() []*MetaData { + mdOpt, ok := rule.GetOption("metadata") + if !ok || mdOpt == nil { + return nil + } + + md := []*MetaData{} + + parts := strings.Split(*mdOpt, ",") + for _, part := range parts { + part = strings.TrimSuffix(strings.TrimSpace(part), ",") + kv := strings.SplitN(part, " ", 2) + if len(kv) == 1 { + kv = append(kv, "") + } + + md = append(md, &MetaData{Key: strings.TrimSpace(kv[0]), Value: strings.TrimSpace(kv[1])}) + } + + return md +} + func (rule *SuricataRule) String() string { opts := make([]string, 0, len(rule.Options)) + md := rule.ParseMetaData() for _, opt := range rule.Options { - if opt.Value == nil { + if opt.Name == "metadata" && len(md) != 0 { + value := strings.Join(lo.Map(md, func(m *MetaData, _ int) string { + return fmt.Sprintf("%s %s", m.Key, m.Value) + }), ", ") + opts = append(opts, fmt.Sprintf("%s:%s;", opt.Name, value)) + } else if opt.Value == nil { opts = append(opts, fmt.Sprintf("%s;", opt.Name)) } else { opts = append(opts, fmt.Sprintf("%s:%s;", opt.Name, *opt.Value)) diff --git a/server/modules/suricata/validate_test.go b/server/modules/suricata/validate_test.go index 261cc8b4f..1217cf3fa 100644 --- a/server/modules/suricata/validate_test.go +++ b/server/modules/suricata/validate_test.go @@ -114,7 +114,7 @@ func TestSuricataRule(t *testing.T) { assert.NotNil(t, opt) assert.Equal(t, `"9"`, *opt) - opt, ok = rule.GetOption("noalert") + opt, ok = rule.GetOption("NoAlErT") assert.True(t, ok) assert.Nil(t, opt) From 654b4386dd48d328d88649babe6758619d0cac73 Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Fri, 29 Sep 2023 15:07:16 -0600 Subject: [PATCH 010/102] WIP: Community Detections Sync Removed the sync endpoint from detections handler. Instead of so-rule-update attempting to upload the file or trigger a sync, the SuricataEngine module will watch files on disks for changes and respond to them. The DetectionEngine interface was cleaned up in response to this change. All sync logic was moved the SuricataEngine module. The module now maintains a long-lived goroutine that checks a configurable file at a configurable interval for changes and applies them when seen. A fingerprint of the rules file is saved in a configurable location. TODO: Tests. --- config/config.go | 7 +- server/detectionengine.go | 2 - server/detectionhandler.go | 46 ---------- server/modules/suricata/suricata.go | 107 +++++++++++++++++++++-- server/modules/suricata/suricata_test.go | 2 +- 5 files changed, 109 insertions(+), 55 deletions(-) diff --git a/config/config.go b/config/config.go index 75ce93534..9558286a8 100644 --- a/config/config.go +++ b/config/config.go @@ -45,7 +45,12 @@ func LoadConfig(filename string, version string, buildTime time.Time) (*Config, } } - cfg.Server.Modules["suricataengine"] = map[string]interface{}{} // TODO: remove when put in config file + // TODO: remove when put in config file + cfg.Server.Modules["suricataengine"] = map[string]interface{}{ + "communityRulesFile": "/nsm/rules/suricata/emerging-all.rules", + "rulesFingerprintFile": "/opt/so/conf/soc/emerging-all.fingerprint", + "communityRulesImportFrequencySeconds": float64(5), + } return cfg, err } diff --git a/server/detectionengine.go b/server/detectionengine.go index 63bcf88b8..e6f1d4c1a 100644 --- a/server/detectionengine.go +++ b/server/detectionengine.go @@ -8,7 +8,5 @@ import ( type DetectionEngine interface { ValidateRule(rule string) (string, error) - ParseRules(content string) ([]*model.Detection, error) SyncLocalDetections(ctx context.Context, detections []*model.Detection) (errMap map[string]string, err error) - SyncCommunityDetections(ctx context.Context, detections []*model.Detection) (errMap map[string]string, err error) } diff --git a/server/detectionhandler.go b/server/detectionhandler.go index c3313f1d7..7743da32b 100644 --- a/server/detectionhandler.go +++ b/server/detectionhandler.go @@ -9,7 +9,6 @@ package server import ( "context" "fmt" - "io" "net/http" "strings" @@ -39,7 +38,6 @@ func RegisterDetectionRoutes(srv *Server, r chi.Router, prefix string) { r.Delete("/{id}", h.deleteDetection) r.Post("/bulk/{newStatus}", h.bulkUpdateDetection) - r.Post("/sync/{engine}", h.syncCommunityDetections) }) } @@ -236,50 +234,6 @@ func (h *DetectionHandler) bulkUpdateDetection(w http.ResponseWriter, r *http.Re web.Respond(w, r, http.StatusOK, errMap) } -func (h *DetectionHandler) syncCommunityDetections(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() - - engineParam := chi.URLParam(r, "engine") - - engine, ok := h.server.DetectionEngines[model.EngineName(engineParam)] - if !ok { - web.Respond(w, r, http.StatusBadRequest, fmt.Errorf("invalid engine")) - return - } - - err := r.ParseMultipartForm(int64(h.server.Config.MaxUploadSizeBytes)) - if err != nil { - web.Respond(w, r, http.StatusBadRequest, err) - return - } - - file, _, err := r.FormFile("file") - if err != nil { - web.Respond(w, r, http.StatusBadRequest, err) - return - } - - content, err := io.ReadAll(file) - if err != nil { - web.Respond(w, r, http.StatusBadRequest, err) - return - } - - detections, err := engine.ParseRules(string(content)) - if err != nil { - web.Respond(w, r, http.StatusBadRequest, err) - return - } - - errMap, err := engine.SyncCommunityDetections(ctx, detections) - if err != nil { - web.Respond(w, r, http.StatusInternalServerError, err) - return - } - - web.Respond(w, r, http.StatusOK, errMap) -} - func SyncLocalDetections(ctx context.Context, srv *Server, detections []*model.Detection) (errMap map[string]string, err error) { defer func() { if len(errMap) == 0 { diff --git a/server/modules/suricata/suricata.go b/server/modules/suricata/suricata.go index 6c8705c82..f7cf2573f 100644 --- a/server/modules/suricata/suricata.go +++ b/server/modules/suricata/suricata.go @@ -2,10 +2,16 @@ package suricata import ( "context" + "crypto/sha256" + "encoding/hex" "fmt" + "io" + "os" "regexp" "strconv" "strings" + "sync" + "time" "github.com/apex/log" "github.com/samber/lo" @@ -20,7 +26,12 @@ var sidExtracter = regexp.MustCompile(`(?i)\bsid: ?['"]?(.*?)['"]?;`) const modifyFromTo = `"flowbits" "noalert; flowbits"` type SuricataEngine struct { - srv *server.Server + srv *server.Server + communityRulesFile string + rulesFingerprintFile string + communityRulesImportFrequencySeconds int + isRunning bool + thread *sync.WaitGroup } func NewSuricataEngine(srv *server.Server) *SuricataEngine { @@ -33,21 +44,107 @@ func (s *SuricataEngine) PrerequisiteModules() []string { return nil } -func (s *SuricataEngine) Init(config module.ModuleConfig) error { +func (s *SuricataEngine) Init(config module.ModuleConfig) (err error) { + s.communityRulesFile = module.GetStringDefault(config, "communityRulesFile", "/nsm/rules/suricata/emerging-all.rules") + s.rulesFingerprintFile = module.GetStringDefault(config, "rulesFingerprintFile", "/opt/so/conf/soc/emerging-all.fingerprint") + s.communityRulesImportFrequencySeconds = module.GetIntDefault(config, "communityRulesImportFrequencySeconds", 5) + return nil } func (s *SuricataEngine) Start() error { s.srv.DetectionEngines[model.EngineNameSuricata] = s + s.thread = &sync.WaitGroup{} + s.thread.Add(1) + s.isRunning = true + + go s.watchCommunityRules() + return nil } func (s *SuricataEngine) Stop() error { + s.isRunning = false + s.thread.Wait() + return nil } func (s *SuricataEngine) IsRunning() bool { - return false + return s.isRunning +} + +func (s *SuricataEngine) watchCommunityRules() { + defer func() { + s.thread.Done() + s.isRunning = false + }() + + for s.isRunning { + time.Sleep(time.Second * time.Duration(s.communityRulesImportFrequencySeconds)) + if !s.isRunning { + break + } + + rules, hash, err := readAndHash(s.communityRulesFile) + if err != nil { + log.WithError(err).Error("unable to read community rules file") + continue + } + + haveFP := true + + fingerprint, err := os.ReadFile(s.rulesFingerprintFile) + if err != nil { + if !os.IsNotExist(err) { + haveFP = false + } else { + log.WithError(err).Error("unable to read rules fingerprint file") + continue + } + } + + if haveFP && strings.EqualFold(string(fingerprint), hash) { + // if we have a fingerprint and the hashes are equal, there's nothing to do + continue + } + + commDetections, err := s.parseRules(rules) + if err != nil { + log.WithError(err).Error("unable to parse community rules") + continue + } + + errMap, err := s.syncCommunityDetections(context.Background(), commDetections) + if err != nil { + log.WithError(err).Error("unable to sync community detections") + continue + } + + if len(errMap) > 0 { + log.WithFields(log.Fields{ + "errors": errMap, + }).Error("unable to sync all community detections") + } + } +} + +func readAndHash(path string) (content string, sha256Hash string, err error) { + f, err := os.Open(path) + if err != nil { + return "", "", err + } + defer f.Close() + + hasher := sha256.New() + data := io.TeeReader(f, hasher) + + raw, err := io.ReadAll(data) + if err != nil { + return "", "", err + } + + return string(raw), hex.EncodeToString(hasher.Sum(nil)), nil } func (s *SuricataEngine) ValidateRule(rule string) (string, error) { @@ -59,7 +156,7 @@ func (s *SuricataEngine) ValidateRule(rule string) (string, error) { return parsed.String(), nil } -func (s *SuricataEngine) ParseRules(content string) ([]*model.Detection, error) { +func (s *SuricataEngine) parseRules(content string) ([]*model.Detection, error) { // expecting one rule per line lines := strings.Split(content, "\n") dets := []*model.Detection{} @@ -297,7 +394,7 @@ func (s *SuricataEngine) SyncLocalDetections(ctx context.Context, detections []* return errMap, nil } -func (s *SuricataEngine) SyncCommunityDetections(ctx context.Context, detections []*model.Detection) (errMap map[string]string, err error) { +func (s *SuricataEngine) syncCommunityDetections(ctx context.Context, detections []*model.Detection) (errMap map[string]string, err error) { defer func() { if len(errMap) == 0 { errMap = nil diff --git a/server/modules/suricata/suricata_test.go b/server/modules/suricata/suricata_test.go index 215adc34a..d21faafc2 100644 --- a/server/modules/suricata/suricata_test.go +++ b/server/modules/suricata/suricata_test.go @@ -313,7 +313,7 @@ func TestParse(t *testing.T) { data := strings.Join(test.Lines, "\n") - detections, err := mod.ParseRules(data) + detections, err := mod.parseRules(data) if test.ExpectedError == nil { assert.NoError(t, err) assert.Equal(t, test.ExpectedDetections, detections) From 5cb992534928f5d7ee55c83de8ba63930c477fe0 Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Tue, 3 Oct 2023 16:31:41 -0600 Subject: [PATCH 011/102] Service Account Ctx, Community Rules, permissions The context that the server initializes is being put to use and slightly modified. As I find new permissions I need, I'm adding roles to cover them. When the dust settles, I'll re-evaluate. Search criteria now determine what permissions they need to check the user account for. Hints can be given but the search criteria's index and kind are tested to see if alternate permissions need to be checked. detection/read and detection/write were quickly added to the rbac/permissions file. The roles they're attached to has not been finalized and will almost certainly change. Lengthened the allowed title as many community rules have titles longer than 100 chars. When RoundTrip'ing with ElasticTransport, do not add the es-security-runas-user header when the server's agent is in the ctx. Added 2 new test rules to TestValidate inspired by community rules that couldn't parse but should've. Successfully imported 30,000+ community rules! --- config/config.go | 2 +- model/event.go | 38 +++++++++ rbac/permissions | 5 +- .../modules/elastic/elasticdetectionstore.go | 2 +- server/modules/elastic/elasticeventstore.go | 29 ++++--- server/modules/elastic/elastictransport.go | 23 ++++-- .../staticrbac/staticrbacauthorizer.go | 2 + server/modules/suricata/suricata.go | 82 +++++++++++++++---- server/modules/suricata/suricata_test.go | 24 +++++- server/modules/suricata/validate.go | 17 +++- server/server.go | 6 +- 11 files changed, 187 insertions(+), 43 deletions(-) diff --git a/config/config.go b/config/config.go index 9558286a8..f19f83053 100644 --- a/config/config.go +++ b/config/config.go @@ -48,7 +48,7 @@ func LoadConfig(filename string, version string, buildTime time.Time) (*Config, // TODO: remove when put in config file cfg.Server.Modules["suricataengine"] = map[string]interface{}{ "communityRulesFile": "/nsm/rules/suricata/emerging-all.rules", - "rulesFingerprintFile": "/opt/so/conf/soc/emerging-all.fingerprint", + "rulesFingerprintFile": "/tmp/socdev/so/conf/soc/emerging-all.fingerprint", // "/opt/so/conf/soc/emerging-all.fingerprint", "communityRulesImportFrequencySeconds": float64(5), } diff --git a/model/event.go b/model/event.go index e75abc2fc..0f8f50036 100644 --- a/model/event.go +++ b/model/event.go @@ -7,6 +7,7 @@ package model import ( + "regexp" "strconv" "strings" "time" @@ -14,6 +15,9 @@ import ( "github.com/apex/log" ) +var indexExtractor = regexp.MustCompile(`_index:[ \t]?"([^ \t]+)"`) +var kindExtractor = regexp.MustCompile(`so_kind:[ \t]?"([^ \t]+)"`) + type EventResults struct { CreateTime time.Time `json:"createTime"` CompleteTime time.Time `json:"completeTime"` @@ -115,6 +119,40 @@ func (criteria *EventSearchCriteria) Populate(query string, dateRange string, da return err } +func (criteria *EventSearchCriteria) DeterminePermissions(hintVerb string, hintNoun string) (verb string, noun string) { + var index, kind string + + for _, seg := range criteria.ParsedQuery.Segments { + segStr := seg.String() + + indexMatches := indexExtractor.FindStringSubmatch(segStr) + if len(indexMatches) != 0 { + index = indexMatches[1] + } + + kindMatches := kindExtractor.FindStringSubmatch(segStr) + if len(kindMatches) != 0 { + kind = kindMatches[1] + } + + if index != "" && kind != "" { + break + } + } + + index = strings.ToLower(strings.TrimPrefix(strings.TrimSpace(index), "*:")) + kind = strings.ToLower(strings.TrimSpace(kind)) + + switch index { + case "so-detection": + return hintVerb, "detection" + } + + _ = kind + + return hintVerb, hintNoun +} + type EventMetric struct { Keys []interface{} `json:"keys"` Value int `json:"value"` diff --git a/rbac/permissions b/rbac/permissions index 603820686..8869dc270 100644 --- a/rbac/permissions +++ b/rbac/permissions @@ -29,7 +29,10 @@ roles/write: user-admin users/read: user-monitor users/write: user-admin users/delete: user-admin - +detection/read: agent +detection/write: agent +detection/read: event-monitor +detection/write: event-admin # Define low-level permission set inheritence relationships # Syntax => roleB: roleA diff --git a/server/modules/elastic/elasticdetectionstore.go b/server/modules/elastic/elasticdetectionstore.go index 34383adc6..918a5c3b2 100644 --- a/server/modules/elastic/elasticdetectionstore.go +++ b/server/modules/elastic/elasticdetectionstore.go @@ -78,7 +78,7 @@ func (store *ElasticDetectionstore) validateDetection(detect *model.Detection) e } if err == nil && detect.Title != "" { - err = store.validateString(detect.Title, SHORT_STRING_MAX, "title") + err = store.validateString(detect.Title, LONG_STRING_MAX, "title") } if err == nil && detect.Severity != "" { diff --git a/server/modules/elastic/elasticeventstore.go b/server/modules/elastic/elasticeventstore.go index 017826323..0e1ab03d2 100644 --- a/server/modules/elastic/elasticeventstore.go +++ b/server/modules/elastic/elasticeventstore.go @@ -176,21 +176,28 @@ func (store *ElasticEventstore) unmapElasticField(field string) string { func (store *ElasticEventstore) Search(ctx context.Context, criteria *model.EventSearchCriteria) (*model.EventSearchResults, error) { var err error results := model.NewEventSearchResults() - if err = store.server.CheckAuthorized(ctx, "read", "events"); err == nil { - store.refreshCache(ctx) + verb, noun := criteria.DeterminePermissions("read", "events") - var query string - query, err = convertToElasticRequest(store, criteria) + err = store.server.CheckAuthorized(ctx, verb, noun) + if err != nil { + return nil, err + } + + store.refreshCache(ctx) + + var query string + query, err = convertToElasticRequest(store, criteria) + if err == nil { + var response string + response, err = store.luceneSearch(ctx, query) if err == nil { - var response string - response, err = store.luceneSearch(ctx, query) - if err == nil { - err = convertFromElasticResults(store, response, results) - results.Criteria = criteria - } + err = convertFromElasticResults(store, response, results) + results.Criteria = criteria } } + results.Complete() + return results, err } @@ -335,7 +342,9 @@ func (store *ElasticEventstore) indexSearch(ctx context.Context, query string, i "query": store.truncate(query), "requestId": ctx.Value(web.ContextKeyRequestId), }).Info("Searching Elasticsearch") + var json string + res, err := store.esClient.Search( store.esClient.Search.WithContext(ctx), store.esClient.Search.WithIndex(indexes...), diff --git a/server/modules/elastic/elastictransport.go b/server/modules/elastic/elastictransport.go index a4c8c6ace..e5606fb74 100644 --- a/server/modules/elastic/elastictransport.go +++ b/server/modules/elastic/elastictransport.go @@ -14,6 +14,7 @@ import ( "github.com/apex/log" "github.com/security-onion-solutions/securityonion-soc/model" + "github.com/security-onion-solutions/securityonion-soc/server" "github.com/security-onion-solutions/securityonion-soc/web" ) @@ -42,16 +43,20 @@ func NewElasticTransport(user string, pass string, timeoutMs time.Duration, veri func (transport *ElasticTransport) RoundTrip(req *http.Request) (*http.Response, error) { if user, ok := req.Context().Value(web.ContextKeyRequestor).(*model.User); ok { - log.WithFields(log.Fields{ - "username": user.Email, - "searchUsername": user.SearchUsername, - "requestId": req.Context().Value(web.ContextKeyRequestId), - }).Debug("Executing Elastic request on behalf of user") - username := user.Email - if user.SearchUsername != "" { - username = user.SearchUsername + if user.Id != server.AGENT_ID { + log.WithFields(log.Fields{ + "username": user.Email, + "searchUsername": user.SearchUsername, + "requestId": req.Context().Value(web.ContextKeyRequestId), + }).Debug("Executing Elastic request on behalf of user") + username := user.Email + if user.SearchUsername != "" { + username = user.SearchUsername + } + req.Header.Set("es-security-runas-user", username) + } else { + log.Info("Executing Elastic request without es-security-runas-user") } - req.Header.Set("es-security-runas-user", username) } else { log.Warn("User not found in context") } diff --git a/server/modules/staticrbac/staticrbacauthorizer.go b/server/modules/staticrbac/staticrbacauthorizer.go index a1a5463cf..5c1029b90 100644 --- a/server/modules/staticrbac/staticrbacauthorizer.go +++ b/server/modules/staticrbac/staticrbacauthorizer.go @@ -294,6 +294,8 @@ func (impl *StaticRbacAuthorizer) scanNow() { // Ensure agent user/role exists impl.AddRoleToUser(impl.server.Agent, "agent") + impl.AddRoleToUser(impl.server.Agent, "config-admin") + impl.AddRoleToUser(impl.server.Agent, "event-admin") impl.previousUserHash = hash } diff --git a/server/modules/suricata/suricata.go b/server/modules/suricata/suricata.go index f7cf2573f..1b234278a 100644 --- a/server/modules/suricata/suricata.go +++ b/server/modules/suricata/suricata.go @@ -80,31 +80,29 @@ func (s *SuricataEngine) watchCommunityRules() { s.isRunning = false }() + ctx := s.srv.Context + for s.isRunning { time.Sleep(time.Second * time.Duration(s.communityRulesImportFrequencySeconds)) if !s.isRunning { break } + start := time.Now() + rules, hash, err := readAndHash(s.communityRulesFile) if err != nil { log.WithError(err).Error("unable to read community rules file") continue } - haveFP := true - - fingerprint, err := os.ReadFile(s.rulesFingerprintFile) + fingerprint, haveFP, err := readFingerprint(s.rulesFingerprintFile) if err != nil { - if !os.IsNotExist(err) { - haveFP = false - } else { - log.WithError(err).Error("unable to read rules fingerprint file") - continue - } + log.WithError(err).Error("unable to read rules fingerprint file") + continue } - if haveFP && strings.EqualFold(string(fingerprint), hash) { + if haveFP && strings.EqualFold(*fingerprint, hash) { // if we have a fingerprint and the hashes are equal, there's nothing to do continue } @@ -115,7 +113,7 @@ func (s *SuricataEngine) watchCommunityRules() { continue } - errMap, err := s.syncCommunityDetections(context.Background(), commDetections) + errMap, err := s.syncCommunityDetections(ctx, commDetections) if err != nil { log.WithError(err).Error("unable to sync community detections") continue @@ -125,7 +123,18 @@ func (s *SuricataEngine) watchCommunityRules() { log.WithFields(log.Fields{ "errors": errMap, }).Error("unable to sync all community detections") + } else { + err = os.WriteFile(s.rulesFingerprintFile, []byte(hash), 0644) + if err != nil { + log.WithError(err).WithField("path", s.rulesFingerprintFile).Error("unable to write rules fingerprint file") + } } + + dur := time.Since(start) + + log.WithFields(log.Fields{ + "durationSeconds": dur.Seconds(), + }).Info("Suricata community rules synced") } } @@ -147,6 +156,26 @@ func readAndHash(path string) (content string, sha256Hash string, err error) { return string(raw), hex.EncodeToString(hasher.Sum(nil)), nil } +func readFingerprint(path string) (fingerprint *string, ok bool, err error) { + _, err = os.Stat(path) + if err != nil { + if os.IsNotExist(err) { + return nil, false, nil + } + + return nil, false, err + } + + raw, err := os.ReadFile(path) + if err != nil { + return nil, false, err + } + + fingerprint = util.Ptr(strings.TrimSpace(string(raw))) + + return fingerprint, true, nil +} + func (s *SuricataEngine) ValidateRule(rule string) (string, error) { parsed, err := ParseSuricataRule(rule) if err != nil { @@ -279,8 +308,8 @@ func (s *SuricataEngine) SyncLocalDetections(ctx context.Context, detections []* modifyLines := strings.Split(modify.Value, "\n") localIndex := indexLocal(localLines) - enabledIndex := indexEnabled(enabledLines) - disabledIndex := indexEnabled(disabledLines) + enabledIndex := indexEnabled(enabledLines, false) + disabledIndex := indexEnabled(disabledLines, false) modifyIndex := indexModify(modifyLines) errMap = map[string]string{} // map[sid]error @@ -408,6 +437,19 @@ func (s *SuricataEngine) syncCommunityDetections(ctx context.Context, detections Removed int }{} + allSettings, err := s.srv.Configstore.GetSettings(ctx) + if err != nil { + return nil, err + } + + disabled := settingByID(allSettings, "idstools.sids.disabled") + if disabled == nil { + return nil, fmt.Errorf("unable to find disabled setting") + } + + disabledLines := strings.Split(disabled.Value, "\n") + disabledIndex := indexEnabled(disabledLines, true) + commSIDs, err := s.srv.Detectionstore.GetAllCommunitySIDs(ctx) if err != nil { return nil, err @@ -419,6 +461,9 @@ func (s *SuricataEngine) syncCommunityDetections(ctx context.Context, detections } for _, detect := range detections { + _, disabled := disabledIndex[detect.PublicID] + detect.IsEnabled = !disabled + id, exists := commSIDs[detect.PublicID] if exists { detect.Id = id @@ -454,7 +499,7 @@ func (s *SuricataEngine) syncCommunityDetections(ctx context.Context, detections "updated": results.Updated, "removed": results.Removed, "errors": errMap, - }).Info("Suricata community detections synced") + }).Info("suricata community diff") return errMap, nil } @@ -494,11 +539,16 @@ func indexLocal(lines []string) map[string]int { return index } -func indexEnabled(lines []string) map[string]int { +func indexEnabled(lines []string, ignoreComments bool) map[string]int { index := map[string]int{} for i, line := range lines { - line = strings.TrimSpace(strings.TrimLeft(line, "# \t")) + line = strings.TrimSpace(line) + if strings.HasPrefix(line, "#") && ignoreComments { + continue + } + + line = strings.TrimLeft(line, "# \t") if line != "" { index[line] = i } diff --git a/server/modules/suricata/suricata_test.go b/server/modules/suricata/suricata_test.go index d21faafc2..e816e4562 100644 --- a/server/modules/suricata/suricata_test.go +++ b/server/modules/suricata/suricata_test.go @@ -155,7 +155,7 @@ func TestIndexEnabled(t *testing.T) { "# 24adee9b-6010-46ed-9c4a-9cb7a9c972a1", } - output := indexEnabled(lines) + output := indexEnabled(lines, false) assert.Equal(t, 7, len(output)) assert.Contains(t, output, "10000") @@ -172,6 +172,20 @@ func TestIndexEnabled(t *testing.T) { assert.Equal(t, output["not a number"], 6) assert.Contains(t, output, "24adee9b-6010-46ed-9c4a-9cb7a9c972a1") assert.Equal(t, output["24adee9b-6010-46ed-9c4a-9cb7a9c972a1"], 7) + + output = indexEnabled(lines, true) + assert.Equal(t, 4, len(output)) + assert.Contains(t, output, "10000") + assert.Equal(t, output["10000"], 0) + assert.NotContains(t, output, "20000") + assert.Contains(t, output, "30000") + assert.Equal(t, output["30000"], 3) + assert.NotContains(t, output, "40000") + assert.Contains(t, output, "50000") + assert.Equal(t, output["50000"], 5) + assert.Contains(t, output, "not a number") + assert.Equal(t, output["not a number"], 6) + assert.NotContains(t, output, "24adee9b-6010-46ed-9c4a-9cb7a9c972a1") } func TestIndexModify(t *testing.T) { @@ -239,6 +253,14 @@ func TestValidate(t *testing.T) { Input: "x", ExpectedErr: util.Ptr("invalid rule, unexpected end of rule"), }, + { + Name: "Parentheses in Unquoted Option", + Input: `alert http $HOME_NET any -> $EXTERNAL_NET any (msg:"ET ADWARE_PUP WinSoftware.com Spyware User-Agent (WinSoftware)"; flow:to_server,established; http.user_agent; content:"WinSoftware"; nocase; depth:11; reference:url,research.sunbelt-software.com/threatdisplay.aspx?name=WinSoftware%20Corporation%2c%20Inc.%20(v)&threatid=90037; reference:url,doc.emergingthreats.net/2003527; classtype:pup-activity; sid:2003527; rev:12; metadata:attack_target Client_Endpoint, created_at 2010_07_30, deployment Perimeter, former_category ADWARE_PUP, signature_severity Minor, tag Spyware_User_Agent, updated_at 2020_10_13;)`, + }, + { + Name: "Unescaped Double Quote in PCRE Option", + Input: `alert http $EXTERNAL_NET any -> $HOME_NET any (msg:"ET PHISHING Common Unhidebody Function Observed in Phishing Landing"; flow:established,to_client; file.data; content:"function unhideBody()"; nocase; fast_pattern; content:"var bodyElems = document.getElementsByTagName(|22|body|22|)|3b|"; nocase; content:"bodyElems[0].style.visibility =|20 22|visible|22 3b|"; nocase; distance:0; content:"onload=|22|unhideBody()|22|"; content:"method="; nocase; pcre:"/^["']?post/Ri"; classtype:social-engineering; sid:2029732; rev:2; metadata:affected_product Web_Browsers, attack_target Client_Endpoint, created_at 2020_03_24, deployment Perimeter, signature_severity Minor, tag Phishing, updated_at 2020_03_24;)`, + }, } for _, test := range table { diff --git a/server/modules/suricata/validate.go b/server/modules/suricata/validate.go index 8f9d1b89f..f3ee8e707 100644 --- a/server/modules/suricata/validate.go +++ b/server/modules/suricata/validate.go @@ -24,7 +24,7 @@ type RuleOption struct { } type MetaData struct { - Key string + Key string Value string } @@ -101,7 +101,7 @@ func ParseSuricataRule(rule string) (*SuricataRule, error) { buf.WriteRune(ch) } case stateOptions: - if ch == ')' && !inQuotes && !isEscaping { + if ch == ')' && !inQuotes && !isEscaping && len(strings.TrimSpace(buf.String())) == 0 { if r.Len() != 0 { // end of options, but not end of rule? return nil, fmt.Errorf("invalid rule, expected end of rule, got %d more bytes", r.Len()) @@ -125,7 +125,18 @@ func ParseSuricataRule(rule string) (*SuricataRule, error) { if isEscaping { isEscaping = false } else { - inQuotes = !inQuotes + if strings.Contains(buf.String(), "pcre:") { + // is the current option a regular expression? + // if so, only end the quotes if the next character is a semicolon + next, _, _ := r.ReadRune() + r.UnreadRune() + + if next == ';' { + inQuotes = false + } + } else { + inQuotes = !inQuotes + } } } else if ch == '\\' { isEscaping = true diff --git a/server/server.go b/server/server.go index 4b281b5e5..9a41e1a77 100644 --- a/server/server.go +++ b/server/server.go @@ -63,7 +63,11 @@ func (server *Server) initContext() { server.Agent = model.NewUser() server.Agent.Id = AGENT_ID server.Agent.Email = server.Agent.Id - server.Context = context.WithValue(context.Background(), web.ContextKeyRequestor, server.Agent) + + ctx := context.WithValue(context.Background(), web.ContextKeyRequestor, server.Agent) + ctx = context.WithValue(ctx, web.ContextKeyRequestorId, AGENT_ID) + + server.Context = ctx } func (server *Server) Start() { From c4c66fb913040763bafc5e724d68016c73a7389f Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Thu, 5 Oct 2023 12:23:43 -0600 Subject: [PATCH 012/102] WIP: GetAll that gets all Uses elasticsearch's `search_after` to page through results when the max documents of -1 is passed in. --- model/event.go | 2 + server/modules/elastic/converter.go | 14 ++++ .../modules/elastic/elasticdetectionstore.go | 72 ++++++++++++++----- server/modules/suricata/suricata.go | 28 ++++++-- 4 files changed, 93 insertions(+), 23 deletions(-) diff --git a/model/event.go b/model/event.go index 0f8f50036..9f9306e8c 100644 --- a/model/event.go +++ b/model/event.go @@ -66,6 +66,7 @@ type EventSearchCriteria struct { CreateTime time.Time ParsedQuery *Query SortFields []*SortCriteria + SearchAfter []interface{} } func (criteria *EventSearchCriteria) initSearchCriteria() { @@ -166,6 +167,7 @@ type EventRecord struct { Type string `json:"type"` Score float64 `json:"score"` Payload map[string]interface{} `json:"payload"` + Sort []interface{} `json:"sort"` } type EventUpdateCriteria struct { diff --git a/server/modules/elastic/converter.go b/server/modules/elastic/converter.go index 8b8fbeaf5..549e42e3e 100644 --- a/server/modules/elastic/converter.go +++ b/server/modules/elastic/converter.go @@ -196,6 +196,10 @@ func convertToElasticRequest(store *ElasticEventstore, criteria *model.EventSear esMap["size"] = criteria.EventLimit esMap["query"] = makeQuery(store, criteria.ParsedQuery, criteria.BeginTime, criteria.EndTime) + if len(criteria.SearchAfter) != 0 { + esMap["search_after"] = criteria.SearchAfter + } + aggregations := make(map[string]interface{}) if criteria.MetricLimit > 0 { @@ -244,6 +248,13 @@ func convertToElasticRequest(store *ElasticEventstore, criteria *model.EventSear } esMap["sort"] = sorting } + } else { + sort := map[string]string{} + for _, field := range criteria.SortFields { + sort[field.Field] = field.Order + } + + esMap["sort"] = sort } bytes, err := json.WriteJson(esMap) @@ -342,6 +353,9 @@ func convertFromElasticResults(store *ElasticEventstore, esJson string, results event.Score = esRecord["_score"].(float64) } event.Payload = flatten(store, esRecord["_source"].(map[string]interface{})) + if esRecord["sort"] != nil { + event.Sort = esRecord["sort"].([]interface{}) + } if event.Payload["@timestamp"] != nil { event.Time, _ = time.Parse(time.RFC3339, event.Payload["@timestamp"].(string)) diff --git a/server/modules/elastic/elasticdetectionstore.go b/server/modules/elastic/elasticdetectionstore.go index 918a5c3b2..c2bd36003 100644 --- a/server/modules/elastic/elasticdetectionstore.go +++ b/server/modules/elastic/elasticdetectionstore.go @@ -196,30 +196,65 @@ func (store *ElasticDetectionstore) getAll(ctx context.Context, query string, ma endTime := now.Format(format) zone := now.Location().String() - err = criteria.Populate(query, - zeroTimeStr+" - "+endTime, // timeframe range - format, // timeframe format - zone, // timezone - "0", // no metrics - strconv.Itoa(max)) + unlimited := false + if max == -1 { + max = 10000 + unlimited = true + } + + sort := []interface{}{} + + for { + err = criteria.Populate(query, + zeroTimeStr+" - "+endTime, // timeframe range + format, // timeframe format + zone, // timezone + "0", // no metrics + strconv.Itoa(max)) + + if err != nil { + return nil, err + } + + if unlimited { + // need a deterministic sort order for paging + criteria.SortFields = []*model.SortCriteria{ + { + Field: "@timestamp", + Order: "desc", + }, + } + + if len(sort) != 0 { + criteria.SearchAfter = sort + } + } - if err == nil { var results *model.EventSearchResults results, err = store.server.Eventstore.Search(ctx, criteria) - if err == nil { - for _, event := range results.Events { - var obj interface{} - - obj, err = convertElasticEventToObject(event, store.schemaPrefix) - if err == nil { - objects = append(objects, obj) - } else { - log.WithField("event", event).WithError(err).Error("Unable to convert case object") - } + if err != nil { + return nil, err + } + + for _, event := range results.Events { + var obj interface{} + + obj, err = convertElasticEventToObject(event, store.schemaPrefix) + if err == nil { + objects = append(objects, obj) + } else { + log.WithField("event", event).WithError(err).Error("Unable to convert case object") } } + + if !unlimited || len(results.Events) == 0 { + break + } + + sort = results.Events[len(results.Events)-1].Sort } + // } return objects, err @@ -360,8 +395,7 @@ func (store *ElasticDetectionstore) DeleteDetection(ctx context.Context, onionID } func (store *ElasticDetectionstore) GetAllCommunitySIDs(ctx context.Context) (map[string]string, error) { - // TODO: 10000? Is that enough? - all, err := store.getAll(ctx, fmt.Sprintf(`_index:"%s" AND %skind:"%s"`, store.index, store.schemaPrefix, "detection"), 10000) + all, err := store.getAll(ctx, fmt.Sprintf(`_index:"%s" AND %skind:"%s"`, store.index, store.schemaPrefix, "detection"), -1) if err != nil { return nil, err } diff --git a/server/modules/suricata/suricata.go b/server/modules/suricata/suricata.go index 1b234278a..0b641e706 100644 --- a/server/modules/suricata/suricata.go +++ b/server/modules/suricata/suricata.go @@ -25,6 +25,8 @@ var sidExtracter = regexp.MustCompile(`(?i)\bsid: ?['"]?(.*?)['"]?;`) const modifyFromTo = `"flowbits" "noalert; flowbits"` +var errModuleStopped = fmt.Errorf("module has stopped running") + type SuricataEngine struct { srv *server.Server communityRulesFile string @@ -115,6 +117,11 @@ func (s *SuricataEngine) watchCommunityRules() { errMap, err := s.syncCommunityDetections(ctx, commDetections) if err != nil { + if err == errModuleStopped { + log.Info("incomplete sync of suricata community detections due to module stopping") + return + } + log.WithError(err).Error("unable to sync community detections") continue } @@ -134,7 +141,7 @@ func (s *SuricataEngine) watchCommunityRules() { log.WithFields(log.Fields{ "durationSeconds": dur.Seconds(), - }).Info("Suricata community rules synced") + }).Info("suricata community rules synced") } } @@ -447,8 +454,16 @@ func (s *SuricataEngine) syncCommunityDetections(ctx context.Context, detections return nil, fmt.Errorf("unable to find disabled setting") } + modify := settingByID(allSettings, "idstools.sids.modify") + if modify == nil { + return nil, fmt.Errorf("unable to find modify setting") + } + disabledLines := strings.Split(disabled.Value, "\n") + modifyLines := strings.Split(modify.Value, "\n") + disabledIndex := indexEnabled(disabledLines, true) + modifyIndex := indexModify(modifyLines) commSIDs, err := s.srv.Detectionstore.GetAllCommunitySIDs(ctx) if err != nil { @@ -461,8 +476,13 @@ func (s *SuricataEngine) syncCommunityDetections(ctx context.Context, detections } for _, detect := range detections { + if !s.isRunning { + return errMap, errModuleStopped + } + _, disabled := disabledIndex[detect.PublicID] - detect.IsEnabled = !disabled + _, modified := modifyIndex[detect.PublicID] + detect.IsEnabled = !(disabled || modified) id, exists := commSIDs[detect.PublicID] if exists { @@ -486,9 +506,9 @@ func (s *SuricataEngine) syncCommunityDetections(ctx context.Context, detections } for sid := range toDelete { - _, err = s.srv.Detectionstore.DeleteDetection(ctx, sid) + _, err = s.srv.Detectionstore.DeleteDetection(ctx, commSIDs[sid]) if err != nil { - errMap[sid] = fmt.Sprintf("unable to update detection; reason=%s", err.Error()) + errMap[sid] = fmt.Sprintf("unable to delete detection; reason=%s", err.Error()) } else { results.Removed++ } From 77ecfb0ba5b594edc4e631933d97c0406ea90f31 Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Wed, 11 Oct 2023 15:48:18 -0600 Subject: [PATCH 013/102] WIP: Tightening Suricata Bolts Added logic to disallow editing of community rules. Adjusted detection's form validation rules so that community rules aren't validated at the form level. Removed "Details" Detection tab and reordered the controls under it to the "Summary" and "Signature" tabs. Rearranged some tabs. Cleaned up the strings used in dropdowns (engine, severity) with capitalization. Cleaner UI, but the correct casing is still passed over the wire. Refactored PublicID's "Generate" button to be "Extract" so that it fills the field using the SID in the rule. May eventually make this field readonly except for this modification. Updated infohandler's response to use the same severity values the rules use. Converted more strings to i18n references. `indexDocument` and `deleteDocument` on the ElasticEventstore now pass the ctx in. This removes the WARN log message about making a request without a user. Needs more thorough testing to be sure there's no unwanted side effects. --- html/index.html | 86 ++++++++------------- html/js/i18n.js | 17 +++- html/js/routes/detection.js | 81 ++++++++++++++----- server/infohandler.go | 2 +- server/modules/elastic/converter.go | 2 +- server/modules/elastic/elasticeventstore.go | 6 +- server/modules/suricata/suricata.go | 2 + 7 files changed, 118 insertions(+), 78 deletions(-) diff --git a/html/index.html b/html/index.html index e2d9c7902..d10c0ae23 100644 --- a/html/index.html +++ b/html/index.html @@ -965,7 +965,7 @@

{{ i18n.eventTotal }} {{ totalEvents.toLocaleString() }}

{{ detect.title }}

- +
@@ -975,10 +975,10 @@

- + - + @@ -1018,18 +1018,14 @@

Summary

- - fa-circle-info -
Details
+ + fa-signature +
Signature
 fa-pencil
Notes
 - - fa-signature -
Signature
- - Description:
+ {{i18n.description}}:
{{ detect.description }} - +

- + + + + + - + - +
{{i18n.duplicate}} - + {{i18n.delete}}
- -
+ +
- -