Skip to content

Commit

Permalink
[FIlebeat] add strict_date_optional_time_nanos date format to PanOS m…
Browse files Browse the repository at this point in the history
…odule (#26158) (#26525)

* #26033: add strict_date_optional_time_nanos format

* update changelog

* added new sample log, need to check CSV parsing

* update sample data

Co-authored-by: Marius Iversen <marius.iversen@elastic.co>

Co-authored-by: Alex Resnick <adr8292@gmail.com>
Co-authored-by: Marius Iversen <marius.iversen@elastic.co>
  • Loading branch information
3 people committed Jun 29, 2021
1 parent 86352bf commit 9c89a29
Show file tree
Hide file tree
Showing 6 changed files with 130 additions and 16 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.next.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -594,6 +594,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
- Add `include_s3_metadata` config option to the `aws-s3` input for including object metadata in events. {pull}26267[26267]
- RFC 5424 and UNIX socket support in the Syslog input are now GA {pull}26293[26293]
- Update grok patterns for HA Proxy module {issue}25827[25827] {pull}25835[25835]
- Update PanOS module's date processor formats to parse `strict_date_optional_time_nanos`. {issue}26033[26033] {pull}26158[26158]
- Update Okta module to parse additional fields to `okta.debug_context.debug_data`. {issue}25689[25689] {pull}25818[25818]
- Added dataset `anomalithreatstream` to the `threatintel` module to ingest indicators from Anomali ThreatStream {pull}26350[26350]

Expand Down
6 changes: 6 additions & 0 deletions x-pack/filebeat/module/panw/panos/ingest/pipeline.yml
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,14 @@ processors:
field: "_temp_.generated_time"
formats:
- "yyyy/MM/dd HH:mm:ss"
- "strict_date_optional_time_nanos"
on_failure: [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}]
- date:
if: "ctx.event.timezone != null"
field: "_temp_.generated_time"
formats:
- "yyyy/MM/dd HH:mm:ss"
- "strict_date_optional_time_nanos"
timezone: "{{ event.timezone }}"
on_failure: [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}]

Expand All @@ -39,13 +41,15 @@ processors:
target_field: "event.created"
formats:
- "yyyy/MM/dd HH:mm:ss"
- "strict_date_optional_time_nanos"
on_failure: [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}]
- date:
if: "ctx.event.timezone != null && ctx.event.created != null "
field: "event.created"
target_field: "event.created"
formats:
- "yyyy/MM/dd HH:mm:ss"
- "strict_date_optional_time_nanos"
timezone: "{{ event.timezone }}"
on_failure: [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}]

Expand All @@ -56,6 +60,7 @@ processors:
target_field: "event.start"
formats:
- "yyyy/MM/dd HH:mm:ss"
- "strict_date_optional_time_nanos"
on_failure: [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}]
- date:
if: "ctx.event.timezone != null && ctx.event.start != null"
Expand All @@ -64,6 +69,7 @@ processors:
timezone: "{{ event.timezone }}"
formats:
- "yyyy/MM/dd HH:mm:ss"
- "strict_date_optional_time_nanos"
on_failure: [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}]

# convert integer fields as the output of the CSV processor is always a string.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -89,8 +89,8 @@
"panw.panos.type": "GLOBALPROTECT",
"panw.panos.virtual_sys": "vsys1",
"related.hosts": [
"GlobalProtect_GW",
"CP935"
"CP935",
"GlobalProtect_GW"
],
"related.ip": [
"10.20.13.217",
Expand Down Expand Up @@ -368,8 +368,8 @@
"source.nat.ip": "10.20.30.40",
"source.user.name": "maxmustermann",
"tags": [
"pan-os",
"forwarded"
"forwarded",
"pan-os"
],
"user.name": "maxmustermann"
},
Expand Down Expand Up @@ -432,8 +432,8 @@
"source.user.domain": "domain",
"source.user.name": "musterman",
"tags": [
"pan-os",
"forwarded"
"forwarded",
"pan-os"
],
"user.domain": "domain",
"user.name": "musterman"
Expand Down Expand Up @@ -493,8 +493,8 @@
"source.user.domain": "domain.de",
"source.user.name": "Max.Mustermann",
"tags": [
"pan-os",
"forwarded"
"forwarded",
"pan-os"
],
"user.domain": "domain.de",
"user.name": "Max.Mustermann"
Expand Down Expand Up @@ -559,8 +559,8 @@
"source.user.domain": "domain",
"source.user.name": "maxmustermann",
"tags": [
"pan-os",
"forwarded"
"forwarded",
"pan-os"
],
"user.domain": "domain",
"user.name": "maxmustermann"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,8 @@
"panw.panos.virtual_sys": "vsys1",
"panw.panos.vsys_id": "1",
"related.hosts": [
"de-firewall",
"PC12345"
"PC12345",
"de-firewall"
],
"related.ip": [
"10.20.30.40"
Expand All @@ -48,8 +48,8 @@
"source.user.domain": "domain",
"source.user.name": "mustermanm",
"tags": [
"pan-os",
"forwarded"
"forwarded",
"pan-os"
],
"user.domain": "domain",
"user.name": "mustermanm"
Expand Down Expand Up @@ -113,8 +113,8 @@
"source.ip": "67.240.185.235",
"source.user.name": "ira",
"tags": [
"pan-os",
"forwarded"
"forwarded",
"pan-os"
],
"user.name": "ira"
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Oct 30 09:46:42 1,2021-05-26T16:27:07.000000Z,no-serial,TRAFFIC,end,9.1,2021-05-26T16:26:47.000000Z,127.0.0.0,127.0.0.1,0.0.0.0,0.0.0.0,intrazone-default,,,web-browsing,vsys1,untrust,untrust,ethernet1/1,ethernet1/1,Cortex Data Lake,,688290,1,35834,443,35834,20077,0x1400070,tcp,allow,7291,1696,5595,21,2021-05-26T16:26:30.000000Z,1,medium-risk,,620386,0x8800000000000000,US,SG,,14,7,tcp-fin,22,18,0,0,,GP cloud service,from-policy,,,0,,0,1970-01-01T00:00:00.000000Z,N/A,0,0,0,0,6a2f6161-88f2-4afc-8dd5-256bc4505a64,0,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
[
{
"@timestamp": "2021-05-26T16:26:47.000Z",
"client.bytes": 1696,
"client.ip": "127.0.0.0",
"client.nat.ip": "0.0.0.0",
"client.nat.port": 35834,
"client.packets": 14,
"client.port": 35834,
"destination.address": "127.0.0.1",
"destination.bytes": 5595,
"destination.ip": "127.0.0.1",
"destination.nat.ip": "0.0.0.0",
"destination.nat.port": 20077,
"destination.packets": 7,
"destination.port": 443,
"event.action": "flow_terminated",
"event.category": [
"network",
"network_traffic"
],
"event.dataset": "panw.panos",
"event.duration": 1000000000,
"event.end": "2021-05-26T16:26:31.000Z",
"event.kind": "event",
"event.module": "panw",
"event.outcome": "success",
"event.start": "2021-05-26T16:26:30.000Z",
"event.timezone": "-02:00",
"event.type": [
"allowed",
"connection",
"end"
],
"fileset.name": "panos",
"input.type": "log",
"labels.nat_translated": true,
"labels.ssl_decrypted": true,
"log.offset": 0,
"log.original": "Oct 30 09:46:42 1,2021-05-26T16:27:07.000000Z,no-serial,TRAFFIC,end,9.1,2021-05-26T16:26:47.000000Z,127.0.0.0,127.0.0.1,0.0.0.0,0.0.0.0,intrazone-default,,,web-browsing,vsys1,untrust,untrust,ethernet1/1,ethernet1/1,Cortex Data Lake,,688290,1,35834,443,35834,20077,0x1400070,tcp,allow,7291,1696,5595,21,2021-05-26T16:26:30.000000Z,1,medium-risk,,620386,0x8800000000000000,US,SG,,14,7,tcp-fin,22,18,0,0,,GP cloud service,from-policy,,,0,,0,1970-01-01T00:00:00.000000Z,N/A,0,0,0,0,6a2f6161-88f2-4afc-8dd5-256bc4505a64,0,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,",
"network.application": "web-browsing",
"network.bytes": 7291,
"network.community_id": [
"1:lME0D6scndGsx6dABDTbtWkIb3E=",
"1:q5HHCKGDtoHfI//AqHbOlmLMsRQ="
],
"network.direction": "external",
"network.packets": 21,
"network.transport": "tcp",
"network.type": "ipv4",
"observer.egress.interface.name": "ethernet1/1",
"observer.egress.zone": "untrust",
"observer.hostname": "GP cloud service",
"observer.ingress.interface.name": "ethernet1/1",
"observer.ingress.zone": "untrust",
"observer.product": "PAN-OS",
"observer.serial_number": "no-serial",
"observer.type": "firewall",
"observer.vendor": "Palo Alto Networks",
"panw.panos.action": "allow",
"panw.panos.destination.interface": "ethernet1/1",
"panw.panos.destination.nat.ip": "0.0.0.0",
"panw.panos.destination.nat.port": 20077,
"panw.panos.destination.zone": "untrust",
"panw.panos.endreason": "tcp-fin",
"panw.panos.flow_id": "688290",
"panw.panos.network.nat.community_id": "1:lME0D6scndGsx6dABDTbtWkIb3E=",
"panw.panos.ruleset": "intrazone-default",
"panw.panos.sequence_number": 620386,
"panw.panos.source.interface": "ethernet1/1",
"panw.panos.source.nat.ip": "0.0.0.0",
"panw.panos.source.nat.port": 35834,
"panw.panos.source.zone": "untrust",
"panw.panos.sub_type": "end",
"panw.panos.type": "TRAFFIC",
"panw.panos.url.category": "medium-risk",
"panw.panos.virtual_sys": "vsys1",
"related.hosts": [
"GP cloud service"
],
"related.ip": [
"0.0.0.0",
"127.0.0.0",
"127.0.0.1"
],
"rule.name": "intrazone-default",
"server.bytes": 5595,
"server.ip": "127.0.0.1",
"server.nat.ip": "0.0.0.0",
"server.nat.port": 20077,
"server.packets": 7,
"server.port": 443,
"service.type": "panw",
"source.address": "127.0.0.0",
"source.bytes": 1696,
"source.ip": "127.0.0.0",
"source.nat.ip": "0.0.0.0",
"source.nat.port": 35834,
"source.packets": 14,
"source.port": 35834,
"tags": [
"forwarded",
"pan-os"
]
}
]

0 comments on commit 9c89a29

Please sign in to comment.