elastic · adriansr · Feb 1, 2022 · Jan 11, 2022 · Jan 26, 2022 · Jan 26, 2022
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -175,6 +175,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add support in httpjson input for oAuth2ProviderDefault of password grant_type. {pull}29087[29087]
 - Add support for filtering in journald input with `unit`, `kernel`, `identifiers` and `include_matches`. {pull}29294[29294]
 - Add new `userAgent` and `beatInfo` template functions for httpjson input {pull}29528[29528]
+- threatintel module: Add new Recorded Future integration. {pull}30030[30030]
 
 *Heartbeat*
 

diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc
@@ -151873,157 +151873,34 @@ Fields for Recorded Future Threat Intel
 
 
 
-[float]
-=== entity
-
-Entity that represents a threat.
-
-
-
-*`recordedfuture.entity.id`*::
-+
---
-Entity ID.
-
-
-type: keyword
-
-example: ip:192.0.2.13
-
---
-
-*`recordedfuture.entity.name`*::
-+
---
-Entity name. Value for the entity.
-
-
-type: keyword
-
-example: 192.0.2.13
-
---
-
-*`recordedfuture.entity.type`*::
-+
---
-Entity type.
-
-
-type: keyword
-
-example: IpAddress
-
---
-
-*`recordedfuture.intelCard`*::
-+
---
-Link to the Recorded Future Intelligence Card for to this indicator.
-
-
-type: keyword
-
---
-
-*`recordedfuture.ip_range`*::
-+
---
-Range of IPs for this indicator.
-
-
-type: ip_range
-
-example: 192.0.2.0/16
-
---
-
-[float]
-=== risk
-
-Risk fields.
-
-
-
-*`recordedfuture.risk.criticality`*::
+*`recordedfuture.evidence_details`*::
 +
 --
-Risk criticality (0-4).
-
-
-type: byte
-
---
-
-*`recordedfuture.risk.criticalityLabel`*::
-+
---
-Risk criticality label. One of None, Unusual, Suspicious, Malicious, Very Malicious.
-
-
-type: keyword
-
---
-
-*`recordedfuture.risk.evidenceDetails`*::
-+
---
-Risk's evidence details.
+List of sightings used as evidence for this indicator.
 
 
 type: flattened
 
 --
 
-*`recordedfuture.risk.score`*::
-+
---
-Risk score (0-99).
-
-
-type: short
-
---
-
-*`recordedfuture.risk.riskString`*::
+*`recordedfuture.name`*::
 +
 --
-Number of Risk Rules observed as a factor of total number of rules.
+Indicator value.
 
 
 type: keyword
 
-example: 1/54
-
 --
 
-*`recordedfuture.risk.riskSummary`*::
+*`recordedfuture.risk_string`*::
 +
 --
-Risk summary.
+Details of risk rules observed.
 
 
 type: keyword
 
-example: 1 of 54 Risk Rules currently observed.
-
---
-
-*`recordedfuture.risk.riskSummary.text`*::
-+
---
-type: text
-
---
-
-*`recordedfuture.risk.rules`*::
-+
---
-Number of rules observed.
-
-
-type: long
-
 --
 
 [float]

diff --git a/filebeat/docs/modules/threatintel.asciidoc b/filebeat/docs/modules/threatintel.asciidoc
@@ -519,14 +519,12 @@ Anomali ThreatStream fields are mapped to the following ECS fields:
 [float]
 ==== `recordedfuture` fileset settings
 
-The `recordedfuture` fileset fetches intelligence from the Recorded Future Connect API.
-It supports `domain`, `hash`, `ip` and `url` data types.
+The `recordedfuture` fileset fetches risklists from the Recorded Future Connect API.
+It supports `domain`, `hash`, `ip` and `url` entities.
 
-To enable it you need to define the URL to fetch data from. You can construct this URL
-using the https://api.recordedfuture.com/index.html[Recorded Future API Explorer.] The URL
-must point to the `/search` endpoint and contain a suitable `limit`
-(how many records to return from a single request) and `fields` parameters.
-The `entity` and `timestamps` fields are required.
+In order to use it you need to define the `entity` and `list` to fetch. Check the
+https://api.recordedfuture.com/index.html[Recorded Future API Explorer] for the
+available lists for each entity.
 
 Sample configuration:
 [source,yaml]
@@ -535,56 +533,92 @@ Sample configuration:
   recordedfuture:
     enabled: true
     var.input: httpjson
-    var.interval: 5m
-    var.first_interval: 168h
-    var.url: "https://api.recordedfuture.com/v2/ip/search?limit=200&fields=entity,timestamps,risk,intelCard,location&metadata=false"
+    var.interval: 1h
     var.api_token: "<RF_TOKEN>"
+    var.list: default
+    var.entity: domain
 ----
 
-To fetch threat intelligence from multiple data types, you must define more than
-one instance of the module:
+To fetch threat intelligence from multiple entities and/or lists, you must define more
+than one instance of the module. The following configuration fetches the default list
+for domains every hour and the rfTrending list for hashes every 12 hours:
+
 [source,yaml]
 ----
 - module: threatintel
   recordedfuture:
     enabled: true
     var.input: httpjson
-    var.interval: 5m
-    var.first_interval: 168h
-    var.url: "https://api.recordedfuture.com/v2/ip/search?limit=200&fields=entity,timestamps,risk,intelCard,location&metadata=false"
+    var.interval: 1h
     var.api_token: "<RF_TOKEN>"
+    var.list: default
+    var.entity: domain
 - module: threatintel
   recordedfuture:
     enabled: true
     var.input: httpjson
-    var.interval: 1m
-    var.first_interval: 168h
-    var.url: "https://api.recordedfuture.com/v2/hash/search?limit=200&fields=entity,fileHashes,timestamps,risk,intelCard,location&metadata=false"
+    var.interval: 12h
     var.api_token: "<RF_TOKEN>"
+    var.entity: hash
+    var.list: rfTrending
 ----
 
-*`var.url`*::
+Alternatively, you can use the module to fetch custom Fusion files by setting
+`var.custom_url` to the URL of the Fusion File:
 
-The URL of the API endpoint to connect with.
+[source,yaml]
+----
+- module: threatintel
+  recordedfuture:
+    enabled: true
+    var.input: httpjson
+    var.interval: 1h
+    var.api_token: "<RF_TOKEN>"
+    var.custom_url: 'https://api.recordedfuture.com/v2/fusion/files/?path=%2Fpublic%2Frisklists%2Fdefault_domain_risklist.csv'
+----
+
+It's also possible to load CSV risklists from a file:
+
+[source,yaml]
+----
+- module: threatintel
+  recordedfuture:
+    enabled: true
+    var.input: file
+    var.paths:
+        - /path/to/risklist.csv
+----
+
+*`var.input`*::
+
+The input to use to fetch indicators. Use `httpjson` to query
+Recorded Future API or `file` to load the indicators from a file.
 
 *`var.api_token`*::
 
-The API token used to access Recorded Future API.
+The API token used to access Recorded Future API (RF-Token).
 
 *`var.interval`*::
 
-How often the API is polled for updated information.
+How often the API is polled for updated information. It is recommended to set this
+to `1h`. For `hash` entities, it's recommended to set this to `12h`.
 
-*`var.first_interval`*::
+*`var.entity`*::
 
-How far back to search when retrieving events the first time {beatname_uc} starts up.
-After the first interval has passed the module itself will use the timestamp
-from the last response as the filter when retrieving new events.
+The type of entity to fetch. One of `domain`, `hash`, `ip` or `url`.
+
+*`var.list`*::
+
+The indicator list to fetch.
 
 *`var.proxy_url`*::
 
 Optional URL to use as HTTP proxy.
 
+*`var.custom_url`*::
+
+An alternative URL pointing to a CSV risklist. Use this option
+to fetch custom Fusion Files.
 
 Recorded Future fields are mapped to the following ECS fields:
 
@@ -594,13 +628,7 @@ Recorded Future fields are mapped to the following ECS fields:
 | entity.name            | threat.indicator.{url,ip,domain,file.hash}
 | entity.type            | threat.indicator.type
 | fileHashes             | threat.indicator.file.hash
-| intelCard              | event.reference
-| location.asn           | threat.indicator.as.number
-| location.location      | threat.indicator.geo
-| location.organization  | threat.indicator.as.organization.name
 | risk.score             | event.risk_score
-| timestamps.firstSeen   | threat.indicator.first_seen
-| timestamps.lastSeen    | threat.indicator.last_seen
 |=============================================================
 
 :has-dashboards!:

diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml
@@ -2231,31 +2231,25 @@ filebeat.modules:
     # Input used for ingesting threat intel data
     var.input: httpjson
 
-    # The interval to poll the API for updates
-    var.interval: 5m
-
-    # How far back in time to start fetching intelligence when run for the
-    # first time. Value must be in hours. Default: 168h (1 week).
-    var.first_interval: 168h
+    # Set your API Token.
+    var.api_token: "<RF_TOKEN>"
 
-    # The URL used for Threat Intel API calls.
-    # Must include the `limit` parameter and at least `entity` and `timestamps` fields.
-    # See the Connect API Explorer for a list of possible parameters.
-    #
-    # For `ip` entities:
-    var.url: "https://api.recordedfuture.com/v2/ip/search?limit=200&fields=entity,timestamps,risk,intelCard,location&metadata=false"
+    # The interval to poll the API for updates
+    var.interval: 1h
 
-    # For `domain` entities:
-    # var.url: "https://api.recordedfuture.com/v2/domain/search?limit=200&fields=entity,timestamps,risk,intelCard,location&metadata=false"
+    # The kind of entity to fetch. One of domain, hash, ip or url.
+    var.entity: domain
 
-    # For `hash` entities:
-    # var.url: "https://api.recordedfuture.com/v2/hash/search?limit=200&fields=entity,fileHashes,timestamps,risk,intelCard,location&metadata=false"
+    # The list to fetch. See the Recorded Future API Explorer for
+    # valid lists for each kind of entity.
+    var.list: default
 
-    # For `url` entities:
-    # var.url: "https://api.recordedfuture.com/v2/url/search?limit=200&fields=entity,timestamps,risk&metadata=false"
+    # Uncomment to use a different API endpoint.
+    # The API endpoint used for Recorded Future API calls.
+    # var.endpoint: "https://api.recordedfuture.com/v2"
 
-    # Set your API Token.
-    var.api_token: "<RF_TOKEN>"
+    # Uncomment to fetch a custom CSV file via URL. Useful for custom Fusion Files.
+    # var.custom_url: "https://api.recordedfuture.com/v2/fusion/files/?path=%2Fhome"
 
   threatq:
     enabled: false