Fix Issue 2765 - rule option: migrate to GeoIP2 database and libmaxminddb API.

appveyor.yml

@@ -31,7 +31,7 @@ init:
 install:
     - ps: 'Start-FileDownload "http://cygwin.com/$env:CYG_SETUP" -FileName "$env:CYG_SETUP"'
     - '%CYG_SETUP% -gqnNdO --quiet-mode --no-shortcuts --only-site --root "%CYG_ROOT%" --site "%CYG_MIRROR%" --local-package-dir "%CYG_CACHE%" > NUL 2>&1'
-    - '%CYG_SETUP% --quiet-mode --no-shortcuts --only-site --root "%CYG_ROOT%" --site "%CYG_MIRROR%" --local-package-dir "%CYG_CACHE%" --packages automake,bison,gcc-core,libtool,make,gettext-devel,gettext,intltool,pkg-config,clang,llvm,libpcre-devel,file-devel,wget,zlib-devel,libnss-devel,libnspr-devel,libGeoIP-devel,libyaml-devel,luajit-devel,unzip,libiconv,libiconv-devel > NUL 2>&1'
+    - '%CYG_SETUP% --quiet-mode --no-shortcuts --only-site --root "%CYG_ROOT%" --site "%CYG_MIRROR%" --local-package-dir "%CYG_CACHE%" --packages automake,bison,gcc-core,libtool,make,gettext-devel,gettext,intltool,pkg-config,clang,llvm,libpcre-devel,file-devel,wget,zlib-devel,libnss-devel,libnspr-devel,libmaxminddb-dev,libyaml-devel,luajit-devel,unzip,libiconv,libiconv-devel > NUL 2>&1'
     - '%CYG_BASH% -lc "cygcheck -dc cygwin"'
     - '%CYG_BASH% -lc "wget https://www.winpcap.org/install/bin/WpdPack_4_1_2.zip && ls && unzip WpdPack_4_1_2.zip"'
     - '%CYG_BASH% -lc "cp WpdPack/Lib/libpacket.a /usr/lib/"'

configure.ac

@@ -2031,43 +2031,43 @@
 
     AM_CONDITIONAL([HAVE_LUA], [test "x$enable_lua" != "xno"])
 
-  # libgeoip
+  # libmaxminddb
     AC_ARG_ENABLE(geoip,
-	        AS_HELP_STRING([--enable-geoip],[Enable GeoIP support]),
+	        AS_HELP_STRING([--enable-geoip],[Enable GeoIP2 support]),
 	        [ enable_geoip="yes"],
 	        [ enable_geoip="no"])
-    AC_ARG_WITH(libgeoip_includes,
-            [  --with-libgeoip-includes=DIR  libgeoip include directory],
-            [with_libgeoip_includes="$withval"],[with_libgeoip_includes="no"])
-    AC_ARG_WITH(libgeoip_libraries,
-            [  --with-libgeoip-libraries=DIR    libgeoip library directory],
-            [with_libgeoip_libraries="$withval"],[with_libgeoip_libraries="no"])
+    AC_ARG_WITH(libmaxminddb_includes,
+            [  --with-libmaxminddb-includes=DIR  libmaxminddb include directory],
+            [with_libmaxminddb_includes="$withval"],[with_libmaxminddb_includes="no"])
+    AC_ARG_WITH(libmaxminddb_libraries,
+            [  --with-libmaxminddb-libraries=DIR    libmaxminddb library directory],
+            [with_libmaxminddb_libraries="$withval"],[with_libmaxminddb_libraries="no"])
 
     if test "$enable_geoip" = "yes"; then
-        if test "$with_libgeoip_includes" != "no"; then
-            CPPFLAGS="${CPPFLAGS} -I${with_libgeoip_includes}"
+        if test "$with_libmaxminddb_includes" != "no"; then
+            CPPFLAGS="${CPPFLAGS} -I${with_libmaxminddb_includes}"
         fi
 
-        AC_CHECK_HEADER(GeoIP.h,GEOIP="yes",GEOIP="no")
+        AC_CHECK_HEADER(maxminddb.h,GEOIP="yes",GEOIP="no")
         if test "$GEOIP" = "yes"; then
-            if test "$with_libgeoip_libraries" != "no"; then
-                LDFLAGS="${LDFLAGS} -L${with_libgeoip_libraries}"
+            if test "$with_libmaxminddb_libraries" != "no"; then
+                LDFLAGS="${LDFLAGS} -L${with_libmaxminddb_libraries}"
             fi
-            AC_CHECK_LIB(GeoIP, GeoIP_country_code_by_ipnum,, GEOIP="no")
+            AC_CHECK_LIB(maxminddb, MMDB_open,, GEOIP="no")
         fi
         if test "$GEOIP" = "no"; then
             echo
-            echo "   ERROR!  libgeoip library not found, go get it"
-            echo "   from http://www.maxmind.com/en/geolite or your distribution:"
+            echo "   ERROR!  libmaxminddb GeoIP2 library not found, go get it"
+            echo "   from https://github.com/maxmind/libmaxminddb or your distribution:"
             echo
-            echo "   Ubuntu: apt-get install libgeoip-dev"
-            echo "   Fedora: dnf install GeoIP-devel"
-            echo "   CentOS/RHEL: yum install GeoIP-devel"
+            echo "   Ubuntu: apt-get install libmaxminddb-dev"
+            echo "   Fedora: dnf install libmaxminddb-devel"
+            echo "   CentOS/RHEL: yum install libmaxminddb-devel"
             echo
             exit 1
         fi
 
-        AC_DEFINE([HAVE_GEOIP],[1],[libgeoip available])
+        AC_DEFINE([HAVE_GEOIP],[1],[libmaxminddb available])
         enable_geoip="yes"
     fi
 
@@ -2473,7 +2473,7 @@ SURICATA_BUILD_CONF="Suricata Configuration:
   PCRE jit:                                ${pcre_jit_available}
   LUA support:                             ${enable_lua}
   libluajit:                               ${enable_luajit}
-  libgeoip:                                ${enable_geoip}
+  GeoIP2 support:                          ${enable_geoip}
   Non-bundled htp:                         ${enable_non_bundled_htp}
   Old barnyard2 support:                   ${enable_old_barnyard2}
   Hyperscan support:                       ${enable_hyperscan}

doc/userguide/rules/header-keywords.rst

@@ -137,7 +137,7 @@ geoip
 ^^^^^
 The geoip keyword enables (you) to match on the source, destination or
 source and destination IP addresses of network traffic, and to see to
-which country it belongs. To be able to do this, Suricata uses GeoIP
+which country it belongs. To be able to do this, Suricata uses GeoIP2
 API of Maxmind.
 
 The syntax of geoip::
@@ -156,8 +156,16 @@ direction you would like to match::
   dest: if the destination matches with the given geoip.
   src: the source matches with the given geoip.
 
-The keyword only supports IPv4. As it uses the GeoIP API of Maxmind,
-libgeoip must be compiled in.
+The keyword only supports IPv4. As it uses the GeoIP2 API of MaxMind,
+libmaxminddb must be compiled in. You must download and install the
+GeoIP2 or GeoLite2 database editions desired. Visit the MaxMind site 
+at https://dev.maxmind.com/geoip/geoip2/geolite2/ for details.
+
+You must also supply the location of the GeoIP2 or GeoLite2 database 
+file on the local system in the YAML-file configuration (for example)::
+
+geoip-database: /usr/local/share/GeoLite2/GeoLite2-Country.mmdb
+
 
 fragbits (IP fragmentation)
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^

src/detect-geoip.c

@@ -59,7 +59,7 @@ void DetectGeoipRegister(void)
 
 #else /* HAVE_GEOIP */
 
-#include <GeoIP.h>
+#include <maxminddb.h>
 
 static int DetectGeoipMatch(ThreadVars *, DetectEngineThreadCtx *, Packet *,
                             const Signature *, const SigMatchCtx *);
@@ -84,27 +84,85 @@ void DetectGeoipRegister(void)
  * \internal
  * \brief This function is used to initialize the geolocation MaxMind engine
  *
- * \retval NULL if the engine couldn't be initialized
- * \retval (GeoIP *) to the geolocation engine
+ * \retval FALSE if the engine couldn't be initialized
  */
-static GeoIP *InitGeolocationEngine(void)
+static int InitGeolocationEngine(DetectGeoipData *geoipdata)
 {
-    return GeoIP_new(GEOIP_MEMORY_CACHE);
+    const char *filename = NULL;
+
+    /* Get location and name of GeoIP2 database from YAML conf */
+    (void)ConfGet("geoip-database", &filename);
+
+    if (filename == NULL) {
+        SCLogWarning(SC_ERR_INVALID_ARGUMENT, "Unable to locate a GeoIP2"  
+                     "database filename in YAML conf.  GeoIP rule matching "
+                     "is disabled.");
+        geoipdata->mmdb_status = MMDB_FILE_OPEN_ERROR;
+        return FALSE;
+    }
+
+    /* Attempt to open MaxMind DB and save file handle if successful */
+    int status = MMDB_open(filename, MMDB_MODE_MMAP, &geoipdata->mmdb);
+
+    if (status == MMDB_SUCCESS) {
+        geoipdata->mmdb_status = status;
+        return TRUE;
+    }
+
+    SCLogWarning(SC_ERR_INVALID_ARGUMENT, "Failed to open GeoIP2 database: %s. " 
+                 "Error was: %s.  GeoIP rule matching is disabled.", filename, 
+                 MMDB_strerror(status));
+    geoipdata->mmdb_status = status;
+    return FALSE;
 }
 
 /**
  * \internal
  * \brief This function is used to geolocate the IP using the MaxMind libraries
  *
- * \param ip IP to geolocate (uint32_t ip)
+ * \param ip IPv4 to geolocate (uint32_t ip)
  *
  * \retval NULL if it couldn't be geolocated
  * \retval ptr (const char *) to the country code string
  */
-static const char *GeolocateIPv4(GeoIP *geoengine, uint32_t ip)
+static const char *GeolocateIPv4(const DetectGeoipData *geoipdata, uint32_t ip)
 {
-    if (geoengine != NULL)
-        return GeoIP_country_code_by_ipnum(geoengine,  SCNtohl(ip));
+    int mmdb_error;
+    struct sockaddr_in sa;
+    sa.sin_family = AF_INET;
+    sa.sin_port = 0;
+    sa.sin_addr.s_addr = ip;
+    MMDB_lookup_result_s result;
+    MMDB_entry_data_s entry_data;
+
+    /* Return if no GeoIP database access available */
+    if (geoipdata->mmdb_status != MMDB_SUCCESS)
+        return NULL;
+
+    /* Attempt to find the IPv4 address in the database */
+    result = MMDB_lookup_sockaddr((MMDB_s *)&geoipdata->mmdb, 
+                                  (struct sockaddr*)&sa, &mmdb_error);
+    if (mmdb_error != MMDB_SUCCESS)
+        return NULL;
+
+    /* The IPv4 address was found, so grab ISO country code if available */
+    if (result.found_entry) {
+        mmdb_error = MMDB_get_value(&result.entry, &entry_data, "country", 
+                                    "iso_code", NULL);
+        if (mmdb_error != MMDB_SUCCESS)
+            return NULL;
+
+        /* If ISO country code was found, then return it */
+        if (entry_data.has_data) {
+            if (entry_data.type == MMDB_DATA_TYPE_UTF8_STRING) {
+                    char *country_code = strndup((char *)entry_data.utf8_string, 
+                                                  entry_data.data_size);
+                    return country_code;
+            }
+        }
+    }
+
+    /* The country code for the IP was not found */
     return NULL;
 }
 
@@ -125,29 +183,44 @@ static const char *GeolocateIPv4(GeoIP *geoengine, uint32_t ip)
  * \internal
  * \brief This function is used to geolocate the IP using the MaxMind libraries
  *
- * \param ip IP to geolocate (uint32_t ip)
+ * \param ip IPv4 to geolocate (uint32_t ip)
  *
  * \retval 0 no match
  * \retval 1 match
  */
 static int CheckGeoMatchIPv4(const DetectGeoipData *geoipdata, uint32_t ip)
 {
-    const char *country;
+    const char *country = NULL;
     int i;
-    country = GeolocateIPv4(geoipdata->geoengine, ip);
+
+    /* Attempt country code lookup for the IP address */
+    country = GeolocateIPv4(geoipdata, ip);
+
+    /* Skip further checks if did not find a country code */
+    if (country == NULL)
+        return 0;
+
     /* Check if NOT NEGATED match-on condition */
     if ((geoipdata->flags & GEOIP_MATCH_NEGATED) == 0)
     {
-        for (i = 0; i < geoipdata->nlocations; i++)
-            if (country != NULL && strcmp(country, (char *)geoipdata->location[i])==0)
+        for (i = 0; i < geoipdata->nlocations; i++) {
+            if (country != NULL && strcmp(country, (char *)geoipdata->location[i])==0) {
+                free(country);
                 return 1;
+            }
+        }
     } else {
         /* Check if NEGATED match-on condition */
-        for (i = 0; i < geoipdata->nlocations; i++)
-            if (country != NULL && strcmp(country, (char *)geoipdata->location[i])==0)
+        for (i = 0; i < geoipdata->nlocations; i++) {
+            if (country != NULL && strcmp(country, (char *)geoipdata->location[i])==0) {
+                free(country);
                 return 0; /* if one matches, rule does NOT match (negated) */
+            }
+        }
+        free(country);
         return 1; /* returns 1 if no location matches (negated) */
     }
+    free(country);
     return 0;
 }
 
@@ -299,8 +372,7 @@ static DetectGeoipData *DetectGeoipDataParse (const char *str)
     }
 
     /* Initialize the geolocation engine */
-    geoipdata->geoengine = InitGeolocationEngine();
-    if (geoipdata->geoengine == NULL)
+    if (InitGeolocationEngine(geoipdata) == FALSE)
         goto error;
 
     return geoipdata;
@@ -362,6 +434,8 @@ static void DetectGeoipDataFree(void *ptr)
 {
     if (ptr != NULL) {
         DetectGeoipData *geoipdata = (DetectGeoipData *)ptr;
+        if (geoipdata->mmdb_status == MMDB_SUCCESS)
+            MMDB_close(&geoipdata->mmdb);
         SCFree(geoipdata);
     }
 }

src/detect-geoip.h

@@ -26,17 +26,18 @@
 
 #ifdef HAVE_GEOIP
 
-#include <GeoIP.h>
+#include <maxminddb.h>
 #include "util-spm-bm.h"
 
 #define GEOOPTION_MAXSIZE 3 /* Country Code (2 chars) + NULL */
 #define GEOOPTION_MAXLOCATIONS 64
 
 typedef struct DetectGeoipData_ {
     uint8_t location[GEOOPTION_MAXLOCATIONS][GEOOPTION_MAXSIZE];  /** country code for now, null term.*/
-    int nlocations; /** number of location strings parsed */
+    int nlocations;  /** number of location strings parsed */
     uint32_t flags;
-    GeoIP *geoengine;
+    int mmdb_status; /** Status of DB open call, MMDB_SUCCESS or error */
+    MMDB_s mmdb;     /** MaxMind DB file handle structure */
 } DetectGeoipData;
 
 #endif

suricata.yaml.in

@@ -1122,6 +1122,10 @@ unix-command:
 #magic-file: /usr/share/file/magic
 @e_magic_file_comment@magic-file: @e_magic_file@
 
+# GeoIP2 database file. Specify path and filename of GeoIP2 database
+# if using rules with "geoip" rule option.
+#geoip-database: /usr/local/share/GeoLite2/GeoLite2-Country.mmdb
+
 legacy:
   uricontent: enabled