Merge remote-tracking branch 'origin/topic/christian/mmdb-configurabi…

…lity'

* origin/topic/christian/mmdb-configurability:
  Modernize various C++/Zeek-isms in the MMDB code.
  Fix MMDB code to re-open explicitly opened DBs correctly
  Add btest to verify behavior of re-opened MMDBs opened directly via BIFs
  Simplify MMDB code by moving more lookup functionality into MMDB class
  Move MMDB logic out of mmdb.bif and into MMDB.cc/h.
  Fix mmdb.temporary-error testcase when MMDBs are installed on system
  Adapt MMDB BiF code to new script-layer variables
  Update btest baselines to reflect introduction of mmdb.bif
  Move MaxMind/GeoIP BiF functionality into separate file
  Provide script-level configurability of MaxMind DB placement on disk
  Sort toplevel .bif list in CMakeLists

CHANGES

@@ -1,3 +1,60 @@
+6.2.0-dev.375 | 2024-01-12 09:27:58 +0100
+
+  * Modernize various C++/Zeek-isms in the MMDB code. (Christian Kreibich, Corelight)
+
+  * Fix MMDB code to re-open explicitly opened DBs correctly (Christian Kreibich, Corelight)
+
+    The filename from which a DB first gets opened (either via an explicitly
+    specified filename, or via the path sequence now configurable at the script
+    layer) is now "sticky", meaning re-opening won't switch to a different file.
+
+    This was easiest by moving most state into the MMDB class itself. The previous
+    approach of tracking the two DB instances via a smart pointer and blowing the
+    pointed-to objects away as needed is now instead one of two objects fixed over
+    the lifetime of Zeek, able to open/close/reopen their underlying Maxmind DBs.
+
+    The MMDB class now only has one Lookup() method since there was no need to break
+    them apart -- it saves the return of a MMDB_lookup_result_s over the stack and
+    there's no need for throwing an exception.
+
+  * Add btest to verify behavior of re-opened MMDBs opened directly via BIFs (Christian Kreibich, Corelight)
+
+    The mmdb_open_location_db() and mmdb_open_asn_db() BiFs were untested, and Zeek
+    has a bug that makes any DBs opened that way fall back to looking up DBs via the
+    existing script-level config mechanism (via mmdb_dir), which is at least
+    unexpected and might well be unconfigured if somebody uses the direct BiFs.
+
+  * Simplify MMDB code by moving more lookup functionality into MMDB class (Christian Kreibich, Corelight)
+
+  * Move MMDB logic out of mmdb.bif and into MMDB.cc/h. (Christian Kreibich, Corelight)
+
+    This does not change the implementation except for some light renaming where
+    things are now naturally scoped within MMDB.cc.
+
+  * Fix mmdb.temporary-error testcase when MMDBs are installed on system (Christian Kreibich, Corelight)
+
+    The test would previously fail in settings where the user has Maxmind DBs
+    installed in the hardwired system locations, because the fallback logic still
+    picked those up.
+
+  * Adapt MMDB BiF code to new script-layer variables (Christian Kreibich, Corelight)
+
+  * Update btest baselines to reflect introduction of mmdb.bif (Christian Kreibich, Corelight)
+
+  * Move MaxMind/GeoIP BiF functionality into separate file (Christian Kreibich, Corelight)
+
+  * Provide script-level configurability of MaxMind DB placement on disk (Christian Kreibich, Corelight)
+
+    This lifts the list of fallback directories in which Zeek will look for Maxmind
+    DBs into the script layer, and makes the names of the DB files themselves
+    (previously hardwired) configurable as well.
+
+    This does not yet change the in-core code; that commit follows.
+
+  * Sort toplevel .bif list in CMakeLists (Christian Kreibich, Corelight)
+
+    Not important, but here it is safe to do so and the list is getting lengthy.
+
 6.2.0-dev.362 | 2024-01-11 20:17:45 +0100
 
   * GH-3540: Known: Keep &create_expire on local tables/sets valid (Arne Welzel, Corelight)

VERSION

@@ -1 +1 @@
-6.2.0-dev.362
+6.2.0-dev.375

scripts/base/init-bare.zeek

@@ -1107,6 +1107,25 @@ type geo_autonomous_system: record {
 ## The directory containing MaxMind DB (.mmdb) files to use for GeoIP support.
 const mmdb_dir: string = "" &redef;
 
+## Default name of the MaxMind City database file:
+const mmdb_city_db: string = "GeoLite2-City.mmdb" &redef;
+## Default name of the MaxMind Country database file:
+const mmdb_country_db: string = "GeoLite2-Country.mmdb" &redef;
+## Default name of the MaxMind ASN database file:
+const mmdb_asn_db: string = "GeoLite2-ASN.mmdb" &redef;
+
+## Fallback locations for MaxMind databases. Zeek attempts these when
+## :zeek:see:`mmdb_dir` is not set, or it cannot read a DB file from it. For
+## geolocation lookups, Zeek will first attempt to locate the city database in
+## each of the fallback locations, and should this fail, attempt to locate the
+## country one.
+const mmdb_dir_fallbacks: vector of string = vector(
+	"/usr/share/GeoIP",
+	"/var/lib/GeoIP",
+	"/usr/local/share/GeoIP",
+	"/usr/local/var/GeoIP",
+) &redef;
+
 ## Sets the interval for MaxMind DB file staleness checks. When Zeek detects a
 ## change in inode or modification time, the database is re-opened. Setting
 ## a negative interval disables staleness checks.
@@ -2094,6 +2113,7 @@ type gtp_delete_pdp_ctx_response_elements: record {
 @load base/bif/supervisor.bif
 @load base/bif/packet_analysis.bif
 @load base/bif/CPP-load.bif
+@load base/bif/mmdb.bif
 
 ## Internal function.
 function add_interface(iold: string, inew: string): string

src/CMakeLists.txt

@@ -111,27 +111,28 @@ include(BifCl)
 set(SUPERVISOR_SRCS supervisor/Supervisor.cc Pipe.cc)
 
 set(BIF_SRCS
-    zeek.bif
     communityid.bif
-    stats.bif
-    event.bif
     const.bif
-    types.bif
-    strings.bif
-    reporter.bif
+    event.bif
+    mmdb.bif
     option.bif
-    # Note: the supervisor BIF file is treated like other top-level BIFs instead
-    # of contained in its own subdirectory CMake logic because subdirectory BIFs
-    # are treated differently and don't support being called *during* parsing
-    # (e.g. within an @if directive).
-    supervisor/supervisor.bif
+    reporter.bif
+    stats.bif
+    strings.bif
+    types.bif
+    zeek.bif
     # The packet analysis BIF is treated like other top-level BIFs because it's
     # needed before parsing the packet protocol scripts, which happen very near
     # to the start of parsing.
     packet_analysis/packet_analysis.bif
     # The C++ loading BIF is treated like other top-level BIFs to give us
     # flexibility regarding when it's called.
-    script_opt/CPP/CPP-load.bif)
+    script_opt/CPP/CPP-load.bif
+    # Note: the supervisor BIF file is treated like other top-level BIFs instead
+    # of contained in its own subdirectory CMake logic because subdirectory BIFs
+    # are treated differently and don't support being called *during* parsing
+    # (e.g. within an @if directive).
+    supervisor/supervisor.bif)
 
 foreach (bift ${BIF_SRCS})
     bif_target(${bift} "standard")
@@ -328,6 +329,7 @@ set(MAIN_SRCS
     IP.cc
     IPAddr.cc
     List.cc
+    MMDB.cc
     Reporter.cc
     NFA.cc
     NetVar.cc

src/Func.cc

@@ -52,6 +52,7 @@
 #include "supervisor.bif.func_h"
 #include "packet_analysis.bif.func_h"
 #include "CPP-load.bif.func_h"
+#include "mmdb.bif.func_h"
 
 #include "zeek.bif.func_def"
 #include "communityid.bif.func_def"
@@ -62,6 +63,7 @@
 #include "supervisor.bif.func_def"
 #include "packet_analysis.bif.func_def"
 #include "CPP-load.bif.func_def"
+#include "mmdb.bif.func_def"
 // clang-format on
 
 extern RETSIGTYPE sig_handler(int signo);
@@ -1049,6 +1051,7 @@ void init_primary_bifs() {
 
 #include "CPP-load.bif.func_init"
 #include "communityid.bif.func_init"
+#include "mmdb.bif.func_init"
 #include "option.bif.func_init"
 #include "packet_analysis.bif.func_init"
 #include "reporter.bif.func_init"