feature/geoip download verify and auto updates (#33)

cmd/alice/flags.go

@@ -260,9 +260,26 @@ func flagsInitialization() []cli.Flag {
 		&cli.BoolFlag{
 			Name:     "geoip-download-sha256-skip",
 			Category: "GeoIP",
+			Usage: `sha256 helps to check database contents of the mmdb database
+			and avoid unnecessary requests to MaxMind CDN`,
+		},
+		&cli.DurationFlag{
+			Name:     "geoip-update-frequency",
+			Category: "GeoIP",
+			Usage: `when geoip-maxmind-permalink is selected and geoip-db-path is empty,
+			once within a certain 'PERIOD' of time app will update the geoip database;
+			do not forget about maxmind donwload limits; set to 0s if want to disable`,
+			Value: 24 * time.Hour,
+		},
+		&cli.DurationFlag{
+			Name:     "geoip-update-retry-frequency",
+			Category: "GeoIP",
+			Value:    1 * time.Hour,
 		},
 		&cli.BoolFlag{
 			Name:               "geoip-skip-database-verify",
+			Category:           "GeoIP",
+			Usage:              "skip mmdb contents validation by vendor function db.Verify()",
 			DisableDefaultText: true,
 		},
 

go.mod

@@ -11,7 +11,7 @@ require (
 	github.com/mailru/easyjson v0.7.7
 	github.com/oschwald/maxminddb-golang v1.13.1
 	github.com/rs/zerolog v1.33.0
-	github.com/urfave/cli/v2 v2.27.2
+	github.com/urfave/cli/v2 v2.27.3
 	github.com/valyala/bytebufferpool v1.0.0
 	github.com/valyala/fasthttp v1.55.0
 )
@@ -30,7 +30,7 @@ require (
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/tinylib/msgp v1.1.8 // indirect
 	github.com/valyala/tcplisten v1.0.0 // indirect
-	github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 // indirect
+	github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
 	go.etcd.io/bbolt v1.3.9 // indirect
 	golang.org/x/sys v0.21.0 // indirect
 )

go.sum

@@ -47,16 +47,16 @@ github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/tinylib/msgp v1.1.8 h1:FCXC1xanKO4I8plpHGH2P7koL/RzZs12l/+r7vakfm0=
 github.com/tinylib/msgp v1.1.8/go.mod h1:qkpG+2ldGg4xRFmx+jfTvZPxfGFhi64BcnL9vkCm/Tw=
-github.com/urfave/cli/v2 v2.27.2 h1:6e0H+AkS+zDckwPCUrZkKX38mRaau4nL2uipkJpbkcI=
-github.com/urfave/cli/v2 v2.27.2/go.mod h1:g0+79LmHHATl7DAcHO99smiR/T7uGLw84w8Y42x+4eM=
+github.com/urfave/cli/v2 v2.27.3 h1:/POWahRmdh7uztQ3CYnaDddk0Rm90PyOgIxgW2rr41M=
+github.com/urfave/cli/v2 v2.27.3/go.mod h1:m4QzxcD2qpra4z7WhzEGn74WZLViBnMpb1ToCAKdGRQ=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
 github.com/valyala/fasthttp v1.55.0 h1:Zkefzgt6a7+bVKHnu/YaYSOPfNYNisSVBo/unVCf8k8=
 github.com/valyala/fasthttp v1.55.0/go.mod h1:NkY9JtkrpPKmgwV3HTaS2HWaJss9RSIsRVfcxxoHiOM=
 github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8=
 github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
-github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 h1:+qGGcbkzsfDQNPPe9UDgpxAWQrhbbBXOYJFQDq/dtJw=
-github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913/go.mod h1:4aEEwZQutDLsQv2Deui4iYQ6DWTxR14g6m8Wv88+Xqk=
+github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4=
+github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 go.etcd.io/bbolt v1.3.9 h1:8x7aARPEXiXbHmtUwAIv7eV2fQFHrLLavdiJ3uzJXoI=
 go.etcd.io/bbolt v1.3.9/go.mod h1:zaO32+Ti0PK1ivdPtgMESzuzL2VPoIG1PCQNvOdo/dE=

internal/geoip/extract.go

@@ -0,0 +1,50 @@
+package geoip
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"io"
+	"strings"
+
+	"github.com/rs/zerolog"
+)
+
+func extractTarGzArchive(log *zerolog.Logger, dst io.Writer, src io.Reader) (e error) {
+	var rd *gzip.Reader
+	if rd, e = gzip.NewReader(src); e != nil {
+		return
+	}
+
+	return extractTarArchive(log, dst, rd)
+}
+
+func extractTarArchive(log *zerolog.Logger, dst io.Writer, src io.Reader) (e error) {
+	tr := tar.NewReader(src)
+	for {
+		var hdr *tar.Header
+		hdr, e = tr.Next()
+
+		if e == io.EOF {
+			break // End of archive
+		} else if e != nil {
+			return
+		}
+
+		log.Trace().Msg("found file in maxmind tar archive - " + hdr.Name)
+		if !strings.HasSuffix(hdr.Name, "mmdb") {
+			continue
+		}
+
+		log.Trace().Msg("found mmdb file, copy to temporary file")
+
+		var written int64
+		if written, e = io.Copy(dst, tr); e != nil { // skipcq: GO-S2110 decompression bomb isn't possible here
+			return
+		}
+
+		log.Debug().Msgf("parsed response has written in temporary file with %d bytes", written)
+		break
+	}
+
+	return
+}

internal/geoip/geoip_file.go → internal/geoip/file.go

@@ -12,13 +12,14 @@ import (
 )
 
 type GeoIPFileClient struct {
+	mu sync.RWMutex
 	*maxminddb.Reader
 
 	appname, tempdir string
 	skipVerify       bool
 
-	mu      sync.RWMutex
-	isReady bool
+	muReady sync.RWMutex
+	ready   bool
 
 	log *zerolog.Logger
 
@@ -64,13 +65,14 @@ func (m *GeoIPFileClient) Bootstrap() {
 }
 
 func (m *GeoIPFileClient) LookupCountryISO(ip string) (string, error) {
-	return lookupISOByIP(m.Reader, ip)
+	return lookupISOByIP(&m.mu, m.Reader, ip)
 }
 
 func (m *GeoIPFileClient) IsReady() bool {
-	m.mu.RLock()
-	defer m.mu.RUnlock()
-	return m.isReady
+	m.muReady.RLock()
+	defer m.muReady.RUnlock()
+
+	return m.ready
 }
 
 //
@@ -82,7 +84,8 @@ func (m *GeoIPFileClient) destroy() {
 }
 
 func (m *GeoIPFileClient) setReady(ready bool) {
-	m.mu.Lock()
-	m.isReady = ready
-	m.mu.Unlock()
+	m.muReady.Lock()
+	defer m.muReady.Unlock()
+
+	m.ready = ready
 }

internal/geoip/geoip.go

@@ -37,7 +37,13 @@ func (m *geoIPRecord) reset() {
 	*m = geoIPRecord{}
 }
 
-func lookupISOByIP(mxrd *maxminddb.Reader, rawip string) (iso string, e error) {
+func lookupISOByIP(mu *sync.RWMutex, mxrd *maxminddb.Reader, rawip string) (iso string, e error) {
+	if !mu.TryRLock() {
+		e = errors.New("could not get lock for geoip lookup()")
+		return
+	}
+	defer mu.RUnlock()
+
 	var ip netip.Addr
 	if ip, e = netip.ParseAddr(rawip); e != nil {
 		e = errors.New(fmt.Sprintf("could not parse ip addr with netip library, ip - %+v - ", rawip) + e.Error())