feat(ipinfo): support GeoIP (#26)

Co-authored-by: E99p1ant <[email protected]>

internal/cli/config.tpl.yaml

@@ -1,4 +1,4 @@
-version: 1.2
+version: 1.3
 addr: :10000
 token:
 domain:
@@ -7,6 +7,10 @@ admin_path_prefix: "/revsuit"
 database: revsuit.db
 log_level: info
 
+ip_location_database:
+  database: "qqwry"                 # qqwry or geoip.
+  geo_license_key: ""               # Mandatory field, if you choose to use GeoIP.
+
 http:
   ip_header:
 dns:

internal/ipinfo/config.go

@@ -0,0 +1,6 @@
+package ipinfo
+
+type Config struct {
+	Database      string
+	GeoLicenseKey string `yaml:"geo_license_key"`
+}

internal/ipinfo/database.go

@@ -0,0 +1,5 @@
+package ipinfo
+
+type Database interface {
+	Area(string) string
+}

internal/ipinfo/geoip/download.go

@@ -0,0 +1,86 @@
+package geoip
+
+import (
+	"archive/tar"
+	"bytes"
+	"compress/gzip"
+	"crypto/tls"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"strings"
+	"time"
+
+	log "unknwon.dev/clog/v2"
+)
+
+const (
+	Url = "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-City&license_key=%s&suffix=tar.gz"
+)
+
+func get(url string) (b []byte, err error) {
+	client := http.Client{
+		Timeout: 90 * time.Second,
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, // disable verify
+		}}
+	request, _ := http.NewRequest(http.MethodGet, url, nil)
+	request.Header.Add("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36")
+
+	resp, err := client.Do(request)
+	if err != nil {
+		return
+	}
+	defer resp.Body.Close()
+
+	return io.ReadAll(resp.Body)
+}
+
+func extractTarGz(gzipStream io.Reader) error {
+	uncompressedStream, err := gzip.NewReader(gzipStream)
+	if err != nil {
+		return err
+	}
+
+	tarReader := tar.NewReader(uncompressedStream)
+
+	for {
+		header, err := tarReader.Next()
+
+		if err == io.EOF {
+			break
+		}
+
+		if err != nil {
+			return err
+		}
+
+		switch header.Typeflag {
+		case tar.TypeReg:
+			log.Trace(header.Name)
+			if !strings.HasSuffix(header.Name, "GeoLite2-City.mmdb") {
+				continue
+			}
+			outFile, err := os.Create("GeoLite2-City.mmdb")
+			if err != nil {
+				return err
+			}
+			defer outFile.Close()
+			if _, err := io.Copy(outFile, tarReader); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func download(licenseKey string) (err error) {
+	var GeoLite2TarGz []byte
+	log.Info("Downloading GeoLite2-City.mmdb...")
+	if GeoLite2TarGz, err = get(fmt.Sprintf(Url, licenseKey)); err != nil {
+		return err
+	}
+
+	return extractTarGz(bytes.NewReader(GeoLite2TarGz))
+}

internal/ipinfo/geoip/geo.go

@@ -0,0 +1,67 @@
+package geoip
+
+import (
+	"fmt"
+	"net"
+	"os"
+	"sync"
+	"time"
+
+	"github.com/oschwald/geoip2-golang"
+	log "unknwon.dev/clog/v2"
+)
+
+type Database struct {
+	geo *geoip2.Reader
+}
+
+var geo *geoip2.Reader
+var once sync.Once
+
+// Area returns IpArea according to ip
+func (db *Database) Area(ip string) string {
+	defer func() {
+		_ = recover()
+	}()
+	record, err := db.geo.City(net.ParseIP(ip))
+	if err != nil {
+		return ""
+	}
+
+	country := record.Country.Names["en"]
+	city := record.City.Names["en"]
+	if city == "" {
+		city = record.Location.TimeZone
+	}
+	return fmt.Sprintf("%s %s", country, city)
+}
+
+func checkUpdate(licenseKey string) {
+	info, err := os.Stat("GeoLite2-City.mmdb")
+	if err != nil {
+		if os.IsNotExist(err) {
+			err := download(licenseKey)
+			if err != nil {
+				log.Warn("Download GeoLite2-City.mmdb failed, caused by:%v, recommend to download it by yourself otherwise the `IpArea` will be null", err)
+			}
+		}
+	} else if -time.Until(info.ModTime()) > 7*24*time.Hour {
+		log.Info("Updating GeoLite2-City.mmdb...")
+		err := download(licenseKey)
+		if err != nil {
+			log.Warn("Update GeoLite2-City.mmdb failed, please download GeoLite2-City.mmdb by yourself")
+		}
+	}
+}
+
+func New(licenseKey string) *Database {
+	once.Do(func() {
+		var err error
+		checkUpdate(licenseKey)
+		geo, err = geoip2.Open("GeoLite2-City.mmdb")
+		if err != nil {
+			log.Error("Load GeoLite2-City.mmdb failed, `IpArea` will be null")
+		}
+	})
+	return &Database{geo: geo}
+}

internal/ipinfo/ipinfo.go

@@ -0,0 +1,27 @@
+package ipinfo
+
+import (
+	"github.com/li4n0/revsuit/internal/ipinfo/geoip"
+	"github.com/li4n0/revsuit/internal/ipinfo/qqwry"
+	log "unknwon.dev/clog/v2"
+)
+
+var db Database
+
+func Area(ip string) string {
+	if db != nil {
+		return db.Area(ip)
+	}
+	return ""
+}
+
+func Init(config Config) {
+	switch config.Database {
+	case "qqwry":
+		db = qqwry.New()
+	case "geoip":
+		db = geoip.New(config.GeoLicenseKey)
+	default:
+		log.Fatal("wrong ip location database type: %q", config.Database)
+	}
+}

internal/ipinfo/qqwry/download.go

@@ -6,8 +6,9 @@ import (
 	"crypto/tls"
 	"encoding/binary"
 	"errors"
-	"io/ioutil"
+	"io"
 	"net/http"
+	"os"
 	"sync"
 	"time"
 
@@ -16,7 +17,7 @@ import (
 
 const (
 	CopyWriteUrl = "https://qqwry.mirror.noc.one/copywrite.rar"
-	QqwryUrl     = "https://qqwry.mirror.noc.one/qqwry.rar"
+	Url          = "https://qqwry.mirror.noc.one/qqwry.rar"
 )
 
 func get(url string) (b []byte, err error) {
@@ -34,8 +35,7 @@ func get(url string) (b []byte, err error) {
 	}
 	defer resp.Body.Close()
 
-	b, err = ioutil.ReadAll(resp.Body)
-	return b, err
+	return io.ReadAll(resp.Body)
 }
 
 func getKey(b []byte) (key uint32, err error) {
@@ -58,15 +58,17 @@ func decrypt(b []byte, key uint32) (_ []byte, err error) {
 		return
 	}
 	defer rc.Close()
-	return ioutil.ReadAll(rc)
+	return io.ReadAll(rc)
 }
 
 func download() (err error) {
 	var (
 		copyWriteData, qqwryData []byte
 		wg                       sync.WaitGroup
+		key                      uint32
 	)
 	log.Info("Downloading qqwry.dat...")
+
 	wg.Add(2)
 	go func() {
 		defer wg.Done()
@@ -77,23 +79,21 @@ func download() (err error) {
 
 	go func() {
 		defer wg.Done()
-		if qqwryData, err = get(QqwryUrl); err != nil {
+		if qqwryData, err = get(Url); err != nil {
 			return
 		}
 	}()
 	wg.Wait()
-	if err != nil {
-		return err
-	}
-	var key uint32
-	if key, err = getKey(copyWriteData); err != nil {
-		return
-	}
-	b, err := decrypt(qqwryData, key)
-	if err != nil {
-		return err
-	}
-	_ = ioutil.WriteFile("qqwry.dat", b, 0644)
 
-	return nil
+	if err != nil {
+		return err
+	}
+	if key, err = getKey(copyWriteData); err != nil {
+		return err
+	}
+	if b, err := decrypt(qqwryData, key); err != nil {
+		return err
+	} else {
+		return os.WriteFile("qqwry.dat", b, 0644)
+	}
 }

internal/ipinfo/qqwry/wry.go

@@ -9,14 +9,29 @@ import (
 	log "unknwon.dev/clog/v2"
 )
 
+type Database struct {
+	wry *qqwry.QQwry
+}
+
+// Area returns IpArea according to ip
+func (db *Database) Area(ip string) string {
+	defer func() {
+		_ = recover()
+	}()
+	if db.wry == nil {
+		return ""
+	}
+	ipData := db.wry.SearchByIPv4(ip)
+	if ipData.Area == " CZ88.NET" {
+		return ipData.Country
+	}
+	return ipData.Country + " " + ipData.Area
+}
+
 var wry *qqwry.QQwry
 var once sync.Once
 
-func init() {
-	_ = log.NewConsole(100,
-		log.ConsoleConfig{
-			Level: log.LevelInfo,
-		})
+func checkUpdate() {
 	info, err := os.Stat("qqwry.dat")
 	if err != nil {
 		if os.IsNotExist(err) {
@@ -25,7 +40,7 @@ func init() {
 				log.Warn("Download qqwry.dat failed, caused by:%v, recommend to download it by yourself otherwise the `IpArea` will be null", err)
 			}
 		}
-	} else if -time.Until(info.ModTime()) > 5*24*time.Hour {
+	} else if -time.Until(info.ModTime()) > 7*24*time.Hour {
 		log.Info("Updating qqwry.dat...")
 		err := download()
 		if err != nil {
@@ -34,8 +49,9 @@ func init() {
 	}
 }
 
-func GetQQWry() *qqwry.QQwry {
+func New() *Database {
 	once.Do(func() {
+		checkUpdate()
 		qqwry.DatData.FilePath = "qqwry.dat"
 		init := qqwry.DatData.InitDatFile()
 		if v, ok := init.(error); ok {
@@ -46,6 +62,5 @@ func GetQQWry() *qqwry.QQwry {
 		}
 		wry = qqwry.NewQQwry()
 	})
-
-	return wry
+	return &Database{wry: wry}
 }

internal/qqwry/query.go

@@ -1,16 +0,0 @@
-package qqwry
-
-// Area returns IpArea according to ip
-func Area(ip string) string {
-	defer func() {
-		_ = recover()
-	}()
-	if GetQQWry() == nil {
-		return ""
-	}
-	ipData := GetQQWry().SearchByIPv4(ip)
-	if ipData.Area == " CZ88.NET" {
-		return ipData.Country
-	}
-	return ipData.Country + " " + ipData.Area
-}