added geolite2 server

1 year ago · 74c5a1b70f
parent 4e7c1a2fbb
commit 74c5a1b70f
9 changed files with 397 additions and 0 deletions
--- a/.dockerignore
+++ b/.dockerignore
@ -0,0 +1,6 @@
+# flyctl launch added from .gitignore
+data/*
+!data/README.txt
+dist/*
+!dist/README.txt
+fly.toml
--- a/.gitignore
+++ b/.gitignore
@ -0,0 +1,4 @@
+/data/*
+!/data/README.txt
+/dist/*
+!/dist/README.txt
--- a/18
+++ b/18
@ -0,0 +1,18 @@
+ARG GO_VERSION=1
+FROM golang:${GO_VERSION}-bookworm as builder
+
+WORKDIR /usr/src/app
+COPY go.mod go.sum ./
+RUN go mod download && go mod verify
+COPY . .
+RUN go build -v -o /run-app .
+
+
+FROM debian:bookworm
+
+RUN apt update
+
+RUN apt install -y ca-certificates
+
+COPY --from=builder /run-app /usr/local/bin/
+CMD ["run-app"]
--- a/data/README.txt
+++ b/data/README.txt
@ -0,0 +1,2 @@
+this directory will contain the geolite2 db file
+the server will only read GeoLite2-City.mmdb
--- a/dist/README.txt
+++ b/dist/README.txt
@ -0,0 +1,3 @@
+this directory is used to download and extract the latest geolite2 db file
+
+the content including this README will be delete during the update
--- a/fly.toml
+++ b/fly.toml
@ -0,0 +1,25 @@
+# fly.toml app configuration file generated for geoip on 2024-08-06T18:24:17+08:00
+#
+# See https://fly.io/docs/reference/configuration/ for information about how to use this file.
+#
+
+app = 'geoip'
+primary_region = 'iad'
+
+[build]
+  [build.args]
+    GO_VERSION = '1.22.5'
+
+[env]
+  PORT = '8080'
+
+[http_service]
+  internal_port = 3000
+  force_https = true
+  auto_stop_machines = 'stop'
+  auto_start_machines = true
+  min_machines_running = 0
+  processes = ['app']
+
+[[vm]]
+  size = 'shared-cpu-1x'
--- a/go.mod
+++ b/go.mod
@ -0,0 +1,24 @@
+module tzgyn/geolite2server
+
+go 1.22.5
+
+require (
+	github.com/gofiber/fiber/v2 v2.52.5
+	github.com/oschwald/geoip2-golang v1.11.0
+	github.com/robfig/cron/v3 v3.0.0
+)
+
+require (
+	github.com/andybalholm/brotli v1.0.5 // indirect
+	github.com/google/uuid v1.5.0 // indirect
+	github.com/klauspost/compress v1.17.0 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/mattn/go-runewidth v0.0.15 // indirect
+	github.com/oschwald/maxminddb-golang v1.13.0 // indirect
+	github.com/rivo/uniseg v0.2.0 // indirect
+	github.com/valyala/bytebufferpool v1.0.0 // indirect
+	github.com/valyala/fasthttp v1.51.0 // indirect
+	github.com/valyala/tcplisten v1.0.0 // indirect
+	golang.org/x/sys v0.20.0 // indirect
+)
--- a/go.sum
+++ b/go.sum
@ -0,0 +1,41 @@
+github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs=
+github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/gofiber/fiber/v2 v2.52.5 h1:tWoP1MJQjGEe4GB5TUGOi7P2E0ZMMRx5ZTG4rT+yGMo=
+github.com/gofiber/fiber/v2 v2.52.5/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ=
+github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU=
+github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM=
+github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
+github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
+github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
+github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
+github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
+github.com/oschwald/geoip2-golang v1.11.0 h1:hNENhCn1Uyzhf9PTmquXENiWS6AlxAEnBII6r8krA3w=
+github.com/oschwald/geoip2-golang v1.11.0/go.mod h1:P9zG+54KPEFOliZ29i7SeYZ/GM6tfEL+rgSn03hYuUo=
+github.com/oschwald/maxminddb-golang v1.13.0 h1:R8xBorY71s84yO06NgTmQvqvTvlS/bnYZrrWX1MElnU=
+github.com/oschwald/maxminddb-golang v1.13.0/go.mod h1:BU0z8BfFVhi1LQaonTwwGQlsHUEu9pWNdMfmq4ztm0o=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
+github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+github.com/robfig/cron/v3 v3.0.0 h1:kQ6Cb7aHOHTSzNVNEhmp8EcWKLb4CbiMW9h9VyIhO4E=
+github.com/robfig/cron/v3 v3.0.0/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
+github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
+github.com/valyala/fasthttp v1.51.0 h1:8b30A5JlZ6C7AS81RsWjYMQmrZG6feChmgAolCl1SqA=
+github.com/valyala/fasthttp v1.51.0/go.mod h1:oI2XroL+lI7vdXyYoQk03bXBThfFl2cVdIA3Xl7cH8g=
+github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8=
+github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
+golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
+golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
--- a/main.go
+++ b/main.go
@ -0,0 +1,274 @@
+package main
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"encoding/base64"
+	"errors"
+	"fmt"
+	"io"
+	"log"
+	"net"
+	"net/http"
+	"os"
+	"os/signal"
+	"path/filepath"
+	"syscall"
+
+	"github.com/gofiber/fiber/v2"
+	"github.com/gofiber/fiber/v2/middleware/logger"
+	"github.com/oschwald/geoip2-golang"
+	"github.com/robfig/cron/v3"
+)
+
+func main() {
+	fmt.Println("Initializing GeoLite2 DB...")
+	err := downloadAndExtractDB()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	db, err := geoip2.Open("./data/GeoLite2-City.mmdb")
+	if err != nil {
+		log.Fatal("Failed to initialize GeoLite2 DB")
+	}
+
+	fmt.Println("Finished initializing GeoLite2 DB")
+
+	app := fiber.New()
+
+	app.Use(logger.New())
+
+	app.Get("/", func(c *fiber.Ctx) error {
+		return c.SendString("Hello, World!")
+	})
+
+	app.Get("/me", func(c *fiber.Ctx) error {
+		ip := c.IPs()[0]
+
+		record, err := getCity(db, ip)
+		if err != nil {
+			c.JSON(fiber.Map{
+				"success": false,
+				"message": err.Error(),
+			})
+		}
+
+		return c.JSON(record)
+	})
+
+	app.Get("/:ip", func(c *fiber.Ctx) error {
+		queryIp := c.Params("ip")
+
+		record, err := getCity(db, queryIp)
+		if err != nil {
+			c.JSON(fiber.Map{
+				"success": false,
+				"message": err.Error(),
+			})
+		}
+
+		return c.JSON(record)
+	})
+
+	c := cron.New()
+	c.AddFunc("@weekly", func() {
+		fmt.Println("Updating GeoLite2 DB...")
+		err := downloadAndExtractDB()
+		if err != nil {
+			log.Fatal(err)
+		}
+		fmt.Println("Finished updating GeoLite2 DB")
+	})
+
+	c.Start()
+
+	done := make(chan os.Signal, 1)
+	signal.Notify(done, syscall.SIGINT, syscall.SIGTERM, os.Interrupt)
+
+	go func() {
+		<-done
+		fmt.Println("Gracefully shutting down...")
+		_ = app.Shutdown()
+	}()
+
+	if err := app.Listen(":3000"); err != nil {
+		log.Panic(err)
+	}
+
+	fmt.Println("Running cleanup tasks...")
+	fmt.Println("Stopping cronjobs...")
+	c.Stop()
+	db.Close()
+}
+
+func getCity(db *geoip2.Reader, queryIp string) (*geoip2.City, error) {
+	// If you are using strings that may be invalid, check that ip is not nil
+	ip := net.ParseIP(queryIp)
+	if ip == nil {
+		return nil, errors.New("invalid ip")
+	}
+
+	record, err := db.City(ip)
+	if err != nil {
+		return nil, errors.New("no data")
+	}
+	return record, nil
+}
+
+func downloadAndExtractDB() error {
+	_, err := os.ReadDir("./data")
+	if err != nil {
+		os.MkdirAll("./data", os.ModePerm)
+	}
+
+	_, err = os.ReadDir("./dist")
+	if err != nil {
+		os.MkdirAll("./dist", os.ModePerm)
+	}
+
+	err = downloadDB()
+	if err != nil {
+		return err
+	}
+
+	r, err := os.Open("./db.tar.gz")
+	if err != nil {
+		return err
+	}
+
+	err = Untar("./dist", r)
+	if err != nil {
+		return err
+	}
+
+	files, err := os.ReadDir("./dist")
+	if err != nil {
+		return err
+	}
+
+	fileInfo := files[0]
+
+	err = os.Rename("./dist/"+fileInfo.Name()+"/GeoLite2-City.mmdb", "./data/GeoLite2-City.mmdb")
+	if err != nil {
+		return err
+	}
+
+	err = os.RemoveAll("./dist/")
+	if err != nil {
+		return err
+	}
+
+	err = os.MkdirAll("./dist/", os.ModePerm)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func downloadDB() error {
+	accountID := os.Getenv("ACCOUNT_ID")
+	if len(accountID) == 0 {
+		return errors.New("please provide ACCOUNT_ID as env")
+	}
+
+	licenseKey := os.Getenv("LICENSE_KEY")
+	if len(licenseKey) == 0 {
+		return errors.New("please provide LICENSE_KEY as env")
+	}
+
+	out, err := os.Create("db.tar.gz")
+	if err != nil {
+		return err
+	}
+
+	defer out.Close()
+
+	auth := base64.StdEncoding.EncodeToString([]byte(accountID + ":" + licenseKey))
+
+	req, err := http.NewRequest(http.MethodGet, "https://download.maxmind.com/geoip/databases/GeoLite2-City/download?suffix=tar.gz", nil)
+	if err != nil {
+		return err
+	}
+	req.Header.Add("Authorization", "Basic "+auth)
+
+	res, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return err
+	}
+
+	defer res.Body.Close()
+
+	_, err = io.Copy(out, res.Body)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func Untar(dst string, r io.Reader) error {
+
+	gzr, err := gzip.NewReader(r)
+	if err != nil {
+		return err
+	}
+	defer gzr.Close()
+
+	tr := tar.NewReader(gzr)
+
+	for {
+		header, err := tr.Next()
+
+		switch {
+
+		// if no more files are found return
+		case err == io.EOF:
+			return nil
+
+		// return any other error
+		case err != nil:
+			return err
+
+		// if the header is nil, just skip it (not sure how this happens)
+		case header == nil:
+			continue
+		}
+
+		// the target location where the dir/file should be created
+		target := filepath.Join(dst, header.Name)
+
+		// the following switch could also be done using fi.Mode(), not sure if there
+		// a benefit of using one vs. the other.
+		// fi := header.FileInfo()
+
+		// check the file type
+		switch header.Typeflag {
+
+		// if its a dir and it doesn't exist create it
+		case tar.TypeDir:
+			if _, err := os.Stat(target); err != nil {
+				if err := os.MkdirAll(target, 0755); err != nil {
+					return err
+				}
+			}
+
+		// if it's a file create it
+		case tar.TypeReg:
+			f, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode))
+			if err != nil {
+				return err
+			}
+
+			// copy over contents
+			if _, err := io.Copy(f, tr); err != nil {
+				return err
+			}
+
+			// manually close here after each file operation; defering would cause each file close
+			// to wait until all operations have completed.
+			f.Close()
+		}
+	}
+}