added self hostable geolite2 server with weekly auto update

1 year ago · ba4a075036
parent 8a6aef7430
commit ba4a075036
11 changed files with 403 additions and 13 deletions
--- a/geolite2/.dockerignore
+++ b/geolite2/.dockerignore
@ -0,0 +1,2 @@
+/data/*
+/dist/*
--- a/geolite2/.gitignore
+++ b/geolite2/.gitignore
@ -0,0 +1,4 @@
+/data/*
+!/data/README.txt
+/dist/*
+!/dist/README.txt
--- a/geolite2/Dockerfile
+++ b/geolite2/Dockerfile
@ -0,0 +1,18 @@
+ARG GO_VERSION=1
+FROM golang:${GO_VERSION}-bookworm as builder
+
+WORKDIR /usr/src/app
+COPY go.mod go.sum ./
+RUN go mod download && go mod verify
+COPY . .
+RUN go build -v -o /run-app .
+
+
+FROM debian:bookworm
+
+RUN apt update
+
+RUN apt install -y ca-certificates
+
+COPY --from=builder /run-app /usr/local/bin/
+CMD ["run-app"]
--- a/geolite2/data/README.txt
+++ b/geolite2/data/README.txt
@ -0,0 +1,2 @@
+this directory will contain the geolite2 db file
+the server will only read GeoLite2-City.mmdb
--- a/geolite2/dist/README.txt
+++ b/geolite2/dist/README.txt
@ -0,0 +1,3 @@
+this directory is used to download and extract the latest geolite2 db file
+
+the content including this README will be delete during the update
--- a/geolite2/fly.toml
+++ b/geolite2/fly.toml
@ -0,0 +1,25 @@
+# fly.toml app configuration file generated for geolite2 on 2024-08-04T12:09:00+08:00
+#
+# See https://fly.io/docs/reference/configuration/ for information about how to use this file.
+#
+
+app = 'geolite2'
+primary_region = 'iad'
+
+[build]
+  [build.args]
+    GO_VERSION = '1.22.5'
+
+[env]
+  PORT = '8080'
+
+[http_service]
+  internal_port = 3000
+  force_https = true
+  auto_stop_machines = 'stop'
+  auto_start_machines = true
+  min_machines_running = 0
+  processes = ['app']
+
+[[vm]]
+  size = 'shared-cpu-1x'
--- a/geolite2/go.mod
+++ b/geolite2/go.mod
@ -0,0 +1,24 @@
+module tzgyn/geolite2server
+
+go 1.22.5
+
+require (
+	github.com/gofiber/fiber/v2 v2.52.5
+	github.com/oschwald/geoip2-golang v1.11.0
+	github.com/robfig/cron/v3 v3.0.0
+)
+
+require (
+	github.com/andybalholm/brotli v1.0.5 // indirect
+	github.com/google/uuid v1.5.0 // indirect
+	github.com/klauspost/compress v1.17.0 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/mattn/go-runewidth v0.0.15 // indirect
+	github.com/oschwald/maxminddb-golang v1.13.0 // indirect
+	github.com/rivo/uniseg v0.2.0 // indirect
+	github.com/valyala/bytebufferpool v1.0.0 // indirect
+	github.com/valyala/fasthttp v1.51.0 // indirect
+	github.com/valyala/tcplisten v1.0.0 // indirect
+	golang.org/x/sys v0.20.0 // indirect
+)
--- a/geolite2/go.sum
+++ b/geolite2/go.sum
@ -0,0 +1,41 @@
+github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs=
+github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/gofiber/fiber/v2 v2.52.5 h1:tWoP1MJQjGEe4GB5TUGOi7P2E0ZMMRx5ZTG4rT+yGMo=
+github.com/gofiber/fiber/v2 v2.52.5/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ=
+github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU=
+github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM=
+github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
+github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
+github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
+github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
+github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
+github.com/oschwald/geoip2-golang v1.11.0 h1:hNENhCn1Uyzhf9PTmquXENiWS6AlxAEnBII6r8krA3w=
+github.com/oschwald/geoip2-golang v1.11.0/go.mod h1:P9zG+54KPEFOliZ29i7SeYZ/GM6tfEL+rgSn03hYuUo=
+github.com/oschwald/maxminddb-golang v1.13.0 h1:R8xBorY71s84yO06NgTmQvqvTvlS/bnYZrrWX1MElnU=
+github.com/oschwald/maxminddb-golang v1.13.0/go.mod h1:BU0z8BfFVhi1LQaonTwwGQlsHUEu9pWNdMfmq4ztm0o=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
+github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+github.com/robfig/cron/v3 v3.0.0 h1:kQ6Cb7aHOHTSzNVNEhmp8EcWKLb4CbiMW9h9VyIhO4E=
+github.com/robfig/cron/v3 v3.0.0/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
+github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
+github.com/valyala/fasthttp v1.51.0 h1:8b30A5JlZ6C7AS81RsWjYMQmrZG6feChmgAolCl1SqA=
+github.com/valyala/fasthttp v1.51.0/go.mod h1:oI2XroL+lI7vdXyYoQk03bXBThfFl2cVdIA3Xl7cH8g=
+github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8=
+github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
+golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
+golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
--- a/geolite2/main.go
+++ b/geolite2/main.go
@ -0,0 +1,249 @@
+package main
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"encoding/base64"
+	"errors"
+	"fmt"
+	"io"
+	"log"
+	"net"
+	"net/http"
+	"os"
+	"os/signal"
+	"path/filepath"
+	"syscall"
+
+	"github.com/gofiber/fiber/v2"
+	"github.com/gofiber/fiber/v2/middleware/logger"
+	"github.com/oschwald/geoip2-golang"
+	"github.com/robfig/cron/v3"
+)
+
+func main() {
+	fmt.Println("Initializing GeoLite2 DB...")
+	err := downloadAndExtractDB()
+	if err != nil {
+		log.Fatal(err)
+	}
+	fmt.Println("Finished initializing GeoLite2 DB")
+
+	app := fiber.New()
+
+	app.Use(logger.New())
+
+	app.Get("/", func(c *fiber.Ctx) error {
+		return c.SendString("Hello, World!")
+	})
+
+	app.Get("/:ip", func(c *fiber.Ctx) error {
+		queryIp := c.Params("ip")
+
+		db, err := geoip2.Open("./data/GeoLite2-City.mmdb")
+		if err != nil {
+			log.Fatal(err)
+		}
+		defer db.Close()
+
+		// If you are using strings that may be invalid, check that ip is not nil
+		ip := net.ParseIP(queryIp)
+		if ip == nil {
+			return c.JSON(fiber.Map{
+				"success": false, "message": "Please provide a valid ip",
+			})
+		}
+		record, err := db.City(ip)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		return c.JSON(record)
+	})
+
+	c := cron.New()
+	c.AddFunc("@weekly", func() {
+		fmt.Println("Updating GeoLite2 DB...")
+		err := downloadAndExtractDB()
+		if err != nil {
+			log.Fatal(err)
+		}
+		fmt.Println("Finished updating GeoLite2 DB")
+	})
+
+	c.Start()
+
+	done := make(chan os.Signal, 1)
+	signal.Notify(done, syscall.SIGINT, syscall.SIGTERM, os.Interrupt)
+
+	go func() {
+		<-done
+		fmt.Println("Gracefully shutting down...")
+		_ = app.Shutdown()
+	}()
+
+	if err := app.Listen(":3000"); err != nil {
+		log.Panic(err)
+	}
+
+	fmt.Println("Running cleanup tasks...")
+	fmt.Println("Stopping cronjobs...")
+	c.Stop()
+}
+
+func downloadAndExtractDB() error {
+	_, err := os.ReadDir("./data")
+	if err != nil {
+		os.MkdirAll("./data", os.ModePerm)
+	}
+
+	_, err = os.ReadDir("./dist")
+	if err != nil {
+		os.MkdirAll("./dist", os.ModePerm)
+	}
+
+	err = downloadDB()
+	if err != nil {
+		return err
+	}
+
+	r, err := os.Open("./db.tar.gz")
+	if err != nil {
+		return err
+	}
+
+	err = Untar("./dist", r)
+	if err != nil {
+		return err
+	}
+
+	files, err := os.ReadDir("./dist")
+	if err != nil {
+		return err
+	}
+
+	fileInfo := files[0]
+
+	err = os.Rename("./dist/"+fileInfo.Name()+"/GeoLite2-City.mmdb", "./data/GeoLite2-City.mmdb")
+	if err != nil {
+		return err
+	}
+
+	err = os.RemoveAll("./dist/")
+	if err != nil {
+		return err
+	}
+
+	err = os.MkdirAll("./dist/", os.ModePerm)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func downloadDB() error {
+	accountID := os.Getenv("ACCOUNT_ID")
+	if len(accountID) == 0 {
+		return errors.New("please provide ACCOUNT_ID as env")
+	}
+
+	licenseKey := os.Getenv("LICENSE_KEY")
+	if len(licenseKey) == 0 {
+		return errors.New("please provide LICENSE_KEY as env")
+	}
+
+	out, err := os.Create("db.tar.gz")
+	if err != nil {
+		return err
+	}
+
+	defer out.Close()
+
+	auth := base64.StdEncoding.EncodeToString([]byte(accountID + ":" + licenseKey))
+
+	req, err := http.NewRequest(http.MethodGet, "https://download.maxmind.com/geoip/databases/GeoLite2-City/download?suffix=tar.gz", nil)
+	if err != nil {
+		return err
+	}
+	req.Header.Add("Authorization", "Basic "+auth)
+
+	res, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return err
+	}
+
+	defer res.Body.Close()
+
+	_, err = io.Copy(out, res.Body)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func Untar(dst string, r io.Reader) error {
+
+	gzr, err := gzip.NewReader(r)
+	if err != nil {
+		return err
+	}
+	defer gzr.Close()
+
+	tr := tar.NewReader(gzr)
+
+	for {
+		header, err := tr.Next()
+
+		switch {
+
+		// if no more files are found return
+		case err == io.EOF:
+			return nil
+
+		// return any other error
+		case err != nil:
+			return err
+
+		// if the header is nil, just skip it (not sure how this happens)
+		case header == nil:
+			continue
+		}
+
+		// the target location where the dir/file should be created
+		target := filepath.Join(dst, header.Name)
+
+		// the following switch could also be done using fi.Mode(), not sure if there
+		// a benefit of using one vs. the other.
+		// fi := header.FileInfo()
+
+		// check the file type
+		switch header.Typeflag {
+
+		// if its a dir and it doesn't exist create it
+		case tar.TypeDir:
+			if _, err := os.Stat(target); err != nil {
+				if err := os.MkdirAll(target, 0755); err != nil {
+					return err
+				}
+			}
+
+		// if it's a file create it
+		case tar.TypeReg:
+			f, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode))
+			if err != nil {
+				return err
+			}
+
+			// copy over contents
+			if _, err := io.Copy(f, tr); err != nil {
+				return err
+			}
+
+			// manually close here after each file operation; defering would cause each file close
+			// to wait until all operations have completed.
+			f.Close()
+		}
+	}
+}
--- a/redirect/.env.example
+++ b/redirect/.env.example
@ -1,5 +1,8 @@
 FALLBACK_URL=https://app.kon.sh
 DATABASE_URL=postgres://postgres:password@0.0.0.0:5432/link-shortener
 APP_URL=kon.sh
+GEOIPAPI= # self host geolite2 api url
+
+# Geolite2 web service (1000 request per day limit for free account)
 GEOIPUPDATE_ACCOUNT_ID= 
 GEOIPUPDATE_LICENSE_KEY=
--- a/redirect/src/index.ts
+++ b/redirect/src/index.ts
@ -12,12 +12,6 @@ const fallback_url = Bun.env.FALLBACK_URL ?? 'https://app.kon.sh'
 const app_url = Bun.env.APP_URL ?? 'kon.sh'
 const hosting_provider = Bun.env.HOSTING_PROVIDER

-const client = new WebServiceClient(
-	Bun.env.GEOIPUPDATE_ACCOUNT_ID || '',
-	Bun.env.GEOIPUPDATE_LICENSE_KEY || '',
-	{ host: 'geolite.info' }
-)
-
 const clickLimiter = new LRUCache({
 	ttl: 60 * 60 * 60 * 1000, // 1 hr
 	ttlAutopurge: true,
@ -91,20 +85,45 @@ app.get(

 			clickLimiter.set(clickKey, 1)

-			const geolocation = await client.city(ip || '')
+			const geo = {
+				country: '',
+				country_code: '',
+				city: '',
+			}
+			if (Bun.env.GEOIPAPI) {
+				const response = await fetch(Bun.env.GEOIPAPI + '/' + ip)
+				const data = await response.json()
+
+				if (data.success && data.success == false) {
+					return
+				}
+				geo.country = data.Country.Names?.en || ''
+				geo.country_code = data.Country.IsoCode
+				geo.city = data.City.Names?.en || ''
+			} else {
+				const client = new WebServiceClient(
+					Bun.env.GEOIPUPDATE_ACCOUNT_ID || '',
+					Bun.env.GEOIPUPDATE_LICENSE_KEY || '',
+					{ host: 'geolite.info' }
+				)
+				const geolocation = await client.city(ip || '')
+				geo.country = geolocation.country?.names?.en || ''
+				geo.country_code = geolocation.country?.isoCode || ''
+				geo.city = geolocation.city?.names?.en || ''
+			}

 			const visitor_data = {
 				shortener_id: shortener[0].id,
-				country: geolocation.country!.names.en,
-				country_code: geolocation.country!.isoCode,
-				city: geolocation.city!.names.en,
 				device_type: ua_parser.getDevice().type || 'desktop',
 				device_vendor: ua_parser.getDevice().vendor,
 				browser: ua_parser.getBrowser().name,
 				os: ua_parser.getOS().name,
 			}

-			await db.insertInto('visitor').values(visitor_data).execute()
+			await db
+				.insertInto('visitor')
+				.values({ ...visitor_data, ...geo })
+				.execute()
 		} catch (error) {
 			console.error(error)
 			set.redirect = fallback_url