From ba4a07503609121604411cd3fa730fffc5ac2b91 Mon Sep 17 00:00:00 2001 From: TZGyn Date: Sun, 4 Aug 2024 15:35:31 +0800 Subject: [PATCH] added self hostable geolite2 server with weekly auto update --- geolite2/.dockerignore | 2 + geolite2/.gitignore | 4 + geolite2/Dockerfile | 18 +++ geolite2/data/README.txt | 2 + geolite2/dist/README.txt | 3 + geolite2/fly.toml | 25 ++++ geolite2/go.mod | 24 ++++ geolite2/go.sum | 41 +++++++ geolite2/main.go | 249 +++++++++++++++++++++++++++++++++++++++ redirect/.env.example | 7 +- redirect/src/index.ts | 41 +++++-- 11 files changed, 403 insertions(+), 13 deletions(-) create mode 100644 geolite2/.dockerignore create mode 100644 geolite2/.gitignore create mode 100644 geolite2/Dockerfile create mode 100644 geolite2/data/README.txt create mode 100644 geolite2/dist/README.txt create mode 100644 geolite2/fly.toml create mode 100644 geolite2/go.mod create mode 100644 geolite2/go.sum create mode 100644 geolite2/main.go diff --git a/geolite2/.dockerignore b/geolite2/.dockerignore new file mode 100644 index 0000000..2eceffd --- /dev/null +++ b/geolite2/.dockerignore @@ -0,0 +1,2 @@ +/data/* +/dist/* \ No newline at end of file diff --git a/geolite2/.gitignore b/geolite2/.gitignore new file mode 100644 index 0000000..8143896 --- /dev/null +++ b/geolite2/.gitignore @@ -0,0 +1,4 @@ +/data/* +!/data/README.txt +/dist/* +!/dist/README.txt \ No newline at end of file diff --git a/geolite2/Dockerfile b/geolite2/Dockerfile new file mode 100644 index 0000000..fc4e491 --- /dev/null +++ b/geolite2/Dockerfile @@ -0,0 +1,18 @@ +ARG GO_VERSION=1 +FROM golang:${GO_VERSION}-bookworm as builder + +WORKDIR /usr/src/app +COPY go.mod go.sum ./ +RUN go mod download && go mod verify +COPY . . +RUN go build -v -o /run-app . + + +FROM debian:bookworm + +RUN apt update + +RUN apt install -y ca-certificates + +COPY --from=builder /run-app /usr/local/bin/ +CMD ["run-app"] diff --git a/geolite2/data/README.txt b/geolite2/data/README.txt new file mode 100644 index 0000000..a3503dc --- /dev/null +++ b/geolite2/data/README.txt @@ -0,0 +1,2 @@ +this directory will contain the geolite2 db file +the server will only read GeoLite2-City.mmdb \ No newline at end of file diff --git a/geolite2/dist/README.txt b/geolite2/dist/README.txt new file mode 100644 index 0000000..4e65ab0 --- /dev/null +++ b/geolite2/dist/README.txt @@ -0,0 +1,3 @@ +this directory is used to download and extract the latest geolite2 db file + +the content including this README will be delete during the update \ No newline at end of file diff --git a/geolite2/fly.toml b/geolite2/fly.toml new file mode 100644 index 0000000..37dec38 --- /dev/null +++ b/geolite2/fly.toml @@ -0,0 +1,25 @@ +# fly.toml app configuration file generated for geolite2 on 2024-08-04T12:09:00+08:00 +# +# See https://fly.io/docs/reference/configuration/ for information about how to use this file. +# + +app = 'geolite2' +primary_region = 'iad' + +[build] + [build.args] + GO_VERSION = '1.22.5' + +[env] + PORT = '8080' + +[http_service] + internal_port = 3000 + force_https = true + auto_stop_machines = 'stop' + auto_start_machines = true + min_machines_running = 0 + processes = ['app'] + +[[vm]] + size = 'shared-cpu-1x' diff --git a/geolite2/go.mod b/geolite2/go.mod new file mode 100644 index 0000000..ac4b523 --- /dev/null +++ b/geolite2/go.mod @@ -0,0 +1,24 @@ +module tzgyn/geolite2server + +go 1.22.5 + +require ( + github.com/gofiber/fiber/v2 v2.52.5 + github.com/oschwald/geoip2-golang v1.11.0 + github.com/robfig/cron/v3 v3.0.0 +) + +require ( + github.com/andybalholm/brotli v1.0.5 // indirect + github.com/google/uuid v1.5.0 // indirect + github.com/klauspost/compress v1.17.0 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-runewidth v0.0.15 // indirect + github.com/oschwald/maxminddb-golang v1.13.0 // indirect + github.com/rivo/uniseg v0.2.0 // indirect + github.com/valyala/bytebufferpool v1.0.0 // indirect + github.com/valyala/fasthttp v1.51.0 // indirect + github.com/valyala/tcplisten v1.0.0 // indirect + golang.org/x/sys v0.20.0 // indirect +) diff --git a/geolite2/go.sum b/geolite2/go.sum new file mode 100644 index 0000000..c47327e --- /dev/null +++ b/geolite2/go.sum @@ -0,0 +1,41 @@ +github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= +github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/gofiber/fiber/v2 v2.52.5 h1:tWoP1MJQjGEe4GB5TUGOi7P2E0ZMMRx5ZTG4rT+yGMo= +github.com/gofiber/fiber/v2 v2.52.5/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ= +github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= +github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM= +github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= +github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/oschwald/geoip2-golang v1.11.0 h1:hNENhCn1Uyzhf9PTmquXENiWS6AlxAEnBII6r8krA3w= +github.com/oschwald/geoip2-golang v1.11.0/go.mod h1:P9zG+54KPEFOliZ29i7SeYZ/GM6tfEL+rgSn03hYuUo= +github.com/oschwald/maxminddb-golang v1.13.0 h1:R8xBorY71s84yO06NgTmQvqvTvlS/bnYZrrWX1MElnU= +github.com/oschwald/maxminddb-golang v1.13.0/go.mod h1:BU0z8BfFVhi1LQaonTwwGQlsHUEu9pWNdMfmq4ztm0o= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/robfig/cron/v3 v3.0.0 h1:kQ6Cb7aHOHTSzNVNEhmp8EcWKLb4CbiMW9h9VyIhO4E= +github.com/robfig/cron/v3 v3.0.0/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasthttp v1.51.0 h1:8b30A5JlZ6C7AS81RsWjYMQmrZG6feChmgAolCl1SqA= +github.com/valyala/fasthttp v1.51.0/go.mod h1:oI2XroL+lI7vdXyYoQk03bXBThfFl2cVdIA3Xl7cH8g= +github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8= +github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/geolite2/main.go b/geolite2/main.go new file mode 100644 index 0000000..ff6391e --- /dev/null +++ b/geolite2/main.go @@ -0,0 +1,249 @@ +package main + +import ( + "archive/tar" + "compress/gzip" + "encoding/base64" + "errors" + "fmt" + "io" + "log" + "net" + "net/http" + "os" + "os/signal" + "path/filepath" + "syscall" + + "github.com/gofiber/fiber/v2" + "github.com/gofiber/fiber/v2/middleware/logger" + "github.com/oschwald/geoip2-golang" + "github.com/robfig/cron/v3" +) + +func main() { + fmt.Println("Initializing GeoLite2 DB...") + err := downloadAndExtractDB() + if err != nil { + log.Fatal(err) + } + fmt.Println("Finished initializing GeoLite2 DB") + + app := fiber.New() + + app.Use(logger.New()) + + app.Get("/", func(c *fiber.Ctx) error { + return c.SendString("Hello, World!") + }) + + app.Get("/:ip", func(c *fiber.Ctx) error { + queryIp := c.Params("ip") + + db, err := geoip2.Open("./data/GeoLite2-City.mmdb") + if err != nil { + log.Fatal(err) + } + defer db.Close() + + // If you are using strings that may be invalid, check that ip is not nil + ip := net.ParseIP(queryIp) + if ip == nil { + return c.JSON(fiber.Map{ + "success": false, "message": "Please provide a valid ip", + }) + } + record, err := db.City(ip) + if err != nil { + log.Fatal(err) + } + + return c.JSON(record) + }) + + c := cron.New() + c.AddFunc("@weekly", func() { + fmt.Println("Updating GeoLite2 DB...") + err := downloadAndExtractDB() + if err != nil { + log.Fatal(err) + } + fmt.Println("Finished updating GeoLite2 DB") + }) + + c.Start() + + done := make(chan os.Signal, 1) + signal.Notify(done, syscall.SIGINT, syscall.SIGTERM, os.Interrupt) + + go func() { + <-done + fmt.Println("Gracefully shutting down...") + _ = app.Shutdown() + }() + + if err := app.Listen(":3000"); err != nil { + log.Panic(err) + } + + fmt.Println("Running cleanup tasks...") + fmt.Println("Stopping cronjobs...") + c.Stop() +} + +func downloadAndExtractDB() error { + _, err := os.ReadDir("./data") + if err != nil { + os.MkdirAll("./data", os.ModePerm) + } + + _, err = os.ReadDir("./dist") + if err != nil { + os.MkdirAll("./dist", os.ModePerm) + } + + err = downloadDB() + if err != nil { + return err + } + + r, err := os.Open("./db.tar.gz") + if err != nil { + return err + } + + err = Untar("./dist", r) + if err != nil { + return err + } + + files, err := os.ReadDir("./dist") + if err != nil { + return err + } + + fileInfo := files[0] + + err = os.Rename("./dist/"+fileInfo.Name()+"/GeoLite2-City.mmdb", "./data/GeoLite2-City.mmdb") + if err != nil { + return err + } + + err = os.RemoveAll("./dist/") + if err != nil { + return err + } + + err = os.MkdirAll("./dist/", os.ModePerm) + if err != nil { + return err + } + + return nil +} + +func downloadDB() error { + accountID := os.Getenv("ACCOUNT_ID") + if len(accountID) == 0 { + return errors.New("please provide ACCOUNT_ID as env") + } + + licenseKey := os.Getenv("LICENSE_KEY") + if len(licenseKey) == 0 { + return errors.New("please provide LICENSE_KEY as env") + } + + out, err := os.Create("db.tar.gz") + if err != nil { + return err + } + + defer out.Close() + + auth := base64.StdEncoding.EncodeToString([]byte(accountID + ":" + licenseKey)) + + req, err := http.NewRequest(http.MethodGet, "https://download.maxmind.com/geoip/databases/GeoLite2-City/download?suffix=tar.gz", nil) + if err != nil { + return err + } + req.Header.Add("Authorization", "Basic "+auth) + + res, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + + defer res.Body.Close() + + _, err = io.Copy(out, res.Body) + if err != nil { + return err + } + + return nil +} + +func Untar(dst string, r io.Reader) error { + + gzr, err := gzip.NewReader(r) + if err != nil { + return err + } + defer gzr.Close() + + tr := tar.NewReader(gzr) + + for { + header, err := tr.Next() + + switch { + + // if no more files are found return + case err == io.EOF: + return nil + + // return any other error + case err != nil: + return err + + // if the header is nil, just skip it (not sure how this happens) + case header == nil: + continue + } + + // the target location where the dir/file should be created + target := filepath.Join(dst, header.Name) + + // the following switch could also be done using fi.Mode(), not sure if there + // a benefit of using one vs. the other. + // fi := header.FileInfo() + + // check the file type + switch header.Typeflag { + + // if its a dir and it doesn't exist create it + case tar.TypeDir: + if _, err := os.Stat(target); err != nil { + if err := os.MkdirAll(target, 0755); err != nil { + return err + } + } + + // if it's a file create it + case tar.TypeReg: + f, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode)) + if err != nil { + return err + } + + // copy over contents + if _, err := io.Copy(f, tr); err != nil { + return err + } + + // manually close here after each file operation; defering would cause each file close + // to wait until all operations have completed. + f.Close() + } + } +} diff --git a/redirect/.env.example b/redirect/.env.example index 9c42531..cb63fa7 100644 --- a/redirect/.env.example +++ b/redirect/.env.example @@ -1,5 +1,8 @@ FALLBACK_URL=https://app.kon.sh DATABASE_URL=postgres://postgres:password@0.0.0.0:5432/link-shortener APP_URL=kon.sh -GEOIPUPDATE_ACCOUNT_ID= -GEOIPUPDATE_LICENSE_KEY= +GEOIPAPI= # self host geolite2 api url + +# Geolite2 web service (1000 request per day limit for free account) +GEOIPUPDATE_ACCOUNT_ID= +GEOIPUPDATE_LICENSE_KEY= \ No newline at end of file diff --git a/redirect/src/index.ts b/redirect/src/index.ts index a2f65de..afd35e6 100644 --- a/redirect/src/index.ts +++ b/redirect/src/index.ts @@ -12,12 +12,6 @@ const fallback_url = Bun.env.FALLBACK_URL ?? 'https://app.kon.sh' const app_url = Bun.env.APP_URL ?? 'kon.sh' const hosting_provider = Bun.env.HOSTING_PROVIDER -const client = new WebServiceClient( - Bun.env.GEOIPUPDATE_ACCOUNT_ID || '', - Bun.env.GEOIPUPDATE_LICENSE_KEY || '', - { host: 'geolite.info' } -) - const clickLimiter = new LRUCache({ ttl: 60 * 60 * 60 * 1000, // 1 hr ttlAutopurge: true, @@ -91,20 +85,45 @@ app.get( clickLimiter.set(clickKey, 1) - const geolocation = await client.city(ip || '') + const geo = { + country: '', + country_code: '', + city: '', + } + if (Bun.env.GEOIPAPI) { + const response = await fetch(Bun.env.GEOIPAPI + '/' + ip) + const data = await response.json() + + if (data.success && data.success == false) { + return + } + geo.country = data.Country.Names?.en || '' + geo.country_code = data.Country.IsoCode + geo.city = data.City.Names?.en || '' + } else { + const client = new WebServiceClient( + Bun.env.GEOIPUPDATE_ACCOUNT_ID || '', + Bun.env.GEOIPUPDATE_LICENSE_KEY || '', + { host: 'geolite.info' } + ) + const geolocation = await client.city(ip || '') + geo.country = geolocation.country?.names?.en || '' + geo.country_code = geolocation.country?.isoCode || '' + geo.city = geolocation.city?.names?.en || '' + } const visitor_data = { shortener_id: shortener[0].id, - country: geolocation.country!.names.en, - country_code: geolocation.country!.isoCode, - city: geolocation.city!.names.en, device_type: ua_parser.getDevice().type || 'desktop', device_vendor: ua_parser.getDevice().vendor, browser: ua_parser.getBrowser().name, os: ua_parser.getOS().name, } - await db.insertInto('visitor').values(visitor_data).execute() + await db + .insertInto('visitor') + .values({ ...visitor_data, ...geo }) + .execute() } catch (error) { console.error(error) set.redirect = fallback_url