From 74c5a1b70f7130cfa1a62dde2243598f673ea989 Mon Sep 17 00:00:00 2001 From: TZGyn Date: Tue, 6 Aug 2024 21:03:23 +0800 Subject: [PATCH] added geolite2 server --- .dockerignore | 6 ++ .gitignore | 4 + Dockerfile | 18 ++++ data/README.txt | 2 + dist/README.txt | 3 + fly.toml | 25 +++++ go.mod | 24 +++++ go.sum | 41 ++++++++ main.go | 274 ++++++++++++++++++++++++++++++++++++++++++++++++ 9 files changed, 397 insertions(+) create mode 100644 .dockerignore create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 data/README.txt create mode 100644 dist/README.txt create mode 100644 fly.toml create mode 100644 go.mod create mode 100644 go.sum create mode 100644 main.go diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..30dafff --- /dev/null +++ b/.dockerignore @@ -0,0 +1,6 @@ +# flyctl launch added from .gitignore +data/* +!data/README.txt +dist/* +!dist/README.txt +fly.toml diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8143896 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +/data/* +!/data/README.txt +/dist/* +!/dist/README.txt \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..fc4e491 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,18 @@ +ARG GO_VERSION=1 +FROM golang:${GO_VERSION}-bookworm as builder + +WORKDIR /usr/src/app +COPY go.mod go.sum ./ +RUN go mod download && go mod verify +COPY . . +RUN go build -v -o /run-app . + + +FROM debian:bookworm + +RUN apt update + +RUN apt install -y ca-certificates + +COPY --from=builder /run-app /usr/local/bin/ +CMD ["run-app"] diff --git a/data/README.txt b/data/README.txt new file mode 100644 index 0000000..a3503dc --- /dev/null +++ b/data/README.txt @@ -0,0 +1,2 @@ +this directory will contain the geolite2 db file +the server will only read GeoLite2-City.mmdb \ No newline at end of file diff --git a/dist/README.txt b/dist/README.txt new file mode 100644 index 0000000..4e65ab0 --- /dev/null +++ b/dist/README.txt @@ -0,0 +1,3 @@ +this directory is used to download and extract the latest geolite2 db file + +the content including this README will be delete during the update \ No newline at end of file diff --git a/fly.toml b/fly.toml new file mode 100644 index 0000000..0657bd3 --- /dev/null +++ b/fly.toml @@ -0,0 +1,25 @@ +# fly.toml app configuration file generated for geoip on 2024-08-06T18:24:17+08:00 +# +# See https://fly.io/docs/reference/configuration/ for information about how to use this file. +# + +app = 'geoip' +primary_region = 'iad' + +[build] + [build.args] + GO_VERSION = '1.22.5' + +[env] + PORT = '8080' + +[http_service] + internal_port = 3000 + force_https = true + auto_stop_machines = 'stop' + auto_start_machines = true + min_machines_running = 0 + processes = ['app'] + +[[vm]] + size = 'shared-cpu-1x' diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..ac4b523 --- /dev/null +++ b/go.mod @@ -0,0 +1,24 @@ +module tzgyn/geolite2server + +go 1.22.5 + +require ( + github.com/gofiber/fiber/v2 v2.52.5 + github.com/oschwald/geoip2-golang v1.11.0 + github.com/robfig/cron/v3 v3.0.0 +) + +require ( + github.com/andybalholm/brotli v1.0.5 // indirect + github.com/google/uuid v1.5.0 // indirect + github.com/klauspost/compress v1.17.0 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-runewidth v0.0.15 // indirect + github.com/oschwald/maxminddb-golang v1.13.0 // indirect + github.com/rivo/uniseg v0.2.0 // indirect + github.com/valyala/bytebufferpool v1.0.0 // indirect + github.com/valyala/fasthttp v1.51.0 // indirect + github.com/valyala/tcplisten v1.0.0 // indirect + golang.org/x/sys v0.20.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..c47327e --- /dev/null +++ b/go.sum @@ -0,0 +1,41 @@ +github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= +github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/gofiber/fiber/v2 v2.52.5 h1:tWoP1MJQjGEe4GB5TUGOi7P2E0ZMMRx5ZTG4rT+yGMo= +github.com/gofiber/fiber/v2 v2.52.5/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ= +github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= +github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM= +github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= +github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/oschwald/geoip2-golang v1.11.0 h1:hNENhCn1Uyzhf9PTmquXENiWS6AlxAEnBII6r8krA3w= +github.com/oschwald/geoip2-golang v1.11.0/go.mod h1:P9zG+54KPEFOliZ29i7SeYZ/GM6tfEL+rgSn03hYuUo= +github.com/oschwald/maxminddb-golang v1.13.0 h1:R8xBorY71s84yO06NgTmQvqvTvlS/bnYZrrWX1MElnU= +github.com/oschwald/maxminddb-golang v1.13.0/go.mod h1:BU0z8BfFVhi1LQaonTwwGQlsHUEu9pWNdMfmq4ztm0o= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/robfig/cron/v3 v3.0.0 h1:kQ6Cb7aHOHTSzNVNEhmp8EcWKLb4CbiMW9h9VyIhO4E= +github.com/robfig/cron/v3 v3.0.0/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasthttp v1.51.0 h1:8b30A5JlZ6C7AS81RsWjYMQmrZG6feChmgAolCl1SqA= +github.com/valyala/fasthttp v1.51.0/go.mod h1:oI2XroL+lI7vdXyYoQk03bXBThfFl2cVdIA3Xl7cH8g= +github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8= +github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/main.go b/main.go new file mode 100644 index 0000000..7c17033 --- /dev/null +++ b/main.go @@ -0,0 +1,274 @@ +package main + +import ( + "archive/tar" + "compress/gzip" + "encoding/base64" + "errors" + "fmt" + "io" + "log" + "net" + "net/http" + "os" + "os/signal" + "path/filepath" + "syscall" + + "github.com/gofiber/fiber/v2" + "github.com/gofiber/fiber/v2/middleware/logger" + "github.com/oschwald/geoip2-golang" + "github.com/robfig/cron/v3" +) + +func main() { + fmt.Println("Initializing GeoLite2 DB...") + err := downloadAndExtractDB() + if err != nil { + log.Fatal(err) + } + + db, err := geoip2.Open("./data/GeoLite2-City.mmdb") + if err != nil { + log.Fatal("Failed to initialize GeoLite2 DB") + } + + fmt.Println("Finished initializing GeoLite2 DB") + + app := fiber.New() + + app.Use(logger.New()) + + app.Get("/", func(c *fiber.Ctx) error { + return c.SendString("Hello, World!") + }) + + app.Get("/me", func(c *fiber.Ctx) error { + ip := c.IPs()[0] + + record, err := getCity(db, ip) + if err != nil { + c.JSON(fiber.Map{ + "success": false, + "message": err.Error(), + }) + } + + return c.JSON(record) + }) + + app.Get("/:ip", func(c *fiber.Ctx) error { + queryIp := c.Params("ip") + + record, err := getCity(db, queryIp) + if err != nil { + c.JSON(fiber.Map{ + "success": false, + "message": err.Error(), + }) + } + + return c.JSON(record) + }) + + c := cron.New() + c.AddFunc("@weekly", func() { + fmt.Println("Updating GeoLite2 DB...") + err := downloadAndExtractDB() + if err != nil { + log.Fatal(err) + } + fmt.Println("Finished updating GeoLite2 DB") + }) + + c.Start() + + done := make(chan os.Signal, 1) + signal.Notify(done, syscall.SIGINT, syscall.SIGTERM, os.Interrupt) + + go func() { + <-done + fmt.Println("Gracefully shutting down...") + _ = app.Shutdown() + }() + + if err := app.Listen(":3000"); err != nil { + log.Panic(err) + } + + fmt.Println("Running cleanup tasks...") + fmt.Println("Stopping cronjobs...") + c.Stop() + db.Close() +} + +func getCity(db *geoip2.Reader, queryIp string) (*geoip2.City, error) { + // If you are using strings that may be invalid, check that ip is not nil + ip := net.ParseIP(queryIp) + if ip == nil { + return nil, errors.New("invalid ip") + } + + record, err := db.City(ip) + if err != nil { + return nil, errors.New("no data") + } + return record, nil +} + +func downloadAndExtractDB() error { + _, err := os.ReadDir("./data") + if err != nil { + os.MkdirAll("./data", os.ModePerm) + } + + _, err = os.ReadDir("./dist") + if err != nil { + os.MkdirAll("./dist", os.ModePerm) + } + + err = downloadDB() + if err != nil { + return err + } + + r, err := os.Open("./db.tar.gz") + if err != nil { + return err + } + + err = Untar("./dist", r) + if err != nil { + return err + } + + files, err := os.ReadDir("./dist") + if err != nil { + return err + } + + fileInfo := files[0] + + err = os.Rename("./dist/"+fileInfo.Name()+"/GeoLite2-City.mmdb", "./data/GeoLite2-City.mmdb") + if err != nil { + return err + } + + err = os.RemoveAll("./dist/") + if err != nil { + return err + } + + err = os.MkdirAll("./dist/", os.ModePerm) + if err != nil { + return err + } + + return nil +} + +func downloadDB() error { + accountID := os.Getenv("ACCOUNT_ID") + if len(accountID) == 0 { + return errors.New("please provide ACCOUNT_ID as env") + } + + licenseKey := os.Getenv("LICENSE_KEY") + if len(licenseKey) == 0 { + return errors.New("please provide LICENSE_KEY as env") + } + + out, err := os.Create("db.tar.gz") + if err != nil { + return err + } + + defer out.Close() + + auth := base64.StdEncoding.EncodeToString([]byte(accountID + ":" + licenseKey)) + + req, err := http.NewRequest(http.MethodGet, "https://download.maxmind.com/geoip/databases/GeoLite2-City/download?suffix=tar.gz", nil) + if err != nil { + return err + } + req.Header.Add("Authorization", "Basic "+auth) + + res, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + + defer res.Body.Close() + + _, err = io.Copy(out, res.Body) + if err != nil { + return err + } + + return nil +} + +func Untar(dst string, r io.Reader) error { + + gzr, err := gzip.NewReader(r) + if err != nil { + return err + } + defer gzr.Close() + + tr := tar.NewReader(gzr) + + for { + header, err := tr.Next() + + switch { + + // if no more files are found return + case err == io.EOF: + return nil + + // return any other error + case err != nil: + return err + + // if the header is nil, just skip it (not sure how this happens) + case header == nil: + continue + } + + // the target location where the dir/file should be created + target := filepath.Join(dst, header.Name) + + // the following switch could also be done using fi.Mode(), not sure if there + // a benefit of using one vs. the other. + // fi := header.FileInfo() + + // check the file type + switch header.Typeflag { + + // if its a dir and it doesn't exist create it + case tar.TypeDir: + if _, err := os.Stat(target); err != nil { + if err := os.MkdirAll(target, 0755); err != nil { + return err + } + } + + // if it's a file create it + case tar.TypeReg: + f, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode)) + if err != nil { + return err + } + + // copy over contents + if _, err := io.Copy(f, tr); err != nil { + return err + } + + // manually close here after each file operation; defering would cause each file close + // to wait until all operations have completed. + f.Close() + } + } +}