feat: save location information in logs

Signed-off-by: Arnav Gupta <championswimmer@gmail.com>

.gitignore

@@ -194,3 +194,4 @@ fabric.properties
 /events.db
 /events.db.wal
 /duckdb-driver
+/GeoLite2-City.mmdb

go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/golang-jwt/jwt/v5 v5.2.0
 	github.com/google/uuid v1.5.0
 	github.com/joho/godotenv v1.5.1
+	github.com/oschwald/geoip2-golang v1.9.0
 	github.com/samber/lo v1.39.0
 	github.com/stretchr/testify v1.8.4
 	github.com/swaggo/swag v1.16.2
@@ -47,6 +48,7 @@ require (
 	github.com/mattn/go-runewidth v0.0.15 // indirect
 	github.com/mattn/go-sqlite3 v1.14.22 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/oschwald/maxminddb-golang v1.11.0 // indirect
 	github.com/paulmach/orb v0.11.0 // indirect
 	github.com/pierrec/lz4/v4 v4.1.21 // indirect
 	github.com/pkg/errors v0.9.1 // indirect

go.sum

@@ -749,8 +749,6 @@ github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghf
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/championswimmer/duckdb-driver v0.1.0/go.mod h1:wkHe/zl2ERZDFN/G+ZGVe9Ip+QWPwbsI4dXrRixBRc4=
-github.com/championswimmer/duckdb-driver v0.2.0/go.mod h1:zoGrzT9RpdOfCEOCDx4fZ1s87Rpx3D1QUIEzwxHWGLo=
 github.com/championswimmer/duckdb-driver v0.2.1 h1:1wjIhRLE3NQjwMLntAPRXw5IdkxdMPkDxxr7i3vW4vE=
 github.com/championswimmer/duckdb-driver v0.2.1/go.mod h1:zoGrzT9RpdOfCEOCDx4fZ1s87Rpx3D1QUIEzwxHWGLo=
 github.com/checkpoint-restore/go-criu/v4 v4.1.0/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw=
@@ -1588,6 +1586,10 @@ github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuh
 github.com/opencontainers/selinux v1.10.1/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI=
 github.com/opencontainers/selinux v1.11.0/go.mod h1:E5dMC3VPuVvVHDYmi78qvhJp8+M586T4DlDRYpFkyec=
 github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
+github.com/oschwald/geoip2-golang v1.9.0 h1:uvD3O6fXAXs+usU+UGExshpdP13GAqp4GBrzN7IgKZc=
+github.com/oschwald/geoip2-golang v1.9.0/go.mod h1:BHK6TvDyATVQhKNbQBdrj9eAvuwOMi2zSFXizL3K81Y=
+github.com/oschwald/maxminddb-golang v1.11.0 h1:aSXMqYR/EPNjGE8epgqwDay+P30hCBZIveY0WZbAWh0=
+github.com/oschwald/maxminddb-golang v1.11.0/go.mod h1:YmVI+H0zh3ySFR3w+oz8PCfglAFj3PuCmui13+P9zDg=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pascaldekloe/name v1.0.0/go.mod h1:Z//MfYJnH4jVpQ9wkclwu2I2MkHmXTlT9wR5UZScttM=
 github.com/pascaldekloe/name v1.0.1/go.mod h1:Z//MfYJnH4jVpQ9wkclwu2I2MkHmXTlT9wR5UZScttM=
@@ -2815,12 +2817,8 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gorm.io/driver/clickhouse v0.6.0 h1:nyhaeQ92qFEqf47B5N/vwPnnqV2DAuSHPC0QmlZrVZI=
 gorm.io/driver/clickhouse v0.6.0/go.mod h1:UtkbKNA4ibWTCzVkuFY80hBsb82nTH335JUVUKvT9YY=
-gorm.io/driver/postgres v1.5.4 h1:Iyrp9Meh3GmbSuyIAGyjkN+n9K+GHX9b9MqsTL4EJCo=
-gorm.io/driver/postgres v1.5.4/go.mod h1:Bgo89+h0CRcdA33Y6frlaHHVuTdOf87pmyzwW9C/BH0=
 gorm.io/driver/postgres v1.5.6 h1:ydr9xEd5YAM0vxVDY0X139dyzNz10spDiDlC7+ibLeU=
 gorm.io/driver/postgres v1.5.6/go.mod h1:3e019WlBaYI5o5LIdNV+LyxCMNtLOQETBXL2h4chKpA=
-gorm.io/driver/sqlite v1.5.4 h1:IqXwXi8M/ZlPzH/947tn5uik3aYQslP9BVveoax0nV0=
-gorm.io/driver/sqlite v1.5.4/go.mod h1:qxAuCol+2r6PannQDpOP1FP6ag3mKi4esLnB/jHed+4=
 gorm.io/driver/sqlite v1.5.5 h1:7MDMtUZhV065SilG62E0MquljeArQZNfJnjd9i9gx3E=
 gorm.io/driver/sqlite v1.5.5/go.mod h1:6NgQ7sQWAIFsPrJJl1lSNSu2TABh0ZZ/zm5fosATavE=
 gorm.io/gorm v1.24.6/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k=

src/config/consts.go

@@ -0,0 +1,5 @@
+package config
+
+const (
+	LOCALS_USER = "user"
+)

src/controllers/events.go

@@ -1,9 +1,12 @@
 package controllers
 
 import (
+	"fmt"
 	"github.com/google/uuid"
+	"github.com/oschwald/geoip2-golang"
 	"github.com/samber/lo"
 	"gorm.io/gorm"
+	"net"
 	"onepixel_backend/src/db"
 	"onepixel_backend/src/db/models"
 	"onepixel_backend/src/utils/applogger"
@@ -12,12 +15,13 @@ import (
 type EventsController struct {
 	// event logging eventDb (not the main app eventDb)
 	eventDb *gorm.DB
+	geoipDB *geoip2.Reader
 }
 
 func CreateEventsController() *EventsController {
-	eventsDB := lo.Must(db.GetEventsDB())
 	return &EventsController{
-		eventDb: eventsDB,
+		eventDb: db.GetEventsDB(),
+		geoipDB: db.GetGeoIPDB(),
 	}
 }
 
@@ -34,6 +38,7 @@ type EventRedirectDTO struct {
 func (c *EventsController) LogRedirectAsync(redirData *EventRedirectDTO) {
 
 	lo.Async(func() uuid.UUID {
+
 		event := &models.EventRedirect{
 			ID:         uuid.New(),
 			ShortURL:   redirData.ShortURL,
@@ -44,6 +49,24 @@ func (c *EventsController) LogRedirectAsync(redirData *EventRedirectDTO) {
 			IPAddress:  redirData.IPAddress,
 			Referer:    redirData.Referer,
 		}
+		ip := net.ParseIP(redirData.IPAddress)
+
+		if ip != nil {
+			city, err := c.geoipDB.City(ip)
+			if err == nil {
+				if city.Country.Names["en"] != "" {
+					event.LocationCountry = fmt.Sprintf("%s (%s)", city.Country.Names["en"], city.Country.IsoCode)
+				}
+
+				if city.Subdivisions[0].Names["en"] != "" {
+					event.LocationRegion = fmt.Sprintf("%s (%s)", city.Subdivisions[0].Names["en"], city.Subdivisions[0].IsoCode)
+				}
+
+				if city.City.Names["en"] != "" {
+					event.LocationCity = city.City.Names["en"]
+				}
+			}
+		}
 		lo.Try(func() error {
 			tx := c.eventDb.Create(event)
 			return tx.Error
@@ -53,20 +76,22 @@ func (c *EventsController) LogRedirectAsync(redirData *EventRedirectDTO) {
 
 }
 
-func (c *EventsController) GetRedirectsCountForUserId(userId string) []models.EventRedirectCountView {
+func (c *EventsController) GetRedirectsCountForUserId(userId uint64) ([]models.EventRedirectCountView, error) {
 	rows, err := c.eventDb.Model(&models.EventRedirect{}).
 		Select("count(id) as redirects, short_url").
+		Where("creator_id = ?", userId).
 		Group("short_url").
 		Rows()
 
 	if err != nil {
-		applogger.Panic("GetRedirectsCountForUserId: ", err)
+		applogger.Error("GetRedirectsCountForUserId: ", err)
+		return nil, err
 	}
 	data := make([]models.EventRedirectCountView, 0)
 	for rows.Next() {
 		var d models.EventRedirectCountView
 		lo.Must0(c.eventDb.Model(&models.EventRedirect{}).ScanRows(rows, &d))
 		data = append(data, d)
 	}
-	return data
+	return data, nil
 }

src/controllers/urls.go

@@ -79,7 +79,7 @@ func (c *UrlsController) initDefaultUrlGroup() {
 }
 
 func CreateUrlsController() *UrlsController {
-	appDb := lo.Must(db.GetAppDB())
+	appDb := db.GetAppDB()
 	ctrl := &UrlsController{
 		db: appDb,
 	}

src/controllers/users.go

@@ -3,7 +3,6 @@ package controllers
 import (
 	"errors"
 	"github.com/google/uuid"
-	"github.com/samber/lo"
 	"gorm.io/gorm"
 	"onepixel_backend/src/db"
 	"onepixel_backend/src/db/models"
@@ -57,7 +56,7 @@ func (c *UsersController) initDefaultUser() {
 var initDefaultUserOnce sync.Once
 
 func CreateUsersController() *UsersController {
-	appDb := lo.Must(db.GetAppDB())
+	appDb := db.GetAppDB()
 	ctrl := &UsersController{
 		db: appDb,
 	}

src/db/init.go

@@ -1,21 +1,25 @@
 package db
 
 import (
-	"errors"
+	"github.com/oschwald/geoip2-golang"
 	"onepixel_backend/src/config"
 	"onepixel_backend/src/db/models"
+	"onepixel_backend/src/utils"
 	"onepixel_backend/src/utils/applogger"
+	"os"
 	"sync"
 
 	"github.com/samber/lo"
 	"gorm.io/gorm"
 	"gorm.io/gorm/logger"
 )
 
-var appDb *gorm.DB    // singleton
-var eventsDb *gorm.DB // singleton
+var appDb *gorm.DB        // singleton
+var eventsDb *gorm.DB     // singleton
+var reader *geoip2.Reader // singleton
 var createAppDbOnce sync.Once
 var createEventsDbOnce sync.Once
+var createGeoIPDbOnce sync.Once
 
 func getGormConfig() (dbConfig *gorm.Config) {
 	dbConfig = &gorm.Config{
@@ -44,7 +48,7 @@ func init() {
 	InjectDBProvider("clickhouse", ProvideClickhouseDB)
 }
 
-func GetAppDB() (*gorm.DB, error) {
+func GetAppDB() *gorm.DB {
 
 	createAppDbOnce.Do(func() {
 		applogger.Warn("App: Initialising database")
@@ -64,10 +68,10 @@ func GetAppDB() (*gorm.DB, error) {
 		lo.Must0(appDb.AutoMigrate(&models.Url{}))
 	})
 
-	return appDb, nil
+	return appDb
 }
 
-func GetEventsDB() (*gorm.DB, error) {
+func GetEventsDB() *gorm.DB {
 	createEventsDbOnce.Do(func() {
 		applogger.Warn("Events: Initialising database")
 
@@ -83,23 +87,37 @@ func GetEventsDB() (*gorm.DB, error) {
 		}
 
 		// automigrate table if we cannot get column types
-		lo.TryCatchWithErrorValue(func() error {
-			if eventsDb.Migrator().HasTable((&models.EventRedirect{}).TableName()) {
-				applogger.Info("Events: table exists")
-				return nil
-			} else {
-				return errors.New("table not found")
-			}
-			//_, err := eventsDb.Migrator().ColumnTypes((&models.EventRedirect{}).TableName())
-			//return err
-		}, func(e any) {
-			applogger.Error("Error reading column types of eventsdb: " + e.(error).Error())
-			lo.Must0(eventsDb.AutoMigrate(&models.EventRedirect{}))
-			applogger.Info("Events: table automigrated")
-
+		err, success := lo.TryWithErrorValue(func() error {
+			return eventsDb.AutoMigrate(&models.EventRedirect{})
 		})
+		if !success {
+			applogger.Error("Events: AutoMigrate failed", err)
+		}
+
+	})
+
+	return eventsDb
+}
+
+func GetGeoIPDB() *geoip2.Reader {
+
+	// download file : https://git.io/GeoLite2-City.mmdb
+	createGeoIPDbOnce.Do(func() {
+		applogger.Warn("GeoIP: Initialising database")
+		fresh := utils.IsFileFresh(30, "GeoLite2-City.mmdb")
+		if !fresh {
+			applogger.Error("GeoIP: GeoLite2-City.mmdb is not fresh; downloading again")
+			lo.Try(func() error {
+				return os.Remove("GeoLite2-City.mmdb")
+			})
+			lo.Must0(utils.DownloadFile("https://git.io/GeoLite2-City.mmdb", "GeoLite2-City.mmdb"))
+			applogger.Info("GeoIP: GeoLite2-City.mmdb downloaded")
+		}
+
+		reader = lo.Must(geoip2.Open("GeoLite2-City.mmdb"))
 
 	})
 
-	return eventsDb, nil
+	return reader
+
 }

src/db/models/event_redirect.go

@@ -19,7 +19,10 @@ type EventRedirect struct {
 	// user agent
 	UserAgent string `gorm:"type:string"`
 	// ip address
-	IPAddress string `gorm:"type:string"`
+	IPAddress       string `gorm:"type:string"`
+	LocationCity    string `gorm:"type:string"`
+	LocationRegion  string `gorm:"type:string"`
+	LocationCountry string `gorm:"type:string"`
 	// referer
 	Referer string `gorm:"type:string"`
 }

src/dtos/http_responses.go

@@ -23,6 +23,9 @@ type ErrorResponse struct {
 	Message string `json:"message" example:"Something went wrong"`
 }
 
+type RedirectCountStatsResponse struct {
+}
+
 func CreateUserResponseFromUser(user *models.User, token *string) UserResponse {
 	return UserResponse{
 		ID:    user.ID,

src/main.go

@@ -19,8 +19,9 @@ import (
 
 func main() {
 	// Initialize the database
-	appDb := lo.Must(db.GetAppDB())
-	_ = lo.Must(db.GetEventsDB())
+	appDb := db.GetAppDB()
+	eventDb := db.GetEventsDB()
+	geoipDb := db.GetGeoIPDB()
 
 	// Create the app
 	adminApp := server.CreateAdminApp()
@@ -58,6 +59,9 @@ func main() {
 		defer cancel()
 
 		lo.Must0(lo.Must(appDb.DB()).Close())
+		lo.Must0(lo.Must(eventDb.DB()).Close())
+		lo.Must0(geoipDb.Close())
+
 		lo.Must0(adminApp.ShutdownWithContext(ctx))
 		lo.Must0(mainApp.ShutdownWithContext(ctx))
 		lo.Must0(app.ShutdownWithContext(ctx))

src/routes/api/stats.go

@@ -1,9 +1,14 @@
 package api
 
 import (
-	"github.com/gofiber/fiber/v2"
+	"onepixel_backend/src/config"
 	"onepixel_backend/src/controllers"
+	"onepixel_backend/src/db/models"
+	"onepixel_backend/src/dtos"
+	"onepixel_backend/src/security"
 	"onepixel_backend/src/utils/applogger"
+
+	"github.com/gofiber/fiber/v2"
 )
 
 var eventsController *controllers.EventsController
@@ -14,12 +19,27 @@ func StatsRoute() func(router fiber.Router) {
 	eventsController = controllers.CreateEventsController()
 
 	return func(router fiber.Router) {
-		router.Get("/", getStats)
+		router.Get("/", security.OptionalJwtAuthMiddleware, getAllStats)
+		router.Get("/:shortcode" /*security.MandatoryJwtAuthMiddleware,*/, getStatsForShortCode)
+		// TODO: add stats for grouped shortcodes
+	}
+}
+
+func getAllStats(ctx *fiber.Ctx) error {
+	// TODO: handle null case
+	user := ctx.Locals(config.LOCALS_USER).(*models.User)
+	stats, err := eventsController.GetRedirectsCountForUserId(user.ID)
+
+	if err != nil {
+		applogger.Error(err)
+		return ctx.Status(fiber.StatusInternalServerError).JSON(dtos.CreateErrorResponse(fiber.StatusInternalServerError, "something went wrong"))
 	}
+
+	return ctx.Status(fiber.StatusOK).JSON(stats)
 }
 
-func getStats(ctx *fiber.Ctx) error {
-	stats := eventsController.GetRedirectsCountForUserId("")
-	applogger.Info("Stats: ", len(stats))
+func getStatsForShortCode(ctx *fiber.Ctx) error {
+	// stats := eventsController.GetRedirectsCountForUserId("")
+	// applogger.Info("Stats: ", len(stats))
 	return ctx.SendString("GetStats")
 }