Merge pull request #773 from openmultiplayer/amir/file-cached-servers

Cached servers using json file

app/resources/server/db.go

@@ -2,21 +2,25 @@ package server
 
 import (
 	"context"
+	"os"
 	"time"
 
+	"github.com/goccy/go-json"
 	"github.com/pkg/errors"
 
+	"github.com/openmultiplayer/web/internal/config"
 	"github.com/openmultiplayer/web/internal/db"
 )
 
 var _ Repository = &DB{}
 
 type DB struct {
 	client *db.PrismaClient
+	cfg    config.Config
 }
 
-func New(client *db.PrismaClient) Repository {
-	return &DB{client}
+func New(client *db.PrismaClient, cfg config.Config) Repository {
+	return &DB{client, cfg}
 }
 
 func (s *DB) Upsert(ctx context.Context, e All) error {
@@ -164,3 +168,33 @@ func (s *DB) SetDeleted(ctx context.Context, ip string, at *time.Time) (*All, er
 	}
 	return dbToAPI(*result), err
 }
+
+func (s *DB) GetAllCached(updatedSince time.Duration) ([]All, error) {
+	result := []All{}
+	dat, err := os.ReadFile(s.cfg.CachedServers)
+	if err != nil {
+		return nil, err
+	}
+
+	err = json.Unmarshal(dat, &result)
+	if err != nil {
+		return nil, err
+	}
+
+	return result, nil
+}
+
+func (s *DB) GetByAddressCached(address string) (*All, error) {
+	result, err := s.GetAllCached(-120 * time.Hour)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, n := range result {
+		if address == n.IP {
+			return &n, nil
+		}
+	}
+
+	return nil, errors.New("server_not_found")
+}

app/resources/server/repository.go

@@ -14,4 +14,6 @@ type Repository interface {
 	GetServersToQuery(context.Context, time.Duration) ([]string, error)
 	GetAll(context.Context, time.Duration) ([]All, error)
 	SetDeleted(context.Context, string, *time.Time) (*All, error)
+	GetAllCached(time.Duration) ([]All, error)
+	GetByAddressCached(string) (*All, error)
 }

app/services/serverworker/serverworker.go

@@ -2,8 +2,10 @@ package serverworker
 
 import (
 	"context"
+	"os"
 	"time"
 
+	"github.com/goccy/go-json"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
 	"go.uber.org/fx"
@@ -34,8 +36,9 @@ var (
 )
 
 type Worker struct {
-	db server.Repository
-	sc scraper.Scraper
+	db  server.Repository
+	sc  scraper.Scraper
+	cfg config.Config
 }
 
 func Build() fx.Option {
@@ -46,7 +49,7 @@ func Build() fx.Option {
 		),
 
 		fx.Invoke(func(lc fx.Lifecycle, db server.Repository, sc scraper.Scraper, cfg config.Config) *Worker {
-			w := &Worker{db, sc}
+			w := &Worker{db, sc, cfg}
 
 			lc.Append(fx.Hook{
 				OnStart: func(ctx context.Context) error {
@@ -114,6 +117,28 @@ func (w *Worker) Run(ctx context.Context, window time.Duration) error {
 			continue
 		}
 
+		// Let's save all servers info our cache file to be used in our API data processing instead of DB
+		jsonData, err := json.Marshal(all)
+		if err != nil {
+			zap.L().Error("There was an error converting native array of servers to JSON data",
+				zap.Error(err))
+			continue
+		}
+
+		cacheFile, err := os.Create(w.cfg.CachedServers)
+		if err != nil {
+			zap.L().Error("Could not create server cache file",
+				zap.Error(err))
+			continue
+		}
+
+		_, err = cacheFile.Write(jsonData)
+		if err != nil {
+			zap.L().Error("There was an error writing collected servers into cache file",
+				zap.Error(err))
+			continue
+		}
+
 		active := 0
 		inactive := 0
 		for _, s := range all {

app/transports/api/servers/api.go

@@ -60,6 +60,7 @@ func Build() fx.Option {
 
 			rtr.Get("/{address}", s.get)
 			rtr.Get("/", s.list)
+			rtr.Get("/full", s.fulllist)
 			rtr.Post("/", s.add)
 			rtr.
 				With(authentication.MustBeAuthenticated).

app/transports/api/servers/h_fulllist.go

@@ -0,0 +1,32 @@
+package servers
+
+import (
+	"net/http"
+	"strconv"
+	"time"
+
+	"github.com/pkg/errors"
+
+	"github.com/openmultiplayer/web/internal/web"
+)
+
+func (s *service) fulllist(w http.ResponseWriter, r *http.Request) {
+	queries := r.URL.Query()
+	since, err := strconv.Atoi(queries.Get("activeSince"))
+	if err != nil {
+		since = 12
+	}
+
+	// This used to be for getting servers from database directly
+	// list, err := s.storer.GetAll(r.Context(), time.Duration(-since)*time.Hour)
+
+	// Let's use cached servers, instead of getting them directly from database
+	// This way we can save a lot DB processing
+	list, err := s.storer.GetAllCached(time.Duration(-since) * time.Hour)
+	if err != nil {
+		web.StatusInternalServerError(w, errors.Wrap(err, "failed to get list of servers"))
+		return
+	}
+
+	web.Write(w, list)
+}

app/transports/api/servers/h_get.go

@@ -11,7 +11,12 @@ import (
 )
 
 func (s *service) get(w http.ResponseWriter, r *http.Request) {
-	result, err := s.storer.GetByAddress(r.Context(), chi.URLParam(r, "address"))
+	// This used to be for getting servers from database directly
+	// result, err := s.storer.GetByAddress(r.Context(), chi.URLParam(r, "address"))
+
+	// Let's use cached servers, instead of getting them directly from database
+	// This way we can save a lot DB processing
+	result, err := s.storer.GetByAddressCached(chi.URLParam(r, "address"))
 	if err != nil {
 		if errors.Is(err, db.ErrNotFound) {
 			web.StatusNotFound(w, err)

app/transports/api/servers/h_list.go

@@ -18,7 +18,12 @@ func (s *service) list(w http.ResponseWriter, r *http.Request) {
 		since = 12
 	}
 
-	list, err := s.storer.GetAll(r.Context(), time.Duration(-since)*time.Hour)
+	// This used to be for getting servers from database directly
+	// list, err := s.storer.GetAll(r.Context(), time.Duration(-since)*time.Hour)
+
+	// Let's use cached servers, instead of getting them directly from database
+	// This way we can save a lot DB processing
+	list, err := s.storer.GetAllCached(time.Duration(-since) * time.Hour)
 	if err != nil {
 		web.StatusInternalServerError(w, errors.Wrap(err, "failed to get list of servers"))
 		return

fly.toml

@@ -11,10 +11,11 @@ processes    = []
     auto_rollback        = true
 
 [env]
-    LOG_LEVEL          = "debug"
-    PACKAGES_DB        = "/data/packages.db"
-    PRODUCTION         = "true"
-    PUBLIC_WEB_ADDRESS = "https://www.open.mp"
+    LOG_LEVEL           = "debug"
+    PACKAGES_DB         = "/data/packages.db"
+    CACHED_SERVERS_FILE = "/data/cachedServers.db"
+    PRODUCTION          = "true"
+    PUBLIC_WEB_ADDRESS  = "https://www.open.mp"
 
 [mounts]
     destination = "/data"

go.mod

@@ -17,6 +17,7 @@ require (
 	github.com/forPelevin/gomoji v0.0.0-20210718160015-5fcf0e405128
 	github.com/go-chi/chi v4.1.2+incompatible
 	github.com/go-chi/cors v1.1.1
+	github.com/goccy/go-json v0.10.2 // indirect
 	github.com/gomarkdown/markdown v0.0.0-20211105120026-16f708f914c3
 	github.com/google/go-github v0.0.0-20180819205025-d7732128a00e
 	github.com/google/go-github/v28 v28.1.1

go.sum

@@ -283,6 +283,8 @@ github.com/gobwas/pool v0.2.0 h1:QEmUOlnSjWtnpRGHF3SauEiOsy82Cup83Vf2LcMlnc8=
 github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
 github.com/gobwas/ws v1.0.2 h1:CoAavW/wd/kulfZmSIBt6p24n4j7tHgNVCjsfHVNUbo=
 github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM=
+github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
+github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
 github.com/gofrs/flock v0.8.0/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=

internal/config/config.go

@@ -27,7 +27,8 @@ type Config struct {
 	DocsSourcesPath       string        `envconfig:"DOCS_SOURCES_PATH"      required:"false" default:"docs/"`
 	DocsIndexPath         string        `envconfig:"DOCS_INDEX_PATH"        required:"false" default:"docs.bleve"`
 	PackagesDB            string        `envconfig:"PACKAGES_DB"            required:"false" default:"data/packages.db"`
-	ServerScrapeInterval  time.Duration `envconfig:"SERVER_SCRAPE_INTERVAL" required:"false" default:"45m"`
+	CachedServers         string        `envconfig:"CACHED_SERVERS_FILE"    required:"false" default:"data/cachedServers.json"`
+	ServerScrapeInterval  time.Duration `envconfig:"SERVER_SCRAPE_INTERVAL" required:"false" default:"30m"`
 	PackageSearchInterval time.Duration `envconfig:"PACKAGE_SEARCH_INTERVAL" required:"false" default:"24h"`
 	PackageScrapeInterval time.Duration `envconfig:"PACKAGE_SCRAPE_INTERVAL" required:"false" default:"24h"`
 }