improve dev mode pretty logger

README.md

@@ -71,6 +71,7 @@ deny_routes:
         - /helpers/endorsing_rights
         - /chains/.*/blocks/.*/context/contracts(/?)$
         - /chains/.*/blocks/.*/context/raw/bytes
+dev_mode: false
 gc:
     optimize_memory_store: true
     percent: 100
@@ -101,7 +102,7 @@ tezos_host:
 
 You can also configure or overwrite TzProxy with environment variables, using the same structure:
 
-
+- `TZPROXY_DEV_MODE` is a flag to enable dev features like pretty logger.
 - `TZPROXY_HOST` is the host of the proxy.
 - `TZPROXY_TEZOS_HOST` are the hosts of the tezos nodes.
 - `TZPROXY_TEZOS_HOST_RETRY` is the host used when finding a 404 or 410. It's recommended use full or archive nodes.

balancers/iphash.go

@@ -56,6 +56,7 @@ func (b *ipHashBalancer) RemoveTarget(name string) bool {
 func (b *ipHashBalancer) Next(c echo.Context) *middleware.ProxyTarget {
 	b.mutex.Lock()
 	defer b.mutex.Unlock()
+	c.Logger().Info("retrying")
 
 	if len(b.targets) == 0 {
 		return nil
@@ -64,6 +65,7 @@ func (b *ipHashBalancer) Next(c echo.Context) *middleware.ProxyTarget {
 	}
 
 	if c.Get("retry") != nil {
+		c.Logger().Debug("retrying")
 		return b.retryTarget
 	}
 

config/config.go

@@ -0,0 +1,189 @@
+package config
+
+import (
+	"net/http"
+	"net/url"
+	"os"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/coocood/freecache"
+	echocache "github.com/fraidev/go-echo-cache"
+	"github.com/labstack/echo/v4"
+	"github.com/labstack/echo/v4/middleware"
+	"github.com/marigold-dev/tzproxy/balancers"
+	"github.com/redis/go-redis/v9"
+	"github.com/rs/zerolog"
+	"github.com/rs/zerolog/diode"
+	"github.com/rs/zerolog/log"
+	"github.com/ulule/limiter/v3"
+)
+
+func NewConfig() *Config {
+	configFile := initViper()
+
+	if configFile.GC.OptimizeMemoryStore {
+		if !configFile.Redis.Enabled {
+			configFile.GC.Percent = 20
+		}
+	}
+
+	var targets = []*middleware.ProxyTarget{}
+	var retryTarget *middleware.ProxyTarget = nil
+	if configFile.TezosHostRetry != "" {
+		retryTarget = hostToTarget(configFile.TezosHostRetry)
+	}
+	for _, host := range configFile.TezosHost {
+		targets = append(targets, hostToTarget(host))
+	}
+
+	var redisClient *redis.Client
+	if configFile.Redis.Enabled {
+		redisClient = redis.NewClient(&redis.Options{
+			Addr: configFile.Redis.Host,
+		})
+	}
+	store := buildStore(configFile, redisClient)
+	balancer := balancers.NewIPHashBalancer(targets, retryTarget, configFile.LoadBalancer.TTL, store)
+	logger := buildLogger(configFile.DevMode)
+	proxyConfig := middleware.ProxyConfig{
+		Skipper:    middleware.DefaultSkipper,
+		ContextKey: "target",
+		RetryCount: 0,
+		Balancer:   balancer,
+		RetryFilter: func(c echo.Context, err error) bool {
+			if httpErr, ok := err.(*echo.HTTPError); ok {
+				if httpErr.Code == http.StatusBadGateway || httpErr.Code == http.StatusNotFound || httpErr.Code == http.StatusGone {
+					return true
+				}
+			}
+
+			return false
+		},
+		ErrorHandler: func(c echo.Context, err error) error {
+			logger.Error().Err(err).Msg("proxy error")
+			c.Logger().Error(err)
+			return err
+		},
+	}
+
+	config := &Config{
+		ConfigFile: configFile,
+		DenyListTable: func() map[string]bool {
+			table := make(map[string]bool)
+			for _, ip := range configFile.DenyList.Values {
+				table[ip] = true
+			}
+			return table
+		}(),
+		Rate: &limiter.Rate{
+			Period: time.Duration(configFile.RateLimit.Minutes) * time.Minute,
+			Limit:  int64(configFile.RateLimit.Max),
+		},
+		Store:       store,
+		CacheTTL:    time.Duration(configFile.Cache.TTL) * (time.Second),
+		ProxyConfig: &proxyConfig,
+		Redis:       redisClient,
+	}
+
+	for _, route := range config.ConfigFile.DenyRoutes.Values {
+		regex, err := regexp.Compile(route)
+		if err != nil {
+			log.Fatal().Err(err).Msg("unable to compile regex")
+		}
+		config.BlockRoutesRegex = append(config.BlockRoutesRegex, regex)
+	}
+
+	for _, route := range config.ConfigFile.Cache.DisabledRoutes {
+		regex, err := regexp.Compile(route)
+		if err != nil {
+			log.Fatal().Err(err).Msg("unable to compile regex")
+		}
+		config.CacheDisabledRoutesRegex = append(config.CacheDisabledRoutesRegex, regex)
+	}
+
+	config.Logger = logger
+
+	config.RequestLoggerConfig = &middleware.RequestLoggerConfig{
+		LogLatency:      true,
+		LogProtocol:     true,
+		LogRemoteIP:     true,
+		LogMethod:       true,
+		LogURI:          true,
+		LogUserAgent:    true,
+		LogStatus:       true,
+		LogError:        true,
+		HandleError:     true,
+		LogResponseSize: true,
+		LogReferer:      true,
+		LogValuesFunc: func(c echo.Context, v middleware.RequestLoggerValues) error {
+			if v.Error != nil {
+				config.Logger.Error().
+					Err(v.Error).
+					Str("ip", v.RemoteIP).
+					Int("status", v.Status).
+					Str("method", v.Method).
+					Str("uri", v.URI).
+					Str("user_agent", v.UserAgent).
+					Msg("request error")
+				return v.Error
+			}
+
+			config.Logger.Info().
+				Str("ip", v.RemoteIP).
+				Str("protocol", v.Protocol).
+				Int("status", v.Status).
+				Str("method", v.Method).
+				Str("uri", v.URI).
+				Int64("elapsed", int64(v.Latency)).
+				Str("user_agent", v.UserAgent).
+				Str("referer", v.Referer).
+				Int64("response_size", v.ResponseSize).
+				Msg("request")
+
+			return nil
+		},
+	}
+
+	return config
+}
+
+func buildStore(cf *ConfigFile, redis *redis.Client) echocache.Cache {
+	if cf.Redis.Enabled {
+		redisStore := echocache.NewRedisCache(redis)
+		return &redisStore
+	}
+
+	freeCache := freecache.NewCache(cf.Cache.SizeMB * 1024 * 1024)
+	memoryStore := echocache.NewMemoryCache(freeCache)
+	return &memoryStore
+}
+
+func buildLogger(devMode bool) zerolog.Logger {
+	if !devMode {
+		bunchWriter := diode.NewWriter(
+			os.Stdout,
+			1000,
+			time.Second, func(missed int) {
+				log.Printf("Logger Dropped %d messages", missed)
+			})
+
+		return zerolog.New(bunchWriter)
+	}
+
+	return log.Output(zerolog.ConsoleWriter{Out: os.Stderr})
+}
+
+func hostToTarget(host string) *middleware.ProxyTarget {
+	hostWithScheme := host
+	if !strings.Contains(host, "http") {
+		hostWithScheme = "http://" + host
+	}
+	targetURL, err := url.ParseRequestURI(hostWithScheme)
+	if err != nil {
+		log.Fatal().Err(err).Msg("unable to parse host")
+	}
+
+	return &middleware.ProxyTarget{URL: targetURL}
+}

config/default.go

@@ -0,0 +1,64 @@
+package config
+
+var defaultConfig = &ConfigFile{
+	DevMode:        false,
+	Host:           "0.0.0.0:8080",
+	TezosHost:      []string{"127.0.0.1:8732"},
+	TezosHostRetry: "",
+	Redis: Redis{
+		Host:    "",
+		Enabled: false,
+	},
+	LoadBalancer: LoadBalancer{
+		TTL: 600,
+	},
+	Logger: Logger{
+		BunchSize:           1000,
+		PoolIntervalSeconds: 1,
+	},
+	RateLimit: RateLimit{
+		Enabled: false,
+		Minutes: 1,
+		Max:     300,
+	},
+	Cache: Cache{
+		Enabled: true,
+		TTL:     5,
+		DisabledRoutes: []string{
+			"/monitor/.*",
+			"/chains/.*/mempool",
+			"/chains/.*/blocks.*head",
+		},
+		SizeMB: 100,
+	},
+	DenyList: DenyList{
+		Enabled: false,
+		Values:  []string{},
+	},
+	DenyRoutes: DenyRoutes{
+		Enabled: true,
+		Values: []string{
+			"/injection/block", "/injection/protocol", "/network.*", "/workers.*",
+			"/worker.*", "/stats.*", "/config", "/chains/.*/blocks/.*/helpers/baking_rights",
+			"/chains/.*/blocks/.*/helpers/endorsing_rights",
+			"/helpers/baking_rights", "/helpers/endorsing_rights",
+			"/chains/.*/blocks/.*/context/contracts(/?)$",
+			"/chains/.*/blocks/.*/context/raw/bytes",
+		},
+	},
+	Metrics: Metrics{
+		Host:    "0.0.0.0:9000",
+		Enabled: true,
+		Pprof:   false,
+	},
+	GC: GC{
+		OptimizeMemoryStore: true,
+		Percent:             100,
+	},
+	GZIP: GZIP{
+		Enabled: true,
+	},
+	CORS: CORS{
+		Enabled: true,
+	},
+}

config/model.go

@@ -0,0 +1,102 @@
+package config
+
+import (
+	"regexp"
+	"time"
+
+	echocache "github.com/fraidev/go-echo-cache"
+	"github.com/labstack/echo/v4/middleware"
+	"github.com/redis/go-redis/v9"
+	"github.com/rs/zerolog"
+	"github.com/ulule/limiter/v3"
+)
+
+type Config struct {
+	Level                    uint
+	HashBlock                string
+	ConfigFile               *ConfigFile
+	DenyListTable            map[string]bool
+	Rate                     *limiter.Rate
+	CacheDisabledRoutesRegex []*regexp.Regexp
+	BlockRoutesRegex         []*regexp.Regexp
+	Store                    echocache.Cache
+	CacheTTL                 time.Duration
+	RequestLoggerConfig      *middleware.RequestLoggerConfig
+	ProxyConfig              *middleware.ProxyConfig
+	Redis                    *redis.Client
+	Logger                   zerolog.Logger
+}
+
+type Logger struct {
+	BunchSize           int `mapstructure:"bunch_size"`
+	PoolIntervalSeconds int `mapstructure:"pool_interval_seconds"`
+}
+
+type RateLimit struct {
+	Enabled bool    `mapstructure:"enabled"`
+	Minutes float64 `mapstructure:"minutes"`
+	Max     int     `mapstructure:"max"`
+}
+
+type Cache struct {
+	Enabled        bool     `mapstructure:"enabled"`
+	TTL            int      `mapstructure:"ttl"`
+	DisabledRoutes []string `mapstructure:"disabled_routes"`
+	SizeMB         int      `mapstructure:"size_mb"`
+}
+
+type DenyList struct {
+	Enabled bool     `mapstructure:"enabled"`
+	Values  []string `mapstructure:"values"`
+}
+
+type DenyRoutes struct {
+	Enabled bool     `mapstructure:"enabled"`
+	Values  []string `mapstructure:"values"`
+}
+
+type Metrics struct {
+	Host    string `mapstructure:"host"`
+	Enabled bool   `mapstructure:"enabled"`
+	Pprof   bool   `mapstructure:"pprof"`
+}
+
+type GC struct {
+	OptimizeMemoryStore bool `mapstructure:"optimize_memory_store"`
+	Percent             int  `mapstructure:"percent"`
+}
+
+type CORS struct {
+	Enabled bool `mapstructure:"enabled"`
+}
+
+type GZIP struct {
+	Enabled bool `mapstructure:"enabled"`
+}
+
+type Redis struct {
+	Host    string `mapstructure:"host"`
+	Enabled bool   `mapstructure:"enabled"`
+}
+
+type LoadBalancer struct {
+	TTL int `mapstructure:"ttl"`
+}
+
+type ConfigFile struct {
+	DevMode        bool         `mapstructure:"dev_mode"`
+	LoadBalancer   LoadBalancer `mapstructure:"load_balancer"`
+	Redis          Redis        `mapstructure:"redis"`
+	Logger         Logger       `mapstructure:"logger"`
+	RateLimit      RateLimit    `mapstructure:"rate_limit"`
+	Cache          Cache        `mapstructure:"cache"`
+	DenyList       DenyList     `mapstructure:"deny_list"`
+	DenyRoutes     DenyRoutes   `mapstructure:"deny_routes"`
+	Metrics        Metrics      `mapstructure:"metrics"`
+	GC             GC           `mapstructure:"gc"`
+	CORS           CORS         `mapstructure:"cors"`
+	GZIP           GZIP         `mapstructure:"gzip"`
+	Host           string       `mapstructure:"host"`
+	TezosHost      []string     `mapstructure:"tezos_host"`
+	TezosHostRetry string       `mapstructure:"tezos_host_retry"`
+}