Merge pull request #5 from justcoded/develop

Develop

.gitignore

@@ -40,3 +40,5 @@ fabric.properties
 /storage
 .env
 docker-compose.yml
+/bin
+/pkg

README.md

@@ -35,26 +35,26 @@ To prerender some URL you need to specify it as a query parameter, like this:
 http://localhost:3000/render?url=https://www.example.com/
 ```
 
-#### Purge cache
+#### Clear cache
 
 Cached pages are valid for 7 days by default, but you can force clear it.
 
-To purge all cached pages you need to pass the `Cache-Control` header
-with the value `must-revalidate` or the `Clear-Site-Data` header with any value.
+To purge all cached pages you need to send the`Clear-Site-Data` header with the value `*`. To remove only one page from
+cache, just pass `Cache-Control` header with the value `must-revalidate`.
 
 Just send a usual request to
 
 ```
 http://localhost:3000/render?url=https://www.example.com/
 ```
 
-With this header
+With this header, to remove the cached page
 
 ```
 Cache-Control: must-revalidate
 ```
 
-or
+or, to flush all cache
 
 ```
 Clear-Site-Data: *

docker-compose.example.yml

@@ -5,9 +5,8 @@ services:
     build:
       context: ./
       dockerfile: ./docker/server/Dockerfile
-    environment:
-      STORAGE_URL: 'storage:50051'
-      PAGE_WAIT_TIME: 5
+    env_file:
+      - src/.env
     ports:
       - '${SERVER_PORT}:3000'
     links:
@@ -17,5 +16,7 @@ services:
     build:
       context: ./
       dockerfile: ./docker/storage/Dockerfile
+    env_file:
+      - src/.env
     ports:
       - '${STORAGE_PORT}:50051'

src/.env.example

@@ -1,2 +1,3 @@
 STORAGE_URL='storage:50051'
 PAGE_WAIT_TIME=5
+CACHE_EXPIRATION_TIME=168

src/api/proto/storage.proto

@@ -14,6 +14,8 @@ service Storage {
   // Get a request from storage
   rpc Get(GetRequest) returns (GetReplay) {}
   //
+  rpc Remove(RemoveRequest) returns (RemoveReply) {}
+  //
   rpc Len(LenRequest) returns (LenReplay) {}
   //
   rpc Purge(PurgeRequest) returns (PurgeReply) {}
@@ -48,12 +50,20 @@ message GetRequest {
   string hash = 1;
 }
 
+message RemoveRequest {
+  string hash = 1;
+}
+
 // Tre replay message
 message GetReplay {
   bytes data = 1;
   bool result = 2;
 }
 
+message RemoveReply {
+  bool result = 2;
+}
+
 message LenRequest {}
 
 message LenReplay {

src/cmd/server/main.go

@@ -20,6 +20,7 @@ import (
 	"prerender/internal/healthcheck"
 	"prerender/internal/renderer"
 	"prerender/internal/sitemap"
+	"prerender/internal/urlparser"
 	"prerender/pkg/api/storage"
 	prLog "prerender/pkg/log"
 	"time"
@@ -149,7 +150,6 @@ func startCroneRefresh(ctx context.Context, c *cron.Cron, pc cachers.Сacher, lo
 
 func handleRequest(ctx context.Context, pc cachers.Сacher, logger prLog.Logger) routing.Handler {
 	return func(c *routing.Context) error {
-
 		queryString := c.Request.URL.Query().Get("url")
 
 		newTabCtx, cancel := chromedp.NewContext(ctx)
@@ -158,8 +158,18 @@ func handleRequest(ctx context.Context, pc cachers.Сacher, logger prLog.Logger)
 
 		req := c.Request
 
-		if req.Header.Get("Cache-Control") == "must-revalidate" || req.Header.Get("Clear-Site-Data") != "" {
+		if req.Header.Get("Clear-Site-Data") == "*" {
 			pc.Purge()
+		} else if req.Header.Get("Cache-Control") == "must-revalidate" {
+			key, err := urlparser.GetHashKey(queryString)
+			if err != nil {
+				return err
+			}
+
+			_, err = pc.Remove(key)
+			if err != nil {
+				return err
+			}
 		}
 
 		res, err := renderer.DoRender(ctx, queryString, pc, false, logger)

src/cmd/storage/main.go

@@ -11,15 +11,17 @@ import (
 	"os"
 	"os/signal"
 	"prerender/pkg/api/storage"
+	"strconv"
 	"syscall"
 	"time"
 )
 
 const (
-	port     = ":50051"
-	duration = time.Hour * 24 * 7
+	port = ":50051"
 )
 
+var duration = time.Hour * 24 * 7
+
 var gc = gcache.New(100000).
 	LRU().
 	Build()
@@ -42,6 +44,7 @@ func (s *server) Store(ctx context.Context, in *storage.StoreRequest) (*storage.
 func (s *server) Get(ctx context.Context, in *storage.GetRequest) (*storage.GetReplay, error) {
 	//log.Printf("Received: %v", in.GetHash())
 	value, err := gc.Get(in.Hash)
+
 	if err != nil {
 		return nil, status.Error(codes.NotFound, "not found")
 	}
@@ -50,6 +53,14 @@ func (s *server) Get(ctx context.Context, in *storage.GetRequest) (*storage.GetR
 	}, nil
 }
 
+func (s *server) Remove(ctx context.Context, in *storage.RemoveRequest) (*storage.RemoveReply, error) {
+	//log.Printf("Received: %v", in.GetHash())
+	result := gc.Remove(in.Hash)
+	return &storage.RemoveReply{
+		Result: result,
+	}, nil
+}
+
 func (s *server) Len(ctx context.Context, in *storage.LenRequest) (*storage.LenReplay, error) {
 	//log.Printf("Received: Len request")
 	return &storage.LenReplay{Length: int32(gc.Len(true))}, nil
@@ -62,6 +73,16 @@ func (s *server) Purge(ctx context.Context, in *storage.PurgeRequest) (*storage.
 }
 
 func main() {
+	durationStr, exists := os.LookupEnv("CACHE_EXPIRATION_TIME")
+	if exists && durationStr != "" {
+		durationHours, err := strconv.Atoi(durationStr)
+		if err != nil {
+			log.Fatalf("failed to parse env: %v", err)
+		}
+
+		duration = time.Hour * time.Duration(durationHours)
+	}
+
 	ctx := context.Background()
 	lis, err := net.Listen("tcp", port)
 	if err != nil {

src/internal/cachers/cacher.go

@@ -3,6 +3,7 @@ package cachers
 type Сacher interface {
 	Put(key string, data []byte) error
 	Get(key string) ([]byte, error)
+	Remove(key string) (bool, error)
 	Len() int
 	Purge()
 }

src/internal/cachers/inmemory/repository.go

@@ -27,6 +27,10 @@ func (r repository) Get(key string) ([]byte, error) {
 	return value.([]byte), nil
 }
 
+func (r repository) Remove(key string) (bool, error) {
+	return r.gc.Remove(key), nil
+}
+
 func (r repository) Len() int {
 	return r.gc.Len(true)
 }

src/internal/cachers/rstorage/service.go

@@ -42,6 +42,13 @@ func (s server) Get(key string) ([]byte, error) {
 	return result.GetData(), err
 }
 
+func (s server) Remove(key string) (bool, error) {
+	ctx := context.Background()
+	req := storage.RemoveRequest{Hash: key}
+	result, err := s.gw.Remove(ctx, &req)
+	return result.GetResult(), err
+}
+
 func (s server) Len() int {
 	ctx := context.Background()
 	req := storage.LenRequest{}

src/internal/renderer/renderer.go

@@ -2,28 +2,25 @@ package renderer
 
 import (
 	"context"
-	"crypto/sha256"
 	"encoding/json"
 	"fmt"
 	"github.com/chromedp/cdproto/dom"
 	"github.com/chromedp/chromedp"
 	"net/http"
-	"net/url"
 	"os"
 	"prerender/internal/archive"
 	"prerender/internal/cachers"
+	"prerender/internal/urlparser"
 	"prerender/pkg/log"
 	"strconv"
-	"strings"
 	"time"
 )
 
 func DoRender(ctx context.Context, queryString string, pc cachers.Сacher, force bool, logger log.Logger) (string, error) {
 	waitSecondsStr, exists := os.LookupEnv("PAGE_WAIT_TIME")
-
 	waitSeconds := 5
 
-	if exists {
+	if exists && waitSecondsStr != "" {
 		var err error
 		waitSeconds, err = strconv.Atoi(waitSecondsStr)
 
@@ -32,26 +29,19 @@ func DoRender(ctx context.Context, queryString string, pc cachers.Сacher, force
 		}
 	}
 
-	u, err := url.Parse(queryString)
+	requestURL, hostPath, err := urlparser.ParseUrl(queryString)
 	if err != nil {
 		logger.Error(err)
 	}
 
-	requestURL := ""
-	hostPath := ""
-
-	if u.Path != "/" && strings.HasSuffix(u.Path, "/") {
-		path := strings.TrimRight(u.Path, "/")
-		requestURL = u.Scheme + "://" + u.Host + path
-		hostPath = u.Host + path
-	} else {
-		requestURL = queryString
-		hostPath = u.Host + u.Path
+	key, err := urlparser.GetHashKey(queryString)
+	if err != nil {
+		logger.Error(err)
 	}
 
-	var res string
-	key := fmt.Sprintf("%x", sha256.Sum256([]byte(hostPath)))
 	value, err := pc.Get(key)
+	var res string
+
 	if force || err != nil {
 		err := chromedp.Run(ctx,
 			chromedp.Navigate(requestURL),

src/internal/urlparser/urlparser.go

@@ -0,0 +1,35 @@
+package urlparser
+
+import (
+	"crypto/sha256"
+	"fmt"
+	"net/url"
+	"strings"
+)
+
+func ParseUrl(queryString string) (string, string, error) {
+	u, err := url.Parse(queryString)
+	if err != nil {
+		return "", "", err
+	}
+
+	requestURL := ""
+	hostPath := ""
+
+	if u.Path != "/" && strings.HasSuffix(u.Path, "/") {
+		path := strings.TrimRight(u.Path, "/")
+		requestURL = u.Scheme + "://" + u.Host + path
+		hostPath = u.Host + path
+	} else {
+		requestURL = queryString
+		hostPath = u.Host + u.Path
+	}
+
+	return requestURL, hostPath, err
+}
+
+func GetHashKey(queryString string) (string, error) {
+	_, hostPath, err := ParseUrl(queryString)
+
+	return fmt.Sprintf("%x", sha256.Sum256([]byte(hostPath))), err
+}