Skip to content

Commit

Permalink
big speed improvements and remove redis dependency
Browse files Browse the repository at this point in the history
  • Loading branch information
pluja committed Feb 20, 2024
1 parent d9284e7 commit cd34d09
Show file tree
Hide file tree
Showing 14 changed files with 427 additions and 249 deletions.
3 changes: 2 additions & 1 deletion .dockerignore
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
node_modules
*.yml
articles
Dockerfile
Dockerfile
content
3 changes: 2 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -11,4 +11,5 @@ go.work
.env
articles
TODO.md
style.css
style.css
content
111 changes: 31 additions & 80 deletions blogo/articles.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,19 +2,18 @@ package main

import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"html/template"
"os"
"path"
"path/filepath"
"strconv"
"strings"
"time"

chromahtml "github.com/alecthomas/chroma/v2/formatters/html"
"github.com/redis/go-redis/v9"
"github.com/rs/zerolog/log"
"github.com/yuin/goldmark"
highlighting "github.com/yuin/goldmark-highlighting/v2"
Expand Down Expand Up @@ -59,13 +58,24 @@ func InitGoldmark() {
func LoadArticles() error {
InitGoldmark()
var slugs []string
err := filepath.Walk(fmt.Sprintf("%v/articles/", os.Getenv("CONTENT_PATH")), func(fpath string, info os.FileInfo, err error) error {
err := filepath.Walk(path.Join(os.Getenv("CONTENT_PATH"), "/articles/"), func(fpath string, info os.FileInfo, err error) error {
if err != nil {
return err
}

if !info.IsDir() && strings.HasSuffix(info.Name(), ".md") {
err := LoadArticle(fpath)
article, err := GetArticleFromFile(fpath)
if err != nil {
log.Error().Msgf("Could not get article from file %v", fpath)
return err
}

err = LoadArticle(article)
if err != nil {
return err
}

err = GenerateArticleStatic(article)
if err != nil {
return err
}
Expand All @@ -79,14 +89,15 @@ func LoadArticles() error {
}

// Remove articles that are no longer in the articles folder from Redis
ctx := context.Background()
articleSlugs, err := RedisDb.SMembers(ctx, "articles").Result()
articleSlugs, err := Badger.GetAllArticleSlugs()
if err != nil {
log.Err(err).Msg("Error getting articles from Redis")
articleSlugs = []string{}
}
left, _ := Difference(articleSlugs, slugs)
log.Printf("Removing %v articles from Redis. Left articles: %v", left, slugs)
if len(left) > 0 {
log.Printf("Removing %v articles from Badger. Left articles: %v", len(left), slugs)
}
for _, articleSlug := range left {
if !StringInSlice(articleSlug, slugs) {
RemoveArticle(fmt.Sprintf("%v/articles/%v.md", os.Getenv("CONTENT_PATH"), articleSlug))
Expand Down Expand Up @@ -121,15 +132,8 @@ func GetArticleContent(filename string) (template.HTML, string, error) {
}

// Loads an article from a markdown file and stores it in Redis
func LoadArticle(filepath string) (err error) {
slug, extension := ParseFilePath(filepath)
filename := fmt.Sprintf("%v%v", slug, extension)

article, err := GetArticleFromFile(filename)
if err != nil {
return err
}
switch slug {
func LoadArticle(article ArticleData) (err error) {
switch article.Slug {
case "about":
About.Slug = "about"
About.Data = article
Expand All @@ -139,19 +143,7 @@ func LoadArticle(filepath string) (err error) {
if err != nil {
return fmt.Errorf("error while marshalling article to JSON: %v", err)
}

// Store the article in Redis with the slug as the key
ctx := context.Background()
err = RedisDb.Set(ctx, slug, articleJson, 0).Err()
if err != nil {
return fmt.Errorf("error while setting article to Redis: %v", err)
}
RedisDb.SAdd(ctx, "articles", slug)

// Add article to a set for each tag
for _, tag := range article.Tags {
RedisDb.SAdd(ctx, fmt.Sprintf("tag:%v", tag), slug)
}
Badger.Set("post_"+article.Slug, articleJson)
}

if article.NostrUrl == "" && article.Slug != "about" {
Expand All @@ -168,59 +160,13 @@ func LoadArticle(filepath string) (err error) {
func RemoveArticle(filename string) {
log.Printf("Removing article: %v", filename)
slug, _ := ParseFilePath(filename)

ctx := context.Background()
// Get the article from Redis
article, err := RedisDb.Get(ctx, slug).Result()
if err != nil || err == redis.Nil {
log.Err(err).Msgf("Could not get article %v from Redis", slug)
return
}

// Unmarshal the article data
var articleData ArticleData
err = json.Unmarshal([]byte(article), &articleData)
if err != nil {
log.Err(err).Msgf("Could not unmarshal article %v from Redis", slug)
return
}

pipe := RedisDb.Pipeline()

// Delete the article
delCmd := pipe.Del(ctx, slug)

// Remove from the "articles" set
sRemCmd := pipe.SRem(ctx, "articles", slug)

// Remove from each tag set
tagRemCmds := make([]*redis.IntCmd, len(articleData.Tags))
for i, tag := range articleData.Tags {
tagRemCmds[i] = pipe.SRem(ctx, fmt.Sprintf("tag:%v", tag), slug)
}

_, err = pipe.Exec(ctx)
if err != nil {
log.Err(err).Msgf("Could not execute pipelined commands for article %v", slug)
return
}

// Check for command errors
if delCmd.Err() != nil || sRemCmd.Err() != nil {
log.Err(err).Msg("Failed to delete article or remove from 'articles' set")
}
for _, cmd := range tagRemCmds {
if cmd.Err() != nil {
log.Err(cmd.Err()).Msg("Failed to remove article from a tag set")
}
}
Badger.DeleteArticle(slug)
}

// Returns an ArticleData struct from a markdown file
func GetArticleFromFile(filename string) (ArticleData, error) {
log.Printf("Getting article from file: %v", filename)
filepath := fmt.Sprintf("%v/articles/%v", os.Getenv("CONTENT_PATH"), filename)
slug, _ := ParseFilePath(filepath)
func GetArticleFromFile(filepath string) (ArticleData, error) {
slug, extension := ParseFilePath(filepath)
filename := slug + extension

var article ArticleData
// Read the markdown file
Expand Down Expand Up @@ -362,7 +308,12 @@ func AddMetadataToFile(filename, key, value string) error {
return err
}

LoadArticle(filename)
article, err := GetArticleFromFile(path.Join(os.Getenv("CONTENT_PATH"), "/articles", filename))
if err != nil {
log.Error().Msgf("Could not get article from file %v", filePath)
return err
}
LoadArticle(article)
return nil
}

Expand Down
184 changes: 184 additions & 0 deletions blogo/badger.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,184 @@
package main

import (
"encoding/json"
"strings"

"github.com/rs/zerolog/log"

badger "github.com/dgraph-io/badger/v4"
)

var Badger Database

type Database struct {
*badger.DB
}

func InitBadger() {
opts := badger.DefaultOptions("").WithInMemory(true)
opts.Logger = nil // Disable logging
var err error
Badger.DB, err = badger.Open(opts)
if err != nil {
log.Fatal().Err(err)
}
}

// BLOGO SPECIFIC FUNCTIONS

func (d *Database) GetAllArticleSlugs() ([]string, error) {
var keys []string
err := d.View(func(txn *badger.Txn) error {
opts := badger.DefaultIteratorOptions
opts.Prefix = []byte("post_")
it := txn.NewIterator(opts)
defer it.Close()

for it.Rewind(); it.Valid(); it.Next() {
item := it.Item()
key := item.Key()
keys = append(keys, strings.Trim(string(key), "post_"))
}
return nil
})
return keys, err
}

func (d *Database) GetPostBySlug(slug string) (ArticleData, error) {
var value []byte
var aData ArticleData
err := d.View(func(txn *badger.Txn) error {
item, err := txn.Get([]byte("post_" + slug))
if err != nil {
return err
}

err = item.Value(func(val []byte) error {
value = val
return nil
})
return err
})
if err != nil {
return aData, err
}

err = json.Unmarshal(value, &aData)
if err != nil {
return aData, err
}
return aData, err
}

func (d *Database) GetAllArticles() []ArticleData {
var articles []ArticleData
articleBytes := d.GetValuesWithPrefix("post_")
for _, ab := range articleBytes {
var article ArticleData
err := json.Unmarshal(ab, &article)
if err != nil {
log.Error().Err(err).Msg("Error unmarshalling article from Badger:")
continue
}
articles = append(articles, article)
}
return articles
}

func (d *Database) DeleteArticle(key string) error {
return d.Delete("post_" + key)
}

// GENERIC FUNCTIONS

func (d *Database) Set(key string, value []byte) error {
return d.Update(func(txn *badger.Txn) error {
return txn.Set([]byte(key), value)
})
}

func (d *Database) Get(key string) ([]byte, error) {
var value []byte
err := d.View(func(txn *badger.Txn) error {
item, err := txn.Get([]byte(key))
if err != nil {
return err
}

err = item.Value(func(val []byte) error {
value = val
return nil
})
return err
})
return value, err
}

func (d *Database) Delete(key string) error {
return d.Update(func(txn *badger.Txn) error {
return txn.Delete([]byte(key))
})
}

func (d *Database) GetAllKeys() ([]string, error) {
var keys []string
err := d.View(func(txn *badger.Txn) error {
opts := badger.DefaultIteratorOptions
it := txn.NewIterator(opts)
defer it.Close()

for it.Rewind(); it.Valid(); it.Next() {
item := it.Item()
key := item.Key()
keys = append(keys, string(key))
}
return nil
})
return keys, err
}

func (d *Database) GetValues() ([][]byte, error) {
var values [][]byte
err := d.View(func(txn *badger.Txn) error {
opts := badger.DefaultIteratorOptions
it := txn.NewIterator(opts)
defer it.Close()

for it.Rewind(); it.ValidForPrefix(opts.Prefix); it.Next() {
item := it.Item()
err := item.Value(func(val []byte) error {
values = append(values, val)
return nil
})
if err != nil {
return err
}
}
return nil
})
return values, err
}

func (d *Database) GetValuesWithPrefix(prefix string) [][]byte {
var values [][]byte
d.View(func(txn *badger.Txn) error {
opts := badger.DefaultIteratorOptions
opts.Prefix = []byte(prefix)
it := txn.NewIterator(opts)
defer it.Close()

for it.Rewind(); it.ValidForPrefix(opts.Prefix); it.Next() {
item := it.Item()
err := item.Value(func(val []byte) error {
values = append(values, val)
return nil
})
if err != nil {
return err
}
}
return nil
})
return values
}
Loading

0 comments on commit cd34d09

Please sign in to comment.