Skip to content

Commit

Permalink
Keep panel images and html in-memory (#53)
Browse files Browse the repository at this point in the history 
* Avoid FS access for PNGs and HTML report

---------

Signed-off-by: Jan-Otto Kröpke <joe@cloudeteer.de>
  • Loading branch information
@jkroepke
jkroepke authored Jul 10, 2024
1 parent 4ef76d8 commit 05efab8
Show file tree
Hide file tree
Showing 3 changed files with 75 additions and 88 deletions.
105 changes: 53 additions & 52 deletions pkg/plugin/report.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"bytes"
"context"
"embed"
"encoding/base64"
"fmt"
"html/template"
"io"
Expand Down Expand Up @@ -39,7 +40,8 @@ type Report interface {
type templateData struct {
Dashboard
ReportOptions
Date string
Images PanelImages
Date string
}

// Report options
Expand All @@ -54,6 +56,8 @@ type ReportOptions struct {
footer string
}

type PanelImages map[int]template.URL

// Is layout grid?
func (o ReportOptions) IsGridLayout() bool {
return (o.config.Layout == "grid")
Expand Down Expand Up @@ -135,17 +139,19 @@ func (r *report) Generate() ([]byte, error) {
r.options.dashTitle = dash.Title

// Render panel PNGs in parallel using max workers configured in plugin
if err = r.renderPNGsParallel(dash); err != nil {
images, err := r.renderPNGsParallel(dash)
if err != nil {
return nil, fmt.Errorf("error rendering PNGs in parallel for dashboard %s: %v", dash.Title, err)
}

// Generate HTML file with fetched panel PNGs
if err = r.generateHTMLFile(dash); err != nil {
htmlReport, err := r.generateHTMLFile(dash, images)
if err != nil {
return nil, fmt.Errorf("error generating HTML file for dashboard %s: %v", dash.Title, err)
}

// Print HTML page into PDF
return r.renderPDF()
return r.renderPDF(htmlReport)
}

// Title returns the dashboard title parsed from the dashboard definition
Expand Down Expand Up @@ -179,15 +185,17 @@ func (r *report) htmlPath() string {
return filepath.Join(r.options.reportsDir, reportHTML)
}

// Render panel PNGs in parallel using configured number of workers
func (r *report) renderPNGsParallel(dash Dashboard) error {
// renderPNGsParallel render panel PNGs in parallel using configured amount workers.
func (r *report) renderPNGsParallel(dash Dashboard) (PanelImages, error) {
// buffer all panels on a channel
panels := make(chan Panel, len(dash.Panels))
for _, p := range dash.Panels {
panels <- p
}
close(panels)

panelImages := make(PanelImages, len(dash.Panels))

// fetch images in parallel form Grafana sever.
// limit concurrency using a worker pool to avoid overwhelming grafana
// for dashboards with many panels.
Expand All @@ -199,10 +207,12 @@ func (r *report) renderPNGsParallel(dash Dashboard) error {
go func(panels <-chan Panel, errs chan<- error) {
defer wg.Done()
for p := range panels {
err := r.renderPNG(p)
image, err := r.renderPNG(p)
if err != nil {
errs <- err
}

panelImages[p.ID] = image
}
}(panels, errs)
}
Expand All @@ -211,44 +221,36 @@ func (r *report) renderPNGsParallel(dash Dashboard) error {

for err := range errs {
if err != nil {
return err
return nil, err
}
}
return nil
return panelImages, nil
}

// Render a single panel into PNG
func (r *report) renderPNG(p Panel) error {
func (r *report) renderPNG(p Panel) (template.URL, error) {
var body io.ReadCloser
var file afero.File
var err error

// Get panel
if body, err = r.client.PanelPNG(p, r.options.dashUID, r.options.timeRange); err != nil {
return fmt.Errorf("error getting panel %s: %v", p.Title, err)
return "", fmt.Errorf("error getting panel %s: %w", p.Title, err)
}
defer body.Close()

// Create directory to store PNG files and get file handler
if err = r.options.vfs.MkdirAll(r.imgDirPath(), 0750); err != nil {
return fmt.Errorf("error creating img directory: %v", err)
}
imgFileName := fmt.Sprintf("image%d.png", p.ID)
if file, err = r.options.vfs.Create(filepath.Join(r.imgDirPath(), imgFileName)); err != nil {
return fmt.Errorf("error creating image file: %v", err)
imageContent, err := io.ReadAll(body)
if err != nil {
return "", fmt.Errorf("error reading image content for panel %s: %w", p.Title, err)
}
defer file.Close()

// Copy PNG to file
if _, err = io.Copy(file, body); err != nil {
return fmt.Errorf("error copying body to file: %v", err)
}
return nil
imageContentBase64 := make([]byte, base64.StdEncoding.EncodedLen(len(imageContent)))
base64.StdEncoding.Encode(imageContentBase64, imageContent)

return template.URL("data:image/png;charset=utf-8;base64," + string(imageContentBase64)), nil
}

// Generate HTML file(s) for dashboard
func (r *report) generateHTMLFile(dash Dashboard) error {
var file afero.File
func (r *report) generateHTMLFile(dash Dashboard, images PanelImages) (string, error) {
var tmpl *template.Template
var err error

Expand All @@ -264,53 +266,49 @@ func (r *report) generateHTMLFile(dash Dashboard) error {
},
}

// Make a file handle for HTML file
if file, err = r.options.vfs.Create(r.htmlPath()); err != nil {
return fmt.Errorf("error creating HTML file at %v : %v", r.htmlPath(), err)
}
defer file.Close()

// Make a new template for body of the report
if tmpl, err = template.New("report").Funcs(funcMap).ParseFS(templateFS, "templates/report.gohtml"); err != nil {
return fmt.Errorf("error parsing report template: %v", err)
return "", fmt.Errorf("error parsing report template: %w", err)
}

// Template data
data := templateData{dash, *r.options, time.Now().Local().In(r.options.location()).Format(time.RFC850)}
data := templateData{dash, *r.options, images, time.Now().Local().In(r.options.location()).Format(time.RFC850)}

// Render the template for body of the report
if err = tmpl.ExecuteTemplate(file, "report.gohtml", data); err != nil {
return fmt.Errorf("error executing report template: %v", err)
bufReport := &bytes.Buffer{}
if err = tmpl.ExecuteTemplate(bufReport, "report.gohtml", data); err != nil {
return "", fmt.Errorf("error executing report template: %w", err)
}

// Make a new template for header of the report
if tmpl, err = template.New("header").Funcs(funcMap).ParseFS(templateFS, "templates/header.gohtml"); err != nil {
return fmt.Errorf("error parsing header template: %v", err)
return "", fmt.Errorf("error parsing header template: %w", err)
}

// Render the template for header of the report
bufHeader := &bytes.Buffer{}
if err = tmpl.ExecuteTemplate(bufHeader, "header.gohtml", data); err != nil {
return fmt.Errorf("error executing header template: %v", err)
return "", fmt.Errorf("error executing header template: %w", err)
}
r.options.header = bufHeader.String()

// Make a new template for footer of the report
if tmpl, err = template.New("footer").Funcs(funcMap).ParseFS(templateFS, "templates/footer.gohtml"); err != nil {
return fmt.Errorf("error parsing footer template: %v", err)
return "", fmt.Errorf("error parsing footer template: %w", err)
}

// Render the template for footer of the report
bufFooter := &bytes.Buffer{}
if err = tmpl.ExecuteTemplate(bufFooter, "footer.gohtml", data); err != nil {
return fmt.Errorf("error executing footer template: %v", err)
return "", fmt.Errorf("error executing footer template: %w", err)
}
r.options.footer = bufFooter.String()
return nil

return bufReport.String(), nil
}

// Render HTML page into PDF using Chromium
func (r *report) renderPDF() ([]byte, error) {
func (r *report) renderPDF(htmlReport string) ([]byte, error) {
var realPath string
var err error

Expand All @@ -326,14 +324,9 @@ func (r *report) renderPDF() ([]byte, error) {
defer cancel()

// capture pdf
// NOTE: We can improve this by using in memory base64 encoded PNG images and
// using SetDocumentContent of chromedp without having to have access to underlying
// filesystem.
// This will need a bit of refactoring of the code tho
// Ref: https://github.com/chromedp/chromedp/issues/941#issuecomment-961181348
var buf []byte
if err := chromedp.Run(
ctx, r.printToPDF(fmt.Sprintf("file://%s", filepath.Join(realPath, reportHTML)), &buf),
if err = chromedp.Run(
ctx, r.printToPDF(htmlReport, &buf),
); err != nil {
return nil, fmt.Errorf("error rendering PDF: %v", err)
}
Expand All @@ -348,9 +341,17 @@ func (r *report) renderPDF() ([]byte, error) {
}

// Print to PDF using headless Chromium
func (r *report) printToPDF(url string, res *[]byte) chromedp.Tasks {
func (r *report) printToPDF(html string, res *[]byte) chromedp.Tasks {
return chromedp.Tasks{
chromedp.Navigate(url),
chromedp.Navigate("about:blank"),
chromedp.ActionFunc(func(ctx context.Context) error {
frameTree, err := page.GetFrameTree().Do(ctx)
if err != nil {
return err
}

return page.SetDocumentContent(frameTree.Frame.ID, html).Do(ctx)
}),
chromedp.ActionFunc(func(ctx context.Context) error {

var pageParams *page.PrintToPDFParams
Expand Down
54 changes: 20 additions & 34 deletions pkg/plugin/report_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"io"
"net/url"
"os"
"strings"
"testing"

"github.com/grafana/grafana-plugin-sdk-go/backend/log"
Expand Down Expand Up @@ -63,56 +64,49 @@ func TestReport(t *testing.T) {

Convey("When rendering images", func() {
dashboard, _ := gClient.Dashboard("")
rep.renderPNGsParallel(dashboard)
pngs, err := rep.renderPNGsParallel(dashboard)
So(err, ShouldBeNil)
So(pngs, ShouldHaveLength, 9)

Convey("It should create a temporary folder", func() {
_, err := rep.options.vfs.Stat(rep.options.reportsDir)
So(err, ShouldBeNil)
})

Convey("It should copy the file to the image folder", func() {
_, err := rep.options.vfs.Stat(rep.imgDirPath() + "/image1.png")
So(err, ShouldBeNil)
})

Convey("It shoud call getPanelPng once per panel", func() {
Convey("It should call getPanelPng once per panel", func() {
So(gClient.getPanelCallCount, ShouldEqual, 9)
})

Convey("It should create one file per panel", func() {
f, _ := rep.options.vfs.Open(rep.imgDirPath())
defer f.Close()
files, err := f.Readdir(0)
So(files, ShouldHaveLength, 9)
So(err, ShouldBeNil)
})
})

Convey("When genereting the HTML files", func() {
dashboard, _ := gClient.Dashboard("")
rep.generateHTMLFile(dashboard)
f, err := rep.options.vfs.Open(rep.htmlPath())
defer f.Close()
pngs, err := rep.renderPNGsParallel(dashboard)
So(err, ShouldBeNil)
So(pngs, ShouldHaveLength, 9)

html, err := rep.generateHTMLFile(dashboard, pngs)
So(err, ShouldBeNil)

Convey("It should create a file in the temporary folder", func() {
So(err, ShouldBeNil)
})

Convey("The file should contain reference to the template data", func() {
var buf bytes.Buffer
io.Copy(&buf, f)
s := buf.String()
s := html

So(err, ShouldBeNil)
Convey("Including the Title", func() {
So(rep.options.header, ShouldContainSubstring, "My first dashboard")

})
Convey("Including the varialbe values", func() {
Convey("Including the variable values", func() {
So(rep.options.header, ShouldContainSubstring, "testvarvalue")

})
Convey("and the images", func() {
So(s, ShouldContainSubstring, "data:image/png")
So(strings.Count(s, "data:image/png"), ShouldEqual, 9)

So(s, ShouldContainSubstring, "image1")
So(s, ShouldContainSubstring, "image22")
So(s, ShouldContainSubstring, "image33")
Expand All @@ -124,8 +118,8 @@ func TestReport(t *testing.T) {
So(s, ShouldContainSubstring, "image99")
})
Convey("and the time range", func() {
//server time zone by shift hours timestamp
//so just test for day and year
// server time zone by shift hours timestamp
// so just test for day and year
So(rep.options.header, ShouldContainSubstring, "Tue Jan 19")
So(rep.options.header, ShouldContainSubstring, "2016")
})
Expand Down Expand Up @@ -174,22 +168,14 @@ func TestReportErrorHandling(t *testing.T) {

Convey("When rendering images", func() {
dashboard, _ := gClient.Dashboard("")
err := rep.renderPNGsParallel(dashboard)
_, err := rep.renderPNGsParallel(dashboard)

Convey("It shoud call getPanelPng once per panel", func() {
So(gClient.getPanelCallCount, ShouldEqual, 9)
})

Convey("It should create one less image file than the total number of panels", func() {
f, _ := rep.options.vfs.Open(rep.imgDirPath())
defer f.Close()
files, err := f.Readdir(0)
So(files, ShouldHaveLength, 8) // one less than the total number of im
So(err, ShouldBeNil)
})

Convey(
"If any panels return errors, renderPNGsParralel should return the error message from one panel",
"If any panels return errors, renderPNGsParallel should return the error message from one panel",
func() {
So(err, ShouldNotBeNil)
So(err.Error(), ShouldContainSubstring, "The second panel has some problem")
Expand Down
4 changes: 2 additions & 2 deletions pkg/plugin/templates/report.gohtml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,9 +76,9 @@
<div class="grid">
{{- range $i, $v := .Panels}}
<figure class="grid-image grid-image-{{$i}}">
<img src="images/image{{$v.ID}}.png" alt="{{$v.Title}}" class="grid-image">
<img src="{{ index $.Images $v.ID }}" id="image{{ $v.ID }}" alt="{{ $v.Title }}" class="grid-image">
</figure>
{{end}}
{{- end }}
</div>
</div>
</body>
Expand Down

0 comments on commit 05efab8

Please sign in to comment.