feat: Support rendering tables

* The feature adds support for adding tabular data of panels at the end of the report. Users can choose which panel data must be included in the report by query parameters.

Signed-off-by: Mahendra Paipuri <mahendra.paipuri@gmail.com>

README.md

@@ -29,8 +29,11 @@ improvements and modernization.
 - The plugin is capable of including all the repeated rows and/or panels in the
   generated report.
 
-- The plugin can be configured by Admins and users either from
-  [Configuration Page](./src/img/light.png) or query parameters to the report API.
+- The plugin can include selected panels data into report which can be chosen using
+  query parameters to the report API.
+
+- The plugin can be configured by Admins from [Configuration Page](./src/img/light.png)
+  and users using query parameters to the report API.
 
 ## Documentation
 

pkg/plugin/chrome/tab.go

@@ -5,17 +5,24 @@ import (
 	"io"
 	"net/http"
 	"os"
+	"time"
 
 	"github.com/chromedp/cdproto/network"
 	"github.com/chromedp/cdproto/page"
+	"github.com/chromedp/cdproto/runtime"
 	"github.com/chromedp/chromedp"
 	"github.com/grafana/grafana-plugin-sdk-go/backend/log"
 	"golang.org/x/net/context"
 )
 
+var WithAwaitPromise = func(p *runtime.EvaluateParams) *runtime.EvaluateParams {
+	return p.WithAwaitPromise(true)
+}
+
 // Tab is container for a browser tab.
 type Tab struct {
-	ctx context.Context
+	ctx    context.Context
+	cancel context.CancelFunc
 }
 
 // Close releases the resources of the current browser tab.
@@ -31,19 +38,25 @@ func (t *Tab) Close(logger log.Logger) {
 		if err = chromedp.Cancel(t.ctx); err != nil {
 			logger.Error("got error from cancel tab context", "error", err)
 		}
+
+		if t.cancel != nil {
+			t.cancel()
+		}
 	}
 }
 
 // NavigateAndWaitFor navigates to the given address and waits for the given event to be fired on the page.
 func (t *Tab) NavigateAndWaitFor(addr string, headers map[string]any, eventName string) error {
-	err := t.Run(enableLifeCycleEvents())
-	if err != nil {
+	if err := t.Run(
+		// block some URLs to avoid unnecessary requests
+		network.SetBlockedURLS([]string{"*/api/frontend-metrics", "*/api/live/ws", "*/api/user/*"}),
+		enableLifeCycleEvents(),
+	); err != nil {
 		return fmt.Errorf("error enable lifecycle events: %w", err)
 	}
 
 	if headers != nil {
-		err = t.Run(setHeaders(headers))
-		if err != nil {
+		if err := t.Run(setHeaders(headers)); err != nil {
 			return fmt.Errorf("error set headers: %w", err)
 		}
 	}
@@ -57,17 +70,31 @@ func (t *Tab) NavigateAndWaitFor(addr string, headers map[string]any, eventName
 		return fmt.Errorf("status code is %d:%s", resp.Status, resp.StatusText)
 	}
 
-	err = t.Run(waitFor(eventName))
-	if err != nil {
+	if err = t.Run(waitFor(eventName)); err != nil {
 		return fmt.Errorf("error waiting for %s on page %s: %w", eventName, addr, err)
 	}
 
 	return nil
 }
 
+// WithTimeout set the timeout for the actions in the current tab.
+func (t *Tab) WithTimeout(timeout time.Duration) {
+	t.ctx, t.cancel = context.WithTimeout(t.ctx, timeout)
+}
+
 // Run executes the actions in the current tab.
-func (t *Tab) Run(actions chromedp.Action) error {
-	return chromedp.Run(t.ctx, actions)
+func (t *Tab) Run(actions ...chromedp.Action) error {
+	return chromedp.Run(t.ctx, actions...)
+}
+
+// RunWithTimeout executes the actions in the current tab.
+func (t *Tab) RunWithTimeout(timeout time.Duration, actions ...chromedp.Action) error {
+	ctx, cancel := context.WithTimeout(t.ctx, timeout)
+	err := chromedp.Run(ctx, actions...)
+
+	cancel()
+
+	return err //nolint:wrapcheck
 }
 
 // Context returns the current tab's context.

pkg/plugin/client/client.go

@@ -4,15 +4,19 @@ import (
 	"bytes"
 	"context"
 	"encoding/base64"
+	"encoding/csv"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
 	"net/url"
 	"reflect"
 	"strconv"
+	"strings"
 	"sync"
 	"time"
 
+	"github.com/chromedp/cdproto/browser"
 	"github.com/chromedp/chromedp"
 	"github.com/grafana/grafana-plugin-sdk-go/backend/log"
 	"github.com/mahendrapaipuri/grafana-dashboard-reporter-app/pkg/plugin/chrome"
@@ -34,6 +38,16 @@ var (
 	dashboardDataJS = `[...document.getElementsByClassName('react-grid-item')].map((e) => ({"width": e.style.width, "height": e.style.height, "transform": e.style.transform, "id": e.getAttribute("data-panelid")}))`
 )
 
+// Tables related javascripts.
+const (
+	selPageScrollbar                                 = `#page-scrollbar`
+	selTimePickerTimeRangeToolTip                    = `div[role="tooltip"]`
+	selTimePickerButton                              = `button[aria-controls="TimePickerContent"]`
+	selDownloadCSVButton                             = `div[aria-label="Panel inspector Data content"] button[type="button"][aria-disabled="false"]`
+	selInspectPanelDataTabExpandDataOptions          = `div[role='dialog'] button[aria-expanded=false]`
+	selInspectPanelDataTabApplyTransformationsToggle = `div[data-testid="dataOptions"] input:not(#excel-toggle):not(#formatted-data-toggle) + label`
+)
+
 // Browser vars.
 var (
 	// We must set a view port to browser to ensure chromedp (or chromium)
@@ -57,6 +71,7 @@ var getPanelRetrySleepTime = time.Duration(10) * time.Second
 type Grafana interface {
 	Dashboard(ctx context.Context, dashUID string) (dashboard.Dashboard, error)
 	PanelPNG(ctx context.Context, dashUID string, p dashboard.Panel, t dashboard.TimeRange) (dashboard.PanelImage, error)
+	PanelCSV(ctx context.Context, dashUID string, p dashboard.Panel, t dashboard.TimeRange) (dashboard.CSVData, error)
 }
 
 type Credential struct {
@@ -246,8 +261,9 @@ func (g GrafanaClient) dashboardFromBrowser(dashUID string) ([]interface{}, erro
 	return dashboardData, nil
 }
 
+// PanelPNG returns encoded PNG image of a given panel.
 func (g GrafanaClient) PanelPNG(ctx context.Context, dashUID string, p dashboard.Panel, t dashboard.TimeRange) (dashboard.PanelImage, error) {
-	panelURL := g.getPanelURL(p, dashUID, t)
+	panelURL := g.getPanelPNGURL(p, dashUID, t)
 
 	// Create a new request for panel
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, panelURL, nil)
@@ -309,7 +325,8 @@ func (g GrafanaClient) PanelPNG(ctx context.Context, dashUID string, p dashboard
 	}, nil
 }
 
-func (g GrafanaClient) getPanelURL(p dashboard.Panel, dashUID string, t dashboard.TimeRange) string {
+// getPanelPNGURL returns the URL to fetch panel PNG.
+func (g GrafanaClient) getPanelPNGURL(p dashboard.Panel, dashUID string, t dashboard.TimeRange) string {
 	values := url.Values{}
 	values.Add("theme", g.conf.Theme)
 	values.Add("panelId", strconv.Itoa(p.ID))
@@ -343,3 +360,149 @@ func (g GrafanaClient) getPanelURL(p dashboard.Panel, dashUID string, t dashboar
 	// Get Panel API endpoint
 	return fmt.Sprintf("%s/render/d-solo/%s/_?%s", g.appURL, dashUID, values.Encode())
 }
+
+// PanelCSV returns CSV data of a given panel.
+func (g GrafanaClient) PanelCSV(_ context.Context, dashUID string, p dashboard.Panel, t dashboard.TimeRange) (dashboard.CSVData, error) {
+	panelURL := g.getPanelCSVURL(p, dashUID, t)
+	// Create a new tab
+	tab := g.chromeInstance.NewTab(g.logger, g.conf)
+	// Set a timeout for the tab
+	// Fail-safe for newer Grafana versions, if css has been changed.
+	tab.WithTimeout(60 * time.Second)
+	defer tab.Close(g.logger)
+
+	var headers map[string]any
+	if g.credential.HeaderName != "" {
+		headers = map[string]any{
+			g.credential.HeaderName: g.credential.HeaderValue,
+		}
+	}
+
+	g.logger.Debug("fetch table data via browser", "url", panelURL)
+
+	err := tab.NavigateAndWaitFor(panelURL, headers, "networkIdle")
+	if err != nil {
+		return nil, fmt.Errorf("NavigateAndWaitFor: %w", err)
+	}
+
+	// this will be used to capture the blob URL of the CSV download
+	blobURLCh := make(chan string, 1)
+
+	// If an error occurs on the way to fetching the CSV data, it will be sent to this channel
+	errCh := make(chan error, 1)
+
+	// Listen for download events. Downloading from JavaScript won't emit any network events.
+	chromedp.ListenTarget(tab.Context(), func(event interface{}) {
+		if eventDownloadWillBegin, ok := event.(*browser.EventDownloadWillBegin); ok {
+			g.logger.Debug("got CSV download URL", "url", eventDownloadWillBegin.URL)
+			// once we have the download URL, we can fetch the CSV data via JavaScript.
+			blobURLCh <- eventDownloadWillBegin.URL
+		}
+	})
+
+	downTasks := chromedp.Tasks{
+		// Downloads needs to be allowed, otherwise the CSV request will be denied.
+		// Allow download events to emit so we can get the download URL.
+		browser.SetDownloadBehavior(browser.SetDownloadBehaviorBehaviorAllowAndName).
+			WithDownloadPath("/dev/null").
+			WithEventsEnabled(true),
+	}
+
+	if err = tab.RunWithTimeout(2*time.Second, downTasks); err != nil {
+		return nil, fmt.Errorf("error setting download behavior: %w", err)
+	}
+
+	if err = tab.RunWithTimeout(2*time.Second, chromedp.WaitVisible(selDownloadCSVButton, chromedp.ByQuery)); err != nil {
+		return nil, fmt.Errorf("error waiting for download CSV button: %w", err)
+	}
+
+	if err = tab.RunWithTimeout(2*time.Second, chromedp.Click(selInspectPanelDataTabExpandDataOptions, chromedp.ByQuery)); err != nil {
+		return nil, fmt.Errorf("error clicking on expand data options: %w", err)
+	}
+
+	if err = tab.RunWithTimeout(1*time.Second, chromedp.Click(selInspectPanelDataTabApplyTransformationsToggle, chromedp.ByQuery)); err != nil && !errors.Is(err, context.DeadlineExceeded) {
+		return nil, fmt.Errorf("error clicking on apply transformations toggle: %w", err)
+	}
+
+	if err = tab.RunWithTimeout(1*time.Second, chromedp.Click(selInspectPanelDataTabApplyTransformationsToggle, chromedp.ByQuery)); err != nil && !errors.Is(err, context.DeadlineExceeded) {
+		return nil, fmt.Errorf("error clicking on apply transformations toggle: %w", err)
+	}
+
+	// Run all tasks in a goroutine.
+	// If an error occurs, it will be sent to the errCh channel.
+	// If a element can't be found, a timeout will occur and the context will be canceled.
+	go func() {
+		task := chromedp.Click(selDownloadCSVButton, chromedp.ByQuery)
+		if err := tab.Run(task); err != nil {
+			errCh <- fmt.Errorf("error fetching dashboard URL from browser %s: %w", panelURL, err)
+		}
+	}()
+
+	var blobURL string
+
+	select {
+	case blobURL = <-blobURLCh:
+		if blobURL == "" {
+			return nil, fmt.Errorf("error fetching CSV data from URL from browser %s: %w", panelURL, ErrEmptyBlobURL)
+		}
+	case err := <-errCh:
+		return nil, fmt.Errorf("error fetching CSV data from URL from browser %s: %w", panelURL, err)
+	case <-tab.Context().Done():
+		return nil, fmt.Errorf("error fetching CSV data from URL from browser %s: %w", panelURL, tab.Context().Err())
+	}
+
+	close(blobURLCh)
+	close(errCh)
+
+	var buf []byte
+
+	task := chromedp.Evaluate(
+		// fetch the CSV data from the blob URL, using Javascript.
+		fmt.Sprintf("fetch('%s').then(r => r.blob()).then(b => new Response(b).text()).then(t => t)", blobURL),
+		&buf,
+		chrome.WithAwaitPromise,
+	)
+
+	if err := tab.RunWithTimeout(45*time.Second, task); err != nil {
+		return nil, fmt.Errorf("error fetching CSV data from URL from browser %s: %w", panelURL, err)
+	}
+
+	if len(buf) == 0 {
+		return nil, fmt.Errorf("error fetching CSV data from URL from browser %s: %w", panelURL, ErrEmptyCSVData)
+	}
+
+	csvStringData, err := strconv.Unquote(string(buf))
+	if err != nil {
+		return nil, fmt.Errorf("error unquoting CSV data: %w", err)
+	}
+
+	reader := csv.NewReader(strings.NewReader(csvStringData))
+
+	csvData, err := reader.ReadAll()
+	if err != nil {
+		return nil, fmt.Errorf("error reading CSV data: %w", err)
+	}
+
+	return csvData, nil
+}
+
+// getPanelCSVURL returns URL to fetch panel's CSV data.
+func (g GrafanaClient) getPanelCSVURL(p dashboard.Panel, dashUID string, t dashboard.TimeRange) string {
+	values := url.Values{}
+	values.Add("theme", g.conf.Theme)
+	values.Add("viewPanel", strconv.Itoa(p.ID))
+	values.Add("from", t.From)
+	values.Add("to", t.To)
+	values.Add("inspect", strconv.Itoa(p.ID))
+	values.Add("inspectTab", "data")
+
+	// Add templated queryParams to URL
+	for k, v := range g.queryParams {
+		for _, singleValue := range v {
+			values.Add(k, singleValue)
+		}
+	}
+
+	// Get Panel API endpoint
+	return fmt.Sprintf("%s/d/%s/_?%s", g.appURL, dashUID, values.Encode())
+}

pkg/plugin/client/errors.go

@@ -5,4 +5,6 @@ import "errors"
 var (
 	ErrJavaScriptReturnedNoData = errors.New("javascript did not return any dashboard data")
 	ErrDashboardHTTPError       = errors.New("dashboard request does not return 200 OK")
+	ErrEmptyBlobURL             = errors.New("empty blob URL")
+	ErrEmptyCSVData             = errors.New("empty csv data")
 )