From 036cb20d43b8f2d3cdb4de79d11f97ff63831025 Mon Sep 17 00:00:00 2001 From: Mateusz Sekara Date: Mon, 9 Dec 2024 12:18:32 +0100 Subject: [PATCH] CCIP-4447 Publishing status of the OCR3 plugin (based on the config digest) (#15544) * Publishing status of the OCR3 plugin (based on the config digest) * Publishing status of the OCR3 plugin (based on the config digest) --- .changeset/dull-readers-rush.md | 5 ++++ core/services/ocr3/promwrapper/factory.go | 2 ++ core/services/ocr3/promwrapper/plugin.go | 28 +++++++++++++++---- core/services/ocr3/promwrapper/plugin_test.go | 19 +++++++++---- core/services/ocr3/promwrapper/types.go | 7 +++++ 5 files changed, 49 insertions(+), 12 deletions(-) create mode 100644 .changeset/dull-readers-rush.md diff --git a/.changeset/dull-readers-rush.md b/.changeset/dull-readers-rush.md new file mode 100644 index 00000000000..3b6f2ae8758 --- /dev/null +++ b/.changeset/dull-readers-rush.md @@ -0,0 +1,5 @@ +--- +"chainlink": patch +--- + +Reporting number of OCR3 instances running using promwrapper #internal diff --git a/core/services/ocr3/promwrapper/factory.go b/core/services/ocr3/promwrapper/factory.go index 84269cf3779..6518cea3c0d 100644 --- a/core/services/ocr3/promwrapper/factory.go +++ b/core/services/ocr3/promwrapper/factory.go @@ -44,8 +44,10 @@ func (r ReportingPluginFactory[RI]) NewReportingPlugin(ctx context.Context, conf plugin, r.chainID, r.plugin, + config.ConfigDigest.String(), promOCR3ReportsGenerated, promOCR3Durations, + promOCR3PluginStatus, ) return wrapped, info, err } diff --git a/core/services/ocr3/promwrapper/plugin.go b/core/services/ocr3/promwrapper/plugin.go index e4e0c3d35d5..dcee5050d1e 100644 --- a/core/services/ocr3/promwrapper/plugin.go +++ b/core/services/ocr3/promwrapper/plugin.go @@ -14,27 +14,33 @@ var _ ocr3types.ReportingPlugin[any] = &reportingPlugin[any]{} type reportingPlugin[RI any] struct { ocr3types.ReportingPlugin[RI] - chainID string - plugin string + chainID string + plugin string + configDigest string // Prometheus components for tracking metrics reportsGenerated *prometheus.CounterVec durations *prometheus.HistogramVec + status *prometheus.GaugeVec } func newReportingPlugin[RI any]( origin ocr3types.ReportingPlugin[RI], chainID string, plugin string, + configDigest string, reportsGenerated *prometheus.CounterVec, durations *prometheus.HistogramVec, + status *prometheus.GaugeVec, ) *reportingPlugin[RI] { return &reportingPlugin[RI]{ ReportingPlugin: origin, chainID: chainID, plugin: plugin, + configDigest: configDigest, reportsGenerated: reportsGenerated, durations: durations, + status: status, } } @@ -88,15 +94,23 @@ func (p *reportingPlugin[RI]) ShouldTransmitAcceptedReport(ctx context.Context, return result, err } -func (p *reportingPlugin[RI]) trackReports( - function functionType, - count int, -) { +func (p *reportingPlugin[RI]) Close() error { + p.updateStatus(false) + return p.ReportingPlugin.Close() +} + +func (p *reportingPlugin[RI]) trackReports(function functionType, count int) { p.reportsGenerated. WithLabelValues(p.chainID, p.plugin, string(function)). Add(float64(count)) } +func (p *reportingPlugin[RI]) updateStatus(status bool) { + p.status. + WithLabelValues(p.chainID, p.plugin, p.configDigest). + Set(float64(boolToInt(status))) +} + func boolToInt(arg bool) int { if arg { return 1 @@ -118,5 +132,7 @@ func withObservedExecution[RI, R any]( WithLabelValues(p.chainID, p.plugin, string(function), strconv.FormatBool(success)). Observe(float64(time.Since(start))) + p.updateStatus(true) + return result, err } diff --git a/core/services/ocr3/promwrapper/plugin_test.go b/core/services/ocr3/promwrapper/plugin_test.go index 35a97d109aa..9a7b6f2e648 100644 --- a/core/services/ocr3/promwrapper/plugin_test.go +++ b/core/services/ocr3/promwrapper/plugin_test.go @@ -19,15 +19,15 @@ import ( func Test_ReportsGeneratedGauge(t *testing.T) { plugin1 := newReportingPlugin( fakePlugin[uint]{reports: make([]ocr3types.ReportPlus[uint], 2)}, - "123", "empty", promOCR3ReportsGenerated, promOCR3Durations, + "123", "empty", "abc", promOCR3ReportsGenerated, promOCR3Durations, promOCR3PluginStatus, ) plugin2 := newReportingPlugin( fakePlugin[bool]{reports: make([]ocr3types.ReportPlus[bool], 10)}, - "solana", "different_plugin", promOCR3ReportsGenerated, promOCR3Durations, + "solana", "different_plugin", "abc", promOCR3ReportsGenerated, promOCR3Durations, promOCR3PluginStatus, ) plugin3 := newReportingPlugin( fakePlugin[string]{err: errors.New("error")}, - "1234", "empty", promOCR3ReportsGenerated, promOCR3Durations, + "1234", "empty", "abc", promOCR3ReportsGenerated, promOCR3Durations, promOCR3PluginStatus, ) r1, err := plugin1.Reports(tests.Context(t), 1, nil) @@ -57,20 +57,27 @@ func Test_ReportsGeneratedGauge(t *testing.T) { g4 := testutil.ToFloat64(promOCR3ReportsGenerated.WithLabelValues("1234", "empty", "reports")) require.Equal(t, 0, int(g4)) + + pluginHealth := testutil.ToFloat64(promOCR3PluginStatus.WithLabelValues("123", "empty", "abc")) + require.Equal(t, 1, int(pluginHealth)) + + require.NoError(t, plugin1.Close()) + pluginHealth = testutil.ToFloat64(promOCR3PluginStatus.WithLabelValues("123", "empty", "abc")) + require.Equal(t, 0, int(pluginHealth)) } func Test_DurationHistograms(t *testing.T) { plugin1 := newReportingPlugin( fakePlugin[uint]{}, - "123", "empty", promOCR3ReportsGenerated, promOCR3Durations, + "123", "empty", "abc", promOCR3ReportsGenerated, promOCR3Durations, promOCR3PluginStatus, ) plugin2 := newReportingPlugin( fakePlugin[uint]{err: errors.New("error")}, - "123", "empty", promOCR3ReportsGenerated, promOCR3Durations, + "123", "empty", "abc", promOCR3ReportsGenerated, promOCR3Durations, promOCR3PluginStatus, ) plugin3 := newReportingPlugin( fakePlugin[uint]{}, - "solana", "commit", promOCR3ReportsGenerated, promOCR3Durations, + "solana", "commit", "abc", promOCR3ReportsGenerated, promOCR3Durations, promOCR3PluginStatus, ) for _, p := range []*reportingPlugin[uint]{plugin1, plugin2, plugin3} { diff --git a/core/services/ocr3/promwrapper/types.go b/core/services/ocr3/promwrapper/types.go index bf6a1b2a39c..2fa29dcdf20 100644 --- a/core/services/ocr3/promwrapper/types.go +++ b/core/services/ocr3/promwrapper/types.go @@ -48,4 +48,11 @@ var ( }, []string{"chainID", "plugin", "function", "success"}, ) + promOCR3PluginStatus = promauto.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "ocr3_reporting_plugin_status", + Help: "Gauge indicating whether plugin is up and running or not", + }, + []string{"chainID", "plugin", "configDigest"}, + ) )