From 2330feb4cba0ca9eb7c6e785c123bdc295f530a3 Mon Sep 17 00:00:00 2001 From: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> Date: Thu, 5 Sep 2024 17:01:07 -0400 Subject: [PATCH 01/13] HPCC-32603 prevent Grid network races adds AbortController signals to the Grid's queries, so that subsequent requests will properly abort any pending request Signed-off-by: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> --- esp/src/package-lock.json | 9 ++++----- esp/src/package.json | 2 +- esp/src/src-react/components/controls/Grid.tsx | 11 ++++++++++- esp/src/src/ESPWorkunit.ts | 4 ++-- esp/src/src/store/Deferred.ts | 2 +- esp/src/src/store/Paged.ts | 6 +++--- esp/src/src/store/Store.ts | 11 ++++++++--- 7 files changed, 29 insertions(+), 16 deletions(-) diff --git a/esp/src/package-lock.json b/esp/src/package-lock.json index 3391efbb526..d38e8aecddd 100644 --- a/esp/src/package-lock.json +++ b/esp/src/package-lock.json @@ -18,7 +18,7 @@ "@hpcc-js/chart": "2.84.1", "@hpcc-js/codemirror": "2.63.0", "@hpcc-js/common": "2.72.0", - "@hpcc-js/comms": "2.95.0", + "@hpcc-js/comms": "2.96.0", "@hpcc-js/dataflow": "8.1.7", "@hpcc-js/eclwatch": "2.75.3", "@hpcc-js/graph": "2.86.0", @@ -2082,10 +2082,9 @@ } }, "node_modules/@hpcc-js/comms": { - "version": "2.95.0", - "resolved": "https://registry.npmjs.org/@hpcc-js/comms/-/comms-2.95.0.tgz", - "integrity": "sha512-kzEyDxf1Msus5rhU0yO826JxgIl2kh/bI7yNTxHAlCfLKp4SU//zrj/MK5SuEDs2lUHvmTkfx1jbfGUAK3RGFg==", - "license": "Apache-2.0", + "version": "2.96.0", + "resolved": "https://registry.npmjs.org/@hpcc-js/comms/-/comms-2.96.0.tgz", + "integrity": "sha512-8faDDSBvH22lN1LHhD6xtoT/FSV3YGu7hZahuRtKp4JG27ai73AFcNOJNxyqUZ9SLN7R3oNrujqp04xpyQ6nIA==", "dependencies": { "@hpcc-js/ddl-shim": "^2.21.0", "@hpcc-js/util": "^2.52.0", diff --git a/esp/src/package.json b/esp/src/package.json index de3511485ec..9c023a0d207 100644 --- a/esp/src/package.json +++ b/esp/src/package.json @@ -44,7 +44,7 @@ "@hpcc-js/chart": "2.84.1", "@hpcc-js/codemirror": "2.63.0", "@hpcc-js/common": "2.72.0", - "@hpcc-js/comms": "2.95.0", + "@hpcc-js/comms": "2.96.0", "@hpcc-js/dataflow": "8.1.7", "@hpcc-js/eclwatch": "2.75.3", "@hpcc-js/graph": "2.86.0", diff --git a/esp/src/src-react/components/controls/Grid.tsx b/esp/src/src-react/components/controls/Grid.tsx index 129271d47b3..277e1ad261b 100644 --- a/esp/src/src-react/components/controls/Grid.tsx +++ b/esp/src/src-react/components/controls/Grid.tsx @@ -237,12 +237,21 @@ const FluentStoreGrid: React.FunctionComponent = ({ }); }); + const abortController = React.useRef(); + + React.useEffect(() => { + if (abortController.current) { + abortController.current.abort({ message: "Grid aborting stale request" }); + } + abortController.current = new AbortController(); + }, [query]); + const refreshTable = useDeepCallback((clearSelection = false) => { if (isNaN(start) || isNaN(count)) return; if (clearSelection) { selectionHandler.setItems([], true); } - const storeQuery = store.query({ ...query }, { start, count, sort: sorted ? [sorted] : undefined }); + const storeQuery = store.query({ ...query }, { start, count, sort: sorted ? [sorted] : undefined }, abortController.current.signal); storeQuery.total.then(total => { setTotal(total); }); diff --git a/esp/src/src/ESPWorkunit.ts b/esp/src/src/ESPWorkunit.ts index 1efadc5afc1..5cb9b91aeb5 100644 --- a/esp/src/src/ESPWorkunit.ts +++ b/esp/src/src/ESPWorkunit.ts @@ -1082,11 +1082,11 @@ export function CreateWUQueryStore(): BaseStore { + }, "Wuid", (request, abortSignal) => { if (request.Sortby && request.Sortby === "TotalClusterTime") { request.Sortby = "ClusterTime"; } - return service.WUQuery(request).then(response => { + return service.WUQuery(request, abortSignal).then(response => { const page = { start: undefined, end: undefined diff --git a/esp/src/src/store/Deferred.ts b/esp/src/src/store/Deferred.ts index db7c3288666..a0f4d0c65ce 100644 --- a/esp/src/src/store/Deferred.ts +++ b/esp/src/src/store/Deferred.ts @@ -68,7 +68,7 @@ export class Deferred implements Thenable { } } -export class DeferredResponse extends Deferred { +export class DeferredResponse extends Deferred { // --- Legacy Dojo Support (fake QeuryResults) --- forEach(callback) { diff --git a/esp/src/src/store/Paged.ts b/esp/src/src/store/Paged.ts index eb57375376a..b4c1025e8db 100644 --- a/esp/src/src/store/Paged.ts +++ b/esp/src/src/store/Paged.ts @@ -16,7 +16,7 @@ function dataPage(array: T[], start: number, size: number, total: number): Pa return retVal; } -type FetchData = (query: QueryRequest) => Thenable<{ data: T[], total: number }>; +type FetchData = (query: QueryRequest, abortSignal?: AbortSignal) => Thenable<{ data: T[], total: number }>; export interface RequestFields { start: keyof T; @@ -43,7 +43,7 @@ export class Paged ext this._fetchData = fetchData; } - fetchData(request: QueryRequest, options: QueryOptions): ThenableResponse { + fetchData(request: QueryRequest, options: QueryOptions, abortSignal?: AbortSignal): ThenableResponse { if (options.start !== undefined && options.count !== undefined) { request[this._requestFields.start] = options.start as any; request[this._requestFields.count] = options.count as any; @@ -52,7 +52,7 @@ export class Paged ext request[this._requestFields.sortBy] = options.sort[0].attribute as any; request[this._requestFields.descending] = options.sort[0].descending as any; } - return this._fetchData(request).then(response => { + return this._fetchData(request, abortSignal).then(response => { response.data.forEach(row => { this.index[this.getIdentity(row)] = row; }); diff --git a/esp/src/src/store/Store.ts b/esp/src/src/store/Store.ts index d41d24e893c..9e8aab02b15 100644 --- a/esp/src/src/store/Store.ts +++ b/esp/src/src/store/Store.ts @@ -1,5 +1,8 @@ import * as QueryResults from "dojo/store/util/QueryResults"; import { DeferredResponse, Thenable } from "./Deferred"; +import { scopedLogger } from "@hpcc-js/util"; + +const logger = scopedLogger("src/store/Store.ts"); // Query --- export type Key = string | number | symbol; @@ -32,7 +35,7 @@ export abstract class BaseStore { this.responseIDField = responseIDField; } - protected abstract fetchData(request: QueryRequest, options: QueryOptions): ThenableResponse; + protected abstract fetchData(request: QueryRequest, options: QueryOptions, abortSignal?: AbortSignal): ThenableResponse; abstract get(id: string | number): T; @@ -40,11 +43,13 @@ export abstract class BaseStore { return object[this.responseIDField]; } - protected query(request: QueryRequest, options: QueryOptions): DeferredResponse { + protected query(request: QueryRequest, options: QueryOptions, abortSignal?: AbortSignal): DeferredResponse { const retVal = new DeferredResponse(); - this.fetchData(request, options).then((data: QueryResponse) => { + this.fetchData(request, options, abortSignal).then((data: QueryResponse) => { retVal.total.resolve(data.total); retVal.resolve(data); + }, (err) => { + logger.debug(err); }); return QueryResults(retVal); } From 3b27444f99b7d2a21d835750ead11bf9d977facf Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Fri, 13 Sep 2024 15:44:48 +0100 Subject: [PATCH 02/13] Split off 9.2.122 Signed-off-by: Jake Smith --- helm/hpcc/Chart.yaml | 4 ++-- helm/hpcc/templates/_helpers.tpl | 2 +- version.cmake | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/helm/hpcc/Chart.yaml b/helm/hpcc/Chart.yaml index ebfbe3c60c4..bbf494b7028 100644 --- a/helm/hpcc/Chart.yaml +++ b/helm/hpcc/Chart.yaml @@ -6,9 +6,9 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 9.2.121-closedown0 +version: 9.2.123-closedown0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 9.2.121-closedown0 +appVersion: 9.2.123-closedown0 diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl index a59efa7c3dd..496bda4e36f 100644 --- a/helm/hpcc/templates/_helpers.tpl +++ b/helm/hpcc/templates/_helpers.tpl @@ -1361,7 +1361,7 @@ Pass in dict with .root, .visibility defined {{- end -}} {{- define "hpcc.generateHelmVersion" -}} -helmVersion: 9.2.121-closedown0 +helmVersion: 9.2.123-closedown0 {{- end -}} {{/* diff --git a/version.cmake b/version.cmake index 1fa7a287d76..7095875c96e 100644 --- a/version.cmake +++ b/version.cmake @@ -5,8 +5,8 @@ set ( HPCC_NAME "Community Edition" ) set ( HPCC_PROJECT "community" ) set ( HPCC_MAJOR 9 ) set ( HPCC_MINOR 2 ) -set ( HPCC_POINT 121 ) +set ( HPCC_POINT 123 ) set ( HPCC_MATURITY "closedown" ) set ( HPCC_SEQUENCE 0 ) -set ( HPCC_TAG_TIMESTAMP "2024-09-05T16:41:58Z" ) +set ( HPCC_TAG_TIMESTAMP "2024-09-13T14:44:48Z" ) ### From 7d088d3d75ce4095ffeb354be013c4843fc884cf Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Fri, 13 Sep 2024 15:45:53 +0100 Subject: [PATCH 03/13] Split off 9.4.96 Signed-off-by: Jake Smith --- helm/hpcc/Chart.yaml | 4 ++-- helm/hpcc/templates/_helpers.tpl | 2 +- version.cmake | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/helm/hpcc/Chart.yaml b/helm/hpcc/Chart.yaml index fcfaf9d730d..76b4546e55f 100644 --- a/helm/hpcc/Chart.yaml +++ b/helm/hpcc/Chart.yaml @@ -6,9 +6,9 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 9.4.95-closedown0 +version: 9.4.97-closedown0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 9.4.95-closedown0 +appVersion: 9.4.97-closedown0 diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl index b633efc6628..b67c30179de 100644 --- a/helm/hpcc/templates/_helpers.tpl +++ b/helm/hpcc/templates/_helpers.tpl @@ -1473,7 +1473,7 @@ Pass in dict with .root, .visibility defined {{- end -}} {{- define "hpcc.generateHelmVersion" -}} -helmVersion: 9.4.95-closedown0 +helmVersion: 9.4.97-closedown0 {{- end -}} {{/* diff --git a/version.cmake b/version.cmake index e3a2fdcb7f0..46525ea8c6a 100644 --- a/version.cmake +++ b/version.cmake @@ -5,8 +5,8 @@ set ( HPCC_NAME "Community Edition" ) set ( HPCC_PROJECT "community" ) set ( HPCC_MAJOR 9 ) set ( HPCC_MINOR 4 ) -set ( HPCC_POINT 95 ) +set ( HPCC_POINT 97 ) set ( HPCC_MATURITY "closedown" ) set ( HPCC_SEQUENCE 0 ) -set ( HPCC_TAG_TIMESTAMP "2024-09-05T16:41:07Z" ) +set ( HPCC_TAG_TIMESTAMP "2024-09-13T14:45:53Z" ) ### From 699f0a0baa058223cfba0717ceaf35516a118d6e Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Fri, 13 Sep 2024 15:47:59 +0100 Subject: [PATCH 04/13] Split off 9.6.48 Signed-off-by: Jake Smith --- helm/hpcc/Chart.yaml | 4 ++-- helm/hpcc/templates/_helpers.tpl | 2 +- version.cmake | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/helm/hpcc/Chart.yaml b/helm/hpcc/Chart.yaml index 06e07c17893..f7d4cf394e0 100644 --- a/helm/hpcc/Chart.yaml +++ b/helm/hpcc/Chart.yaml @@ -6,9 +6,9 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 9.6.47-closedown0 +version: 9.6.49-closedown0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 9.6.47-closedown0 +appVersion: 9.6.49-closedown0 diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl index d18d890ec59..5dd48fa279d 100644 --- a/helm/hpcc/templates/_helpers.tpl +++ b/helm/hpcc/templates/_helpers.tpl @@ -1477,7 +1477,7 @@ Pass in dict with .root, .visibility defined {{- end -}} {{- define "hpcc.generateHelmVersion" -}} -helmVersion: 9.6.47-closedown0 +helmVersion: 9.6.49-closedown0 {{- end -}} {{/* diff --git a/version.cmake b/version.cmake index a4eeb6f1952..5808dec055e 100644 --- a/version.cmake +++ b/version.cmake @@ -5,8 +5,8 @@ set ( HPCC_NAME "Community Edition" ) set ( HPCC_PROJECT "community" ) set ( HPCC_MAJOR 9 ) set ( HPCC_MINOR 6 ) -set ( HPCC_POINT 47 ) +set ( HPCC_POINT 49 ) set ( HPCC_MATURITY "closedown" ) set ( HPCC_SEQUENCE 0 ) -set ( HPCC_TAG_TIMESTAMP "2024-09-05T16:28:41Z" ) +set ( HPCC_TAG_TIMESTAMP "2024-09-13T14:47:59Z" ) ### From 5b67f034e088ae5ca01b2ff643c9eeee6a84fc63 Mon Sep 17 00:00:00 2001 From: Shamser Ahmed Date: Wed, 3 Jul 2024 12:04:04 +0100 Subject: [PATCH 05/13] HPCC-32193 Fix some issues with spill stats in smart join activity The current temp file statistics for smartjoin did not include all stats from all temp files: 1) temp files were closed before its sizes were recorded in the stats 2) stats from some types of temp files were not being tracked such as overflowWriteFile from RHS 3) stats from temp files that were closed in CSpillableStreamBase were not preserved 4) peak temp file size was not tracked in CThorSpillableRowArray 5) make CThorRowCollectorBase use of stats from CThorSpillableRowArray for more accurate and simpler temp stats tracking. Signed-off-by: Shamser Ahmed HPCC-32193 Close overflowWriteFileIO when it's no longer needed Signed-off-by: Shamser Ahmed --- .../lookupjoin/thlookupjoinslave.cpp | 59 +++++++++++++++---- .../activities/nsplitter/thnsplitterslave.cpp | 2 +- thorlcr/msort/tsorts.cpp | 2 +- thorlcr/thorutil/thbuf.cpp | 14 ++--- thorlcr/thorutil/thmem.cpp | 48 +++++++-------- thorlcr/thorutil/thmem.hpp | 8 ++- thorlcr/thorutil/thormisc.cpp | 1 + thorlcr/thorutil/thormisc.hpp | 2 + 8 files changed, 86 insertions(+), 50 deletions(-) diff --git a/thorlcr/activities/lookupjoin/thlookupjoinslave.cpp b/thorlcr/activities/lookupjoin/thlookupjoinslave.cpp index 0441534af72..189e39c56e3 100644 --- a/thorlcr/activities/lookupjoin/thlookupjoinslave.cpp +++ b/thorlcr/activities/lookupjoin/thlookupjoinslave.cpp @@ -950,7 +950,7 @@ class CInMemJoinBase : public CSlaveActivity, public CAllOrLookupHelper, } } *rowProcessor; - CriticalSection rhsRowLock; + mutable CriticalSection rhsRowLock; Owned broadcaster; CBroadcaster *channel0Broadcaster; CriticalSection *broadcastLock; @@ -1092,7 +1092,10 @@ class CInMemJoinBase : public CSlaveActivity, public CAllOrLookupHelper, { CThorSpillableRowArray *rows = rhsSlaveRows.item(a); if (rows) + { + mergeRemappedStats(inactiveStats, rows, diskToTempStatsMap); rows->kill(); + } } rhs.kill(); } @@ -1808,6 +1811,8 @@ class CLookupJoinActivityBase : public CInMemJoinBase overflowWriteFile; Owned overflowWriteStream; + OwnedIFileIO overflowWriteFileIO; + mutable CriticalSection critOverflowWriteFileIO; rowcount_t overflowWriteCount; OwnedMalloc channelDistributors; unsigned nextRhsToSpill = 0; @@ -2096,7 +2101,6 @@ class CLookupJoinActivityBase : public CInMemJoinBaseaddRowBuffer(this); } @@ -2107,7 +2111,15 @@ class CLookupJoinActivityBase : public CInMemJoinBasenoteSize(overflowWriteFileIO->getStatistic(StSizeDiskWrite)); + overflowWriteFileIO.clear(); + } overflowWriteStream.clear(); // broadcast has finished, no more can be written + } } if (!hasFailedOverToLocal()) { @@ -2155,6 +2167,7 @@ class CLookupJoinActivityBase : public CInMemJoinBase rowLoader = createThorRowLoader(*this, queryRowInterfaces(leftITDL), helper->isLeftAlreadyLocallySorted() ? NULL : compareLeft); rowLoader->setTracingPrefix("Join left"); left.setown(rowLoader->load(left, abortSoon, false)); + mergeRemappedStats(PARENT::inactiveStats, rowLoader, diskToTempStatsMap); leftITDL = queryInput(0); // reset ActPrintLog("LHS loaded/sorted"); @@ -2550,6 +2564,7 @@ class CLookupJoinActivityBase : public CInMemJoinBase rightCollector; + Owned exception; try { CMarker marker(*this); @@ -2674,12 +2689,19 @@ class CLookupJoinActivityBase : public CInMemJoinBasequeryFromActivity()->queryContainer().queryHelper()->queryOutputMeta(); - // rows may either be in separate slave row arrays or in single rhs array, or split. - rowcount_t total = rightCollector ? rightCollector->numRows() : (getGlobalRHSTotal() + rhs.ordinality()); - throw checkAndCreateOOMContextException(this, e, "gathering RHS rows for lookup join", total, inputOutputMeta, NULL); + exception.setown(e); + else + { + IOutputMetaData *inputOutputMeta = rightITDL->queryFromActivity()->queryContainer().queryHelper()->queryOutputMeta(); + // rows may either be in separate slave row arrays or in single rhs array, or split. + rowcount_t total = rightCollector ? rightCollector->numRows() : (getGlobalRHSTotal() + rhs.ordinality()); + exception.setown(checkAndCreateOOMContextException(this, e, "gathering RHS rows for lookup join", total, inputOutputMeta, NULL)); + } } + if (rightCollector && rightCollector->hasSpilt()) + mergeRemappedStats(PARENT::inactiveStats, rightCollector, diskToTempStatsMap); + if (exception) + throw exception.getClear(); } public: static bool needDedup(IHThorHashJoinArg *helper) @@ -2943,7 +2965,7 @@ class CLookupJoinActivityBase : public CInMemJoinBaseputRow(rhsInRowsTemp.getClear(r)); - overflowWriteFile->noteSize(overflowWriteStream->getStatistic(StSizeDiskWrite)); + overflowWriteFile->noteSize(overflowWriteFileIO->getStatistic(StSizeDiskWrite)); return true; } if (hasFailedOverToLocal()) @@ -3001,12 +3023,13 @@ class CLookupJoinActivityBase : public CInMemJoinBasecreateOwnedTempFile(tempFilename.str())); - overflowWriteStream.setown(createRowWriter(&(overflowWriteFile->queryIFile()), queryRowInterfaces(rightITDL), rwFlags)); + overflowWriteFileIO.setown(overflowWriteFile->queryIFile().open(IFOcreate)); + overflowWriteStream.setown(createRowWriter(overflowWriteFileIO, queryRowInterfaces(rightITDL), rwFlags)); overflowWriteCount += rhsInRowsTemp.ordinality(); ForEachItemIn(r, rhsInRowsTemp) overflowWriteStream->putRow(rhsInRowsTemp.getClear(r)); - overflowWriteFile->noteSize(overflowWriteStream->getStatistic(StSizeDiskWrite)); + overflowWriteFile->noteSize(overflowWriteFileIO->getStatistic(StSizeDiskWrite)); return true; } virtual void gatherActiveStats(CRuntimeStatisticCollection &activeStats) const @@ -3018,6 +3041,19 @@ class CLookupJoinActivityBase : public CInMemJoinBase ForEachItemIn(a, rhsSlaveRows) { CThorSpillableRowArray &rows = *rhsSlaveRows.item(a); + mergeRemappedStats(PARENT::inactiveStats, &rows, diskToTempStatsMap); rhs.appendRows(rows, true); rows.kill(); // free up ptr table asap } diff --git a/thorlcr/activities/nsplitter/thnsplitterslave.cpp b/thorlcr/activities/nsplitter/thnsplitterslave.cpp index ef47bcc27bf..50da35c613f 100644 --- a/thorlcr/activities/nsplitter/thnsplitterslave.cpp +++ b/thorlcr/activities/nsplitter/thnsplitterslave.cpp @@ -405,7 +405,7 @@ class NSplitterSlaveActivity : public CSlaveActivity, implements ISharedSmartBuf { PARENT::gatherActiveStats(activeStats); if (sharedRowStream) - ::mergeStats(activeStats, sharedRowStream); + mergeRemappedStats(activeStats, sharedRowStream, diskToTempStatsMap); } // ISharedSmartBufferCallback impl. virtual void paged() { pagedOut = true; } diff --git a/thorlcr/msort/tsorts.cpp b/thorlcr/msort/tsorts.cpp index 7e58fa50ea1..10a95e6af0d 100644 --- a/thorlcr/msort/tsorts.cpp +++ b/thorlcr/msort/tsorts.cpp @@ -1338,7 +1338,7 @@ class CThorSorter : public CSimpleInterface, implements IThorSorter, implements throw; } - mergeStats(spillStats, sortedloader); + mergeRemappedStats(spillStats, sortedloader, diskToTempStatsMap); if (!abort) { diff --git a/thorlcr/thorutil/thbuf.cpp b/thorlcr/thorutil/thbuf.cpp index 2e45c7c78d8..ad2d42bdb60 100644 --- a/thorlcr/thorutil/thbuf.cpp +++ b/thorlcr/thorutil/thbuf.cpp @@ -2449,7 +2449,7 @@ class CSharedFullSpillingWriteAhead : public CInterfaceOfflush(); tempFileOwner->noteSize(iFileIO->getStatistic(StSizeDiskWrite)); - updateRemappedStatsDelta(inactiveStats, previousFileStats, iFileIO, diskToTempStatsMap); // NB: also updates prev to current + updateStatsDelta(stats, previousFileStats, iFileIO); // NB: also updates prev to current previousFileStats.reset(); iFileIO.clear(); } } void createOutputStream() { - closeWriter(); // Ensure stats from closing files are preserved in inactiveStats + closeWriter(); // Ensure stats from closing files are preserved in stats // NB: Called once, when spilling starts. tempFileOwner.setown(activity.createOwnedTempFile(baseTmpFilename)); auto res = createSerialOutputStream(&(tempFileOwner->queryIFile()), compressHandler, options, numOutputs + 1); outputStream.setown(std::get<0>(res)); iFileIO.setown(std::get<1>(res)); totalInputRowsRead = inMemTotalRows; - inactiveStats.addStatistic(StNumSpills, 1); + stats.addStatistic(StNumSpills, 1); } void writeRowsFromInput() { @@ -2539,7 +2539,7 @@ class CSharedFullSpillingWriteAhead : public CInterfaceOfflush(); totalInputRowsRead.fetch_add(newRowsWritten); tempFileOwner->noteSize(iFileIO->getStatistic(StSizeDiskWrite)); - updateRemappedStatsDelta(inactiveStats, previousFileStats, iFileIO, diskToTempStatsMap); // NB: also updates prev to current + updateStatsDelta(stats, previousFileStats, iFileIO); // NB: also updates prev to current // JCSMORE - could track size written, and start new file at this point (e.g. every 100MB), // and track their starting points (by row #) in a vector // We could then tell if/when the readers catch up, and remove consumed files as they do. @@ -2553,7 +2553,7 @@ class CSharedFullSpillingWriteAhead : public CInterfaceOfqueryRowMetaData()), serializer(rowIf->queryRowSerializer()), allocator(rowIf->queryRowAllocator()), deserializer(rowIf->queryRowDeserializer()), - inactiveStats(spillStatistics), previousFileStats(spillStatistics) + stats(tempFileStatistics), previousFileStats(tempFileStatistics) { assertex(input); @@ -2717,7 +2717,7 @@ class CSharedFullSpillingWriteAhead : public CInterfaceOf spillFile; + CRuntimeStatisticCollection stats; bool spillRows() { @@ -248,13 +249,13 @@ class CSpillableStreamBase : public CSpillable spillFile.setown(activity.createOwnedTempFile(tempName.str())); VStringBuffer spillPrefixStr("SpillableStream(%u)", spillPriority); rows.save(*spillFile, spillCompInfo, false, spillPrefixStr.str()); // saves committed rows + mergeStats(stats, &rows); rows.kill(); // no longer needed, readers will pull from spillFile. NB: ok to kill array as rows is never written to or expanded - spillFile->noteSize(spillFile->queryIFile().size()); return true; } public: CSpillableStreamBase(CActivityBase &_activity, CThorSpillableRowArray &inRows, IThorRowInterfaces *_rowIf, EmptyRowSemantics _emptyRowSemantics, unsigned _spillPriority) - : CSpillable(_activity, _rowIf, _spillPriority), rows(_activity), emptyRowSemantics(_emptyRowSemantics) + : CSpillable(_activity, _rowIf, _spillPriority), rows(_activity), emptyRowSemantics(_emptyRowSemantics), stats(diskRemoteStatistics) { assertex(inRows.isFlushed()); spillCompInfo = 0x0; @@ -265,6 +266,10 @@ class CSpillableStreamBase : public CSpillable { ensureSpillingCallbackRemoved(); } + unsigned __int64 getStatistic(StatisticKind kind) const + { + return stats.getStatisticValue(kind); + } // IBufferedRowCallback virtual bool freeBufferedRows(bool critical) override { @@ -1328,13 +1333,13 @@ void CThorSpillableRowArray::safeUnregisterWriteCallback(IWritePosCallback &cb) } CThorSpillableRowArray::CThorSpillableRowArray(CActivityBase &activity) - : CThorExpandingRowArray(activity) + : CThorExpandingRowArray(activity), stats(tempFileStatistics) { throwOnOom = false; } CThorSpillableRowArray::CThorSpillableRowArray(CActivityBase &activity, IThorRowInterfaces *rowIf, EmptyRowSemantics emptyRowSemantics, StableSortFlag stableSort, rowidx_t initialSize, size32_t _commitDelta) - : CThorExpandingRowArray(activity, rowIf, ers_forbidden, stableSort, false, initialSize), commitDelta(_commitDelta) + : CThorExpandingRowArray(activity, rowIf, ers_forbidden, stableSort, false, initialSize), commitDelta(_commitDelta), stats(tempFileStatistics) { } @@ -1363,6 +1368,7 @@ void CThorSpillableRowArray::kill() { clearRows(); CThorExpandingRowArray::kill(); + stats.reset(); } void CThorSpillableRowArray::sort(ICompare &compare, unsigned maxCores) @@ -1413,7 +1419,8 @@ rowidx_t CThorSpillableRowArray::save(CFileOwner &iFileOwner, unsigned _spillCom nextCB = &cbCopy.popGet(); nextCBI = nextCB->queryRecordNumber(); } - Owned writer = createRowWriter(&iFileOwner.queryIFile(), rowIf, rwFlags, nullptr, compBlkSz); + OwnedIFileIO iFileIO = iFileOwner.queryIFile().open(IFOcreate); + Owned writer = createRowWriter(iFileIO, rowIf, rwFlags, nullptr, compBlkSz); rowidx_t i=0; rowidx_t rowsWritten=0; try @@ -1452,7 +1459,6 @@ rowidx_t CThorSpillableRowArray::save(CFileOwner &iFileOwner, unsigned _spillCom ++i; } writer->flush(NULL); - iFileOwner.noteSize(writer->getStatistic(StSizeDiskWrite)); } catch (IException *e) { @@ -1463,6 +1469,10 @@ rowidx_t CThorSpillableRowArray::save(CFileOwner &iFileOwner, unsigned _spillCom firstRow += n; offset_t bytesWritten = writer->getPosition(); writer.clear(); + mergeStats(stats, iFileIO); + offset_t sizeTempFile = iFileIO->getStatistic(StSizeDiskWrite); + iFileOwner.noteSize(sizeTempFile); + stats.addStatistic(StNumSpills, 1); ActPrintLog(&activity, "%s: CThorSpillableRowArray::save done, rows written = %" RIPF "u, bytes = %" I64F "u, firstRow = %u", _tracingPrefix, rowsWritten, (__int64)bytesWritten, firstRow); return rowsWritten; } @@ -1638,11 +1648,7 @@ class CThorRowCollectorBase : public CSpillable Owned spillableRowSet; unsigned options = 0; unsigned spillCompInfo = 0; - RelaxedAtomic statOverflowCount{0}; - RelaxedAtomic statSizeSpill{0}; - RelaxedAtomic<__uint64> statSpillCycles{0}; RelaxedAtomic<__uint64> statSortCycles{0}; - bool spillRows(bool critical) { //This must only be called while a lock is held on spillableRows @@ -1668,11 +1674,6 @@ class CThorRowCollectorBase : public CSpillable spillableRows.save(*tempFileOwner, spillCompInfo, false, spillPrefixStr.str()); // saves committed rows spillFiles.append(tempFileOwner.getLink()); ++overflowCount; - statOverflowCount.fastAdd(1); // NB: this is total over multiple uses of this class - offset_t tempFileSize = tempFileOwner->queryIFile().size(); - statSizeSpill.fastAdd(tempFileSize); - tempFileOwner->noteSize(tempFileSize); - statSpillCycles.fastAdd(spillTimer.elapsedCycles()); return true; } void setEmptyRowSemantics(EmptyRowSemantics _emptyRowSemantics) @@ -1960,26 +1961,17 @@ class CThorRowCollectorBase : public CSpillable { options = _options; } - unsigned __int64 getStatistic(StatisticKind kind) + unsigned __int64 getStatistic(StatisticKind kind) const { switch (kind) { - case StCycleSpillElapsedCycles: - return statSpillCycles; case StCycleSortElapsedCycles: return statSortCycles; - case StTimeSpillElapsed: - return cycle_to_nanosec(statSpillCycles); case StTimeSortElapsed: return cycle_to_nanosec(statSortCycles); - case StNumSpills: - return statOverflowCount; - case StSizeSpillFile: - return statSizeSpill; default: - break; + return spillableRows.getStatistic(kind); } - return 0; } bool hasSpilt() const { return overflowCount >= 1; } @@ -2048,7 +2040,7 @@ class CThorRowLoader : public CThorRowCollectorBase, implements IThorRowLoader } virtual void resize(rowidx_t max) override { CThorRowCollectorBase::resize(max); } virtual void setOptions(unsigned options) override { CThorRowCollectorBase::setOptions(options); } - virtual unsigned __int64 getStatistic(StatisticKind kind) override { return CThorRowCollectorBase::getStatistic(kind); } + virtual unsigned __int64 getStatistic(StatisticKind kind) const override { return CThorRowCollectorBase::getStatistic(kind); } virtual bool hasSpilt() const override { return CThorRowCollectorBase::hasSpilt(); } virtual void setTracingPrefix(const char *tracing) override { CThorRowCollectorBase::setTracingPrefix(tracing); } virtual void reset() override { CThorRowCollectorBase::reset(); } @@ -2103,7 +2095,7 @@ class CThorRowCollector : public CThorRowCollectorBase, implements IThorRowColle } virtual void resize(rowidx_t max) override { CThorRowCollectorBase::resize(max); } virtual void setOptions(unsigned options) override { CThorRowCollectorBase::setOptions(options); } - virtual unsigned __int64 getStatistic(StatisticKind kind) override { return CThorRowCollectorBase::getStatistic(kind); } + virtual unsigned __int64 getStatistic(StatisticKind kind) const override { return CThorRowCollectorBase::getStatistic(kind); } virtual bool hasSpilt() const override { return CThorRowCollectorBase::hasSpilt(); } virtual void setTracingPrefix(const char *tracing) override { CThorRowCollectorBase::setTracingPrefix(tracing); } // IThorArrayLock diff --git a/thorlcr/thorutil/thmem.hpp b/thorlcr/thorutil/thmem.hpp index 8e4f1b896a8..a89d4cdd0f2 100644 --- a/thorlcr/thorutil/thmem.hpp +++ b/thorlcr/thorutil/thmem.hpp @@ -413,7 +413,7 @@ class graph_decl CThorSpillableRowArray : private CThorExpandingRowArray, implem mutable CriticalSection cs; ICopyArrayOf writeCallbacks; size32_t compBlkSz = 0; // means use default - + CRuntimeStatisticCollection stats; // reset after each kill bool _flush(bool force); void doFlush(); inline bool needToMoveRows(bool force) { return (firstRow != 0 && (force || (firstRow >= commitRows/2))); } @@ -484,6 +484,10 @@ class graph_decl CThorSpillableRowArray : private CThorExpandingRowArray, implem inline rowidx_t numCommitted() const { return commitRows - firstRow; } //MORE::Not convinced this is very safe! inline rowidx_t queryTotalRows() const { return CThorExpandingRowArray::ordinality(); } // includes uncommited rows + inline unsigned __int64 getStatistic(StatisticKind kind) const + { + return stats.getStatisticValue(kind); + } // access to void swap(CThorSpillableRowArray &src); @@ -542,7 +546,7 @@ interface IThorRowCollectorCommon : extends IInterface, extends IThorArrayLock virtual void setup(ICompare *iCompare, StableSortFlag stableSort=stableSort_none, RowCollectorSpillFlags diskMemMix=rc_mixed, unsigned spillPriority=50) = 0; virtual void resize(rowidx_t max) = 0; virtual void setOptions(unsigned options) = 0; - virtual unsigned __int64 getStatistic(StatisticKind kind) = 0; + virtual unsigned __int64 getStatistic(StatisticKind kind) const = 0; virtual bool hasSpilt() const = 0; // equivalent to numOverlows() >= 1 virtual void setTracingPrefix(const char *tracing) = 0; virtual void reset() = 0; diff --git a/thorlcr/thorutil/thormisc.cpp b/thorlcr/thorutil/thormisc.cpp index a9e493e02e4..70bdedb9361 100644 --- a/thorlcr/thorutil/thormisc.cpp +++ b/thorlcr/thorutil/thormisc.cpp @@ -97,6 +97,7 @@ const StatisticsMapping hashDedupActivityStatistics({}, spillStatistics, diskWri const StatisticsMapping hashDistribActivityStatistics({StNumLocalRows, StNumRemoteRows, StSizeRemoteWrite}, basicActivityStatistics); const StatisticsMapping loopActivityStatistics({StNumIterations}, basicActivityStatistics); const StatisticsMapping graphStatistics({StNumExecutions, StSizeSpillFile, StSizeGraphSpill, StSizePeakTempDisk, StSizePeakEphemeralDisk, StTimeUser, StTimeSystem, StNumContextSwitches, StSizeMemory, StSizePeakMemory, StSizeRowMemory, StSizePeakRowMemory}, executeStatistics); +const StatisticsMapping tempFileStatistics({StNumSpills}, diskRemoteStatistics); const StatKindMap diskToTempStatsMap ={ {StSizeDiskWrite, StSizeSpillFile}, diff --git a/thorlcr/thorutil/thormisc.hpp b/thorlcr/thorutil/thormisc.hpp index 3159d58ad48..17ebe967e8b 100644 --- a/thorlcr/thorutil/thormisc.hpp +++ b/thorlcr/thorutil/thormisc.hpp @@ -167,6 +167,8 @@ extern graph_decl const StatisticsMapping soapcallActivityStatistics; extern graph_decl const StatisticsMapping indexReadFileStatistics; extern graph_decl const StatisticsMapping hashDedupActivityStatistics; extern graph_decl const StatisticsMapping hashDistribActivityStatistics; +extern graph_decl const StatisticsMapping tempFileStatistics; + // Maps disk related stats to spill stats extern graph_decl const std::map diskToTempStatsMap; From 81e878937a28c110480cc3b4ec8532233bd0c427 Mon Sep 17 00:00:00 2001 From: Gordon Smith Date: Mon, 16 Sep 2024 14:39:37 +0100 Subject: [PATCH 06/13] HPCC-32641 Metrics occasionally shows a blank page Removed lazy loading of metrics page. Signed-off-by: Gordon Smith --- esp/src/src-react/components/WorkunitDetails.tsx | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/esp/src/src-react/components/WorkunitDetails.tsx b/esp/src/src-react/components/WorkunitDetails.tsx index 8e595bf7b16..f30bbf974ae 100644 --- a/esp/src/src-react/components/WorkunitDetails.tsx +++ b/esp/src/src-react/components/WorkunitDetails.tsx @@ -27,8 +27,7 @@ import { Workflows } from "./Workflows"; import { WorkunitSummary } from "./WorkunitSummary"; import { TabInfo, DelayLoadedPanel, OverflowTabList } from "./controls/TabbedPanes/index"; import { ECLArchive } from "./ECLArchive"; - -const Metrics = React.lazy(() => import("./Metrics").then(mod => ({ default: mod.Metrics }))); +import { Metrics } from "./Metrics"; const logger = scopedLogger("src-react/components/WorkunitDetails.tsx"); From 2e65a77954391f51af2dc232b68d25f4ee144c57 Mon Sep 17 00:00:00 2001 From: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> Date: Mon, 16 Sep 2024 10:30:46 -0400 Subject: [PATCH 07/13] HPCC-32669 ECL Watch v9 fix grids missing command bars fixes an issue where the CommandBar components for some WU and File details tabs were hidden by the contents of the grid contents Signed-off-by: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> --- esp/src/src-react/components/Helpers.tsx | 31 +++++++-------- esp/src/src-react/components/InfoGrid.tsx | 2 +- esp/src/src-react/components/LogViewer.tsx | 18 ++++----- .../src-react/components/MetricsOptions.tsx | 22 ++++++----- esp/src/src-react/components/Resources.tsx | 37 +++++++++--------- esp/src/src-react/components/Results.tsx | 33 ++++++++-------- esp/src/src-react/components/SourceFiles.tsx | 37 +++++++++--------- esp/src/src-react/components/SubFiles.tsx | 37 +++++++++--------- esp/src/src-react/components/SuperFiles.tsx | 31 +++++++-------- esp/src/src-react/components/Variables.tsx | 33 ++++++++-------- esp/src/src-react/components/Workflows.tsx | 33 ++++++++-------- .../src-react/components/controls/Grid.tsx | 38 ++++++++++--------- 12 files changed, 182 insertions(+), 170 deletions(-) diff --git a/esp/src/src-react/components/Helpers.tsx b/esp/src/src-react/components/Helpers.tsx index 458308debd8..73fed185eca 100644 --- a/esp/src/src-react/components/Helpers.tsx +++ b/esp/src/src-react/components/Helpers.tsx @@ -1,8 +1,9 @@ import * as React from "react"; -import { CommandBar, ContextualMenuItemType, ICommandBarItemProps, Link, ScrollablePane, Sticky } from "@fluentui/react"; +import { CommandBar, ContextualMenuItemType, ICommandBarItemProps, Link } from "@fluentui/react"; import * as ESPRequest from "src/ESPRequest"; import nlsHPCC from "src/nlsHPCC"; import { HelperRow, useWorkunitHelpers } from "../hooks/workunit"; +import { HolyGrail } from "../layouts/HolyGrail"; import { FluentGrid, useCopyButtons, useFluentStoreState, FluentColumns } from "./controls/Grid"; import { ShortVerticalDivider } from "./Common"; @@ -223,18 +224,18 @@ export const Helpers: React.FunctionComponent = ({ setData(helpers); }, [helpers]); - return - - - - - ; + return } + main={ + + } + />; }; diff --git a/esp/src/src-react/components/InfoGrid.tsx b/esp/src/src-react/components/InfoGrid.tsx index ec028305d8b..5860ef755bc 100644 --- a/esp/src/src-react/components/InfoGrid.tsx +++ b/esp/src/src-react/components/InfoGrid.tsx @@ -236,7 +236,7 @@ export const InfoGrid: React.FunctionComponent = ({ } }, [data.length]); - return
+ return
{({ size }) => = ({ return } main={ -
- -
+ } />; }; diff --git a/esp/src/src-react/components/MetricsOptions.tsx b/esp/src/src-react/components/MetricsOptions.tsx index ce48f07513e..aa0d20a108f 100644 --- a/esp/src/src-react/components/MetricsOptions.tsx +++ b/esp/src/src-react/components/MetricsOptions.tsx @@ -67,16 +67,18 @@ const GridOptions: React.FunctionComponent = ({ } }, [selectionHandler, strSelection]); - return ; + return
+ +
; }; interface AddLabelProps { diff --git a/esp/src/src-react/components/Resources.tsx b/esp/src/src-react/components/Resources.tsx index d3cb0a73a53..ccae64dfa5e 100644 --- a/esp/src/src-react/components/Resources.tsx +++ b/esp/src/src-react/components/Resources.tsx @@ -1,9 +1,10 @@ import * as React from "react"; -import { CommandBar, ContextualMenuItemType, ICommandBarItemProps, Link, ScrollablePane, Sticky } from "@fluentui/react"; +import { CommandBar, ContextualMenuItemType, ICommandBarItemProps, Link } from "@fluentui/react"; import nlsHPCC from "src/nlsHPCC"; import { QuerySortItem } from "src/store/Store"; import { useWorkunitResources } from "../hooks/workunit"; import { updateParam } from "../util/history"; +import { HolyGrail } from "../layouts/HolyGrail"; import { FluentGrid, useCopyButtons, useFluentStoreState, FluentColumns } from "./controls/Grid"; import { ShortVerticalDivider } from "./Common"; import { IFrame } from "./IFrame"; @@ -108,21 +109,21 @@ export const Resources: React.FunctionComponent = ({ })); }, [resources, wuid]); - return - - - - {preview && webUrl ? -