diff --git a/cmake_modules/options.cmake b/cmake_modules/options.cmake index 9dce9f2f474..0ecda9cc436 100644 --- a/cmake_modules/options.cmake +++ b/cmake_modules/options.cmake @@ -65,6 +65,7 @@ option(INSTALL_VCPKG_CATALOG "Install vcpkg-catalog.txt" ON) option(PORTALURL "Set url to hpccsystems portal download page") option(PROFILING "Set to true if planning to profile so stacks are informative" OFF) option(COLLECT_SERVICE_METRICS "Set to true to gather metrics for HIDL services by default" OFF) +option(VCPKG_ECLBLAS_DYNAMIC_ARCH "Set to ON to build eclblas with dynamic architecture" ON) set(CUSTOM_LABEL "" CACHE STRING "Appends a custom label to the final package name") diff --git a/cmake_modules/plugins.cmake b/cmake_modules/plugins.cmake index 334bd302f15..1892188c756 100644 --- a/cmake_modules/plugins.cmake +++ b/cmake_modules/plugins.cmake @@ -150,6 +150,12 @@ if (USE_ZLIB) set(VCPKG_ZLIB "${VCPKG_INCLUDE}") endif() +if (VCPKG_ECLBLAS_DYNAMIC_ARCH) + set(VCPKG_ECLBLAS_DYNAMIC_ARCH_FEATURE "\"dynamic-arch\",") +else () + set(VCPKG_ECLBLAS_DYNAMIC_ARCH_FEATURE "") +endif() + configure_file("${HPCC_SOURCE_DIR}/vcpkg.json.in" "${HPCC_SOURCE_DIR}/vcpkg.json") endif() diff --git a/ecl/hql/hqlexpr.cpp b/ecl/hql/hqlexpr.cpp index 90e2ef1d91a..395f2a64b55 100644 --- a/ecl/hql/hqlexpr.cpp +++ b/ecl/hql/hqlexpr.cpp @@ -11030,7 +11030,9 @@ IHqlExpression *CHqlDelayedCall::clone(HqlExprArray &newkids) IHqlExpression * CHqlDelayedCall::makeDelayedCall(IHqlExpression * _funcdef, HqlExprArray &operands) { - ITypeInfo * returnType = _funcdef->queryType()->queryChildType(); + ITypeInfo * funcType = _funcdef->queryType(); + assertex(funcType->getTypeCode() == type_function); + ITypeInfo * returnType = funcType->queryChildType(); CHqlDelayedCall * ret; switch (returnType->getTypeCode()) { diff --git a/ecl/hql/hqlgram2.cpp b/ecl/hql/hqlgram2.cpp index 4838a9bc262..162439587ad 100644 --- a/ecl/hql/hqlgram2.cpp +++ b/ecl/hql/hqlgram2.cpp @@ -3638,6 +3638,30 @@ IHqlExpression * HqlGram::implementInterfaceFromModule(const attribute & modpos, { HqlExprArray parameters; bool isParametered = extractSymbolParameters(parameters, &baseSym); + if (baseSym.isFunction() != match->isFunction()) + { + if (isParametered) + { + //Convert the value to a function definition - the parameters will be ignored + IHqlExpression * formals = queryFunctionParameters(&baseSym); + IHqlExpression * defaults = queryFunctionDefaults(&baseSym); + OwnedHqlExpr funcdef = createFunctionDefinition(id, LINK(match->queryBody()), LINK(formals), LINK(defaults), NULL); + match.setown(match->cloneAllAnnotations(funcdef)); + } + else + { + //Convert a function into a value - possible if all parameters (including none) have default values + if (!allParametersHaveDefaults(match)) + { + reportError(ERR_EXPECTED_ATTRIBUTE, ipos, "Symbol %s is defined as a value in the base scope, but a function with non-default parameters in the interface", str(id)); + } + else + { + HqlExprArray actuals; + match.setown(createBoundFunction(nullptr, match, actuals, nullptr, false)); + } + } + } checkDerivedCompatible(id, newScopeExpr, match, isParametered, parameters, modpos); newScope->defineSymbol(LINK(match)); diff --git a/ecl/regress/iproject.ecl b/ecl/regress/iproject.ecl new file mode 100644 index 00000000000..3822182065d --- /dev/null +++ b/ecl/regress/iproject.ecl @@ -0,0 +1,50 @@ +/*############################################################################## + + HPCC SYSTEMS software Copyright (C) 2024 HPCC Systems®. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +############################################################################## */ + +myModule := MODULE + + EXPORT value1 := 100; + EXPORT value2 := 200; + EXPORT value3(unsigned x = 10) := 300; + EXPORT value4() := 400; + +END; + +myInterface := INTERFACE + + EXPORT value1 := 0; + EXPORT value2(unsigned unknown) := 0; + EXPORT value3 := 0; + EXPORT value4() := 0; + +END; + + + +display(myInterface x) := FUNCTION + RETURN + ORDERED( + OUTPUT(x.value1); + OUTPUT(x.value2(99999)); + OUTPUT(x.value3); + OUTPUT(x.value4()); + ); +END; + + +mappedModule := PROJECT(myModule, myInterface); +display(mappedModule); \ No newline at end of file diff --git a/ecl/regress/iproject_err.ecl b/ecl/regress/iproject_err.ecl new file mode 100644 index 00000000000..cb4d64b7c74 --- /dev/null +++ b/ecl/regress/iproject_err.ecl @@ -0,0 +1,50 @@ +/*############################################################################## + + HPCC SYSTEMS software Copyright (C) 2024 HPCC Systems®. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +############################################################################## */ + +myModule := MODULE + + EXPORT value1 := 100; + EXPORT value2 := 200; + EXPORT value3(unsigned i) := 300; // invalid - no default value supplied + EXPORT value4() := 400; + +END; + +myInterface := INTERFACE + + EXPORT value1 := 0; + EXPORT value2() := 0; + EXPORT value3 := 0; + EXPORT value4() := 0; + +END; + + + +display(myInterface x) := FUNCTION + RETURN + ORDERED( + OUTPUT(x.value1); + OUTPUT(x.value2()); + OUTPUT(x.value3); + OUTPUT(x.value4()); + ); +END; + + +mappedModule := PROJECT(myModule, myInterface); +display(mappedModule); \ No newline at end of file diff --git a/esp/src/package-lock.json b/esp/src/package-lock.json index 3281d6c476e..262f9708232 100644 --- a/esp/src/package-lock.json +++ b/esp/src/package-lock.json @@ -18,7 +18,7 @@ "@hpcc-js/chart": "2.84.1", "@hpcc-js/codemirror": "2.63.0", "@hpcc-js/common": "2.72.0", - "@hpcc-js/comms": "2.95.1", + "@hpcc-js/comms": "2.96.1", "@hpcc-js/dataflow": "8.1.7", "@hpcc-js/eclwatch": "2.75.3", "@hpcc-js/graph": "2.86.0", @@ -2144,10 +2144,9 @@ } }, "node_modules/@hpcc-js/comms": { - "version": "2.95.1", - "resolved": "https://registry.npmjs.org/@hpcc-js/comms/-/comms-2.95.1.tgz", - "integrity": "sha512-zygjJDGYzh0hhZhVf+RAnEXu4jPtlEbGB9+23klrPCvrKXzzQv426N9+Ui7i82HFtzJzQg+orjfI8Tu2NavsuQ==", - "license": "Apache-2.0", + "version": "2.96.1", + "resolved": "https://registry.npmjs.org/@hpcc-js/comms/-/comms-2.96.1.tgz", + "integrity": "sha512-38vIe8foZa5fYtrj65oeWyYWUDZmQTbKetHG5HXWZWMu0Lfmln8uG5/J7mO0ilw3ls2oZj7xOk5T/4xvg7v43w==", "dependencies": { "@hpcc-js/ddl-shim": "^2.21.0", "@hpcc-js/util": "^2.52.0", diff --git a/esp/src/package.json b/esp/src/package.json index 6432e2fd582..f3f5270f0e7 100644 --- a/esp/src/package.json +++ b/esp/src/package.json @@ -44,7 +44,7 @@ "@hpcc-js/chart": "2.84.1", "@hpcc-js/codemirror": "2.63.0", "@hpcc-js/common": "2.72.0", - "@hpcc-js/comms": "2.95.1", + "@hpcc-js/comms": "2.96.1", "@hpcc-js/dataflow": "8.1.7", "@hpcc-js/eclwatch": "2.75.3", "@hpcc-js/graph": "2.86.0", @@ -115,4 +115,4 @@ "type": "git", "url": "https://github.com/hpcc-systems/HPCC-Platform" } -} \ No newline at end of file +} diff --git a/esp/src/src-react/components/Helpers.tsx b/esp/src/src-react/components/Helpers.tsx index 458308debd8..73fed185eca 100644 --- a/esp/src/src-react/components/Helpers.tsx +++ b/esp/src/src-react/components/Helpers.tsx @@ -1,8 +1,9 @@ import * as React from "react"; -import { CommandBar, ContextualMenuItemType, ICommandBarItemProps, Link, ScrollablePane, Sticky } from "@fluentui/react"; +import { CommandBar, ContextualMenuItemType, ICommandBarItemProps, Link } from "@fluentui/react"; import * as ESPRequest from "src/ESPRequest"; import nlsHPCC from "src/nlsHPCC"; import { HelperRow, useWorkunitHelpers } from "../hooks/workunit"; +import { HolyGrail } from "../layouts/HolyGrail"; import { FluentGrid, useCopyButtons, useFluentStoreState, FluentColumns } from "./controls/Grid"; import { ShortVerticalDivider } from "./Common"; @@ -223,18 +224,18 @@ export const Helpers: React.FunctionComponent<HelpersProps> = ({ setData(helpers); }, [helpers]); - return <ScrollablePane> - <Sticky> - <CommandBar items={buttons} farItems={copyButtons} /> - </Sticky> - <FluentGrid - data={data} - primaryID={"id"} - alphaNumColumns={{ Value: true }} - columns={columns} - setSelection={setSelection} - setTotal={setTotal} - refresh={refreshTable} - ></FluentGrid> - </ScrollablePane>; + return <HolyGrail + header={<CommandBar items={buttons} farItems={copyButtons} />} + main={ + <FluentGrid + data={data} + primaryID={"id"} + alphaNumColumns={{ Value: true }} + columns={columns} + setSelection={setSelection} + setTotal={setTotal} + refresh={refreshTable} + ></FluentGrid> + } + />; }; diff --git a/esp/src/src-react/components/InfoGrid.tsx b/esp/src/src-react/components/InfoGrid.tsx index da3a212ba8b..99cccb3fafc 100644 --- a/esp/src/src-react/components/InfoGrid.tsx +++ b/esp/src/src-react/components/InfoGrid.tsx @@ -236,7 +236,7 @@ export const InfoGrid: React.FunctionComponent<InfoGridProps> = ({ } }, [data.length]); - return <div style={{ height: "100%" }}> + return <div style={{ height: "100%", overflow: "hidden" }}> <CommandBar items={buttons} farItems={copyButtons} /> <SizeMe monitorHeight >{({ size }) => <FluentGrid diff --git a/esp/src/src-react/components/LogViewer.tsx b/esp/src/src-react/components/LogViewer.tsx index 2e44c369051..4f5e5cdcf41 100644 --- a/esp/src/src-react/components/LogViewer.tsx +++ b/esp/src/src-react/components/LogViewer.tsx @@ -102,16 +102,14 @@ export const LogViewer: React.FunctionComponent<LogViewerProps> = ({ return <HolyGrail header={<CommandBar items={buttons} farItems={copyButtons} />} main={ - <div style={{ position: "relative", height: "100%" }}> - <FluentGrid - data={data} - primaryID={"dateTime"} - columns={columns} - setSelection={setSelection} - setTotal={setTotal} - refresh={refreshTable} - ></FluentGrid> - </div> + <FluentGrid + data={data} + primaryID={"dateTime"} + columns={columns} + setSelection={setSelection} + setTotal={setTotal} + refresh={refreshTable} + ></FluentGrid> } />; }; diff --git a/esp/src/src-react/components/Logs.tsx b/esp/src/src-react/components/Logs.tsx index d9703c0e74f..86f7297abc1 100644 --- a/esp/src/src-react/components/Logs.tsx +++ b/esp/src/src-react/components/Logs.tsx @@ -182,7 +182,7 @@ export const Logs: React.FunctionComponent<LogsProps> = ({ return <HolyGrail header={<CommandBar items={buttons} farItems={copyButtons} />} main={ - <> + <div style={{ position: "relative", height: "100%" }}> <FluentPagedGrid store={gridStore} query={query} @@ -200,7 +200,7 @@ export const Logs: React.FunctionComponent<LogsProps> = ({ refresh={refreshTable} ></FluentPagedGrid> <Filter showFilter={showFilter} setShowFilter={setShowFilter} filterFields={filterFields} onApply={pushParams} /> - </> + </div> } footer={<FluentPagedFooter persistID={"cloudlogs"} diff --git a/esp/src/src-react/components/MetricsOptions.tsx b/esp/src/src-react/components/MetricsOptions.tsx index ce48f07513e..aa0d20a108f 100644 --- a/esp/src/src-react/components/MetricsOptions.tsx +++ b/esp/src/src-react/components/MetricsOptions.tsx @@ -67,16 +67,18 @@ const GridOptions: React.FunctionComponent<GridOptionsProps> = ({ } }, [selectionHandler, strSelection]); - return <FluentGrid - data={data} - primaryID={"id"} - columns={columns} - selectionMode={SelectionMode.multiple} - setSelection={selectionHandler} - setTotal={setTotal} - refresh={refreshTable} - height={`${innerHeight}px`} - ></FluentGrid>; + return <div style={{ position: "relative", height: 400 }}> + <FluentGrid + data={data} + primaryID={"id"} + columns={columns} + selectionMode={SelectionMode.multiple} + setSelection={selectionHandler} + setTotal={setTotal} + refresh={refreshTable} + height={`${innerHeight}px`} + ></FluentGrid> + </div>; }; interface AddLabelProps { diff --git a/esp/src/src-react/components/Resources.tsx b/esp/src/src-react/components/Resources.tsx index d3cb0a73a53..ccae64dfa5e 100644 --- a/esp/src/src-react/components/Resources.tsx +++ b/esp/src/src-react/components/Resources.tsx @@ -1,9 +1,10 @@ import * as React from "react"; -import { CommandBar, ContextualMenuItemType, ICommandBarItemProps, Link, ScrollablePane, Sticky } from "@fluentui/react"; +import { CommandBar, ContextualMenuItemType, ICommandBarItemProps, Link } from "@fluentui/react"; import nlsHPCC from "src/nlsHPCC"; import { QuerySortItem } from "src/store/Store"; import { useWorkunitResources } from "../hooks/workunit"; import { updateParam } from "../util/history"; +import { HolyGrail } from "../layouts/HolyGrail"; import { FluentGrid, useCopyButtons, useFluentStoreState, FluentColumns } from "./controls/Grid"; import { ShortVerticalDivider } from "./Common"; import { IFrame } from "./IFrame"; @@ -108,21 +109,21 @@ export const Resources: React.FunctionComponent<ResourcesProps> = ({ })); }, [resources, wuid]); - return <ScrollablePane> - <Sticky> - <CommandBar items={buttons} farItems={copyButtons} /> - </Sticky> - {preview && webUrl ? - <IFrame src={webUrl} /> : - <FluentGrid - data={data} - primaryID={"DisplayPath"} - alphaNumColumns={{ Value: true }} - sort={sort} - columns={columns} - setSelection={setSelection} - setTotal={setTotal} - refresh={refreshTable} - ></FluentGrid>} - </ScrollablePane>; + return <HolyGrail + header={<CommandBar items={buttons} farItems={copyButtons} />} + main={ + preview && webUrl ? + <IFrame src={webUrl} /> : + <FluentGrid + data={data} + primaryID={"DisplayPath"} + alphaNumColumns={{ Value: true }} + sort={sort} + columns={columns} + setSelection={setSelection} + setTotal={setTotal} + refresh={refreshTable} + ></FluentGrid> + } + />; }; diff --git a/esp/src/src-react/components/Results.tsx b/esp/src/src-react/components/Results.tsx index 02595519fd9..1343c84268d 100644 --- a/esp/src/src-react/components/Results.tsx +++ b/esp/src/src-react/components/Results.tsx @@ -1,9 +1,10 @@ import * as React from "react"; -import { CommandBar, ContextualMenuItemType, ICommandBarItemProps, Link, Pivot, PivotItem, ScrollablePane, Sticky } from "@fluentui/react"; +import { CommandBar, ContextualMenuItemType, ICommandBarItemProps, Link, Pivot, PivotItem } from "@fluentui/react"; import { SizeMe } from "react-sizeme"; import nlsHPCC from "src/nlsHPCC"; import { QuerySortItem } from "src/store/Store"; import { useWorkunitResults } from "../hooks/workunit"; +import { HolyGrail } from "../layouts/HolyGrail"; import { pivotItemStyle } from "../layouts/pivot"; import { FluentGrid, useCopyButtons, useFluentStoreState, FluentColumns } from "./controls/Grid"; import { ShortVerticalDivider } from "./Common"; @@ -141,21 +142,21 @@ export const Results: React.FunctionComponent<ResultsProps> = ({ })); }, [results]); - return <ScrollablePane> - <Sticky> - <CommandBar items={buttons} farItems={copyButtons} /> - </Sticky> - <FluentGrid - data={data} - primaryID={"__hpcc_id"} - alphaNumColumns={{ Name: true, Value: true }} - sort={sort} - columns={columns} - setSelection={setSelection} - setTotal={setTotal} - refresh={refreshTable} - ></FluentGrid> - </ScrollablePane >; + return <HolyGrail + header={<CommandBar items={buttons} farItems={copyButtons} />} + main={ + <FluentGrid + data={data} + primaryID={"__hpcc_id"} + alphaNumColumns={{ Name: true, Value: true }} + sort={sort} + columns={columns} + setSelection={setSelection} + setTotal={setTotal} + refresh={refreshTable} + ></FluentGrid> + } + />; }; interface TabbedResultsProps { diff --git a/esp/src/src-react/components/SourceFiles.tsx b/esp/src/src-react/components/SourceFiles.tsx index cda748e8845..2c1b1c6578b 100644 --- a/esp/src/src-react/components/SourceFiles.tsx +++ b/esp/src/src-react/components/SourceFiles.tsx @@ -1,10 +1,11 @@ import * as React from "react"; -import { CommandBar, ContextualMenuItemType, ICommandBarItemProps, Image, Link, ScrollablePane, Sticky } from "@fluentui/react"; +import { CommandBar, ContextualMenuItemType, ICommandBarItemProps, Image, Link } from "@fluentui/react"; import * as Utility from "src/Utility"; import { QuerySortItem } from "src/store/Store"; import nlsHPCC from "src/nlsHPCC"; import { useWorkunitSourceFiles } from "../hooks/workunit"; import { pushParams } from "../util/history"; +import { HolyGrail } from "../layouts/HolyGrail"; import { FluentGrid, useCopyButtons, useFluentStoreState, FluentColumns } from "./controls/Grid"; import { Fields } from "./forms/Fields"; import { Filter } from "./forms/Filter"; @@ -131,20 +132,22 @@ export const SourceFiles: React.FunctionComponent<SourceFilesProps> = ({ setData(files); }, [filter, sourceFiles]); - return <ScrollablePane> - <Sticky> - <CommandBar items={buttons} farItems={copyButtons} /> - </Sticky> - <FluentGrid - data={data} - primaryID={"Name"} - alphaNumColumns={{ Value: true }} - sort={sort} - columns={columns} - setSelection={setSelection} - setTotal={setTotal} - refresh={refreshTable} - ></FluentGrid> - <Filter showFilter={showFilter} setShowFilter={setShowFilter} filterFields={filterFields} onApply={pushParams} /> - </ScrollablePane>; + return <HolyGrail + header={<CommandBar items={buttons} farItems={copyButtons} />} + main={ + <div style={{ position: "relative", height: "100%" }}> + <FluentGrid + data={data} + primaryID={"Name"} + alphaNumColumns={{ Value: true }} + sort={sort} + columns={columns} + setSelection={setSelection} + setTotal={setTotal} + refresh={refreshTable} + ></FluentGrid> + <Filter showFilter={showFilter} setShowFilter={setShowFilter} filterFields={filterFields} onApply={pushParams} /> + </div> + } + />; }; diff --git a/esp/src/src-react/components/SubFiles.tsx b/esp/src/src-react/components/SubFiles.tsx index e7088869472..5b1e2f3a0c7 100644 --- a/esp/src/src-react/components/SubFiles.tsx +++ b/esp/src/src-react/components/SubFiles.tsx @@ -1,11 +1,12 @@ import * as React from "react"; -import { CommandBar, ContextualMenuItemType, FontIcon, ICommandBarItemProps, Link, ScrollablePane, Sticky } from "@fluentui/react"; +import { CommandBar, ContextualMenuItemType, FontIcon, ICommandBarItemProps, Link } from "@fluentui/react"; import * as ESPLogicalFile from "src/ESPLogicalFile"; import nlsHPCC from "src/nlsHPCC"; import { QuerySortItem } from "src/store/Store"; import * as WsDfu from "src/WsDfu"; import { useConfirm } from "../hooks/confirm"; import { useFile, useSubfiles } from "../hooks/file"; +import { HolyGrail } from "../layouts/HolyGrail"; import { FluentGrid, useCopyButtons, useFluentStoreState, FluentColumns } from "./controls/Grid"; import { ShortVerticalDivider } from "./Common"; import { pushUrl } from "../util/history"; @@ -159,21 +160,21 @@ export const SubFiles: React.FunctionComponent<SubFilesProps> = ({ setUIState(state); }, [selection]); - return <> - <ScrollablePane> - <Sticky> - <CommandBar items={buttons} farItems={copyButtons} /> - </Sticky> - <FluentGrid - data={data} - primaryID={"Name"} - columns={columns} - alphaNumColumns={{ RecordCount: true, Totalsize: true }} - setSelection={setSelection} - setTotal={setTotal} - refresh={refreshTable} - ></FluentGrid> - </ScrollablePane > - <DeleteSubfilesConfirm /> - </>; + return <HolyGrail + header={<CommandBar items={buttons} farItems={copyButtons} />} + main={ + <div style={{ position: "relative", height: "100%" }}> + <FluentGrid + data={data} + primaryID={"Name"} + columns={columns} + alphaNumColumns={{ RecordCount: true, Totalsize: true }} + setSelection={setSelection} + setTotal={setTotal} + refresh={refreshTable} + ></FluentGrid> + <DeleteSubfilesConfirm /> + </div> + } + />; }; \ No newline at end of file diff --git a/esp/src/src-react/components/SuperFiles.tsx b/esp/src/src-react/components/SuperFiles.tsx index b9cb69e99e4..49e83e1a68b 100644 --- a/esp/src/src-react/components/SuperFiles.tsx +++ b/esp/src/src-react/components/SuperFiles.tsx @@ -1,8 +1,9 @@ import * as React from "react"; -import { CommandBar, ContextualMenuItemType, ICommandBarItemProps, Link, ScrollablePane, Sticky } from "@fluentui/react"; +import { CommandBar, ContextualMenuItemType, ICommandBarItemProps, Link } from "@fluentui/react"; import nlsHPCC from "src/nlsHPCC"; import { QuerySortItem } from "src/store/Store"; import { useFile } from "../hooks/file"; +import { HolyGrail } from "../layouts/HolyGrail"; import { FluentGrid, useCopyButtons, useFluentStoreState, FluentColumns } from "./controls/Grid"; import { ShortVerticalDivider } from "./Common"; @@ -88,18 +89,18 @@ export const SuperFiles: React.FunctionComponent<SuperFilesProps> = ({ } }, [file]); - return <ScrollablePane> - <Sticky> - <CommandBar items={buttons} farItems={copyButtons} /> - </Sticky> - <FluentGrid - data={data} - primaryID={"Name"} - sort={sort} - columns={columns} - setSelection={setSelection} - setTotal={setTotal} - refresh={refreshTable} - ></FluentGrid> - </ScrollablePane>; + return <HolyGrail + header={<CommandBar items={buttons} farItems={copyButtons} />} + main={ + <FluentGrid + data={data} + primaryID={"Name"} + sort={sort} + columns={columns} + setSelection={setSelection} + setTotal={setTotal} + refresh={refreshTable} + ></FluentGrid> + } + />; }; diff --git a/esp/src/src-react/components/Variables.tsx b/esp/src/src-react/components/Variables.tsx index 35c3231f328..6cb7f193df6 100644 --- a/esp/src/src-react/components/Variables.tsx +++ b/esp/src/src-react/components/Variables.tsx @@ -1,8 +1,9 @@ import * as React from "react"; -import { CommandBar, ContextualMenuItemType, ICommandBarItemProps, ScrollablePane, Sticky } from "@fluentui/react"; +import { CommandBar, ContextualMenuItemType, ICommandBarItemProps } from "@fluentui/react"; import nlsHPCC from "src/nlsHPCC"; import { QuerySortItem } from "src/store/Store"; import { useWorkunitVariables } from "../hooks/workunit"; +import { HolyGrail } from "../layouts/HolyGrail"; import { FluentGrid, useCopyButtons, useFluentStoreState, FluentColumns } from "./controls/Grid"; import { ShortVerticalDivider } from "./Common"; @@ -54,19 +55,19 @@ export const Variables: React.FunctionComponent<VariablesProps> = ({ const copyButtons = useCopyButtons(columns, selection, "variables"); - return <ScrollablePane> - <Sticky> - <CommandBar items={buttons} farItems={copyButtons} /> - </Sticky> - <FluentGrid - data={data} - primaryID={"__hpcc_id"} - alphaNumColumns={{ Value: true }} - sort={sort} - columns={columns} - setSelection={setSelection} - setTotal={setTotal} - refresh={refreshTable} - ></FluentGrid> - </ScrollablePane>; + return <HolyGrail + header={<CommandBar items={buttons} farItems={copyButtons} />} + main={ + <FluentGrid + data={data} + primaryID={"__hpcc_id"} + alphaNumColumns={{ Value: true }} + sort={sort} + columns={columns} + setSelection={setSelection} + setTotal={setTotal} + refresh={refreshTable} + ></FluentGrid> + } + />; }; diff --git a/esp/src/src-react/components/Workflows.tsx b/esp/src/src-react/components/Workflows.tsx index 7c12d4415f8..c372e2e3c81 100644 --- a/esp/src/src-react/components/Workflows.tsx +++ b/esp/src/src-react/components/Workflows.tsx @@ -1,8 +1,9 @@ import * as React from "react"; -import { CommandBar, ContextualMenuItemType, ICommandBarItemProps, ScrollablePane, Sticky } from "@fluentui/react"; +import { CommandBar, ContextualMenuItemType, ICommandBarItemProps } from "@fluentui/react"; import nlsHPCC from "src/nlsHPCC"; import { QuerySortItem } from "src/store/Store"; import { useWorkunitWorkflows } from "../hooks/workunit"; +import { HolyGrail } from "../layouts/HolyGrail"; import { FluentGrid, useCopyButtons, useFluentStoreState, FluentColumns } from "./controls/Grid"; import { ShortVerticalDivider } from "./Common"; @@ -73,19 +74,19 @@ export const Workflows: React.FunctionComponent<WorkflowsProps> = ({ const copyButtons = useCopyButtons(columns, selection, "workflows"); - return <ScrollablePane> - <Sticky> - <CommandBar items={buttons} farItems={copyButtons} /> - </Sticky> - <FluentGrid - data={data} - primaryID={"__hpcc_id"} - alphaNumColumns={{ Name: true, Value: true }} - sort={sort} - columns={columns} - setSelection={setSelection} - setTotal={setTotal} - refresh={refreshTable} - ></FluentGrid> - </ScrollablePane>; + return <HolyGrail + header={<CommandBar items={buttons} farItems={copyButtons} />} + main={ + <FluentGrid + data={data} + primaryID={"__hpcc_id"} + alphaNumColumns={{ Name: true, Value: true }} + sort={sort} + columns={columns} + setSelection={setSelection} + setTotal={setTotal} + refresh={refreshTable} + ></FluentGrid> + } + />; }; diff --git a/esp/src/src-react/components/WorkunitDetails.tsx b/esp/src/src-react/components/WorkunitDetails.tsx index 8e595bf7b16..f30bbf974ae 100644 --- a/esp/src/src-react/components/WorkunitDetails.tsx +++ b/esp/src/src-react/components/WorkunitDetails.tsx @@ -27,8 +27,7 @@ import { Workflows } from "./Workflows"; import { WorkunitSummary } from "./WorkunitSummary"; import { TabInfo, DelayLoadedPanel, OverflowTabList } from "./controls/TabbedPanes/index"; import { ECLArchive } from "./ECLArchive"; - -const Metrics = React.lazy(() => import("./Metrics").then(mod => ({ default: mod.Metrics }))); +import { Metrics } from "./Metrics"; const logger = scopedLogger("src-react/components/WorkunitDetails.tsx"); diff --git a/esp/src/src-react/components/controls/Grid.tsx b/esp/src/src-react/components/controls/Grid.tsx index 129271d47b3..e0ddc78c7e0 100644 --- a/esp/src/src-react/components/controls/Grid.tsx +++ b/esp/src/src-react/components/controls/Grid.tsx @@ -237,12 +237,21 @@ const FluentStoreGrid: React.FunctionComponent<FluentStoreGridProps> = ({ }); }); + const abortController = React.useRef<AbortController>(); + + React.useEffect(() => { + if (abortController.current) { + abortController.current.abort({ message: "Grid aborting stale request" }); + } + abortController.current = new AbortController(); + }, [query]); + const refreshTable = useDeepCallback((clearSelection = false) => { if (isNaN(start) || isNaN(count)) return; if (clearSelection) { selectionHandler.setItems([], true); } - const storeQuery = store.query({ ...query }, { start, count, sort: sorted ? [sorted] : undefined }); + const storeQuery = store.query({ ...query }, { start, count, sort: sorted ? [sorted] : undefined }, abortController.current.signal); storeQuery.total.then(total => { setTotal(total); }); @@ -308,24 +317,26 @@ const FluentStoreGrid: React.FunctionComponent<FluentStoreGridProps> = ({ columnWidths.set(column.key, newWidth); }, [columnWidths]); - return <ScrollablePane> - <DetailsList - compact={true} - items={items} - columns={fluentColumns} - layoutMode={DetailsListLayoutMode.fixedColumns} - constrainMode={ConstrainMode.unconstrained} - selection={selectionHandler} - isSelectedOnFocus={false} - selectionPreservedOnEmptyClick={true} - onColumnHeaderClick={onColumnClick} - onRenderDetailsHeader={renderDetailsHeader} - onColumnResize={columnResize} - onRenderRow={onRenderRow} - styles={gridStyles(height)} - selectionMode={selectionMode} - /> - </ScrollablePane>; + return <div style={{ position: "relative", height: "100%" }}> + <ScrollablePane> + <DetailsList + compact={true} + items={items} + columns={fluentColumns} + layoutMode={DetailsListLayoutMode.fixedColumns} + constrainMode={ConstrainMode.unconstrained} + selection={selectionHandler} + isSelectedOnFocus={false} + selectionPreservedOnEmptyClick={true} + onColumnHeaderClick={onColumnClick} + onRenderDetailsHeader={renderDetailsHeader} + onColumnResize={columnResize} + onRenderRow={onRenderRow} + styles={gridStyles(height)} + selectionMode={selectionMode} + /> + </ScrollablePane> + </div>; }; interface FluentGridProps { diff --git a/esp/src/src/ESPWorkunit.ts b/esp/src/src/ESPWorkunit.ts index 1efadc5afc1..5cb9b91aeb5 100644 --- a/esp/src/src/ESPWorkunit.ts +++ b/esp/src/src/ESPWorkunit.ts @@ -1082,11 +1082,11 @@ export function CreateWUQueryStore(): BaseStore<WsWorkunitsNS.WUQuery, typeof Wo count: "PageSize", sortBy: "Sortby", descending: "Descending" - }, "Wuid", request => { + }, "Wuid", (request, abortSignal) => { if (request.Sortby && request.Sortby === "TotalClusterTime") { request.Sortby = "ClusterTime"; } - return service.WUQuery(request).then(response => { + return service.WUQuery(request, abortSignal).then(response => { const page = { start: undefined, end: undefined diff --git a/esp/src/src/store/Deferred.ts b/esp/src/src/store/Deferred.ts index db7c3288666..a0f4d0c65ce 100644 --- a/esp/src/src/store/Deferred.ts +++ b/esp/src/src/store/Deferred.ts @@ -68,7 +68,7 @@ export class Deferred<T> implements Thenable<T> { } } -export class DeferredResponse<T> extends Deferred<T[]> { +export class DeferredResponse<T> extends Deferred<T[]> { // --- Legacy Dojo Support (fake QeuryResults) --- forEach(callback) { diff --git a/esp/src/src/store/Paged.ts b/esp/src/src/store/Paged.ts index eb57375376a..b4c1025e8db 100644 --- a/esp/src/src/store/Paged.ts +++ b/esp/src/src/store/Paged.ts @@ -16,7 +16,7 @@ function dataPage<T>(array: T[], start: number, size: number, total: number): Pa return retVal; } -type FetchData<R, T> = (query: QueryRequest<R>) => Thenable<{ data: T[], total: number }>; +type FetchData<R, T> = (query: QueryRequest<R>, abortSignal?: AbortSignal) => Thenable<{ data: T[], total: number }>; export interface RequestFields<T extends BaseRow> { start: keyof T; @@ -43,7 +43,7 @@ export class Paged<R extends BaseRow = BaseRow, T extends BaseRow = BaseRow> ext this._fetchData = fetchData; } - fetchData(request: QueryRequest<R>, options: QueryOptions<T>): ThenableResponse<T> { + fetchData(request: QueryRequest<R>, options: QueryOptions<T>, abortSignal?: AbortSignal): ThenableResponse<T> { if (options.start !== undefined && options.count !== undefined) { request[this._requestFields.start] = options.start as any; request[this._requestFields.count] = options.count as any; @@ -52,7 +52,7 @@ export class Paged<R extends BaseRow = BaseRow, T extends BaseRow = BaseRow> ext request[this._requestFields.sortBy] = options.sort[0].attribute as any; request[this._requestFields.descending] = options.sort[0].descending as any; } - return this._fetchData(request).then(response => { + return this._fetchData(request, abortSignal).then(response => { response.data.forEach(row => { this.index[this.getIdentity(row)] = row; }); diff --git a/esp/src/src/store/Store.ts b/esp/src/src/store/Store.ts index d41d24e893c..9e8aab02b15 100644 --- a/esp/src/src/store/Store.ts +++ b/esp/src/src/store/Store.ts @@ -1,5 +1,8 @@ import * as QueryResults from "dojo/store/util/QueryResults"; import { DeferredResponse, Thenable } from "./Deferred"; +import { scopedLogger } from "@hpcc-js/util"; + +const logger = scopedLogger("src/store/Store.ts"); // Query --- export type Key = string | number | symbol; @@ -32,7 +35,7 @@ export abstract class BaseStore<R extends BaseRow, T extends BaseRow> { this.responseIDField = responseIDField; } - protected abstract fetchData(request: QueryRequest<R>, options: QueryOptions<T>): ThenableResponse<T>; + protected abstract fetchData(request: QueryRequest<R>, options: QueryOptions<T>, abortSignal?: AbortSignal): ThenableResponse<T>; abstract get(id: string | number): T; @@ -40,11 +43,13 @@ export abstract class BaseStore<R extends BaseRow, T extends BaseRow> { return object[this.responseIDField]; } - protected query(request: QueryRequest<R>, options: QueryOptions<T>): DeferredResponse<T> { + protected query(request: QueryRequest<R>, options: QueryOptions<T>, abortSignal?: AbortSignal): DeferredResponse<T> { const retVal = new DeferredResponse<T>(); - this.fetchData(request, options).then((data: QueryResponse<T>) => { + this.fetchData(request, options, abortSignal).then((data: QueryResponse<T>) => { retVal.total.resolve(data.total); retVal.resolve(data); + }, (err) => { + logger.debug(err); }); return QueryResults(retVal); } diff --git a/fs/dafsserver/dafsserver.cpp b/fs/dafsserver/dafsserver.cpp index 22d953e8a50..246318a127f 100644 --- a/fs/dafsserver/dafsserver.cpp +++ b/fs/dafsserver/dafsserver.cpp @@ -1176,7 +1176,7 @@ class CRemoteDiskBaseActivity : public CSimpleInterfaceOf<IRemoteReadActivity>, bool opened = false; bool eofSeen = false; const RtlRecord *record = nullptr; - RowFilter filters; + RowFilter filter; RtlDynRow *filterRow = nullptr; // virtual field values StringAttr logicalFilename; @@ -1186,8 +1186,8 @@ class CRemoteDiskBaseActivity : public CSimpleInterfaceOf<IRemoteReadActivity>, { if (filterRow) { - filterRow->setRow(buffer, filters.getNumFieldsRequired()); - return filters.matches(*filterRow); + filterRow->setRow(buffer, filter.getNumFieldsRequired()); + return filter.matches(*filterRow); } else return true; @@ -1217,7 +1217,7 @@ class CRemoteDiskBaseActivity : public CSimpleInterfaceOf<IRemoteReadActivity>, filterRow = new RtlDynRow(*record); Owned<IPropertyTreeIterator> filterIter = config.getElements("keyFilter"); ForEach(*filterIter) - filters.addFilter(*record, filterIter->query().queryProp(nullptr)); + filter.addFilter(*record, filterIter->query().queryProp(nullptr)); } } // IRemoteReadActivity impl. @@ -2228,6 +2228,7 @@ class CRemoteIndexBaseActivity : public CRemoteDiskBaseActivity unsigned fileCrc = 0; Owned<IKeyIndex> keyIndex; Owned<IKeyManager> keyManager; + RowFilter keyFilter; void checkOpen() { @@ -2243,7 +2244,7 @@ class CRemoteIndexBaseActivity : public CRemoteDiskBaseActivity keyIndex.setown(createKeyIndex(fileName, crc, isTlk, 0)); keyManager.setown(createLocalKeyManager(*record, keyIndex, nullptr, true, false)); - filters.createSegmentMonitors(keyManager); + keyFilter.createSegmentMonitors(keyManager); keyManager->finishSegmentMonitors(); keyManager->reset(); @@ -2260,6 +2261,7 @@ class CRemoteIndexBaseActivity : public CRemoteDiskBaseActivity CRemoteIndexBaseActivity(IPropertyTree &config, IFileDescriptor *fileDesc) : PARENT(config, fileDesc) { setupInputMeta(config, getTypeInfoOutputMetaData(config, "input", false)); + filter.splitIntoKeyFilter(*record, keyFilter); isTlk = config.getPropBool("isTlk"); fileCrc = config.getPropInt("crc"); diff --git a/rtl/eclrtl/rtlnewkey.cpp b/rtl/eclrtl/rtlnewkey.cpp index 48af5a62e52..138edccc6b9 100644 --- a/rtl/eclrtl/rtlnewkey.cpp +++ b/rtl/eclrtl/rtlnewkey.cpp @@ -2245,6 +2245,27 @@ void RowFilter::extractMemKeyFilter(const RtlRecord & record, const UnsignedArra } } +void RowFilter::splitIntoKeyFilter(const RtlRecord & record, RowFilter &keyFilter) +{ + if (0 == filters.ordinality()) + return; + + unsigned numKeyedFields = record.getNumKeyedFields(); + ForEachItemInRev(i, filters) + { + const IFieldFilter & cur = filters.item(i); + if (cur.queryFieldIndex() < numKeyedFields) + { + keyFilter.addFilter(OLINK(cur)); + filters.remove(i); + } + } + keyFilter.sortByFieldOrder(); // NB: need to be ordered ahead of keyFilter.createSegmentMonitors being used + //There is either a payload filter, in which case numFieldsRequired will not change, or now no filter + if (filters.ordinality() == 0) + numFieldsRequired = 0; +} + const IFieldFilter *RowFilter::findFilter(unsigned fieldNum) const { ForEachItemIn(i, filters) @@ -2298,6 +2319,13 @@ void RowFilter::remapField(unsigned filterIdx, unsigned newFieldNum) filters.replace(*filters.item(filterIdx).remap(newFieldNum), filterIdx); } +void RowFilter::sortByFieldOrder() +{ + if (0 == filters.ordinality()) + return; + filters.sort(compareFieldFilters); +} + //--------------------------------------------------------------------------------------------------------------------- bool RowCursor::setRowForward(const byte * row) diff --git a/rtl/eclrtl/rtlnewkey.hpp b/rtl/eclrtl/rtlnewkey.hpp index a6aadc9b787..aeccb9fd5a2 100644 --- a/rtl/eclrtl/rtlnewkey.hpp +++ b/rtl/eclrtl/rtlnewkey.hpp @@ -65,6 +65,7 @@ class ECLRTL_API RowFilter void createSegmentMonitors(IIndexReadContext *irc); void extractKeyFilter(const RtlRecord & record, IConstArrayOf<IFieldFilter> & keyFilters) const; void extractMemKeyFilter(const RtlRecord & record, const UnsignedArray &sortOrder, IConstArrayOf<IFieldFilter> & keyFilters) const; + void splitIntoKeyFilter(const RtlRecord & record, RowFilter &keyFilter); unsigned numFilterFields() const { return filters.ordinality(); } const IFieldFilter & queryFilter(unsigned i) const { return filters.item(i); } const IFieldFilter *findFilter(unsigned fieldIdx) const; @@ -75,6 +76,7 @@ class ECLRTL_API RowFilter void remove(unsigned idx); RowFilter & clear(); void appendFilters(const IConstArrayOf<IFieldFilter> &_filters); + void sortByFieldOrder(); protected: IConstArrayOf<IFieldFilter> filters; unsigned numFieldsRequired = 0; diff --git a/system/jlib/jmutex.hpp b/system/jlib/jmutex.hpp index 4e3f51cdc7f..39956c6868b 100644 --- a/system/jlib/jmutex.hpp +++ b/system/jlib/jmutex.hpp @@ -1080,4 +1080,118 @@ class Singleton CriticalSection cs; }; +// Similar to class Shared<X>, but thread safe versions of the functions (avoid the need for critical sections) +// Optional atomic query(std::function<CLASS *()> factoryFunc, CriticalSection & cs) allows for a singleton initialization. +template <class CLASS> class AtomicShared +{ +public: + inline AtomicShared() { ptr.store(nullptr, std::memory_order_relaxed); } + inline AtomicShared(CLASS * _ptr, bool owned) { ptr.store(_ptr, std::memory_order_relaxed); if (!owned && _ptr) _ptr->Link(); } + inline AtomicShared(const AtomicShared & other) { ptr.store(other.getLinkNonAtomic(), std::memory_order_relaxed); } +#if defined(__cplusplus) && __cplusplus >= 201100 + inline AtomicShared(AtomicShared && other) { ptr.store(other.getClear(), std::memory_order_relaxed); } +#endif + inline ~AtomicShared() { ::Release(ptr.load(std::memory_order_relaxed)); } + inline AtomicShared<CLASS> & operator = (const AtomicShared<CLASS> & other) { this->setown(other.getLinkNonAtomic()); return *this; } + + inline void clear() { ::Release(getClear()); } + inline CLASS * getClear() + { + return ptr.exchange(nullptr); + } + inline CLASS * getClearNonAtomic() + { + CLASS * result = ptr.load(); + ptr.store(nullptr); + return result; + } + + //The getLink() function cannot be implemented in a thread safe way - e.g. if clear is called concurrently + //then temp will point to a freed object. (Might be possible with support for transactional memory...) + inline CLASS * getLinkNonAtomic() const + { + CLASS * temp = ptr; + if (temp) + temp->Link(); + return temp; + } + inline CLASS * query() const + { + return ptr.load(std::memory_order_acquire); + } + template <typename FUNC> + inline CLASS * query(FUNC factoryFunc, CriticalSection & cs) + { + CLASS * result = ptr.load(std::memory_order_acquire); + if (result) + return result; + CriticalBlock block(cs); + if (ptr.load(std::memory_order_acquire)) + return ptr.load(std::memory_order_acquire); + result = factoryFunc(); + ptr.store(result, std::memory_order_release); + return result; + } + inline bool isSet() const { return ptr != nullptr; } + inline void set(CLASS * _ptr) + { + if (ptr != _ptr) + { + LINK(_ptr); + this->setown(_ptr); + } + } + inline bool setownIfNull(CLASS * _ptr) + { + if (!_ptr) + return false; + + CLASS * expected = nullptr; + if (ptr.compare_exchange_strong(expected, _ptr)) + return true; + _ptr->Release(); + return false; + } + inline bool setIfNull(CLASS * _ptr) + { + if (!_ptr) + return false; + + CLASS * expected = nullptr; + if (ptr.compare_exchange_strong(expected, _ptr)) + { + _ptr->Link(); + return true; + } + return false; + } + inline void setown(CLASS * _ptr) + { + CLASS * temp = ptr.exchange(_ptr); + ::Release(temp); + } + inline CLASS * swap(CLASS * _ptr) + { + return ptr.exchange(_ptr); + } + //swap - this will only update this once, but other can temporarily have a null value + inline void swap(AtomicShared<CLASS> & other) + { + CLASS * temp = other.getClear(); + temp = this->swap(temp); + temp = other.swap(temp); + ::Release(temp); + } + +protected: + inline AtomicShared(CLASS * _ptr) { ptr = _ptr; } // deliberately protected + +private: + inline void setown(const AtomicShared<CLASS> &other); // illegal - going to cause a -ve leak + inline AtomicShared<CLASS> & operator = (const CLASS * other); + +private: + std::atomic<CLASS *> ptr; +}; + #endif diff --git a/system/jlib/jscm.hpp b/system/jlib/jscm.hpp index 718d6f3a215..263d9b6f069 100644 --- a/system/jlib/jscm.hpp +++ b/system/jlib/jscm.hpp @@ -110,102 +110,6 @@ template <class CLASS> class Shared CLASS * ptr; }; -// Similar to class Shared<X>, but thread safe versions of the functions (avoid the need for critical sections) -template <class CLASS> class AtomicShared -{ -public: - inline AtomicShared() { ptr.store(nullptr, std::memory_order_relaxed); } - inline AtomicShared(CLASS * _ptr, bool owned) { ptr.store(_ptr, std::memory_order_relaxed); if (!owned && _ptr) _ptr->Link(); } - inline AtomicShared(const AtomicShared & other) { ptr.store(other.getLinkNonAtomic(), std::memory_order_relaxed); } -#if defined(__cplusplus) && __cplusplus >= 201100 - inline AtomicShared(AtomicShared && other) { ptr.store(other.getClear(), std::memory_order_relaxed); } -#endif - inline ~AtomicShared() { ::Release(ptr.load(std::memory_order_relaxed)); } - inline AtomicShared<CLASS> & operator = (const AtomicShared<CLASS> & other) { this->setown(other.getLinkNonAtomic()); return *this; } - - inline void clear() { ::Release(getClear()); } - inline CLASS * getClear() - { - return ptr.exchange(nullptr); - } - inline CLASS * getClearNonAtomic() - { - CLASS * result = ptr.load(); - ptr.store(nullptr); - return result; - } - - //The getLink() function cannot be implemented in a thread safe way - e.g. if clear is called concurrently - //then temp will point to a freed object. (Might be possible with support for transactional memory...) - inline CLASS * getLinkNonAtomic() const - { - CLASS * temp = ptr; - if (temp) - temp->Link(); - return temp; - } - inline bool isSet() const { return ptr != nullptr; } - inline void set(CLASS * _ptr) - { - if (ptr != _ptr) - { - LINK(_ptr); - this->setown(_ptr); - } - } - inline bool setownIfNull(CLASS * _ptr) - { - if (!_ptr) - return false; - - CLASS * expected = nullptr; - if (ptr.compare_exchange_strong(expected, _ptr)) - return true; - _ptr->Release(); - return false; - } - inline bool setIfNull(CLASS * _ptr) - { - if (!_ptr) - return false; - - CLASS * expected = nullptr; - if (ptr.compare_exchange_strong(expected, _ptr)) - { - _ptr->Link(); - return true; - } - return false; - } - inline void setown(CLASS * _ptr) - { - CLASS * temp = ptr.exchange(_ptr); - ::Release(temp); - } - inline CLASS * swap(CLASS * _ptr) - { - return ptr.exchange(_ptr); - } - //swap - this will only update this once, but other can temporarily have a null value - inline void swap(AtomicShared<CLASS> & other) - { - CLASS * temp = other.getClear(); - temp = this->swap(temp); - temp = other.swap(temp); - ::Release(temp); - } - -protected: - inline AtomicShared(CLASS * _ptr) { ptr = _ptr; } // deliberately protected - -private: - inline void setown(const AtomicShared<CLASS> &other); // illegal - going to cause a -ve leak - inline AtomicShared<CLASS> & operator = (const CLASS * other); - -private: - std::atomic<CLASS *> ptr; -}; - //An Owned Shared object takes ownership of the pointer that is passed in the constructor. template <class CLASS> class Owned : public Shared<CLASS> diff --git a/thorlcr/activities/lookupjoin/thlookupjoinslave.cpp b/thorlcr/activities/lookupjoin/thlookupjoinslave.cpp index 2236833cc2b..c1e62740c2f 100644 --- a/thorlcr/activities/lookupjoin/thlookupjoinslave.cpp +++ b/thorlcr/activities/lookupjoin/thlookupjoinslave.cpp @@ -957,7 +957,7 @@ class CInMemJoinBase : public CSlaveActivity, public CAllOrLookupHelper<HELPER>, } } *rowProcessor; - CriticalSection rhsRowLock; + mutable CriticalSection rhsRowLock; Owned<CBroadcaster> broadcaster; CBroadcaster *channel0Broadcaster; CriticalSection *broadcastLock; @@ -1099,7 +1099,10 @@ class CInMemJoinBase : public CSlaveActivity, public CAllOrLookupHelper<HELPER>, { CThorSpillableRowArray *rows = rhsSlaveRows.item(a); if (rows) + { + mergeRemappedStats(inactiveStats, rows, diskToTempStatsMap); rows->kill(); + } } rhs.kill(); } @@ -1815,6 +1818,8 @@ class CLookupJoinActivityBase : public CInMemJoinBase<HTHELPER, IHThorHashJoinAr // NB: Only used by channel 0 Owned<CFileOwner> overflowWriteFile; Owned<IExtRowWriter> overflowWriteStream; + OwnedIFileIO overflowWriteFileIO; + mutable CriticalSection critOverflowWriteFileIO; rowcount_t overflowWriteCount; OwnedMalloc<IChannelDistributor *> channelDistributors; unsigned nextRhsToSpill = 0; @@ -2103,7 +2108,6 @@ class CLookupJoinActivityBase : public CInMemJoinBase<HTHELPER, IHThorHashJoinAr if (isSmart()) { overflowWriteCount = 0; - overflowWriteFile.clear(); overflowWriteStream.clear(); rightRowManager->addRowBuffer(this); } @@ -2114,7 +2118,15 @@ class CLookupJoinActivityBase : public CInMemJoinBase<HTHELPER, IHThorHashJoinAr CriticalBlock b(broadcastSpillingLock); rhsRows = getGlobalRHSTotal(); // flushes all rhsSlaveRows arrays to calculate total. if (hasFailedOverToLocal()) + { + if (overflowWriteFileIO) + { + mergeRemappedStats(PARENT::inactiveStats, overflowWriteFileIO, diskToTempStatsMap); + overflowWriteFile->noteSize(overflowWriteFileIO->getStatistic(StSizeDiskWrite)); + overflowWriteFileIO.clear(); + } overflowWriteStream.clear(); // broadcast has finished, no more can be written + } } if (!hasFailedOverToLocal()) { @@ -2162,6 +2174,7 @@ class CLookupJoinActivityBase : public CInMemJoinBase<HTHELPER, IHThorHashJoinAr ForEachItemIn(a, rhsSlaveRows) { CThorSpillableRowArray &rows = *rhsSlaveRows.item(a); + mergeRemappedStats(PARENT::inactiveStats, &rows, diskToTempStatsMap); rhs.appendRows(rows, true); // NB: This should not cause spilling, rhs is already sized and we are only copying ptrs in rows.kill(); // free up ptr table asap } @@ -2486,6 +2499,7 @@ class CLookupJoinActivityBase : public CInMemJoinBase<HTHELPER, IHThorHashJoinAr Owned<IThorRowLoader> rowLoader = createThorRowLoader(*this, queryRowInterfaces(leftITDL), helper->isLeftAlreadyLocallySorted() ? NULL : compareLeft); rowLoader->setTracingPrefix("Join left"); left.setown(rowLoader->load(left, abortSoon, false)); + mergeRemappedStats(PARENT::inactiveStats, rowLoader, diskToTempStatsMap); leftITDL = queryInput(0); // reset ActPrintLog("LHS loaded/sorted"); @@ -2557,6 +2571,7 @@ class CLookupJoinActivityBase : public CInMemJoinBase<HTHELPER, IHThorHashJoinAr */ Owned<IThorRowCollector> rightCollector; + Owned<IException> exception; try { CMarker marker(*this); @@ -2681,12 +2696,19 @@ class CLookupJoinActivityBase : public CInMemJoinBase<HTHELPER, IHThorHashJoinAr catch (IException *e) { if (!isOOMException(e)) - throw e; - IOutputMetaData *inputOutputMeta = rightITDL->queryFromActivity()->queryContainer().queryHelper()->queryOutputMeta(); - // rows may either be in separate slave row arrays or in single rhs array, or split. - rowcount_t total = rightCollector ? rightCollector->numRows() : (getGlobalRHSTotal() + rhs.ordinality()); - throw checkAndCreateOOMContextException(this, e, "gathering RHS rows for lookup join", total, inputOutputMeta, NULL); + exception.setown(e); + else + { + IOutputMetaData *inputOutputMeta = rightITDL->queryFromActivity()->queryContainer().queryHelper()->queryOutputMeta(); + // rows may either be in separate slave row arrays or in single rhs array, or split. + rowcount_t total = rightCollector ? rightCollector->numRows() : (getGlobalRHSTotal() + rhs.ordinality()); + exception.setown(checkAndCreateOOMContextException(this, e, "gathering RHS rows for lookup join", total, inputOutputMeta, NULL)); + } } + if (rightCollector && rightCollector->hasSpilt()) + mergeRemappedStats(PARENT::inactiveStats, rightCollector, diskToTempStatsMap); + if (exception) + throw exception.getClear(); } public: static bool needDedup(IHThorHashJoinArg *helper) @@ -2950,7 +2972,7 @@ class CLookupJoinActivityBase : public CInMemJoinBase<HTHELPER, IHThorHashJoinAr return true; } CriticalBlock b(rhsRowLock); - if (overflowWriteFile) + if (overflowWriteFileIO) { /* Tried to do outside crit above, but if empty, and now overflow, need to inside * Will be one off if at all @@ -2963,7 +2985,7 @@ class CLookupJoinActivityBase : public CInMemJoinBase<HTHELPER, IHThorHashJoinAr overflowWriteCount += rhsInRowsTemp.ordinality(); ForEachItemIn(r, rhsInRowsTemp) overflowWriteStream->putRow(rhsInRowsTemp.getClear(r)); - overflowWriteFile->noteSize(overflowWriteStream->getStatistic(StSizeDiskWrite)); + overflowWriteFile->noteSize(overflowWriteFileIO->getStatistic(StSizeDiskWrite)); return true; } if (hasFailedOverToLocal()) @@ -3008,12 +3030,13 @@ class CLookupJoinActivityBase : public CInMemJoinBase<HTHELPER, IHThorHashJoinAr GetTempFilePath(tempFilename, "lookup_local"); ActPrintLog("Overflowing RHS broadcast rows to spill file: %s", tempFilename.str()); overflowWriteFile.setown(container.queryActivity()->createOwnedTempFile(tempFilename.str())); - overflowWriteStream.setown(createRowWriter(&(overflowWriteFile->queryIFile()), queryRowInterfaces(rightITDL), rwFlags)); + overflowWriteFileIO.setown(overflowWriteFile->queryIFile().open(IFOcreate)); + overflowWriteStream.setown(createRowWriter(overflowWriteFileIO, queryRowInterfaces(rightITDL), rwFlags)); overflowWriteCount += rhsInRowsTemp.ordinality(); ForEachItemIn(r, rhsInRowsTemp) overflowWriteStream->putRow(rhsInRowsTemp.getClear(r)); - overflowWriteFile->noteSize(overflowWriteStream->getStatistic(StSizeDiskWrite)); + overflowWriteFile->noteSize(overflowWriteFileIO->getStatistic(StSizeDiskWrite)); return true; } virtual void gatherActiveStats(CRuntimeStatisticCollection &activeStats) const @@ -3025,6 +3048,19 @@ class CLookupJoinActivityBase : public CInMemJoinBase<HTHELPER, IHThorHashJoinAr activeStats.setStatistic(StNumSmartJoinDegradedToLocal, aggregateFailoversToLocal); // NB: is going to be same for all slaves. activeStats.setStatistic(StNumSmartJoinSlavesDegradedToStd, aggregateFailoversToStandard); } + { + CriticalBlock b(critOverflowWriteFileIO); + if (overflowWriteFileIO) + mergeRemappedStats(activeStats, overflowWriteFileIO, diskToTempStatsMap); + } + { + CriticalBlock b(rhsRowLock); + ForEachItemIn(a, rhsSlaveRows) + { + CThorSpillableRowArray &rows = *rhsSlaveRows.item(a); + mergeRemappedStats(activeStats, &rows, diskToTempStatsMap); + } + } } }; @@ -3367,6 +3403,7 @@ class CAllJoinSlaveActivity : public CInMemJoinBase<CAllTable, IHThorAllJoinArg> ForEachItemIn(a, rhsSlaveRows) { CThorSpillableRowArray &rows = *rhsSlaveRows.item(a); + mergeRemappedStats(PARENT::inactiveStats, &rows, diskToTempStatsMap); rhs.appendRows(rows, true); rows.kill(); // free up ptr table asap } diff --git a/thorlcr/activities/nsplitter/thnsplitterslave.cpp b/thorlcr/activities/nsplitter/thnsplitterslave.cpp index d733aa41a64..2cb322e6730 100644 --- a/thorlcr/activities/nsplitter/thnsplitterslave.cpp +++ b/thorlcr/activities/nsplitter/thnsplitterslave.cpp @@ -405,7 +405,7 @@ class NSplitterSlaveActivity : public CSlaveActivity, implements ISharedSmartBuf { PARENT::gatherActiveStats(activeStats); if (sharedRowStream) - ::mergeStats(activeStats, sharedRowStream); + mergeRemappedStats(activeStats, sharedRowStream, diskToTempStatsMap); } // ISharedSmartBufferCallback impl. virtual void paged() { pagedOut = true; } diff --git a/thorlcr/graph/thgraph.cpp b/thorlcr/graph/thgraph.cpp index 72c6b25bd13..414b2ec22e2 100644 --- a/thorlcr/graph/thgraph.cpp +++ b/thorlcr/graph/thgraph.cpp @@ -1210,6 +1210,7 @@ void traceMemUsage() } ///// +static CriticalSection tempFileSizeTrackerCrit; // shared amongst all, because very unlikely to contend CGraphBase::CGraphBase(CJobChannel &_jobChannel) : jobChannel(_jobChannel), job(_jobChannel.queryJob()), progressUpdated(false) { @@ -2288,6 +2289,10 @@ IThorGraphResults *CGraphBase::createThorGraphResults(unsigned num) return new CThorGraphResults(num); } +CFileSizeTracker * CGraphBase::queryTempFileSizeTracker() +{ + return tempFileSizeTracker.query([] { return new CFileSizeTracker; }, tempFileSizeTrackerCrit); +} //// @@ -3281,6 +3286,12 @@ IThorRowInterfaces * CActivityBase::createRowInterfaces(IOutputMetaData * meta, return createThorRowInterfaces(queryRowManager(), meta, id, heapFlags, queryCodeContext()); } +CFileSizeTracker * CActivityBase::queryTempFileSizeTracker() +{ + return tempFileSizeTracker.query([&] { return new CFileSizeTracker(queryGraph().queryParent()->queryTempFileSizeTracker()); }, tempFileSizeTrackerCrit); +} + + bool CActivityBase::fireException(IException *e) { Owned<IThorException> _te; diff --git a/thorlcr/graph/thgraph.hpp b/thorlcr/graph/thgraph.hpp index b1045fda17a..8a3ce04df21 100644 --- a/thorlcr/graph/thgraph.hpp +++ b/thorlcr/graph/thgraph.hpp @@ -630,7 +630,7 @@ class graph_decl CGraphBase : public CGraphStub, implements IEclGraphResults CChildGraphTable childGraphsTable; CGraphStubArrayCopy orderedChildGraphs; Owned<IGraphTempHandler> tmpHandler; - Owned<CFileSizeTracker> tempFileSizeTracker; + AtomicShared<CFileSizeTracker> tempFileSizeTracker; void clean(); protected: @@ -805,23 +805,16 @@ class graph_decl CGraphBase : public CGraphStub, implements IEclGraphResults virtual void end(); virtual void abort(IException *e) override; virtual IThorGraphResults *createThorGraphResults(unsigned num); - CFileSizeTracker * queryTempFileSizeTracker() - { - if (!tempFileSizeTracker) - tempFileSizeTracker.setown(new CFileSizeTracker); - return tempFileSizeTracker; - } + CFileSizeTracker * queryTempFileSizeTracker(); offset_t queryPeakTempSize() { - if (tempFileSizeTracker) - return tempFileSizeTracker->queryPeakSize(); - return 0; + CFileSizeTracker *tracker = tempFileSizeTracker.query(); + return tracker ? tracker->queryPeakSize() : 0; } offset_t queryActiveTempSize() { - if (tempFileSizeTracker) - return tempFileSizeTracker->queryActiveSize(); - return 0; + CFileSizeTracker *tracker = tempFileSizeTracker.query(); + return tracker ? tracker->queryActiveSize() : 0; } // IExceptionHandler virtual bool fireException(IException *e); @@ -1121,7 +1114,7 @@ class graph_decl CActivityBase : implements CInterfaceOf<IThorRowInterfaces>, im CSingletonLock CABserializerlock; CSingletonLock CABdeserializerlock; roxiemem::RoxieHeapFlags defaultRoxieMemHeapFlags = roxiemem::RHFnone; - Owned<CFileSizeTracker> tempFileSizeTracker; + AtomicShared<CFileSizeTracker> tempFileSizeTracker; protected: CGraphElementBase &container; @@ -1189,19 +1182,16 @@ class graph_decl CActivityBase : implements CInterfaceOf<IThorRowInterfaces>, im IThorRowInterfaces * createRowInterfaces(IOutputMetaData * meta, byte seq=0); IThorRowInterfaces * createRowInterfaces(IOutputMetaData * meta, roxiemem::RoxieHeapFlags heapFlags, byte seq=0); - CFileSizeTracker * queryTempFileSizeTracker() - { - if (!tempFileSizeTracker) - tempFileSizeTracker.setown(new CFileSizeTracker(queryGraph().queryParent()->queryTempFileSizeTracker())); - return tempFileSizeTracker; - } + CFileSizeTracker * queryTempFileSizeTracker(); offset_t queryActiveTempSize() const { - return tempFileSizeTracker ? tempFileSizeTracker->queryActiveSize() : 0; + CFileSizeTracker *tracker = tempFileSizeTracker.query(); + return tracker ? tracker->queryActiveSize() : 0; } offset_t queryPeakTempSize() const { - return tempFileSizeTracker ? tempFileSizeTracker->queryPeakSize() : 0; + CFileSizeTracker *tracker = tempFileSizeTracker.query(); + return tracker ? tracker->queryPeakSize() : 0; } CFileOwner * createOwnedTempFile(const char *fileName) { diff --git a/thorlcr/msort/tsorts.cpp b/thorlcr/msort/tsorts.cpp index 9047eb719d2..fe9ff077883 100644 --- a/thorlcr/msort/tsorts.cpp +++ b/thorlcr/msort/tsorts.cpp @@ -1340,7 +1340,7 @@ class CThorSorter : public CSimpleInterface, implements IThorSorter, implements throw; } - mergeStats(spillStats, sortedloader); + mergeRemappedStats(spillStats, sortedloader, diskToTempStatsMap); if (!abort) { diff --git a/thorlcr/thorutil/thbuf.cpp b/thorlcr/thorutil/thbuf.cpp index 7c5d533fa77..96915818937 100644 --- a/thorlcr/thorutil/thbuf.cpp +++ b/thorlcr/thorutil/thbuf.cpp @@ -2449,7 +2449,7 @@ class CSharedFullSpillingWriteAhead : public CInterfaceOf<ISharedRowStreamReader bool inputGrouped = false; SharedRowStreamReaderOptions options; size32_t inMemReadAheadGranularity = 0; - CRuntimeStatisticCollection inactiveStats; + CRuntimeStatisticCollection stats; CRuntimeStatisticCollection previousFileStats; StringAttr baseTmpFilename; @@ -2483,21 +2483,21 @@ class CSharedFullSpillingWriteAhead : public CInterfaceOf<ISharedRowStreamReader outputStream.clear(); iFileIO->flush(); tempFileOwner->noteSize(iFileIO->getStatistic(StSizeDiskWrite)); - updateRemappedStatsDelta(inactiveStats, previousFileStats, iFileIO, diskToTempStatsMap); // NB: also updates prev to current + updateStatsDelta(stats, previousFileStats, iFileIO); // NB: also updates prev to current previousFileStats.reset(); iFileIO.clear(); } } void createOutputStream() { - closeWriter(); // Ensure stats from closing files are preserved in inactiveStats + closeWriter(); // Ensure stats from closing files are preserved in stats // NB: Called once, when spilling starts. tempFileOwner.setown(activity.createOwnedTempFile(baseTmpFilename)); auto res = createSerialOutputStream(&(tempFileOwner->queryIFile()), compressHandler, options, numOutputs + 1); outputStream.setown(std::get<0>(res)); iFileIO.setown(std::get<1>(res)); totalInputRowsRead = inMemTotalRows; - inactiveStats.addStatistic(StNumSpills, 1); + stats.addStatistic(StNumSpills, 1); } void writeRowsFromInput() { @@ -2539,7 +2539,7 @@ class CSharedFullSpillingWriteAhead : public CInterfaceOf<ISharedRowStreamReader outputStream->flush(); totalInputRowsRead.fetch_add(newRowsWritten); tempFileOwner->noteSize(iFileIO->getStatistic(StSizeDiskWrite)); - updateRemappedStatsDelta(inactiveStats, previousFileStats, iFileIO, diskToTempStatsMap); // NB: also updates prev to current + updateStatsDelta(stats, previousFileStats, iFileIO); // NB: also updates prev to current // JCSMORE - could track size written, and start new file at this point (e.g. every 100MB), // and track their starting points (by row #) in a vector // We could then tell if/when the readers catch up, and remove consumed files as they do. @@ -2553,7 +2553,7 @@ class CSharedFullSpillingWriteAhead : public CInterfaceOf<ISharedRowStreamReader explicit CSharedFullSpillingWriteAhead(CActivityBase *_activity, unsigned _numOutputs, IRowStream *_input, bool _inputGrouped, const SharedRowStreamReaderOptions &_options, IThorRowInterfaces *rowIf, const char *_baseTmpFilename, ICompressHandler *_compressHandler) : activity(*_activity), numOutputs(_numOutputs), input(_input), inputGrouped(_inputGrouped), options(_options), compressHandler(_compressHandler), baseTmpFilename(_baseTmpFilename), meta(rowIf->queryRowMetaData()), serializer(rowIf->queryRowSerializer()), allocator(rowIf->queryRowAllocator()), deserializer(rowIf->queryRowDeserializer()), - inactiveStats(spillStatistics), previousFileStats(spillStatistics) + stats(tempFileStatistics), previousFileStats(tempFileStatistics) { assertex(input); @@ -2717,7 +2717,7 @@ class CSharedFullSpillingWriteAhead : public CInterfaceOf<ISharedRowStreamReader } virtual unsigned __int64 getStatistic(StatisticKind kind) const override { - return inactiveStats.getStatisticValue(kind); + return stats.getStatisticValue(kind); } }; diff --git a/thorlcr/thorutil/thmem.cpp b/thorlcr/thorutil/thmem.cpp index 6f2c7e4fcfd..630b95f68d7 100644 --- a/thorlcr/thorutil/thmem.cpp +++ b/thorlcr/thorutil/thmem.cpp @@ -234,6 +234,7 @@ class CSpillableStreamBase : public CSpillable unsigned spillCompInfo; CThorSpillableRowArray rows; Owned<CFileOwner> spillFile; + CRuntimeStatisticCollection stats; bool spillRows() { @@ -248,13 +249,13 @@ class CSpillableStreamBase : public CSpillable spillFile.setown(activity.createOwnedTempFile(tempName.str())); VStringBuffer spillPrefixStr("SpillableStream(%u)", spillPriority); rows.save(*spillFile, spillCompInfo, false, spillPrefixStr.str()); // saves committed rows + mergeStats(stats, &rows); rows.kill(); // no longer needed, readers will pull from spillFile. NB: ok to kill array as rows is never written to or expanded - spillFile->noteSize(spillFile->queryIFile().size()); return true; } public: CSpillableStreamBase(CActivityBase &_activity, CThorSpillableRowArray &inRows, IThorRowInterfaces *_rowIf, EmptyRowSemantics _emptyRowSemantics, unsigned _spillPriority) - : CSpillable(_activity, _rowIf, _spillPriority), rows(_activity), emptyRowSemantics(_emptyRowSemantics) + : CSpillable(_activity, _rowIf, _spillPriority), rows(_activity), emptyRowSemantics(_emptyRowSemantics), stats(diskRemoteStatistics) { assertex(inRows.isFlushed()); spillCompInfo = 0x0; @@ -265,6 +266,10 @@ class CSpillableStreamBase : public CSpillable { ensureSpillingCallbackRemoved(); } + unsigned __int64 getStatistic(StatisticKind kind) const + { + return stats.getStatisticValue(kind); + } // IBufferedRowCallback virtual bool freeBufferedRows(bool critical) override { @@ -1328,13 +1333,13 @@ void CThorSpillableRowArray::safeUnregisterWriteCallback(IWritePosCallback &cb) } CThorSpillableRowArray::CThorSpillableRowArray(CActivityBase &activity) - : CThorExpandingRowArray(activity) + : CThorExpandingRowArray(activity), stats(tempFileStatistics) { throwOnOom = false; } CThorSpillableRowArray::CThorSpillableRowArray(CActivityBase &activity, IThorRowInterfaces *rowIf, EmptyRowSemantics emptyRowSemantics, StableSortFlag stableSort, rowidx_t initialSize, size32_t _commitDelta) - : CThorExpandingRowArray(activity, rowIf, ers_forbidden, stableSort, false, initialSize), commitDelta(_commitDelta) + : CThorExpandingRowArray(activity, rowIf, ers_forbidden, stableSort, false, initialSize), commitDelta(_commitDelta), stats(tempFileStatistics) { } @@ -1363,6 +1368,7 @@ void CThorSpillableRowArray::kill() { clearRows(); CThorExpandingRowArray::kill(); + stats.reset(); } void CThorSpillableRowArray::sort(ICompare &compare, unsigned maxCores) @@ -1413,7 +1419,8 @@ rowidx_t CThorSpillableRowArray::save(CFileOwner &iFileOwner, unsigned _spillCom nextCB = &cbCopy.popGet(); nextCBI = nextCB->queryRecordNumber(); } - Owned<IExtRowWriter> writer = createRowWriter(&iFileOwner.queryIFile(), rowIf, rwFlags, nullptr, compBlkSz); + OwnedIFileIO iFileIO = iFileOwner.queryIFile().open(IFOcreate); + Owned<IExtRowWriter> writer = createRowWriter(iFileIO, rowIf, rwFlags, nullptr, compBlkSz); rowidx_t i=0; rowidx_t rowsWritten=0; try @@ -1452,7 +1459,6 @@ rowidx_t CThorSpillableRowArray::save(CFileOwner &iFileOwner, unsigned _spillCom ++i; } writer->flush(NULL); - iFileOwner.noteSize(writer->getStatistic(StSizeDiskWrite)); } catch (IException *e) { @@ -1463,6 +1469,10 @@ rowidx_t CThorSpillableRowArray::save(CFileOwner &iFileOwner, unsigned _spillCom firstRow += n; offset_t bytesWritten = writer->getPosition(); writer.clear(); + mergeStats(stats, iFileIO); + offset_t sizeTempFile = iFileIO->getStatistic(StSizeDiskWrite); + iFileOwner.noteSize(sizeTempFile); + stats.addStatistic(StNumSpills, 1); ActPrintLog(&activity, "%s: CThorSpillableRowArray::save done, rows written = %" RIPF "u, bytes = %" I64F "u, firstRow = %u", _tracingPrefix, rowsWritten, (__int64)bytesWritten, firstRow); return rowsWritten; } @@ -1638,11 +1648,7 @@ class CThorRowCollectorBase : public CSpillable Owned<CSharedSpillableRowSet> spillableRowSet; unsigned options = 0; unsigned spillCompInfo = 0; - RelaxedAtomic<unsigned> statOverflowCount{0}; - RelaxedAtomic<offset_t> statSizeSpill{0}; - RelaxedAtomic<__uint64> statSpillCycles{0}; RelaxedAtomic<__uint64> statSortCycles{0}; - bool spillRows(bool critical) { //This must only be called while a lock is held on spillableRows @@ -1668,11 +1674,6 @@ class CThorRowCollectorBase : public CSpillable spillableRows.save(*tempFileOwner, spillCompInfo, false, spillPrefixStr.str()); // saves committed rows spillFiles.append(tempFileOwner.getLink()); ++overflowCount; - statOverflowCount.fastAdd(1); // NB: this is total over multiple uses of this class - offset_t tempFileSize = tempFileOwner->queryIFile().size(); - statSizeSpill.fastAdd(tempFileSize); - tempFileOwner->noteSize(tempFileSize); - statSpillCycles.fastAdd(spillTimer.elapsedCycles()); return true; } void setEmptyRowSemantics(EmptyRowSemantics _emptyRowSemantics) @@ -1960,26 +1961,17 @@ class CThorRowCollectorBase : public CSpillable { options = _options; } - unsigned __int64 getStatistic(StatisticKind kind) + unsigned __int64 getStatistic(StatisticKind kind) const { switch (kind) { - case StCycleSpillElapsedCycles: - return statSpillCycles; case StCycleSortElapsedCycles: return statSortCycles; - case StTimeSpillElapsed: - return cycle_to_nanosec(statSpillCycles); case StTimeSortElapsed: return cycle_to_nanosec(statSortCycles); - case StNumSpills: - return statOverflowCount; - case StSizeSpillFile: - return statSizeSpill; default: - break; + return spillableRows.getStatistic(kind); } - return 0; } bool hasSpilt() const { return overflowCount >= 1; } @@ -2048,7 +2040,7 @@ class CThorRowLoader : public CThorRowCollectorBase, implements IThorRowLoader } virtual void resize(rowidx_t max) override { CThorRowCollectorBase::resize(max); } virtual void setOptions(unsigned options) override { CThorRowCollectorBase::setOptions(options); } - virtual unsigned __int64 getStatistic(StatisticKind kind) override { return CThorRowCollectorBase::getStatistic(kind); } + virtual unsigned __int64 getStatistic(StatisticKind kind) const override { return CThorRowCollectorBase::getStatistic(kind); } virtual bool hasSpilt() const override { return CThorRowCollectorBase::hasSpilt(); } virtual void setTracingPrefix(const char *tracing) override { CThorRowCollectorBase::setTracingPrefix(tracing); } virtual void reset() override { CThorRowCollectorBase::reset(); } @@ -2103,7 +2095,7 @@ class CThorRowCollector : public CThorRowCollectorBase, implements IThorRowColle } virtual void resize(rowidx_t max) override { CThorRowCollectorBase::resize(max); } virtual void setOptions(unsigned options) override { CThorRowCollectorBase::setOptions(options); } - virtual unsigned __int64 getStatistic(StatisticKind kind) override { return CThorRowCollectorBase::getStatistic(kind); } + virtual unsigned __int64 getStatistic(StatisticKind kind) const override { return CThorRowCollectorBase::getStatistic(kind); } virtual bool hasSpilt() const override { return CThorRowCollectorBase::hasSpilt(); } virtual void setTracingPrefix(const char *tracing) override { CThorRowCollectorBase::setTracingPrefix(tracing); } // IThorArrayLock diff --git a/thorlcr/thorutil/thmem.hpp b/thorlcr/thorutil/thmem.hpp index 8e4f1b896a8..a89d4cdd0f2 100644 --- a/thorlcr/thorutil/thmem.hpp +++ b/thorlcr/thorutil/thmem.hpp @@ -413,7 +413,7 @@ class graph_decl CThorSpillableRowArray : private CThorExpandingRowArray, implem mutable CriticalSection cs; ICopyArrayOf<IWritePosCallback> writeCallbacks; size32_t compBlkSz = 0; // means use default - + CRuntimeStatisticCollection stats; // reset after each kill bool _flush(bool force); void doFlush(); inline bool needToMoveRows(bool force) { return (firstRow != 0 && (force || (firstRow >= commitRows/2))); } @@ -484,6 +484,10 @@ class graph_decl CThorSpillableRowArray : private CThorExpandingRowArray, implem inline rowidx_t numCommitted() const { return commitRows - firstRow; } //MORE::Not convinced this is very safe! inline rowidx_t queryTotalRows() const { return CThorExpandingRowArray::ordinality(); } // includes uncommited rows + inline unsigned __int64 getStatistic(StatisticKind kind) const + { + return stats.getStatisticValue(kind); + } // access to void swap(CThorSpillableRowArray &src); @@ -542,7 +546,7 @@ interface IThorRowCollectorCommon : extends IInterface, extends IThorArrayLock virtual void setup(ICompare *iCompare, StableSortFlag stableSort=stableSort_none, RowCollectorSpillFlags diskMemMix=rc_mixed, unsigned spillPriority=50) = 0; virtual void resize(rowidx_t max) = 0; virtual void setOptions(unsigned options) = 0; - virtual unsigned __int64 getStatistic(StatisticKind kind) = 0; + virtual unsigned __int64 getStatistic(StatisticKind kind) const = 0; virtual bool hasSpilt() const = 0; // equivalent to numOverlows() >= 1 virtual void setTracingPrefix(const char *tracing) = 0; virtual void reset() = 0; diff --git a/thorlcr/thorutil/thormisc.cpp b/thorlcr/thorutil/thormisc.cpp index 25e62473115..dfdb1b243b1 100644 --- a/thorlcr/thorutil/thormisc.cpp +++ b/thorlcr/thorutil/thormisc.cpp @@ -97,6 +97,7 @@ const StatisticsMapping hashDedupActivityStatistics({}, diskWriteRemoteStatistic const StatisticsMapping hashDistribActivityStatistics({StNumLocalRows, StNumRemoteRows, StSizeRemoteWrite}, basicActivityStatistics); const StatisticsMapping loopActivityStatistics({StNumIterations}, basicActivityStatistics); const StatisticsMapping graphStatistics({StNumExecutions, StSizeSpillFile, StSizeGraphSpill, StSizePeakTempDisk, StSizePeakEphemeralDisk, StTimeUser, StTimeSystem, StNumContextSwitches, StSizeMemory, StSizePeakMemory, StSizeRowMemory, StSizePeakRowMemory}, executeStatistics); +const StatisticsMapping tempFileStatistics({StNumSpills}, diskRemoteStatistics); const StatKindMap diskToTempStatsMap ={ {StSizeDiskWrite, StSizeSpillFile}, diff --git a/thorlcr/thorutil/thormisc.hpp b/thorlcr/thorutil/thormisc.hpp index 430d979c000..260b58a5b55 100644 --- a/thorlcr/thorutil/thormisc.hpp +++ b/thorlcr/thorutil/thormisc.hpp @@ -175,6 +175,8 @@ extern graph_decl const StatisticsMapping soapcallActivityStatistics; extern graph_decl const StatisticsMapping indexReadFileStatistics; extern graph_decl const StatisticsMapping hashDedupActivityStatistics; extern graph_decl const StatisticsMapping hashDistribActivityStatistics; +extern graph_decl const StatisticsMapping tempFileStatistics; + // Maps disk related stats to spill stats extern graph_decl const std::map<StatisticKind, StatisticKind> diskToTempStatsMap; diff --git a/vcpkg.json.in b/vcpkg.json.in index e86f4f52dfe..f2e23d68ff5 100644 --- a/vcpkg.json.in +++ b/vcpkg.json.in @@ -146,7 +146,7 @@ { "name": "openblas", "features": [ - "dynamic-arch", + @VCPKG_ECLBLAS_DYNAMIC_ARCH_FEATURE@ "threads" ], "platform": "@VCPKG_ECLBLAS@ & !windows"