Merge remote-tracking branch 'origin/candidate-9.8.x'

Signed-off-by: Gavin Halliday <gavin.halliday@lexisnexis.com>

dali/base/dadfs.cpp

@@ -2912,7 +2912,7 @@ class CDistributedFileBase : implements INTERFACE, public CInterface
         if (history)
             queryAttributes().removeTree(history);
     }
-    void lockFileAttrLock(CFileAttrLock & attrLock)
+    virtual void lockFileAttrLock(CFileAttrLock & attrLock)
     {
         if (!attrLock.init(logicalName, DXB_File, RTM_LOCK_WRITE, conn, defaultTimeout, "CDistributedFile::lockFileAttrLock"))
         {
@@ -6482,6 +6482,19 @@ class CDistributedSuperFile: public CDistributedFileBase<IDistributedSuperFile>
         return new cSubFileIterator(subfiles,supersub);
     }
 
+    virtual void lockFileAttrLock(CFileAttrLock & attrLock) override
+    {
+        if (!attrLock.init(logicalName, DXB_SuperFile, RTM_LOCK_WRITE, conn, defaultTimeout, "CDistributedFile::lockFileAttrLock"))
+        {
+            // In unlikely event File/Attr doesn't exist, must ensure created, commited and root connection is reloaded.
+            verifyex(attrLock.init(logicalName, DXB_SuperFile, RTM_LOCK_WRITE|RTM_CREATE_QUERY, conn, defaultTimeout, "CDistributedFile::lockFileAttrLock"));
+            attrLock.commit();
+            conn->commit();
+            conn->reload();
+            root.setown(conn->getRoot());
+        }
+    }
+
     void updateFileAttrs()
     {
         if (subfiles.ordinality()==0) {

dali/daliadmin/daadmin.cpp

@@ -3385,4 +3385,43 @@ void removeOrphanedGlobalVariables(bool dryrun, bool reconstruct)
     }
 }
 
+void cleanJobQueues(bool dryRun)
+{
+    Owned<IRemoteConnection> conn = querySDS().connect("/JobQueues", myProcessSession(), 0, SDS_LOCK_TIMEOUT);
+    if (!conn)
+    {
+        WARNLOG("Failed to connect to /JobQueues");
+        return;
+    }
+    Owned<IPropertyTreeIterator> queueIter = conn->queryRoot()->getElements("Queue");
+    ForEach(*queueIter)
+    {
+        IPropertyTree &queue = queueIter->query();
+        const char *name = queue.queryProp("@name");
+        if (isEmptyString(name)) // should not be blank, but guard
+            continue;
+        PROGLOG("Processing queue: %s", name);
+        VStringBuffer queuePath("/JobQueues/Queue[@name=\"%s\"]", name);
+        Owned<IRemoteConnection> queueConn = querySDS().connect(queuePath, myProcessSession(), RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT);
+        IPropertyTree *queueRoot = queueConn->queryRoot();
+
+        Owned<IPropertyTreeIterator> clientIter = queueRoot->getElements("Client");
+        std::vector<IPropertyTree *> toRemove;
+        ForEach (*clientIter)
+        {
+            IPropertyTree &client = clientIter->query();
+            if (client.getPropInt("@connected") == 0)
+                toRemove.push_back(&client);
+        }
+        if (!dryRun)
+        {
+            for (auto &client: toRemove)
+                queue.removeTree(client);
+        }
+        PROGLOG("Job queue '%s': %s %u stale client entries", name, dryRun ? "dryrun, there are" : "removed", (unsigned)toRemove.size());
+        queueConn->commit();
+        queueConn.clear();
+    }
+}
+
 } // namespace daadmin

dali/daliadmin/daadmin.hpp

@@ -95,5 +95,6 @@ extern DALIADMIN_API void daliping(const char *dalis, unsigned connecttime, unsi
 
 extern DALIADMIN_API void validateStore(bool fix, bool deleteFiles, bool verbose);
 extern DALIADMIN_API void removeOrphanedGlobalVariables(bool dryrun, bool reconstruct);
+extern DALIADMIN_API void cleanJobQueues(bool dryRun);
 
 } // namespace daadmin

dali/daliadmin/daliadmin.cpp

@@ -93,6 +93,7 @@ void usage(const char *exe)
   printf("Other dali server and misc commands:\n");
   printf("  auditlog <fromdate> <todate> <match>\n");
   printf("  cleanglobalwuid [dryrun] [noreconstruct]\n");
+  printf("  cleanjobqueues [dryrun]\n");
   printf("  clusterlist <mask>              -- list clusters   (mask optional)\n");
   printf("  coalesce                        -- force transaction coalesce\n");
   printf("  dalilocks [ <ip-pattern> ] [ files ] -- get all locked files/xpaths\n");
@@ -576,6 +577,11 @@ int main(int argc, const char* argv[])
                         }
                         removeOrphanedGlobalVariables(dryrun, reconstruct);
                     }
+                    else if (strieq(cmd, "cleanjobqueues"))
+                    {
+                        bool dryRun = np>0 && strieq("dryrun", params.item(1));
+                        cleanJobQueues(dryRun);
+                    }
                     else if (strieq(cmd, "remotetest"))
                         remoteTest(params.item(1), true);
                     else

dali/dfu/dfuutil.cpp

@@ -530,13 +530,18 @@ class CFileCloner
         if (iskey&&!cluster2.isEmpty())
             dstfdesc->addCluster(cluster2,grp2,spec2);
 
-        for (unsigned pn=0; pn<numParts; pn++) {
-            offset_t sz = srcfdesc->queryPart(pn)->queryProperties().getPropInt64("@size",-1);
+        for (unsigned pn=0; pn<numParts; pn++)
+        {
+            IPropertyTree &srcProps = srcfdesc->queryPart(pn)->queryProperties();
+            IPropertyTree &dstProps = dstfdesc->queryPart(pn)->queryProperties();
+            offset_t sz = srcProps.getPropInt64("@size",-1);
             if (sz!=(offset_t)-1)
-                dstfdesc->queryPart(pn)->queryProperties().setPropInt64("@size",sz);
+                dstProps.setPropInt64("@size",sz);
             StringBuffer dates;
-            if (srcfdesc->queryPart(pn)->queryProperties().getProp("@modified",dates))
-                dstfdesc->queryPart(pn)->queryProperties().setProp("@modified",dates.str());
+            if (srcProps.getProp("@modified",dates))
+                dstProps.setProp("@modified",dates.str());
+            if (srcProps.hasProp("@kind"))
+                dstProps.setProp("@kind", srcProps.queryProp("@kind"));
         }
 
         if (!copyphysical) //cloneFrom tells roxie where to copy from.. it's unnecessary if we already did the copy

devdoc/userdoc/copilot/CopilotPromptTips.md

@@ -0,0 +1,84 @@
+# Copilot Prompt Tips
+Unlock the full potential of GitHub Copilot with these carefully curated prompts designed to streamline your workflow and enhance productivity. 
+
+Whether you're summarizing complex texts, brainstorming innovative ideas, or creating detailed guides, these prompts will help you get the most out of Copilot's capabilities. 
+
+Dive in and discover how these simple yet powerful prompts can save you time and effort in your daily tasks.
+
+## Generic Prompts
+
+Here are a few simple prompts that can save time:
+
+- Provide a brief summary of the text below. Show the key points in a bullet list.
+
+- List [N (number)] ways to [accomplish a specific goal or solve a problem]? Include a short description for each approach in a bullet list.
+
+  - **Example**: List 10 ways to reduce redundancy in text. Include a short description for each approach in a bullet list.
+
+- What are the key differences/similarities between [concept A] and [concept B]? Present the information in a table format.
+
+  - **Example:** What are the key differences/similarities between a quicksort ad a bubble sort? Present the information in a table format.
+
+- Explain [complex topic] in simple terms. Use analogies or examples to help make it easier to understand.
+
+  - **Example:** Explain generative AI in simple terms. Use analogies or examples to help make it easier to understand. 
+
+- Brainstorm [N (number)] ideas for [a specific project or topic]? Include a short description for each idea in a bullet list.
+
+  - **Example:** Brainstorm 7 ideas for an instructional video for new programmers? Include a short description for each idea in a bullet list.
+
+- Create a template for [a specific type of document, such as a business email, proposal, etc.]. Include the key elements to include in a bullet list.
+
+  - **Example:** Create a template for product version release announcement. Include the key elements to include in a bullet list.
+
+- Write a step-by-step guide on how to [specific task or procedure]. Number the steps to improve clarity.
+
+  - **Example:** Write a step-by-step guide on how to make a PB & J sandwich. Number the steps to improve clarity.
+
+## Specific Prompts
+
+These prompts are more focused and are meant to accomplish a specific purpose. They are included here to spark your imagination.
+
+If you think of others that could be included here to share with the world, please send your ideas to <docfeedback@hpccsystems.com>. 
+
+- Write comments for this ECL code in javadoc format
+
+- Create an ECL File Definition, record structure, and inline dataset with [N (number)] of records with the following fields [list of fields]
+
+  - **Example:** Create an ECL file definition with a record structure and an inline dataset with 20 records for a dataset of animals with the following fields: ReferenceNumber, AnimalName, ClassName, FamilyName, Color, Size, and LifeExpectancy.
+
+- Write ECL code to classify records into [categories].
+
+  - **Example:** Write ECL code to classify customer feedback into neutral, negative, or positive categories.
+
+
+## How to Avoid AI Hallucinations with Good Prompts
+AI hallucinations refer to instances where artificial intelligence systems generate information or responses that are incorrect, misleading, or entirely fabricated. 
+
+Hallucinations can occur due to various reasons, such as limitations in the training data, inherent biases, or the AI's attempt to provide an answer even when it lacks sufficient context or knowledge.
+
+Understanding and mitigating AI hallucinations is crucial for ensuring the reliability and accuracy of AI-driven applications.
+
+Creating effective prompts is essential to minimize AI hallucinations. Here are some tips to help you craft prompts that lead to accurate and reliable responses:
+
+- **Be Specific and Clear**: Ambiguous prompts can lead to incorrect or irrelevant answers. Clearly define the task and provide specific instructions.
+
+  - **Example**: Instead of asking "What is AI?", ask "Explain the concept of artificial intelligence and its primary applications in healthcare."
+
+- **Provide Context**: Give the AI enough background information to understand the task. This helps in generating more accurate responses.
+
+  - **Example**: "Given the following text about climate change, summarize the key points in a bullet list."
+
+- **Use Constraints**: Limit the scope of the response by specifying constraints such as word count, format, or specific details to include.
+
+  - **Example**: "List 5 benefits of renewable energy in a bullet list, each point not exceeding 20 words."
+
+- **Ask for Evidence or Sources**: Encourage the AI to provide evidence or cite sources for the information it generates.
+
+  - **Example**: "Explain the impact of social media on mental health and provide references to recent studies."
+
+- **Iterative Refinement**: Start with a broad prompt and refine it based on the initial responses to get more accurate results.
+
+  - **Example**: Begin with "Describe the process of photosynthesis." If the response is too vague, refine it to "Describe the process of photosynthesis in plants, including the role of chlorophyll and sunlight."
+
+By following these guidelines, you can reduce the likelihood of AI hallucinations and ensure that the responses generated are accurate and useful.

ecl/eclcc/eclcc.cpp

@@ -1537,8 +1537,11 @@ void EclCC::processSingleQuery(const EclRepositoryManager & localRepositoryManag
 
             updateWorkunitStat(instance.wu, SSToperation, ">compile:>parse", StTimeElapsed, NULL, parseTimeNs);
             stat_type sourceDownloadTime = localRepositoryManager.getStatistic(StTimeElapsed);
+            stat_type sourceDownloadBlockedTime = localRepositoryManager.getStatistic(StTimeBlocked);
             if (sourceDownloadTime)
                 updateWorkunitStat(instance.wu, SSToperation, ">compile:>parse:>download", StTimeElapsed, NULL, sourceDownloadTime);
+            if (sourceDownloadBlockedTime)
+                updateWorkunitStat(instance.wu, SSToperation, ">compile:>parse:>download", StTimeBlocked, NULL, sourceDownloadBlockedTime);
 
             if (optExtraStats)
             {

ecl/hql/hqlrepository.cpp

@@ -676,6 +676,8 @@ unsigned __int64 EclRepositoryManager::getStatistic(StatisticKind kind) const
     {
     case StTimeElapsed:
         return cycle_to_nanosec(gitDownloadCycles);
+    case StTimeBlocked:
+        return cycle_to_nanosec(gitDownloadBlockedCycles);
     }
     return 0;
 }
@@ -823,7 +825,12 @@ IEclSourceCollection * EclRepositoryManager::resolveGitCollection(const char * r
         throw makeStringExceptionV(99, "Unsupported repository link format '%s'", defaultUrl);
 
     bool alreadyExists = false;
+
+    CCycleTimer gitDownloadTimer;
     Owned<IInterface> gitUpdateLock(getGitUpdateLock(repoPath));
+    cycle_t blockedCycles = gitDownloadTimer.elapsedCycles();
+    gitDownloadBlockedCycles += blockedCycles;
+
     if (checkDirExists(repoPath))
     {
         if (options.cleanRepos)
@@ -853,7 +860,6 @@ IEclSourceCollection * EclRepositoryManager::resolveGitCollection(const char * r
 
     bool ok = false;
     Owned<IError> error;
-    CCycleTimer gitDownloadTimer;
     if (alreadyExists)
     {
         if (options.updateRepos)
@@ -890,7 +896,8 @@ IEclSourceCollection * EclRepositoryManager::resolveGitCollection(const char * r
     //this could become a read/write lock if that proved to be an issue.
     gitUpdateLock.clear();
 
-    gitDownloadCycles += gitDownloadTimer.elapsedCycles();
+    gitDownloadCycles += (gitDownloadTimer.elapsedCycles() - blockedCycles);
+
     if (error)
     {
         if (errorReceiver)

ecl/hql/hqlrepository.hpp

@@ -105,6 +105,7 @@ class HQL_API EclRepositoryManager
     IArrayOf<IEclRepository> overrideSources;   // -D options
     IArrayOf<IEclRepository> allSources;        // also includes -D options
     cycle_t gitDownloadCycles = 0;
+    cycle_t gitDownloadBlockedCycles = 0;
 
     //Include all options in a nested struct to make it easy to ensure they are cloned
     struct {

esp/src/src-react/components/Metrics.tsx

@@ -56,6 +56,9 @@ export const Metrics: React.FunctionComponent<MetricsProps> = ({
     selection,
     fullscreen = false
 }) => {
+    if (querySet && queryId) {
+        wuid = "";
+    }
     const [_uiState, _setUIState] = React.useState({ ...defaultUIState });
     const [selectedMetricsSource, setSelectedMetricsSource] = React.useState<SelectedMetricsSource>("");
     const [selectedMetrics, setSelectedMetrics] = React.useState<IScope[]>([]);