Build filetree with missing directories to support --only-findings scans

Signed-off-by: Omkar Phansopkar <omkarphansopkar@gmail.com>
aboutcode-org · Jan 1, 2024 · 19d3d6a · 19d3d6a
1 parent 99a1525
commit 19d3d6a
Show file tree

Hide file tree

Showing 10 changed files with 343 additions and 46 deletions.
diff --git a/src/App.tsx b/src/App.tsx
@@ -29,6 +29,7 @@ import "react-toastify/dist/ReactToastify.css";
 import "bootstrap/dist/css/bootstrap.min.css";
 import "react-tooltip/dist/react-tooltip.css";
 import "./styles/app.css";
+import "./styles/toastify.css";
 import "./styles/dashStyles.css";
 import "./styles/customFaColors.css";
 

diff --git a/src/contexts/dbContext.tsx b/src/contexts/dbContext.tsx
@@ -183,12 +183,8 @@ export const WorkbenchDBProvider = (
         // Check that the database has the correct header information.
         if (!infoHeader) {
           const errTitle = "Invalid SQLite file";
-          const errMessage =
-            "Invalid SQLite file: " +
-            sqliteFilePath +
-            "\n" +
-            "The SQLite file is invalid. Try re-importing the ScanCode JSON " +
-            "file and creating a new SQLite file.";
+          const errMessage = `Invalid SQLite file: ${sqliteFilePath}
+The SQLite file is invalid. Try re-importing the ScanCode JSON file and creating a new SQLite file.`;
 
           console.error("Handled invalid sqlite import", {
             title: errTitle,
@@ -239,8 +235,7 @@ export const WorkbenchDBProvider = (
           .then((db) => db.File.findOne({ where: { parent: "#" } }))
           .then(async (root) => {
             if (!root) {
-              console.error("Root path not found !!!!", root);
-              return;
+              throw new Error("Root path not found !!");
             }
 
             const defaultPath = root.getDataValue("path");
@@ -271,35 +266,20 @@ export const WorkbenchDBProvider = (
             );
 
             if (!preventNavigation) changeRouteOnImport();
-          })
-          .catch((err) => {
-            const foundInvalidHistoryItem = GetHistory().find(
-              (historyItem) => historyItem.sqlite_path === sqliteFilePath
-            );
-            if (foundInvalidHistoryItem) {
-              RemoveEntry(foundInvalidHistoryItem);
-            }
-            console.error("Err trying to import sqlite:");
-            console.error(err);
-            toast.error(
-              `Unexpected error while importing json \nPlease check console for more info`
-            );
-            abortImport();
           });
       })
-      .catch((err) => {
+      .catch((err: Error) => {
+        abortImport();
         const foundInvalidHistoryItem = GetHistory().find(
           (historyItem) => historyItem.sqlite_path === sqliteFilePath
         );
         if (foundInvalidHistoryItem) {
           RemoveEntry(foundInvalidHistoryItem);
         }
-        console.error("Err trying to import sqlite:");
-        console.error(err);
+        console.error("Err trying to import sqlite:", err);
         toast.error(
-          `Unexpected error while finalising json import \nPlease check console for more info`
+          `Sqlite file is outdated or corrupt\nPlease try importing json file again`
         );
-        abortImport();
       });
   }
 
@@ -341,11 +321,10 @@ export const WorkbenchDBProvider = (
           .then((db) => db.File.findOne({ where: { parent: "#" } }))
           .then(async (root) => {
             if (!root) {
-              console.error("Root path not found !!!!");
               console.error("Root:", root);
-              abortImport();
-              return;
+              throw new Error("Root path not found !!!!");
             }
+
             const defaultPath = root.getDataValue("path");
 
             await updateWorkbenchDB(newWorkbenchDB, sqliteFilePath);
@@ -369,16 +348,29 @@ export const WorkbenchDBProvider = (
             );
 
             if (!preventNavigation) changeRouteOnImport();
+          })
+          .catch((err) => {
+            abortImport();
+            const foundInvalidHistoryItem = GetHistory().find(
+              (historyItem) => historyItem.sqlite_path === sqliteFilePath
+            );
+            if (foundInvalidHistoryItem) {
+              RemoveEntry(foundInvalidHistoryItem);
+            }
+            console.error(err);
+            toast.error(
+              `Can't resolve root directory \nPlease check console for more info`
+            );
           });
       })
       .catch((err) => {
         abortImport();
         console.error(
-          "Some error parsing data (caught in workbenchContext) !!",
+          "Some error parsing json data (caught in dbContext)",
           err
         );
         toast.error(
-          "Some error parsing data !! \nPlease check console for more info"
+          "Some error parsing json data !! \nPlease check console for more info"
         );
       });
   }

diff --git a/src/pages/Licenses/Licenses.tsx b/src/pages/Licenses/Licenses.tsx
@@ -171,7 +171,13 @@ const LicenseDetections = () => {
           activateLicenseClue(newLicenseClues[0]);
         }
       }
-    })().then(endProcessing);
+    })()
+      .then(endProcessing)
+      .catch((err) => {
+        endProcessing();
+        console.error("Error getting license contents", err);
+        toast.error("Error loading license data.");
+      });
   }, []);
 
   const handleItemToggle = (

diff --git a/src/pages/TableView/TableView.tsx b/src/pages/TableView/TableView.tsx
@@ -1,4 +1,5 @@
 import { Op } from "sequelize";
+import { toast } from "react-toastify";
 import React, { useEffect, useState } from "react";
 import { ColDef, ColumnApi, GridApi, GridReadyEvent } from "ag-grid-community";
 
@@ -101,6 +102,11 @@ const TableView = () => {
           if (prevColDefs.length > 0) return prevColDefs; // Don't mutate cols, if already set
           return [...COLUMN_GROUPS.DEFAULT];
         });
+      })
+      .catch((err) => {
+        endProcessing();
+        console.error("Error getting tableview contents", err);
+        toast.error("Error loading table data.");
       });
 
     // Update set filters whenever new db is loaded or path is changed

diff --git a/src/services/models/header.ts b/src/services/models/header.ts
@@ -53,15 +53,33 @@ export default function headerModel(sequelize: Sequelize) {
         primaryKey: true,
         type: DataTypes.INTEGER,
       },
-      json_file_name: DataTypes.STRING,
-      tool_name: DataTypes.STRING,
-      tool_version: DataTypes.STRING,
-      notice: DataTypes.STRING,
-      duration: DataTypes.DOUBLE,
+      json_file_name: {
+        type: DataTypes.STRING,
+        defaultValue: null,
+      },
+      tool_name: {
+        type: DataTypes.STRING,
+        defaultValue: null,
+      },
+      tool_version: {
+        type: DataTypes.STRING,
+        defaultValue: null,
+      },
+      notice: {
+        type: DataTypes.STRING,
+        defaultValue: null,
+      },
+      duration: {
+        type: DataTypes.DOUBLE,
+        defaultValue: null,
+      },
       options: jsonDataType("options", {}),
       input: jsonDataType("input", []),
       header_content: DataTypes.STRING,
-      files_count: DataTypes.INTEGER,
+      files_count: {
+        type: DataTypes.INTEGER,
+        defaultValue: 0,
+      },
       output_format_version: {
         type: DataTypes.STRING,
         defaultValue: null,

diff --git a/src/services/workbenchDB.ts b/src/services/workbenchDB.ts
@@ -282,12 +282,12 @@ export class WorkbenchDB {
     fileList.forEach((file) => {
       const fileParentPath = file.getDataValue("parent");
       const fileNode = pathToNodeMap.get(file.getDataValue("path"));
-      if (Number(file.getDataValue("id")) !== 0) {
+      if (file.getDataValue("parent") === "#") {
+        roots.push(fileNode);
+      } else {
         if (pathToNodeMap.has(fileParentPath)) {
           pathToNodeMap.get(fileParentPath).children?.push(fileNode);
         }
-      } else {
-        roots.push(fileNode);
       }
     });
 
@@ -401,6 +401,7 @@ export class WorkbenchDB {
                 this.pause();
 
                 promiseChain = promiseChain
+                  .then(() => this._imputeMissingIntermediateDirectories(files))
                   .then(() => primaryPromise._batchCreateFiles(files))
                   .then(() => {
                     const currentProgress = Math.round(
@@ -427,7 +428,8 @@ export class WorkbenchDB {
               // See https://github.com/nexB/scancode-toolkit/issues/543
               promiseChain
                 .then(() => {
-                  if (rootPath && !hasRootPath) {
+                  if (!hasRootPath) {
+                    rootPath = rootPath || "no-files";
                     files.push({
                       path: rootPath,
                       name: rootPath,
@@ -436,6 +438,7 @@ export class WorkbenchDB {
                     });
                   }
                 })
+                .then(() => this._imputeIntermediateDirectories(files))
                 .then(() => this._batchCreateFiles(files))
                 .then(() => this.db.Header.create(TopLevelData.parsedHeader))
                 .then(() => {
@@ -600,7 +603,7 @@ export class WorkbenchDB {
       duration: header.duration,
       options: header?.options || {},
       input,
-      files_count: header.extra_data?.files_count,
+      files_count: header.extra_data?.files_count || 0,
       output_format_version: header.output_format_version,
       spdx_license_list_version: header.extra_data?.spdx_license_list_version,
       operating_system: header.extra_data?.system_environment?.operating_system,
@@ -880,6 +883,37 @@ export class WorkbenchDB {
     });
   }
 
+  // Adds & modifies files array in place, adding missing intermediate directories
+  _imputeIntermediateDirectories(files: Resource[]) {
+    const availableFiles = new Set(files.map((file) => file.path));
+    const intermediateDirectories: Resource[] = [];
+
+    files.forEach((file) => {
+      file.parent = parentPath(file.path);
+
+      // Add intermediate directories if parent not available in files
+      if (!availableFiles.has(file.parent)) {
+        for (
+          let currentDir = file.parent;
+          currentDir !== parentPath(currentDir) &&
+          !availableFiles.has(currentDir);
+          currentDir = parentPath(currentDir)
+        ) {
+          availableFiles.add(currentDir);
+          intermediateDirectories.push({
+            path: currentDir,
+            parent: parentPath(currentDir),
+            name: path.basename(currentDir),
+            type: "directory",
+            files_count: 0,
+          });
+        }
+        availableFiles.add(file.parent);
+      }
+    });
+    files.push(...intermediateDirectories);
+  }
+
   _batchCreateFiles(files: Resource[]) {
     // Add batched files to the DB
     return this._addFlattenedFiles(files).then(() => this._addFiles(files));

diff --git a/src/styles/toastify.css b/src/styles/toastify.css
@@ -0,0 +1,10 @@
+.Toastify__toast-body {
+  white-space: pre;
+  width: fit-content;
+  padding-right: 10px;
+}
+
+.Toastify__toast-container,
+.Toastify__toast-body>div:last-child {
+  width: max-content;
+}