Merge pull request #312 from Canner/develop

Bump to 0.10.0

README.md

@@ -19,28 +19,32 @@
     <img alt="" src="https://img.shields.io/badge/-@vulcansql-orange?style=for-the-badge&logo=twitter&logoColor=white&labelColor=gray&logoWidth=20">
   </a>
   <a href="https://img.shields.io/codecov/c/gh/Canner/vulcan-sql" > 
-    <img alt="" src="https://img.shields.io/codecov/c/gh/Canner/vulcan-sql?style=for-the-badge&labelColor=gray&logoColor=white&logoWidth=20"/> 
+    <img alt="" src="https://img.shields.io/codecov/c/gh/Canner/vulcan-sql?style=for-the-badge&color=orange&labelColor=gray&logoColor=white&logoWidth=20"/> 
   </a>
 </p>
 
-## What is VulcanSQL
+## What is VulcanSQL?
 
-**[VulcanSQL](https://vulcansql.com/) is a Data API Framework for data applications** that helps data folks create and share data APIs faster. It turns your SQL templates into data APIs. No backend skills required.
+**[VulcanSQL](https://vulcansql.com/) is an Analytical Data API Framework for data apps**. It aims to help data professionals deliver RESTful APIs from databases, data warehouses or data lakes much easier and faster. It turns your SQL queries into REST APIs in no time!
 
 ![overview of VulcanSQL](https://i.imgur.com/JvCIZQ1.png)
 
-## Online Playground
+## What Problems does VulcanSQL aim to solve?
 
-Use [Online Playground](https://codesandbox.io/p/sandbox/vulcansql-demo-wfd834) to get a taste of VulcanSQL!
+Given the vast amount of analytical data in databases, data warehouses, and data lakes, there is currently no easy method for data professionals to share data with relevant stakeholders for operational business use cases.
 
-## Examples
+## Online Playground
 
-Need Inspiration?! Discover a [selected compilation of examples](https://github.com/Canner/vulcan-sql-examples) showcasing the use of VulcanSQL!
+Use [Online Playground](https://codesandbox.io/p/sandbox/vulcansql-demo-wfd834) to get a taste of VulcanSQL!
 
 ## Installation
 
 Please visit [the installation guide](https://vulcansql.com/docs/get-started/installation).
 
+## Examples
+
+Need inspiration? Here are a [selected compilation of examples](https://github.com/Canner/vulcan-sql-examples) showcasing how you can use VulcanSQL!
+
 ## How VulcanSQL works?
 
  💻 **Build**
@@ -49,7 +53,7 @@ VulcanSQL offers a development experience similar to dbt. Just insert variables
 
 🚀 **Accelerate**
 
-VulcanSQL uses DuckDB as a caching layer, boosting your query speed and API response time . This means faster, smoother data APIs for you and less strain on your data sources.
+VulcanSQL uses DuckDB as a caching layer, boosting your query speed and reducing API response time. This means faster, smoother data APIs for you and less strain on your data sources.
 
 🔥 **Deploy**
 
@@ -82,7 +86,7 @@ Below are some common scenarios that you may be interested:
 
 👏 **Data sharing**: Sharing data with partners, vendors, or customers, which requires a secure and scalable way to expose data.
 
-⚙️ **Internal tools**: Integration with internal tools like AppSmith and Retools, etc.
+⚙️ **Internal tools**: Integration with internal tools like Zapier, AppSmith and Retools, etc.
 
 ## Community
 

codecov.yml

@@ -3,3 +3,13 @@ flag_management:
     # Reference of past coverage for tests that are not run on current commit.
     # https://docs.codecov.com/docs/carryforward-flags
     carryforward: true
+coverage:
+  status:
+    patch: false
+    project:
+      default:
+        target: auto
+        threshold: "80%"
+        base: auto 
+        flags: 
+          - unit

packages/build/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@vulcan-sql/build",
   "description": "VulcanSQL package for building projects",
-  "version": "0.9.1",
+  "version": "0.10.0",
   "type": "commonjs",
   "publishConfig": {
     "access": "public"
@@ -22,6 +22,6 @@
   },
   "license": "Apache-2.0",
   "peerDependencies": {
-    "@vulcan-sql/core": "~0.9.1-0"
+    "@vulcan-sql/core": "~0.10.0-0"
   }
 }

packages/catalog-server/lib/api.ts

@@ -1,5 +1,8 @@
 import axios from 'axios';
 import { errorCode } from '@vulcan-sql/catalog-server/utils/errorCode';
+import getConfig from 'next/config';
+
+const { publicRuntimeConfig } = getConfig();
 
 enum API {
   Login = '/api/auth/login',
@@ -22,7 +25,7 @@ const handleError = ({ statusCode, errorMessage }) => {
 };
 
 export const axiosInstance = axios.create({
-  baseURL: process.env.API_URL || 'http://localhost:4200',
+  baseURL: publicRuntimeConfig.baseUrl,
   responseType: 'json',
   timeout: 30000,
   headers: {

packages/catalog-server/lib/apollo.ts

@@ -7,7 +7,7 @@ import {
 import { setContext } from '@apollo/client/link/context';
 
 const httpLink = createHttpLink({
-  uri: process.env.GQL_API_URL || 'http://localhost:4200/api/graphql',
+  uri: '/api/graphql',
 });
 
 const authLink = setContext((_, { headers }) => {

packages/catalog-server/next.config.js

@@ -12,6 +12,9 @@ const nextConfig = {
   compiler: {
     styledComponents: true,
   },
+  publicRuntimeConfig: {
+    baseUrl: process.env.BASE_URL || 'http://localhost:4200',
+  },
   serverRuntimeConfig: {
     // Will only be available on the server side
     vulcanSQLHost: process.env.VULCAN_SQL_HOST || 'http://localhost:3000',

packages/catalog-server/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@vulcan-sql/catalog-server",
   "description": "Catalog server for VulcanSQL",
-  "version": "0.9.1",
+  "version": "0.10.0",
   "publishConfig": {
     "access": "public"
   },

packages/cli/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@vulcan-sql/cli",
   "description": "CLI tools for VulcanSQL",
-  "version": "0.9.1",
+  "version": "0.10.0",
   "type": "commonjs",
   "bin": {
     "vulcan": "./src/index.js"

packages/core/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@vulcan-sql/core",
   "description": "Core package of VulcanSQL",
-  "version": "0.9.1",
+  "version": "0.10.0",
   "type": "commonjs",
   "publishConfig": {
     "access": "public"

packages/core/src/containers/modules/extension.ts

@@ -3,6 +3,7 @@ import { ExtensionLoader } from '../../lib/extension-loader';
 import { ICoreOptions } from '../../models/coreOptions';
 import templateEngineModules from '../../lib/template-engine/built-in-extensions';
 import validatorModule from '../../lib/validators/built-in-validators';
+import LoggerModule from '../../lib/loggers';
 import {
   builtInCodeLoader,
   builtInTemplateProvider,
@@ -23,6 +24,7 @@ export const extensionModule = (options: ICoreOptions) =>
     for (const templateEngineModule of templateEngineModules) {
       loader.loadInternalExtensionModule(templateEngineModule);
     }
+    loader.loadInternalExtensionModule(LoggerModule);
     // Validator (single module)
     loader.loadInternalExtensionModule(validatorModule);
     // Template provider (single module)

packages/core/src/containers/types.ts

@@ -52,4 +52,6 @@ export const TYPES = {
   Extension_CompilerLoader: Symbol.for('Extension_CompilerLoader'),
   Extension_DataSource: Symbol.for('Extension_DataSource'),
   Extension_ProfileReader: Symbol.for('ProfileReader'),
+  // Logger
+  Extension_ActivityLogger: Symbol.for('Extension_ActivityLogger'),
 };

packages/core/src/index.ts

@@ -1,5 +1,6 @@
 export * from './lib/utils';
 export * from './lib/validators';
+export * from './lib/loggers';
 export * from './lib/template-engine';
 export * from './lib/artifact-builder';
 export * from './lib/data-query';

packages/core/src/lib/cache-layer/cacheLayerLoader.ts

@@ -22,7 +22,6 @@ export class CacheLayerLoader implements ICacheLayerLoader {
   private options: ICacheLayerOptions;
   private cacheStorage: DataSource;
   private logger = getLogger({ scopeName: 'CORE' });
-
   constructor(
     @inject(TYPES.CacheLayerOptions) options: CacheLayerOptions,
     @inject(TYPES.Factory_DataSource)
@@ -43,7 +42,14 @@ export class CacheLayerLoader implements ICacheLayerLoader {
     templateName: string,
     cache: CacheLayerInfo
   ): Promise<void> {
-    const { cacheTableName, sql, profile, indexes, folderSubpath } = cache;
+    const {
+      cacheTableName,
+      sql,
+      profile,
+      indexes,
+      folderSubpath,
+      options: cacheOptions,
+    } = cache;
     const type = this.options.type!;
     const dataSource = this.dataSourceFactory(profile);
 
@@ -82,6 +88,7 @@ export class CacheLayerLoader implements ICacheLayerLoader {
         directory,
         profileName: profile,
         type,
+        options: cacheOptions,
       });
     } else {
       this.logger.debug(

packages/core/src/lib/cache-layer/cacheLayerRefresher.ts

@@ -1,12 +1,24 @@
 import ms, { StringValue } from 'ms';
 import { uniq } from 'lodash';
 import { ToadScheduler, SimpleIntervalJob, AsyncTask } from 'toad-scheduler';
-import { inject, injectable } from 'inversify';
+import { inject, injectable, multiInject } from 'inversify';
 import { TYPES } from '@vulcan-sql/core/types';
-import { APISchema } from '@vulcan-sql/core/models';
+import {
+  APISchema,
+  ActivityLogContentOptions,
+  ActivityLogType,
+  CacheLayerInfo,
+  IActivityLogger,
+} from '@vulcan-sql/core/models';
 import { ConfigurationError } from '../utils/errors';
 import { ICacheLayerLoader } from './cacheLayerLoader';
+import { getLogger } from '../utils';
+import moment = require('moment');
 
+enum RefreshResult {
+  SUCCESS = 'SUCCESS',
+  FAILED = 'FAILED',
+}
 export interface ICacheLayerRefresher {
   /**
    * Start the job to load the data source to cache storage and created tables from cache settings in schemas
@@ -22,9 +34,16 @@ export interface ICacheLayerRefresher {
 export class CacheLayerRefresher implements ICacheLayerRefresher {
   private cacheLoader: ICacheLayerLoader;
   private scheduler = new ToadScheduler();
+  private activityLoggers: IActivityLogger[];
+  private logger = getLogger({ scopeName: 'CORE' });
 
-  constructor(@inject(TYPES.CacheLayerLoader) loader: ICacheLayerLoader) {
+  constructor(
+    @inject(TYPES.CacheLayerLoader) loader: ICacheLayerLoader,
+    @multiInject(TYPES.Extension_ActivityLogger)
+    activityLoggers: IActivityLogger[]
+  ) {
     this.cacheLoader = loader;
+    this.activityLoggers = activityLoggers;
   }
 
   public async start(
@@ -53,16 +72,14 @@ export class CacheLayerRefresher implements ICacheLayerRefresher {
               const refreshJob = new SimpleIntervalJob(
                 { milliseconds, runImmediately },
                 new AsyncTask(workerId, async () => {
-                  // load data the to cache storage
-
-                  await this.cacheLoader.load(templateName, cache);
+                  await this.loadCacheAndSendActivityLog(schema, cache);
                 }),
                 { preventOverrun: true, id: workerId }
               );
               // add the job to schedule cache refresh task
               this.scheduler.addIntervalJob(refreshJob);
             } else {
-              await this.cacheLoader.load(templateName, cache);
+              await this.loadCacheAndSendActivityLog(schema, cache);
             }
           })
         );
@@ -77,6 +94,44 @@ export class CacheLayerRefresher implements ICacheLayerRefresher {
     this.scheduler.stop();
   }
 
+  private async loadCacheAndSendActivityLog(
+    schema: APISchema,
+    cache: CacheLayerInfo
+  ) {
+    const { urlPath } = schema;
+    const { sql } = cache;
+    let refreshResult = RefreshResult.SUCCESS;
+    const now = moment.utc().format('YYYY-MM-DD HH:mm:ss');
+    const templateName = schema.templateSource.replace('/', '_');
+    try {
+      // get the current time in format of UTC
+      await this.cacheLoader.load(templateName, cache);
+    } catch (error: any) {
+      refreshResult = RefreshResult.FAILED;
+      this.logger.debug(`Failed to refresh cache: ${error}`);
+    } finally {
+      // send activity log
+      const content = {
+        isSuccess: refreshResult === RefreshResult.SUCCESS ? true : false,
+        activityLogType: ActivityLogType.CACHE_REFRESH,
+        logTime: now,
+        urlPath,
+        sql,
+      } as ActivityLogContentOptions;
+      const activityLoggers = this.getActivityLoggers();
+      for (const activityLogger of activityLoggers)
+        activityLogger.log(content).catch((err: any) => {
+          this.logger.debug(
+            `Failed to log activity after refreshing cache: ${err}`
+          );
+        });
+    }
+  }
+
+  private getActivityLoggers(): IActivityLogger[] {
+    return this.activityLoggers.filter((logger) => logger.isEnabled());
+  }
+
   private checkDuplicateCacheTableName(schemas: APISchema[]) {
     const tableNames = schemas
       // => [[table1, table2], [table1, table3], [table4]]