impl runtime log (#535)

* fix(server): fix oss external endpoint in env

Signed-off-by: maslow <wangfugen@126.com>

* feat(runtime): implement func logging in node runtime

Signed-off-by: maslow <wangfugen@126.com>

Signed-off-by: maslow <wangfugen@126.com>

runtimes/nodejs/.env.template

@@ -1,5 +1,9 @@
-DB_URI=mongodb://root:password123@localhost:27017/?authSource=admin&replicaSet=laf&writeConcern=majority&directConnection=true
-SERVER_SALT=abcdefg1234567!@#$%^&sadfqwef&*^*#!@^
-LOG_LEVEL=trace
-ENABLE_CLOUD_FUNCTION_LOG = always
-FLAGS=--max_old_space_size=256
+DB_URI=mongodb://tgvsxb:wNE5eARjN5mJUXMDlBxUpoM12qk9B4JS1JKkukSkkM1oXj1KnCaX5pGVlXQ5oe4Q@mongo.laf-system.svc.cluster.local:27017/tgvsxb?authSource=tgvsxb&replicaSet=rs0&writeConcern=majority&directConnection=true
+APP_ID=tgvsxb
+OSS_ACCESS_KEY=tgvsxb
+OSS_ACCESS_SECRET=IlV2xfR2YEPM4ULvHhRLqBAR0sRaAGxbibtxjvbcS4uoLazV3b0NKyXSHNrcT8hi
+OSS_INTERNAL_ENDPOINT=laf-minio.laf-system.svc.cluster.local:9000
+OSS_EXTERNAL_ENDPOINT=laf-minio.laf-system.svc.cluster.local:9000
+OSS_REGION=default
+FLAGS=--max_old_space_size=204
+SERVER_SECRET=RJIDKwdN9psSqPILtpCC9lLmsrlvSEQUkWCjvFVd48yzJZmGl6Rq5jw0CjyuuzeO

runtimes/nodejs/.prettierrc

@@ -0,0 +1,5 @@
+{
+ "singleQuote": true,
+ "trailingComma": "all",
+ "semi": false
+}

runtimes/nodejs/package.json

@@ -38,11 +38,11 @@
  "fs-extra": "^9.1.0",
  "jsonwebtoken": "^8.5.1",
  "lodash": "^4.17.21",
- "log4js": "^6.3.0",
+ "log4js": "^6.7.1",
  "minio": "^7.0.28",
  "mongodb": "^4.1.3",
  "mongodb-uri": "^0.9.7",
- "multer": "^1.4.4",
+ "multer": "^1.4.5-lts.1",
  "node-modules-utils": "^0.8.2",
  "nodemailer": "^6.6.3",
  "validator": "^13.7.0",

runtimes/nodejs/src/cloud-sdk/index.ts

@@ -6,7 +6,6 @@ import { DatabaseAgent } from "../db";
 import request from "axios";
 import { SchedulerInstance } from "../support/scheduler";
 import { getToken, parseToken } from "../support/token";
-import { addFunctionLog } from "../support/function-log";
 import { WebSocket } from "ws";
 import { WebSocketAgent } from "../support/ws";
 import Config from "../config";
@@ -108,7 +107,7 @@ export interface CloudSdkInterface {
 
  env: {
  DB_URI?: string;
- SERVER_SECRET_SALT?: string;
+ SERVER_SECRET?: string;
  APP_ID?: string;
  OSS_ACCESS_KEY?: string;
  OSS_ACCESS_SECRET?: string;
@@ -155,7 +154,7 @@ export function create() {
  appid: Config.APP_ID,
  env: {
  DB_URI: Config.DB_URI,
- SERVER_SECRET_SALT: Config.SERVER_SECRET,
+ SERVER_SECRET: Config.SERVER_SECRET,
  APP_ID: process.env.APP_ID,
  OSS_ACCESS_KEY: process.env.APP_ID,
  OSS_ACCESS_SECRET: process.env.OSS_ACCESS_SECRET,
@@ -194,17 +193,6 @@ async function invokeInFunction(name: string, param?: FunctionContext) {
 
  const result = await func.invoke(param);
 
- await addFunctionLog({
- requestId: param.requestId,
- method: param.method,
- func_id: new mongodb.ObjectId(func.id),
- func_name: name,
- logs: result.logs,
- time_usage: result.time_usage,
- data: result.data,
- error: result.error,
- });
-
  if (result.error) {
  throw result.error;
  }

runtimes/nodejs/src/handler/debug-func.ts

@@ -8,8 +8,6 @@
 import { Request, Response } from 'express'
 import { FunctionContext } from '../support/function-engine'
 import { logger } from '../support/logger'
-import { addFunctionLog } from '../support/function-log'
-import { ObjectId } from 'bson'
 import { CloudFunction } from '../support/function-engine'
 
 /**
@@ -48,25 +46,10 @@ export async function handleDebugFunction(req: Request, res: Response) {
  }
  const result = await func.invoke(ctx)
 
- // log this execution to db
- await addFunctionLog({
- requestId: requestId,
- method: req.method,
- func_id: new ObjectId(func.id),
- func_name: func_name,
- logs: result.logs,
- time_usage: result.time_usage,
- created_by: req['auth']?.uid,
- data: result.data,
- error: result.error,
- debug: true
- })
-
  if (result.error) {
  logger.error(requestId, `debug function ${func_name} error: `, result)
  return res.send({
  error: 'invoke function got error: ' + result.error.toString(),
- logs: result.logs,
  time_usage: result.time_usage,
  requestId
  })

runtimes/nodejs/src/handler/invoke-func.ts

@@ -7,11 +7,8 @@
 
 import { Request, Response } from "express";
 import { FunctionContext } from "../support/function-engine";
-import Config from "../config";
 import { logger } from "../support/logger";
-import { addFunctionLog } from "../support/function-log";
 import { CloudFunction } from "../support/function-engine";
-import { ObjectId } from "mongodb";
 
 const DEFAULT_FUNCTION_NAME = "__default__";
 
@@ -58,22 +55,6 @@ export async function handleInvokeFunction(req: Request, res: Response) {
  };
  const result = await func.invoke(ctx);
 
- // log this execution to db
- if (Config.ENABLE_CLOUD_FUNCTION_LOG === "always") {
- await addFunctionLog({
- requestId: requestId,
- method: req.method,
- func_id: new ObjectId(func.id),
- func_name: func_name,
- logs: result.logs,
- time_usage: result.time_usage,
- created_by: req["auth"]?.uid,
- data: result.data,
- error: result.error,
- debug: false,
- });
- }
-
  if (result.error) {
  logger.error(
  requestId,

runtimes/nodejs/src/index.ts

@@ -15,9 +15,10 @@ import { generateUUID } from './support/utils'
 import { WebSocketAgent } from './support/ws'
 import { DatabaseAgent } from './db'
 import { SchedulerInstance } from './support/scheduler'
-// const xmlparser = require('express-xml-bodyparser')
 import * as xmlparser from 'express-xml-bodyparser'
 
+import "./support/function-log"
+
 /**
  * Just for generating declaration type files for `@/cloud-sdk` which used in cloud function
  */

runtimes/nodejs/src/support/function-engine/console.ts

@@ -1,17 +1,22 @@
 import * as util from 'util'
-import * as dayjs from 'dayjs'
 
 export class FunctionConsole {
- private _logs: string[] = []
+ requestId: string = ''
 
- get logs() {
- return this._logs
+ static write: (message: string, requestId: string) => void = console.log
+
+ constructor(requestId: string) {
+ this.requestId = requestId
  }
 
  private _log(...params: any[]) {
- const date = dayjs().format("YYYY/MM/DD HH:mm:ss")
- const r = util.format("[%s] -", date, ...params)
- this._logs.push(r)
+ const content = params
+ .map((param) => {
+ return util.inspect(param, { depth: 30 })
+ })
+ .join(' ')
+
+ FunctionConsole.write(content, this.requestId)
  }
 
  debug(...params: any[]) {

runtimes/nodejs/src/support/function-engine/engine.ts

@@ -4,8 +4,12 @@ import { URL } from 'node:url'
 import * as vm from 'vm'
 import { nanosecond2ms } from '../utils'
 import { FunctionConsole } from './console'
-import { FunctionContext, FunctionResult, RequireFuncType, RuntimeContext } from './types'
-
+import {
+ FunctionContext,
+ FunctionResult,
+ RequireFuncType,
+ RuntimeContext,
+} from './types'
 
 /**
  * Default require function
@@ -18,7 +22,6 @@ const defaultRequireFunction: RequireFuncType = (module): any => {
  * Function engine
  */
 export class FunctionEngine {
-
  require_func: RequireFuncType
 
  constructor(require_func?: RequireFuncType) {
@@ -27,9 +30,13 @@ export class FunctionEngine {
 
  /**
  * Execute function
- * @returns 
+ * @returns
  */
- async run(code: string, context: FunctionContext, options: vm.RunningScriptOptions): Promise<FunctionResult> {
+ async run(
+ code: string,
+ context: FunctionContext,
+ options: vm.RunningScriptOptions,
+ ): Promise<FunctionResult> {
  const sandbox = this.buildSandbox(context)
  const wrapped = this.wrap(code)
  const fconsole = sandbox.console
@@ -48,34 +55,30 @@ export class FunctionEngine {
  const time_usage = nanosecond2ms(_end_time - _start_time)
  return {
  data,
- logs: fconsole.logs,
- time_usage
+ time_usage,
  }
  } catch (error) {
- fconsole.log(error.message)
- fconsole.log(error.stack)
+ fconsole.log(error.message, error.stack)
 
  const _end_time = process.hrtime.bigint()
  const time_usage = nanosecond2ms(_end_time - _start_time)
 
  return {
  error: error,
- logs: fconsole.logs,
- time_usage
+ time_usage,
  }
  }
  }
 
-
  /**
  * build sandbox
- * @returns 
+ * @returns
  */
  buildSandbox(functionContext: FunctionContext): RuntimeContext {
- const fconsole = new FunctionConsole()
+ const fconsole = new FunctionConsole(functionContext.requestId)
 
  const _module = {
- exports: {}
+ exports: {},
  }
  const sandbox = {
  __context__: functionContext,
@@ -114,4 +117,4 @@ export class FunctionEngine {
  `
  return wrapped
  }
-}
+}

runtimes/nodejs/src/support/function-engine/trigger-scheduler.ts

@@ -6,9 +6,6 @@ import { Trigger } from "./trigger"
 import assert = require('node:assert')
 import WebSocket = require('ws')
 import { IncomingMessage } from 'node:http'
-import { addFunctionLog } from '../function-log'
-import { ObjectId } from 'mongodb'
-
 
 export class TriggerScheduler {
  private _triggers: Trigger[] = []
@@ -73,7 +70,7 @@ export class TriggerScheduler {
  method: 'trigger',
  requestId: `trigger_${tri.id}`
  }
- this.executeFunction(tri.func_id, param, tri)
+ this.executeFunction(tri.func_id, param)
  }
  }
 
@@ -111,29 +108,16 @@ export class TriggerScheduler {
  socket,
  headers: request?.headers
  }
- this.executeFunction(tri.func_id, param, tri)
+ this.executeFunction(tri.func_id, param)
  }
  }
 
  /**
  * Execute function
  */
- protected async executeFunction(func_id: string, param: FunctionContext, trigger: Trigger) {
+ protected async executeFunction(func_id: string, param: FunctionContext) {
  const func = await this.getFunctionById(func_id)
- const result = await func.invoke(param)
-
- // save function log
- result.logs.unshift(`invoked by trigger: ${trigger.name} (${trigger.id})`)
- await addFunctionLog({
- requestId: `trigger_${trigger.id}`,
- method: param.method,
- func_id: new ObjectId(func_id),
- func_name: func.name,
- logs: result.logs,
- time_usage: result.time_usage,
- created_by: `trigger_${trigger.id}`,
- trigger_id: trigger.id
- })
+ await func.invoke(param)
  }
 
  /**
@@ -188,7 +172,7 @@ export class TriggerScheduler {
  requestId: `trigger_${tri.id}`
  }
  // execute function
- this.executeFunction(tri.func_id, param, tri)
+ this.executeFunction(tri.func_id, param)
 
  // update last exec time
  tri.last_exec_time = Date.now()

runtimes/nodejs/src/support/function-engine/types.ts

@@ -45,11 +45,10 @@ export interface FunctionContext {
 }
 
 /**
- * 运行函数返回的结果对象
+ * Result object returned by the running function
  */
 export interface FunctionResult {
  data?: any;
- logs: any[];
  error?: Error;
  time_usage: number;
 }
@@ -89,26 +88,6 @@ export interface ICloudFunctionData {
  createdBy: string;
 }
 
-// export interface ICloudFunctionData {
-// _id: ObjectId
-// description: string
-// tags: string[]
-// label: string
-// name: string
-// triggers: any[]
-// version: number
-// hash: string
-// status: FunctionStatus
-// enableHTTP: boolean
-// appid: string
-// debugParams: string
-// code: string
-// compiledCode: string
-// created_at: Date
-// updated_at: Date
-// created_by: any
-// }
-
 /** Object containing file metadata and access information. */
 interface File {
  /** Name of the form field associated with this file. */