feat: support datapack export based on configuration definitions in a…

… YAML definitions file

Change default service type to transient

packages/cli/package.json

@@ -54,6 +54,7 @@
  "homepage": "https://github.com/Codeneos/vlocode#readme",
  "devDependencies": {
  "@types/jest": "^29.5.11",
+ "@types/js-yaml": "^4.0.2",
  "@types/node": "^20.4.2",
  "@vlocode/apex": "workspace:*",
  "@vlocode/core": "workspace:*",
@@ -67,6 +68,7 @@
  "esbuild-loader": "^4.0.3",
  "glob": "^7.1.7",
  "jest": "^29.7.0",
+ "js-yaml": "^4.1.0",
  "log-symbols": "^4.0.0",
  "shx": "^0.3.4",
  "source-map-support": "^0.5.21",

packages/cli/src/commands/deploy.ts

@@ -2,6 +2,7 @@ import logSymbols from 'log-symbols';
 import { join } from 'path';
 import chalk from 'chalk';
 import { existsSync } from 'fs';
+import { stat } from 'fs/promises';
 
 import { Logger, LogLevel, LogManager } from '@vlocode/core';
 import { DatapackDeployer, ForkedSassCompiler, DatapackDeploymentOptions } from '@vlocode/vlocity-deploy';
@@ -16,7 +17,7 @@ export default class extends SalesforceCommand {
  static description = 'Deploy datapacks to Salesforce';
 
  static args = [
- new Argument('<folders...>', 'path to a folder containing the datapacks to be deployed').argParser((value, previous: string[] | undefined) => {
+ new Argument('<paths..>', 'path of the folders containing the datapacks or datapack files to be deployed').argParser((value, previous: string[] | undefined) => {
  if (!existsSync(value)) {
  throw new Error('No such folder exists');
  }
@@ -77,9 +78,9 @@ export default class extends SalesforceCommand {
  super();
  }
 
- public async run(folders: string[], options: any) {
+ public async run(paths: string[], options: any) {
  // Load datapacks
- const datapacks = await this.loadDatapacksFromFolders(folders);
+ const datapacks = await this.loadDatapacks(paths);
  if (!datapacks.length) {
  return;
  }
@@ -132,15 +133,22 @@ export default class extends SalesforceCommand {
  }
  }
 
- private async loadDatapacksFromFolders(folders: string[]) {
- this.logger.info(`Load datapacks from: "${folders.join('", "')}"`);
+ private async loadDatapacks(paths: string[]) {
+ this.logger.info(`Load datapacks: "${paths.join('", "')}"`);
 
  const datapackLoadTimer = new Timer();
  const loader = this.container.get(DatapackLoader);
- const datapacks = (await mapAsync(folders, folder => loader.loadDatapacksFromFolder(folder))).flat();
+ const datapacks = (await mapAsync(paths, async path => {
+ const fileInfo = await stat(path);
+ if (fileInfo.isDirectory()) {
+ return loader.loadDatapacksFromFolder(path);
+ } else {
+ return [ await loader.loadDatapack(path) ];
+ }
+ })).flat();
 
  if (datapacks.length == 0) {
- this.logger.error(`No datapacks found in specified folders: "${folders.join('", "')}"`);
+ this.logger.error(`No datapacks found in specified paths: "${paths.join('", "')}"`);
  } else {
  this.logger.info(`Loaded ${datapacks.length} datapacks in [${datapackLoadTimer.stop()}]`);
  }

packages/cli/src/commands/export.ts

@@ -0,0 +1,92 @@
+import { join } from 'path';
+import * as fs from 'fs-extra';
+import * as yaml from 'js-yaml';
+
+import { Logger, LogManager } from '@vlocode/core';
+import { DatapackExpander, DatapackExportDefinitionStore, DatapackExporter } from '@vlocode/vlocity-deploy';
+
+import { Argument, Option } from '../command';
+import { SalesforceCommand } from '../salesforceCommand';
+
+export default class extends SalesforceCommand {
+
+ static description = 'Export an objecr as datapack from Salesforce';
+
+ static args = [
+ new Argument('<ids...>', 'list of object IDs to export')
+ ];
+
+ static options = [
+ ...SalesforceCommand.options,
+ new Option(
+ '-d, --export-definitions <file>', 
+ 'path of the YAML or JSON file containing the export definitions that define how objects are exported'
+ ).argParser((value) => {
+ if (!fs.existsSync(value)) {
+ throw new Error('Specified definitions file does not exists');
+ }
+ return value;
+ }),
+ new Option(
+ '-e, --expand',
+ 'expand the datapack once exported into separate files according to the export definitions'
+ )
+ ];
+
+ constructor(private logger: Logger = LogManager.get('datapack-export')) {
+ super();
+ }
+
+ public async run(ids: string[], options: any) {
+ const definitions = await this.loadDefinitions(options.exportDefinitions);
+ this.container.get(DatapackExportDefinitionStore).load(definitions);
+
+ const exporter = this.container.create(DatapackExporter);
+ const expander = this.container.create(DatapackExpander);
+
+ for (const id of ids) {
+ const result = await exporter.exportObject(id);
+ if (options.expand) {
+ const expanded = expander.expandDatapack(result.datapack);
+ for (const [fileName, fileData] of Object.entries(expanded.files)) {
+ await this.writeFile(
+ join(expanded.objectType, expanded.folder ?? id, fileName),
+ fileData
+ );
+ }
+ } else {
+ const baseName = result.datapack.VlocityRecordSObjectType + '_' + id;
+ await this.writeFile(baseName + '_DataPack.json', result.datapack);
+ await this.writeFile(baseName + '_ParentKeys.json', result.parentKeys);
+ }
+ }
+ }
+
+ public async writeFile(fileName: string, data: object | string | Buffer) {
+ if (typeof data === 'object' && !Buffer.isBuffer(data)) {
+ data = JSON.stringify(data, null, 4);
+ }
+
+ try {
+ await fs.outputFile(fileName, data);
+ this.logger.info(`Output file: ${fileName}`);
+ } catch (err) {
+ this.logger.warn(`Failed to write file ${fileName}: ${err.message}`);
+ }
+ }
+
+ public async loadDefinitions(filePath: string) {
+ this.logger.info(`Loading export definitions from ${filePath}`);
+ try {
+ if (/\.json$/i.test(filePath)) {
+ return fs.readJson(filePath, { encoding: 'utf-8' });
+ } else if (/\.ya?ml$/i.test(filePath)) {
+ return yaml.load(await fs.readFile(filePath, { encoding: 'utf-8' }));
+ } else {
+ throw new Error('Unsupported file format, expected a YAML or JSON file');
+ }
+ } catch (err) {
+ this.logger.error(`Failed to load export definitions from ${filePath}: ${err.message}`);
+ }
+ }
+}

packages/core/src/container.ts

@@ -1,5 +1,5 @@
 import 'reflect-metadata';
-import { singleton, Iterable, arrayMapPush, asArray, getParameterTypes, getPropertyType } from '@vlocode/util';
+import { singleton, Iterable, arrayMapPush, asArray, getParameterTypes, getPropertyKey } from '@vlocode/util';
 import { uniqueNamesGenerator, Config as uniqueNamesGeneratorConfig, adjectives, animals } from 'unique-names-generator';
 import { LogManager } from './logging';
 import { createServiceProxy, serviceIsResolved, proxyTarget, isServiceProxy, ProxyTarget } from './serviceProxy';
@@ -58,7 +58,7 @@ export const ServiceGuidSymbol = Symbol('Container:ServiceGuid');
 const ContainerRootSymbol = Symbol('Container:IsRoot');
 
 const defaultServiceOptions: Readonly<ServiceOptions> = Object.seal({
- lifecycle: LifecyclePolicy.singleton,
+ lifecycle: LifecyclePolicy.transient,
  priority: 0
 });
 
@@ -381,7 +381,7 @@ export class Container {
  continue;
  }
  // Resolve property to an actual service
- const typeInfo = getPropertyType(prototype, property);
+ const typeInfo = getPropertyKey(prototype, property);
  if (!typeInfo) {
  throw new Error('Code compiled with emitting required type metadata');
  }

packages/core/src/fs/nodeFileSystem.ts

@@ -1,13 +1,13 @@
 import * as fs from 'fs';
 import * as path from 'path';
 import globby from 'globby';
-import { injectable, LifecyclePolicy } from '../index';
+import { injectable } from '../index';
 import { FileInfo, FileStat, FileSystem, StatsOptions, WriteOptions } from './fileSystem';
 
 /**
  * Basic class that can wrap any API implementation the NodeJS FS API into a more reduced FileSystem interface.
  */
-@injectable({ provides: FileSystem, lifecycle: LifecyclePolicy.singleton })
+@injectable.singleton({ provides: FileSystem })
 export class NodeFileSystem extends FileSystem {
 
  constructor(protected readonly innerFs: typeof fs = fs) {

packages/omniscript/src/omniScriptLwcCompiler.ts

@@ -28,7 +28,7 @@ interface OmniCompiler {
 /**
  * Compiler that transforms activated OmniScripts into LWC components that can easily be deployed or written to the disk.
  */
-@injectable()
+@injectable.singleton()
 export class OmniScriptLwcCompiler {
 
  private readonly lwcCompilerResource = 'OmniscriptLwcCompiler';

packages/omniscript/src/scriptDefinitionProvider.ts

@@ -3,7 +3,7 @@ import { injectable } from '@vlocode/core';
 import { OmniScriptDefinition, OmniScriptSpecification } from './types';
 import { OmniScriptDefinitionProvider } from './omniScriptDefinitionProvider';
 
-@injectable()
+@injectable.singleton()
 export class ScriptDefinitionProvider implements OmniScriptDefinitionProvider {
  constructor(
  private readonly queryService: QueryService

packages/salesforce/src/connection/metadata/metadataSchemas.ts

@@ -6006,8 +6006,8 @@ const ExperienceBundleSettingsSchema = {
  }
 }
 
-const ExperiencePropertyTypeBundleSchema = {
- name: 'ExperiencePropertyTypeBundle',
+const ExperiencePropertyKeyBundleSchema = {
+ name: 'ExperiencePropertyKeyBundle',
  extends: 'Metadata',
  fields: {
  defaultDesignConfigMCTBody: { array: false, nullable: false, optional: true, type: 'string' },
@@ -14624,7 +14624,7 @@ export const Schemas: Record<string, Schema> = {
  'ExperienceResources': ExperienceResourcesSchema,
  'ExperienceResource': ExperienceResourceSchema,
  'ExperienceBundleSettings': ExperienceBundleSettingsSchema,
- 'ExperiencePropertyTypeBundle': ExperiencePropertyTypeBundleSchema,
+ 'ExperiencePropertyKeyBundle': ExperiencePropertyKeyBundleSchema,
  'ExplainabilityActionDefinition': ExplainabilityActionDefinitionSchema,
  'ExplainabilityActionVersion': ExplainabilityActionVersionSchema,
  'ExternalClientAppSettings': ExternalClientAppSettingsSchema,

packages/salesforce/src/connection/metadata/metadataTypes.ts

@@ -228,7 +228,7 @@ export interface MetadataTypes {
  'EventSettings': Metadata.EventSettingsMetadata;
  'ExperienceBundle': Metadata.ExperienceBundleMetadata;
  'ExperienceBundleSettings': Metadata.ExperienceBundleSettingsMetadata;
- 'ExperiencePropertyTypeBundle': Metadata.ExperiencePropertyTypeBundleMetadata;
+ 'ExperiencePropertyKeyBundle': Metadata.ExperiencePropertyKeyBundleMetadata;
  'ExplainabilityActionDefinition': Metadata.ExplainabilityActionDefinitionMetadata;
  'ExplainabilityActionVersion': Metadata.ExplainabilityActionVersionMetadata;
  'ExternalClientAppSettings': Metadata.ExternalClientAppSettingsMetadata;

packages/salesforce/src/connection/providers/interactiveConnectionProvider.ts

@@ -4,7 +4,7 @@ import { SfdxConnectionProvider } from './sfdxConnectionProvider';
 import { injectable, Logger } from '@vlocode/core';
 import { SalesforceConnection, SalesforceConnectionOptions } from '../salesforceConnection';
 
-@injectable()
+@injectable.singleton()
 export class InteractiveConnectionProvider implements SalesforceConnectionProvider {
 
  private sfdxProvider: SalesforceConnectionProvider;

packages/salesforce/src/deploymentPackageBuilder.ts

@@ -2,7 +2,7 @@ import * as path from 'path';
 import chalk from 'chalk';
 import ZipArchive from 'jszip';
 
-import { Logger, injectable , LifecyclePolicy, CachedFileSystemAdapter , FileSystem, Container, container } from '@vlocode/core';
+import { Logger, injectable, CachedFileSystemAdapter , FileSystem, Container, container } from '@vlocode/core';
 import { cache, substringAfterLast , Iterable, XML, CancellationToken, FileSystemUri, substringBeforeLast, stringEquals } from '@vlocode/util';
 
 import { PackageManifest } from './deploy/packageXml';
@@ -49,7 +49,7 @@ interface MetadataObject {
  data: Record<string, unknown>;
 }
 
-@injectable( { lifecycle: LifecyclePolicy.transient } )
+@injectable()
 export class SalesforcePackageBuilder {
 
  /**

packages/salesforce/src/metadataRegistry.ts

@@ -30,7 +30,7 @@ export interface MetadataType extends RegistryMetadataType {
 }
 
 @singletonMixin
-@injectable( { lifecycle: LifecyclePolicy.singleton } )
+@injectable.singleton()
 export class MetadataRegistry {
 
  private readonly registry = new Array<MetadataType>();

packages/salesforce/src/namespaceService.ts

@@ -1,7 +1,7 @@
 import { injectable } from "@vlocode/core";
 import { visitObject } from "@vlocode/util";
 
-@injectable()
+@injectable.singleton()
 export class NamespaceService {
  /**
  * Replaces a namespace place holder with the actual namespace in the target org

packages/salesforce/src/queryRecordFactory.ts

@@ -213,7 +213,7 @@ export class RecordFactory {
  }
 }
 
-@injectable({ lifecycle: LifecyclePolicy.transient })
+@injectable()
 export class Query2Service {
  // private wrapRecord<T extends object>(record: T) {
  // const getPropertyKey = (target: T, name: string | number | symbol) => {

packages/salesforce/src/queryService.ts

@@ -1,14 +1,14 @@
 import { DateTime } from 'luxon';
 import { Logger, injectable, LifecyclePolicy } from '@vlocode/core';
-import { CancellationToken, PropertyAccessor } from '@vlocode/util';
+import { CancellationToken } from '@vlocode/util';
 
 import { SalesforceConnectionProvider } from './connection';
 import { SObjectRecord, FieldType } from './types';
 import { NamespaceService } from './namespaceService';
 import { QueryCache } from './queryCache';
 import { RecordFactory } from './queryRecordFactory';
 
-export type QueryResult<TBase, TProps extends PropertyAccessor = any> = TBase & SObjectRecord & { [P in TProps]: any; };
+export type QueryResult<TBase, TProps extends PropertyKey = any> = TBase & SObjectRecord & { [P in TProps]: any; };
 
 export interface QueryOptions {
  /**
@@ -56,19 +56,20 @@ export class QueryService {
  }
 
  /**
- * Disables the query cache functions; overrides useCache and cacheDefault disabling all caching.
- * @param disabled Disable or enable query cache;
+ * Sets the query cache options. When the query cache is enabled, the query results are stored in memory and reused when the same query is executed again.
+ * When changing cache state the current cache is cleared.
+ * @param options - The query cache options.
+ * @param options.enabled - Indicates whether the query cache is enabled.
+ * @param options.default - Indicates what the default behavior should be when the useCache paramter is not explicitly set by the caller.
+ * @returns The current instance of the QueryService.
  */
- public disableCache(disabled: boolean): this {
- this.queryCacheEnabled = !disabled;
- return this;
- }
-
- /**
- * Changes the default behavior for caching queries, when true if no explicit useCache parameters are passed the caching is decided based on the default cache parameter;
- */
- public setCacheDefault(enabled: boolean): this {
- this.queryCacheDefault = enabled;
+ public setQueryCache(options: { enabled: boolean, default?: boolean }): this {
+ if (options.enabled !== this.queryCacheEnabled) {
+ this.logger.verbose(`Query cache ${options.enabled ? 'enabled' : 'disabled'}`);
+ this.clearCache();
+ }
+ this.queryCacheEnabled = options.enabled === true;
+ this.queryCacheDefault = options.default ?? false;
  return this;
  }
 
@@ -86,7 +87,7 @@ export class QueryService {
  * @param query Query string
  * @param useCache Store the query in the internal query cache or retrieve the cached version of the response if it exists
  */
- public query<T extends object = object, K extends PropertyAccessor = keyof T>(query: string, useCache?: boolean, cancelToken?: CancellationToken) : Promise<QueryResult<T, K>[]> {
+ public query<T extends object = object, K extends PropertyKey = keyof T>(query: string, useCache?: boolean, cancelToken?: CancellationToken) : Promise<QueryResult<T, K>[]> {
  return this.execute(query, { cache: useCache, cancelToken });
  }
 
@@ -95,11 +96,11 @@ export class QueryService {
  * @param query Query string
  * @param useCache Store the query in the internal query cache or retrieve the cached version of the response if it exists
  */
- public queryTooling<T extends object = object, K extends PropertyAccessor = keyof T>(query: string, useCache?: boolean, cancelToken?: CancellationToken) : Promise<QueryResult<T, K>[]> {
+ public queryTooling<T extends object = object, K extends PropertyKey = keyof T>(query: string, useCache?: boolean, cancelToken?: CancellationToken) : Promise<QueryResult<T, K>[]> {
  return this.execute(query, { cache: useCache, cancelToken, toolingApi: true });
  }
 
- public execute<T extends object = object, K extends PropertyAccessor = keyof T>(query: string, options?: QueryOptions) : Promise<QueryResult<T, K>[]> {
+ public execute<T extends object = object, K extends PropertyKey = keyof T>(query: string, options?: QueryOptions) : Promise<QueryResult<T, K>[]> {
  const nsNormalizedQuery = this.nsService?.updateNamespace(query) ?? query;
  const enableCache = this.queryCacheEnabled && (options?.cache ?? this.queryCacheDefault);
  const queryExecutor = async () => {