From 286ceaaf84eebc52aecd7bc1611bb37e9fef59bb Mon Sep 17 00:00:00 2001 From: Franklin Koch Date: Wed, 5 Jun 2024 10:36:25 -0600 Subject: [PATCH] =?UTF-8?q?=F0=9F=8C=8E=20Fetch=20config=20extensions=20fr?= =?UTF-8?q?om=20URL=20(#1251)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🌎 Fetch config fields from URL * 🔧 Fix cache path construction * 🔧 Make remote config paths an option --- .changeset/hot-actors-destroy.md | 6 + packages/myst-cli/src/build/build.ts | 5 +- packages/myst-cli/src/build/init.ts | 6 +- packages/myst-cli/src/build/legacy.ts | 14 +- packages/myst-cli/src/build/site/manifest.ts | 40 +-- packages/myst-cli/src/build/site/start.ts | 2 +- .../src/build/utils/collectExportOptions.ts | 2 +- .../src/build/utils/localArticleExport.ts | 6 +- packages/myst-cli/src/config.ts | 263 +++++++++++------- packages/myst-cli/src/process/site.ts | 2 +- packages/myst-cli/src/project/load.ts | 21 +- packages/myst-cli/src/project/toTOC.ts | 2 +- packages/myst-cli/src/project/toc.spec.ts | 4 +- packages/myst-cli/src/session/session.spec.ts | 4 +- packages/myst-cli/src/session/session.ts | 12 +- packages/myst-cli/src/session/types.ts | 4 +- packages/mystmd/src/clirun.ts | 1 + 17 files changed, 238 insertions(+), 156 deletions(-) create mode 100644 .changeset/hot-actors-destroy.md diff --git a/.changeset/hot-actors-destroy.md b/.changeset/hot-actors-destroy.md new file mode 100644 index 000000000..2eafa6b9d --- /dev/null +++ b/.changeset/hot-actors-destroy.md @@ -0,0 +1,6 @@ +--- +'myst-cli': patch +'mystmd': patch +--- + +Fetch config files from url diff --git a/packages/myst-cli/src/build/build.ts b/packages/myst-cli/src/build/build.ts index 3c3481f69..3849df053 100644 --- a/packages/myst-cli/src/build/build.ts +++ b/packages/myst-cli/src/build/build.ts @@ -126,7 +126,10 @@ export async function collectAllBuildExportOptions( throw new Error(`When specifying output, you can only request one format`); } let exportOptionsList: ExportWithInputOutput[]; - const projectPath = findCurrentProjectAndLoad(session, files[0] ? path.dirname(files[0]) : '.'); + const projectPath = await findCurrentProjectAndLoad( + session, + files[0] ? path.dirname(files[0]) : '.', + ); if (projectPath) await loadProjectFromDisk(session, projectPath); if (output) { session.log.debug(`Exporting formats: "${requestedFormats.join('", "')}"`); diff --git a/packages/myst-cli/src/build/init.ts b/packages/myst-cli/src/build/init.ts index ffe8ceed8..46c0d5c7e 100644 --- a/packages/myst-cli/src/build/init.ts +++ b/packages/myst-cli/src/build/init.ts @@ -69,7 +69,7 @@ export async function init(session: ISession, opts: InitOptions) { if (!project && !site && !writeTOC) { session.log.info(WELCOME()); } - loadConfig(session, '.'); + await loadConfig(session, '.'); const state = session.store.getState(); const existingRawConfig = selectors.selectLocalRawConfig(state, '.'); const existingProjectConfig = selectors.selectLocalProjectConfig(state, '.'); @@ -96,7 +96,7 @@ export async function init(session: ISession, opts: InitOptions) { } if (siteConfig || projectConfig) { session.log.info(`💾 Updating config file: ${existingConfigFile}`); - writeConfigs(session, '.', { siteConfig, projectConfig }); + await writeConfigs(session, '.', { siteConfig, projectConfig }); } } else { // If no config is present, write it explicitly to include comments. @@ -119,7 +119,7 @@ export async function init(session: ISession, opts: InitOptions) { fs.writeFileSync(configFile, configData); } if (writeTOC) { - loadConfig(session, '.'); + await loadConfig(session, '.'); await loadProjectFromDisk(session, '.', { writeTOC }); } // If we have any options, this command is complete! diff --git a/packages/myst-cli/src/build/legacy.ts b/packages/myst-cli/src/build/legacy.ts index 491099af2..0820cdf3e 100644 --- a/packages/myst-cli/src/build/legacy.ts +++ b/packages/myst-cli/src/build/legacy.ts @@ -47,7 +47,7 @@ export async function localArticleToWord( extraLinkTransformers?: LinkTransformer[], ): Promise { let { projectPath } = opts; - if (!projectPath) projectPath = findCurrentProjectAndLoad(session, path.dirname(file)); + if (!projectPath) projectPath = await findCurrentProjectAndLoad(session, path.dirname(file)); if (projectPath) await loadProjectFromDisk(session, projectPath); const exportOptionsList = ( await legacyCollectExportOptions(session, file, 'docx', [ExportFormats.docx], projectPath, opts) @@ -81,7 +81,7 @@ export async function localArticleToJats( extraLinkTransformers?: LinkTransformer[], ) { let { projectPath } = opts; - if (!projectPath) projectPath = findCurrentProjectAndLoad(session, path.dirname(file)); + if (!projectPath) projectPath = await findCurrentProjectAndLoad(session, path.dirname(file)); if (projectPath) await loadProjectFromDisk(session, projectPath); const exportOptionsList = ( await legacyCollectExportOptions(session, file, 'xml', [ExportFormats.xml], projectPath, opts) @@ -115,7 +115,7 @@ export async function localArticleToMd( extraLinkTransformers?: LinkTransformer[], ) { let { projectPath } = opts; - if (!projectPath) projectPath = findCurrentProjectAndLoad(session, path.dirname(file)); + if (!projectPath) projectPath = await findCurrentProjectAndLoad(session, path.dirname(file)); if (projectPath) await loadProjectFromDisk(session, projectPath); const exportOptionsList = ( await legacyCollectExportOptions(session, file, 'md', [ExportFormats.md], projectPath, opts) @@ -146,7 +146,7 @@ export async function localProjectToMeca( extraLinkTransformers?: LinkTransformer[], ) { let { projectPath } = opts; - if (!projectPath) projectPath = findCurrentProjectAndLoad(session, path.dirname(file)); + if (!projectPath) projectPath = await findCurrentProjectAndLoad(session, path.dirname(file)); if (projectPath) await loadProjectFromDisk(session, projectPath); const exportOptionsList = ( await legacyCollectExportOptions(session, file, 'zip', [ExportFormats.meca], projectPath, opts) @@ -179,7 +179,7 @@ export async function localArticleToPdf( templateOptions?: Record, ): Promise { let { projectPath } = opts; - if (!projectPath) projectPath = findCurrentProjectAndLoad(session, path.dirname(file)); + if (!projectPath) projectPath = await findCurrentProjectAndLoad(session, path.dirname(file)); if (projectPath) await loadProjectFromDisk(session, projectPath); const pdfExportOptionsList = ( await legacyCollectExportOptions( @@ -249,7 +249,7 @@ export async function localArticleToTex( extraLinkTransformers?: LinkTransformer[], ): Promise { let { projectPath } = opts; - if (!projectPath) projectPath = findCurrentProjectAndLoad(session, path.dirname(file)); + if (!projectPath) projectPath = await findCurrentProjectAndLoad(session, path.dirname(file)); if (projectPath) await loadProjectFromDisk(session, projectPath); const exportOptionsList = ( await legacyCollectExportOptions(session, file, 'tex', [ExportFormats.tex], projectPath, opts) @@ -323,7 +323,7 @@ export async function localArticleToTypst( extraLinkTransformers?: LinkTransformer[], ): Promise { let { projectPath } = opts; - if (!projectPath) projectPath = findCurrentProjectAndLoad(session, path.dirname(file)); + if (!projectPath) projectPath = await findCurrentProjectAndLoad(session, path.dirname(file)); if (projectPath) await loadProjectFromDisk(session, projectPath); const exportOptionsList = ( await legacyCollectExportOptions(session, file, 'typ', [ExportFormats.typst], projectPath, opts) diff --git a/packages/myst-cli/src/build/site/manifest.ts b/packages/myst-cli/src/build/site/manifest.ts index 8ffb5f4df..be52c5e6d 100644 --- a/packages/myst-cli/src/build/site/manifest.ts +++ b/packages/myst-cli/src/build/site/manifest.ts @@ -211,25 +211,27 @@ async function resolveTemplateFileOptions( options: Record, ) { const resolvedOptions = { ...options }; - mystTemplate.getValidatedTemplateYml().options?.forEach((option) => { - if (option.type === TemplateOptionType.file && options[option.id]) { - const configPath = selectors.selectCurrentSitePath(session.store.getState()); - const absPath = configPath - ? resolveToAbsolute(session, configPath, options[option.id]) - : options[option.id]; - const fileHash = hashAndCopyStaticFile( - session, - absPath, - session.publicPath(), - (m: string) => { - addWarningForFile(session, options[option.id], m, 'error', { - ruleId: RuleId.templateFileCopied, - }); - }, - ); - resolvedOptions[option.id] = `/${fileHash}`; - } - }); + await Promise.all( + (mystTemplate.getValidatedTemplateYml().options ?? []).map(async (option) => { + if (option.type === TemplateOptionType.file && options[option.id]) { + const configPath = selectors.selectCurrentSitePath(session.store.getState()); + const absPath = configPath + ? await resolveToAbsolute(session, configPath, options[option.id]) + : options[option.id]; + const fileHash = hashAndCopyStaticFile( + session, + absPath, + session.publicPath(), + (m: string) => { + addWarningForFile(session, options[option.id], m, 'error', { + ruleId: RuleId.templateFileCopied, + }); + }, + ); + resolvedOptions[option.id] = `/${fileHash}`; + } + }), + ); return resolvedOptions; } diff --git a/packages/myst-cli/src/build/site/start.ts b/packages/myst-cli/src/build/site/start.ts index 88a856c4f..e1b66eb68 100644 --- a/packages/myst-cli/src/build/site/start.ts +++ b/packages/myst-cli/src/build/site/start.ts @@ -120,7 +120,7 @@ export async function startServer( opts: StartOptions, ): Promise { // Ensure we are on the latest version of the configs - session.reload(); + await session.reload(); warnOnHostEnvironmentVariable(session, opts); const mystTemplate = await getMystTemplate(session, opts); if (!opts.headless) await installSiteTemplate(session, mystTemplate); diff --git a/packages/myst-cli/src/build/utils/collectExportOptions.ts b/packages/myst-cli/src/build/utils/collectExportOptions.ts index aa55419e3..7524d7694 100644 --- a/packages/myst-cli/src/build/utils/collectExportOptions.ts +++ b/packages/myst-cli/src/build/utils/collectExportOptions.ts @@ -471,7 +471,7 @@ export async function collectExportOptions( sourceFiles.map(async (file) => { let fileProjectPath: string | undefined; if (!projectPath) { - fileProjectPath = findCurrentProjectAndLoad(session, path.dirname(file)); + fileProjectPath = await findCurrentProjectAndLoad(session, path.dirname(file)); if (fileProjectPath) await loadProjectFromDisk(session, fileProjectPath); } else { fileProjectPath = projectPath; diff --git a/packages/myst-cli/src/build/utils/localArticleExport.ts b/packages/myst-cli/src/build/utils/localArticleExport.ts index 4613546e5..e88563dba 100644 --- a/packages/myst-cli/src/build/utils/localArticleExport.ts +++ b/packages/myst-cli/src/build/utils/localArticleExport.ts @@ -82,9 +82,11 @@ async function _localArticleExport( exportOptionsList.map(async (exportOptionsWithFile) => { const { $file, $project, ...exportOptions } = exportOptionsWithFile; const { format, output } = exportOptions; - const sessionClone = session.clone(); + const sessionClone = await session.clone(); const fileProjectPath = - projectPath ?? $project ?? findCurrentProjectAndLoad(sessionClone, path.dirname($file)); + projectPath ?? + $project ?? + (await findCurrentProjectAndLoad(sessionClone, path.dirname($file))); if (fileProjectPath) { await loadProjectFromDisk(sessionClone, fileProjectPath); diff --git a/packages/myst-cli/src/config.ts b/packages/myst-cli/src/config.ts index 0614b1002..49dd0e617 100644 --- a/packages/myst-cli/src/config.ts +++ b/packages/myst-cli/src/config.ts @@ -1,7 +1,7 @@ import fs from 'node:fs'; -import { dirname, join, relative, resolve } from 'node:path'; +import { dirname, extname, join, relative, resolve } from 'node:path'; import yaml from 'js-yaml'; -import { writeFileToFolder } from 'myst-cli-utils'; +import { writeFileToFolder, isUrl, computeHash } from 'myst-cli-utils'; import { fileError, fileWarn, RuleId } from 'myst-common'; import type { Config, ProjectConfig, SiteConfig, SiteProject } from 'myst-config'; import { validateProjectConfig, validateSiteConfig } from 'myst-config'; @@ -17,6 +17,7 @@ import { } from 'simple-validators'; import { VFile } from 'vfile'; import { prepareToWrite } from './frontmatter.js'; +import { cachePath, loadFromCache, writeToCache } from './session/cache.js'; import type { ISession } from './session/types.js'; import { selectors } from './store/index.js'; import { config } from './store/reducers.js'; @@ -94,7 +95,7 @@ function fillSiteConfig(base: SiteConfig, filler: SiteConfig) { * * Throws errors config file is malformed or invalid. */ -function getValidatedConfigsFromFile( +async function getValidatedConfigsFromFile( session: ISession, file: string, vfile?: VFile, @@ -155,43 +156,55 @@ function getValidatedConfigsFromFile( const projectOpts = configValidationOpts(vfile, 'config.project', RuleId.validProjectConfig); let extend: string[] | undefined; if (conf.extend) { - extend = validateList( - conf.extend, - { coerce: true, ...incrementOptions('extend', opts) }, - (item, index) => { - const relativeFile = validateString(item, incrementOptions(`extend.${index}`, opts)); - if (!relativeFile) return relativeFile; - return resolveToAbsolute(session, dirname(file), relativeFile); - }, + extend = await Promise.all( + ( + validateList( + conf.extend, + { coerce: true, ...incrementOptions('extend', opts) }, + (item, index) => { + return validateString(item, incrementOptions(`extend.${index}`, opts)); + }, + ) ?? [] + ).map(async (extendFile) => { + const resolvedFile = await resolveToAbsolute(session, dirname(file), extendFile, { + allowRemote: true, + }); + return resolvedFile; + }), ); stack = [...(stack ?? []), file]; - extend?.forEach((extFile) => { - if (stack?.includes(extFile)) { - fileError(vfile, 'Circular dependency encountered during "config.extend" resolution', { - ruleId: RuleId.validConfigStructure, - note: [...stack, extFile].map((f) => resolveToRelative(session, '.', f)).join(' > '), - }); - return; - } - const { site: extSite, project: extProject } = getValidatedConfigsFromFile( - session, - extFile, - vfile, - stack, - ); - session.store.dispatch(config.actions.receiveConfigExtension({ file: extFile })); - if (extSite) { - site = site ? fillSiteConfig(extSite, site) : extSite; - } - if (extProject) { - project = project ? fillProjectFrontmatter(extProject, project, projectOpts) : extProject; - } - }); + await Promise.all( + (extend ?? []).map(async (extFile) => { + if (stack?.includes(extFile)) { + fileError(vfile, 'Circular dependency encountered during "config.extend" resolution', { + ruleId: RuleId.validConfigStructure, + note: [...stack, extFile].map((f) => resolveToRelative(session, '.', f)).join(' > '), + }); + return; + } + const { site: extSite, project: extProject } = await getValidatedConfigsFromFile( + session, + extFile, + vfile, + stack, + ); + session.store.dispatch(config.actions.receiveConfigExtension({ file: extFile })); + if (extSite) { + site = site ? fillSiteConfig(extSite, site) : extSite; + } + if (extProject) { + project = project ? fillProjectFrontmatter(extProject, project, projectOpts) : extProject; + } + }), + ); } const { site: rawSite, project: rawProject } = conf ?? {}; const path = dirname(file); if (rawSite) { - site = fillSiteConfig(validateSiteConfigAndThrow(session, path, vfile, rawSite), site ?? {}); + site = fillSiteConfig( + await validateSiteConfigAndThrow(session, path, vfile, rawSite), + site ?? {}, + ); } if (site) { session.log.debug(`Loaded site config from ${file}`); @@ -200,7 +213,7 @@ function getValidatedConfigsFromFile( } if (rawProject) { project = fillProjectFrontmatter( - validateProjectConfigAndThrow(session, path, vfile, rawProject), + await validateProjectConfigAndThrow(session, path, vfile, rawProject), project ?? {}, projectOpts, ); @@ -219,7 +232,11 @@ function getValidatedConfigsFromFile( * * Errors if config file does not exist or if config file exists but is invalid. */ -export function loadConfig(session: ISession, path: string, opts?: { reloadProject?: boolean }) { +export async function loadConfig( + session: ISession, + path: string, + opts?: { reloadProject?: boolean }, +) { const file = configFromPath(session, path); if (!file) { session.log.debug(`No config loaded from path: ${path}`); @@ -232,7 +249,7 @@ export function loadConfig(session: ISession, path: string, opts?: { reloadProje return existingConf.validated; } } - const { site, project, extend } = getValidatedConfigsFromFile(session, file); + const { site, project, extend } = await getValidatedConfigsFromFile(session, file); const validated = { ...rawConf, site, project, extend }; session.store.dispatch( config.actions.receiveRawConfig({ @@ -247,21 +264,40 @@ export function loadConfig(session: ISession, path: string, opts?: { reloadProje return validated; } -export function resolveToAbsolute( +export async function resolveToAbsolute( session: ISession, basePath: string, relativePath: string, - checkExists = true, + opts?: { + allowNotExist?: boolean; + allowRemote?: boolean; + }, ) { - let message: string; + let message: string | undefined; + if (opts?.allowRemote && isUrl(relativePath)) { + const cacheFilename = `config-item-${computeHash(relativePath)}${extname(relativePath)}`; + if (!loadFromCache(session, cacheFilename, { maxAge: 30 })) { + try { + const resp = await session.fetch(relativePath); + if (resp.ok) { + writeToCache(session, cacheFilename, await resp.text()); + } else { + message = `Bad response from config URL: ${relativePath}`; + } + } catch { + message = `Error fetching config URL: ${relativePath}`; + } + } + relativePath = cachePath(session, cacheFilename); + } try { - const absPath = resolve(join(basePath, relativePath)); - if (!checkExists || fs.existsSync(absPath)) { + const absPath = resolve(basePath, relativePath); + if (opts?.allowNotExist || fs.existsSync(absPath)) { return absPath; } - message = `Does not exist as local path: ${absPath}`; + message = message ?? `Does not exist as local path: ${absPath}`; } catch { - message = `Unable to resolve as local path: ${relativePath}`; + message = message ?? `Unable to resolve as local path: ${relativePath}`; } session.log.debug(message); return relativePath; @@ -271,11 +307,13 @@ function resolveToRelative( session: ISession, basePath: string, absPath: string, - checkExists = true, + opts?: { + allowNotExist?: boolean; + }, ) { let message: string; try { - if (!checkExists || fs.existsSync(absPath)) { + if (opts?.allowNotExist || fs.existsSync(absPath)) { // If it is the same path, use a '.' return relative(basePath, absPath) || '.'; } @@ -287,7 +325,7 @@ function resolveToRelative( return absPath; } -function resolveSiteConfigPaths( +async function resolveSiteConfigPaths( session: ISession, path: string, siteConfig: SiteConfig, @@ -295,25 +333,30 @@ function resolveSiteConfigPaths( session: ISession, basePath: string, path: string, - checkExists?: boolean, - ) => string, + opts?: { + allowNotExist?: boolean; + allowRemote?: boolean; + }, + ) => string | Promise, ) { const resolvedFields: SiteConfig = {}; if (siteConfig.projects) { - resolvedFields.projects = siteConfig.projects.map((proj) => { - if (proj.path) { - return { ...proj, path: resolutionFn(session, path, proj.path) }; - } - return proj; - }); + resolvedFields.projects = await Promise.all( + siteConfig.projects.map(async (proj) => { + if (proj.path) { + return { ...proj, path: await resolutionFn(session, path, proj.path) }; + } + return proj; + }), + ); } if (siteConfig.favicon) { - resolvedFields.favicon = resolutionFn(session, path, siteConfig.favicon); + resolvedFields.favicon = await resolutionFn(session, path, siteConfig.favicon); } return { ...siteConfig, ...resolvedFields }; } -function resolveProjectConfigPaths( +async function resolveProjectConfigPaths( session: ISession, path: string, projectConfig: ProjectConfig, @@ -321,39 +364,50 @@ function resolveProjectConfigPaths( session: ISession, basePath: string, path: string, - checkExists?: boolean, - ) => string, + opts?: { + allowNotExist?: boolean; + allowRemote?: boolean; + }, + ) => string | Promise, ) { const resolvedFields: ProjectConfig = {}; if (projectConfig.bibliography) { - resolvedFields.bibliography = projectConfig.bibliography.map((file) => { - return resolutionFn(session, path, file); - }); + resolvedFields.bibliography = await Promise.all( + projectConfig.bibliography.map(async (file) => { + const resolved = await resolutionFn(session, path, file); + return resolved; + }), + ); } if (projectConfig.index) { - resolvedFields.index = resolutionFn(session, path, projectConfig.index); + resolvedFields.index = await resolutionFn(session, path, projectConfig.index); } if (projectConfig.exclude) { - resolvedFields.exclude = projectConfig.exclude.map((file) => { - return resolutionFn(session, path, file, false); - }); + resolvedFields.exclude = await Promise.all( + projectConfig.exclude.map(async (file) => { + const resolved = await resolutionFn(session, path, file, { allowNotExist: true }); + return resolved; + }), + ); } if (projectConfig.plugins) { - resolvedFields.plugins = projectConfig.plugins.map((file) => { - const resolved = resolutionFn(session, path, file); - if (fs.existsSync(resolved)) return resolved; - return file; - }); + resolvedFields.plugins = await Promise.all( + projectConfig.plugins.map(async (file) => { + const resolved = await resolutionFn(session, path, file); + if (fs.existsSync(resolved)) return resolved; + return file; + }), + ); } return { ...projectConfig, ...resolvedFields }; } -function validateSiteConfigAndThrow( +async function validateSiteConfigAndThrow( session: ISession, path: string, vfile: VFile, rawSite: Record, -): SiteConfig { +): Promise { const site = validateSiteConfig( rawSite, configValidationOpts(vfile, 'config.site', RuleId.validSiteConfig), @@ -370,12 +424,12 @@ function saveSiteConfig(session: ISession, path: string, site: SiteConfig) { session.store.dispatch(config.actions.receiveSiteConfig({ path, ...site })); } -function validateProjectConfigAndThrow( +async function validateProjectConfigAndThrow( session: ISession, path: string, vfile: VFile, rawProject: Record, -): ProjectConfig { +): Promise { const project = validateProjectConfig( rawProject, configValidationOpts(vfile, 'config.project', RuleId.validProjectConfig), @@ -401,7 +455,7 @@ function saveProjectConfig(session: ISession, path: string, project: ProjectConf * If a config file exists on the path, this will override the * site portion of the config and leave the rest. */ -export function writeConfigs( +export async function writeConfigs( session: ISession, path: string, newConfigs?: { @@ -417,24 +471,33 @@ export function writeConfigs( const vfile = new VFile(); vfile.path = file; if (siteConfig) { - saveSiteConfig(session, path, validateSiteConfigAndThrow(session, path, vfile, siteConfig)); + saveSiteConfig( + session, + path, + await validateSiteConfigAndThrow(session, path, vfile, siteConfig), + ); } siteConfig = selectors.selectLocalSiteConfig(session.store.getState(), path); if (siteConfig) { - siteConfig = resolveSiteConfigPaths(session, path, siteConfig, resolveToRelative); + siteConfig = await resolveSiteConfigPaths(session, path, siteConfig, resolveToRelative); } // Get project config to save if (projectConfig) { saveProjectConfig( session, path, - validateProjectConfigAndThrow(session, path, vfile, projectConfig), + await validateProjectConfigAndThrow(session, path, vfile, projectConfig), ); } projectConfig = selectors.selectLocalProjectConfig(session.store.getState(), path); if (projectConfig) { projectConfig = prepareToWrite(projectConfig); - projectConfig = resolveProjectConfigPaths(session, path, projectConfig, resolveToRelative); + projectConfig = await resolveProjectConfigPaths( + session, + path, + projectConfig, + resolveToRelative, + ); } // Return early if nothing new to save if (!siteConfig && !projectConfig) { @@ -442,7 +505,7 @@ export function writeConfigs( return; } // Get raw config to override - const validatedRawConfig = loadConfig(session, path) ?? emptyConfig(); + const validatedRawConfig = (await loadConfig(session, path)) ?? emptyConfig(); let logContent: string; if (siteConfig && projectConfig) { logContent = 'site and project configs'; @@ -459,10 +522,10 @@ export function writeConfigs( writeFileToFolder(file, yaml.dump(newConfig), 'utf-8'); } -export function findCurrentProjectAndLoad(session: ISession, path: string): string | undefined { +export async function findCurrentProjectAndLoad(session: ISession, path: string) { path = resolve(path); if (configFromPath(session, path)) { - loadConfig(session, path); + await loadConfig(session, path); const project = selectors.selectLocalProjectConfig(session.store.getState(), path); if (project) { session.store.dispatch(config.actions.receiveCurrentProjectPath({ path: path })); @@ -475,10 +538,10 @@ export function findCurrentProjectAndLoad(session: ISession, path: string): stri return findCurrentProjectAndLoad(session, dirname(path)); } -export function findCurrentSiteAndLoad(session: ISession, path: string): string | undefined { +export async function findCurrentSiteAndLoad(session: ISession, path: string) { path = resolve(path); if (configFromPath(session, path)) { - loadConfig(session, path); + await loadConfig(session, path); const site = selectors.selectLocalSiteConfig(session.store.getState(), path); if (site) { session.store.dispatch(config.actions.receiveCurrentSitePath({ path: path })); @@ -491,7 +554,7 @@ export function findCurrentSiteAndLoad(session: ISession, path: string): string return findCurrentSiteAndLoad(session, dirname(path)); } -export function reloadAllConfigsForCurrentSite(session: ISession) { +export async function reloadAllConfigsForCurrentSite(session: ISession) { const state = session.store.getState(); const sitePath = selectors.selectCurrentSitePath(state); const file = @@ -502,19 +565,21 @@ export function reloadAllConfigsForCurrentSite(session: ISession) { addWarningForFile(session, file, message, 'error', { ruleId: RuleId.siteConfigExists }); throw Error(message); } - loadConfig(session, sitePath); + await loadConfig(session, sitePath); const siteConfig = selectors.selectCurrentSiteConfig(session.store.getState()); if (!siteConfig?.projects) return; - siteConfig.projects - .filter((project): project is SiteProject & { path: string } => { - return Boolean(project.path); - }) - .forEach((project) => { - try { - loadConfig(session, project.path); - } catch (error) { - // TODO: what error? - session.log.debug(`Failed to find or load project config from "${project.path}"`); - } - }); + await Promise.all( + siteConfig.projects + .filter((project): project is SiteProject & { path: string } => { + return Boolean(project.path); + }) + .map(async (project) => { + try { + await loadConfig(session, project.path); + } catch (error) { + // TODO: what error? + session.log.debug(`Failed to find or load project config from "${project.path}"`); + } + }), + ); } diff --git a/packages/myst-cli/src/process/site.ts b/packages/myst-cli/src/process/site.ts index cf303d4d5..3821dc05b 100644 --- a/packages/myst-cli/src/process/site.ts +++ b/packages/myst-cli/src/process/site.ts @@ -466,7 +466,7 @@ export async function processProject( export async function processSite(session: ISession, opts?: ProcessSiteOptions): Promise { try { - reloadAllConfigsForCurrentSite(session); + await reloadAllConfigsForCurrentSite(session); } catch (error) { session.log.debug(`\n\n${(error as Error)?.stack}\n\n`); const prefix = (error as Error)?.message diff --git a/packages/myst-cli/src/project/load.ts b/packages/myst-cli/src/project/load.ts index 4205e61d8..17f34a117 100644 --- a/packages/myst-cli/src/project/load.ts +++ b/packages/myst-cli/src/project/load.ts @@ -38,7 +38,7 @@ export async function loadProjectFromDisk( const cachedProject = selectors.selectLocalProject(session.store.getState(), path); if (cachedProject) return cachedProject; } - loadConfig(session, path, opts); + await loadConfig(session, path, opts); const state = session.store.getState(); const projectConfig = selectors.selectLocalProjectConfig(state, path); const projectConfigFile = @@ -139,21 +139,24 @@ export async function loadProjectFromDisk( return project; } -export function findProjectsOnPath(session: ISession, path: string) { +export async function findProjectsOnPath(session: ISession, path: string) { let projectPaths: string[] = []; const content = fs.readdirSync(path); if (session.configFiles.filter((file) => content.includes(file)).length) { - loadConfig(session, path); + await loadConfig(session, path); if (selectors.selectLocalProjectConfig(session.store.getState(), path)) { projectPaths.push(path); } } - content - .map((dir) => join(path, dir)) - .filter((file) => isDirectory(file)) - .forEach((dir) => { - projectPaths = projectPaths.concat(findProjectsOnPath(session, dir)); - }); + const projs = await Promise.all( + content + .map((dir) => join(path, dir)) + .filter((file) => isDirectory(file)) + .map(async (dir) => await findProjectsOnPath(session, dir)), + ); + projs.forEach((p) => { + projectPaths = projectPaths.concat(p); + }); return projectPaths; } diff --git a/packages/myst-cli/src/project/toTOC.ts b/packages/myst-cli/src/project/toTOC.ts index cafe8788b..c6eda92ba 100644 --- a/packages/myst-cli/src/project/toTOC.ts +++ b/packages/myst-cli/src/project/toTOC.ts @@ -49,7 +49,7 @@ function tocFromPages(pages: (LocalProjectFolder | LocalProjectPage)[], path: st } export function tocFromProject( - project: Omit, + project: Pick, dir: string = '.', ): any { return [ diff --git a/packages/myst-cli/src/project/toc.spec.ts b/packages/myst-cli/src/project/toc.spec.ts index 00b2a3487..233de8a3f 100644 --- a/packages/myst-cli/src/project/toc.spec.ts +++ b/packages/myst-cli/src/project/toc.spec.ts @@ -576,7 +576,7 @@ describe('findProjectPaths', () => { 'folder/newproj/page.md': '', 'folder/newproj/myst.yml': SITE_CONFIG, }); - expect(findProjectsOnPath(session, '.')).toEqual([]); + expect(await findProjectsOnPath(session, '.')).toEqual([]); }); it('project myst.ymls', async () => { memfs.vol.fromJSON({ @@ -587,7 +587,7 @@ describe('findProjectPaths', () => { 'folder/newproj/page.md': '', 'folder/newproj/myst.yml': PROJECT_CONFIG, }); - expect(findProjectsOnPath(session, '.')).toEqual(['.', 'folder/newproj']); + expect(await findProjectsOnPath(session, '.')).toEqual(['.', 'folder/newproj']); }); }); diff --git a/packages/myst-cli/src/session/session.spec.ts b/packages/myst-cli/src/session/session.spec.ts index bf6ae2d02..a4a5b94ae 100644 --- a/packages/myst-cli/src/session/session.spec.ts +++ b/packages/myst-cli/src/session/session.spec.ts @@ -25,7 +25,7 @@ describe('session warnings', () => { }); it('getAllWarnings returns clone warnings', async () => { const session = new Session(); - const clone = session.clone(); + const clone = await session.clone(); addWarningForFile(session, 'my-file-0', 'my message', 'error', { ruleId: RuleId.bibFileExists, }); @@ -55,7 +55,7 @@ describe('session warnings', () => { }); it('getAllWarnings deduplicates clone warnings', async () => { const session = new Session(); - const clone = session.clone(); + const clone = await session.clone(); addWarningForFile(session, 'my-file', 'my message', 'error', { ruleId: RuleId.bibFileExists }); addWarningForFile(clone, 'my-file', 'my message', 'error', { ruleId: RuleId.bibFileExists }); expect(session.getAllWarnings(RuleId.bibFileExists)).toEqual([ diff --git a/packages/myst-cli/src/session/session.ts b/packages/myst-cli/src/session/session.ts index 1f50e1d74..88bc233bd 100644 --- a/packages/myst-cli/src/session/session.ts +++ b/packages/myst-cli/src/session/session.ts @@ -91,7 +91,6 @@ export class Session implements ISession { const proxyUrl = process.env.HTTPS_PROXY; if (proxyUrl) this.proxyAgent = new HttpsProxyAgent(proxyUrl); this.store = createStore(rootReducer); - this.reload(); // Allow the latest version to be loaded latestVersion('mystmd') .then((latest) => { @@ -113,11 +112,11 @@ export class Session implements ISession { this._shownUpgrade = true; } - reload() { - findCurrentProjectAndLoad(this, '.'); - findCurrentSiteAndLoad(this, '.'); + async reload() { + await findCurrentProjectAndLoad(this, '.'); + await findCurrentSiteAndLoad(this, '.'); if (selectors.selectCurrentSitePath(this.store.getState())) { - reloadAllConfigsForCurrentSite(this); + await reloadAllConfigsForCurrentSite(this); } return this; } @@ -172,8 +171,9 @@ export class Session implements ISession { _clones: ISession[] = []; - clone(): Session { + async clone() { const cloneSession = new Session({ logger: this.log }); + await cloneSession.reload(); // TODO: clean this up through better state handling cloneSession._jupyterSessionManagerPromise = this._jupyterSessionManagerPromise; this._clones.push(cloneSession); diff --git a/packages/myst-cli/src/session/types.ts b/packages/myst-cli/src/session/types.ts index b1a95e91d..3934ccf5b 100644 --- a/packages/myst-cli/src/session/types.ts +++ b/packages/myst-cli/src/session/types.ts @@ -14,8 +14,8 @@ export type ISession = { configFiles: string[]; store: Store; log: Logger; - reload(): ISession; - clone(): ISession; + reload(): Promise; + clone(): Promise; sourcePath(): string; buildPath(): string; sitePath(): string; diff --git a/packages/mystmd/src/clirun.ts b/packages/mystmd/src/clirun.ts index 49d90315b..87932b7ca 100644 --- a/packages/mystmd/src/clirun.ts +++ b/packages/mystmd/src/clirun.ts @@ -19,6 +19,7 @@ export function clirun( const opts = program.opts() as SessionOpts; const logger = chalkLogger(opts?.debug ? LogLevel.debug : LogLevel.info, process.cwd()); const session = new sessionClass({ logger }); + await session.reload(); const versions = await getNodeVersion(session); logVersions(session, versions); const versionsInstalled = await checkNodeVersion(session);