🌎 Fetch config extensions from URL (#1251)

* 🌎 Fetch config fields from URL * 🔧 Fix cache path construction * 🔧 Make remote config paths an option
jupyter-book · Jun 5, 2024 · 286ceaa · 286ceaa
1 parent a665cb1
commit 286ceaa
Show file tree

Hide file tree

Showing 17 changed files with 238 additions and 156 deletions.
diff --git a/.changeset/hot-actors-destroy.md b/.changeset/hot-actors-destroy.md
@@ -0,0 +1,6 @@
+---
+'myst-cli': patch
+'mystmd': patch
+---
+
+Fetch config files from url
diff --git a/packages/myst-cli/src/build/build.ts b/packages/myst-cli/src/build/build.ts
@@ -126,7 +126,10 @@ export async function collectAllBuildExportOptions(
     throw new Error(`When specifying output, you can only request one format`);
   }
   let exportOptionsList: ExportWithInputOutput[];
-  const projectPath = findCurrentProjectAndLoad(session, files[0] ? path.dirname(files[0]) : '.');
+  const projectPath = await findCurrentProjectAndLoad(
+    session,
+    files[0] ? path.dirname(files[0]) : '.',
+  );
   if (projectPath) await loadProjectFromDisk(session, projectPath);
   if (output) {
     session.log.debug(`Exporting formats: "${requestedFormats.join('", "')}"`);

diff --git a/packages/myst-cli/src/build/init.ts b/packages/myst-cli/src/build/init.ts
@@ -69,7 +69,7 @@ export async function init(session: ISession, opts: InitOptions) {
   if (!project && !site && !writeTOC) {
     session.log.info(WELCOME());
   }
-  loadConfig(session, '.');
+  await loadConfig(session, '.');
   const state = session.store.getState();
   const existingRawConfig = selectors.selectLocalRawConfig(state, '.');
   const existingProjectConfig = selectors.selectLocalProjectConfig(state, '.');
@@ -96,7 +96,7 @@ export async function init(session: ISession, opts: InitOptions) {
     }
     if (siteConfig || projectConfig) {
       session.log.info(`💾 Updating config file: ${existingConfigFile}`);
-      writeConfigs(session, '.', { siteConfig, projectConfig });
+      await writeConfigs(session, '.', { siteConfig, projectConfig });
     }
   } else {
     // If no config is present, write it explicitly to include comments.
@@ -119,7 +119,7 @@ export async function init(session: ISession, opts: InitOptions) {
     fs.writeFileSync(configFile, configData);
   }
   if (writeTOC) {
-    loadConfig(session, '.');
+    await loadConfig(session, '.');
     await loadProjectFromDisk(session, '.', { writeTOC });
   }
   // If we have any options, this command is complete!

diff --git a/packages/myst-cli/src/build/legacy.ts b/packages/myst-cli/src/build/legacy.ts
@@ -47,7 +47,7 @@ export async function localArticleToWord(
   extraLinkTransformers?: LinkTransformer[],
 ): Promise<ExportResults> {
   let { projectPath } = opts;
-  if (!projectPath) projectPath = findCurrentProjectAndLoad(session, path.dirname(file));
+  if (!projectPath) projectPath = await findCurrentProjectAndLoad(session, path.dirname(file));
   if (projectPath) await loadProjectFromDisk(session, projectPath);
   const exportOptionsList = (
     await legacyCollectExportOptions(session, file, 'docx', [ExportFormats.docx], projectPath, opts)
@@ -81,7 +81,7 @@ export async function localArticleToJats(
   extraLinkTransformers?: LinkTransformer[],
 ) {
   let { projectPath } = opts;
-  if (!projectPath) projectPath = findCurrentProjectAndLoad(session, path.dirname(file));
+  if (!projectPath) projectPath = await findCurrentProjectAndLoad(session, path.dirname(file));
   if (projectPath) await loadProjectFromDisk(session, projectPath);
   const exportOptionsList = (
     await legacyCollectExportOptions(session, file, 'xml', [ExportFormats.xml], projectPath, opts)
@@ -115,7 +115,7 @@ export async function localArticleToMd(
   extraLinkTransformers?: LinkTransformer[],
 ) {
   let { projectPath } = opts;
-  if (!projectPath) projectPath = findCurrentProjectAndLoad(session, path.dirname(file));
+  if (!projectPath) projectPath = await findCurrentProjectAndLoad(session, path.dirname(file));
   if (projectPath) await loadProjectFromDisk(session, projectPath);
   const exportOptionsList = (
     await legacyCollectExportOptions(session, file, 'md', [ExportFormats.md], projectPath, opts)
@@ -146,7 +146,7 @@ export async function localProjectToMeca(
   extraLinkTransformers?: LinkTransformer[],
 ) {
   let { projectPath } = opts;
-  if (!projectPath) projectPath = findCurrentProjectAndLoad(session, path.dirname(file));
+  if (!projectPath) projectPath = await findCurrentProjectAndLoad(session, path.dirname(file));
   if (projectPath) await loadProjectFromDisk(session, projectPath);
   const exportOptionsList = (
     await legacyCollectExportOptions(session, file, 'zip', [ExportFormats.meca], projectPath, opts)
@@ -179,7 +179,7 @@ export async function localArticleToPdf(
   templateOptions?: Record<string, any>,
 ): Promise<ExportResults> {
   let { projectPath } = opts;
-  if (!projectPath) projectPath = findCurrentProjectAndLoad(session, path.dirname(file));
+  if (!projectPath) projectPath = await findCurrentProjectAndLoad(session, path.dirname(file));
   if (projectPath) await loadProjectFromDisk(session, projectPath);
   const pdfExportOptionsList = (
     await legacyCollectExportOptions(
@@ -249,7 +249,7 @@ export async function localArticleToTex(
   extraLinkTransformers?: LinkTransformer[],
 ): Promise<ExportResults> {
   let { projectPath } = opts;
-  if (!projectPath) projectPath = findCurrentProjectAndLoad(session, path.dirname(file));
+  if (!projectPath) projectPath = await findCurrentProjectAndLoad(session, path.dirname(file));
   if (projectPath) await loadProjectFromDisk(session, projectPath);
   const exportOptionsList = (
     await legacyCollectExportOptions(session, file, 'tex', [ExportFormats.tex], projectPath, opts)
@@ -323,7 +323,7 @@ export async function localArticleToTypst(
   extraLinkTransformers?: LinkTransformer[],
 ): Promise<ExportResults> {
   let { projectPath } = opts;
-  if (!projectPath) projectPath = findCurrentProjectAndLoad(session, path.dirname(file));
+  if (!projectPath) projectPath = await findCurrentProjectAndLoad(session, path.dirname(file));
   if (projectPath) await loadProjectFromDisk(session, projectPath);
   const exportOptionsList = (
     await legacyCollectExportOptions(session, file, 'typ', [ExportFormats.typst], projectPath, opts)

diff --git a/packages/myst-cli/src/build/site/manifest.ts b/packages/myst-cli/src/build/site/manifest.ts
@@ -211,25 +211,27 @@ async function resolveTemplateFileOptions(
   options: Record<string, any>,
 ) {
   const resolvedOptions = { ...options };
-  mystTemplate.getValidatedTemplateYml().options?.forEach((option) => {
-    if (option.type === TemplateOptionType.file && options[option.id]) {
-      const configPath = selectors.selectCurrentSitePath(session.store.getState());
-      const absPath = configPath
-        ? resolveToAbsolute(session, configPath, options[option.id])
-        : options[option.id];
-      const fileHash = hashAndCopyStaticFile(
-        session,
-        absPath,
-        session.publicPath(),
-        (m: string) => {
-          addWarningForFile(session, options[option.id], m, 'error', {
-            ruleId: RuleId.templateFileCopied,
-          });
-        },
-      );
-      resolvedOptions[option.id] = `/${fileHash}`;
-    }
-  });
+  await Promise.all(
+    (mystTemplate.getValidatedTemplateYml().options ?? []).map(async (option) => {
+      if (option.type === TemplateOptionType.file && options[option.id]) {
+        const configPath = selectors.selectCurrentSitePath(session.store.getState());
+        const absPath = configPath
+          ? await resolveToAbsolute(session, configPath, options[option.id])
+          : options[option.id];
+        const fileHash = hashAndCopyStaticFile(
+          session,
+          absPath,
+          session.publicPath(),
+          (m: string) => {
+            addWarningForFile(session, options[option.id], m, 'error', {
+              ruleId: RuleId.templateFileCopied,
+            });
+          },
+        );
+        resolvedOptions[option.id] = `/${fileHash}`;
+      }
+    }),
+  );
   return resolvedOptions;
 }
 

diff --git a/packages/myst-cli/src/build/site/start.ts b/packages/myst-cli/src/build/site/start.ts
@@ -120,7 +120,7 @@ export async function startServer(
   opts: StartOptions,
 ): Promise<AppServer | undefined> {
   // Ensure we are on the latest version of the configs
-  session.reload();
+  await session.reload();
   warnOnHostEnvironmentVariable(session, opts);
   const mystTemplate = await getMystTemplate(session, opts);
   if (!opts.headless) await installSiteTemplate(session, mystTemplate);

diff --git a/packages/myst-cli/src/build/utils/collectExportOptions.ts b/packages/myst-cli/src/build/utils/collectExportOptions.ts
@@ -471,7 +471,7 @@ export async function collectExportOptions(
     sourceFiles.map(async (file) => {
       let fileProjectPath: string | undefined;
       if (!projectPath) {
-        fileProjectPath = findCurrentProjectAndLoad(session, path.dirname(file));
+        fileProjectPath = await findCurrentProjectAndLoad(session, path.dirname(file));
         if (fileProjectPath) await loadProjectFromDisk(session, fileProjectPath);
       } else {
         fileProjectPath = projectPath;

diff --git a/packages/myst-cli/src/build/utils/localArticleExport.ts b/packages/myst-cli/src/build/utils/localArticleExport.ts
@@ -82,9 +82,11 @@ async function _localArticleExport(
     exportOptionsList.map(async (exportOptionsWithFile) => {
       const { $file, $project, ...exportOptions } = exportOptionsWithFile;
       const { format, output } = exportOptions;
-      const sessionClone = session.clone();
+      const sessionClone = await session.clone();
       const fileProjectPath =
-        projectPath ?? $project ?? findCurrentProjectAndLoad(sessionClone, path.dirname($file));
+        projectPath ??
+        $project ??
+        (await findCurrentProjectAndLoad(sessionClone, path.dirname($file)));
 
       if (fileProjectPath) {
         await loadProjectFromDisk(sessionClone, fileProjectPath);