🌎 Allow remote files for template/site options (#1306)

jupyter-book · Jun 13, 2024 · fec350a · fec350a
1 parent 7190846
commit fec350a
Show file tree

Hide file tree

Showing 5 changed files with 21 additions and 6 deletions.
diff --git a/.changeset/yellow-ghosts-deny.md b/.changeset/yellow-ghosts-deny.md
@@ -0,0 +1,6 @@
+---
+'myst-templates': patch
+'myst-cli': patch
+---
+
+Allow remote files for template/site options
diff --git a/packages/myst-cli/src/build/site/manifest.ts b/packages/myst-cli/src/build/site/manifest.ts
@@ -216,7 +216,7 @@ async function resolveTemplateFileOptions(
       if (option.type === TemplateOptionType.file && options[option.id]) {
         const configPath = selectors.selectCurrentSitePath(session.store.getState());
         const absPath = configPath
-          ? await resolveToAbsolute(session, configPath, options[option.id])
+          ? await resolveToAbsolute(session, configPath, options[option.id], { allowRemote: true })
           : options[option.id];
         const fileHash = hashAndCopyStaticFile(
           session,
@@ -388,6 +388,7 @@ export async function getSiteManifest(
   const validatedOptions = mystTemplate.validateOptions(
     siteFrontmatter.options ?? {},
     siteConfigFile,
+    { allowRemote: true },
   );
   const validatedFrontmatter = mystTemplate.validateDoc(
     siteFrontmatter,

diff --git a/packages/myst-cli/src/config.ts b/packages/myst-cli/src/config.ts
@@ -275,12 +275,12 @@ export async function resolveToAbsolute(
 ) {
   let message: string | undefined;
   if (opts?.allowRemote && isUrl(relativePath)) {
-    const cacheFilename = `config-item-${computeHash(relativePath)}${extname(relativePath)}`;
+    const cacheFilename = `config-item-${computeHash(relativePath)}${extname(new URL(relativePath).pathname)}`;
     if (!loadFromCache(session, cacheFilename, { maxAge: 30 })) {
       try {
         const resp = await session.fetch(relativePath);
         if (resp.ok) {
-          writeToCache(session, cacheFilename, await resp.text());
+          writeToCache(session, cacheFilename, Buffer.from(await resp.arrayBuffer()));
         } else {
           message = `Bad response from config URL: ${relativePath}`;
         }

diff --git a/packages/myst-cli/src/session/cache.ts b/packages/myst-cli/src/session/cache.ts
@@ -30,7 +30,11 @@ export function cachePath(session: ISession, filename: string) {
 /**
  * Write data to file on-disk cache
  */
-export function writeToCache(session: ISession, filename: string, data: string) {
+export function writeToCache(
+  session: ISession,
+  filename: string,
+  data: string | NodeJS.ArrayBufferView,
+) {
   const file = cachePath(session, filename);
   session.log.debug(`Writing cache file: ${file}`);
   writeFileToFolder(file, data);

diff --git a/packages/myst-templates/src/validators.ts b/packages/myst-templates/src/validators.ts
@@ -1,7 +1,7 @@
 import fs from 'node:fs';
 import path from 'node:path';
 import { globSync } from 'glob';
-import { hashAndCopyStaticFile, isDirectory } from 'myst-cli-utils';
+import { hashAndCopyStaticFile, isDirectory, isUrl } from 'myst-cli-utils';
 import { TemplateKind, TemplateOptionType } from 'myst-common';
 import type { ReferenceStash } from 'myst-frontmatter';
 import {
@@ -41,18 +41,22 @@ import type {
 } from './types.js';
 import { KIND_TO_EXT } from './download.js';
 
-export type FileOptions = { copyFolder?: string; relativePathFrom?: string };
+export type FileOptions = { copyFolder?: string; relativePathFrom?: string; allowRemote?: boolean };
 
 export type FileValidationOptions = ValidationOptions & FileOptions;
 
 /** Validate that input is an existing file name
  *
  * Resolved relative to the file cached on validation options.
  * Full resolved path is returned.
+ *
+ * If opts.allowRemote is true, input may be a URL.
+ * In this case, the URL is returned unchanged.
  */
 function validateFile(session: ISession, input: any, opts: FileValidationOptions) {
   const filename = validateString(input, opts);
   if (!filename) return;
+  if (opts.allowRemote && isUrl(filename)) return filename;
   let resolvedFile: string;
   if (opts.file) {
     resolvedFile = path.resolve(path.dirname(opts.file), filename);