🐱 Add support for pre-parsed mdast-type files (.myst.json) (#1398)

Co-authored-by: Franklin Koch <franklinwkoch@gmail.com>

packages/myst-cli/src/build/utils/collectExportOptions.ts

@@ -29,7 +29,7 @@ import { getExportListFromRawFrontmatter } from '../../frontmatter.js';
 import { getDefaultExportFilename, getDefaultExportFolder } from './defaultNames.js';
 import { getRawFrontmatterFromFile } from '../../process/file.js';
 
-export const SOURCE_EXTENSIONS = ['.ipynb', '.md', '.tex'];
+export const SOURCE_EXTENSIONS = ['.ipynb', '.md', '.tex', '.myst.json'];
 
 type ResolvedArticles = Pick<Export, 'articles' | 'sub_articles'>;
 
@@ -197,7 +197,7 @@ export function resolveArticles(
     }
   }
   // If no articles are specified, use the sourceFile for article
-  if (!resolved.articles && SOURCE_EXTENSIONS.includes(path.extname(sourceFile))) {
+  if (!resolved.articles && SOURCE_EXTENSIONS.some((ext) => sourceFile.endsWith(ext))) {
     resolved.articles = [{ file: path.resolve(sourceFile) }];
   }
   // If there is only one article with no explicit level, it should be 0, making the first section depth 1.

packages/myst-cli/src/build/utils/getFileContent.ts

@@ -40,7 +40,7 @@ export async function getFileContent(
   await Promise.all([
     // Load all citations (.bib)
     ...project.bibliography.map((path) => loadFile(session, path, projectPath, '.bib')),
-    // Load all content (.md and .ipynb)
+    // Load all content (.md, .tex, .myst.json, or .ipynb)
     ...allFiles.map((file, ind) => {
       const preFrontmatter = Array.isArray(preFrontmatters)
         ? preFrontmatters?.[ind]

packages/myst-cli/src/process/file.ts

@@ -108,6 +108,11 @@ export function loadTexFile(
   return { kind: SourceFileKind.Article, mdast: tex.ast as GenericParent, frontmatter };
 }
 
+export function loadMySTJSON(content: string) {
+  const { mdast, frontmatter, kind } = JSON.parse(content);
+  return { kind: kind ?? SourceFileKind.Article, mdast, frontmatter };
+}
+
 /**
  * Attempt to load a file into the current session. Unsupported files with
  * issue a warning
@@ -129,7 +134,7 @@ export async function loadFile(
   session: ISession,
   file: string,
   projectPath?: string,
-  extension?: '.md' | '.ipynb' | '.tex' | '.bib',
+  extension?: '.md' | '.ipynb' | '.tex' | '.bib' | '.myst.json',
   opts?: LoadFileOptions,
 ): Promise<PreRendererData | undefined> {
   await session.loadPlugins();
@@ -177,6 +182,17 @@ export async function loadFile(
         });
         break;
       }
+      case '.json': {
+        if (file.endsWith('.myst.json')) {
+          loadResult = loadMySTJSON(content);
+          break;
+        }
+        // This MUST be the final case before `default`, as
+        // we rely on falling through to the `default` case if
+        // a non-MyST JSON file is encountered here
+        //
+        // falls through
+      }
       default:
         addWarningForFile(session, file, 'Unrecognized extension', 'error', {
           ruleId: RuleId.mystFileLoads,

packages/myst-cli/src/process/site.ts

@@ -400,7 +400,7 @@ export async function processProject(
     await Promise.all([
       // Load all citations (.bib)
       ...project.bibliography.map((path) => loadFile(session, path, siteProject.path, '.bib')),
-      // Load all content (.md and .ipynb)
+      // Load all content (.md, .ipynb, .tex, and .myst.json)
       ...pages.map((page) => loadFile(session, page.file, siteProject.path, undefined)),
       // Load up all the intersphinx references
       loadReferences(session, { projectPath: siteProject.path }),

packages/myst-cli/src/project/fromPath.ts

@@ -127,8 +127,11 @@ function indexFileFromPages(pages: (LocalProjectFolder | LocalProjectPage)[], pa
   if (!indexFile) indexFile = matcher('.md');
   if (!indexFile) indexFile = matcher('.tex');
   if (!indexFile) indexFile = matcher('.ipynb');
+  if (!indexFile) indexFile = matcher('.myst.json');
   if (!indexFile) [indexFile] = files.filter((file) => extname(file) === '.md');
   if (!indexFile) [indexFile] = files.filter((file) => extname(file) === '.tex');
+  if (!indexFile) [indexFile] = files.filter((file) => extname(file) === '.ipynb');
+  if (!indexFile) [indexFile] = files.filter((file) => file.endsWith('.myst.json'));
   if (!indexFile) [indexFile] = files;
   return indexFile;
 }

packages/myst-cli/src/project/fromTOC.ts

@@ -32,7 +32,7 @@ import { isDirectory } from 'myst-cli-utils';
 
 export const DEFAULT_INDEX_FILENAMES = ['index', 'readme', 'main'];
 
-const DEFAULT_INDEX_WITH_EXT = ['.md', '.ipynb']
+const DEFAULT_INDEX_WITH_EXT = ['.md', '.ipynb', '.myst.json']
   .map((ext) => DEFAULT_INDEX_FILENAMES.map((file) => `${file}${ext}`))
   .flat();
 
@@ -222,6 +222,7 @@ export function patternsToFileEntries(
         const { pattern } = entry as PatternEntry;
         const dirStructure = patternToDirectoryStructure(pattern, path);
         const newEntries = directoryStructureToFileEntries(dirStructure, path, ignore);
+
         if (newEntries.length === 0) {
           addWarningForFile(
             session,

packages/myst-cli/src/utils/resolveExtension.ts

@@ -16,19 +16,21 @@ export enum ImageExtensions {
   mp4 = '.mp4', // A moving image!
 }
 export const KNOWN_IMAGE_EXTENSIONS = Object.values(ImageExtensions);
-export const VALID_FILE_EXTENSIONS = ['.md', '.ipynb', '.tex'];
-export const KNOWN_FAST_BUILDS = new Set(['.ipynb', '.md', '.tex']);
+
+export const VALID_FILE_EXTENSIONS = ['.md', '.ipynb', '.tex', '.myst.json'];
+export const KNOWN_FAST_BUILDS = new Set(['.ipynb', '.md', '.tex', '.myst.json']);
 
 /** Return true if file has a valid extension for MyST content */
 export function isValidFile(file: string): boolean {
-  return VALID_FILE_EXTENSIONS.includes(path.extname(file).toLowerCase());
+  const lowerCasePath = file.toLowerCase();
+  return VALID_FILE_EXTENSIONS.some((ext) => lowerCasePath.endsWith(ext));
 }
 
 /**
  * Given a file with resolved path and filename, match to md, ipynb, or tex files
  *
  * If the file already has an extension and exists, it is returned as is.
- * If the file with any of the extensions .md, .ipynb, and .tex appended exists,
+ * If the file with any of the extensions .md, .ipynb, .myst.json, or .tex appended exists,
  * the existing file is returned.
  *
  * This will log an error if no match is found or multiple valid matches are found.