From da6bf80b0b51fa75d9c2e92a60144a37467685a6 Mon Sep 17 00:00:00 2001 From: RheeseyB <1044774+Rheeseyb@users.noreply.github.com> Date: Fri, 11 Oct 2024 11:48:53 +0100 Subject: [PATCH] Use node module resolution (#6425) **Problem** Utopia's module resolution logic is based on the [2019 node module resolution](https://web.archive.org/web/20190213102857/https://nodejs.org/api/modules.html#modules_all_together), but that logic has significantly evolved since then. This means that there are plenty of cases where the module resolution whilst running a project in Utopia would not match that whilst running elsewhere, causing certain projects to break in unexpected ways. **Fix:** I've implemented the latest node.js module resolution logic from https://nodejs.org/api/modules.html#all-together. Since this is of course running in the browser, this also still needs to take into account the [`browser` field of the `package.json`](https://github.com/defunctzombie/package-browser-field-spec), which was already implemented but I wanted to call it out as that is one area where this differs from the node.js spec. There is still a remaining `FIXME` in here about partial path matching when checking the `imports` and `exports` fields. I'll tackle that in a followup PR. Fixes #6187 --- .../components/custom-code/code-file.spec.ts | 90 ++- .../src/components/custom-code/code-file.ts | 3 + .../core/es-modules/evaluator/evaluator.ts | 1 + .../package-manager/module-resolution-esm.ts | 228 ++++++ .../module-resolution-utils.ts | 216 ++++++ .../package-manager/module-resolution.spec.ts | 130 +++- .../package-manager/module-resolution.ts | 665 ++++++++---------- .../package-manager/package-manager.ts | 14 +- .../module-resolution-examples.json | 53 +- .../core/frameworks/framework-hooks-vite.ts | 5 +- editor/src/utils/path-utils.ts | 13 +- 11 files changed, 1006 insertions(+), 412 deletions(-) create mode 100644 editor/src/core/es-modules/package-manager/module-resolution-esm.ts create mode 100644 editor/src/core/es-modules/package-manager/module-resolution-utils.ts diff --git a/editor/src/components/custom-code/code-file.spec.ts b/editor/src/components/custom-code/code-file.spec.ts index 4562aa933e10..7cb6d28929d9 100644 --- a/editor/src/components/custom-code/code-file.spec.ts +++ b/editor/src/components/custom-code/code-file.spec.ts @@ -26,7 +26,12 @@ import { textFileContents, unparsed, } from '../../core/shared/project-file-types' -import { addFileToProjectContents, getTextFileByPath } from '../assets' +import { + addFileToProjectContents, + contentsToTree, + getTextFileByPath, + type ProjectContentTreeRoot, +} from '../assets' import type { ExportsInfo, MultiFileBuildResult } from '../../core/workers/common/worker-types' import { createBuiltInDependenciesList } from '../../core/es-modules/package-manager/built-in-dependencies-list' @@ -53,6 +58,15 @@ function transpileCode( const appJSCode = "\nimport * as React from 'react'\nimport {\n Ellipse,\n HelperFunctions,\n Image,\n NodeImplementations,\n Rectangle,\n Text,\n View,\n} from 'utopia-api'\nimport {\n colorTheme,\n Button,\n Dialog,\n Icn,\n Icons,\n LargerIcons,\n FunctionIcons,\n MenuIcons,\n Isolator,\n TabComponent,\n Tooltip,\n ActionSheet,\n Avatar,\n ControlledTextArea,\n Title,\n H1,\n H2,\n H3,\n Subdued,\n InspectorSectionHeader,\n InspectorSubsectionHeader,\n FlexColumn,\n FlexRow,\n ResizableFlexColumn,\n PopupList,\n Section,\n SectionTitleRow,\n SectionBodyArea,\n UtopiaListSelect,\n UtopiaListItem,\n CheckboxInput,\n NumberInput,\n StringInput,\n OnClickOutsideHOC,\n} from 'uuiui'\n\nexport var canvasMetadata = {\n specialNodes: [],\n nodeMetadata: {},\n scenes: [\n {\n component: 'App',\n frame: { height: 812, left: 0, width: 375, top: 0 },\n props: { layout: { top: 0, left: 0, bottom: 0, right: 0 } },\n container: { layoutSystem: 'pinSystem' },\n },\n ],\n elementMetadata: {},\n}\n\nexport var App = (props) => {\n return (\n \n )\n}\n\n" +const SampleSingleFileProjectContents: ProjectContentTreeRoot = contentsToTree({ + '/app.js': textFile( + textFileContents(appJSCode, unparsed, RevisionsState.CodeAhead), + null, + null, + 0, + ), +}) + const SampleSingleFileBuildResult = transpileCode(['/app.js'], { '/app.js': appJSCode, }) @@ -84,15 +98,40 @@ const SampleSingleFileExportsInfo = [ }, ] +const multiFileAppJSCode = + "\nimport * as React from 'react'\nimport {\n Ellipse,\n HelperFunctions,\n Image,\n NodeImplementations,\n Rectangle,\n Text,\n View,\n} from 'utopia-api'\nimport {\n colorTheme,\n Button,\n Dialog,\n Icn,\n Icons,\n LargerIcons,\n FunctionIcons,\n MenuIcons,\n Isolator,\n TabComponent,\n Tooltip,\n ActionSheet,\n Avatar,\n ControlledTextArea,\n Title,\n H1,\n H2,\n H3,\n Subdued,\n InspectorSectionHeader,\n InspectorSubsectionHeader,\n FlexColumn,\n FlexRow,\n ResizableFlexColumn,\n PopupList,\n Section,\n SectionTitleRow,\n SectionBodyArea,\n UtopiaListSelect,\n UtopiaListItem,\n CheckboxInput,\n NumberInput,\n StringInput,\n OnClickOutsideHOC,\n} from 'uuiui'\n\nexport var canvasMetadata = {\n specialNodes: [],\n nodeMetadata: {},\n scenes: [\n {\n component: 'App',\n frame: { height: 812, left: 0, width: 375, top: 0 },\n props: { layout: { top: 0, left: 0, bottom: 0, right: 0 } },\n container: { layoutSystem: 'pinSystem' },\n },\n ],\n elementMetadata: {},\n}\n\n\nexport var App = (props) => {\n return (\n \n )\n}\n\n" +const multiFileComponentsJSCode = + "// component library\nimport * as React from 'react'\nimport { Text, View } from 'utopia-api'\n\nexport default (props) => (\n \n \n \n)\n\nexport const LABEL = 'press me! 😉'\n\nexport const ComponentWithProps = (props) => {\n return (\n \n {(props.text + ' ').repeat(props.num)}\n \n )\n}\n\nComponentWithProps.propertyControls = {\n text: {\n type: 'string',\n title: 'Title',\n defaultValue: 'Change me',\n },\n num: {\n type: 'number',\n title: 'amount',\n defaultValue: 2,\n },\n pink: {\n type: 'boolean',\n title: 'Enabled',\n defaultValue: true,\n },\n}\n\n" +const multiFilePreviewJSXCode = + 'import * as React from "react";\nimport * as ReactDOM from "react-dom";\nimport { App } from "../app";\n\nconst root = document.getElementById("root");\nif (root != null) {\nReactDOM.render(, root);\n}' + +const SampleMultiFileProjectContents: ProjectContentTreeRoot = contentsToTree({ + '/app.js': textFile( + textFileContents(multiFileAppJSCode, unparsed, RevisionsState.CodeAhead), + null, + null, + 0, + ), + '/src/components.js': textFile( + textFileContents(multiFileComponentsJSCode, unparsed, RevisionsState.CodeAhead), + null, + null, + 0, + ), + '/public/preview.jsx': textFile( + textFileContents(multiFilePreviewJSXCode, unparsed, RevisionsState.CodeAhead), + null, + null, + 0, + ), +}) + const SampleMultiFileBuildResult = transpileCode( ['/app.js', '/src/components.js', '/public/preview.jsx'], { - '/app.js': - "\nimport * as React from 'react'\nimport {\n Ellipse,\n HelperFunctions,\n Image,\n NodeImplementations,\n Rectangle,\n Text,\n View,\n} from 'utopia-api'\nimport {\n colorTheme,\n Button,\n Dialog,\n Icn,\n Icons,\n LargerIcons,\n FunctionIcons,\n MenuIcons,\n Isolator,\n TabComponent,\n Tooltip,\n ActionSheet,\n Avatar,\n ControlledTextArea,\n Title,\n H1,\n H2,\n H3,\n Subdued,\n InspectorSectionHeader,\n InspectorSubsectionHeader,\n FlexColumn,\n FlexRow,\n ResizableFlexColumn,\n PopupList,\n Section,\n SectionTitleRow,\n SectionBodyArea,\n UtopiaListSelect,\n UtopiaListItem,\n CheckboxInput,\n NumberInput,\n StringInput,\n OnClickOutsideHOC,\n} from 'uuiui'\n\nexport var canvasMetadata = {\n specialNodes: [],\n nodeMetadata: {},\n scenes: [\n {\n component: 'App',\n frame: { height: 812, left: 0, width: 375, top: 0 },\n props: { layout: { top: 0, left: 0, bottom: 0, right: 0 } },\n container: { layoutSystem: 'pinSystem' },\n },\n ],\n elementMetadata: {},\n}\n\n\nexport var App = (props) => {\n return (\n \n )\n}\n\n", - '/src/components.js': - "// component library\nimport * as React from 'react'\nimport { Text, View } from 'utopia-api'\n\nexport default (props) => (\n \n \n \n)\n\nexport const LABEL = 'press me! 😉'\n\nexport const ComponentWithProps = (props) => {\n return (\n \n {(props.text + ' ').repeat(props.num)}\n \n )\n}\n\nComponentWithProps.propertyControls = {\n text: {\n type: 'string',\n title: 'Title',\n defaultValue: 'Change me',\n },\n num: {\n type: 'number',\n title: 'amount',\n defaultValue: 2,\n },\n pink: {\n type: 'boolean',\n title: 'Enabled',\n defaultValue: true,\n },\n}\n\n", - '/public/preview.jsx': - 'import * as React from "react";\nimport * as ReactDOM from "react-dom";\nimport { App } from "../app";\n\nconst root = document.getElementById("root");\nif (root != null) {\nReactDOM.render(, root);\n}', + '/app.js': multiFileAppJSCode, + '/src/components.js': multiFileComponentsJSCode, + '/public/preview.jsx': multiFilePreviewJSXCode, }, ) @@ -216,14 +255,25 @@ const SampleExportsInfoWithError = [ }, ] +const exceptionThrowingCodeJSFile = "export const boom = (() => {\n throw Error('booom')\n})()\n" + +const SampleExceptionThrowingProjectContents: ProjectContentTreeRoot = contentsToTree({ + '/src/code.js': textFile( + textFileContents(exceptionThrowingCodeJSFile, unparsed, RevisionsState.CodeAhead), + null, + null, + 0, + ), +}) + const SampleBuildResultWithException = transpileCode(['/src/code.js'], { - '/src/code.js': "export const boom = (() => {\n throw Error('booom')\n})()\n", + '/src/code.js': exceptionThrowingCodeJSFile, }) const SampleExportsInfoWithException = [ { filename: '/src/code.js', - code: "export const boom = (() => {\n throw Error('booom')\n})()\n", + code: exceptionThrowingCodeJSFile, exportTypes: { boom: { type: 'never', @@ -390,7 +440,9 @@ describe('Creating require function', () => { createBuiltInDependenciesList(null), ) - expect(codeResultCache.curriedRequireFn({})('/', './app', false)).toMatchSnapshot() + expect( + codeResultCache.curriedRequireFn(SampleSingleFileProjectContents)('/', './app', false), + ).toMatchSnapshot() }) it('Creates require function for multi file build result', () => { const codeResultCache = generateCodeResultCache( @@ -403,8 +455,16 @@ describe('Creating require function', () => { createBuiltInDependenciesList(null), ) - expect(codeResultCache.curriedRequireFn({})('/', './app', false)).toMatchSnapshot() - expect(codeResultCache.curriedRequireFn({})('/', './src/components', false)).toMatchSnapshot() + expect( + codeResultCache.curriedRequireFn(SampleMultiFileProjectContents)('/', './app', false), + ).toMatchSnapshot() + expect( + codeResultCache.curriedRequireFn(SampleMultiFileProjectContents)( + '/', + './src/components', + false, + ), + ).toMatchSnapshot() }) it('Require throws exception for module code', () => { const codeResultCache = generateCodeResultCache( @@ -418,7 +478,11 @@ describe('Creating require function', () => { ) expect(() => - codeResultCache.curriedRequireFn({})('/', './src/code', false), + codeResultCache.curriedRequireFn(SampleExceptionThrowingProjectContents)( + '/', + './src/code', + false, + ), ).toThrowErrorMatchingSnapshot() }) it('Require throws exception for import from non-existing module', () => { diff --git a/editor/src/components/custom-code/code-file.ts b/editor/src/components/custom-code/code-file.ts index db71fa3fa70c..373b778925d3 100644 --- a/editor/src/components/custom-code/code-file.ts +++ b/editor/src/components/custom-code/code-file.ts @@ -355,6 +355,9 @@ export function generateCodeResultCache( delete evaluationCache[fileToDelete] } + // FIXME This actually isn't doing anything ever, because it appears that `updatedModules` above is always empty, + // however the require and resolve functions are still used, so we need to figure out which part of this code is + // still alive and which is dead // MUTATION ALERT! This function is mutating editorState.nodeModules.files by inserting the project files into it. incorporateBuildResult(nodeModules, projectContents, projectModules) diff --git a/editor/src/core/es-modules/evaluator/evaluator.ts b/editor/src/core/es-modules/evaluator/evaluator.ts index 089651ed16a7..2c1ff0134993 100644 --- a/editor/src/core/es-modules/evaluator/evaluator.ts +++ b/editor/src/core/es-modules/evaluator/evaluator.ts @@ -92,6 +92,7 @@ function evaluateJs( function firstErrorHandler(error: Error): void { if (isEsModuleError(error)) { + // This is effectively implementing `MAYBE_DETECT_AND_LOAD` from https://nodejs.org/api/modules.html#all-together const { transpiledCode, sourceMap } = transformToCommonJS(filePath, moduleCode) evaluateWithHandler(transpiledCode, sourceMap, secondErrorHandler) } else { diff --git a/editor/src/core/es-modules/package-manager/module-resolution-esm.ts b/editor/src/core/es-modules/package-manager/module-resolution-esm.ts new file mode 100644 index 000000000000..4483cf4bedca --- /dev/null +++ b/editor/src/core/es-modules/package-manager/module-resolution-esm.ts @@ -0,0 +1,228 @@ +import { getPartsFromPath, makePathFromParts, normalizePath } from '../../../utils/path-utils' +import type { ParseResult } from '../../../utils/value-parser-utils' +import { foldEither } from '../../shared/either' +import { isEsCodeFile } from '../../shared/project-file-types' +import type { + ExportsField, + FileForPathParts, + FileLookupFn, + FileLookupResult, + PartialPackageJsonDefinition, +} from './module-resolution-utils' +import { + findClosestPackageScopeToPath, + getPartialPackageJson, + isResolveSuccess, + resolveNotPresent, +} from './module-resolution-utils' + +// Using the logic from https://nodejs.org/api/modules.html#all-together +// This is separated to emphasize that these functions act an ESM compatibility +// layer in CommonJS + +// LOAD_PACKAGE_IMPORTS(X, DIR) +export function loadPackageImports( + path: string, + originPath: string, + packageJsonLookupFn: FileForPathParts, + lookupFn: FileLookupFn, +): FileLookupResult { + // Loading a module from the imports field mapping https://nodejs.org/api/packages.html#imports + // Find the closest package scope SCOPE to DIR. + const { packageJsonFileResult, packageJsonDir } = findClosestPackageScopeToPath( + originPath, + packageJsonLookupFn, + ) + + // If no scope was found, return + if ( + !(isResolveSuccess(packageJsonFileResult) && isEsCodeFile(packageJsonFileResult.success.file)) + ) { + return resolveNotPresent + } + + let possiblePackageJson: ParseResult + try { + possiblePackageJson = getPartialPackageJson(packageJsonFileResult.success.file.fileContents) + } catch { + return resolveNotPresent + } + + return foldEither( + (_) => resolveNotPresent, + (packageJson) => { + // If the SCOPE/package.json "imports" is null or undefined, return + const importsEntry = packageJson['imports'] + if (importsEntry == null) { + return resolveNotPresent + } + + return packageImportsExportsResolve(path, importsEntry, packageJsonDir, lookupFn) + }, + possiblePackageJson, + ) +} + +// LOAD_PACKAGE_EXPORTS(X, DIR) +export function loadPackageExports( + path: string, + originPath: string, + packageJsonLookupFn: FileForPathParts, + lookupFn: FileLookupFn, +): FileLookupResult { + // Try to interpret X as a combination of NAME and SUBPATH where the name + // may have a @scope/ prefix and the subpath begins with a slash (`/`) + const regex = /((?:@[\w|\-|\.]+\/)*[\w|\-|\.]+)(\/[\w|\-|\.]*)*/ + const matches = path.match(regex) + if (matches == null) { + // If X does not match this pattern, return + return resolveNotPresent + } + + const [_combined, name, subpath] = matches + + // If DIR/NAME/package.json is not a file, return + const pathToModule = normalizePath([...getPartsFromPath(originPath), name]) + const packageJsonPathParts = pathToModule.concat('package.json') + const packageJsonFileResult = packageJsonLookupFn(packageJsonPathParts) + if ( + !(isResolveSuccess(packageJsonFileResult) && isEsCodeFile(packageJsonFileResult.success.file)) + ) { + return resolveNotPresent + } + + // Parse DIR/NAME/package.json, and look for "exports" field + let possiblePackageJson: ParseResult + try { + possiblePackageJson = getPartialPackageJson(packageJsonFileResult.success.file.fileContents) + } catch { + return resolveNotPresent + } + + return foldEither( + (_) => resolveNotPresent, + (packageJson) => { + // If "exports" is null or undefined, return. + const exportsEntry = packageJson['exports'] + if (exportsEntry == null) { + return resolveNotPresent + } + + return packageImportsExportsResolve( + `.${subpath}`, + exportsEntry, + makePathFromParts(pathToModule), + lookupFn, + ) + }, + possiblePackageJson, + ) +} + +// LOAD_PACKAGE_SELF(X, DIR) +export function loadPackageSelf( + path: string, + originPath: string, + packageJsonLookupFn: FileForPathParts, + lookupFn: FileLookupFn, +): FileLookupResult { + // Find the closest package scope SCOPE to DIR. + const { packageJsonFileResult, packageJsonDir } = findClosestPackageScopeToPath( + originPath, + packageJsonLookupFn, + ) + + // If no scope was found, return. + if ( + !(isResolveSuccess(packageJsonFileResult) && isEsCodeFile(packageJsonFileResult.success.file)) + ) { + return resolveNotPresent + } + + let possiblePackageJson: ParseResult + try { + possiblePackageJson = getPartialPackageJson(packageJsonFileResult.success.file.fileContents) + } catch { + return resolveNotPresent + } + return foldEither( + (_) => resolveNotPresent, + (packageJson) => { + // If "exports" is null or undefined, return. + const exportsEntry = packageJson['exports'] + if (exportsEntry == null) { + return resolveNotPresent + } + + // If "name" is not the first segment of X, return + const nameEntry = packageJson.name + if (nameEntry == null || !path.startsWith(nameEntry)) { + return resolveNotPresent + } + + return packageImportsExportsResolve( + `.${path.slice(nameEntry.length)}`, + exportsEntry, + packageJsonDir, + lookupFn, + ) + }, + possiblePackageJson, + ) +} + +// PACKAGE_IMPORTS_EXPORTS_RESOLVE +export function packageImportsExportsResolve( + path: string, + importsOrExportsEntry: ExportsField, + packageJsonDir: string, + lookupFn: FileLookupFn, +): FileLookupResult { + // FIXME partial path lookup because the exports field could be along the lines of `{ '@thing': { stuff: { ... }} }` + // in which case we would want to match that against the import `@thing/stuff` + const importsOrExportsResult = + typeof importsOrExportsEntry === 'string' ? importsOrExportsEntry : importsOrExportsEntry[path] + if (importsOrExportsResult == null) { + return resolveNotPresent + } + + if (typeof importsOrExportsResult === 'string') { + // Lookup the import relative to the package.json we have found + return lookupFn(importsOrExportsResult, packageJsonDir) + } + + // Otherwise importsOrExportsResult is an object, so now we need to check for the specific keys we're interested in + const defaultImportsOrExportsEntry = importsOrExportsResult['default'] + const browserImportsOrExportsEntry = importsOrExportsResult['browser'] // The nodejs algorithm uses `node` here + const requireImportsOrExportsEntry = importsOrExportsResult['require'] + + if (defaultImportsOrExportsEntry != null) { + const fileToLookup = + typeof defaultImportsOrExportsEntry === 'string' + ? defaultImportsOrExportsEntry + : defaultImportsOrExportsEntry['default'] + if (fileToLookup != null) { + return lookupFn(fileToLookup, packageJsonDir) + } + } + if (browserImportsOrExportsEntry != null) { + const fileToLookup = + typeof browserImportsOrExportsEntry === 'string' + ? browserImportsOrExportsEntry + : browserImportsOrExportsEntry['default'] + if (fileToLookup != null) { + return lookupFn(fileToLookup, packageJsonDir) + } + } + if (requireImportsOrExportsEntry != null) { + const fileToLookup = + typeof requireImportsOrExportsEntry === 'string' + ? requireImportsOrExportsEntry + : requireImportsOrExportsEntry['default'] + if (fileToLookup != null) { + return lookupFn(fileToLookup, packageJsonDir) + } + } + + return resolveNotPresent +} diff --git a/editor/src/core/es-modules/package-manager/module-resolution-utils.ts b/editor/src/core/es-modules/package-manager/module-resolution-utils.ts new file mode 100644 index 000000000000..0a2bc939216b --- /dev/null +++ b/editor/src/core/es-modules/package-manager/module-resolution-utils.ts @@ -0,0 +1,216 @@ +import type { MapLike } from 'typescript' +import { getPartsFromPath, makePathFromParts, normalizePath } from '../../../utils/path-utils' +import type { ParseResult } from '../../../utils/value-parser-utils' +import { + optionalObjectKeyParser, + parseAlternative, + parseFalse, + parseNull, + parseObject, + parseString, +} from '../../../utils/value-parser-utils' +import { applicative6Either } from '../../shared/either' +import { setOptionalProp } from '../../shared/object-utils' +import type { ESCodeFile, ESRemoteDependencyPlaceholder } from '../../shared/project-file-types' + +import LRU from 'lru-cache' + +export type ImportsExportsObjectField = MapLike< + string | MapLike> +> +export type ExportsField = string | ImportsExportsObjectField + +export interface PartialPackageJsonDefinition { + name?: string + main?: string + module?: string + browser?: string | MapLike + imports?: ImportsExportsObjectField + exports?: ExportsField +} + +const partialPackageJsonCache: LRU> = new LRU({ + max: 20, +}) +export function getPartialPackageJson(contents: string): ParseResult { + const fromCache = partialPackageJsonCache.get(contents) + if (fromCache == null) { + const jsonParsed = JSON.parse(contents) + const result = parsePartialPackageJsonDefinition(jsonParsed) + partialPackageJsonCache.set(contents, result) + return result + } else { + return fromCache + } +} + +export interface ResolveSuccess { + type: 'RESOLVE_SUCCESS' + success: T +} + +export function resolveSuccess(success: T): ResolveSuccess { + return { + type: 'RESOLVE_SUCCESS', + success: success, + } +} + +export interface ResolveNotPresent { + type: 'RESOLVE_NOT_PRESENT' +} + +export const resolveNotPresent: ResolveNotPresent = { + type: 'RESOLVE_NOT_PRESENT', +} + +export interface ResolveSuccessIgnoreModule { + type: 'RESOLVE_SUCCESS_IGNORE_MODULE' +} + +export function isResolveSuccessIgnoreModule( + resolveResult: ResolveResult, +): resolveResult is ResolveSuccessIgnoreModule { + return resolveResult.type === 'RESOLVE_SUCCESS_IGNORE_MODULE' +} + +export const resolveSuccessIgnoreModule: ResolveSuccessIgnoreModule = { + type: 'RESOLVE_SUCCESS_IGNORE_MODULE', +} + +export type ResolveResult = ResolveNotPresent | ResolveSuccessIgnoreModule | ResolveSuccess + +export function isResolveSuccess( + resolveResult: ResolveResult, +): resolveResult is ResolveSuccess { + return resolveResult.type === 'RESOLVE_SUCCESS' +} + +export function isResolveNotPresent( + resolveResult: ResolveResult, +): resolveResult is ResolveNotPresent { + return resolveResult.type === 'RESOLVE_NOT_PRESENT' +} + +export interface FoundFile { + path: string + file: ESCodeFile | ESRemoteDependencyPlaceholder +} + +export type FileLookupResult = ResolveResult + +export function fileLookupResult( + path: string, + file: ESCodeFile | ESRemoteDependencyPlaceholder | null, +): FileLookupResult { + if (file == null) { + return resolveNotPresent + } else { + return resolveSuccess({ + path: path, + file: file, + }) + } +} + +export type FileLookupFn = (path: string, importOrigin: string) => FileLookupResult +export type FileForPathParts = (pathParts: string[]) => FileLookupResult + +const importsExportsObjectParser = parseObject( + parseAlternative>>( + [ + parseString, + parseObject( + parseAlternative>( + [ + parseString, + parseNull, + parseObject( + parseAlternative( + [parseString, parseNull], + `package.imports and package.exports replacement entries must be either string or null`, + ), + ), + ], + `package.imports and package.exports replacement entries must be either string or null`, + ), + ), + ], + 'package.imports and package.exports object must be an object with type {[key: string]: string | null}', + ), +) + +export const exportsParser = parseAlternative( + [parseString, importsExportsObjectParser], + 'package.imports and package.exports field must either be a string or an object with type {[key: string]: string | null}', +) + +export function parsePartialPackageJsonDefinition( + value: unknown, +): ParseResult { + return applicative6Either( + (name, main, module, browser, imports, exports) => { + let result: PartialPackageJsonDefinition = {} + setOptionalProp(result, 'name', name) + setOptionalProp(result, 'main', main) + setOptionalProp(result, 'module', module) + setOptionalProp(result, 'browser', browser) + setOptionalProp(result, 'imports', imports) + setOptionalProp(result, 'exports', exports) + return result + }, + optionalObjectKeyParser(parseString, 'name')(value), + optionalObjectKeyParser(parseString, 'main')(value), + optionalObjectKeyParser(parseString, 'module')(value), + optionalObjectKeyParser( + parseAlternative>( + [ + parseString, + parseObject( + parseAlternative( + [parseString, parseFalse], + `package.browser replacement entries must be either string or 'false' for ignoring a package`, + ), + ), + ], + 'package.browser field must either be a string or an object with type {[key: string]: string | false}', + ), + 'browser', + )(value), + optionalObjectKeyParser(importsExportsObjectParser, 'imports')(value), + optionalObjectKeyParser(exportsParser, 'exports')(value), + ) +} + +export type PackageLookupResult = { + packageJsonFileResult: FileLookupResult + packageJsonDir: string +} +export function findClosestPackageScopeToPath( + path: string, + localFileLookup: FileForPathParts, +): PackageLookupResult { + const originPathPartsToTest = normalizePath(getPartsFromPath(path)) + const allPossiblePackageJsonPaths = [['package.json']] + .concat( + originPathPartsToTest.map((_part, index) => + originPathPartsToTest.slice(0, index + 1).concat('package.json'), + ), + ) + .reverse() + + for (const possiblePath of allPossiblePackageJsonPaths) { + const packageJsonFileResult = localFileLookup(possiblePath) + if (isResolveSuccess(packageJsonFileResult)) { + return { + packageJsonFileResult: packageJsonFileResult, + packageJsonDir: makePathFromParts(possiblePath.slice(0, -1)), + } + } + } + + return { + packageJsonFileResult: resolveNotPresent, + packageJsonDir: '', + } +} diff --git a/editor/src/core/es-modules/package-manager/module-resolution.spec.ts b/editor/src/core/es-modules/package-manager/module-resolution.spec.ts index aa22947b72e7..eb394f94f2c2 100644 --- a/editor/src/core/es-modules/package-manager/module-resolution.spec.ts +++ b/editor/src/core/es-modules/package-manager/module-resolution.spec.ts @@ -7,7 +7,8 @@ import { unparsed, } from '../../shared/project-file-types' import * as moduleResolutionExamples from '../test-cases/module-resolution-examples.json' -import { isResolveSuccess, resolveModule } from './module-resolution' +import { resolveModule } from './module-resolution' +import { isResolveSuccess } from './module-resolution-utils' import { createNodeModules } from './test-utils' const sampleProjectContents: ProjectContentTreeRoot = contentsToTree({ @@ -33,6 +34,34 @@ const sampleProjectContents: ProjectContentTreeRoot = contentsToTree({ null, 0, ), + '/package.json': textFile( + textFileContents( + ` + { + "name": "sample-project", + "exports": { + "./exported-module": "./exported-module-main.js" + }, + "imports": { + "#dep-object": { + "default": "./src/dep-object.js" + }, + "#dep-simple": "./src/dep-simple.js" + } + }`, + unparsed, + RevisionsState.CodeAhead, + ), + null, + null, + 0, + ), + '/exported-module-main.js': textFile( + textFileContents('export const ExportedModuleMain = 1', unparsed, RevisionsState.CodeAhead), + null, + null, + 0, + ), '/a/some/nested/file.js': textFile( textFileContents('export const Cake = "tasty"', unparsed, RevisionsState.CodeAhead), null, @@ -51,6 +80,18 @@ const sampleProjectContents: ProjectContentTreeRoot = contentsToTree({ null, 0, ), + '/src/dep-object.js': textFile( + textFileContents('export const DepObject = 1', unparsed, RevisionsState.CodeAhead), + null, + null, + 0, + ), + '/src/dep-simple.js': textFile( + textFileContents('export const DepSimple = 1', unparsed, RevisionsState.CodeAhead), + null, + null, + 0, + ), '/src/icon.png': { type: 'ASSET_FILE', }, @@ -60,6 +101,12 @@ const sampleProjectContents: ProjectContentTreeRoot = contentsToTree({ null, 0, ), + '/utopia/layout.utopia.js': textFile( + textFileContents('export const Layout = 1', unparsed, RevisionsState.CodeAhead), + null, + null, + 0, + ), }) describe('ES Package Manager Module Resolution', () => { @@ -244,20 +291,69 @@ describe('ES Package Manager Module Resolution', () => { expect(resolveResult.type).toEqual('RESOLVE_SUCCESS_IGNORE_MODULE') }) - // it('loads self references', () => { - // expect( - // resolveModule( - // createNodeModules(moduleResolutionExamples.contents), - // 'my-package-with-package-root/src/deep-folder/moduleA.js', - // 'my-package-with-package-root', - // ), - // ).toEqual('my-package-with-package-root/index.js') - // expect( - // resolveModule( - // createNodeModules(moduleResolutionExamples.contents), - // 'my-package-with-package-root/src/deep-folder/moduleA.js', - // 'my-package-with-package-root/src/other-folder/moduleB', - // ), - // ).toEqual('my-package-with-package-root/src/other-folder/moduleB.js') - // }) + it('resolves package exports for local package', () => { + expect(resolve('/src/app.js', 'sample-project/exported-module')).toEqual( + '/exported-module-main.js', + ) + }) + + it('resolves package imports simple for local package', () => { + expect(resolve('/src/app.js', '#dep-simple')).toEqual('/src/dep-simple.js') + }) + it('resolves package imports object for local package', () => { + expect(resolve('/src/app.js', '#dep-object')).toEqual('/src/dep-object.js') + }) + + it('resolves package main entry for node module with exports with no subpath', () => { + expect(resolve('/src/app.js', 'module-with-exports-and-imports')).toEqual( + '/node_modules/module-with-exports-and-imports/index.main.js', + ) + }) + + it('resolves package exports object main entry for node module', () => { + expect(resolve('/src/app.js', 'module-with-exports-and-imports-objects')).toEqual( + '/node_modules/module-with-exports-and-imports-objects/index.main.js', + ) + }) + it('resolves package exports object submodule export for node module', () => { + expect(resolve('/src/app.js', 'module-with-exports-and-imports-objects/submodule')).toEqual( + '/node_modules/module-with-exports-and-imports-objects/src/submodule.js', + ) + }) + + it('resolves package imports for node module', () => { + expect( + resolve('/node_modules/module-with-exports-and-imports/src/something.js', '#dep'), + ).toEqual('/node_modules/module-with-exports-and-imports/dep.js') + }) + it('resolves package imports object for node module', () => { + expect( + resolve('/node_modules/module-with-exports-and-imports-objects/src/something.js', '#dep'), + ).toEqual('/node_modules/module-with-exports-and-imports-objects/dep.js') + }) + + it('loads self references', () => { + expect( + resolve( + '/node_modules/my-package-with-package-root/src/deep-folder/moduleA.js', + 'my-package-with-package-root', + ), + ).toEqual('/node_modules/my-package-with-package-root/index.js') + expect( + resolve( + '/node_modules/my-package-with-package-root/src/deep-folder/moduleA.js', + 'my-package-with-package-root/src/other-folder/moduleB', + ), + ).toEqual('/node_modules/my-package-with-package-root/src/other-folder/moduleB.js') + }) + + it('resolves clsx', () => { + expect(resolve('/app/components/hydrogen/Button.jsx', 'clsx')).toEqual( + '/node_modules/clsx/dist/clsx.js', + ) + }) + + it('resolves /utopia/layout.utopia', () => { + expect(resolve('/utopia/text.utopia.js', './layout.utopia')).toEqual('/utopia/layout.utopia.js') + }) }) diff --git a/editor/src/core/es-modules/package-manager/module-resolution.ts b/editor/src/core/es-modules/package-manager/module-resolution.ts index 8904005017ff..b570990de635 100644 --- a/editor/src/core/es-modules/package-manager/module-resolution.ts +++ b/editor/src/core/es-modules/package-manager/module-resolution.ts @@ -1,180 +1,261 @@ -import type { - ProjectFile, - NodeModules, - ESCodeFile, - ESRemoteDependencyPlaceholder, -} from '../../shared/project-file-types' -import { isEsCodeFile, esCodeFile } from '../../shared/project-file-types' -import type { ParseResult } from '../../../utils/value-parser-utils' -import { - optionalObjectKeyParser, - parseString, - parseAlternative, - parseObject, - parseFalse, -} from '../../../utils/value-parser-utils' -import type { Either } from '../../shared/either' -import { - applicative3Either, - applicative4Either, - foldEither, - isLeft, - isRight, - left, - right, -} from '../../shared/either' -import { setOptionalProp } from '../../shared/object-utils' +import type { MapLike } from 'typescript' import type { ProjectContentTreeRoot } from '../../../components/assets' import { getContentsTreeFileFromElements } from '../../../components/assets' +import { applyFilePathMappingsToFilePath } from '../../../core/workers/common/project-file-utils' +import { + getParentDirectory, + getPartsFromPath, + makePathFromParts, + normalizePath, + stripTrailingSlash, +} from '../../../utils/path-utils' +import type { ParseResult } from '../../../utils/value-parser-utils' +import { getFilePathMappings } from '../../model/project-file-utils' import { dropLast, last } from '../../shared/array-utils' -import { getPartsFromPath, makePathFromParts, normalizePath } from '../../../utils/path-utils' -import type { MapLike } from 'typescript' - -import LRU from 'lru-cache' +import type { Either } from '../../shared/either' +import { foldEither, isLeft, isRight, left, right } from '../../shared/either' +import type { NodeModules, ProjectFile } from '../../shared/project-file-types' +import { esCodeFile, isEsCodeFile } from '../../shared/project-file-types' import type { BuiltInDependencies } from './built-in-dependencies-list' -import { getFilePathMappings } from '../../model/project-file-utils' -import { applyFilePathMappingsToFilePath } from '../../../core/workers/common/project-file-utils' +import { loadPackageSelf, loadPackageImports, loadPackageExports } from './module-resolution-esm' +import type { + FileForPathParts, + FileLookupFn, + FileLookupResult, + PartialPackageJsonDefinition, +} from './module-resolution-utils' +import { + fileLookupResult, + getPartialPackageJson, + isResolveSuccess, + resolveNotPresent, + resolveSuccessIgnoreModule, +} from './module-resolution-utils' -const partialPackageJsonCache: LRU> = new LRU({ - max: 20, -}) -function getPartialPackageJson(contents: string): ParseResult { - const fromCache = partialPackageJsonCache.get(contents) - if (fromCache == null) { - const jsonParsed = JSON.parse(contents) - const result = parsePartialPackageJsonDefinition(jsonParsed) - partialPackageJsonCache.set(contents, result) - return result - } else { - return fromCache - } -} +// Using the logic from https://nodejs.org/api/modules.html#all-together -interface ResolveSuccess { - type: 'RESOLVE_SUCCESS' - success: T +function loadAsFile( + projectContents: ProjectContentTreeRoot, + pathParts: string[], +): FileLookupResult { + return abstractLoadAsFile(projectContentsFileForPathParts(projectContents), pathParts) } -function resolveSuccess(success: T): ResolveSuccess { - return { - type: 'RESOLVE_SUCCESS', - success: success, +// LOAD_AS_FILE(X) +function abstractLoadAsFile( + fileForPathParts: FileForPathParts, + pathParts: string[], +): FileLookupResult { + const normalisedPathParts = normalizePath(pathParts) + const pathToFile = dropLast(normalisedPathParts) + const lastPart = last(normalisedPathParts) + if (lastPart == null) { + return resolveNotPresent } -} -interface ResolveNotPresent { - type: 'RESOLVE_NOT_PRESENT' -} + // If X is a file, load X as its file extension format + const asFileResult = fileForPathParts(normalisedPathParts) + if (isResolveSuccess(asFileResult)) { + return asFileResult + } -const resolveNotPresent: ResolveNotPresent = { - type: 'RESOLVE_NOT_PRESENT', -} + // If X.js(x) is a file + const withJs = lastPart + '.js' + const withJsResult = fileForPathParts(pathToFile.concat(withJs)) + if (isResolveSuccess(withJsResult)) { + return withJsResult + } -interface ResolveSuccessIgnoreModule { - type: 'RESOLVE_SUCCESS_IGNORE_MODULE' -} + const withJsx = lastPart + '.jsx' + const withJsxResult = fileForPathParts(pathToFile.concat(withJsx)) + if (isResolveSuccess(withJsxResult)) { + return withJsxResult + } -export function isResolveSuccessIgnoreModule( - resolveResult: ResolveResult, -): resolveResult is ResolveSuccessIgnoreModule { - return resolveResult.type === 'RESOLVE_SUCCESS_IGNORE_MODULE' -} + // If X.json is a file, load X.json to a JavaScript Object + const withJson = lastPart + '.json' + const withJsonResult = fileForPathParts(pathToFile.concat(withJson)) + if (isResolveSuccess(withJsonResult)) { + return withJsonResult + } -const resolveSuccessIgnoreModule: ResolveSuccessIgnoreModule = { - type: 'RESOLVE_SUCCESS_IGNORE_MODULE', + return resolveNotPresent } -type ResolveResult = ResolveNotPresent | ResolveSuccessIgnoreModule | ResolveSuccess +function processPackageJson(potentiallyJsonCode: string): string | null { + let possiblePackageJson: ParseResult + try { + possiblePackageJson = getPartialPackageJson(potentiallyJsonCode) + } catch { + return null + } + return foldEither( + (_) => null, + (packageJson) => { + // const moduleName: string | null = packageJson.name ?? null + const browserEntry: string | MapLike | null = packageJson.browser ?? null + const mainEntry: string | null = packageJson.main ?? null + // const moduleEntry: string | null = packageJson.module ?? null -export function isResolveSuccess( - resolveResult: ResolveResult, -): resolveResult is ResolveSuccess { - return resolveResult.type === 'RESOLVE_SUCCESS' -} + if (browserEntry != null && typeof browserEntry === 'string') { + return browserEntry + } else if (mainEntry != null) { + return mainEntry + // } else if (moduleEntry != null) { + // return moduleEntry + // } else if (moduleName != null) { + // return moduleName + } -export function isResolveNotPresent( - resolveResult: ResolveResult, -): resolveResult is ResolveNotPresent { - return resolveResult.type === 'RESOLVE_NOT_PRESENT' + return null + }, + possiblePackageJson, + ) } -interface FoundFile { - path: string - file: ESCodeFile | ESRemoteDependencyPlaceholder +function loadAsDirectory( + projectContents: ProjectContentTreeRoot, + pathParts: string[], +): FileLookupResult { + return abstractLoadAsDirectory(projectContentsFileForPathParts(projectContents), pathParts) } -export type FileLookupResult = ResolveResult - -function fileLookupResult( - path: string, - file: ESCodeFile | ESRemoteDependencyPlaceholder | null, +// LOAD_AS_DIRECTORY(X) +function abstractLoadAsDirectory( + fileForPathParts: FileForPathParts, + pathParts: string[], ): FileLookupResult { - if (file == null) { - return resolveNotPresent - } else { - return resolveSuccess({ - path: path, - file: file, - }) - } -} + const normalisedPathParts = normalizePath(pathParts) // X -type FileLookupFn = (path: string[], direct: boolean) => FileLookupResult + // If X/package.json is a file + const jsonFileResult = fileForPathParts(normalisedPathParts.concat('package.json')) + if (isResolveSuccess(jsonFileResult) && isEsCodeFile(jsonFileResult.success.file)) { + // Parse X/package.json, and look for "browser" or "main" field + const mainEntryPath = processPackageJson(jsonFileResult.success.file.fileContents) -function fallbackLookup( - lookupFn: FileLookupFn, - path: Array, - direct: boolean, -): FileLookupResult { - const asIsPath = normalizePath(path) - const pathToFile = dropLast(asIsPath) - const lastPart = last(asIsPath) - if (lastPart == null) { - return resolveNotPresent - } else { - const asIsResult = lookupFn(asIsPath, direct) - if (isResolveSuccess(asIsResult)) { - return asIsResult - } + if (mainEntryPath != null) { + // let M = X + (json main field) + const mainEntryPathParts = [...normalisedPathParts, ...getPartsFromPath(mainEntryPath)] - if (!direct) { - const withJs = lastPart + '.js' - const withJsResult = lookupFn(pathToFile.concat(withJs), direct) - if (isResolveSuccess(withJsResult)) { - return withJsResult + // LOAD_AS_FILE(M) + const mainEntryResult = abstractLoadAsFile(fileForPathParts, mainEntryPathParts) + if (isResolveSuccess(mainEntryResult)) { + return mainEntryResult } - const withJsx = lastPart + '.jsx' - const withJsxResult = lookupFn(pathToFile.concat(withJsx), direct) - if (isResolveSuccess(withJsxResult)) { - return withJsxResult + // LOAD_INDEX(M) + const loadIndexMResult = loadIndex(fileForPathParts, mainEntryPathParts) + if (isResolveSuccess(loadIndexMResult)) { + return loadIndexMResult } - // TODO this also needs JSON parsing - const withJson = lastPart + '.json' - const withJsonResult = lookupFn(pathToFile.concat(withJson), direct) - if (isResolveSuccess(withJsonResult)) { - return withJsonResult - } + // LOAD_INDEX(X) DEPRECATED - note that this is only deprecated in the case where "main" is truthy + return loadIndex(fileForPathParts, normalisedPathParts) } + } - return resolveNotPresent + // If "main" is a falsy value + // LOAD_INDEX(X) + return loadIndex(fileForPathParts, normalisedPathParts) +} + +// LOAD_INDEX(X) +function loadIndex(fileForPathParts: FileForPathParts, pathParts: string[]): FileLookupResult { + // 1. If X/index.js is a file + // a. Find the closest package scope SCOPE to X. + // b. If no scope was found, load X/index.js as a CommonJS module. STOP. + // c. If the SCOPE/package.json contains "type" field, + // 1. If the "type" field is "module", load X/index.js as an ECMAScript module. STOP. + // 2. Else, load X/index.js as an CommonJS module. STOP. + // Note: We don't attempt to load as an ECMAScript, as we can't - instead we'll try to load whatever + // it is as CommonJS, and if it turns out to be an esm we'll attempt to transpile it to CommonJS + + const indexJSResult = abstractLoadAsFile(fileForPathParts, pathParts.concat('index.js')) + if (isResolveSuccess(indexJSResult)) { + return indexJSResult } + + // If X/index.json is a file, parse X/index.json to a JavaScript object + return abstractLoadAsFile(fileForPathParts, pathParts.concat('index.json')) + + // If X/index.node is a file, load X/index.node as binary addon - not applicable +} + +// NODE_MODULES_PATHS +function nodeModulesPaths(path: string): Array { + // let PARTS = path split(START) + const pathParts = getPartsFromPath(path) + + // let DIRS = [] + let dirs: Array = [] + + // The algorithm builds dirs in descending order, but we'll do it in ascending and reverse + pathParts.forEach((part, index) => { + // if PARTS[I] = "node_modules" CONTINUE + if (part === 'node_modules') { + return + } + + // DIR = path join(PARTS[0 .. I] + "node_modules") + const dir = makePathFromParts(pathParts.slice(0, index + 1).concat('node_modules')) + + // DIRS = DIR + DIRS + dirs.push(dir) + }) + dirs.reverse() + + // return DIRS + GLOBAL_FOLDERS + return dirs.concat('/node_modules/') } -function nodeModulesFileLookup( +// LOAD_NODE_MODULES +function loadNodeModules( nodeModules: NodeModules, - path: Array, - direct: boolean, + path: string, + originPath: string, + lookupFn: FileLookupFn, ): FileLookupResult { - return fallbackLookup( - (innerPath: string[], innerDirect: boolean) => { - const filename = makePathFromParts(innerPath) - return fileLookupResult(filename, nodeModules[filename]) - }, - path, - direct, - ) + const packageJsonLookupFn = nodeModulesFileForPathParts(nodeModules) + + // let DIRS = NODE_MODULES_PATHS(START) + const dirs = nodeModulesPaths(originPath).reverse() + const targetOrigins = [originPath, ...dirs] + + for (const targetOrigin of targetOrigins) { + // for each DIR in DIRS: + + // LOAD_PACKAGE_EXPORTS(X, DIR) + const packageExportsResult = loadPackageExports( + path, + targetOrigin, + packageJsonLookupFn, + lookupFn, + ) + if (isResolveSuccess(packageExportsResult)) { + return packageExportsResult + } + + const pathForLookup = path.startsWith('/') + ? path + : `${stripTrailingSlash(targetOrigin)}/${path}` + + const pathParts = getPartsFromPath(pathForLookup) + const normalisedPathParts = normalizePath(pathParts) + + // LOAD_AS_FILE(DIR/X) + const asFileResult = abstractLoadAsFile(packageJsonLookupFn, normalisedPathParts) + if (isResolveSuccess(asFileResult)) { + return asFileResult + } + + // LOAD_AS_DIRECTORY(DIR/X) + const asDirectoryResult = abstractLoadAsDirectory(packageJsonLookupFn, normalisedPathParts) + if (isResolveSuccess(asDirectoryResult)) { + return asDirectoryResult + } + } + + return resolveNotPresent } function getProjectFileContentsAsString(file: ProjectFile): string | null { @@ -193,240 +274,52 @@ function getProjectFileContentsAsString(file: ProjectFile): string | null { } } -function projectContentsFileLookup( +function projectContentsFileForPathParts( projectContents: ProjectContentTreeRoot, - path: Array, - direct: boolean, -): FileLookupResult { - return fallbackLookup( - (innerPath: string[]) => { - const projectFile = getContentsTreeFileFromElements(projectContents, innerPath) - if (projectFile == null) { +): FileForPathParts { + return (pathParts: string[]) => { + const projectFile = getContentsTreeFileFromElements(projectContents, pathParts) + if (projectFile == null) { + return resolveNotPresent + } else { + const fileContents = getProjectFileContentsAsString(projectFile) + if (fileContents == null) { return resolveNotPresent } else { - const fileContents = getProjectFileContentsAsString(projectFile) - if (fileContents == null) { - return resolveNotPresent - } else { - const filename = makePathFromParts(innerPath) - return fileLookupResult(filename, esCodeFile(fileContents, 'PROJECT_CONTENTS', filename)) - } + const filename = makePathFromParts(pathParts) + return fileLookupResult(filename, esCodeFile(fileContents, 'PROJECT_CONTENTS', filename)) } - }, - path, - direct, - ) -} - -interface PartialPackageJsonDefinition { - name?: string - main?: string - module?: string - browser?: string | MapLike -} - -export function parsePartialPackageJsonDefinition( - value: unknown, -): ParseResult { - return applicative4Either( - (name, main, module, browser) => { - let result: PartialPackageJsonDefinition = {} - setOptionalProp(result, 'name', name) - setOptionalProp(result, 'main', main) - setOptionalProp(result, 'module', module) - setOptionalProp(result, 'browser', browser) - return result - }, - optionalObjectKeyParser(parseString, 'name')(value), - optionalObjectKeyParser(parseString, 'main')(value), - optionalObjectKeyParser(parseString, 'module')(value), - optionalObjectKeyParser( - parseAlternative>( - [ - parseString, - parseObject( - parseAlternative( - [parseString, parseFalse], - `package.browser replacement entries must be either string or 'false' for ignoring a package`, - ), - ), - ], - 'package.browser field must either be a string or an object with type {[key: string]: string}', - ), - 'browser', - )(value), - ) -} - -function processPackageJson( - potentiallyJsonCode: string, - containerFolder: string[], -): ResolveResult> { - let possiblePackageJson: ParseResult - try { - possiblePackageJson = getPartialPackageJson(potentiallyJsonCode) - } catch { - return resolveNotPresent + } } - return foldEither( - (_) => resolveNotPresent, - (packageJson) => { - const moduleName: string | null = packageJson.name ?? null - const browserEntry: string | MapLike | null = packageJson.browser ?? null - const mainEntry: string | null = packageJson.main ?? null - const moduleEntry: string | null = packageJson.module ?? null - - if (browserEntry != null && typeof browserEntry === 'string') { - return resolveSuccess( - normalizePath([...containerFolder, ...getPartsFromPath(browserEntry)]), - ) - } else if (mainEntry != null) { - return resolveSuccess(normalizePath([...containerFolder, ...getPartsFromPath(mainEntry)])) - } else if (moduleEntry != null) { - return resolveSuccess(normalizePath([...containerFolder, ...getPartsFromPath(moduleEntry)])) - } else if (moduleName != null) { - return resolveSuccess(normalizePath([...containerFolder, ...getPartsFromPath(moduleName)])) - } else if (containerFolder.length > 0) { - return resolveSuccess(normalizePath(containerFolder)) - } - - return resolveNotPresent - }, - possiblePackageJson, - ) } -function resolvePackageJson( - fileLookupFn: FileLookupFn, - packageJsonFolder: string[], -): FileLookupResult { - const normalizedFolderPath = normalizePath(packageJsonFolder) - const folderPackageJson = fileLookupFn([...normalizedFolderPath, 'package.json'], true) - if (isResolveSuccess(folderPackageJson) && isEsCodeFile(folderPackageJson.success.file)) { - const mainEntryPath = processPackageJson( - folderPackageJson.success.file.fileContents, - normalizedFolderPath, - ) - if (isResolveSuccess(mainEntryPath)) { - // try loading the entry path as a file - const mainEntryResult = fileLookupFn(mainEntryPath.success, false) - if (isResolveSuccess(mainEntryResult)) { - return mainEntryResult - } else { - // fallback to loading it as a folder with an index.js - const indexJsPath = [...mainEntryPath.success, 'index'] - return fileLookupFn(indexJsPath, false) - } - } else { +function nodeModulesFileForPathParts(nodeModules: NodeModules): FileForPathParts { + return (pathParts: string[]) => { + const filename = makePathFromParts(pathParts) + const nodeModulesFile = nodeModules[filename] + if (nodeModulesFile == null) { return resolveNotPresent + } else { + return fileLookupResult(filename, nodeModulesFile) } } - return resolveNotPresent } -function findPackageJsonForPath(fileLookupFn: FileLookupFn, origin: string[]): FileLookupResult { +function findPackageJsonForPath( + fileLookupFn: FileLookupFn, + originPathParts: string[], +): FileLookupResult { // 1. look for /package.json - const packageJsonResult = fileLookupFn([...origin, 'package.json'], true) + const packageJsonResult = fileLookupFn(`./package.json`, makePathFromParts(originPathParts)) if (isResolveSuccess(packageJsonResult)) { return packageJsonResult } else { // 2. repeat in the parent folder - if (origin.length === 0) { + if (originPathParts.length === 0) { // we exhausted all folders without success return resolveNotPresent } else { - return findPackageJsonForPath(fileLookupFn, origin.slice(0, -1)) - } - } -} - -function resolveNonRelativeModule( - fileLookupFn: FileLookupFn, - importOrigin: string[], - toImport: string[], -): FileLookupResult { - const pathElements = [...importOrigin, 'node_modules', ...toImport] - // 1. look for ./node_modules/.js - const packageNameResult = fileLookupFn(pathElements, false) - if (isResolveSuccess(packageNameResult)) { - return packageNameResult - } else { - // 2. look for ./node_modules//package.json - const packageJsonResult = resolvePackageJson(fileLookupFn, pathElements) - if (isResolveSuccess(packageJsonResult)) { - return packageJsonResult - } else { - // 3. look for ./node_modules//index.js - const indexJsPath = [...pathElements, 'index'] - const indexJSResult = fileLookupFn(indexJsPath, false) - if (isResolveSuccess(indexJSResult)) { - return indexJSResult - } else { - // 4. repeat in the parent folder - if (importOrigin.length === 0) { - // we exhausted all folders without success - return resolveNotPresent - } else { - return resolveNonRelativeModule(fileLookupFn, importOrigin.slice(0, -1), toImport) - } - } - } - } -} - -function resolveRelativeModule( - fileLookupFn: FileLookupFn, - importOrigin: string[], - toImport: string[], -): FileLookupResult { - const pathElements = [...importOrigin, ...toImport] - // 1. look for a file named - const importNameResult = fileLookupFn(pathElements, false) - if (isResolveSuccess(importNameResult)) { - return importNameResult - } else { - // 2. look for /package.json - const packageJsonResult = resolvePackageJson(fileLookupFn, pathElements) - if (isResolveSuccess(packageJsonResult)) { - return packageJsonResult - } else { - // 3. look for /index.js - return fileLookupFn([...pathElements, 'index'], false) - } - } -} - -// Module resolution logic based on what Node / Typescript does https://www.typescriptlang.org/docs/handbook/module-resolution.html#node - -// TODO there's way more module resolution rules here https://parceljs.org/module_resolution.html#module-resolution - -// TODO an even more comprehensive writeup https://nodejs.org/api/modules.html#modules_all_together - -function resolveModuleInternal( - fileLookupFn: FileLookupFn, - importOrigin: string, - toImport: string, -): FileLookupResult { - if (toImport.startsWith('/')) { - // absolute import - return resolveRelativeModule( - fileLookupFn, - [], // this import is relative to the root - getPartsFromPath(toImport), - ) - } else { - if (toImport.startsWith('.')) { - return resolveRelativeModule( - fileLookupFn, - getPartsFromPath(importOrigin).slice(0, -1), - getPartsFromPath(toImport), - ) - } else { - return resolveNonRelativeModule( - fileLookupFn, - getPartsFromPath(importOrigin).slice(0, -1), - getPartsFromPath(toImport), - ) + return findPackageJsonForPath(fileLookupFn, originPathParts.slice(0, -1)) } } } @@ -482,15 +375,64 @@ function resolveModuleAndApplySubstitutions( importOrigin: string, toImport: string, ): FileLookupResult { - function lookupFn(path: string[], direct: boolean): FileLookupResult { - const nodeModulesResult = nodeModulesFileLookup(nodeModules, path, direct) - if (isResolveSuccess(nodeModulesResult)) { - return nodeModulesResult - } else { - return projectContentsFileLookup(projectContents, path, direct) + // require(X) from module at path Y + function lookupFn(path: string, innerImportOrigin: string): FileLookupResult { + // 1 Check if it's a core module - this first step is irrelevant to us, since the browser has no core modules + if (path.startsWith('/') || path.startsWith('./') || path.startsWith('../')) { + // pathForLookup here is Y + X + const pathForLookup = path.startsWith('/') // If X begins with '/' set Y to be the file system root + ? path + : `${stripTrailingSlash(innerImportOrigin)}/${path}` + const pathParts = getPartsFromPath(pathForLookup) + + // LOAD_AS_FILE(Y + X) + const projectContentsFileLookupResult = loadAsFile(projectContents, pathParts) + + if (isResolveSuccess(projectContentsFileLookupResult)) { + return projectContentsFileLookupResult + } + + // LOAD_AS_DIRECTORY(Y + X) + const projectContentsDirectoryLookupResult = loadAsDirectory(projectContents, pathParts) + if (isResolveSuccess(projectContentsDirectoryLookupResult)) { + return projectContentsDirectoryLookupResult + } + } else if (path.startsWith('#')) { + // If X begins with '#' + // LOAD_PACKAGE_IMPORTS(X, dirname(Y)) + return loadPackageImports( + path, + innerImportOrigin, + innerImportOrigin.startsWith('/node_modules') + ? nodeModulesFileForPathParts(nodeModules) + : projectContentsFileForPathParts(projectContents), + lookupFn, + ) + } + + // LOAD_PACKAGE_SELF(X, dirname(Y)) + const packageExportsResult = loadPackageSelf( + path, + innerImportOrigin, + projectContentsFileForPathParts(projectContents), + lookupFn, + ) + + if (isResolveSuccess(packageExportsResult)) { + return packageExportsResult } + + // LOAD_NODE_MODULES(X, dirname(Y)), returning the result here either way in place of `throw 'not fount'` + return loadNodeModules( + nodeModules, + path, + innerImportOrigin.startsWith('/node_modules') ? innerImportOrigin : '/node_modules', + lookupFn, + ) } + // We first need to check for the presence of a `browser` field in the current module's package.json, as that + // can suggest an alternative mapping to use for modules it is importing https://github.com/defunctzombie/package-browser-field-spec const substitutedImport: Either = findSubstitutionsForImport( lookupFn, importOrigin, @@ -505,7 +447,8 @@ function resolveModuleAndApplySubstitutions( substitutedImport.value, filePathMappings, ) - return resolveModuleInternal(lookupFn, importOrigin, unAliasedImport) + + return lookupFn(unAliasedImport, getParentDirectory(importOrigin)) } } diff --git a/editor/src/core/es-modules/package-manager/package-manager.ts b/editor/src/core/es-modules/package-manager/package-manager.ts index 0e90fe32c94a..e7971183db65 100644 --- a/editor/src/core/es-modules/package-manager/package-manager.ts +++ b/editor/src/core/es-modules/package-manager/package-manager.ts @@ -1,12 +1,7 @@ import type { NodeModules, ESCodeFile } from '../../shared/project-file-types' import { isEsCodeFile, isEsRemoteDependencyPlaceholder } from '../../shared/project-file-types' import type { RequireFn, TypeDefinitions } from '../../shared/npm-dependency-types' -import { - isResolveNotPresent, - isResolveSuccess, - isResolveSuccessIgnoreModule, - resolveModule, -} from './module-resolution' +import { resolveModule } from './module-resolution' import { evaluator } from '../evaluator/evaluator' import { fetchMissingFileDependency } from './fetch-packages' import type { EditorDispatch } from '../../../components/editor/action-types' @@ -17,10 +12,13 @@ import { utopiaApiTypings } from './utopia-api-typings' import { resolveBuiltInDependency } from './built-in-dependencies' import type { ProjectContentTreeRoot } from '../../../components/assets' import { applyLoaders } from '../../webpack-loaders/loaders' -import { string } from 'prop-types' -import { Either } from '../../shared/either' import type { CurriedUtopiaRequireFn } from '../../../components/custom-code/code-file' import type { BuiltInDependencies } from './built-in-dependencies-list' +import { + isResolveNotPresent, + isResolveSuccess, + isResolveSuccessIgnoreModule, +} from './module-resolution-utils' import type { FrameworkHooks } from '../../frameworks/framework-hooks' import { getFrameworkHooks } from '../../frameworks/framework-hooks' diff --git a/editor/src/core/es-modules/test-cases/module-resolution-examples.json b/editor/src/core/es-modules/test-cases/module-resolution-examples.json index a0bf7334fa06..f094135cf88f 100644 --- a/editor/src/core/es-modules/test-cases/module-resolution-examples.json +++ b/editor/src/core/es-modules/test-cases/module-resolution-examples.json @@ -230,25 +230,70 @@ "isModule": false, "requires": [] }, - "my-package-with-package-root/src/deep-folder/moduleA.js": { + "/node_modules/module-with-exports-and-imports/package.json": { + "content": "{\"name\": \"module-with-exports-and-imports\", \"main\": \"index.main.js\", \"exports\": \"index.exports.js\", \"imports\": {\"#dep\": \"./dep.js\"}}", + "isModule": false, + "requires": [] + }, + "/node_modules/module-with-exports-and-imports/index.main.js": { "content": "exports.cica = 'kitten';", "isModule": false, "requires": [] }, - "my-package-with-package-root/src/other-folder/moduleB.js": { + "/node_modules/module-with-exports-and-imports/dep.js": { "content": "exports.cica = 'kitten';", "isModule": false, "requires": [] }, - "my-package-with-package-root/index.js": { + "/node_modules/module-with-exports-and-imports-objects/package.json": { + "content": "{\"name\": \"module-with-exports-and-imports-objects\", \"main\": \"index.main.js\", \"exports\": {\".\": \"index.exports.js\", \"./submodule\": \"./src/submodule.js\"}, \"imports\": {\"#dep\": {\"default\": \"./dep.js\"}}}", + "isModule": false, + "requires": [] + }, + "/node_modules/module-with-exports-and-imports-objects/index.main.js": { "content": "exports.cica = 'kitten';", "isModule": false, "requires": [] }, - "my-package-with-package-root/package.json": { + "/node_modules/module-with-exports-and-imports-objects/dep.js": { + "content": "exports.cica = 'kitten';", + "isModule": false, + "requires": [] + }, + "/node_modules/module-with-exports-and-imports-objects/src/submodule.js": { + "content": "exports.cica = 'kitten';", + "isModule": false, + "requires": [] + }, + "/node_modules/my-package-with-package-root/src/deep-folder/moduleA.js": { + "content": "exports.cica = 'kitten';", + "isModule": false, + "requires": [] + }, + "/node_modules/my-package-with-package-root/src/other-folder/moduleB.js": { + "content": "exports.cica = 'kitten';", + "isModule": false, + "requires": [] + }, + "/node_modules/my-package-with-package-root/index.js": { + "content": "exports.cica = 'kitten';", + "isModule": false, + "requires": [] + }, + "/node_modules/my-package-with-package-root/package.json": { "content": "{\"name\": \"my-package-with-package-root\", \"main\": \".index.js\"}", "isModule": false, "requires": [] + }, + "/node_modules/clsx/package.json": { + "content": "{\"name\": \"clsx\", \"main\": \"dist/clsx.js\", \"exports\": {\".\": {\"default\": { \"default\": \"./dist/clsx.js\"}}}}", + "isModule": false, + "requires": [] + }, + "/node_modules/clsx/dist/clsx.js": { + "content": "exports.cica = 'kitten';", + "isModule": false, + "requires": [] } }, "dependency": { diff --git a/editor/src/core/frameworks/framework-hooks-vite.ts b/editor/src/core/frameworks/framework-hooks-vite.ts index c5b67b7cfe2f..fd6fd2992af5 100644 --- a/editor/src/core/frameworks/framework-hooks-vite.ts +++ b/editor/src/core/frameworks/framework-hooks-vite.ts @@ -9,8 +9,9 @@ import type { ComponentToImport, CreationDataFromProject } from '../model/storyb import { namedComponentToImport, PossiblyMainComponentNames } from '../model/storyboard-utils' import { mergeImports } from '../workers/common/project-file-utils' import { absolutePathFromRelativePath } from '../../utils/path-utils' -import type { FileLookupResult } from '../es-modules/package-manager/module-resolution' -import { isResolveSuccess, resolveModule } from '../es-modules/package-manager/module-resolution' +import type { FileLookupResult } from '../es-modules/package-manager/module-resolution-utils' +import { isResolveSuccess } from '../es-modules/package-manager/module-resolution-utils' +import { resolveModule } from '../es-modules/package-manager/module-resolution' import { getMainScriptElement, getRootElement, parseHtml } from '../shared/dom-utils' export class ViteFrameworkHooks implements FrameworkHooks { diff --git a/editor/src/utils/path-utils.ts b/editor/src/utils/path-utils.ts index 13dad8741a71..33fbc1fd77e9 100644 --- a/editor/src/utils/path-utils.ts +++ b/editor/src/utils/path-utils.ts @@ -33,7 +33,7 @@ export function getParentDirectory(filepath: string): string { return makePathFromParts(getPartsFromPath(filepath).slice(0, -1)) } -type SubPathCache = { [key: string]: PathCache } +type SubPathCache = Map interface PathCache { cachedString: string @@ -47,7 +47,7 @@ function pathCache(cachedString: string, subPathCache: SubPathCache): PathCache } } -let rootPathCache: SubPathCache = {} +let rootPathCache: SubPathCache = new Map() function getPathFromCache(parts: Array): string { let workingSubCache: SubPathCache = rootPathCache @@ -55,15 +55,14 @@ function getPathFromCache(parts: Array): string { let partsSoFar: Array = [] for (const part of parts) { partsSoFar.push(part) - if (part in workingSubCache) { - // The `in` check above proves this does not return `undefined`. - workingPathCache = workingSubCache[part]! + if (workingSubCache.has(part)) { + workingPathCache = workingSubCache.get(part)! workingSubCache = workingPathCache.subPathCache } else { const cachedString = `/${partsSoFar.join('/')}` - const newPathCache = pathCache(cachedString, {}) + const newPathCache = pathCache(cachedString, new Map()) workingPathCache = newPathCache - workingSubCache[part] = newPathCache + workingSubCache.set(part, newPathCache) workingSubCache = newPathCache.subPathCache } }