🪪 Allow custom licenses outside SPDX (#1563)

.changeset/orange-dingos-promise.md

@@ -0,0 +1,6 @@
+---
+'myst-to-md': patch
+'myst-cli': patch
+---
+
+Consume new license simplification function

.changeset/unlucky-ties-reflect.md

@@ -0,0 +1,5 @@
+---
+'myst-frontmatter': patch
+---
+
+Allow custom licenses outside SPDX

docs/frontmatter.md

@@ -636,7 +636,7 @@ This field can be set to a string value directly or to a License object.
 
 Available fields in the License object are `content` and `code` allowing licenses to be set separately for these two forms of content, as often different subsets of licenses are applicable to each. If you only wish to apply a single license to your page or project use the string form rather than an object.
 
-String values for licenses should be a valid “Identifier” string from the [SPDX License List](https://spdx.org/licenses/). Identifiers for well-known licenses are easily recognizable (e.g. `MIT` or `BSD`) and MyST will attempt to infer the specific identifier if an ambiguous license is specified (e.g. `GPL` will be interpreted as `GPL-3.0+` and a warning raised letting you know of this interpretation). Some common licenses are:
+If selecting a license from the [SPDX License List](https://spdx.org/licenses/), you may simply use the “Identifier” string; MyST will expand these identifiers into objects with `name`, `url`, and additional metadata related to open access ([OSI-approved](https://opensource.org/licenses), [FSF free](https://www.gnu.org/licenses/license-list.en.html), and [CC](https://creativecommons.org/)). Identifiers for well-known licenses are easily recognizable (e.g. `MIT` or `BSD`) and MyST will attempt to infer the specific identifier if an ambiguous license is specified (e.g. `GPL` will be interpreted as `GPL-3.0+` and a warning raised letting you know of this interpretation). Some common licenses are:
 
 ```{list-table}
 :header-rows: 1
@@ -658,7 +658,25 @@ String values for licenses should be a valid “Identifier” string from the [S
     - `AGPL`
 ```
 
-By using the correct SPDX Identifier, your website will automatically use the appropriate icon for the license and link to the license definition.
+By using the correct SPDX Identifier, your website will automatically use the appropriate icon for the license and link to the license definition.  The simplest and most common example is something like:
+
+```yaml
+license: CC-BY-4.0
+```
+
+### Nonstandard licenses
+
+Although not recommended, you may specify nonstandard licenses not found on the SPDX License List. For these, you may provide an object where available fields are `id`, `name`, `url`, and `note`. You can also extend the default SPDX Licenses by providing modified values for these fields. Here is a more complex example where content and code have different licenses; content uses an SPDX License with an additional note, and code uses a totally custom license.
+
+```yaml
+license:
+  content:
+    id: CC-BY-4.0
+    note: When attributing this content, please indicate the Source was MyST Documentation.
+  code:
+    name: I Am Not A Lawyer License
+    url: https://example.com/i-am-not-a-lawyer
+```
 
 (frontmatter:funding)=
 

packages/myst-cli/src/frontmatter.ts

@@ -3,7 +3,7 @@ import type { Export, Licenses, PageFrontmatter } from 'myst-frontmatter';
 import {
   validateExportsList,
   fillPageFrontmatter,
-  licensesToString,
+  simplifyLicenses,
   unnestKernelSpec,
   validatePageFrontmatter,
 } from 'myst-frontmatter';
@@ -102,7 +102,7 @@ export function processPageFrontmatter(
 
 export function prepareToWrite(frontmatter: { license?: Licenses }) {
   if (!frontmatter.license) return { ...frontmatter };
-  return { ...frontmatter, license: licensesToString(frontmatter.license) };
+  return { ...frontmatter, license: simplifyLicenses(frontmatter.license) };
 }
 
 export function getExportListFromRawFrontmatter(

packages/myst-frontmatter/package.json

@@ -43,6 +43,7 @@
     "@types/spdx-correct": "^3.1.0",
     "glob": "^10.3.1",
     "js-yaml": "^4.1.0",
-    "moment": "^2.29.4"
+    "moment": "^2.29.4",
+    "node-fetch": "^3.3.2"
   }
 }

packages/myst-frontmatter/src/licenses/licenses.spec.ts

@@ -2,7 +2,12 @@ import { describe, expect, beforeEach, it } from 'vitest';
 import type { ValidationOptions } from 'simple-validators';
 import licenses from './licenses';
 import fetch from 'node-fetch';
-import { licensesToString, validateLicense, validateLicenses } from './validators';
+import {
+  licensesToString,
+  simplifyLicenses,
+  validateLicense,
+  validateLicenses,
+} from './validators';
 
 const TEST_LICENSE = {
   name: 'Creative Commons Attribution 4.0 International',
@@ -20,7 +25,7 @@ beforeEach(() => {
 
 describe('licenses are upto date with SPDX', () => {
   it('compare with https://spdx.org/licenses/licenses.json', async () => {
-    const data = await (await fetch('https://spdx.org/licenses/licenses.json')).json();
+    const data: any = await (await fetch('https://spdx.org/licenses/licenses.json')).json();
     const onlineLicenses = Object.fromEntries(
       (data.licenses as any[])
         .filter((l) => !l.isDeprecatedLicenseId)
@@ -41,28 +46,29 @@ describe('licenses are upto date with SPDX', () => {
 });
 
 describe('validateLicense', () => {
-  it('invalid string errors', () => {
-    expect(validateLicense('', opts)).toEqual(undefined);
-    expect(opts.messages.errors?.length).toEqual(1);
+  it('non-spdx string warns', () => {
+    expect(validateLicense('', opts)).toEqual({ id: '' });
+    expect(opts.messages.warnings?.length).toEqual(1);
   });
   it('valid string coerces', () => {
     expect(validateLicense('CC-BY-4.0', opts)).toEqual(TEST_LICENSE);
   });
   it('valid license object passes', () => {
     expect(validateLicense(TEST_LICENSE, opts)).toEqual(TEST_LICENSE);
   });
-  it('invalid license object fails', () => {
-    expect(validateLicense({ ...TEST_LICENSE, url: 'https://example.com' }, opts)).toEqual(
-      undefined,
-    );
-    expect(opts.messages.errors?.length).toEqual(1);
+  it('spdx license with incorrect url warns', () => {
+    expect(validateLicense({ ...TEST_LICENSE, url: 'https://example.com' }, opts)).toEqual({
+      ...TEST_LICENSE,
+      url: 'https://example.com',
+    });
+    expect(opts.messages.warnings?.length).toEqual(1);
   });
 });
 
 describe('validateLicenses', () => {
-  it('invalid string errors', () => {
-    expect(validateLicenses('', opts)).toEqual({});
-    expect(opts.messages.errors?.length).toEqual(1);
+  it('non-spdx string warns', () => {
+    expect(validateLicenses('', opts)).toEqual({ content: { id: '' } });
+    expect(opts.messages.warnings?.length).toEqual(1);
   });
   it('corrects known licenses', () => {
     expect(validateLicenses('apache 2', opts)).toEqual({
@@ -75,12 +81,13 @@ describe('validateLicenses', () => {
       },
     });
   });
-  it('invalid content string errors', () => {
-    expect(validateLicenses({ content: '' }, opts)).toEqual({});
-    expect(opts.messages.errors?.length).toEqual(1);
+  it('non-spdx content string warns', () => {
+    expect(validateLicenses({ content: '' }, opts)).toEqual({ content: { id: '' } });
+    expect(opts.messages.warnings?.length).toEqual(1);
   });
-  it('empty object returns self', () => {
-    expect(validateLicenses({}, opts)).toEqual({});
+  it('empty object warns and returns undefined', () => {
+    expect(validateLicenses({}, opts)).toEqual(undefined);
+    expect(opts.messages.warnings?.length).toEqual(1);
   });
   it('valid string coerces', () => {
     expect(validateLicenses('CC-BY-4.0', opts)).toEqual({
@@ -180,4 +187,183 @@ describe('licensesToString', () => {
       }),
     ).toEqual({ content: 'CC-BY-SA-4.0', code: 'CC-BY-ND-4.0' });
   });
+  it('custom licenses lose info', () => {
+    expect(
+      licensesToString({
+        content: {
+          name: 'My Custom License',
+          id: 'my-custom-license',
+          url: 'https://example.com',
+        },
+        code: {
+          name: 'Creative Commons Attribution No Derivatives 4.0 International',
+          id: 'CC-BY-ND-4.0',
+          CC: true,
+          url: 'https://example.com',
+          note: 'Using a CC license was probably a better idea...',
+        },
+      }),
+    ).toEqual({ content: 'my-custom-license', code: 'CC-BY-ND-4.0' });
+  });
+  it('license without id is lost', () => {
+    expect(
+      licensesToString({
+        content: {
+          name: 'Creative Commons Attribution Share Alike 4.0 International',
+          CC: true,
+          free: true,
+          url: 'https://example.com',
+        },
+        code: {
+          name: 'Creative Commons Attribution No Derivatives 4.0 International',
+          id: 'CC-BY-ND-4.0',
+          CC: true,
+          url: 'https://example.com',
+        },
+      }),
+    ).toEqual({ code: 'CC-BY-ND-4.0' });
+  });
+  it('licenses without ids are lost', () => {
+    expect(
+      licensesToString({
+        content: {
+          name: 'Creative Commons Attribution Share Alike 4.0 International',
+          CC: true,
+          free: true,
+          url: 'https://example.com',
+        },
+        code: {
+          name: 'Creative Commons Attribution No Derivatives 4.0 International',
+          CC: true,
+          url: 'https://example.com',
+        },
+      }),
+    ).toEqual(undefined);
+  });
+});
+
+describe('simplifyLicenses', () => {
+  it('empty licenses returns self', () => {
+    expect(simplifyLicenses({})).toEqual({});
+  });
+  it('content licenses returns content string only', () => {
+    expect(
+      simplifyLicenses({
+        content: TEST_LICENSE,
+      }),
+    ).toEqual(TEST_LICENSE.id);
+  });
+  it('code licenses returns code string', () => {
+    expect(
+      simplifyLicenses({
+        code: TEST_LICENSE,
+      }),
+    ).toEqual({ code: TEST_LICENSE.id });
+  });
+  it('matching content/code licenses returns string', () => {
+    expect(
+      simplifyLicenses({
+        content: TEST_LICENSE,
+        code: TEST_LICENSE,
+      }),
+    ).toEqual(TEST_LICENSE.id);
+  });
+  it('content/code licenses returns content/code strings', () => {
+    expect(
+      simplifyLicenses({
+        content: TEST_LICENSE,
+        code: {
+          name: 'Creative Commons Attribution No Derivatives 4.0 International',
+          id: 'CC-BY-ND-4.0',
+          CC: true,
+          url: 'https://creativecommons.org/licenses/by-nd/4.0/',
+        },
+      }),
+    ).toEqual({ content: TEST_LICENSE.id, code: 'CC-BY-ND-4.0' });
+  });
+  it('custom licenses do not lose info!', () => {
+    expect(
+      simplifyLicenses({
+        content: {
+          name: 'My Custom License',
+          id: 'my-custom-license',
+          url: 'https://example.com',
+        },
+        code: {
+          ...TEST_LICENSE,
+          url: 'https://example.com',
+          note: 'Using a CC license was probably a better idea...',
+        },
+      }),
+    ).toEqual({
+      content: {
+        name: 'My Custom License',
+        id: 'my-custom-license',
+        url: 'https://example.com',
+      },
+      code: {
+        id: TEST_LICENSE.id,
+        url: 'https://example.com',
+        note: 'Using a CC license was probably a better idea...',
+      },
+    });
+  });
+  it('license without id persist!', () => {
+    expect(
+      simplifyLicenses({
+        content: {
+          name: 'Creative Commons Attribution Share Alike 4.0 International',
+          CC: true,
+          free: true,
+          url: 'https://example.com',
+        },
+        code: {
+          name: 'Creative Commons Attribution No Derivatives 4.0 International',
+          id: 'CC-BY-ND-4.0',
+          CC: true,
+          url: 'https://example.com',
+        },
+      }),
+    ).toEqual({
+      content: {
+        name: 'Creative Commons Attribution Share Alike 4.0 International',
+        CC: true,
+        free: true,
+        url: 'https://example.com',
+      },
+      code: {
+        id: 'CC-BY-ND-4.0',
+        url: 'https://example.com',
+      },
+    });
+  });
+  it('licenses without ids persist!', () => {
+    expect(
+      simplifyLicenses({
+        content: {
+          name: 'Creative Commons Attribution Share Alike 4.0 International',
+          CC: true,
+          free: true,
+          url: 'https://example.com',
+        },
+        code: {
+          name: 'Creative Commons Attribution No Derivatives 4.0 International',
+          CC: true,
+          url: 'https://example.com',
+        },
+      }),
+    ).toEqual({
+      content: {
+        name: 'Creative Commons Attribution Share Alike 4.0 International',
+        CC: true,
+        free: true,
+        url: 'https://example.com',
+      },
+      code: {
+        name: 'Creative Commons Attribution No Derivatives 4.0 International',
+        CC: true,
+        url: 'https://example.com',
+      },
+    });
+  });
 });