diff --git a/bun.lockb b/bun.lockb index 836a209..3b0fc62 100755 Binary files a/bun.lockb and b/bun.lockb differ diff --git a/package.json b/package.json index 5088bd4..31024c4 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "google-font-metadata", "description": "A metadata generator for Google Fonts.", - "version": "6.0.2", + "version": "6.0.3", "author": "Ayuhito ", "main": "./dist/index.js", "module": "./dist/index.mjs", @@ -25,14 +25,13 @@ "unicode range" ], "dependencies": { + "@evan/concurrency": "^0.0.3", "@octokit/core": "^6.1.2", - "@types/stylis": "^4.2.7", "cac": "^6.7.14", "consola": "^3.3.3", "deepmerge": "^4.3.1", "json-stringify-pretty-compact": "^4.0.0", "linkedom": "^0.18.6", - "p-queue": "^8.0.1", "pathe": "^1.1.2", "picocolors": "^1.1.1", "playwright": "^1.49.1", @@ -41,9 +40,9 @@ }, "devDependencies": { "@biomejs/biome": "1.9.4", - "@playwright/test": "^1.49.1", "@types/bun": "latest", "@types/node": "^22.10.2", + "@types/stylis": "^4.2.7", "c8": "^10.1.3", "magic-string": "^0.30.17", "msw": "^2.7.0", @@ -58,8 +57,7 @@ "test": "vitest", "test:generate-fixtures": "bun run ./tests/utils/generate-css-fixtures", "coverage": "vitest --coverage", - "format": "biome format --fix", - "lint": "biome lint --fix", + "lint": "biome check --fix", "prepublishOnly": "bunx biome ci && bun run build" }, "files": ["dist/*", "data/*"], diff --git a/src/api-parser-v1.ts b/src/api-parser-v1.ts index 0e734fb..2dba267 100644 --- a/src/api-parser-v1.ts +++ b/src/api-parser-v1.ts @@ -1,19 +1,23 @@ import * as fs from 'node:fs/promises'; import { fileURLToPath } from 'node:url'; +import { Limiter } from '@evan/concurrency'; import { consola } from 'consola'; import stringify from 'json-stringify-pretty-compact'; -import PQueue from 'p-queue'; import { dirname, join } from 'pathe'; import { compile } from 'stylis'; import { apiv1 as userAgents } from '../data/user-agents.json'; import { APIDirect, APIv1 } from './data'; +import { LOOP_LIMIT, addError, checkErrors } from './errors'; import type { APIResponse, FontObjectV1 } from './types'; import { orderObject, weightListGen } from './utils'; import { validate } from './validate'; const baseurl = 'https://fonts.googleapis.com/css?subset='; +const queue = Limiter(18); + +const results: FontObjectV1[] = []; export const fetchCSS = async ( font: APIResponse, @@ -49,10 +53,10 @@ export const fetchCSS = async ( return cssMap.join(''); }; +// Download CSS stylesheets for each file format export const fetchAllCSS = async ( font: APIResponse, ): Promise<[string, string, string]> => - // Download CSS stylesheets for each file format await Promise.all([ fetchCSS(font, userAgents.woff2), fetchCSS(font, userAgents.woff), @@ -176,35 +180,32 @@ export const processCSS = ( return fontObject; }; -const results: FontObjectV1[] = []; - -const processQueue = async (font: APIResponse, force: boolean) => { - const id = font.family.replaceAll(/\s/g, '-').toLowerCase(); - - // If last-modified matches latest API, skip fetching CSS and processing. - if ( - APIv1[id] !== undefined && - font.lastModified === APIv1[id].lastModified && - !force - ) { - results.push({ [id]: APIv1[id] }); - } else { - const css = await fetchAllCSS(font); - const fontObject = processCSS(css, font); - results.push(fontObject); - consola.info(`Updated ${id}`); +const processQueue = async ( + font: APIResponse, + force: boolean, +): Promise => { + try { + const id = font.family.replaceAll(/\s/g, '-').toLowerCase(); + + // If last-modified matches latest API, skip fetching CSS and processing. + if ( + APIv1[id] !== undefined && + font.lastModified === APIv1[id].lastModified && + !force + ) { + results.push({ [id]: APIv1[id] }); + } else { + const css = await fetchAllCSS(font); + const fontObject = processCSS(css, font); + results.push(fontObject); + consola.info(`Updated ${id}`); + } + consola.success(`Parsed ${id}`); + } catch (error) { + addError(`${font.family} experienced an error. ${String(error)}`); } - consola.success(`Parsed ${id}`); }; -// Queue control -const queue = new PQueue({ concurrency: 18 }); - -// @ts-ignore - rollup-plugin-dts fails to compile this typing -queue.on('error', (error: Error) => { - consola.error(error); -}); - /** * Parses the fetched API data and writes it to the APIv1 JSON dataset. * @param force - Force update all fonts without using cache. @@ -212,33 +213,30 @@ queue.on('error', (error: Error) => { */ export const parsev1 = async (force: boolean, noValidate: boolean) => { for (const font of APIDirect) { - try { - queue.add(async () => { - await processQueue(font, force); - }); - } catch (error) { - throw new Error(`${font.family} experienced an error. ${String(error)}`); - } + checkErrors(LOOP_LIMIT); + queue.add(() => processQueue(font, force)); } - await queue.onIdle().then(async () => { - // Order the font objects alphabetically for consistency and not create huge diffs - const unordered: FontObjectV1 = Object.assign({}, ...results); - const ordered = orderObject(unordered); - if (!noValidate) { - validate('v1', ordered); - } + await queue.flush(); + checkErrors(); - await fs.writeFile( - join( - dirname(fileURLToPath(import.meta.url)), - '../data/google-fonts-v1.json', - ), - stringify(ordered), - ); - - consola.success( - `All ${results.length} font datapoints using CSS APIv1 have been generated.`, - ); - }); + // Order the font objects alphabetically for consistency and not create huge diffs + const unordered: FontObjectV1 = Object.assign({}, ...results); + const ordered = orderObject(unordered); + + if (!noValidate) { + validate('v1', ordered); + } + + await fs.writeFile( + join( + dirname(fileURLToPath(import.meta.url)), + '../data/google-fonts-v1.json', + ), + stringify(ordered), + ); + + consola.success( + `All ${results.length} font datapoints using CSS APIv1 have been generated.`, + ); }; diff --git a/src/api-parser-v2.ts b/src/api-parser-v2.ts index e49d15b..7e50b66 100644 --- a/src/api-parser-v2.ts +++ b/src/api-parser-v2.ts @@ -1,19 +1,23 @@ import * as fs from 'node:fs/promises'; import { fileURLToPath } from 'node:url'; +import { Limiter } from '@evan/concurrency'; import { consola } from 'consola'; import stringify from 'json-stringify-pretty-compact'; -import PQueue from 'p-queue'; import { dirname, join } from 'pathe'; import { compile } from 'stylis'; import { apiv2 as userAgents } from '../data/user-agents.json'; import { APIDirect, APIv2 } from './data'; +import { LOOP_LIMIT, addError, checkErrors } from './errors'; import type { APIResponse, FontObjectV2 } from './types'; import { orderObject, weightListGen } from './utils'; import { validate } from './validate'; const baseurl = 'https://fonts.googleapis.com/css2?family='; +const queue = Limiter(18); + +const results: FontObjectV2[] = []; export const fetchCSS = async ( fontFamily: string, @@ -234,35 +238,32 @@ export const processCSS = ( return fontObject; }; -const results: FontObjectV2[] = []; - -const processQueue = async (font: APIResponse, force: boolean) => { - const id = font.family.replaceAll(/\s/g, '-').toLowerCase(); - - // If last-modified matches latest API, skip fetching CSS and processing. - if ( - APIv2[id] !== undefined && - font.lastModified === APIv2[id].lastModified && - !force - ) { - results.push({ [id]: APIv2[id] }); - } else { - const css = await fetchAllCSS(font); - const fontObject = processCSS(css, font); - results.push(fontObject); - consola.info(`Updated ${id}`); +const processQueue = async ( + font: APIResponse, + force: boolean, +): Promise => { + try { + const id = font.family.replaceAll(/\s/g, '-').toLowerCase(); + + // If last-modified matches latest API, skip fetching CSS and processing. + if ( + APIv2[id] !== undefined && + font.lastModified === APIv2[id].lastModified && + !force + ) { + results.push({ [id]: APIv2[id] }); + } else { + const css = await fetchAllCSS(font); + const fontObject = processCSS(css, font); + results.push(fontObject); + consola.info(`Updated ${id}`); + } + consola.success(`Parsed ${id}`); + } catch (error) { + addError(`${font.family} experienced an error. ${String(error)}`); } - consola.success(`Parsed ${id}`); }; -// Queue control -const queue = new PQueue({ concurrency: 18 }); - -// @ts-ignore - rollup-plugin-dts fails to compile this typing -queue.on('error', (error: Error) => { - consola.error(error); -}); - /** * Parses the fetched API and writes it to the APIv2 dataset. * @param force - Force update all fonts without using cache. @@ -270,33 +271,30 @@ queue.on('error', (error: Error) => { */ export const parsev2 = async (force: boolean, noValidate: boolean) => { for (const font of APIDirect) { - try { - queue.add(async () => { - await processQueue(font, force); - }); - } catch (error) { - throw new Error(`${font.family} experienced an error. ${String(error)}`); - } + checkErrors(LOOP_LIMIT); + queue.add(() => processQueue(font, force)); } - await queue.onIdle().then(async () => { - // Order the font objects alphabetically for consistency and not create huge diffs - const unordered: FontObjectV2 = Object.assign({}, ...results); - const ordered = orderObject(unordered); - if (!noValidate) { - validate('v2', ordered); - } + await queue.flush(); + checkErrors(); - await fs.writeFile( - join( - dirname(fileURLToPath(import.meta.url)), - '../data/google-fonts-v2.json', - ), - stringify(ordered), - ); + // Order the font objects alphabetically for consistency and not create huge diffs + const unordered: FontObjectV2 = Object.assign({}, ...results); + const ordered = orderObject(unordered); - consola.success( - `All ${results.length} font datapoints using CSS APIv2 have been generated.`, - ); - }); + if (!noValidate) { + validate('v2', ordered); + } + + await fs.writeFile( + join( + dirname(fileURLToPath(import.meta.url)), + '../data/google-fonts-v2.json', + ), + stringify(ordered), + ); + + consola.success( + `All ${results.length} font datapoints using CSS APIv2 have been generated.`, + ); }; diff --git a/src/errors.ts b/src/errors.ts new file mode 100644 index 0000000..3734b8e --- /dev/null +++ b/src/errors.ts @@ -0,0 +1,23 @@ +import consola from 'consola'; + +const errs: string[] = []; + +export const LOOP_LIMIT = 5; + +export const addError = (error: string) => { + errs.push(error); +}; + +export const checkErrors = (limit = 0) => { + if (errs.length > limit) { + for (const err of errs) { + consola.error(err); + } + + if (limit > 0) { + throw new Error('Too many errors occurred during parsing. Stopping...'); + } + + throw new Error('Some fonts experienced errors during parsing.'); + } +}; diff --git a/src/icons-parser.ts b/src/icons-parser.ts index 0f95a96..0f3ffaf 100644 --- a/src/icons-parser.ts +++ b/src/icons-parser.ts @@ -1,9 +1,9 @@ import * as fs from 'node:fs/promises'; import { fileURLToPath } from 'node:url'; +import { Limiter } from '@evan/concurrency'; import { consola } from 'consola'; import stringify from 'json-stringify-pretty-compact'; -import PQueue from 'p-queue'; import { dirname, join } from 'pathe'; import { @@ -11,6 +11,7 @@ import { processCSS as processV2CSS, } from './api-parser-v2'; import { APIIconDirect, APIIconStatic, APIIconVariable } from './data'; +import { LOOP_LIMIT, addError, checkErrors } from './errors'; import type { APIIconResponse, FontObjectV2, @@ -23,63 +24,61 @@ import { parseCSS as parseVariableCSS, } from './variable-parser'; +const queue = Limiter(18); + const resultsStatic: FontObjectV2[] = []; const resultsVariable: FontObjectVariable = {}; const processQueue = async (icon: APIIconResponse, force: boolean) => { - const id = icon.family.replaceAll(/\s/g, '-').toLowerCase(); - - // We need to get defSubset to parse out the fallback subset - let defSubset: string | undefined; - - // If last-modified matches latest API, skip fetching CSS and processing. - if ( - APIIconStatic[id] !== undefined && - icon.lastModified === APIIconStatic[id].lastModified && - !force - ) { - resultsStatic.push({ [id]: APIIconStatic[id] }); - defSubset = APIIconStatic[id].defSubset; - } else { - const css = await fetchAllV2CSS(icon); - const iconObject = processV2CSS(css, icon); - resultsStatic.push(iconObject); - defSubset = iconObject[id].defSubset; - consola.info(`Updated static ${id}`); - } + try { + const id = icon.family.replaceAll(/\s/g, '-').toLowerCase(); - // If has variable axes, fetch CSS and process. - if (icon.axes !== undefined) { + // We need to get defSubset to parse out the fallback subset + let defSubset: string | undefined; + + // If last-modified matches latest API, skip fetching CSS and processing. if ( - APIIconVariable[id] !== undefined && + APIIconStatic[id] !== undefined && icon.lastModified === APIIconStatic[id].lastModified && !force ) { - resultsVariable[id] = { ...APIIconVariable[id] }; + resultsStatic.push({ [id]: APIIconStatic[id] }); + defSubset = APIIconStatic[id].defSubset; } else { - const obj = { - family: icon.family, - id, - axes: icon.axes, - }; - - const cssLinks = generateCSSLinks(obj); - const cssTuple = await fetchAllVariableCSS(cssLinks); - const variantsObject = parseVariableCSS(cssTuple, defSubset); - resultsVariable[id] = { ...obj, variants: variantsObject }; - consola.info(`Updated variable ${id}`); + const css = await fetchAllV2CSS(icon); + const iconObject = processV2CSS(css, icon); + resultsStatic.push(iconObject); + defSubset = iconObject[id].defSubset; + consola.info(`Updated static ${id}`); } - } - consola.success(`Parsed ${id}`); -}; -// Queue control -const queue = new PQueue({ concurrency: 18 }); + // If has variable axes, fetch CSS and process. + if (icon.axes !== undefined) { + if ( + APIIconVariable[id] !== undefined && + icon.lastModified === APIIconStatic[id].lastModified && + !force + ) { + resultsVariable[id] = { ...APIIconVariable[id] }; + } else { + const obj = { + family: icon.family, + id, + axes: icon.axes, + }; -// @ts-ignore - rollup-plugin-dts fails to compile this typing -queue.on('error', (error: Error) => { - consola.error(error); -}); + const cssLinks = generateCSSLinks(obj); + const cssTuple = await fetchAllVariableCSS(cssLinks); + const variantsObject = parseVariableCSS(cssTuple, defSubset); + resultsVariable[id] = { ...obj, variants: variantsObject }; + consola.info(`Updated variable ${id}`); + } + } + consola.success(`Parsed ${id}`); + } catch (error) { + addError(`${icon.family} experienced an error. ${String(error)}`); + } +}; /** * Parses the fetched API and writes it to the APIv2 dataset. @@ -88,42 +87,36 @@ queue.on('error', (error: Error) => { */ export const parseIcons = async (force: boolean) => { for (const icon of APIIconDirect) { - try { - queue.add(async () => { - await processQueue(icon, force); - }); - } catch (error) { - throw new Error(`${icon.family} experienced an error. ${String(error)}`); - } + checkErrors(LOOP_LIMIT); + queue.add(() => processQueue(icon, force)); } - await queue.onIdle().then(async () => { - // Order the font objects alphabetically for consistency and not create huge diffs - const unorderedStatic: FontObjectV2 = Object.assign({}, ...resultsStatic); - const orderedStatic = orderObject(unorderedStatic); - - const unorderedVariable: FontObjectVariable = resultsVariable; - const orderedVariable = orderObject(unorderedVariable); - - await fs.writeFile( - join( - dirname(fileURLToPath(import.meta.url)), - '../data/icons-static.json', - ), - stringify(orderedStatic), - ); - - await fs.writeFile( - join( - dirname(fileURLToPath(import.meta.url)), - '../data/icons-variable.json', - ), - stringify(orderedVariable), - ); - - consola.success( - `All ${resultsStatic.length} static + ${ - Object.keys(resultsVariable).length - } variable icon datapoints have been generated.`, - ); - }); + + await queue.flush(); + checkErrors(); + + // Order the font objects alphabetically for consistency and not create huge diffs + const unorderedStatic: FontObjectV2 = Object.assign({}, ...resultsStatic); + const orderedStatic = orderObject(unorderedStatic); + + const unorderedVariable: FontObjectVariable = resultsVariable; + const orderedVariable = orderObject(unorderedVariable); + + await fs.writeFile( + join(dirname(fileURLToPath(import.meta.url)), '../data/icons-static.json'), + stringify(orderedStatic), + ); + + await fs.writeFile( + join( + dirname(fileURLToPath(import.meta.url)), + '../data/icons-variable.json', + ), + stringify(orderedVariable), + ); + + consola.success( + `All ${resultsStatic.length} static + ${ + Object.keys(resultsVariable).length + } variable icon datapoints have been generated.`, + ); }; diff --git a/src/variable-parser.ts b/src/variable-parser.ts index fbeaa30..dda8905 100644 --- a/src/variable-parser.ts +++ b/src/variable-parser.ts @@ -1,14 +1,15 @@ import * as fs from 'node:fs/promises'; import { fileURLToPath } from 'node:url'; +import { Limiter } from '@evan/concurrency'; import { consola } from 'consola'; import stringify from 'json-stringify-pretty-compact'; -import PQueue from 'p-queue'; import { dirname, join } from 'pathe'; import { compile } from 'stylis'; import { apiv2 as userAgents } from '../data/user-agents.json'; import { APIVariableDirect } from './data'; +import { LOOP_LIMIT, addError, checkErrors } from './errors'; import type { FontObjectVariable, FontObjectVariableDirect, @@ -20,6 +21,10 @@ import { validate } from './validate'; export type Links = Record; +const queue = Limiter(10); + +const results: FontObjectVariable = {}; + // CSS API needs axes to given in alphabetical order or request throws e.g. (a,b,c,A,B,C) export const sortAxes = (axesArr: string[]) => { const upper = axesArr @@ -170,9 +175,8 @@ export const generateCSSLinks = (font: FontObjectVariableDirect): Links => { return links; }; +// Download CSS stylesheets using Google Fonts APIv2 export const fetchCSS = async (url: string) => { - // Download CSS stylesheets using Google Fonts APIv2 - const response = await fetch(url, { headers: { 'User-Agent': userAgents.variable, @@ -252,53 +256,44 @@ export const parseCSS = (cssTuple: string[][], defSubset?: string) => { return fontVariants; }; -const results: FontObjectVariable = {}; - const processQueue = async (font: FontObjectVariableDirect) => { - const cssLinks = generateCSSLinks(font); - const cssTuple = await fetchAllCSS(cssLinks); - const variantsObject = parseCSS(cssTuple); - results[font.id] = { ...font, variants: variantsObject }; - consola.success(`Parsed ${font.id}`); + try { + const cssLinks = generateCSSLinks(font); + const cssTuple = await fetchAllCSS(cssLinks); + const variantsObject = parseCSS(cssTuple); + results[font.id] = { ...font, variants: variantsObject }; + consola.success(`Parsed ${font.id}`); + } catch (error) { + addError(`${font.family} experienced an error. ${String(error)}`); + } }; -// Queue control -const queue = new PQueue({ concurrency: 10 }); - -// @ts-ignore - rollup-plugin-dts fails to compile this typing -queue.on('error', (error: Error) => { - consola.error(error); -}); - /** * Parses the scraped variable font data into a usable APIVariable dataset, * @param noValidate - Skip automatic validation of parsed dataset. */ export const parseVariable = async (noValidate: boolean) => { for (const font of APIVariableDirect) { - try { - queue.add(async () => { - await processQueue(font); - }); - } catch (error) { - throw new Error(`${font.family} experienced an error. ${String(error)}`); - } + checkErrors(LOOP_LIMIT); + queue.add(() => processQueue(font)); } - await queue.onIdle().then(async () => { - if (!noValidate) { - validate('variable', results); - } - const ordered = orderObject(results); - await fs.writeFile( - join(dirname(fileURLToPath(import.meta.url)), '../data/variable.json'), - stringify(ordered), - ); + await queue.flush(); + checkErrors(); - consola.success( - `All ${ - Object.keys(results).length - } variable font datapoints have been generated.`, - ); - }); + if (!noValidate) { + validate('variable', results); + } + + const ordered = orderObject(results); + await fs.writeFile( + join(dirname(fileURLToPath(import.meta.url)), '../data/variable.json'), + stringify(ordered), + ); + + consola.success( + `All ${ + Object.keys(results).length + } variable font datapoints have been generated.`, + ); };