Skip to content

Commit

Permalink
perf: replace p-queue with evan/concurrency (#143)
Browse files Browse the repository at this point in the history 
* perf: replace p-queue with evan/concurrency

* refactor: unify queue error handling
  • Loading branch information
@ayuhito
ayuhito authored Jan 5, 2025
1 parent 52574b8 commit b8cafd8
Show file tree
Hide file tree
Showing 7 changed files with 239 additions and 234 deletions.
Binary file modified bun.lockb
View file
Binary file not shown.
10 changes: 4 additions & 6 deletions package.json
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"name": "google-font-metadata",
"description": "A metadata generator for Google Fonts.",
"version": "6.0.2",
"version": "6.0.3",
"author": "Ayuhito <hello@ayuhito.com>",
"main": "./dist/index.js",
"module": "./dist/index.mjs",
Expand All @@ -25,14 +25,13 @@
"unicode range"
],
"dependencies": {
"@evan/concurrency": "^0.0.3",
"@octokit/core": "^6.1.2",
"@types/stylis": "^4.2.7",
"cac": "^6.7.14",
"consola": "^3.3.3",
"deepmerge": "^4.3.1",
"json-stringify-pretty-compact": "^4.0.0",
"linkedom": "^0.18.6",
"p-queue": "^8.0.1",
"pathe": "^1.1.2",
"picocolors": "^1.1.1",
"playwright": "^1.49.1",
Expand All @@ -41,9 +40,9 @@
},
"devDependencies": {
"@biomejs/biome": "1.9.4",
"@playwright/test": "^1.49.1",
"@types/bun": "latest",
"@types/node": "^22.10.2",
"@types/stylis": "^4.2.7",
"c8": "^10.1.3",
"magic-string": "^0.30.17",
"msw": "^2.7.0",
Expand All @@ -58,8 +57,7 @@
"test": "vitest",
"test:generate-fixtures": "bun run ./tests/utils/generate-css-fixtures",
"coverage": "vitest --coverage",
"format": "biome format --fix",
"lint": "biome lint --fix",
"lint": "biome check --fix",
"prepublishOnly": "bunx biome ci && bun run build"
},
"files": ["dist/*", "data/*"],
Expand Down
106 changes: 52 additions & 54 deletions src/api-parser-v1.ts
View file
Original file line number Diff line number Diff line change
@@ -1,19 +1,23 @@
import * as fs from 'node:fs/promises';
import { fileURLToPath } from 'node:url';

import { Limiter } from '@evan/concurrency';
import { consola } from 'consola';
import stringify from 'json-stringify-pretty-compact';
import PQueue from 'p-queue';
import { dirname, join } from 'pathe';
import { compile } from 'stylis';

import { apiv1 as userAgents } from '../data/user-agents.json';
import { APIDirect, APIv1 } from './data';
import { LOOP_LIMIT, addError, checkErrors } from './errors';
import type { APIResponse, FontObjectV1 } from './types';
import { orderObject, weightListGen } from './utils';
import { validate } from './validate';

const baseurl = 'https://fonts.googleapis.com/css?subset=';
const queue = Limiter(18);

const results: FontObjectV1[] = [];

export const fetchCSS = async (
font: APIResponse,
Expand Down Expand Up @@ -49,10 +53,10 @@ export const fetchCSS = async (
return cssMap.join('');
};

// Download CSS stylesheets for each file format
export const fetchAllCSS = async (
font: APIResponse,
): Promise<[string, string, string]> =>
// Download CSS stylesheets for each file format
await Promise.all([
fetchCSS(font, userAgents.woff2),
fetchCSS(font, userAgents.woff),
Expand Down Expand Up @@ -176,69 +180,63 @@ export const processCSS = (
return fontObject;
};

const results: FontObjectV1[] = [];

const processQueue = async (font: APIResponse, force: boolean) => {
const id = font.family.replaceAll(/\s/g, '-').toLowerCase();

// If last-modified matches latest API, skip fetching CSS and processing.
if (
APIv1[id] !== undefined &&
font.lastModified === APIv1[id].lastModified &&
!force
) {
results.push({ [id]: APIv1[id] });
} else {
const css = await fetchAllCSS(font);
const fontObject = processCSS(css, font);
results.push(fontObject);
consola.info(`Updated ${id}`);
const processQueue = async (
font: APIResponse,
force: boolean,
): Promise<void> => {
try {
const id = font.family.replaceAll(/\s/g, '-').toLowerCase();

// If last-modified matches latest API, skip fetching CSS and processing.
if (
APIv1[id] !== undefined &&
font.lastModified === APIv1[id].lastModified &&
!force
) {
results.push({ [id]: APIv1[id] });
} else {
const css = await fetchAllCSS(font);
const fontObject = processCSS(css, font);
results.push(fontObject);
consola.info(`Updated ${id}`);
}
consola.success(`Parsed ${id}`);
} catch (error) {
addError(`${font.family} experienced an error. ${String(error)}`);
}
consola.success(`Parsed ${id}`);
};

// Queue control
const queue = new PQueue({ concurrency: 18 });

// @ts-ignore - rollup-plugin-dts fails to compile this typing
queue.on('error', (error: Error) => {
consola.error(error);
});

/**
* Parses the fetched API data and writes it to the APIv1 JSON dataset.
* @param force - Force update all fonts without using cache.
* @param noValidate - Skip automatic validation of generated data.
*/
export const parsev1 = async (force: boolean, noValidate: boolean) => {
for (const font of APIDirect) {
try {
queue.add(async () => {
await processQueue(font, force);
});
} catch (error) {
throw new Error(`${font.family} experienced an error. ${String(error)}`);
}
checkErrors(LOOP_LIMIT);
queue.add(() => processQueue(font, force));
}
await queue.onIdle().then(async () => {
// Order the font objects alphabetically for consistency and not create huge diffs
const unordered: FontObjectV1 = Object.assign({}, ...results);
const ordered = orderObject(unordered);

if (!noValidate) {
validate('v1', ordered);
}
await queue.flush();
checkErrors();

await fs.writeFile(
join(
dirname(fileURLToPath(import.meta.url)),
'../data/google-fonts-v1.json',
),
stringify(ordered),
);

consola.success(
`All ${results.length} font datapoints using CSS APIv1 have been generated.`,
);
});
// Order the font objects alphabetically for consistency and not create huge diffs
const unordered: FontObjectV1 = Object.assign({}, ...results);
const ordered = orderObject(unordered);

if (!noValidate) {
validate('v1', ordered);
}

await fs.writeFile(
join(
dirname(fileURLToPath(import.meta.url)),
'../data/google-fonts-v1.json',
),
stringify(ordered),
);

consola.success(
`All ${results.length} font datapoints using CSS APIv1 have been generated.`,
);
};
102 changes: 50 additions & 52 deletions src/api-parser-v2.ts
View file
Original file line number Diff line number Diff line change
@@ -1,19 +1,23 @@
import * as fs from 'node:fs/promises';
import { fileURLToPath } from 'node:url';

import { Limiter } from '@evan/concurrency';
import { consola } from 'consola';
import stringify from 'json-stringify-pretty-compact';
import PQueue from 'p-queue';
import { dirname, join } from 'pathe';
import { compile } from 'stylis';

import { apiv2 as userAgents } from '../data/user-agents.json';
import { APIDirect, APIv2 } from './data';
import { LOOP_LIMIT, addError, checkErrors } from './errors';
import type { APIResponse, FontObjectV2 } from './types';
import { orderObject, weightListGen } from './utils';
import { validate } from './validate';

const baseurl = 'https://fonts.googleapis.com/css2?family=';
const queue = Limiter(18);

const results: FontObjectV2[] = [];

export const fetchCSS = async (
fontFamily: string,
Expand Down Expand Up @@ -234,69 +238,63 @@ export const processCSS = (
return fontObject;
};

const results: FontObjectV2[] = [];

const processQueue = async (font: APIResponse, force: boolean) => {
const id = font.family.replaceAll(/\s/g, '-').toLowerCase();

// If last-modified matches latest API, skip fetching CSS and processing.
if (
APIv2[id] !== undefined &&
font.lastModified === APIv2[id].lastModified &&
!force
) {
results.push({ [id]: APIv2[id] });
} else {
const css = await fetchAllCSS(font);
const fontObject = processCSS(css, font);
results.push(fontObject);
consola.info(`Updated ${id}`);
const processQueue = async (
font: APIResponse,
force: boolean,
): Promise<void> => {
try {
const id = font.family.replaceAll(/\s/g, '-').toLowerCase();

// If last-modified matches latest API, skip fetching CSS and processing.
if (
APIv2[id] !== undefined &&
font.lastModified === APIv2[id].lastModified &&
!force
) {
results.push({ [id]: APIv2[id] });
} else {
const css = await fetchAllCSS(font);
const fontObject = processCSS(css, font);
results.push(fontObject);
consola.info(`Updated ${id}`);
}
consola.success(`Parsed ${id}`);
} catch (error) {
addError(`${font.family} experienced an error. ${String(error)}`);
}
consola.success(`Parsed ${id}`);
};

// Queue control
const queue = new PQueue({ concurrency: 18 });

// @ts-ignore - rollup-plugin-dts fails to compile this typing
queue.on('error', (error: Error) => {
consola.error(error);
});

/**
* Parses the fetched API and writes it to the APIv2 dataset.
* @param force - Force update all fonts without using cache.
* @param noValidate - Skip automatic validation of generated data.
*/
export const parsev2 = async (force: boolean, noValidate: boolean) => {
for (const font of APIDirect) {
try {
queue.add(async () => {
await processQueue(font, force);
});
} catch (error) {
throw new Error(`${font.family} experienced an error. ${String(error)}`);
}
checkErrors(LOOP_LIMIT);
queue.add(() => processQueue(font, force));
}
await queue.onIdle().then(async () => {
// Order the font objects alphabetically for consistency and not create huge diffs
const unordered: FontObjectV2 = Object.assign({}, ...results);
const ordered = orderObject(unordered);

if (!noValidate) {
validate('v2', ordered);
}
await queue.flush();
checkErrors();

await fs.writeFile(
join(
dirname(fileURLToPath(import.meta.url)),
'../data/google-fonts-v2.json',
),
stringify(ordered),
);
// Order the font objects alphabetically for consistency and not create huge diffs
const unordered: FontObjectV2 = Object.assign({}, ...results);
const ordered = orderObject(unordered);

consola.success(
`All ${results.length} font datapoints using CSS APIv2 have been generated.`,
);
});
if (!noValidate) {
validate('v2', ordered);
}

await fs.writeFile(
join(
dirname(fileURLToPath(import.meta.url)),
'../data/google-fonts-v2.json',
),
stringify(ordered),
);

consola.success(
`All ${results.length} font datapoints using CSS APIv2 have been generated.`,
);
};
23 changes: 23 additions & 0 deletions src/errors.ts
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
import consola from 'consola';

const errs: string[] = [];

export const LOOP_LIMIT = 5;

export const addError = (error: string) => {
errs.push(error);
};

export const checkErrors = (limit = 0) => {
if (errs.length > limit) {
for (const err of errs) {
consola.error(err);
}

if (limit > 0) {
throw new Error('Too many errors occurred during parsing. Stopping...');
}

throw new Error('Some fonts experienced errors during parsing.');
}
};
Loading

0 comments on commit b8cafd8

Please sign in to comment.