From 78b9517d2e4301566726b882a9d2f1e87f5002ca Mon Sep 17 00:00:00 2001 From: Chris Burnell Date: Tue, 13 Aug 2024 03:00:22 +0800 Subject: [PATCH] fix!: correctly utilise per-page in WM.io fetch --- .editorconfig | 2 +- .github/workflows/npm-publish.yml | 2 +- .prettierrc | 3 +- README.md | 20 +--- eleventy-cache-webmentions.js | 155 +++++++++++++++++------------- package.json | 2 +- 6 files changed, 95 insertions(+), 89 deletions(-) diff --git a/.editorconfig b/.editorconfig index d75be91..22f19c5 100644 --- a/.editorconfig +++ b/.editorconfig @@ -8,7 +8,7 @@ end_of_line = lf insert_final_newline = true trim_trailing_whitespace = true -[*.{json,svg,toml,yaml,yml}] +[*.{json,svg,yaml,yml}] indent_style = space indent_size = 2 diff --git a/.github/workflows/npm-publish.yml b/.github/workflows/npm-publish.yml index 0c159af..f2198cb 100644 --- a/.github/workflows/npm-publish.yml +++ b/.github/workflows/npm-publish.yml @@ -16,7 +16,7 @@ jobs: with: node-version: 20 registry-url: https://registry.npmjs.org/ - - run: npm install + - run: npm ci - if: ${{ github.event.release.tag_name != '' && env.NPM_PUBLISH_TAG != '' }} run: npm publish --provenance --access=public --tag=${{ env.NPM_PUBLISH_TAG }} env: diff --git a/.prettierrc b/.prettierrc index cef22be..7565d91 100644 --- a/.prettierrc +++ b/.prettierrc @@ -1,3 +1,4 @@ { - "tabWidth": 4 + "tabWidth": 4, + "trailingComma": "all" } diff --git a/README.md b/README.md index b3c9ebb..1b7ec96 100644 --- a/README.md +++ b/README.md @@ -205,9 +205,6 @@ webmentions.forEach((webmention) => { Liquid / Nunjucks ```twig -{# get Array of Webmentions for a given URL #} -{% set webmentions = ("https://example.com" + page.url) | getWebmentions %} - {# filter Webmentions by their response type #} {{ set responses = webmentions | getWebmentionsByTypes(['mention-of', 'in-reply-to']) }} @@ -227,7 +224,7 @@ webmentions.forEach((webmention) => { -## Attach Webmentions to Pages using Directory Data +### Attach Webmentions to Pages using Directory Data Using [Eleventy’s Data Cascade](https://www.11ty.dev/docs/data-cascade/), you can attach Webmentions to each page by using [Directory Specific Data Files](https://www.11ty.dev/docs/data-template-dir/). @@ -281,18 +278,7 @@ module.exports = (eleventyConfig) => { } ``` -## Without Directory Data - -If you would rather get Webmentions for a given page directly from a Layout/Include/Page itself, you can do so using the Filter, `getWebmentions`: - -```twig -{% set webmentions = ("https://example.com" + page.url) | getWebmentions %} -{% for webmention in webmentions %} - ... -{% endfor %} -``` - -## Get specific types of Webmentions +### Get specific types of Webmentions Instead of getting all the Webmentions for a given page, you may want to grab only certain types of Webmentions. This is useful if you want to display different types of Webmentions separately, e.g.: @@ -304,7 +290,7 @@ Instead of getting all the Webmentions for a given page, you may want to grab on {% set replies = webmentions | getWebmentionsByTypes(['mention-of', 'in-reply-to']) %} ``` -## Get all Webmentions at once +### Get all Webmentions at once If you need it, the plugin also makes available an Object containing your cached Webmentions organised in key:value pairs, where each key is a full URL on your website and its value is an Array of Webmentions sent to that URL: diff --git a/eleventy-cache-webmentions.js b/eleventy-cache-webmentions.js index d0bd932..df35489 100644 --- a/eleventy-cache-webmentions.js +++ b/eleventy-cache-webmentions.js @@ -92,6 +92,12 @@ const getType = (webmention) => { ); }; +const getByType = (webmentions, allowedType) => { + return webmentions.filter((webmention) => { + return allowedTypes === getType(webmention); + }); +}; + const getByTypes = (webmentions, allowedTypes) => { return webmentions.filter((webmention) => { return allowedTypes.includes(getType(webmention)); @@ -117,61 +123,73 @@ const defaults = { const performFetch = async (options, webmentions, url) => { return await fetch(url) .then(async (response) => { - if (response.ok) { - const feed = await response.json(); - if (feed[options.key].length) { - // Combine newly-fetched Webmentions with cached Webmentions - webmentions = feed[options.key].concat(webmentions); - // Remove duplicates by source URL - webmentions = uniqBy( - [...feed[options.key], ...webmentions], - (webmention) => { - return getSource(webmention); - }, - ); - // Process the blocklist, if it has any entries - if (options.blocklist.length) { - webmentions = webmentions.filter((webmention) => { - let sourceUrl = getSource(webmention); - for (let url of options.blocklist) { - if ( - sourceUrl.includes(url.replace(/\/?$/, "/")) - ) { - return false; - } - } - return true; - }); - } - // Process the allowlist, if it has any entries - if (options.allowlist.length) { - webmentions = webmentions.filter((webmention) => { - let sourceUrl = getSource(webmention); - for (let url of options.allowlist) { - if ( - sourceUrl.includes(url.replace(/\/?$/, "/")) - ) { - return true; - } - } + if (!response.ok) { + return Promise.reject(response); + } + + const feed = await response.json(); + + if (!options.key in feed) { + console.log( + `[${hostname(options.domain)}] ${ + options.key + } was not found as a key in the response from ${hostname( + options.feed, + )}!`, + ); + return Promise.reject(response); + } + + // Combine newly-fetched Webmentions with cached Webmentions + webmentions = feed[options.key].concat(webmentions); + // Remove duplicates by source URL + webmentions = uniqBy( + [...feed[options.key], ...webmentions], + (webmention) => { + return getSource(webmention); + }, + ); + // Process the blocklist, if it has any entries + if (options.blocklist.length) { + webmentions = webmentions.filter((webmention) => { + let sourceUrl = getSource(webmention); + for (let url of options.blocklist) { + if (sourceUrl.includes(url.replace(/\/?$/, "/"))) { return false; - }); + } } - // Sort webmentions by received date for getting most recent Webmention on subsequent requests - webmentions = webmentions.sort((a, b) => { - return epoch(getReceived(b)) - epoch(getReceived(a)); - }); - } - return { - found: feed[options.key].length, - filtered: webmentions, - }; + return true; + }); } - return Promise.reject(response); + // Process the allowlist, if it has any entries + if (options.allowlist.length) { + webmentions = webmentions.filter((webmention) => { + let sourceUrl = getSource(webmention); + for (let url of options.allowlist) { + if (sourceUrl.includes(url.replace(/\/?$/, "/"))) { + return true; + } + } + return false; + }); + } + // Sort webmentions by received date for getting most recent Webmention on subsequent requests + webmentions = webmentions.sort((a, b) => { + return epoch(getReceived(b)) - epoch(getReceived(a)); + }); + + return { + found: feed[options.key].length, + filtered: webmentions, + }; }) .catch((error) => { console.log( - `[${hostname(options.domain)}] Something went wrong with your request to ${hostname(options.feed)}!`, + `[${hostname( + options.domain, + )}] Something went wrong with your request to ${hostname( + options.feed, + )}!`, error, ); }); @@ -212,11 +230,15 @@ const fetchWebmentions = async (options) => { // If there is a cached file but it is outside of expiry, fetch fresh // results since the most recent Webmention - if (!asset.isCacheValid(options.duration)) { + if (!asset.isCacheValid("1s")) { // Get the received date of the most recent Webmention, if it exists const since = webmentions.length ? getReceived(webmentions[0]) : false; // Build the URL for the fetch request - const url = `${options.feed}${since ? `${options.feed.includes("?") ? "&" : "?"}since=${since}` : ""}`; + const url = `${options.feed}${ + since + ? `${options.feed.includes("?") ? "&" : "?"}since=${since}` + : "" + }`; // If using webmention.io, loop through pages until no results found if (url.includes("https://webmention.io")) { @@ -226,7 +248,7 @@ const fetchWebmentions = async (options) => { while (true) { const perPage = Number(new URL(url).searchParams.get("per-page")) || 1000; - const urlPaginated = url + `&page=${page}`; + const urlPaginated = url + `&per-page=${perPage}&page=${page}`; const fetched = await performFetch( options, webmentions, @@ -266,7 +288,9 @@ const fetchWebmentions = async (options) => { // Add a console message with the number of fetched and processed Webmentions, if any if (webmentionsCachedLength < webmentions.length) { console.log( - `[${hostname(options.domain)}] ${webmentions.length - webmentionsCachedLength} new Webmentions fetched into cache.`, + `[${hostname(options.domain)}] ${ + webmentions.length - webmentionsCachedLength + } new Webmentions fetched into cache.`, ); } } @@ -329,7 +353,11 @@ const getWebmentions = async (options, url, allowedTypes = {}) => { options.allowedHTML, ); if (html.length > options.maximumHtmlLength) { - entry.contentSanitized = `${options.maximumHtmlText} ${getSource(entry)}`; + entry.contentSanitized = `${ + options.maximumHtmlText + } ${getSource( + entry, + )}`; } } @@ -342,20 +370,12 @@ const getWebmentions = async (options, url, allowedTypes = {}) => { ); }; -const getWebmentionsFilter = async (options, url, allowedTypes, callback) => { - if (typeof callback !== "function") { - callback = allowedTypes; - allowedTypes = {}; - } - const webmentions = await getWebmentions(options, url, allowedTypes); - callback(null, webmentions); -}; - const eleventyCacheWebmentions = (eleventyConfig, options = {}) => { options = Object.assign(defaults, options); // Global Data eleventyConfig.addGlobalData("webmentionsDefaults", defaults); + eleventyConfig.addGlobalData("webmentionsOptions", options); const filtered = async () => await filteredWebmentions(options); eleventyConfig.addGlobalData("webmentionsByUrl", filtered); const unfiltered = async () => @@ -368,7 +388,7 @@ const eleventyCacheWebmentions = (eleventyConfig, options = {}) => { eleventyConfig.addGlobalData("webmentionsAll", unfiltered); // Liquid Filters - eleventyConfig.addLiquidFilter("getWebmentions", getWebmentionsFilter); + eleventyConfig.addLiquidFilter("getWebmentionsByType", getByType); eleventyConfig.addLiquidFilter("getWebmentionsByTypes", getByTypes); eleventyConfig.addLiquidFilter("getWebmentionPublished", getPublished); eleventyConfig.addLiquidFilter("getWebmentionReceived", getReceived); @@ -378,10 +398,7 @@ const eleventyCacheWebmentions = (eleventyConfig, options = {}) => { eleventyConfig.addLiquidFilter("getWebmentionType", getType); // Nunjucks Filters - eleventyConfig.addNunjucksAsyncFilter( - "getWebmentions", - getWebmentionsFilter, - ); + eleventyConfig.addNunjucksFilter("getWebmentionsByType", getByType); eleventyConfig.addNunjucksFilter("getWebmentionsByTypes", getByTypes); eleventyConfig.addNunjucksFilter("getWebmentionPublished", getPublished); eleventyConfig.addNunjucksFilter("getWebmentionReceived", getReceived); @@ -397,6 +414,8 @@ module.exports.filteredWebmentions = filteredWebmentions; module.exports.webmentionsByUrl = filteredWebmentions; module.exports.fetchWebmentions = fetchWebmentions; module.exports.getWebmentions = getWebmentions; +module.exports.getByType = getByType; +module.exports.getWebmentionsByType = getByType; module.exports.getByTypes = getByTypes; module.exports.getWebmentionsByTypes = getByTypes; module.exports.getPublished = getPublished; diff --git a/package.json b/package.json index 3eddc16..301fa92 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@chrisburnell/eleventy-cache-webmentions", - "version": "2.1.2", + "version": "2.1.3-beta.1", "description": "Cache webmentions using eleventy-fetch and make them available to use in collections, layouts, pages, etc. in Eleventy.", "license": "MIT", "repository": {