fix!: correctly utilise per-page in WM.io fetch

.editorconfig

@@ -8,7 +8,7 @@ end_of_line = lf
 insert_final_newline = true
 trim_trailing_whitespace = true
 
-[*.{json,svg,toml,yaml,yml}]
+[*.{json,svg,yaml,yml}]
 indent_style = space
 indent_size = 2
 

.github/workflows/npm-publish.yml

@@ -16,7 +16,7 @@ jobs:
  with:
  node-version: 20
  registry-url: https://registry.npmjs.org/
- - run: npm install
+ - run: npm ci
  - if: ${{ github.event.release.tag_name != '' && env.NPM_PUBLISH_TAG != '' }}
  run: npm publish --provenance --access=public --tag=${{ env.NPM_PUBLISH_TAG }}
  env:

.prettierrc

@@ -1,3 +1,4 @@
 {
- "tabWidth": 4
+ "tabWidth": 4,
+ "trailingComma": "all"
 }

README.md

@@ -205,9 +205,6 @@ webmentions.forEach((webmention) => {
 <summary>Liquid / Nunjucks</summary>
 
 ```twig
-{# get Array of Webmentions for a given URL #}
-{% set webmentions = ("https://example.com" + page.url) | getWebmentions %}
-
 {# filter Webmentions by their response type #}
 {{ set responses = webmentions | getWebmentionsByTypes(['mention-of', 'in-reply-to']) }}
 
@@ -227,7 +224,7 @@ webmentions.forEach((webmention) => {
 
 </details>
 
-## Attach Webmentions to Pages using Directory Data
+### Attach Webmentions to Pages using Directory Data
 
 Using [Eleventy’s Data Cascade](https://www.11ty.dev/docs/data-cascade/), you can attach Webmentions to each page by using [Directory Specific Data Files](https://www.11ty.dev/docs/data-template-dir/).
 
@@ -281,18 +278,7 @@ module.exports = (eleventyConfig) => {
 }
 ```
 
-## Without Directory Data
-
-If you would rather get Webmentions for a given page directly from a Layout/Include/Page itself, you can do so using the Filter, `getWebmentions`:
-
-```twig
-{% set webmentions = ("https://example.com" + page.url) | getWebmentions %}
-{% for webmention in webmentions %}
- ...
-{% endfor %}
-```
-
-## Get specific types of Webmentions
+### Get specific types of Webmentions
 
 Instead of getting all the Webmentions for a given page, you may want to grab only certain types of Webmentions. This is useful if you want to display different types of Webmentions separately, e.g.:
 
@@ -304,7 +290,7 @@ Instead of getting all the Webmentions for a given page, you may want to grab on
 {% set replies = webmentions | getWebmentionsByTypes(['mention-of', 'in-reply-to']) %}
 ```
 
-## Get all Webmentions at once
+### Get all Webmentions at once
 
 If you need it, the plugin also makes available an Object containing your cached Webmentions organised in key:value pairs, where each key is a full URL on your website and its value is an Array of Webmentions sent to that URL:
 

eleventy-cache-webmentions.js

@@ -92,6 +92,12 @@ const getType = (webmention) => {
  );
 };
 
+const getByType = (webmentions, allowedType) => {
+ return webmentions.filter((webmention) => {
+ return allowedTypes === getType(webmention);
+ });
+};
+
 const getByTypes = (webmentions, allowedTypes) => {
  return webmentions.filter((webmention) => {
  return allowedTypes.includes(getType(webmention));
@@ -117,61 +123,73 @@ const defaults = {
 const performFetch = async (options, webmentions, url) => {
  return await fetch(url)
  .then(async (response) => {
- if (response.ok) {
- const feed = await response.json();
- if (feed[options.key].length) {
- // Combine newly-fetched Webmentions with cached Webmentions
- webmentions = feed[options.key].concat(webmentions);
- // Remove duplicates by source URL
- webmentions = uniqBy(
- [...feed[options.key], ...webmentions],
- (webmention) => {
- return getSource(webmention);
- },
- );
- // Process the blocklist, if it has any entries
- if (options.blocklist.length) {
- webmentions = webmentions.filter((webmention) => {
- let sourceUrl = getSource(webmention);
- for (let url of options.blocklist) {
- if (
- sourceUrl.includes(url.replace(/\/?$/, "/"))
- ) {
- return false;
- }
- }
- return true;
- });
- }
- // Process the allowlist, if it has any entries
- if (options.allowlist.length) {
- webmentions = webmentions.filter((webmention) => {
- let sourceUrl = getSource(webmention);
- for (let url of options.allowlist) {
- if (
- sourceUrl.includes(url.replace(/\/?$/, "/"))
- ) {
- return true;
- }
- }
+ if (!response.ok) {
+ return Promise.reject(response);
+ }
+
+ const feed = await response.json();
+
+ if (!options.key in feed) {
+ console.log(
+ `[${hostname(options.domain)}] ${
+ options.key
+ } was not found as a key in the response from ${hostname(
+ options.feed,
+ )}!`,
+ );
+ return Promise.reject(response);
+ }
+
+ // Combine newly-fetched Webmentions with cached Webmentions
+ webmentions = feed[options.key].concat(webmentions);
+ // Remove duplicates by source URL
+ webmentions = uniqBy(
+ [...feed[options.key], ...webmentions],
+ (webmention) => {
+ return getSource(webmention);
+ },
+ );
+ // Process the blocklist, if it has any entries
+ if (options.blocklist.length) {
+ webmentions = webmentions.filter((webmention) => {
+ let sourceUrl = getSource(webmention);
+ for (let url of options.blocklist) {
+ if (sourceUrl.includes(url.replace(/\/?$/, "/"))) {
  return false;
- });
+ }
  }
- // Sort webmentions by received date for getting most recent Webmention on subsequent requests
- webmentions = webmentions.sort((a, b) => {
- return epoch(getReceived(b)) - epoch(getReceived(a));
- });
- }
- return {
- found: feed[options.key].length,
- filtered: webmentions,
- };
+ return true;
+ });
  }
- return Promise.reject(response);
+ // Process the allowlist, if it has any entries
+ if (options.allowlist.length) {
+ webmentions = webmentions.filter((webmention) => {
+ let sourceUrl = getSource(webmention);
+ for (let url of options.allowlist) {
+ if (sourceUrl.includes(url.replace(/\/?$/, "/"))) {
+ return true;
+ }
+ }
+ return false;
+ });
+ }
+ // Sort webmentions by received date for getting most recent Webmention on subsequent requests
+ webmentions = webmentions.sort((a, b) => {
+ return epoch(getReceived(b)) - epoch(getReceived(a));
+ });
+
+ return {
+ found: feed[options.key].length,
+ filtered: webmentions,
+ };
  })
  .catch((error) => {
  console.log(
- `[${hostname(options.domain)}] Something went wrong with your request to ${hostname(options.feed)}!`,
+ `[${hostname(
+ options.domain,
+ )}] Something went wrong with your request to ${hostname(
+ options.feed,
+ )}!`,
  error,
  );
  });
@@ -212,11 +230,15 @@ const fetchWebmentions = async (options) => {
 
  // If there is a cached file but it is outside of expiry, fetch fresh
  // results since the most recent Webmention
- if (!asset.isCacheValid(options.duration)) {
+ if (!asset.isCacheValid("1s")) {
  // Get the received date of the most recent Webmention, if it exists
  const since = webmentions.length ? getReceived(webmentions[0]) : false;
  // Build the URL for the fetch request
- const url = `${options.feed}${since ? `${options.feed.includes("?") ? "&" : "?"}since=${since}` : ""}`;
+ const url = `${options.feed}${
+ since
+ ? `${options.feed.includes("?") ? "&" : "?"}since=${since}`
+ : ""
+ }`;
 
  // If using webmention.io, loop through pages until no results found
  if (url.includes("https://webmention.io")) {
@@ -226,7 +248,7 @@ const fetchWebmentions = async (options) => {
  while (true) {
  const perPage =
  Number(new URL(url).searchParams.get("per-page")) || 1000;
- const urlPaginated = url + `&page=${page}`;
+ const urlPaginated = url + `&per-page=${perPage}&page=${page}`;
  const fetched = await performFetch(
  options,
  webmentions,
@@ -266,7 +288,9 @@ const fetchWebmentions = async (options) => {
  // Add a console message with the number of fetched and processed Webmentions, if any
  if (webmentionsCachedLength < webmentions.length) {
  console.log(
- `[${hostname(options.domain)}] ${webmentions.length - webmentionsCachedLength} new Webmentions fetched into cache.`,
+ `[${hostname(options.domain)}] ${
+ webmentions.length - webmentionsCachedLength
+ } new Webmentions fetched into cache.`,
  );
  }
  }
@@ -329,7 +353,11 @@ const getWebmentions = async (options, url, allowedTypes = {}) => {
  options.allowedHTML,
  );
  if (html.length > options.maximumHtmlLength) {
- entry.contentSanitized = `${options.maximumHtmlText} <a href="${getSource(entry)}">${getSource(entry)}</a>`;
+ entry.contentSanitized = `${
+ options.maximumHtmlText
+ } <a href="${getSource(entry)}">${getSource(
+ entry,
+ )}</a>`;
  }
  }
 
@@ -342,20 +370,12 @@ const getWebmentions = async (options, url, allowedTypes = {}) => {
  );
 };
 
-const getWebmentionsFilter = async (options, url, allowedTypes, callback) => {
- if (typeof callback !== "function") {
- callback = allowedTypes;
- allowedTypes = {};
- }
- const webmentions = await getWebmentions(options, url, allowedTypes);
- callback(null, webmentions);
-};
-
 const eleventyCacheWebmentions = (eleventyConfig, options = {}) => {
  options = Object.assign(defaults, options);
 
  // Global Data
  eleventyConfig.addGlobalData("webmentionsDefaults", defaults);
+ eleventyConfig.addGlobalData("webmentionsOptions", options);
  const filtered = async () => await filteredWebmentions(options);
  eleventyConfig.addGlobalData("webmentionsByUrl", filtered);
  const unfiltered = async () =>
@@ -368,7 +388,7 @@ const eleventyCacheWebmentions = (eleventyConfig, options = {}) => {
  eleventyConfig.addGlobalData("webmentionsAll", unfiltered);
 
  // Liquid Filters
- eleventyConfig.addLiquidFilter("getWebmentions", getWebmentionsFilter);
+ eleventyConfig.addLiquidFilter("getWebmentionsByType", getByType);
  eleventyConfig.addLiquidFilter("getWebmentionsByTypes", getByTypes);
  eleventyConfig.addLiquidFilter("getWebmentionPublished", getPublished);
  eleventyConfig.addLiquidFilter("getWebmentionReceived", getReceived);
@@ -378,10 +398,7 @@ const eleventyCacheWebmentions = (eleventyConfig, options = {}) => {
  eleventyConfig.addLiquidFilter("getWebmentionType", getType);
 
  // Nunjucks Filters
- eleventyConfig.addNunjucksAsyncFilter(
- "getWebmentions",
- getWebmentionsFilter,
- );
+ eleventyConfig.addNunjucksFilter("getWebmentionsByType", getByType);
  eleventyConfig.addNunjucksFilter("getWebmentionsByTypes", getByTypes);
  eleventyConfig.addNunjucksFilter("getWebmentionPublished", getPublished);
  eleventyConfig.addNunjucksFilter("getWebmentionReceived", getReceived);
@@ -397,6 +414,8 @@ module.exports.filteredWebmentions = filteredWebmentions;
 module.exports.webmentionsByUrl = filteredWebmentions;
 module.exports.fetchWebmentions = fetchWebmentions;
 module.exports.getWebmentions = getWebmentions;
+module.exports.getByType = getByType;
+module.exports.getWebmentionsByType = getByType;
 module.exports.getByTypes = getByTypes;
 module.exports.getWebmentionsByTypes = getByTypes;
 module.exports.getPublished = getPublished;

package.json

@@ -1,6 +1,6 @@
 {
  "name": "@chrisburnell/eleventy-cache-webmentions",
- "version": "2.1.2",
+ "version": "2.1.3-beta.1",
  "description": "Cache webmentions using eleventy-fetch and make them available to use in collections, layouts, pages, etc. in Eleventy.",
  "license": "MIT",
  "repository": {