diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml index 0e0a792..98f38a5 100644 --- a/.github/FUNDING.yml +++ b/.github/FUNDING.yml @@ -1,4 +1,5 @@ # Docs for this file can be found here: # https://docs.github.com/en/github/administering-a-repository/displaying-a-sponsor-button-in-your-repository -github: ["chrisburnell"] +github: chrisburnell +buy_me_a_coffee: chrisburnell diff --git a/.github/workflows/npm-publish.yml b/.github/workflows/npm-publish.yml index 31f0e25..9a16536 100644 --- a/.github/workflows/npm-publish.yml +++ b/.github/workflows/npm-publish.yml @@ -14,6 +14,8 @@ jobs: node-version: 19 registry-url: https://registry.npmjs.org/ - run: npm install - - run: npm publish + - run: | + TAG=$(echo $GITHUB_REF_NAME | grep -oP '^v\d+\.\d+\.\d+-?\K(\w+)?') + npm publish --tag ${TAG:-latest} env: NODE_AUTH_TOKEN: ${{secrets.npm_token}} diff --git a/eleventy-cache-webmentions.js b/eleventy-cache-webmentions.js index 24716b4..4b038b6 100644 --- a/eleventy-cache-webmentions.js +++ b/eleventy-cache-webmentions.js @@ -83,6 +83,59 @@ const defaults = { maximumHtmlText: "mentioned this in", } +const performFetch = async (options, webmentions, url) => { + return await fetch(url) + .then(async (response) => { + if (response.ok) { + const feed = await response.json() + if (feed[options.key].length) { + // Combine newly-fetched Webmentions with cached Webmentions + webmentions = feed[options.key].concat(webmentions) + // Remove duplicates by source URL + webmentions = uniqBy([...feed[options.key], ...webmentions], (webmention) => { + return getSource(webmention) + }) + // Process the blocklist, if it has any entries + if (options.blocklist.length) { + webmentions = webmentions.filter((webmention) => { + let sourceUrl = getSource(webmention) + for (let url of options.blocklist) { + if (sourceUrl.includes(url.replace(/\/?$/, "/"))) { + return false + } + } + return true + }) + } + // Process the allowlist, if it has any entries + if (options.allowlist.length) { + webmentions = webmentions.filter((webmention) => { + let sourceUrl = getSource(webmention) + for (let url of options.allowlist) { + if (sourceUrl.includes(url.replace(/\/?$/, "/"))) { + return true + } + } + return false + }) + } + // Sort webmentions by received date for getting most recent Webmention on subsequent requests + webmentions = webmentions.sort((a, b) => { + return epoch(getReceived(b)) - epoch(getReceived(a)) + }) + } + return { + found: feed[options.key].length, + filtered: webmentions, + } + } + return Promise.reject(response) + }) + .catch((error) => { + console.log(`[${hostname(options.domain)}] Something went wrong with your request to ${hostname(options.feed)}!`, error) + }) +} + const fetchWebmentions = async (options) => { if (!options.domain) { throw new Error("`domain` is a required field when attempting to retrieve Webmentions. See https://www.npmjs.com/package/@chrisburnell/eleventy-cache-webmentions#installation for more information.") @@ -117,58 +170,51 @@ const fetchWebmentions = async (options) => { const since = webmentions.length ? getReceived(webmentions[0]) : false // Build the URL for the fetch request const url = `${options.feed}${since ? `${options.feed.includes("?") ? "&" : "?"}since=${since}` : ""}` - await fetch(url) - .then(async (response) => { - if (response.ok) { - const feed = await response.json() - if (feed[options.key].length) { - // Combine newly-fetched Webmentions with cached Webmentions - webmentions = feed[options.key].concat(webmentions) - // Remove duplicates by source URL - webmentions = uniqBy([...feed[options.key], ...webmentions], (webmention) => { - return getSource(webmention) - }) - // Process the blocklist, if it has any entries - if (options.blocklist.length) { - webmentions = webmentions.filter((webmention) => { - let sourceUrl = getSource(webmention) - for (let url of options.blocklist) { - if (sourceUrl.includes(url.replace(/\/?$/, "/"))) { - return false - } - } - return true - }) - } - // Process the allowlist, if it has any entries - if (options.allowlist.length) { - webmentions = webmentions.filter((webmention) => { - let sourceUrl = getSource(webmention) - for (let url of options.allowlist) { - if (sourceUrl.includes(url.replace(/\/?$/, "/"))) { - return true - } - } - return false - }) - } - // Sort webmentions by received date for getting most recent Webmention on subsequent requests - webmentions = webmentions.sort((a, b) => { - return epoch(getReceived(b)) - epoch(getReceived(a)) - }) - // Add a console message with the number of fetched and processed Webmentions, if any - if (webmentionsCachedLength < webmentions.length) { - console.log(`[${hostname(options.domain)}] ${webmentions.length - webmentionsCachedLength} new Webmentions fetched into cache.`) - } - } - await asset.save(webmentions, "json") - return webmentions + + // If using webmention.io, loop through pages until no results found + if (url.includes("https://webmention.io")) { + // Start on page 0, to increment per subsequent request + let page = 0 + // Loop until a break condition is hit + while (true) { + const perPage = Number(new URL(url).searchParams.get("per-page")) || 20 + const urlPaginated = url + `&page=${page}` + const fetched = await performFetch(options, webmentions, urlPaginated) + + // An error occurred during fetching paged results → break + if (!fetched && !fetched.found && !fetched.filtered) { + break } - return Promise.reject(response) - }) - .catch((error) => { - console.log(`[${hostname(options.domain)}] Something went wrong with your request to ${hostname(options.feed)}!`, error) - }) + + // Page has no Webmentions → break + if (fetched.found === 0) { + break + } + + webmentions = fetched.filtered + + // If there are less Webmentions found than should be in each + // page → break + if (fetched.found < perPage) { + break + } + + // Increment page + page += 1 + // Throttle next request + await new Promise(resolve => setTimeout(resolve, 1000)) + } + } else { + const fetched = await performFetch(options, webmentions, url) + webmentions = fetched.filtered + } + + await asset.save(webmentions, "json") + + // Add a console message with the number of fetched and processed Webmentions, if any + if (webmentionsCachedLength < webmentions.length) { + console.log(`[${hostname(options.domain)}] ${webmentions.length - webmentionsCachedLength} new Webmentions fetched into cache.`) + } } return webmentions diff --git a/package.json b/package.json index fee5ee4..9e884be 100644 --- a/package.json +++ b/package.json @@ -1,16 +1,32 @@ { "name": "@chrisburnell/eleventy-cache-webmentions", - "version": "2.0.6", + "version": "2.1.0-beta.1", "description": "Cache webmentions using eleventy-fetch and make them available to use in collections, layouts, pages, etc. in Eleventy.", - "author": "Chris Burnell ", "license": "MIT", "repository": { "type": "git", "url": "git@github.com:chrisburnell/eleventy-cache-webmentions.git" }, + "homepage": "https://chrisburnell.com/eleventy-cache-webmentions/", "bugs": { "url": "https://github.com/chrisburnell/eleventy-cache-webmentions/issues" }, + "author": "Chris Burnell ", + "contributors": [ + { + "name": "Chris Burnell", + "email": "me@chrisburnell.com", + "url": "https://chrisburnell.com", + "mastodon": "@chrisburnell@repc.co" + } + ], + "publishConfig": { + "access": "public" + }, + "main": "eleventy-cache-webmentions.js", + "scripts": { + "lint": "eslint eleventy-cache-webmentions.js" + }, "keywords": [ "eleventy", "eleventy-plugin", @@ -19,17 +35,13 @@ "js", "webmention" ], - "devDependencies": { - "eslint": "^9.2.0" - }, "dependencies": { "@11ty/eleventy-fetch": "^4.0.1", "lodash": "^4.17.21", "node-fetch": "2.6.7", "sanitize-html": "^2.13.0" }, - "main": "eleventy-cache-webmentions.js", - "scripts": { - "lint": "eslint eleventy-cache-webmentions.js" + "devDependencies": { + "eslint": "^9.2.0" } }