From 5df18a6ac4014efb4c58ea1abac9a20e0e2d4eea Mon Sep 17 00:00:00 2001 From: Chris Burnell Date: Tue, 30 May 2023 19:15:44 +0100 Subject: [PATCH] fix: add duration to defaults, remove uniqueKey check, add extra console message for acquiring filteredWebmentions, README updates --- README.md | 111 +++++++++++++++++++++++------- eleventy-cache-webmentions.js | 123 ++++++++-------------------------- package.json | 2 +- 3 files changed, 113 insertions(+), 123 deletions(-) diff --git a/README.md b/README.md index 4b0ed6d..716e4a0 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,91 @@ module.exports = function (eleventyConfig) { ## Options + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
optiondefault valuedescriptionversion added
domain
required		—The website you’re fetching Webmentions for.0.0.1
feed
required		—The URL of your Webmention server’s feed for your domain.0.2.0
key
required		—The key in the above feed whose value is an Array of Webmentions.0.0.1
directory".cache"See Eleventy Fetch’s Cache Directory for more information.1.1.2
duration"1d" or 1 daySee Eleventy Fetch’s Cache Duration for more information.0.0.1
uniqueKey"webmentions"The name of the file generated by Eleventy Fetch.0.1.9
allowedHTMLSee code example belowSee the sanitize-html package for more information.0.0.1
allowlist[]An Array of root URLs from which Webmentions are kept.1.1.0
blocklist[]An Array of root URLs from which Webmentions are discarded.1.1.0
urlReplacements{}An Object of key-value string pairs containing from-to URL replacements on this domain.0.0.3
maximumHtmlLength2000Maximum number of characters in a Webmention’s HTML content, beyond which point a different message is shown, referring to the original source.0.0.1
maximumHtmlText"mentioned this in"The glue-y part of the message displayed when a Webmention content’s character count exceeds maximumHtmlLength.0.1.0
+ Advanced control over how the Webmentions are cached and processed is done by passing `options` into the plugin when using `addPlugin`: ```javascript @@ -39,48 +124,22 @@ const pluginWebmentions = require("@chrisburnell/eleventy-cache-webmentions") module.exports = function (eleventyConfig) { eleventyConfig.addPlugin(pluginWebmentions, { - // domain: required or the plugin will not function - // this is the website that you want to pull in Webmentions for domain: "https://example.com", - // feed: required or the plugin will not function - // defines the URL of your Webmention server where a feed of Webmentions for your domain can be found feed: "https://webmentions.example.com?token=S3cr3tT0k3n", - // key: required or the plugin will not function - // dictates the key inside the feed where the array of Webmentions is located key: "children", - // directory: ".cache" by default - // see https://www.11ty.dev/docs/plugins/cache/#cache-directory for more info directory: ".cache", - // duration: "1d" by default - // see https://www.11ty.dev/docs/plugins/cache/#change-the-cache-duration for more info duration: "1d", - // uniquekey: "webmentions" by default - // dictates the name sent to eleventy-fetch to name the file uniqueKey: "webmentions", - // allowedHTML: Object by default - // see https://www.npmjs.com/package/sanitize-html for more info allowedHTML: { allowedTags: ["b", "i", "em", "strong", "a"], allowedAttributes: { a: ["href"], }, }, - // allowlist: [] by default - // array of root URLs from which webmentions are wanted exclusively allowlist: [], - // blocklist: [] by default - // array of root URLs from which webmentions are not wanted - // exclusively blocklist: [], - // urlReplacements: {} by default - // object of key:value pairs containing from:to URL replacements urlReplacements: {}, - // maximumHtmlLength: 2000 by default - // number of characters in the HTML content at which a different - // message is shown instead of the content maximumHtmlLength: 2000, - // maximumHtmlText: "mentioned this in" by default - // message shown when maximumHtmlLength is reached maximumHtmlText: "mentioned this in", }) } diff --git a/eleventy-cache-webmentions.js b/eleventy-cache-webmentions.js index 62e5ac5..057fb70 100644 --- a/eleventy-cache-webmentions.js +++ b/eleventy-cache-webmentions.js @@ -7,9 +7,7 @@ const absoluteURL = (url, domain) => { try { return new URL(url, domain).toString() } catch (e) { - console.log( - `Trying to convert ${url} to be an absolute url with base ${domain} and failed.` - ) + console.log(`Trying to convert ${url} to be an absolute url with base ${domain} and failed.`) return url } } @@ -21,13 +19,10 @@ const baseUrl = (url) => { } const fixUrl = (url, urlReplacements) => { - return Object.entries(urlReplacements).reduce( - (accumulator, [key, value]) => { - const regex = new RegExp(key, "g") - return accumulator.replace(regex, value) - }, - url - ) + return Object.entries(urlReplacements).reduce((accumulator, [key, value]) => { + const regex = new RegExp(key, "g") + return accumulator.replace(regex, value) + }, url) } const hostname = (value) => { @@ -43,30 +38,15 @@ const epoch = (value) => { } const getPublished = (webmention) => { - return ( - webmention["wm-received"] || - webmention?.["data"]["published"] || - webmention["verified_date"] || - webmention["published"] - ) + return webmention["wm-received"] || webmention?.["data"]["published"] || webmention["verified_date"] || webmention["published"] } const getContent = (webmention) => { - return ( - webmention?.["content"]?.["html"] || - webmention?.["content"] || - webmention?.["data"]?.["content"] || - "" - ) + return webmention?.["content"]?.["html"] || webmention?.["content"] || webmention?.["data"]?.["content"] || "" } const getSource = (webmention) => { - return ( - webmention["url"] || - webmention?.["data"]["url"] || - webmention["wm-source"] || - webmention["source"] - ) + return webmention["url"] || webmention?.["data"]["url"] || webmention["wm-source"] || webmention["source"] } const getTarget = (webmention) => { @@ -74,14 +54,11 @@ const getTarget = (webmention) => { } const getType = (webmention) => { - return ( - webmention["wm-property"] || - webmention?.["activity"]["type"] || - webmention["type"] - ) + return webmention["wm-property"] || webmention?.["activity"]["type"] || webmention["type"] } const defaults = { + duration: "1d", uniqueKey: "webmentions", allowedHTML: { allowedTags: ["b", "i", "em", "strong", "a"], @@ -98,27 +75,15 @@ const defaults = { const fetchWebmentions = async (options) => { if (!options.domain) { - throw new Error( - "domain is a required field when attempting to retrieve Webmentions. See https://www.npmjs.com/package/@chrisburnell/eleventy-cache-webmentions#installation for more information." - ) + throw new Error("domain is a required field when attempting to retrieve Webmentions. See https://www.npmjs.com/package/@chrisburnell/eleventy-cache-webmentions#installation for more information.") } if (!options.feed) { - throw new Error( - "feed is a required field when attempting to retrieve Webmentions. See https://www.npmjs.com/package/@chrisburnell/eleventy-cache-webmentions#installation for more information." - ) + throw new Error("feed is a required field when attempting to retrieve Webmentions. See https://www.npmjs.com/package/@chrisburnell/eleventy-cache-webmentions#installation for more information.") } if (!options.key) { - throw new Error( - "key is a required field when attempting to retrieve Webmentions. See https://www.npmjs.com/package/@chrisburnell/eleventy-cache-webmentions#installation for more information." - ) - } - - if (!options.uniqueKey) { - throw new Error( - "uniqueKey is a required field when attempting to retrieve Webmentions. See https://www.npmjs.com/package/@chrisburnell/eleventy-cache-webmentions#installation for more information." - ) + throw new Error("key is a required field when attempting to retrieve Webmentions. See https://www.npmjs.com/package/@chrisburnell/eleventy-cache-webmentions#installation for more information.") } let asset = new AssetCache(options.uniqueKey, options.directory) @@ -135,27 +100,16 @@ const fetchWebmentions = async (options) => { // results since the most recent webmention if (!asset.isCacheValid(options.duration)) { const since = webmentions.length ? getPublished(webmentions[0]) : false - const url = `${options.feed}${ - since - ? `${options.feed.includes("?") ? "&" : "?"}since=${since}` - : "" - }` + const url = `${options.feed}${since ? `${options.feed.includes("?") ? "&" : "?"}since=${since}` : ""}` await fetch(url) .then(async (response) => { if (response.ok) { const feed = await response.json() if (feed[options.key].length) { - webmentions = uniqBy( - [...feed[options.key], ...webmentions], - (entry) => { - return getSource(entry) - } - ) - console.log( - `[${hostname(options.domain)}] ${ - feed[options.key].length - } new Webmentions fetched into cache.` - ) + webmentions = uniqBy([...feed[options.key], ...webmentions], (entry) => { + return getSource(entry) + }) + console.log(`[${hostname(options.domain)}] ${feed[options.key].length} new Webmentions fetched into cache.`) } await asset.save(webmentions, "json") return webmentions @@ -163,14 +117,7 @@ const fetchWebmentions = async (options) => { return Promise.reject(response) }) .catch((error) => { - console.log( - `[${hostname( - options.domain - )}] Something went wrong with your request to ${hostname( - options.feed - )}!`, - error - ) + console.log(`[${hostname(options.domain)}] Something went wrong with your request to ${hostname(options.feed)}!`, error) }) } @@ -218,12 +165,7 @@ const filteredWebmentions = async (options) => { // Fix local URLs based on urlReplacements and sort Webmentions into groups // by target base URL rawWebmentions.forEach((webmention) => { - let url = baseUrl( - fixUrl( - getTarget(webmention).replace(/\/?$/, "/"), - options.urlReplacements - ) - ) + let url = baseUrl(fixUrl(getTarget(webmention).replace(/\/?$/, "/"), options.urlReplacements)) if (!filtered[url]) { filtered[url] = [] @@ -239,6 +181,9 @@ const filteredWebmentions = async (options) => { }) } + const filteredCount = Object.values(filtered).reduce((count, webmentions) => count + webmentions.length, 0) + console.log(`[${hostname(options.domain)}] ${filteredCount} filtered Webmentions pulled from cache.`) + return filtered } @@ -254,10 +199,7 @@ const getWebmentions = async (options, url, allowedTypes = {}) => { webmentions[url] // filter webmentions by allowed response post types .filter((entry) => { - return typeof allowedTypes === "object" && - Object.keys(allowedTypes).length - ? allowedTypes.includes(getType(entry)) - : true + return typeof allowedTypes === "object" && Object.keys(allowedTypes).length ? allowedTypes.includes(getType(entry)) : true }) // sanitize content of webmentions against HTML limit .map((entry) => { @@ -266,9 +208,7 @@ const getWebmentions = async (options, url, allowedTypes = {}) => { if (html.length) { entry.content = sanitizeHTML(html, options.allowedHTML) if (html.length > options.maximumHtmlLength) { - entry.content = `${ - options.maximumHtmlText - } ${getSource(entry)}` + entry.content = `${options.maximumHtmlText} ${getSource(entry)}` } } @@ -297,13 +237,7 @@ module.exports = (eleventyConfig, options = {}) => { // Global Data const filtered = async () => await filteredWebmentions(options) eleventyConfig.addGlobalData("webmentionsByUrl", filtered) - const unfiltered = async () => - await filteredWebmentions(options).then((filtered) => - Object.values(filtered).reduce( - (array, webmentions) => [...array, ...webmentions], - [] - ) - ) + const unfiltered = async () => await filteredWebmentions(options).then((filtered) => Object.values(filtered).reduce((array, webmentions) => [...array, ...webmentions], [])) eleventyConfig.addGlobalData("webmentionsAll", unfiltered) eleventyConfig.addGlobalData("webmentions", unfiltered) @@ -311,10 +245,7 @@ module.exports = (eleventyConfig, options = {}) => { eleventyConfig.addLiquidFilter("getWebmentions", getWebmentionsFilter) // Nunjucks Filter - eleventyConfig.addNunjucksAsyncFilter( - "getWebmentions", - getWebmentionsFilter - ) + eleventyConfig.addNunjucksAsyncFilter("getWebmentions", getWebmentionsFilter) } return { diff --git a/package.json b/package.json index 8f459e5..661e0b7 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@chrisburnell/eleventy-cache-webmentions", - "version": "1.2.1", + "version": "1.2.2", "description": "Fetch and cache webmentions using eleventy-fetch.", "author": "Chris Burnell ", "license": "MIT",