Skip to content

Commit

Permalink
fix: add duration to defaults, remove uniqueKey check, add extra cons…
Browse files Browse the repository at this point in the history 
…ole message for acquiring filteredWebmentions, README updates
  • Loading branch information
@chrisburnell
chrisburnell committed May 30, 2023
1 parent fc09792 commit 5df18a6
Show file tree
Hide file tree
Showing 3 changed files with 113 additions and 123 deletions.
111 changes: 85 additions & 26 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,55 +32,114 @@ module.exports = function (eleventyConfig) {

## Options

<table>
<thead>
<tr>
<th>option</th>
<th>default value</th>
<th>description</th>
<th>version added</th>
</tr>
</thead>
<tbody>
<tr>
<td><code>domain</code><br><em>required</em></td>
<td>—</td>
<td>The website you’re fetching Webmentions for.</td>
<td>0.0.1</td>
</tr>
<tr>
<td><code>feed</code><br><em>required</em></td>
<td>—</td>
<td>The URL of your Webmention server’s feed for your <code>domain</code>.</td>
<td>0.2.0</td>
</tr>
<tr>
<td><code>key</code><br><em>required</em></td>
<td>—</td>
<td>The key in the above <code>feed</code> whose value is an Array of Webmentions.</td>
<td>0.0.1</td>
</tr>
<tr>
<td><code>directory</code></td>
<td><code>".cache"</code></td>
<td>See <a href="https://www.11ty.dev/docs/plugins/cache/#cache-directory">Eleventy Fetch’s Cache Directory</a> for more information.</td>
<td>1.1.2</td>
</tr>
<tr>
<td><code>duration</code></td>
<td><code>"1d"</code> <em>or</em> 1 day</td>
<td>See <a href="https://www.11ty.dev/docs/plugins/cache/#change-the-cache-duration">Eleventy Fetch’s Cache Duration</a> for more information.</td>
<td>0.0.1</td>
</tr>
<tr>
<td><code>uniqueKey</code></td>
<td><code>"webmentions"</code></td>
<td>The name of the file generated by Eleventy Fetch.</td>
<td>0.1.9</td>
</tr>
<tr>
<td><code>allowedHTML</code></td>
<td>See code example below</td>
<td>See the <a href="https://www.npmjs.com/package/sanitize-html">sanitize-html</a> package for more information.</td>
<td>0.0.1</td>
</tr>
<tr>
<td><code>allowlist</code></td>
<td><code>[]</code></td>
<td>An Array of root URLs from which Webmentions are kept.</td>
<td>1.1.0</td>
</tr>
<tr>
<td><code>blocklist</code></td>
<td><code>[]</code></td>
<td>An Array of root URLs from which Webmentions are discarded.</td>
<td>1.1.0</td>
</tr>
<tr>
<td><code>urlReplacements</code></td>
<td><code>{}</code></td>
<td>An Object of key-value string pairs containing from-to URL replacements on this <code>domain</code>.</td>
<td>0.0.3</td>
</tr>
<tr>
<td><code>maximumHtmlLength</code></td>
<td><code>2000</code></td>
<td>Maximum number of characters in a Webmention’s HTML content, beyond which point a different message is shown, referring to the original source.</td>
<td>0.0.1</td>
</tr>
<tr>
<td><code>maximumHtmlText</code></td>
<td><code>"mentioned this in"</code></td>
<td>The glue-y part of the message displayed when a Webmention content’s character count exceeds <code>maximumHtmlLength</code>.</td>
<td>0.1.0</td>
</tr>
</tbody>
</table>

Advanced control over how the Webmentions are cached and processed is done by passing `options` into the plugin when using `addPlugin`:

```javascript
const pluginWebmentions = require("@chrisburnell/eleventy-cache-webmentions")

module.exports = function (eleventyConfig) {
eleventyConfig.addPlugin(pluginWebmentions, {
// domain: required or the plugin will not function
// this is the website that you want to pull in Webmentions for
domain: "https://example.com",
// feed: required or the plugin will not function
// defines the URL of your Webmention server where a feed of Webmentions for your domain can be found
feed: "https://webmentions.example.com?token=S3cr3tT0k3n",
// key: required or the plugin will not function
// dictates the key inside the feed where the array of Webmentions is located
key: "children",
// directory: ".cache" by default
// see https://www.11ty.dev/docs/plugins/cache/#cache-directory for more info
directory: ".cache",
// duration: "1d" by default
// see https://www.11ty.dev/docs/plugins/cache/#change-the-cache-duration for more info
duration: "1d",
// uniquekey: "webmentions" by default
// dictates the name sent to eleventy-fetch to name the file
uniqueKey: "webmentions",
// allowedHTML: Object by default
// see https://www.npmjs.com/package/sanitize-html for more info
allowedHTML: {
allowedTags: ["b", "i", "em", "strong", "a"],
allowedAttributes: {
a: ["href"],
},
},
// allowlist: [] by default
// array of root URLs from which webmentions are wanted exclusively
allowlist: [],
// blocklist: [] by default
// array of root URLs from which webmentions are not wanted
// exclusively
blocklist: [],
// urlReplacements: {} by default
// object of key:value pairs containing from:to URL replacements
urlReplacements: {},
// maximumHtmlLength: 2000 by default
// number of characters in the HTML content at which a different
// message is shown instead of the content
maximumHtmlLength: 2000,
// maximumHtmlText: "mentioned this in" by default
// message shown when maximumHtmlLength is reached
maximumHtmlText: "mentioned this in",
})
}
Expand Down
123 changes: 27 additions & 96 deletions eleventy-cache-webmentions.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,7 @@ const absoluteURL = (url, domain) => {
try {
return new URL(url, domain).toString()
} catch (e) {
console.log(
`Trying to convert ${url} to be an absolute url with base ${domain} and failed.`
)
console.log(`Trying to convert ${url} to be an absolute url with base ${domain} and failed.`)
return url
}
}
Expand All @@ -21,13 +19,10 @@ const baseUrl = (url) => {
}

const fixUrl = (url, urlReplacements) => {
return Object.entries(urlReplacements).reduce(
(accumulator, [key, value]) => {
const regex = new RegExp(key, "g")
return accumulator.replace(regex, value)
},
url
)
return Object.entries(urlReplacements).reduce((accumulator, [key, value]) => {
const regex = new RegExp(key, "g")
return accumulator.replace(regex, value)
}, url)
}

const hostname = (value) => {
Expand All @@ -43,45 +38,27 @@ const epoch = (value) => {
}

const getPublished = (webmention) => {
return (
webmention["wm-received"] ||
webmention?.["data"]["published"] ||
webmention["verified_date"] ||
webmention["published"]
)
return webmention["wm-received"] || webmention?.["data"]["published"] || webmention["verified_date"] || webmention["published"]
}

const getContent = (webmention) => {
return (
webmention?.["content"]?.["html"] ||
webmention?.["content"] ||
webmention?.["data"]?.["content"] ||
""
)
return webmention?.["content"]?.["html"] || webmention?.["content"] || webmention?.["data"]?.["content"] || ""
}

const getSource = (webmention) => {
return (
webmention["url"] ||
webmention?.["data"]["url"] ||
webmention["wm-source"] ||
webmention["source"]
)
return webmention["url"] || webmention?.["data"]["url"] || webmention["wm-source"] || webmention["source"]
}

const getTarget = (webmention) => {
return webmention["wm-target"] || webmention["target"]
}

const getType = (webmention) => {
return (
webmention["wm-property"] ||
webmention?.["activity"]["type"] ||
webmention["type"]
)
return webmention["wm-property"] || webmention?.["activity"]["type"] || webmention["type"]
}

const defaults = {
duration: "1d",
uniqueKey: "webmentions",
allowedHTML: {
allowedTags: ["b", "i", "em", "strong", "a"],
Expand All @@ -98,27 +75,15 @@ const defaults = {

const fetchWebmentions = async (options) => {
if (!options.domain) {
throw new Error(
"domain is a required field when attempting to retrieve Webmentions. See https://www.npmjs.com/package/@chrisburnell/eleventy-cache-webmentions#installation for more information."
)
throw new Error("domain is a required field when attempting to retrieve Webmentions. See https://www.npmjs.com/package/@chrisburnell/eleventy-cache-webmentions#installation for more information.")
}

if (!options.feed) {
throw new Error(
"feed is a required field when attempting to retrieve Webmentions. See https://www.npmjs.com/package/@chrisburnell/eleventy-cache-webmentions#installation for more information."
)
throw new Error("feed is a required field when attempting to retrieve Webmentions. See https://www.npmjs.com/package/@chrisburnell/eleventy-cache-webmentions#installation for more information.")
}

if (!options.key) {
throw new Error(
"key is a required field when attempting to retrieve Webmentions. See https://www.npmjs.com/package/@chrisburnell/eleventy-cache-webmentions#installation for more information."
)
}

if (!options.uniqueKey) {
throw new Error(
"uniqueKey is a required field when attempting to retrieve Webmentions. See https://www.npmjs.com/package/@chrisburnell/eleventy-cache-webmentions#installation for more information."
)
throw new Error("key is a required field when attempting to retrieve Webmentions. See https://www.npmjs.com/package/@chrisburnell/eleventy-cache-webmentions#installation for more information.")
}

let asset = new AssetCache(options.uniqueKey, options.directory)
Expand All @@ -135,42 +100,24 @@ const fetchWebmentions = async (options) => {
// results since the most recent webmention
if (!asset.isCacheValid(options.duration)) {
const since = webmentions.length ? getPublished(webmentions[0]) : false
const url = `${options.feed}${
since
? `${options.feed.includes("?") ? "&" : "?"}since=${since}`
: ""
}`
const url = `${options.feed}${since ? `${options.feed.includes("?") ? "&" : "?"}since=${since}` : ""}`
await fetch(url)
.then(async (response) => {
if (response.ok) {
const feed = await response.json()
if (feed[options.key].length) {
webmentions = uniqBy(
[...feed[options.key], ...webmentions],
(entry) => {
return getSource(entry)
}
)
console.log(
`[${hostname(options.domain)}] ${
feed[options.key].length
} new Webmentions fetched into cache.`
)
webmentions = uniqBy([...feed[options.key], ...webmentions], (entry) => {
return getSource(entry)
})
console.log(`[${hostname(options.domain)}] ${feed[options.key].length} new Webmentions fetched into cache.`)
}
await asset.save(webmentions, "json")
return webmentions
}
return Promise.reject(response)
})
.catch((error) => {
console.log(
`[${hostname(
options.domain
)}] Something went wrong with your request to ${hostname(
options.feed
)}!`,
error
)
console.log(`[${hostname(options.domain)}] Something went wrong with your request to ${hostname(options.feed)}!`, error)
})
}

Expand Down Expand Up @@ -218,12 +165,7 @@ const filteredWebmentions = async (options) => {
// Fix local URLs based on urlReplacements and sort Webmentions into groups
// by target base URL
rawWebmentions.forEach((webmention) => {
let url = baseUrl(
fixUrl(
getTarget(webmention).replace(/\/?$/, "/"),
options.urlReplacements
)
)
let url = baseUrl(fixUrl(getTarget(webmention).replace(/\/?$/, "/"), options.urlReplacements))

if (!filtered[url]) {
filtered[url] = []
Expand All @@ -239,6 +181,9 @@ const filteredWebmentions = async (options) => {
})
}

const filteredCount = Object.values(filtered).reduce((count, webmentions) => count + webmentions.length, 0)
console.log(`[${hostname(options.domain)}] ${filteredCount} filtered Webmentions pulled from cache.`)

return filtered
}

Expand All @@ -254,10 +199,7 @@ const getWebmentions = async (options, url, allowedTypes = {}) => {
webmentions[url]
// filter webmentions by allowed response post types
.filter((entry) => {
return typeof allowedTypes === "object" &&
Object.keys(allowedTypes).length
? allowedTypes.includes(getType(entry))
: true
return typeof allowedTypes === "object" && Object.keys(allowedTypes).length ? allowedTypes.includes(getType(entry)) : true
})
// sanitize content of webmentions against HTML limit
.map((entry) => {
Expand All @@ -266,9 +208,7 @@ const getWebmentions = async (options, url, allowedTypes = {}) => {
if (html.length) {
entry.content = sanitizeHTML(html, options.allowedHTML)
if (html.length > options.maximumHtmlLength) {
entry.content = `${
options.maximumHtmlText
} <a href="${getSource(entry)}">${getSource(entry)}</a>`
entry.content = `${options.maximumHtmlText} <a href="${getSource(entry)}">${getSource(entry)}</a>`
}
}

Expand Down Expand Up @@ -297,24 +237,15 @@ module.exports = (eleventyConfig, options = {}) => {
// Global Data
const filtered = async () => await filteredWebmentions(options)
eleventyConfig.addGlobalData("webmentionsByUrl", filtered)
const unfiltered = async () =>
await filteredWebmentions(options).then((filtered) =>
Object.values(filtered).reduce(
(array, webmentions) => [...array, ...webmentions],
[]
)
)
const unfiltered = async () => await filteredWebmentions(options).then((filtered) => Object.values(filtered).reduce((array, webmentions) => [...array, ...webmentions], []))
eleventyConfig.addGlobalData("webmentionsAll", unfiltered)
eleventyConfig.addGlobalData("webmentions", unfiltered)

// Liquid Filter
eleventyConfig.addLiquidFilter("getWebmentions", getWebmentionsFilter)

// Nunjucks Filter
eleventyConfig.addNunjucksAsyncFilter(
"getWebmentions",
getWebmentionsFilter
)
eleventyConfig.addNunjucksAsyncFilter("getWebmentions", getWebmentionsFilter)
}

return {
Expand Down
2 changes: 1 addition & 1 deletion package.json
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@chrisburnell/eleventy-cache-webmentions",
"version": "1.2.1",
"version": "1.2.2",
"description": "Fetch and cache webmentions using eleventy-fetch.",
"author": "Chris Burnell <me@chrisburnell.com>",
"license": "MIT",
Expand Down

0 comments on commit 5df18a6

Please sign in to comment.