From ea93b5e14db1699ee0535d89c4c4415025b77151 Mon Sep 17 00:00:00 2001 From: Stephen Allen <720493+stetime@users.noreply.github.com> Date: Tue, 21 Nov 2023 19:05:38 +0000 Subject: [PATCH 1/2] Implement HTTP Conditional Requests in parseURL --- lib/parser.js | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/lib/parser.js b/lib/parser.js index 7d71431..b77509b 100644 --- a/lib/parser.js +++ b/lib/parser.js @@ -26,6 +26,8 @@ class Parser { if (!options.timeout) options.timeout = DEFAULT_TIMEOUT; this.options = options; this.xmlParser = new xml2js.Parser(this.options.xml2js); + this.etags = {}; + this.lastModified = {}; } parseString(xml, callback) { @@ -73,6 +75,12 @@ class Parser { let get = feedUrl.indexOf('https') === 0 ? https.get : http.get; let urlParts = url.parse(feedUrl); let headers = Object.assign({}, DEFAULT_HEADERS, this.options.headers); + if (this.etags[feedUrl]) { + headers['If-None-Match'] = this.etags[feedUrl]; + } + if (this.lastModified[feedUrl]) { + headers['If-Modified-Since'] = this.lastModified[feedUrl]; + } let timeout = null; let prom = new Promise((resolve, reject) => { const requestOpts = Object.assign({headers}, urlParts, this.options.requestOptions); @@ -84,9 +92,21 @@ class Parser { const newLocation = url.resolve(feedUrl, res.headers['location']); return this.parseURL(newLocation, null, redirectCount + 1).then(resolve, reject); } - } else if (res.statusCode >= 300) { - return reject(new Error("Status code " + res.statusCode)) + } else if (res.statusCode === 304) { + return resolve(null); + } + else if (res.statusCode >= 300) { + return reject(new Error("Status code " + res.statusCode)); + } + + if (res.headers['etag']) { + this.etags[feedUrl] = res.headers['etag']; + } + + if (res.headers['last-modified']) { + this.lastModified[feedUrl] = res.headers['last-modified']; } + let encoding = utils.getEncodingFromContentType(res.headers['content-type']); res.setEncoding(encoding); res.on('data', (chunk) => { From 67e48002a5b1fa2634077cdedd83c823b327cca0 Mon Sep 17 00:00:00 2001 From: Stephen Allen <720493+stetime@users.noreply.github.com> Date: Wed, 29 Nov 2023 21:29:57 +0000 Subject: [PATCH 2/2] Bugfix: Handle absence of media:content in XML parsing --- lib/parser.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/parser.js b/lib/parser.js index b77509b..0f6e55a 100644 --- a/lib/parser.js +++ b/lib/parser.js @@ -246,7 +246,7 @@ class Parser { } if (xmlItem.category) item.categories = xmlItem.category; - var mediaContent = xmlItem['media:content'][0].$; + var mediaContent = xmlItem['media:content']?.[0]?.$ ?? null; if(mediaContent) item.mediaContent = mediaContent; this.setISODate(item);