From 2939fc721bef02f74755566a4dff929fc52e6ccd Mon Sep 17 00:00:00 2001 From: Ng Guoyou Date: Sat, 8 Sep 2018 23:33:43 +0800 Subject: [PATCH] Add option to set `Referer` header on downloads #66 --- .eslintrc | 2 ++ CHANGELOG.md | 4 +++ README.md | 1 + _locales/en/messages.json | 18 +++++++++++++- manifest.json | 7 ++++-- package.json | 2 +- src/download.js | 2 ++ src/headers.js | 51 +++++++++++++++++++++++++++++++++++++++ src/index.js | 2 ++ src/option.js | 8 +++++- src/options/options.html | 11 +++++++++ 11 files changed, 103 insertions(+), 5 deletions(-) create mode 100644 src/headers.js diff --git a/.eslintrc b/.eslintrc index 88b867d..d5a6490 100644 --- a/.eslintrc +++ b/.eslintrc @@ -11,6 +11,7 @@ "no-param-reassign": 0 }, "globals": { + "globalChromeState": false, "getFilenameFromContentDispositionHeader": false, "Path": false, @@ -20,6 +21,7 @@ "OptionsManagement": false, "Notification": false, "Menus": false, + "Headers": false, "Shortcut": false, "Router": false, "options": false, diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c1f85c..9f02d50 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +# 3.2.0 + +* Add option to set `Referer` header on downloads, disabled by default. Should fix errors while downloading for sites that check this, especially pixiv.net in Chrome. Requires new permissions. (#66) + # 3.1.3 * Fix submenus not tracking parent menu item diff --git a/README.md b/README.md index 41aae66..19473d0 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,7 @@ Make sure the actual directories exist, or downloads will silently fail. * permission is used to get around CORS on HTTP HEAD requests (to check for Content-Disposition headers) * tabs permission is used to get the active page's title. +* webRequest permissions are required to inject the Referer header on downloads (disabled by default) Configure before use. diff --git a/_locales/en/messages.json b/_locales/en/messages.json index 06dc0ee..d9ef2fc 100644 --- a/_locales/en/messages.json +++ b/_locales/en/messages.json @@ -529,7 +529,23 @@ }, "o_cFetchViaContentScriptHelp": { - "message": "Send headers from page with download request. Some sites such as pixiv require this. Slower and prone to failure. Success/failure notifications do not work in this mode. Requires page refresh to take effect." + "message": "Do not enable this if you have no problems downloading. Download from the content page and not the extension, sending headers from the page itself. Some sites (eg. pixiv.net) require this to bypass server-side protections. Slower and prone to failure. Success/failure notifications do not work in this mode. Requires page refresh to take effect." + }, + + "o_cSetRefererHeader": { + "message": "Set the \"Referer\" header to page URL if it is missing" + }, + + "o_cSetRefererHeaderHelp": { + "message": "Do not enable this if you have no problems downloading, and try using the content script option before this. Some sites (eg. pixiv.net) check HTTP Referer headers on the server. Enabling this will set the HTTP Referer header to the page URL if it does not already exist." + }, + + "o_cSetRefererHeaderFilter": { + "message": "Set Referer headers for the following sites" + }, + + "o_cSetRefererHeaderFilterHelp": { + "message": "Whitelist of match patterns to match against. One per line. Headers will only be modified for downloads (source URLs) that match this whitelist." }, "o_cImportSettings": { diff --git a/manifest.json b/manifest.json index 5283431..6cb7192 100644 --- a/manifest.json +++ b/manifest.json @@ -2,7 +2,7 @@ "manifest_version": 2, "name": "__MSG_extensionName__", "description": "__MSG_extensionDescription__", - "version": "3.1.3", + "version": "3.2.0", "default_locale": "en", "applications": { @@ -30,6 +30,7 @@ "src/router.js", "src/shortcut.js", "src/messaging.js", + "src/headers.js", "src/variable.js", "src/menu.js", "src/option.js", @@ -51,7 +52,9 @@ "contextMenus", "downloads", "notifications", - "storage" + "storage", + "webRequest", + "webRequestBlocking" ], "options_ui": { diff --git a/package.json b/package.json index 04d81b3..03ddfe9 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "save-in", - "version": "3.1.3", + "version": "3.2.0", "license": "MIT", "scripts": { "build": "env -u WEB_EXT_API_KEY -u WEB_EXT_API_SECRET web-ext build --overwrite-dest -i test docs yarn.lock yarn-error.log", diff --git a/src/download.js b/src/download.js index 2bb94ad..43ce604 100644 --- a/src/download.js +++ b/src/download.js @@ -151,6 +151,8 @@ const Download = { globalChromeState = state; download(state); } else { + // Set globalChromeState as well for headers + globalChromeState = state; fetch(state.info.url, { method: "HEAD", credentials: "include" }) .then(res => { if (res.headers.has("Content-Disposition")) { diff --git a/src/headers.js b/src/headers.js new file mode 100644 index 0000000..86816ed --- /dev/null +++ b/src/headers.js @@ -0,0 +1,51 @@ +const Headers = { + refererListener: details => { + // TODO: option to ignore or rewrite referer, check if needed + const existingReferer = details.requestHeaders.find( + h => h.name === "Referer" + ); + if (existingReferer) { + return {}; + } + + if (!globalChromeState || !globalChromeState.info) { + return {}; + } + + const { pageUrl } = globalChromeState.info; + if (!pageUrl) { + return {}; + } + + const referer = { + name: "Referer", + value: pageUrl + }; + details.requestHeaders.push(referer); + + return { requestHeaders: details.requestHeaders }; + }, + + addRequestListener: () => { + browser.webRequest.onBeforeSendHeaders.removeListener( + Headers.refererListener + ); + + if (options.setRefererHeader) { + const filterList = options.setRefererHeaderFilter || ""; + + const urls = filterList.split("\n").map(s => s.trim()); + + browser.webRequest.onBeforeSendHeaders.addListener( + Headers.refererListener, + { urls }, + ["blocking", "requestHeaders"] + ); + } + } +}; + +// Export for testing +if (typeof module !== "undefined") { + module.exports = Headers; +} diff --git a/src/index.js b/src/index.js index 1a68e4d..e0f9a53 100644 --- a/src/index.js +++ b/src/index.js @@ -10,6 +10,8 @@ window.init = () => { OptionsManagement.loadOptions() .then(browser.contextMenus.removeAll()) .then(() => { + Headers.addRequestListener(); + Notification.addNotifications({ notifyOnSuccess: options.notifyOnSuccess, notifyOnFailure: options.notifyOnFailure, diff --git a/src/option.js b/src/option.js index 3c858be..0cf64c9 100644 --- a/src/option.js +++ b/src/option.js @@ -63,7 +63,13 @@ const OptionsManagement = { }, { name: "truncateLength", type: T.VALUE, default: 240 }, { name: "fetchViaContent", type: T.BOOL, default: false }, - { name: "tabEnabled", type: T.BOOL, default: false } + { name: "tabEnabled", type: T.BOOL, default: false }, + { name: "setRefererHeader", type: T.BOOL, default: false }, + { + name: "setRefererHeaderFilter", + type: T.VALUE, + default: "*://i.pximg.net/*" + } ], getKeys: () => diff --git a/src/options/options.html b/src/options/options.html index 8a4a25d..b42fac6 100644 --- a/src/options/options.html +++ b/src/options/options.html @@ -461,6 +461,17 @@

__MSG_o_sMoreOptions__
__MSG_o_cFetchViaContentScriptHelp__
+ +
__MSG_o_cImportSettings__
__MSG_o_cExportSettings__