Add option to set Referer header on downloads

#66

.eslintrc

@@ -11,6 +11,7 @@
     "no-param-reassign": 0
   },
   "globals": {
+    "globalChromeState": false,
     "getFilenameFromContentDispositionHeader": false,
 
     "Path": false,
@@ -20,6 +21,7 @@
     "OptionsManagement": false,
     "Notification": false,
     "Menus": false,
+    "Headers": false,
     "Shortcut": false,
     "Router": false,
     "options": false,

CHANGELOG.md

@@ -1,3 +1,7 @@
+# 3.2.0
+
+* Add option to set `Referer` header on downloads, disabled by default. Should fix errors while downloading for sites that check this, especially pixiv.net in Chrome. Requires new permissions. (#66)
+
 # 3.1.3
 
 * Fix submenus not tracking parent menu item

README.md

@@ -32,6 +32,7 @@ Make sure the actual directories exist, or downloads will silently fail.
 
 * <all_urls> permission is used to get around CORS on HTTP HEAD requests (to check for Content-Disposition headers)
 * tabs permission is used to get the active page's title.
+* webRequest permissions are required to inject the Referer header on downloads (disabled by default)
 
 Configure before use.
 

_locales/en/messages.json

@@ -529,7 +529,23 @@
   },
 
   "o_cFetchViaContentScriptHelp": {
-    "message": "Send headers from page with download request. Some sites such as pixiv require this. Slower and prone to failure. Success/failure notifications do not work in this mode. Requires page refresh to take effect."
+    "message": "Do not enable this if you have no problems downloading. Download from the content page and not the extension, sending headers from the page itself. Some sites (eg. pixiv.net) require this to bypass server-side protections. Slower and prone to failure. Success/failure notifications do not work in this mode. Requires page refresh to take effect."
+  },
+
+  "o_cSetRefererHeader": {
+    "message": "Set the \"Referer\" header to page URL if it is missing"
+  },
+
+  "o_cSetRefererHeaderHelp": {
+    "message": "Do not enable this if you have no problems downloading, and try using the content script option before this. Some sites (eg. pixiv.net) check HTTP Referer headers on the server. Enabling this will set the HTTP Referer header to the page URL if it does not already exist."
+  },
+
+  "o_cSetRefererHeaderFilter": {
+    "message": "Set Referer headers for the following sites"
+  },
+
+  "o_cSetRefererHeaderFilterHelp": {
+    "message": "Whitelist of match patterns to match against. One per line. Headers will only be modified for downloads (source URLs) that match this whitelist."
   },
 
   "o_cImportSettings": {

manifest.json

@@ -2,7 +2,7 @@
   "manifest_version": 2,
   "name": "__MSG_extensionName__",
   "description": "__MSG_extensionDescription__",
-  "version": "3.1.3",
+  "version": "3.2.0",
   "default_locale": "en",
 
   "applications": {
@@ -30,6 +30,7 @@
       "src/router.js",
       "src/shortcut.js",
       "src/messaging.js",
+      "src/headers.js",
       "src/variable.js",
       "src/menu.js",
       "src/option.js",
@@ -51,7 +52,9 @@
     "contextMenus",
     "downloads",
     "notifications",
-    "storage"
+    "storage",
+    "webRequest",
+    "webRequestBlocking"
   ],
 
   "options_ui": {

package.json

@@ -1,6 +1,6 @@
 {
   "name": "save-in",
-  "version": "3.1.3",
+  "version": "3.2.0",
   "license": "MIT",
   "scripts": {
     "build": "env -u WEB_EXT_API_KEY -u WEB_EXT_API_SECRET web-ext build --overwrite-dest -i test docs yarn.lock yarn-error.log",

src/download.js

@@ -151,6 +151,8 @@ const Download = {
       globalChromeState = state;
       download(state);
     } else {
+      // Set globalChromeState as well for headers
+      globalChromeState = state;
       fetch(state.info.url, { method: "HEAD", credentials: "include" })
         .then(res => {
           if (res.headers.has("Content-Disposition")) {

src/headers.js

@@ -0,0 +1,51 @@
+const Headers = {
+  refererListener: details => {
+    // TODO: option to ignore or rewrite referer, check if needed
+    const existingReferer = details.requestHeaders.find(
+      h => h.name === "Referer"
+    );
+    if (existingReferer) {
+      return {};
+    }
+
+    if (!globalChromeState || !globalChromeState.info) {
+      return {};
+    }
+
+    const { pageUrl } = globalChromeState.info;
+    if (!pageUrl) {
+      return {};
+    }
+
+    const referer = {
+      name: "Referer",
+      value: pageUrl
+    };
+    details.requestHeaders.push(referer);
+
+    return { requestHeaders: details.requestHeaders };
+  },
+
+  addRequestListener: () => {
+    browser.webRequest.onBeforeSendHeaders.removeListener(
+      Headers.refererListener
+    );
+
+    if (options.setRefererHeader) {
+      const filterList = options.setRefererHeaderFilter || "";
+
+      const urls = filterList.split("\n").map(s => s.trim());
+
+      browser.webRequest.onBeforeSendHeaders.addListener(
+        Headers.refererListener,
+        { urls },
+        ["blocking", "requestHeaders"]
+      );
+    }
+  }
+};
+
+// Export for testing
+if (typeof module !== "undefined") {
+  module.exports = Headers;
+}

src/index.js

@@ -10,6 +10,8 @@ window.init = () => {
   OptionsManagement.loadOptions()
     .then(browser.contextMenus.removeAll())
     .then(() => {
+      Headers.addRequestListener();
+
       Notification.addNotifications({
         notifyOnSuccess: options.notifyOnSuccess,
         notifyOnFailure: options.notifyOnFailure,

src/option.js

@@ -63,7 +63,13 @@ const OptionsManagement = {
     },
     { name: "truncateLength", type: T.VALUE, default: 240 },
     { name: "fetchViaContent", type: T.BOOL, default: false },
-    { name: "tabEnabled", type: T.BOOL, default: false }
+    { name: "tabEnabled", type: T.BOOL, default: false },
+    { name: "setRefererHeader", type: T.BOOL, default: false },
+    {
+      name: "setRefererHeaderFilter",
+      type: T.VALUE,
+      default: "*://i.pximg.net/*"
+    }
   ],
 
   getKeys: () =>

src/options/options.html

@@ -461,6 +461,17 @@ <h2 id="section-more-options">__MSG_o_sMoreOptions__
       <div class="caption caption-line">__MSG_o_cFetchViaContentScriptHelp__</div>
     </label>
 
+    <label>
+      <input type="checkbox" id="setRefererHeader" />
+      __MSG_o_cSetRefererHeader__ <span class="warning badge">Experimental</span>
+      <div class="caption caption-line">__MSG_o_cSetRefererHeaderHelp__</div>
+      <div class="caption-line">
+        <div>__MSG_o_cSetRefererHeaderFilter__</div>
+        <textarea style="width: 100%" type="textarea" id="setRefererHeaderFilter" rows=3 placeholder="" spellcheck="false"></textarea>
+      </div>
+      <div class="caption caption-line">__MSG_o_cSetRefererHeaderFilterHelp__ <a href="https://developer.mozilla.org/en-US/docs/Mozilla/Add-ons/WebExtensions/Match_patterns">MDN: Match patterns</a></div>
+    </label>
+
     <div style="display: flex; justify-content: flex-start; align-items: center;">
       <div id="settings-import" class="button">__MSG_o_cImportSettings__</div>
       <div style="margin-left: 8px;" id="settings-export" class="button">__MSG_o_cExportSettings__</div>