downloader: adjust queue concurrency

Adjust queue concurrency based off number of links finished in last time period
Close #11

adjust-concurrency.js

@@ -0,0 +1,32 @@
+
+/**
+ * @param {Downloader} downloader
+ */
+const adjust = (downloader) => {
+  if (!downloader.firstPeriodCount) {
+    downloader.firstPeriodCount = Object.keys(downloader.downloadedLinks).length;
+    downloader.lastPeriodTotalCount =
+      downloader.currentPreiodCount =
+        downloader.lastPeriodCount =
+          downloader.firstPeriodCount;
+    return;
+  }
+  const total = Object.keys(downloader.downloadedLinks).length;
+  downloader.lastPeriodCount = downloader.currentPreiodCount;
+  downloader.currentPreiodCount = total - downloader.lastPeriodTotalCount;
+  downloader.lastPeriodTotalCount = total;
+  if (downloader.currentPreiodCount < 2) {
+    downloader.queue.concurrency += 8;
+  } else if (downloader.currentPreiodCount < downloader.lastPeriodCount >> 1) {
+    downloader.queue.concurrency += 4;
+  }
+  if (downloader.currentPreiodCount < downloader.firstPeriodCount >> 2) {
+    downloader.queue.concurrency += 2;
+  }
+  if (downloader.currentPreiodCount > downloader.firstPeriodCount) {
+    downloader.queue.concurrency -= 4;
+  }
+  console.info('concurrency', downloader.queue.concurrency);
+};
+
+module.exports = adjust;

downloader.js

@@ -97,6 +97,14 @@ class Downloader {
   }
   start() {
     this.finished = 0;
+    if (this.adjustConcurrencyTimer) {
+      clearInterval(this.adjustConcurrencyTimer);
+    }
+    if (this.options.adjustConcurrencyPeriod > 0) {
+      this.adjustConcurrencyTimer = setInterval(() => {
+        this.options.adjustConcurrencyFunc(this);
+      }, this.options.adjustConcurrencyPeriod);
+    }
     this.queue.onIdle().then(() => {
       this.finished = 1;
       if (typeof this.options.onSuccess === 'function') {
@@ -107,6 +115,9 @@ class Downloader {
   }
 
   stop() {
+    if (this.adjustConcurrencyTimer) {
+      clearInterval(this.adjustConcurrencyTimer);
+    }
     this.queue.pause();
   }
 }

mdn-local.d.ts

@@ -22,6 +22,8 @@ interface Options {
     onSuccess?: Function;
     onError?: Function;
     detectIncompleteHtml?: '</html>' | '</body>' | string;
+    adjustConcurrencyPeriod: number;
+    adjustConcurrencyFunc?(downloader: any): void;
 }
 
 interface Encoding {

options.js

@@ -1,3 +1,4 @@
+const adjust = require('./adjust-concurrency');
 const defaultOptions = {
   req: {
     retry : 20,
@@ -11,8 +12,8 @@ const defaultOptions = {
       connect: 400,
       secureConnect: 500,
       send: 700,
-      response: 35000,
-      request: 40000
+      response: 20000,
+      request: 23000
     }
   },
   encoding: {
@@ -59,7 +60,7 @@ const defaultOptions = {
   /**
    * @type number
    */
-  concurrency: 16,
+  concurrency: 12,
   /**
    * @type number
    */
@@ -74,6 +75,8 @@ const defaultOptions = {
   /**
    * @type * | function
    */
-  onError: null
+  onError: null,
+  adjustConcurrencyPeriod: 60000,
+  adjustConcurrencyFunc: adjust
 };
 module.exports = defaultOptions;