From ce0eec25a0b6e74864c860f856c720e69065d23d Mon Sep 17 00:00:00 2001
From: myfreeer <myfreeer@users.noreply.github.com>
Date: Wed, 18 Sep 2019 21:36:52 +0800
Subject: [PATCH] downloader: adjust queue concurrency

Adjust queue concurrency based off number of links finished in last time period
Close https://github.com/myfreeer/mdn-local/issues/11
---
 adjust-concurrency.js | 32 ++++++++++++++++++++++++++++++++
 downloader.js         | 11 +++++++++++
 mdn-local.d.ts        |  2 ++
 options.js            | 11 +++++++----
 4 files changed, 52 insertions(+), 4 deletions(-)
 create mode 100644 adjust-concurrency.js

diff --git a/adjust-concurrency.js b/adjust-concurrency.js
new file mode 100644
index 00000000..77da639e
--- /dev/null
+++ b/adjust-concurrency.js
@@ -0,0 +1,32 @@
+
+/**
+ * @param {Downloader} downloader
+ */
+const adjust = (downloader) => {
+  if (!downloader.firstPeriodCount) {
+    downloader.firstPeriodCount = Object.keys(downloader.downloadedLinks).length;
+    downloader.lastPeriodTotalCount =
+      downloader.currentPreiodCount =
+        downloader.lastPeriodCount =
+          downloader.firstPeriodCount;
+    return;
+  }
+  const total = Object.keys(downloader.downloadedLinks).length;
+  downloader.lastPeriodCount = downloader.currentPreiodCount;
+  downloader.currentPreiodCount = total - downloader.lastPeriodTotalCount;
+  downloader.lastPeriodTotalCount = total;
+  if (downloader.currentPreiodCount < 2) {
+    downloader.queue.concurrency += 8;
+  } else if (downloader.currentPreiodCount < downloader.lastPeriodCount >> 1) {
+    downloader.queue.concurrency += 4;
+  }
+  if (downloader.currentPreiodCount < downloader.firstPeriodCount >> 2) {
+    downloader.queue.concurrency += 2;
+  }
+  if (downloader.currentPreiodCount > downloader.firstPeriodCount) {
+    downloader.queue.concurrency -= 4;
+  }
+  console.info('concurrency', downloader.queue.concurrency);
+};
+
+module.exports = adjust;
diff --git a/downloader.js b/downloader.js
index 2ddf5639..2c50603d 100644
--- a/downloader.js
+++ b/downloader.js
@@ -97,6 +97,14 @@ class Downloader {
   }
   start() {
     this.finished = 0;
+    if (this.adjustConcurrencyTimer) {
+      clearInterval(this.adjustConcurrencyTimer);
+    }
+    if (this.options.adjustConcurrencyPeriod > 0) {
+      this.adjustConcurrencyTimer = setInterval(() => {
+        this.options.adjustConcurrencyFunc(this);
+      }, this.options.adjustConcurrencyPeriod);
+    }
     this.queue.onIdle().then(() => {
       this.finished = 1;
       if (typeof this.options.onSuccess === 'function') {
@@ -107,6 +115,9 @@ class Downloader {
   }
 
   stop() {
+    if (this.adjustConcurrencyTimer) {
+      clearInterval(this.adjustConcurrencyTimer);
+    }
     this.queue.pause();
   }
 }
diff --git a/mdn-local.d.ts b/mdn-local.d.ts
index 6b5ea128..ef5e801a 100644
--- a/mdn-local.d.ts
+++ b/mdn-local.d.ts
@@ -22,6 +22,8 @@ interface Options {
     onSuccess?: Function;
     onError?: Function;
     detectIncompleteHtml?: '</html>' | '</body>' | string;
+    adjustConcurrencyPeriod: number;
+    adjustConcurrencyFunc?(downloader: any): void;
 }
 
 interface Encoding {
diff --git a/options.js b/options.js
index 4c0ffca9..d285ee64 100644
--- a/options.js
+++ b/options.js
@@ -1,3 +1,4 @@
+const adjust = require('./adjust-concurrency');
 const defaultOptions = {
   req: {
     retry : 20,
@@ -11,8 +12,8 @@ const defaultOptions = {
       connect: 400,
       secureConnect: 500,
       send: 700,
-      response: 35000,
-      request: 40000
+      response: 20000,
+      request: 23000
     }
   },
   encoding: {
@@ -59,7 +60,7 @@ const defaultOptions = {
   /**
    * @type number
    */
-  concurrency: 16,
+  concurrency: 12,
   /**
    * @type number
    */
@@ -74,6 +75,8 @@ const defaultOptions = {
   /**
    * @type * | function
    */
-  onError: null
+  onError: null,
+  adjustConcurrencyPeriod: 60000,
+  adjustConcurrencyFunc: adjust
 };
 module.exports = defaultOptions;
\ No newline at end of file