Skip to content

Commit

Permalink
Drop obsolete logic from the downloadFile function in `test/downloa…
Browse files Browse the repository at this point in the history
…dutils.js`

This code is old and predates the improvements we made to the test
manifest to only contain working URLs (either Web Archive or
GitHub/Bugzilla links), so the fallback logic to try the Web Archive is
no longer necessary. This greatly simplifies the function and also
makes sure that we fail directly in case a bad URL is added to the
manifest, instead of having it work "accidentally" because of this
logic, since we want the manifest to be correct at all times (and
otherwise fail loudly).
  • Loading branch information
timvandermeij committed May 22, 2021
1 parent 0df1a56 commit 9943022
Showing 1 changed file with 5 additions and 35 deletions.
40 changes: 5 additions & 35 deletions test/downloadutils.js
Original file line number Diff line number Diff line change
Expand Up @@ -38,11 +38,9 @@ function rewriteWebArchiveUrl(url) {
function downloadFile(file, url, callback, redirects) {
url = rewriteWebArchiveUrl(url);

var completed = false;
var protocol = /^https:\/\//.test(url) ? https : http;
protocol
.get(url, function (response) {
var redirectTo;
if (
response.statusCode === 301 ||
response.statusCode === 302 ||
Expand All @@ -52,56 +50,28 @@ function downloadFile(file, url, callback, redirects) {
if (redirects > 10) {
callback("Too many redirects");
}
redirectTo = response.headers.location;
var redirectTo = response.headers.location;
redirectTo = require("url").resolve(url, redirectTo);
downloadFile(file, redirectTo, callback, (redirects || 0) + 1);
return;
}
if (response.statusCode === 404 && !url.includes("web.archive.org")) {
// trying waybackmachine
redirectTo = "http://web.archive.org/web/" + url;
downloadFile(file, redirectTo, callback, (redirects || 0) + 1);
return;
}

if (response.statusCode !== 200) {
if (!completed) {
completed = true;
callback("HTTP " + response.statusCode);
}
callback("HTTP " + response.statusCode);
return;
}
var stream = fs.createWriteStream(file);
stream.on("error", function (err) {
if (!completed) {
completed = true;
callback(err);
}
callback(err);
});
response.pipe(stream);
stream.on("finish", function () {
stream.end();
if (!completed) {
completed = true;
callback();
}
callback();
});
})
.on("error", function (err) {
if (!completed) {
if (
typeof err === "object" &&
err.errno === "ENOTFOUND" &&
!url.includes("web.archive.org")
) {
// trying waybackmachine
var redirectTo = "http://web.archive.org/web/" + url;
downloadFile(file, redirectTo, callback, (redirects || 0) + 1);
return;
}
completed = true;
callback(err);
}
callback(err);
});
}

Expand Down

0 comments on commit 9943022

Please sign in to comment.