fix: bad_keywords.txt apparently contains some blacklisted websites

src/domain_stats.ts

@@ -35,25 +35,32 @@ export class Domains {
  // Thanks tripleee!
  // https://github.com/Charcoal-SE/halflife/blob/ab0fa5fc2a048b9e17762ceb6e3472e4d9c65317/halflife.py#L77
  const [
- watchedCall, blacklistedCall, prsCall, whitelistedCall, redirectorsCall
+ watchedCall, blacklistedCall, prsCall, whitelistedCall, redirectorsCall, badCall
  ] = await Promise.all(([
  fetch(githubUrls.watched),
  fetch(githubUrls.blacklisted),
  fetch(githubUrls.api),
  fetch(githubUrls.whitelisted),
- fetch(githubUrls.redirectors)
+ fetch(githubUrls.redirectors),
+ fetch(githubUrls.bad)
  ]));
 
- const [watched, blacklisted, prs, whitelisted, redirectors] = await Promise.all([
+ const [
+ watched, blacklisted, prs, whitelisted, redirectors, bad
+ ] = await Promise.all([
  watchedCall.text(),
  blacklistedCall.text(),
  prsCall.json() as Promise<GithubApiResponse[]>,
  whitelistedCall.text(),
- redirectorsCall.text()
+ redirectorsCall.text(),
+ badCall.text()
  ]);
 
+ const badRegexes = getRegexesFromTxtFile(blacklisted, 0);
+ const blacklistedRegexes = getRegexesFromTxtFile(bad, 0);
+
  this.watched = getRegexesFromTxtFile(watched, 2);
- this.blacklisted = getRegexesFromTxtFile(blacklisted, 0);
+ this.blacklisted = badRegexes.concat(blacklistedRegexes);
  this.pullRequests = parseApiResponse(prs);
 
  this.whitelisted = whitelisted.split('\n');

src/github.ts

@@ -22,7 +22,8 @@ export const githubUrls = {
  whitelisted: 'https://raw.githubusercontent.com/userscripters/fire-extra-functionality/master/ini/whitelisted_domains.txt',
  redirectors: 'https://raw.githubusercontent.com/userscripters/fire-extra-functionality/master/ini/redirectors.txt',
  watched: 'https://raw.githubusercontent.com/Charcoal-SE/SmokeDetector/master/watched_keywords.txt',
- blacklisted: 'https://raw.githubusercontent.com/Charcoal-SE/SmokeDetector/master/blacklisted_websites.txt'
+ blacklisted: 'https://raw.githubusercontent.com/Charcoal-SE/SmokeDetector/master/blacklisted_websites.txt',
+ bad: 'https://raw.githubusercontent.com/Charcoal-SE/SmokeDetector/master/bad_keywords.txt'
 };
 
 function makeRegexESCompatible(keyword: string): RegExp[] {

test/index.spec.ts

@@ -76,8 +76,21 @@ describe('index helpers', () => {
  validWatches.forEach(keyword => expect(isWatched(keyword)).to.be.true);
  invalidWatches.forEach(keyword => expect(isWatched(keyword)).to.be.false);
 
- const validBlacklists = ['powerigfaustralia', 'ewebtonic.in', 'healthcaresup', 'd680adc632091138ed9fd09659e15dc9'];
- const invalidBlacklists = invalidWatches;
+ const validBlacklists = [
+ // blacklisted websites
+ 'powerigfaustralia',
+ 'ewebtonic.in',
+ 'healthcaresup',
+ 'd680adc632091138ed9fd09659e15dc9',
+
+ // bad keywords
+ 'orvigomax',
+ 'opstree.com'
+ ];
+ const invalidBlacklists = [
+ ...invalidWatches,
+ 'blog.opstree.com' // test negative lookbehind
+ ];
 
  validBlacklists.forEach(keyword => expect(isBlacklisted(keyword)).to.be.true);
  invalidBlacklists.forEach(keyword => expect(isBlacklisted(keyword)).to.be.false);