Skip to content
This repository has been archived by the owner on Jul 7, 2023. It is now read-only.

Commit

Permalink
add another fallback for portuguese scraping
Browse files Browse the repository at this point in the history
  • Loading branch information
BenShelton committed Feb 13, 2019
1 parent 925cdcc commit 9f644a8
Showing 1 changed file with 8 additions and 1 deletion.
9 changes: 8 additions & 1 deletion database/scraper.js
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import cheerio from 'cheerio'

const LANGUAGE_OPTIONS = {
en: {
months: ['january', 'february', 'march', 'april', 'may', 'june', 'july', 'august', 'september', 'october', 'november', 'december'],
addressConstructor: date => {
return ['https://wol.jw.org/en/wol/dt/r1/lp-e/' + date.replace(/-/g, '/')]
},
Expand Down Expand Up @@ -35,6 +36,9 @@ const LANGUAGE_OPTIONS = {
workbookWeeks.push(sD + '-' + eD + sMonth)
workbookWeeks.push(sD + 'a' + eD + '-' + sMonth)
workbookWeeks.push(sD + '-' + eD + '-' + sMonth)
// just in case of an accidental english abbreviation instead (e.g. /programa-reuniao-22a28-apr/)
const englishMonth = LANGUAGE_OPTIONS.en.months[sM - 1].substr(0, 3)
workbookWeeks.push(sD + 'a' + eD + '-' + englishMonth)
} else {
const eMonth = months[eM - 1].substr(0, 3)
workbookWeeks.push(sD + sMonth + '-' + eD + eMonth)
Expand Down Expand Up @@ -93,7 +97,10 @@ export default function scrapeWOL (date, language) {
)
}))
.then(
() => { throw new Error('404') },
() => {
console.log('No URIs matched: ', uris)
throw new Error('404')
},
val => Promise.resolve(val)
)
.then($ => {
Expand Down

0 comments on commit 9f644a8

Please sign in to comment.