diff --git a/1_read_paper_json.py b/1_read_paper_json.py index bc1b05e..47fb182 100644 --- a/1_read_paper_json.py +++ b/1_read_paper_json.py @@ -1,4 +1,5 @@ import os +import sys import argparse import datetime import scrapy @@ -88,3 +89,7 @@ process = CrawlerProcess(settings) process.crawl(OparlSpider, **spargs) process.start() + +failed = process.stats.get_value('custom/failed_job') +if failed: + sys.exit(1)