fail when crawler fails

CodeforLeipzig · Oct 11, 2023 · cc12988 · cc12988
1 parent 8bf0a23
commit cc12988
Showing 1 changed file with 5 additions and 0 deletions.
diff --git a/1_read_paper_json.py b/1_read_paper_json.py
@@ -1,4 +1,5 @@
 import os
+import sys
 import argparse
 import datetime
 import scrapy
@@ -88,3 +89,7 @@
 process = CrawlerProcess(settings)
 process.crawl(OparlSpider, **spargs)
 process.start()
+
+failed = process.stats.get_value('custom/failed_job')
+if failed:
+    sys.exit(1)