From cc12988837a7fc8a7108133d4ce5f7440441b7d3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Reichert?=
 <de.abg.reichert.joerg@googlemail.com>
Date: Wed, 11 Oct 2023 22:01:20 +0200
Subject: [PATCH] fail when crawler fails

---
 1_read_paper_json.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/1_read_paper_json.py b/1_read_paper_json.py
index bc1b05e..47fb182 100644
--- a/1_read_paper_json.py
+++ b/1_read_paper_json.py
@@ -1,4 +1,5 @@
 import os
+import sys
 import argparse
 import datetime
 import scrapy
@@ -88,3 +89,7 @@
 process = CrawlerProcess(settings)
 process.crawl(OparlSpider, **spargs)
 process.start()
+
+failed = process.stats.get_value('custom/failed_job')
+if failed:
+    sys.exit(1)