Boljše preverjanje datumov pri uvozu MIZŠ podatkov

csv/schools-absences.csv

@@ -7601,8 +7601,8 @@ date,absent.from,absent.to,school_type,school,person_type,person_class,reason
 2020-10-24,2020-10-25,2020-11-03,PV,19190,E,ZAVU,ZAVOKU
 2020-10-24,2020-10-22,2020-11-01,PV,24276,E,PVV,PVOKU
 2020-10-24,2020-10-20,2020-10-30,PV,3710,E,PVDS,OSOKU
-2020-10-24,2020-10-21T00:00:00,2020-10-22,PV,9598,A,PVSO2,OSSUM
-2020-10-24,2020-10-21T00:00:00,2020-10-22,PV,9598,A,PVSO2,OSSUM
+2020-10-24,2020-10-21,2020-10-22,PV,9598,A,PVSO2,OSSUM
+2020-10-24,2020-10-21,2020-10-22,PV,9598,A,PVSO2,OSSUM
 2020-10-24,2020-10-19,2020-10-23,ZAV,1001,E,ZAVU,ZAVSUM
 2020-10-24,2020-10-19,2020-10-23,ZAV,1001,E,ZAVU,ZAVSUM
 2020-10-24,2020-10-19,2020-10-23,ZAV,1001,E,ZAVU,ZAVSUM
@@ -17448,7 +17448,7 @@ date,absent.from,absent.to,school_type,school,person_type,person_class,reason
 2021-02-10,2021-02-05,2021-02-14,PV,20816,A,PVSO2,PVSUM
 2021-02-10,2021-02-09,2021-02-18,PV,20816,A,PVSO2,PVSUM
 2021-02-10,2021-02-09,2021-02-18,PV,20816,E,PVV,PVOKU
-2021-02-10,2021-02-10,2201-02-18,PV,20816,E,PVV,PVSUM
+2021-02-10,2021-02-10,2021-02-18,PV,20816,E,PVV,PVSUM
 2021-02-10,2021-02-11,2021-02-14,PV,20816,E,PVV,PVSUM
 2021-02-10,2021-02-11,2021-02-14,PV,20816,E,PVV,PVSUM
 2021-02-10,2021-02-05,2021-02-15,PV,20915,E,PVV,OSOKU
@@ -18063,8 +18063,8 @@ date,absent.from,absent.to,school_type,school,person_type,person_class,reason
 2021-02-15,2021-02-09,2021-02-11,PV,10050,E,PVV,PVSUM
 2021-02-15,2021-02-09,2021-02-11,PV,10050,E,PVDS,PVSUM
 2021-02-15,2021-02-09,2021-02-11,PV,10050,E,PVDS,PVSUM
-2021-02-15,2021-02-12,2121-02-21,PV,10730,E,PVV,PVSUM
-2021-02-15,2021-02-12,2121-02-21,PV,10730,E,PVDS,PVSUM
+2021-02-15,2021-02-12,2021-02-21,PV,10730,E,PVV,PVSUM
+2021-02-15,2021-02-12,2021-02-21,PV,10730,E,PVDS,PVSUM
 2021-02-15,2021-02-09,2021-02-18,PV,1091,E,PVV,PVOKU
 2021-02-15,2021-02-15,2021-02-25,PV,1135,E,PVDS,OSOKU
 2021-02-15,2021-02-10,2021-02-19,PV,1150,A,PVSO1,OSSUM

csv/schools-cases.csv.timestamp

@@ -1 +1,5 @@
+<<<<<<< HEAD
 1613568164
+=======
+1613568229
+>>>>>>> d5e22fd290f15082b09a8f100618b4fe493b8a1f

csv/schools-regimes.csv

@@ -2193,7 +2193,7 @@ date,changed.from,changed.to,school_type,school,person_class,attendees,regime,re
 2021-02-15,2021-02-09,2021-02-18,OSPP,529,OSPP9,8,OSPPKAR,OSPPOKUU
 2021-02-15,2021-02-09,2021-02-11,PV,10050,PVSO1,7,PVKAR,PVSUMV
 2021-02-15,2021-02-09,2021-02-11,PV,10050,PVSO1,10,PVKAR,PVSUMV
-2021-02-15,2021-02-12,2121-02-21,PV,10730,PVSO2,5,PVKAR,PVOKUUC
+2021-02-15,2021-02-12,2021-02-21,PV,10730,PVSO2,5,PVKAR,PVOKUUC
 2021-02-15,2021-02-12,2021-02-21,PV,1126,PVSO2,24,OSKAR,OSOKUUC
 2021-02-15,2021-02-10,2021-02-19,PV,11691,PVSO2,11,PVKAR,PVOKUV
 2021-02-15,2021-02-09,2021-02-10,PV,11909,PVSO1,6,PVKAR,PVSUMV

ostanizdrav/merged.csv.timestamp

@@ -1 +1,5 @@
+<<<<<<< HEAD
 1613696247
+=======
+1613696318
+>>>>>>> d5e22fd290f15082b09a8f100618b4fe493b8a1f

schools/parse_mizs_csvs.py

@@ -1,6 +1,6 @@
 #! /bin/python
 
-from datetime import datetime, timedelta
+from datetime import datetime
 from operator import itemgetter
 import codecs
 import csv
@@ -61,19 +61,45 @@ def get_sledilnik_key(d, mizs_key):
 
 def reformat_dates(date_columns, row):
     """
-    Reformat the dates from a human-readable d.m.Y form into
-    the standard YMD form.
+    Reformat dates (DAT_OD, DAT_DO, DAT_POROCANJA) from a
+    human-readable d.m.Y form into the standard YMD form.
     """
+
+    # parse dates
     for i in date_columns:
         date = dateutil.parser.parse(row[i], dayfirst=True).date()
-        # fix errornous "0020" year entries
+
+        # the reporting app doen't seem to validate the year field
+        # sometimes, typos can happen
+        # fix some common ones
         if date.year == 20:
-            date = datetime(2020, date.month, date.day)
+            date = datetime(2020, date.month, date.day).date()
+        elif date.year == 2121:
+            date = datetime(2021, date.month, date.day).date()
+        elif date.year == 2201:
+            date = datetime(2021, date.month, date.day).date()
 
-        if date.year < 2020 or date.year > 2021:
-            logger.warning("Suspicious date found in line: \n{}\n".format(row))
         row[i] = date
 
+    # do some extra checks on the start and stop dates
+    start_date = row[date_columns[0]]
+    stop_date = row[date_columns[1]]
+    # start date should of course not come after the stop date
+    if start_date > stop_date:
+        logger.warning(
+            "Start date {} is after the stop date {} in line:\n{}".format(
+                start_date, stop_date, row
+            )
+        )
+    # both should not be too far into the past nor the future
+    now = datetime.now().date()
+    if start_date.year < 2020 or (start_date - now).days > 10:
+        logger.warning(
+            "Weird start date {} found in line: \n{}".format(start_date, row)
+        )
+    elif stop_date.year < 2020 or (stop_date - now).days > 90:
+        logger.warning("Weird stop date {} found in line: \n{}".format(stop_date, row))
+
 
 def parse_csv(url):
     """
@@ -215,6 +241,7 @@ def school_regimes_csv(outfile):
         csvwriter.writeheader()
         csvwriter.writerows(regimes)
 
+
 if __name__ == "__main__":
     dicts = load_dicts()