fix non utf8 chars (#713)

cid/helpers/csv2view.py

@@ -30,7 +30,7 @@ def read_csv(input_file_name):
         first_character = open(input_file_name).read(1)
         encoding = 'utf-8-sig' if first_character == '\ufeff' else 'utf-8'
 
-        with open(input_file_name, encoding=encoding) as file_:
+        with open(input_file_name, encoding=encoding, errors='ignore') as file_:
             text = '\n'.join([line for line in read_nonblank_lines(file_)]) # AWS Organization produces a CSV with empty lines
             dialect = sniffer.sniff(text)
             data = [row for row in csv.DictReader(StringIO(text), dialect=dialect, skipinitialspace=True)]
@@ -48,7 +48,6 @@ def read_csv(input_file_name):
 def csv2view(input_file_name: str, name: str, output_file_name: str=None) -> None:
     """ Make an sql mapping from sql """
     logger.debug(f"input {input_file_name}")
-
     data = read_csv(input_file_name)
     lines = []
     for line in data:
@@ -57,9 +56,9 @@ def csv2view(input_file_name: str, name: str, output_file_name: str=None) -> Non
 
     if not lines:
         CidCritical(f'There is no data to write, exiting"')
-        
+
     headers = data[0].keys()
-    
+
     row_lines = '\n, '.join(lines)
     cols = ', '.join([escape_sql(c.lower()) for c in headers ])