From 394fb2c06b1741d3aad0fb51ce7e185e94d9d7f0 Mon Sep 17 00:00:00 2001 From: Iakov GAN <82834333+iakov-aws@users.noreply.github.com> Date: Wed, 10 Jan 2024 14:18:39 +0100 Subject: [PATCH] fix non utf8 chars (#713) --- cid/helpers/csv2view.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/cid/helpers/csv2view.py b/cid/helpers/csv2view.py index c6e32467..17265195 100644 --- a/cid/helpers/csv2view.py +++ b/cid/helpers/csv2view.py @@ -30,7 +30,7 @@ def read_csv(input_file_name): first_character = open(input_file_name).read(1) encoding = 'utf-8-sig' if first_character == '\ufeff' else 'utf-8' - with open(input_file_name, encoding=encoding) as file_: + with open(input_file_name, encoding=encoding, errors='ignore') as file_: text = '\n'.join([line for line in read_nonblank_lines(file_)]) # AWS Organization produces a CSV with empty lines dialect = sniffer.sniff(text) data = [row for row in csv.DictReader(StringIO(text), dialect=dialect, skipinitialspace=True)] @@ -48,7 +48,6 @@ def read_csv(input_file_name): def csv2view(input_file_name: str, name: str, output_file_name: str=None) -> None: """ Make an sql mapping from sql """ logger.debug(f"input {input_file_name}") - data = read_csv(input_file_name) lines = [] for line in data: @@ -57,9 +56,9 @@ def csv2view(input_file_name: str, name: str, output_file_name: str=None) -> Non if not lines: CidCritical(f'There is no data to write, exiting"') - + headers = data[0].keys() - + row_lines = '\n, '.join(lines) cols = ', '.join([escape_sql(c.lower()) for c in headers ])