fix: Numbers to floats (#106)

Closes #103

- Replaces #104
- This implementations is based on this comment
#103 (comment).
Wise treats all `number` json types as floats in snowflake.
- As part of this I reverted the changes from
#97 since they arent
needed anymore. We'll let the json schema validation enforce this vs
having strict snowflake schema enforcement.

Warning: I think this is a breaking change because I believe the target
will now try to update the type of existing columns from NUMBER to
DOUBLE which throws a `cannot change column NUM_COL from type
NUMBER(38,0) to FLOAT`.

@edgarrmondragon how do you think we should handle this in terms of
breaking changes? Should this be the 1.0 release?

target_snowflake/connector.py

@@ -14,8 +14,6 @@
 from target_snowflake.snowflake_types import NUMBER, TIMESTAMP_NTZ, VARIANT
 
 SNOWFLAKE_MAX_STRING_LENGTH = 16777216
-SNOWFLAKE_MAX_NUMBER_PRECISION = 38
-SNOWFLAKE_MAX_NUMBER_SCALE = 0
 
 class TypeMap:
  def __init__(self, operator, map_value, match_value=None):
@@ -95,7 +93,7 @@ def _convert_type(self, sql_type):
  if isinstance(sql_type, sct.TIMESTAMP_NTZ):
  return TIMESTAMP_NTZ
  elif isinstance(sql_type, sct.NUMBER):
- return NUMBER(precision=sql_type.precision, scale=sql_type.scale)
+ return NUMBER
  elif isinstance(sql_type, sct.VARIANT):
  return VARIANT
  else:
@@ -220,28 +218,6 @@ def _conform_max_length(jsonschema_type):
  jsonschema_type["maxLength"] = SNOWFLAKE_MAX_STRING_LENGTH
  return jsonschema_type
 
- @staticmethod
- def _get_numeric_precision(jsonschema_type):
- return SNOWFLAKE_MAX_NUMBER_PRECISION
-
- @staticmethod
- def _get_numeric_scale(jsonschema_type):
- precision = SNOWFLAKE_MAX_NUMBER_SCALE
- if jsonschema_type.get("exclusiveMinimum"):
- if str(jsonschema_type[attrib])[-1] == 1:
- return len(str(jsonschema_type[attrib]).split(".")[1]) - 1
- else:
- return len(str(jsonschema_type[attrib]).split(".")[1])
- attribs_to_check = [
- "multipleOf",
- "min",
- "max",
- ]
- for attrib in attribs_to_check:
- if jsonschema_type.get(attrib):
- precision = max(precision, len(str(jsonschema_type[attrib]).split(".")[1]))
- return precision
-
  @staticmethod
  def to_sql_type(jsonschema_type: dict) -> sqlalchemy.types.TypeEngine:
  """Return a JSON Schema representation of the provided type.
@@ -260,8 +236,6 @@ def to_sql_type(jsonschema_type: dict) -> sqlalchemy.types.TypeEngine:
  # snowflake max and default varchar length
  # https://docs.snowflake.com/en/sql-reference/intro-summary-data-types.html
  maxlength = jsonschema_type.get("maxLength", SNOWFLAKE_MAX_STRING_LENGTH)
- num_precision = SnowflakeConnector._get_numeric_precision(jsonschema_type)
- num_scale = SnowflakeConnector._get_numeric_scale(jsonschema_type)
  # define type maps
  string_submaps = [
  TypeMap(eq, TIMESTAMP_NTZ(), "date-time"),
@@ -273,7 +247,7 @@ def to_sql_type(jsonschema_type: dict) -> sqlalchemy.types.TypeEngine:
  TypeMap(th._jsonschema_type_check, NUMBER(), ("integer",)),
  TypeMap(th._jsonschema_type_check, VARIANT(), ("object",)),
  TypeMap(th._jsonschema_type_check, VARIANT(), ("array",)),
- TypeMap(th._jsonschema_type_check, NUMBER(precision=num_precision, scale=num_scale), ("number",)),
+ TypeMap(th._jsonschema_type_check, sct.DOUBLE(), ("number",)),
  ]
  # apply type maps
  if th._jsonschema_type_check(jsonschema_type, ("string",)):

tests/core.py

@@ -255,7 +255,7 @@ def validate(self) -> None:
  table_schema = connector.get_table(table)
  expected_types = {
  "id": sct.NUMBER,
- "a1": sct.NUMBER,
+ "a1": sct.DOUBLE,
  "a2": sct.STRING,
  "a3": sqlalchemy.types.BOOLEAN,
  "a4": sct.VARIANT,
@@ -435,7 +435,8 @@ def validate(self) -> None:
  expected_types = {
  "id": sct.NUMBER,
  "col_max_length_str": sct.STRING,
- "col_multiple_of": sct.NUMBER,
+ "col_multiple_of": sct.DOUBLE,
+ "col_multiple_of_int": sct.DOUBLE,
  "_sdc_extracted_at": sct.TIMESTAMP_NTZ,
  "_sdc_batched_at": sct.TIMESTAMP_NTZ,
  "_sdc_received_at": sct.TIMESTAMP_NTZ,
@@ -446,10 +447,6 @@ def validate(self) -> None:
  for column in table_schema.columns:
  assert column.name in expected_types
  isinstance(column.type, expected_types[column.name])
- if column.name == "col_multiple_of":
- assert column.type.precision == 38
- assert column.type.scale == 4
-
 
 target_tests = TestSuite(
  kind="target",

tests/target_test_streams/type_edge_cases.singer

@@ -1,2 +1,2 @@
-{"type": "SCHEMA", "stream": "type_edge_cases", "key_properties": ["id"], "schema": {"required": ["id"], "type": "object", "properties": {"id": {"type": "integer"}, "col_max_length_str": {"maxLength": 4294967295, "type": [ "null", "string" ] }, "col_multiple_of": {"multipleOf": 0.0001, "type": [ "null", "number" ] }}}}
-{"type": "RECORD", "stream": "type_edge_cases", "record": {"id": 1, "col_max_length_str": "foo", "col_multiple_of": 123.456}}
+{"type": "SCHEMA", "stream": "type_edge_cases", "key_properties": ["id"], "schema": {"required": ["id"], "type": "object", "properties": {"id": {"type": "integer"}, "col_max_length_str": {"maxLength": 4294967295, "type": [ "null", "string" ] }, "col_multiple_of": {"multipleOf": 0.0001, "type": [ "null", "number" ] }, "col_multiple_of_int": {"multipleOf": 10, "type": [ "null", "number" ] }}}}
+{"type": "RECORD", "stream": "type_edge_cases", "record": {"id": 1, "col_max_length_str": "foo", "col_multiple_of": 123.456, "col_multiple_of_int": 100}}