Feature store 1.0.3 (#284)

oracle · Sep 21, 2023 · 856b5f3 · 856b5f3
2 parents ba93f40 + 754e964
commit 856b5f3
Show file tree

Hide file tree

Showing 42 changed files with 1,840 additions and 178 deletions.
diff --git a/THIRD_PARTY_LICENSES.txt b/THIRD_PARTY_LICENSES.txt
@@ -229,6 +229,12 @@ pandavro
 * Source code: https://github.com/ynqa/pandavro
 * Project home: https://github.com/ynqa/pandavro
 
+plotly
+* Copyright (c) 2016-2018 Plotly, Inc
+* License: MIT License
+* Source code: https://github.com/plotly/plotly.py
+* Project home: https://plotly.com/
+
 protobuf
 * Copyright 2008 Google Inc.  All rights reserved.
 * License: Google Protobuf License

diff --git a/ads/common/decorator/runtime_dependency.py b/ads/common/decorator/runtime_dependency.py
@@ -64,10 +64,9 @@ class OptionalDependency:
     OPTUNA = "oracle-ads[optuna]"
     SPARK = "oracle-ads[spark]"
     HUGGINGFACE = "oracle-ads[huggingface]"
-    GREAT_EXPECTATIONS = "oracle-ads[great-expectations]"
+    FEATURE_STORE = "oracle-ads[feature-store]"
     GRAPHVIZ = "oracle-ads[graphviz]"
     MLM_INSIGHTS = "oracle-ads[mlm_insights]"
-    PYARROW = "oracle-ads[pyarrow]"
 
 
 def runtime_dependency(

diff --git a/ads/feature_store/.readthedocs.yaml b/ads/feature_store/.readthedocs.yaml
@@ -0,0 +1,19 @@
+# Read the Docs configuration file
+
+# Required
+version: 2
+
+# Set the version of Python and other tools you might need
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.9"
+
+# Build documentation in the docs/ directory with Sphinx
+sphinx:
+   configuration: ads/feature_store/docs/source/conf.py
+
+# Optionally declare the Python requirements required to build your docs
+python:
+   install:
+   - requirements: ads/feature_store/docs/requirements.txt
diff --git a/ads/feature_store/common/spark_session_singleton.py b/ads/feature_store/common/spark_session_singleton.py
@@ -84,6 +84,7 @@ def __init__(self, metastore_id: str = None):
             )
             .enableHiveSupport()
         )
+        _managed_table_location = None
 
         if not developer_enabled() and metastore_id:
             # Get the authentication credentials for the OCI data catalog service
@@ -94,12 +95,11 @@ def __init__(self, metastore_id: str = None):
 
             data_catalog_client = OCIClientFactory(**auth).data_catalog
             metastore = data_catalog_client.get_metastore(metastore_id).data
+            _managed_table_location = metastore.default_managed_table_location
             # Configure the Spark session builder object to use the specified metastore
             spark_builder.config(
                 "spark.hadoop.oracle.dcat.metastore.id", metastore_id
-            ).config(
-                "spark.sql.warehouse.dir", metastore.default_managed_table_location
-            ).config(
+            ).config("spark.sql.warehouse.dir", _managed_table_location).config(
                 "spark.driver.memory", "16G"
             )
 
@@ -114,7 +114,12 @@ def __init__(self, metastore_id: str = None):
 
         self.spark_session.conf.set("spark.sql.execution.arrow.pyspark.enabled", "true")
         self.spark_session.sparkContext.setLogLevel("OFF")
+        self.managed_table_location = _managed_table_location
 
     def get_spark_session(self):
         """Access method to get the spark session."""
         return self.spark_session
+
+    def get_managed_table_location(self):
+        """Returns the managed table location for the spark"""
+        return self.managed_table_location
diff --git a/ads/feature_store/common/utils/transformation_utils.py b/ads/feature_store/common/utils/transformation_utils.py
@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8; -*-
 import json
+
 # Copyright (c) 2023 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
@@ -64,9 +65,13 @@ def apply_transformation(
             dataframe.createOrReplaceTempView(temporary_table_view)
 
             transformed_data = spark.sql(
-                transformation_function_caller(temporary_table_view, **transformation_kwargs_dict)
+                transformation_function_caller(
+                    temporary_table_view, **transformation_kwargs_dict
+                )
             )
         elif transformation.transformation_mode == TransformationMode.PANDAS.value:
-            transformed_data = transformation_function_caller(dataframe, **transformation_kwargs_dict)
+            transformed_data = transformation_function_caller(
+                dataframe, **transformation_kwargs_dict
+            )
 
         return transformed_data
diff --git a/ads/feature_store/data_validation/great_expectation.py b/ads/feature_store/data_validation/great_expectation.py
@@ -21,8 +21,8 @@
     from great_expectations.validator.validator import Validator
 except ModuleNotFoundError:
     raise ModuleNotFoundError(
-        f"The `great-expectations` module was not found. Please run `pip install "
-        f"{OptionalDependency.GREAT_EXPECTATIONS}`."
+        f"The `feature-store` module was not found. Please run `pip install "
+        f"{OptionalDependency.FEATURE_STORE}`."
     )
 except Exception as e:
     raise