Merge remote-tracking branch 'origin/main' into feature/feature-store

.github/workflows/add-3plicense-warning.yml

@@ -3,7 +3,7 @@ name: "Add 3P License Warning to PR"
 on:
   pull_request:
     paths:
-      - setup.py
+      - pyproject.toml
 
 # Cancel in progress workflows on pull_requests.
 # https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value
@@ -23,9 +23,9 @@ jobs:
     steps:
       - run: |
           BODY_MSG=$(cat << EOF
-          ⚠️ This PR changed **setup.py** file. ⚠️ 
-            - PR Creator must update 📃 THIRD_PARTY_LICENSES.txt, if any 📚 library added/removed in **setup.py**.
-            - PR Approver must confirm 📃 THIRD_PARTY_LICENSES.txt updated, if any 📚 library added/removed in **setup.py**.
+          ⚠️ This PR changed **pyproject.toml** file. ⚠️
+            - PR Creator must update 📃 THIRD_PARTY_LICENSES.txt, if any 📚 library added/removed in **pyproject.toml**.
+            - PR Approver must confirm 📃 THIRD_PARTY_LICENSES.txt updated, if any 📚 library added/removed in **pyproject.toml**.
           EOF
           )
           echo "BODY_MSG<<EOF" >> $GITHUB_ENV

.github/workflows/publish-to-pypi.yml

@@ -16,15 +16,15 @@ jobs:
           python-version: "3.x"
       - name: Build distribution 📦
         run: |
-          pip install wheel
+          pip install build
           make dist
       - name: Validate
         run: |
           pip install dist/*.whl
           python -c "import ads;"
-## To run publish to test PyPI secret with token needs to be added,
+## To run publish to test PyPI a secret with token needs to be added,
 ##    this one GH_ADS_TESTPYPI_TOKEN - removed after initial test.
-## Project name also needed to be updated in setup.py - setup(name="test_oracle_ads", ...),
+## Project name also needed to be updated in pyproject.toml - name = "test_oracle_ads" in [project] section
 ##    regular name is occupied by former developer and can't be used for testing
 #      - name: Publish distribution 📦 to Test PyPI
 #        env:

.github/workflows/run-unittests-default_setup.yml

@@ -9,8 +9,7 @@ on:
       - develop
     paths:
       - "ads/**"
-      - "!ads/ads_version.json"
-      - setup.py
+      - pyproject.toml
       - "**requirements.txt"
       - .github/workflows/run-unittests.yml
       - .github/workflows/run-unittests-default_setup.yml

.github/workflows/run-unittests.yml

@@ -9,8 +9,7 @@ on:
       - develop
     paths:
       - "ads/**"
-      - "!ads/ads_version.json"
-      - setup.py
+      - pyproject.toml
       - "**requirements.txt"
       - .github/workflows/run-unittests.yml
       - .github/workflows/run-unittests-default_setup.yml
@@ -39,9 +38,9 @@ jobs:
       fail-fast: false
       matrix:
         python-version: ["3.8", "3.9", "3.10"]
-        test-path: ["tests/unitary", "tests/unitary/with_extras/model"]
+        test-path: ["tests/unitary/with_extras tests/unitary/default_setup", "tests/unitary/with_extras/model"]
         include:
-          - test-path: "tests/unitary"
+          - test-path: "tests/unitary/with_extras tests/unitary/default_setup"
             ignore-path: "--ignore tests/unitary/with_extras/model --ignore tests/unitary/with_extras/feature_store"
             name: "unitary"
           - test-path: "tests/unitary/with_extras/model"
@@ -148,15 +147,15 @@ jobs:
         run: |
           set -x # print commands that are executed
 
-          # Prepare default cov body text 
+          # Prepare default cov body text
           COV_BODY_INTRO="📌 Overall coverage:\n\n"
           echo COV_BODY="$COV_BODY_INTRO No success to gather report. 😿" >> $GITHUB_ENV
 
           # Prepare file paths to .coverage files
           # Filenames taken from job.test last step with name - "Save coverage files"
-          FILE_UNITARY="cov-reports-unitary/.coverage"; [[ ! -f $FILE_UNITARY ]] && FILE_UNITARY="" 
+          FILE_UNITARY="cov-reports-unitary/.coverage"; [[ ! -f $FILE_UNITARY ]] && FILE_UNITARY=""
           FILE_MODEL="cov-reports-model/.coverage"; [[ ! -f $FILE_MODEL ]] && FILE_MODEL=""
-          
+
           # Combine coverage files
           pip install coverage
           coverage combine $FILE_UNITARY $FILE_MODEL
@@ -166,7 +165,7 @@ jobs:
 
           # Calculate overall coverage and update body message
           COV=$(grep -E 'pc_cov' htmlcov/index.html | cut -d'>' -f 2 | cut -d'%' -f 1)
-          if [[ ! -z $COV ]]; then 
+          if [[ ! -z $COV ]]; then
             if [[ $COV -lt 50 ]]; then COLOR=red; elif [[ $COV -lt 80 ]]; then COLOR=yellow; else COLOR=green; fi
             echo COV_BODY="$COV_BODY_INTRO ![Coverage-$COV%](https://img.shields.io/badge/coverage-$COV%25-$COLOR)" >> $GITHUB_ENV
           fi
@@ -176,23 +175,23 @@ jobs:
         run: |
           set -x # print commands that are executed
 
-          # Prepare default diff body text 
+          # Prepare default diff body text
           DIFF_BODY_INTRO="📌 Cov diff with **${{ env.COMPARE_BRANCH }}**:\n\n"
           echo DIFF_BODY="$DIFF_BODY_INTRO No success to gather report. 😿" >> $GITHUB_ENV
 
           # Prepare file paths to coverage xml files
           # Filenames taken from job.test last step with name - "Save coverage files"
-          FILE1="cov-reports-unitary/coverage.xml"; [[ ! -f $FILE1 ]] && FILE1="" 
+          FILE1="cov-reports-unitary/coverage.xml"; [[ ! -f $FILE1 ]] && FILE1=""
           FILE2="cov-reports-model/coverage.xml"; [[ ! -f $FILE2 ]] && FILE2=""
           echo "FILE1=$FILE1" >> $GITHUB_ENV
           echo "FILE2=$FILE2" >> $GITHUB_ENV
 
           # Calculate coverage diff and update body message
           pip install diff_cover
-          diff-cover $FILE1 $FILE2 --compare-branch=origin/${{ env.COMPARE_BRANCH }} 
+          diff-cover $FILE1 $FILE2 --compare-branch=origin/${{ env.COMPARE_BRANCH }}
           DIFF=$(diff-cover $FILE1 $FILE2 \
             --compare-branch=origin/${{ env.COMPARE_BRANCH }} | grep Coverage: | cut -d' ' -f 2 | cut -d'%' -f 1)
-          if [[ -z $DIFF ]]; then 
+          if [[ -z $DIFF ]]; then
             DIFF_INFO=$(diff-cover $FILE1 $FILE2 \
               --compare-branch=origin/${{ env.COMPARE_BRANCH }}  | grep "No lines");
             echo DIFF_BODY="$DIFF_BODY_INTRO $DIFF_INFO">> $GITHUB_ENV

MANIFEST.in

@@ -1,6 +1,5 @@
 include LICENSE.txt
 include THIRD_PARTY_LICENSES.txt
-include ads/ads_version.json
 include ads/common/*.json
 include ads/feature_engineering/*.json
 include ads/templates/*.jinja2
@@ -15,6 +14,7 @@ include ads/opctl/conda/manifest_template.yaml
 include ads/opctl/conda/config.yaml
 include ads/opctl/config/diagnostics/**/*.yaml
 include ads/opctl/templates/*.jinja2
+include pyproject.toml
 global-exclude tests/**
 global-exclude notebooks/**
 exclude tests/**
@@ -23,4 +23,4 @@ exclude build/lib/tests/**
 exclude build/lib/notebooks/**
 exclude benchmark/**
 include ads/ads
-include ads/model/common/*.*
+include ads/model/common/*.*

Makefile

@@ -2,7 +2,7 @@ RELEASE_BRANCH := release/ads
 DOCS_RELEASE_BRANCH := release
 CLONE_DIR := /tmp/advanced-ds
 DOCS_CLONE_DIR := /tmp/ads-docs
-COPY_INVENTORY := setup.py CONTRIBUTING.md LICENSE.txt MANIFEST.in README-development.md README.md SECURITY.md THIRD_PARTY_LICENSES.txt
+COPY_INVENTORY := CONTRIBUTING.md LICENSE.txt MANIFEST.in README-development.md README.md SECURITY.md THIRD_PARTY_LICENSES.txt
 
 prepare-release-branch: clean
 	@git checkout master
@@ -13,31 +13,32 @@ prepare-release-branch: clean
 prepare-ads:
 	@echo "Started advanced-ds clone at $$(date)"
 	@git clone ssh://git@bitbucket.oci.oraclecorp.com:7999/odsc/advanced-ds.git --branch $(RELEASE_BRANCH)  --depth 1 $(CLONE_DIR)
-	@echo "Finished cloning at $$(date)" 
+	@echo "Finished cloning at $$(date)"
 	cp -r $(CLONE_DIR)/ads .
 	$(foreach var,$(COPY_INVENTORY),cp $(CLONE_DIR)/$(var) .;)
 
-prepare-docs: 
+prepare-docs:
 	@echo "Started ads_docs clone at $$(date)"
 	@git clone ssh://git@bitbucket.oci.oraclecorp.com:7999/odsc/ads_docs.git --branch $(DOCS_RELEASE_BRANCH)  --depth 1 $(DOCS_CLONE_DIR)
-	@echo "Finished cloning at $$(date)" 
+	@echo "Finished cloning at $$(date)"
 	cp -r $(DOCS_CLONE_DIR)/source docs/ && cp $(DOCS_CLONE_DIR)/requirements.txt docs
 
 prepare: prepare-release-branch prepare-ads prepare-docs
 
 push: clean
-	@bash -c 'if [[ $$(git branch | grep \*) == "* release/$(RELEASE_VERSION)" ]];then echo "Version matching current branch"; else echo "Set proper value to RELEASE_VERSION";exit 1 ; fi'	
+	@bash -c 'if [[ $$(git branch | grep \*) == "* release/$(RELEASE_VERSION)" ]];then echo "Version matching current branch"; else echo "Set proper value to RELEASE_VERSION";exit 1 ; fi'
 	@git add .
 	@git commit -m "Release version: $(RELEASE_VERSION)"
 	@git push --set-upstream origin release/$(RELEASE_VERSION)
 
 dist: clean
-	@python3 setup.py sdist bdist_wheel
+	@python3 -m build
 
 publish: dist
 	@twine upload dist/*
 
 clean:
+	@echo "Cleaning - removing dist, *.pyc, Thumbs.db and other files"
 	@rm -rf dist build oracle_ads.egg-info
 	@find ./ -name '*.pyc' -exec rm -f {} \;
 	@find ./ -name 'Thumbs.db' -exec rm -f {} \;

README-development.md

@@ -58,7 +58,7 @@ Open the destination folder where you want to clone ADS library, and install dep
     python3 -m pip install -e .
 ```
 
-To which packages were installed and their version numbers, run:
+To view which packages were installed and their version numbers, run:
 
 ```bash
     python3 -m pip freeze
@@ -71,10 +71,11 @@ You should also set up configuration files, see the [SDK and CLI Configuration F
 
 ### Step 5: Versioning and generation the wheel
 
-Use `ads_version.json` for versioning. The ADS SDK is packaged as a wheel. To generate the wheel, you can run:
+Bump the versions in `pyproject.toml`. The ADS SDK using [build](https://pypa-build.readthedocs.io/en/stable/index.html) as build frontend. To generate sdist and wheel, you can run:
 
 ```bash
-    python3 setup.py sdist bdist_wheel
+    pip install build
+    python3 -m build
 ```
 
 This wheel can then be installed using `pip`.
@@ -85,7 +86,7 @@ The SDK uses pytest as its test framework.
 
 ### Running default setup tests
 
-Default setup tests for testing ADS SDK without extra dependencies, specified in setup.py.
+Default setup tests for testing ADS SDK without extra dependencies, specified in `pyproject.toml` in `[project.optional-dependencies]`.
 
 ```bash
   # Update your environment with tests dependencies

README.md

@@ -29,12 +29,6 @@ You have various options when installing ADS.
 
 ### Installing extras libraries
 
-The `all-optional` module will install all optional dependencies. Note the single quotes around installation of extra libraries.
-
-```bash
-  python3 -m pip install 'oracle-ads[all-optional]'
-```
-
 To work with gradient boosting models, install the `boosted` module. This module includes XGBoost and LightGBM model classes.
 
 ```bash
@@ -107,6 +101,8 @@ Install the `viz` module to include libraries for visualization tasks. Some of t
   python3 -m pip install 'oracle-ads[viz]'
 ```
 
+See `pyproject.toml` file `[project.optional-dependencies]` section for full list of modules and its list of extra libraries.
+
 **Note**
 
 Multiple extra dependencies can be installed together. For example:

SECURITY.md

@@ -21,7 +21,7 @@ security features are welcome on GitHub Issues.
 
 Security updates will be released on a regular cadence. Many of our projects
 will typically release security fixes in conjunction with the
-[Oracle Critical Patch Update][3] program. Additional
+Oracle Critical Patch Update program. Additional
 information, including past advisories, is available on our [security alerts][4]
 page.
 

ads/__init__.py

@@ -6,15 +6,18 @@
 
 from __future__ import print_function, division, absolute_import
 import os
+import sys
 import logging
 import json
 from typing import Callable, Dict, Optional, Union
 
-__version__ = ""
-with open(
-    os.path.join(os.path.dirname(os.path.abspath(__file__)), "ads_version.json")
-) as version_file:
-    __version__ = json.load(version_file)["version"]
+# https://packaging.python.org/en/latest/guides/single-sourcing-package-version/#single-sourcing-the-package-version
+if sys.version_info >= (3, 8):
+    from importlib import metadata
+else:
+    import importlib_metadata as metadata
+
+__version__ = metadata.version("oracle_ads")
 import oci
 
 import matplotlib.font_manager  # causes matplotlib to regenerate its fonts
@@ -138,7 +141,8 @@ def hello():
 oci v{oci.__version__}
 ocifs v{ocifs.__version__}
 
-""")
+"""
+    )
 
 
 configure_plotting()

ads/cli.py

@@ -1,10 +1,13 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*--
 
-# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
+# Copyright (c) 2021, 2023 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-from ads.common import logger
+
 import traceback
+import sys
+
+from ads.common import logger
 
 try:
     import click
@@ -22,11 +25,13 @@
     logger.debug(traceback.format_exc())
     exit()
 
+# https://packaging.python.org/en/latest/guides/single-sourcing-package-version/#single-sourcing-the-package-version
+if sys.version_info >= (3, 8):
+    from importlib import metadata
+else:
+    import importlib_metadata as metadata
 
-with open(
-    os.path.join(os.path.dirname(os.path.abspath(__file__)), "ads_version.json")
-) as version_file:
-    ADS_VERSION = json.load(version_file)["version"]
+ADS_VERSION = metadata.version("oracle_ads")
 
 
 @click.group()

ads/common/auth.py

@@ -889,11 +889,12 @@ def _read_security_token_file(self, security_token_file: str) -> str:
         str:
             Security token string.
         """
-        if not os.path.isfile(security_token_file):
+        expanded_path = os.path.expanduser(security_token_file)
+        if not os.path.isfile(expanded_path):
             raise ValueError("Invalid `security_token_file`. Specify a valid path.")
         try:
             token = None
-            with open(security_token_file, 'r') as f:
+            with open(expanded_path, 'r') as f:
                 token = f.read()
             return token
         except:

ads/common/model_export_util.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8; -*-
 
-# Copyright (c) 2020, 2022 Oracle and/or its affiliates.
+# Copyright (c) 2020, 2023 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 import warnings
 
@@ -261,7 +261,7 @@ def prepare_generic_model(
 
         progress.update("Updating requirements.txt")
         if fn_artifact_files_included:
-            # fdk removed from dependency list in setup.py (fn deployments deprecated)
+            # fdk removed from dependency list in pyproject.toml (fn deployments deprecated)
             # before we request versions we want to check if fdk installed by user
             # and provide support in error message, if not installed
             try:
@@ -449,7 +449,6 @@ def _sklearn_to_onnx(model=None, target_dir=None, X=None, y=None, **kwargs):
 
 
 def _automl_to_pkl(model=None, target_dir=None, **kwargs):
-
     with open(os.path.join(target_dir, "model.pkl"), "wb") as outfile:
         cloudpickle.dump(model, outfile)
 
@@ -484,7 +483,6 @@ def _lightgbm_to_onnx(model=None, target_dir=None, X=None, y=None, **kwargs):
             options={"nocl": [True, False], "zipmap": [True, False]},
         )
     elif lightgbm.sklearn.LGBMRegressor in model_est_types:
-
         from onnxmltools.convert.lightgbm.operator_converters.LightGbm import (
             convert_lightgbm,
         )
@@ -527,7 +525,6 @@ def _xgboost_to_onnx(model=None, target_dir=None, X=None, y=None, **kwargs):
     else:
         model_est_types = [type(model.est)]
     if xgboost.sklearn.XGBClassifier in model_est_types:
-
         from onnxmltools.convert.xgboost.operator_converters.XGBoost import (
             convert_xgboost,
         )