From c8a5f222ea330f3b33049c7719841f5b985865ea Mon Sep 17 00:00:00 2001
From: Khor Shu Heng <32997938+khorshuheng@users.noreply.github.com>
Date: Mon, 8 Apr 2024 16:03:08 +0800
Subject: [PATCH] bug: skip requirement comparison for pip reqs that is not a
 package (#570)

<!--  Thanks for sending a pull request!  Here are some tips for you:

1. Run unit tests and ensure that they are passing
2. If your change introduces any API changes, make sure to update the
e2e tests
3. Make sure documentation is updated for your PR!

-->
# Description
<!-- Briefly describe the motivation for the change. Please include
illustrations where appropriate. -->
While the SDK is processing the conda env, and additional merlin
dependencies are required, such as merlin-pyfunc-server, the SDK will
check the existing conda env to verify if the dependency is already
present. If not, the additional requirement will be appended as part of
the dependencies.

Unfortunately, this workflow assuems that every line under the `pip`
configurations is a python package, which is not necessarily the case.
For example, the user might specify trusted host or repository under the
pip section.

# Modifications
<!-- Summarize the key code changes. -->
Skip comparing pip requirements that is not a package while attempting
to check if extra merlin dependencies are present in the conda.yaml
file.

# Tests
<!-- Besides the existing / updated automated tests, what specific
scenarios should be tested? Consider the backward compatibility of the
changes, whether corner cases are covered, etc. Please describe the
tests and check the ones that have been completed. Eg:
- [x] Deploying new and existing standard models
- [ ] Deploying PyFunc models
-->

# Checklist
- [ ] Added PR label
- [ ] Added unit test, integration, and/or e2e tests
- [ ] Tested locally
- [ ] Updated documentation
- [ ] Update Swagger spec if the PR introduce API changes
- [ ] Regenerated Golang and Python client if the PR introduces API
changes

# Release Notes
<!--
Does this PR introduce a user-facing change?
If no, just write "NONE" in the release-note block below.
If yes, a release note is required. Enter your extended release note in
the block below.
If the PR requires additional action from users switching to the new
release, include the string "action required".

For more information about release notes, see kubernetes' guide here:
http://git.k8s.io/community/contributors/guide/release-notes.md
-->

```release-note

```
---
 python/sdk/merlin/requirements.py                      | 7 ++++++-
 python/sdk/test/requirements/non_package_reqs_in.yaml  | 6 ++++++
 python/sdk/test/requirements/non_package_reqs_out.yaml | 7 +++++++
 python/sdk/test/requirements_test.py                   | 5 +++++
 4 files changed, 24 insertions(+), 1 deletion(-)
 create mode 100644 python/sdk/test/requirements/non_package_reqs_in.yaml
 create mode 100644 python/sdk/test/requirements/non_package_reqs_out.yaml

diff --git a/python/sdk/merlin/requirements.py b/python/sdk/merlin/requirements.py
index a4849f6f1..328e9a064 100644
--- a/python/sdk/merlin/requirements.py
+++ b/python/sdk/merlin/requirements.py
@@ -5,7 +5,7 @@
 from typing import Any, List, Optional
 
 import yaml
-from packaging.requirements import Requirement
+from packaging.requirements import Requirement, InvalidRequirement
 
 _CONSTRAINTS_FILE_NAME = "constraints.txt"
 
@@ -51,6 +51,11 @@ def process_conda_env(
         for additional_merlin_req in additional_merlin_reqs:
             exist = False
             for pip_req in pip_reqs:
+                # Skip requirement comparison for pip reqs that is not a package, which can happen if the user specifies
+                # repository or trusted host as part of the pip file
+                if pip_req.strip().startswith("--"):
+                    continue
+
                 pip_req_obj = Requirement(pip_req)
                 additional_merlin_req_obj = Requirement(additional_merlin_req)
                 if pip_req_obj.name.lower() == additional_merlin_req_obj.name.lower():
diff --git a/python/sdk/test/requirements/non_package_reqs_in.yaml b/python/sdk/test/requirements/non_package_reqs_in.yaml
new file mode 100644
index 000000000..56d14156b
--- /dev/null
+++ b/python/sdk/test/requirements/non_package_reqs_in.yaml
@@ -0,0 +1,6 @@
+dependencies:
+  - python=3.10.*
+  - pip:
+      - --extra-index-url=http://artifactory.com
+      - --trusted-host=artifactory.com
+      - scikit-learn==1.4.0
diff --git a/python/sdk/test/requirements/non_package_reqs_out.yaml b/python/sdk/test/requirements/non_package_reqs_out.yaml
new file mode 100644
index 000000000..ff10b4a6a
--- /dev/null
+++ b/python/sdk/test/requirements/non_package_reqs_out.yaml
@@ -0,0 +1,7 @@
+dependencies:
+  - python=3.10.*
+  - pip:
+      - merlin-pyfunc-server<0.42.0
+      - --extra-index-url=http://artifactory.com
+      - --trusted-host=artifactory.com
+      - scikit-learn==1.4.0
diff --git a/python/sdk/test/requirements_test.py b/python/sdk/test/requirements_test.py
index 9805d5c66..837ca1154 100644
--- a/python/sdk/test/requirements_test.py
+++ b/python/sdk/test/requirements_test.py
@@ -27,6 +27,11 @@
             "test/requirements/pyfunc_server_without_version_out.yaml",
             ["merlin-pyfunc-server<0.42.0"],
         ),
+        (
+            "test/requirements/non_package_reqs_in.yaml",
+            "test/requirements/non_package_reqs_out.yaml",
+            ["merlin-pyfunc-server<0.42.0"],
+        ),
         (
             "test/requirements/other_reqs_in.yaml",
             "test/requirements/other_reqs_out.yaml",