Add in-development version support (#4865)

This adds support for in-development versioning of jars and wheels. Specifically, this PR translates the java version qualifier (anything after "-") to a python "local version label" (<public version identifier>[+<local version label>]). For example, the java version "1.2.3-SNAPSHOT" becomes python version "1.2.3+snapshot"; the java version "4.5.6-my-custom-build123" becomes python version "4.5.6+my.custom.build123". By using a "local version identifier", these wheels become non-public artifacts - they can't be publicly published. While we could contemplate using "development release segments" (ie, python versios that look like "X.Y.Z.devN"), we wouldn't be able to generally translate java versions to python versions without enforcing more structure onto our java versions.

As an added benefit, this PR also reduces some of the duplication of version numbers by sourcing the version number as appropriate during the wheel building process; and then also in the runtime python code, sourcing the version information via the appropriate importlib metadata APIs.

This also improves our versioning support (now supported via property changes instead of hard-coded buildSrc changes), and gives re-builders options when defining their own version.

See https://packaging.python.org/en/latest/specifications/version-specifiers/#
See https://peps.python.org/pep-0440/

Partial #3466
Fixes #4654

.github/workflows/publish-ci.yml

@@ -63,6 +63,7 @@ jobs:
           ORG_GRADLE_PROJECT_signingKey: ${{ secrets.CI_AT_DEEPHAVEN_KEY }}
           ORG_GRADLE_PROJECT_signingPassword: ${{ secrets.CI_AT_DEEPHAVEN_PASSWORD }}
           ORG_GRADLE_PROJECT_signingRequired: true
+          ORG_GRADLE_PROJECT_deephavenBaseQualifier: ""
 
       - name: Upload Artifacts
         if: ${{ startsWith(github.ref, 'refs/heads/release/v') }}

RELEASE.md

@@ -261,18 +261,9 @@ mention the version explicitly. These files are listed below:
 
 ```
 #
-# Edit files for version change, updating from 0.31.0 to 0.32.0
+# Edit files for version change
 #
-authorization-codegen/protoc-gen-contextual-auth-wiring
-authorization-codegen/protoc-gen-service-auth-wiring
-buildSrc/src/main/groovy/io.deephaven.common-conventions.gradle
-py/client-ticking/README.md
-py/client-ticking/setup.py
-py/client/README.md
-py/client/pydeephaven/__init__.py
-py/client/setup.py
-py/embedded-server/deephaven_server/__init__.py
-py/server/deephaven/__init__.py
+gradle.properties
 R/rdeephaven/DESCRIPTION
 ```
 

authorization-codegen/protoc-gen-contextual-auth-wiring

@@ -1,2 +1,2 @@
 # protoc-gen-contextual-auth-wiring
-java -cp authorization-codegen/build/libs/deephaven-authorization-codegen-0.31.0-all.jar io.deephaven.auth.codegen.GenerateContextualAuthWiring
+java -cp "authorization-codegen/build/libs/deephaven-authorization-codegen-${DEEPHAVEN_VERSION}-all.jar" io.deephaven.auth.codegen.GenerateContextualAuthWiring

authorization-codegen/protoc-gen-service-auth-wiring

@@ -1,2 +1,2 @@
 # protoc-gen-service-auth-wiring
-java -cp authorization-codegen/build/libs/deephaven-authorization-codegen-0.31.0-all.jar io.deephaven.auth.codegen.GenerateServiceAuthWiring
+java -cp "authorization-codegen/build/libs/deephaven-authorization-codegen-${DEEPHAVEN_VERSION}-all.jar" io.deephaven.auth.codegen.GenerateServiceAuthWiring

authorization/README.md

@@ -27,11 +27,12 @@ Here is a sample bash script to generate the provided authorizing wiring if you
 ```bash
 ./gradlew :authorization-codegen:shadowJar
 
+DEEPHAVEN_VERSION="$(./gradlew printVersion -q)"
 OUT_DIR=authorization/src/main/java/
 PROTO_DIR=proto/proto-backplane-grpc/src/main/proto/
 ROOT_DIR=$PROTO_DIR/deephaven/proto
 
-PATH=authorization-codegen:$PATH protoc --service-auth-wiring_out=$OUT_DIR -I $PROTO_DIR    \
+DEEPHAVEN_VERSION=${DEEPHAVEN_VERSION} PATH=authorization-codegen:$PATH protoc --service-auth-wiring_out=$OUT_DIR -I $PROTO_DIR    \
      $ROOT_DIR/application.proto                                        \
      $ROOT_DIR/console.proto                                            \
      $ROOT_DIR/config.proto                                             \
@@ -41,7 +42,7 @@ PATH=authorization-codegen:$PATH protoc --service-auth-wiring_out=$OUT_DIR -I $P
      $ROOT_DIR/storage.proto                                            \
      $ROOT_DIR/ticket.proto
 
-PATH=authorization-codegen:$PATH protoc --contextual-auth-wiring_out=$OUT_DIR -I $PROTO_DIR \
+DEEPHAVEN_VERSION=${DEEPHAVEN_VERSION} PATH=authorization-codegen:$PATH protoc --contextual-auth-wiring_out=$OUT_DIR -I $PROTO_DIR \
      $ROOT_DIR/table.proto                                              \
      $ROOT_DIR/inputtable.proto                                         \
      $ROOT_DIR/partitionedtable.proto                                   \
@@ -51,6 +52,8 @@ OUT_DIR=authorization/src/main/java/
 PROTO_DIR=../grpc/src/proto/grpc/health/v1/
 ROOT_DIR=$PROTO_DIR
 
-PATH=authorization-codegen:$PATH protoc --service-auth-wiring_out=$OUT_DIR -I $PROTO_DIR    \
+DEEPHAVEN_VERSION=${DEEPHAVEN_VERSION}  PATH=authorization-codegen:$PATH protoc --service-auth-wiring_out=$OUT_DIR -I $PROTO_DIR    \
      $ROOT_DIR/health.proto
+
+./gradlew :authorization:spotlessApply
 ```

buildSrc/src/main/groovy/io.deephaven.common-conventions.gradle

@@ -5,7 +5,11 @@ plugins {
 }
 
 group = 'io.deephaven'
-version = '0.31.0'
+
+def deephavenBaseVersion = project.property('deephavenBaseVersion').toString().trim()
+def deephavenBaseQualifier = project.property('deephavenBaseQualifier').toString().trim()
+def versionSeparator = deephavenBaseQualifier.isEmpty() ? "" : "-"
+version = "${deephavenBaseVersion}${versionSeparator}${deephavenBaseQualifier}"
 
 if (!name.startsWith('deephaven-')) {
     archivesBaseName = "deephaven-${name}"

gradle.properties

@@ -1,3 +1,17 @@
+# This structure gives DHC and re-builders control over the version. When deephavenBaseQualifier is non-empty, the
+# project version will be "${deephavenBaseVersion}-${deephavenBaseQualifier}". When deephavenBaseQualifier is empty, the
+# project version will be deephavenBaseVersion.
+#
+# The default values will represent the in-development version for the next release: "X.Y.Z-SNAPSHOT".
+#
+# During the normal DHC release process, the publishing code will set -PdeephavenBaseQualifier="": "X.Y.Z".
+#
+# Re-builders who want to inherit the base version, but have their own qualifier can set -PdeephavenBaseQualifier="customQualifier": "X.Y.Z-customQualifier".
+#
+# Re-builders who want a fully custom version can set -PdeephavenBaseVersion="customVersion" -PdeephavenBaseQualifier="": "customVersion".
+deephavenBaseVersion=0.31.0
+deephavenBaseQualifier=SNAPSHOT
+
 #org.gradle.debug
 ## Enable to attach debugger to port 5005 when running gradle
 ## Note, you can use -Dorg.gradle.debug=true as well.

py/client-ticking/README.md

@@ -66,7 +66,7 @@ Then install the package.
 Note the actual name of the `.whl` file may be different depending on system details.
 
 ```
-pip3 install --force --no-deps dist/pydeephaven_ticking-0.31.0-cp310-cp310-linux_x86_64.whl
+pip3 install --force --no-deps dist/pydeephaven_ticking-<x.y.z>-cp310-cp310-linux_x86_64.whl
 ```
 
 The reason for the "--force" flag is to overwrite any previously-built version of the package that

py/client-ticking/build.gradle

@@ -56,15 +56,16 @@ def buildPyClientTicking = Docker.registerDockerTask(project, 'pyClientTicking')
         copyFile('README.md', "${prefix}/src/py-client-ticking")
         copyFile('src/', "${prefix}/src/py-client-ticking/src/")
         copyFile('wheels/', "${prefix}/src/py-client-ticking/in-wheels")
-        runCommand("PREFIX=${prefix}; " +
+        runCommand("PREFIX=${prefix}; DEEPHAVEN_VERSION=${project.version};" +
                    '''set -eux ; \
                      cd "${PREFIX}/src/py-client-ticking"; \
                      . "${PREFIX}/env.sh"; \
                      MAKEFLAGS="-j${NCPUS}" \
                        CFLAGS="-I${DHCPP}/include" \
                        LDFLAGS="-L${DHCPP}/lib" \
+                       DEEPHAVEN_VERSION="${DEEPHAVEN_VERSION}" \
                        python3 setup.py build_ext -i; \
-                     python3 setup.py bdist_wheel; \
+                     DEEPHAVEN_VERSION="${DEEPHAVEN_VERSION}" python3 setup.py bdist_wheel; \
                      pip3 install in-wheels/*.whl; \
                      pip3 install --force --no-deps dist/*.whl; \
                      ln dist/*.whl /out; \

py/client-ticking/setup.py

@@ -1,21 +1,40 @@
 #
 #  Copyright (c) 2016-2023 Deephaven Data Labs and Patent Pending
 #
+import os
 import pathlib
+
+# Note: pkg_resources is deprecated https://setuptools.pypa.io/en/latest/pkg_resources.html, and it is suggested
+# to use an external library `packaging`. From the context of building a wheel though, we'd prefer to not have to
+# install extra dependencies, at least until we can more properly manage the build environment (pyproject.toml).
+# TODO(deephaven-core#2233): upgrade setup.py to pyproject.toml
+from pkg_resources import parse_version
 from setuptools import find_packages, setup, Extension
 from Cython.Build import cythonize
 
-# The directory containing this file
-HERE = pathlib.Path(__file__).parent
+def _get_readme() -> str:
+    # The directory containing this file
+    HERE = pathlib.Path(__file__).parent
+    # The text of the README file
+    return (HERE / "README.md").read_text(encoding="utf-8")
+
+def _normalize_version(java_version):
+    partitions = java_version.partition("-")
+    regular_version = partitions[0]
+    local_segment = partitions[2]
+    python_version = f"{regular_version}+{local_segment}" if local_segment else regular_version
+    return str(parse_version(python_version))
+
+def _compute_version():
+    return _normalize_version(os.environ['DEEPHAVEN_VERSION'])
 
-# The text of the README file
-README = (HERE / "README.md").read_text()
+_version = _compute_version()
 
 setup(
     name='pydeephaven-ticking',
-    version='0.30.0.dev0',
+    version=_version,
     description='The Deephaven Python Client for Ticking Tables',
-    long_description=README,
+    long_description=_get_readme(),
     long_description_content_type="text/markdown",
     packages=find_packages(where="src", exclude=("tests",)),
     package_dir={"": "src"},
@@ -44,5 +63,5 @@
                    libraries=["dhcore_static"]
         )]),
     python_requires='>=3.8',
-    install_requires=['pydeephaven==0.31.0']
+    install_requires=[f"pydeephaven=={_version}"]
 )

py/client/README.md

@@ -38,7 +38,7 @@ $ python3 -m examples.demo_asof_join
 Note the actual name of the `.whl` file may be different depending on system details.
 
 ``` shell
-$ pip3 install dist/pydeephaven-0.31.0-py3-none-any.whl
+$ pip3 install dist/pydeephaven-<x.y.z>-py3-none-any.whl
 ```
 ## Quick start
 

py/client/pydeephaven/__init__.py

@@ -23,6 +23,8 @@
     >>> session.close()
 """
 
+import importlib.metadata
+
 from .session import Session
 from .dherror import DHError
 from ._table_interface import SortDirection
@@ -35,4 +37,8 @@
     pass
 
 __all__ = ["Session", "DHError", "SortDirection"]
-__version__ = "0.31.0"
+
+# Note: this is the _distribution_ name, not the _package_ name. Until 3.10, there is not an easy way to get the
+# distribution name from the package name.
+# https://docs.python.org/3/library/importlib.metadata.html#package-distributions
+__version__ = importlib.metadata.version('pydeephaven')

py/client/setup.py

@@ -1,20 +1,37 @@
 #
 #  Copyright (c) 2016-2021 Deephaven Data Labs and Patent Pending
 #
+import os
 import pathlib
+
+# Note: pkg_resources is deprecated https://setuptools.pypa.io/en/latest/pkg_resources.html, and it is suggested
+# to use an external library `packaging`. From the context of building a wheel though, we'd prefer to not have to
+# install extra dependencies, at least until we can more properly manage the build environment (pyproject.toml).
+# TODO(deephaven-core#2233): upgrade setup.py to pyproject.toml
+from pkg_resources import parse_version
 from setuptools import find_packages, setup
 
-# The directory containing this file
-HERE = pathlib.Path(__file__).parent
+def _get_readme() -> str:
+    # The directory containing this file
+    HERE = pathlib.Path(__file__).parent
+    # The text of the README file
+    return (HERE / "README.md").read_text(encoding="utf-8")
+
+def _normalize_version(java_version):
+    partitions = java_version.partition("-")
+    regular_version = partitions[0]
+    local_segment = partitions[2]
+    python_version = f"{regular_version}+{local_segment}" if local_segment else regular_version
+    return str(parse_version(python_version))
 
-# The text of the README file
-README = (HERE / "README.md").read_text()
+def _compute_version():
+    return _normalize_version(os.environ['DEEPHAVEN_VERSION'])
 
 setup(
     name='pydeephaven',
-    version='0.31.0',
+    version=_compute_version(),
     description='The Deephaven Python Client',
-    long_description=README,
+    long_description=_get_readme(),
     long_description_content_type="text/markdown",
     packages=find_packages(exclude=("tests",)),
     url='https://deephaven.io/',

py/embedded-server/deephaven_server/__init__.py

@@ -1,10 +1,16 @@
 #
 # Copyright (c) 2016-2022 Deephaven Data Labs and Patent Pending
 #
-__version__ = "0.31.0"
+
+import importlib.metadata
 
 from .start_jvm import DEFAULT_JVM_PROPERTIES, DEFAULT_JVM_ARGS, start_jvm
 from .server import Server
 
 from deephaven_internal.jvm import check_py_env
 check_py_env()
+
+# Note: this is the _distribution_ name, not the _package_ name. Until 3.10, there is not an easy way to get the
+# distribution name from the package name.
+# https://docs.python.org/3/library/importlib.metadata.html#package-distributions
+__version__ = importlib.metadata.version('deephaven-server')

py/embedded-server/setup.py

@@ -3,33 +3,37 @@
 #
 import os
 import pathlib
-from setuptools.extern import packaging
-from setuptools import find_namespace_packages, setup
-
-# The directory containing this file
-HERE = pathlib.Path(__file__).parent
 
-# The text of the README file
-README = (HERE / "README_PyPi.md").read_text()
+# Note: pkg_resources is deprecated https://setuptools.pypa.io/en/latest/pkg_resources.html, and it is suggested
+# to use an external library `packaging`. From the context of building a wheel though, we'd prefer to not have to
+# install extra dependencies, at least until we can more properly manage the build environment (pyproject.toml).
+# TODO(deephaven-core#2233): upgrade setup.py to pyproject.toml
+from pkg_resources import parse_version
+from setuptools import find_namespace_packages, setup
 
+def _get_readme() -> str:
+    # The directory containing this file
+    HERE = pathlib.Path(__file__).parent
+    # The text of the README file
+    return (HERE / "README_PyPi.md").read_text(encoding="utf-8")
 
-# Versions should comply with PEP440.  For a discussion on single-sourcing
-# the version across setup.py and the project code, see
-# https://packaging.python.org/en/latest/single_source_version.html
-# todo: does DH versions align w/ PEP440?
-# see https://github.com/pypa/setuptools/blob/v40.8.0/setuptools/dist.py#L470
-def normalize_version(version):
-    return str(packaging.version.Version(version))
+def _normalize_version(java_version) -> str:
+    partitions = java_version.partition("-")
+    regular_version = partitions[0]
+    local_segment = partitions[2]
+    python_version = f"{regular_version}+{local_segment}" if local_segment else regular_version
+    return str(parse_version(python_version))
 
+def _compute_version():
+    return _normalize_version(os.environ['DEEPHAVEN_VERSION'])
 
-__deephaven_version__ = os.environ['DEEPHAVEN_VERSION']
-__normalized_version__ = normalize_version(__deephaven_version__)
+_version = _compute_version()
 
 setup(
     name='deephaven-server',
-    version=__normalized_version__,
+    version=_version,
     description='Deephaven Embedded Server Python Package',
-    long_description=README,
+    long_description=_get_readme(),
     long_description_content_type='text/markdown',
     packages=find_namespace_packages(exclude=("tests")),
     package_data={'deephaven_server': ['jars/*']},
@@ -56,6 +60,6 @@ def normalize_version(version):
     install_requires=[
         'jpy>=0.14.0',
         "java-utilities",
-        f"deephaven-core[autocomplete]=={__normalized_version__}",
+        f"deephaven-core[autocomplete]=={_version}",
     ]
 )

py/server/deephaven/__init__.py

@@ -7,7 +7,7 @@
 
 """
 
-__version__ = "0.31.0"
+import importlib.metadata
 
 from deephaven_internal import jvm
 
@@ -31,3 +31,8 @@
 __all__ = ["read_csv", "write_csv", "kafka_consumer", "kafka_producer", "empty_table", "time_table", "merge",
            "merge_sorted", "new_table", "input_table", "ring_table", "function_generated_table", "DynamicTableWriter",
            "TableReplayer", "garbage_collect", "read_sql", "DHError", "SortDirection"]
+
+# Note: this is the _distribution_ name, not the _package_ name. Until 3.10, there is not an easy way to get the
+# distribution name from the package name.
+# https://docs.python.org/3/library/importlib.metadata.html#package-distributions
+__version__ = importlib.metadata.version('deephaven-core')