From eb152b0c5e7badaf46461e55df2002d508f9af66 Mon Sep 17 00:00:00 2001 From: Phillip Cloud <417981+cpcloud@users.noreply.github.com> Date: Wed, 28 Dec 2022 17:24:19 -0500 Subject: [PATCH] feat(protoc): support using `grpc_tools.protoc` --- .github/workflows/ci.yml | 4 +- poetry.lock | 84 ++++++++++++++++++++++++++++++++- protoletariat/fdsetgen.py | 3 +- protoletariat/tests/conftest.py | 51 +++++++++++++++++++- pyproject.toml | 9 +++- 5 files changed, 143 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 386b5d90..37017257 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -158,7 +158,7 @@ jobs: python-version: ${{ matrix.python-version }} environment-file: environment.yml - - run: mamba install grpc-cpp + - run: mamba install grpc-cpp grpcio-tools - run: pip install . - run: pytest -ra conda-windows: @@ -203,7 +203,7 @@ jobs: python-version: ${{ matrix.python-version }} environment-file: environment.yml - - run: mamba install grpc-cpp pip + - run: mamba install grpc-cpp grpcio-tools pip - run: pip install . - run: pytest -ra tensorflow: diff --git a/poetry.lock b/poetry.lock index 6f2a8b34..a92bbbcf 100644 --- a/poetry.lock +++ b/poetry.lock @@ -146,7 +146,7 @@ grpcio = "*" name = "grpcio" version = "1.51.3" description = "HTTP/2-based RPC framework" -category = "dev" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -200,6 +200,66 @@ files = [ [package.extras] protobuf = ["grpcio-tools (>=1.51.3)"] +[[package]] +name = "grpcio-tools" +version = "1.51.3" +description = "Protobuf code generator for gRPC" +category = "main" +optional = true +python-versions = ">=3.7" +files = [ + {file = "grpcio-tools-1.51.3.tar.gz", hash = "sha256:4fea28e3dd31871579a57058796a78093c75b74b74e9de2f2b7a7fd9a443d403"}, + {file = "grpcio_tools-1.51.3-cp310-cp310-linux_armv7l.whl", hash = "sha256:779ac1ad2258b8debaa45595bfb3814806ed8880e3ea7f194e551d76a6255969"}, + {file = "grpcio_tools-1.51.3-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:83bf605fe2b3591d3c8a78646f37c72c5832c4dd84b5f92405c17cb10b136be6"}, + {file = "grpcio_tools-1.51.3-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:35f885c5afd8e6a77d320f5a9624b439a93f9be2b87fa7b7948c1ad7b2ba0894"}, + {file = "grpcio_tools-1.51.3-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:253b639fb79a4d28ce494ae40e5695bf1e2cb4a05f205fc433c46b2049ab4d99"}, + {file = "grpcio_tools-1.51.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c6b145587d6062e2335f0b3286501dd6853a1ea50bd466a913351b7c48e5f20"}, + {file = "grpcio_tools-1.51.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:046c0b1e372d4acf552aa0c8f5e830f019d67b75f25aeb0968d15fbdd3eaabd3"}, + {file = "grpcio_tools-1.51.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:efc90b0287908c46281eb61933acaa1b96a575d0160fc98b5c64b9dec46f60d1"}, + {file = "grpcio_tools-1.51.3-cp310-cp310-win32.whl", hash = "sha256:8e9df40db7a0edd403b539cc142d6114270e35debf723a5b4a7a93d5c30fffc0"}, + {file = "grpcio_tools-1.51.3-cp310-cp310-win_amd64.whl", hash = "sha256:077adaee431c2b040dd77923964577087c32e828908e8fa2e53f8e003ad408c9"}, + {file = "grpcio_tools-1.51.3-cp311-cp311-linux_armv7l.whl", hash = "sha256:b50f9b8a6482a90c1a41e731a879a130f7dea267065d0a06f47c9160ce5d01c3"}, + {file = "grpcio_tools-1.51.3-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:89a68adcb4238aba69f3a364ac02c9a46e55b9e3fd8af1c6f384079abfa9347c"}, + {file = "grpcio_tools-1.51.3-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:78d177da43e7f6fde6715df4a3015ae13158166bc2845ac7f9cfb526eafb41b8"}, + {file = "grpcio_tools-1.51.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:793f9edef82f600a3324f8a3d8cd8318a8d02f28fb54f8236cbb35ce0928d186"}, + {file = "grpcio_tools-1.51.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:f7583735542ced7d30baec6cc21bffeaffcec1523bf807e8f8f0047113b6d30a"}, + {file = "grpcio_tools-1.51.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f2df233a3e7db23d9b516cb5e2bfe029465f40a72978bee0584e44e7860ea73f"}, + {file = "grpcio_tools-1.51.3-cp311-cp311-win32.whl", hash = "sha256:7427939455735fbf2ea88c37f1585c9c8b809eec7b447642f34465eb4d26020b"}, + {file = "grpcio_tools-1.51.3-cp311-cp311-win_amd64.whl", hash = "sha256:ba76d15fd149b575170fa32a1f6a9ff2b38ff9db223229a8ad6f53450a452688"}, + {file = "grpcio_tools-1.51.3-cp37-cp37m-linux_armv7l.whl", hash = "sha256:d2212c682529263b3c9e903092d0ccbb9fc6afba820e4c2fa52c2c27720cdcae"}, + {file = "grpcio_tools-1.51.3-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:405656b3cf9639427e6c30a795570cba4a7c06b88a3145866f7d2c05b7e048b4"}, + {file = "grpcio_tools-1.51.3-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:3c445a064b2ef3d3475e26e2add8ddb4ac2933741ecddf71d5b071a3ad078db4"}, + {file = "grpcio_tools-1.51.3-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c7b3374f4a6579c58d16a5fab2e6b4e9bb8625a034a7f4cd6024f4d1cc12f2a0"}, + {file = "grpcio_tools-1.51.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:46e8df08b65f9379c3f103147b29542b0141ca84e77d0eee9114ca5f9b3f0d23"}, + {file = "grpcio_tools-1.51.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2fade12de08923b350475ca16d0d0bd68578c30fce89147aa0f94ef5759bc5a9"}, + {file = "grpcio_tools-1.51.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d4ffb6325ed489065dbdca764cf37c3a29376bc657874116c9af788d7a0d2ee4"}, + {file = "grpcio_tools-1.51.3-cp37-cp37m-win32.whl", hash = "sha256:f8d17271fc58ed3503dd571c79917e126deca51f85f093770a9606e806aac9dc"}, + {file = "grpcio_tools-1.51.3-cp37-cp37m-win_amd64.whl", hash = "sha256:ef849687c7f2bd7f3277edc7c7cafc7042823d0fb078e3c01c861eb0c96ed181"}, + {file = "grpcio_tools-1.51.3-cp38-cp38-linux_armv7l.whl", hash = "sha256:7fd18d8d211fbfd337fc12e5bdd57e62368f636addf901d290e68a39f1dfea38"}, + {file = "grpcio_tools-1.51.3-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:233fc56f054424232e2086f444004413e33c699174ce6ee0e279c25227243fec"}, + {file = "grpcio_tools-1.51.3-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:867fa1973fa8b0772077c15425f122f672a18b1c53709a8a2bff9d056db4c20e"}, + {file = "grpcio_tools-1.51.3-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b486a99bdf2722e68a9d59769389e2fb86878b6f293be5111f7678e364a0c359"}, + {file = "grpcio_tools-1.51.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c8bbf412c357999f88d87f421fd48b4b114fc037fec7bbaed0cb7620c24a5e44"}, + {file = "grpcio_tools-1.51.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1166744c40821bb0aa605d2af2287fac367756f858a3d18f4c3d25bc0b92757b"}, + {file = "grpcio_tools-1.51.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:781896c488e07b9463196045e6725e52d018cd7d0e1062d4ab1eee2647ca9170"}, + {file = "grpcio_tools-1.51.3-cp38-cp38-win32.whl", hash = "sha256:35c1ee7c766eb586f04ba41fa7711eb847767eb277a1737998374ac57768f1f0"}, + {file = "grpcio_tools-1.51.3-cp38-cp38-win_amd64.whl", hash = "sha256:584b201fb39307dcb1affcf2647656a0e6244423ef1659cc6caa3ff85c5ae5c1"}, + {file = "grpcio_tools-1.51.3-cp39-cp39-linux_armv7l.whl", hash = "sha256:e02231e21029f716a1d23a0b5e664fa243d147da33a3f55088a9529b860aa4ac"}, + {file = "grpcio_tools-1.51.3-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:fbb742e10bd548031b8d80f7c28eb70c7c3a9850f8e99c98cd496f19a05f9fee"}, + {file = "grpcio_tools-1.51.3-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:a836a72c657f751244cdb358c3461a89627e6d02654079d2450cfe361800428c"}, + {file = "grpcio_tools-1.51.3-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bb554408e0ec5ff5201013f268726d9eef8e5bd1fd4b4e09c46c0b4a9de8b64c"}, + {file = "grpcio_tools-1.51.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:158c5bfe7e157fd9a944bde9f7dfe3b468416666e4fade77cd17caa3edc8bd81"}, + {file = "grpcio_tools-1.51.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:715c792679224171c0584e9f235b921d76f8990deb38b0d1215d0469301d9cd9"}, + {file = "grpcio_tools-1.51.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ece44f42b10e0bceb49235be1e361e1ee69afee7f938c82fb656a601a4a720e3"}, + {file = "grpcio_tools-1.51.3-cp39-cp39-win32.whl", hash = "sha256:980e632710ba05e04364c6f276e905d5d367437f1ce2265ce7b96b5c1eac5693"}, + {file = "grpcio_tools-1.51.3-cp39-cp39-win_amd64.whl", hash = "sha256:5f4c47b14e66f80365cd5667ecc2f7fb0eb91e02c4e54362041b758feaa00511"}, +] + +[package.dependencies] +grpcio = ">=1.51.3" +protobuf = ">=4.21.6,<5.0dev" +setuptools = "*" + [[package]] name = "importlib-metadata" version = "6.0.0" @@ -453,6 +513,23 @@ files = [ {file = "ruff-0.0.257.tar.gz", hash = "sha256:fedfd06a37ddc17449203c3e38fc83fb68de7f20b5daa0ee4e60d3599b38bab0"}, ] +[[package]] +name = "setuptools" +version = "67.6.0" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +category = "main" +optional = true +python-versions = ">=3.7" +files = [ + {file = "setuptools-67.6.0-py3-none-any.whl", hash = "sha256:b78aaa36f6b90a074c1fa651168723acbf45d14cb1196b6f02c0fd07f17623b2"}, + {file = "setuptools-67.6.0.tar.gz", hash = "sha256:2ee892cd5f29f3373097f5a814697e397cf3ce313616df0af11231e2ad118077"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + [[package]] name = "six" version = "1.16.0" @@ -532,7 +609,10 @@ files = [ docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] +[extras] +grpcio-tools = ["grpcio-tools"] + [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "22ae0bf6ce1bacc6057af3f9d9d6efd129363f3305f34454f62611bfcd3d0cfd" +content-hash = "c9907fa8901dd0577ce5ffa000af73f85f331f003190f5f27a3f0bb19410703b" diff --git a/protoletariat/fdsetgen.py b/protoletariat/fdsetgen.py index 6bceef2d..fc076dcf 100644 --- a/protoletariat/fdsetgen.py +++ b/protoletariat/fdsetgen.py @@ -4,6 +4,7 @@ import fnmatch import itertools import re +import shlex import subprocess import tempfile from pathlib import Path @@ -154,7 +155,7 @@ def generate_file_descriptor_set_bytes(self) -> bytes: filename = Path(f.name) subprocess.check_output( [ - self.protoc_path, + *shlex.split(self.protoc_path), "--include_imports", f"--descriptor_set_out={filename}", *map("--proto_path={}".format, self.proto_paths), diff --git a/protoletariat/tests/conftest.py b/protoletariat/tests/conftest.py index 7f64516a..6fc24ba7 100644 --- a/protoletariat/tests/conftest.py +++ b/protoletariat/tests/conftest.py @@ -5,8 +5,10 @@ import itertools import json import os +import shlex import shutil import subprocess +import sys import tempfile from functools import partial from pathlib import Path @@ -138,6 +140,8 @@ def do_generate(self, cli: CliRunner, *, args: Iterable[str] = ()) -> Result: class ProtocFixture(ProtoletariatFixture): + protoc_exe = ("protoc",) + def __init__( self, *, @@ -161,7 +165,7 @@ def __init__( def do_generate(self, cli: CliRunner, *, args: Iterable[str] = ()) -> Result: protoc_args = [ - "protoc", + *self.protoc_exe, "--proto_path", str(self.base_dir), "--python_out", @@ -190,6 +194,8 @@ def do_generate(self, cli: CliRunner, *, args: Iterable[str] = ()) -> Result: str(self.package_dir), *args, "protoc", + "--protoc-path", + shlex.join(self.protoc_exe), "--proto-path", str(self.base_dir), *(str(filename) for filename, _ in self.proto_texts), @@ -198,6 +204,10 @@ def do_generate(self, cli: CliRunner, *, args: Iterable[str] = ()) -> Result: ) +class GrpcIoToolsFixture(ProtocFixture): + protoc_exe = sys.executable, "-m", "grpc_tools.protoc" + + class RawFixture(ProtoletariatFixture): def __init__( self, @@ -319,6 +329,10 @@ def basic_cli_texts(request: SubRequest) -> list[ProtoFile]: partial(ProtocFixture, package="basic_cli"), id="basic_cli_protoc", ), + pytest.param( + partial(GrpcIoToolsFixture, package="basic_cli"), + id="basic_cli_grpc_io_tools", + ), pytest.param( partial(RawFixture, package="basic_cli"), id="basic_cli_raw", @@ -398,6 +412,10 @@ def thing_service_texts(request: SubRequest) -> list[ProtoFile]: partial(ProtocFixture, package="thing_service", grpc=True), id="thing_service_protoc", ), + pytest.param( + partial(GrpcIoToolsFixture, package="thing_service", grpc=True), + id="thing_service_grpc_io_tools", + ), pytest.param( partial(RawFixture, package="thing_service", grpc=True), id="thing_service_raw", @@ -453,6 +471,9 @@ def nested_texts() -> list[ProtoFile]: id="nested_buf", ), pytest.param(partial(ProtocFixture, package="nested"), id="nested_protoc"), + pytest.param( + partial(GrpcIoToolsFixture, package="nested"), id="nested_grpc_io_tools" + ), pytest.param(partial(RawFixture, package="nested"), id="nested_raw"), ] ) @@ -512,6 +533,16 @@ def no_imports_service_texts(request: SubRequest) -> list[ProtoFile]: ), id="no_imports_service_protoc", ), + pytest.param( + partial( + GrpcIoToolsFixture, + package="no_imports_service", + grpc=True, + mypy=True, + mypy_grpc=True, + ), + id="no_imports_service_grpc_io_tools", + ), pytest.param( partial( RawFixture, @@ -601,6 +632,16 @@ def imports_service_texts(request: SubRequest) -> list[ProtoFile]: ), id="imports_service_protoc", ), + pytest.param( + partial( + GrpcIoToolsFixture, + package="imports_service", + grpc=True, + mypy=True, + mypy_grpc=True, + ), + id="imports_service_grpc_io_tools", + ), pytest.param( partial( RawFixture, @@ -659,6 +700,10 @@ def long_names_texts() -> list[ProtoFile]: partial(ProtocFixture, package="long_names", mypy=True), id="long_names_protoc", ), + pytest.param( + partial(GrpcIoToolsFixture, package="long_names", mypy=True), + id="long_names_grpc_io_tools", + ), pytest.param( partial(RawFixture, package="long_names", mypy=True), id="long_names_raw", @@ -710,6 +755,10 @@ def ignored_import_texts(request: SubRequest) -> list[ProtoFile]: partial(ProtocFixture, package="ignored_imports"), id="ignored_imports_protoc", ), + pytest.param( + partial(GrpcIoToolsFixture, package="ignored_imports"), + id="ignored_imports_grpc_io_tools", + ), pytest.param( partial(RawFixture, package="ignored_imports"), id="ignored_imports_raw", diff --git a/pyproject.toml b/pyproject.toml index d3a75e02..d9c15df3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,13 +16,18 @@ classifiers = [ "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", ] +[tool.poetry.scripts] +protol = "protoletariat.__main__:main" + [tool.poetry.dependencies] python = "^3.8" astunparse = { version = ">=1.6.3,<2", python = "<3.9" } click = ">=8,<9" protobuf = ">=3.19.1,<5" +grpcio-tools = { version = ">=1.42.0,<2", optional = true } [tool.poetry.dev-dependencies] black = ">=22,<24" @@ -35,8 +40,8 @@ pytest-randomly = ">=3.10.1,<4" ruff = ">=0.0.198,<1" types-protobuf = ">=4.21,<5" -[tool.poetry.scripts] -protol = "protoletariat.__main__:main" +[tool.poetry.extras] +grpcio-tools = ["grpcio-tools"] [tool.pytest.ini_options] xfail_strict = true