feat: release v2.4.3 with proto3 optional support (#834)

* feat: release v2.4.3 with proto3 optional support

* fix: grpc-tools v1.30.2 for ruby

* chore: remove unsupported APIs from Circle config

* chore: update path for tests

.circleci/config.yml

@@ -43,35 +43,19 @@ jobs:
           command: |
             EXIT_STATUS=0
             docker run -it -e RUNNING_IN_ARTMAN_DOCKER=True --workdir /tmp artman /bin/bash -c "artman --local --config=google/pubsub/artman_pubsub.yaml --root-dir=/googleapis generate csharp_gapic" || EXIT_STATUS=$?
-            docker run -it -e RUNNING_IN_ARTMAN_DOCKER=True --workdir /tmp artman /bin/bash -c "artman --local --config=google/cloud/speech/artman_speech_v1.yaml --root-dir=/googleapis generate csharp_gapic" || EXIT_STATUS=$?
+            docker run -it -e RUNNING_IN_ARTMAN_DOCKER=True --workdir /tmp artman /bin/bash -c "artman --local --config=google/spanner/artman_spanner.yaml --root-dir=/googleapis generate csharp_gapic" || EXIT_STATUS=$?
             docker run -it -e RUNNING_IN_ARTMAN_DOCKER=True --workdir /googleapis artman /bin/bash -c "mkdir /tmp/pubsub-go; artman --local --output-dir /tmp/pubsub-go --config=google/pubsub/artman_pubsub.yaml generate go_gapic" || EXIT_STATUS=$?
-            docker run -it -e RUNNING_IN_ARTMAN_DOCKER=True --workdir /googleapis artman /bin/bash -c "mkdir /tmp/speech-go; artman --local --output-dir /tmp/speech-go --config=google/cloud/speech/artman_speech_v1.yaml generate go_gapic" || EXIT_STATUS=$?
+            docker run -it -e RUNNING_IN_ARTMAN_DOCKER=True --workdir /googleapis artman /bin/bash -c "mkdir /tmp/spanner-go; artman --local --output-dir /tmp/spanner-go --config=google/spanner/artman_spanner.yaml generate go_gapic" || EXIT_STATUS=$?
             docker run -it -e RUNNING_IN_ARTMAN_DOCKER=True --workdir /googleapis artman /bin/bash -c "mkdir /tmp/pubsub-java; artman --local --output-dir /tmp/pubsub-java --config=google/pubsub/artman_pubsub.yaml generate java_gapic" || EXIT_STATUS=$?
-            docker run -it -e RUNNING_IN_ARTMAN_DOCKER=True --workdir /googleapis artman /bin/bash -c "mkdir /tmp/speech-java; artman --local --output-dir /tmp/speech-java --config=google/cloud/speech/artman_speech_v1.yaml generate java_gapic" || EXIT_STATUS=$?
+            docker run -it -e RUNNING_IN_ARTMAN_DOCKER=True --workdir /googleapis artman /bin/bash -c "mkdir /tmp/spanner-java; artman --local --output-dir /tmp/spanner-java --config=google/spanner/artman_spanner.yaml generate java_gapic" || EXIT_STATUS=$?
             docker run -it -e RUNNING_IN_ARTMAN_DOCKER=True --workdir /googleapis artman /bin/bash -c "mkdir /tmp/pubsub-node;artman --local --output-dir /tmp/pubsub-node --config=google/pubsub/artman_pubsub.yaml generate nodejs_gapic" || EXIT_STATUS=$?
-            docker run -it -e RUNNING_IN_ARTMAN_DOCKER=True --workdir /googleapis artman /bin/bash -c "mkdir /tmp/speech-node;artman --local --output-dir speech-node --config=google/cloud/speech/artman_speech_v1.yaml generate nodejs_gapic" || EXIT_STATUS=$?
+            docker run -it -e RUNNING_IN_ARTMAN_DOCKER=True --workdir /googleapis artman /bin/bash -c "mkdir /tmp/spanner-node;artman --local --output-dir /tmp/spanner-node --config=google/spanner/artman_spanner.yaml generate nodejs_gapic" || EXIT_STATUS=$?
             docker run -it -e RUNNING_IN_ARTMAN_DOCKER=True --workdir /googleapis artman /bin/bash -c "mkdir /tmp/pubsub-php; artman --local --output-dir /tmp/pubsub-php --config=google/pubsub/artman_pubsub.yaml generate php_gapic" || EXIT_STATUS=$?
-            docker run -it -e RUNNING_IN_ARTMAN_DOCKER=True --workdir /googleapis artman /bin/bash -c "mkdir /tmp/speech-php; artman --local --output-dir /tmp/speech-php --config=google/cloud/speech/artman_speech_v1.yaml generate php_gapic" || EXIT_STATUS=$?
+            docker run -it -e RUNNING_IN_ARTMAN_DOCKER=True --workdir /googleapis artman /bin/bash -c "mkdir /tmp/spanner-php; artman --local --output-dir /tmp/spanner-php --config=google/spanner/artman_spanner.yaml generate php_gapic" || EXIT_STATUS=$?
             docker run -it -e RUNNING_IN_ARTMAN_DOCKER=True --workdir /googleapis artman /bin/bash -c "mkdir /tmp/pubsub-python; artman --local --output-dir /tmp/pubsub-python --config=google/pubsub/artman_pubsub.yaml generate python_gapic" || EXIT_STATUS=$?
-            docker run -it -e RUNNING_IN_ARTMAN_DOCKER=True --workdir /googleapis artman /bin/bash -c "mkdir /tmp/speech-python; artman --local --output-dir /tmp/speech-python --config=google/cloud/speech/artman_speech_v1.yaml generate python_gapic" || EXIT_STATUS=$?
+            docker run -it -e RUNNING_IN_ARTMAN_DOCKER=True --workdir /googleapis artman /bin/bash -c "mkdir /tmp/spanner-python; artman --local --output-dir /tmp/spanner-python --config=google/spanner/artman_spanner.yaml generate python_gapic" || EXIT_STATUS=$?
             docker run -it -e RUNNING_IN_ARTMAN_DOCKER=True --workdir /googleapis artman /bin/bash -c "mkdir /tmp/pubsub-ruby; artman --local --output-dir /tmp/pubsub-ruby --config=google/pubsub/artman_pubsub.yaml generate ruby_gapic" || EXIT_STATUS=$?
-            docker run -it -e RUNNING_IN_ARTMAN_DOCKER=True --workdir /googleapis artman /bin/bash -c "mkdir /tmp/speech-ruby; artman --local --output-dir /tmp/speech-ruby --config=google/cloud/speech/artman_speech_v1.yaml generate ruby_gapic" || EXIT_STATUS=$?
-            exit $EXIT_STATUS
-      - run:
-          name: Check samples are generated for selected APIs
-          command: |
-            EXIT_STATUS=0
-            docker run -it --rm -d --name samplegen --workdir /tmp -e RUNNING_IN_ARTMAN_DOCKER=True artman
-            docker exec samplegen mkdir /tmp/language-python
-            docker exec samplegen artman --local --output-dir /tmp/language-python --config=google/cloud/language/artman_language_v1.yaml --root-dir=/googleapis --generator-args='--dev_samples' generate python_gapic || EXIT_STATUS=$?
-            docker exec samplegen find /tmp/language-python/python/language-v1/samples/v1 -mindepth 1 | read || EXIT_STATUS=$?
-            docker exec samplegen mkdir /tmp/talent-php
-            docker exec samplegen artman --local --output-dir /tmp/talent-php --config=google/cloud/talent/artman_talent_v4beta1.yaml --root-dir=/googleapis --generator-args='--dev_samples' generate php_gapic || EXIT_STATUS=$?
-            docker exec samplegen find /tmp/talent-php/php/google-cloud-talent-v4beta1/samples/V4beta1 -mindepth 1 | read || EXIT_STATUS=$?
-            docker exec samplegen mkdir /tmp/speech-nodejs
-            docker exec samplegen artman --local --output-dir /tmp/speech-nodejs --config=google/cloud/speech/artman_speech_v1p1beta1.yaml --root-dir=/googleapis --generator-args='--dev_samples' generate nodejs_gapic || EXIT_STATUS=$?
-            docker exec samplegen find /tmp/speech-nodejs/js/speech-v1p1beta1/samples/v1p1beta1 -mindepth 1 | read || EXIT_STATUS=$?
-            docker stop samplegen
+            docker run -it -e RUNNING_IN_ARTMAN_DOCKER=True --workdir /googleapis artman /bin/bash -c "mkdir /tmp/spanner-ruby; artman --local --output-dir /tmp/spanner-ruby --config=google/spanner/artman_spanner.yaml generate ruby_gapic" || EXIT_STATUS=$?
             exit $EXIT_STATUS
       - run:
           name: Conditionally run smoketests against all APIs

Dockerfile

@@ -1,11 +1,11 @@
 FROM ubuntu:16.04
 
 # Release parameters
-ENV GOOGLEAPIS_HASH e47fdd266542386e5e7346697f90476e96dc7ee8
-ENV GAPIC_GENERATOR_HASH 8d53d31494bebc1e1da4ed0d1352f1f35441d439
+ENV GOOGLEAPIS_HASH e26e1839a45445d13cd45b1be3b1523defb72fee
+ENV GAPIC_GENERATOR_HASH v2.4.3
 # Define version number below. The ARTMAN_VERSION line is parsed by
 # .circleci/config.yml and setup.py, please keep the format.
-ENV ARTMAN_VERSION 2.0.0
+ENV ARTMAN_VERSION 2.4.3
 
 ENV DEBIAN_FRONTEND noninteractive
 
@@ -90,8 +90,8 @@ RUN export JAVA_HOME
 # Install Go.
 RUN mkdir -p /golang \
   && cd /golang \
-  && curl https://dl.google.com/go/go1.11.linux-amd64.tar.gz > go.tar.gz \
-  && (echo 'b3fcf280ff86558e0559e185b601c9eade0fd24c900b4c63cd14d1d38613e499 go.tar.gz' | sha256sum -c) \
+  && curl -L https://golang.org/dl/go1.14.6.linux-amd64.tar.gz > go.tar.gz \
+  && (echo '5c566ddc2e0bcfc25c26a5dc44a440fcc0177f7350c1f01952b34d5989a0d287 go.tar.gz' | sha256sum -c) \
   && tar xzf go.tar.gz \
   && rm go.tar.gz \
   && cd /
@@ -110,7 +110,7 @@ RUN gem install rake --no-ri --no-rdoc \
   && gem install rubocop --version '= 0.39.0' --no-ri --no-rdoc \
   && gem install bundler --version '= 1.12.1' --no-ri --no-rdoc \
   && gem install rake --version '= 10.5.0' --no-ri --no-rdoc \
-  && gem install grpc-tools --version '=1.17.1' --no-ri --no-rdoc
+  && gem install grpc-tools --version '= 1.30.2' --no-ri --no-rdoc
 
 # Install grpc_php_plugin
 RUN git clone -b v1.17.1 --recurse-submodules --depth=1 https://github.com/grpc/grpc.git /temp/grpc \

artman/utils/protoc_utils.py

@@ -260,7 +260,7 @@ def protoc_header_params(proto_path,
                           toolkit_path):
     proto_path = proto_path[:]
     proto_path.append(_find_protobuf_path(toolkit_path))
-    return (['--proto_path=' + path for path in proto_path])
+    return (['--experimental_allow_proto3_optional'] + ['--proto_path=' + path for path in proto_path])
 
 
 def protoc_desc_params(output_dir, desc_out_file):

install_protoc.sh

@@ -10,15 +10,16 @@ declare -A protobuf_versions
 declare -A override_download_location
 
 # Please adhere to this format, as artman parses these lines for the protobuf versions.
-protobuf_versions[nodejs]=3.11.2
-protobuf_versions[go]=3.11.2
-protobuf_versions[python]=3.11.2
-protobuf_versions[ruby]=3.11.2
-protobuf_versions[php]=3.11.2
-protobuf_versions[csharp]=3.11.2
+protobuf_versions[nodejs]=3.12.3
+protobuf_versions[go]=3.12.3
+# Protobuf Python should match the latest version available on PyPI
+protobuf_versions[python]=3.12.2
+protobuf_versions[ruby]=3.12.3
+protobuf_versions[php]=3.12.3
+protobuf_versions[csharp]=3.12.3
 # Protobuf Java dependency must match grpc-java's protobuf dep.
-# https://github.com/grpc/grpc-java/blob/18e099d9d37163905fc61febd2aee983e298a066/build.gradle#L51
-protobuf_versions[java]=3.11.0
+# https://github.com/grpc/grpc-java/blob/master/build.gradle#L61
+protobuf_versions[java]=3.12.0
 
 # Install each unique protobuf version.
 for i in "${protobuf_versions[@]}"
@@ -38,6 +39,6 @@ done
 
 # Install GRPC and Protobuf.
 pip3 install --upgrade pip==20.0.2 setuptools==39.2.0 \
-  && hash -r pip3 && pip3 install grpcio>=1.21.1 \
-    grpcio-tools==1.21.1 \
+  && hash -r pip3 && pip3 install grpcio==1.30.0 \
+    grpcio-tools==1.30.0 \
     protobuf==${protobuf_versions[python]}