From a30bdf74c2e908caafb5c50dce95b5f2a0754b51 Mon Sep 17 00:00:00 2001 From: Sergiy Zaschipas Date: Wed, 5 Oct 2022 23:32:37 +0300 Subject: [PATCH 01/25] Add support of references/dependencies --- .gitignore | 1 + karapace/dependency.py | 21 + karapace/errors.py | 18 + karapace/protobuf/dependency.py | 56 ++ karapace/protobuf/exception.py | 4 + karapace/protobuf/google/protobuf/any.proto | 158 +++ .../protobuf/google/protobuf/any_test.proto | 44 + karapace/protobuf/google/protobuf/api.proto | 208 ++++ .../protobuf/google/protobuf/descriptor.proto | 918 ++++++++++++++++++ .../protobuf/google/protobuf/duration.proto | 116 +++ karapace/protobuf/google/protobuf/empty.proto | 51 + .../protobuf/google/protobuf/field_mask.proto | 245 +++++ .../google/protobuf/map_lite_unittest.proto | 131 +++ .../google/protobuf/map_proto2_unittest.proto | 91 ++ .../google/protobuf/map_unittest.proto | 125 +++ .../google/protobuf/source_context.proto | 48 + .../protobuf/google/protobuf/struct.proto | 95 ++ .../protobuf/google/protobuf/timestamp.proto | 147 +++ karapace/protobuf/google/protobuf/type.proto | 187 ++++ .../protobuf/google/protobuf/wrappers.proto | 123 +++ karapace/protobuf/google/type/BUILD.bazel | 536 ++++++++++ karapace/protobuf/google/type/README.md | 16 + .../google/type/calendar_period.proto | 56 ++ karapace/protobuf/google/type/color.proto | 174 ++++ karapace/protobuf/google/type/date.proto | 52 + karapace/protobuf/google/type/datetime.proto | 104 ++ karapace/protobuf/google/type/dayofweek.proto | 50 + karapace/protobuf/google/type/decimal.proto | 95 ++ karapace/protobuf/google/type/expr.proto | 73 ++ karapace/protobuf/google/type/fraction.proto | 33 + karapace/protobuf/google/type/interval.proto | 46 + karapace/protobuf/google/type/latlng.proto | 37 + .../protobuf/google/type/localized_text.proto | 36 + karapace/protobuf/google/type/money.proto | 42 + karapace/protobuf/google/type/month.proto | 65 ++ .../protobuf/google/type/phone_number.proto | 113 +++ .../protobuf/google/type/postal_address.proto | 134 +++ .../protobuf/google/type/quaternion.proto | 94 ++ karapace/protobuf/google/type/timeofday.proto | 44 + karapace/protobuf/google/type/type.yaml | 40 + karapace/protobuf/known_dependency.py | 154 +++ karapace/protobuf/proto_file_element.py | 19 +- karapace/protobuf/proto_parser.py | 9 +- karapace/protobuf/schema.py | 63 +- karapace/schema_models.py | 76 +- karapace/schema_reader.py | 102 +- karapace/schema_references.py | 27 + karapace/schema_registry.py | 59 +- karapace/schema_registry_apis.py | 181 +++- karapace/schema_type.py | 8 + karapace/serialization.py | 25 +- karapace/utils.py | 9 + tests/__init__.py | 0 tests/integration/test_client_protobuf.py | 4 +- tests/integration/test_schema.py | 11 + tests/integration/test_schema_protobuf.py | 177 ++++ tests/unit/test_dependency_verifier.py | 54 ++ 57 files changed, 5539 insertions(+), 66 deletions(-) create mode 100644 karapace/dependency.py create mode 100644 karapace/protobuf/dependency.py create mode 100644 karapace/protobuf/google/protobuf/any.proto create mode 100644 karapace/protobuf/google/protobuf/any_test.proto create mode 100644 karapace/protobuf/google/protobuf/api.proto create mode 100644 karapace/protobuf/google/protobuf/descriptor.proto create mode 100644 karapace/protobuf/google/protobuf/duration.proto create mode 100644 karapace/protobuf/google/protobuf/empty.proto create mode 100644 karapace/protobuf/google/protobuf/field_mask.proto create mode 100644 karapace/protobuf/google/protobuf/map_lite_unittest.proto create mode 100644 karapace/protobuf/google/protobuf/map_proto2_unittest.proto create mode 100644 karapace/protobuf/google/protobuf/map_unittest.proto create mode 100644 karapace/protobuf/google/protobuf/source_context.proto create mode 100644 karapace/protobuf/google/protobuf/struct.proto create mode 100644 karapace/protobuf/google/protobuf/timestamp.proto create mode 100644 karapace/protobuf/google/protobuf/type.proto create mode 100644 karapace/protobuf/google/protobuf/wrappers.proto create mode 100644 karapace/protobuf/google/type/BUILD.bazel create mode 100644 karapace/protobuf/google/type/README.md create mode 100644 karapace/protobuf/google/type/calendar_period.proto create mode 100644 karapace/protobuf/google/type/color.proto create mode 100644 karapace/protobuf/google/type/date.proto create mode 100644 karapace/protobuf/google/type/datetime.proto create mode 100644 karapace/protobuf/google/type/dayofweek.proto create mode 100644 karapace/protobuf/google/type/decimal.proto create mode 100644 karapace/protobuf/google/type/expr.proto create mode 100644 karapace/protobuf/google/type/fraction.proto create mode 100644 karapace/protobuf/google/type/interval.proto create mode 100644 karapace/protobuf/google/type/latlng.proto create mode 100644 karapace/protobuf/google/type/localized_text.proto create mode 100644 karapace/protobuf/google/type/money.proto create mode 100644 karapace/protobuf/google/type/month.proto create mode 100644 karapace/protobuf/google/type/phone_number.proto create mode 100644 karapace/protobuf/google/type/postal_address.proto create mode 100644 karapace/protobuf/google/type/quaternion.proto create mode 100644 karapace/protobuf/google/type/timeofday.proto create mode 100644 karapace/protobuf/google/type/type.yaml create mode 100644 karapace/protobuf/known_dependency.py create mode 100644 karapace/schema_references.py create mode 100644 karapace/schema_type.py create mode 100644 tests/__init__.py create mode 100644 tests/unit/test_dependency_verifier.py diff --git a/.gitignore b/.gitignore index 67b386bbf..4a162fa59 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,4 @@ __pycache__/ /kafka_*/ venv /karapace/version.py +.run diff --git a/karapace/dependency.py b/karapace/dependency.py new file mode 100644 index 000000000..19736c02f --- /dev/null +++ b/karapace/dependency.py @@ -0,0 +1,21 @@ +from typing import Optional, TYPE_CHECKING + +if TYPE_CHECKING: + from karapace.schema_models import ValidatedTypedSchema + + +class Dependency: + def __init__(self, name: str, subject: str, version: int, schema: "ValidatedTypedSchema") -> None: + self.name = name + self.subject = subject + self.version = version + self.schema = schema + + def identifier(self) -> str: + return self.name + "_" + self.subject + "_" + str(self.version) + + +class DependencyVerifierResult: + def __init__(self, result: bool, message: Optional[str] = ""): + self.result = result + self.message = message diff --git a/karapace/errors.py b/karapace/errors.py index af1542500..188da45d0 100644 --- a/karapace/errors.py +++ b/karapace/errors.py @@ -1,3 +1,6 @@ +from typing import List, Union + + class VersionNotFoundException(Exception): pass @@ -18,6 +21,14 @@ class InvalidSchemaType(Exception): pass +class InvalidReferences(Exception): + pass + + +class ReferencesNotSupportedException(Exception): + pass + + class SchemasNotFoundException(Exception): pass @@ -38,6 +49,13 @@ class SubjectNotSoftDeletedException(Exception): pass +class ReferenceExistsException(Exception): + def __init__(self, referenced_by: List, version: Union[int, str]): + super().__init__() + self.version = version + self.referenced_by = referenced_by + + class SubjectSoftDeletedException(Exception): pass diff --git a/karapace/protobuf/dependency.py b/karapace/protobuf/dependency.py new file mode 100644 index 000000000..126b7c887 --- /dev/null +++ b/karapace/protobuf/dependency.py @@ -0,0 +1,56 @@ +from karapace.dependency import DependencyVerifierResult +from karapace.protobuf.known_dependency import DependenciesHardcoded, KnownDependency +from karapace.protobuf.one_of_element import OneOfElement +from typing import List + + +class ProtobufDependencyVerifier: + def __init__(self) -> None: + self.declared_types: List[str] = [] + self.used_types: List[str] = [] + self.import_path: List[str] = [] + + def add_declared_type(self, full_name: str) -> None: + self.declared_types.append(full_name) + + def add_used_type(self, parent: str, element_type: str) -> None: + if element_type.find("map<") == 0: + end = element_type.find(">") + virgule = element_type.find(",") + key = element_type[4:virgule] + value = element_type[virgule + 1 : end] + value = value.strip() + self.used_types.append(parent + ";" + key) + self.used_types.append(parent + ";" + value) + else: + self.used_types.append(parent + ";" + element_type) + + def add_import(self, import_name: str) -> None: + self.import_path.append(import_name) + + def verify(self) -> DependencyVerifierResult: + declared_index = set(self.declared_types) + for used_type in self.used_types: + delimiter = used_type.rfind(";") + used_type_with_scope = "" + if delimiter != -1: + used_type_with_scope = used_type[:delimiter] + "." + used_type[delimiter + 1 :] + used_type = used_type[delimiter + 1 :] + + if not ( + used_type in DependenciesHardcoded.index + or KnownDependency.index_simple.get(used_type) is not None + or KnownDependency.index.get(used_type) is not None + or used_type in declared_index + or (delimiter != -1 and used_type_with_scope in declared_index) + or "." + used_type in declared_index + ): + return DependencyVerifierResult(False, f"type {used_type} is not defined") + + return DependencyVerifierResult(True) + + +def _process_one_of(verifier: ProtobufDependencyVerifier, package_name: str, parent_name: str, one_of: OneOfElement) -> None: + parent = package_name + "." + parent_name + for field in one_of.fields: + verifier.add_used_type(parent, field.element_type) diff --git a/karapace/protobuf/exception.py b/karapace/protobuf/exception.py index b42d6e141..7f286c343 100644 --- a/karapace/protobuf/exception.py +++ b/karapace/protobuf/exception.py @@ -25,6 +25,10 @@ class ProtobufTypeException(Error): """Generic Protobuf type error.""" +class ProtobufUnresolvedDependencyException(ProtobufException): + """a Protobuf schema has unresolved dependency""" + + class SchemaParseException(ProtobufException): """Error while parsing a Protobuf schema descriptor.""" diff --git a/karapace/protobuf/google/protobuf/any.proto b/karapace/protobuf/google/protobuf/any.proto new file mode 100644 index 000000000..e2c2042fd --- /dev/null +++ b/karapace/protobuf/google/protobuf/any.proto @@ -0,0 +1,158 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package google.protobuf; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option go_package = "google.golang.org/protobuf/types/known/anypb"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "AnyProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; + +// `Any` contains an arbitrary serialized protocol buffer message along with a +// URL that describes the type of the serialized message. +// +// Protobuf library provides support to pack/unpack Any values in the form +// of utility functions or additional generated methods of the Any type. +// +// Example 1: Pack and unpack a message in C++. +// +// Foo foo = ...; +// Any any; +// any.PackFrom(foo); +// ... +// if (any.UnpackTo(&foo)) { +// ... +// } +// +// Example 2: Pack and unpack a message in Java. +// +// Foo foo = ...; +// Any any = Any.pack(foo); +// ... +// if (any.is(Foo.class)) { +// foo = any.unpack(Foo.class); +// } +// +// Example 3: Pack and unpack a message in Python. +// +// foo = Foo(...) +// any = Any() +// any.Pack(foo) +// ... +// if any.Is(Foo.DESCRIPTOR): +// any.Unpack(foo) +// ... +// +// Example 4: Pack and unpack a message in Go +// +// foo := &pb.Foo{...} +// any, err := anypb.New(foo) +// if err != nil { +// ... +// } +// ... +// foo := &pb.Foo{} +// if err := any.UnmarshalTo(foo); err != nil { +// ... +// } +// +// The pack methods provided by protobuf library will by default use +// 'type.googleapis.com/full.type.name' as the type URL and the unpack +// methods only use the fully qualified type name after the last '/' +// in the type URL, for example "foo.bar.com/x/y.z" will yield type +// name "y.z". +// +// +// JSON +// +// The JSON representation of an `Any` value uses the regular +// representation of the deserialized, embedded message, with an +// additional field `@type` which contains the type URL. Example: +// +// package google.profile; +// message Person { +// string first_name = 1; +// string last_name = 2; +// } +// +// { +// "@type": "type.googleapis.com/google.profile.Person", +// "firstName": , +// "lastName": +// } +// +// If the embedded message type is well-known and has a custom JSON +// representation, that representation will be embedded adding a field +// `value` which holds the custom JSON in addition to the `@type` +// field. Example (for message [google.protobuf.Duration][]): +// +// { +// "@type": "type.googleapis.com/google.protobuf.Duration", +// "value": "1.212s" +// } +// +message Any { + // A URL/resource name that uniquely identifies the type of the serialized + // protocol buffer message. This string must contain at least + // one "/" character. The last segment of the URL's path must represent + // the fully qualified name of the type (as in + // `path/google.protobuf.Duration`). The name should be in a canonical form + // (e.g., leading "." is not accepted). + // + // In practice, teams usually precompile into the binary all types that they + // expect it to use in the context of Any. However, for URLs which use the + // scheme `http`, `https`, or no scheme, one can optionally set up a type + // server that maps type URLs to message definitions as follows: + // + // * If no scheme is provided, `https` is assumed. + // * An HTTP GET on the URL must yield a [google.protobuf.Type][] + // value in binary format, or produce an error. + // * Applications are allowed to cache lookup results based on the + // URL, or have them precompiled into a binary to avoid any + // lookup. Therefore, binary compatibility needs to be preserved + // on changes to types. (Use versioned type names to manage + // breaking changes.) + // + // Note: this functionality is not currently available in the official + // protobuf release, and it is not used for type URLs beginning with + // type.googleapis.com. + // + // Schemes other than `http`, `https` (or the empty scheme) might be + // used with implementation specific semantics. + // + string type_url = 1; + + // Must be a valid serialized protocol buffer of the above specified type. + bytes value = 2; +} diff --git a/karapace/protobuf/google/protobuf/any_test.proto b/karapace/protobuf/google/protobuf/any_test.proto new file mode 100644 index 000000000..256035b44 --- /dev/null +++ b/karapace/protobuf/google/protobuf/any_test.proto @@ -0,0 +1,44 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package protobuf_unittest; + +import "google/protobuf/any.proto"; + +option java_outer_classname = "TestAnyProto"; + +message TestAny { + int32 int32_value = 1; + google.protobuf.Any any_value = 2; + repeated google.protobuf.Any repeated_any_value = 3; + string text = 4; +} diff --git a/karapace/protobuf/google/protobuf/api.proto b/karapace/protobuf/google/protobuf/api.proto new file mode 100644 index 000000000..3d598fc85 --- /dev/null +++ b/karapace/protobuf/google/protobuf/api.proto @@ -0,0 +1,208 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package google.protobuf; + +import "google/protobuf/source_context.proto"; +import "google/protobuf/type.proto"; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "ApiProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; +option go_package = "google.golang.org/protobuf/types/known/apipb"; + +// Api is a light-weight descriptor for an API Interface. +// +// Interfaces are also described as "protocol buffer services" in some contexts, +// such as by the "service" keyword in a .proto file, but they are different +// from API Services, which represent a concrete implementation of an interface +// as opposed to simply a description of methods and bindings. They are also +// sometimes simply referred to as "APIs" in other contexts, such as the name of +// this message itself. See https://cloud.google.com/apis/design/glossary for +// detailed terminology. +message Api { + // The fully qualified name of this interface, including package name + // followed by the interface's simple name. + string name = 1; + + // The methods of this interface, in unspecified order. + repeated Method methods = 2; + + // Any metadata attached to the interface. + repeated Option options = 3; + + // A version string for this interface. If specified, must have the form + // `major-version.minor-version`, as in `1.10`. If the minor version is + // omitted, it defaults to zero. If the entire version field is empty, the + // major version is derived from the package name, as outlined below. If the + // field is not empty, the version in the package name will be verified to be + // consistent with what is provided here. + // + // The versioning schema uses [semantic + // versioning](http://semver.org) where the major version number + // indicates a breaking change and the minor version an additive, + // non-breaking change. Both version numbers are signals to users + // what to expect from different versions, and should be carefully + // chosen based on the product plan. + // + // The major version is also reflected in the package name of the + // interface, which must end in `v`, as in + // `google.feature.v1`. For major versions 0 and 1, the suffix can + // be omitted. Zero major versions must only be used for + // experimental, non-GA interfaces. + // + // + string version = 4; + + // Source context for the protocol buffer service represented by this + // message. + SourceContext source_context = 5; + + // Included interfaces. See [Mixin][]. + repeated Mixin mixins = 6; + + // The source syntax of the service. + Syntax syntax = 7; +} + +// Method represents a method of an API interface. +message Method { + // The simple name of this method. + string name = 1; + + // A URL of the input message type. + string request_type_url = 2; + + // If true, the request is streamed. + bool request_streaming = 3; + + // The URL of the output message type. + string response_type_url = 4; + + // If true, the response is streamed. + bool response_streaming = 5; + + // Any metadata attached to the method. + repeated Option options = 6; + + // The source syntax of this method. + Syntax syntax = 7; +} + +// Declares an API Interface to be included in this interface. The including +// interface must redeclare all the methods from the included interface, but +// documentation and options are inherited as follows: +// +// - If after comment and whitespace stripping, the documentation +// string of the redeclared method is empty, it will be inherited +// from the original method. +// +// - Each annotation belonging to the service config (http, +// visibility) which is not set in the redeclared method will be +// inherited. +// +// - If an http annotation is inherited, the path pattern will be +// modified as follows. Any version prefix will be replaced by the +// version of the including interface plus the [root][] path if +// specified. +// +// Example of a simple mixin: +// +// package google.acl.v1; +// service AccessControl { +// // Get the underlying ACL object. +// rpc GetAcl(GetAclRequest) returns (Acl) { +// option (google.api.http).get = "/v1/{resource=**}:getAcl"; +// } +// } +// +// package google.storage.v2; +// service Storage { +// rpc GetAcl(GetAclRequest) returns (Acl); +// +// // Get a data record. +// rpc GetData(GetDataRequest) returns (Data) { +// option (google.api.http).get = "/v2/{resource=**}"; +// } +// } +// +// Example of a mixin configuration: +// +// apis: +// - name: google.storage.v2.Storage +// mixins: +// - name: google.acl.v1.AccessControl +// +// The mixin construct implies that all methods in `AccessControl` are +// also declared with same name and request/response types in +// `Storage`. A documentation generator or annotation processor will +// see the effective `Storage.GetAcl` method after inheriting +// documentation and annotations as follows: +// +// service Storage { +// // Get the underlying ACL object. +// rpc GetAcl(GetAclRequest) returns (Acl) { +// option (google.api.http).get = "/v2/{resource=**}:getAcl"; +// } +// ... +// } +// +// Note how the version in the path pattern changed from `v1` to `v2`. +// +// If the `root` field in the mixin is specified, it should be a +// relative path under which inherited HTTP paths are placed. Example: +// +// apis: +// - name: google.storage.v2.Storage +// mixins: +// - name: google.acl.v1.AccessControl +// root: acls +// +// This implies the following inherited HTTP annotation: +// +// service Storage { +// // Get the underlying ACL object. +// rpc GetAcl(GetAclRequest) returns (Acl) { +// option (google.api.http).get = "/v2/acls/{resource=**}:getAcl"; +// } +// ... +// } +message Mixin { + // The fully qualified name of the interface which is included. + string name = 1; + + // If non-empty specifies a path under which inherited HTTP paths + // are rooted. + string root = 2; +} diff --git a/karapace/protobuf/google/protobuf/descriptor.proto b/karapace/protobuf/google/protobuf/descriptor.proto new file mode 100644 index 000000000..5b4d06b41 --- /dev/null +++ b/karapace/protobuf/google/protobuf/descriptor.proto @@ -0,0 +1,918 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Author: kenton@google.com (Kenton Varda) +// Based on original Protocol Buffers design by +// Sanjay Ghemawat, Jeff Dean, and others. +// +// The messages in this file describe the definitions found in .proto files. +// A valid .proto file can be translated directly to a FileDescriptorProto +// without any other information (e.g. without reading its imports). + + +syntax = "proto2"; + +package google.protobuf; + +option go_package = "google.golang.org/protobuf/types/descriptorpb"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "DescriptorProtos"; +option csharp_namespace = "Google.Protobuf.Reflection"; +option objc_class_prefix = "GPB"; +option cc_enable_arenas = true; + +// descriptor.proto must be optimized for speed because reflection-based +// algorithms don't work during bootstrapping. +option optimize_for = SPEED; + +// The protocol compiler can output a FileDescriptorSet containing the .proto +// files it parses. +message FileDescriptorSet { + repeated FileDescriptorProto file = 1; +} + +// Describes a complete .proto file. +message FileDescriptorProto { + optional string name = 1; // file name, relative to root of source tree + optional string package = 2; // e.g. "foo", "foo.bar", etc. + + // Names of files imported by this file. + repeated string dependency = 3; + // Indexes of the public imported files in the dependency list above. + repeated int32 public_dependency = 10; + // Indexes of the weak imported files in the dependency list. + // For Google-internal migration only. Do not use. + repeated int32 weak_dependency = 11; + + // All top-level definitions in this file. + repeated DescriptorProto message_type = 4; + repeated EnumDescriptorProto enum_type = 5; + repeated ServiceDescriptorProto service = 6; + repeated FieldDescriptorProto extension = 7; + + optional FileOptions options = 8; + + // This field contains optional information about the original source code. + // You may safely remove this entire field without harming runtime + // functionality of the descriptors -- the information is needed only by + // development tools. + optional SourceCodeInfo source_code_info = 9; + + // The syntax of the proto file. + // The supported values are "proto2" and "proto3". + optional string syntax = 12; +} + +// Describes a message type. +message DescriptorProto { + optional string name = 1; + + repeated FieldDescriptorProto field = 2; + repeated FieldDescriptorProto extension = 6; + + repeated DescriptorProto nested_type = 3; + repeated EnumDescriptorProto enum_type = 4; + + message ExtensionRange { + optional int32 start = 1; // Inclusive. + optional int32 end = 2; // Exclusive. + + optional ExtensionRangeOptions options = 3; + } + repeated ExtensionRange extension_range = 5; + + repeated OneofDescriptorProto oneof_decl = 8; + + optional MessageOptions options = 7; + + // Range of reserved tag numbers. Reserved tag numbers may not be used by + // fields or extension ranges in the same message. Reserved ranges may + // not overlap. + message ReservedRange { + optional int32 start = 1; // Inclusive. + optional int32 end = 2; // Exclusive. + } + repeated ReservedRange reserved_range = 9; + // Reserved field names, which may not be used by fields in the same message. + // A given name may only be reserved once. + repeated string reserved_name = 10; +} + +message ExtensionRangeOptions { + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; +} + +// Describes a field within a message. +message FieldDescriptorProto { + enum Type { + // 0 is reserved for errors. + // Order is weird for historical reasons. + TYPE_DOUBLE = 1; + TYPE_FLOAT = 2; + // Not ZigZag encoded. Negative numbers take 10 bytes. Use TYPE_SINT64 if + // negative values are likely. + TYPE_INT64 = 3; + TYPE_UINT64 = 4; + // Not ZigZag encoded. Negative numbers take 10 bytes. Use TYPE_SINT32 if + // negative values are likely. + TYPE_INT32 = 5; + TYPE_FIXED64 = 6; + TYPE_FIXED32 = 7; + TYPE_BOOL = 8; + TYPE_STRING = 9; + // Tag-delimited aggregate. + // Group type is deprecated and not supported in proto3. However, Proto3 + // implementations should still be able to parse the group wire format and + // treat group fields as unknown fields. + TYPE_GROUP = 10; + TYPE_MESSAGE = 11; // Length-delimited aggregate. + + // New in version 2. + TYPE_BYTES = 12; + TYPE_UINT32 = 13; + TYPE_ENUM = 14; + TYPE_SFIXED32 = 15; + TYPE_SFIXED64 = 16; + TYPE_SINT32 = 17; // Uses ZigZag encoding. + TYPE_SINT64 = 18; // Uses ZigZag encoding. + } + + enum Label { + // 0 is reserved for errors + LABEL_OPTIONAL = 1; + LABEL_REQUIRED = 2; + LABEL_REPEATED = 3; + } + + optional string name = 1; + optional int32 number = 3; + optional Label label = 4; + + // If type_name is set, this need not be set. If both this and type_name + // are set, this must be one of TYPE_ENUM, TYPE_MESSAGE or TYPE_GROUP. + optional Type type = 5; + + // For message and enum types, this is the name of the type. If the name + // starts with a '.', it is fully-qualified. Otherwise, C++-like scoping + // rules are used to find the type (i.e. first the nested types within this + // message are searched, then within the parent, on up to the root + // namespace). + optional string type_name = 6; + + // For extensions, this is the name of the type being extended. It is + // resolved in the same manner as type_name. + optional string extendee = 2; + + // For numeric types, contains the original text representation of the value. + // For booleans, "true" or "false". + // For strings, contains the default text contents (not escaped in any way). + // For bytes, contains the C escaped value. All bytes >= 128 are escaped. + optional string default_value = 7; + + // If set, gives the index of a oneof in the containing type's oneof_decl + // list. This field is a member of that oneof. + optional int32 oneof_index = 9; + + // JSON name of this field. The value is set by protocol compiler. If the + // user has set a "json_name" option on this field, that option's value + // will be used. Otherwise, it's deduced from the field's name by converting + // it to camelCase. + optional string json_name = 10; + + optional FieldOptions options = 8; + + // If true, this is a proto3 "optional". When a proto3 field is optional, it + // tracks presence regardless of field type. + // + // When proto3_optional is true, this field must be belong to a oneof to + // signal to old proto3 clients that presence is tracked for this field. This + // oneof is known as a "synthetic" oneof, and this field must be its sole + // member (each proto3 optional field gets its own synthetic oneof). Synthetic + // oneofs exist in the descriptor only, and do not generate any API. Synthetic + // oneofs must be ordered after all "real" oneofs. + // + // For message fields, proto3_optional doesn't create any semantic change, + // since non-repeated message fields always track presence. However it still + // indicates the semantic detail of whether the user wrote "optional" or not. + // This can be useful for round-tripping the .proto file. For consistency we + // give message fields a synthetic oneof also, even though it is not required + // to track presence. This is especially important because the parser can't + // tell if a field is a message or an enum, so it must always create a + // synthetic oneof. + // + // Proto2 optional fields do not set this flag, because they already indicate + // optional with `LABEL_OPTIONAL`. + optional bool proto3_optional = 17; +} + +// Describes a oneof. +message OneofDescriptorProto { + optional string name = 1; + optional OneofOptions options = 2; +} + +// Describes an enum type. +message EnumDescriptorProto { + optional string name = 1; + + repeated EnumValueDescriptorProto value = 2; + + optional EnumOptions options = 3; + + // Range of reserved numeric values. Reserved values may not be used by + // entries in the same enum. Reserved ranges may not overlap. + // + // Note that this is distinct from DescriptorProto.ReservedRange in that it + // is inclusive such that it can appropriately represent the entire int32 + // domain. + message EnumReservedRange { + optional int32 start = 1; // Inclusive. + optional int32 end = 2; // Inclusive. + } + + // Range of reserved numeric values. Reserved numeric values may not be used + // by enum values in the same enum declaration. Reserved ranges may not + // overlap. + repeated EnumReservedRange reserved_range = 4; + + // Reserved enum value names, which may not be reused. A given name may only + // be reserved once. + repeated string reserved_name = 5; +} + +// Describes a value within an enum. +message EnumValueDescriptorProto { + optional string name = 1; + optional int32 number = 2; + + optional EnumValueOptions options = 3; +} + +// Describes a service. +message ServiceDescriptorProto { + optional string name = 1; + repeated MethodDescriptorProto method = 2; + + optional ServiceOptions options = 3; +} + +// Describes a method of a service. +message MethodDescriptorProto { + optional string name = 1; + + // Input and output type names. These are resolved in the same way as + // FieldDescriptorProto.type_name, but must refer to a message type. + optional string input_type = 2; + optional string output_type = 3; + + optional MethodOptions options = 4; + + // Identifies if client streams multiple client messages + optional bool client_streaming = 5 [default = false]; + // Identifies if server streams multiple server messages + optional bool server_streaming = 6 [default = false]; +} + + +// =================================================================== +// Options + +// Each of the definitions above may have "options" attached. These are +// just annotations which may cause code to be generated slightly differently +// or may contain hints for code that manipulates protocol messages. +// +// Clients may define custom options as extensions of the *Options messages. +// These extensions may not yet be known at parsing time, so the parser cannot +// store the values in them. Instead it stores them in a field in the *Options +// message called uninterpreted_option. This field must have the same name +// across all *Options messages. We then use this field to populate the +// extensions when we build a descriptor, at which point all protos have been +// parsed and so all extensions are known. +// +// Extension numbers for custom options may be chosen as follows: +// * For options which will only be used within a single application or +// organization, or for experimental options, use field numbers 50000 +// through 99999. It is up to you to ensure that you do not use the +// same number for multiple options. +// * For options which will be published and used publicly by multiple +// independent entities, e-mail protobuf-global-extension-registry@google.com +// to reserve extension numbers. Simply provide your project name (e.g. +// Objective-C plugin) and your project website (if available) -- there's no +// need to explain how you intend to use them. Usually you only need one +// extension number. You can declare multiple options with only one extension +// number by putting them in a sub-message. See the Custom Options section of +// the docs for examples: +// https://developers.google.com/protocol-buffers/docs/proto#options +// If this turns out to be popular, a web service will be set up +// to automatically assign option numbers. + +message FileOptions { + + // Sets the Java package where classes generated from this .proto will be + // placed. By default, the proto package is used, but this is often + // inappropriate because proto packages do not normally start with backwards + // domain names. + optional string java_package = 1; + + + // Controls the name of the wrapper Java class generated for the .proto file. + // That class will always contain the .proto file's getDescriptor() method as + // well as any top-level extensions defined in the .proto file. + // If java_multiple_files is disabled, then all the other classes from the + // .proto file will be nested inside the single wrapper outer class. + optional string java_outer_classname = 8; + + // If enabled, then the Java code generator will generate a separate .java + // file for each top-level message, enum, and service defined in the .proto + // file. Thus, these types will *not* be nested inside the wrapper class + // named by java_outer_classname. However, the wrapper class will still be + // generated to contain the file's getDescriptor() method as well as any + // top-level extensions defined in the file. + optional bool java_multiple_files = 10 [default = false]; + + // This option does nothing. + optional bool java_generate_equals_and_hash = 20 [deprecated=true]; + + // If set true, then the Java2 code generator will generate code that + // throws an exception whenever an attempt is made to assign a non-UTF-8 + // byte sequence to a string field. + // Message reflection will do the same. + // However, an extension field still accepts non-UTF-8 byte sequences. + // This option has no effect on when used with the lite runtime. + optional bool java_string_check_utf8 = 27 [default = false]; + + + // Generated classes can be optimized for speed or code size. + enum OptimizeMode { + SPEED = 1; // Generate complete code for parsing, serialization, + // etc. + CODE_SIZE = 2; // Use ReflectionOps to implement these methods. + LITE_RUNTIME = 3; // Generate code using MessageLite and the lite runtime. + } + optional OptimizeMode optimize_for = 9 [default = SPEED]; + + // Sets the Go package where structs generated from this .proto will be + // placed. If omitted, the Go package will be derived from the following: + // - The basename of the package import path, if provided. + // - Otherwise, the package statement in the .proto file, if present. + // - Otherwise, the basename of the .proto file, without extension. + optional string go_package = 11; + + + + + // Should generic services be generated in each language? "Generic" services + // are not specific to any particular RPC system. They are generated by the + // main code generators in each language (without additional plugins). + // Generic services were the only kind of service generation supported by + // early versions of google.protobuf. + // + // Generic services are now considered deprecated in favor of using plugins + // that generate code specific to your particular RPC system. Therefore, + // these default to false. Old code which depends on generic services should + // explicitly set them to true. + optional bool cc_generic_services = 16 [default = false]; + optional bool java_generic_services = 17 [default = false]; + optional bool py_generic_services = 18 [default = false]; + optional bool php_generic_services = 42 [default = false]; + + // Is this file deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for everything in the file, or it will be completely ignored; in the very + // least, this is a formalization for deprecating files. + optional bool deprecated = 23 [default = false]; + + // Enables the use of arenas for the proto messages in this file. This applies + // only to generated classes for C++. + optional bool cc_enable_arenas = 31 [default = true]; + + + // Sets the objective c class prefix which is prepended to all objective c + // generated classes from this .proto. There is no default. + optional string objc_class_prefix = 36; + + // Namespace for generated classes; defaults to the package. + optional string csharp_namespace = 37; + + // By default Swift generators will take the proto package and CamelCase it + // replacing '.' with underscore and use that to prefix the types/symbols + // defined. When this options is provided, they will use this value instead + // to prefix the types/symbols defined. + optional string swift_prefix = 39; + + // Sets the php class prefix which is prepended to all php generated classes + // from this .proto. Default is empty. + optional string php_class_prefix = 40; + + // Use this option to change the namespace of php generated classes. Default + // is empty. When this option is empty, the package name will be used for + // determining the namespace. + optional string php_namespace = 41; + + // Use this option to change the namespace of php generated metadata classes. + // Default is empty. When this option is empty, the proto file name will be + // used for determining the namespace. + optional string php_metadata_namespace = 44; + + // Use this option to change the package of ruby generated classes. Default + // is empty. When this option is not set, the package name will be used for + // determining the ruby package. + optional string ruby_package = 45; + + + // The parser stores options it doesn't recognize here. + // See the documentation for the "Options" section above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. + // See the documentation for the "Options" section above. + extensions 1000 to max; + + reserved 38; +} + +message MessageOptions { + // Set true to use the old proto1 MessageSet wire format for extensions. + // This is provided for backwards-compatibility with the MessageSet wire + // format. You should not use this for any other reason: It's less + // efficient, has fewer features, and is more complicated. + // + // The message must be defined exactly as follows: + // message Foo { + // option message_set_wire_format = true; + // extensions 4 to max; + // } + // Note that the message cannot have any defined fields; MessageSets only + // have extensions. + // + // All extensions of your type must be singular messages; e.g. they cannot + // be int32s, enums, or repeated messages. + // + // Because this is an option, the above two restrictions are not enforced by + // the protocol compiler. + optional bool message_set_wire_format = 1 [default = false]; + + // Disables the generation of the standard "descriptor()" accessor, which can + // conflict with a field of the same name. This is meant to make migration + // from proto1 easier; new code should avoid fields named "descriptor". + optional bool no_standard_descriptor_accessor = 2 [default = false]; + + // Is this message deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for the message, or it will be completely ignored; in the very least, + // this is a formalization for deprecating messages. + optional bool deprecated = 3 [default = false]; + + reserved 4, 5, 6; + + // Whether the message is an automatically generated map entry type for the + // maps field. + // + // For maps fields: + // map map_field = 1; + // The parsed descriptor looks like: + // message MapFieldEntry { + // option map_entry = true; + // optional KeyType key = 1; + // optional ValueType value = 2; + // } + // repeated MapFieldEntry map_field = 1; + // + // Implementations may choose not to generate the map_entry=true message, but + // use a native map in the target language to hold the keys and values. + // The reflection APIs in such implementations still need to work as + // if the field is a repeated message field. + // + // NOTE: Do not set the option in .proto files. Always use the maps syntax + // instead. The option should only be implicitly set by the proto compiler + // parser. + optional bool map_entry = 7; + + reserved 8; // javalite_serializable + reserved 9; // javanano_as_lite + + + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; +} + +message FieldOptions { + // The ctype option instructs the C++ code generator to use a different + // representation of the field than it normally would. See the specific + // options below. This option is not yet implemented in the open source + // release -- sorry, we'll try to include it in a future version! + optional CType ctype = 1 [default = STRING]; + enum CType { + // Default mode. + STRING = 0; + + CORD = 1; + + STRING_PIECE = 2; + } + // The packed option can be enabled for repeated primitive fields to enable + // a more efficient representation on the wire. Rather than repeatedly + // writing the tag and type for each element, the entire array is encoded as + // a single length-delimited blob. In proto3, only explicit setting it to + // false will avoid using packed encoding. + optional bool packed = 2; + + // The jstype option determines the JavaScript type used for values of the + // field. The option is permitted only for 64 bit integral and fixed types + // (int64, uint64, sint64, fixed64, sfixed64). A field with jstype JS_STRING + // is represented as JavaScript string, which avoids loss of precision that + // can happen when a large value is converted to a floating point JavaScript. + // Specifying JS_NUMBER for the jstype causes the generated JavaScript code to + // use the JavaScript "number" type. The behavior of the default option + // JS_NORMAL is implementation dependent. + // + // This option is an enum to permit additional types to be added, e.g. + // goog.math.Integer. + optional JSType jstype = 6 [default = JS_NORMAL]; + enum JSType { + // Use the default type. + JS_NORMAL = 0; + + // Use JavaScript strings. + JS_STRING = 1; + + // Use JavaScript numbers. + JS_NUMBER = 2; + } + + // Should this field be parsed lazily? Lazy applies only to message-type + // fields. It means that when the outer message is initially parsed, the + // inner message's contents will not be parsed but instead stored in encoded + // form. The inner message will actually be parsed when it is first accessed. + // + // This is only a hint. Implementations are free to choose whether to use + // eager or lazy parsing regardless of the value of this option. However, + // setting this option true suggests that the protocol author believes that + // using lazy parsing on this field is worth the additional bookkeeping + // overhead typically needed to implement it. + // + // This option does not affect the public interface of any generated code; + // all method signatures remain the same. Furthermore, thread-safety of the + // interface is not affected by this option; const methods remain safe to + // call from multiple threads concurrently, while non-const methods continue + // to require exclusive access. + // + // + // Note that implementations may choose not to check required fields within + // a lazy sub-message. That is, calling IsInitialized() on the outer message + // may return true even if the inner message has missing required fields. + // This is necessary because otherwise the inner message would have to be + // parsed in order to perform the check, defeating the purpose of lazy + // parsing. An implementation which chooses not to check required fields + // must be consistent about it. That is, for any particular sub-message, the + // implementation must either *always* check its required fields, or *never* + // check its required fields, regardless of whether or not the message has + // been parsed. + // + // As of May 2022, lazy verifies the contents of the byte stream during + // parsing. An invalid byte stream will cause the overall parsing to fail. + optional bool lazy = 5 [default = false]; + + // unverified_lazy does no correctness checks on the byte stream. This should + // only be used where lazy with verification is prohibitive for performance + // reasons. + optional bool unverified_lazy = 15 [default = false]; + + // Is this field deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for accessors, or it will be completely ignored; in the very least, this + // is a formalization for deprecating fields. + optional bool deprecated = 3 [default = false]; + + // For Google-internal migration only. Do not use. + optional bool weak = 10 [default = false]; + + + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; + + reserved 4; // removed jtype +} + +message OneofOptions { + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; +} + +message EnumOptions { + + // Set this option to true to allow mapping different tag names to the same + // value. + optional bool allow_alias = 2; + + // Is this enum deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for the enum, or it will be completely ignored; in the very least, this + // is a formalization for deprecating enums. + optional bool deprecated = 3 [default = false]; + + reserved 5; // javanano_as_lite + + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; +} + +message EnumValueOptions { + // Is this enum value deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for the enum value, or it will be completely ignored; in the very least, + // this is a formalization for deprecating enum values. + optional bool deprecated = 1 [default = false]; + + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; +} + +message ServiceOptions { + + // Note: Field numbers 1 through 32 are reserved for Google's internal RPC + // framework. We apologize for hoarding these numbers to ourselves, but + // we were already using them long before we decided to release Protocol + // Buffers. + + // Is this service deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for the service, or it will be completely ignored; in the very least, + // this is a formalization for deprecating services. + optional bool deprecated = 33 [default = false]; + + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; +} + +message MethodOptions { + + // Note: Field numbers 1 through 32 are reserved for Google's internal RPC + // framework. We apologize for hoarding these numbers to ourselves, but + // we were already using them long before we decided to release Protocol + // Buffers. + + // Is this method deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for the method, or it will be completely ignored; in the very least, + // this is a formalization for deprecating methods. + optional bool deprecated = 33 [default = false]; + + // Is this method side-effect-free (or safe in HTTP parlance), or idempotent, + // or neither? HTTP based RPC implementation may choose GET verb for safe + // methods, and PUT verb for idempotent methods instead of the default POST. + enum IdempotencyLevel { + IDEMPOTENCY_UNKNOWN = 0; + NO_SIDE_EFFECTS = 1; // implies idempotent + IDEMPOTENT = 2; // idempotent, but may have side effects + } + optional IdempotencyLevel idempotency_level = 34 + [default = IDEMPOTENCY_UNKNOWN]; + + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; +} + + +// A message representing a option the parser does not recognize. This only +// appears in options protos created by the compiler::Parser class. +// DescriptorPool resolves these when building Descriptor objects. Therefore, +// options protos in descriptor objects (e.g. returned by Descriptor::options(), +// or produced by Descriptor::CopyTo()) will never have UninterpretedOptions +// in them. +message UninterpretedOption { + // The name of the uninterpreted option. Each string represents a segment in + // a dot-separated name. is_extension is true iff a segment represents an + // extension (denoted with parentheses in options specs in .proto files). + // E.g.,{ ["foo", false], ["bar.baz", true], ["moo", false] } represents + // "foo.(bar.baz).moo". + message NamePart { + required string name_part = 1; + required bool is_extension = 2; + } + repeated NamePart name = 2; + + // The value of the uninterpreted option, in whatever type the tokenizer + // identified it as during parsing. Exactly one of these should be set. + optional string identifier_value = 3; + optional uint64 positive_int_value = 4; + optional int64 negative_int_value = 5; + optional double double_value = 6; + optional bytes string_value = 7; + optional string aggregate_value = 8; +} + +// =================================================================== +// Optional source code info + +// Encapsulates information about the original source file from which a +// FileDescriptorProto was generated. +message SourceCodeInfo { + // A Location identifies a piece of source code in a .proto file which + // corresponds to a particular definition. This information is intended + // to be useful to IDEs, code indexers, documentation generators, and similar + // tools. + // + // For example, say we have a file like: + // message Foo { + // optional string foo = 1; + // } + // Let's look at just the field definition: + // optional string foo = 1; + // ^ ^^ ^^ ^ ^^^ + // a bc de f ghi + // We have the following locations: + // span path represents + // [a,i) [ 4, 0, 2, 0 ] The whole field definition. + // [a,b) [ 4, 0, 2, 0, 4 ] The label (optional). + // [c,d) [ 4, 0, 2, 0, 5 ] The type (string). + // [e,f) [ 4, 0, 2, 0, 1 ] The name (foo). + // [g,h) [ 4, 0, 2, 0, 3 ] The number (1). + // + // Notes: + // - A location may refer to a repeated field itself (i.e. not to any + // particular index within it). This is used whenever a set of elements are + // logically enclosed in a single code segment. For example, an entire + // extend block (possibly containing multiple extension definitions) will + // have an outer location whose path refers to the "extensions" repeated + // field without an index. + // - Multiple locations may have the same path. This happens when a single + // logical declaration is spread out across multiple places. The most + // obvious example is the "extend" block again -- there may be multiple + // extend blocks in the same scope, each of which will have the same path. + // - A location's span is not always a subset of its parent's span. For + // example, the "extendee" of an extension declaration appears at the + // beginning of the "extend" block and is shared by all extensions within + // the block. + // - Just because a location's span is a subset of some other location's span + // does not mean that it is a descendant. For example, a "group" defines + // both a type and a field in a single declaration. Thus, the locations + // corresponding to the type and field and their components will overlap. + // - Code which tries to interpret locations should probably be designed to + // ignore those that it doesn't understand, as more types of locations could + // be recorded in the future. + repeated Location location = 1; + message Location { + // Identifies which part of the FileDescriptorProto was defined at this + // location. + // + // Each element is a field number or an index. They form a path from + // the root FileDescriptorProto to the place where the definition occurs. + // For example, this path: + // [ 4, 3, 2, 7, 1 ] + // refers to: + // file.message_type(3) // 4, 3 + // .field(7) // 2, 7 + // .name() // 1 + // This is because FileDescriptorProto.message_type has field number 4: + // repeated DescriptorProto message_type = 4; + // and DescriptorProto.field has field number 2: + // repeated FieldDescriptorProto field = 2; + // and FieldDescriptorProto.name has field number 1: + // optional string name = 1; + // + // Thus, the above path gives the location of a field name. If we removed + // the last element: + // [ 4, 3, 2, 7 ] + // this path refers to the whole field declaration (from the beginning + // of the label to the terminating semicolon). + repeated int32 path = 1 [packed = true]; + + // Always has exactly three or four elements: start line, start column, + // end line (optional, otherwise assumed same as start line), end column. + // These are packed into a single field for efficiency. Note that line + // and column numbers are zero-based -- typically you will want to add + // 1 to each before displaying to a user. + repeated int32 span = 2 [packed = true]; + + // If this SourceCodeInfo represents a complete declaration, these are any + // comments appearing before and after the declaration which appear to be + // attached to the declaration. + // + // A series of line comments appearing on consecutive lines, with no other + // tokens appearing on those lines, will be treated as a single comment. + // + // leading_detached_comments will keep paragraphs of comments that appear + // before (but not connected to) the current element. Each paragraph, + // separated by empty lines, will be one comment element in the repeated + // field. + // + // Only the comment content is provided; comment markers (e.g. //) are + // stripped out. For block comments, leading whitespace and an asterisk + // will be stripped from the beginning of each line other than the first. + // Newlines are included in the output. + // + // Examples: + // + // optional int32 foo = 1; // Comment attached to foo. + // // Comment attached to bar. + // optional int32 bar = 2; + // + // optional string baz = 3; + // // Comment attached to baz. + // // Another line attached to baz. + // + // // Comment attached to moo. + // // + // // Another line attached to moo. + // optional double moo = 4; + // + // // Detached comment for corge. This is not leading or trailing comments + // // to moo or corge because there are blank lines separating it from + // // both. + // + // // Detached comment for corge paragraph 2. + // + // optional string corge = 5; + // /* Block comment attached + // * to corge. Leading asterisks + // * will be removed. */ + // /* Block comment attached to + // * grault. */ + // optional int32 grault = 6; + // + // // ignored detached comments. + optional string leading_comments = 3; + optional string trailing_comments = 4; + repeated string leading_detached_comments = 6; + } +} + +// Describes the relationship between generated code and its original source +// file. A GeneratedCodeInfo message is associated with only one generated +// source file, but may contain references to different source .proto files. +message GeneratedCodeInfo { + // An Annotation connects some span of text in generated code to an element + // of its generating .proto file. + repeated Annotation annotation = 1; + message Annotation { + // Identifies the element in the original source .proto file. This field + // is formatted the same as SourceCodeInfo.Location.path. + repeated int32 path = 1 [packed = true]; + + // Identifies the filesystem path to the original source .proto. + optional string source_file = 2; + + // Identifies the starting offset in bytes in the generated code + // that relates to the identified object. + optional int32 begin = 3; + + // Identifies the ending offset in bytes in the generated code that + // relates to the identified offset. The end offset should be one past + // the last relevant byte (so the length of the text = end - begin). + optional int32 end = 4; + } +} diff --git a/karapace/protobuf/google/protobuf/duration.proto b/karapace/protobuf/google/protobuf/duration.proto new file mode 100644 index 000000000..81c3e369f --- /dev/null +++ b/karapace/protobuf/google/protobuf/duration.proto @@ -0,0 +1,116 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package google.protobuf; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option cc_enable_arenas = true; +option go_package = "google.golang.org/protobuf/types/known/durationpb"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "DurationProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; + +// A Duration represents a signed, fixed-length span of time represented +// as a count of seconds and fractions of seconds at nanosecond +// resolution. It is independent of any calendar and concepts like "day" +// or "month". It is related to Timestamp in that the difference between +// two Timestamp values is a Duration and it can be added or subtracted +// from a Timestamp. Range is approximately +-10,000 years. +// +// # Examples +// +// Example 1: Compute Duration from two Timestamps in pseudo code. +// +// Timestamp start = ...; +// Timestamp end = ...; +// Duration duration = ...; +// +// duration.seconds = end.seconds - start.seconds; +// duration.nanos = end.nanos - start.nanos; +// +// if (duration.seconds < 0 && duration.nanos > 0) { +// duration.seconds += 1; +// duration.nanos -= 1000000000; +// } else if (duration.seconds > 0 && duration.nanos < 0) { +// duration.seconds -= 1; +// duration.nanos += 1000000000; +// } +// +// Example 2: Compute Timestamp from Timestamp + Duration in pseudo code. +// +// Timestamp start = ...; +// Duration duration = ...; +// Timestamp end = ...; +// +// end.seconds = start.seconds + duration.seconds; +// end.nanos = start.nanos + duration.nanos; +// +// if (end.nanos < 0) { +// end.seconds -= 1; +// end.nanos += 1000000000; +// } else if (end.nanos >= 1000000000) { +// end.seconds += 1; +// end.nanos -= 1000000000; +// } +// +// Example 3: Compute Duration from datetime.timedelta in Python. +// +// td = datetime.timedelta(days=3, minutes=10) +// duration = Duration() +// duration.FromTimedelta(td) +// +// # JSON Mapping +// +// In JSON format, the Duration type is encoded as a string rather than an +// object, where the string ends in the suffix "s" (indicating seconds) and +// is preceded by the number of seconds, with nanoseconds expressed as +// fractional seconds. For example, 3 seconds with 0 nanoseconds should be +// encoded in JSON format as "3s", while 3 seconds and 1 nanosecond should +// be expressed in JSON format as "3.000000001s", and 3 seconds and 1 +// microsecond should be expressed in JSON format as "3.000001s". +// +// +message Duration { + // Signed seconds of the span of time. Must be from -315,576,000,000 + // to +315,576,000,000 inclusive. Note: these bounds are computed from: + // 60 sec/min * 60 min/hr * 24 hr/day * 365.25 days/year * 10000 years + int64 seconds = 1; + + // Signed fractions of a second at nanosecond resolution of the span + // of time. Durations less than one second are represented with a 0 + // `seconds` field and a positive or negative `nanos` field. For durations + // of one second or more, a non-zero value for the `nanos` field must be + // of the same sign as the `seconds` field. Must be from -999,999,999 + // to +999,999,999 inclusive. + int32 nanos = 2; +} diff --git a/karapace/protobuf/google/protobuf/empty.proto b/karapace/protobuf/google/protobuf/empty.proto new file mode 100644 index 000000000..222746219 --- /dev/null +++ b/karapace/protobuf/google/protobuf/empty.proto @@ -0,0 +1,51 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package google.protobuf; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option go_package = "google.golang.org/protobuf/types/known/emptypb"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "EmptyProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; +option cc_enable_arenas = true; + +// A generic empty message that you can re-use to avoid defining duplicated +// empty messages in your APIs. A typical example is to use it as the request +// or the response type of an API method. For instance: +// +// service Foo { +// rpc Bar(google.protobuf.Empty) returns (google.protobuf.Empty); +// } +// +message Empty {} diff --git a/karapace/protobuf/google/protobuf/field_mask.proto b/karapace/protobuf/google/protobuf/field_mask.proto new file mode 100644 index 000000000..6b5104f18 --- /dev/null +++ b/karapace/protobuf/google/protobuf/field_mask.proto @@ -0,0 +1,245 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package google.protobuf; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "FieldMaskProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; +option go_package = "google.golang.org/protobuf/types/known/fieldmaskpb"; +option cc_enable_arenas = true; + +// `FieldMask` represents a set of symbolic field paths, for example: +// +// paths: "f.a" +// paths: "f.b.d" +// +// Here `f` represents a field in some root message, `a` and `b` +// fields in the message found in `f`, and `d` a field found in the +// message in `f.b`. +// +// Field masks are used to specify a subset of fields that should be +// returned by a get operation or modified by an update operation. +// Field masks also have a custom JSON encoding (see below). +// +// # Field Masks in Projections +// +// When used in the context of a projection, a response message or +// sub-message is filtered by the API to only contain those fields as +// specified in the mask. For example, if the mask in the previous +// example is applied to a response message as follows: +// +// f { +// a : 22 +// b { +// d : 1 +// x : 2 +// } +// y : 13 +// } +// z: 8 +// +// The result will not contain specific values for fields x,y and z +// (their value will be set to the default, and omitted in proto text +// output): +// +// +// f { +// a : 22 +// b { +// d : 1 +// } +// } +// +// A repeated field is not allowed except at the last position of a +// paths string. +// +// If a FieldMask object is not present in a get operation, the +// operation applies to all fields (as if a FieldMask of all fields +// had been specified). +// +// Note that a field mask does not necessarily apply to the +// top-level response message. In case of a REST get operation, the +// field mask applies directly to the response, but in case of a REST +// list operation, the mask instead applies to each individual message +// in the returned resource list. In case of a REST custom method, +// other definitions may be used. Where the mask applies will be +// clearly documented together with its declaration in the API. In +// any case, the effect on the returned resource/resources is required +// behavior for APIs. +// +// # Field Masks in Update Operations +// +// A field mask in update operations specifies which fields of the +// targeted resource are going to be updated. The API is required +// to only change the values of the fields as specified in the mask +// and leave the others untouched. If a resource is passed in to +// describe the updated values, the API ignores the values of all +// fields not covered by the mask. +// +// If a repeated field is specified for an update operation, new values will +// be appended to the existing repeated field in the target resource. Note that +// a repeated field is only allowed in the last position of a `paths` string. +// +// If a sub-message is specified in the last position of the field mask for an +// update operation, then new value will be merged into the existing sub-message +// in the target resource. +// +// For example, given the target message: +// +// f { +// b { +// d: 1 +// x: 2 +// } +// c: [1] +// } +// +// And an update message: +// +// f { +// b { +// d: 10 +// } +// c: [2] +// } +// +// then if the field mask is: +// +// paths: ["f.b", "f.c"] +// +// then the result will be: +// +// f { +// b { +// d: 10 +// x: 2 +// } +// c: [1, 2] +// } +// +// An implementation may provide options to override this default behavior for +// repeated and message fields. +// +// In order to reset a field's value to the default, the field must +// be in the mask and set to the default value in the provided resource. +// Hence, in order to reset all fields of a resource, provide a default +// instance of the resource and set all fields in the mask, or do +// not provide a mask as described below. +// +// If a field mask is not present on update, the operation applies to +// all fields (as if a field mask of all fields has been specified). +// Note that in the presence of schema evolution, this may mean that +// fields the client does not know and has therefore not filled into +// the request will be reset to their default. If this is unwanted +// behavior, a specific service may require a client to always specify +// a field mask, producing an error if not. +// +// As with get operations, the location of the resource which +// describes the updated values in the request message depends on the +// operation kind. In any case, the effect of the field mask is +// required to be honored by the API. +// +// ## Considerations for HTTP REST +// +// The HTTP kind of an update operation which uses a field mask must +// be set to PATCH instead of PUT in order to satisfy HTTP semantics +// (PUT must only be used for full updates). +// +// # JSON Encoding of Field Masks +// +// In JSON, a field mask is encoded as a single string where paths are +// separated by a comma. Fields name in each path are converted +// to/from lower-camel naming conventions. +// +// As an example, consider the following message declarations: +// +// message Profile { +// User user = 1; +// Photo photo = 2; +// } +// message User { +// string display_name = 1; +// string address = 2; +// } +// +// In proto a field mask for `Profile` may look as such: +// +// mask { +// paths: "user.display_name" +// paths: "photo" +// } +// +// In JSON, the same mask is represented as below: +// +// { +// mask: "user.displayName,photo" +// } +// +// # Field Masks and Oneof Fields +// +// Field masks treat fields in oneofs just as regular fields. Consider the +// following message: +// +// message SampleMessage { +// oneof test_oneof { +// string name = 4; +// SubMessage sub_message = 9; +// } +// } +// +// The field mask can be: +// +// mask { +// paths: "name" +// } +// +// Or: +// +// mask { +// paths: "sub_message" +// } +// +// Note that oneof type names ("test_oneof" in this case) cannot be used in +// paths. +// +// ## Field Mask Verification +// +// The implementation of any API method which has a FieldMask type field in the +// request should verify the included field paths, and return an +// `INVALID_ARGUMENT` error if any path is unmappable. +message FieldMask { + // The set of field mask paths. + repeated string paths = 1; +} diff --git a/karapace/protobuf/google/protobuf/map_lite_unittest.proto b/karapace/protobuf/google/protobuf/map_lite_unittest.proto new file mode 100644 index 000000000..7f104315c --- /dev/null +++ b/karapace/protobuf/google/protobuf/map_lite_unittest.proto @@ -0,0 +1,131 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto2"; + +package protobuf_unittest; + +import "google/protobuf/unittest_lite.proto"; + +option cc_enable_arenas = true; +option optimize_for = LITE_RUNTIME; + +message TestMapLite { + map map_int32_int32 = 1; + map map_int64_int64 = 2; + map map_uint32_uint32 = 3; + map map_uint64_uint64 = 4; + map map_sint32_sint32 = 5; + map map_sint64_sint64 = 6; + map map_fixed32_fixed32 = 7; + map map_fixed64_fixed64 = 8; + map map_sfixed32_sfixed32 = 9; + map map_sfixed64_sfixed64 = 10; + map map_int32_float = 11; + map map_int32_double = 12; + map map_bool_bool = 13; + map map_string_string = 14; + map map_int32_bytes = 15; + map map_int32_enum = 16; + map map_int32_foreign_message = 17; + map teboring = 18; +} + +message TestArenaMapLite { + map map_int32_int32 = 1; + map map_int64_int64 = 2; + map map_uint32_uint32 = 3; + map map_uint64_uint64 = 4; + map map_sint32_sint32 = 5; + map map_sint64_sint64 = 6; + map map_fixed32_fixed32 = 7; + map map_fixed64_fixed64 = 8; + map map_sfixed32_sfixed32 = 9; + map map_sfixed64_sfixed64 = 10; + map map_int32_float = 11; + map map_int32_double = 12; + map map_bool_bool = 13; + map map_string_string = 14; + map map_int32_bytes = 15; + map map_int32_enum = 16; + map map_int32_foreign_message = 17; +} + +// Test embedded message with required fields +message TestRequiredMessageMapLite { + map map_field = 1; +} + +message TestEnumMapLite { + map known_map_field = 101; + map unknown_map_field = 102; +} + +message TestEnumMapPlusExtraLite { + map known_map_field = 101; + map unknown_map_field = 102; +} + +message TestMessageMapLite { + map map_int32_message = 1; +} + +enum Proto2MapEnumLite { + PROTO2_MAP_ENUM_FOO_LITE = 0; + PROTO2_MAP_ENUM_BAR_LITE = 1; + PROTO2_MAP_ENUM_BAZ_LITE = 2; +} + +enum Proto2MapEnumPlusExtraLite { + E_PROTO2_MAP_ENUM_FOO_LITE = 0; + E_PROTO2_MAP_ENUM_BAR_LITE = 1; + E_PROTO2_MAP_ENUM_BAZ_LITE = 2; + E_PROTO2_MAP_ENUM_EXTRA_LITE = 3; +} + +enum MapEnumLite { + MAP_ENUM_FOO_LITE = 0; + MAP_ENUM_BAR_LITE = 1; + MAP_ENUM_BAZ_LITE = 2; +} + +message TestRequiredLite { + required int32 a = 1; + required int32 b = 2; + required int32 c = 3; + + extend TestAllExtensionsLite { + optional TestRequiredLite single = 1000; + } +} + +message ForeignMessageArenaLite { + optional int32 c = 1; +} diff --git a/karapace/protobuf/google/protobuf/map_proto2_unittest.proto b/karapace/protobuf/google/protobuf/map_proto2_unittest.proto new file mode 100644 index 000000000..20d58f903 --- /dev/null +++ b/karapace/protobuf/google/protobuf/map_proto2_unittest.proto @@ -0,0 +1,91 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto2"; +option cc_enable_arenas = true; + +import "google/protobuf/unittest_import.proto"; + +// We don't put this in a package within proto2 because we need to make sure +// that the generated code doesn't depend on being in the proto2 namespace. +// In map_test_util.h we do "using namespace unittest = protobuf_unittest". +package protobuf_unittest; + +enum Proto2MapEnum { + PROTO2_MAP_ENUM_FOO = 0; + PROTO2_MAP_ENUM_BAR = 1; + PROTO2_MAP_ENUM_BAZ = 2; +} + +enum Proto2MapEnumPlusExtra { + E_PROTO2_MAP_ENUM_FOO = 0; + E_PROTO2_MAP_ENUM_BAR = 1; + E_PROTO2_MAP_ENUM_BAZ = 2; + E_PROTO2_MAP_ENUM_EXTRA = 3; +} + +message TestEnumMap { + map known_map_field = 101; + map unknown_map_field = 102; +} + +message TestEnumMapPlusExtra { + map known_map_field = 101; + map unknown_map_field = 102; +} + +message TestImportEnumMap { + map import_enum_amp = 1; +} + +message TestIntIntMap { + map m = 1; +} + +// Test all key types: string, plus the non-floating-point scalars. +message TestMaps { + map m_int32 = 1; + map m_int64 = 2; + map m_uint32 = 3; + map m_uint64 = 4; + map m_sint32 = 5; + map m_sint64 = 6; + map m_fixed32 = 7; + map m_fixed64 = 8; + map m_sfixed32 = 9; + map m_sfixed64 = 10; + map m_bool = 11; + map m_string = 12; +} + +// Test maps in submessages. +message TestSubmessageMaps { + optional TestMaps m = 1; +} diff --git a/karapace/protobuf/google/protobuf/map_unittest.proto b/karapace/protobuf/google/protobuf/map_unittest.proto new file mode 100644 index 000000000..263ef61f8 --- /dev/null +++ b/karapace/protobuf/google/protobuf/map_unittest.proto @@ -0,0 +1,125 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +option cc_enable_arenas = true; + +import "google/protobuf/unittest.proto"; + +// We don't put this in a package within proto2 because we need to make sure +// that the generated code doesn't depend on being in the proto2 namespace. +// In map_test_util.h we do "using namespace unittest = protobuf_unittest". +package protobuf_unittest; + +// Tests maps. +message TestMap { + map map_int32_int32 = 1; + map map_int64_int64 = 2; + map map_uint32_uint32 = 3; + map map_uint64_uint64 = 4; + map map_sint32_sint32 = 5; + map map_sint64_sint64 = 6; + map map_fixed32_fixed32 = 7; + map map_fixed64_fixed64 = 8; + map map_sfixed32_sfixed32 = 9; + map map_sfixed64_sfixed64 = 10; + map map_int32_float = 11; + map map_int32_double = 12; + map map_bool_bool = 13; + map map_string_string = 14; + map map_int32_bytes = 15; + map map_int32_enum = 16; + map map_int32_foreign_message = 17; + map map_string_foreign_message = 18; + map map_int32_all_types = 19; +} + +message TestMapSubmessage { + TestMap test_map = 1; +} + +message TestMessageMap { + map map_int32_message = 1; +} + +// Two map fields share the same entry default instance. +message TestSameTypeMap { + map map1 = 1; + map map2 = 2; +} + + +enum MapEnum { + MAP_ENUM_FOO = 0; + MAP_ENUM_BAR = 1; + MAP_ENUM_BAZ = 2; +} + +// Test embedded message with required fields +message TestRequiredMessageMap { + map map_field = 1; +} + +message TestArenaMap { + map map_int32_int32 = 1; + map map_int64_int64 = 2; + map map_uint32_uint32 = 3; + map map_uint64_uint64 = 4; + map map_sint32_sint32 = 5; + map map_sint64_sint64 = 6; + map map_fixed32_fixed32 = 7; + map map_fixed64_fixed64 = 8; + map map_sfixed32_sfixed32 = 9; + map map_sfixed64_sfixed64 = 10; + map map_int32_float = 11; + map map_int32_double = 12; + map map_bool_bool = 13; + map map_string_string = 14; + map map_int32_bytes = 15; + map map_int32_enum = 16; + map map_int32_foreign_message = 17; +} + +// Previously, message containing enum called Type cannot be used as value of +// map field. +message MessageContainingEnumCalledType { + enum Type { TYPE_FOO = 0; } + map type = 1; +} + +// Previously, message cannot contain map field called "entry". +message MessageContainingMapCalledEntry { + map entry = 1; +} + +message TestRecursiveMapMessage { + map a = 1; +} diff --git a/karapace/protobuf/google/protobuf/source_context.proto b/karapace/protobuf/google/protobuf/source_context.proto new file mode 100644 index 000000000..06bfc43a7 --- /dev/null +++ b/karapace/protobuf/google/protobuf/source_context.proto @@ -0,0 +1,48 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package google.protobuf; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "SourceContextProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; +option go_package = "google.golang.org/protobuf/types/known/sourcecontextpb"; + +// `SourceContext` represents information about the source of a +// protobuf element, like the file in which it is defined. +message SourceContext { + // The path-qualified name of the .proto file that contained the associated + // protobuf element. For example: `"google/protobuf/source_context.proto"`. + string file_name = 1; +} diff --git a/karapace/protobuf/google/protobuf/struct.proto b/karapace/protobuf/google/protobuf/struct.proto new file mode 100644 index 000000000..0ac843ca0 --- /dev/null +++ b/karapace/protobuf/google/protobuf/struct.proto @@ -0,0 +1,95 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package google.protobuf; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option cc_enable_arenas = true; +option go_package = "google.golang.org/protobuf/types/known/structpb"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "StructProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; + +// `Struct` represents a structured data value, consisting of fields +// which map to dynamically typed values. In some languages, `Struct` +// might be supported by a native representation. For example, in +// scripting languages like JS a struct is represented as an +// object. The details of that representation are described together +// with the proto support for the language. +// +// The JSON representation for `Struct` is JSON object. +message Struct { + // Unordered map of dynamically typed values. + map fields = 1; +} + +// `Value` represents a dynamically typed value which can be either +// null, a number, a string, a boolean, a recursive struct value, or a +// list of values. A producer of value is expected to set one of these +// variants. Absence of any variant indicates an error. +// +// The JSON representation for `Value` is JSON value. +message Value { + // The kind of value. + oneof kind { + // Represents a null value. + NullValue null_value = 1; + // Represents a double value. + double number_value = 2; + // Represents a string value. + string string_value = 3; + // Represents a boolean value. + bool bool_value = 4; + // Represents a structured value. + Struct struct_value = 5; + // Represents a repeated `Value`. + ListValue list_value = 6; + } +} + +// `NullValue` is a singleton enumeration to represent the null value for the +// `Value` type union. +// +// The JSON representation for `NullValue` is JSON `null`. +enum NullValue { + // Null value. + NULL_VALUE = 0; +} + +// `ListValue` is a wrapper around a repeated field of values. +// +// The JSON representation for `ListValue` is JSON array. +message ListValue { + // Repeated field of dynamically typed values. + repeated Value values = 1; +} diff --git a/karapace/protobuf/google/protobuf/timestamp.proto b/karapace/protobuf/google/protobuf/timestamp.proto new file mode 100644 index 000000000..3b2df6d91 --- /dev/null +++ b/karapace/protobuf/google/protobuf/timestamp.proto @@ -0,0 +1,147 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package google.protobuf; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option cc_enable_arenas = true; +option go_package = "google.golang.org/protobuf/types/known/timestamppb"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "TimestampProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; + +// A Timestamp represents a point in time independent of any time zone or local +// calendar, encoded as a count of seconds and fractions of seconds at +// nanosecond resolution. The count is relative to an epoch at UTC midnight on +// January 1, 1970, in the proleptic Gregorian calendar which extends the +// Gregorian calendar backwards to year one. +// +// All minutes are 60 seconds long. Leap seconds are "smeared" so that no leap +// second table is needed for interpretation, using a [24-hour linear +// smear](https://developers.google.com/time/smear). +// +// The range is from 0001-01-01T00:00:00Z to 9999-12-31T23:59:59.999999999Z. By +// restricting to that range, we ensure that we can convert to and from [RFC +// 3339](https://www.ietf.org/rfc/rfc3339.txt) date strings. +// +// # Examples +// +// Example 1: Compute Timestamp from POSIX `time()`. +// +// Timestamp timestamp; +// timestamp.set_seconds(time(NULL)); +// timestamp.set_nanos(0); +// +// Example 2: Compute Timestamp from POSIX `gettimeofday()`. +// +// struct timeval tv; +// gettimeofday(&tv, NULL); +// +// Timestamp timestamp; +// timestamp.set_seconds(tv.tv_sec); +// timestamp.set_nanos(tv.tv_usec * 1000); +// +// Example 3: Compute Timestamp from Win32 `GetSystemTimeAsFileTime()`. +// +// FILETIME ft; +// GetSystemTimeAsFileTime(&ft); +// UINT64 ticks = (((UINT64)ft.dwHighDateTime) << 32) | ft.dwLowDateTime; +// +// // A Windows tick is 100 nanoseconds. Windows epoch 1601-01-01T00:00:00Z +// // is 11644473600 seconds before Unix epoch 1970-01-01T00:00:00Z. +// Timestamp timestamp; +// timestamp.set_seconds((INT64) ((ticks / 10000000) - 11644473600LL)); +// timestamp.set_nanos((INT32) ((ticks % 10000000) * 100)); +// +// Example 4: Compute Timestamp from Java `System.currentTimeMillis()`. +// +// long millis = System.currentTimeMillis(); +// +// Timestamp timestamp = Timestamp.newBuilder().setSeconds(millis / 1000) +// .setNanos((int) ((millis % 1000) * 1000000)).build(); +// +// +// Example 5: Compute Timestamp from Java `Instant.now()`. +// +// Instant now = Instant.now(); +// +// Timestamp timestamp = +// Timestamp.newBuilder().setSeconds(now.getEpochSecond()) +// .setNanos(now.getNano()).build(); +// +// +// Example 6: Compute Timestamp from current time in Python. +// +// timestamp = Timestamp() +// timestamp.GetCurrentTime() +// +// # JSON Mapping +// +// In JSON format, the Timestamp type is encoded as a string in the +// [RFC 3339](https://www.ietf.org/rfc/rfc3339.txt) format. That is, the +// format is "{year}-{month}-{day}T{hour}:{min}:{sec}[.{frac_sec}]Z" +// where {year} is always expressed using four digits while {month}, {day}, +// {hour}, {min}, and {sec} are zero-padded to two digits each. The fractional +// seconds, which can go up to 9 digits (i.e. up to 1 nanosecond resolution), +// are optional. The "Z" suffix indicates the timezone ("UTC"); the timezone +// is required. A proto3 JSON serializer should always use UTC (as indicated by +// "Z") when printing the Timestamp type and a proto3 JSON parser should be +// able to accept both UTC and other timezones (as indicated by an offset). +// +// For example, "2017-01-15T01:30:15.01Z" encodes 15.01 seconds past +// 01:30 UTC on January 15, 2017. +// +// In JavaScript, one can convert a Date object to this format using the +// standard +// [toISOString()](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Date/toISOString) +// method. In Python, a standard `datetime.datetime` object can be converted +// to this format using +// [`strftime`](https://docs.python.org/2/library/time.html#time.strftime) with +// the time format spec '%Y-%m-%dT%H:%M:%S.%fZ'. Likewise, in Java, one can use +// the Joda Time's [`ISODateTimeFormat.dateTime()`]( +// http://www.joda.org/joda-time/apidocs/org/joda/time/format/ISODateTimeFormat.html#dateTime%2D%2D +// ) to obtain a formatter capable of generating timestamps in this format. +// +// +message Timestamp { + // Represents seconds of UTC time since Unix epoch + // 1970-01-01T00:00:00Z. Must be from 0001-01-01T00:00:00Z to + // 9999-12-31T23:59:59Z inclusive. + int64 seconds = 1; + + // Non-negative fractions of a second at nanosecond resolution. Negative + // second values with fractions must still have non-negative nanos values + // that count forward in time. Must be from 0 to 999,999,999 + // inclusive. + int32 nanos = 2; +} diff --git a/karapace/protobuf/google/protobuf/type.proto b/karapace/protobuf/google/protobuf/type.proto new file mode 100644 index 000000000..d3f6a68b8 --- /dev/null +++ b/karapace/protobuf/google/protobuf/type.proto @@ -0,0 +1,187 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package google.protobuf; + +import "google/protobuf/any.proto"; +import "google/protobuf/source_context.proto"; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option cc_enable_arenas = true; +option java_package = "com.google.protobuf"; +option java_outer_classname = "TypeProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; +option go_package = "google.golang.org/protobuf/types/known/typepb"; + +// A protocol buffer message type. +message Type { + // The fully qualified message name. + string name = 1; + // The list of fields. + repeated Field fields = 2; + // The list of types appearing in `oneof` definitions in this type. + repeated string oneofs = 3; + // The protocol buffer options. + repeated Option options = 4; + // The source context. + SourceContext source_context = 5; + // The source syntax. + Syntax syntax = 6; +} + +// A single field of a message type. +message Field { + // Basic field types. + enum Kind { + // Field type unknown. + TYPE_UNKNOWN = 0; + // Field type double. + TYPE_DOUBLE = 1; + // Field type float. + TYPE_FLOAT = 2; + // Field type int64. + TYPE_INT64 = 3; + // Field type uint64. + TYPE_UINT64 = 4; + // Field type int32. + TYPE_INT32 = 5; + // Field type fixed64. + TYPE_FIXED64 = 6; + // Field type fixed32. + TYPE_FIXED32 = 7; + // Field type bool. + TYPE_BOOL = 8; + // Field type string. + TYPE_STRING = 9; + // Field type group. Proto2 syntax only, and deprecated. + TYPE_GROUP = 10; + // Field type message. + TYPE_MESSAGE = 11; + // Field type bytes. + TYPE_BYTES = 12; + // Field type uint32. + TYPE_UINT32 = 13; + // Field type enum. + TYPE_ENUM = 14; + // Field type sfixed32. + TYPE_SFIXED32 = 15; + // Field type sfixed64. + TYPE_SFIXED64 = 16; + // Field type sint32. + TYPE_SINT32 = 17; + // Field type sint64. + TYPE_SINT64 = 18; + } + + // Whether a field is optional, required, or repeated. + enum Cardinality { + // For fields with unknown cardinality. + CARDINALITY_UNKNOWN = 0; + // For optional fields. + CARDINALITY_OPTIONAL = 1; + // For required fields. Proto2 syntax only. + CARDINALITY_REQUIRED = 2; + // For repeated fields. + CARDINALITY_REPEATED = 3; + } + + // The field type. + Kind kind = 1; + // The field cardinality. + Cardinality cardinality = 2; + // The field number. + int32 number = 3; + // The field name. + string name = 4; + // The field type URL, without the scheme, for message or enumeration + // types. Example: `"type.googleapis.com/google.protobuf.Timestamp"`. + string type_url = 6; + // The index of the field type in `Type.oneofs`, for message or enumeration + // types. The first type has index 1; zero means the type is not in the list. + int32 oneof_index = 7; + // Whether to use alternative packed wire representation. + bool packed = 8; + // The protocol buffer options. + repeated Option options = 9; + // The field JSON name. + string json_name = 10; + // The string value of the default value of this field. Proto2 syntax only. + string default_value = 11; +} + +// Enum type definition. +message Enum { + // Enum type name. + string name = 1; + // Enum value definitions. + repeated EnumValue enumvalue = 2; + // Protocol buffer options. + repeated Option options = 3; + // The source context. + SourceContext source_context = 4; + // The source syntax. + Syntax syntax = 5; +} + +// Enum value definition. +message EnumValue { + // Enum value name. + string name = 1; + // Enum value number. + int32 number = 2; + // Protocol buffer options. + repeated Option options = 3; +} + +// A protocol buffer option, which can be attached to a message, field, +// enumeration, etc. +message Option { + // The option's name. For protobuf built-in options (options defined in + // descriptor.proto), this is the short name. For example, `"map_entry"`. + // For custom options, it should be the fully-qualified name. For example, + // `"google.api.http"`. + string name = 1; + // The option's value packed in an Any message. If the value is a primitive, + // the corresponding wrapper type defined in google/protobuf/wrappers.proto + // should be used. If the value is an enum, it should be stored as an int32 + // value using the google.protobuf.Int32Value type. + Any value = 2; +} + +// The syntax in which a protocol buffer element is defined. +enum Syntax { + // Syntax `proto2`. + SYNTAX_PROTO2 = 0; + // Syntax `proto3`. + SYNTAX_PROTO3 = 1; +} diff --git a/karapace/protobuf/google/protobuf/wrappers.proto b/karapace/protobuf/google/protobuf/wrappers.proto new file mode 100644 index 000000000..d49dd53c8 --- /dev/null +++ b/karapace/protobuf/google/protobuf/wrappers.proto @@ -0,0 +1,123 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Wrappers for primitive (non-message) types. These types are useful +// for embedding primitives in the `google.protobuf.Any` type and for places +// where we need to distinguish between the absence of a primitive +// typed field and its default value. +// +// These wrappers have no meaningful use within repeated fields as they lack +// the ability to detect presence on individual elements. +// These wrappers have no meaningful use within a map or a oneof since +// individual entries of a map or fields of a oneof can already detect presence. + +syntax = "proto3"; + +package google.protobuf; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option cc_enable_arenas = true; +option go_package = "google.golang.org/protobuf/types/known/wrapperspb"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "WrappersProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; + +// Wrapper message for `double`. +// +// The JSON representation for `DoubleValue` is JSON number. +message DoubleValue { + // The double value. + double value = 1; +} + +// Wrapper message for `float`. +// +// The JSON representation for `FloatValue` is JSON number. +message FloatValue { + // The float value. + float value = 1; +} + +// Wrapper message for `int64`. +// +// The JSON representation for `Int64Value` is JSON string. +message Int64Value { + // The int64 value. + int64 value = 1; +} + +// Wrapper message for `uint64`. +// +// The JSON representation for `UInt64Value` is JSON string. +message UInt64Value { + // The uint64 value. + uint64 value = 1; +} + +// Wrapper message for `int32`. +// +// The JSON representation for `Int32Value` is JSON number. +message Int32Value { + // The int32 value. + int32 value = 1; +} + +// Wrapper message for `uint32`. +// +// The JSON representation for `UInt32Value` is JSON number. +message UInt32Value { + // The uint32 value. + uint32 value = 1; +} + +// Wrapper message for `bool`. +// +// The JSON representation for `BoolValue` is JSON `true` and `false`. +message BoolValue { + // The bool value. + bool value = 1; +} + +// Wrapper message for `string`. +// +// The JSON representation for `StringValue` is JSON string. +message StringValue { + // The string value. + string value = 1; +} + +// Wrapper message for `bytes`. +// +// The JSON representation for `BytesValue` is JSON string. +message BytesValue { + // The bytes value. + bytes value = 1; +} diff --git a/karapace/protobuf/google/type/BUILD.bazel b/karapace/protobuf/google/type/BUILD.bazel new file mode 100644 index 000000000..a7ad4d4d2 --- /dev/null +++ b/karapace/protobuf/google/type/BUILD.bazel @@ -0,0 +1,536 @@ +load("@rules_proto//proto:defs.bzl", "proto_library") + +# This is an API workspace, having public visibility by default makes perfect sense. +package(default_visibility = ["//visibility:public"]) + +############################################################################## +# Common +############################################################################## +proto_library( + name = "calendar_period_proto", + srcs = ["calendar_period.proto"], +) + +proto_library( + name = "color_proto", + srcs = ["color.proto"], + deps = [ + "@com_google_protobuf//:wrappers_proto", + ], +) + +proto_library( + name = "date_proto", + srcs = ["date.proto"], +) + +proto_library( + name = "datetime_proto", + srcs = ["datetime.proto"], + deps = [ + "@com_google_protobuf//:duration_proto", + ], +) + +proto_library( + name = "dayofweek_proto", + srcs = ["dayofweek.proto"], +) + +proto_library( + name = "decimal_proto", + srcs = ["decimal.proto"], +) + +proto_library( + name = "expr_proto", + srcs = ["expr.proto"], +) + +proto_library( + name = "fraction_proto", + srcs = ["fraction.proto"], +) + +proto_library( + name = "interval_proto", + srcs = ["interval.proto"], + deps = [ + "@com_google_protobuf//:timestamp_proto", + ], +) + +proto_library( + name = "latlng_proto", + srcs = ["latlng.proto"], +) + +proto_library( + name = "localized_text_proto", + srcs = ["localized_text.proto"], +) + +proto_library( + name = "money_proto", + srcs = ["money.proto"], +) + +proto_library( + name = "month_proto", + srcs = ["month.proto"], +) + +proto_library( + name = "phone_number_proto", + srcs = ["phone_number.proto"], +) + +proto_library( + name = "postal_address_proto", + srcs = ["postal_address.proto"], +) + +proto_library( + name = "quaternion_proto", + srcs = ["quaternion.proto"], +) + +proto_library( + name = "timeofday_proto", + srcs = ["timeofday.proto"], +) + +############################################################################## +# Java +############################################################################## +load("@com_google_googleapis_imports//:imports.bzl", + "java_gapic_assembly_gradle_pkg", + "java_proto_library") + +java_proto_library( + name = "type_java_proto", + deps = [ + ":calendar_period_proto", + ":color_proto", + ":date_proto", + ":datetime_proto", + ":dayofweek_proto", + ":decimal_proto", + ":expr_proto", + ":fraction_proto", + ":interval_proto", + ":latlng_proto", + ":localized_text_proto", + ":money_proto", + ":month_proto", + ":phone_number_proto", + ":postal_address_proto", + ":quaternion_proto", + ":timeofday_proto", + ], +) + +# Please DO-NOT-REMOVE this section. +# This is required to generate java files for these protos. +# Open Source Packages +java_gapic_assembly_gradle_pkg( + name = "google-type-java", + deps = [ + ":type_java_proto", + ":calendar_period_proto", + ":color_proto", + ":date_proto", + ":datetime_proto", + ":dayofweek_proto", + ":decimal_proto", + ":expr_proto", + ":fraction_proto", + ":interval_proto", + ":latlng_proto", + ":localized_text_proto", + ":money_proto", + ":month_proto", + ":phone_number_proto", + ":postal_address_proto", + ":quaternion_proto", + ":timeofday_proto", + ], +) + + +############################################################################## +# Go +############################################################################## +load("@com_google_googleapis_imports//:imports.bzl", "go_proto_library") + +go_proto_library( + name = "calendar_period_go_proto", + importpath = "google.golang.org/genproto/googleapis/type/calendarperiod", + protos = [":calendar_period_proto"], +) + +go_proto_library( + name = "color_go_proto", + importpath = "google.golang.org/genproto/googleapis/type/color", + protos = [":color_proto"], +) + +go_proto_library( + name = "date_go_proto", + importpath = "google.golang.org/genproto/googleapis/type/date", + protos = [":date_proto"], +) + +go_proto_library( + name = "datetime_go_proto", + importpath = "google.golang.org/genproto/googleapis/type/datetime", + protos = [":datetime_proto"], +) + +go_proto_library( + name = "dayofweek_go_proto", + importpath = "google.golang.org/genproto/googleapis/type/dayofweek", + protos = [":dayofweek_proto"], +) + +go_proto_library( + name = "decimal_go_proto", + importpath = "google.golang.org/genproto/googleapis/type/decimal", + protos = [":decimal_proto"], +) + +go_proto_library( + name = "expr_go_proto", + importpath = "google.golang.org/genproto/googleapis/type/expr", + protos = [":expr_proto"], +) + +go_proto_library( + name = "fraction_go_proto", + importpath = "google.golang.org/genproto/googleapis/type/fraction", + protos = [":fraction_proto"], +) + +go_proto_library( + name = "interval_go_proto", + importpath = "google.golang.org/genproto/googleapis/type/interval", + protos = [":interval_proto"], +) + +go_proto_library( + name = "latlng_go_proto", + importpath = "google.golang.org/genproto/googleapis/type/latlng", + protos = [":latlng_proto"], +) + +go_proto_library( + name = "localized_text_go_proto", + importpath = "google.golang.org/genproto/googleapis/type/localized_text", + protos = [":localized_text_proto"], +) + +go_proto_library( + name = "money_go_proto", + importpath = "google.golang.org/genproto/googleapis/type/money", + protos = [":money_proto"], +) + +go_proto_library( + name = "month_go_proto", + importpath = "google.golang.org/genproto/googleapis/type/month", + protos = [":month_proto"], +) + +go_proto_library( + name = "phone_number_go_proto", + importpath = "google.golang.org/genproto/googleapis/type/phone_number", + protos = [":phone_number_proto"], +) + +go_proto_library( + name = "postaladdress_go_proto", + importpath = "google.golang.org/genproto/googleapis/type/postaladdress", + protos = [":postal_address_proto"], +) + +go_proto_library( + name = "quaternion_go_proto", + importpath = "google.golang.org/genproto/googleapis/type/quaternion", + protos = [":quaternion_proto"], +) + +go_proto_library( + name = "timeofday_go_proto", + importpath = "google.golang.org/genproto/googleapis/type/timeofday", + protos = [":timeofday_proto"], +) + +############################################################################## +# C++ +############################################################################## +load( + "@com_google_googleapis_imports//:imports.bzl", + "cc_proto_library", +) + +cc_proto_library( + name = "calendar_period_cc_proto", + deps = [":calendar_period_proto"], +) + +cc_proto_library( + name = "color_cc_proto", + deps = [":color_proto"], +) + +cc_proto_library( + name = "date_cc_proto", + deps = [":date_proto"], +) + +cc_proto_library( + name = "datetime_cc_proto", + deps = [":datetime_proto"], +) + +cc_proto_library( + name = "dayofweek_cc_proto", + deps = [":dayofweek_proto"], +) + +cc_proto_library( + name = "decimal_cc_proto", + deps = [":decimal_proto"], +) + +cc_proto_library( + name = "expr_cc_proto", + deps = [":expr_proto"], +) + +cc_proto_library( + name = "fraction_cc_proto", + deps = [":fraction_proto"], +) + +cc_proto_library( + name = "interval_cc_proto", + deps = [":interval_proto"], +) + +cc_proto_library( + name = "latlng_cc_proto", + deps = [":latlng_proto"], +) + +cc_proto_library( + name = "money_cc_proto", + deps = [":money_proto"], +) + +cc_proto_library( + name = "month_cc_proto", + deps = [":month_proto"], +) + +cc_proto_library( + name = "phone_number_cc_proto", + deps = [":phone_number_proto"], +) + +cc_proto_library( + name = "postal_address_cc_proto", + deps = [":postal_address_proto"], +) + +cc_proto_library( + name = "quaternion_cc_proto", + deps = [":quaternion_proto"], +) + +cc_proto_library( + name = "timeofday_cc_proto", + deps = [":timeofday_proto"], +) + +############################################################################## +# Python +############################################################################## +load( + "@com_google_googleapis_imports//:imports.bzl", + "py_proto_library", +) + +py_proto_library( + name = "calendar_period_py_proto", + deps = [":calendar_period_proto"], +) + +py_proto_library( + name = "color_py_proto", + deps = [":color_proto"], +) + +py_proto_library( + name = "date_py_proto", + deps = [":date_proto"], +) + +py_proto_library( + name = "datetime_py_proto", + deps = [":datetime_proto"], +) + +py_proto_library( + name = "dayofweek_py_proto", + deps = [":dayofweek_proto"], +) + +py_proto_library( + name = "decimal_py_proto", + deps = [":decimal_proto"], +) + +py_proto_library( + name = "expr_py_proto", + deps = [":expr_proto"], +) + +py_proto_library( + name = "fraction_py_proto", + deps = [":fraction_proto"], +) + +py_proto_library( + name = "interval_py_proto", + deps = [":interval_proto"], +) + +py_proto_library( + name = "latlng_py_proto", + deps = [":latlng_proto"], +) + +py_proto_library( + name = "localized_text_py_proto", + deps = [":localized_text_proto"], +) + +py_proto_library( + name = "money_py_proto", + deps = [":money_proto"], +) + +py_proto_library( + name = "month_py_proto", + deps = [":month_proto"], +) + +py_proto_library( + name = "phone_number_py_proto", + deps = [":phone_number_proto"], +) + +py_proto_library( + name = "postal_address_py_proto", + deps = [":postal_address_proto"], +) + +py_proto_library( + name = "quaternion_py_proto", + deps = [":quaternion_proto"], +) + +py_proto_library( + name = "timeofday_py_proto", + deps = [":timeofday_proto"], +) + +############################################################################## +# C# +############################################################################## + +load( + "@com_google_googleapis_imports//:imports.bzl", + "csharp_proto_library", +) + +csharp_proto_library( + name = "calendar_period_csharp_proto", + deps = [":calendar_period_proto"], +) + +csharp_proto_library( + name = "color_csharp_proto", + deps = [":color_proto"], +) + +csharp_proto_library( + name = "date_csharp_proto", + deps = [":date_proto"], +) + +csharp_proto_library( + name = "datetime_csharp_proto", + deps = [":datetime_proto"], +) + +csharp_proto_library( + name = "dayofweek_csharp_proto", + deps = [":dayofweek_proto"], +) + +csharp_proto_library( + name = "decimal_csharp_proto", + deps = [":decimal_proto"], +) + +csharp_proto_library( + name = "expr_csharp_proto", + deps = [":expr_proto"], +) + +csharp_proto_library( + name = "fraction_csharp_proto", + deps = [":fraction_proto"], +) + +csharp_proto_library( + name = "interval_csharp_proto", + deps = [":interval_proto"], +) + +csharp_proto_library( + name = "latlng_csharp_proto", + deps = [":latlng_proto"], +) + +csharp_proto_library( + name = "localized_text_csharp_proto", + deps = [":localized_text_proto"], +) + +csharp_proto_library( + name = "money_csharp_proto", + deps = [":money_proto"], +) + +csharp_proto_library( + name = "month_csharp_proto", + deps = [":month_proto"], +) + +csharp_proto_library( + name = "phone_number_csharp_proto", + deps = [":phone_number_proto"], +) + +csharp_proto_library( + name = "postal_address_csharp_proto", + deps = [":postal_address_proto"], +) + +csharp_proto_library( + name = "quaternion_csharp_proto", + deps = [":quaternion_proto"], +) diff --git a/karapace/protobuf/google/type/README.md b/karapace/protobuf/google/type/README.md new file mode 100644 index 000000000..de6a835d7 --- /dev/null +++ b/karapace/protobuf/google/type/README.md @@ -0,0 +1,16 @@ +# Google Common Types + +This package contains definitions of common types for Google APIs. +All types defined in this package are suitable for different APIs to +exchange data, and will never break binary compatibility. They should +have design quality comparable to major programming languages like +Java and C#. + +NOTE: Some common types are defined in the package `google.protobuf` +as they are directly supported by Protocol Buffers compiler and +runtime. Those types are called Well-Known Types. + +## Java Utilities + +A set of Java utilities for the Common Types are provided in the +`//java/com/google/type/util/` package. diff --git a/karapace/protobuf/google/type/calendar_period.proto b/karapace/protobuf/google/type/calendar_period.proto new file mode 100644 index 000000000..82f5690b7 --- /dev/null +++ b/karapace/protobuf/google/type/calendar_period.proto @@ -0,0 +1,56 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option go_package = "google.golang.org/genproto/googleapis/type/calendarperiod;calendarperiod"; +option java_multiple_files = true; +option java_outer_classname = "CalendarPeriodProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// A `CalendarPeriod` represents the abstract concept of a time period that has +// a canonical start. Grammatically, "the start of the current +// `CalendarPeriod`." All calendar times begin at midnight UTC. +enum CalendarPeriod { + // Undefined period, raises an error. + CALENDAR_PERIOD_UNSPECIFIED = 0; + + // A day. + DAY = 1; + + // A week. Weeks begin on Monday, following + // [ISO 8601](https://en.wikipedia.org/wiki/ISO_week_date). + WEEK = 2; + + // A fortnight. The first calendar fortnight of the year begins at the start + // of week 1 according to + // [ISO 8601](https://en.wikipedia.org/wiki/ISO_week_date). + FORTNIGHT = 3; + + // A month. + MONTH = 4; + + // A quarter. Quarters start on dates 1-Jan, 1-Apr, 1-Jul, and 1-Oct of each + // year. + QUARTER = 5; + + // A half-year. Half-years start on dates 1-Jan and 1-Jul. + HALF = 6; + + // A year. + YEAR = 7; +} diff --git a/karapace/protobuf/google/type/color.proto b/karapace/protobuf/google/type/color.proto new file mode 100644 index 000000000..5dc85a6a3 --- /dev/null +++ b/karapace/protobuf/google/type/color.proto @@ -0,0 +1,174 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +import "google/protobuf/wrappers.proto"; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/color;color"; +option java_multiple_files = true; +option java_outer_classname = "ColorProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a color in the RGBA color space. This representation is designed +// for simplicity of conversion to/from color representations in various +// languages over compactness. For example, the fields of this representation +// can be trivially provided to the constructor of `java.awt.Color` in Java; it +// can also be trivially provided to UIColor's `+colorWithRed:green:blue:alpha` +// method in iOS; and, with just a little work, it can be easily formatted into +// a CSS `rgba()` string in JavaScript. +// +// This reference page doesn't carry information about the absolute color +// space +// that should be used to interpret the RGB value (e.g. sRGB, Adobe RGB, +// DCI-P3, BT.2020, etc.). By default, applications should assume the sRGB color +// space. +// +// When color equality needs to be decided, implementations, unless +// documented otherwise, treat two colors as equal if all their red, +// green, blue, and alpha values each differ by at most 1e-5. +// +// Example (Java): +// +// import com.google.type.Color; +// +// // ... +// public static java.awt.Color fromProto(Color protocolor) { +// float alpha = protocolor.hasAlpha() +// ? protocolor.getAlpha().getValue() +// : 1.0; +// +// return new java.awt.Color( +// protocolor.getRed(), +// protocolor.getGreen(), +// protocolor.getBlue(), +// alpha); +// } +// +// public static Color toProto(java.awt.Color color) { +// float red = (float) color.getRed(); +// float green = (float) color.getGreen(); +// float blue = (float) color.getBlue(); +// float denominator = 255.0; +// Color.Builder resultBuilder = +// Color +// .newBuilder() +// .setRed(red / denominator) +// .setGreen(green / denominator) +// .setBlue(blue / denominator); +// int alpha = color.getAlpha(); +// if (alpha != 255) { +// result.setAlpha( +// FloatValue +// .newBuilder() +// .setValue(((float) alpha) / denominator) +// .build()); +// } +// return resultBuilder.build(); +// } +// // ... +// +// Example (iOS / Obj-C): +// +// // ... +// static UIColor* fromProto(Color* protocolor) { +// float red = [protocolor red]; +// float green = [protocolor green]; +// float blue = [protocolor blue]; +// FloatValue* alpha_wrapper = [protocolor alpha]; +// float alpha = 1.0; +// if (alpha_wrapper != nil) { +// alpha = [alpha_wrapper value]; +// } +// return [UIColor colorWithRed:red green:green blue:blue alpha:alpha]; +// } +// +// static Color* toProto(UIColor* color) { +// CGFloat red, green, blue, alpha; +// if (![color getRed:&red green:&green blue:&blue alpha:&alpha]) { +// return nil; +// } +// Color* result = [[Color alloc] init]; +// [result setRed:red]; +// [result setGreen:green]; +// [result setBlue:blue]; +// if (alpha <= 0.9999) { +// [result setAlpha:floatWrapperWithValue(alpha)]; +// } +// [result autorelease]; +// return result; +// } +// // ... +// +// Example (JavaScript): +// +// // ... +// +// var protoToCssColor = function(rgb_color) { +// var redFrac = rgb_color.red || 0.0; +// var greenFrac = rgb_color.green || 0.0; +// var blueFrac = rgb_color.blue || 0.0; +// var red = Math.floor(redFrac * 255); +// var green = Math.floor(greenFrac * 255); +// var blue = Math.floor(blueFrac * 255); +// +// if (!('alpha' in rgb_color)) { +// return rgbToCssColor(red, green, blue); +// } +// +// var alphaFrac = rgb_color.alpha.value || 0.0; +// var rgbParams = [red, green, blue].join(','); +// return ['rgba(', rgbParams, ',', alphaFrac, ')'].join(''); +// }; +// +// var rgbToCssColor = function(red, green, blue) { +// var rgbNumber = new Number((red << 16) | (green << 8) | blue); +// var hexString = rgbNumber.toString(16); +// var missingZeros = 6 - hexString.length; +// var resultBuilder = ['#']; +// for (var i = 0; i < missingZeros; i++) { +// resultBuilder.push('0'); +// } +// resultBuilder.push(hexString); +// return resultBuilder.join(''); +// }; +// +// // ... +message Color { + // The amount of red in the color as a value in the interval [0, 1]. + float red = 1; + + // The amount of green in the color as a value in the interval [0, 1]. + float green = 2; + + // The amount of blue in the color as a value in the interval [0, 1]. + float blue = 3; + + // The fraction of this color that should be applied to the pixel. That is, + // the final pixel color is defined by the equation: + // + // `pixel color = alpha * (this color) + (1.0 - alpha) * (background color)` + // + // This means that a value of 1.0 corresponds to a solid color, whereas + // a value of 0.0 corresponds to a completely transparent color. This + // uses a wrapper message rather than a simple float scalar so that it is + // possible to distinguish between a default value and the value being unset. + // If omitted, this color object is rendered as a solid color + // (as if the alpha value had been explicitly given a value of 1.0). + google.protobuf.FloatValue alpha = 4; +} diff --git a/karapace/protobuf/google/type/date.proto b/karapace/protobuf/google/type/date.proto new file mode 100644 index 000000000..e4e730e6f --- /dev/null +++ b/karapace/protobuf/google/type/date.proto @@ -0,0 +1,52 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/date;date"; +option java_multiple_files = true; +option java_outer_classname = "DateProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a whole or partial calendar date, such as a birthday. The time of +// day and time zone are either specified elsewhere or are insignificant. The +// date is relative to the Gregorian Calendar. This can represent one of the +// following: +// +// * A full date, with non-zero year, month, and day values +// * A month and day value, with a zero year, such as an anniversary +// * A year on its own, with zero month and day values +// * A year and month value, with a zero day, such as a credit card expiration +// date +// +// Related types are [google.type.TimeOfDay][google.type.TimeOfDay] and +// `google.protobuf.Timestamp`. +message Date { + // Year of the date. Must be from 1 to 9999, or 0 to specify a date without + // a year. + int32 year = 1; + + // Month of a year. Must be from 1 to 12, or 0 to specify a year without a + // month and day. + int32 month = 2; + + // Day of a month. Must be from 1 to 31 and valid for the year and month, or 0 + // to specify a year by itself or a year and month where the day isn't + // significant. + int32 day = 3; +} diff --git a/karapace/protobuf/google/type/datetime.proto b/karapace/protobuf/google/type/datetime.proto new file mode 100644 index 000000000..cfed85d70 --- /dev/null +++ b/karapace/protobuf/google/type/datetime.proto @@ -0,0 +1,104 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +import "google/protobuf/duration.proto"; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/datetime;datetime"; +option java_multiple_files = true; +option java_outer_classname = "DateTimeProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents civil time (or occasionally physical time). +// +// This type can represent a civil time in one of a few possible ways: +// +// * When utc_offset is set and time_zone is unset: a civil time on a calendar +// day with a particular offset from UTC. +// * When time_zone is set and utc_offset is unset: a civil time on a calendar +// day in a particular time zone. +// * When neither time_zone nor utc_offset is set: a civil time on a calendar +// day in local time. +// +// The date is relative to the Proleptic Gregorian Calendar. +// +// If year is 0, the DateTime is considered not to have a specific year. month +// and day must have valid, non-zero values. +// +// This type may also be used to represent a physical time if all the date and +// time fields are set and either case of the `time_offset` oneof is set. +// Consider using `Timestamp` message for physical time instead. If your use +// case also would like to store the user's timezone, that can be done in +// another field. +// +// This type is more flexible than some applications may want. Make sure to +// document and validate your application's limitations. +message DateTime { + // Optional. Year of date. Must be from 1 to 9999, or 0 if specifying a + // datetime without a year. + int32 year = 1; + + // Required. Month of year. Must be from 1 to 12. + int32 month = 2; + + // Required. Day of month. Must be from 1 to 31 and valid for the year and + // month. + int32 day = 3; + + // Required. Hours of day in 24 hour format. Should be from 0 to 23. An API + // may choose to allow the value "24:00:00" for scenarios like business + // closing time. + int32 hours = 4; + + // Required. Minutes of hour of day. Must be from 0 to 59. + int32 minutes = 5; + + // Required. Seconds of minutes of the time. Must normally be from 0 to 59. An + // API may allow the value 60 if it allows leap-seconds. + int32 seconds = 6; + + // Required. Fractions of seconds in nanoseconds. Must be from 0 to + // 999,999,999. + int32 nanos = 7; + + // Optional. Specifies either the UTC offset or the time zone of the DateTime. + // Choose carefully between them, considering that time zone data may change + // in the future (for example, a country modifies their DST start/end dates, + // and future DateTimes in the affected range had already been stored). + // If omitted, the DateTime is considered to be in local time. + oneof time_offset { + // UTC offset. Must be whole seconds, between -18 hours and +18 hours. + // For example, a UTC offset of -4:00 would be represented as + // { seconds: -14400 }. + google.protobuf.Duration utc_offset = 8; + + // Time zone. + TimeZone time_zone = 9; + } +} + +// Represents a time zone from the +// [IANA Time Zone Database](https://www.iana.org/time-zones). +message TimeZone { + // IANA Time Zone Database time zone, e.g. "America/New_York". + string id = 1; + + // Optional. IANA Time Zone Database version number, e.g. "2019a". + string version = 2; +} diff --git a/karapace/protobuf/google/type/dayofweek.proto b/karapace/protobuf/google/type/dayofweek.proto new file mode 100644 index 000000000..4c80c62ec --- /dev/null +++ b/karapace/protobuf/google/type/dayofweek.proto @@ -0,0 +1,50 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option go_package = "google.golang.org/genproto/googleapis/type/dayofweek;dayofweek"; +option java_multiple_files = true; +option java_outer_classname = "DayOfWeekProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a day of the week. +enum DayOfWeek { + // The day of the week is unspecified. + DAY_OF_WEEK_UNSPECIFIED = 0; + + // Monday + MONDAY = 1; + + // Tuesday + TUESDAY = 2; + + // Wednesday + WEDNESDAY = 3; + + // Thursday + THURSDAY = 4; + + // Friday + FRIDAY = 5; + + // Saturday + SATURDAY = 6; + + // Sunday + SUNDAY = 7; +} diff --git a/karapace/protobuf/google/type/decimal.proto b/karapace/protobuf/google/type/decimal.proto new file mode 100644 index 000000000..beb18a5d8 --- /dev/null +++ b/karapace/protobuf/google/type/decimal.proto @@ -0,0 +1,95 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/decimal;decimal"; +option java_multiple_files = true; +option java_outer_classname = "DecimalProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// A representation of a decimal value, such as 2.5. Clients may convert values +// into language-native decimal formats, such as Java's [BigDecimal][] or +// Python's [decimal.Decimal][]. +// +// [BigDecimal]: +// https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/math/BigDecimal.html +// [decimal.Decimal]: https://docs.python.org/3/library/decimal.html +message Decimal { + // The decimal value, as a string. + // + // The string representation consists of an optional sign, `+` (`U+002B`) + // or `-` (`U+002D`), followed by a sequence of zero or more decimal digits + // ("the integer"), optionally followed by a fraction, optionally followed + // by an exponent. + // + // The fraction consists of a decimal point followed by zero or more decimal + // digits. The string must contain at least one digit in either the integer + // or the fraction. The number formed by the sign, the integer and the + // fraction is referred to as the significand. + // + // The exponent consists of the character `e` (`U+0065`) or `E` (`U+0045`) + // followed by one or more decimal digits. + // + // Services **should** normalize decimal values before storing them by: + // + // - Removing an explicitly-provided `+` sign (`+2.5` -> `2.5`). + // - Replacing a zero-length integer value with `0` (`.5` -> `0.5`). + // - Coercing the exponent character to lower-case (`2.5E8` -> `2.5e8`). + // - Removing an explicitly-provided zero exponent (`2.5e0` -> `2.5`). + // + // Services **may** perform additional normalization based on its own needs + // and the internal decimal implementation selected, such as shifting the + // decimal point and exponent value together (example: `2.5e-1` <-> `0.25`). + // Additionally, services **may** preserve trailing zeroes in the fraction + // to indicate increased precision, but are not required to do so. + // + // Note that only the `.` character is supported to divide the integer + // and the fraction; `,` **should not** be supported regardless of locale. + // Additionally, thousand separators **should not** be supported. If a + // service does support them, values **must** be normalized. + // + // The ENBF grammar is: + // + // DecimalString = + // [Sign] Significand [Exponent]; + // + // Sign = '+' | '-'; + // + // Significand = + // Digits ['.'] [Digits] | [Digits] '.' Digits; + // + // Exponent = ('e' | 'E') [Sign] Digits; + // + // Digits = { '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' }; + // + // Services **should** clearly document the range of supported values, the + // maximum supported precision (total number of digits), and, if applicable, + // the scale (number of digits after the decimal point), as well as how it + // behaves when receiving out-of-bounds values. + // + // Services **may** choose to accept values passed as input even when the + // value has a higher precision or scale than the service supports, and + // **should** round the value to fit the supported scale. Alternatively, the + // service **may** error with `400 Bad Request` (`INVALID_ARGUMENT` in gRPC) + // if precision would be lost. + // + // Services **should** error with `400 Bad Request` (`INVALID_ARGUMENT` in + // gRPC) if the service receives a value outside of the supported range. + string value = 1; +} diff --git a/karapace/protobuf/google/type/expr.proto b/karapace/protobuf/google/type/expr.proto new file mode 100644 index 000000000..af0778cf9 --- /dev/null +++ b/karapace/protobuf/google/type/expr.proto @@ -0,0 +1,73 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option go_package = "google.golang.org/genproto/googleapis/type/expr;expr"; +option java_multiple_files = true; +option java_outer_classname = "ExprProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a textual expression in the Common Expression Language (CEL) +// syntax. CEL is a C-like expression language. The syntax and semantics of CEL +// are documented at https://github.com/google/cel-spec. +// +// Example (Comparison): +// +// title: "Summary size limit" +// description: "Determines if a summary is less than 100 chars" +// expression: "document.summary.size() < 100" +// +// Example (Equality): +// +// title: "Requestor is owner" +// description: "Determines if requestor is the document owner" +// expression: "document.owner == request.auth.claims.email" +// +// Example (Logic): +// +// title: "Public documents" +// description: "Determine whether the document should be publicly visible" +// expression: "document.type != 'private' && document.type != 'internal'" +// +// Example (Data Manipulation): +// +// title: "Notification string" +// description: "Create a notification string with a timestamp." +// expression: "'New message received at ' + string(document.create_time)" +// +// The exact variables and functions that may be referenced within an expression +// are determined by the service that evaluates it. See the service +// documentation for additional information. +message Expr { + // Textual representation of an expression in Common Expression Language + // syntax. + string expression = 1; + + // Optional. Title for the expression, i.e. a short string describing + // its purpose. This can be used e.g. in UIs which allow to enter the + // expression. + string title = 2; + + // Optional. Description of the expression. This is a longer text which + // describes the expression, e.g. when hovered over it in a UI. + string description = 3; + + // Optional. String indicating the location of the expression for error + // reporting, e.g. a file name and a position in the file. + string location = 4; +} diff --git a/karapace/protobuf/google/type/fraction.proto b/karapace/protobuf/google/type/fraction.proto new file mode 100644 index 000000000..6c5ae6e2a --- /dev/null +++ b/karapace/protobuf/google/type/fraction.proto @@ -0,0 +1,33 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option go_package = "google.golang.org/genproto/googleapis/type/fraction;fraction"; +option java_multiple_files = true; +option java_outer_classname = "FractionProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a fraction in terms of a numerator divided by a denominator. +message Fraction { + // The numerator in the fraction, e.g. 2 in 2/3. + int64 numerator = 1; + + // The value by which the numerator is divided, e.g. 3 in 2/3. Must be + // positive. + int64 denominator = 2; +} diff --git a/karapace/protobuf/google/type/interval.proto b/karapace/protobuf/google/type/interval.proto new file mode 100644 index 000000000..9702324cd --- /dev/null +++ b/karapace/protobuf/google/type/interval.proto @@ -0,0 +1,46 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +import "google/protobuf/timestamp.proto"; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/interval;interval"; +option java_multiple_files = true; +option java_outer_classname = "IntervalProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a time interval, encoded as a Timestamp start (inclusive) and a +// Timestamp end (exclusive). +// +// The start must be less than or equal to the end. +// When the start equals the end, the interval is empty (matches no time). +// When both start and end are unspecified, the interval matches any time. +message Interval { + // Optional. Inclusive start of the interval. + // + // If specified, a Timestamp matching this interval will have to be the same + // or after the start. + google.protobuf.Timestamp start_time = 1; + + // Optional. Exclusive end of the interval. + // + // If specified, a Timestamp matching this interval will have to be before the + // end. + google.protobuf.Timestamp end_time = 2; +} diff --git a/karapace/protobuf/google/type/latlng.proto b/karapace/protobuf/google/type/latlng.proto new file mode 100644 index 000000000..9231456e3 --- /dev/null +++ b/karapace/protobuf/google/type/latlng.proto @@ -0,0 +1,37 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/latlng;latlng"; +option java_multiple_files = true; +option java_outer_classname = "LatLngProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// An object that represents a latitude/longitude pair. This is expressed as a +// pair of doubles to represent degrees latitude and degrees longitude. Unless +// specified otherwise, this must conform to the +// WGS84 +// standard. Values must be within normalized ranges. +message LatLng { + // The latitude in degrees. It must be in the range [-90.0, +90.0]. + double latitude = 1; + + // The longitude in degrees. It must be in the range [-180.0, +180.0]. + double longitude = 2; +} diff --git a/karapace/protobuf/google/type/localized_text.proto b/karapace/protobuf/google/type/localized_text.proto new file mode 100644 index 000000000..5c6922b8c --- /dev/null +++ b/karapace/protobuf/google/type/localized_text.proto @@ -0,0 +1,36 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/localized_text;localized_text"; +option java_multiple_files = true; +option java_outer_classname = "LocalizedTextProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Localized variant of a text in a particular language. +message LocalizedText { + // Localized string in the language corresponding to `language_code' below. + string text = 1; + + // The text's BCP-47 language code, such as "en-US" or "sr-Latn". + // + // For more information, see + // http://www.unicode.org/reports/tr35/#Unicode_locale_identifier. + string language_code = 2; +} diff --git a/karapace/protobuf/google/type/money.proto b/karapace/protobuf/google/type/money.proto new file mode 100644 index 000000000..98d6494e4 --- /dev/null +++ b/karapace/protobuf/google/type/money.proto @@ -0,0 +1,42 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/money;money"; +option java_multiple_files = true; +option java_outer_classname = "MoneyProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents an amount of money with its currency type. +message Money { + // The three-letter currency code defined in ISO 4217. + string currency_code = 1; + + // The whole units of the amount. + // For example if `currencyCode` is `"USD"`, then 1 unit is one US dollar. + int64 units = 2; + + // Number of nano (10^-9) units of the amount. + // The value must be between -999,999,999 and +999,999,999 inclusive. + // If `units` is positive, `nanos` must be positive or zero. + // If `units` is zero, `nanos` can be positive, zero, or negative. + // If `units` is negative, `nanos` must be negative or zero. + // For example $-1.75 is represented as `units`=-1 and `nanos`=-750,000,000. + int32 nanos = 3; +} diff --git a/karapace/protobuf/google/type/month.proto b/karapace/protobuf/google/type/month.proto new file mode 100644 index 000000000..99e7551b1 --- /dev/null +++ b/karapace/protobuf/google/type/month.proto @@ -0,0 +1,65 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option go_package = "google.golang.org/genproto/googleapis/type/month;month"; +option java_multiple_files = true; +option java_outer_classname = "MonthProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a month in the Gregorian calendar. +enum Month { + // The unspecified month. + MONTH_UNSPECIFIED = 0; + + // The month of January. + JANUARY = 1; + + // The month of February. + FEBRUARY = 2; + + // The month of March. + MARCH = 3; + + // The month of April. + APRIL = 4; + + // The month of May. + MAY = 5; + + // The month of June. + JUNE = 6; + + // The month of July. + JULY = 7; + + // The month of August. + AUGUST = 8; + + // The month of September. + SEPTEMBER = 9; + + // The month of October. + OCTOBER = 10; + + // The month of November. + NOVEMBER = 11; + + // The month of December. + DECEMBER = 12; +} diff --git a/karapace/protobuf/google/type/phone_number.proto b/karapace/protobuf/google/type/phone_number.proto new file mode 100644 index 000000000..7bbb7d873 --- /dev/null +++ b/karapace/protobuf/google/type/phone_number.proto @@ -0,0 +1,113 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/phone_number;phone_number"; +option java_multiple_files = true; +option java_outer_classname = "PhoneNumberProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// An object representing a phone number, suitable as an API wire format. +// +// This representation: +// +// - should not be used for locale-specific formatting of a phone number, such +// as "+1 (650) 253-0000 ext. 123" +// +// - is not designed for efficient storage +// - may not be suitable for dialing - specialized libraries (see references) +// should be used to parse the number for that purpose +// +// To do something meaningful with this number, such as format it for various +// use-cases, convert it to an `i18n.phonenumbers.PhoneNumber` object first. +// +// For instance, in Java this would be: +// +// com.google.type.PhoneNumber wireProto = +// com.google.type.PhoneNumber.newBuilder().build(); +// com.google.i18n.phonenumbers.Phonenumber.PhoneNumber phoneNumber = +// PhoneNumberUtil.getInstance().parse(wireProto.getE164Number(), "ZZ"); +// if (!wireProto.getExtension().isEmpty()) { +// phoneNumber.setExtension(wireProto.getExtension()); +// } +// +// Reference(s): +// - https://github.com/google/libphonenumber +message PhoneNumber { + // An object representing a short code, which is a phone number that is + // typically much shorter than regular phone numbers and can be used to + // address messages in MMS and SMS systems, as well as for abbreviated dialing + // (e.g. "Text 611 to see how many minutes you have remaining on your plan."). + // + // Short codes are restricted to a region and are not internationally + // dialable, which means the same short code can exist in different regions, + // with different usage and pricing, even if those regions share the same + // country calling code (e.g. US and CA). + message ShortCode { + // Required. The BCP-47 region code of the location where calls to this + // short code can be made, such as "US" and "BB". + // + // Reference(s): + // - http://www.unicode.org/reports/tr35/#unicode_region_subtag + string region_code = 1; + + // Required. The short code digits, without a leading plus ('+') or country + // calling code, e.g. "611". + string number = 2; + } + + // Required. Either a regular number, or a short code. New fields may be + // added to the oneof below in the future, so clients should ignore phone + // numbers for which none of the fields they coded against are set. + oneof kind { + // The phone number, represented as a leading plus sign ('+'), followed by a + // phone number that uses a relaxed ITU E.164 format consisting of the + // country calling code (1 to 3 digits) and the subscriber number, with no + // additional spaces or formatting, e.g.: + // - correct: "+15552220123" + // - incorrect: "+1 (555) 222-01234 x123". + // + // The ITU E.164 format limits the latter to 12 digits, but in practice not + // all countries respect that, so we relax that restriction here. + // National-only numbers are not allowed. + // + // References: + // - https://www.itu.int/rec/T-REC-E.164-201011-I + // - https://en.wikipedia.org/wiki/E.164. + // - https://en.wikipedia.org/wiki/List_of_country_calling_codes + string e164_number = 1; + + // A short code. + // + // Reference(s): + // - https://en.wikipedia.org/wiki/Short_code + ShortCode short_code = 2; + } + + // The phone number's extension. The extension is not standardized in ITU + // recommendations, except for being defined as a series of numbers with a + // maximum length of 40 digits. Other than digits, some other dialing + // characters such as ',' (indicating a wait) or '#' may be stored here. + // + // Note that no regions currently use extensions with short codes, so this + // field is normally only set in conjunction with an E.164 number. It is held + // separately from the E.164 number to allow for short code extensions in the + // future. + string extension = 3; +} diff --git a/karapace/protobuf/google/type/postal_address.proto b/karapace/protobuf/google/type/postal_address.proto new file mode 100644 index 000000000..c57c7c31a --- /dev/null +++ b/karapace/protobuf/google/type/postal_address.proto @@ -0,0 +1,134 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/postaladdress;postaladdress"; +option java_multiple_files = true; +option java_outer_classname = "PostalAddressProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a postal address, e.g. for postal delivery or payments addresses. +// Given a postal address, a postal service can deliver items to a premise, P.O. +// Box or similar. +// It is not intended to model geographical locations (roads, towns, +// mountains). +// +// In typical usage an address would be created via user input or from importing +// existing data, depending on the type of process. +// +// Advice on address input / editing: +// - Use an i18n-ready address widget such as +// https://github.com/google/libaddressinput) +// - Users should not be presented with UI elements for input or editing of +// fields outside countries where that field is used. +// +// For more guidance on how to use this schema, please see: +// https://support.google.com/business/answer/6397478 +message PostalAddress { + // The schema revision of the `PostalAddress`. This must be set to 0, which is + // the latest revision. + // + // All new revisions **must** be backward compatible with old revisions. + int32 revision = 1; + + // Required. CLDR region code of the country/region of the address. This + // is never inferred and it is up to the user to ensure the value is + // correct. See http://cldr.unicode.org/ and + // http://www.unicode.org/cldr/charts/30/supplemental/territory_information.html + // for details. Example: "CH" for Switzerland. + string region_code = 2; + + // Optional. BCP-47 language code of the contents of this address (if + // known). This is often the UI language of the input form or is expected + // to match one of the languages used in the address' country/region, or their + // transliterated equivalents. + // This can affect formatting in certain countries, but is not critical + // to the correctness of the data and will never affect any validation or + // other non-formatting related operations. + // + // If this value is not known, it should be omitted (rather than specifying a + // possibly incorrect default). + // + // Examples: "zh-Hant", "ja", "ja-Latn", "en". + string language_code = 3; + + // Optional. Postal code of the address. Not all countries use or require + // postal codes to be present, but where they are used, they may trigger + // additional validation with other parts of the address (e.g. state/zip + // validation in the U.S.A.). + string postal_code = 4; + + // Optional. Additional, country-specific, sorting code. This is not used + // in most regions. Where it is used, the value is either a string like + // "CEDEX", optionally followed by a number (e.g. "CEDEX 7"), or just a number + // alone, representing the "sector code" (Jamaica), "delivery area indicator" + // (Malawi) or "post office indicator" (e.g. Côte d'Ivoire). + string sorting_code = 5; + + // Optional. Highest administrative subdivision which is used for postal + // addresses of a country or region. + // For example, this can be a state, a province, an oblast, or a prefecture. + // Specifically, for Spain this is the province and not the autonomous + // community (e.g. "Barcelona" and not "Catalonia"). + // Many countries don't use an administrative area in postal addresses. E.g. + // in Switzerland this should be left unpopulated. + string administrative_area = 6; + + // Optional. Generally refers to the city/town portion of the address. + // Examples: US city, IT comune, UK post town. + // In regions of the world where localities are not well defined or do not fit + // into this structure well, leave locality empty and use address_lines. + string locality = 7; + + // Optional. Sublocality of the address. + // For example, this can be neighborhoods, boroughs, districts. + string sublocality = 8; + + // Unstructured address lines describing the lower levels of an address. + // + // Because values in address_lines do not have type information and may + // sometimes contain multiple values in a single field (e.g. + // "Austin, TX"), it is important that the line order is clear. The order of + // address lines should be "envelope order" for the country/region of the + // address. In places where this can vary (e.g. Japan), address_language is + // used to make it explicit (e.g. "ja" for large-to-small ordering and + // "ja-Latn" or "en" for small-to-large). This way, the most specific line of + // an address can be selected based on the language. + // + // The minimum permitted structural representation of an address consists + // of a region_code with all remaining information placed in the + // address_lines. It would be possible to format such an address very + // approximately without geocoding, but no semantic reasoning could be + // made about any of the address components until it was at least + // partially resolved. + // + // Creating an address only containing a region_code and address_lines, and + // then geocoding is the recommended way to handle completely unstructured + // addresses (as opposed to guessing which parts of the address should be + // localities or administrative areas). + repeated string address_lines = 9; + + // Optional. The recipient at the address. + // This field may, under certain circumstances, contain multiline information. + // For example, it might contain "care of" information. + repeated string recipients = 10; + + // Optional. The name of the organization at the address. + string organization = 11; +} diff --git a/karapace/protobuf/google/type/quaternion.proto b/karapace/protobuf/google/type/quaternion.proto new file mode 100644 index 000000000..dfb822def --- /dev/null +++ b/karapace/protobuf/google/type/quaternion.proto @@ -0,0 +1,94 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/quaternion;quaternion"; +option java_multiple_files = true; +option java_outer_classname = "QuaternionProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// A quaternion is defined as the quotient of two directed lines in a +// three-dimensional space or equivalently as the quotient of two Euclidean +// vectors (https://en.wikipedia.org/wiki/Quaternion). +// +// Quaternions are often used in calculations involving three-dimensional +// rotations (https://en.wikipedia.org/wiki/Quaternions_and_spatial_rotation), +// as they provide greater mathematical robustness by avoiding the gimbal lock +// problems that can be encountered when using Euler angles +// (https://en.wikipedia.org/wiki/Gimbal_lock). +// +// Quaternions are generally represented in this form: +// +// w + xi + yj + zk +// +// where x, y, z, and w are real numbers, and i, j, and k are three imaginary +// numbers. +// +// Our naming choice `(x, y, z, w)` comes from the desire to avoid confusion for +// those interested in the geometric properties of the quaternion in the 3D +// Cartesian space. Other texts often use alternative names or subscripts, such +// as `(a, b, c, d)`, `(1, i, j, k)`, or `(0, 1, 2, 3)`, which are perhaps +// better suited for mathematical interpretations. +// +// To avoid any confusion, as well as to maintain compatibility with a large +// number of software libraries, the quaternions represented using the protocol +// buffer below *must* follow the Hamilton convention, which defines `ij = k` +// (i.e. a right-handed algebra), and therefore: +// +// i^2 = j^2 = k^2 = ijk = −1 +// ij = −ji = k +// jk = −kj = i +// ki = −ik = j +// +// Please DO NOT use this to represent quaternions that follow the JPL +// convention, or any of the other quaternion flavors out there. +// +// Definitions: +// +// - Quaternion norm (or magnitude): `sqrt(x^2 + y^2 + z^2 + w^2)`. +// - Unit (or normalized) quaternion: a quaternion whose norm is 1. +// - Pure quaternion: a quaternion whose scalar component (`w`) is 0. +// - Rotation quaternion: a unit quaternion used to represent rotation. +// - Orientation quaternion: a unit quaternion used to represent orientation. +// +// A quaternion can be normalized by dividing it by its norm. The resulting +// quaternion maintains the same direction, but has a norm of 1, i.e. it moves +// on the unit sphere. This is generally necessary for rotation and orientation +// quaternions, to avoid rounding errors: +// https://en.wikipedia.org/wiki/Rotation_formalisms_in_three_dimensions +// +// Note that `(x, y, z, w)` and `(-x, -y, -z, -w)` represent the same rotation, +// but normalization would be even more useful, e.g. for comparison purposes, if +// it would produce a unique representation. It is thus recommended that `w` be +// kept positive, which can be achieved by changing all the signs when `w` is +// negative. +// +message Quaternion { + // The x component. + double x = 1; + + // The y component. + double y = 2; + + // The z component. + double z = 3; + + // The scalar component. + double w = 4; +} diff --git a/karapace/protobuf/google/type/timeofday.proto b/karapace/protobuf/google/type/timeofday.proto new file mode 100644 index 000000000..5cb48aa93 --- /dev/null +++ b/karapace/protobuf/google/type/timeofday.proto @@ -0,0 +1,44 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/timeofday;timeofday"; +option java_multiple_files = true; +option java_outer_classname = "TimeOfDayProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a time of day. The date and time zone are either not significant +// or are specified elsewhere. An API may choose to allow leap seconds. Related +// types are [google.type.Date][google.type.Date] and +// `google.protobuf.Timestamp`. +message TimeOfDay { + // Hours of day in 24 hour format. Should be from 0 to 23. An API may choose + // to allow the value "24:00:00" for scenarios like business closing time. + int32 hours = 1; + + // Minutes of hour of day. Must be from 0 to 59. + int32 minutes = 2; + + // Seconds of minutes of the time. Must normally be from 0 to 59. An API may + // allow the value 60 if it allows leap-seconds. + int32 seconds = 3; + + // Fractions of seconds in nanoseconds. Must be from 0 to 999,999,999. + int32 nanos = 4; +} diff --git a/karapace/protobuf/google/type/type.yaml b/karapace/protobuf/google/type/type.yaml new file mode 100644 index 000000000..d5c71364d --- /dev/null +++ b/karapace/protobuf/google/type/type.yaml @@ -0,0 +1,40 @@ +type: google.api.Service +config_version: 3 +name: type.googleapis.com +title: Common Types + +types: +- name: google.type.Color +- name: google.type.Date +- name: google.type.DateTime +- name: google.type.Decimal +- name: google.type.Expr +- name: google.type.Fraction +- name: google.type.Interval +- name: google.type.LatLng +- name: google.type.LocalizedText +- name: google.type.Money +- name: google.type.PhoneNumber +- name: google.type.PostalAddress +- name: google.type.Quaternion +- name: google.type.TimeOfDay + +enums: +- name: google.type.CalendarPeriod +- name: google.type.DayOfWeek +- name: google.type.Month + +documentation: + summary: Defines common types for Google APIs. + overview: |- + # Google Common Types + + This package contains definitions of common types for Google APIs. + All types defined in this package are suitable for different APIs to + exchange data, and will never break binary compatibility. They should + have design quality comparable to major programming languages like + Java and C#. + + NOTE: Some common types are defined in the package `google.protobuf` + as they are directly supported by Protocol Buffers compiler and + runtime. Those types are called Well-Known Types. diff --git a/karapace/protobuf/known_dependency.py b/karapace/protobuf/known_dependency.py new file mode 100644 index 000000000..5c3e74a00 --- /dev/null +++ b/karapace/protobuf/known_dependency.py @@ -0,0 +1,154 @@ +# Support of known dependencies + +from enum import Enum +from typing import Any, Dict, Set + + +def static_init(cls: Any) -> object: + if getattr(cls, "static_init", None): + cls.static_init() + return cls + + +class KnownDependencyLocation(Enum): + ANY_LOCATION = "google/protobuf/any.proto" + API_LOCATION = "google/protobuf/api.proto" + DESCRIPTOR_LOCATION = "google/protobuf/descriptor.proto" + DURATION_LOCATION = "google/protobuf/duration.proto" + EMPTY_LOCATION = "google/protobuf/empty.proto" + FIELD_MASK_LOCATION = "google/protobuf/field_mask.proto" + SOURCE_CONTEXT_LOCATION = "google/protobuf/source_context.proto" + STRUCT_LOCATION = "google/protobuf/struct.proto" + TIMESTAMP_LOCATION = "google/protobuf/timestamp.proto" + TYPE_LOCATION = "google/protobuf/type.proto" + WRAPPER_LOCATION = "google/protobuf/wrappers.proto" + CALENDAR_PERIOD_LOCATION = "google/type/calendar_period.proto" + COLOR_LOCATION = "google/type/color.proto" + DATE_LOCATION = "google/type/date.proto" + DATETIME_LOCATION = "google/type/datetime.proto" + DAY_OF_WEEK_LOCATION = "google/type/dayofweek.proto" + DECIMAL_LOCATION = "google/type/decimal.proto" + EXPR_LOCATION = "google/type/expr.proto" + FRACTION_LOCATION = "google/type/fraction.proto" + INTERVAL_LOCATION = "google/type/interval.proto" + LATLNG_LOCATION = "google/type/latlng.proto" + MONEY_LOCATION = "google/type/money.proto" + MONTH_LOCATION = "google/type/month.proto" + PHONE_NUMBER_LOCATION = "google/type/phone_number.proto" + POSTAL_ADDRESS_LOCATION = "google/type/postal_address.proto" + QUATERNION_LOCATION = "google/type/quaternion.proto" + TIME_OF_DAY_LOCATION = "google/type/timeofday.proto" + + +@static_init +class KnownDependency: + index: Dict = dict() + index_simple: Dict = dict() + map: Dict = { + "google/protobuf/any.proto": [".google.protobuf.Any"], + "google/protobuf/api.proto": [".google.protobuf.Api", ".google.protobuf.Method", ".google.protobuf.Mixin"], + "google/protobuf/descriptor.proto": [ + ".google.protobuf.FileDescriptorSet", + ".google.protobuf.FileDescriptorProto", + ".google.protobuf.DescriptorProto", + ".google.protobuf.ExtensionRangeOptions", + ".google.protobuf.FieldDescriptorProto", + ".google.protobuf.OneofDescriptorProto", + ".google.protobuf.EnumDescriptorProto", + ".google.protobuf.EnumValueDescriptorProto", + ".google.protobuf.ServiceDescriptorProto", + ".google.protobuf.MethodDescriptorProto", + ".google.protobuf.FileOptions", + ".google.protobuf.MessageOptions", + ".google.protobuf.FieldOptions", + ".google.protobuf.OneofOptions", + ".google.protobuf.EnumOptions", + ".google.protobuf.EnumValueOptions", + ".google.protobuf.ServiceOptions", + ".google.protobuf.MethodOptions", + ".google.protobuf.UninterpretedOption", + ".google.protobuf.SourceCodeInfo", + ".google.protobuf.GeneratedCodeInfo", + ], + "google/protobuf/duration.proto": [".google.protobuf.Duration"], + "google/protobuf/empty.proto": [".google.protobuf.Empty"], + "google/protobuf/field_mask.proto": [".google.protobuf.FieldMask"], + "google/protobuf/source_context.proto": [".google.protobuf.SourceContext"], + "google/protobuf/struct.proto": [ + ".google.protobuf.Struct", + ".google.protobuf.Value", + ".google.protobuf.NullValue", + ".google.protobuf.ListValue", + ], + "google/protobuf/timestamp.proto": [".google.protobuf.Timestamp"], + "google/protobuf/type.proto": [ + ".google.protobuf.Type", + ".google.protobuf.Field", + ".google.protobuf.Enum", + ".google.protobuf.EnumValue", + ".google.protobuf.Option", + ".google.protobuf.Syntax", + ], + "google/protobuf/wrappers.proto": [ + ".google.protobuf.DoubleValue", + ".google.protobuf.FloatValue", + ".google.protobuf.Int64Value", + ".google.protobuf.UInt64Value", + ".google.protobuf.Int32Value", + ".google.protobuf.UInt32Value", + ".google.protobuf.BoolValue", + ".google.protobuf.StringValue", + ".google.protobuf.BytesValue", + ], + "google/type/calendar_period.proto": [".google.type.CalendarPeriod"], + "google/type/color.proto": [".google.type.Color"], + "google/type/date.proto": [".google.type.Date"], + "google/type/datetime.proto": [".google.type.DateTime", ".google.type.TimeZone"], + "google/type/dayofweek.proto": [".google.type.DayOfWeek"], + "google/type/decimal.proto": [".google.type.Decimal"], + "google/type/expr.proto": [".google.type.Expr"], + "google/type/fraction.proto": [".google.type.Fraction"], + "google/type/interval.proto": [".google.type.Interval"], + "google/type/latlng.proto": [".google.type.LatLng"], + "google/type/money.proto": [".google.type.Money"], + "google/type/month.proto": [".google.type.Month"], + "google/type/phone_number.proto": [".google.type.PhoneNumber"], + "google/type/postal_address.proto": [".google.type.PostalAddress"], + "google/type/quaternion.proto": [".google.type.Quaternion"], + "google/type/timeofday.proto": [".google.type.TimeOfDay"], + "confluent/meta.proto": [".confluent.Meta"], + "confluent/type/decimal.proto": [".confluent.type.Decimal"], + } + + @classmethod + def static_init(cls) -> None: + for key, value in cls.map.items(): + for item in value: + cls.index[item] = key + dot = item.rfind(".") + cls.index_simple[item[dot + 1 :]] = key + + +@static_init +class DependenciesHardcoded: + index: Set = set() + + @classmethod + def static_init(cls) -> None: + cls.index = { + "bool", + "bytes", + "double", + "float", + "fixed32", + "fixed64", + "int32", + "int64", + "sfixed32", + "sfixed64", + "sint32", + "sint64", + "string", + "uint32", + "uint64", + } diff --git a/karapace/protobuf/proto_file_element.py b/karapace/protobuf/proto_file_element.py index 4774a8c67..fa43dd9e8 100644 --- a/karapace/protobuf/proto_file_element.py +++ b/karapace/protobuf/proto_file_element.py @@ -9,23 +9,24 @@ from karapace.protobuf.message_element import MessageElement from karapace.protobuf.syntax import Syntax from karapace.protobuf.type_element import TypeElement +from typing import List, Optional class ProtoFileElement: def __init__( self, location: Location, - package_name: str = None, - syntax: Syntax = None, - imports: list = None, - public_imports: list = None, - types=None, - services: list = None, - extend_declarations: list = None, - options: list = None, + package_name: Optional[str] = None, + syntax: Optional[Syntax] = None, + imports: Optional[list] = None, + public_imports: Optional[list] = None, + types: Optional[List[TypeElement]] = None, + services: Optional[list] = None, + extend_declarations: Optional[list] = None, + options: Optional[list] = None, ) -> None: if types is None: - types = [] + types = list() self.location = location self.package_name = package_name self.syntax = syntax diff --git a/karapace/protobuf/proto_parser.py b/karapace/protobuf/proto_parser.py index 81c333af4..27cc59f11 100644 --- a/karapace/protobuf/proto_parser.py +++ b/karapace/protobuf/proto_parser.py @@ -25,7 +25,7 @@ from karapace.protobuf.syntax_reader import SyntaxReader from karapace.protobuf.type_element import TypeElement from karapace.protobuf.utils import MAX_TAG_VALUE -from typing import List, Union +from typing import List, Optional, Union class Context(Enum): @@ -71,13 +71,13 @@ class ProtoParser: def __init__(self, location: Location, data: str) -> None: self.location = location self.imports: List[str] = [] - self.nested_types: List[str] = [] + self.nested_types: List[TypeElement] = [] self.services: List[str] = [] self.extends_list: List[str] = [] self.options: List[str] = [] self.declaration_count = 0 - self.syntax: Union[Syntax, None] = None - self.package_name: Union[str, None] = None + self.syntax: Optional[Syntax] = None + self.package_name: Optional[str] = None self.prefix = "" self.data = data self.public_imports: List[str] = [] @@ -176,7 +176,6 @@ def read_declaration( import_string = self.reader.read_string() if import_string == "public": self.public_imports.append(self.reader.read_string()) - else: self.imports.append(import_string) self.reader.require(";") diff --git a/karapace/protobuf/schema.py b/karapace/protobuf/schema.py index ae723be1b..76e319019 100644 --- a/karapace/protobuf/schema.py +++ b/karapace/protobuf/schema.py @@ -1,7 +1,10 @@ # Ported from square/wire: +# Ported from square/wire: # wire-library/wire-schema/src/commonMain/kotlin/com/squareup/wire/schema/Schema.kt # Ported partially for required functionality. +from karapace.dependency import Dependency, DependencyVerifierResult from karapace.protobuf.compare_result import CompareResult +from karapace.protobuf.dependency import _process_one_of, ProtobufDependencyVerifier from karapace.protobuf.enum_element import EnumElement from karapace.protobuf.exception import IllegalArgumentException from karapace.protobuf.location import Location @@ -9,7 +12,10 @@ from karapace.protobuf.option_element import OptionElement from karapace.protobuf.proto_file_element import ProtoFileElement from karapace.protobuf.proto_parser import ProtoParser +from karapace.protobuf.type_element import TypeElement from karapace.protobuf.utils import append_documentation, append_indented +from karapace.schema_references import References +from typing import Dict, Optional def add_slashes(text: str) -> str: @@ -100,12 +106,67 @@ def option_element_string(option: OptionElement) -> str: class ProtobufSchema: DEFAULT_LOCATION = Location.get("") - def __init__(self, schema: str) -> None: + def __init__( + self, schema: str, references: Optional[References] = None, dependencies: Optional[Dict[str, Dependency]] = None + ) -> None: if type(schema).__name__ != "str": raise IllegalArgumentException("Non str type of schema string") self.dirty = schema self.cache_string = "" self.proto_file_element = ProtoParser.parse(self.DEFAULT_LOCATION, schema) + self.references = references + self.dependencies = dependencies + + def gather_deps(self) -> ProtobufDependencyVerifier: + verifier = ProtobufDependencyVerifier() + self.collect_dependencies(verifier) + return verifier + + def verify_schema_dependencies(self) -> DependencyVerifierResult: + verifier = ProtobufDependencyVerifier() + self.collect_dependencies(verifier) + return verifier.verify() + + def collect_dependencies(self, verifier: ProtobufDependencyVerifier): + + if self.dependencies: + for key in self.dependencies: + self.dependencies[key].schema.schema.collect_dependencies(verifier) + # verifier.add_import?? we have no access to own Kafka structure from this class... + # but we need data to analyse imports to avoid ciclyc dependencies... + + package_name = self.proto_file_element.package_name + if package_name is None: + package_name = "" + else: + package_name = "." + package_name + for element_type in self.proto_file_element.types: + type_name = element_type.name + full_name = package_name + "." + type_name + verifier.add_declared_type(full_name) + verifier.add_declared_type(type_name) + if isinstance(element_type, MessageElement): + for one_of in element_type.one_ofs: + _process_one_of(verifier, package_name, type_name, one_of) + for field in element_type.fields: + verifier.add_used_type(full_name, field.element_type) + for nested_type in element_type.nested_types: + self._process_nested_type(verifier, package_name, type_name, nested_type) + + def _process_nested_type( + self, verifier: ProtobufDependencyVerifier, package_name: str, parent_name, element_type: TypeElement + ): + + verifier.add_declared_type(package_name + "." + parent_name + "." + element_type.name) + verifier.add_declared_type(parent_name + "." + element_type.name) + + if isinstance(element_type, MessageElement): + for one_of in element_type.one_ofs: + _process_one_of(verifier, package_name, parent_name, one_of) + for field in element_type.fields: + verifier.add_used_type(parent_name, field.element_type) + for nested_type in element_type.nested_types: + self._process_nested_type(verifier, package_name, parent_name + "." + element_type.name, nested_type) def __str__(self) -> str: if not self.cache_string: diff --git a/karapace/schema_models.py b/karapace/schema_models.py index 4ac732d02..7448c4a8f 100644 --- a/karapace/schema_models.py +++ b/karapace/schema_models.py @@ -1,6 +1,5 @@ from avro.errors import SchemaParseException from avro.schema import parse as avro_parse, Schema as AvroSchema -from enum import Enum, unique from jsonschema import Draft7Validator from jsonschema.exceptions import SchemaError from karapace.errors import InvalidSchema @@ -10,9 +9,12 @@ IllegalStateException, ProtobufException, ProtobufParserRuntimeException, + ProtobufUnresolvedDependencyException, SchemaParseException as ProtobufSchemaParseException, ) -from karapace.protobuf.schema import ProtobufSchema +from karapace.protobuf.schema import Dependency, ProtobufSchema +from karapace.schema_references import References +from karapace.schema_type import SchemaType from karapace.utils import json_encode from typing import Any, Dict, Optional, Union @@ -49,36 +51,42 @@ def parse_jsonschema_definition(schema_definition: str) -> Draft7Validator: return Draft7Validator(schema) -def parse_protobuf_schema_definition(schema_definition: str) -> ProtobufSchema: +def parse_protobuf_schema_definition( + schema_definition: str, references: Optional[References] = None, dependencies: Optional[Dict[str, Dependency]] = None +) -> ProtobufSchema: """Parses and validates `schema_definition`. Raises: Nothing yet. """ - - return ProtobufSchema(schema_definition) - - -@unique -class SchemaType(str, Enum): - AVRO = "AVRO" - JSONSCHEMA = "JSON" - PROTOBUF = "PROTOBUF" + protobuf_schema = ProtobufSchema(schema_definition, references, dependencies) + result = protobuf_schema.verify_schema_dependencies() + if not result.result: + raise ProtobufUnresolvedDependencyException(f"{result.message}") + return protobuf_schema class TypedSchema: - def __init__(self, schema_type: SchemaType, schema_str: str): + def __init__( + self, + schema_type: SchemaType, + schema_str: str, + references: Optional[References] = None, + dependencies: Optional[Dict[str, Dependency]] = None, + ): """Schema with type information Args: schema_type (SchemaType): The type of the schema schema_str (str): The original schema string + references(References): The references of schema """ self.schema_type = schema_type self.schema_str = schema_str + self.references = references + self.dependencies = dependencies self.max_id: Optional[int] = None - self._str_cached: Optional[str] = None def to_dict(self) -> Dict[str, Any]: @@ -97,17 +105,39 @@ def __str__(self) -> str: def __repr__(self) -> str: return f"TypedSchema(type={self.schema_type}, schema={str(self)})" + def get_references(self) -> Optional["References"]: + return self.references + def __eq__(self, other: Any) -> bool: - return isinstance(other, TypedSchema) and self.__str__() == other.__str__() and self.schema_type is other.schema_type + schema_is_equal = ( + isinstance(other, TypedSchema) and self.schema_type is other.schema_type and self.__str__() == other.__str__() + ) + if not schema_is_equal: + return False + if self.references is not None: + return self.references == other.references + return other.references is None class ValidatedTypedSchema(TypedSchema): - def __init__(self, schema_type: SchemaType, schema_str: str, schema: Union[Draft7Validator, AvroSchema, ProtobufSchema]): - super().__init__(schema_type=schema_type, schema_str=schema_str) + def __init__( + self, + schema_type: SchemaType, + schema_str: str, + schema: Union[Draft7Validator, AvroSchema, ProtobufSchema], + references: Optional["References"] = None, + dependencies: Optional[Dict[str, Dependency]] = None, + ): + super().__init__(schema_type=schema_type, schema_str=schema_str, references=references, dependencies=dependencies) self.schema = schema @staticmethod - def parse(schema_type: SchemaType, schema_str: str) -> "ValidatedTypedSchema": + def parse( + schema_type: SchemaType, + schema_str: str, + references: Optional["References"] = None, + dependencies: Optional[Dict[str, Dependency]] = None, + ) -> "ValidatedTypedSchema": if schema_type not in [SchemaType.AVRO, SchemaType.JSONSCHEMA, SchemaType.PROTOBUF]: raise InvalidSchema(f"Unknown parser {schema_type} for {schema_str}") @@ -127,7 +157,7 @@ def parse(schema_type: SchemaType, schema_str: str) -> "ValidatedTypedSchema": elif schema_type is SchemaType.PROTOBUF: try: - parsed_schema = parse_protobuf_schema_definition(schema_str) + parsed_schema = parse_protobuf_schema_definition(schema_str, references, dependencies) except ( TypeError, SchemaError, @@ -143,7 +173,13 @@ def parse(schema_type: SchemaType, schema_str: str) -> "ValidatedTypedSchema": else: raise InvalidSchema(f"Unknown parser {schema_type} for {schema_str}") - return ValidatedTypedSchema(schema_type=schema_type, schema_str=schema_str, schema=parsed_schema) + return ValidatedTypedSchema( + schema_type=schema_type, + schema_str=schema_str, + schema=parsed_schema, + references=references, + dependencies=dependencies, + ) def __str__(self) -> str: if self.schema_type == SchemaType.PROTOBUF: diff --git a/karapace/schema_reader.py b/karapace/schema_reader.py index adab80466..f7fe60f08 100644 --- a/karapace/schema_reader.py +++ b/karapace/schema_reader.py @@ -10,15 +10,17 @@ from kafka.errors import KafkaConfigurationError, NoBrokersAvailable, NodeNotReadyError, TopicAlreadyExistsError from karapace import constants from karapace.config import Config -from karapace.errors import InvalidSchema +from karapace.dependency import Dependency +from karapace.errors import InvalidReferences, InvalidSchema, ReferencesNotSupportedException from karapace.key_format import is_key_in_canonical_format, KeyFormatter, KeyMode from karapace.master_coordinator import MasterCoordinator from karapace.schema_models import SchemaType, TypedSchema, ValidatedTypedSchema +from karapace.schema_references import References from karapace.statsd import StatsClient from karapace.typing import SubjectData -from karapace.utils import KarapaceKafkaClient +from karapace.utils import KarapaceKafkaClient, reference_key from threading import Condition, Event, Lock, Thread -from typing import Any, Dict, Optional +from typing import Any, Dict, List, Optional, Tuple import json import logging @@ -27,6 +29,7 @@ Subject = str Version = int Schema = Dict[str, Any] +Referents = List SchemaId = int # The value `0` is a valid offset and it represents the first message produced @@ -37,7 +40,6 @@ KAFKA_CLIENT_CREATION_TIMEOUT_SECONDS = 2.0 SCHEMA_TOPIC_CREATION_TIMEOUT_SECONDS = 5.0 - # Metric names METRIC_SCHEMA_TOPIC_RECORDS_PROCESSED_COUNT = "karapace_schema_reader_records_processed" METRIC_SCHEMA_TOPIC_RECORDS_PER_KEYMODE_GAUGE = "karapace_schema_reader_records_per_keymode" @@ -137,6 +139,7 @@ def __init__( self.config = config self.subjects: Dict[Subject, SubjectData] = {} self.schemas: Dict[int, TypedSchema] = {} + self.referenced_by: Dict[str, Referents] = {} self.global_schema_id = 0 self.admin_client: Optional[KafkaAdminClient] = None self.topic_replication_factor = self.config["replication_factor"] @@ -377,7 +380,6 @@ def _handle_msg_config(self, key: dict, value: Optional[dict]) -> None: if subject not in self.subjects: LOG.info("Adding first version of subject: %r with no schemas", subject) self.subjects[subject] = {"schemas": {}} - if not value: LOG.info("Deleting compatibility config completely for subject: %r", subject) self.subjects[subject].pop("compatibility", None) @@ -427,6 +429,7 @@ def _handle_msg_schema(self, key: dict, value: Optional[dict]) -> None: schema_id = value["id"] schema_version = value["version"] schema_deleted = value.get("deleted", False) + schema_references = value.get("references", None) try: schema_type_parsed = SchemaType(schema_type) @@ -448,7 +451,10 @@ def _handle_msg_schema(self, key: dict, value: Optional[dict]) -> None: return elif schema_type_parsed == SchemaType.PROTOBUF: try: - parsed_schema = ValidatedTypedSchema.parse(SchemaType.PROTOBUF, schema_str) + resolved_references, resolved_dependencies = self.resolve_schema_references(value) + parsed_schema = ValidatedTypedSchema.parse( + SchemaType.PROTOBUF, schema_str, resolved_references, resolved_dependencies + ) schema_str = str(parsed_schema) except InvalidSchema: LOG.exception("Schema is not valid ProtoBuf definition") @@ -464,21 +470,34 @@ def _handle_msg_schema(self, key: dict, value: Optional[dict]) -> None: # dedup schemas to reduce memory pressure schema_str = self._hash_to_schema.setdefault(hash(schema_str), schema_str) - if schema_version in subjects_schemas: - LOG.info("Updating entry subject: %r version: %r id: %r", schema_subject, schema_version, schema_id) - else: - LOG.info("Adding entry subject: %r version: %r id: %r", schema_subject, schema_version, schema_id) - typed_schema = TypedSchema( schema_type=schema_type_parsed, schema_str=schema_str, + references=schema_references, ) - subjects_schemas[schema_version] = { + schema = { "schema": typed_schema, "version": schema_version, "id": schema_id, "deleted": schema_deleted, } + if schema_references: + schema["references"] = schema_references + + if schema_version in subjects_schemas: + LOG.info("Updating entry subject: %r version: %r id: %r", schema_subject, schema_version, schema_id) + else: + LOG.info("Adding entry subject: %r version: %r id: %r", schema_subject, schema_version, schema_id) + + subjects_schemas[schema_version] = schema + if schema_references: + for ref in schema_references: + ref_str = reference_key(ref["subject"], ref["version"]) + referents = self.referenced_by.get(ref_str, None) + if referents: + referents.append(schema_id) + else: + self.referenced_by[ref_str] = [schema_id] self.schemas[schema_id] = typed_schema self.global_schema_id = max(self.global_schema_id, schema_id) @@ -531,3 +550,62 @@ def get_schemas_list(self, *, include_deleted: bool, latest_only: bool) -> Dict[ selected_schemas = [schema for schema in selected_schemas if schema.get("deleted", False) is False] res_schemas[subject] = selected_schemas return res_schemas + + def remove_referenced_by(self, schema_id: SchemaId, references: List): + for ref in references: + key = reference_key(ref["subject"], ref["version"]) + if self.referenced_by.get(key, None) and schema_id in self.referenced_by[key]: + self.referenced_by[key].remove(schema_id) + + def resolve_references(self, references: Optional["References"] = None) -> Optional[Dict[str, Dependency]]: + if references is None: + return None + dependencies = dict() + + for r in references.val(): + subject = r["subject"] + version = r["version"] + name = r["name"] + subject_data = self.subjects.get(subject) + if subject_data is not None: + schema_data = subject_data["schemas"][version] + schema_references, schema_dependencies = self.resolve_schema_references(schema_data) + else: + raise InvalidReferences + + parsed_schema = ValidatedTypedSchema.parse( + schema_type=schema_data["schema"].schema_type, + schema_str=schema_data["schema"].schema_str, + references=schema_references, + dependencies=schema_dependencies, + ) + dependencies[name] = Dependency(name, subject, version, parsed_schema) + return dependencies + + def resolve_schema_references( + self, schema_data: Optional[dict] + ) -> Tuple[Optional[References], Optional[Dict[str, Dependency]]]: + + if schema_data is None: + raise InvalidSchema + + schema_references = schema_data.get("references") + if schema_references is None: + return None, None + + schema_type = schema_data.get("schemaType") + if schema_type is None: + schema = schema_data.get("schema") + if schema is None: + raise InvalidReferences + if isinstance(schema, TypedSchema): + schema_type = schema.schema_type + else: + schema_type = None + if schema_type != SchemaType.PROTOBUF: + raise ReferencesNotSupportedException + + schema_references = References(schema_type, schema_references) + schema_dependencies = self.resolve_references(schema_references) + + return schema_references, schema_dependencies diff --git a/karapace/schema_references.py b/karapace/schema_references.py new file mode 100644 index 000000000..6663a3d12 --- /dev/null +++ b/karapace/schema_references.py @@ -0,0 +1,27 @@ +from karapace.schema_type import SchemaType +from karapace.typing import JsonData +from karapace.utils import json_encode +from typing import Any + + +class References: + def __init__(self, schema_type: SchemaType, references: JsonData): + """Schema with type information + + Args: + schema_type (SchemaType): The type of the schema + references (str): The original schema string + """ + self.schema_type = schema_type + self.references = references + + def val(self) -> JsonData: + return self.references + + def json(self) -> str: + return str(json_encode(self.references, sort_keys=True)) + + def __eq__(self, other: Any) -> bool: + if other is None or not isinstance(other, References): + return False + return self.json() == other.json() diff --git a/karapace/schema_registry.py b/karapace/schema_registry.py index 6d30bb971..6317b36c2 100644 --- a/karapace/schema_registry.py +++ b/karapace/schema_registry.py @@ -4,9 +4,11 @@ from karapace.compatibility import check_compatibility, CompatibilityModes from karapace.compatibility.jsonschema.checks import is_incompatible from karapace.config import Config +from karapace.dependency import Dependency from karapace.errors import ( IncompatibleSchema, InvalidVersion, + ReferenceExistsException, SchemasNotFoundException, SchemaTooLargeException, SchemaVersionNotSoftDeletedException, @@ -20,8 +22,9 @@ from karapace.master_coordinator import MasterCoordinator from karapace.schema_models import SchemaType, TypedSchema, ValidatedTypedSchema from karapace.schema_reader import KafkaSchemaReader +from karapace.schema_references import References from karapace.typing import ResolvedVersion, Subject, SubjectData, Version -from karapace.utils import json_encode, KarapaceKafkaClient +from karapace.utils import json_encode, KarapaceKafkaClient, reference_key from typing import Dict, List, Optional, Tuple, Union import asyncio @@ -181,14 +184,27 @@ async def subject_delete_local(self, subject: str, permanent: bool) -> List[Reso latest_schema_id = 0 version_list = [] if permanent: + + for version, value in list(subject_schemas_all.items()): + referenced_by = self.schema_reader.referenced_by.get(reference_key(subject, version), None) + if referenced_by and len(referenced_by) > 0: + raise ReferenceExistsException(referenced_by, version) + version_list = list(subject_schemas_all) - latest_schema_id = version_list[-1] for version, value in list(subject_schemas_all.items()): schema_id = value.get("id") + references = value.get("references", None) LOG.info("Permanently deleting subject '%s' version %s (schema id=%s)", subject, version, schema_id) self.send_schema_message( - subject=subject, schema=None, schema_id=schema_id, version=version, deleted=True + subject=subject, + schema=None, + schema_id=schema_id, + version=version, + deleted=True, + references=references, ) + if references and len(references) > 0: + self.schema_reader.remove_referenced_by(schema_id, references) else: try: subject_data_live = self.subject_get(subject, include_deleted=False) @@ -197,6 +213,11 @@ async def subject_delete_local(self, subject: str, permanent: bool) -> List[Reso latest_schema_id = version_list[-1] except SchemasNotFoundException: pass + + referenced_by = self.schema_reader.referenced_by.get(reference_key(subject, latest_schema_id), None) + if referenced_by and len(referenced_by) > 0: + raise ReferenceExistsException(referenced_by, latest_schema_id) + self.send_delete_subject_message(subject, latest_schema_id) return version_list @@ -220,15 +241,23 @@ async def subject_version_delete_local(self, subject: Subject, version: Version, if permanent and not subject_schema_data.get("deleted", False): raise SchemaVersionNotSoftDeletedException() + referenced_by = self.schema_reader.referenced_by.get(reference_key(subject, int(resolved_version)), None) + if referenced_by and len(referenced_by) > 0: + raise ReferenceExistsException(referenced_by, version) + schema_id = subject_schema_data["id"] schema = subject_schema_data["schema"] + references = subject_schema_data.get("references", None) self.send_schema_message( subject=subject, schema=None if permanent else schema, schema_id=schema_id, version=resolved_version, deleted=True, + references=None, ) + if references and len(references) > 0: + self.schema_reader.remove_referenced_by(schema_id, references) return resolved_version def subject_get(self, subject: Subject, include_deleted: bool = False) -> SubjectData: @@ -265,6 +294,7 @@ async def subject_version_get(self, subject: Subject, version: Version, *, inclu "id": schema_id, "schema": schema.schema_str, } + if schema.schema_type is not SchemaType.AVRO: ret["schemaType"] = schema.schema_type # Return also compatibility information to compatibility check @@ -276,6 +306,7 @@ async def write_new_schema_local( self, subject: Subject, new_schema: ValidatedTypedSchema, + new_schema_references: Optional[References], ) -> int: """Write new schema and return new id or return id of matching existing schema @@ -315,6 +346,7 @@ async def write_new_schema_local( schema_id=schema_id, version=version, deleted=False, + references=new_schema_references, ) return schema_id @@ -336,8 +368,15 @@ async def write_new_schema_local( for old_version in check_against: old_schema = subject_data["schemas"][old_version]["schema"] + old_schema_references, old_schema_dependencies = self.resolve_schema_references( + subject_data["schemas"][old_version], + ) + validated_old_schema = ValidatedTypedSchema.parse( - schema_type=old_schema.schema_type, schema_str=old_schema.schema_str + schema_type=old_schema.schema_type, + schema_str=old_schema.schema_str, + references=old_schema_references, + dependencies=old_schema_dependencies, ) result = check_compatibility( old_schema=validated_old_schema, @@ -379,6 +418,7 @@ async def write_new_schema_local( schema_id=schema_id, version=version, deleted=False, + references=new_schema_references, ) return schema_id @@ -440,6 +480,7 @@ def send_schema_message( schema_id: int, version: int, deleted: bool, + references: Optional[References], ) -> FutureRecordMetadata: key = self.key_formatter.format_key( {"subject": subject, "version": version, "magic": 1, "keytype": "SCHEMA"}, @@ -452,6 +493,8 @@ def send_schema_message( "schema": schema.schema_str, "deleted": deleted, } + if references: + valuedict["references"] = references.val() if schema.schema_type is not SchemaType.AVRO: valuedict["schemaType"] = schema.schema_type value = json_encode(valuedict) @@ -492,3 +535,11 @@ def send_delete_subject_message(self, subject: Subject, version: Version) -> Fut ) value = '{{"subject":"{}","version":{}}}'.format(subject, version) return self.send_kafka_message(key, value) + + def resolve_references(self, references: Optional["References"] = None) -> Optional[Dict[str, Dependency]]: + return self.schema_reader.resolve_references(references) + + def resolve_schema_references( + self, schema_data: Optional[dict] + ) -> Tuple[Optional[References], Optional[Dict[str, Dependency]]]: + return self.schema_reader.resolve_schema_references(schema_data) diff --git a/karapace/schema_registry_apis.py b/karapace/schema_registry_apis.py index 27d2b1954..3200338bd 100644 --- a/karapace/schema_registry_apis.py +++ b/karapace/schema_registry_apis.py @@ -8,9 +8,12 @@ from karapace.config import Config from karapace.errors import ( IncompatibleSchema, + InvalidReferences, InvalidSchema, InvalidSchemaType, InvalidVersion, + ReferenceExistsException, + ReferencesNotSupportedException, SchemasNotFoundException, SchemaTooLargeException, SchemaVersionNotSoftDeletedException, @@ -25,6 +28,7 @@ from karapace.schema_models import SchemaType, TypedSchema, ValidatedTypedSchema from karapace.schema_registry import KarapaceSchemaRegistry, validate_version from karapace.typing import JsonData +from karapace.utils import reference_key from typing import Any, Dict, Optional, Union import aiohttp @@ -51,6 +55,9 @@ class SchemaErrorCodes(Enum): INVALID_SCHEMA = 42201 INVALID_SUBJECT = 42208 SCHEMA_TOO_LARGE_ERROR_CODE = 42209 + INVALID_REFERENCES = 44301 + REFERENCES_SUPPORT_NOT_IMPLEMENTED = 44302 + REFERENCE_EXISTS = 42206 NO_MASTER_ERROR = 50003 @@ -63,6 +70,7 @@ class SchemaErrorMessages(Enum): "full_transitive" ) SUBJECT_LEVEL_COMPATIBILITY_NOT_CONFIGURED_FMT = "Subject '%s' does not have subject-level compatibility configured" + REFERENCES_SUPPORT_NOT_IMPLEMENTED = "Schema references are not supported for '{schema_type}' schema type" class KarapaceSchemaRegistryController(KarapaceBase): @@ -226,6 +234,12 @@ def _add_schema_registry_routes(self) -> None: schema_request=True, auth=self._auth, ) + self.route( + "/subjects//versions//referencedby", + callback=self.subject_version_referencedby_get, + method="GET", + schema_request=True, + ) self.route( "/subjects/", callback=self.subject_delete, @@ -296,7 +310,13 @@ async def compatibility_check( body = request.json schema_type = self._validate_schema_type(content_type=content_type, data=body) try: - new_schema = ValidatedTypedSchema.parse(schema_type, body["schema"]) + new_schema_references, new_schema_dependencies = self.schema_registry.resolve_schema_references(body) + new_schema = ValidatedTypedSchema.parse( + schema_type=schema_type, + schema_str=body["schema"], + references=new_schema_references, + dependencies=new_schema_dependencies, + ) except InvalidSchema: self.r( body={ @@ -325,7 +345,13 @@ async def compatibility_check( old_schema_type = self._validate_schema_type(content_type=content_type, data=old) try: - old_schema = ValidatedTypedSchema.parse(old_schema_type, old["schema"]) + old_schema_references, old_schema_dependencies = self.schema_registry.resolve_schema_references(old) + old_schema = ValidatedTypedSchema.parse( + schema_type=old_schema_type, + schema_str=old["schema"], + references=old_schema_references, + dependencies=old_schema_dependencies, + ) except InvalidSchema: self.r( body={ @@ -631,6 +657,19 @@ async def subject_delete( content_type=content_type, status=HTTPStatus.NOT_FOUND, ) + + except ReferenceExistsException as arg: + self.r( + body={ + "error_code": SchemaErrorCodes.REFERENCE_EXISTS.value, + "message": ( + f"One or more references exist to the schema " + f"{{magic=1,keytype=SCHEMA,subject={subject},version={arg.version}}}" + ), + }, + content_type=content_type, + status=HTTPStatus.NOT_FOUND, + ) elif not master_url: self.no_master_error(content_type) else: @@ -722,6 +761,18 @@ async def subject_version_delete( content_type=content_type, status=HTTPStatus.NOT_FOUND, ) + except ReferenceExistsException as arg: + self.r( + body={ + "error_code": SchemaErrorCodes.REFERENCE_EXISTS.value, + "message": ( + f"One or more references exist to the schema " + f"{{magic=1,keytype=SCHEMA,subject={subject},version={arg.version}}}" + ), + }, + content_type=content_type, + status=HTTPStatus.NOT_FOUND, + ) elif not master_url: self.no_master_error(content_type) else: @@ -757,9 +808,41 @@ async def subject_version_schema_get( status=HTTPStatus.NOT_FOUND, ) + async def subject_version_referencedby_get(self, content_type, *, subject, version, user: Optional[User] = None): + self._check_authorization(user, Operation.Read, f"Subject:{subject}") + + try: + subject_data = await self.schema_registry.subject_version_get(subject, version) + except (SubjectNotFoundException, SchemasNotFoundException): + self.r( + body={ + "error_code": SchemaErrorCodes.SUBJECT_NOT_FOUND.value, + "message": SchemaErrorMessages.SUBJECT_NOT_FOUND_FMT.value.format(subject=subject), + }, + content_type=content_type, + status=HTTPStatus.NOT_FOUND, + ) + except VersionNotFoundException: + self.r( + body={ + "error_code": SchemaErrorCodes.VERSION_NOT_FOUND.value, + "message": f"Version {version} not found.", + }, + content_type=content_type, + status=HTTPStatus.NOT_FOUND, + ) + except InvalidVersion: + self._invalid_version(content_type, version) + + referenced_by = self.schema_registry.schema_reader.referenced_by.get( + reference_key(subject_data["subject"], subject_data["version"]), [] + ) + self.r(list(referenced_by), content_type, status=HTTPStatus.OK) + async def subject_versions_list( self, content_type: str, *, subject: str, request: HTTPRequest, user: Optional[User] = None ) -> None: + self._check_authorization(user, Operation.Read, f"Subject:{subject}") deleted = request.query.get("deleted", "false").lower() == "true" try: @@ -798,12 +881,12 @@ def _validate_schema_request_body(self, content_type: str, body: Union[dict, Any content_type=content_type, status=HTTPStatus.INTERNAL_SERVER_ERROR, ) - for attr in body: - if attr not in {"schema", "schemaType"}: + for field in body: + if field not in {"schema", "schemaType", "references"}: self.r( body={ "error_code": SchemaErrorCodes.HTTP_UNPROCESSABLE_ENTITY.value, - "message": f"Unrecognized field: {attr}", + "message": f"Unrecognized field: {field}", }, content_type=content_type, status=HTTPStatus.UNPROCESSABLE_ENTITY, @@ -854,7 +937,7 @@ async def subjects_schema_post( content_type=content_type, status=HTTPStatus.NOT_FOUND, ) - new_schema = None + if "schema" not in body: self.r( body={ @@ -866,8 +949,39 @@ async def subjects_schema_post( ) schema_str = body["schema"] schema_type = self._validate_schema_type(content_type=content_type, data=body) + try: - new_schema = ValidatedTypedSchema.parse(schema_type, schema_str) + new_schema_references, new_schema_dependencies = self.schema_registry.resolve_schema_references(body) + new_schema = ValidatedTypedSchema.parse( + schema_type=schema_type, + schema_str=schema_str, + references=new_schema_references, + dependencies=new_schema_dependencies, + ) + for schema in subject_data["schemas"].values(): + validated_typed_schema = ValidatedTypedSchema.parse( + schema["schema"].schema_type, schema["schema"].schema_str + ) + if schema_type is SchemaType.JSONSCHEMA: + schema_valid = validated_typed_schema.to_dict() == new_schema.to_dict() + else: + schema_valid = validated_typed_schema.schema == new_schema.schema + if ( + validated_typed_schema.schema_type == new_schema.schema_type + and schema_valid + and schema.get("references", None) == new_schema_references + ): + ret = { + "subject": subject, + "version": schema["version"], + "id": schema["id"], + "schema": validated_typed_schema.schema_str, + } + if schema_type is not SchemaType.AVRO: + ret["schemaType"] = schema_type + self.r(ret, content_type) + else: + self.log.debug("Schema %r did not match %r", schema, validated_typed_schema) except InvalidSchema: self.log.exception("No proper parser found") self.r( @@ -878,6 +992,27 @@ async def subjects_schema_post( content_type=content_type, status=HTTPStatus.INTERNAL_SERVER_ERROR, ) + except ReferencesNotSupportedException: + self.r( + body={ + "error_code": SchemaErrorCodes.REFERENCES_SUPPORT_NOT_IMPLEMENTED.value, + "message": SchemaErrorMessages.REFERENCES_SUPPORT_NOT_IMPLEMENTED.value.format( + schema_type=schema_type.value + ), + }, + content_type=content_type, + status=HTTPStatus.UNPROCESSABLE_ENTITY, + ) + except InvalidReferences: + human_error = "Provided references is not valid" + self.r( + body={ + "error_code": SchemaErrorCodes.INVALID_REFERENCES.value, + "message": f"Invalid {schema_type} references. Error: {human_error}", + }, + content_type=content_type, + status=HTTPStatus.UNPROCESSABLE_ENTITY, + ) # Match schemas based on version from latest to oldest for schema in sorted(subject_data["schemas"].values(), key=lambda item: item["version"], reverse=True): @@ -898,6 +1033,7 @@ async def subjects_schema_post( self.r(ret, content_type) else: self.log.debug("Schema %r did not match %r", schema, validated_typed_schema) + self.r( body={ "error_code": SchemaErrorCodes.SCHEMA_NOT_FOUND.value, @@ -920,7 +1056,13 @@ async def subject_post( self._validate_schema_key(content_type, body) try: - new_schema = ValidatedTypedSchema.parse(schema_type=schema_type, schema_str=body["schema"]) + new_schema_references, new_schema_dependencies = self.schema_registry.resolve_schema_references(body) + new_schema = ValidatedTypedSchema.parse( + schema_type=schema_type, + schema_str=body["schema"], + references=new_schema_references, + dependencies=new_schema_dependencies, + ) except (InvalidSchema, InvalidSchemaType) as e: self.log.warning("Invalid schema: %r", body["schema"], exc_info=True) if isinstance(e.__cause__, (SchemaParseException, json.JSONDecodeError)): @@ -935,6 +1077,27 @@ async def subject_post( content_type=content_type, status=HTTPStatus.UNPROCESSABLE_ENTITY, ) + except ReferencesNotSupportedException: + self.r( + body={ + "error_code": SchemaErrorCodes.REFERENCES_SUPPORT_NOT_IMPLEMENTED.value, + "message": SchemaErrorMessages.REFERENCES_SUPPORT_NOT_IMPLEMENTED.value.format( + schema_type=schema_type.value + ), + }, + content_type=content_type, + status=HTTPStatus.UNPROCESSABLE_ENTITY, + ) + except InvalidReferences: + human_error = "Provided references is not valid" + self.r( + body={ + "error_code": SchemaErrorCodes.INVALID_REFERENCES.value, + "message": f"Invalid {schema_type} references. Error: {human_error}", + }, + content_type=content_type, + status=HTTPStatus.UNPROCESSABLE_ENTITY, + ) schema_id = self.get_schema_id_if_exists(subject=subject, schema=new_schema) if schema_id is not None: @@ -951,7 +1114,7 @@ async def subject_post( are_we_master, master_url = await self.schema_registry.get_master() if are_we_master: try: - schema_id = await self.schema_registry.write_new_schema_local(subject, new_schema) + schema_id = await self.schema_registry.write_new_schema_local(subject, new_schema, new_schema_references) self.r( body={"id": schema_id}, content_type=content_type, diff --git a/karapace/schema_type.py b/karapace/schema_type.py new file mode 100644 index 000000000..2881c4f3f --- /dev/null +++ b/karapace/schema_type.py @@ -0,0 +1,8 @@ +from enum import Enum, unique + + +@unique +class SchemaType(str, Enum): + AVRO = "AVRO" + JSONSCHEMA = "JSON" + PROTOBUF = "PROTOBUF" diff --git a/karapace/serialization.py b/karapace/serialization.py index fa5c16fa1..4b272d9e4 100644 --- a/karapace/serialization.py +++ b/karapace/serialization.py @@ -5,7 +5,7 @@ from karapace.client import Client from karapace.protobuf.exception import ProtobufTypeException from karapace.protobuf.io import ProtobufDatumReader, ProtobufDatumWriter -from karapace.schema_models import InvalidSchema, SchemaType, TypedSchema, ValidatedTypedSchema +from karapace.schema_models import InvalidSchema, References, SchemaType, TypedSchema, ValidatedTypedSchema from karapace.utils import json_encode from typing import Any, Dict, Optional, Tuple from urllib.parse import quote @@ -75,9 +75,14 @@ def __init__( self.client = Client(server_uri=schema_registry_url, server_ca=server_ca, session_auth=session_auth) self.base_url = schema_registry_url - async def post_new_schema(self, subject: str, schema: ValidatedTypedSchema) -> int: + async def post_new_schema( + self, subject: str, schema: ValidatedTypedSchema, references: Optional[References] = None + ) -> int: if schema.schema_type is SchemaType.PROTOBUF: - payload = {"schema": str(schema), "schemaType": schema.schema_type.value} + if references: + payload = {"schema": str(schema), "schemaType": schema.schema_type.value, "references": references.json()} + else: + payload = {"schema": str(schema), "schemaType": schema.schema_type.value} else: payload = {"schema": json_encode(schema.to_dict()), "schemaType": schema.schema_type.value} result = await self.client.post(f"subjects/{quote(subject)}/versions", json=payload) @@ -94,11 +99,13 @@ async def get_latest_schema(self, subject: str) -> Tuple[int, ValidatedTypedSche raise SchemaRetrievalError(f"Invalid result format: {json_result}") try: schema_type = SchemaType(json_result.get("schemaType", "AVRO")) + return json_result["id"], ValidatedTypedSchema.parse(schema_type, json_result["schema"]) + except InvalidSchema as e: raise SchemaRetrievalError(f"Failed to parse schema string from response: {json_result}") from e - async def get_schema_for_id(self, schema_id: int) -> ValidatedTypedSchema: + async def get_schema_for_id(self, schema_id: int) -> Tuple[ValidatedTypedSchema, Optional[References]]: result = await self.client.get(f"schemas/ids/{schema_id}") if not result.ok: raise SchemaRetrievalError(result.json()["message"]) @@ -107,7 +114,15 @@ async def get_schema_for_id(self, schema_id: int) -> ValidatedTypedSchema: raise SchemaRetrievalError(f"Invalid result format: {json_result}") try: schema_type = SchemaType(json_result.get("schemaType", "AVRO")) + references_str = json_result.get("references") + if references_str: + references = References(schema_type, references_str) + else: + references = None + if references: + return ValidatedTypedSchema.parse(schema_type, json_result["schema"]), references return ValidatedTypedSchema.parse(schema_type, json_result["schema"]) + except InvalidSchema as e: raise SchemaRetrievalError(f"Failed to parse schema string from response: {json_result}") from e @@ -160,6 +175,7 @@ def get_subject_name(self, topic_name: str, schema: str, subject_type: str, sche async def get_schema_for_subject(self, subject: str) -> TypedSchema: assert self.registry_client, "must not call this method after the object is closed." + schema_id, schema = await self.registry_client.get_latest_schema(subject) async with self.state_lock: schema_ser = schema.__str__() @@ -177,6 +193,7 @@ async def get_id_for_schema(self, schema: str, subject: str, schema_type: Schema if schema_ser in self.schemas_to_ids: return self.schemas_to_ids[schema_ser] schema_id = await self.registry_client.post_new_schema(subject, schema_typed) + async with self.state_lock: self.schemas_to_ids[schema_ser] = schema_id self.ids_to_schemas[schema_id] = schema_typed diff --git a/karapace/utils.py b/karapace/utils.py index b8b077744..a3c0c229b 100644 --- a/karapace/utils.py +++ b/karapace/utils.py @@ -12,6 +12,7 @@ from decimal import Decimal from http import HTTPStatus from kafka.client_async import BrokerConnection, KafkaClient, MetadataRequest +from pathlib import Path from types import MappingProxyType from typing import NoReturn, overload, Union @@ -55,6 +56,10 @@ def default_json_serialization(obj: MappingProxyType) -> dict: ... +def reference_key(subject: str, version: int) -> str: + return hash((subject, version)) + + def default_json_serialization( # pylint: disable=inconsistent-return-statements obj: Union[datetime, timedelta, Decimal, MappingProxyType], ) -> Union[str, float, dict]: @@ -88,6 +93,10 @@ def assert_never(value: NoReturn) -> NoReturn: raise RuntimeError(f"This code should never be reached, got: {value}") +def get_project_root() -> Path: + return Path(__file__).parent.parent + + class Timeout(Exception): pass diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integration/test_client_protobuf.py b/tests/integration/test_client_protobuf.py index 476dbd2de..f924b536f 100644 --- a/tests/integration/test_client_protobuf.py +++ b/tests/integration/test_client_protobuf.py @@ -10,7 +10,7 @@ async def test_remote_client_protobuf(registry_async_client): reg_cli = SchemaRegistryClient() reg_cli.client = registry_async_client subject = new_random_name("subject") - sc_id = await reg_cli.post_new_schema(subject, schema_protobuf) + sc_id = await reg_cli.post_new_schema(subject, schema_protobuf, None) assert sc_id >= 0 stored_schema = await reg_cli.get_schema_for_id(sc_id) assert stored_schema == schema_protobuf, f"stored schema {stored_schema} is not {schema_protobuf}" @@ -25,7 +25,7 @@ async def test_remote_client_protobuf2(registry_async_client): reg_cli = SchemaRegistryClient() reg_cli.client = registry_async_client subject = new_random_name("subject") - sc_id = await reg_cli.post_new_schema(subject, schema_protobuf) + sc_id = await reg_cli.post_new_schema(subject, schema_protobuf, None) assert sc_id >= 0 stored_schema = await reg_cli.get_schema_for_id(sc_id) assert stored_schema == schema_protobuf, f"stored schema {stored_schema} is not {schema_protobuf}" diff --git a/tests/integration/test_schema.py b/tests/integration/test_schema.py index f95f111d6..6809df83f 100644 --- a/tests/integration/test_schema.py +++ b/tests/integration/test_schema.py @@ -1389,6 +1389,17 @@ async def test_schema_subject_post_invalid(registry_async_client: Client) -> Non assert res.json()["error_code"] == 40401 assert res.json()["message"] == f"Subject '{subject_3}' not found." + schema_str = json.dumps({"type": "string"}) + # Create the subject + subject_1 = subject_name_factory() + res = await registry_async_client.post( + f"subjects/{subject_1}/versions", + json={"schema": schema_str, "references": [{"name": "Customer.avro", "subject": "customer", "version": 1}]}, + ) + assert res.status_code == 422 + assert res.json()["error_code"] == 44302 + assert res.json()["message"] == "Schema references are not supported for 'AVRO' schema type" + @pytest.mark.parametrize("trail", ["", "/"]) async def test_schema_lifecycle(registry_async_client: Client, trail: str) -> None: diff --git a/tests/integration/test_schema_protobuf.py b/tests/integration/test_schema_protobuf.py index 1b58d4616..7cc1dd082 100644 --- a/tests/integration/test_schema_protobuf.py +++ b/tests/integration/test_schema_protobuf.py @@ -167,3 +167,180 @@ async def test_protobuf_schema_normalization(registry_async_client: Client, trai assert "id" in res.json() assert "schema" in res.json() assert evolved_id == res.json()["id"], "Check returns evolved id" + + +async def test_protobuf_schema_references(registry_async_client: Client) -> None: + + customer_schema = """ + |syntax = "proto3"; + |package a1; + |import "Place.proto"; + |message Customer { + | string name = 1; + | int32 code = 2; + | Place place = 3; + |} + |""" + + customer_schema = trim_margin(customer_schema) + + place_schema = """ + |syntax = "proto3"; + |package a1; + |message Place { + | string city = 1; + | int32 zone = 2; + |} + |""" + + place_schema = trim_margin(place_schema) + res = await registry_async_client.post( + "subjects/place/versions", json={"schemaType": "PROTOBUF", "schema": place_schema} + ) + assert res.status_code == 200 + + assert "id" in res.json() + + customer_references = [{"name": "Place.proto", "subject": "place", "version": 1}] + res = await registry_async_client.post( + "subjects/customer/versions", + json={"schemaType": "PROTOBUF", "schema": customer_schema, "references": customer_references}, + ) + assert res.status_code == 200 + + assert "id" in res.json() + + original_schema = """ + |syntax = "proto3"; + |package a1; + |import "Customer.proto"; + |message TestMessage { + | enum Enum { + | HIGH = 0; + | MIDDLE = 1; + | LOW = 2; + | } + | message Value { + | message Label{ + | int32 Id = 1; + | string name = 2; + | } + | Customer customer = 1; + | int32 x = 2; + | } + | string test = 1; + | .a1.TestMessage.Value val = 2; + | oneof page_info { + | option (my_option) = true; + | int32 page_number = 3; + | int32 result_per_page = 4; + | } + |} + |""" + + original_schema = trim_margin(original_schema) + references = [{"name": "Customer.proto", "subject": "customer", "version": 1}] + res = await registry_async_client.post( + "subjects/test_schema/versions", + json={"schemaType": "PROTOBUF", "schema": original_schema, "references": references}, + ) + assert res.status_code == 200 + + assert "id" in res.json() + + res = await registry_async_client.get("subjects/customer/versions/latest/referencedby", json={}) + assert res.status_code == 200 + + myjson = res.json() + referents = [3] + assert not any(x != y for x, y in zip(myjson, referents)) + + res = await registry_async_client.get("subjects/place/versions/latest/referencedby", json={}) + assert res.status_code == 200 + + myjson = res.json() + res = await registry_async_client.delete("subjects/customer/versions/1") + assert res.status_code == 404 + + match_msg = "One or more references exist to the schema {magic=1,keytype=SCHEMA,subject=customer,version=1}" + myjson = res.json() + assert myjson["error_code"] == 42206 and myjson["message"] == match_msg + + res = await registry_async_client.delete("subjects/test_schema/versions/1") + assert res.status_code == 200 + + res = await registry_async_client.delete("subjects/customer/versions/1") + assert res.status_code == 200 + + +async def test_protobuf_schema_verifier(registry_async_client: Client) -> None: + customer_schema = """ + |syntax = "proto3"; + |package a1; + |message Customer { + | string name = 1; + | int32 code = 2; + |} + |""" + + customer_schema = trim_margin(customer_schema) + res = await registry_async_client.post( + "subjects/customer/versions", json={"schemaType": "PROTOBUF", "schema": customer_schema} + ) + assert res.status_code == 200 + assert "id" in res.json() + original_schema = """ + |syntax = "proto3"; + |package a1; + |import "Customer.proto"; + |message TestMessage { + | enum Enum { + | HIGH = 0; + | MIDDLE = 1; + | LOW = 2; + | } + | message Value { + | message Label{ + | int32 Id = 1; + | string name = 2; + | } + | Customer customer = 1; + | int32 x = 2; + | } + | string test = 1; + | .a1.TestMessage.Value val = 2; + | TestMessage.Value valx = 3; + | + | oneof page_info { + | option (my_option) = true; + | int32 page_number = 5; + | int32 result_per_page = 6; + | } + |} + |""" + + original_schema = trim_margin(original_schema) + references = [{"name": "Customer.proto", "subject": "customer", "version": 1}] + res = await registry_async_client.post( + "subjects/test_schema/versions", + json={"schemaType": "PROTOBUF", "schema": original_schema, "references": references}, + ) + assert res.status_code == 200 + assert "id" in res.json() + res = await registry_async_client.get("subjects/customer/versions/latest/referencedby", json={}) + assert res.status_code == 200 + myjson = res.json() + referents = [2] + assert not any(x != y for x, y in zip(myjson, referents)) + + res = await registry_async_client.delete("subjects/customer/versions/1") + assert res.status_code == 404 + match_msg = "One or more references exist to the schema {magic=1,keytype=SCHEMA,subject=customer,version=1}" + myjson = res.json() + assert myjson["error_code"] == 42206 and myjson["message"] == match_msg + + res = await registry_async_client.delete("subjects/test_schema/versions/1") + assert res.status_code == 200 + + res = await registry_async_client.delete("subjects/customer/versions/1") + assert res.status_code == 200 diff --git a/tests/unit/test_dependency_verifier.py b/tests/unit/test_dependency_verifier.py new file mode 100644 index 000000000..49a5ddf6b --- /dev/null +++ b/tests/unit/test_dependency_verifier.py @@ -0,0 +1,54 @@ +from karapace.protobuf.dependency import ProtobufDependencyVerifier + +import logging + +log = logging.getLogger(__name__) + + +async def test_protobuf_dependency_verifier(): + declared_types = [ + ".a1.Place", + "Place", + ".a1.Customer", + "Customer", + ".a1.TestMessage", + "TestMessage", + ".a1", + ".TestMessage", + ".Enum", + "TestMessage.Enum", + ".a1.TestMessage.Value", + "TestMessage.Value", + ".a1.TestMessage.Value.Label", + "TestMessage", + ".Value.Label", + ] + + used_types = [ + ".a1.Place;string", + ".a1.Place;int32", + ".a1.Customer;string", + ".a1.Customer;int32", + ".a1.Customer;Place", + ".a1.TestMessage;int32", + ".a1.TestMessage;int32", + ".a1.TestMessage;string", + ".a1.TestMessage;.a1.TestMessage.Value", + "TestMessage;Customer", + "TestMessage;int32", + "TestMessage.Value;int32", + "TestMessage.Value;string", + ] + + verifier = ProtobufDependencyVerifier() + for declared in declared_types: + verifier.add_declared_type(declared) + for used in used_types: + x = used.split(";") + verifier.add_used_type(x[0], x[1]) + + result = verifier.verify() + assert result.result, True + + verifier.add_used_type("TestMessage.Delta", "Tag") + assert result.result, False From 7c71af472414f5bf555c69ad410c0e2eb4bf1cad Mon Sep 17 00:00:00 2001 From: Sergiy Zaschipas Date: Sat, 8 Oct 2022 00:05:48 +0300 Subject: [PATCH 02/25] add support of dependencies to compatibility check --- karapace/dependency.py | 3 + karapace/protobuf/compare_type_lists.py | 68 ++++++ karapace/protobuf/compare_type_storage.py | 16 +- karapace/protobuf/field_element.py | 6 + karapace/protobuf/message_element.py | 5 +- karapace/protobuf/proto_file_element.py | 90 +++----- karapace/protobuf/schema.py | 7 +- karapace/protobuf/type_element.py | 5 + karapace/schema_registry.py | 3 +- ...est_dependencies_compatibility_protobuf.py | 211 ++++++++++++++++++ 10 files changed, 343 insertions(+), 71 deletions(-) create mode 100644 karapace/protobuf/compare_type_lists.py create mode 100644 tests/integration/test_dependencies_compatibility_protobuf.py diff --git a/karapace/dependency.py b/karapace/dependency.py index 19736c02f..ef0e62a4c 100644 --- a/karapace/dependency.py +++ b/karapace/dependency.py @@ -11,6 +11,9 @@ def __init__(self, name: str, subject: str, version: int, schema: "ValidatedType self.version = version self.schema = schema + def get_schema(self) -> "ValidatedTypedSchema": + return self.schema + def identifier(self) -> str: return self.name + "_" + self.subject + "_" + str(self.version) diff --git a/karapace/protobuf/compare_type_lists.py b/karapace/protobuf/compare_type_lists.py new file mode 100644 index 000000000..3819949a6 --- /dev/null +++ b/karapace/protobuf/compare_type_lists.py @@ -0,0 +1,68 @@ +from itertools import chain +from karapace.protobuf.compare_result import CompareResult, Modification +from karapace.protobuf.compare_type_storage import CompareTypes +from karapace.protobuf.enum_element import EnumElement +from karapace.protobuf.exception import IllegalStateException +from karapace.protobuf.message_element import MessageElement +from karapace.protobuf.type_element import TypeElement +from typing import List + + +def compare_type_lists( + self_types_list: List[TypeElement], + other_types_list: List[TypeElement], + result: CompareResult, + compare_types: CompareTypes, +) -> CompareResult: + self_types = {} + other_types = {} + self_indexes = {} + other_indexes = {} + + type_: TypeElement + for i, type_ in enumerate(self_types_list): + self_types[type_.name] = type_ + self_indexes[type_.name] = i + compare_types.add_self_type(compare_types.self_package_name, type_) + + for i, type_ in enumerate(other_types_list): + other_types[type_.name] = type_ + other_indexes[type_.name] = i + compare_types.add_other_type(compare_types.other_package_name, type_) + + for name in chain(self_types.keys(), other_types.keys() - self_types.keys()): + + result.push_path(str(name), True) + + if self_types.get(name) is None and other_types.get(name) is not None: + if isinstance(other_types[name], MessageElement): + result.add_modification(Modification.MESSAGE_ADD) + elif isinstance(other_types[name], EnumElement): + result.add_modification(Modification.ENUM_ADD) + else: + raise IllegalStateException("Instance of element is not applicable") + elif self_types.get(name) is not None and other_types.get(name) is None: + if isinstance(self_types[name], MessageElement): + result.add_modification(Modification.MESSAGE_DROP) + elif isinstance(self_types[name], EnumElement): + result.add_modification(Modification.ENUM_DROP) + else: + raise IllegalStateException("Instance of element is not applicable") + else: + if other_indexes[name] != self_indexes[name]: + if isinstance(self_types[name], MessageElement): + # incompatible type + result.add_modification(Modification.MESSAGE_MOVE) + else: + raise IllegalStateException("Instance of element is not applicable") + else: + if isinstance(self_types[name], MessageElement) and isinstance(other_types[name], MessageElement): + self_types[name].compare(other_types[name], result, compare_types) + elif isinstance(self_types[name], EnumElement) and isinstance(other_types[name], EnumElement): + self_types[name].compare(other_types[name], result, compare_types) + else: + # incompatible type + result.add_modification(Modification.TYPE_ALTER) + result.pop_path(True) + + return result diff --git a/karapace/protobuf/compare_type_storage.py b/karapace/protobuf/compare_type_storage.py index 3a8df52e5..e727b8ce8 100644 --- a/karapace/protobuf/compare_type_storage.py +++ b/karapace/protobuf/compare_type_storage.py @@ -2,12 +2,12 @@ from karapace.protobuf.compare_result import CompareResult from karapace.protobuf.exception import IllegalArgumentException from karapace.protobuf.proto_type import ProtoType -from karapace.protobuf.type_element import TypeElement from typing import Dict, List, Optional, TYPE_CHECKING, Union if TYPE_CHECKING: from karapace.protobuf.field_element import FieldElement from karapace.protobuf.message_element import MessageElement + from karapace.protobuf.type_element import TypeElement def compute_name(t: ProtoType, result_path: List[str], package_name: str, types: dict) -> Optional[str]: @@ -35,15 +35,15 @@ def compute_name(t: ProtoType, result_path: List[str], package_name: str, types: class CompareTypes: def __init__(self, self_package_name: str, other_package_name: str, result: CompareResult) -> None: - self.self_package_name = self_package_name - self.other_package_name = other_package_name + self.self_package_name = self_package_name or "" + self.other_package_name = other_package_name or "" self.self_types: Dict[str, Union[TypeRecord, TypeRecordMap]] = {} self.other_types: Dict[str, Union[TypeRecord, TypeRecordMap]] = {} self.locked_messages: List["MessageElement"] = [] self.environment: List["MessageElement"] = [] self.result = result - def add_a_type(self, prefix: str, package_name: str, type_element: TypeElement, types: dict) -> None: + def add_a_type(self, prefix: str, package_name: str, type_element: "TypeElement", types: dict) -> None: name: str if prefix: name = prefix + "." + type_element.name @@ -65,10 +65,10 @@ def add_a_type(self, prefix: str, package_name: str, type_element: TypeElement, for t in type_element.nested_types: self.add_a_type(name, package_name, t, types) - def add_self_type(self, package_name: str, type_element: TypeElement) -> None: + def add_self_type(self, package_name: str, type_element: "TypeElement") -> None: self.add_a_type(package_name, package_name, type_element, self.self_types) - def add_other_type(self, package_name: str, type_element: TypeElement) -> None: + def add_other_type(self, package_name: str, type_element: "TypeElement") -> None: self.add_a_type(package_name, package_name, type_element, self.other_types) def get_self_type(self, t: ProtoType) -> Union[None, "TypeRecord", "TypeRecordMap"]: @@ -119,12 +119,12 @@ def unlock_message(self, message: "MessageElement") -> bool: @dataclass class TypeRecord: package_name: str - type_element: TypeElement + type_element: "TypeElement" class TypeRecordMap(TypeRecord): def __init__( - self, package_name: str, type_element: TypeElement, key: Optional["FieldElement"], value: Optional["FieldElement"] + self, package_name: str, type_element: "TypeElement", key: Optional["FieldElement"], value: Optional["FieldElement"] ) -> None: super().__init__(package_name, type_element) self.key = key diff --git a/karapace/protobuf/field_element.py b/karapace/protobuf/field_element.py index 9cc247fbd..c6dd9b0a9 100644 --- a/karapace/protobuf/field_element.py +++ b/karapace/protobuf/field_element.py @@ -1,5 +1,6 @@ # Ported from square/wire: # wire-library/wire-schema/src/commonMain/kotlin/com/squareup/wire/schema/internal/parser/FieldElement.kt +from karapace.errors import InvalidSchema from karapace.protobuf.compare_result import CompareResult, Modification from karapace.protobuf.compare_type_storage import TypeRecordMap from karapace.protobuf.field import Field @@ -143,6 +144,11 @@ def compare_message( self_type_record = types.get_self_type(self_type) other_type_record = types.get_other_type(other_type) + if self_type_record is None: + raise InvalidSchema(f"Cannot resolve type {self_type}") + if other_type_record is None: + raise InvalidSchema(f"Cannot resolve type {other_type}") + self_type_element: MessageElement = self_type_record.type_element other_type_element: MessageElement = other_type_record.type_element diff --git a/karapace/protobuf/message_element.py b/karapace/protobuf/message_element.py index d0acecd5c..054166816 100644 --- a/karapace/protobuf/message_element.py +++ b/karapace/protobuf/message_element.py @@ -22,7 +22,7 @@ def __init__( location: Location, name: str, documentation: str = "", - nested_types: List[str] = None, + nested_types: List[TypeElement] = None, options: List[OptionElement] = None, reserveds: List[ReservedElement] = None, fields: List[FieldElement] = None, @@ -80,6 +80,7 @@ def to_schema(self) -> str: return "".join(result) def compare(self, other: "MessageElement", result: CompareResult, types: CompareTypes) -> None: + from karapace.protobuf.compare_type_lists import compare_type_lists if types.lock_message(self): field: FieldElement @@ -138,5 +139,5 @@ def compare(self, other: "MessageElement", result: CompareResult, types: Compare self_one_ofs[name].compare(other_one_ofs[name], result, types) result.pop_path() - + compare_type_lists(self.nested_types, other.nested_types, result, types) types.unlock_message(self) diff --git a/karapace/protobuf/proto_file_element.py b/karapace/protobuf/proto_file_element.py index fa43dd9e8..d27e01076 100644 --- a/karapace/protobuf/proto_file_element.py +++ b/karapace/protobuf/proto_file_element.py @@ -1,15 +1,31 @@ # Ported from square/wire: # wire-library/wire-schema/src/commonMain/kotlin/com/squareup/wire/schema/internal/parser/ProtoFileElement.kt -from itertools import chain +from karapace.dependency import Dependency from karapace.protobuf.compare_result import CompareResult, Modification +from karapace.protobuf.compare_type_lists import compare_type_lists from karapace.protobuf.compare_type_storage import CompareTypes -from karapace.protobuf.enum_element import EnumElement -from karapace.protobuf.exception import IllegalStateException from karapace.protobuf.location import Location -from karapace.protobuf.message_element import MessageElement from karapace.protobuf.syntax import Syntax from karapace.protobuf.type_element import TypeElement -from typing import List, Optional +from typing import Dict, List, Optional + + +def collect_dependencies_types(compare_types: CompareTypes, dependencies: Optional[Dict[str, Dependency]], is_self: bool): + if dependencies is None: + return + for dep in dependencies.values(): + types: List[TypeElement] = dep.schema.schema.proto_file_element.types + sub_deps = dep.schema.schema.dependencies + package_name = dep.schema.schema.proto_file_element.package_name + type_: TypeElement + for type_ in types: + if is_self: + compare_types.add_self_type(package_name, type_) + else: + compare_types.add_other_type(package_name, type_) + if sub_deps is None: + return + collect_dependencies_types(compare_types, sub_deps, is_self) class ProtoFileElement: @@ -99,7 +115,13 @@ def __eq__(self, other: "ProtoFileElement") -> bool: # type: ignore def __repr__(self) -> str: return self.to_schema() - def compare(self, other: "ProtoFileElement", result: CompareResult) -> CompareResult: + def compare( + self, + other: "ProtoFileElement", + result: CompareResult, + self_dependencies: Optional[Dict[str, Dependency]] = None, + other_dependencies: Optional[Dict[str, Dependency]] = None, + ) -> CompareResult: if self.package_name != other.package_name: result.add_modification(Modification.PACKAGE_ALTER) @@ -107,57 +129,7 @@ def compare(self, other: "ProtoFileElement", result: CompareResult) -> CompareRe if self.syntax != other.syntax: result.add_modification(Modification.SYNTAX_ALTER) - self_types = {} - other_types = {} - self_indexes = {} - other_indexes = {} compare_types = CompareTypes(self.package_name, other.package_name, result) - type_: TypeElement - for i, type_ in enumerate(self.types): - self_types[type_.name] = type_ - self_indexes[type_.name] = i - package_name = self.package_name or "" - compare_types.add_self_type(package_name, type_) - - for i, type_ in enumerate(other.types): - other_types[type_.name] = type_ - other_indexes[type_.name] = i - package_name = other.package_name or "" - compare_types.add_other_type(package_name, type_) - - for name in chain(self_types.keys(), other_types.keys() - self_types.keys()): - - result.push_path(str(name), True) - - if self_types.get(name) is None and other_types.get(name) is not None: - if isinstance(other_types[name], MessageElement): - result.add_modification(Modification.MESSAGE_ADD) - elif isinstance(other_types[name], EnumElement): - result.add_modification(Modification.ENUM_ADD) - else: - raise IllegalStateException("Instance of element is not applicable") - elif self_types.get(name) is not None and other_types.get(name) is None: - if isinstance(self_types[name], MessageElement): - result.add_modification(Modification.MESSAGE_DROP) - elif isinstance(self_types[name], EnumElement): - result.add_modification(Modification.ENUM_DROP) - else: - raise IllegalStateException("Instance of element is not applicable") - else: - if other_indexes[name] != self_indexes[name]: - if isinstance(self_types[name], MessageElement): - # incompatible type - result.add_modification(Modification.MESSAGE_MOVE) - else: - raise IllegalStateException("Instance of element is not applicable") - else: - if isinstance(self_types[name], MessageElement) and isinstance(other_types[name], MessageElement): - self_types[name].compare(other_types[name], result, compare_types) - elif isinstance(self_types[name], EnumElement) and isinstance(other_types[name], EnumElement): - self_types[name].compare(other_types[name], result, compare_types) - else: - # incompatible type - result.add_modification(Modification.TYPE_ALTER) - result.pop_path(True) - - return result + collect_dependencies_types(compare_types, self_dependencies, True) + collect_dependencies_types(compare_types, other_dependencies, False) + return compare_type_lists(self.types, other.types, result, compare_types) diff --git a/karapace/protobuf/schema.py b/karapace/protobuf/schema.py index 76e319019..3a8c76720 100644 --- a/karapace/protobuf/schema.py +++ b/karapace/protobuf/schema.py @@ -223,4 +223,9 @@ def to_schema(self) -> str: return "".join(strings) def compare(self, other: "ProtobufSchema", result: CompareResult) -> CompareResult: - self.proto_file_element.compare(other.proto_file_element, result) + return self.proto_file_element.compare( + other.proto_file_element, + result, + self.dependencies, + other.dependencies, + ) diff --git a/karapace/protobuf/type_element.py b/karapace/protobuf/type_element.py index 568c6e4d5..58be0369b 100644 --- a/karapace/protobuf/type_element.py +++ b/karapace/protobuf/type_element.py @@ -1,6 +1,8 @@ # Ported from square/wire: # wire-library/wire-schema/src/commonMain/kotlin/com/squareup/wire/schema/internal/parser/TypeElement.kt from dataclasses import dataclass +from karapace.protobuf.compare_result import CompareResult +from karapace.protobuf.compare_type_storage import CompareTypes from karapace.protobuf.location import Location from typing import List, TYPE_CHECKING @@ -30,3 +32,6 @@ def __repr__(self) -> str: def __str__(self) -> str: mytype = type(self) return f"{mytype}({self.to_schema()})" + + def compare(self, other: "TypeElement", result: CompareResult, types: CompareTypes) -> None: + pass diff --git a/karapace/schema_registry.py b/karapace/schema_registry.py index 6317b36c2..715675c1f 100644 --- a/karapace/schema_registry.py +++ b/karapace/schema_registry.py @@ -294,7 +294,8 @@ async def subject_version_get(self, subject: Subject, version: Version, *, inclu "id": schema_id, "schema": schema.schema_str, } - + if schema.references is not None: + ret["references"] = schema.references if schema.schema_type is not SchemaType.AVRO: ret["schemaType"] = schema.schema_type # Return also compatibility information to compatibility check diff --git a/tests/integration/test_dependencies_compatibility_protobuf.py b/tests/integration/test_dependencies_compatibility_protobuf.py new file mode 100644 index 000000000..72cca9f6a --- /dev/null +++ b/tests/integration/test_dependencies_compatibility_protobuf.py @@ -0,0 +1,211 @@ +""" +karapace - schema tests + +Copyright (c) 2019 Aiven Ltd +See LICENSE for details +""" +from karapace.protobuf.kotlin_wrapper import trim_margin +from karapace.client import Client +from tests.utils import create_subject_name_factory + +import logging +import pytest + + + +@pytest.mark.parametrize("trail", ["", "/"]) +async def test_protobuf_schema_compatibility(registry_async_client: Client, trail: str) -> None: + subject = create_subject_name_factory(f"test_protobuf_schema_compatibility-{trail}")() + + res = await registry_async_client.put(f"config/{subject}{trail}", json={"compatibility": "BACKWARD"}) + assert res.status_code == 200 + + + original_dependencies =""" + |syntax = "proto3"; + |package a1; + |message container { + | message Hint { + | string hint_str = 1; + | } + |} + |""" + + evolved_dependencies = """ + |syntax = "proto3"; + |package a1; + |message container { + | message Hint { + | string hint_str = 1; + | } + |} + |""" + + original_dependencies = trim_margin(original_dependencies) + res = await registry_async_client.post( + "subjects/container1/versions", json={"schemaType": "PROTOBUF", "schema": original_dependencies} + ) + assert res.status_code == 200 + assert "id" in res.json() + + evolved_dependencies = trim_margin(evolved_dependencies) + res = await registry_async_client.post( + "subjects/container2/versions", json={"schemaType": "PROTOBUF", "schema": evolved_dependencies} + ) + assert res.status_code == 200 + assert "id" in res.json() + + original_schema = """ + |syntax = "proto3"; + |package a1; + |import "container1.proto"; + |message TestMessage { + | message Value { + | .a1.container.Hint hint = 1; + | int32 x = 2; + | } + | string test = 1; + | .a1.TestMessage.Value val = 2; + |} + |""" + + original_schema = trim_margin(original_schema) + + original_references = [{"name": "container1.proto", "subject": "container1", "version": 1}] + res = await registry_async_client.post( + f"subjects/{subject}/versions{trail}", json={ + "schemaType": "PROTOBUF", + "schema": original_schema, + "references": original_references + }, + ) + assert res.status_code == 200 + assert "id" in res.json() + + evolved_schema = """ + |syntax = "proto3"; + |package a1; + |import "container2.proto"; + |message TestMessage { + | message Value { + | .a1.container.Hint hint = 1; + | int32 x = 2; + | } + | string test = 1; + | .a1.TestMessage.Value val = 2; + |} + |""" + evolved_schema = trim_margin(evolved_schema) + evolved_references = [{"name": "container2.proto", "subject": "container2", "version": 1}] + res = await registry_async_client.post( + f"compatibility/subjects/{subject}/versions/latest{trail}", + json={ + "schemaType": "PROTOBUF", + "schema": evolved_schema, + "references": evolved_references + }, + ) + assert res.status_code == 200 + assert res.json() == {"is_compatible": True} + + +@pytest.mark.parametrize("trail", ["", "/"]) +async def test_protobuf_schema_compatibility2(registry_async_client: Client, trail: str) -> None: + subject = create_subject_name_factory(f"test_protobuf_schema_compatibility-{trail}")() + + res = await registry_async_client.put(f"config/{subject}{trail}", json={"compatibility": "BACKWARD"}) + assert res.status_code == 200 + + + original_dependencies =""" + |syntax = "proto3"; + |package a1; + |message container { + | message Hint { + | string hint_str = 1; + | } + |} + |""" + + evolved_dependencies = """ + |syntax = "proto3"; + |package a1; + |message container { + | message Hint { + | int32 hint_str = 1; + | } + |} + |""" + + original_dependencies = trim_margin(original_dependencies) + res = await registry_async_client.post( + "subjects/container1/versions", json={"schemaType": "PROTOBUF", "schema": original_dependencies} + ) + assert res.status_code == 200 + assert "id" in res.json() + + evolved_dependencies = trim_margin(evolved_dependencies) + res = await registry_async_client.post( + "subjects/container2/versions", json={"schemaType": "PROTOBUF", "schema": evolved_dependencies} + ) + assert res.status_code == 200 + assert "id" in res.json() + + original_schema = """ + |syntax = "proto3"; + |package a1; + |import "container1.proto"; + |message TestMessage { + | message Value { + | .a1.container.Hint hint = 1; + | int32 x = 2; + | } + | string test = 1; + | .a1.TestMessage.Value val = 2; + |} + |""" + + original_schema = trim_margin(original_schema) + + original_references = [{"name": "container1.proto", "subject": "container1", "version": 1}] + res = await registry_async_client.post( + f"subjects/{subject}/versions{trail}", json={ + "schemaType": "PROTOBUF", + "schema": original_schema, + "references": original_references + }, + ) + assert res.status_code == 200 + assert "id" in res.json() + + evolved_schema = """ + |syntax = "proto3"; + |package a1; + |import "container2.proto"; + |message TestMessage { + | message Value { + | .a1.container.Hint hint = 1; + | int32 x = 2; + | } + | string test = 1; + | .a1.TestMessage.Value val = 2; + |} + |""" + evolved_schema = trim_margin(evolved_schema) + evolved_references = [{"name": "container2.proto", "subject": "container2", "version": 1}] + res = await registry_async_client.post( + f"compatibility/subjects/{subject}/versions/latest{trail}", + json={ + "schemaType": "PROTOBUF", + "schema": evolved_schema, + "references": evolved_references + }, + ) + assert res.status_code == 200 + assert res.json() == { + "is_compatible": False + } + + + + From c85ea891d5871fc6e8cc4b9b8526a7fe32e9e4ee Mon Sep 17 00:00:00 2001 From: Sergiy Zaschipas Date: Tue, 18 Oct 2022 18:32:32 +0300 Subject: [PATCH 03/25] add few tests for workaround --- karapace/protobuf/schema.py | 1 - tests/integration/test_schema_protobuf.py | 121 +++++++++++++++++++++- 2 files changed, 118 insertions(+), 4 deletions(-) diff --git a/karapace/protobuf/schema.py b/karapace/protobuf/schema.py index 76e319019..c1fd57df4 100644 --- a/karapace/protobuf/schema.py +++ b/karapace/protobuf/schema.py @@ -1,5 +1,4 @@ # Ported from square/wire: -# Ported from square/wire: # wire-library/wire-schema/src/commonMain/kotlin/com/squareup/wire/schema/Schema.kt # Ported partially for required functionality. from karapace.dependency import Dependency, DependencyVerifierResult diff --git a/tests/integration/test_schema_protobuf.py b/tests/integration/test_schema_protobuf.py index 7cc1dd082..db2da6eb5 100644 --- a/tests/integration/test_schema_protobuf.py +++ b/tests/integration/test_schema_protobuf.py @@ -170,16 +170,19 @@ async def test_protobuf_schema_normalization(registry_async_client: Client, trai async def test_protobuf_schema_references(registry_async_client: Client) -> None: - customer_schema = """ |syntax = "proto3"; |package a1; |import "Place.proto"; + |import "google/protobuf/duration.proto"; + |import "google/type/color.proto"; |message Customer { | string name = 1; | int32 code = 2; | Place place = 3; - |} + | google.protobuf.Duration dur = 4; + | google.type.Color color = 5; + |} |""" customer_schema = trim_margin(customer_schema) @@ -258,7 +261,6 @@ async def test_protobuf_schema_references(registry_async_client: Client) -> None res = await registry_async_client.get("subjects/place/versions/latest/referencedby", json={}) assert res.status_code == 200 - myjson = res.json() res = await registry_async_client.delete("subjects/customer/versions/1") assert res.status_code == 404 @@ -273,6 +275,119 @@ async def test_protobuf_schema_references(registry_async_client: Client) -> None assert res.status_code == 200 +async def test_protobuf_schema_jjaakola_one(registry_async_client: Client) -> None: + no_ref = """ + |syntax = "proto3"; + | + |message NoReference { + | string name = 1; + |} + |""" + + no_ref = trim_margin(no_ref) + res = await registry_async_client.post("subjects/sub1/versions", json={"schemaType": "PROTOBUF", "schema": no_ref}) + assert res.status_code == 200 + assert "id" in res.json() + + + with_first_ref = """ + |syntax = "proto3"; + | + |import "NoReference.proto"; + | + |message WithReference { + | string name = 1; + | NoReference ref = 2; + |}""" + + with_first_ref = trim_margin(with_first_ref) + references = [{"name": "NoReference.proto", "subject": "sub1", "version": 1}] + + res = await registry_async_client.post( + "subjects/sub2/versions", + json={"schemaType": "PROTOBUF", "schema": with_first_ref, "references": references}, + ) + assert res.status_code == 200 + assert "id" in res.json() + + no_ref_second = """ + |syntax = "proto3"; + | + |message NoReferenceTwo { + | string name = 1; + |} + |""" + + no_ref_second = trim_margin(no_ref_second) + res = await registry_async_client.post( + "subjects/sub3/versions", json={"schemaType": "PROTOBUF", "schema": no_ref_second} + ) + assert res.status_code == 200 + assert "id" in res.json() + + add_new_ref_in_sub2 = """ + |syntax = "proto3"; + |import "NoReference.proto"; + |import "NoReferenceTwo.proto"; + |message WithReference { + | string name = 1; + | NoReference ref = 2; + | NoReferenceTwo refTwo = 3; + |} + |""" + + add_new_ref_in_sub2 = trim_margin(add_new_ref_in_sub2) + + references = [ + {"name": "NoReference.proto", "subject": "sub1", "version": 1}, + {"name": "NoReferenceTwo.proto", "subject": "sub3", "version": 1}, + ] + + res = await registry_async_client.post( + "subjects/sub2/versions", + json={"schemaType": "PROTOBUF", "schema": add_new_ref_in_sub2, "references": references}, + ) + assert res.status_code == 200 + assert "id" in res.json() + + +async def test_protobuf_schema_jjaakola_two(registry_async_client: Client) -> None: + no_ref = """ + |syntax = "proto3"; + | + |message NoReference { + | string name = 1; + |} + |""" + + no_ref = trim_margin(no_ref) + res = await registry_async_client.post("subjects/sub1/versions", json={"schemaType": "PROTOBUF", "schema": no_ref}) + assert res.status_code == 200 + assert "id" in res.json() + + with_first_ref = """ + |syntax = "proto3"; + | + |import "NoReference.proto"; + | + |message WithReference { + | string name = 1; + | NoReference ref = 2; + |}""" + + with_first_ref = trim_margin(with_first_ref) + references = [{"name": "NoReference.proto", "subject": "sub1", "version": 1}] + res = await registry_async_client.post( + "subjects/sub2/versions", + json={"schemaType": "PROTOBUF", "schema": with_first_ref, "references": references}, + ) + assert res.status_code == 200 + assert "id" in res.json() + + res = await registry_async_client.delete("subjects/subject2/versions/1") + assert res.status_code == 404 + + async def test_protobuf_schema_verifier(registry_async_client: Client) -> None: customer_schema = """ |syntax = "proto3"; From bdb677cf68e5f5788ca3b71233ed566b16c4c7eb Mon Sep 17 00:00:00 2001 From: Sergiy Zaschipas Date: Tue, 18 Oct 2022 23:39:55 +0300 Subject: [PATCH 04/25] fix bugs and add more tests --- karapace/protobuf/known_dependency.py | 2 + karapace/schema_registry.py | 48 +++++++++++-------- ...est_dependencies_compatibility_protobuf.py | 44 ++++------------- tests/integration/test_schema_protobuf.py | 9 ++-- 4 files changed, 45 insertions(+), 58 deletions(-) diff --git a/karapace/protobuf/known_dependency.py b/karapace/protobuf/known_dependency.py index 5c3e74a00..fd862c98a 100644 --- a/karapace/protobuf/known_dependency.py +++ b/karapace/protobuf/known_dependency.py @@ -127,6 +127,8 @@ def static_init(cls) -> None: cls.index[item] = key dot = item.rfind(".") cls.index_simple[item[dot + 1 :]] = key + cls.index_simple[item] = key + cls.index_simple[item[1:]] = key @static_init diff --git a/karapace/schema_registry.py b/karapace/schema_registry.py index 715675c1f..daf4af26b 100644 --- a/karapace/schema_registry.py +++ b/karapace/schema_registry.py @@ -222,12 +222,14 @@ async def subject_delete_local(self, subject: str, permanent: bool) -> List[Reso return version_list - async def subject_version_delete_local(self, subject: Subject, version: Version, permanent: bool) -> ResolvedVersion: + async def subject_version_delete_local(self, subject: Subject, version: Version, + permanent: bool) -> ResolvedVersion: async with self.schema_lock: subject_data = self.subject_get(subject, include_deleted=True) if not permanent and isinstance(version, str) and version == "latest": subject_data["schemas"] = { - key: value for (key, value) in subject_data["schemas"].items() if value.get("deleted", False) is False + key: value for (key, value) in subject_data["schemas"].items() if + value.get("deleted", False) is False } resolved_version = _resolve_version(subject_data=subject_data, version=version) subject_schema_data = subject_data["schemas"].get(resolved_version, None) @@ -247,17 +249,20 @@ async def subject_version_delete_local(self, subject: Subject, version: Version, schema_id = subject_schema_data["id"] schema = subject_schema_data["schema"] - references = subject_schema_data.get("references", None) + references_list = subject_schema_data.get("references", None) + references = None + if references_list: + references = References(schema.schema_type, references_list) self.send_schema_message( subject=subject, schema=None if permanent else schema, schema_id=schema_id, version=resolved_version, deleted=True, - references=None, + references=references ) - if references and len(references) > 0: - self.schema_reader.remove_referenced_by(schema_id, references) + if references_list and len(references_list) > 0: + self.schema_reader.remove_referenced_by(schema_id, references_list) return resolved_version def subject_get(self, subject: Subject, include_deleted: bool = False) -> SubjectData: @@ -273,7 +278,8 @@ def subject_get(self, subject: Subject, include_deleted: bool = False) -> Subjec subject_data["schemas"] = schemas return subject_data - async def subject_version_get(self, subject: Subject, version: Version, *, include_deleted: bool = False) -> SubjectData: + async def subject_version_get(self, subject: Subject, version: Version, *, + include_deleted: bool = False) -> SubjectData: validate_version(version) subject_data = self.subject_get(subject, include_deleted=include_deleted) if not subject_data: @@ -304,10 +310,10 @@ async def subject_version_get(self, subject: Subject, version: Version, *, inclu return ret async def write_new_schema_local( - self, - subject: Subject, - new_schema: ValidatedTypedSchema, - new_schema_references: Optional[References], + self, + subject: Subject, + new_schema: ValidatedTypedSchema, + new_schema_references: Optional[References], ) -> int: """Write new schema and return new id or return id of matching existing schema @@ -474,14 +480,14 @@ def send_kafka_message(self, key: Union[bytes, str], value: Union[bytes, str]) - return future def send_schema_message( - self, - *, - subject: Subject, - schema: Optional[TypedSchema], - schema_id: int, - version: int, - deleted: bool, - references: Optional[References], + self, + *, + subject: Subject, + schema: Optional[TypedSchema], + schema_id: int, + version: int, + deleted: bool, + references: Optional[References], ) -> FutureRecordMetadata: key = self.key_formatter.format_key( {"subject": subject, "version": version, "magic": 1, "keytype": "SCHEMA"}, @@ -504,7 +510,7 @@ def send_schema_message( return self.send_kafka_message(key, value) def send_config_message( - self, compatibility_level: CompatibilityModes, subject: Optional[Subject] = None + self, compatibility_level: CompatibilityModes, subject: Optional[Subject] = None ) -> FutureRecordMetadata: key = self.key_formatter.format_key( { @@ -541,6 +547,6 @@ def resolve_references(self, references: Optional["References"] = None) -> Optio return self.schema_reader.resolve_references(references) def resolve_schema_references( - self, schema_data: Optional[dict] + self, schema_data: Optional[dict] ) -> Tuple[Optional[References], Optional[Dict[str, Dependency]]]: return self.schema_reader.resolve_schema_references(schema_data) diff --git a/tests/integration/test_dependencies_compatibility_protobuf.py b/tests/integration/test_dependencies_compatibility_protobuf.py index 72cca9f6a..091ea164e 100644 --- a/tests/integration/test_dependencies_compatibility_protobuf.py +++ b/tests/integration/test_dependencies_compatibility_protobuf.py @@ -4,15 +4,13 @@ Copyright (c) 2019 Aiven Ltd See LICENSE for details """ -from karapace.protobuf.kotlin_wrapper import trim_margin from karapace.client import Client +from karapace.protobuf.kotlin_wrapper import trim_margin from tests.utils import create_subject_name_factory -import logging import pytest - @pytest.mark.parametrize("trail", ["", "/"]) async def test_protobuf_schema_compatibility(registry_async_client: Client, trail: str) -> None: subject = create_subject_name_factory(f"test_protobuf_schema_compatibility-{trail}")() @@ -20,8 +18,7 @@ async def test_protobuf_schema_compatibility(registry_async_client: Client, trai res = await registry_async_client.put(f"config/{subject}{trail}", json={"compatibility": "BACKWARD"}) assert res.status_code == 200 - - original_dependencies =""" + original_dependencies = """ |syntax = "proto3"; |package a1; |message container { @@ -73,11 +70,8 @@ async def test_protobuf_schema_compatibility(registry_async_client: Client, trai original_references = [{"name": "container1.proto", "subject": "container1", "version": 1}] res = await registry_async_client.post( - f"subjects/{subject}/versions{trail}", json={ - "schemaType": "PROTOBUF", - "schema": original_schema, - "references": original_references - }, + f"subjects/{subject}/versions{trail}", + json={"schemaType": "PROTOBUF", "schema": original_schema, "references": original_references}, ) assert res.status_code == 200 assert "id" in res.json() @@ -99,11 +93,7 @@ async def test_protobuf_schema_compatibility(registry_async_client: Client, trai evolved_references = [{"name": "container2.proto", "subject": "container2", "version": 1}] res = await registry_async_client.post( f"compatibility/subjects/{subject}/versions/latest{trail}", - json={ - "schemaType": "PROTOBUF", - "schema": evolved_schema, - "references": evolved_references - }, + json={"schemaType": "PROTOBUF", "schema": evolved_schema, "references": evolved_references}, ) assert res.status_code == 200 assert res.json() == {"is_compatible": True} @@ -116,8 +106,7 @@ async def test_protobuf_schema_compatibility2(registry_async_client: Client, tra res = await registry_async_client.put(f"config/{subject}{trail}", json={"compatibility": "BACKWARD"}) assert res.status_code == 200 - - original_dependencies =""" + original_dependencies = """ |syntax = "proto3"; |package a1; |message container { @@ -169,11 +158,8 @@ async def test_protobuf_schema_compatibility2(registry_async_client: Client, tra original_references = [{"name": "container1.proto", "subject": "container1", "version": 1}] res = await registry_async_client.post( - f"subjects/{subject}/versions{trail}", json={ - "schemaType": "PROTOBUF", - "schema": original_schema, - "references": original_references - }, + f"subjects/{subject}/versions{trail}", + json={"schemaType": "PROTOBUF", "schema": original_schema, "references": original_references}, ) assert res.status_code == 200 assert "id" in res.json() @@ -195,17 +181,7 @@ async def test_protobuf_schema_compatibility2(registry_async_client: Client, tra evolved_references = [{"name": "container2.proto", "subject": "container2", "version": 1}] res = await registry_async_client.post( f"compatibility/subjects/{subject}/versions/latest{trail}", - json={ - "schemaType": "PROTOBUF", - "schema": evolved_schema, - "references": evolved_references - }, + json={"schemaType": "PROTOBUF", "schema": evolved_schema, "references": evolved_references}, ) assert res.status_code == 200 - assert res.json() == { - "is_compatible": False - } - - - - + assert res.json() == {"is_compatible": False} diff --git a/tests/integration/test_schema_protobuf.py b/tests/integration/test_schema_protobuf.py index db2da6eb5..af36f261c 100644 --- a/tests/integration/test_schema_protobuf.py +++ b/tests/integration/test_schema_protobuf.py @@ -182,7 +182,7 @@ async def test_protobuf_schema_references(registry_async_client: Client) -> None | Place place = 3; | google.protobuf.Duration dur = 4; | google.type.Color color = 5; - |} + |} |""" customer_schema = trim_margin(customer_schema) @@ -289,7 +289,6 @@ async def test_protobuf_schema_jjaakola_one(registry_async_client: Client) -> No assert res.status_code == 200 assert "id" in res.json() - with_first_ref = """ |syntax = "proto3"; | @@ -384,7 +383,11 @@ async def test_protobuf_schema_jjaakola_two(registry_async_client: Client) -> No assert res.status_code == 200 assert "id" in res.json() - res = await registry_async_client.delete("subjects/subject2/versions/1") + res = await registry_async_client.delete("subjects/sub2/versions/1") + assert res.status_code == 200 + res = await registry_async_client.get("subjects/sub2/versions/1") + assert res.status_code == 404 + res = await registry_async_client.delete("subjects/sub2/versions/1") assert res.status_code == 404 From 492fe934b05b910d023443d690b1fe685c5ccaec Mon Sep 17 00:00:00 2001 From: Sergiy Zaschipas Date: Wed, 5 Oct 2022 23:32:37 +0300 Subject: [PATCH 05/25] Add support of references/dependencies --- .gitignore | 1 + karapace/dependency.py | 21 + karapace/errors.py | 18 + karapace/protobuf/dependency.py | 56 ++ karapace/protobuf/exception.py | 4 + karapace/protobuf/google/protobuf/any.proto | 158 +++ .../protobuf/google/protobuf/any_test.proto | 44 + karapace/protobuf/google/protobuf/api.proto | 208 ++++ .../protobuf/google/protobuf/descriptor.proto | 918 ++++++++++++++++++ .../protobuf/google/protobuf/duration.proto | 116 +++ karapace/protobuf/google/protobuf/empty.proto | 51 + .../protobuf/google/protobuf/field_mask.proto | 245 +++++ .../google/protobuf/map_lite_unittest.proto | 131 +++ .../google/protobuf/map_proto2_unittest.proto | 91 ++ .../google/protobuf/map_unittest.proto | 125 +++ .../google/protobuf/source_context.proto | 48 + .../protobuf/google/protobuf/struct.proto | 95 ++ .../protobuf/google/protobuf/timestamp.proto | 147 +++ karapace/protobuf/google/protobuf/type.proto | 187 ++++ .../protobuf/google/protobuf/wrappers.proto | 123 +++ karapace/protobuf/google/type/BUILD.bazel | 536 ++++++++++ karapace/protobuf/google/type/README.md | 16 + .../google/type/calendar_period.proto | 56 ++ karapace/protobuf/google/type/color.proto | 174 ++++ karapace/protobuf/google/type/date.proto | 52 + karapace/protobuf/google/type/datetime.proto | 104 ++ karapace/protobuf/google/type/dayofweek.proto | 50 + karapace/protobuf/google/type/decimal.proto | 95 ++ karapace/protobuf/google/type/expr.proto | 73 ++ karapace/protobuf/google/type/fraction.proto | 33 + karapace/protobuf/google/type/interval.proto | 46 + karapace/protobuf/google/type/latlng.proto | 37 + .../protobuf/google/type/localized_text.proto | 36 + karapace/protobuf/google/type/money.proto | 42 + karapace/protobuf/google/type/month.proto | 65 ++ .../protobuf/google/type/phone_number.proto | 113 +++ .../protobuf/google/type/postal_address.proto | 134 +++ .../protobuf/google/type/quaternion.proto | 94 ++ karapace/protobuf/google/type/timeofday.proto | 44 + karapace/protobuf/google/type/type.yaml | 40 + karapace/protobuf/known_dependency.py | 154 +++ karapace/protobuf/proto_file_element.py | 19 +- karapace/protobuf/proto_parser.py | 9 +- karapace/protobuf/schema.py | 63 +- karapace/schema_models.py | 76 +- karapace/schema_reader.py | 102 +- karapace/schema_references.py | 27 + karapace/schema_registry.py | 59 +- karapace/schema_registry_apis.py | 181 +++- karapace/schema_type.py | 8 + karapace/serialization.py | 25 +- karapace/utils.py | 9 + tests/__init__.py | 0 tests/integration/test_client_protobuf.py | 4 +- tests/integration/test_schema.py | 11 + tests/integration/test_schema_protobuf.py | 177 ++++ tests/unit/test_dependency_verifier.py | 54 ++ 57 files changed, 5539 insertions(+), 66 deletions(-) create mode 100644 karapace/dependency.py create mode 100644 karapace/protobuf/dependency.py create mode 100644 karapace/protobuf/google/protobuf/any.proto create mode 100644 karapace/protobuf/google/protobuf/any_test.proto create mode 100644 karapace/protobuf/google/protobuf/api.proto create mode 100644 karapace/protobuf/google/protobuf/descriptor.proto create mode 100644 karapace/protobuf/google/protobuf/duration.proto create mode 100644 karapace/protobuf/google/protobuf/empty.proto create mode 100644 karapace/protobuf/google/protobuf/field_mask.proto create mode 100644 karapace/protobuf/google/protobuf/map_lite_unittest.proto create mode 100644 karapace/protobuf/google/protobuf/map_proto2_unittest.proto create mode 100644 karapace/protobuf/google/protobuf/map_unittest.proto create mode 100644 karapace/protobuf/google/protobuf/source_context.proto create mode 100644 karapace/protobuf/google/protobuf/struct.proto create mode 100644 karapace/protobuf/google/protobuf/timestamp.proto create mode 100644 karapace/protobuf/google/protobuf/type.proto create mode 100644 karapace/protobuf/google/protobuf/wrappers.proto create mode 100644 karapace/protobuf/google/type/BUILD.bazel create mode 100644 karapace/protobuf/google/type/README.md create mode 100644 karapace/protobuf/google/type/calendar_period.proto create mode 100644 karapace/protobuf/google/type/color.proto create mode 100644 karapace/protobuf/google/type/date.proto create mode 100644 karapace/protobuf/google/type/datetime.proto create mode 100644 karapace/protobuf/google/type/dayofweek.proto create mode 100644 karapace/protobuf/google/type/decimal.proto create mode 100644 karapace/protobuf/google/type/expr.proto create mode 100644 karapace/protobuf/google/type/fraction.proto create mode 100644 karapace/protobuf/google/type/interval.proto create mode 100644 karapace/protobuf/google/type/latlng.proto create mode 100644 karapace/protobuf/google/type/localized_text.proto create mode 100644 karapace/protobuf/google/type/money.proto create mode 100644 karapace/protobuf/google/type/month.proto create mode 100644 karapace/protobuf/google/type/phone_number.proto create mode 100644 karapace/protobuf/google/type/postal_address.proto create mode 100644 karapace/protobuf/google/type/quaternion.proto create mode 100644 karapace/protobuf/google/type/timeofday.proto create mode 100644 karapace/protobuf/google/type/type.yaml create mode 100644 karapace/protobuf/known_dependency.py create mode 100644 karapace/schema_references.py create mode 100644 karapace/schema_type.py create mode 100644 tests/__init__.py create mode 100644 tests/unit/test_dependency_verifier.py diff --git a/.gitignore b/.gitignore index 67b386bbf..4a162fa59 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,4 @@ __pycache__/ /kafka_*/ venv /karapace/version.py +.run diff --git a/karapace/dependency.py b/karapace/dependency.py new file mode 100644 index 000000000..19736c02f --- /dev/null +++ b/karapace/dependency.py @@ -0,0 +1,21 @@ +from typing import Optional, TYPE_CHECKING + +if TYPE_CHECKING: + from karapace.schema_models import ValidatedTypedSchema + + +class Dependency: + def __init__(self, name: str, subject: str, version: int, schema: "ValidatedTypedSchema") -> None: + self.name = name + self.subject = subject + self.version = version + self.schema = schema + + def identifier(self) -> str: + return self.name + "_" + self.subject + "_" + str(self.version) + + +class DependencyVerifierResult: + def __init__(self, result: bool, message: Optional[str] = ""): + self.result = result + self.message = message diff --git a/karapace/errors.py b/karapace/errors.py index af1542500..188da45d0 100644 --- a/karapace/errors.py +++ b/karapace/errors.py @@ -1,3 +1,6 @@ +from typing import List, Union + + class VersionNotFoundException(Exception): pass @@ -18,6 +21,14 @@ class InvalidSchemaType(Exception): pass +class InvalidReferences(Exception): + pass + + +class ReferencesNotSupportedException(Exception): + pass + + class SchemasNotFoundException(Exception): pass @@ -38,6 +49,13 @@ class SubjectNotSoftDeletedException(Exception): pass +class ReferenceExistsException(Exception): + def __init__(self, referenced_by: List, version: Union[int, str]): + super().__init__() + self.version = version + self.referenced_by = referenced_by + + class SubjectSoftDeletedException(Exception): pass diff --git a/karapace/protobuf/dependency.py b/karapace/protobuf/dependency.py new file mode 100644 index 000000000..126b7c887 --- /dev/null +++ b/karapace/protobuf/dependency.py @@ -0,0 +1,56 @@ +from karapace.dependency import DependencyVerifierResult +from karapace.protobuf.known_dependency import DependenciesHardcoded, KnownDependency +from karapace.protobuf.one_of_element import OneOfElement +from typing import List + + +class ProtobufDependencyVerifier: + def __init__(self) -> None: + self.declared_types: List[str] = [] + self.used_types: List[str] = [] + self.import_path: List[str] = [] + + def add_declared_type(self, full_name: str) -> None: + self.declared_types.append(full_name) + + def add_used_type(self, parent: str, element_type: str) -> None: + if element_type.find("map<") == 0: + end = element_type.find(">") + virgule = element_type.find(",") + key = element_type[4:virgule] + value = element_type[virgule + 1 : end] + value = value.strip() + self.used_types.append(parent + ";" + key) + self.used_types.append(parent + ";" + value) + else: + self.used_types.append(parent + ";" + element_type) + + def add_import(self, import_name: str) -> None: + self.import_path.append(import_name) + + def verify(self) -> DependencyVerifierResult: + declared_index = set(self.declared_types) + for used_type in self.used_types: + delimiter = used_type.rfind(";") + used_type_with_scope = "" + if delimiter != -1: + used_type_with_scope = used_type[:delimiter] + "." + used_type[delimiter + 1 :] + used_type = used_type[delimiter + 1 :] + + if not ( + used_type in DependenciesHardcoded.index + or KnownDependency.index_simple.get(used_type) is not None + or KnownDependency.index.get(used_type) is not None + or used_type in declared_index + or (delimiter != -1 and used_type_with_scope in declared_index) + or "." + used_type in declared_index + ): + return DependencyVerifierResult(False, f"type {used_type} is not defined") + + return DependencyVerifierResult(True) + + +def _process_one_of(verifier: ProtobufDependencyVerifier, package_name: str, parent_name: str, one_of: OneOfElement) -> None: + parent = package_name + "." + parent_name + for field in one_of.fields: + verifier.add_used_type(parent, field.element_type) diff --git a/karapace/protobuf/exception.py b/karapace/protobuf/exception.py index b42d6e141..7f286c343 100644 --- a/karapace/protobuf/exception.py +++ b/karapace/protobuf/exception.py @@ -25,6 +25,10 @@ class ProtobufTypeException(Error): """Generic Protobuf type error.""" +class ProtobufUnresolvedDependencyException(ProtobufException): + """a Protobuf schema has unresolved dependency""" + + class SchemaParseException(ProtobufException): """Error while parsing a Protobuf schema descriptor.""" diff --git a/karapace/protobuf/google/protobuf/any.proto b/karapace/protobuf/google/protobuf/any.proto new file mode 100644 index 000000000..e2c2042fd --- /dev/null +++ b/karapace/protobuf/google/protobuf/any.proto @@ -0,0 +1,158 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package google.protobuf; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option go_package = "google.golang.org/protobuf/types/known/anypb"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "AnyProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; + +// `Any` contains an arbitrary serialized protocol buffer message along with a +// URL that describes the type of the serialized message. +// +// Protobuf library provides support to pack/unpack Any values in the form +// of utility functions or additional generated methods of the Any type. +// +// Example 1: Pack and unpack a message in C++. +// +// Foo foo = ...; +// Any any; +// any.PackFrom(foo); +// ... +// if (any.UnpackTo(&foo)) { +// ... +// } +// +// Example 2: Pack and unpack a message in Java. +// +// Foo foo = ...; +// Any any = Any.pack(foo); +// ... +// if (any.is(Foo.class)) { +// foo = any.unpack(Foo.class); +// } +// +// Example 3: Pack and unpack a message in Python. +// +// foo = Foo(...) +// any = Any() +// any.Pack(foo) +// ... +// if any.Is(Foo.DESCRIPTOR): +// any.Unpack(foo) +// ... +// +// Example 4: Pack and unpack a message in Go +// +// foo := &pb.Foo{...} +// any, err := anypb.New(foo) +// if err != nil { +// ... +// } +// ... +// foo := &pb.Foo{} +// if err := any.UnmarshalTo(foo); err != nil { +// ... +// } +// +// The pack methods provided by protobuf library will by default use +// 'type.googleapis.com/full.type.name' as the type URL and the unpack +// methods only use the fully qualified type name after the last '/' +// in the type URL, for example "foo.bar.com/x/y.z" will yield type +// name "y.z". +// +// +// JSON +// +// The JSON representation of an `Any` value uses the regular +// representation of the deserialized, embedded message, with an +// additional field `@type` which contains the type URL. Example: +// +// package google.profile; +// message Person { +// string first_name = 1; +// string last_name = 2; +// } +// +// { +// "@type": "type.googleapis.com/google.profile.Person", +// "firstName": , +// "lastName": +// } +// +// If the embedded message type is well-known and has a custom JSON +// representation, that representation will be embedded adding a field +// `value` which holds the custom JSON in addition to the `@type` +// field. Example (for message [google.protobuf.Duration][]): +// +// { +// "@type": "type.googleapis.com/google.protobuf.Duration", +// "value": "1.212s" +// } +// +message Any { + // A URL/resource name that uniquely identifies the type of the serialized + // protocol buffer message. This string must contain at least + // one "/" character. The last segment of the URL's path must represent + // the fully qualified name of the type (as in + // `path/google.protobuf.Duration`). The name should be in a canonical form + // (e.g., leading "." is not accepted). + // + // In practice, teams usually precompile into the binary all types that they + // expect it to use in the context of Any. However, for URLs which use the + // scheme `http`, `https`, or no scheme, one can optionally set up a type + // server that maps type URLs to message definitions as follows: + // + // * If no scheme is provided, `https` is assumed. + // * An HTTP GET on the URL must yield a [google.protobuf.Type][] + // value in binary format, or produce an error. + // * Applications are allowed to cache lookup results based on the + // URL, or have them precompiled into a binary to avoid any + // lookup. Therefore, binary compatibility needs to be preserved + // on changes to types. (Use versioned type names to manage + // breaking changes.) + // + // Note: this functionality is not currently available in the official + // protobuf release, and it is not used for type URLs beginning with + // type.googleapis.com. + // + // Schemes other than `http`, `https` (or the empty scheme) might be + // used with implementation specific semantics. + // + string type_url = 1; + + // Must be a valid serialized protocol buffer of the above specified type. + bytes value = 2; +} diff --git a/karapace/protobuf/google/protobuf/any_test.proto b/karapace/protobuf/google/protobuf/any_test.proto new file mode 100644 index 000000000..256035b44 --- /dev/null +++ b/karapace/protobuf/google/protobuf/any_test.proto @@ -0,0 +1,44 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package protobuf_unittest; + +import "google/protobuf/any.proto"; + +option java_outer_classname = "TestAnyProto"; + +message TestAny { + int32 int32_value = 1; + google.protobuf.Any any_value = 2; + repeated google.protobuf.Any repeated_any_value = 3; + string text = 4; +} diff --git a/karapace/protobuf/google/protobuf/api.proto b/karapace/protobuf/google/protobuf/api.proto new file mode 100644 index 000000000..3d598fc85 --- /dev/null +++ b/karapace/protobuf/google/protobuf/api.proto @@ -0,0 +1,208 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package google.protobuf; + +import "google/protobuf/source_context.proto"; +import "google/protobuf/type.proto"; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "ApiProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; +option go_package = "google.golang.org/protobuf/types/known/apipb"; + +// Api is a light-weight descriptor for an API Interface. +// +// Interfaces are also described as "protocol buffer services" in some contexts, +// such as by the "service" keyword in a .proto file, but they are different +// from API Services, which represent a concrete implementation of an interface +// as opposed to simply a description of methods and bindings. They are also +// sometimes simply referred to as "APIs" in other contexts, such as the name of +// this message itself. See https://cloud.google.com/apis/design/glossary for +// detailed terminology. +message Api { + // The fully qualified name of this interface, including package name + // followed by the interface's simple name. + string name = 1; + + // The methods of this interface, in unspecified order. + repeated Method methods = 2; + + // Any metadata attached to the interface. + repeated Option options = 3; + + // A version string for this interface. If specified, must have the form + // `major-version.minor-version`, as in `1.10`. If the minor version is + // omitted, it defaults to zero. If the entire version field is empty, the + // major version is derived from the package name, as outlined below. If the + // field is not empty, the version in the package name will be verified to be + // consistent with what is provided here. + // + // The versioning schema uses [semantic + // versioning](http://semver.org) where the major version number + // indicates a breaking change and the minor version an additive, + // non-breaking change. Both version numbers are signals to users + // what to expect from different versions, and should be carefully + // chosen based on the product plan. + // + // The major version is also reflected in the package name of the + // interface, which must end in `v`, as in + // `google.feature.v1`. For major versions 0 and 1, the suffix can + // be omitted. Zero major versions must only be used for + // experimental, non-GA interfaces. + // + // + string version = 4; + + // Source context for the protocol buffer service represented by this + // message. + SourceContext source_context = 5; + + // Included interfaces. See [Mixin][]. + repeated Mixin mixins = 6; + + // The source syntax of the service. + Syntax syntax = 7; +} + +// Method represents a method of an API interface. +message Method { + // The simple name of this method. + string name = 1; + + // A URL of the input message type. + string request_type_url = 2; + + // If true, the request is streamed. + bool request_streaming = 3; + + // The URL of the output message type. + string response_type_url = 4; + + // If true, the response is streamed. + bool response_streaming = 5; + + // Any metadata attached to the method. + repeated Option options = 6; + + // The source syntax of this method. + Syntax syntax = 7; +} + +// Declares an API Interface to be included in this interface. The including +// interface must redeclare all the methods from the included interface, but +// documentation and options are inherited as follows: +// +// - If after comment and whitespace stripping, the documentation +// string of the redeclared method is empty, it will be inherited +// from the original method. +// +// - Each annotation belonging to the service config (http, +// visibility) which is not set in the redeclared method will be +// inherited. +// +// - If an http annotation is inherited, the path pattern will be +// modified as follows. Any version prefix will be replaced by the +// version of the including interface plus the [root][] path if +// specified. +// +// Example of a simple mixin: +// +// package google.acl.v1; +// service AccessControl { +// // Get the underlying ACL object. +// rpc GetAcl(GetAclRequest) returns (Acl) { +// option (google.api.http).get = "/v1/{resource=**}:getAcl"; +// } +// } +// +// package google.storage.v2; +// service Storage { +// rpc GetAcl(GetAclRequest) returns (Acl); +// +// // Get a data record. +// rpc GetData(GetDataRequest) returns (Data) { +// option (google.api.http).get = "/v2/{resource=**}"; +// } +// } +// +// Example of a mixin configuration: +// +// apis: +// - name: google.storage.v2.Storage +// mixins: +// - name: google.acl.v1.AccessControl +// +// The mixin construct implies that all methods in `AccessControl` are +// also declared with same name and request/response types in +// `Storage`. A documentation generator or annotation processor will +// see the effective `Storage.GetAcl` method after inheriting +// documentation and annotations as follows: +// +// service Storage { +// // Get the underlying ACL object. +// rpc GetAcl(GetAclRequest) returns (Acl) { +// option (google.api.http).get = "/v2/{resource=**}:getAcl"; +// } +// ... +// } +// +// Note how the version in the path pattern changed from `v1` to `v2`. +// +// If the `root` field in the mixin is specified, it should be a +// relative path under which inherited HTTP paths are placed. Example: +// +// apis: +// - name: google.storage.v2.Storage +// mixins: +// - name: google.acl.v1.AccessControl +// root: acls +// +// This implies the following inherited HTTP annotation: +// +// service Storage { +// // Get the underlying ACL object. +// rpc GetAcl(GetAclRequest) returns (Acl) { +// option (google.api.http).get = "/v2/acls/{resource=**}:getAcl"; +// } +// ... +// } +message Mixin { + // The fully qualified name of the interface which is included. + string name = 1; + + // If non-empty specifies a path under which inherited HTTP paths + // are rooted. + string root = 2; +} diff --git a/karapace/protobuf/google/protobuf/descriptor.proto b/karapace/protobuf/google/protobuf/descriptor.proto new file mode 100644 index 000000000..5b4d06b41 --- /dev/null +++ b/karapace/protobuf/google/protobuf/descriptor.proto @@ -0,0 +1,918 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Author: kenton@google.com (Kenton Varda) +// Based on original Protocol Buffers design by +// Sanjay Ghemawat, Jeff Dean, and others. +// +// The messages in this file describe the definitions found in .proto files. +// A valid .proto file can be translated directly to a FileDescriptorProto +// without any other information (e.g. without reading its imports). + + +syntax = "proto2"; + +package google.protobuf; + +option go_package = "google.golang.org/protobuf/types/descriptorpb"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "DescriptorProtos"; +option csharp_namespace = "Google.Protobuf.Reflection"; +option objc_class_prefix = "GPB"; +option cc_enable_arenas = true; + +// descriptor.proto must be optimized for speed because reflection-based +// algorithms don't work during bootstrapping. +option optimize_for = SPEED; + +// The protocol compiler can output a FileDescriptorSet containing the .proto +// files it parses. +message FileDescriptorSet { + repeated FileDescriptorProto file = 1; +} + +// Describes a complete .proto file. +message FileDescriptorProto { + optional string name = 1; // file name, relative to root of source tree + optional string package = 2; // e.g. "foo", "foo.bar", etc. + + // Names of files imported by this file. + repeated string dependency = 3; + // Indexes of the public imported files in the dependency list above. + repeated int32 public_dependency = 10; + // Indexes of the weak imported files in the dependency list. + // For Google-internal migration only. Do not use. + repeated int32 weak_dependency = 11; + + // All top-level definitions in this file. + repeated DescriptorProto message_type = 4; + repeated EnumDescriptorProto enum_type = 5; + repeated ServiceDescriptorProto service = 6; + repeated FieldDescriptorProto extension = 7; + + optional FileOptions options = 8; + + // This field contains optional information about the original source code. + // You may safely remove this entire field without harming runtime + // functionality of the descriptors -- the information is needed only by + // development tools. + optional SourceCodeInfo source_code_info = 9; + + // The syntax of the proto file. + // The supported values are "proto2" and "proto3". + optional string syntax = 12; +} + +// Describes a message type. +message DescriptorProto { + optional string name = 1; + + repeated FieldDescriptorProto field = 2; + repeated FieldDescriptorProto extension = 6; + + repeated DescriptorProto nested_type = 3; + repeated EnumDescriptorProto enum_type = 4; + + message ExtensionRange { + optional int32 start = 1; // Inclusive. + optional int32 end = 2; // Exclusive. + + optional ExtensionRangeOptions options = 3; + } + repeated ExtensionRange extension_range = 5; + + repeated OneofDescriptorProto oneof_decl = 8; + + optional MessageOptions options = 7; + + // Range of reserved tag numbers. Reserved tag numbers may not be used by + // fields or extension ranges in the same message. Reserved ranges may + // not overlap. + message ReservedRange { + optional int32 start = 1; // Inclusive. + optional int32 end = 2; // Exclusive. + } + repeated ReservedRange reserved_range = 9; + // Reserved field names, which may not be used by fields in the same message. + // A given name may only be reserved once. + repeated string reserved_name = 10; +} + +message ExtensionRangeOptions { + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; +} + +// Describes a field within a message. +message FieldDescriptorProto { + enum Type { + // 0 is reserved for errors. + // Order is weird for historical reasons. + TYPE_DOUBLE = 1; + TYPE_FLOAT = 2; + // Not ZigZag encoded. Negative numbers take 10 bytes. Use TYPE_SINT64 if + // negative values are likely. + TYPE_INT64 = 3; + TYPE_UINT64 = 4; + // Not ZigZag encoded. Negative numbers take 10 bytes. Use TYPE_SINT32 if + // negative values are likely. + TYPE_INT32 = 5; + TYPE_FIXED64 = 6; + TYPE_FIXED32 = 7; + TYPE_BOOL = 8; + TYPE_STRING = 9; + // Tag-delimited aggregate. + // Group type is deprecated and not supported in proto3. However, Proto3 + // implementations should still be able to parse the group wire format and + // treat group fields as unknown fields. + TYPE_GROUP = 10; + TYPE_MESSAGE = 11; // Length-delimited aggregate. + + // New in version 2. + TYPE_BYTES = 12; + TYPE_UINT32 = 13; + TYPE_ENUM = 14; + TYPE_SFIXED32 = 15; + TYPE_SFIXED64 = 16; + TYPE_SINT32 = 17; // Uses ZigZag encoding. + TYPE_SINT64 = 18; // Uses ZigZag encoding. + } + + enum Label { + // 0 is reserved for errors + LABEL_OPTIONAL = 1; + LABEL_REQUIRED = 2; + LABEL_REPEATED = 3; + } + + optional string name = 1; + optional int32 number = 3; + optional Label label = 4; + + // If type_name is set, this need not be set. If both this and type_name + // are set, this must be one of TYPE_ENUM, TYPE_MESSAGE or TYPE_GROUP. + optional Type type = 5; + + // For message and enum types, this is the name of the type. If the name + // starts with a '.', it is fully-qualified. Otherwise, C++-like scoping + // rules are used to find the type (i.e. first the nested types within this + // message are searched, then within the parent, on up to the root + // namespace). + optional string type_name = 6; + + // For extensions, this is the name of the type being extended. It is + // resolved in the same manner as type_name. + optional string extendee = 2; + + // For numeric types, contains the original text representation of the value. + // For booleans, "true" or "false". + // For strings, contains the default text contents (not escaped in any way). + // For bytes, contains the C escaped value. All bytes >= 128 are escaped. + optional string default_value = 7; + + // If set, gives the index of a oneof in the containing type's oneof_decl + // list. This field is a member of that oneof. + optional int32 oneof_index = 9; + + // JSON name of this field. The value is set by protocol compiler. If the + // user has set a "json_name" option on this field, that option's value + // will be used. Otherwise, it's deduced from the field's name by converting + // it to camelCase. + optional string json_name = 10; + + optional FieldOptions options = 8; + + // If true, this is a proto3 "optional". When a proto3 field is optional, it + // tracks presence regardless of field type. + // + // When proto3_optional is true, this field must be belong to a oneof to + // signal to old proto3 clients that presence is tracked for this field. This + // oneof is known as a "synthetic" oneof, and this field must be its sole + // member (each proto3 optional field gets its own synthetic oneof). Synthetic + // oneofs exist in the descriptor only, and do not generate any API. Synthetic + // oneofs must be ordered after all "real" oneofs. + // + // For message fields, proto3_optional doesn't create any semantic change, + // since non-repeated message fields always track presence. However it still + // indicates the semantic detail of whether the user wrote "optional" or not. + // This can be useful for round-tripping the .proto file. For consistency we + // give message fields a synthetic oneof also, even though it is not required + // to track presence. This is especially important because the parser can't + // tell if a field is a message or an enum, so it must always create a + // synthetic oneof. + // + // Proto2 optional fields do not set this flag, because they already indicate + // optional with `LABEL_OPTIONAL`. + optional bool proto3_optional = 17; +} + +// Describes a oneof. +message OneofDescriptorProto { + optional string name = 1; + optional OneofOptions options = 2; +} + +// Describes an enum type. +message EnumDescriptorProto { + optional string name = 1; + + repeated EnumValueDescriptorProto value = 2; + + optional EnumOptions options = 3; + + // Range of reserved numeric values. Reserved values may not be used by + // entries in the same enum. Reserved ranges may not overlap. + // + // Note that this is distinct from DescriptorProto.ReservedRange in that it + // is inclusive such that it can appropriately represent the entire int32 + // domain. + message EnumReservedRange { + optional int32 start = 1; // Inclusive. + optional int32 end = 2; // Inclusive. + } + + // Range of reserved numeric values. Reserved numeric values may not be used + // by enum values in the same enum declaration. Reserved ranges may not + // overlap. + repeated EnumReservedRange reserved_range = 4; + + // Reserved enum value names, which may not be reused. A given name may only + // be reserved once. + repeated string reserved_name = 5; +} + +// Describes a value within an enum. +message EnumValueDescriptorProto { + optional string name = 1; + optional int32 number = 2; + + optional EnumValueOptions options = 3; +} + +// Describes a service. +message ServiceDescriptorProto { + optional string name = 1; + repeated MethodDescriptorProto method = 2; + + optional ServiceOptions options = 3; +} + +// Describes a method of a service. +message MethodDescriptorProto { + optional string name = 1; + + // Input and output type names. These are resolved in the same way as + // FieldDescriptorProto.type_name, but must refer to a message type. + optional string input_type = 2; + optional string output_type = 3; + + optional MethodOptions options = 4; + + // Identifies if client streams multiple client messages + optional bool client_streaming = 5 [default = false]; + // Identifies if server streams multiple server messages + optional bool server_streaming = 6 [default = false]; +} + + +// =================================================================== +// Options + +// Each of the definitions above may have "options" attached. These are +// just annotations which may cause code to be generated slightly differently +// or may contain hints for code that manipulates protocol messages. +// +// Clients may define custom options as extensions of the *Options messages. +// These extensions may not yet be known at parsing time, so the parser cannot +// store the values in them. Instead it stores them in a field in the *Options +// message called uninterpreted_option. This field must have the same name +// across all *Options messages. We then use this field to populate the +// extensions when we build a descriptor, at which point all protos have been +// parsed and so all extensions are known. +// +// Extension numbers for custom options may be chosen as follows: +// * For options which will only be used within a single application or +// organization, or for experimental options, use field numbers 50000 +// through 99999. It is up to you to ensure that you do not use the +// same number for multiple options. +// * For options which will be published and used publicly by multiple +// independent entities, e-mail protobuf-global-extension-registry@google.com +// to reserve extension numbers. Simply provide your project name (e.g. +// Objective-C plugin) and your project website (if available) -- there's no +// need to explain how you intend to use them. Usually you only need one +// extension number. You can declare multiple options with only one extension +// number by putting them in a sub-message. See the Custom Options section of +// the docs for examples: +// https://developers.google.com/protocol-buffers/docs/proto#options +// If this turns out to be popular, a web service will be set up +// to automatically assign option numbers. + +message FileOptions { + + // Sets the Java package where classes generated from this .proto will be + // placed. By default, the proto package is used, but this is often + // inappropriate because proto packages do not normally start with backwards + // domain names. + optional string java_package = 1; + + + // Controls the name of the wrapper Java class generated for the .proto file. + // That class will always contain the .proto file's getDescriptor() method as + // well as any top-level extensions defined in the .proto file. + // If java_multiple_files is disabled, then all the other classes from the + // .proto file will be nested inside the single wrapper outer class. + optional string java_outer_classname = 8; + + // If enabled, then the Java code generator will generate a separate .java + // file for each top-level message, enum, and service defined in the .proto + // file. Thus, these types will *not* be nested inside the wrapper class + // named by java_outer_classname. However, the wrapper class will still be + // generated to contain the file's getDescriptor() method as well as any + // top-level extensions defined in the file. + optional bool java_multiple_files = 10 [default = false]; + + // This option does nothing. + optional bool java_generate_equals_and_hash = 20 [deprecated=true]; + + // If set true, then the Java2 code generator will generate code that + // throws an exception whenever an attempt is made to assign a non-UTF-8 + // byte sequence to a string field. + // Message reflection will do the same. + // However, an extension field still accepts non-UTF-8 byte sequences. + // This option has no effect on when used with the lite runtime. + optional bool java_string_check_utf8 = 27 [default = false]; + + + // Generated classes can be optimized for speed or code size. + enum OptimizeMode { + SPEED = 1; // Generate complete code for parsing, serialization, + // etc. + CODE_SIZE = 2; // Use ReflectionOps to implement these methods. + LITE_RUNTIME = 3; // Generate code using MessageLite and the lite runtime. + } + optional OptimizeMode optimize_for = 9 [default = SPEED]; + + // Sets the Go package where structs generated from this .proto will be + // placed. If omitted, the Go package will be derived from the following: + // - The basename of the package import path, if provided. + // - Otherwise, the package statement in the .proto file, if present. + // - Otherwise, the basename of the .proto file, without extension. + optional string go_package = 11; + + + + + // Should generic services be generated in each language? "Generic" services + // are not specific to any particular RPC system. They are generated by the + // main code generators in each language (without additional plugins). + // Generic services were the only kind of service generation supported by + // early versions of google.protobuf. + // + // Generic services are now considered deprecated in favor of using plugins + // that generate code specific to your particular RPC system. Therefore, + // these default to false. Old code which depends on generic services should + // explicitly set them to true. + optional bool cc_generic_services = 16 [default = false]; + optional bool java_generic_services = 17 [default = false]; + optional bool py_generic_services = 18 [default = false]; + optional bool php_generic_services = 42 [default = false]; + + // Is this file deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for everything in the file, or it will be completely ignored; in the very + // least, this is a formalization for deprecating files. + optional bool deprecated = 23 [default = false]; + + // Enables the use of arenas for the proto messages in this file. This applies + // only to generated classes for C++. + optional bool cc_enable_arenas = 31 [default = true]; + + + // Sets the objective c class prefix which is prepended to all objective c + // generated classes from this .proto. There is no default. + optional string objc_class_prefix = 36; + + // Namespace for generated classes; defaults to the package. + optional string csharp_namespace = 37; + + // By default Swift generators will take the proto package and CamelCase it + // replacing '.' with underscore and use that to prefix the types/symbols + // defined. When this options is provided, they will use this value instead + // to prefix the types/symbols defined. + optional string swift_prefix = 39; + + // Sets the php class prefix which is prepended to all php generated classes + // from this .proto. Default is empty. + optional string php_class_prefix = 40; + + // Use this option to change the namespace of php generated classes. Default + // is empty. When this option is empty, the package name will be used for + // determining the namespace. + optional string php_namespace = 41; + + // Use this option to change the namespace of php generated metadata classes. + // Default is empty. When this option is empty, the proto file name will be + // used for determining the namespace. + optional string php_metadata_namespace = 44; + + // Use this option to change the package of ruby generated classes. Default + // is empty. When this option is not set, the package name will be used for + // determining the ruby package. + optional string ruby_package = 45; + + + // The parser stores options it doesn't recognize here. + // See the documentation for the "Options" section above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. + // See the documentation for the "Options" section above. + extensions 1000 to max; + + reserved 38; +} + +message MessageOptions { + // Set true to use the old proto1 MessageSet wire format for extensions. + // This is provided for backwards-compatibility with the MessageSet wire + // format. You should not use this for any other reason: It's less + // efficient, has fewer features, and is more complicated. + // + // The message must be defined exactly as follows: + // message Foo { + // option message_set_wire_format = true; + // extensions 4 to max; + // } + // Note that the message cannot have any defined fields; MessageSets only + // have extensions. + // + // All extensions of your type must be singular messages; e.g. they cannot + // be int32s, enums, or repeated messages. + // + // Because this is an option, the above two restrictions are not enforced by + // the protocol compiler. + optional bool message_set_wire_format = 1 [default = false]; + + // Disables the generation of the standard "descriptor()" accessor, which can + // conflict with a field of the same name. This is meant to make migration + // from proto1 easier; new code should avoid fields named "descriptor". + optional bool no_standard_descriptor_accessor = 2 [default = false]; + + // Is this message deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for the message, or it will be completely ignored; in the very least, + // this is a formalization for deprecating messages. + optional bool deprecated = 3 [default = false]; + + reserved 4, 5, 6; + + // Whether the message is an automatically generated map entry type for the + // maps field. + // + // For maps fields: + // map map_field = 1; + // The parsed descriptor looks like: + // message MapFieldEntry { + // option map_entry = true; + // optional KeyType key = 1; + // optional ValueType value = 2; + // } + // repeated MapFieldEntry map_field = 1; + // + // Implementations may choose not to generate the map_entry=true message, but + // use a native map in the target language to hold the keys and values. + // The reflection APIs in such implementations still need to work as + // if the field is a repeated message field. + // + // NOTE: Do not set the option in .proto files. Always use the maps syntax + // instead. The option should only be implicitly set by the proto compiler + // parser. + optional bool map_entry = 7; + + reserved 8; // javalite_serializable + reserved 9; // javanano_as_lite + + + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; +} + +message FieldOptions { + // The ctype option instructs the C++ code generator to use a different + // representation of the field than it normally would. See the specific + // options below. This option is not yet implemented in the open source + // release -- sorry, we'll try to include it in a future version! + optional CType ctype = 1 [default = STRING]; + enum CType { + // Default mode. + STRING = 0; + + CORD = 1; + + STRING_PIECE = 2; + } + // The packed option can be enabled for repeated primitive fields to enable + // a more efficient representation on the wire. Rather than repeatedly + // writing the tag and type for each element, the entire array is encoded as + // a single length-delimited blob. In proto3, only explicit setting it to + // false will avoid using packed encoding. + optional bool packed = 2; + + // The jstype option determines the JavaScript type used for values of the + // field. The option is permitted only for 64 bit integral and fixed types + // (int64, uint64, sint64, fixed64, sfixed64). A field with jstype JS_STRING + // is represented as JavaScript string, which avoids loss of precision that + // can happen when a large value is converted to a floating point JavaScript. + // Specifying JS_NUMBER for the jstype causes the generated JavaScript code to + // use the JavaScript "number" type. The behavior of the default option + // JS_NORMAL is implementation dependent. + // + // This option is an enum to permit additional types to be added, e.g. + // goog.math.Integer. + optional JSType jstype = 6 [default = JS_NORMAL]; + enum JSType { + // Use the default type. + JS_NORMAL = 0; + + // Use JavaScript strings. + JS_STRING = 1; + + // Use JavaScript numbers. + JS_NUMBER = 2; + } + + // Should this field be parsed lazily? Lazy applies only to message-type + // fields. It means that when the outer message is initially parsed, the + // inner message's contents will not be parsed but instead stored in encoded + // form. The inner message will actually be parsed when it is first accessed. + // + // This is only a hint. Implementations are free to choose whether to use + // eager or lazy parsing regardless of the value of this option. However, + // setting this option true suggests that the protocol author believes that + // using lazy parsing on this field is worth the additional bookkeeping + // overhead typically needed to implement it. + // + // This option does not affect the public interface of any generated code; + // all method signatures remain the same. Furthermore, thread-safety of the + // interface is not affected by this option; const methods remain safe to + // call from multiple threads concurrently, while non-const methods continue + // to require exclusive access. + // + // + // Note that implementations may choose not to check required fields within + // a lazy sub-message. That is, calling IsInitialized() on the outer message + // may return true even if the inner message has missing required fields. + // This is necessary because otherwise the inner message would have to be + // parsed in order to perform the check, defeating the purpose of lazy + // parsing. An implementation which chooses not to check required fields + // must be consistent about it. That is, for any particular sub-message, the + // implementation must either *always* check its required fields, or *never* + // check its required fields, regardless of whether or not the message has + // been parsed. + // + // As of May 2022, lazy verifies the contents of the byte stream during + // parsing. An invalid byte stream will cause the overall parsing to fail. + optional bool lazy = 5 [default = false]; + + // unverified_lazy does no correctness checks on the byte stream. This should + // only be used where lazy with verification is prohibitive for performance + // reasons. + optional bool unverified_lazy = 15 [default = false]; + + // Is this field deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for accessors, or it will be completely ignored; in the very least, this + // is a formalization for deprecating fields. + optional bool deprecated = 3 [default = false]; + + // For Google-internal migration only. Do not use. + optional bool weak = 10 [default = false]; + + + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; + + reserved 4; // removed jtype +} + +message OneofOptions { + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; +} + +message EnumOptions { + + // Set this option to true to allow mapping different tag names to the same + // value. + optional bool allow_alias = 2; + + // Is this enum deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for the enum, or it will be completely ignored; in the very least, this + // is a formalization for deprecating enums. + optional bool deprecated = 3 [default = false]; + + reserved 5; // javanano_as_lite + + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; +} + +message EnumValueOptions { + // Is this enum value deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for the enum value, or it will be completely ignored; in the very least, + // this is a formalization for deprecating enum values. + optional bool deprecated = 1 [default = false]; + + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; +} + +message ServiceOptions { + + // Note: Field numbers 1 through 32 are reserved for Google's internal RPC + // framework. We apologize for hoarding these numbers to ourselves, but + // we were already using them long before we decided to release Protocol + // Buffers. + + // Is this service deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for the service, or it will be completely ignored; in the very least, + // this is a formalization for deprecating services. + optional bool deprecated = 33 [default = false]; + + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; +} + +message MethodOptions { + + // Note: Field numbers 1 through 32 are reserved for Google's internal RPC + // framework. We apologize for hoarding these numbers to ourselves, but + // we were already using them long before we decided to release Protocol + // Buffers. + + // Is this method deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for the method, or it will be completely ignored; in the very least, + // this is a formalization for deprecating methods. + optional bool deprecated = 33 [default = false]; + + // Is this method side-effect-free (or safe in HTTP parlance), or idempotent, + // or neither? HTTP based RPC implementation may choose GET verb for safe + // methods, and PUT verb for idempotent methods instead of the default POST. + enum IdempotencyLevel { + IDEMPOTENCY_UNKNOWN = 0; + NO_SIDE_EFFECTS = 1; // implies idempotent + IDEMPOTENT = 2; // idempotent, but may have side effects + } + optional IdempotencyLevel idempotency_level = 34 + [default = IDEMPOTENCY_UNKNOWN]; + + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; +} + + +// A message representing a option the parser does not recognize. This only +// appears in options protos created by the compiler::Parser class. +// DescriptorPool resolves these when building Descriptor objects. Therefore, +// options protos in descriptor objects (e.g. returned by Descriptor::options(), +// or produced by Descriptor::CopyTo()) will never have UninterpretedOptions +// in them. +message UninterpretedOption { + // The name of the uninterpreted option. Each string represents a segment in + // a dot-separated name. is_extension is true iff a segment represents an + // extension (denoted with parentheses in options specs in .proto files). + // E.g.,{ ["foo", false], ["bar.baz", true], ["moo", false] } represents + // "foo.(bar.baz).moo". + message NamePart { + required string name_part = 1; + required bool is_extension = 2; + } + repeated NamePart name = 2; + + // The value of the uninterpreted option, in whatever type the tokenizer + // identified it as during parsing. Exactly one of these should be set. + optional string identifier_value = 3; + optional uint64 positive_int_value = 4; + optional int64 negative_int_value = 5; + optional double double_value = 6; + optional bytes string_value = 7; + optional string aggregate_value = 8; +} + +// =================================================================== +// Optional source code info + +// Encapsulates information about the original source file from which a +// FileDescriptorProto was generated. +message SourceCodeInfo { + // A Location identifies a piece of source code in a .proto file which + // corresponds to a particular definition. This information is intended + // to be useful to IDEs, code indexers, documentation generators, and similar + // tools. + // + // For example, say we have a file like: + // message Foo { + // optional string foo = 1; + // } + // Let's look at just the field definition: + // optional string foo = 1; + // ^ ^^ ^^ ^ ^^^ + // a bc de f ghi + // We have the following locations: + // span path represents + // [a,i) [ 4, 0, 2, 0 ] The whole field definition. + // [a,b) [ 4, 0, 2, 0, 4 ] The label (optional). + // [c,d) [ 4, 0, 2, 0, 5 ] The type (string). + // [e,f) [ 4, 0, 2, 0, 1 ] The name (foo). + // [g,h) [ 4, 0, 2, 0, 3 ] The number (1). + // + // Notes: + // - A location may refer to a repeated field itself (i.e. not to any + // particular index within it). This is used whenever a set of elements are + // logically enclosed in a single code segment. For example, an entire + // extend block (possibly containing multiple extension definitions) will + // have an outer location whose path refers to the "extensions" repeated + // field without an index. + // - Multiple locations may have the same path. This happens when a single + // logical declaration is spread out across multiple places. The most + // obvious example is the "extend" block again -- there may be multiple + // extend blocks in the same scope, each of which will have the same path. + // - A location's span is not always a subset of its parent's span. For + // example, the "extendee" of an extension declaration appears at the + // beginning of the "extend" block and is shared by all extensions within + // the block. + // - Just because a location's span is a subset of some other location's span + // does not mean that it is a descendant. For example, a "group" defines + // both a type and a field in a single declaration. Thus, the locations + // corresponding to the type and field and their components will overlap. + // - Code which tries to interpret locations should probably be designed to + // ignore those that it doesn't understand, as more types of locations could + // be recorded in the future. + repeated Location location = 1; + message Location { + // Identifies which part of the FileDescriptorProto was defined at this + // location. + // + // Each element is a field number or an index. They form a path from + // the root FileDescriptorProto to the place where the definition occurs. + // For example, this path: + // [ 4, 3, 2, 7, 1 ] + // refers to: + // file.message_type(3) // 4, 3 + // .field(7) // 2, 7 + // .name() // 1 + // This is because FileDescriptorProto.message_type has field number 4: + // repeated DescriptorProto message_type = 4; + // and DescriptorProto.field has field number 2: + // repeated FieldDescriptorProto field = 2; + // and FieldDescriptorProto.name has field number 1: + // optional string name = 1; + // + // Thus, the above path gives the location of a field name. If we removed + // the last element: + // [ 4, 3, 2, 7 ] + // this path refers to the whole field declaration (from the beginning + // of the label to the terminating semicolon). + repeated int32 path = 1 [packed = true]; + + // Always has exactly three or four elements: start line, start column, + // end line (optional, otherwise assumed same as start line), end column. + // These are packed into a single field for efficiency. Note that line + // and column numbers are zero-based -- typically you will want to add + // 1 to each before displaying to a user. + repeated int32 span = 2 [packed = true]; + + // If this SourceCodeInfo represents a complete declaration, these are any + // comments appearing before and after the declaration which appear to be + // attached to the declaration. + // + // A series of line comments appearing on consecutive lines, with no other + // tokens appearing on those lines, will be treated as a single comment. + // + // leading_detached_comments will keep paragraphs of comments that appear + // before (but not connected to) the current element. Each paragraph, + // separated by empty lines, will be one comment element in the repeated + // field. + // + // Only the comment content is provided; comment markers (e.g. //) are + // stripped out. For block comments, leading whitespace and an asterisk + // will be stripped from the beginning of each line other than the first. + // Newlines are included in the output. + // + // Examples: + // + // optional int32 foo = 1; // Comment attached to foo. + // // Comment attached to bar. + // optional int32 bar = 2; + // + // optional string baz = 3; + // // Comment attached to baz. + // // Another line attached to baz. + // + // // Comment attached to moo. + // // + // // Another line attached to moo. + // optional double moo = 4; + // + // // Detached comment for corge. This is not leading or trailing comments + // // to moo or corge because there are blank lines separating it from + // // both. + // + // // Detached comment for corge paragraph 2. + // + // optional string corge = 5; + // /* Block comment attached + // * to corge. Leading asterisks + // * will be removed. */ + // /* Block comment attached to + // * grault. */ + // optional int32 grault = 6; + // + // // ignored detached comments. + optional string leading_comments = 3; + optional string trailing_comments = 4; + repeated string leading_detached_comments = 6; + } +} + +// Describes the relationship between generated code and its original source +// file. A GeneratedCodeInfo message is associated with only one generated +// source file, but may contain references to different source .proto files. +message GeneratedCodeInfo { + // An Annotation connects some span of text in generated code to an element + // of its generating .proto file. + repeated Annotation annotation = 1; + message Annotation { + // Identifies the element in the original source .proto file. This field + // is formatted the same as SourceCodeInfo.Location.path. + repeated int32 path = 1 [packed = true]; + + // Identifies the filesystem path to the original source .proto. + optional string source_file = 2; + + // Identifies the starting offset in bytes in the generated code + // that relates to the identified object. + optional int32 begin = 3; + + // Identifies the ending offset in bytes in the generated code that + // relates to the identified offset. The end offset should be one past + // the last relevant byte (so the length of the text = end - begin). + optional int32 end = 4; + } +} diff --git a/karapace/protobuf/google/protobuf/duration.proto b/karapace/protobuf/google/protobuf/duration.proto new file mode 100644 index 000000000..81c3e369f --- /dev/null +++ b/karapace/protobuf/google/protobuf/duration.proto @@ -0,0 +1,116 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package google.protobuf; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option cc_enable_arenas = true; +option go_package = "google.golang.org/protobuf/types/known/durationpb"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "DurationProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; + +// A Duration represents a signed, fixed-length span of time represented +// as a count of seconds and fractions of seconds at nanosecond +// resolution. It is independent of any calendar and concepts like "day" +// or "month". It is related to Timestamp in that the difference between +// two Timestamp values is a Duration and it can be added or subtracted +// from a Timestamp. Range is approximately +-10,000 years. +// +// # Examples +// +// Example 1: Compute Duration from two Timestamps in pseudo code. +// +// Timestamp start = ...; +// Timestamp end = ...; +// Duration duration = ...; +// +// duration.seconds = end.seconds - start.seconds; +// duration.nanos = end.nanos - start.nanos; +// +// if (duration.seconds < 0 && duration.nanos > 0) { +// duration.seconds += 1; +// duration.nanos -= 1000000000; +// } else if (duration.seconds > 0 && duration.nanos < 0) { +// duration.seconds -= 1; +// duration.nanos += 1000000000; +// } +// +// Example 2: Compute Timestamp from Timestamp + Duration in pseudo code. +// +// Timestamp start = ...; +// Duration duration = ...; +// Timestamp end = ...; +// +// end.seconds = start.seconds + duration.seconds; +// end.nanos = start.nanos + duration.nanos; +// +// if (end.nanos < 0) { +// end.seconds -= 1; +// end.nanos += 1000000000; +// } else if (end.nanos >= 1000000000) { +// end.seconds += 1; +// end.nanos -= 1000000000; +// } +// +// Example 3: Compute Duration from datetime.timedelta in Python. +// +// td = datetime.timedelta(days=3, minutes=10) +// duration = Duration() +// duration.FromTimedelta(td) +// +// # JSON Mapping +// +// In JSON format, the Duration type is encoded as a string rather than an +// object, where the string ends in the suffix "s" (indicating seconds) and +// is preceded by the number of seconds, with nanoseconds expressed as +// fractional seconds. For example, 3 seconds with 0 nanoseconds should be +// encoded in JSON format as "3s", while 3 seconds and 1 nanosecond should +// be expressed in JSON format as "3.000000001s", and 3 seconds and 1 +// microsecond should be expressed in JSON format as "3.000001s". +// +// +message Duration { + // Signed seconds of the span of time. Must be from -315,576,000,000 + // to +315,576,000,000 inclusive. Note: these bounds are computed from: + // 60 sec/min * 60 min/hr * 24 hr/day * 365.25 days/year * 10000 years + int64 seconds = 1; + + // Signed fractions of a second at nanosecond resolution of the span + // of time. Durations less than one second are represented with a 0 + // `seconds` field and a positive or negative `nanos` field. For durations + // of one second or more, a non-zero value for the `nanos` field must be + // of the same sign as the `seconds` field. Must be from -999,999,999 + // to +999,999,999 inclusive. + int32 nanos = 2; +} diff --git a/karapace/protobuf/google/protobuf/empty.proto b/karapace/protobuf/google/protobuf/empty.proto new file mode 100644 index 000000000..222746219 --- /dev/null +++ b/karapace/protobuf/google/protobuf/empty.proto @@ -0,0 +1,51 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package google.protobuf; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option go_package = "google.golang.org/protobuf/types/known/emptypb"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "EmptyProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; +option cc_enable_arenas = true; + +// A generic empty message that you can re-use to avoid defining duplicated +// empty messages in your APIs. A typical example is to use it as the request +// or the response type of an API method. For instance: +// +// service Foo { +// rpc Bar(google.protobuf.Empty) returns (google.protobuf.Empty); +// } +// +message Empty {} diff --git a/karapace/protobuf/google/protobuf/field_mask.proto b/karapace/protobuf/google/protobuf/field_mask.proto new file mode 100644 index 000000000..6b5104f18 --- /dev/null +++ b/karapace/protobuf/google/protobuf/field_mask.proto @@ -0,0 +1,245 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package google.protobuf; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "FieldMaskProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; +option go_package = "google.golang.org/protobuf/types/known/fieldmaskpb"; +option cc_enable_arenas = true; + +// `FieldMask` represents a set of symbolic field paths, for example: +// +// paths: "f.a" +// paths: "f.b.d" +// +// Here `f` represents a field in some root message, `a` and `b` +// fields in the message found in `f`, and `d` a field found in the +// message in `f.b`. +// +// Field masks are used to specify a subset of fields that should be +// returned by a get operation or modified by an update operation. +// Field masks also have a custom JSON encoding (see below). +// +// # Field Masks in Projections +// +// When used in the context of a projection, a response message or +// sub-message is filtered by the API to only contain those fields as +// specified in the mask. For example, if the mask in the previous +// example is applied to a response message as follows: +// +// f { +// a : 22 +// b { +// d : 1 +// x : 2 +// } +// y : 13 +// } +// z: 8 +// +// The result will not contain specific values for fields x,y and z +// (their value will be set to the default, and omitted in proto text +// output): +// +// +// f { +// a : 22 +// b { +// d : 1 +// } +// } +// +// A repeated field is not allowed except at the last position of a +// paths string. +// +// If a FieldMask object is not present in a get operation, the +// operation applies to all fields (as if a FieldMask of all fields +// had been specified). +// +// Note that a field mask does not necessarily apply to the +// top-level response message. In case of a REST get operation, the +// field mask applies directly to the response, but in case of a REST +// list operation, the mask instead applies to each individual message +// in the returned resource list. In case of a REST custom method, +// other definitions may be used. Where the mask applies will be +// clearly documented together with its declaration in the API. In +// any case, the effect on the returned resource/resources is required +// behavior for APIs. +// +// # Field Masks in Update Operations +// +// A field mask in update operations specifies which fields of the +// targeted resource are going to be updated. The API is required +// to only change the values of the fields as specified in the mask +// and leave the others untouched. If a resource is passed in to +// describe the updated values, the API ignores the values of all +// fields not covered by the mask. +// +// If a repeated field is specified for an update operation, new values will +// be appended to the existing repeated field in the target resource. Note that +// a repeated field is only allowed in the last position of a `paths` string. +// +// If a sub-message is specified in the last position of the field mask for an +// update operation, then new value will be merged into the existing sub-message +// in the target resource. +// +// For example, given the target message: +// +// f { +// b { +// d: 1 +// x: 2 +// } +// c: [1] +// } +// +// And an update message: +// +// f { +// b { +// d: 10 +// } +// c: [2] +// } +// +// then if the field mask is: +// +// paths: ["f.b", "f.c"] +// +// then the result will be: +// +// f { +// b { +// d: 10 +// x: 2 +// } +// c: [1, 2] +// } +// +// An implementation may provide options to override this default behavior for +// repeated and message fields. +// +// In order to reset a field's value to the default, the field must +// be in the mask and set to the default value in the provided resource. +// Hence, in order to reset all fields of a resource, provide a default +// instance of the resource and set all fields in the mask, or do +// not provide a mask as described below. +// +// If a field mask is not present on update, the operation applies to +// all fields (as if a field mask of all fields has been specified). +// Note that in the presence of schema evolution, this may mean that +// fields the client does not know and has therefore not filled into +// the request will be reset to their default. If this is unwanted +// behavior, a specific service may require a client to always specify +// a field mask, producing an error if not. +// +// As with get operations, the location of the resource which +// describes the updated values in the request message depends on the +// operation kind. In any case, the effect of the field mask is +// required to be honored by the API. +// +// ## Considerations for HTTP REST +// +// The HTTP kind of an update operation which uses a field mask must +// be set to PATCH instead of PUT in order to satisfy HTTP semantics +// (PUT must only be used for full updates). +// +// # JSON Encoding of Field Masks +// +// In JSON, a field mask is encoded as a single string where paths are +// separated by a comma. Fields name in each path are converted +// to/from lower-camel naming conventions. +// +// As an example, consider the following message declarations: +// +// message Profile { +// User user = 1; +// Photo photo = 2; +// } +// message User { +// string display_name = 1; +// string address = 2; +// } +// +// In proto a field mask for `Profile` may look as such: +// +// mask { +// paths: "user.display_name" +// paths: "photo" +// } +// +// In JSON, the same mask is represented as below: +// +// { +// mask: "user.displayName,photo" +// } +// +// # Field Masks and Oneof Fields +// +// Field masks treat fields in oneofs just as regular fields. Consider the +// following message: +// +// message SampleMessage { +// oneof test_oneof { +// string name = 4; +// SubMessage sub_message = 9; +// } +// } +// +// The field mask can be: +// +// mask { +// paths: "name" +// } +// +// Or: +// +// mask { +// paths: "sub_message" +// } +// +// Note that oneof type names ("test_oneof" in this case) cannot be used in +// paths. +// +// ## Field Mask Verification +// +// The implementation of any API method which has a FieldMask type field in the +// request should verify the included field paths, and return an +// `INVALID_ARGUMENT` error if any path is unmappable. +message FieldMask { + // The set of field mask paths. + repeated string paths = 1; +} diff --git a/karapace/protobuf/google/protobuf/map_lite_unittest.proto b/karapace/protobuf/google/protobuf/map_lite_unittest.proto new file mode 100644 index 000000000..7f104315c --- /dev/null +++ b/karapace/protobuf/google/protobuf/map_lite_unittest.proto @@ -0,0 +1,131 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto2"; + +package protobuf_unittest; + +import "google/protobuf/unittest_lite.proto"; + +option cc_enable_arenas = true; +option optimize_for = LITE_RUNTIME; + +message TestMapLite { + map map_int32_int32 = 1; + map map_int64_int64 = 2; + map map_uint32_uint32 = 3; + map map_uint64_uint64 = 4; + map map_sint32_sint32 = 5; + map map_sint64_sint64 = 6; + map map_fixed32_fixed32 = 7; + map map_fixed64_fixed64 = 8; + map map_sfixed32_sfixed32 = 9; + map map_sfixed64_sfixed64 = 10; + map map_int32_float = 11; + map map_int32_double = 12; + map map_bool_bool = 13; + map map_string_string = 14; + map map_int32_bytes = 15; + map map_int32_enum = 16; + map map_int32_foreign_message = 17; + map teboring = 18; +} + +message TestArenaMapLite { + map map_int32_int32 = 1; + map map_int64_int64 = 2; + map map_uint32_uint32 = 3; + map map_uint64_uint64 = 4; + map map_sint32_sint32 = 5; + map map_sint64_sint64 = 6; + map map_fixed32_fixed32 = 7; + map map_fixed64_fixed64 = 8; + map map_sfixed32_sfixed32 = 9; + map map_sfixed64_sfixed64 = 10; + map map_int32_float = 11; + map map_int32_double = 12; + map map_bool_bool = 13; + map map_string_string = 14; + map map_int32_bytes = 15; + map map_int32_enum = 16; + map map_int32_foreign_message = 17; +} + +// Test embedded message with required fields +message TestRequiredMessageMapLite { + map map_field = 1; +} + +message TestEnumMapLite { + map known_map_field = 101; + map unknown_map_field = 102; +} + +message TestEnumMapPlusExtraLite { + map known_map_field = 101; + map unknown_map_field = 102; +} + +message TestMessageMapLite { + map map_int32_message = 1; +} + +enum Proto2MapEnumLite { + PROTO2_MAP_ENUM_FOO_LITE = 0; + PROTO2_MAP_ENUM_BAR_LITE = 1; + PROTO2_MAP_ENUM_BAZ_LITE = 2; +} + +enum Proto2MapEnumPlusExtraLite { + E_PROTO2_MAP_ENUM_FOO_LITE = 0; + E_PROTO2_MAP_ENUM_BAR_LITE = 1; + E_PROTO2_MAP_ENUM_BAZ_LITE = 2; + E_PROTO2_MAP_ENUM_EXTRA_LITE = 3; +} + +enum MapEnumLite { + MAP_ENUM_FOO_LITE = 0; + MAP_ENUM_BAR_LITE = 1; + MAP_ENUM_BAZ_LITE = 2; +} + +message TestRequiredLite { + required int32 a = 1; + required int32 b = 2; + required int32 c = 3; + + extend TestAllExtensionsLite { + optional TestRequiredLite single = 1000; + } +} + +message ForeignMessageArenaLite { + optional int32 c = 1; +} diff --git a/karapace/protobuf/google/protobuf/map_proto2_unittest.proto b/karapace/protobuf/google/protobuf/map_proto2_unittest.proto new file mode 100644 index 000000000..20d58f903 --- /dev/null +++ b/karapace/protobuf/google/protobuf/map_proto2_unittest.proto @@ -0,0 +1,91 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto2"; +option cc_enable_arenas = true; + +import "google/protobuf/unittest_import.proto"; + +// We don't put this in a package within proto2 because we need to make sure +// that the generated code doesn't depend on being in the proto2 namespace. +// In map_test_util.h we do "using namespace unittest = protobuf_unittest". +package protobuf_unittest; + +enum Proto2MapEnum { + PROTO2_MAP_ENUM_FOO = 0; + PROTO2_MAP_ENUM_BAR = 1; + PROTO2_MAP_ENUM_BAZ = 2; +} + +enum Proto2MapEnumPlusExtra { + E_PROTO2_MAP_ENUM_FOO = 0; + E_PROTO2_MAP_ENUM_BAR = 1; + E_PROTO2_MAP_ENUM_BAZ = 2; + E_PROTO2_MAP_ENUM_EXTRA = 3; +} + +message TestEnumMap { + map known_map_field = 101; + map unknown_map_field = 102; +} + +message TestEnumMapPlusExtra { + map known_map_field = 101; + map unknown_map_field = 102; +} + +message TestImportEnumMap { + map import_enum_amp = 1; +} + +message TestIntIntMap { + map m = 1; +} + +// Test all key types: string, plus the non-floating-point scalars. +message TestMaps { + map m_int32 = 1; + map m_int64 = 2; + map m_uint32 = 3; + map m_uint64 = 4; + map m_sint32 = 5; + map m_sint64 = 6; + map m_fixed32 = 7; + map m_fixed64 = 8; + map m_sfixed32 = 9; + map m_sfixed64 = 10; + map m_bool = 11; + map m_string = 12; +} + +// Test maps in submessages. +message TestSubmessageMaps { + optional TestMaps m = 1; +} diff --git a/karapace/protobuf/google/protobuf/map_unittest.proto b/karapace/protobuf/google/protobuf/map_unittest.proto new file mode 100644 index 000000000..263ef61f8 --- /dev/null +++ b/karapace/protobuf/google/protobuf/map_unittest.proto @@ -0,0 +1,125 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +option cc_enable_arenas = true; + +import "google/protobuf/unittest.proto"; + +// We don't put this in a package within proto2 because we need to make sure +// that the generated code doesn't depend on being in the proto2 namespace. +// In map_test_util.h we do "using namespace unittest = protobuf_unittest". +package protobuf_unittest; + +// Tests maps. +message TestMap { + map map_int32_int32 = 1; + map map_int64_int64 = 2; + map map_uint32_uint32 = 3; + map map_uint64_uint64 = 4; + map map_sint32_sint32 = 5; + map map_sint64_sint64 = 6; + map map_fixed32_fixed32 = 7; + map map_fixed64_fixed64 = 8; + map map_sfixed32_sfixed32 = 9; + map map_sfixed64_sfixed64 = 10; + map map_int32_float = 11; + map map_int32_double = 12; + map map_bool_bool = 13; + map map_string_string = 14; + map map_int32_bytes = 15; + map map_int32_enum = 16; + map map_int32_foreign_message = 17; + map map_string_foreign_message = 18; + map map_int32_all_types = 19; +} + +message TestMapSubmessage { + TestMap test_map = 1; +} + +message TestMessageMap { + map map_int32_message = 1; +} + +// Two map fields share the same entry default instance. +message TestSameTypeMap { + map map1 = 1; + map map2 = 2; +} + + +enum MapEnum { + MAP_ENUM_FOO = 0; + MAP_ENUM_BAR = 1; + MAP_ENUM_BAZ = 2; +} + +// Test embedded message with required fields +message TestRequiredMessageMap { + map map_field = 1; +} + +message TestArenaMap { + map map_int32_int32 = 1; + map map_int64_int64 = 2; + map map_uint32_uint32 = 3; + map map_uint64_uint64 = 4; + map map_sint32_sint32 = 5; + map map_sint64_sint64 = 6; + map map_fixed32_fixed32 = 7; + map map_fixed64_fixed64 = 8; + map map_sfixed32_sfixed32 = 9; + map map_sfixed64_sfixed64 = 10; + map map_int32_float = 11; + map map_int32_double = 12; + map map_bool_bool = 13; + map map_string_string = 14; + map map_int32_bytes = 15; + map map_int32_enum = 16; + map map_int32_foreign_message = 17; +} + +// Previously, message containing enum called Type cannot be used as value of +// map field. +message MessageContainingEnumCalledType { + enum Type { TYPE_FOO = 0; } + map type = 1; +} + +// Previously, message cannot contain map field called "entry". +message MessageContainingMapCalledEntry { + map entry = 1; +} + +message TestRecursiveMapMessage { + map a = 1; +} diff --git a/karapace/protobuf/google/protobuf/source_context.proto b/karapace/protobuf/google/protobuf/source_context.proto new file mode 100644 index 000000000..06bfc43a7 --- /dev/null +++ b/karapace/protobuf/google/protobuf/source_context.proto @@ -0,0 +1,48 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package google.protobuf; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "SourceContextProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; +option go_package = "google.golang.org/protobuf/types/known/sourcecontextpb"; + +// `SourceContext` represents information about the source of a +// protobuf element, like the file in which it is defined. +message SourceContext { + // The path-qualified name of the .proto file that contained the associated + // protobuf element. For example: `"google/protobuf/source_context.proto"`. + string file_name = 1; +} diff --git a/karapace/protobuf/google/protobuf/struct.proto b/karapace/protobuf/google/protobuf/struct.proto new file mode 100644 index 000000000..0ac843ca0 --- /dev/null +++ b/karapace/protobuf/google/protobuf/struct.proto @@ -0,0 +1,95 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package google.protobuf; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option cc_enable_arenas = true; +option go_package = "google.golang.org/protobuf/types/known/structpb"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "StructProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; + +// `Struct` represents a structured data value, consisting of fields +// which map to dynamically typed values. In some languages, `Struct` +// might be supported by a native representation. For example, in +// scripting languages like JS a struct is represented as an +// object. The details of that representation are described together +// with the proto support for the language. +// +// The JSON representation for `Struct` is JSON object. +message Struct { + // Unordered map of dynamically typed values. + map fields = 1; +} + +// `Value` represents a dynamically typed value which can be either +// null, a number, a string, a boolean, a recursive struct value, or a +// list of values. A producer of value is expected to set one of these +// variants. Absence of any variant indicates an error. +// +// The JSON representation for `Value` is JSON value. +message Value { + // The kind of value. + oneof kind { + // Represents a null value. + NullValue null_value = 1; + // Represents a double value. + double number_value = 2; + // Represents a string value. + string string_value = 3; + // Represents a boolean value. + bool bool_value = 4; + // Represents a structured value. + Struct struct_value = 5; + // Represents a repeated `Value`. + ListValue list_value = 6; + } +} + +// `NullValue` is a singleton enumeration to represent the null value for the +// `Value` type union. +// +// The JSON representation for `NullValue` is JSON `null`. +enum NullValue { + // Null value. + NULL_VALUE = 0; +} + +// `ListValue` is a wrapper around a repeated field of values. +// +// The JSON representation for `ListValue` is JSON array. +message ListValue { + // Repeated field of dynamically typed values. + repeated Value values = 1; +} diff --git a/karapace/protobuf/google/protobuf/timestamp.proto b/karapace/protobuf/google/protobuf/timestamp.proto new file mode 100644 index 000000000..3b2df6d91 --- /dev/null +++ b/karapace/protobuf/google/protobuf/timestamp.proto @@ -0,0 +1,147 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package google.protobuf; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option cc_enable_arenas = true; +option go_package = "google.golang.org/protobuf/types/known/timestamppb"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "TimestampProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; + +// A Timestamp represents a point in time independent of any time zone or local +// calendar, encoded as a count of seconds and fractions of seconds at +// nanosecond resolution. The count is relative to an epoch at UTC midnight on +// January 1, 1970, in the proleptic Gregorian calendar which extends the +// Gregorian calendar backwards to year one. +// +// All minutes are 60 seconds long. Leap seconds are "smeared" so that no leap +// second table is needed for interpretation, using a [24-hour linear +// smear](https://developers.google.com/time/smear). +// +// The range is from 0001-01-01T00:00:00Z to 9999-12-31T23:59:59.999999999Z. By +// restricting to that range, we ensure that we can convert to and from [RFC +// 3339](https://www.ietf.org/rfc/rfc3339.txt) date strings. +// +// # Examples +// +// Example 1: Compute Timestamp from POSIX `time()`. +// +// Timestamp timestamp; +// timestamp.set_seconds(time(NULL)); +// timestamp.set_nanos(0); +// +// Example 2: Compute Timestamp from POSIX `gettimeofday()`. +// +// struct timeval tv; +// gettimeofday(&tv, NULL); +// +// Timestamp timestamp; +// timestamp.set_seconds(tv.tv_sec); +// timestamp.set_nanos(tv.tv_usec * 1000); +// +// Example 3: Compute Timestamp from Win32 `GetSystemTimeAsFileTime()`. +// +// FILETIME ft; +// GetSystemTimeAsFileTime(&ft); +// UINT64 ticks = (((UINT64)ft.dwHighDateTime) << 32) | ft.dwLowDateTime; +// +// // A Windows tick is 100 nanoseconds. Windows epoch 1601-01-01T00:00:00Z +// // is 11644473600 seconds before Unix epoch 1970-01-01T00:00:00Z. +// Timestamp timestamp; +// timestamp.set_seconds((INT64) ((ticks / 10000000) - 11644473600LL)); +// timestamp.set_nanos((INT32) ((ticks % 10000000) * 100)); +// +// Example 4: Compute Timestamp from Java `System.currentTimeMillis()`. +// +// long millis = System.currentTimeMillis(); +// +// Timestamp timestamp = Timestamp.newBuilder().setSeconds(millis / 1000) +// .setNanos((int) ((millis % 1000) * 1000000)).build(); +// +// +// Example 5: Compute Timestamp from Java `Instant.now()`. +// +// Instant now = Instant.now(); +// +// Timestamp timestamp = +// Timestamp.newBuilder().setSeconds(now.getEpochSecond()) +// .setNanos(now.getNano()).build(); +// +// +// Example 6: Compute Timestamp from current time in Python. +// +// timestamp = Timestamp() +// timestamp.GetCurrentTime() +// +// # JSON Mapping +// +// In JSON format, the Timestamp type is encoded as a string in the +// [RFC 3339](https://www.ietf.org/rfc/rfc3339.txt) format. That is, the +// format is "{year}-{month}-{day}T{hour}:{min}:{sec}[.{frac_sec}]Z" +// where {year} is always expressed using four digits while {month}, {day}, +// {hour}, {min}, and {sec} are zero-padded to two digits each. The fractional +// seconds, which can go up to 9 digits (i.e. up to 1 nanosecond resolution), +// are optional. The "Z" suffix indicates the timezone ("UTC"); the timezone +// is required. A proto3 JSON serializer should always use UTC (as indicated by +// "Z") when printing the Timestamp type and a proto3 JSON parser should be +// able to accept both UTC and other timezones (as indicated by an offset). +// +// For example, "2017-01-15T01:30:15.01Z" encodes 15.01 seconds past +// 01:30 UTC on January 15, 2017. +// +// In JavaScript, one can convert a Date object to this format using the +// standard +// [toISOString()](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Date/toISOString) +// method. In Python, a standard `datetime.datetime` object can be converted +// to this format using +// [`strftime`](https://docs.python.org/2/library/time.html#time.strftime) with +// the time format spec '%Y-%m-%dT%H:%M:%S.%fZ'. Likewise, in Java, one can use +// the Joda Time's [`ISODateTimeFormat.dateTime()`]( +// http://www.joda.org/joda-time/apidocs/org/joda/time/format/ISODateTimeFormat.html#dateTime%2D%2D +// ) to obtain a formatter capable of generating timestamps in this format. +// +// +message Timestamp { + // Represents seconds of UTC time since Unix epoch + // 1970-01-01T00:00:00Z. Must be from 0001-01-01T00:00:00Z to + // 9999-12-31T23:59:59Z inclusive. + int64 seconds = 1; + + // Non-negative fractions of a second at nanosecond resolution. Negative + // second values with fractions must still have non-negative nanos values + // that count forward in time. Must be from 0 to 999,999,999 + // inclusive. + int32 nanos = 2; +} diff --git a/karapace/protobuf/google/protobuf/type.proto b/karapace/protobuf/google/protobuf/type.proto new file mode 100644 index 000000000..d3f6a68b8 --- /dev/null +++ b/karapace/protobuf/google/protobuf/type.proto @@ -0,0 +1,187 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package google.protobuf; + +import "google/protobuf/any.proto"; +import "google/protobuf/source_context.proto"; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option cc_enable_arenas = true; +option java_package = "com.google.protobuf"; +option java_outer_classname = "TypeProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; +option go_package = "google.golang.org/protobuf/types/known/typepb"; + +// A protocol buffer message type. +message Type { + // The fully qualified message name. + string name = 1; + // The list of fields. + repeated Field fields = 2; + // The list of types appearing in `oneof` definitions in this type. + repeated string oneofs = 3; + // The protocol buffer options. + repeated Option options = 4; + // The source context. + SourceContext source_context = 5; + // The source syntax. + Syntax syntax = 6; +} + +// A single field of a message type. +message Field { + // Basic field types. + enum Kind { + // Field type unknown. + TYPE_UNKNOWN = 0; + // Field type double. + TYPE_DOUBLE = 1; + // Field type float. + TYPE_FLOAT = 2; + // Field type int64. + TYPE_INT64 = 3; + // Field type uint64. + TYPE_UINT64 = 4; + // Field type int32. + TYPE_INT32 = 5; + // Field type fixed64. + TYPE_FIXED64 = 6; + // Field type fixed32. + TYPE_FIXED32 = 7; + // Field type bool. + TYPE_BOOL = 8; + // Field type string. + TYPE_STRING = 9; + // Field type group. Proto2 syntax only, and deprecated. + TYPE_GROUP = 10; + // Field type message. + TYPE_MESSAGE = 11; + // Field type bytes. + TYPE_BYTES = 12; + // Field type uint32. + TYPE_UINT32 = 13; + // Field type enum. + TYPE_ENUM = 14; + // Field type sfixed32. + TYPE_SFIXED32 = 15; + // Field type sfixed64. + TYPE_SFIXED64 = 16; + // Field type sint32. + TYPE_SINT32 = 17; + // Field type sint64. + TYPE_SINT64 = 18; + } + + // Whether a field is optional, required, or repeated. + enum Cardinality { + // For fields with unknown cardinality. + CARDINALITY_UNKNOWN = 0; + // For optional fields. + CARDINALITY_OPTIONAL = 1; + // For required fields. Proto2 syntax only. + CARDINALITY_REQUIRED = 2; + // For repeated fields. + CARDINALITY_REPEATED = 3; + } + + // The field type. + Kind kind = 1; + // The field cardinality. + Cardinality cardinality = 2; + // The field number. + int32 number = 3; + // The field name. + string name = 4; + // The field type URL, without the scheme, for message or enumeration + // types. Example: `"type.googleapis.com/google.protobuf.Timestamp"`. + string type_url = 6; + // The index of the field type in `Type.oneofs`, for message or enumeration + // types. The first type has index 1; zero means the type is not in the list. + int32 oneof_index = 7; + // Whether to use alternative packed wire representation. + bool packed = 8; + // The protocol buffer options. + repeated Option options = 9; + // The field JSON name. + string json_name = 10; + // The string value of the default value of this field. Proto2 syntax only. + string default_value = 11; +} + +// Enum type definition. +message Enum { + // Enum type name. + string name = 1; + // Enum value definitions. + repeated EnumValue enumvalue = 2; + // Protocol buffer options. + repeated Option options = 3; + // The source context. + SourceContext source_context = 4; + // The source syntax. + Syntax syntax = 5; +} + +// Enum value definition. +message EnumValue { + // Enum value name. + string name = 1; + // Enum value number. + int32 number = 2; + // Protocol buffer options. + repeated Option options = 3; +} + +// A protocol buffer option, which can be attached to a message, field, +// enumeration, etc. +message Option { + // The option's name. For protobuf built-in options (options defined in + // descriptor.proto), this is the short name. For example, `"map_entry"`. + // For custom options, it should be the fully-qualified name. For example, + // `"google.api.http"`. + string name = 1; + // The option's value packed in an Any message. If the value is a primitive, + // the corresponding wrapper type defined in google/protobuf/wrappers.proto + // should be used. If the value is an enum, it should be stored as an int32 + // value using the google.protobuf.Int32Value type. + Any value = 2; +} + +// The syntax in which a protocol buffer element is defined. +enum Syntax { + // Syntax `proto2`. + SYNTAX_PROTO2 = 0; + // Syntax `proto3`. + SYNTAX_PROTO3 = 1; +} diff --git a/karapace/protobuf/google/protobuf/wrappers.proto b/karapace/protobuf/google/protobuf/wrappers.proto new file mode 100644 index 000000000..d49dd53c8 --- /dev/null +++ b/karapace/protobuf/google/protobuf/wrappers.proto @@ -0,0 +1,123 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Wrappers for primitive (non-message) types. These types are useful +// for embedding primitives in the `google.protobuf.Any` type and for places +// where we need to distinguish between the absence of a primitive +// typed field and its default value. +// +// These wrappers have no meaningful use within repeated fields as they lack +// the ability to detect presence on individual elements. +// These wrappers have no meaningful use within a map or a oneof since +// individual entries of a map or fields of a oneof can already detect presence. + +syntax = "proto3"; + +package google.protobuf; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option cc_enable_arenas = true; +option go_package = "google.golang.org/protobuf/types/known/wrapperspb"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "WrappersProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; + +// Wrapper message for `double`. +// +// The JSON representation for `DoubleValue` is JSON number. +message DoubleValue { + // The double value. + double value = 1; +} + +// Wrapper message for `float`. +// +// The JSON representation for `FloatValue` is JSON number. +message FloatValue { + // The float value. + float value = 1; +} + +// Wrapper message for `int64`. +// +// The JSON representation for `Int64Value` is JSON string. +message Int64Value { + // The int64 value. + int64 value = 1; +} + +// Wrapper message for `uint64`. +// +// The JSON representation for `UInt64Value` is JSON string. +message UInt64Value { + // The uint64 value. + uint64 value = 1; +} + +// Wrapper message for `int32`. +// +// The JSON representation for `Int32Value` is JSON number. +message Int32Value { + // The int32 value. + int32 value = 1; +} + +// Wrapper message for `uint32`. +// +// The JSON representation for `UInt32Value` is JSON number. +message UInt32Value { + // The uint32 value. + uint32 value = 1; +} + +// Wrapper message for `bool`. +// +// The JSON representation for `BoolValue` is JSON `true` and `false`. +message BoolValue { + // The bool value. + bool value = 1; +} + +// Wrapper message for `string`. +// +// The JSON representation for `StringValue` is JSON string. +message StringValue { + // The string value. + string value = 1; +} + +// Wrapper message for `bytes`. +// +// The JSON representation for `BytesValue` is JSON string. +message BytesValue { + // The bytes value. + bytes value = 1; +} diff --git a/karapace/protobuf/google/type/BUILD.bazel b/karapace/protobuf/google/type/BUILD.bazel new file mode 100644 index 000000000..a7ad4d4d2 --- /dev/null +++ b/karapace/protobuf/google/type/BUILD.bazel @@ -0,0 +1,536 @@ +load("@rules_proto//proto:defs.bzl", "proto_library") + +# This is an API workspace, having public visibility by default makes perfect sense. +package(default_visibility = ["//visibility:public"]) + +############################################################################## +# Common +############################################################################## +proto_library( + name = "calendar_period_proto", + srcs = ["calendar_period.proto"], +) + +proto_library( + name = "color_proto", + srcs = ["color.proto"], + deps = [ + "@com_google_protobuf//:wrappers_proto", + ], +) + +proto_library( + name = "date_proto", + srcs = ["date.proto"], +) + +proto_library( + name = "datetime_proto", + srcs = ["datetime.proto"], + deps = [ + "@com_google_protobuf//:duration_proto", + ], +) + +proto_library( + name = "dayofweek_proto", + srcs = ["dayofweek.proto"], +) + +proto_library( + name = "decimal_proto", + srcs = ["decimal.proto"], +) + +proto_library( + name = "expr_proto", + srcs = ["expr.proto"], +) + +proto_library( + name = "fraction_proto", + srcs = ["fraction.proto"], +) + +proto_library( + name = "interval_proto", + srcs = ["interval.proto"], + deps = [ + "@com_google_protobuf//:timestamp_proto", + ], +) + +proto_library( + name = "latlng_proto", + srcs = ["latlng.proto"], +) + +proto_library( + name = "localized_text_proto", + srcs = ["localized_text.proto"], +) + +proto_library( + name = "money_proto", + srcs = ["money.proto"], +) + +proto_library( + name = "month_proto", + srcs = ["month.proto"], +) + +proto_library( + name = "phone_number_proto", + srcs = ["phone_number.proto"], +) + +proto_library( + name = "postal_address_proto", + srcs = ["postal_address.proto"], +) + +proto_library( + name = "quaternion_proto", + srcs = ["quaternion.proto"], +) + +proto_library( + name = "timeofday_proto", + srcs = ["timeofday.proto"], +) + +############################################################################## +# Java +############################################################################## +load("@com_google_googleapis_imports//:imports.bzl", + "java_gapic_assembly_gradle_pkg", + "java_proto_library") + +java_proto_library( + name = "type_java_proto", + deps = [ + ":calendar_period_proto", + ":color_proto", + ":date_proto", + ":datetime_proto", + ":dayofweek_proto", + ":decimal_proto", + ":expr_proto", + ":fraction_proto", + ":interval_proto", + ":latlng_proto", + ":localized_text_proto", + ":money_proto", + ":month_proto", + ":phone_number_proto", + ":postal_address_proto", + ":quaternion_proto", + ":timeofday_proto", + ], +) + +# Please DO-NOT-REMOVE this section. +# This is required to generate java files for these protos. +# Open Source Packages +java_gapic_assembly_gradle_pkg( + name = "google-type-java", + deps = [ + ":type_java_proto", + ":calendar_period_proto", + ":color_proto", + ":date_proto", + ":datetime_proto", + ":dayofweek_proto", + ":decimal_proto", + ":expr_proto", + ":fraction_proto", + ":interval_proto", + ":latlng_proto", + ":localized_text_proto", + ":money_proto", + ":month_proto", + ":phone_number_proto", + ":postal_address_proto", + ":quaternion_proto", + ":timeofday_proto", + ], +) + + +############################################################################## +# Go +############################################################################## +load("@com_google_googleapis_imports//:imports.bzl", "go_proto_library") + +go_proto_library( + name = "calendar_period_go_proto", + importpath = "google.golang.org/genproto/googleapis/type/calendarperiod", + protos = [":calendar_period_proto"], +) + +go_proto_library( + name = "color_go_proto", + importpath = "google.golang.org/genproto/googleapis/type/color", + protos = [":color_proto"], +) + +go_proto_library( + name = "date_go_proto", + importpath = "google.golang.org/genproto/googleapis/type/date", + protos = [":date_proto"], +) + +go_proto_library( + name = "datetime_go_proto", + importpath = "google.golang.org/genproto/googleapis/type/datetime", + protos = [":datetime_proto"], +) + +go_proto_library( + name = "dayofweek_go_proto", + importpath = "google.golang.org/genproto/googleapis/type/dayofweek", + protos = [":dayofweek_proto"], +) + +go_proto_library( + name = "decimal_go_proto", + importpath = "google.golang.org/genproto/googleapis/type/decimal", + protos = [":decimal_proto"], +) + +go_proto_library( + name = "expr_go_proto", + importpath = "google.golang.org/genproto/googleapis/type/expr", + protos = [":expr_proto"], +) + +go_proto_library( + name = "fraction_go_proto", + importpath = "google.golang.org/genproto/googleapis/type/fraction", + protos = [":fraction_proto"], +) + +go_proto_library( + name = "interval_go_proto", + importpath = "google.golang.org/genproto/googleapis/type/interval", + protos = [":interval_proto"], +) + +go_proto_library( + name = "latlng_go_proto", + importpath = "google.golang.org/genproto/googleapis/type/latlng", + protos = [":latlng_proto"], +) + +go_proto_library( + name = "localized_text_go_proto", + importpath = "google.golang.org/genproto/googleapis/type/localized_text", + protos = [":localized_text_proto"], +) + +go_proto_library( + name = "money_go_proto", + importpath = "google.golang.org/genproto/googleapis/type/money", + protos = [":money_proto"], +) + +go_proto_library( + name = "month_go_proto", + importpath = "google.golang.org/genproto/googleapis/type/month", + protos = [":month_proto"], +) + +go_proto_library( + name = "phone_number_go_proto", + importpath = "google.golang.org/genproto/googleapis/type/phone_number", + protos = [":phone_number_proto"], +) + +go_proto_library( + name = "postaladdress_go_proto", + importpath = "google.golang.org/genproto/googleapis/type/postaladdress", + protos = [":postal_address_proto"], +) + +go_proto_library( + name = "quaternion_go_proto", + importpath = "google.golang.org/genproto/googleapis/type/quaternion", + protos = [":quaternion_proto"], +) + +go_proto_library( + name = "timeofday_go_proto", + importpath = "google.golang.org/genproto/googleapis/type/timeofday", + protos = [":timeofday_proto"], +) + +############################################################################## +# C++ +############################################################################## +load( + "@com_google_googleapis_imports//:imports.bzl", + "cc_proto_library", +) + +cc_proto_library( + name = "calendar_period_cc_proto", + deps = [":calendar_period_proto"], +) + +cc_proto_library( + name = "color_cc_proto", + deps = [":color_proto"], +) + +cc_proto_library( + name = "date_cc_proto", + deps = [":date_proto"], +) + +cc_proto_library( + name = "datetime_cc_proto", + deps = [":datetime_proto"], +) + +cc_proto_library( + name = "dayofweek_cc_proto", + deps = [":dayofweek_proto"], +) + +cc_proto_library( + name = "decimal_cc_proto", + deps = [":decimal_proto"], +) + +cc_proto_library( + name = "expr_cc_proto", + deps = [":expr_proto"], +) + +cc_proto_library( + name = "fraction_cc_proto", + deps = [":fraction_proto"], +) + +cc_proto_library( + name = "interval_cc_proto", + deps = [":interval_proto"], +) + +cc_proto_library( + name = "latlng_cc_proto", + deps = [":latlng_proto"], +) + +cc_proto_library( + name = "money_cc_proto", + deps = [":money_proto"], +) + +cc_proto_library( + name = "month_cc_proto", + deps = [":month_proto"], +) + +cc_proto_library( + name = "phone_number_cc_proto", + deps = [":phone_number_proto"], +) + +cc_proto_library( + name = "postal_address_cc_proto", + deps = [":postal_address_proto"], +) + +cc_proto_library( + name = "quaternion_cc_proto", + deps = [":quaternion_proto"], +) + +cc_proto_library( + name = "timeofday_cc_proto", + deps = [":timeofday_proto"], +) + +############################################################################## +# Python +############################################################################## +load( + "@com_google_googleapis_imports//:imports.bzl", + "py_proto_library", +) + +py_proto_library( + name = "calendar_period_py_proto", + deps = [":calendar_period_proto"], +) + +py_proto_library( + name = "color_py_proto", + deps = [":color_proto"], +) + +py_proto_library( + name = "date_py_proto", + deps = [":date_proto"], +) + +py_proto_library( + name = "datetime_py_proto", + deps = [":datetime_proto"], +) + +py_proto_library( + name = "dayofweek_py_proto", + deps = [":dayofweek_proto"], +) + +py_proto_library( + name = "decimal_py_proto", + deps = [":decimal_proto"], +) + +py_proto_library( + name = "expr_py_proto", + deps = [":expr_proto"], +) + +py_proto_library( + name = "fraction_py_proto", + deps = [":fraction_proto"], +) + +py_proto_library( + name = "interval_py_proto", + deps = [":interval_proto"], +) + +py_proto_library( + name = "latlng_py_proto", + deps = [":latlng_proto"], +) + +py_proto_library( + name = "localized_text_py_proto", + deps = [":localized_text_proto"], +) + +py_proto_library( + name = "money_py_proto", + deps = [":money_proto"], +) + +py_proto_library( + name = "month_py_proto", + deps = [":month_proto"], +) + +py_proto_library( + name = "phone_number_py_proto", + deps = [":phone_number_proto"], +) + +py_proto_library( + name = "postal_address_py_proto", + deps = [":postal_address_proto"], +) + +py_proto_library( + name = "quaternion_py_proto", + deps = [":quaternion_proto"], +) + +py_proto_library( + name = "timeofday_py_proto", + deps = [":timeofday_proto"], +) + +############################################################################## +# C# +############################################################################## + +load( + "@com_google_googleapis_imports//:imports.bzl", + "csharp_proto_library", +) + +csharp_proto_library( + name = "calendar_period_csharp_proto", + deps = [":calendar_period_proto"], +) + +csharp_proto_library( + name = "color_csharp_proto", + deps = [":color_proto"], +) + +csharp_proto_library( + name = "date_csharp_proto", + deps = [":date_proto"], +) + +csharp_proto_library( + name = "datetime_csharp_proto", + deps = [":datetime_proto"], +) + +csharp_proto_library( + name = "dayofweek_csharp_proto", + deps = [":dayofweek_proto"], +) + +csharp_proto_library( + name = "decimal_csharp_proto", + deps = [":decimal_proto"], +) + +csharp_proto_library( + name = "expr_csharp_proto", + deps = [":expr_proto"], +) + +csharp_proto_library( + name = "fraction_csharp_proto", + deps = [":fraction_proto"], +) + +csharp_proto_library( + name = "interval_csharp_proto", + deps = [":interval_proto"], +) + +csharp_proto_library( + name = "latlng_csharp_proto", + deps = [":latlng_proto"], +) + +csharp_proto_library( + name = "localized_text_csharp_proto", + deps = [":localized_text_proto"], +) + +csharp_proto_library( + name = "money_csharp_proto", + deps = [":money_proto"], +) + +csharp_proto_library( + name = "month_csharp_proto", + deps = [":month_proto"], +) + +csharp_proto_library( + name = "phone_number_csharp_proto", + deps = [":phone_number_proto"], +) + +csharp_proto_library( + name = "postal_address_csharp_proto", + deps = [":postal_address_proto"], +) + +csharp_proto_library( + name = "quaternion_csharp_proto", + deps = [":quaternion_proto"], +) diff --git a/karapace/protobuf/google/type/README.md b/karapace/protobuf/google/type/README.md new file mode 100644 index 000000000..de6a835d7 --- /dev/null +++ b/karapace/protobuf/google/type/README.md @@ -0,0 +1,16 @@ +# Google Common Types + +This package contains definitions of common types for Google APIs. +All types defined in this package are suitable for different APIs to +exchange data, and will never break binary compatibility. They should +have design quality comparable to major programming languages like +Java and C#. + +NOTE: Some common types are defined in the package `google.protobuf` +as they are directly supported by Protocol Buffers compiler and +runtime. Those types are called Well-Known Types. + +## Java Utilities + +A set of Java utilities for the Common Types are provided in the +`//java/com/google/type/util/` package. diff --git a/karapace/protobuf/google/type/calendar_period.proto b/karapace/protobuf/google/type/calendar_period.proto new file mode 100644 index 000000000..82f5690b7 --- /dev/null +++ b/karapace/protobuf/google/type/calendar_period.proto @@ -0,0 +1,56 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option go_package = "google.golang.org/genproto/googleapis/type/calendarperiod;calendarperiod"; +option java_multiple_files = true; +option java_outer_classname = "CalendarPeriodProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// A `CalendarPeriod` represents the abstract concept of a time period that has +// a canonical start. Grammatically, "the start of the current +// `CalendarPeriod`." All calendar times begin at midnight UTC. +enum CalendarPeriod { + // Undefined period, raises an error. + CALENDAR_PERIOD_UNSPECIFIED = 0; + + // A day. + DAY = 1; + + // A week. Weeks begin on Monday, following + // [ISO 8601](https://en.wikipedia.org/wiki/ISO_week_date). + WEEK = 2; + + // A fortnight. The first calendar fortnight of the year begins at the start + // of week 1 according to + // [ISO 8601](https://en.wikipedia.org/wiki/ISO_week_date). + FORTNIGHT = 3; + + // A month. + MONTH = 4; + + // A quarter. Quarters start on dates 1-Jan, 1-Apr, 1-Jul, and 1-Oct of each + // year. + QUARTER = 5; + + // A half-year. Half-years start on dates 1-Jan and 1-Jul. + HALF = 6; + + // A year. + YEAR = 7; +} diff --git a/karapace/protobuf/google/type/color.proto b/karapace/protobuf/google/type/color.proto new file mode 100644 index 000000000..5dc85a6a3 --- /dev/null +++ b/karapace/protobuf/google/type/color.proto @@ -0,0 +1,174 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +import "google/protobuf/wrappers.proto"; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/color;color"; +option java_multiple_files = true; +option java_outer_classname = "ColorProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a color in the RGBA color space. This representation is designed +// for simplicity of conversion to/from color representations in various +// languages over compactness. For example, the fields of this representation +// can be trivially provided to the constructor of `java.awt.Color` in Java; it +// can also be trivially provided to UIColor's `+colorWithRed:green:blue:alpha` +// method in iOS; and, with just a little work, it can be easily formatted into +// a CSS `rgba()` string in JavaScript. +// +// This reference page doesn't carry information about the absolute color +// space +// that should be used to interpret the RGB value (e.g. sRGB, Adobe RGB, +// DCI-P3, BT.2020, etc.). By default, applications should assume the sRGB color +// space. +// +// When color equality needs to be decided, implementations, unless +// documented otherwise, treat two colors as equal if all their red, +// green, blue, and alpha values each differ by at most 1e-5. +// +// Example (Java): +// +// import com.google.type.Color; +// +// // ... +// public static java.awt.Color fromProto(Color protocolor) { +// float alpha = protocolor.hasAlpha() +// ? protocolor.getAlpha().getValue() +// : 1.0; +// +// return new java.awt.Color( +// protocolor.getRed(), +// protocolor.getGreen(), +// protocolor.getBlue(), +// alpha); +// } +// +// public static Color toProto(java.awt.Color color) { +// float red = (float) color.getRed(); +// float green = (float) color.getGreen(); +// float blue = (float) color.getBlue(); +// float denominator = 255.0; +// Color.Builder resultBuilder = +// Color +// .newBuilder() +// .setRed(red / denominator) +// .setGreen(green / denominator) +// .setBlue(blue / denominator); +// int alpha = color.getAlpha(); +// if (alpha != 255) { +// result.setAlpha( +// FloatValue +// .newBuilder() +// .setValue(((float) alpha) / denominator) +// .build()); +// } +// return resultBuilder.build(); +// } +// // ... +// +// Example (iOS / Obj-C): +// +// // ... +// static UIColor* fromProto(Color* protocolor) { +// float red = [protocolor red]; +// float green = [protocolor green]; +// float blue = [protocolor blue]; +// FloatValue* alpha_wrapper = [protocolor alpha]; +// float alpha = 1.0; +// if (alpha_wrapper != nil) { +// alpha = [alpha_wrapper value]; +// } +// return [UIColor colorWithRed:red green:green blue:blue alpha:alpha]; +// } +// +// static Color* toProto(UIColor* color) { +// CGFloat red, green, blue, alpha; +// if (![color getRed:&red green:&green blue:&blue alpha:&alpha]) { +// return nil; +// } +// Color* result = [[Color alloc] init]; +// [result setRed:red]; +// [result setGreen:green]; +// [result setBlue:blue]; +// if (alpha <= 0.9999) { +// [result setAlpha:floatWrapperWithValue(alpha)]; +// } +// [result autorelease]; +// return result; +// } +// // ... +// +// Example (JavaScript): +// +// // ... +// +// var protoToCssColor = function(rgb_color) { +// var redFrac = rgb_color.red || 0.0; +// var greenFrac = rgb_color.green || 0.0; +// var blueFrac = rgb_color.blue || 0.0; +// var red = Math.floor(redFrac * 255); +// var green = Math.floor(greenFrac * 255); +// var blue = Math.floor(blueFrac * 255); +// +// if (!('alpha' in rgb_color)) { +// return rgbToCssColor(red, green, blue); +// } +// +// var alphaFrac = rgb_color.alpha.value || 0.0; +// var rgbParams = [red, green, blue].join(','); +// return ['rgba(', rgbParams, ',', alphaFrac, ')'].join(''); +// }; +// +// var rgbToCssColor = function(red, green, blue) { +// var rgbNumber = new Number((red << 16) | (green << 8) | blue); +// var hexString = rgbNumber.toString(16); +// var missingZeros = 6 - hexString.length; +// var resultBuilder = ['#']; +// for (var i = 0; i < missingZeros; i++) { +// resultBuilder.push('0'); +// } +// resultBuilder.push(hexString); +// return resultBuilder.join(''); +// }; +// +// // ... +message Color { + // The amount of red in the color as a value in the interval [0, 1]. + float red = 1; + + // The amount of green in the color as a value in the interval [0, 1]. + float green = 2; + + // The amount of blue in the color as a value in the interval [0, 1]. + float blue = 3; + + // The fraction of this color that should be applied to the pixel. That is, + // the final pixel color is defined by the equation: + // + // `pixel color = alpha * (this color) + (1.0 - alpha) * (background color)` + // + // This means that a value of 1.0 corresponds to a solid color, whereas + // a value of 0.0 corresponds to a completely transparent color. This + // uses a wrapper message rather than a simple float scalar so that it is + // possible to distinguish between a default value and the value being unset. + // If omitted, this color object is rendered as a solid color + // (as if the alpha value had been explicitly given a value of 1.0). + google.protobuf.FloatValue alpha = 4; +} diff --git a/karapace/protobuf/google/type/date.proto b/karapace/protobuf/google/type/date.proto new file mode 100644 index 000000000..e4e730e6f --- /dev/null +++ b/karapace/protobuf/google/type/date.proto @@ -0,0 +1,52 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/date;date"; +option java_multiple_files = true; +option java_outer_classname = "DateProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a whole or partial calendar date, such as a birthday. The time of +// day and time zone are either specified elsewhere or are insignificant. The +// date is relative to the Gregorian Calendar. This can represent one of the +// following: +// +// * A full date, with non-zero year, month, and day values +// * A month and day value, with a zero year, such as an anniversary +// * A year on its own, with zero month and day values +// * A year and month value, with a zero day, such as a credit card expiration +// date +// +// Related types are [google.type.TimeOfDay][google.type.TimeOfDay] and +// `google.protobuf.Timestamp`. +message Date { + // Year of the date. Must be from 1 to 9999, or 0 to specify a date without + // a year. + int32 year = 1; + + // Month of a year. Must be from 1 to 12, or 0 to specify a year without a + // month and day. + int32 month = 2; + + // Day of a month. Must be from 1 to 31 and valid for the year and month, or 0 + // to specify a year by itself or a year and month where the day isn't + // significant. + int32 day = 3; +} diff --git a/karapace/protobuf/google/type/datetime.proto b/karapace/protobuf/google/type/datetime.proto new file mode 100644 index 000000000..cfed85d70 --- /dev/null +++ b/karapace/protobuf/google/type/datetime.proto @@ -0,0 +1,104 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +import "google/protobuf/duration.proto"; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/datetime;datetime"; +option java_multiple_files = true; +option java_outer_classname = "DateTimeProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents civil time (or occasionally physical time). +// +// This type can represent a civil time in one of a few possible ways: +// +// * When utc_offset is set and time_zone is unset: a civil time on a calendar +// day with a particular offset from UTC. +// * When time_zone is set and utc_offset is unset: a civil time on a calendar +// day in a particular time zone. +// * When neither time_zone nor utc_offset is set: a civil time on a calendar +// day in local time. +// +// The date is relative to the Proleptic Gregorian Calendar. +// +// If year is 0, the DateTime is considered not to have a specific year. month +// and day must have valid, non-zero values. +// +// This type may also be used to represent a physical time if all the date and +// time fields are set and either case of the `time_offset` oneof is set. +// Consider using `Timestamp` message for physical time instead. If your use +// case also would like to store the user's timezone, that can be done in +// another field. +// +// This type is more flexible than some applications may want. Make sure to +// document and validate your application's limitations. +message DateTime { + // Optional. Year of date. Must be from 1 to 9999, or 0 if specifying a + // datetime without a year. + int32 year = 1; + + // Required. Month of year. Must be from 1 to 12. + int32 month = 2; + + // Required. Day of month. Must be from 1 to 31 and valid for the year and + // month. + int32 day = 3; + + // Required. Hours of day in 24 hour format. Should be from 0 to 23. An API + // may choose to allow the value "24:00:00" for scenarios like business + // closing time. + int32 hours = 4; + + // Required. Minutes of hour of day. Must be from 0 to 59. + int32 minutes = 5; + + // Required. Seconds of minutes of the time. Must normally be from 0 to 59. An + // API may allow the value 60 if it allows leap-seconds. + int32 seconds = 6; + + // Required. Fractions of seconds in nanoseconds. Must be from 0 to + // 999,999,999. + int32 nanos = 7; + + // Optional. Specifies either the UTC offset or the time zone of the DateTime. + // Choose carefully between them, considering that time zone data may change + // in the future (for example, a country modifies their DST start/end dates, + // and future DateTimes in the affected range had already been stored). + // If omitted, the DateTime is considered to be in local time. + oneof time_offset { + // UTC offset. Must be whole seconds, between -18 hours and +18 hours. + // For example, a UTC offset of -4:00 would be represented as + // { seconds: -14400 }. + google.protobuf.Duration utc_offset = 8; + + // Time zone. + TimeZone time_zone = 9; + } +} + +// Represents a time zone from the +// [IANA Time Zone Database](https://www.iana.org/time-zones). +message TimeZone { + // IANA Time Zone Database time zone, e.g. "America/New_York". + string id = 1; + + // Optional. IANA Time Zone Database version number, e.g. "2019a". + string version = 2; +} diff --git a/karapace/protobuf/google/type/dayofweek.proto b/karapace/protobuf/google/type/dayofweek.proto new file mode 100644 index 000000000..4c80c62ec --- /dev/null +++ b/karapace/protobuf/google/type/dayofweek.proto @@ -0,0 +1,50 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option go_package = "google.golang.org/genproto/googleapis/type/dayofweek;dayofweek"; +option java_multiple_files = true; +option java_outer_classname = "DayOfWeekProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a day of the week. +enum DayOfWeek { + // The day of the week is unspecified. + DAY_OF_WEEK_UNSPECIFIED = 0; + + // Monday + MONDAY = 1; + + // Tuesday + TUESDAY = 2; + + // Wednesday + WEDNESDAY = 3; + + // Thursday + THURSDAY = 4; + + // Friday + FRIDAY = 5; + + // Saturday + SATURDAY = 6; + + // Sunday + SUNDAY = 7; +} diff --git a/karapace/protobuf/google/type/decimal.proto b/karapace/protobuf/google/type/decimal.proto new file mode 100644 index 000000000..beb18a5d8 --- /dev/null +++ b/karapace/protobuf/google/type/decimal.proto @@ -0,0 +1,95 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/decimal;decimal"; +option java_multiple_files = true; +option java_outer_classname = "DecimalProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// A representation of a decimal value, such as 2.5. Clients may convert values +// into language-native decimal formats, such as Java's [BigDecimal][] or +// Python's [decimal.Decimal][]. +// +// [BigDecimal]: +// https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/math/BigDecimal.html +// [decimal.Decimal]: https://docs.python.org/3/library/decimal.html +message Decimal { + // The decimal value, as a string. + // + // The string representation consists of an optional sign, `+` (`U+002B`) + // or `-` (`U+002D`), followed by a sequence of zero or more decimal digits + // ("the integer"), optionally followed by a fraction, optionally followed + // by an exponent. + // + // The fraction consists of a decimal point followed by zero or more decimal + // digits. The string must contain at least one digit in either the integer + // or the fraction. The number formed by the sign, the integer and the + // fraction is referred to as the significand. + // + // The exponent consists of the character `e` (`U+0065`) or `E` (`U+0045`) + // followed by one or more decimal digits. + // + // Services **should** normalize decimal values before storing them by: + // + // - Removing an explicitly-provided `+` sign (`+2.5` -> `2.5`). + // - Replacing a zero-length integer value with `0` (`.5` -> `0.5`). + // - Coercing the exponent character to lower-case (`2.5E8` -> `2.5e8`). + // - Removing an explicitly-provided zero exponent (`2.5e0` -> `2.5`). + // + // Services **may** perform additional normalization based on its own needs + // and the internal decimal implementation selected, such as shifting the + // decimal point and exponent value together (example: `2.5e-1` <-> `0.25`). + // Additionally, services **may** preserve trailing zeroes in the fraction + // to indicate increased precision, but are not required to do so. + // + // Note that only the `.` character is supported to divide the integer + // and the fraction; `,` **should not** be supported regardless of locale. + // Additionally, thousand separators **should not** be supported. If a + // service does support them, values **must** be normalized. + // + // The ENBF grammar is: + // + // DecimalString = + // [Sign] Significand [Exponent]; + // + // Sign = '+' | '-'; + // + // Significand = + // Digits ['.'] [Digits] | [Digits] '.' Digits; + // + // Exponent = ('e' | 'E') [Sign] Digits; + // + // Digits = { '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' }; + // + // Services **should** clearly document the range of supported values, the + // maximum supported precision (total number of digits), and, if applicable, + // the scale (number of digits after the decimal point), as well as how it + // behaves when receiving out-of-bounds values. + // + // Services **may** choose to accept values passed as input even when the + // value has a higher precision or scale than the service supports, and + // **should** round the value to fit the supported scale. Alternatively, the + // service **may** error with `400 Bad Request` (`INVALID_ARGUMENT` in gRPC) + // if precision would be lost. + // + // Services **should** error with `400 Bad Request` (`INVALID_ARGUMENT` in + // gRPC) if the service receives a value outside of the supported range. + string value = 1; +} diff --git a/karapace/protobuf/google/type/expr.proto b/karapace/protobuf/google/type/expr.proto new file mode 100644 index 000000000..af0778cf9 --- /dev/null +++ b/karapace/protobuf/google/type/expr.proto @@ -0,0 +1,73 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option go_package = "google.golang.org/genproto/googleapis/type/expr;expr"; +option java_multiple_files = true; +option java_outer_classname = "ExprProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a textual expression in the Common Expression Language (CEL) +// syntax. CEL is a C-like expression language. The syntax and semantics of CEL +// are documented at https://github.com/google/cel-spec. +// +// Example (Comparison): +// +// title: "Summary size limit" +// description: "Determines if a summary is less than 100 chars" +// expression: "document.summary.size() < 100" +// +// Example (Equality): +// +// title: "Requestor is owner" +// description: "Determines if requestor is the document owner" +// expression: "document.owner == request.auth.claims.email" +// +// Example (Logic): +// +// title: "Public documents" +// description: "Determine whether the document should be publicly visible" +// expression: "document.type != 'private' && document.type != 'internal'" +// +// Example (Data Manipulation): +// +// title: "Notification string" +// description: "Create a notification string with a timestamp." +// expression: "'New message received at ' + string(document.create_time)" +// +// The exact variables and functions that may be referenced within an expression +// are determined by the service that evaluates it. See the service +// documentation for additional information. +message Expr { + // Textual representation of an expression in Common Expression Language + // syntax. + string expression = 1; + + // Optional. Title for the expression, i.e. a short string describing + // its purpose. This can be used e.g. in UIs which allow to enter the + // expression. + string title = 2; + + // Optional. Description of the expression. This is a longer text which + // describes the expression, e.g. when hovered over it in a UI. + string description = 3; + + // Optional. String indicating the location of the expression for error + // reporting, e.g. a file name and a position in the file. + string location = 4; +} diff --git a/karapace/protobuf/google/type/fraction.proto b/karapace/protobuf/google/type/fraction.proto new file mode 100644 index 000000000..6c5ae6e2a --- /dev/null +++ b/karapace/protobuf/google/type/fraction.proto @@ -0,0 +1,33 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option go_package = "google.golang.org/genproto/googleapis/type/fraction;fraction"; +option java_multiple_files = true; +option java_outer_classname = "FractionProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a fraction in terms of a numerator divided by a denominator. +message Fraction { + // The numerator in the fraction, e.g. 2 in 2/3. + int64 numerator = 1; + + // The value by which the numerator is divided, e.g. 3 in 2/3. Must be + // positive. + int64 denominator = 2; +} diff --git a/karapace/protobuf/google/type/interval.proto b/karapace/protobuf/google/type/interval.proto new file mode 100644 index 000000000..9702324cd --- /dev/null +++ b/karapace/protobuf/google/type/interval.proto @@ -0,0 +1,46 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +import "google/protobuf/timestamp.proto"; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/interval;interval"; +option java_multiple_files = true; +option java_outer_classname = "IntervalProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a time interval, encoded as a Timestamp start (inclusive) and a +// Timestamp end (exclusive). +// +// The start must be less than or equal to the end. +// When the start equals the end, the interval is empty (matches no time). +// When both start and end are unspecified, the interval matches any time. +message Interval { + // Optional. Inclusive start of the interval. + // + // If specified, a Timestamp matching this interval will have to be the same + // or after the start. + google.protobuf.Timestamp start_time = 1; + + // Optional. Exclusive end of the interval. + // + // If specified, a Timestamp matching this interval will have to be before the + // end. + google.protobuf.Timestamp end_time = 2; +} diff --git a/karapace/protobuf/google/type/latlng.proto b/karapace/protobuf/google/type/latlng.proto new file mode 100644 index 000000000..9231456e3 --- /dev/null +++ b/karapace/protobuf/google/type/latlng.proto @@ -0,0 +1,37 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/latlng;latlng"; +option java_multiple_files = true; +option java_outer_classname = "LatLngProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// An object that represents a latitude/longitude pair. This is expressed as a +// pair of doubles to represent degrees latitude and degrees longitude. Unless +// specified otherwise, this must conform to the +// WGS84 +// standard. Values must be within normalized ranges. +message LatLng { + // The latitude in degrees. It must be in the range [-90.0, +90.0]. + double latitude = 1; + + // The longitude in degrees. It must be in the range [-180.0, +180.0]. + double longitude = 2; +} diff --git a/karapace/protobuf/google/type/localized_text.proto b/karapace/protobuf/google/type/localized_text.proto new file mode 100644 index 000000000..5c6922b8c --- /dev/null +++ b/karapace/protobuf/google/type/localized_text.proto @@ -0,0 +1,36 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/localized_text;localized_text"; +option java_multiple_files = true; +option java_outer_classname = "LocalizedTextProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Localized variant of a text in a particular language. +message LocalizedText { + // Localized string in the language corresponding to `language_code' below. + string text = 1; + + // The text's BCP-47 language code, such as "en-US" or "sr-Latn". + // + // For more information, see + // http://www.unicode.org/reports/tr35/#Unicode_locale_identifier. + string language_code = 2; +} diff --git a/karapace/protobuf/google/type/money.proto b/karapace/protobuf/google/type/money.proto new file mode 100644 index 000000000..98d6494e4 --- /dev/null +++ b/karapace/protobuf/google/type/money.proto @@ -0,0 +1,42 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/money;money"; +option java_multiple_files = true; +option java_outer_classname = "MoneyProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents an amount of money with its currency type. +message Money { + // The three-letter currency code defined in ISO 4217. + string currency_code = 1; + + // The whole units of the amount. + // For example if `currencyCode` is `"USD"`, then 1 unit is one US dollar. + int64 units = 2; + + // Number of nano (10^-9) units of the amount. + // The value must be between -999,999,999 and +999,999,999 inclusive. + // If `units` is positive, `nanos` must be positive or zero. + // If `units` is zero, `nanos` can be positive, zero, or negative. + // If `units` is negative, `nanos` must be negative or zero. + // For example $-1.75 is represented as `units`=-1 and `nanos`=-750,000,000. + int32 nanos = 3; +} diff --git a/karapace/protobuf/google/type/month.proto b/karapace/protobuf/google/type/month.proto new file mode 100644 index 000000000..99e7551b1 --- /dev/null +++ b/karapace/protobuf/google/type/month.proto @@ -0,0 +1,65 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option go_package = "google.golang.org/genproto/googleapis/type/month;month"; +option java_multiple_files = true; +option java_outer_classname = "MonthProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a month in the Gregorian calendar. +enum Month { + // The unspecified month. + MONTH_UNSPECIFIED = 0; + + // The month of January. + JANUARY = 1; + + // The month of February. + FEBRUARY = 2; + + // The month of March. + MARCH = 3; + + // The month of April. + APRIL = 4; + + // The month of May. + MAY = 5; + + // The month of June. + JUNE = 6; + + // The month of July. + JULY = 7; + + // The month of August. + AUGUST = 8; + + // The month of September. + SEPTEMBER = 9; + + // The month of October. + OCTOBER = 10; + + // The month of November. + NOVEMBER = 11; + + // The month of December. + DECEMBER = 12; +} diff --git a/karapace/protobuf/google/type/phone_number.proto b/karapace/protobuf/google/type/phone_number.proto new file mode 100644 index 000000000..7bbb7d873 --- /dev/null +++ b/karapace/protobuf/google/type/phone_number.proto @@ -0,0 +1,113 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/phone_number;phone_number"; +option java_multiple_files = true; +option java_outer_classname = "PhoneNumberProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// An object representing a phone number, suitable as an API wire format. +// +// This representation: +// +// - should not be used for locale-specific formatting of a phone number, such +// as "+1 (650) 253-0000 ext. 123" +// +// - is not designed for efficient storage +// - may not be suitable for dialing - specialized libraries (see references) +// should be used to parse the number for that purpose +// +// To do something meaningful with this number, such as format it for various +// use-cases, convert it to an `i18n.phonenumbers.PhoneNumber` object first. +// +// For instance, in Java this would be: +// +// com.google.type.PhoneNumber wireProto = +// com.google.type.PhoneNumber.newBuilder().build(); +// com.google.i18n.phonenumbers.Phonenumber.PhoneNumber phoneNumber = +// PhoneNumberUtil.getInstance().parse(wireProto.getE164Number(), "ZZ"); +// if (!wireProto.getExtension().isEmpty()) { +// phoneNumber.setExtension(wireProto.getExtension()); +// } +// +// Reference(s): +// - https://github.com/google/libphonenumber +message PhoneNumber { + // An object representing a short code, which is a phone number that is + // typically much shorter than regular phone numbers and can be used to + // address messages in MMS and SMS systems, as well as for abbreviated dialing + // (e.g. "Text 611 to see how many minutes you have remaining on your plan."). + // + // Short codes are restricted to a region and are not internationally + // dialable, which means the same short code can exist in different regions, + // with different usage and pricing, even if those regions share the same + // country calling code (e.g. US and CA). + message ShortCode { + // Required. The BCP-47 region code of the location where calls to this + // short code can be made, such as "US" and "BB". + // + // Reference(s): + // - http://www.unicode.org/reports/tr35/#unicode_region_subtag + string region_code = 1; + + // Required. The short code digits, without a leading plus ('+') or country + // calling code, e.g. "611". + string number = 2; + } + + // Required. Either a regular number, or a short code. New fields may be + // added to the oneof below in the future, so clients should ignore phone + // numbers for which none of the fields they coded against are set. + oneof kind { + // The phone number, represented as a leading plus sign ('+'), followed by a + // phone number that uses a relaxed ITU E.164 format consisting of the + // country calling code (1 to 3 digits) and the subscriber number, with no + // additional spaces or formatting, e.g.: + // - correct: "+15552220123" + // - incorrect: "+1 (555) 222-01234 x123". + // + // The ITU E.164 format limits the latter to 12 digits, but in practice not + // all countries respect that, so we relax that restriction here. + // National-only numbers are not allowed. + // + // References: + // - https://www.itu.int/rec/T-REC-E.164-201011-I + // - https://en.wikipedia.org/wiki/E.164. + // - https://en.wikipedia.org/wiki/List_of_country_calling_codes + string e164_number = 1; + + // A short code. + // + // Reference(s): + // - https://en.wikipedia.org/wiki/Short_code + ShortCode short_code = 2; + } + + // The phone number's extension. The extension is not standardized in ITU + // recommendations, except for being defined as a series of numbers with a + // maximum length of 40 digits. Other than digits, some other dialing + // characters such as ',' (indicating a wait) or '#' may be stored here. + // + // Note that no regions currently use extensions with short codes, so this + // field is normally only set in conjunction with an E.164 number. It is held + // separately from the E.164 number to allow for short code extensions in the + // future. + string extension = 3; +} diff --git a/karapace/protobuf/google/type/postal_address.proto b/karapace/protobuf/google/type/postal_address.proto new file mode 100644 index 000000000..c57c7c31a --- /dev/null +++ b/karapace/protobuf/google/type/postal_address.proto @@ -0,0 +1,134 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/postaladdress;postaladdress"; +option java_multiple_files = true; +option java_outer_classname = "PostalAddressProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a postal address, e.g. for postal delivery or payments addresses. +// Given a postal address, a postal service can deliver items to a premise, P.O. +// Box or similar. +// It is not intended to model geographical locations (roads, towns, +// mountains). +// +// In typical usage an address would be created via user input or from importing +// existing data, depending on the type of process. +// +// Advice on address input / editing: +// - Use an i18n-ready address widget such as +// https://github.com/google/libaddressinput) +// - Users should not be presented with UI elements for input or editing of +// fields outside countries where that field is used. +// +// For more guidance on how to use this schema, please see: +// https://support.google.com/business/answer/6397478 +message PostalAddress { + // The schema revision of the `PostalAddress`. This must be set to 0, which is + // the latest revision. + // + // All new revisions **must** be backward compatible with old revisions. + int32 revision = 1; + + // Required. CLDR region code of the country/region of the address. This + // is never inferred and it is up to the user to ensure the value is + // correct. See http://cldr.unicode.org/ and + // http://www.unicode.org/cldr/charts/30/supplemental/territory_information.html + // for details. Example: "CH" for Switzerland. + string region_code = 2; + + // Optional. BCP-47 language code of the contents of this address (if + // known). This is often the UI language of the input form or is expected + // to match one of the languages used in the address' country/region, or their + // transliterated equivalents. + // This can affect formatting in certain countries, but is not critical + // to the correctness of the data and will never affect any validation or + // other non-formatting related operations. + // + // If this value is not known, it should be omitted (rather than specifying a + // possibly incorrect default). + // + // Examples: "zh-Hant", "ja", "ja-Latn", "en". + string language_code = 3; + + // Optional. Postal code of the address. Not all countries use or require + // postal codes to be present, but where they are used, they may trigger + // additional validation with other parts of the address (e.g. state/zip + // validation in the U.S.A.). + string postal_code = 4; + + // Optional. Additional, country-specific, sorting code. This is not used + // in most regions. Where it is used, the value is either a string like + // "CEDEX", optionally followed by a number (e.g. "CEDEX 7"), or just a number + // alone, representing the "sector code" (Jamaica), "delivery area indicator" + // (Malawi) or "post office indicator" (e.g. Côte d'Ivoire). + string sorting_code = 5; + + // Optional. Highest administrative subdivision which is used for postal + // addresses of a country or region. + // For example, this can be a state, a province, an oblast, or a prefecture. + // Specifically, for Spain this is the province and not the autonomous + // community (e.g. "Barcelona" and not "Catalonia"). + // Many countries don't use an administrative area in postal addresses. E.g. + // in Switzerland this should be left unpopulated. + string administrative_area = 6; + + // Optional. Generally refers to the city/town portion of the address. + // Examples: US city, IT comune, UK post town. + // In regions of the world where localities are not well defined or do not fit + // into this structure well, leave locality empty and use address_lines. + string locality = 7; + + // Optional. Sublocality of the address. + // For example, this can be neighborhoods, boroughs, districts. + string sublocality = 8; + + // Unstructured address lines describing the lower levels of an address. + // + // Because values in address_lines do not have type information and may + // sometimes contain multiple values in a single field (e.g. + // "Austin, TX"), it is important that the line order is clear. The order of + // address lines should be "envelope order" for the country/region of the + // address. In places where this can vary (e.g. Japan), address_language is + // used to make it explicit (e.g. "ja" for large-to-small ordering and + // "ja-Latn" or "en" for small-to-large). This way, the most specific line of + // an address can be selected based on the language. + // + // The minimum permitted structural representation of an address consists + // of a region_code with all remaining information placed in the + // address_lines. It would be possible to format such an address very + // approximately without geocoding, but no semantic reasoning could be + // made about any of the address components until it was at least + // partially resolved. + // + // Creating an address only containing a region_code and address_lines, and + // then geocoding is the recommended way to handle completely unstructured + // addresses (as opposed to guessing which parts of the address should be + // localities or administrative areas). + repeated string address_lines = 9; + + // Optional. The recipient at the address. + // This field may, under certain circumstances, contain multiline information. + // For example, it might contain "care of" information. + repeated string recipients = 10; + + // Optional. The name of the organization at the address. + string organization = 11; +} diff --git a/karapace/protobuf/google/type/quaternion.proto b/karapace/protobuf/google/type/quaternion.proto new file mode 100644 index 000000000..dfb822def --- /dev/null +++ b/karapace/protobuf/google/type/quaternion.proto @@ -0,0 +1,94 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/quaternion;quaternion"; +option java_multiple_files = true; +option java_outer_classname = "QuaternionProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// A quaternion is defined as the quotient of two directed lines in a +// three-dimensional space or equivalently as the quotient of two Euclidean +// vectors (https://en.wikipedia.org/wiki/Quaternion). +// +// Quaternions are often used in calculations involving three-dimensional +// rotations (https://en.wikipedia.org/wiki/Quaternions_and_spatial_rotation), +// as they provide greater mathematical robustness by avoiding the gimbal lock +// problems that can be encountered when using Euler angles +// (https://en.wikipedia.org/wiki/Gimbal_lock). +// +// Quaternions are generally represented in this form: +// +// w + xi + yj + zk +// +// where x, y, z, and w are real numbers, and i, j, and k are three imaginary +// numbers. +// +// Our naming choice `(x, y, z, w)` comes from the desire to avoid confusion for +// those interested in the geometric properties of the quaternion in the 3D +// Cartesian space. Other texts often use alternative names or subscripts, such +// as `(a, b, c, d)`, `(1, i, j, k)`, or `(0, 1, 2, 3)`, which are perhaps +// better suited for mathematical interpretations. +// +// To avoid any confusion, as well as to maintain compatibility with a large +// number of software libraries, the quaternions represented using the protocol +// buffer below *must* follow the Hamilton convention, which defines `ij = k` +// (i.e. a right-handed algebra), and therefore: +// +// i^2 = j^2 = k^2 = ijk = −1 +// ij = −ji = k +// jk = −kj = i +// ki = −ik = j +// +// Please DO NOT use this to represent quaternions that follow the JPL +// convention, or any of the other quaternion flavors out there. +// +// Definitions: +// +// - Quaternion norm (or magnitude): `sqrt(x^2 + y^2 + z^2 + w^2)`. +// - Unit (or normalized) quaternion: a quaternion whose norm is 1. +// - Pure quaternion: a quaternion whose scalar component (`w`) is 0. +// - Rotation quaternion: a unit quaternion used to represent rotation. +// - Orientation quaternion: a unit quaternion used to represent orientation. +// +// A quaternion can be normalized by dividing it by its norm. The resulting +// quaternion maintains the same direction, but has a norm of 1, i.e. it moves +// on the unit sphere. This is generally necessary for rotation and orientation +// quaternions, to avoid rounding errors: +// https://en.wikipedia.org/wiki/Rotation_formalisms_in_three_dimensions +// +// Note that `(x, y, z, w)` and `(-x, -y, -z, -w)` represent the same rotation, +// but normalization would be even more useful, e.g. for comparison purposes, if +// it would produce a unique representation. It is thus recommended that `w` be +// kept positive, which can be achieved by changing all the signs when `w` is +// negative. +// +message Quaternion { + // The x component. + double x = 1; + + // The y component. + double y = 2; + + // The z component. + double z = 3; + + // The scalar component. + double w = 4; +} diff --git a/karapace/protobuf/google/type/timeofday.proto b/karapace/protobuf/google/type/timeofday.proto new file mode 100644 index 000000000..5cb48aa93 --- /dev/null +++ b/karapace/protobuf/google/type/timeofday.proto @@ -0,0 +1,44 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/timeofday;timeofday"; +option java_multiple_files = true; +option java_outer_classname = "TimeOfDayProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a time of day. The date and time zone are either not significant +// or are specified elsewhere. An API may choose to allow leap seconds. Related +// types are [google.type.Date][google.type.Date] and +// `google.protobuf.Timestamp`. +message TimeOfDay { + // Hours of day in 24 hour format. Should be from 0 to 23. An API may choose + // to allow the value "24:00:00" for scenarios like business closing time. + int32 hours = 1; + + // Minutes of hour of day. Must be from 0 to 59. + int32 minutes = 2; + + // Seconds of minutes of the time. Must normally be from 0 to 59. An API may + // allow the value 60 if it allows leap-seconds. + int32 seconds = 3; + + // Fractions of seconds in nanoseconds. Must be from 0 to 999,999,999. + int32 nanos = 4; +} diff --git a/karapace/protobuf/google/type/type.yaml b/karapace/protobuf/google/type/type.yaml new file mode 100644 index 000000000..d5c71364d --- /dev/null +++ b/karapace/protobuf/google/type/type.yaml @@ -0,0 +1,40 @@ +type: google.api.Service +config_version: 3 +name: type.googleapis.com +title: Common Types + +types: +- name: google.type.Color +- name: google.type.Date +- name: google.type.DateTime +- name: google.type.Decimal +- name: google.type.Expr +- name: google.type.Fraction +- name: google.type.Interval +- name: google.type.LatLng +- name: google.type.LocalizedText +- name: google.type.Money +- name: google.type.PhoneNumber +- name: google.type.PostalAddress +- name: google.type.Quaternion +- name: google.type.TimeOfDay + +enums: +- name: google.type.CalendarPeriod +- name: google.type.DayOfWeek +- name: google.type.Month + +documentation: + summary: Defines common types for Google APIs. + overview: |- + # Google Common Types + + This package contains definitions of common types for Google APIs. + All types defined in this package are suitable for different APIs to + exchange data, and will never break binary compatibility. They should + have design quality comparable to major programming languages like + Java and C#. + + NOTE: Some common types are defined in the package `google.protobuf` + as they are directly supported by Protocol Buffers compiler and + runtime. Those types are called Well-Known Types. diff --git a/karapace/protobuf/known_dependency.py b/karapace/protobuf/known_dependency.py new file mode 100644 index 000000000..5c3e74a00 --- /dev/null +++ b/karapace/protobuf/known_dependency.py @@ -0,0 +1,154 @@ +# Support of known dependencies + +from enum import Enum +from typing import Any, Dict, Set + + +def static_init(cls: Any) -> object: + if getattr(cls, "static_init", None): + cls.static_init() + return cls + + +class KnownDependencyLocation(Enum): + ANY_LOCATION = "google/protobuf/any.proto" + API_LOCATION = "google/protobuf/api.proto" + DESCRIPTOR_LOCATION = "google/protobuf/descriptor.proto" + DURATION_LOCATION = "google/protobuf/duration.proto" + EMPTY_LOCATION = "google/protobuf/empty.proto" + FIELD_MASK_LOCATION = "google/protobuf/field_mask.proto" + SOURCE_CONTEXT_LOCATION = "google/protobuf/source_context.proto" + STRUCT_LOCATION = "google/protobuf/struct.proto" + TIMESTAMP_LOCATION = "google/protobuf/timestamp.proto" + TYPE_LOCATION = "google/protobuf/type.proto" + WRAPPER_LOCATION = "google/protobuf/wrappers.proto" + CALENDAR_PERIOD_LOCATION = "google/type/calendar_period.proto" + COLOR_LOCATION = "google/type/color.proto" + DATE_LOCATION = "google/type/date.proto" + DATETIME_LOCATION = "google/type/datetime.proto" + DAY_OF_WEEK_LOCATION = "google/type/dayofweek.proto" + DECIMAL_LOCATION = "google/type/decimal.proto" + EXPR_LOCATION = "google/type/expr.proto" + FRACTION_LOCATION = "google/type/fraction.proto" + INTERVAL_LOCATION = "google/type/interval.proto" + LATLNG_LOCATION = "google/type/latlng.proto" + MONEY_LOCATION = "google/type/money.proto" + MONTH_LOCATION = "google/type/month.proto" + PHONE_NUMBER_LOCATION = "google/type/phone_number.proto" + POSTAL_ADDRESS_LOCATION = "google/type/postal_address.proto" + QUATERNION_LOCATION = "google/type/quaternion.proto" + TIME_OF_DAY_LOCATION = "google/type/timeofday.proto" + + +@static_init +class KnownDependency: + index: Dict = dict() + index_simple: Dict = dict() + map: Dict = { + "google/protobuf/any.proto": [".google.protobuf.Any"], + "google/protobuf/api.proto": [".google.protobuf.Api", ".google.protobuf.Method", ".google.protobuf.Mixin"], + "google/protobuf/descriptor.proto": [ + ".google.protobuf.FileDescriptorSet", + ".google.protobuf.FileDescriptorProto", + ".google.protobuf.DescriptorProto", + ".google.protobuf.ExtensionRangeOptions", + ".google.protobuf.FieldDescriptorProto", + ".google.protobuf.OneofDescriptorProto", + ".google.protobuf.EnumDescriptorProto", + ".google.protobuf.EnumValueDescriptorProto", + ".google.protobuf.ServiceDescriptorProto", + ".google.protobuf.MethodDescriptorProto", + ".google.protobuf.FileOptions", + ".google.protobuf.MessageOptions", + ".google.protobuf.FieldOptions", + ".google.protobuf.OneofOptions", + ".google.protobuf.EnumOptions", + ".google.protobuf.EnumValueOptions", + ".google.protobuf.ServiceOptions", + ".google.protobuf.MethodOptions", + ".google.protobuf.UninterpretedOption", + ".google.protobuf.SourceCodeInfo", + ".google.protobuf.GeneratedCodeInfo", + ], + "google/protobuf/duration.proto": [".google.protobuf.Duration"], + "google/protobuf/empty.proto": [".google.protobuf.Empty"], + "google/protobuf/field_mask.proto": [".google.protobuf.FieldMask"], + "google/protobuf/source_context.proto": [".google.protobuf.SourceContext"], + "google/protobuf/struct.proto": [ + ".google.protobuf.Struct", + ".google.protobuf.Value", + ".google.protobuf.NullValue", + ".google.protobuf.ListValue", + ], + "google/protobuf/timestamp.proto": [".google.protobuf.Timestamp"], + "google/protobuf/type.proto": [ + ".google.protobuf.Type", + ".google.protobuf.Field", + ".google.protobuf.Enum", + ".google.protobuf.EnumValue", + ".google.protobuf.Option", + ".google.protobuf.Syntax", + ], + "google/protobuf/wrappers.proto": [ + ".google.protobuf.DoubleValue", + ".google.protobuf.FloatValue", + ".google.protobuf.Int64Value", + ".google.protobuf.UInt64Value", + ".google.protobuf.Int32Value", + ".google.protobuf.UInt32Value", + ".google.protobuf.BoolValue", + ".google.protobuf.StringValue", + ".google.protobuf.BytesValue", + ], + "google/type/calendar_period.proto": [".google.type.CalendarPeriod"], + "google/type/color.proto": [".google.type.Color"], + "google/type/date.proto": [".google.type.Date"], + "google/type/datetime.proto": [".google.type.DateTime", ".google.type.TimeZone"], + "google/type/dayofweek.proto": [".google.type.DayOfWeek"], + "google/type/decimal.proto": [".google.type.Decimal"], + "google/type/expr.proto": [".google.type.Expr"], + "google/type/fraction.proto": [".google.type.Fraction"], + "google/type/interval.proto": [".google.type.Interval"], + "google/type/latlng.proto": [".google.type.LatLng"], + "google/type/money.proto": [".google.type.Money"], + "google/type/month.proto": [".google.type.Month"], + "google/type/phone_number.proto": [".google.type.PhoneNumber"], + "google/type/postal_address.proto": [".google.type.PostalAddress"], + "google/type/quaternion.proto": [".google.type.Quaternion"], + "google/type/timeofday.proto": [".google.type.TimeOfDay"], + "confluent/meta.proto": [".confluent.Meta"], + "confluent/type/decimal.proto": [".confluent.type.Decimal"], + } + + @classmethod + def static_init(cls) -> None: + for key, value in cls.map.items(): + for item in value: + cls.index[item] = key + dot = item.rfind(".") + cls.index_simple[item[dot + 1 :]] = key + + +@static_init +class DependenciesHardcoded: + index: Set = set() + + @classmethod + def static_init(cls) -> None: + cls.index = { + "bool", + "bytes", + "double", + "float", + "fixed32", + "fixed64", + "int32", + "int64", + "sfixed32", + "sfixed64", + "sint32", + "sint64", + "string", + "uint32", + "uint64", + } diff --git a/karapace/protobuf/proto_file_element.py b/karapace/protobuf/proto_file_element.py index 4774a8c67..fa43dd9e8 100644 --- a/karapace/protobuf/proto_file_element.py +++ b/karapace/protobuf/proto_file_element.py @@ -9,23 +9,24 @@ from karapace.protobuf.message_element import MessageElement from karapace.protobuf.syntax import Syntax from karapace.protobuf.type_element import TypeElement +from typing import List, Optional class ProtoFileElement: def __init__( self, location: Location, - package_name: str = None, - syntax: Syntax = None, - imports: list = None, - public_imports: list = None, - types=None, - services: list = None, - extend_declarations: list = None, - options: list = None, + package_name: Optional[str] = None, + syntax: Optional[Syntax] = None, + imports: Optional[list] = None, + public_imports: Optional[list] = None, + types: Optional[List[TypeElement]] = None, + services: Optional[list] = None, + extend_declarations: Optional[list] = None, + options: Optional[list] = None, ) -> None: if types is None: - types = [] + types = list() self.location = location self.package_name = package_name self.syntax = syntax diff --git a/karapace/protobuf/proto_parser.py b/karapace/protobuf/proto_parser.py index 81c333af4..27cc59f11 100644 --- a/karapace/protobuf/proto_parser.py +++ b/karapace/protobuf/proto_parser.py @@ -25,7 +25,7 @@ from karapace.protobuf.syntax_reader import SyntaxReader from karapace.protobuf.type_element import TypeElement from karapace.protobuf.utils import MAX_TAG_VALUE -from typing import List, Union +from typing import List, Optional, Union class Context(Enum): @@ -71,13 +71,13 @@ class ProtoParser: def __init__(self, location: Location, data: str) -> None: self.location = location self.imports: List[str] = [] - self.nested_types: List[str] = [] + self.nested_types: List[TypeElement] = [] self.services: List[str] = [] self.extends_list: List[str] = [] self.options: List[str] = [] self.declaration_count = 0 - self.syntax: Union[Syntax, None] = None - self.package_name: Union[str, None] = None + self.syntax: Optional[Syntax] = None + self.package_name: Optional[str] = None self.prefix = "" self.data = data self.public_imports: List[str] = [] @@ -176,7 +176,6 @@ def read_declaration( import_string = self.reader.read_string() if import_string == "public": self.public_imports.append(self.reader.read_string()) - else: self.imports.append(import_string) self.reader.require(";") diff --git a/karapace/protobuf/schema.py b/karapace/protobuf/schema.py index ae723be1b..76e319019 100644 --- a/karapace/protobuf/schema.py +++ b/karapace/protobuf/schema.py @@ -1,7 +1,10 @@ # Ported from square/wire: +# Ported from square/wire: # wire-library/wire-schema/src/commonMain/kotlin/com/squareup/wire/schema/Schema.kt # Ported partially for required functionality. +from karapace.dependency import Dependency, DependencyVerifierResult from karapace.protobuf.compare_result import CompareResult +from karapace.protobuf.dependency import _process_one_of, ProtobufDependencyVerifier from karapace.protobuf.enum_element import EnumElement from karapace.protobuf.exception import IllegalArgumentException from karapace.protobuf.location import Location @@ -9,7 +12,10 @@ from karapace.protobuf.option_element import OptionElement from karapace.protobuf.proto_file_element import ProtoFileElement from karapace.protobuf.proto_parser import ProtoParser +from karapace.protobuf.type_element import TypeElement from karapace.protobuf.utils import append_documentation, append_indented +from karapace.schema_references import References +from typing import Dict, Optional def add_slashes(text: str) -> str: @@ -100,12 +106,67 @@ def option_element_string(option: OptionElement) -> str: class ProtobufSchema: DEFAULT_LOCATION = Location.get("") - def __init__(self, schema: str) -> None: + def __init__( + self, schema: str, references: Optional[References] = None, dependencies: Optional[Dict[str, Dependency]] = None + ) -> None: if type(schema).__name__ != "str": raise IllegalArgumentException("Non str type of schema string") self.dirty = schema self.cache_string = "" self.proto_file_element = ProtoParser.parse(self.DEFAULT_LOCATION, schema) + self.references = references + self.dependencies = dependencies + + def gather_deps(self) -> ProtobufDependencyVerifier: + verifier = ProtobufDependencyVerifier() + self.collect_dependencies(verifier) + return verifier + + def verify_schema_dependencies(self) -> DependencyVerifierResult: + verifier = ProtobufDependencyVerifier() + self.collect_dependencies(verifier) + return verifier.verify() + + def collect_dependencies(self, verifier: ProtobufDependencyVerifier): + + if self.dependencies: + for key in self.dependencies: + self.dependencies[key].schema.schema.collect_dependencies(verifier) + # verifier.add_import?? we have no access to own Kafka structure from this class... + # but we need data to analyse imports to avoid ciclyc dependencies... + + package_name = self.proto_file_element.package_name + if package_name is None: + package_name = "" + else: + package_name = "." + package_name + for element_type in self.proto_file_element.types: + type_name = element_type.name + full_name = package_name + "." + type_name + verifier.add_declared_type(full_name) + verifier.add_declared_type(type_name) + if isinstance(element_type, MessageElement): + for one_of in element_type.one_ofs: + _process_one_of(verifier, package_name, type_name, one_of) + for field in element_type.fields: + verifier.add_used_type(full_name, field.element_type) + for nested_type in element_type.nested_types: + self._process_nested_type(verifier, package_name, type_name, nested_type) + + def _process_nested_type( + self, verifier: ProtobufDependencyVerifier, package_name: str, parent_name, element_type: TypeElement + ): + + verifier.add_declared_type(package_name + "." + parent_name + "." + element_type.name) + verifier.add_declared_type(parent_name + "." + element_type.name) + + if isinstance(element_type, MessageElement): + for one_of in element_type.one_ofs: + _process_one_of(verifier, package_name, parent_name, one_of) + for field in element_type.fields: + verifier.add_used_type(parent_name, field.element_type) + for nested_type in element_type.nested_types: + self._process_nested_type(verifier, package_name, parent_name + "." + element_type.name, nested_type) def __str__(self) -> str: if not self.cache_string: diff --git a/karapace/schema_models.py b/karapace/schema_models.py index 4ac732d02..7448c4a8f 100644 --- a/karapace/schema_models.py +++ b/karapace/schema_models.py @@ -1,6 +1,5 @@ from avro.errors import SchemaParseException from avro.schema import parse as avro_parse, Schema as AvroSchema -from enum import Enum, unique from jsonschema import Draft7Validator from jsonschema.exceptions import SchemaError from karapace.errors import InvalidSchema @@ -10,9 +9,12 @@ IllegalStateException, ProtobufException, ProtobufParserRuntimeException, + ProtobufUnresolvedDependencyException, SchemaParseException as ProtobufSchemaParseException, ) -from karapace.protobuf.schema import ProtobufSchema +from karapace.protobuf.schema import Dependency, ProtobufSchema +from karapace.schema_references import References +from karapace.schema_type import SchemaType from karapace.utils import json_encode from typing import Any, Dict, Optional, Union @@ -49,36 +51,42 @@ def parse_jsonschema_definition(schema_definition: str) -> Draft7Validator: return Draft7Validator(schema) -def parse_protobuf_schema_definition(schema_definition: str) -> ProtobufSchema: +def parse_protobuf_schema_definition( + schema_definition: str, references: Optional[References] = None, dependencies: Optional[Dict[str, Dependency]] = None +) -> ProtobufSchema: """Parses and validates `schema_definition`. Raises: Nothing yet. """ - - return ProtobufSchema(schema_definition) - - -@unique -class SchemaType(str, Enum): - AVRO = "AVRO" - JSONSCHEMA = "JSON" - PROTOBUF = "PROTOBUF" + protobuf_schema = ProtobufSchema(schema_definition, references, dependencies) + result = protobuf_schema.verify_schema_dependencies() + if not result.result: + raise ProtobufUnresolvedDependencyException(f"{result.message}") + return protobuf_schema class TypedSchema: - def __init__(self, schema_type: SchemaType, schema_str: str): + def __init__( + self, + schema_type: SchemaType, + schema_str: str, + references: Optional[References] = None, + dependencies: Optional[Dict[str, Dependency]] = None, + ): """Schema with type information Args: schema_type (SchemaType): The type of the schema schema_str (str): The original schema string + references(References): The references of schema """ self.schema_type = schema_type self.schema_str = schema_str + self.references = references + self.dependencies = dependencies self.max_id: Optional[int] = None - self._str_cached: Optional[str] = None def to_dict(self) -> Dict[str, Any]: @@ -97,17 +105,39 @@ def __str__(self) -> str: def __repr__(self) -> str: return f"TypedSchema(type={self.schema_type}, schema={str(self)})" + def get_references(self) -> Optional["References"]: + return self.references + def __eq__(self, other: Any) -> bool: - return isinstance(other, TypedSchema) and self.__str__() == other.__str__() and self.schema_type is other.schema_type + schema_is_equal = ( + isinstance(other, TypedSchema) and self.schema_type is other.schema_type and self.__str__() == other.__str__() + ) + if not schema_is_equal: + return False + if self.references is not None: + return self.references == other.references + return other.references is None class ValidatedTypedSchema(TypedSchema): - def __init__(self, schema_type: SchemaType, schema_str: str, schema: Union[Draft7Validator, AvroSchema, ProtobufSchema]): - super().__init__(schema_type=schema_type, schema_str=schema_str) + def __init__( + self, + schema_type: SchemaType, + schema_str: str, + schema: Union[Draft7Validator, AvroSchema, ProtobufSchema], + references: Optional["References"] = None, + dependencies: Optional[Dict[str, Dependency]] = None, + ): + super().__init__(schema_type=schema_type, schema_str=schema_str, references=references, dependencies=dependencies) self.schema = schema @staticmethod - def parse(schema_type: SchemaType, schema_str: str) -> "ValidatedTypedSchema": + def parse( + schema_type: SchemaType, + schema_str: str, + references: Optional["References"] = None, + dependencies: Optional[Dict[str, Dependency]] = None, + ) -> "ValidatedTypedSchema": if schema_type not in [SchemaType.AVRO, SchemaType.JSONSCHEMA, SchemaType.PROTOBUF]: raise InvalidSchema(f"Unknown parser {schema_type} for {schema_str}") @@ -127,7 +157,7 @@ def parse(schema_type: SchemaType, schema_str: str) -> "ValidatedTypedSchema": elif schema_type is SchemaType.PROTOBUF: try: - parsed_schema = parse_protobuf_schema_definition(schema_str) + parsed_schema = parse_protobuf_schema_definition(schema_str, references, dependencies) except ( TypeError, SchemaError, @@ -143,7 +173,13 @@ def parse(schema_type: SchemaType, schema_str: str) -> "ValidatedTypedSchema": else: raise InvalidSchema(f"Unknown parser {schema_type} for {schema_str}") - return ValidatedTypedSchema(schema_type=schema_type, schema_str=schema_str, schema=parsed_schema) + return ValidatedTypedSchema( + schema_type=schema_type, + schema_str=schema_str, + schema=parsed_schema, + references=references, + dependencies=dependencies, + ) def __str__(self) -> str: if self.schema_type == SchemaType.PROTOBUF: diff --git a/karapace/schema_reader.py b/karapace/schema_reader.py index a3e051b75..01953f3a7 100644 --- a/karapace/schema_reader.py +++ b/karapace/schema_reader.py @@ -10,15 +10,17 @@ from kafka.errors import KafkaConfigurationError, NoBrokersAvailable, NodeNotReadyError, TopicAlreadyExistsError from karapace import constants from karapace.config import Config -from karapace.errors import InvalidSchema +from karapace.dependency import Dependency +from karapace.errors import InvalidReferences, InvalidSchema, ReferencesNotSupportedException from karapace.key_format import is_key_in_canonical_format, KeyFormatter, KeyMode from karapace.master_coordinator import MasterCoordinator from karapace.schema_models import SchemaType, TypedSchema, ValidatedTypedSchema +from karapace.schema_references import References from karapace.statsd import StatsClient from karapace.typing import SubjectData -from karapace.utils import KarapaceKafkaClient +from karapace.utils import KarapaceKafkaClient, reference_key from threading import Condition, Event, Lock, Thread -from typing import Any, Dict, Optional +from typing import Any, Dict, List, Optional, Tuple import json import logging @@ -27,6 +29,7 @@ Subject = str Version = int Schema = Dict[str, Any] +Referents = List SchemaId = int # The value `0` is a valid offset and it represents the first message produced @@ -37,7 +40,6 @@ KAFKA_CLIENT_CREATION_TIMEOUT_SECONDS = 2.0 SCHEMA_TOPIC_CREATION_TIMEOUT_SECONDS = 5.0 - # Metric names METRIC_SCHEMA_TOPIC_RECORDS_PROCESSED_COUNT = "karapace_schema_reader_records_processed" METRIC_SCHEMA_TOPIC_RECORDS_PER_KEYMODE_GAUGE = "karapace_schema_reader_records_per_keymode" @@ -137,6 +139,7 @@ def __init__( self.config = config self.subjects: Dict[Subject, SubjectData] = {} self.schemas: Dict[int, TypedSchema] = {} + self.referenced_by: Dict[str, Referents] = {} self.global_schema_id = 0 self.admin_client: Optional[KafkaAdminClient] = None self.topic_replication_factor = self.config["replication_factor"] @@ -375,7 +378,6 @@ def _handle_msg_config(self, key: dict, value: Optional[dict]) -> None: if subject not in self.subjects: LOG.info("Adding first version of subject: %r with no schemas", subject) self.subjects[subject] = {"schemas": {}} - if not value: LOG.info("Deleting compatibility config completely for subject: %r", subject) self.subjects[subject].pop("compatibility", None) @@ -428,6 +430,7 @@ def _handle_msg_schema(self, key: dict, value: Optional[dict]) -> None: schema_id = value["id"] schema_version = value["version"] schema_deleted = value.get("deleted", False) + schema_references = value.get("references", None) try: schema_type_parsed = SchemaType(schema_type) @@ -449,7 +452,10 @@ def _handle_msg_schema(self, key: dict, value: Optional[dict]) -> None: return elif schema_type_parsed == SchemaType.PROTOBUF: try: - parsed_schema = ValidatedTypedSchema.parse(SchemaType.PROTOBUF, schema_str) + resolved_references, resolved_dependencies = self.resolve_schema_references(value) + parsed_schema = ValidatedTypedSchema.parse( + SchemaType.PROTOBUF, schema_str, resolved_references, resolved_dependencies + ) schema_str = str(parsed_schema) except InvalidSchema: LOG.exception("Schema is not valid ProtoBuf definition") @@ -465,21 +471,34 @@ def _handle_msg_schema(self, key: dict, value: Optional[dict]) -> None: # dedup schemas to reduce memory pressure schema_str = self._hash_to_schema.setdefault(hash(schema_str), schema_str) - if schema_version in subjects_schemas: - LOG.info("Updating entry subject: %r version: %r id: %r", schema_subject, schema_version, schema_id) - else: - LOG.info("Adding entry subject: %r version: %r id: %r", schema_subject, schema_version, schema_id) - typed_schema = TypedSchema( schema_type=schema_type_parsed, schema_str=schema_str, + references=schema_references, ) - subjects_schemas[schema_version] = { + schema = { "schema": typed_schema, "version": schema_version, "id": schema_id, "deleted": schema_deleted, } + if schema_references: + schema["references"] = schema_references + + if schema_version in subjects_schemas: + LOG.info("Updating entry subject: %r version: %r id: %r", schema_subject, schema_version, schema_id) + else: + LOG.info("Adding entry subject: %r version: %r id: %r", schema_subject, schema_version, schema_id) + + subjects_schemas[schema_version] = schema + if schema_references: + for ref in schema_references: + ref_str = reference_key(ref["subject"], ref["version"]) + referents = self.referenced_by.get(ref_str, None) + if referents: + referents.append(schema_id) + else: + self.referenced_by[ref_str] = [schema_id] self.schemas[schema_id] = typed_schema self.global_schema_id = max(self.global_schema_id, schema_id) @@ -532,3 +551,62 @@ def get_schemas_list(self, *, include_deleted: bool, latest_only: bool) -> Dict[ selected_schemas = [schema for schema in selected_schemas if schema.get("deleted", False) is False] res_schemas[subject] = selected_schemas return res_schemas + + def remove_referenced_by(self, schema_id: SchemaId, references: List): + for ref in references: + key = reference_key(ref["subject"], ref["version"]) + if self.referenced_by.get(key, None) and schema_id in self.referenced_by[key]: + self.referenced_by[key].remove(schema_id) + + def resolve_references(self, references: Optional["References"] = None) -> Optional[Dict[str, Dependency]]: + if references is None: + return None + dependencies = dict() + + for r in references.val(): + subject = r["subject"] + version = r["version"] + name = r["name"] + subject_data = self.subjects.get(subject) + if subject_data is not None: + schema_data = subject_data["schemas"][version] + schema_references, schema_dependencies = self.resolve_schema_references(schema_data) + else: + raise InvalidReferences + + parsed_schema = ValidatedTypedSchema.parse( + schema_type=schema_data["schema"].schema_type, + schema_str=schema_data["schema"].schema_str, + references=schema_references, + dependencies=schema_dependencies, + ) + dependencies[name] = Dependency(name, subject, version, parsed_schema) + return dependencies + + def resolve_schema_references( + self, schema_data: Optional[dict] + ) -> Tuple[Optional[References], Optional[Dict[str, Dependency]]]: + + if schema_data is None: + raise InvalidSchema + + schema_references = schema_data.get("references") + if schema_references is None: + return None, None + + schema_type = schema_data.get("schemaType") + if schema_type is None: + schema = schema_data.get("schema") + if schema is None: + raise InvalidReferences + if isinstance(schema, TypedSchema): + schema_type = schema.schema_type + else: + schema_type = None + if schema_type != SchemaType.PROTOBUF: + raise ReferencesNotSupportedException + + schema_references = References(schema_type, schema_references) + schema_dependencies = self.resolve_references(schema_references) + + return schema_references, schema_dependencies diff --git a/karapace/schema_references.py b/karapace/schema_references.py new file mode 100644 index 000000000..6663a3d12 --- /dev/null +++ b/karapace/schema_references.py @@ -0,0 +1,27 @@ +from karapace.schema_type import SchemaType +from karapace.typing import JsonData +from karapace.utils import json_encode +from typing import Any + + +class References: + def __init__(self, schema_type: SchemaType, references: JsonData): + """Schema with type information + + Args: + schema_type (SchemaType): The type of the schema + references (str): The original schema string + """ + self.schema_type = schema_type + self.references = references + + def val(self) -> JsonData: + return self.references + + def json(self) -> str: + return str(json_encode(self.references, sort_keys=True)) + + def __eq__(self, other: Any) -> bool: + if other is None or not isinstance(other, References): + return False + return self.json() == other.json() diff --git a/karapace/schema_registry.py b/karapace/schema_registry.py index 6d30bb971..6317b36c2 100644 --- a/karapace/schema_registry.py +++ b/karapace/schema_registry.py @@ -4,9 +4,11 @@ from karapace.compatibility import check_compatibility, CompatibilityModes from karapace.compatibility.jsonschema.checks import is_incompatible from karapace.config import Config +from karapace.dependency import Dependency from karapace.errors import ( IncompatibleSchema, InvalidVersion, + ReferenceExistsException, SchemasNotFoundException, SchemaTooLargeException, SchemaVersionNotSoftDeletedException, @@ -20,8 +22,9 @@ from karapace.master_coordinator import MasterCoordinator from karapace.schema_models import SchemaType, TypedSchema, ValidatedTypedSchema from karapace.schema_reader import KafkaSchemaReader +from karapace.schema_references import References from karapace.typing import ResolvedVersion, Subject, SubjectData, Version -from karapace.utils import json_encode, KarapaceKafkaClient +from karapace.utils import json_encode, KarapaceKafkaClient, reference_key from typing import Dict, List, Optional, Tuple, Union import asyncio @@ -181,14 +184,27 @@ async def subject_delete_local(self, subject: str, permanent: bool) -> List[Reso latest_schema_id = 0 version_list = [] if permanent: + + for version, value in list(subject_schemas_all.items()): + referenced_by = self.schema_reader.referenced_by.get(reference_key(subject, version), None) + if referenced_by and len(referenced_by) > 0: + raise ReferenceExistsException(referenced_by, version) + version_list = list(subject_schemas_all) - latest_schema_id = version_list[-1] for version, value in list(subject_schemas_all.items()): schema_id = value.get("id") + references = value.get("references", None) LOG.info("Permanently deleting subject '%s' version %s (schema id=%s)", subject, version, schema_id) self.send_schema_message( - subject=subject, schema=None, schema_id=schema_id, version=version, deleted=True + subject=subject, + schema=None, + schema_id=schema_id, + version=version, + deleted=True, + references=references, ) + if references and len(references) > 0: + self.schema_reader.remove_referenced_by(schema_id, references) else: try: subject_data_live = self.subject_get(subject, include_deleted=False) @@ -197,6 +213,11 @@ async def subject_delete_local(self, subject: str, permanent: bool) -> List[Reso latest_schema_id = version_list[-1] except SchemasNotFoundException: pass + + referenced_by = self.schema_reader.referenced_by.get(reference_key(subject, latest_schema_id), None) + if referenced_by and len(referenced_by) > 0: + raise ReferenceExistsException(referenced_by, latest_schema_id) + self.send_delete_subject_message(subject, latest_schema_id) return version_list @@ -220,15 +241,23 @@ async def subject_version_delete_local(self, subject: Subject, version: Version, if permanent and not subject_schema_data.get("deleted", False): raise SchemaVersionNotSoftDeletedException() + referenced_by = self.schema_reader.referenced_by.get(reference_key(subject, int(resolved_version)), None) + if referenced_by and len(referenced_by) > 0: + raise ReferenceExistsException(referenced_by, version) + schema_id = subject_schema_data["id"] schema = subject_schema_data["schema"] + references = subject_schema_data.get("references", None) self.send_schema_message( subject=subject, schema=None if permanent else schema, schema_id=schema_id, version=resolved_version, deleted=True, + references=None, ) + if references and len(references) > 0: + self.schema_reader.remove_referenced_by(schema_id, references) return resolved_version def subject_get(self, subject: Subject, include_deleted: bool = False) -> SubjectData: @@ -265,6 +294,7 @@ async def subject_version_get(self, subject: Subject, version: Version, *, inclu "id": schema_id, "schema": schema.schema_str, } + if schema.schema_type is not SchemaType.AVRO: ret["schemaType"] = schema.schema_type # Return also compatibility information to compatibility check @@ -276,6 +306,7 @@ async def write_new_schema_local( self, subject: Subject, new_schema: ValidatedTypedSchema, + new_schema_references: Optional[References], ) -> int: """Write new schema and return new id or return id of matching existing schema @@ -315,6 +346,7 @@ async def write_new_schema_local( schema_id=schema_id, version=version, deleted=False, + references=new_schema_references, ) return schema_id @@ -336,8 +368,15 @@ async def write_new_schema_local( for old_version in check_against: old_schema = subject_data["schemas"][old_version]["schema"] + old_schema_references, old_schema_dependencies = self.resolve_schema_references( + subject_data["schemas"][old_version], + ) + validated_old_schema = ValidatedTypedSchema.parse( - schema_type=old_schema.schema_type, schema_str=old_schema.schema_str + schema_type=old_schema.schema_type, + schema_str=old_schema.schema_str, + references=old_schema_references, + dependencies=old_schema_dependencies, ) result = check_compatibility( old_schema=validated_old_schema, @@ -379,6 +418,7 @@ async def write_new_schema_local( schema_id=schema_id, version=version, deleted=False, + references=new_schema_references, ) return schema_id @@ -440,6 +480,7 @@ def send_schema_message( schema_id: int, version: int, deleted: bool, + references: Optional[References], ) -> FutureRecordMetadata: key = self.key_formatter.format_key( {"subject": subject, "version": version, "magic": 1, "keytype": "SCHEMA"}, @@ -452,6 +493,8 @@ def send_schema_message( "schema": schema.schema_str, "deleted": deleted, } + if references: + valuedict["references"] = references.val() if schema.schema_type is not SchemaType.AVRO: valuedict["schemaType"] = schema.schema_type value = json_encode(valuedict) @@ -492,3 +535,11 @@ def send_delete_subject_message(self, subject: Subject, version: Version) -> Fut ) value = '{{"subject":"{}","version":{}}}'.format(subject, version) return self.send_kafka_message(key, value) + + def resolve_references(self, references: Optional["References"] = None) -> Optional[Dict[str, Dependency]]: + return self.schema_reader.resolve_references(references) + + def resolve_schema_references( + self, schema_data: Optional[dict] + ) -> Tuple[Optional[References], Optional[Dict[str, Dependency]]]: + return self.schema_reader.resolve_schema_references(schema_data) diff --git a/karapace/schema_registry_apis.py b/karapace/schema_registry_apis.py index f520f7469..3be4ffd4b 100644 --- a/karapace/schema_registry_apis.py +++ b/karapace/schema_registry_apis.py @@ -8,9 +8,12 @@ from karapace.config import Config from karapace.errors import ( IncompatibleSchema, + InvalidReferences, InvalidSchema, InvalidSchemaType, InvalidVersion, + ReferenceExistsException, + ReferencesNotSupportedException, SchemasNotFoundException, SchemaTooLargeException, SchemaVersionNotSoftDeletedException, @@ -25,6 +28,7 @@ from karapace.schema_models import SchemaType, TypedSchema, ValidatedTypedSchema from karapace.schema_registry import KarapaceSchemaRegistry, validate_version from karapace.typing import JsonData +from karapace.utils import reference_key from typing import Any, Dict, Optional, Union import aiohttp @@ -51,6 +55,9 @@ class SchemaErrorCodes(Enum): INVALID_SCHEMA = 42201 INVALID_SUBJECT = 42208 SCHEMA_TOO_LARGE_ERROR_CODE = 42209 + INVALID_REFERENCES = 44301 + REFERENCES_SUPPORT_NOT_IMPLEMENTED = 44302 + REFERENCE_EXISTS = 42206 NO_MASTER_ERROR = 50003 @@ -63,6 +70,7 @@ class SchemaErrorMessages(Enum): "full_transitive" ) SUBJECT_LEVEL_COMPATIBILITY_NOT_CONFIGURED_FMT = "Subject '%s' does not have subject-level compatibility configured" + REFERENCES_SUPPORT_NOT_IMPLEMENTED = "Schema references are not supported for '{schema_type}' schema type" class KarapaceSchemaRegistryController(KarapaceBase): @@ -226,6 +234,12 @@ def _add_schema_registry_routes(self) -> None: schema_request=True, auth=self._auth, ) + self.route( + "/subjects//versions//referencedby", + callback=self.subject_version_referencedby_get, + method="GET", + schema_request=True, + ) self.route( "/subjects/", callback=self.subject_delete, @@ -296,7 +310,13 @@ async def compatibility_check( body = request.json schema_type = self._validate_schema_type(content_type=content_type, data=body) try: - new_schema = ValidatedTypedSchema.parse(schema_type, body["schema"]) + new_schema_references, new_schema_dependencies = self.schema_registry.resolve_schema_references(body) + new_schema = ValidatedTypedSchema.parse( + schema_type=schema_type, + schema_str=body["schema"], + references=new_schema_references, + dependencies=new_schema_dependencies, + ) except InvalidSchema: self.r( body={ @@ -322,7 +342,13 @@ async def compatibility_check( old_schema_type = self._validate_schema_type(content_type=content_type, data=old) try: - old_schema = ValidatedTypedSchema.parse(old_schema_type, old["schema"]) + old_schema_references, old_schema_dependencies = self.schema_registry.resolve_schema_references(old) + old_schema = ValidatedTypedSchema.parse( + schema_type=old_schema_type, + schema_str=old["schema"], + references=old_schema_references, + dependencies=old_schema_dependencies, + ) except InvalidSchema: self.r( body={ @@ -639,6 +665,19 @@ async def subject_delete( content_type=content_type, status=HTTPStatus.NOT_FOUND, ) + + except ReferenceExistsException as arg: + self.r( + body={ + "error_code": SchemaErrorCodes.REFERENCE_EXISTS.value, + "message": ( + f"One or more references exist to the schema " + f"{{magic=1,keytype=SCHEMA,subject={subject},version={arg.version}}}" + ), + }, + content_type=content_type, + status=HTTPStatus.NOT_FOUND, + ) elif not master_url: self.no_master_error(content_type) else: @@ -730,6 +769,18 @@ async def subject_version_delete( content_type=content_type, status=HTTPStatus.NOT_FOUND, ) + except ReferenceExistsException as arg: + self.r( + body={ + "error_code": SchemaErrorCodes.REFERENCE_EXISTS.value, + "message": ( + f"One or more references exist to the schema " + f"{{magic=1,keytype=SCHEMA,subject={subject},version={arg.version}}}" + ), + }, + content_type=content_type, + status=HTTPStatus.NOT_FOUND, + ) elif not master_url: self.no_master_error(content_type) else: @@ -765,9 +816,41 @@ async def subject_version_schema_get( status=HTTPStatus.NOT_FOUND, ) + async def subject_version_referencedby_get(self, content_type, *, subject, version, user: Optional[User] = None): + self._check_authorization(user, Operation.Read, f"Subject:{subject}") + + try: + subject_data = await self.schema_registry.subject_version_get(subject, version) + except (SubjectNotFoundException, SchemasNotFoundException): + self.r( + body={ + "error_code": SchemaErrorCodes.SUBJECT_NOT_FOUND.value, + "message": SchemaErrorMessages.SUBJECT_NOT_FOUND_FMT.value.format(subject=subject), + }, + content_type=content_type, + status=HTTPStatus.NOT_FOUND, + ) + except VersionNotFoundException: + self.r( + body={ + "error_code": SchemaErrorCodes.VERSION_NOT_FOUND.value, + "message": f"Version {version} not found.", + }, + content_type=content_type, + status=HTTPStatus.NOT_FOUND, + ) + except InvalidVersion: + self._invalid_version(content_type, version) + + referenced_by = self.schema_registry.schema_reader.referenced_by.get( + reference_key(subject_data["subject"], subject_data["version"]), [] + ) + self.r(list(referenced_by), content_type, status=HTTPStatus.OK) + async def subject_versions_list( self, content_type: str, *, subject: str, request: HTTPRequest, user: Optional[User] = None ) -> None: + self._check_authorization(user, Operation.Read, f"Subject:{subject}") deleted = request.query.get("deleted", "false").lower() == "true" try: @@ -806,12 +889,12 @@ def _validate_schema_request_body(self, content_type: str, body: Union[dict, Any content_type=content_type, status=HTTPStatus.INTERNAL_SERVER_ERROR, ) - for attr in body: - if attr not in {"schema", "schemaType"}: + for field in body: + if field not in {"schema", "schemaType", "references"}: self.r( body={ "error_code": SchemaErrorCodes.HTTP_UNPROCESSABLE_ENTITY.value, - "message": f"Unrecognized field: {attr}", + "message": f"Unrecognized field: {field}", }, content_type=content_type, status=HTTPStatus.UNPROCESSABLE_ENTITY, @@ -862,7 +945,7 @@ async def subjects_schema_post( content_type=content_type, status=HTTPStatus.NOT_FOUND, ) - new_schema = None + if "schema" not in body: self.r( body={ @@ -874,8 +957,39 @@ async def subjects_schema_post( ) schema_str = body["schema"] schema_type = self._validate_schema_type(content_type=content_type, data=body) + try: - new_schema = ValidatedTypedSchema.parse(schema_type, schema_str) + new_schema_references, new_schema_dependencies = self.schema_registry.resolve_schema_references(body) + new_schema = ValidatedTypedSchema.parse( + schema_type=schema_type, + schema_str=schema_str, + references=new_schema_references, + dependencies=new_schema_dependencies, + ) + for schema in subject_data["schemas"].values(): + validated_typed_schema = ValidatedTypedSchema.parse( + schema["schema"].schema_type, schema["schema"].schema_str + ) + if schema_type is SchemaType.JSONSCHEMA: + schema_valid = validated_typed_schema.to_dict() == new_schema.to_dict() + else: + schema_valid = validated_typed_schema.schema == new_schema.schema + if ( + validated_typed_schema.schema_type == new_schema.schema_type + and schema_valid + and schema.get("references", None) == new_schema_references + ): + ret = { + "subject": subject, + "version": schema["version"], + "id": schema["id"], + "schema": validated_typed_schema.schema_str, + } + if schema_type is not SchemaType.AVRO: + ret["schemaType"] = schema_type + self.r(ret, content_type) + else: + self.log.debug("Schema %r did not match %r", schema, validated_typed_schema) except InvalidSchema: self.log.exception("No proper parser found") self.r( @@ -886,6 +1000,27 @@ async def subjects_schema_post( content_type=content_type, status=HTTPStatus.INTERNAL_SERVER_ERROR, ) + except ReferencesNotSupportedException: + self.r( + body={ + "error_code": SchemaErrorCodes.REFERENCES_SUPPORT_NOT_IMPLEMENTED.value, + "message": SchemaErrorMessages.REFERENCES_SUPPORT_NOT_IMPLEMENTED.value.format( + schema_type=schema_type.value + ), + }, + content_type=content_type, + status=HTTPStatus.UNPROCESSABLE_ENTITY, + ) + except InvalidReferences: + human_error = "Provided references is not valid" + self.r( + body={ + "error_code": SchemaErrorCodes.INVALID_REFERENCES.value, + "message": f"Invalid {schema_type} references. Error: {human_error}", + }, + content_type=content_type, + status=HTTPStatus.UNPROCESSABLE_ENTITY, + ) # Match schemas based on version from latest to oldest for schema in sorted(subject_data["schemas"].values(), key=lambda item: item["version"], reverse=True): @@ -906,6 +1041,7 @@ async def subjects_schema_post( self.r(ret, content_type) else: self.log.debug("Schema %r did not match %r", schema, validated_typed_schema) + self.r( body={ "error_code": SchemaErrorCodes.SCHEMA_NOT_FOUND.value, @@ -928,7 +1064,13 @@ async def subject_post( self._validate_schema_key(content_type, body) try: - new_schema = ValidatedTypedSchema.parse(schema_type=schema_type, schema_str=body["schema"]) + new_schema_references, new_schema_dependencies = self.schema_registry.resolve_schema_references(body) + new_schema = ValidatedTypedSchema.parse( + schema_type=schema_type, + schema_str=body["schema"], + references=new_schema_references, + dependencies=new_schema_dependencies, + ) except (InvalidSchema, InvalidSchemaType) as e: self.log.warning("Invalid schema: %r", body["schema"], exc_info=True) if isinstance(e.__cause__, (SchemaParseException, json.JSONDecodeError)): @@ -943,6 +1085,27 @@ async def subject_post( content_type=content_type, status=HTTPStatus.UNPROCESSABLE_ENTITY, ) + except ReferencesNotSupportedException: + self.r( + body={ + "error_code": SchemaErrorCodes.REFERENCES_SUPPORT_NOT_IMPLEMENTED.value, + "message": SchemaErrorMessages.REFERENCES_SUPPORT_NOT_IMPLEMENTED.value.format( + schema_type=schema_type.value + ), + }, + content_type=content_type, + status=HTTPStatus.UNPROCESSABLE_ENTITY, + ) + except InvalidReferences: + human_error = "Provided references is not valid" + self.r( + body={ + "error_code": SchemaErrorCodes.INVALID_REFERENCES.value, + "message": f"Invalid {schema_type} references. Error: {human_error}", + }, + content_type=content_type, + status=HTTPStatus.UNPROCESSABLE_ENTITY, + ) schema_id = self.get_schema_id_if_exists(subject=subject, schema=new_schema) if schema_id is not None: @@ -959,7 +1122,7 @@ async def subject_post( are_we_master, master_url = await self.schema_registry.get_master() if are_we_master: try: - schema_id = await self.schema_registry.write_new_schema_local(subject, new_schema) + schema_id = await self.schema_registry.write_new_schema_local(subject, new_schema, new_schema_references) self.r( body={"id": schema_id}, content_type=content_type, diff --git a/karapace/schema_type.py b/karapace/schema_type.py new file mode 100644 index 000000000..2881c4f3f --- /dev/null +++ b/karapace/schema_type.py @@ -0,0 +1,8 @@ +from enum import Enum, unique + + +@unique +class SchemaType(str, Enum): + AVRO = "AVRO" + JSONSCHEMA = "JSON" + PROTOBUF = "PROTOBUF" diff --git a/karapace/serialization.py b/karapace/serialization.py index fa5c16fa1..4b272d9e4 100644 --- a/karapace/serialization.py +++ b/karapace/serialization.py @@ -5,7 +5,7 @@ from karapace.client import Client from karapace.protobuf.exception import ProtobufTypeException from karapace.protobuf.io import ProtobufDatumReader, ProtobufDatumWriter -from karapace.schema_models import InvalidSchema, SchemaType, TypedSchema, ValidatedTypedSchema +from karapace.schema_models import InvalidSchema, References, SchemaType, TypedSchema, ValidatedTypedSchema from karapace.utils import json_encode from typing import Any, Dict, Optional, Tuple from urllib.parse import quote @@ -75,9 +75,14 @@ def __init__( self.client = Client(server_uri=schema_registry_url, server_ca=server_ca, session_auth=session_auth) self.base_url = schema_registry_url - async def post_new_schema(self, subject: str, schema: ValidatedTypedSchema) -> int: + async def post_new_schema( + self, subject: str, schema: ValidatedTypedSchema, references: Optional[References] = None + ) -> int: if schema.schema_type is SchemaType.PROTOBUF: - payload = {"schema": str(schema), "schemaType": schema.schema_type.value} + if references: + payload = {"schema": str(schema), "schemaType": schema.schema_type.value, "references": references.json()} + else: + payload = {"schema": str(schema), "schemaType": schema.schema_type.value} else: payload = {"schema": json_encode(schema.to_dict()), "schemaType": schema.schema_type.value} result = await self.client.post(f"subjects/{quote(subject)}/versions", json=payload) @@ -94,11 +99,13 @@ async def get_latest_schema(self, subject: str) -> Tuple[int, ValidatedTypedSche raise SchemaRetrievalError(f"Invalid result format: {json_result}") try: schema_type = SchemaType(json_result.get("schemaType", "AVRO")) + return json_result["id"], ValidatedTypedSchema.parse(schema_type, json_result["schema"]) + except InvalidSchema as e: raise SchemaRetrievalError(f"Failed to parse schema string from response: {json_result}") from e - async def get_schema_for_id(self, schema_id: int) -> ValidatedTypedSchema: + async def get_schema_for_id(self, schema_id: int) -> Tuple[ValidatedTypedSchema, Optional[References]]: result = await self.client.get(f"schemas/ids/{schema_id}") if not result.ok: raise SchemaRetrievalError(result.json()["message"]) @@ -107,7 +114,15 @@ async def get_schema_for_id(self, schema_id: int) -> ValidatedTypedSchema: raise SchemaRetrievalError(f"Invalid result format: {json_result}") try: schema_type = SchemaType(json_result.get("schemaType", "AVRO")) + references_str = json_result.get("references") + if references_str: + references = References(schema_type, references_str) + else: + references = None + if references: + return ValidatedTypedSchema.parse(schema_type, json_result["schema"]), references return ValidatedTypedSchema.parse(schema_type, json_result["schema"]) + except InvalidSchema as e: raise SchemaRetrievalError(f"Failed to parse schema string from response: {json_result}") from e @@ -160,6 +175,7 @@ def get_subject_name(self, topic_name: str, schema: str, subject_type: str, sche async def get_schema_for_subject(self, subject: str) -> TypedSchema: assert self.registry_client, "must not call this method after the object is closed." + schema_id, schema = await self.registry_client.get_latest_schema(subject) async with self.state_lock: schema_ser = schema.__str__() @@ -177,6 +193,7 @@ async def get_id_for_schema(self, schema: str, subject: str, schema_type: Schema if schema_ser in self.schemas_to_ids: return self.schemas_to_ids[schema_ser] schema_id = await self.registry_client.post_new_schema(subject, schema_typed) + async with self.state_lock: self.schemas_to_ids[schema_ser] = schema_id self.ids_to_schemas[schema_id] = schema_typed diff --git a/karapace/utils.py b/karapace/utils.py index b8b077744..a3c0c229b 100644 --- a/karapace/utils.py +++ b/karapace/utils.py @@ -12,6 +12,7 @@ from decimal import Decimal from http import HTTPStatus from kafka.client_async import BrokerConnection, KafkaClient, MetadataRequest +from pathlib import Path from types import MappingProxyType from typing import NoReturn, overload, Union @@ -55,6 +56,10 @@ def default_json_serialization(obj: MappingProxyType) -> dict: ... +def reference_key(subject: str, version: int) -> str: + return hash((subject, version)) + + def default_json_serialization( # pylint: disable=inconsistent-return-statements obj: Union[datetime, timedelta, Decimal, MappingProxyType], ) -> Union[str, float, dict]: @@ -88,6 +93,10 @@ def assert_never(value: NoReturn) -> NoReturn: raise RuntimeError(f"This code should never be reached, got: {value}") +def get_project_root() -> Path: + return Path(__file__).parent.parent + + class Timeout(Exception): pass diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integration/test_client_protobuf.py b/tests/integration/test_client_protobuf.py index 476dbd2de..f924b536f 100644 --- a/tests/integration/test_client_protobuf.py +++ b/tests/integration/test_client_protobuf.py @@ -10,7 +10,7 @@ async def test_remote_client_protobuf(registry_async_client): reg_cli = SchemaRegistryClient() reg_cli.client = registry_async_client subject = new_random_name("subject") - sc_id = await reg_cli.post_new_schema(subject, schema_protobuf) + sc_id = await reg_cli.post_new_schema(subject, schema_protobuf, None) assert sc_id >= 0 stored_schema = await reg_cli.get_schema_for_id(sc_id) assert stored_schema == schema_protobuf, f"stored schema {stored_schema} is not {schema_protobuf}" @@ -25,7 +25,7 @@ async def test_remote_client_protobuf2(registry_async_client): reg_cli = SchemaRegistryClient() reg_cli.client = registry_async_client subject = new_random_name("subject") - sc_id = await reg_cli.post_new_schema(subject, schema_protobuf) + sc_id = await reg_cli.post_new_schema(subject, schema_protobuf, None) assert sc_id >= 0 stored_schema = await reg_cli.get_schema_for_id(sc_id) assert stored_schema == schema_protobuf, f"stored schema {stored_schema} is not {schema_protobuf}" diff --git a/tests/integration/test_schema.py b/tests/integration/test_schema.py index 3779ea88d..3b0ec3d5d 100644 --- a/tests/integration/test_schema.py +++ b/tests/integration/test_schema.py @@ -1434,6 +1434,17 @@ async def test_schema_subject_post_invalid(registry_async_client: Client) -> Non assert res.json()["error_code"] == 40401 assert res.json()["message"] == f"Subject '{subject_3}' not found." + schema_str = json.dumps({"type": "string"}) + # Create the subject + subject_1 = subject_name_factory() + res = await registry_async_client.post( + f"subjects/{subject_1}/versions", + json={"schema": schema_str, "references": [{"name": "Customer.avro", "subject": "customer", "version": 1}]}, + ) + assert res.status_code == 422 + assert res.json()["error_code"] == 44302 + assert res.json()["message"] == "Schema references are not supported for 'AVRO' schema type" + @pytest.mark.parametrize("trail", ["", "/"]) async def test_schema_lifecycle(registry_async_client: Client, trail: str) -> None: diff --git a/tests/integration/test_schema_protobuf.py b/tests/integration/test_schema_protobuf.py index 1b58d4616..7cc1dd082 100644 --- a/tests/integration/test_schema_protobuf.py +++ b/tests/integration/test_schema_protobuf.py @@ -167,3 +167,180 @@ async def test_protobuf_schema_normalization(registry_async_client: Client, trai assert "id" in res.json() assert "schema" in res.json() assert evolved_id == res.json()["id"], "Check returns evolved id" + + +async def test_protobuf_schema_references(registry_async_client: Client) -> None: + + customer_schema = """ + |syntax = "proto3"; + |package a1; + |import "Place.proto"; + |message Customer { + | string name = 1; + | int32 code = 2; + | Place place = 3; + |} + |""" + + customer_schema = trim_margin(customer_schema) + + place_schema = """ + |syntax = "proto3"; + |package a1; + |message Place { + | string city = 1; + | int32 zone = 2; + |} + |""" + + place_schema = trim_margin(place_schema) + res = await registry_async_client.post( + "subjects/place/versions", json={"schemaType": "PROTOBUF", "schema": place_schema} + ) + assert res.status_code == 200 + + assert "id" in res.json() + + customer_references = [{"name": "Place.proto", "subject": "place", "version": 1}] + res = await registry_async_client.post( + "subjects/customer/versions", + json={"schemaType": "PROTOBUF", "schema": customer_schema, "references": customer_references}, + ) + assert res.status_code == 200 + + assert "id" in res.json() + + original_schema = """ + |syntax = "proto3"; + |package a1; + |import "Customer.proto"; + |message TestMessage { + | enum Enum { + | HIGH = 0; + | MIDDLE = 1; + | LOW = 2; + | } + | message Value { + | message Label{ + | int32 Id = 1; + | string name = 2; + | } + | Customer customer = 1; + | int32 x = 2; + | } + | string test = 1; + | .a1.TestMessage.Value val = 2; + | oneof page_info { + | option (my_option) = true; + | int32 page_number = 3; + | int32 result_per_page = 4; + | } + |} + |""" + + original_schema = trim_margin(original_schema) + references = [{"name": "Customer.proto", "subject": "customer", "version": 1}] + res = await registry_async_client.post( + "subjects/test_schema/versions", + json={"schemaType": "PROTOBUF", "schema": original_schema, "references": references}, + ) + assert res.status_code == 200 + + assert "id" in res.json() + + res = await registry_async_client.get("subjects/customer/versions/latest/referencedby", json={}) + assert res.status_code == 200 + + myjson = res.json() + referents = [3] + assert not any(x != y for x, y in zip(myjson, referents)) + + res = await registry_async_client.get("subjects/place/versions/latest/referencedby", json={}) + assert res.status_code == 200 + + myjson = res.json() + res = await registry_async_client.delete("subjects/customer/versions/1") + assert res.status_code == 404 + + match_msg = "One or more references exist to the schema {magic=1,keytype=SCHEMA,subject=customer,version=1}" + myjson = res.json() + assert myjson["error_code"] == 42206 and myjson["message"] == match_msg + + res = await registry_async_client.delete("subjects/test_schema/versions/1") + assert res.status_code == 200 + + res = await registry_async_client.delete("subjects/customer/versions/1") + assert res.status_code == 200 + + +async def test_protobuf_schema_verifier(registry_async_client: Client) -> None: + customer_schema = """ + |syntax = "proto3"; + |package a1; + |message Customer { + | string name = 1; + | int32 code = 2; + |} + |""" + + customer_schema = trim_margin(customer_schema) + res = await registry_async_client.post( + "subjects/customer/versions", json={"schemaType": "PROTOBUF", "schema": customer_schema} + ) + assert res.status_code == 200 + assert "id" in res.json() + original_schema = """ + |syntax = "proto3"; + |package a1; + |import "Customer.proto"; + |message TestMessage { + | enum Enum { + | HIGH = 0; + | MIDDLE = 1; + | LOW = 2; + | } + | message Value { + | message Label{ + | int32 Id = 1; + | string name = 2; + | } + | Customer customer = 1; + | int32 x = 2; + | } + | string test = 1; + | .a1.TestMessage.Value val = 2; + | TestMessage.Value valx = 3; + | + | oneof page_info { + | option (my_option) = true; + | int32 page_number = 5; + | int32 result_per_page = 6; + | } + |} + |""" + + original_schema = trim_margin(original_schema) + references = [{"name": "Customer.proto", "subject": "customer", "version": 1}] + res = await registry_async_client.post( + "subjects/test_schema/versions", + json={"schemaType": "PROTOBUF", "schema": original_schema, "references": references}, + ) + assert res.status_code == 200 + assert "id" in res.json() + res = await registry_async_client.get("subjects/customer/versions/latest/referencedby", json={}) + assert res.status_code == 200 + myjson = res.json() + referents = [2] + assert not any(x != y for x, y in zip(myjson, referents)) + + res = await registry_async_client.delete("subjects/customer/versions/1") + assert res.status_code == 404 + match_msg = "One or more references exist to the schema {magic=1,keytype=SCHEMA,subject=customer,version=1}" + myjson = res.json() + assert myjson["error_code"] == 42206 and myjson["message"] == match_msg + + res = await registry_async_client.delete("subjects/test_schema/versions/1") + assert res.status_code == 200 + + res = await registry_async_client.delete("subjects/customer/versions/1") + assert res.status_code == 200 diff --git a/tests/unit/test_dependency_verifier.py b/tests/unit/test_dependency_verifier.py new file mode 100644 index 000000000..49a5ddf6b --- /dev/null +++ b/tests/unit/test_dependency_verifier.py @@ -0,0 +1,54 @@ +from karapace.protobuf.dependency import ProtobufDependencyVerifier + +import logging + +log = logging.getLogger(__name__) + + +async def test_protobuf_dependency_verifier(): + declared_types = [ + ".a1.Place", + "Place", + ".a1.Customer", + "Customer", + ".a1.TestMessage", + "TestMessage", + ".a1", + ".TestMessage", + ".Enum", + "TestMessage.Enum", + ".a1.TestMessage.Value", + "TestMessage.Value", + ".a1.TestMessage.Value.Label", + "TestMessage", + ".Value.Label", + ] + + used_types = [ + ".a1.Place;string", + ".a1.Place;int32", + ".a1.Customer;string", + ".a1.Customer;int32", + ".a1.Customer;Place", + ".a1.TestMessage;int32", + ".a1.TestMessage;int32", + ".a1.TestMessage;string", + ".a1.TestMessage;.a1.TestMessage.Value", + "TestMessage;Customer", + "TestMessage;int32", + "TestMessage.Value;int32", + "TestMessage.Value;string", + ] + + verifier = ProtobufDependencyVerifier() + for declared in declared_types: + verifier.add_declared_type(declared) + for used in used_types: + x = used.split(";") + verifier.add_used_type(x[0], x[1]) + + result = verifier.verify() + assert result.result, True + + verifier.add_used_type("TestMessage.Delta", "Tag") + assert result.result, False From 7c36fe0bb3234e2fe371246568fc1bbf588c0708 Mon Sep 17 00:00:00 2001 From: Jarkko Jaakola Date: Tue, 18 Oct 2022 15:04:17 +0300 Subject: [PATCH 06/25] WIP: deleted proto files --- karapace/protobuf/google/protobuf/any.proto | 158 --- .../protobuf/google/protobuf/any_test.proto | 44 - karapace/protobuf/google/protobuf/api.proto | 208 ---- .../protobuf/google/protobuf/descriptor.proto | 918 ------------------ .../protobuf/google/protobuf/duration.proto | 116 --- karapace/protobuf/google/protobuf/empty.proto | 51 - .../protobuf/google/protobuf/field_mask.proto | 245 ----- .../google/protobuf/map_lite_unittest.proto | 131 --- .../google/protobuf/map_proto2_unittest.proto | 91 -- .../google/protobuf/map_unittest.proto | 125 --- .../google/protobuf/source_context.proto | 48 - .../protobuf/google/protobuf/struct.proto | 95 -- .../protobuf/google/protobuf/timestamp.proto | 147 --- karapace/protobuf/google/protobuf/type.proto | 187 ---- .../protobuf/google/protobuf/wrappers.proto | 123 --- karapace/protobuf/google/type/BUILD.bazel | 536 ---------- karapace/protobuf/google/type/README.md | 16 - .../google/type/calendar_period.proto | 56 -- karapace/protobuf/google/type/color.proto | 174 ---- karapace/protobuf/google/type/date.proto | 52 - karapace/protobuf/google/type/datetime.proto | 104 -- karapace/protobuf/google/type/dayofweek.proto | 50 - karapace/protobuf/google/type/decimal.proto | 95 -- karapace/protobuf/google/type/expr.proto | 73 -- karapace/protobuf/google/type/fraction.proto | 33 - karapace/protobuf/google/type/interval.proto | 46 - karapace/protobuf/google/type/latlng.proto | 37 - .../protobuf/google/type/localized_text.proto | 36 - karapace/protobuf/google/type/money.proto | 42 - karapace/protobuf/google/type/month.proto | 65 -- .../protobuf/google/type/phone_number.proto | 113 --- .../protobuf/google/type/postal_address.proto | 134 --- .../protobuf/google/type/quaternion.proto | 94 -- karapace/protobuf/google/type/timeofday.proto | 44 - karapace/protobuf/google/type/type.yaml | 40 - 35 files changed, 4527 deletions(-) delete mode 100644 karapace/protobuf/google/protobuf/any.proto delete mode 100644 karapace/protobuf/google/protobuf/any_test.proto delete mode 100644 karapace/protobuf/google/protobuf/api.proto delete mode 100644 karapace/protobuf/google/protobuf/descriptor.proto delete mode 100644 karapace/protobuf/google/protobuf/duration.proto delete mode 100644 karapace/protobuf/google/protobuf/empty.proto delete mode 100644 karapace/protobuf/google/protobuf/field_mask.proto delete mode 100644 karapace/protobuf/google/protobuf/map_lite_unittest.proto delete mode 100644 karapace/protobuf/google/protobuf/map_proto2_unittest.proto delete mode 100644 karapace/protobuf/google/protobuf/map_unittest.proto delete mode 100644 karapace/protobuf/google/protobuf/source_context.proto delete mode 100644 karapace/protobuf/google/protobuf/struct.proto delete mode 100644 karapace/protobuf/google/protobuf/timestamp.proto delete mode 100644 karapace/protobuf/google/protobuf/type.proto delete mode 100644 karapace/protobuf/google/protobuf/wrappers.proto delete mode 100644 karapace/protobuf/google/type/BUILD.bazel delete mode 100644 karapace/protobuf/google/type/README.md delete mode 100644 karapace/protobuf/google/type/calendar_period.proto delete mode 100644 karapace/protobuf/google/type/color.proto delete mode 100644 karapace/protobuf/google/type/date.proto delete mode 100644 karapace/protobuf/google/type/datetime.proto delete mode 100644 karapace/protobuf/google/type/dayofweek.proto delete mode 100644 karapace/protobuf/google/type/decimal.proto delete mode 100644 karapace/protobuf/google/type/expr.proto delete mode 100644 karapace/protobuf/google/type/fraction.proto delete mode 100644 karapace/protobuf/google/type/interval.proto delete mode 100644 karapace/protobuf/google/type/latlng.proto delete mode 100644 karapace/protobuf/google/type/localized_text.proto delete mode 100644 karapace/protobuf/google/type/money.proto delete mode 100644 karapace/protobuf/google/type/month.proto delete mode 100644 karapace/protobuf/google/type/phone_number.proto delete mode 100644 karapace/protobuf/google/type/postal_address.proto delete mode 100644 karapace/protobuf/google/type/quaternion.proto delete mode 100644 karapace/protobuf/google/type/timeofday.proto delete mode 100644 karapace/protobuf/google/type/type.yaml diff --git a/karapace/protobuf/google/protobuf/any.proto b/karapace/protobuf/google/protobuf/any.proto deleted file mode 100644 index e2c2042fd..000000000 --- a/karapace/protobuf/google/protobuf/any.proto +++ /dev/null @@ -1,158 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2008 Google Inc. All rights reserved. -// https://developers.google.com/protocol-buffers/ -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -syntax = "proto3"; - -package google.protobuf; - -option csharp_namespace = "Google.Protobuf.WellKnownTypes"; -option go_package = "google.golang.org/protobuf/types/known/anypb"; -option java_package = "com.google.protobuf"; -option java_outer_classname = "AnyProto"; -option java_multiple_files = true; -option objc_class_prefix = "GPB"; - -// `Any` contains an arbitrary serialized protocol buffer message along with a -// URL that describes the type of the serialized message. -// -// Protobuf library provides support to pack/unpack Any values in the form -// of utility functions or additional generated methods of the Any type. -// -// Example 1: Pack and unpack a message in C++. -// -// Foo foo = ...; -// Any any; -// any.PackFrom(foo); -// ... -// if (any.UnpackTo(&foo)) { -// ... -// } -// -// Example 2: Pack and unpack a message in Java. -// -// Foo foo = ...; -// Any any = Any.pack(foo); -// ... -// if (any.is(Foo.class)) { -// foo = any.unpack(Foo.class); -// } -// -// Example 3: Pack and unpack a message in Python. -// -// foo = Foo(...) -// any = Any() -// any.Pack(foo) -// ... -// if any.Is(Foo.DESCRIPTOR): -// any.Unpack(foo) -// ... -// -// Example 4: Pack and unpack a message in Go -// -// foo := &pb.Foo{...} -// any, err := anypb.New(foo) -// if err != nil { -// ... -// } -// ... -// foo := &pb.Foo{} -// if err := any.UnmarshalTo(foo); err != nil { -// ... -// } -// -// The pack methods provided by protobuf library will by default use -// 'type.googleapis.com/full.type.name' as the type URL and the unpack -// methods only use the fully qualified type name after the last '/' -// in the type URL, for example "foo.bar.com/x/y.z" will yield type -// name "y.z". -// -// -// JSON -// -// The JSON representation of an `Any` value uses the regular -// representation of the deserialized, embedded message, with an -// additional field `@type` which contains the type URL. Example: -// -// package google.profile; -// message Person { -// string first_name = 1; -// string last_name = 2; -// } -// -// { -// "@type": "type.googleapis.com/google.profile.Person", -// "firstName": , -// "lastName": -// } -// -// If the embedded message type is well-known and has a custom JSON -// representation, that representation will be embedded adding a field -// `value` which holds the custom JSON in addition to the `@type` -// field. Example (for message [google.protobuf.Duration][]): -// -// { -// "@type": "type.googleapis.com/google.protobuf.Duration", -// "value": "1.212s" -// } -// -message Any { - // A URL/resource name that uniquely identifies the type of the serialized - // protocol buffer message. This string must contain at least - // one "/" character. The last segment of the URL's path must represent - // the fully qualified name of the type (as in - // `path/google.protobuf.Duration`). The name should be in a canonical form - // (e.g., leading "." is not accepted). - // - // In practice, teams usually precompile into the binary all types that they - // expect it to use in the context of Any. However, for URLs which use the - // scheme `http`, `https`, or no scheme, one can optionally set up a type - // server that maps type URLs to message definitions as follows: - // - // * If no scheme is provided, `https` is assumed. - // * An HTTP GET on the URL must yield a [google.protobuf.Type][] - // value in binary format, or produce an error. - // * Applications are allowed to cache lookup results based on the - // URL, or have them precompiled into a binary to avoid any - // lookup. Therefore, binary compatibility needs to be preserved - // on changes to types. (Use versioned type names to manage - // breaking changes.) - // - // Note: this functionality is not currently available in the official - // protobuf release, and it is not used for type URLs beginning with - // type.googleapis.com. - // - // Schemes other than `http`, `https` (or the empty scheme) might be - // used with implementation specific semantics. - // - string type_url = 1; - - // Must be a valid serialized protocol buffer of the above specified type. - bytes value = 2; -} diff --git a/karapace/protobuf/google/protobuf/any_test.proto b/karapace/protobuf/google/protobuf/any_test.proto deleted file mode 100644 index 256035b44..000000000 --- a/karapace/protobuf/google/protobuf/any_test.proto +++ /dev/null @@ -1,44 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2008 Google Inc. All rights reserved. -// https://developers.google.com/protocol-buffers/ -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -syntax = "proto3"; - -package protobuf_unittest; - -import "google/protobuf/any.proto"; - -option java_outer_classname = "TestAnyProto"; - -message TestAny { - int32 int32_value = 1; - google.protobuf.Any any_value = 2; - repeated google.protobuf.Any repeated_any_value = 3; - string text = 4; -} diff --git a/karapace/protobuf/google/protobuf/api.proto b/karapace/protobuf/google/protobuf/api.proto deleted file mode 100644 index 3d598fc85..000000000 --- a/karapace/protobuf/google/protobuf/api.proto +++ /dev/null @@ -1,208 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2008 Google Inc. All rights reserved. -// https://developers.google.com/protocol-buffers/ -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -syntax = "proto3"; - -package google.protobuf; - -import "google/protobuf/source_context.proto"; -import "google/protobuf/type.proto"; - -option csharp_namespace = "Google.Protobuf.WellKnownTypes"; -option java_package = "com.google.protobuf"; -option java_outer_classname = "ApiProto"; -option java_multiple_files = true; -option objc_class_prefix = "GPB"; -option go_package = "google.golang.org/protobuf/types/known/apipb"; - -// Api is a light-weight descriptor for an API Interface. -// -// Interfaces are also described as "protocol buffer services" in some contexts, -// such as by the "service" keyword in a .proto file, but they are different -// from API Services, which represent a concrete implementation of an interface -// as opposed to simply a description of methods and bindings. They are also -// sometimes simply referred to as "APIs" in other contexts, such as the name of -// this message itself. See https://cloud.google.com/apis/design/glossary for -// detailed terminology. -message Api { - // The fully qualified name of this interface, including package name - // followed by the interface's simple name. - string name = 1; - - // The methods of this interface, in unspecified order. - repeated Method methods = 2; - - // Any metadata attached to the interface. - repeated Option options = 3; - - // A version string for this interface. If specified, must have the form - // `major-version.minor-version`, as in `1.10`. If the minor version is - // omitted, it defaults to zero. If the entire version field is empty, the - // major version is derived from the package name, as outlined below. If the - // field is not empty, the version in the package name will be verified to be - // consistent with what is provided here. - // - // The versioning schema uses [semantic - // versioning](http://semver.org) where the major version number - // indicates a breaking change and the minor version an additive, - // non-breaking change. Both version numbers are signals to users - // what to expect from different versions, and should be carefully - // chosen based on the product plan. - // - // The major version is also reflected in the package name of the - // interface, which must end in `v`, as in - // `google.feature.v1`. For major versions 0 and 1, the suffix can - // be omitted. Zero major versions must only be used for - // experimental, non-GA interfaces. - // - // - string version = 4; - - // Source context for the protocol buffer service represented by this - // message. - SourceContext source_context = 5; - - // Included interfaces. See [Mixin][]. - repeated Mixin mixins = 6; - - // The source syntax of the service. - Syntax syntax = 7; -} - -// Method represents a method of an API interface. -message Method { - // The simple name of this method. - string name = 1; - - // A URL of the input message type. - string request_type_url = 2; - - // If true, the request is streamed. - bool request_streaming = 3; - - // The URL of the output message type. - string response_type_url = 4; - - // If true, the response is streamed. - bool response_streaming = 5; - - // Any metadata attached to the method. - repeated Option options = 6; - - // The source syntax of this method. - Syntax syntax = 7; -} - -// Declares an API Interface to be included in this interface. The including -// interface must redeclare all the methods from the included interface, but -// documentation and options are inherited as follows: -// -// - If after comment and whitespace stripping, the documentation -// string of the redeclared method is empty, it will be inherited -// from the original method. -// -// - Each annotation belonging to the service config (http, -// visibility) which is not set in the redeclared method will be -// inherited. -// -// - If an http annotation is inherited, the path pattern will be -// modified as follows. Any version prefix will be replaced by the -// version of the including interface plus the [root][] path if -// specified. -// -// Example of a simple mixin: -// -// package google.acl.v1; -// service AccessControl { -// // Get the underlying ACL object. -// rpc GetAcl(GetAclRequest) returns (Acl) { -// option (google.api.http).get = "/v1/{resource=**}:getAcl"; -// } -// } -// -// package google.storage.v2; -// service Storage { -// rpc GetAcl(GetAclRequest) returns (Acl); -// -// // Get a data record. -// rpc GetData(GetDataRequest) returns (Data) { -// option (google.api.http).get = "/v2/{resource=**}"; -// } -// } -// -// Example of a mixin configuration: -// -// apis: -// - name: google.storage.v2.Storage -// mixins: -// - name: google.acl.v1.AccessControl -// -// The mixin construct implies that all methods in `AccessControl` are -// also declared with same name and request/response types in -// `Storage`. A documentation generator or annotation processor will -// see the effective `Storage.GetAcl` method after inheriting -// documentation and annotations as follows: -// -// service Storage { -// // Get the underlying ACL object. -// rpc GetAcl(GetAclRequest) returns (Acl) { -// option (google.api.http).get = "/v2/{resource=**}:getAcl"; -// } -// ... -// } -// -// Note how the version in the path pattern changed from `v1` to `v2`. -// -// If the `root` field in the mixin is specified, it should be a -// relative path under which inherited HTTP paths are placed. Example: -// -// apis: -// - name: google.storage.v2.Storage -// mixins: -// - name: google.acl.v1.AccessControl -// root: acls -// -// This implies the following inherited HTTP annotation: -// -// service Storage { -// // Get the underlying ACL object. -// rpc GetAcl(GetAclRequest) returns (Acl) { -// option (google.api.http).get = "/v2/acls/{resource=**}:getAcl"; -// } -// ... -// } -message Mixin { - // The fully qualified name of the interface which is included. - string name = 1; - - // If non-empty specifies a path under which inherited HTTP paths - // are rooted. - string root = 2; -} diff --git a/karapace/protobuf/google/protobuf/descriptor.proto b/karapace/protobuf/google/protobuf/descriptor.proto deleted file mode 100644 index 5b4d06b41..000000000 --- a/karapace/protobuf/google/protobuf/descriptor.proto +++ /dev/null @@ -1,918 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2008 Google Inc. All rights reserved. -// https://developers.google.com/protocol-buffers/ -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// Author: kenton@google.com (Kenton Varda) -// Based on original Protocol Buffers design by -// Sanjay Ghemawat, Jeff Dean, and others. -// -// The messages in this file describe the definitions found in .proto files. -// A valid .proto file can be translated directly to a FileDescriptorProto -// without any other information (e.g. without reading its imports). - - -syntax = "proto2"; - -package google.protobuf; - -option go_package = "google.golang.org/protobuf/types/descriptorpb"; -option java_package = "com.google.protobuf"; -option java_outer_classname = "DescriptorProtos"; -option csharp_namespace = "Google.Protobuf.Reflection"; -option objc_class_prefix = "GPB"; -option cc_enable_arenas = true; - -// descriptor.proto must be optimized for speed because reflection-based -// algorithms don't work during bootstrapping. -option optimize_for = SPEED; - -// The protocol compiler can output a FileDescriptorSet containing the .proto -// files it parses. -message FileDescriptorSet { - repeated FileDescriptorProto file = 1; -} - -// Describes a complete .proto file. -message FileDescriptorProto { - optional string name = 1; // file name, relative to root of source tree - optional string package = 2; // e.g. "foo", "foo.bar", etc. - - // Names of files imported by this file. - repeated string dependency = 3; - // Indexes of the public imported files in the dependency list above. - repeated int32 public_dependency = 10; - // Indexes of the weak imported files in the dependency list. - // For Google-internal migration only. Do not use. - repeated int32 weak_dependency = 11; - - // All top-level definitions in this file. - repeated DescriptorProto message_type = 4; - repeated EnumDescriptorProto enum_type = 5; - repeated ServiceDescriptorProto service = 6; - repeated FieldDescriptorProto extension = 7; - - optional FileOptions options = 8; - - // This field contains optional information about the original source code. - // You may safely remove this entire field without harming runtime - // functionality of the descriptors -- the information is needed only by - // development tools. - optional SourceCodeInfo source_code_info = 9; - - // The syntax of the proto file. - // The supported values are "proto2" and "proto3". - optional string syntax = 12; -} - -// Describes a message type. -message DescriptorProto { - optional string name = 1; - - repeated FieldDescriptorProto field = 2; - repeated FieldDescriptorProto extension = 6; - - repeated DescriptorProto nested_type = 3; - repeated EnumDescriptorProto enum_type = 4; - - message ExtensionRange { - optional int32 start = 1; // Inclusive. - optional int32 end = 2; // Exclusive. - - optional ExtensionRangeOptions options = 3; - } - repeated ExtensionRange extension_range = 5; - - repeated OneofDescriptorProto oneof_decl = 8; - - optional MessageOptions options = 7; - - // Range of reserved tag numbers. Reserved tag numbers may not be used by - // fields or extension ranges in the same message. Reserved ranges may - // not overlap. - message ReservedRange { - optional int32 start = 1; // Inclusive. - optional int32 end = 2; // Exclusive. - } - repeated ReservedRange reserved_range = 9; - // Reserved field names, which may not be used by fields in the same message. - // A given name may only be reserved once. - repeated string reserved_name = 10; -} - -message ExtensionRangeOptions { - // The parser stores options it doesn't recognize here. See above. - repeated UninterpretedOption uninterpreted_option = 999; - - - // Clients can define custom options in extensions of this message. See above. - extensions 1000 to max; -} - -// Describes a field within a message. -message FieldDescriptorProto { - enum Type { - // 0 is reserved for errors. - // Order is weird for historical reasons. - TYPE_DOUBLE = 1; - TYPE_FLOAT = 2; - // Not ZigZag encoded. Negative numbers take 10 bytes. Use TYPE_SINT64 if - // negative values are likely. - TYPE_INT64 = 3; - TYPE_UINT64 = 4; - // Not ZigZag encoded. Negative numbers take 10 bytes. Use TYPE_SINT32 if - // negative values are likely. - TYPE_INT32 = 5; - TYPE_FIXED64 = 6; - TYPE_FIXED32 = 7; - TYPE_BOOL = 8; - TYPE_STRING = 9; - // Tag-delimited aggregate. - // Group type is deprecated and not supported in proto3. However, Proto3 - // implementations should still be able to parse the group wire format and - // treat group fields as unknown fields. - TYPE_GROUP = 10; - TYPE_MESSAGE = 11; // Length-delimited aggregate. - - // New in version 2. - TYPE_BYTES = 12; - TYPE_UINT32 = 13; - TYPE_ENUM = 14; - TYPE_SFIXED32 = 15; - TYPE_SFIXED64 = 16; - TYPE_SINT32 = 17; // Uses ZigZag encoding. - TYPE_SINT64 = 18; // Uses ZigZag encoding. - } - - enum Label { - // 0 is reserved for errors - LABEL_OPTIONAL = 1; - LABEL_REQUIRED = 2; - LABEL_REPEATED = 3; - } - - optional string name = 1; - optional int32 number = 3; - optional Label label = 4; - - // If type_name is set, this need not be set. If both this and type_name - // are set, this must be one of TYPE_ENUM, TYPE_MESSAGE or TYPE_GROUP. - optional Type type = 5; - - // For message and enum types, this is the name of the type. If the name - // starts with a '.', it is fully-qualified. Otherwise, C++-like scoping - // rules are used to find the type (i.e. first the nested types within this - // message are searched, then within the parent, on up to the root - // namespace). - optional string type_name = 6; - - // For extensions, this is the name of the type being extended. It is - // resolved in the same manner as type_name. - optional string extendee = 2; - - // For numeric types, contains the original text representation of the value. - // For booleans, "true" or "false". - // For strings, contains the default text contents (not escaped in any way). - // For bytes, contains the C escaped value. All bytes >= 128 are escaped. - optional string default_value = 7; - - // If set, gives the index of a oneof in the containing type's oneof_decl - // list. This field is a member of that oneof. - optional int32 oneof_index = 9; - - // JSON name of this field. The value is set by protocol compiler. If the - // user has set a "json_name" option on this field, that option's value - // will be used. Otherwise, it's deduced from the field's name by converting - // it to camelCase. - optional string json_name = 10; - - optional FieldOptions options = 8; - - // If true, this is a proto3 "optional". When a proto3 field is optional, it - // tracks presence regardless of field type. - // - // When proto3_optional is true, this field must be belong to a oneof to - // signal to old proto3 clients that presence is tracked for this field. This - // oneof is known as a "synthetic" oneof, and this field must be its sole - // member (each proto3 optional field gets its own synthetic oneof). Synthetic - // oneofs exist in the descriptor only, and do not generate any API. Synthetic - // oneofs must be ordered after all "real" oneofs. - // - // For message fields, proto3_optional doesn't create any semantic change, - // since non-repeated message fields always track presence. However it still - // indicates the semantic detail of whether the user wrote "optional" or not. - // This can be useful for round-tripping the .proto file. For consistency we - // give message fields a synthetic oneof also, even though it is not required - // to track presence. This is especially important because the parser can't - // tell if a field is a message or an enum, so it must always create a - // synthetic oneof. - // - // Proto2 optional fields do not set this flag, because they already indicate - // optional with `LABEL_OPTIONAL`. - optional bool proto3_optional = 17; -} - -// Describes a oneof. -message OneofDescriptorProto { - optional string name = 1; - optional OneofOptions options = 2; -} - -// Describes an enum type. -message EnumDescriptorProto { - optional string name = 1; - - repeated EnumValueDescriptorProto value = 2; - - optional EnumOptions options = 3; - - // Range of reserved numeric values. Reserved values may not be used by - // entries in the same enum. Reserved ranges may not overlap. - // - // Note that this is distinct from DescriptorProto.ReservedRange in that it - // is inclusive such that it can appropriately represent the entire int32 - // domain. - message EnumReservedRange { - optional int32 start = 1; // Inclusive. - optional int32 end = 2; // Inclusive. - } - - // Range of reserved numeric values. Reserved numeric values may not be used - // by enum values in the same enum declaration. Reserved ranges may not - // overlap. - repeated EnumReservedRange reserved_range = 4; - - // Reserved enum value names, which may not be reused. A given name may only - // be reserved once. - repeated string reserved_name = 5; -} - -// Describes a value within an enum. -message EnumValueDescriptorProto { - optional string name = 1; - optional int32 number = 2; - - optional EnumValueOptions options = 3; -} - -// Describes a service. -message ServiceDescriptorProto { - optional string name = 1; - repeated MethodDescriptorProto method = 2; - - optional ServiceOptions options = 3; -} - -// Describes a method of a service. -message MethodDescriptorProto { - optional string name = 1; - - // Input and output type names. These are resolved in the same way as - // FieldDescriptorProto.type_name, but must refer to a message type. - optional string input_type = 2; - optional string output_type = 3; - - optional MethodOptions options = 4; - - // Identifies if client streams multiple client messages - optional bool client_streaming = 5 [default = false]; - // Identifies if server streams multiple server messages - optional bool server_streaming = 6 [default = false]; -} - - -// =================================================================== -// Options - -// Each of the definitions above may have "options" attached. These are -// just annotations which may cause code to be generated slightly differently -// or may contain hints for code that manipulates protocol messages. -// -// Clients may define custom options as extensions of the *Options messages. -// These extensions may not yet be known at parsing time, so the parser cannot -// store the values in them. Instead it stores them in a field in the *Options -// message called uninterpreted_option. This field must have the same name -// across all *Options messages. We then use this field to populate the -// extensions when we build a descriptor, at which point all protos have been -// parsed and so all extensions are known. -// -// Extension numbers for custom options may be chosen as follows: -// * For options which will only be used within a single application or -// organization, or for experimental options, use field numbers 50000 -// through 99999. It is up to you to ensure that you do not use the -// same number for multiple options. -// * For options which will be published and used publicly by multiple -// independent entities, e-mail protobuf-global-extension-registry@google.com -// to reserve extension numbers. Simply provide your project name (e.g. -// Objective-C plugin) and your project website (if available) -- there's no -// need to explain how you intend to use them. Usually you only need one -// extension number. You can declare multiple options with only one extension -// number by putting them in a sub-message. See the Custom Options section of -// the docs for examples: -// https://developers.google.com/protocol-buffers/docs/proto#options -// If this turns out to be popular, a web service will be set up -// to automatically assign option numbers. - -message FileOptions { - - // Sets the Java package where classes generated from this .proto will be - // placed. By default, the proto package is used, but this is often - // inappropriate because proto packages do not normally start with backwards - // domain names. - optional string java_package = 1; - - - // Controls the name of the wrapper Java class generated for the .proto file. - // That class will always contain the .proto file's getDescriptor() method as - // well as any top-level extensions defined in the .proto file. - // If java_multiple_files is disabled, then all the other classes from the - // .proto file will be nested inside the single wrapper outer class. - optional string java_outer_classname = 8; - - // If enabled, then the Java code generator will generate a separate .java - // file for each top-level message, enum, and service defined in the .proto - // file. Thus, these types will *not* be nested inside the wrapper class - // named by java_outer_classname. However, the wrapper class will still be - // generated to contain the file's getDescriptor() method as well as any - // top-level extensions defined in the file. - optional bool java_multiple_files = 10 [default = false]; - - // This option does nothing. - optional bool java_generate_equals_and_hash = 20 [deprecated=true]; - - // If set true, then the Java2 code generator will generate code that - // throws an exception whenever an attempt is made to assign a non-UTF-8 - // byte sequence to a string field. - // Message reflection will do the same. - // However, an extension field still accepts non-UTF-8 byte sequences. - // This option has no effect on when used with the lite runtime. - optional bool java_string_check_utf8 = 27 [default = false]; - - - // Generated classes can be optimized for speed or code size. - enum OptimizeMode { - SPEED = 1; // Generate complete code for parsing, serialization, - // etc. - CODE_SIZE = 2; // Use ReflectionOps to implement these methods. - LITE_RUNTIME = 3; // Generate code using MessageLite and the lite runtime. - } - optional OptimizeMode optimize_for = 9 [default = SPEED]; - - // Sets the Go package where structs generated from this .proto will be - // placed. If omitted, the Go package will be derived from the following: - // - The basename of the package import path, if provided. - // - Otherwise, the package statement in the .proto file, if present. - // - Otherwise, the basename of the .proto file, without extension. - optional string go_package = 11; - - - - - // Should generic services be generated in each language? "Generic" services - // are not specific to any particular RPC system. They are generated by the - // main code generators in each language (without additional plugins). - // Generic services were the only kind of service generation supported by - // early versions of google.protobuf. - // - // Generic services are now considered deprecated in favor of using plugins - // that generate code specific to your particular RPC system. Therefore, - // these default to false. Old code which depends on generic services should - // explicitly set them to true. - optional bool cc_generic_services = 16 [default = false]; - optional bool java_generic_services = 17 [default = false]; - optional bool py_generic_services = 18 [default = false]; - optional bool php_generic_services = 42 [default = false]; - - // Is this file deprecated? - // Depending on the target platform, this can emit Deprecated annotations - // for everything in the file, or it will be completely ignored; in the very - // least, this is a formalization for deprecating files. - optional bool deprecated = 23 [default = false]; - - // Enables the use of arenas for the proto messages in this file. This applies - // only to generated classes for C++. - optional bool cc_enable_arenas = 31 [default = true]; - - - // Sets the objective c class prefix which is prepended to all objective c - // generated classes from this .proto. There is no default. - optional string objc_class_prefix = 36; - - // Namespace for generated classes; defaults to the package. - optional string csharp_namespace = 37; - - // By default Swift generators will take the proto package and CamelCase it - // replacing '.' with underscore and use that to prefix the types/symbols - // defined. When this options is provided, they will use this value instead - // to prefix the types/symbols defined. - optional string swift_prefix = 39; - - // Sets the php class prefix which is prepended to all php generated classes - // from this .proto. Default is empty. - optional string php_class_prefix = 40; - - // Use this option to change the namespace of php generated classes. Default - // is empty. When this option is empty, the package name will be used for - // determining the namespace. - optional string php_namespace = 41; - - // Use this option to change the namespace of php generated metadata classes. - // Default is empty. When this option is empty, the proto file name will be - // used for determining the namespace. - optional string php_metadata_namespace = 44; - - // Use this option to change the package of ruby generated classes. Default - // is empty. When this option is not set, the package name will be used for - // determining the ruby package. - optional string ruby_package = 45; - - - // The parser stores options it doesn't recognize here. - // See the documentation for the "Options" section above. - repeated UninterpretedOption uninterpreted_option = 999; - - // Clients can define custom options in extensions of this message. - // See the documentation for the "Options" section above. - extensions 1000 to max; - - reserved 38; -} - -message MessageOptions { - // Set true to use the old proto1 MessageSet wire format for extensions. - // This is provided for backwards-compatibility with the MessageSet wire - // format. You should not use this for any other reason: It's less - // efficient, has fewer features, and is more complicated. - // - // The message must be defined exactly as follows: - // message Foo { - // option message_set_wire_format = true; - // extensions 4 to max; - // } - // Note that the message cannot have any defined fields; MessageSets only - // have extensions. - // - // All extensions of your type must be singular messages; e.g. they cannot - // be int32s, enums, or repeated messages. - // - // Because this is an option, the above two restrictions are not enforced by - // the protocol compiler. - optional bool message_set_wire_format = 1 [default = false]; - - // Disables the generation of the standard "descriptor()" accessor, which can - // conflict with a field of the same name. This is meant to make migration - // from proto1 easier; new code should avoid fields named "descriptor". - optional bool no_standard_descriptor_accessor = 2 [default = false]; - - // Is this message deprecated? - // Depending on the target platform, this can emit Deprecated annotations - // for the message, or it will be completely ignored; in the very least, - // this is a formalization for deprecating messages. - optional bool deprecated = 3 [default = false]; - - reserved 4, 5, 6; - - // Whether the message is an automatically generated map entry type for the - // maps field. - // - // For maps fields: - // map map_field = 1; - // The parsed descriptor looks like: - // message MapFieldEntry { - // option map_entry = true; - // optional KeyType key = 1; - // optional ValueType value = 2; - // } - // repeated MapFieldEntry map_field = 1; - // - // Implementations may choose not to generate the map_entry=true message, but - // use a native map in the target language to hold the keys and values. - // The reflection APIs in such implementations still need to work as - // if the field is a repeated message field. - // - // NOTE: Do not set the option in .proto files. Always use the maps syntax - // instead. The option should only be implicitly set by the proto compiler - // parser. - optional bool map_entry = 7; - - reserved 8; // javalite_serializable - reserved 9; // javanano_as_lite - - - // The parser stores options it doesn't recognize here. See above. - repeated UninterpretedOption uninterpreted_option = 999; - - // Clients can define custom options in extensions of this message. See above. - extensions 1000 to max; -} - -message FieldOptions { - // The ctype option instructs the C++ code generator to use a different - // representation of the field than it normally would. See the specific - // options below. This option is not yet implemented in the open source - // release -- sorry, we'll try to include it in a future version! - optional CType ctype = 1 [default = STRING]; - enum CType { - // Default mode. - STRING = 0; - - CORD = 1; - - STRING_PIECE = 2; - } - // The packed option can be enabled for repeated primitive fields to enable - // a more efficient representation on the wire. Rather than repeatedly - // writing the tag and type for each element, the entire array is encoded as - // a single length-delimited blob. In proto3, only explicit setting it to - // false will avoid using packed encoding. - optional bool packed = 2; - - // The jstype option determines the JavaScript type used for values of the - // field. The option is permitted only for 64 bit integral and fixed types - // (int64, uint64, sint64, fixed64, sfixed64). A field with jstype JS_STRING - // is represented as JavaScript string, which avoids loss of precision that - // can happen when a large value is converted to a floating point JavaScript. - // Specifying JS_NUMBER for the jstype causes the generated JavaScript code to - // use the JavaScript "number" type. The behavior of the default option - // JS_NORMAL is implementation dependent. - // - // This option is an enum to permit additional types to be added, e.g. - // goog.math.Integer. - optional JSType jstype = 6 [default = JS_NORMAL]; - enum JSType { - // Use the default type. - JS_NORMAL = 0; - - // Use JavaScript strings. - JS_STRING = 1; - - // Use JavaScript numbers. - JS_NUMBER = 2; - } - - // Should this field be parsed lazily? Lazy applies only to message-type - // fields. It means that when the outer message is initially parsed, the - // inner message's contents will not be parsed but instead stored in encoded - // form. The inner message will actually be parsed when it is first accessed. - // - // This is only a hint. Implementations are free to choose whether to use - // eager or lazy parsing regardless of the value of this option. However, - // setting this option true suggests that the protocol author believes that - // using lazy parsing on this field is worth the additional bookkeeping - // overhead typically needed to implement it. - // - // This option does not affect the public interface of any generated code; - // all method signatures remain the same. Furthermore, thread-safety of the - // interface is not affected by this option; const methods remain safe to - // call from multiple threads concurrently, while non-const methods continue - // to require exclusive access. - // - // - // Note that implementations may choose not to check required fields within - // a lazy sub-message. That is, calling IsInitialized() on the outer message - // may return true even if the inner message has missing required fields. - // This is necessary because otherwise the inner message would have to be - // parsed in order to perform the check, defeating the purpose of lazy - // parsing. An implementation which chooses not to check required fields - // must be consistent about it. That is, for any particular sub-message, the - // implementation must either *always* check its required fields, or *never* - // check its required fields, regardless of whether or not the message has - // been parsed. - // - // As of May 2022, lazy verifies the contents of the byte stream during - // parsing. An invalid byte stream will cause the overall parsing to fail. - optional bool lazy = 5 [default = false]; - - // unverified_lazy does no correctness checks on the byte stream. This should - // only be used where lazy with verification is prohibitive for performance - // reasons. - optional bool unverified_lazy = 15 [default = false]; - - // Is this field deprecated? - // Depending on the target platform, this can emit Deprecated annotations - // for accessors, or it will be completely ignored; in the very least, this - // is a formalization for deprecating fields. - optional bool deprecated = 3 [default = false]; - - // For Google-internal migration only. Do not use. - optional bool weak = 10 [default = false]; - - - // The parser stores options it doesn't recognize here. See above. - repeated UninterpretedOption uninterpreted_option = 999; - - // Clients can define custom options in extensions of this message. See above. - extensions 1000 to max; - - reserved 4; // removed jtype -} - -message OneofOptions { - // The parser stores options it doesn't recognize here. See above. - repeated UninterpretedOption uninterpreted_option = 999; - - // Clients can define custom options in extensions of this message. See above. - extensions 1000 to max; -} - -message EnumOptions { - - // Set this option to true to allow mapping different tag names to the same - // value. - optional bool allow_alias = 2; - - // Is this enum deprecated? - // Depending on the target platform, this can emit Deprecated annotations - // for the enum, or it will be completely ignored; in the very least, this - // is a formalization for deprecating enums. - optional bool deprecated = 3 [default = false]; - - reserved 5; // javanano_as_lite - - // The parser stores options it doesn't recognize here. See above. - repeated UninterpretedOption uninterpreted_option = 999; - - // Clients can define custom options in extensions of this message. See above. - extensions 1000 to max; -} - -message EnumValueOptions { - // Is this enum value deprecated? - // Depending on the target platform, this can emit Deprecated annotations - // for the enum value, or it will be completely ignored; in the very least, - // this is a formalization for deprecating enum values. - optional bool deprecated = 1 [default = false]; - - // The parser stores options it doesn't recognize here. See above. - repeated UninterpretedOption uninterpreted_option = 999; - - // Clients can define custom options in extensions of this message. See above. - extensions 1000 to max; -} - -message ServiceOptions { - - // Note: Field numbers 1 through 32 are reserved for Google's internal RPC - // framework. We apologize for hoarding these numbers to ourselves, but - // we were already using them long before we decided to release Protocol - // Buffers. - - // Is this service deprecated? - // Depending on the target platform, this can emit Deprecated annotations - // for the service, or it will be completely ignored; in the very least, - // this is a formalization for deprecating services. - optional bool deprecated = 33 [default = false]; - - // The parser stores options it doesn't recognize here. See above. - repeated UninterpretedOption uninterpreted_option = 999; - - // Clients can define custom options in extensions of this message. See above. - extensions 1000 to max; -} - -message MethodOptions { - - // Note: Field numbers 1 through 32 are reserved for Google's internal RPC - // framework. We apologize for hoarding these numbers to ourselves, but - // we were already using them long before we decided to release Protocol - // Buffers. - - // Is this method deprecated? - // Depending on the target platform, this can emit Deprecated annotations - // for the method, or it will be completely ignored; in the very least, - // this is a formalization for deprecating methods. - optional bool deprecated = 33 [default = false]; - - // Is this method side-effect-free (or safe in HTTP parlance), or idempotent, - // or neither? HTTP based RPC implementation may choose GET verb for safe - // methods, and PUT verb for idempotent methods instead of the default POST. - enum IdempotencyLevel { - IDEMPOTENCY_UNKNOWN = 0; - NO_SIDE_EFFECTS = 1; // implies idempotent - IDEMPOTENT = 2; // idempotent, but may have side effects - } - optional IdempotencyLevel idempotency_level = 34 - [default = IDEMPOTENCY_UNKNOWN]; - - // The parser stores options it doesn't recognize here. See above. - repeated UninterpretedOption uninterpreted_option = 999; - - // Clients can define custom options in extensions of this message. See above. - extensions 1000 to max; -} - - -// A message representing a option the parser does not recognize. This only -// appears in options protos created by the compiler::Parser class. -// DescriptorPool resolves these when building Descriptor objects. Therefore, -// options protos in descriptor objects (e.g. returned by Descriptor::options(), -// or produced by Descriptor::CopyTo()) will never have UninterpretedOptions -// in them. -message UninterpretedOption { - // The name of the uninterpreted option. Each string represents a segment in - // a dot-separated name. is_extension is true iff a segment represents an - // extension (denoted with parentheses in options specs in .proto files). - // E.g.,{ ["foo", false], ["bar.baz", true], ["moo", false] } represents - // "foo.(bar.baz).moo". - message NamePart { - required string name_part = 1; - required bool is_extension = 2; - } - repeated NamePart name = 2; - - // The value of the uninterpreted option, in whatever type the tokenizer - // identified it as during parsing. Exactly one of these should be set. - optional string identifier_value = 3; - optional uint64 positive_int_value = 4; - optional int64 negative_int_value = 5; - optional double double_value = 6; - optional bytes string_value = 7; - optional string aggregate_value = 8; -} - -// =================================================================== -// Optional source code info - -// Encapsulates information about the original source file from which a -// FileDescriptorProto was generated. -message SourceCodeInfo { - // A Location identifies a piece of source code in a .proto file which - // corresponds to a particular definition. This information is intended - // to be useful to IDEs, code indexers, documentation generators, and similar - // tools. - // - // For example, say we have a file like: - // message Foo { - // optional string foo = 1; - // } - // Let's look at just the field definition: - // optional string foo = 1; - // ^ ^^ ^^ ^ ^^^ - // a bc de f ghi - // We have the following locations: - // span path represents - // [a,i) [ 4, 0, 2, 0 ] The whole field definition. - // [a,b) [ 4, 0, 2, 0, 4 ] The label (optional). - // [c,d) [ 4, 0, 2, 0, 5 ] The type (string). - // [e,f) [ 4, 0, 2, 0, 1 ] The name (foo). - // [g,h) [ 4, 0, 2, 0, 3 ] The number (1). - // - // Notes: - // - A location may refer to a repeated field itself (i.e. not to any - // particular index within it). This is used whenever a set of elements are - // logically enclosed in a single code segment. For example, an entire - // extend block (possibly containing multiple extension definitions) will - // have an outer location whose path refers to the "extensions" repeated - // field without an index. - // - Multiple locations may have the same path. This happens when a single - // logical declaration is spread out across multiple places. The most - // obvious example is the "extend" block again -- there may be multiple - // extend blocks in the same scope, each of which will have the same path. - // - A location's span is not always a subset of its parent's span. For - // example, the "extendee" of an extension declaration appears at the - // beginning of the "extend" block and is shared by all extensions within - // the block. - // - Just because a location's span is a subset of some other location's span - // does not mean that it is a descendant. For example, a "group" defines - // both a type and a field in a single declaration. Thus, the locations - // corresponding to the type and field and their components will overlap. - // - Code which tries to interpret locations should probably be designed to - // ignore those that it doesn't understand, as more types of locations could - // be recorded in the future. - repeated Location location = 1; - message Location { - // Identifies which part of the FileDescriptorProto was defined at this - // location. - // - // Each element is a field number or an index. They form a path from - // the root FileDescriptorProto to the place where the definition occurs. - // For example, this path: - // [ 4, 3, 2, 7, 1 ] - // refers to: - // file.message_type(3) // 4, 3 - // .field(7) // 2, 7 - // .name() // 1 - // This is because FileDescriptorProto.message_type has field number 4: - // repeated DescriptorProto message_type = 4; - // and DescriptorProto.field has field number 2: - // repeated FieldDescriptorProto field = 2; - // and FieldDescriptorProto.name has field number 1: - // optional string name = 1; - // - // Thus, the above path gives the location of a field name. If we removed - // the last element: - // [ 4, 3, 2, 7 ] - // this path refers to the whole field declaration (from the beginning - // of the label to the terminating semicolon). - repeated int32 path = 1 [packed = true]; - - // Always has exactly three or four elements: start line, start column, - // end line (optional, otherwise assumed same as start line), end column. - // These are packed into a single field for efficiency. Note that line - // and column numbers are zero-based -- typically you will want to add - // 1 to each before displaying to a user. - repeated int32 span = 2 [packed = true]; - - // If this SourceCodeInfo represents a complete declaration, these are any - // comments appearing before and after the declaration which appear to be - // attached to the declaration. - // - // A series of line comments appearing on consecutive lines, with no other - // tokens appearing on those lines, will be treated as a single comment. - // - // leading_detached_comments will keep paragraphs of comments that appear - // before (but not connected to) the current element. Each paragraph, - // separated by empty lines, will be one comment element in the repeated - // field. - // - // Only the comment content is provided; comment markers (e.g. //) are - // stripped out. For block comments, leading whitespace and an asterisk - // will be stripped from the beginning of each line other than the first. - // Newlines are included in the output. - // - // Examples: - // - // optional int32 foo = 1; // Comment attached to foo. - // // Comment attached to bar. - // optional int32 bar = 2; - // - // optional string baz = 3; - // // Comment attached to baz. - // // Another line attached to baz. - // - // // Comment attached to moo. - // // - // // Another line attached to moo. - // optional double moo = 4; - // - // // Detached comment for corge. This is not leading or trailing comments - // // to moo or corge because there are blank lines separating it from - // // both. - // - // // Detached comment for corge paragraph 2. - // - // optional string corge = 5; - // /* Block comment attached - // * to corge. Leading asterisks - // * will be removed. */ - // /* Block comment attached to - // * grault. */ - // optional int32 grault = 6; - // - // // ignored detached comments. - optional string leading_comments = 3; - optional string trailing_comments = 4; - repeated string leading_detached_comments = 6; - } -} - -// Describes the relationship between generated code and its original source -// file. A GeneratedCodeInfo message is associated with only one generated -// source file, but may contain references to different source .proto files. -message GeneratedCodeInfo { - // An Annotation connects some span of text in generated code to an element - // of its generating .proto file. - repeated Annotation annotation = 1; - message Annotation { - // Identifies the element in the original source .proto file. This field - // is formatted the same as SourceCodeInfo.Location.path. - repeated int32 path = 1 [packed = true]; - - // Identifies the filesystem path to the original source .proto. - optional string source_file = 2; - - // Identifies the starting offset in bytes in the generated code - // that relates to the identified object. - optional int32 begin = 3; - - // Identifies the ending offset in bytes in the generated code that - // relates to the identified offset. The end offset should be one past - // the last relevant byte (so the length of the text = end - begin). - optional int32 end = 4; - } -} diff --git a/karapace/protobuf/google/protobuf/duration.proto b/karapace/protobuf/google/protobuf/duration.proto deleted file mode 100644 index 81c3e369f..000000000 --- a/karapace/protobuf/google/protobuf/duration.proto +++ /dev/null @@ -1,116 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2008 Google Inc. All rights reserved. -// https://developers.google.com/protocol-buffers/ -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -syntax = "proto3"; - -package google.protobuf; - -option csharp_namespace = "Google.Protobuf.WellKnownTypes"; -option cc_enable_arenas = true; -option go_package = "google.golang.org/protobuf/types/known/durationpb"; -option java_package = "com.google.protobuf"; -option java_outer_classname = "DurationProto"; -option java_multiple_files = true; -option objc_class_prefix = "GPB"; - -// A Duration represents a signed, fixed-length span of time represented -// as a count of seconds and fractions of seconds at nanosecond -// resolution. It is independent of any calendar and concepts like "day" -// or "month". It is related to Timestamp in that the difference between -// two Timestamp values is a Duration and it can be added or subtracted -// from a Timestamp. Range is approximately +-10,000 years. -// -// # Examples -// -// Example 1: Compute Duration from two Timestamps in pseudo code. -// -// Timestamp start = ...; -// Timestamp end = ...; -// Duration duration = ...; -// -// duration.seconds = end.seconds - start.seconds; -// duration.nanos = end.nanos - start.nanos; -// -// if (duration.seconds < 0 && duration.nanos > 0) { -// duration.seconds += 1; -// duration.nanos -= 1000000000; -// } else if (duration.seconds > 0 && duration.nanos < 0) { -// duration.seconds -= 1; -// duration.nanos += 1000000000; -// } -// -// Example 2: Compute Timestamp from Timestamp + Duration in pseudo code. -// -// Timestamp start = ...; -// Duration duration = ...; -// Timestamp end = ...; -// -// end.seconds = start.seconds + duration.seconds; -// end.nanos = start.nanos + duration.nanos; -// -// if (end.nanos < 0) { -// end.seconds -= 1; -// end.nanos += 1000000000; -// } else if (end.nanos >= 1000000000) { -// end.seconds += 1; -// end.nanos -= 1000000000; -// } -// -// Example 3: Compute Duration from datetime.timedelta in Python. -// -// td = datetime.timedelta(days=3, minutes=10) -// duration = Duration() -// duration.FromTimedelta(td) -// -// # JSON Mapping -// -// In JSON format, the Duration type is encoded as a string rather than an -// object, where the string ends in the suffix "s" (indicating seconds) and -// is preceded by the number of seconds, with nanoseconds expressed as -// fractional seconds. For example, 3 seconds with 0 nanoseconds should be -// encoded in JSON format as "3s", while 3 seconds and 1 nanosecond should -// be expressed in JSON format as "3.000000001s", and 3 seconds and 1 -// microsecond should be expressed in JSON format as "3.000001s". -// -// -message Duration { - // Signed seconds of the span of time. Must be from -315,576,000,000 - // to +315,576,000,000 inclusive. Note: these bounds are computed from: - // 60 sec/min * 60 min/hr * 24 hr/day * 365.25 days/year * 10000 years - int64 seconds = 1; - - // Signed fractions of a second at nanosecond resolution of the span - // of time. Durations less than one second are represented with a 0 - // `seconds` field and a positive or negative `nanos` field. For durations - // of one second or more, a non-zero value for the `nanos` field must be - // of the same sign as the `seconds` field. Must be from -999,999,999 - // to +999,999,999 inclusive. - int32 nanos = 2; -} diff --git a/karapace/protobuf/google/protobuf/empty.proto b/karapace/protobuf/google/protobuf/empty.proto deleted file mode 100644 index 222746219..000000000 --- a/karapace/protobuf/google/protobuf/empty.proto +++ /dev/null @@ -1,51 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2008 Google Inc. All rights reserved. -// https://developers.google.com/protocol-buffers/ -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -syntax = "proto3"; - -package google.protobuf; - -option csharp_namespace = "Google.Protobuf.WellKnownTypes"; -option go_package = "google.golang.org/protobuf/types/known/emptypb"; -option java_package = "com.google.protobuf"; -option java_outer_classname = "EmptyProto"; -option java_multiple_files = true; -option objc_class_prefix = "GPB"; -option cc_enable_arenas = true; - -// A generic empty message that you can re-use to avoid defining duplicated -// empty messages in your APIs. A typical example is to use it as the request -// or the response type of an API method. For instance: -// -// service Foo { -// rpc Bar(google.protobuf.Empty) returns (google.protobuf.Empty); -// } -// -message Empty {} diff --git a/karapace/protobuf/google/protobuf/field_mask.proto b/karapace/protobuf/google/protobuf/field_mask.proto deleted file mode 100644 index 6b5104f18..000000000 --- a/karapace/protobuf/google/protobuf/field_mask.proto +++ /dev/null @@ -1,245 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2008 Google Inc. All rights reserved. -// https://developers.google.com/protocol-buffers/ -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -syntax = "proto3"; - -package google.protobuf; - -option csharp_namespace = "Google.Protobuf.WellKnownTypes"; -option java_package = "com.google.protobuf"; -option java_outer_classname = "FieldMaskProto"; -option java_multiple_files = true; -option objc_class_prefix = "GPB"; -option go_package = "google.golang.org/protobuf/types/known/fieldmaskpb"; -option cc_enable_arenas = true; - -// `FieldMask` represents a set of symbolic field paths, for example: -// -// paths: "f.a" -// paths: "f.b.d" -// -// Here `f` represents a field in some root message, `a` and `b` -// fields in the message found in `f`, and `d` a field found in the -// message in `f.b`. -// -// Field masks are used to specify a subset of fields that should be -// returned by a get operation or modified by an update operation. -// Field masks also have a custom JSON encoding (see below). -// -// # Field Masks in Projections -// -// When used in the context of a projection, a response message or -// sub-message is filtered by the API to only contain those fields as -// specified in the mask. For example, if the mask in the previous -// example is applied to a response message as follows: -// -// f { -// a : 22 -// b { -// d : 1 -// x : 2 -// } -// y : 13 -// } -// z: 8 -// -// The result will not contain specific values for fields x,y and z -// (their value will be set to the default, and omitted in proto text -// output): -// -// -// f { -// a : 22 -// b { -// d : 1 -// } -// } -// -// A repeated field is not allowed except at the last position of a -// paths string. -// -// If a FieldMask object is not present in a get operation, the -// operation applies to all fields (as if a FieldMask of all fields -// had been specified). -// -// Note that a field mask does not necessarily apply to the -// top-level response message. In case of a REST get operation, the -// field mask applies directly to the response, but in case of a REST -// list operation, the mask instead applies to each individual message -// in the returned resource list. In case of a REST custom method, -// other definitions may be used. Where the mask applies will be -// clearly documented together with its declaration in the API. In -// any case, the effect on the returned resource/resources is required -// behavior for APIs. -// -// # Field Masks in Update Operations -// -// A field mask in update operations specifies which fields of the -// targeted resource are going to be updated. The API is required -// to only change the values of the fields as specified in the mask -// and leave the others untouched. If a resource is passed in to -// describe the updated values, the API ignores the values of all -// fields not covered by the mask. -// -// If a repeated field is specified for an update operation, new values will -// be appended to the existing repeated field in the target resource. Note that -// a repeated field is only allowed in the last position of a `paths` string. -// -// If a sub-message is specified in the last position of the field mask for an -// update operation, then new value will be merged into the existing sub-message -// in the target resource. -// -// For example, given the target message: -// -// f { -// b { -// d: 1 -// x: 2 -// } -// c: [1] -// } -// -// And an update message: -// -// f { -// b { -// d: 10 -// } -// c: [2] -// } -// -// then if the field mask is: -// -// paths: ["f.b", "f.c"] -// -// then the result will be: -// -// f { -// b { -// d: 10 -// x: 2 -// } -// c: [1, 2] -// } -// -// An implementation may provide options to override this default behavior for -// repeated and message fields. -// -// In order to reset a field's value to the default, the field must -// be in the mask and set to the default value in the provided resource. -// Hence, in order to reset all fields of a resource, provide a default -// instance of the resource and set all fields in the mask, or do -// not provide a mask as described below. -// -// If a field mask is not present on update, the operation applies to -// all fields (as if a field mask of all fields has been specified). -// Note that in the presence of schema evolution, this may mean that -// fields the client does not know and has therefore not filled into -// the request will be reset to their default. If this is unwanted -// behavior, a specific service may require a client to always specify -// a field mask, producing an error if not. -// -// As with get operations, the location of the resource which -// describes the updated values in the request message depends on the -// operation kind. In any case, the effect of the field mask is -// required to be honored by the API. -// -// ## Considerations for HTTP REST -// -// The HTTP kind of an update operation which uses a field mask must -// be set to PATCH instead of PUT in order to satisfy HTTP semantics -// (PUT must only be used for full updates). -// -// # JSON Encoding of Field Masks -// -// In JSON, a field mask is encoded as a single string where paths are -// separated by a comma. Fields name in each path are converted -// to/from lower-camel naming conventions. -// -// As an example, consider the following message declarations: -// -// message Profile { -// User user = 1; -// Photo photo = 2; -// } -// message User { -// string display_name = 1; -// string address = 2; -// } -// -// In proto a field mask for `Profile` may look as such: -// -// mask { -// paths: "user.display_name" -// paths: "photo" -// } -// -// In JSON, the same mask is represented as below: -// -// { -// mask: "user.displayName,photo" -// } -// -// # Field Masks and Oneof Fields -// -// Field masks treat fields in oneofs just as regular fields. Consider the -// following message: -// -// message SampleMessage { -// oneof test_oneof { -// string name = 4; -// SubMessage sub_message = 9; -// } -// } -// -// The field mask can be: -// -// mask { -// paths: "name" -// } -// -// Or: -// -// mask { -// paths: "sub_message" -// } -// -// Note that oneof type names ("test_oneof" in this case) cannot be used in -// paths. -// -// ## Field Mask Verification -// -// The implementation of any API method which has a FieldMask type field in the -// request should verify the included field paths, and return an -// `INVALID_ARGUMENT` error if any path is unmappable. -message FieldMask { - // The set of field mask paths. - repeated string paths = 1; -} diff --git a/karapace/protobuf/google/protobuf/map_lite_unittest.proto b/karapace/protobuf/google/protobuf/map_lite_unittest.proto deleted file mode 100644 index 7f104315c..000000000 --- a/karapace/protobuf/google/protobuf/map_lite_unittest.proto +++ /dev/null @@ -1,131 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2008 Google Inc. All rights reserved. -// https://developers.google.com/protocol-buffers/ -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -syntax = "proto2"; - -package protobuf_unittest; - -import "google/protobuf/unittest_lite.proto"; - -option cc_enable_arenas = true; -option optimize_for = LITE_RUNTIME; - -message TestMapLite { - map map_int32_int32 = 1; - map map_int64_int64 = 2; - map map_uint32_uint32 = 3; - map map_uint64_uint64 = 4; - map map_sint32_sint32 = 5; - map map_sint64_sint64 = 6; - map map_fixed32_fixed32 = 7; - map map_fixed64_fixed64 = 8; - map map_sfixed32_sfixed32 = 9; - map map_sfixed64_sfixed64 = 10; - map map_int32_float = 11; - map map_int32_double = 12; - map map_bool_bool = 13; - map map_string_string = 14; - map map_int32_bytes = 15; - map map_int32_enum = 16; - map map_int32_foreign_message = 17; - map teboring = 18; -} - -message TestArenaMapLite { - map map_int32_int32 = 1; - map map_int64_int64 = 2; - map map_uint32_uint32 = 3; - map map_uint64_uint64 = 4; - map map_sint32_sint32 = 5; - map map_sint64_sint64 = 6; - map map_fixed32_fixed32 = 7; - map map_fixed64_fixed64 = 8; - map map_sfixed32_sfixed32 = 9; - map map_sfixed64_sfixed64 = 10; - map map_int32_float = 11; - map map_int32_double = 12; - map map_bool_bool = 13; - map map_string_string = 14; - map map_int32_bytes = 15; - map map_int32_enum = 16; - map map_int32_foreign_message = 17; -} - -// Test embedded message with required fields -message TestRequiredMessageMapLite { - map map_field = 1; -} - -message TestEnumMapLite { - map known_map_field = 101; - map unknown_map_field = 102; -} - -message TestEnumMapPlusExtraLite { - map known_map_field = 101; - map unknown_map_field = 102; -} - -message TestMessageMapLite { - map map_int32_message = 1; -} - -enum Proto2MapEnumLite { - PROTO2_MAP_ENUM_FOO_LITE = 0; - PROTO2_MAP_ENUM_BAR_LITE = 1; - PROTO2_MAP_ENUM_BAZ_LITE = 2; -} - -enum Proto2MapEnumPlusExtraLite { - E_PROTO2_MAP_ENUM_FOO_LITE = 0; - E_PROTO2_MAP_ENUM_BAR_LITE = 1; - E_PROTO2_MAP_ENUM_BAZ_LITE = 2; - E_PROTO2_MAP_ENUM_EXTRA_LITE = 3; -} - -enum MapEnumLite { - MAP_ENUM_FOO_LITE = 0; - MAP_ENUM_BAR_LITE = 1; - MAP_ENUM_BAZ_LITE = 2; -} - -message TestRequiredLite { - required int32 a = 1; - required int32 b = 2; - required int32 c = 3; - - extend TestAllExtensionsLite { - optional TestRequiredLite single = 1000; - } -} - -message ForeignMessageArenaLite { - optional int32 c = 1; -} diff --git a/karapace/protobuf/google/protobuf/map_proto2_unittest.proto b/karapace/protobuf/google/protobuf/map_proto2_unittest.proto deleted file mode 100644 index 20d58f903..000000000 --- a/karapace/protobuf/google/protobuf/map_proto2_unittest.proto +++ /dev/null @@ -1,91 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2008 Google Inc. All rights reserved. -// https://developers.google.com/protocol-buffers/ -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -syntax = "proto2"; -option cc_enable_arenas = true; - -import "google/protobuf/unittest_import.proto"; - -// We don't put this in a package within proto2 because we need to make sure -// that the generated code doesn't depend on being in the proto2 namespace. -// In map_test_util.h we do "using namespace unittest = protobuf_unittest". -package protobuf_unittest; - -enum Proto2MapEnum { - PROTO2_MAP_ENUM_FOO = 0; - PROTO2_MAP_ENUM_BAR = 1; - PROTO2_MAP_ENUM_BAZ = 2; -} - -enum Proto2MapEnumPlusExtra { - E_PROTO2_MAP_ENUM_FOO = 0; - E_PROTO2_MAP_ENUM_BAR = 1; - E_PROTO2_MAP_ENUM_BAZ = 2; - E_PROTO2_MAP_ENUM_EXTRA = 3; -} - -message TestEnumMap { - map known_map_field = 101; - map unknown_map_field = 102; -} - -message TestEnumMapPlusExtra { - map known_map_field = 101; - map unknown_map_field = 102; -} - -message TestImportEnumMap { - map import_enum_amp = 1; -} - -message TestIntIntMap { - map m = 1; -} - -// Test all key types: string, plus the non-floating-point scalars. -message TestMaps { - map m_int32 = 1; - map m_int64 = 2; - map m_uint32 = 3; - map m_uint64 = 4; - map m_sint32 = 5; - map m_sint64 = 6; - map m_fixed32 = 7; - map m_fixed64 = 8; - map m_sfixed32 = 9; - map m_sfixed64 = 10; - map m_bool = 11; - map m_string = 12; -} - -// Test maps in submessages. -message TestSubmessageMaps { - optional TestMaps m = 1; -} diff --git a/karapace/protobuf/google/protobuf/map_unittest.proto b/karapace/protobuf/google/protobuf/map_unittest.proto deleted file mode 100644 index 263ef61f8..000000000 --- a/karapace/protobuf/google/protobuf/map_unittest.proto +++ /dev/null @@ -1,125 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2008 Google Inc. All rights reserved. -// https://developers.google.com/protocol-buffers/ -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -syntax = "proto3"; - -option cc_enable_arenas = true; - -import "google/protobuf/unittest.proto"; - -// We don't put this in a package within proto2 because we need to make sure -// that the generated code doesn't depend on being in the proto2 namespace. -// In map_test_util.h we do "using namespace unittest = protobuf_unittest". -package protobuf_unittest; - -// Tests maps. -message TestMap { - map map_int32_int32 = 1; - map map_int64_int64 = 2; - map map_uint32_uint32 = 3; - map map_uint64_uint64 = 4; - map map_sint32_sint32 = 5; - map map_sint64_sint64 = 6; - map map_fixed32_fixed32 = 7; - map map_fixed64_fixed64 = 8; - map map_sfixed32_sfixed32 = 9; - map map_sfixed64_sfixed64 = 10; - map map_int32_float = 11; - map map_int32_double = 12; - map map_bool_bool = 13; - map map_string_string = 14; - map map_int32_bytes = 15; - map map_int32_enum = 16; - map map_int32_foreign_message = 17; - map map_string_foreign_message = 18; - map map_int32_all_types = 19; -} - -message TestMapSubmessage { - TestMap test_map = 1; -} - -message TestMessageMap { - map map_int32_message = 1; -} - -// Two map fields share the same entry default instance. -message TestSameTypeMap { - map map1 = 1; - map map2 = 2; -} - - -enum MapEnum { - MAP_ENUM_FOO = 0; - MAP_ENUM_BAR = 1; - MAP_ENUM_BAZ = 2; -} - -// Test embedded message with required fields -message TestRequiredMessageMap { - map map_field = 1; -} - -message TestArenaMap { - map map_int32_int32 = 1; - map map_int64_int64 = 2; - map map_uint32_uint32 = 3; - map map_uint64_uint64 = 4; - map map_sint32_sint32 = 5; - map map_sint64_sint64 = 6; - map map_fixed32_fixed32 = 7; - map map_fixed64_fixed64 = 8; - map map_sfixed32_sfixed32 = 9; - map map_sfixed64_sfixed64 = 10; - map map_int32_float = 11; - map map_int32_double = 12; - map map_bool_bool = 13; - map map_string_string = 14; - map map_int32_bytes = 15; - map map_int32_enum = 16; - map map_int32_foreign_message = 17; -} - -// Previously, message containing enum called Type cannot be used as value of -// map field. -message MessageContainingEnumCalledType { - enum Type { TYPE_FOO = 0; } - map type = 1; -} - -// Previously, message cannot contain map field called "entry". -message MessageContainingMapCalledEntry { - map entry = 1; -} - -message TestRecursiveMapMessage { - map a = 1; -} diff --git a/karapace/protobuf/google/protobuf/source_context.proto b/karapace/protobuf/google/protobuf/source_context.proto deleted file mode 100644 index 06bfc43a7..000000000 --- a/karapace/protobuf/google/protobuf/source_context.proto +++ /dev/null @@ -1,48 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2008 Google Inc. All rights reserved. -// https://developers.google.com/protocol-buffers/ -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -syntax = "proto3"; - -package google.protobuf; - -option csharp_namespace = "Google.Protobuf.WellKnownTypes"; -option java_package = "com.google.protobuf"; -option java_outer_classname = "SourceContextProto"; -option java_multiple_files = true; -option objc_class_prefix = "GPB"; -option go_package = "google.golang.org/protobuf/types/known/sourcecontextpb"; - -// `SourceContext` represents information about the source of a -// protobuf element, like the file in which it is defined. -message SourceContext { - // The path-qualified name of the .proto file that contained the associated - // protobuf element. For example: `"google/protobuf/source_context.proto"`. - string file_name = 1; -} diff --git a/karapace/protobuf/google/protobuf/struct.proto b/karapace/protobuf/google/protobuf/struct.proto deleted file mode 100644 index 0ac843ca0..000000000 --- a/karapace/protobuf/google/protobuf/struct.proto +++ /dev/null @@ -1,95 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2008 Google Inc. All rights reserved. -// https://developers.google.com/protocol-buffers/ -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -syntax = "proto3"; - -package google.protobuf; - -option csharp_namespace = "Google.Protobuf.WellKnownTypes"; -option cc_enable_arenas = true; -option go_package = "google.golang.org/protobuf/types/known/structpb"; -option java_package = "com.google.protobuf"; -option java_outer_classname = "StructProto"; -option java_multiple_files = true; -option objc_class_prefix = "GPB"; - -// `Struct` represents a structured data value, consisting of fields -// which map to dynamically typed values. In some languages, `Struct` -// might be supported by a native representation. For example, in -// scripting languages like JS a struct is represented as an -// object. The details of that representation are described together -// with the proto support for the language. -// -// The JSON representation for `Struct` is JSON object. -message Struct { - // Unordered map of dynamically typed values. - map fields = 1; -} - -// `Value` represents a dynamically typed value which can be either -// null, a number, a string, a boolean, a recursive struct value, or a -// list of values. A producer of value is expected to set one of these -// variants. Absence of any variant indicates an error. -// -// The JSON representation for `Value` is JSON value. -message Value { - // The kind of value. - oneof kind { - // Represents a null value. - NullValue null_value = 1; - // Represents a double value. - double number_value = 2; - // Represents a string value. - string string_value = 3; - // Represents a boolean value. - bool bool_value = 4; - // Represents a structured value. - Struct struct_value = 5; - // Represents a repeated `Value`. - ListValue list_value = 6; - } -} - -// `NullValue` is a singleton enumeration to represent the null value for the -// `Value` type union. -// -// The JSON representation for `NullValue` is JSON `null`. -enum NullValue { - // Null value. - NULL_VALUE = 0; -} - -// `ListValue` is a wrapper around a repeated field of values. -// -// The JSON representation for `ListValue` is JSON array. -message ListValue { - // Repeated field of dynamically typed values. - repeated Value values = 1; -} diff --git a/karapace/protobuf/google/protobuf/timestamp.proto b/karapace/protobuf/google/protobuf/timestamp.proto deleted file mode 100644 index 3b2df6d91..000000000 --- a/karapace/protobuf/google/protobuf/timestamp.proto +++ /dev/null @@ -1,147 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2008 Google Inc. All rights reserved. -// https://developers.google.com/protocol-buffers/ -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -syntax = "proto3"; - -package google.protobuf; - -option csharp_namespace = "Google.Protobuf.WellKnownTypes"; -option cc_enable_arenas = true; -option go_package = "google.golang.org/protobuf/types/known/timestamppb"; -option java_package = "com.google.protobuf"; -option java_outer_classname = "TimestampProto"; -option java_multiple_files = true; -option objc_class_prefix = "GPB"; - -// A Timestamp represents a point in time independent of any time zone or local -// calendar, encoded as a count of seconds and fractions of seconds at -// nanosecond resolution. The count is relative to an epoch at UTC midnight on -// January 1, 1970, in the proleptic Gregorian calendar which extends the -// Gregorian calendar backwards to year one. -// -// All minutes are 60 seconds long. Leap seconds are "smeared" so that no leap -// second table is needed for interpretation, using a [24-hour linear -// smear](https://developers.google.com/time/smear). -// -// The range is from 0001-01-01T00:00:00Z to 9999-12-31T23:59:59.999999999Z. By -// restricting to that range, we ensure that we can convert to and from [RFC -// 3339](https://www.ietf.org/rfc/rfc3339.txt) date strings. -// -// # Examples -// -// Example 1: Compute Timestamp from POSIX `time()`. -// -// Timestamp timestamp; -// timestamp.set_seconds(time(NULL)); -// timestamp.set_nanos(0); -// -// Example 2: Compute Timestamp from POSIX `gettimeofday()`. -// -// struct timeval tv; -// gettimeofday(&tv, NULL); -// -// Timestamp timestamp; -// timestamp.set_seconds(tv.tv_sec); -// timestamp.set_nanos(tv.tv_usec * 1000); -// -// Example 3: Compute Timestamp from Win32 `GetSystemTimeAsFileTime()`. -// -// FILETIME ft; -// GetSystemTimeAsFileTime(&ft); -// UINT64 ticks = (((UINT64)ft.dwHighDateTime) << 32) | ft.dwLowDateTime; -// -// // A Windows tick is 100 nanoseconds. Windows epoch 1601-01-01T00:00:00Z -// // is 11644473600 seconds before Unix epoch 1970-01-01T00:00:00Z. -// Timestamp timestamp; -// timestamp.set_seconds((INT64) ((ticks / 10000000) - 11644473600LL)); -// timestamp.set_nanos((INT32) ((ticks % 10000000) * 100)); -// -// Example 4: Compute Timestamp from Java `System.currentTimeMillis()`. -// -// long millis = System.currentTimeMillis(); -// -// Timestamp timestamp = Timestamp.newBuilder().setSeconds(millis / 1000) -// .setNanos((int) ((millis % 1000) * 1000000)).build(); -// -// -// Example 5: Compute Timestamp from Java `Instant.now()`. -// -// Instant now = Instant.now(); -// -// Timestamp timestamp = -// Timestamp.newBuilder().setSeconds(now.getEpochSecond()) -// .setNanos(now.getNano()).build(); -// -// -// Example 6: Compute Timestamp from current time in Python. -// -// timestamp = Timestamp() -// timestamp.GetCurrentTime() -// -// # JSON Mapping -// -// In JSON format, the Timestamp type is encoded as a string in the -// [RFC 3339](https://www.ietf.org/rfc/rfc3339.txt) format. That is, the -// format is "{year}-{month}-{day}T{hour}:{min}:{sec}[.{frac_sec}]Z" -// where {year} is always expressed using four digits while {month}, {day}, -// {hour}, {min}, and {sec} are zero-padded to two digits each. The fractional -// seconds, which can go up to 9 digits (i.e. up to 1 nanosecond resolution), -// are optional. The "Z" suffix indicates the timezone ("UTC"); the timezone -// is required. A proto3 JSON serializer should always use UTC (as indicated by -// "Z") when printing the Timestamp type and a proto3 JSON parser should be -// able to accept both UTC and other timezones (as indicated by an offset). -// -// For example, "2017-01-15T01:30:15.01Z" encodes 15.01 seconds past -// 01:30 UTC on January 15, 2017. -// -// In JavaScript, one can convert a Date object to this format using the -// standard -// [toISOString()](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Date/toISOString) -// method. In Python, a standard `datetime.datetime` object can be converted -// to this format using -// [`strftime`](https://docs.python.org/2/library/time.html#time.strftime) with -// the time format spec '%Y-%m-%dT%H:%M:%S.%fZ'. Likewise, in Java, one can use -// the Joda Time's [`ISODateTimeFormat.dateTime()`]( -// http://www.joda.org/joda-time/apidocs/org/joda/time/format/ISODateTimeFormat.html#dateTime%2D%2D -// ) to obtain a formatter capable of generating timestamps in this format. -// -// -message Timestamp { - // Represents seconds of UTC time since Unix epoch - // 1970-01-01T00:00:00Z. Must be from 0001-01-01T00:00:00Z to - // 9999-12-31T23:59:59Z inclusive. - int64 seconds = 1; - - // Non-negative fractions of a second at nanosecond resolution. Negative - // second values with fractions must still have non-negative nanos values - // that count forward in time. Must be from 0 to 999,999,999 - // inclusive. - int32 nanos = 2; -} diff --git a/karapace/protobuf/google/protobuf/type.proto b/karapace/protobuf/google/protobuf/type.proto deleted file mode 100644 index d3f6a68b8..000000000 --- a/karapace/protobuf/google/protobuf/type.proto +++ /dev/null @@ -1,187 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2008 Google Inc. All rights reserved. -// https://developers.google.com/protocol-buffers/ -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -syntax = "proto3"; - -package google.protobuf; - -import "google/protobuf/any.proto"; -import "google/protobuf/source_context.proto"; - -option csharp_namespace = "Google.Protobuf.WellKnownTypes"; -option cc_enable_arenas = true; -option java_package = "com.google.protobuf"; -option java_outer_classname = "TypeProto"; -option java_multiple_files = true; -option objc_class_prefix = "GPB"; -option go_package = "google.golang.org/protobuf/types/known/typepb"; - -// A protocol buffer message type. -message Type { - // The fully qualified message name. - string name = 1; - // The list of fields. - repeated Field fields = 2; - // The list of types appearing in `oneof` definitions in this type. - repeated string oneofs = 3; - // The protocol buffer options. - repeated Option options = 4; - // The source context. - SourceContext source_context = 5; - // The source syntax. - Syntax syntax = 6; -} - -// A single field of a message type. -message Field { - // Basic field types. - enum Kind { - // Field type unknown. - TYPE_UNKNOWN = 0; - // Field type double. - TYPE_DOUBLE = 1; - // Field type float. - TYPE_FLOAT = 2; - // Field type int64. - TYPE_INT64 = 3; - // Field type uint64. - TYPE_UINT64 = 4; - // Field type int32. - TYPE_INT32 = 5; - // Field type fixed64. - TYPE_FIXED64 = 6; - // Field type fixed32. - TYPE_FIXED32 = 7; - // Field type bool. - TYPE_BOOL = 8; - // Field type string. - TYPE_STRING = 9; - // Field type group. Proto2 syntax only, and deprecated. - TYPE_GROUP = 10; - // Field type message. - TYPE_MESSAGE = 11; - // Field type bytes. - TYPE_BYTES = 12; - // Field type uint32. - TYPE_UINT32 = 13; - // Field type enum. - TYPE_ENUM = 14; - // Field type sfixed32. - TYPE_SFIXED32 = 15; - // Field type sfixed64. - TYPE_SFIXED64 = 16; - // Field type sint32. - TYPE_SINT32 = 17; - // Field type sint64. - TYPE_SINT64 = 18; - } - - // Whether a field is optional, required, or repeated. - enum Cardinality { - // For fields with unknown cardinality. - CARDINALITY_UNKNOWN = 0; - // For optional fields. - CARDINALITY_OPTIONAL = 1; - // For required fields. Proto2 syntax only. - CARDINALITY_REQUIRED = 2; - // For repeated fields. - CARDINALITY_REPEATED = 3; - } - - // The field type. - Kind kind = 1; - // The field cardinality. - Cardinality cardinality = 2; - // The field number. - int32 number = 3; - // The field name. - string name = 4; - // The field type URL, without the scheme, for message or enumeration - // types. Example: `"type.googleapis.com/google.protobuf.Timestamp"`. - string type_url = 6; - // The index of the field type in `Type.oneofs`, for message or enumeration - // types. The first type has index 1; zero means the type is not in the list. - int32 oneof_index = 7; - // Whether to use alternative packed wire representation. - bool packed = 8; - // The protocol buffer options. - repeated Option options = 9; - // The field JSON name. - string json_name = 10; - // The string value of the default value of this field. Proto2 syntax only. - string default_value = 11; -} - -// Enum type definition. -message Enum { - // Enum type name. - string name = 1; - // Enum value definitions. - repeated EnumValue enumvalue = 2; - // Protocol buffer options. - repeated Option options = 3; - // The source context. - SourceContext source_context = 4; - // The source syntax. - Syntax syntax = 5; -} - -// Enum value definition. -message EnumValue { - // Enum value name. - string name = 1; - // Enum value number. - int32 number = 2; - // Protocol buffer options. - repeated Option options = 3; -} - -// A protocol buffer option, which can be attached to a message, field, -// enumeration, etc. -message Option { - // The option's name. For protobuf built-in options (options defined in - // descriptor.proto), this is the short name. For example, `"map_entry"`. - // For custom options, it should be the fully-qualified name. For example, - // `"google.api.http"`. - string name = 1; - // The option's value packed in an Any message. If the value is a primitive, - // the corresponding wrapper type defined in google/protobuf/wrappers.proto - // should be used. If the value is an enum, it should be stored as an int32 - // value using the google.protobuf.Int32Value type. - Any value = 2; -} - -// The syntax in which a protocol buffer element is defined. -enum Syntax { - // Syntax `proto2`. - SYNTAX_PROTO2 = 0; - // Syntax `proto3`. - SYNTAX_PROTO3 = 1; -} diff --git a/karapace/protobuf/google/protobuf/wrappers.proto b/karapace/protobuf/google/protobuf/wrappers.proto deleted file mode 100644 index d49dd53c8..000000000 --- a/karapace/protobuf/google/protobuf/wrappers.proto +++ /dev/null @@ -1,123 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2008 Google Inc. All rights reserved. -// https://developers.google.com/protocol-buffers/ -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// Wrappers for primitive (non-message) types. These types are useful -// for embedding primitives in the `google.protobuf.Any` type and for places -// where we need to distinguish between the absence of a primitive -// typed field and its default value. -// -// These wrappers have no meaningful use within repeated fields as they lack -// the ability to detect presence on individual elements. -// These wrappers have no meaningful use within a map or a oneof since -// individual entries of a map or fields of a oneof can already detect presence. - -syntax = "proto3"; - -package google.protobuf; - -option csharp_namespace = "Google.Protobuf.WellKnownTypes"; -option cc_enable_arenas = true; -option go_package = "google.golang.org/protobuf/types/known/wrapperspb"; -option java_package = "com.google.protobuf"; -option java_outer_classname = "WrappersProto"; -option java_multiple_files = true; -option objc_class_prefix = "GPB"; - -// Wrapper message for `double`. -// -// The JSON representation for `DoubleValue` is JSON number. -message DoubleValue { - // The double value. - double value = 1; -} - -// Wrapper message for `float`. -// -// The JSON representation for `FloatValue` is JSON number. -message FloatValue { - // The float value. - float value = 1; -} - -// Wrapper message for `int64`. -// -// The JSON representation for `Int64Value` is JSON string. -message Int64Value { - // The int64 value. - int64 value = 1; -} - -// Wrapper message for `uint64`. -// -// The JSON representation for `UInt64Value` is JSON string. -message UInt64Value { - // The uint64 value. - uint64 value = 1; -} - -// Wrapper message for `int32`. -// -// The JSON representation for `Int32Value` is JSON number. -message Int32Value { - // The int32 value. - int32 value = 1; -} - -// Wrapper message for `uint32`. -// -// The JSON representation for `UInt32Value` is JSON number. -message UInt32Value { - // The uint32 value. - uint32 value = 1; -} - -// Wrapper message for `bool`. -// -// The JSON representation for `BoolValue` is JSON `true` and `false`. -message BoolValue { - // The bool value. - bool value = 1; -} - -// Wrapper message for `string`. -// -// The JSON representation for `StringValue` is JSON string. -message StringValue { - // The string value. - string value = 1; -} - -// Wrapper message for `bytes`. -// -// The JSON representation for `BytesValue` is JSON string. -message BytesValue { - // The bytes value. - bytes value = 1; -} diff --git a/karapace/protobuf/google/type/BUILD.bazel b/karapace/protobuf/google/type/BUILD.bazel deleted file mode 100644 index a7ad4d4d2..000000000 --- a/karapace/protobuf/google/type/BUILD.bazel +++ /dev/null @@ -1,536 +0,0 @@ -load("@rules_proto//proto:defs.bzl", "proto_library") - -# This is an API workspace, having public visibility by default makes perfect sense. -package(default_visibility = ["//visibility:public"]) - -############################################################################## -# Common -############################################################################## -proto_library( - name = "calendar_period_proto", - srcs = ["calendar_period.proto"], -) - -proto_library( - name = "color_proto", - srcs = ["color.proto"], - deps = [ - "@com_google_protobuf//:wrappers_proto", - ], -) - -proto_library( - name = "date_proto", - srcs = ["date.proto"], -) - -proto_library( - name = "datetime_proto", - srcs = ["datetime.proto"], - deps = [ - "@com_google_protobuf//:duration_proto", - ], -) - -proto_library( - name = "dayofweek_proto", - srcs = ["dayofweek.proto"], -) - -proto_library( - name = "decimal_proto", - srcs = ["decimal.proto"], -) - -proto_library( - name = "expr_proto", - srcs = ["expr.proto"], -) - -proto_library( - name = "fraction_proto", - srcs = ["fraction.proto"], -) - -proto_library( - name = "interval_proto", - srcs = ["interval.proto"], - deps = [ - "@com_google_protobuf//:timestamp_proto", - ], -) - -proto_library( - name = "latlng_proto", - srcs = ["latlng.proto"], -) - -proto_library( - name = "localized_text_proto", - srcs = ["localized_text.proto"], -) - -proto_library( - name = "money_proto", - srcs = ["money.proto"], -) - -proto_library( - name = "month_proto", - srcs = ["month.proto"], -) - -proto_library( - name = "phone_number_proto", - srcs = ["phone_number.proto"], -) - -proto_library( - name = "postal_address_proto", - srcs = ["postal_address.proto"], -) - -proto_library( - name = "quaternion_proto", - srcs = ["quaternion.proto"], -) - -proto_library( - name = "timeofday_proto", - srcs = ["timeofday.proto"], -) - -############################################################################## -# Java -############################################################################## -load("@com_google_googleapis_imports//:imports.bzl", - "java_gapic_assembly_gradle_pkg", - "java_proto_library") - -java_proto_library( - name = "type_java_proto", - deps = [ - ":calendar_period_proto", - ":color_proto", - ":date_proto", - ":datetime_proto", - ":dayofweek_proto", - ":decimal_proto", - ":expr_proto", - ":fraction_proto", - ":interval_proto", - ":latlng_proto", - ":localized_text_proto", - ":money_proto", - ":month_proto", - ":phone_number_proto", - ":postal_address_proto", - ":quaternion_proto", - ":timeofday_proto", - ], -) - -# Please DO-NOT-REMOVE this section. -# This is required to generate java files for these protos. -# Open Source Packages -java_gapic_assembly_gradle_pkg( - name = "google-type-java", - deps = [ - ":type_java_proto", - ":calendar_period_proto", - ":color_proto", - ":date_proto", - ":datetime_proto", - ":dayofweek_proto", - ":decimal_proto", - ":expr_proto", - ":fraction_proto", - ":interval_proto", - ":latlng_proto", - ":localized_text_proto", - ":money_proto", - ":month_proto", - ":phone_number_proto", - ":postal_address_proto", - ":quaternion_proto", - ":timeofday_proto", - ], -) - - -############################################################################## -# Go -############################################################################## -load("@com_google_googleapis_imports//:imports.bzl", "go_proto_library") - -go_proto_library( - name = "calendar_period_go_proto", - importpath = "google.golang.org/genproto/googleapis/type/calendarperiod", - protos = [":calendar_period_proto"], -) - -go_proto_library( - name = "color_go_proto", - importpath = "google.golang.org/genproto/googleapis/type/color", - protos = [":color_proto"], -) - -go_proto_library( - name = "date_go_proto", - importpath = "google.golang.org/genproto/googleapis/type/date", - protos = [":date_proto"], -) - -go_proto_library( - name = "datetime_go_proto", - importpath = "google.golang.org/genproto/googleapis/type/datetime", - protos = [":datetime_proto"], -) - -go_proto_library( - name = "dayofweek_go_proto", - importpath = "google.golang.org/genproto/googleapis/type/dayofweek", - protos = [":dayofweek_proto"], -) - -go_proto_library( - name = "decimal_go_proto", - importpath = "google.golang.org/genproto/googleapis/type/decimal", - protos = [":decimal_proto"], -) - -go_proto_library( - name = "expr_go_proto", - importpath = "google.golang.org/genproto/googleapis/type/expr", - protos = [":expr_proto"], -) - -go_proto_library( - name = "fraction_go_proto", - importpath = "google.golang.org/genproto/googleapis/type/fraction", - protos = [":fraction_proto"], -) - -go_proto_library( - name = "interval_go_proto", - importpath = "google.golang.org/genproto/googleapis/type/interval", - protos = [":interval_proto"], -) - -go_proto_library( - name = "latlng_go_proto", - importpath = "google.golang.org/genproto/googleapis/type/latlng", - protos = [":latlng_proto"], -) - -go_proto_library( - name = "localized_text_go_proto", - importpath = "google.golang.org/genproto/googleapis/type/localized_text", - protos = [":localized_text_proto"], -) - -go_proto_library( - name = "money_go_proto", - importpath = "google.golang.org/genproto/googleapis/type/money", - protos = [":money_proto"], -) - -go_proto_library( - name = "month_go_proto", - importpath = "google.golang.org/genproto/googleapis/type/month", - protos = [":month_proto"], -) - -go_proto_library( - name = "phone_number_go_proto", - importpath = "google.golang.org/genproto/googleapis/type/phone_number", - protos = [":phone_number_proto"], -) - -go_proto_library( - name = "postaladdress_go_proto", - importpath = "google.golang.org/genproto/googleapis/type/postaladdress", - protos = [":postal_address_proto"], -) - -go_proto_library( - name = "quaternion_go_proto", - importpath = "google.golang.org/genproto/googleapis/type/quaternion", - protos = [":quaternion_proto"], -) - -go_proto_library( - name = "timeofday_go_proto", - importpath = "google.golang.org/genproto/googleapis/type/timeofday", - protos = [":timeofday_proto"], -) - -############################################################################## -# C++ -############################################################################## -load( - "@com_google_googleapis_imports//:imports.bzl", - "cc_proto_library", -) - -cc_proto_library( - name = "calendar_period_cc_proto", - deps = [":calendar_period_proto"], -) - -cc_proto_library( - name = "color_cc_proto", - deps = [":color_proto"], -) - -cc_proto_library( - name = "date_cc_proto", - deps = [":date_proto"], -) - -cc_proto_library( - name = "datetime_cc_proto", - deps = [":datetime_proto"], -) - -cc_proto_library( - name = "dayofweek_cc_proto", - deps = [":dayofweek_proto"], -) - -cc_proto_library( - name = "decimal_cc_proto", - deps = [":decimal_proto"], -) - -cc_proto_library( - name = "expr_cc_proto", - deps = [":expr_proto"], -) - -cc_proto_library( - name = "fraction_cc_proto", - deps = [":fraction_proto"], -) - -cc_proto_library( - name = "interval_cc_proto", - deps = [":interval_proto"], -) - -cc_proto_library( - name = "latlng_cc_proto", - deps = [":latlng_proto"], -) - -cc_proto_library( - name = "money_cc_proto", - deps = [":money_proto"], -) - -cc_proto_library( - name = "month_cc_proto", - deps = [":month_proto"], -) - -cc_proto_library( - name = "phone_number_cc_proto", - deps = [":phone_number_proto"], -) - -cc_proto_library( - name = "postal_address_cc_proto", - deps = [":postal_address_proto"], -) - -cc_proto_library( - name = "quaternion_cc_proto", - deps = [":quaternion_proto"], -) - -cc_proto_library( - name = "timeofday_cc_proto", - deps = [":timeofday_proto"], -) - -############################################################################## -# Python -############################################################################## -load( - "@com_google_googleapis_imports//:imports.bzl", - "py_proto_library", -) - -py_proto_library( - name = "calendar_period_py_proto", - deps = [":calendar_period_proto"], -) - -py_proto_library( - name = "color_py_proto", - deps = [":color_proto"], -) - -py_proto_library( - name = "date_py_proto", - deps = [":date_proto"], -) - -py_proto_library( - name = "datetime_py_proto", - deps = [":datetime_proto"], -) - -py_proto_library( - name = "dayofweek_py_proto", - deps = [":dayofweek_proto"], -) - -py_proto_library( - name = "decimal_py_proto", - deps = [":decimal_proto"], -) - -py_proto_library( - name = "expr_py_proto", - deps = [":expr_proto"], -) - -py_proto_library( - name = "fraction_py_proto", - deps = [":fraction_proto"], -) - -py_proto_library( - name = "interval_py_proto", - deps = [":interval_proto"], -) - -py_proto_library( - name = "latlng_py_proto", - deps = [":latlng_proto"], -) - -py_proto_library( - name = "localized_text_py_proto", - deps = [":localized_text_proto"], -) - -py_proto_library( - name = "money_py_proto", - deps = [":money_proto"], -) - -py_proto_library( - name = "month_py_proto", - deps = [":month_proto"], -) - -py_proto_library( - name = "phone_number_py_proto", - deps = [":phone_number_proto"], -) - -py_proto_library( - name = "postal_address_py_proto", - deps = [":postal_address_proto"], -) - -py_proto_library( - name = "quaternion_py_proto", - deps = [":quaternion_proto"], -) - -py_proto_library( - name = "timeofday_py_proto", - deps = [":timeofday_proto"], -) - -############################################################################## -# C# -############################################################################## - -load( - "@com_google_googleapis_imports//:imports.bzl", - "csharp_proto_library", -) - -csharp_proto_library( - name = "calendar_period_csharp_proto", - deps = [":calendar_period_proto"], -) - -csharp_proto_library( - name = "color_csharp_proto", - deps = [":color_proto"], -) - -csharp_proto_library( - name = "date_csharp_proto", - deps = [":date_proto"], -) - -csharp_proto_library( - name = "datetime_csharp_proto", - deps = [":datetime_proto"], -) - -csharp_proto_library( - name = "dayofweek_csharp_proto", - deps = [":dayofweek_proto"], -) - -csharp_proto_library( - name = "decimal_csharp_proto", - deps = [":decimal_proto"], -) - -csharp_proto_library( - name = "expr_csharp_proto", - deps = [":expr_proto"], -) - -csharp_proto_library( - name = "fraction_csharp_proto", - deps = [":fraction_proto"], -) - -csharp_proto_library( - name = "interval_csharp_proto", - deps = [":interval_proto"], -) - -csharp_proto_library( - name = "latlng_csharp_proto", - deps = [":latlng_proto"], -) - -csharp_proto_library( - name = "localized_text_csharp_proto", - deps = [":localized_text_proto"], -) - -csharp_proto_library( - name = "money_csharp_proto", - deps = [":money_proto"], -) - -csharp_proto_library( - name = "month_csharp_proto", - deps = [":month_proto"], -) - -csharp_proto_library( - name = "phone_number_csharp_proto", - deps = [":phone_number_proto"], -) - -csharp_proto_library( - name = "postal_address_csharp_proto", - deps = [":postal_address_proto"], -) - -csharp_proto_library( - name = "quaternion_csharp_proto", - deps = [":quaternion_proto"], -) diff --git a/karapace/protobuf/google/type/README.md b/karapace/protobuf/google/type/README.md deleted file mode 100644 index de6a835d7..000000000 --- a/karapace/protobuf/google/type/README.md +++ /dev/null @@ -1,16 +0,0 @@ -# Google Common Types - -This package contains definitions of common types for Google APIs. -All types defined in this package are suitable for different APIs to -exchange data, and will never break binary compatibility. They should -have design quality comparable to major programming languages like -Java and C#. - -NOTE: Some common types are defined in the package `google.protobuf` -as they are directly supported by Protocol Buffers compiler and -runtime. Those types are called Well-Known Types. - -## Java Utilities - -A set of Java utilities for the Common Types are provided in the -`//java/com/google/type/util/` package. diff --git a/karapace/protobuf/google/type/calendar_period.proto b/karapace/protobuf/google/type/calendar_period.proto deleted file mode 100644 index 82f5690b7..000000000 --- a/karapace/protobuf/google/type/calendar_period.proto +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright 2021 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package google.type; - -option go_package = "google.golang.org/genproto/googleapis/type/calendarperiod;calendarperiod"; -option java_multiple_files = true; -option java_outer_classname = "CalendarPeriodProto"; -option java_package = "com.google.type"; -option objc_class_prefix = "GTP"; - -// A `CalendarPeriod` represents the abstract concept of a time period that has -// a canonical start. Grammatically, "the start of the current -// `CalendarPeriod`." All calendar times begin at midnight UTC. -enum CalendarPeriod { - // Undefined period, raises an error. - CALENDAR_PERIOD_UNSPECIFIED = 0; - - // A day. - DAY = 1; - - // A week. Weeks begin on Monday, following - // [ISO 8601](https://en.wikipedia.org/wiki/ISO_week_date). - WEEK = 2; - - // A fortnight. The first calendar fortnight of the year begins at the start - // of week 1 according to - // [ISO 8601](https://en.wikipedia.org/wiki/ISO_week_date). - FORTNIGHT = 3; - - // A month. - MONTH = 4; - - // A quarter. Quarters start on dates 1-Jan, 1-Apr, 1-Jul, and 1-Oct of each - // year. - QUARTER = 5; - - // A half-year. Half-years start on dates 1-Jan and 1-Jul. - HALF = 6; - - // A year. - YEAR = 7; -} diff --git a/karapace/protobuf/google/type/color.proto b/karapace/protobuf/google/type/color.proto deleted file mode 100644 index 5dc85a6a3..000000000 --- a/karapace/protobuf/google/type/color.proto +++ /dev/null @@ -1,174 +0,0 @@ -// Copyright 2021 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package google.type; - -import "google/protobuf/wrappers.proto"; - -option cc_enable_arenas = true; -option go_package = "google.golang.org/genproto/googleapis/type/color;color"; -option java_multiple_files = true; -option java_outer_classname = "ColorProto"; -option java_package = "com.google.type"; -option objc_class_prefix = "GTP"; - -// Represents a color in the RGBA color space. This representation is designed -// for simplicity of conversion to/from color representations in various -// languages over compactness. For example, the fields of this representation -// can be trivially provided to the constructor of `java.awt.Color` in Java; it -// can also be trivially provided to UIColor's `+colorWithRed:green:blue:alpha` -// method in iOS; and, with just a little work, it can be easily formatted into -// a CSS `rgba()` string in JavaScript. -// -// This reference page doesn't carry information about the absolute color -// space -// that should be used to interpret the RGB value (e.g. sRGB, Adobe RGB, -// DCI-P3, BT.2020, etc.). By default, applications should assume the sRGB color -// space. -// -// When color equality needs to be decided, implementations, unless -// documented otherwise, treat two colors as equal if all their red, -// green, blue, and alpha values each differ by at most 1e-5. -// -// Example (Java): -// -// import com.google.type.Color; -// -// // ... -// public static java.awt.Color fromProto(Color protocolor) { -// float alpha = protocolor.hasAlpha() -// ? protocolor.getAlpha().getValue() -// : 1.0; -// -// return new java.awt.Color( -// protocolor.getRed(), -// protocolor.getGreen(), -// protocolor.getBlue(), -// alpha); -// } -// -// public static Color toProto(java.awt.Color color) { -// float red = (float) color.getRed(); -// float green = (float) color.getGreen(); -// float blue = (float) color.getBlue(); -// float denominator = 255.0; -// Color.Builder resultBuilder = -// Color -// .newBuilder() -// .setRed(red / denominator) -// .setGreen(green / denominator) -// .setBlue(blue / denominator); -// int alpha = color.getAlpha(); -// if (alpha != 255) { -// result.setAlpha( -// FloatValue -// .newBuilder() -// .setValue(((float) alpha) / denominator) -// .build()); -// } -// return resultBuilder.build(); -// } -// // ... -// -// Example (iOS / Obj-C): -// -// // ... -// static UIColor* fromProto(Color* protocolor) { -// float red = [protocolor red]; -// float green = [protocolor green]; -// float blue = [protocolor blue]; -// FloatValue* alpha_wrapper = [protocolor alpha]; -// float alpha = 1.0; -// if (alpha_wrapper != nil) { -// alpha = [alpha_wrapper value]; -// } -// return [UIColor colorWithRed:red green:green blue:blue alpha:alpha]; -// } -// -// static Color* toProto(UIColor* color) { -// CGFloat red, green, blue, alpha; -// if (![color getRed:&red green:&green blue:&blue alpha:&alpha]) { -// return nil; -// } -// Color* result = [[Color alloc] init]; -// [result setRed:red]; -// [result setGreen:green]; -// [result setBlue:blue]; -// if (alpha <= 0.9999) { -// [result setAlpha:floatWrapperWithValue(alpha)]; -// } -// [result autorelease]; -// return result; -// } -// // ... -// -// Example (JavaScript): -// -// // ... -// -// var protoToCssColor = function(rgb_color) { -// var redFrac = rgb_color.red || 0.0; -// var greenFrac = rgb_color.green || 0.0; -// var blueFrac = rgb_color.blue || 0.0; -// var red = Math.floor(redFrac * 255); -// var green = Math.floor(greenFrac * 255); -// var blue = Math.floor(blueFrac * 255); -// -// if (!('alpha' in rgb_color)) { -// return rgbToCssColor(red, green, blue); -// } -// -// var alphaFrac = rgb_color.alpha.value || 0.0; -// var rgbParams = [red, green, blue].join(','); -// return ['rgba(', rgbParams, ',', alphaFrac, ')'].join(''); -// }; -// -// var rgbToCssColor = function(red, green, blue) { -// var rgbNumber = new Number((red << 16) | (green << 8) | blue); -// var hexString = rgbNumber.toString(16); -// var missingZeros = 6 - hexString.length; -// var resultBuilder = ['#']; -// for (var i = 0; i < missingZeros; i++) { -// resultBuilder.push('0'); -// } -// resultBuilder.push(hexString); -// return resultBuilder.join(''); -// }; -// -// // ... -message Color { - // The amount of red in the color as a value in the interval [0, 1]. - float red = 1; - - // The amount of green in the color as a value in the interval [0, 1]. - float green = 2; - - // The amount of blue in the color as a value in the interval [0, 1]. - float blue = 3; - - // The fraction of this color that should be applied to the pixel. That is, - // the final pixel color is defined by the equation: - // - // `pixel color = alpha * (this color) + (1.0 - alpha) * (background color)` - // - // This means that a value of 1.0 corresponds to a solid color, whereas - // a value of 0.0 corresponds to a completely transparent color. This - // uses a wrapper message rather than a simple float scalar so that it is - // possible to distinguish between a default value and the value being unset. - // If omitted, this color object is rendered as a solid color - // (as if the alpha value had been explicitly given a value of 1.0). - google.protobuf.FloatValue alpha = 4; -} diff --git a/karapace/protobuf/google/type/date.proto b/karapace/protobuf/google/type/date.proto deleted file mode 100644 index e4e730e6f..000000000 --- a/karapace/protobuf/google/type/date.proto +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright 2021 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package google.type; - -option cc_enable_arenas = true; -option go_package = "google.golang.org/genproto/googleapis/type/date;date"; -option java_multiple_files = true; -option java_outer_classname = "DateProto"; -option java_package = "com.google.type"; -option objc_class_prefix = "GTP"; - -// Represents a whole or partial calendar date, such as a birthday. The time of -// day and time zone are either specified elsewhere or are insignificant. The -// date is relative to the Gregorian Calendar. This can represent one of the -// following: -// -// * A full date, with non-zero year, month, and day values -// * A month and day value, with a zero year, such as an anniversary -// * A year on its own, with zero month and day values -// * A year and month value, with a zero day, such as a credit card expiration -// date -// -// Related types are [google.type.TimeOfDay][google.type.TimeOfDay] and -// `google.protobuf.Timestamp`. -message Date { - // Year of the date. Must be from 1 to 9999, or 0 to specify a date without - // a year. - int32 year = 1; - - // Month of a year. Must be from 1 to 12, or 0 to specify a year without a - // month and day. - int32 month = 2; - - // Day of a month. Must be from 1 to 31 and valid for the year and month, or 0 - // to specify a year by itself or a year and month where the day isn't - // significant. - int32 day = 3; -} diff --git a/karapace/protobuf/google/type/datetime.proto b/karapace/protobuf/google/type/datetime.proto deleted file mode 100644 index cfed85d70..000000000 --- a/karapace/protobuf/google/type/datetime.proto +++ /dev/null @@ -1,104 +0,0 @@ -// Copyright 2021 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package google.type; - -import "google/protobuf/duration.proto"; - -option cc_enable_arenas = true; -option go_package = "google.golang.org/genproto/googleapis/type/datetime;datetime"; -option java_multiple_files = true; -option java_outer_classname = "DateTimeProto"; -option java_package = "com.google.type"; -option objc_class_prefix = "GTP"; - -// Represents civil time (or occasionally physical time). -// -// This type can represent a civil time in one of a few possible ways: -// -// * When utc_offset is set and time_zone is unset: a civil time on a calendar -// day with a particular offset from UTC. -// * When time_zone is set and utc_offset is unset: a civil time on a calendar -// day in a particular time zone. -// * When neither time_zone nor utc_offset is set: a civil time on a calendar -// day in local time. -// -// The date is relative to the Proleptic Gregorian Calendar. -// -// If year is 0, the DateTime is considered not to have a specific year. month -// and day must have valid, non-zero values. -// -// This type may also be used to represent a physical time if all the date and -// time fields are set and either case of the `time_offset` oneof is set. -// Consider using `Timestamp` message for physical time instead. If your use -// case also would like to store the user's timezone, that can be done in -// another field. -// -// This type is more flexible than some applications may want. Make sure to -// document and validate your application's limitations. -message DateTime { - // Optional. Year of date. Must be from 1 to 9999, or 0 if specifying a - // datetime without a year. - int32 year = 1; - - // Required. Month of year. Must be from 1 to 12. - int32 month = 2; - - // Required. Day of month. Must be from 1 to 31 and valid for the year and - // month. - int32 day = 3; - - // Required. Hours of day in 24 hour format. Should be from 0 to 23. An API - // may choose to allow the value "24:00:00" for scenarios like business - // closing time. - int32 hours = 4; - - // Required. Minutes of hour of day. Must be from 0 to 59. - int32 minutes = 5; - - // Required. Seconds of minutes of the time. Must normally be from 0 to 59. An - // API may allow the value 60 if it allows leap-seconds. - int32 seconds = 6; - - // Required. Fractions of seconds in nanoseconds. Must be from 0 to - // 999,999,999. - int32 nanos = 7; - - // Optional. Specifies either the UTC offset or the time zone of the DateTime. - // Choose carefully between them, considering that time zone data may change - // in the future (for example, a country modifies their DST start/end dates, - // and future DateTimes in the affected range had already been stored). - // If omitted, the DateTime is considered to be in local time. - oneof time_offset { - // UTC offset. Must be whole seconds, between -18 hours and +18 hours. - // For example, a UTC offset of -4:00 would be represented as - // { seconds: -14400 }. - google.protobuf.Duration utc_offset = 8; - - // Time zone. - TimeZone time_zone = 9; - } -} - -// Represents a time zone from the -// [IANA Time Zone Database](https://www.iana.org/time-zones). -message TimeZone { - // IANA Time Zone Database time zone, e.g. "America/New_York". - string id = 1; - - // Optional. IANA Time Zone Database version number, e.g. "2019a". - string version = 2; -} diff --git a/karapace/protobuf/google/type/dayofweek.proto b/karapace/protobuf/google/type/dayofweek.proto deleted file mode 100644 index 4c80c62ec..000000000 --- a/karapace/protobuf/google/type/dayofweek.proto +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2021 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package google.type; - -option go_package = "google.golang.org/genproto/googleapis/type/dayofweek;dayofweek"; -option java_multiple_files = true; -option java_outer_classname = "DayOfWeekProto"; -option java_package = "com.google.type"; -option objc_class_prefix = "GTP"; - -// Represents a day of the week. -enum DayOfWeek { - // The day of the week is unspecified. - DAY_OF_WEEK_UNSPECIFIED = 0; - - // Monday - MONDAY = 1; - - // Tuesday - TUESDAY = 2; - - // Wednesday - WEDNESDAY = 3; - - // Thursday - THURSDAY = 4; - - // Friday - FRIDAY = 5; - - // Saturday - SATURDAY = 6; - - // Sunday - SUNDAY = 7; -} diff --git a/karapace/protobuf/google/type/decimal.proto b/karapace/protobuf/google/type/decimal.proto deleted file mode 100644 index beb18a5d8..000000000 --- a/karapace/protobuf/google/type/decimal.proto +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright 2021 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package google.type; - -option cc_enable_arenas = true; -option go_package = "google.golang.org/genproto/googleapis/type/decimal;decimal"; -option java_multiple_files = true; -option java_outer_classname = "DecimalProto"; -option java_package = "com.google.type"; -option objc_class_prefix = "GTP"; - -// A representation of a decimal value, such as 2.5. Clients may convert values -// into language-native decimal formats, such as Java's [BigDecimal][] or -// Python's [decimal.Decimal][]. -// -// [BigDecimal]: -// https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/math/BigDecimal.html -// [decimal.Decimal]: https://docs.python.org/3/library/decimal.html -message Decimal { - // The decimal value, as a string. - // - // The string representation consists of an optional sign, `+` (`U+002B`) - // or `-` (`U+002D`), followed by a sequence of zero or more decimal digits - // ("the integer"), optionally followed by a fraction, optionally followed - // by an exponent. - // - // The fraction consists of a decimal point followed by zero or more decimal - // digits. The string must contain at least one digit in either the integer - // or the fraction. The number formed by the sign, the integer and the - // fraction is referred to as the significand. - // - // The exponent consists of the character `e` (`U+0065`) or `E` (`U+0045`) - // followed by one or more decimal digits. - // - // Services **should** normalize decimal values before storing them by: - // - // - Removing an explicitly-provided `+` sign (`+2.5` -> `2.5`). - // - Replacing a zero-length integer value with `0` (`.5` -> `0.5`). - // - Coercing the exponent character to lower-case (`2.5E8` -> `2.5e8`). - // - Removing an explicitly-provided zero exponent (`2.5e0` -> `2.5`). - // - // Services **may** perform additional normalization based on its own needs - // and the internal decimal implementation selected, such as shifting the - // decimal point and exponent value together (example: `2.5e-1` <-> `0.25`). - // Additionally, services **may** preserve trailing zeroes in the fraction - // to indicate increased precision, but are not required to do so. - // - // Note that only the `.` character is supported to divide the integer - // and the fraction; `,` **should not** be supported regardless of locale. - // Additionally, thousand separators **should not** be supported. If a - // service does support them, values **must** be normalized. - // - // The ENBF grammar is: - // - // DecimalString = - // [Sign] Significand [Exponent]; - // - // Sign = '+' | '-'; - // - // Significand = - // Digits ['.'] [Digits] | [Digits] '.' Digits; - // - // Exponent = ('e' | 'E') [Sign] Digits; - // - // Digits = { '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' }; - // - // Services **should** clearly document the range of supported values, the - // maximum supported precision (total number of digits), and, if applicable, - // the scale (number of digits after the decimal point), as well as how it - // behaves when receiving out-of-bounds values. - // - // Services **may** choose to accept values passed as input even when the - // value has a higher precision or scale than the service supports, and - // **should** round the value to fit the supported scale. Alternatively, the - // service **may** error with `400 Bad Request` (`INVALID_ARGUMENT` in gRPC) - // if precision would be lost. - // - // Services **should** error with `400 Bad Request` (`INVALID_ARGUMENT` in - // gRPC) if the service receives a value outside of the supported range. - string value = 1; -} diff --git a/karapace/protobuf/google/type/expr.proto b/karapace/protobuf/google/type/expr.proto deleted file mode 100644 index af0778cf9..000000000 --- a/karapace/protobuf/google/type/expr.proto +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright 2021 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package google.type; - -option go_package = "google.golang.org/genproto/googleapis/type/expr;expr"; -option java_multiple_files = true; -option java_outer_classname = "ExprProto"; -option java_package = "com.google.type"; -option objc_class_prefix = "GTP"; - -// Represents a textual expression in the Common Expression Language (CEL) -// syntax. CEL is a C-like expression language. The syntax and semantics of CEL -// are documented at https://github.com/google/cel-spec. -// -// Example (Comparison): -// -// title: "Summary size limit" -// description: "Determines if a summary is less than 100 chars" -// expression: "document.summary.size() < 100" -// -// Example (Equality): -// -// title: "Requestor is owner" -// description: "Determines if requestor is the document owner" -// expression: "document.owner == request.auth.claims.email" -// -// Example (Logic): -// -// title: "Public documents" -// description: "Determine whether the document should be publicly visible" -// expression: "document.type != 'private' && document.type != 'internal'" -// -// Example (Data Manipulation): -// -// title: "Notification string" -// description: "Create a notification string with a timestamp." -// expression: "'New message received at ' + string(document.create_time)" -// -// The exact variables and functions that may be referenced within an expression -// are determined by the service that evaluates it. See the service -// documentation for additional information. -message Expr { - // Textual representation of an expression in Common Expression Language - // syntax. - string expression = 1; - - // Optional. Title for the expression, i.e. a short string describing - // its purpose. This can be used e.g. in UIs which allow to enter the - // expression. - string title = 2; - - // Optional. Description of the expression. This is a longer text which - // describes the expression, e.g. when hovered over it in a UI. - string description = 3; - - // Optional. String indicating the location of the expression for error - // reporting, e.g. a file name and a position in the file. - string location = 4; -} diff --git a/karapace/protobuf/google/type/fraction.proto b/karapace/protobuf/google/type/fraction.proto deleted file mode 100644 index 6c5ae6e2a..000000000 --- a/karapace/protobuf/google/type/fraction.proto +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2021 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package google.type; - -option go_package = "google.golang.org/genproto/googleapis/type/fraction;fraction"; -option java_multiple_files = true; -option java_outer_classname = "FractionProto"; -option java_package = "com.google.type"; -option objc_class_prefix = "GTP"; - -// Represents a fraction in terms of a numerator divided by a denominator. -message Fraction { - // The numerator in the fraction, e.g. 2 in 2/3. - int64 numerator = 1; - - // The value by which the numerator is divided, e.g. 3 in 2/3. Must be - // positive. - int64 denominator = 2; -} diff --git a/karapace/protobuf/google/type/interval.proto b/karapace/protobuf/google/type/interval.proto deleted file mode 100644 index 9702324cd..000000000 --- a/karapace/protobuf/google/type/interval.proto +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2021 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package google.type; - -import "google/protobuf/timestamp.proto"; - -option cc_enable_arenas = true; -option go_package = "google.golang.org/genproto/googleapis/type/interval;interval"; -option java_multiple_files = true; -option java_outer_classname = "IntervalProto"; -option java_package = "com.google.type"; -option objc_class_prefix = "GTP"; - -// Represents a time interval, encoded as a Timestamp start (inclusive) and a -// Timestamp end (exclusive). -// -// The start must be less than or equal to the end. -// When the start equals the end, the interval is empty (matches no time). -// When both start and end are unspecified, the interval matches any time. -message Interval { - // Optional. Inclusive start of the interval. - // - // If specified, a Timestamp matching this interval will have to be the same - // or after the start. - google.protobuf.Timestamp start_time = 1; - - // Optional. Exclusive end of the interval. - // - // If specified, a Timestamp matching this interval will have to be before the - // end. - google.protobuf.Timestamp end_time = 2; -} diff --git a/karapace/protobuf/google/type/latlng.proto b/karapace/protobuf/google/type/latlng.proto deleted file mode 100644 index 9231456e3..000000000 --- a/karapace/protobuf/google/type/latlng.proto +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2021 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package google.type; - -option cc_enable_arenas = true; -option go_package = "google.golang.org/genproto/googleapis/type/latlng;latlng"; -option java_multiple_files = true; -option java_outer_classname = "LatLngProto"; -option java_package = "com.google.type"; -option objc_class_prefix = "GTP"; - -// An object that represents a latitude/longitude pair. This is expressed as a -// pair of doubles to represent degrees latitude and degrees longitude. Unless -// specified otherwise, this must conform to the -// WGS84 -// standard. Values must be within normalized ranges. -message LatLng { - // The latitude in degrees. It must be in the range [-90.0, +90.0]. - double latitude = 1; - - // The longitude in degrees. It must be in the range [-180.0, +180.0]. - double longitude = 2; -} diff --git a/karapace/protobuf/google/type/localized_text.proto b/karapace/protobuf/google/type/localized_text.proto deleted file mode 100644 index 5c6922b8c..000000000 --- a/karapace/protobuf/google/type/localized_text.proto +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright 2021 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package google.type; - -option cc_enable_arenas = true; -option go_package = "google.golang.org/genproto/googleapis/type/localized_text;localized_text"; -option java_multiple_files = true; -option java_outer_classname = "LocalizedTextProto"; -option java_package = "com.google.type"; -option objc_class_prefix = "GTP"; - -// Localized variant of a text in a particular language. -message LocalizedText { - // Localized string in the language corresponding to `language_code' below. - string text = 1; - - // The text's BCP-47 language code, such as "en-US" or "sr-Latn". - // - // For more information, see - // http://www.unicode.org/reports/tr35/#Unicode_locale_identifier. - string language_code = 2; -} diff --git a/karapace/protobuf/google/type/money.proto b/karapace/protobuf/google/type/money.proto deleted file mode 100644 index 98d6494e4..000000000 --- a/karapace/protobuf/google/type/money.proto +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2021 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package google.type; - -option cc_enable_arenas = true; -option go_package = "google.golang.org/genproto/googleapis/type/money;money"; -option java_multiple_files = true; -option java_outer_classname = "MoneyProto"; -option java_package = "com.google.type"; -option objc_class_prefix = "GTP"; - -// Represents an amount of money with its currency type. -message Money { - // The three-letter currency code defined in ISO 4217. - string currency_code = 1; - - // The whole units of the amount. - // For example if `currencyCode` is `"USD"`, then 1 unit is one US dollar. - int64 units = 2; - - // Number of nano (10^-9) units of the amount. - // The value must be between -999,999,999 and +999,999,999 inclusive. - // If `units` is positive, `nanos` must be positive or zero. - // If `units` is zero, `nanos` can be positive, zero, or negative. - // If `units` is negative, `nanos` must be negative or zero. - // For example $-1.75 is represented as `units`=-1 and `nanos`=-750,000,000. - int32 nanos = 3; -} diff --git a/karapace/protobuf/google/type/month.proto b/karapace/protobuf/google/type/month.proto deleted file mode 100644 index 99e7551b1..000000000 --- a/karapace/protobuf/google/type/month.proto +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright 2021 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package google.type; - -option go_package = "google.golang.org/genproto/googleapis/type/month;month"; -option java_multiple_files = true; -option java_outer_classname = "MonthProto"; -option java_package = "com.google.type"; -option objc_class_prefix = "GTP"; - -// Represents a month in the Gregorian calendar. -enum Month { - // The unspecified month. - MONTH_UNSPECIFIED = 0; - - // The month of January. - JANUARY = 1; - - // The month of February. - FEBRUARY = 2; - - // The month of March. - MARCH = 3; - - // The month of April. - APRIL = 4; - - // The month of May. - MAY = 5; - - // The month of June. - JUNE = 6; - - // The month of July. - JULY = 7; - - // The month of August. - AUGUST = 8; - - // The month of September. - SEPTEMBER = 9; - - // The month of October. - OCTOBER = 10; - - // The month of November. - NOVEMBER = 11; - - // The month of December. - DECEMBER = 12; -} diff --git a/karapace/protobuf/google/type/phone_number.proto b/karapace/protobuf/google/type/phone_number.proto deleted file mode 100644 index 7bbb7d873..000000000 --- a/karapace/protobuf/google/type/phone_number.proto +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright 2021 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package google.type; - -option cc_enable_arenas = true; -option go_package = "google.golang.org/genproto/googleapis/type/phone_number;phone_number"; -option java_multiple_files = true; -option java_outer_classname = "PhoneNumberProto"; -option java_package = "com.google.type"; -option objc_class_prefix = "GTP"; - -// An object representing a phone number, suitable as an API wire format. -// -// This representation: -// -// - should not be used for locale-specific formatting of a phone number, such -// as "+1 (650) 253-0000 ext. 123" -// -// - is not designed for efficient storage -// - may not be suitable for dialing - specialized libraries (see references) -// should be used to parse the number for that purpose -// -// To do something meaningful with this number, such as format it for various -// use-cases, convert it to an `i18n.phonenumbers.PhoneNumber` object first. -// -// For instance, in Java this would be: -// -// com.google.type.PhoneNumber wireProto = -// com.google.type.PhoneNumber.newBuilder().build(); -// com.google.i18n.phonenumbers.Phonenumber.PhoneNumber phoneNumber = -// PhoneNumberUtil.getInstance().parse(wireProto.getE164Number(), "ZZ"); -// if (!wireProto.getExtension().isEmpty()) { -// phoneNumber.setExtension(wireProto.getExtension()); -// } -// -// Reference(s): -// - https://github.com/google/libphonenumber -message PhoneNumber { - // An object representing a short code, which is a phone number that is - // typically much shorter than regular phone numbers and can be used to - // address messages in MMS and SMS systems, as well as for abbreviated dialing - // (e.g. "Text 611 to see how many minutes you have remaining on your plan."). - // - // Short codes are restricted to a region and are not internationally - // dialable, which means the same short code can exist in different regions, - // with different usage and pricing, even if those regions share the same - // country calling code (e.g. US and CA). - message ShortCode { - // Required. The BCP-47 region code of the location where calls to this - // short code can be made, such as "US" and "BB". - // - // Reference(s): - // - http://www.unicode.org/reports/tr35/#unicode_region_subtag - string region_code = 1; - - // Required. The short code digits, without a leading plus ('+') or country - // calling code, e.g. "611". - string number = 2; - } - - // Required. Either a regular number, or a short code. New fields may be - // added to the oneof below in the future, so clients should ignore phone - // numbers for which none of the fields they coded against are set. - oneof kind { - // The phone number, represented as a leading plus sign ('+'), followed by a - // phone number that uses a relaxed ITU E.164 format consisting of the - // country calling code (1 to 3 digits) and the subscriber number, with no - // additional spaces or formatting, e.g.: - // - correct: "+15552220123" - // - incorrect: "+1 (555) 222-01234 x123". - // - // The ITU E.164 format limits the latter to 12 digits, but in practice not - // all countries respect that, so we relax that restriction here. - // National-only numbers are not allowed. - // - // References: - // - https://www.itu.int/rec/T-REC-E.164-201011-I - // - https://en.wikipedia.org/wiki/E.164. - // - https://en.wikipedia.org/wiki/List_of_country_calling_codes - string e164_number = 1; - - // A short code. - // - // Reference(s): - // - https://en.wikipedia.org/wiki/Short_code - ShortCode short_code = 2; - } - - // The phone number's extension. The extension is not standardized in ITU - // recommendations, except for being defined as a series of numbers with a - // maximum length of 40 digits. Other than digits, some other dialing - // characters such as ',' (indicating a wait) or '#' may be stored here. - // - // Note that no regions currently use extensions with short codes, so this - // field is normally only set in conjunction with an E.164 number. It is held - // separately from the E.164 number to allow for short code extensions in the - // future. - string extension = 3; -} diff --git a/karapace/protobuf/google/type/postal_address.proto b/karapace/protobuf/google/type/postal_address.proto deleted file mode 100644 index c57c7c31a..000000000 --- a/karapace/protobuf/google/type/postal_address.proto +++ /dev/null @@ -1,134 +0,0 @@ -// Copyright 2021 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package google.type; - -option cc_enable_arenas = true; -option go_package = "google.golang.org/genproto/googleapis/type/postaladdress;postaladdress"; -option java_multiple_files = true; -option java_outer_classname = "PostalAddressProto"; -option java_package = "com.google.type"; -option objc_class_prefix = "GTP"; - -// Represents a postal address, e.g. for postal delivery or payments addresses. -// Given a postal address, a postal service can deliver items to a premise, P.O. -// Box or similar. -// It is not intended to model geographical locations (roads, towns, -// mountains). -// -// In typical usage an address would be created via user input or from importing -// existing data, depending on the type of process. -// -// Advice on address input / editing: -// - Use an i18n-ready address widget such as -// https://github.com/google/libaddressinput) -// - Users should not be presented with UI elements for input or editing of -// fields outside countries where that field is used. -// -// For more guidance on how to use this schema, please see: -// https://support.google.com/business/answer/6397478 -message PostalAddress { - // The schema revision of the `PostalAddress`. This must be set to 0, which is - // the latest revision. - // - // All new revisions **must** be backward compatible with old revisions. - int32 revision = 1; - - // Required. CLDR region code of the country/region of the address. This - // is never inferred and it is up to the user to ensure the value is - // correct. See http://cldr.unicode.org/ and - // http://www.unicode.org/cldr/charts/30/supplemental/territory_information.html - // for details. Example: "CH" for Switzerland. - string region_code = 2; - - // Optional. BCP-47 language code of the contents of this address (if - // known). This is often the UI language of the input form or is expected - // to match one of the languages used in the address' country/region, or their - // transliterated equivalents. - // This can affect formatting in certain countries, but is not critical - // to the correctness of the data and will never affect any validation or - // other non-formatting related operations. - // - // If this value is not known, it should be omitted (rather than specifying a - // possibly incorrect default). - // - // Examples: "zh-Hant", "ja", "ja-Latn", "en". - string language_code = 3; - - // Optional. Postal code of the address. Not all countries use or require - // postal codes to be present, but where they are used, they may trigger - // additional validation with other parts of the address (e.g. state/zip - // validation in the U.S.A.). - string postal_code = 4; - - // Optional. Additional, country-specific, sorting code. This is not used - // in most regions. Where it is used, the value is either a string like - // "CEDEX", optionally followed by a number (e.g. "CEDEX 7"), or just a number - // alone, representing the "sector code" (Jamaica), "delivery area indicator" - // (Malawi) or "post office indicator" (e.g. Côte d'Ivoire). - string sorting_code = 5; - - // Optional. Highest administrative subdivision which is used for postal - // addresses of a country or region. - // For example, this can be a state, a province, an oblast, or a prefecture. - // Specifically, for Spain this is the province and not the autonomous - // community (e.g. "Barcelona" and not "Catalonia"). - // Many countries don't use an administrative area in postal addresses. E.g. - // in Switzerland this should be left unpopulated. - string administrative_area = 6; - - // Optional. Generally refers to the city/town portion of the address. - // Examples: US city, IT comune, UK post town. - // In regions of the world where localities are not well defined or do not fit - // into this structure well, leave locality empty and use address_lines. - string locality = 7; - - // Optional. Sublocality of the address. - // For example, this can be neighborhoods, boroughs, districts. - string sublocality = 8; - - // Unstructured address lines describing the lower levels of an address. - // - // Because values in address_lines do not have type information and may - // sometimes contain multiple values in a single field (e.g. - // "Austin, TX"), it is important that the line order is clear. The order of - // address lines should be "envelope order" for the country/region of the - // address. In places where this can vary (e.g. Japan), address_language is - // used to make it explicit (e.g. "ja" for large-to-small ordering and - // "ja-Latn" or "en" for small-to-large). This way, the most specific line of - // an address can be selected based on the language. - // - // The minimum permitted structural representation of an address consists - // of a region_code with all remaining information placed in the - // address_lines. It would be possible to format such an address very - // approximately without geocoding, but no semantic reasoning could be - // made about any of the address components until it was at least - // partially resolved. - // - // Creating an address only containing a region_code and address_lines, and - // then geocoding is the recommended way to handle completely unstructured - // addresses (as opposed to guessing which parts of the address should be - // localities or administrative areas). - repeated string address_lines = 9; - - // Optional. The recipient at the address. - // This field may, under certain circumstances, contain multiline information. - // For example, it might contain "care of" information. - repeated string recipients = 10; - - // Optional. The name of the organization at the address. - string organization = 11; -} diff --git a/karapace/protobuf/google/type/quaternion.proto b/karapace/protobuf/google/type/quaternion.proto deleted file mode 100644 index dfb822def..000000000 --- a/karapace/protobuf/google/type/quaternion.proto +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright 2021 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package google.type; - -option cc_enable_arenas = true; -option go_package = "google.golang.org/genproto/googleapis/type/quaternion;quaternion"; -option java_multiple_files = true; -option java_outer_classname = "QuaternionProto"; -option java_package = "com.google.type"; -option objc_class_prefix = "GTP"; - -// A quaternion is defined as the quotient of two directed lines in a -// three-dimensional space or equivalently as the quotient of two Euclidean -// vectors (https://en.wikipedia.org/wiki/Quaternion). -// -// Quaternions are often used in calculations involving three-dimensional -// rotations (https://en.wikipedia.org/wiki/Quaternions_and_spatial_rotation), -// as they provide greater mathematical robustness by avoiding the gimbal lock -// problems that can be encountered when using Euler angles -// (https://en.wikipedia.org/wiki/Gimbal_lock). -// -// Quaternions are generally represented in this form: -// -// w + xi + yj + zk -// -// where x, y, z, and w are real numbers, and i, j, and k are three imaginary -// numbers. -// -// Our naming choice `(x, y, z, w)` comes from the desire to avoid confusion for -// those interested in the geometric properties of the quaternion in the 3D -// Cartesian space. Other texts often use alternative names or subscripts, such -// as `(a, b, c, d)`, `(1, i, j, k)`, or `(0, 1, 2, 3)`, which are perhaps -// better suited for mathematical interpretations. -// -// To avoid any confusion, as well as to maintain compatibility with a large -// number of software libraries, the quaternions represented using the protocol -// buffer below *must* follow the Hamilton convention, which defines `ij = k` -// (i.e. a right-handed algebra), and therefore: -// -// i^2 = j^2 = k^2 = ijk = −1 -// ij = −ji = k -// jk = −kj = i -// ki = −ik = j -// -// Please DO NOT use this to represent quaternions that follow the JPL -// convention, or any of the other quaternion flavors out there. -// -// Definitions: -// -// - Quaternion norm (or magnitude): `sqrt(x^2 + y^2 + z^2 + w^2)`. -// - Unit (or normalized) quaternion: a quaternion whose norm is 1. -// - Pure quaternion: a quaternion whose scalar component (`w`) is 0. -// - Rotation quaternion: a unit quaternion used to represent rotation. -// - Orientation quaternion: a unit quaternion used to represent orientation. -// -// A quaternion can be normalized by dividing it by its norm. The resulting -// quaternion maintains the same direction, but has a norm of 1, i.e. it moves -// on the unit sphere. This is generally necessary for rotation and orientation -// quaternions, to avoid rounding errors: -// https://en.wikipedia.org/wiki/Rotation_formalisms_in_three_dimensions -// -// Note that `(x, y, z, w)` and `(-x, -y, -z, -w)` represent the same rotation, -// but normalization would be even more useful, e.g. for comparison purposes, if -// it would produce a unique representation. It is thus recommended that `w` be -// kept positive, which can be achieved by changing all the signs when `w` is -// negative. -// -message Quaternion { - // The x component. - double x = 1; - - // The y component. - double y = 2; - - // The z component. - double z = 3; - - // The scalar component. - double w = 4; -} diff --git a/karapace/protobuf/google/type/timeofday.proto b/karapace/protobuf/google/type/timeofday.proto deleted file mode 100644 index 5cb48aa93..000000000 --- a/karapace/protobuf/google/type/timeofday.proto +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright 2021 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package google.type; - -option cc_enable_arenas = true; -option go_package = "google.golang.org/genproto/googleapis/type/timeofday;timeofday"; -option java_multiple_files = true; -option java_outer_classname = "TimeOfDayProto"; -option java_package = "com.google.type"; -option objc_class_prefix = "GTP"; - -// Represents a time of day. The date and time zone are either not significant -// or are specified elsewhere. An API may choose to allow leap seconds. Related -// types are [google.type.Date][google.type.Date] and -// `google.protobuf.Timestamp`. -message TimeOfDay { - // Hours of day in 24 hour format. Should be from 0 to 23. An API may choose - // to allow the value "24:00:00" for scenarios like business closing time. - int32 hours = 1; - - // Minutes of hour of day. Must be from 0 to 59. - int32 minutes = 2; - - // Seconds of minutes of the time. Must normally be from 0 to 59. An API may - // allow the value 60 if it allows leap-seconds. - int32 seconds = 3; - - // Fractions of seconds in nanoseconds. Must be from 0 to 999,999,999. - int32 nanos = 4; -} diff --git a/karapace/protobuf/google/type/type.yaml b/karapace/protobuf/google/type/type.yaml deleted file mode 100644 index d5c71364d..000000000 --- a/karapace/protobuf/google/type/type.yaml +++ /dev/null @@ -1,40 +0,0 @@ -type: google.api.Service -config_version: 3 -name: type.googleapis.com -title: Common Types - -types: -- name: google.type.Color -- name: google.type.Date -- name: google.type.DateTime -- name: google.type.Decimal -- name: google.type.Expr -- name: google.type.Fraction -- name: google.type.Interval -- name: google.type.LatLng -- name: google.type.LocalizedText -- name: google.type.Money -- name: google.type.PhoneNumber -- name: google.type.PostalAddress -- name: google.type.Quaternion -- name: google.type.TimeOfDay - -enums: -- name: google.type.CalendarPeriod -- name: google.type.DayOfWeek -- name: google.type.Month - -documentation: - summary: Defines common types for Google APIs. - overview: |- - # Google Common Types - - This package contains definitions of common types for Google APIs. - All types defined in this package are suitable for different APIs to - exchange data, and will never break binary compatibility. They should - have design quality comparable to major programming languages like - Java and C#. - - NOTE: Some common types are defined in the package `google.protobuf` - as they are directly supported by Protocol Buffers compiler and - runtime. Those types are called Well-Known Types. From 984d794b73f308b762965eea3ddabbcd2e5116a3 Mon Sep 17 00:00:00 2001 From: Jarkko Jaakola Date: Thu, 20 Oct 2022 12:09:05 +0300 Subject: [PATCH 07/25] WIP: Add tests and fixes for protobuf references support --- karapace/dependency.py | 40 +- karapace/errors.py | 9 +- karapace/protobuf/field_element.py | 7 + karapace/protobuf/known_dependency.py | 126 +++--- karapace/protobuf/proto_file_element.py | 25 +- karapace/protobuf/schema.py | 32 +- karapace/schema_models.py | 86 ++-- karapace/schema_reader.py | 133 +++--- karapace/schema_references.py | 39 +- karapace/schema_registry.py | 41 +- karapace/schema_registry_apis.py | 117 ++--- karapace/serialization.py | 10 +- tests/integration/test_schema_protobuf.py | 514 +++++++++++++++++++++- 13 files changed, 888 insertions(+), 291 deletions(-) diff --git a/karapace/dependency.py b/karapace/dependency.py index 19736c02f..3cc2a6ff0 100644 --- a/karapace/dependency.py +++ b/karapace/dependency.py @@ -1,21 +1,47 @@ -from typing import Optional, TYPE_CHECKING +from karapace.schema_references import Reference +from karapace.typing import JsonData, Subject, Version +from typing import Any, Optional, TYPE_CHECKING if TYPE_CHECKING: from karapace.schema_models import ValidatedTypedSchema +class DependencyVerifierResult: + def __init__(self, result: bool, message: Optional[str] = "") -> None: + self.result = result + self.message = message + + class Dependency: - def __init__(self, name: str, subject: str, version: int, schema: "ValidatedTypedSchema") -> None: + def __init__(self, name: str, subject: Subject, version: Version, target_schema: "ValidatedTypedSchema") -> None: self.name = name self.subject = subject self.version = version - self.schema = schema + self.schema = target_schema + + @staticmethod + def of(reference: Reference, target_schema: "ValidatedTypedSchema") -> "Dependency": + return Dependency(reference.name, reference.subject, reference.version, target_schema) + + def to_dict(self) -> JsonData: + return { + "name": self.name, + "subject": self.subject, + "version": self.version, + } def identifier(self) -> str: return self.name + "_" + self.subject + "_" + str(self.version) + def __hash__(self) -> int: + return hash((self.name, self.subject, self.version, self.schema)) -class DependencyVerifierResult: - def __init__(self, result: bool, message: Optional[str] = ""): - self.result = result - self.message = message + def __eq__(self, other: Any) -> bool: + if other is None or not isinstance(other, Dependency): + return False + return ( + self.name == other.name + and self.subject == other.subject + and self.version == other.version + and self.schema == other.schema + ) diff --git a/karapace/errors.py b/karapace/errors.py index 188da45d0..4a2eb138d 100644 --- a/karapace/errors.py +++ b/karapace/errors.py @@ -1,4 +1,5 @@ -from typing import List, Union +from karapace.typing import Version +from typing import List class VersionNotFoundException(Exception): @@ -25,10 +26,6 @@ class InvalidReferences(Exception): pass -class ReferencesNotSupportedException(Exception): - pass - - class SchemasNotFoundException(Exception): pass @@ -50,7 +47,7 @@ class SubjectNotSoftDeletedException(Exception): class ReferenceExistsException(Exception): - def __init__(self, referenced_by: List, version: Union[int, str]): + def __init__(self, referenced_by: List, version: Version): super().__init__() self.version = version self.referenced_by = referenced_by diff --git a/karapace/protobuf/field_element.py b/karapace/protobuf/field_element.py index 9cc247fbd..4816b2728 100644 --- a/karapace/protobuf/field_element.py +++ b/karapace/protobuf/field_element.py @@ -143,6 +143,7 @@ def compare_message( self_type_record = types.get_self_type(self_type) other_type_record = types.get_other_type(other_type) + self_type_element: MessageElement = self_type_record.type_element other_type_element: MessageElement = other_type_record.type_element @@ -150,3 +151,9 @@ def compare_message( result.add_modification(Modification.FIELD_NAME_ALTER) else: self_type_element.compare(other_type_element, result, types) + + def __repr__(self): + return f"{self.element_type} {self.name} = {self.tag}" + + def __str__(self): + return f"{self.element_type} {self.name} = {self.tag}" diff --git a/karapace/protobuf/known_dependency.py b/karapace/protobuf/known_dependency.py index 5c3e74a00..0ba9559f7 100644 --- a/karapace/protobuf/known_dependency.py +++ b/karapace/protobuf/known_dependency.py @@ -45,77 +45,77 @@ class KnownDependency: index: Dict = dict() index_simple: Dict = dict() map: Dict = { - "google/protobuf/any.proto": [".google.protobuf.Any"], - "google/protobuf/api.proto": [".google.protobuf.Api", ".google.protobuf.Method", ".google.protobuf.Mixin"], + "google/protobuf/any.proto": ["google.protobuf.Any"], + "google/protobuf/api.proto": ["google.protobuf.Api", "google.protobuf.Method", "google.protobuf.Mixin"], "google/protobuf/descriptor.proto": [ - ".google.protobuf.FileDescriptorSet", - ".google.protobuf.FileDescriptorProto", - ".google.protobuf.DescriptorProto", - ".google.protobuf.ExtensionRangeOptions", - ".google.protobuf.FieldDescriptorProto", - ".google.protobuf.OneofDescriptorProto", - ".google.protobuf.EnumDescriptorProto", - ".google.protobuf.EnumValueDescriptorProto", - ".google.protobuf.ServiceDescriptorProto", - ".google.protobuf.MethodDescriptorProto", - ".google.protobuf.FileOptions", - ".google.protobuf.MessageOptions", - ".google.protobuf.FieldOptions", - ".google.protobuf.OneofOptions", - ".google.protobuf.EnumOptions", - ".google.protobuf.EnumValueOptions", - ".google.protobuf.ServiceOptions", - ".google.protobuf.MethodOptions", - ".google.protobuf.UninterpretedOption", - ".google.protobuf.SourceCodeInfo", - ".google.protobuf.GeneratedCodeInfo", + "google.protobuf.FileDescriptorSet", + "google.protobuf.FileDescriptorProto", + "google.protobuf.DescriptorProto", + "google.protobuf.ExtensionRangeOptions", + "google.protobuf.FieldDescriptorProto", + "google.protobuf.OneofDescriptorProto", + "google.protobuf.EnumDescriptorProto", + "google.protobuf.EnumValueDescriptorProto", + "google.protobuf.ServiceDescriptorProto", + "google.protobuf.MethodDescriptorProto", + "google.protobuf.FileOptions", + "google.protobuf.MessageOptions", + "google.protobuf.FieldOptions", + "google.protobuf.OneofOptions", + "google.protobuf.EnumOptions", + "google.protobuf.EnumValueOptions", + "google.protobuf.ServiceOptions", + "google.protobuf.MethodOptions", + "google.protobuf.UninterpretedOption", + "google.protobuf.SourceCodeInfo", + "google.protobuf.GeneratedCodeInfo", ], - "google/protobuf/duration.proto": [".google.protobuf.Duration"], - "google/protobuf/empty.proto": [".google.protobuf.Empty"], - "google/protobuf/field_mask.proto": [".google.protobuf.FieldMask"], - "google/protobuf/source_context.proto": [".google.protobuf.SourceContext"], + "google/protobuf/duration.proto": ["google.protobuf.Duration"], + "google/protobuf/empty.proto": ["google.protobuf.Empty"], + "google/protobuf/field_mask.proto": ["google.protobuf.FieldMask"], + "google/protobuf/source_context.proto": ["google.protobuf.SourceContext"], "google/protobuf/struct.proto": [ - ".google.protobuf.Struct", - ".google.protobuf.Value", - ".google.protobuf.NullValue", - ".google.protobuf.ListValue", + "google.protobuf.Struct", + "google.protobuf.Value", + "google.protobuf.NullValue", + "google.protobuf.ListValue", ], - "google/protobuf/timestamp.proto": [".google.protobuf.Timestamp"], + "google/protobuf/timestamp.proto": ["google.protobuf.Timestamp"], "google/protobuf/type.proto": [ - ".google.protobuf.Type", - ".google.protobuf.Field", - ".google.protobuf.Enum", - ".google.protobuf.EnumValue", - ".google.protobuf.Option", - ".google.protobuf.Syntax", + "google.protobuf.Type", + "google.protobuf.Field", + "google.protobuf.Enum", + "google.protobuf.EnumValue", + "google.protobuf.Option", + "google.protobuf.Syntax", ], "google/protobuf/wrappers.proto": [ - ".google.protobuf.DoubleValue", - ".google.protobuf.FloatValue", - ".google.protobuf.Int64Value", - ".google.protobuf.UInt64Value", - ".google.protobuf.Int32Value", - ".google.protobuf.UInt32Value", - ".google.protobuf.BoolValue", - ".google.protobuf.StringValue", - ".google.protobuf.BytesValue", + "google.protobuf.DoubleValue", + "google.protobuf.FloatValue", + "google.protobuf.Int64Value", + "google.protobuf.UInt64Value", + "google.protobuf.Int32Value", + "google.protobuf.UInt32Value", + "google.protobuf.BoolValue", + "google.protobuf.StringValue", + "google.protobuf.BytesValue", ], - "google/type/calendar_period.proto": [".google.type.CalendarPeriod"], - "google/type/color.proto": [".google.type.Color"], - "google/type/date.proto": [".google.type.Date"], - "google/type/datetime.proto": [".google.type.DateTime", ".google.type.TimeZone"], - "google/type/dayofweek.proto": [".google.type.DayOfWeek"], - "google/type/decimal.proto": [".google.type.Decimal"], - "google/type/expr.proto": [".google.type.Expr"], - "google/type/fraction.proto": [".google.type.Fraction"], - "google/type/interval.proto": [".google.type.Interval"], - "google/type/latlng.proto": [".google.type.LatLng"], - "google/type/money.proto": [".google.type.Money"], - "google/type/month.proto": [".google.type.Month"], - "google/type/phone_number.proto": [".google.type.PhoneNumber"], - "google/type/postal_address.proto": [".google.type.PostalAddress"], - "google/type/quaternion.proto": [".google.type.Quaternion"], - "google/type/timeofday.proto": [".google.type.TimeOfDay"], + "google/type/calendar_period.proto": ["google.type.CalendarPeriod"], + "google/type/color.proto": ["google.type.Color"], + "google/type/date.proto": ["google.type.Date"], + "google/type/datetime.proto": ["google.type.DateTime", "google.type.TimeZone"], + "google/type/dayofweek.proto": ["google.type.DayOfWeek"], + "google/type/decimal.proto": ["google.type.Decimal"], + "google/type/expr.proto": ["google.type.Expr"], + "google/type/fraction.proto": ["google.type.Fraction"], + "google/type/interval.proto": ["google.type.Interval"], + "google/type/latlng.proto": ["google.type.LatLng"], + "google/type/money.proto": ["google.type.Money"], + "google/type/month.proto": ["google.type.Month"], + "google/type/phone_number.proto": ["google.type.PhoneNumber"], + "google/type/postal_address.proto": ["google.type.PostalAddress"], + "google/type/quaternion.proto": ["google.type.Quaternion"], + "google/type/timeofday.proto": ["google.type.TimeOfDay"], "confluent/meta.proto": [".confluent.Meta"], "confluent/type/decimal.proto": [".confluent.type.Decimal"], } diff --git a/karapace/protobuf/proto_file_element.py b/karapace/protobuf/proto_file_element.py index fa43dd9e8..f227d63c0 100644 --- a/karapace/protobuf/proto_file_element.py +++ b/karapace/protobuf/proto_file_element.py @@ -99,8 +99,13 @@ def __eq__(self, other: "ProtoFileElement") -> bool: # type: ignore def __repr__(self) -> str: return self.to_schema() - def compare(self, other: "ProtoFileElement", result: CompareResult) -> CompareResult: - + def compare( + self, + other: "ProtoFileElement", + result: CompareResult, + self_dependency_types: Optional[List[TypeElement]] = None, + other_dependency_types: Optional[List[TypeElement]] = None, + ) -> CompareResult: if self.package_name != other.package_name: result.add_modification(Modification.PACKAGE_ALTER) # TODO: do we need syntax check? @@ -125,6 +130,22 @@ def compare(self, other: "ProtoFileElement", result: CompareResult) -> CompareRe package_name = other.package_name or "" compare_types.add_other_type(package_name, type_) + # If there are dependencies declared, add the types for both. + if self_dependency_types: + for i, type_ in enumerate(self_dependency_types): + package_name = "" + + self_types[type_.name] = type_ + self_indexes[type_.name] = i + compare_types.add_self_type(package_name, type_) + + if other_dependency_types: + for i, type_ in enumerate(other_dependency_types): + package_name = "" + other_types[type_.name] = type_ + other_indexes[type_.name] = i + compare_types.add_other_type(package_name, type_) + for name in chain(self_types.keys(), other_types.keys() - self_types.keys()): result.push_path(str(name), True) diff --git a/karapace/protobuf/schema.py b/karapace/protobuf/schema.py index 76e319019..e85b3df35 100644 --- a/karapace/protobuf/schema.py +++ b/karapace/protobuf/schema.py @@ -14,8 +14,7 @@ from karapace.protobuf.proto_parser import ProtoParser from karapace.protobuf.type_element import TypeElement from karapace.protobuf.utils import append_documentation, append_indented -from karapace.schema_references import References -from typing import Dict, Optional +from typing import List, Optional def add_slashes(text: str) -> str: @@ -106,15 +105,12 @@ def option_element_string(option: OptionElement) -> str: class ProtobufSchema: DEFAULT_LOCATION = Location.get("") - def __init__( - self, schema: str, references: Optional[References] = None, dependencies: Optional[Dict[str, Dependency]] = None - ) -> None: + def __init__(self, schema: str, dependencies: Optional[List[Dependency]] = None) -> None: if type(schema).__name__ != "str": raise IllegalArgumentException("Non str type of schema string") self.dirty = schema self.cache_string = "" self.proto_file_element = ProtoParser.parse(self.DEFAULT_LOCATION, schema) - self.references = references self.dependencies = dependencies def gather_deps(self) -> ProtobufDependencyVerifier: @@ -128,10 +124,9 @@ def verify_schema_dependencies(self) -> DependencyVerifierResult: return verifier.verify() def collect_dependencies(self, verifier: ProtobufDependencyVerifier): - if self.dependencies: - for key in self.dependencies: - self.dependencies[key].schema.schema.collect_dependencies(verifier) + for dependency in self.dependencies: + dependency.schema.schema.collect_dependencies(verifier) # verifier.add_import?? we have no access to own Kafka structure from this class... # but we need data to analyse imports to avoid ciclyc dependencies... @@ -223,4 +218,21 @@ def to_schema(self) -> str: return "".join(strings) def compare(self, other: "ProtobufSchema", result: CompareResult) -> CompareResult: - self.proto_file_element.compare(other.proto_file_element, result) + self_dependency_types: List[TypeElement] = [] + other_dependency_types: List[TypeElement] = [] + if self.dependencies: + for dependency in self.dependencies: + schema = dependency.schema.schema + if schema.proto_file_element.types: + self_dependency_types += schema.proto_file_element.types + if other.dependencies: + for dependency in other.dependencies: + schema = dependency.schema.schema + if schema.proto_file_element.types: + other_dependency_types += schema.proto_file_element.types + self.proto_file_element.compare( + other.proto_file_element, + result, + self_dependency_types=self_dependency_types, + other_dependency_types=other_dependency_types, + ) diff --git a/karapace/schema_models.py b/karapace/schema_models.py index 7448c4a8f..8ad8b43fc 100644 --- a/karapace/schema_models.py +++ b/karapace/schema_models.py @@ -2,6 +2,7 @@ from avro.schema import parse as avro_parse, Schema as AvroSchema from jsonschema import Draft7Validator from jsonschema.exceptions import SchemaError +from karapace.dependency import Dependency from karapace.errors import InvalidSchema from karapace.protobuf.exception import ( Error as ProtobufError, @@ -12,11 +13,10 @@ ProtobufUnresolvedDependencyException, SchemaParseException as ProtobufSchemaParseException, ) -from karapace.protobuf.schema import Dependency, ProtobufSchema -from karapace.schema_references import References +from karapace.protobuf.schema import ProtobufSchema from karapace.schema_type import SchemaType from karapace.utils import json_encode -from typing import Any, Dict, Optional, Union +from typing import Any, Dict, List, Optional, Union import json @@ -52,7 +52,9 @@ def parse_jsonschema_definition(schema_definition: str) -> Draft7Validator: def parse_protobuf_schema_definition( - schema_definition: str, references: Optional[References] = None, dependencies: Optional[Dict[str, Dependency]] = None + schema_definition: str, + references: Optional[List[Dependency]] = None, + validate_references: bool = True, ) -> ProtobufSchema: """Parses and validates `schema_definition`. @@ -60,34 +62,38 @@ def parse_protobuf_schema_definition( Nothing yet. """ - protobuf_schema = ProtobufSchema(schema_definition, references, dependencies) - result = protobuf_schema.verify_schema_dependencies() - if not result.result: - raise ProtobufUnresolvedDependencyException(f"{result.message}") + protobuf_schema = ProtobufSchema(schema_definition, references) + if validate_references: + result = protobuf_schema.verify_schema_dependencies() + if not result.result: + raise ProtobufUnresolvedDependencyException(f"{result.message}") return protobuf_schema class TypedSchema: def __init__( self, + *, schema_type: SchemaType, schema_str: str, - references: Optional[References] = None, - dependencies: Optional[Dict[str, Dependency]] = None, + schema: Optional[Union[Draft7Validator, AvroSchema, ProtobufSchema]] = None, + references: Optional[List[Dependency]] = None, ): """Schema with type information Args: schema_type (SchemaType): The type of the schema schema_str (str): The original schema string - references(References): The references of schema + schema (Optional[Union[Draft7Validator, AvroSchema, ProtobufSchema]]): The parsed and validated schema + references (Optional[List[Dependency]]): The references of schema """ self.schema_type = schema_type self.schema_str = schema_str self.references = references - self.dependencies = dependencies self.max_id: Optional[int] = None + self._str_cached: Optional[str] = None + self._schema_cached: Optional[Union[Draft7Validator, AvroSchema, ProtobufSchema]] = schema def to_dict(self) -> Dict[str, Any]: if self.schema_type is SchemaType.PROTOBUF: @@ -105,7 +111,43 @@ def __str__(self) -> str: def __repr__(self) -> str: return f"TypedSchema(type={self.schema_type}, schema={str(self)})" - def get_references(self) -> Optional["References"]: + @property + def schema(self) -> Union[Draft7Validator, AvroSchema, ProtobufSchema]: + if self._schema_cached is not None: + return self._schema_cached + if self.schema_type is SchemaType.AVRO: + try: + self._schema_cached = parse_avro_schema_definition(self.schema_str) + except (SchemaParseException, json.JSONDecodeError, TypeError) as e: + raise InvalidSchema from e + + elif self.schema_type is SchemaType.JSONSCHEMA: + try: + self._schema_cached = parse_jsonschema_definition(self.schema_str) + # TypeError - Raised when the user forgets to encode the schema as a string. + except (TypeError, json.JSONDecodeError, SchemaError, AssertionError) as e: + raise InvalidSchema from e + + elif self.schema_type is SchemaType.PROTOBUF: + try: + self._schema_cached = parse_protobuf_schema_definition(self.schema_str, self.references) + except ( + TypeError, + SchemaError, + AssertionError, + ProtobufParserRuntimeException, + IllegalStateException, + IllegalArgumentException, + ProtobufError, + ProtobufException, + ProtobufSchemaParseException, + ) as e: + raise InvalidSchema from e + else: + raise InvalidSchema(f"Unknown parser {self.schema_type} for {self.schema_str}") + return self._schema_cached + + def get_references(self) -> Optional[List[Dependency]]: return self.references def __eq__(self, other: Any) -> bool: @@ -114,9 +156,7 @@ def __eq__(self, other: Any) -> bool: ) if not schema_is_equal: return False - if self.references is not None: - return self.references == other.references - return other.references is None + return self.references == other.references class ValidatedTypedSchema(TypedSchema): @@ -125,18 +165,15 @@ def __init__( schema_type: SchemaType, schema_str: str, schema: Union[Draft7Validator, AvroSchema, ProtobufSchema], - references: Optional["References"] = None, - dependencies: Optional[Dict[str, Dependency]] = None, + references: Optional[List[Dependency]] = None, ): - super().__init__(schema_type=schema_type, schema_str=schema_str, references=references, dependencies=dependencies) - self.schema = schema + super().__init__(schema_type=schema_type, schema_str=schema_str, references=references, schema=schema) @staticmethod def parse( schema_type: SchemaType, schema_str: str, - references: Optional["References"] = None, - dependencies: Optional[Dict[str, Dependency]] = None, + references: Optional[List[Dependency]] = None, ) -> "ValidatedTypedSchema": if schema_type not in [SchemaType.AVRO, SchemaType.JSONSCHEMA, SchemaType.PROTOBUF]: raise InvalidSchema(f"Unknown parser {schema_type} for {schema_str}") @@ -157,14 +194,14 @@ def parse( elif schema_type is SchemaType.PROTOBUF: try: - parsed_schema = parse_protobuf_schema_definition(schema_str, references, dependencies) + parsed_schema = parse_protobuf_schema_definition(schema_str, references) except ( TypeError, SchemaError, AssertionError, ProtobufParserRuntimeException, - IllegalStateException, IllegalArgumentException, + IllegalStateException, ProtobufError, ProtobufException, ProtobufSchemaParseException, @@ -178,7 +215,6 @@ def parse( schema_str=schema_str, schema=parsed_schema, references=references, - dependencies=dependencies, ) def __str__(self) -> str: diff --git a/karapace/schema_reader.py b/karapace/schema_reader.py index 01953f3a7..1274a97f6 100644 --- a/karapace/schema_reader.py +++ b/karapace/schema_reader.py @@ -11,17 +11,18 @@ from karapace import constants from karapace.config import Config from karapace.dependency import Dependency -from karapace.errors import InvalidReferences, InvalidSchema, ReferencesNotSupportedException +from karapace.errors import InvalidReferences, InvalidSchema from karapace.key_format import is_key_in_canonical_format, KeyFormatter, KeyMode from karapace.master_coordinator import MasterCoordinator -from karapace.schema_models import SchemaType, TypedSchema, ValidatedTypedSchema -from karapace.schema_references import References +from karapace.schema_models import parse_protobuf_schema_definition, SchemaType, TypedSchema +from karapace.schema_references import Reference from karapace.statsd import StatsClient -from karapace.typing import SubjectData +from karapace.typing import JsonData, SubjectData from karapace.utils import KarapaceKafkaClient, reference_key from threading import Condition, Event, Lock, Thread -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Dict, List, Optional +import hashlib import json import logging @@ -314,6 +315,7 @@ def handle_messages(self) -> None: schema_records_processed_keymode_canonical, schema_records_processed_keymode_deprecated_karapace, ) + self.log_state() def _report_schema_metrics( self, @@ -432,6 +434,8 @@ def _handle_msg_schema(self, key: dict, value: Optional[dict]) -> None: schema_deleted = value.get("deleted", False) schema_references = value.get("references", None) + resolved_references = None + try: schema_type_parsed = SchemaType(schema_type) except ValueError: @@ -452,10 +456,15 @@ def _handle_msg_schema(self, key: dict, value: Optional[dict]) -> None: return elif schema_type_parsed == SchemaType.PROTOBUF: try: - resolved_references, resolved_dependencies = self.resolve_schema_references(value) - parsed_schema = ValidatedTypedSchema.parse( - SchemaType.PROTOBUF, schema_str, resolved_references, resolved_dependencies - ) + references = None + if schema_references: + references = [ + Reference(reference["name"], reference["subject"], reference["version"]) + for reference in schema_references + ] + if references: + resolved_references = self.resolve_references(references) + parsed_schema = parse_protobuf_schema_definition(schema_str, resolved_references, validate_references=False) schema_str = str(parsed_schema) except InvalidSchema: LOG.exception("Schema is not valid ProtoBuf definition") @@ -474,7 +483,7 @@ def _handle_msg_schema(self, key: dict, value: Optional[dict]) -> None: typed_schema = TypedSchema( schema_type=schema_type_parsed, schema_str=schema_str, - references=schema_references, + references=resolved_references, ) schema = { "schema": typed_schema, @@ -511,6 +520,7 @@ def handle_msg(self, key: dict, value: Optional[dict]) -> None: if key["keytype"] == "CONFIG": self._handle_msg_config(key, value) elif key["keytype"] == "SCHEMA": + LOG.error("HANDLING SCHEMA MESSAGE") self._handle_msg_schema(key, value) elif key["keytype"] == "DELETE_SUBJECT": self._handle_msg_delete_subject(key, value) @@ -558,55 +568,60 @@ def remove_referenced_by(self, schema_id: SchemaId, references: List): if self.referenced_by.get(key, None) and schema_id in self.referenced_by[key]: self.referenced_by[key].remove(schema_id) - def resolve_references(self, references: Optional["References"] = None) -> Optional[Dict[str, Dependency]]: - if references is None: - return None - dependencies = dict() - - for r in references.val(): - subject = r["subject"] - version = r["version"] - name = r["name"] - subject_data = self.subjects.get(subject) - if subject_data is not None: - schema_data = subject_data["schemas"][version] - schema_references, schema_dependencies = self.resolve_schema_references(schema_data) + def _resolve_reference(self, reference: Reference) -> List[Dependency]: + subject_data = self.subjects.get(reference.subject) + if not subject_data: + raise InvalidReferences(f"Subject not found {reference.subject}.") + schema = subject_data["schemas"].get(reference.version, {}).get("schema", None) + if not schema: + raise InvalidReferences(f"No schema in {reference.subject} with version {reference.version}.") + + resolved_references = [Dependency.of(reference, schema)] + if schema.references: + resolved_references += self.resolve_references(schema.references) + return resolved_references + + def resolve_references(self, references: List[Reference]) -> List[Dependency]: + resolved_references = [] + for reference in references: + resolved_references += self._resolve_reference(reference) + return resolved_references + + def _build_state_dict(self) -> JsonData: + state = {"schemas": [], "subjects": {}} + for schema_id, schema in self.schemas.items(): + if schema.schema_type == SchemaType.AVRO: + schema_str = json.dumps(json.loads(schema.schema_str), sort_keys=True) else: - raise InvalidReferences - - parsed_schema = ValidatedTypedSchema.parse( - schema_type=schema_data["schema"].schema_type, - schema_str=schema_data["schema"].schema_str, - references=schema_references, - dependencies=schema_dependencies, + schema_str = schema.schema_str + schema_hash = hashlib.sha1(schema_str.encode("utf8")).hexdigest() + state["schemas"].append( + { + "id": schema_id, + "schema_hash": schema_hash, + } ) - dependencies[name] = Dependency(name, subject, version, parsed_schema) - return dependencies - - def resolve_schema_references( - self, schema_data: Optional[dict] - ) -> Tuple[Optional[References], Optional[Dict[str, Dependency]]]: - - if schema_data is None: - raise InvalidSchema - - schema_references = schema_data.get("references") - if schema_references is None: - return None, None - - schema_type = schema_data.get("schemaType") - if schema_type is None: - schema = schema_data.get("schema") - if schema is None: - raise InvalidReferences - if isinstance(schema, TypedSchema): - schema_type = schema.schema_type - else: - schema_type = None - if schema_type != SchemaType.PROTOBUF: - raise ReferencesNotSupportedException - - schema_references = References(schema_type, schema_references) - schema_dependencies = self.resolve_references(schema_references) + for subject, subject_data in self.subjects.items(): + subject_state = [] + state["subjects"][subject] = subject_state + for _, schema in subject_data.get("schemas", {}).items(): + schema_id = schema.get("id") + version = schema.get("version") + if schema.get("schema").schema_type == SchemaType.AVRO: + schema_str = json.dumps(json.loads(schema.get("schema").schema_str), sort_keys=True) + else: + schema_str = schema.get("schema").schema_str + schema_hash = hashlib.sha1(schema_str.encode("utf8")).hexdigest() + subject_state.append( + { + "id": schema_id, + "version": version, + "schema_hash": schema_hash, + "references": schema.get("references"), + } + ) + return json.dumps(state, sort_keys=True, indent=2) - return schema_references, schema_dependencies + def log_state(self) -> None: + state_str = self._build_state_dict() + LOG.log(level=100, msg=state_str) diff --git a/karapace/schema_references.py b/karapace/schema_references.py index 6663a3d12..46823fb4c 100644 --- a/karapace/schema_references.py +++ b/karapace/schema_references.py @@ -1,27 +1,30 @@ -from karapace.schema_type import SchemaType -from karapace.typing import JsonData -from karapace.utils import json_encode +from karapace.typing import JsonData, Subject, Version from typing import Any -class References: - def __init__(self, schema_type: SchemaType, references: JsonData): - """Schema with type information +class Reference: + def __init__(self, name: str, subject: Subject, version: Version): + self.name = name + self.subject = subject + self.version = version - Args: - schema_type (SchemaType): The type of the schema - references (str): The original schema string - """ - self.schema_type = schema_type - self.references = references + def identifier(self) -> str: + return self.name + "_" + self.subject + "_" + str(self.version) - def val(self) -> JsonData: - return self.references + def to_dict(self) -> JsonData: + return { + "name": self.name, + "subject": self.subject, + "version": self.version, + } - def json(self) -> str: - return str(json_encode(self.references, sort_keys=True)) + def __repr__(self) -> str: + return f"{{name='{self.name}', subject='{self.subject}', version={self.version}}}" + + def __hash__(self) -> int: + return hash((self.name, self.subject, self.version)) def __eq__(self, other: Any) -> bool: - if other is None or not isinstance(other, References): + if other is None or not isinstance(other, Reference): return False - return self.json() == other.json() + return self.name == other.name and self.subject == other.subject and self.version == other.version diff --git a/karapace/schema_registry.py b/karapace/schema_registry.py index 6317b36c2..eddc6eee6 100644 --- a/karapace/schema_registry.py +++ b/karapace/schema_registry.py @@ -22,7 +22,7 @@ from karapace.master_coordinator import MasterCoordinator from karapace.schema_models import SchemaType, TypedSchema, ValidatedTypedSchema from karapace.schema_reader import KafkaSchemaReader -from karapace.schema_references import References +from karapace.schema_references import Reference from karapace.typing import ResolvedVersion, Subject, SubjectData, Version from karapace.utils import json_encode, KarapaceKafkaClient, reference_key from typing import Dict, List, Optional, Tuple, Union @@ -306,7 +306,7 @@ async def write_new_schema_local( self, subject: Subject, new_schema: ValidatedTypedSchema, - new_schema_references: Optional[References], + new_schema_references: Optional[List[Dependency]], ) -> int: """Write new schema and return new id or return id of matching existing schema @@ -368,18 +368,8 @@ async def write_new_schema_local( for old_version in check_against: old_schema = subject_data["schemas"][old_version]["schema"] - old_schema_references, old_schema_dependencies = self.resolve_schema_references( - subject_data["schemas"][old_version], - ) - - validated_old_schema = ValidatedTypedSchema.parse( - schema_type=old_schema.schema_type, - schema_str=old_schema.schema_str, - references=old_schema_references, - dependencies=old_schema_dependencies, - ) result = check_compatibility( - old_schema=validated_old_schema, + old_schema=old_schema, new_schema=new_schema, compatibility_mode=compatibility_mode, ) @@ -480,7 +470,7 @@ def send_schema_message( schema_id: int, version: int, deleted: bool, - references: Optional[References], + references: Optional[List[Dependency]], ) -> FutureRecordMetadata: key = self.key_formatter.format_key( {"subject": subject, "version": version, "magic": 1, "keytype": "SCHEMA"}, @@ -494,7 +484,7 @@ def send_schema_message( "deleted": deleted, } if references: - valuedict["references"] = references.val() + valuedict["references"] = [reference.to_dict() for reference in references] if schema.schema_type is not SchemaType.AVRO: valuedict["schemaType"] = schema.schema_type value = json_encode(valuedict) @@ -536,10 +526,19 @@ def send_delete_subject_message(self, subject: Subject, version: Version) -> Fut value = '{{"subject":"{}","version":{}}}'.format(subject, version) return self.send_kafka_message(key, value) - def resolve_references(self, references: Optional["References"] = None) -> Optional[Dict[str, Dependency]]: - return self.schema_reader.resolve_references(references) + def resolve_references(self, references: Optional[List[Reference]]) -> Optional[List[Dependency]]: + if references: + return self.schema_reader.resolve_references(references) + return None + + +# def resolve_references(self, references: Optional["References"] = None) -> Optional[Dict[str, Dependency]]: +# return self.schema_reader.resolve_references(references) - def resolve_schema_references( - self, schema_data: Optional[dict] - ) -> Tuple[Optional[References], Optional[Dict[str, Dependency]]]: - return self.schema_reader.resolve_schema_references(schema_data) +# def resolve_schema_references( +# self, schema_data: Optional[dict] +# ) -> Tuple[Optional[References], Optional[Dict[str, Dependency]]]: +# LOG.error("FROM REGISTRY", stack_info=True) +# res = self.schema_reader.resolve_schema_references(schema_data) +# LOG.error("FROM REGISTRY DONE") +# return res diff --git a/karapace/schema_registry_apis.py b/karapace/schema_registry_apis.py index 3be4ffd4b..216ed02e2 100644 --- a/karapace/schema_registry_apis.py +++ b/karapace/schema_registry_apis.py @@ -13,7 +13,6 @@ InvalidSchemaType, InvalidVersion, ReferenceExistsException, - ReferencesNotSupportedException, SchemasNotFoundException, SchemaTooLargeException, SchemaVersionNotSoftDeletedException, @@ -26,10 +25,11 @@ from karapace.karapace import KarapaceBase from karapace.rapu import HTTPRequest, JSON_CONTENT_TYPE, SERVER_NAME from karapace.schema_models import SchemaType, TypedSchema, ValidatedTypedSchema +from karapace.schema_references import Reference from karapace.schema_registry import KarapaceSchemaRegistry, validate_version from karapace.typing import JsonData from karapace.utils import reference_key -from typing import Any, Dict, Optional, Union +from typing import Any, Dict, List, Optional, Union import aiohttp import async_timeout @@ -55,7 +55,6 @@ class SchemaErrorCodes(Enum): INVALID_SCHEMA = 42201 INVALID_SUBJECT = 42208 SCHEMA_TOO_LARGE_ERROR_CODE = 42209 - INVALID_REFERENCES = 44301 REFERENCES_SUPPORT_NOT_IMPLEMENTED = 44302 REFERENCE_EXISTS = 42206 NO_MASTER_ERROR = 50003 @@ -309,13 +308,13 @@ async def compatibility_check( body = request.json schema_type = self._validate_schema_type(content_type=content_type, data=body) + references = self._validate_references(content_type, schema_type, body) try: - new_schema_references, new_schema_dependencies = self.schema_registry.resolve_schema_references(body) + new_schema_references = self.schema_registry.resolve_references(references) new_schema = ValidatedTypedSchema.parse( schema_type=schema_type, schema_str=body["schema"], references=new_schema_references, - dependencies=new_schema_dependencies, ) except InvalidSchema: self.r( @@ -327,7 +326,7 @@ async def compatibility_check( status=HTTPStatus.UNPROCESSABLE_ENTITY, ) try: - old = await self.schema_registry.subject_version_get(subject=subject, version=version) + old_subject_data = await self.schema_registry.subject_version_get(subject=subject, version=version) except InvalidVersion: self._invalid_version(content_type, version) except (VersionNotFoundException, SchemasNotFoundException, SubjectNotFoundException): @@ -340,27 +339,8 @@ async def compatibility_check( status=HTTPStatus.NOT_FOUND, ) - old_schema_type = self._validate_schema_type(content_type=content_type, data=old) try: - old_schema_references, old_schema_dependencies = self.schema_registry.resolve_schema_references(old) - old_schema = ValidatedTypedSchema.parse( - schema_type=old_schema_type, - schema_str=old["schema"], - references=old_schema_references, - dependencies=old_schema_dependencies, - ) - except InvalidSchema: - self.r( - body={ - "error_code": SchemaErrorCodes.INVALID_SCHEMA.value, - "message": f"Found an invalid {old_schema_type} schema registered", - }, - content_type=content_type, - status=HTTPStatus.UNPROCESSABLE_ENTITY, - ) - - try: - compatibility_mode = self.schema_registry.get_compatibility_mode(subject=old) + compatibility_mode = self.schema_registry.get_compatibility_mode(subject=old_subject_data) except ValueError as ex: # Using INTERNAL_SERVER_ERROR because the subject and configuration # should have been validated before. @@ -373,6 +353,7 @@ async def compatibility_check( status=HTTPStatus.INTERNAL_SERVER_ERROR, ) + old_schema = self.schema_registry.schemas_get(old_subject_data.get("id")) result = check_compatibility( old_schema=old_schema, new_schema=new_schema, @@ -672,11 +653,11 @@ async def subject_delete( "error_code": SchemaErrorCodes.REFERENCE_EXISTS.value, "message": ( f"One or more references exist to the schema " - f"{{magic=1,keytype=SCHEMA,subject={subject},version={arg.version}}}" + f"{{magic=1,keytype=SCHEMA,subject={subject},version={arg.version}}}." ), }, content_type=content_type, - status=HTTPStatus.NOT_FOUND, + status=HTTPStatus.UNPROCESSABLE_ENTITY, ) elif not master_url: self.no_master_error(content_type) @@ -775,11 +756,11 @@ async def subject_version_delete( "error_code": SchemaErrorCodes.REFERENCE_EXISTS.value, "message": ( f"One or more references exist to the schema " - f"{{magic=1,keytype=SCHEMA,subject={subject},version={arg.version}}}" + f"{{magic=1,keytype=SCHEMA,subject={subject},version={arg.version}}}." ), }, content_type=content_type, - status=HTTPStatus.NOT_FOUND, + status=HTTPStatus.UNPROCESSABLE_ENTITY, ) elif not master_url: self.no_master_error(content_type) @@ -926,6 +907,29 @@ def _validate_schema_key(self, content_type: str, body: dict) -> None: status=HTTPStatus.UNPROCESSABLE_ENTITY, ) + def _validate_references(self, content_type: str, schema_type: SchemaType, body: JsonData) -> List[Reference]: + references = body.get("references", []) + if references and schema_type != SchemaType.PROTOBUF: + self.r( + body={ + "error_code": SchemaErrorCodes.REFERENCES_SUPPORT_NOT_IMPLEMENTED.value, + "message": SchemaErrorMessages.REFERENCES_SUPPORT_NOT_IMPLEMENTED.value.format( + schema_type=schema_type.value + ), + }, + content_type=content_type, + status=HTTPStatus.UNPROCESSABLE_ENTITY, + ) + + validated_references = [] + for reference in references: + if ["name", "subject", "version"] != sorted(reference.keys()): + raise InvalidReferences() + validated_references.append( + Reference(name=reference["name"], subject=reference["subject"], version=reference["version"]) + ) + return validated_references + async def subjects_schema_post( self, content_type: str, *, subject: str, request: HTTPRequest, user: Optional[User] = None ) -> None: @@ -957,14 +961,14 @@ async def subjects_schema_post( ) schema_str = body["schema"] schema_type = self._validate_schema_type(content_type=content_type, data=body) + references = self._validate_references(content_type, schema_type, body) try: - new_schema_references, new_schema_dependencies = self.schema_registry.resolve_schema_references(body) + new_schema_references = self.schema_registry.resolve_references(references) new_schema = ValidatedTypedSchema.parse( schema_type=schema_type, schema_str=schema_str, references=new_schema_references, - dependencies=new_schema_dependencies, ) for schema in subject_data["schemas"].values(): validated_typed_schema = ValidatedTypedSchema.parse( @@ -1000,22 +1004,11 @@ async def subjects_schema_post( content_type=content_type, status=HTTPStatus.INTERNAL_SERVER_ERROR, ) - except ReferencesNotSupportedException: - self.r( - body={ - "error_code": SchemaErrorCodes.REFERENCES_SUPPORT_NOT_IMPLEMENTED.value, - "message": SchemaErrorMessages.REFERENCES_SUPPORT_NOT_IMPLEMENTED.value.format( - schema_type=schema_type.value - ), - }, - content_type=content_type, - status=HTTPStatus.UNPROCESSABLE_ENTITY, - ) except InvalidReferences: human_error = "Provided references is not valid" self.r( body={ - "error_code": SchemaErrorCodes.INVALID_REFERENCES.value, + "error_code": SchemaErrorCodes.INVALID_SCHEMA.value, "message": f"Invalid {schema_type} references. Error: {human_error}", }, content_type=content_type, @@ -1062,46 +1055,26 @@ async def subject_post( self._validate_schema_request_body(content_type, body) schema_type = self._validate_schema_type(content_type, body) self._validate_schema_key(content_type, body) + references = self._validate_references(content_type, schema_type, body) try: - new_schema_references, new_schema_dependencies = self.schema_registry.resolve_schema_references(body) + resolved_references = self.schema_registry.resolve_references(references) new_schema = ValidatedTypedSchema.parse( schema_type=schema_type, schema_str=body["schema"], - references=new_schema_references, - dependencies=new_schema_dependencies, + references=resolved_references, ) - except (InvalidSchema, InvalidSchemaType) as e: + except (InvalidReferences, InvalidSchema, InvalidSchemaType) as e: self.log.warning("Invalid schema: %r", body["schema"], exc_info=True) if isinstance(e.__cause__, (SchemaParseException, json.JSONDecodeError)): human_error = f"{e.__cause__.args[0]}" # pylint: disable=no-member else: - human_error = "Provided schema is not valid" + from_body_schema_str = body["schema"] + human_error = f"Invalid schema {from_body_schema_str} with refs {references} of type {schema_type}" self.r( body={ "error_code": SchemaErrorCodes.INVALID_SCHEMA.value, - "message": f"Invalid {schema_type} schema. Error: {human_error}", - }, - content_type=content_type, - status=HTTPStatus.UNPROCESSABLE_ENTITY, - ) - except ReferencesNotSupportedException: - self.r( - body={ - "error_code": SchemaErrorCodes.REFERENCES_SUPPORT_NOT_IMPLEMENTED.value, - "message": SchemaErrorMessages.REFERENCES_SUPPORT_NOT_IMPLEMENTED.value.format( - schema_type=schema_type.value - ), - }, - content_type=content_type, - status=HTTPStatus.UNPROCESSABLE_ENTITY, - ) - except InvalidReferences: - human_error = "Provided references is not valid" - self.r( - body={ - "error_code": SchemaErrorCodes.INVALID_REFERENCES.value, - "message": f"Invalid {schema_type} references. Error: {human_error}", + "message": human_error, }, content_type=content_type, status=HTTPStatus.UNPROCESSABLE_ENTITY, @@ -1122,7 +1095,7 @@ async def subject_post( are_we_master, master_url = await self.schema_registry.get_master() if are_we_master: try: - schema_id = await self.schema_registry.write_new_schema_local(subject, new_schema, new_schema_references) + schema_id = await self.schema_registry.write_new_schema_local(subject, new_schema, resolved_references) self.r( body={"id": schema_id}, content_type=content_type, diff --git a/karapace/serialization.py b/karapace/serialization.py index 4b272d9e4..3e9616a47 100644 --- a/karapace/serialization.py +++ b/karapace/serialization.py @@ -3,11 +3,12 @@ from google.protobuf.message import DecodeError from jsonschema import ValidationError from karapace.client import Client +from karapace.dependency import Dependency from karapace.protobuf.exception import ProtobufTypeException from karapace.protobuf.io import ProtobufDatumReader, ProtobufDatumWriter -from karapace.schema_models import InvalidSchema, References, SchemaType, TypedSchema, ValidatedTypedSchema +from karapace.schema_models import InvalidSchema, SchemaType, TypedSchema, ValidatedTypedSchema from karapace.utils import json_encode -from typing import Any, Dict, Optional, Tuple +from typing import Any, Dict, List, Optional, Tuple from urllib.parse import quote import asyncio @@ -76,7 +77,7 @@ def __init__( self.base_url = schema_registry_url async def post_new_schema( - self, subject: str, schema: ValidatedTypedSchema, references: Optional[References] = None + self, subject: str, schema: ValidatedTypedSchema, references: Optional[Dependency] = None ) -> int: if schema.schema_type is SchemaType.PROTOBUF: if references: @@ -105,7 +106,7 @@ async def get_latest_schema(self, subject: str) -> Tuple[int, ValidatedTypedSche except InvalidSchema as e: raise SchemaRetrievalError(f"Failed to parse schema string from response: {json_result}") from e - async def get_schema_for_id(self, schema_id: int) -> Tuple[ValidatedTypedSchema, Optional[References]]: + async def get_schema_for_id(self, schema_id: int) -> Tuple[ValidatedTypedSchema, Optional[List[Dependency]]]: result = await self.client.get(f"schemas/ids/{schema_id}") if not result.ok: raise SchemaRetrievalError(result.json()["message"]) @@ -116,6 +117,7 @@ async def get_schema_for_id(self, schema_id: int) -> Tuple[ValidatedTypedSchema, schema_type = SchemaType(json_result.get("schemaType", "AVRO")) references_str = json_result.get("references") if references_str: + # FIXME: Way to resolve the references? references = References(schema_type, references_str) else: references = None diff --git a/tests/integration/test_schema_protobuf.py b/tests/integration/test_schema_protobuf.py index 7cc1dd082..8095aee2d 100644 --- a/tests/integration/test_schema_protobuf.py +++ b/tests/integration/test_schema_protobuf.py @@ -4,9 +4,14 @@ Copyright (c) 2019 Aiven Ltd See LICENSE for details """ +from dataclasses import dataclass from karapace.client import Client from karapace.protobuf.kotlin_wrapper import trim_margin +from karapace.schema_type import SchemaType +from karapace.typing import JsonData +from tests.base_testcase import BaseTestCase from tests.utils import create_subject_name_factory +from typing import List, Optional, Union import logging import pytest @@ -175,10 +180,14 @@ async def test_protobuf_schema_references(registry_async_client: Client) -> None |syntax = "proto3"; |package a1; |import "Place.proto"; + |import "google/protobuf/duration.proto"; + |import "google/type/color.proto"; |message Customer { | string name = 1; | int32 code = 2; | Place place = 3; + | google.protobuf.Duration dur = 4; + | google.type.Color color = 5; |} |""" @@ -260,9 +269,9 @@ async def test_protobuf_schema_references(registry_async_client: Client) -> None myjson = res.json() res = await registry_async_client.delete("subjects/customer/versions/1") - assert res.status_code == 404 + assert res.status_code == 422 - match_msg = "One or more references exist to the schema {magic=1,keytype=SCHEMA,subject=customer,version=1}" + match_msg = "One or more references exist to the schema {magic=1,keytype=SCHEMA,subject=customer,version=1}." myjson = res.json() assert myjson["error_code"] == 42206 and myjson["message"] == match_msg @@ -334,8 +343,8 @@ async def test_protobuf_schema_verifier(registry_async_client: Client) -> None: assert not any(x != y for x, y in zip(myjson, referents)) res = await registry_async_client.delete("subjects/customer/versions/1") - assert res.status_code == 404 - match_msg = "One or more references exist to the schema {magic=1,keytype=SCHEMA,subject=customer,version=1}" + assert res.status_code == 422 + match_msg = "One or more references exist to the schema {magic=1,keytype=SCHEMA,subject=customer,version=1}." myjson = res.json() assert myjson["error_code"] == 42206 and myjson["message"] == match_msg @@ -344,3 +353,500 @@ async def test_protobuf_schema_verifier(registry_async_client: Client) -> None: res = await registry_async_client.delete("subjects/customer/versions/1") assert res.status_code == 200 + + +@dataclass +class TestCaseSchema: + schema_type: SchemaType + schema_str: str + subject: str + references: Optional[List[JsonData]] = None + expected: int = 200 + expected_msg: str = "" + expected_error_code: Optional[int] = None + + +TestCaseSchema.__test__ = False + + +@dataclass +class TestCaseDeleteSchema: + subject: str + version: int + schema_id: int + expected: int = 200 + expected_msg: str = "" + expected_error_code: Optional[int] = None + + +TestCaseDeleteSchema.__test__ = False + + +@dataclass +class ReferenceTestCase(BaseTestCase): + schemas: List[Union[TestCaseSchema, TestCaseDeleteSchema]] + + +# Base case +SCHEMA_NO_REF = """\ +syntax = "proto3"; + +message NoReference { + string name = 1; +} +""" + +SCHEMA_NO_REF_TWO = """\ +syntax = "proto3"; + +message NoReferenceTwo { + string name = 1; +} +""" + +SCHEMA_WITH_REF = """\ +syntax = "proto3"; + +import "NoReference.proto"; + +message WithReference { + string name = 1; + NoReference ref = 2; +} +""" + +SCHEMA_WITH_2ND_LEVEL_REF = """\ +syntax = "proto3"; + +import "WithReference.proto"; + +message With2ndLevelReference { + string name = 1; + WithReference ref = 2; +} +""" + +SCHEMA_REMOVES_REFERENCED_FIELD_INCOMPATIBLE = """\ +syntax = "proto3"; + +message WithReference { + string name = 1; +} +""" + +SCHEMA_ADDS_NEW_REFERENCE = """\ +syntax = "proto3"; + +import "NoReference.proto"; +import "NoReferenceTwo.proto"; + +message WithReference { + string name = 1; + NoReference ref = 2; + NoReferenceTwo refTwo = 3; +} +""" + +# Invalid schema +SCHEMA_INVALID_MISSING_CLOSING_BRACE = """\ +syntax = "proto3"; + +import "NoReference.proto"; + +message SchemaMissingClosingBrace { + string name = 1; + NoReference ref = 2; + +""" + +# Schema having multiple messages +SCHEMA_NO_REF_TWO_MESSAGES = """\ +syntax = "proto3"; + +message NoReferenceOne { + string nameOne = 1; +} + +message NoReferenceTwo { + string nameTwo = 1; +} +""" + +SCHEMA_WITH_REF_TO_NO_REFERENCE_TWO = """\ +syntax = "proto3"; + +import "NoReferenceTwo.proto"; + +message WithReference { + string name = 1; + NoReferenceTwo ref = 2; +} +""" + + +# Nested references +SCHEMA_NO_REF_NESTED_MESSAGE = """\ +syntax = "proto3"; + +message NoReference { + message NoReferenceNested { + string nameNested = 1; + } + string name = 1; + NoReferenceNested ref = 2; +} +""" + +SCHEMA_WITH_REF_TO_NESTED = """\ +syntax = "proto3"; + +import "NoReferenceNested.proto"; + +message WithReference { + string name = 1; + NoReference.NoReferenceNested ref = 2; +} +""" + + +@pytest.mark.parametrize( + "testcase", + [ + ReferenceTestCase( + test_name="No references", + schemas=[ + TestCaseSchema( + schema_type=SchemaType.PROTOBUF, + schema_str=SCHEMA_NO_REF, + subject="nr_s1", + references=[], + expected=200, + ), + TestCaseDeleteSchema( + subject="nr_s1", + schema_id=1, + version=1, + expected=200, + ), + ], + ), + # Better error message should be given back, now it is only InvalidSchema + ReferenceTestCase( + test_name="With reference, ref schema does not exist", + schemas=[ + TestCaseSchema( + schema_type=SchemaType.PROTOBUF, + schema_str=SCHEMA_WITH_REF, + subject="wr_nonexisting_s1", + references=[{"name": "NoReference.proto", "subject": "wr_not_found", "version": 1}], + expected=422, + expected_msg=( + f"Invalid schema {SCHEMA_WITH_REF} " + "with refs [{name='NoReference.proto', subject='wr_not_found', version=1}] of type PROTOBUF" + ), + expected_error_code=42201, + ), + ], + ), + ReferenceTestCase( + test_name="With reference, references not given", + schemas=[ + TestCaseSchema( + schema_type=SchemaType.PROTOBUF, + schema_str=SCHEMA_WITH_REF, + subject="wr_nonexisting_s1_missing_references", + references=[], + expected=422, + expected_msg=f"Invalid schema {SCHEMA_WITH_REF} with refs [] of type PROTOBUF", + expected_error_code=42201, + ), + ], + ), + ReferenceTestCase( + test_name="With reference", + schemas=[ + TestCaseSchema( + schema_type=SchemaType.PROTOBUF, + schema_str=SCHEMA_NO_REF, + subject="wr_s1", + references=[], + expected=200, + ), + TestCaseSchema( + schema_type=SchemaType.PROTOBUF, + schema_str=SCHEMA_WITH_REF, + subject="wr_s2", + references=[{"name": "NoReference.proto", "subject": "wr_s1", "version": 1}], + expected=200, + ), + TestCaseDeleteSchema( + subject="wr_s1", + schema_id=1, + version=1, + expected=422, + expected_msg=( + "One or more references exist to the schema {magic=1,keytype=SCHEMA,subject=wr_s1,version=1}." + ), + expected_error_code=42206, + ), + TestCaseDeleteSchema( + subject="wr_s2", + schema_id=2, + version=1, + expected=200, + ), + TestCaseDeleteSchema( + subject="wr_s1", + schema_id=1, + version=1, + expected=200, + ), + ], + ), + ReferenceTestCase( + test_name="With reference, remove referenced field causes incompatible schema", + schemas=[ + TestCaseSchema( + schema_type=SchemaType.PROTOBUF, + schema_str=SCHEMA_NO_REF, + subject="wr_s1_test_incompatible_change", + references=[], + expected=200, + ), + TestCaseSchema( + schema_type=SchemaType.PROTOBUF, + schema_str=SCHEMA_WITH_REF, + subject="wr_s2_test_incompatible_change", + references=[{"name": "NoReference.proto", "subject": "wr_s1_test_incompatible_change", "version": 1}], + expected=200, + ), + TestCaseSchema( + schema_type=SchemaType.PROTOBUF, + schema_str=SCHEMA_REMOVES_REFERENCED_FIELD_INCOMPATIBLE, + subject="wr_s2_test_incompatible_change", + references=[], + expected=409, + expected_msg=( + "Incompatible schema, compatibility_mode=BACKWARD " + "Incompatible modification Modification.MESSAGE_DROP found" + ), + ), + ], + ), + ReferenceTestCase( + test_name="With reference, add new referenced field", + schemas=[ + TestCaseSchema( + schema_type=SchemaType.PROTOBUF, + schema_str=SCHEMA_NO_REF, + subject="wr_s1_add_new_reference", + references=[], + expected=200, + ), + TestCaseSchema( + schema_type=SchemaType.PROTOBUF, + schema_str=SCHEMA_WITH_REF, + subject="wr_s2_add_new_reference", + references=[{"name": "NoReference.proto", "subject": "wr_s1_add_new_reference", "version": 1}], + expected=200, + ), + TestCaseSchema( + schema_type=SchemaType.PROTOBUF, + schema_str=SCHEMA_NO_REF_TWO, + subject="wr_s3_the_new_reference", + references=[], + expected=200, + ), + TestCaseSchema( + schema_type=SchemaType.PROTOBUF, + schema_str=SCHEMA_ADDS_NEW_REFERENCE, + subject="wr_s2_add_new_reference", + references=[ + {"name": "NoReference.proto", "subject": "wr_s1_add_new_reference", "version": 1}, + {"name": "NoReferenceTwo.proto", "subject": "wr_s3_the_new_reference", "version": 1}, + ], + expected=200, + ), + ], + ), + ReferenceTestCase( + test_name="With reference chain, nonexisting schema", + schemas=[ + TestCaseSchema( + schema_type=SchemaType.PROTOBUF, + schema_str=SCHEMA_NO_REF, + subject="wr_chain_s1", + references=[], + expected=200, + ), + TestCaseSchema( + schema_type=SchemaType.PROTOBUF, + schema_str=SCHEMA_WITH_REF, + subject="wr_chain_s2", + references=[ + {"name": "NoReference.proto", "subject": "wr_chain_s1", "version": 1}, + {"name": "NotFoundReference.proto", "subject": "wr_chain_nonexisting", "version": 1}, + ], + expected=422, + expected_msg=( + f"Invalid schema {SCHEMA_WITH_REF} " + "with refs [{name='NoReference.proto', subject='wr_chain_s1', version=1}, " + "{name='NotFoundReference.proto', subject='wr_chain_nonexisting', version=1}] " + "of type PROTOBUF" + ), + expected_error_code=42201, + ), + ], + ), + ReferenceTestCase( + test_name="With reference chain", + schemas=[ + TestCaseSchema( + schema_type=SchemaType.PROTOBUF, + schema_str=SCHEMA_NO_REF, + subject="wr_chain_s1", + references=[], + expected=200, + ), + TestCaseSchema( + schema_type=SchemaType.PROTOBUF, + schema_str=SCHEMA_WITH_REF, + subject="wr_chain_s2", + references=[{"name": "NoReference.proto", "subject": "wr_chain_s1", "version": 1}], + expected=200, + ), + TestCaseSchema( + schema_type=SchemaType.PROTOBUF, + schema_str=SCHEMA_WITH_2ND_LEVEL_REF, + subject="wr_chain_s3", + references=[{"name": "WithReference.proto", "subject": "wr_chain_s2", "version": 1}], + expected=200, + ), + TestCaseDeleteSchema( + subject="wr_chain_s1", + schema_id=1, + version=1, + expected=422, + ), + TestCaseDeleteSchema( + subject="wr_chain_s2", + schema_id=2, + version=1, + expected=422, + ), + TestCaseDeleteSchema( + subject="wr_chain_s3", + schema_id=3, + version=1, + expected=200, + ), + TestCaseDeleteSchema( + subject="wr_chain_s2", + schema_id=2, + version=1, + expected=200, + ), + TestCaseDeleteSchema( + subject="wr_chain_s1", + schema_id=1, + version=1, + expected=200, + ), + ], + ), + ReferenceTestCase( + test_name="Invalid schema missing closing brace", + schemas=[ + TestCaseSchema( + schema_type=SchemaType.PROTOBUF, + schema_str=SCHEMA_NO_REF, + subject="wr_invalid_reference_ok_schema", + references=[], + expected=200, + ), + TestCaseSchema( + schema_type=SchemaType.PROTOBUF, + schema_str=SCHEMA_INVALID_MISSING_CLOSING_BRACE, + subject="wr_invalid_missing_closing_brace", + references=[{"name": "NoReference.proto", "subject": "wr_invalid_reference_ok_schema", "version": 1}], + expected=422, + expected_msg=( + f"Invalid schema {SCHEMA_INVALID_MISSING_CLOSING_BRACE} " + "with refs [{name='NoReference.proto', subject='wr_invalid_reference_ok_schema', version=1}] " + "of type PROTOBUF" + ), + expected_error_code=42201, + ), + ], + ), + ReferenceTestCase( + test_name="With reference to message from schema file defining two messages", + schemas=[ + TestCaseSchema( + schema_type=SchemaType.PROTOBUF, + schema_str=SCHEMA_NO_REF_TWO_MESSAGES, + subject="wr_s1_two_messages", + references=[], + expected=200, + ), + TestCaseSchema( + schema_type=SchemaType.PROTOBUF, + schema_str=SCHEMA_WITH_REF_TO_NO_REFERENCE_TWO, + subject="wr_s2_referencing_message_two", + references=[{"name": "NoReferenceTwo.proto", "subject": "wr_s1_two_messages", "version": 1}], + expected=200, + ), + ], + ), + ReferenceTestCase( + test_name="With reference to nested message", + schemas=[ + TestCaseSchema( + schema_type=SchemaType.PROTOBUF, + schema_str=SCHEMA_NO_REF_NESTED_MESSAGE, + subject="wr_s1_with_nested_message", + references=[], + expected=200, + ), + TestCaseSchema( + schema_type=SchemaType.PROTOBUF, + schema_str=SCHEMA_WITH_REF_TO_NESTED, + subject="wr_s2_referencing_nested_message", + references=[{"name": "NoReference.proto", "subject": "wr_s1_with_nested_message", "version": 1}], + expected=200, + ), + ], + ), + ], + ids=str, +) +async def test_references(testcase: ReferenceTestCase, registry_async_client: Client): + for testdata in testcase.schemas: + if isinstance(testdata, TestCaseSchema): + print(f"Adding new schema, subject: '{testdata.subject}'\n{testdata.schema_str}") + body = {"schemaType": testdata.schema_type, "schema": testdata.schema_str} + if testdata.references: + body["references"] = testdata.references + res = await registry_async_client.post(f"subjects/{testdata.subject}/versions", json=body) + elif isinstance(testdata, TestCaseDeleteSchema): + print(f"Deleting schema, subject: '{testdata.subject}', version: {testdata.version}") + res = await registry_async_client.delete(f"subjects/{testdata.subject}/versions/{testdata.version}") + else: + raise Exception("Unknown test case.") + assert res.status_code == testdata.expected + if testdata.expected_msg: + assert res.json_result.get("message", None) == testdata.expected_msg + if testdata.expected_error_code: + assert res.json_result.get("error_code") == testdata.expected_error_code + if isinstance(testdata, TestCaseSchema): + if testdata.expected == 200: + schema_id = res.json().get("id") + fetch_schema_res = await registry_async_client.get(f"/schemas/ids/{schema_id}") + assert fetch_schema_res.status_code == 200 + if isinstance(testdata, TestCaseDeleteSchema): + if testdata.expected == 200: + fetch_res = await registry_async_client.get(f"/subjects/{testdata.subject}/versions/{testdata.version}") + assert fetch_res.status_code == 404 + else: + fetch_schema_res = await registry_async_client.get(f"/schemas/ids/{testdata.schema_id}") + assert fetch_schema_res.status_code == 200 From 0c2b5f05b84ae9b925d7730f02f8e88fc14a51b3 Mon Sep 17 00:00:00 2001 From: Jarkko Jaakola Date: Fri, 21 Oct 2022 09:07:12 +0300 Subject: [PATCH 08/25] WIP: fix schema test assumed error message --- tests/integration/test_schema.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/tests/integration/test_schema.py b/tests/integration/test_schema.py index 3b0ec3d5d..1b836ba88 100644 --- a/tests/integration/test_schema.py +++ b/tests/integration/test_schema.py @@ -2273,8 +2273,7 @@ async def test_invalid_namespace(registry_async_client: Client) -> None: json_res = res.json() assert json_res["error_code"] == 42201, json_res expected_message = ( - "Invalid AVRO schema. Error: foo-bar-baz is not a valid Avro name because it does not match the pattern " - "(?:^|\\.)[A-Za-z_][A-Za-z0-9_]*$" + "foo-bar-baz is not a valid Avro name because it does not match the pattern (?:^|\\.)[A-Za-z_][A-Za-z0-9_]*$" ) assert json_res["message"] == expected_message, json_res @@ -2883,7 +2882,7 @@ async def test_invalid_schema_should_provide_good_error_messages(registry_async_ f"subjects/{test_subject}/versions", json={"schema": schema_str[:-1]}, ) - assert res.json()["message"] == "Invalid AVRO schema. Error: Expecting ',' delimiter: line 1 column 18 (char 17)" + assert res.json()["message"] == "Expecting ',' delimiter: line 1 column 18 (char 17)" # Unfortunately the AVRO library doesn't provide a good error message, it just raises an TypeError schema_str = json.dumps({"type": "enum", "name": "error"}) @@ -2891,10 +2890,7 @@ async def test_invalid_schema_should_provide_good_error_messages(registry_async_ f"subjects/{test_subject}/versions", json={"schema": schema_str}, ) - assert ( - res.json()["message"] - == "Invalid AVRO schema. Error: Enum symbols must be a sequence of strings, but it is " - ) + assert res.json()["message"] == "Enum symbols must be a sequence of strings, but it is " # This is an upstream bug in the python AVRO library, until the bug is fixed we should at least have a nice error message schema_str = json.dumps({"type": "enum", "name": "error", "symbols": {}}) @@ -2902,10 +2898,7 @@ async def test_invalid_schema_should_provide_good_error_messages(registry_async_ f"subjects/{test_subject}/versions", json={"schema": schema_str}, ) - assert ( - res.json()["message"] - == "Invalid AVRO schema. Error: Enum symbols must be a sequence of strings, but it is " - ) + assert res.json()["message"] == "Enum symbols must be a sequence of strings, but it is " # This is an upstream bug in the python AVRO library, until the bug is fixed we should at least have a nice error message schema_str = json.dumps({"type": "enum", "name": "error", "symbols": ["A", "B"]}) @@ -2913,4 +2906,4 @@ async def test_invalid_schema_should_provide_good_error_messages(registry_async_ f"subjects/{test_subject}/versions", json={"schema": schema_str}, ) - assert res.json()["message"] == "Invalid AVRO schema. Error: error is a reserved type name." + assert res.json()["message"] == "error is a reserved type name." From 20d5235ed805e1301f2e094055347017decfa327 Mon Sep 17 00:00:00 2001 From: Jarkko Jaakola Date: Fri, 21 Oct 2022 09:29:00 +0300 Subject: [PATCH 09/25] WIP: SchemaRegistryClient to load references --- karapace/serialization.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/karapace/serialization.py b/karapace/serialization.py index 3e9616a47..9c0d988a5 100644 --- a/karapace/serialization.py +++ b/karapace/serialization.py @@ -4,11 +4,13 @@ from jsonschema import ValidationError from karapace.client import Client from karapace.dependency import Dependency +from karapace.errors import InvalidReferences from karapace.protobuf.exception import ProtobufTypeException from karapace.protobuf.io import ProtobufDatumReader, ProtobufDatumWriter from karapace.schema_models import InvalidSchema, SchemaType, TypedSchema, ValidatedTypedSchema +from karapace.schema_references import Reference from karapace.utils import json_encode -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Dict, Optional, Tuple from urllib.parse import quote import asyncio @@ -106,7 +108,7 @@ async def get_latest_schema(self, subject: str) -> Tuple[int, ValidatedTypedSche except InvalidSchema as e: raise SchemaRetrievalError(f"Failed to parse schema string from response: {json_result}") from e - async def get_schema_for_id(self, schema_id: int) -> Tuple[ValidatedTypedSchema, Optional[List[Dependency]]]: + async def get_schema_for_id(self, schema_id: int) -> ValidatedTypedSchema: result = await self.client.get(f"schemas/ids/{schema_id}") if not result.ok: raise SchemaRetrievalError(result.json()["message"]) @@ -115,14 +117,18 @@ async def get_schema_for_id(self, schema_id: int) -> Tuple[ValidatedTypedSchema, raise SchemaRetrievalError(f"Invalid result format: {json_result}") try: schema_type = SchemaType(json_result.get("schemaType", "AVRO")) - references_str = json_result.get("references") - if references_str: - # FIXME: Way to resolve the references? - references = References(schema_type, references_str) - else: - references = None + references = json_result.get("references") + validated_references = None if references: - return ValidatedTypedSchema.parse(schema_type, json_result["schema"]), references + validated_references = [] + for reference in references: + if ["name", "subject", "version"] != sorted(reference.keys()): + raise InvalidReferences() + validated_references.append( + Reference(name=reference["name"], subject=reference["subject"], version=reference["version"]) + ) + if validated_references: + return ValidatedTypedSchema.parse(schema_type, json_result["schema"], references=validated_references) return ValidatedTypedSchema.parse(schema_type, json_result["schema"]) except InvalidSchema as e: From 27868585d533e839542252453ebefab0069911a3 Mon Sep 17 00:00:00 2001 From: Jarkko Jaakola Date: Fri, 21 Oct 2022 16:51:19 +0300 Subject: [PATCH 10/25] fix schema deletion when references used --- karapace/schema_reader.py | 26 +++++++------- karapace/schema_registry.py | 14 +------- tests/integration/test_schema_protobuf.py | 41 ++++++++++++++++++++++- 3 files changed, 55 insertions(+), 26 deletions(-) diff --git a/karapace/schema_reader.py b/karapace/schema_reader.py index 1274a97f6..21bfdf500 100644 --- a/karapace/schema_reader.py +++ b/karapace/schema_reader.py @@ -433,8 +433,7 @@ def _handle_msg_schema(self, key: dict, value: Optional[dict]) -> None: schema_version = value["version"] schema_deleted = value.get("deleted", False) schema_references = value.get("references", None) - - resolved_references = None + resolved_references: Optional[List[Dependency]] = None try: schema_type_parsed = SchemaType(schema_type) @@ -456,13 +455,11 @@ def _handle_msg_schema(self, key: dict, value: Optional[dict]) -> None: return elif schema_type_parsed == SchemaType.PROTOBUF: try: - references = None if schema_references: references = [ Reference(reference["name"], reference["subject"], reference["version"]) for reference in schema_references ] - if references: resolved_references = self.resolve_references(references) parsed_schema = parse_protobuf_schema_definition(schema_str, resolved_references, validate_references=False) schema_str = str(parsed_schema) @@ -491,8 +488,8 @@ def _handle_msg_schema(self, key: dict, value: Optional[dict]) -> None: "id": schema_id, "deleted": schema_deleted, } - if schema_references: - schema["references"] = schema_references + if resolved_references: + schema["references"] = resolved_references if schema_version in subjects_schemas: LOG.info("Updating entry subject: %r version: %r id: %r", schema_subject, schema_version, schema_id) @@ -500,9 +497,9 @@ def _handle_msg_schema(self, key: dict, value: Optional[dict]) -> None: LOG.info("Adding entry subject: %r version: %r id: %r", schema_subject, schema_version, schema_id) subjects_schemas[schema_version] = schema - if schema_references: - for ref in schema_references: - ref_str = reference_key(ref["subject"], ref["version"]) + if resolved_references: + for ref in resolved_references: + ref_str = reference_key(ref.subject, ref.version) referents = self.referenced_by.get(ref_str, None) if referents: referents.append(schema_id) @@ -562,9 +559,9 @@ def get_schemas_list(self, *, include_deleted: bool, latest_only: bool) -> Dict[ res_schemas[subject] = selected_schemas return res_schemas - def remove_referenced_by(self, schema_id: SchemaId, references: List): + def remove_referenced_by(self, schema_id: SchemaId, references: List[Reference]): for ref in references: - key = reference_key(ref["subject"], ref["version"]) + key = reference_key(ref.subject, ref.version) if self.referenced_by.get(key, None) and schema_id in self.referenced_by[key]: self.referenced_by[key].remove(schema_id) @@ -612,12 +609,17 @@ def _build_state_dict(self) -> JsonData: else: schema_str = schema.get("schema").schema_str schema_hash = hashlib.sha1(schema_str.encode("utf8")).hexdigest() + references = schema.get("references", None) + references_list = None + if references: + references_list = [reference.to_dict() for reference in schema["references"]] subject_state.append( { "id": schema_id, "version": version, "schema_hash": schema_hash, - "references": schema.get("references"), + "references": references_list, + "deleted": schema.get("deleted", False), } ) return json.dumps(state, sort_keys=True, indent=2) diff --git a/karapace/schema_registry.py b/karapace/schema_registry.py index eddc6eee6..1b618a481 100644 --- a/karapace/schema_registry.py +++ b/karapace/schema_registry.py @@ -254,7 +254,7 @@ async def subject_version_delete_local(self, subject: Subject, version: Version, schema_id=schema_id, version=resolved_version, deleted=True, - references=None, + references=references, ) if references and len(references) > 0: self.schema_reader.remove_referenced_by(schema_id, references) @@ -530,15 +530,3 @@ def resolve_references(self, references: Optional[List[Reference]]) -> Optional[ if references: return self.schema_reader.resolve_references(references) return None - - -# def resolve_references(self, references: Optional["References"] = None) -> Optional[Dict[str, Dependency]]: -# return self.schema_reader.resolve_references(references) - -# def resolve_schema_references( -# self, schema_data: Optional[dict] -# ) -> Tuple[Optional[References], Optional[Dict[str, Dependency]]]: -# LOG.error("FROM REGISTRY", stack_info=True) -# res = self.schema_reader.resolve_schema_references(schema_data) -# LOG.error("FROM REGISTRY DONE") -# return res diff --git a/tests/integration/test_schema_protobuf.py b/tests/integration/test_schema_protobuf.py index 8095aee2d..8ab438c80 100644 --- a/tests/integration/test_schema_protobuf.py +++ b/tests/integration/test_schema_protobuf.py @@ -278,6 +278,9 @@ async def test_protobuf_schema_references(registry_async_client: Client) -> None res = await registry_async_client.delete("subjects/test_schema/versions/1") assert res.status_code == 200 + res = await registry_async_client.delete("subjects/test_schema/versions/1?permanent=true") + assert res.status_code == 200 + res = await registry_async_client.delete("subjects/customer/versions/1") assert res.status_code == 200 @@ -351,6 +354,9 @@ async def test_protobuf_schema_verifier(registry_async_client: Client) -> None: res = await registry_async_client.delete("subjects/test_schema/versions/1") assert res.status_code == 200 + res = await registry_async_client.delete("subjects/test_schema/versions/1?permanent=true") + assert res.status_code == 200 + res = await registry_async_client.delete("subjects/customer/versions/1") assert res.status_code == 200 @@ -382,6 +388,11 @@ class TestCaseDeleteSchema: TestCaseDeleteSchema.__test__ = False +@dataclass +class TestCaseHardDeleteSchema(TestCaseDeleteSchema): + pass + + @dataclass class ReferenceTestCase(BaseTestCase): schemas: List[Union[TestCaseSchema, TestCaseDeleteSchema]] @@ -595,6 +606,12 @@ class ReferenceTestCase(BaseTestCase): version=1, expected=200, ), + TestCaseHardDeleteSchema( + subject="wr_s2", + schema_id=2, + version=1, + expected=200, + ), TestCaseDeleteSchema( subject="wr_s1", schema_id=1, @@ -740,12 +757,24 @@ class ReferenceTestCase(BaseTestCase): version=1, expected=200, ), + TestCaseHardDeleteSchema( + subject="wr_chain_s3", + schema_id=3, + version=1, + expected=200, + ), TestCaseDeleteSchema( subject="wr_chain_s2", schema_id=2, version=1, expected=200, ), + TestCaseHardDeleteSchema( + subject="wr_chain_s2", + schema_id=2, + version=1, + expected=200, + ), TestCaseDeleteSchema( subject="wr_chain_s1", schema_id=1, @@ -828,8 +857,18 @@ async def test_references(testcase: ReferenceTestCase, registry_async_client: Cl if testdata.references: body["references"] = testdata.references res = await registry_async_client.post(f"subjects/{testdata.subject}/versions", json=body) + elif isinstance(testdata, TestCaseHardDeleteSchema): + print( + f"Permanently deleting schema, subject: '{testdata.subject}, " + f"schema: {testdata.schema_id}, version: {testdata.version}' " + ) + res = await registry_async_client.delete( + f"subjects/{testdata.subject}/versions/{testdata.version}?permanent=true" + ) elif isinstance(testdata, TestCaseDeleteSchema): - print(f"Deleting schema, subject: '{testdata.subject}', version: {testdata.version}") + print( + f"Deleting schema, subject: '{testdata.subject}, schema: {testdata.schema_id}, version: {testdata.version}' " + ) res = await registry_async_client.delete(f"subjects/{testdata.subject}/versions/{testdata.version}") else: raise Exception("Unknown test case.") From 437f7bd3048ae2774defe635c3497fd6153aa829 Mon Sep 17 00:00:00 2001 From: Sergiy Zaschipas Date: Fri, 21 Oct 2022 17:07:25 +0300 Subject: [PATCH 11/25] bugfixes --- karapace/protobuf/compare_type_storage.py | 14 +- karapace/protobuf/google/protobuf/any.proto | 158 --- .../protobuf/google/protobuf/any_test.proto | 44 - karapace/protobuf/google/protobuf/api.proto | 208 ---- .../protobuf/google/protobuf/descriptor.proto | 918 ------------------ .../protobuf/google/protobuf/duration.proto | 116 --- karapace/protobuf/google/protobuf/empty.proto | 51 - .../protobuf/google/protobuf/field_mask.proto | 245 ----- .../google/protobuf/map_lite_unittest.proto | 131 --- .../google/protobuf/map_proto2_unittest.proto | 91 -- .../google/protobuf/map_unittest.proto | 125 --- .../google/protobuf/source_context.proto | 48 - .../protobuf/google/protobuf/struct.proto | 95 -- .../protobuf/google/protobuf/timestamp.proto | 147 --- karapace/protobuf/google/protobuf/type.proto | 187 ---- .../protobuf/google/protobuf/wrappers.proto | 123 --- karapace/protobuf/google/type/BUILD.bazel | 536 ---------- karapace/protobuf/google/type/README.md | 16 - .../google/type/calendar_period.proto | 56 -- karapace/protobuf/google/type/color.proto | 174 ---- karapace/protobuf/google/type/date.proto | 52 - karapace/protobuf/google/type/datetime.proto | 104 -- karapace/protobuf/google/type/dayofweek.proto | 50 - karapace/protobuf/google/type/decimal.proto | 95 -- karapace/protobuf/google/type/expr.proto | 73 -- karapace/protobuf/google/type/fraction.proto | 33 - karapace/protobuf/google/type/interval.proto | 46 - karapace/protobuf/google/type/latlng.proto | 37 - .../protobuf/google/type/localized_text.proto | 36 - karapace/protobuf/google/type/money.proto | 42 - karapace/protobuf/google/type/month.proto | 65 -- .../protobuf/google/type/phone_number.proto | 113 --- .../protobuf/google/type/postal_address.proto | 134 --- .../protobuf/google/type/quaternion.proto | 94 -- karapace/protobuf/google/type/timeofday.proto | 44 - karapace/protobuf/google/type/type.yaml | 40 - karapace/schema_registry.py | 39 +- tests/integration/test_schema_protobuf.py | 21 + 38 files changed, 49 insertions(+), 4552 deletions(-) delete mode 100644 karapace/protobuf/google/protobuf/any.proto delete mode 100644 karapace/protobuf/google/protobuf/any_test.proto delete mode 100644 karapace/protobuf/google/protobuf/api.proto delete mode 100644 karapace/protobuf/google/protobuf/descriptor.proto delete mode 100644 karapace/protobuf/google/protobuf/duration.proto delete mode 100644 karapace/protobuf/google/protobuf/empty.proto delete mode 100644 karapace/protobuf/google/protobuf/field_mask.proto delete mode 100644 karapace/protobuf/google/protobuf/map_lite_unittest.proto delete mode 100644 karapace/protobuf/google/protobuf/map_proto2_unittest.proto delete mode 100644 karapace/protobuf/google/protobuf/map_unittest.proto delete mode 100644 karapace/protobuf/google/protobuf/source_context.proto delete mode 100644 karapace/protobuf/google/protobuf/struct.proto delete mode 100644 karapace/protobuf/google/protobuf/timestamp.proto delete mode 100644 karapace/protobuf/google/protobuf/type.proto delete mode 100644 karapace/protobuf/google/protobuf/wrappers.proto delete mode 100644 karapace/protobuf/google/type/BUILD.bazel delete mode 100644 karapace/protobuf/google/type/README.md delete mode 100644 karapace/protobuf/google/type/calendar_period.proto delete mode 100644 karapace/protobuf/google/type/color.proto delete mode 100644 karapace/protobuf/google/type/date.proto delete mode 100644 karapace/protobuf/google/type/datetime.proto delete mode 100644 karapace/protobuf/google/type/dayofweek.proto delete mode 100644 karapace/protobuf/google/type/decimal.proto delete mode 100644 karapace/protobuf/google/type/expr.proto delete mode 100644 karapace/protobuf/google/type/fraction.proto delete mode 100644 karapace/protobuf/google/type/interval.proto delete mode 100644 karapace/protobuf/google/type/latlng.proto delete mode 100644 karapace/protobuf/google/type/localized_text.proto delete mode 100644 karapace/protobuf/google/type/money.proto delete mode 100644 karapace/protobuf/google/type/month.proto delete mode 100644 karapace/protobuf/google/type/phone_number.proto delete mode 100644 karapace/protobuf/google/type/postal_address.proto delete mode 100644 karapace/protobuf/google/type/quaternion.proto delete mode 100644 karapace/protobuf/google/type/timeofday.proto delete mode 100644 karapace/protobuf/google/type/type.yaml diff --git a/karapace/protobuf/compare_type_storage.py b/karapace/protobuf/compare_type_storage.py index e727b8ce8..68ea6983d 100644 --- a/karapace/protobuf/compare_type_storage.py +++ b/karapace/protobuf/compare_type_storage.py @@ -90,8 +90,11 @@ def self_type_short_name(self, t: ProtoType) -> Optional[str]: if name is None: raise IllegalArgumentException(f"Cannot determine message type {t}") type_record: TypeRecord = self.self_types.get(name) - if name.startswith(type_record.package_name): - return name[(len(type_record.package_name) + 1) :] + package_name = type_record.package_name + if package_name is None: + package_name = "" + if name.startswith(package_name): + return name[(len(package_name) + 1) :] return name def other_type_short_name(self, t: ProtoType) -> Optional[str]: @@ -99,8 +102,11 @@ def other_type_short_name(self, t: ProtoType) -> Optional[str]: if name is None: raise IllegalArgumentException(f"Cannot determine message type {t}") type_record: TypeRecord = self.other_types.get(name) - if name.startswith(type_record.package_name): - return name[(len(type_record.package_name) + 1) :] + package_name = type_record.package_name + if package_name is None: + package_name = "" + if name.startswith(package_name): + return name[(len(package_name) + 1) :] return name def lock_message(self, message: "MessageElement") -> bool: diff --git a/karapace/protobuf/google/protobuf/any.proto b/karapace/protobuf/google/protobuf/any.proto deleted file mode 100644 index e2c2042fd..000000000 --- a/karapace/protobuf/google/protobuf/any.proto +++ /dev/null @@ -1,158 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2008 Google Inc. All rights reserved. -// https://developers.google.com/protocol-buffers/ -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -syntax = "proto3"; - -package google.protobuf; - -option csharp_namespace = "Google.Protobuf.WellKnownTypes"; -option go_package = "google.golang.org/protobuf/types/known/anypb"; -option java_package = "com.google.protobuf"; -option java_outer_classname = "AnyProto"; -option java_multiple_files = true; -option objc_class_prefix = "GPB"; - -// `Any` contains an arbitrary serialized protocol buffer message along with a -// URL that describes the type of the serialized message. -// -// Protobuf library provides support to pack/unpack Any values in the form -// of utility functions or additional generated methods of the Any type. -// -// Example 1: Pack and unpack a message in C++. -// -// Foo foo = ...; -// Any any; -// any.PackFrom(foo); -// ... -// if (any.UnpackTo(&foo)) { -// ... -// } -// -// Example 2: Pack and unpack a message in Java. -// -// Foo foo = ...; -// Any any = Any.pack(foo); -// ... -// if (any.is(Foo.class)) { -// foo = any.unpack(Foo.class); -// } -// -// Example 3: Pack and unpack a message in Python. -// -// foo = Foo(...) -// any = Any() -// any.Pack(foo) -// ... -// if any.Is(Foo.DESCRIPTOR): -// any.Unpack(foo) -// ... -// -// Example 4: Pack and unpack a message in Go -// -// foo := &pb.Foo{...} -// any, err := anypb.New(foo) -// if err != nil { -// ... -// } -// ... -// foo := &pb.Foo{} -// if err := any.UnmarshalTo(foo); err != nil { -// ... -// } -// -// The pack methods provided by protobuf library will by default use -// 'type.googleapis.com/full.type.name' as the type URL and the unpack -// methods only use the fully qualified type name after the last '/' -// in the type URL, for example "foo.bar.com/x/y.z" will yield type -// name "y.z". -// -// -// JSON -// -// The JSON representation of an `Any` value uses the regular -// representation of the deserialized, embedded message, with an -// additional field `@type` which contains the type URL. Example: -// -// package google.profile; -// message Person { -// string first_name = 1; -// string last_name = 2; -// } -// -// { -// "@type": "type.googleapis.com/google.profile.Person", -// "firstName": , -// "lastName": -// } -// -// If the embedded message type is well-known and has a custom JSON -// representation, that representation will be embedded adding a field -// `value` which holds the custom JSON in addition to the `@type` -// field. Example (for message [google.protobuf.Duration][]): -// -// { -// "@type": "type.googleapis.com/google.protobuf.Duration", -// "value": "1.212s" -// } -// -message Any { - // A URL/resource name that uniquely identifies the type of the serialized - // protocol buffer message. This string must contain at least - // one "/" character. The last segment of the URL's path must represent - // the fully qualified name of the type (as in - // `path/google.protobuf.Duration`). The name should be in a canonical form - // (e.g., leading "." is not accepted). - // - // In practice, teams usually precompile into the binary all types that they - // expect it to use in the context of Any. However, for URLs which use the - // scheme `http`, `https`, or no scheme, one can optionally set up a type - // server that maps type URLs to message definitions as follows: - // - // * If no scheme is provided, `https` is assumed. - // * An HTTP GET on the URL must yield a [google.protobuf.Type][] - // value in binary format, or produce an error. - // * Applications are allowed to cache lookup results based on the - // URL, or have them precompiled into a binary to avoid any - // lookup. Therefore, binary compatibility needs to be preserved - // on changes to types. (Use versioned type names to manage - // breaking changes.) - // - // Note: this functionality is not currently available in the official - // protobuf release, and it is not used for type URLs beginning with - // type.googleapis.com. - // - // Schemes other than `http`, `https` (or the empty scheme) might be - // used with implementation specific semantics. - // - string type_url = 1; - - // Must be a valid serialized protocol buffer of the above specified type. - bytes value = 2; -} diff --git a/karapace/protobuf/google/protobuf/any_test.proto b/karapace/protobuf/google/protobuf/any_test.proto deleted file mode 100644 index 256035b44..000000000 --- a/karapace/protobuf/google/protobuf/any_test.proto +++ /dev/null @@ -1,44 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2008 Google Inc. All rights reserved. -// https://developers.google.com/protocol-buffers/ -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -syntax = "proto3"; - -package protobuf_unittest; - -import "google/protobuf/any.proto"; - -option java_outer_classname = "TestAnyProto"; - -message TestAny { - int32 int32_value = 1; - google.protobuf.Any any_value = 2; - repeated google.protobuf.Any repeated_any_value = 3; - string text = 4; -} diff --git a/karapace/protobuf/google/protobuf/api.proto b/karapace/protobuf/google/protobuf/api.proto deleted file mode 100644 index 3d598fc85..000000000 --- a/karapace/protobuf/google/protobuf/api.proto +++ /dev/null @@ -1,208 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2008 Google Inc. All rights reserved. -// https://developers.google.com/protocol-buffers/ -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -syntax = "proto3"; - -package google.protobuf; - -import "google/protobuf/source_context.proto"; -import "google/protobuf/type.proto"; - -option csharp_namespace = "Google.Protobuf.WellKnownTypes"; -option java_package = "com.google.protobuf"; -option java_outer_classname = "ApiProto"; -option java_multiple_files = true; -option objc_class_prefix = "GPB"; -option go_package = "google.golang.org/protobuf/types/known/apipb"; - -// Api is a light-weight descriptor for an API Interface. -// -// Interfaces are also described as "protocol buffer services" in some contexts, -// such as by the "service" keyword in a .proto file, but they are different -// from API Services, which represent a concrete implementation of an interface -// as opposed to simply a description of methods and bindings. They are also -// sometimes simply referred to as "APIs" in other contexts, such as the name of -// this message itself. See https://cloud.google.com/apis/design/glossary for -// detailed terminology. -message Api { - // The fully qualified name of this interface, including package name - // followed by the interface's simple name. - string name = 1; - - // The methods of this interface, in unspecified order. - repeated Method methods = 2; - - // Any metadata attached to the interface. - repeated Option options = 3; - - // A version string for this interface. If specified, must have the form - // `major-version.minor-version`, as in `1.10`. If the minor version is - // omitted, it defaults to zero. If the entire version field is empty, the - // major version is derived from the package name, as outlined below. If the - // field is not empty, the version in the package name will be verified to be - // consistent with what is provided here. - // - // The versioning schema uses [semantic - // versioning](http://semver.org) where the major version number - // indicates a breaking change and the minor version an additive, - // non-breaking change. Both version numbers are signals to users - // what to expect from different versions, and should be carefully - // chosen based on the product plan. - // - // The major version is also reflected in the package name of the - // interface, which must end in `v`, as in - // `google.feature.v1`. For major versions 0 and 1, the suffix can - // be omitted. Zero major versions must only be used for - // experimental, non-GA interfaces. - // - // - string version = 4; - - // Source context for the protocol buffer service represented by this - // message. - SourceContext source_context = 5; - - // Included interfaces. See [Mixin][]. - repeated Mixin mixins = 6; - - // The source syntax of the service. - Syntax syntax = 7; -} - -// Method represents a method of an API interface. -message Method { - // The simple name of this method. - string name = 1; - - // A URL of the input message type. - string request_type_url = 2; - - // If true, the request is streamed. - bool request_streaming = 3; - - // The URL of the output message type. - string response_type_url = 4; - - // If true, the response is streamed. - bool response_streaming = 5; - - // Any metadata attached to the method. - repeated Option options = 6; - - // The source syntax of this method. - Syntax syntax = 7; -} - -// Declares an API Interface to be included in this interface. The including -// interface must redeclare all the methods from the included interface, but -// documentation and options are inherited as follows: -// -// - If after comment and whitespace stripping, the documentation -// string of the redeclared method is empty, it will be inherited -// from the original method. -// -// - Each annotation belonging to the service config (http, -// visibility) which is not set in the redeclared method will be -// inherited. -// -// - If an http annotation is inherited, the path pattern will be -// modified as follows. Any version prefix will be replaced by the -// version of the including interface plus the [root][] path if -// specified. -// -// Example of a simple mixin: -// -// package google.acl.v1; -// service AccessControl { -// // Get the underlying ACL object. -// rpc GetAcl(GetAclRequest) returns (Acl) { -// option (google.api.http).get = "/v1/{resource=**}:getAcl"; -// } -// } -// -// package google.storage.v2; -// service Storage { -// rpc GetAcl(GetAclRequest) returns (Acl); -// -// // Get a data record. -// rpc GetData(GetDataRequest) returns (Data) { -// option (google.api.http).get = "/v2/{resource=**}"; -// } -// } -// -// Example of a mixin configuration: -// -// apis: -// - name: google.storage.v2.Storage -// mixins: -// - name: google.acl.v1.AccessControl -// -// The mixin construct implies that all methods in `AccessControl` are -// also declared with same name and request/response types in -// `Storage`. A documentation generator or annotation processor will -// see the effective `Storage.GetAcl` method after inheriting -// documentation and annotations as follows: -// -// service Storage { -// // Get the underlying ACL object. -// rpc GetAcl(GetAclRequest) returns (Acl) { -// option (google.api.http).get = "/v2/{resource=**}:getAcl"; -// } -// ... -// } -// -// Note how the version in the path pattern changed from `v1` to `v2`. -// -// If the `root` field in the mixin is specified, it should be a -// relative path under which inherited HTTP paths are placed. Example: -// -// apis: -// - name: google.storage.v2.Storage -// mixins: -// - name: google.acl.v1.AccessControl -// root: acls -// -// This implies the following inherited HTTP annotation: -// -// service Storage { -// // Get the underlying ACL object. -// rpc GetAcl(GetAclRequest) returns (Acl) { -// option (google.api.http).get = "/v2/acls/{resource=**}:getAcl"; -// } -// ... -// } -message Mixin { - // The fully qualified name of the interface which is included. - string name = 1; - - // If non-empty specifies a path under which inherited HTTP paths - // are rooted. - string root = 2; -} diff --git a/karapace/protobuf/google/protobuf/descriptor.proto b/karapace/protobuf/google/protobuf/descriptor.proto deleted file mode 100644 index 5b4d06b41..000000000 --- a/karapace/protobuf/google/protobuf/descriptor.proto +++ /dev/null @@ -1,918 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2008 Google Inc. All rights reserved. -// https://developers.google.com/protocol-buffers/ -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// Author: kenton@google.com (Kenton Varda) -// Based on original Protocol Buffers design by -// Sanjay Ghemawat, Jeff Dean, and others. -// -// The messages in this file describe the definitions found in .proto files. -// A valid .proto file can be translated directly to a FileDescriptorProto -// without any other information (e.g. without reading its imports). - - -syntax = "proto2"; - -package google.protobuf; - -option go_package = "google.golang.org/protobuf/types/descriptorpb"; -option java_package = "com.google.protobuf"; -option java_outer_classname = "DescriptorProtos"; -option csharp_namespace = "Google.Protobuf.Reflection"; -option objc_class_prefix = "GPB"; -option cc_enable_arenas = true; - -// descriptor.proto must be optimized for speed because reflection-based -// algorithms don't work during bootstrapping. -option optimize_for = SPEED; - -// The protocol compiler can output a FileDescriptorSet containing the .proto -// files it parses. -message FileDescriptorSet { - repeated FileDescriptorProto file = 1; -} - -// Describes a complete .proto file. -message FileDescriptorProto { - optional string name = 1; // file name, relative to root of source tree - optional string package = 2; // e.g. "foo", "foo.bar", etc. - - // Names of files imported by this file. - repeated string dependency = 3; - // Indexes of the public imported files in the dependency list above. - repeated int32 public_dependency = 10; - // Indexes of the weak imported files in the dependency list. - // For Google-internal migration only. Do not use. - repeated int32 weak_dependency = 11; - - // All top-level definitions in this file. - repeated DescriptorProto message_type = 4; - repeated EnumDescriptorProto enum_type = 5; - repeated ServiceDescriptorProto service = 6; - repeated FieldDescriptorProto extension = 7; - - optional FileOptions options = 8; - - // This field contains optional information about the original source code. - // You may safely remove this entire field without harming runtime - // functionality of the descriptors -- the information is needed only by - // development tools. - optional SourceCodeInfo source_code_info = 9; - - // The syntax of the proto file. - // The supported values are "proto2" and "proto3". - optional string syntax = 12; -} - -// Describes a message type. -message DescriptorProto { - optional string name = 1; - - repeated FieldDescriptorProto field = 2; - repeated FieldDescriptorProto extension = 6; - - repeated DescriptorProto nested_type = 3; - repeated EnumDescriptorProto enum_type = 4; - - message ExtensionRange { - optional int32 start = 1; // Inclusive. - optional int32 end = 2; // Exclusive. - - optional ExtensionRangeOptions options = 3; - } - repeated ExtensionRange extension_range = 5; - - repeated OneofDescriptorProto oneof_decl = 8; - - optional MessageOptions options = 7; - - // Range of reserved tag numbers. Reserved tag numbers may not be used by - // fields or extension ranges in the same message. Reserved ranges may - // not overlap. - message ReservedRange { - optional int32 start = 1; // Inclusive. - optional int32 end = 2; // Exclusive. - } - repeated ReservedRange reserved_range = 9; - // Reserved field names, which may not be used by fields in the same message. - // A given name may only be reserved once. - repeated string reserved_name = 10; -} - -message ExtensionRangeOptions { - // The parser stores options it doesn't recognize here. See above. - repeated UninterpretedOption uninterpreted_option = 999; - - - // Clients can define custom options in extensions of this message. See above. - extensions 1000 to max; -} - -// Describes a field within a message. -message FieldDescriptorProto { - enum Type { - // 0 is reserved for errors. - // Order is weird for historical reasons. - TYPE_DOUBLE = 1; - TYPE_FLOAT = 2; - // Not ZigZag encoded. Negative numbers take 10 bytes. Use TYPE_SINT64 if - // negative values are likely. - TYPE_INT64 = 3; - TYPE_UINT64 = 4; - // Not ZigZag encoded. Negative numbers take 10 bytes. Use TYPE_SINT32 if - // negative values are likely. - TYPE_INT32 = 5; - TYPE_FIXED64 = 6; - TYPE_FIXED32 = 7; - TYPE_BOOL = 8; - TYPE_STRING = 9; - // Tag-delimited aggregate. - // Group type is deprecated and not supported in proto3. However, Proto3 - // implementations should still be able to parse the group wire format and - // treat group fields as unknown fields. - TYPE_GROUP = 10; - TYPE_MESSAGE = 11; // Length-delimited aggregate. - - // New in version 2. - TYPE_BYTES = 12; - TYPE_UINT32 = 13; - TYPE_ENUM = 14; - TYPE_SFIXED32 = 15; - TYPE_SFIXED64 = 16; - TYPE_SINT32 = 17; // Uses ZigZag encoding. - TYPE_SINT64 = 18; // Uses ZigZag encoding. - } - - enum Label { - // 0 is reserved for errors - LABEL_OPTIONAL = 1; - LABEL_REQUIRED = 2; - LABEL_REPEATED = 3; - } - - optional string name = 1; - optional int32 number = 3; - optional Label label = 4; - - // If type_name is set, this need not be set. If both this and type_name - // are set, this must be one of TYPE_ENUM, TYPE_MESSAGE or TYPE_GROUP. - optional Type type = 5; - - // For message and enum types, this is the name of the type. If the name - // starts with a '.', it is fully-qualified. Otherwise, C++-like scoping - // rules are used to find the type (i.e. first the nested types within this - // message are searched, then within the parent, on up to the root - // namespace). - optional string type_name = 6; - - // For extensions, this is the name of the type being extended. It is - // resolved in the same manner as type_name. - optional string extendee = 2; - - // For numeric types, contains the original text representation of the value. - // For booleans, "true" or "false". - // For strings, contains the default text contents (not escaped in any way). - // For bytes, contains the C escaped value. All bytes >= 128 are escaped. - optional string default_value = 7; - - // If set, gives the index of a oneof in the containing type's oneof_decl - // list. This field is a member of that oneof. - optional int32 oneof_index = 9; - - // JSON name of this field. The value is set by protocol compiler. If the - // user has set a "json_name" option on this field, that option's value - // will be used. Otherwise, it's deduced from the field's name by converting - // it to camelCase. - optional string json_name = 10; - - optional FieldOptions options = 8; - - // If true, this is a proto3 "optional". When a proto3 field is optional, it - // tracks presence regardless of field type. - // - // When proto3_optional is true, this field must be belong to a oneof to - // signal to old proto3 clients that presence is tracked for this field. This - // oneof is known as a "synthetic" oneof, and this field must be its sole - // member (each proto3 optional field gets its own synthetic oneof). Synthetic - // oneofs exist in the descriptor only, and do not generate any API. Synthetic - // oneofs must be ordered after all "real" oneofs. - // - // For message fields, proto3_optional doesn't create any semantic change, - // since non-repeated message fields always track presence. However it still - // indicates the semantic detail of whether the user wrote "optional" or not. - // This can be useful for round-tripping the .proto file. For consistency we - // give message fields a synthetic oneof also, even though it is not required - // to track presence. This is especially important because the parser can't - // tell if a field is a message or an enum, so it must always create a - // synthetic oneof. - // - // Proto2 optional fields do not set this flag, because they already indicate - // optional with `LABEL_OPTIONAL`. - optional bool proto3_optional = 17; -} - -// Describes a oneof. -message OneofDescriptorProto { - optional string name = 1; - optional OneofOptions options = 2; -} - -// Describes an enum type. -message EnumDescriptorProto { - optional string name = 1; - - repeated EnumValueDescriptorProto value = 2; - - optional EnumOptions options = 3; - - // Range of reserved numeric values. Reserved values may not be used by - // entries in the same enum. Reserved ranges may not overlap. - // - // Note that this is distinct from DescriptorProto.ReservedRange in that it - // is inclusive such that it can appropriately represent the entire int32 - // domain. - message EnumReservedRange { - optional int32 start = 1; // Inclusive. - optional int32 end = 2; // Inclusive. - } - - // Range of reserved numeric values. Reserved numeric values may not be used - // by enum values in the same enum declaration. Reserved ranges may not - // overlap. - repeated EnumReservedRange reserved_range = 4; - - // Reserved enum value names, which may not be reused. A given name may only - // be reserved once. - repeated string reserved_name = 5; -} - -// Describes a value within an enum. -message EnumValueDescriptorProto { - optional string name = 1; - optional int32 number = 2; - - optional EnumValueOptions options = 3; -} - -// Describes a service. -message ServiceDescriptorProto { - optional string name = 1; - repeated MethodDescriptorProto method = 2; - - optional ServiceOptions options = 3; -} - -// Describes a method of a service. -message MethodDescriptorProto { - optional string name = 1; - - // Input and output type names. These are resolved in the same way as - // FieldDescriptorProto.type_name, but must refer to a message type. - optional string input_type = 2; - optional string output_type = 3; - - optional MethodOptions options = 4; - - // Identifies if client streams multiple client messages - optional bool client_streaming = 5 [default = false]; - // Identifies if server streams multiple server messages - optional bool server_streaming = 6 [default = false]; -} - - -// =================================================================== -// Options - -// Each of the definitions above may have "options" attached. These are -// just annotations which may cause code to be generated slightly differently -// or may contain hints for code that manipulates protocol messages. -// -// Clients may define custom options as extensions of the *Options messages. -// These extensions may not yet be known at parsing time, so the parser cannot -// store the values in them. Instead it stores them in a field in the *Options -// message called uninterpreted_option. This field must have the same name -// across all *Options messages. We then use this field to populate the -// extensions when we build a descriptor, at which point all protos have been -// parsed and so all extensions are known. -// -// Extension numbers for custom options may be chosen as follows: -// * For options which will only be used within a single application or -// organization, or for experimental options, use field numbers 50000 -// through 99999. It is up to you to ensure that you do not use the -// same number for multiple options. -// * For options which will be published and used publicly by multiple -// independent entities, e-mail protobuf-global-extension-registry@google.com -// to reserve extension numbers. Simply provide your project name (e.g. -// Objective-C plugin) and your project website (if available) -- there's no -// need to explain how you intend to use them. Usually you only need one -// extension number. You can declare multiple options with only one extension -// number by putting them in a sub-message. See the Custom Options section of -// the docs for examples: -// https://developers.google.com/protocol-buffers/docs/proto#options -// If this turns out to be popular, a web service will be set up -// to automatically assign option numbers. - -message FileOptions { - - // Sets the Java package where classes generated from this .proto will be - // placed. By default, the proto package is used, but this is often - // inappropriate because proto packages do not normally start with backwards - // domain names. - optional string java_package = 1; - - - // Controls the name of the wrapper Java class generated for the .proto file. - // That class will always contain the .proto file's getDescriptor() method as - // well as any top-level extensions defined in the .proto file. - // If java_multiple_files is disabled, then all the other classes from the - // .proto file will be nested inside the single wrapper outer class. - optional string java_outer_classname = 8; - - // If enabled, then the Java code generator will generate a separate .java - // file for each top-level message, enum, and service defined in the .proto - // file. Thus, these types will *not* be nested inside the wrapper class - // named by java_outer_classname. However, the wrapper class will still be - // generated to contain the file's getDescriptor() method as well as any - // top-level extensions defined in the file. - optional bool java_multiple_files = 10 [default = false]; - - // This option does nothing. - optional bool java_generate_equals_and_hash = 20 [deprecated=true]; - - // If set true, then the Java2 code generator will generate code that - // throws an exception whenever an attempt is made to assign a non-UTF-8 - // byte sequence to a string field. - // Message reflection will do the same. - // However, an extension field still accepts non-UTF-8 byte sequences. - // This option has no effect on when used with the lite runtime. - optional bool java_string_check_utf8 = 27 [default = false]; - - - // Generated classes can be optimized for speed or code size. - enum OptimizeMode { - SPEED = 1; // Generate complete code for parsing, serialization, - // etc. - CODE_SIZE = 2; // Use ReflectionOps to implement these methods. - LITE_RUNTIME = 3; // Generate code using MessageLite and the lite runtime. - } - optional OptimizeMode optimize_for = 9 [default = SPEED]; - - // Sets the Go package where structs generated from this .proto will be - // placed. If omitted, the Go package will be derived from the following: - // - The basename of the package import path, if provided. - // - Otherwise, the package statement in the .proto file, if present. - // - Otherwise, the basename of the .proto file, without extension. - optional string go_package = 11; - - - - - // Should generic services be generated in each language? "Generic" services - // are not specific to any particular RPC system. They are generated by the - // main code generators in each language (without additional plugins). - // Generic services were the only kind of service generation supported by - // early versions of google.protobuf. - // - // Generic services are now considered deprecated in favor of using plugins - // that generate code specific to your particular RPC system. Therefore, - // these default to false. Old code which depends on generic services should - // explicitly set them to true. - optional bool cc_generic_services = 16 [default = false]; - optional bool java_generic_services = 17 [default = false]; - optional bool py_generic_services = 18 [default = false]; - optional bool php_generic_services = 42 [default = false]; - - // Is this file deprecated? - // Depending on the target platform, this can emit Deprecated annotations - // for everything in the file, or it will be completely ignored; in the very - // least, this is a formalization for deprecating files. - optional bool deprecated = 23 [default = false]; - - // Enables the use of arenas for the proto messages in this file. This applies - // only to generated classes for C++. - optional bool cc_enable_arenas = 31 [default = true]; - - - // Sets the objective c class prefix which is prepended to all objective c - // generated classes from this .proto. There is no default. - optional string objc_class_prefix = 36; - - // Namespace for generated classes; defaults to the package. - optional string csharp_namespace = 37; - - // By default Swift generators will take the proto package and CamelCase it - // replacing '.' with underscore and use that to prefix the types/symbols - // defined. When this options is provided, they will use this value instead - // to prefix the types/symbols defined. - optional string swift_prefix = 39; - - // Sets the php class prefix which is prepended to all php generated classes - // from this .proto. Default is empty. - optional string php_class_prefix = 40; - - // Use this option to change the namespace of php generated classes. Default - // is empty. When this option is empty, the package name will be used for - // determining the namespace. - optional string php_namespace = 41; - - // Use this option to change the namespace of php generated metadata classes. - // Default is empty. When this option is empty, the proto file name will be - // used for determining the namespace. - optional string php_metadata_namespace = 44; - - // Use this option to change the package of ruby generated classes. Default - // is empty. When this option is not set, the package name will be used for - // determining the ruby package. - optional string ruby_package = 45; - - - // The parser stores options it doesn't recognize here. - // See the documentation for the "Options" section above. - repeated UninterpretedOption uninterpreted_option = 999; - - // Clients can define custom options in extensions of this message. - // See the documentation for the "Options" section above. - extensions 1000 to max; - - reserved 38; -} - -message MessageOptions { - // Set true to use the old proto1 MessageSet wire format for extensions. - // This is provided for backwards-compatibility with the MessageSet wire - // format. You should not use this for any other reason: It's less - // efficient, has fewer features, and is more complicated. - // - // The message must be defined exactly as follows: - // message Foo { - // option message_set_wire_format = true; - // extensions 4 to max; - // } - // Note that the message cannot have any defined fields; MessageSets only - // have extensions. - // - // All extensions of your type must be singular messages; e.g. they cannot - // be int32s, enums, or repeated messages. - // - // Because this is an option, the above two restrictions are not enforced by - // the protocol compiler. - optional bool message_set_wire_format = 1 [default = false]; - - // Disables the generation of the standard "descriptor()" accessor, which can - // conflict with a field of the same name. This is meant to make migration - // from proto1 easier; new code should avoid fields named "descriptor". - optional bool no_standard_descriptor_accessor = 2 [default = false]; - - // Is this message deprecated? - // Depending on the target platform, this can emit Deprecated annotations - // for the message, or it will be completely ignored; in the very least, - // this is a formalization for deprecating messages. - optional bool deprecated = 3 [default = false]; - - reserved 4, 5, 6; - - // Whether the message is an automatically generated map entry type for the - // maps field. - // - // For maps fields: - // map map_field = 1; - // The parsed descriptor looks like: - // message MapFieldEntry { - // option map_entry = true; - // optional KeyType key = 1; - // optional ValueType value = 2; - // } - // repeated MapFieldEntry map_field = 1; - // - // Implementations may choose not to generate the map_entry=true message, but - // use a native map in the target language to hold the keys and values. - // The reflection APIs in such implementations still need to work as - // if the field is a repeated message field. - // - // NOTE: Do not set the option in .proto files. Always use the maps syntax - // instead. The option should only be implicitly set by the proto compiler - // parser. - optional bool map_entry = 7; - - reserved 8; // javalite_serializable - reserved 9; // javanano_as_lite - - - // The parser stores options it doesn't recognize here. See above. - repeated UninterpretedOption uninterpreted_option = 999; - - // Clients can define custom options in extensions of this message. See above. - extensions 1000 to max; -} - -message FieldOptions { - // The ctype option instructs the C++ code generator to use a different - // representation of the field than it normally would. See the specific - // options below. This option is not yet implemented in the open source - // release -- sorry, we'll try to include it in a future version! - optional CType ctype = 1 [default = STRING]; - enum CType { - // Default mode. - STRING = 0; - - CORD = 1; - - STRING_PIECE = 2; - } - // The packed option can be enabled for repeated primitive fields to enable - // a more efficient representation on the wire. Rather than repeatedly - // writing the tag and type for each element, the entire array is encoded as - // a single length-delimited blob. In proto3, only explicit setting it to - // false will avoid using packed encoding. - optional bool packed = 2; - - // The jstype option determines the JavaScript type used for values of the - // field. The option is permitted only for 64 bit integral and fixed types - // (int64, uint64, sint64, fixed64, sfixed64). A field with jstype JS_STRING - // is represented as JavaScript string, which avoids loss of precision that - // can happen when a large value is converted to a floating point JavaScript. - // Specifying JS_NUMBER for the jstype causes the generated JavaScript code to - // use the JavaScript "number" type. The behavior of the default option - // JS_NORMAL is implementation dependent. - // - // This option is an enum to permit additional types to be added, e.g. - // goog.math.Integer. - optional JSType jstype = 6 [default = JS_NORMAL]; - enum JSType { - // Use the default type. - JS_NORMAL = 0; - - // Use JavaScript strings. - JS_STRING = 1; - - // Use JavaScript numbers. - JS_NUMBER = 2; - } - - // Should this field be parsed lazily? Lazy applies only to message-type - // fields. It means that when the outer message is initially parsed, the - // inner message's contents will not be parsed but instead stored in encoded - // form. The inner message will actually be parsed when it is first accessed. - // - // This is only a hint. Implementations are free to choose whether to use - // eager or lazy parsing regardless of the value of this option. However, - // setting this option true suggests that the protocol author believes that - // using lazy parsing on this field is worth the additional bookkeeping - // overhead typically needed to implement it. - // - // This option does not affect the public interface of any generated code; - // all method signatures remain the same. Furthermore, thread-safety of the - // interface is not affected by this option; const methods remain safe to - // call from multiple threads concurrently, while non-const methods continue - // to require exclusive access. - // - // - // Note that implementations may choose not to check required fields within - // a lazy sub-message. That is, calling IsInitialized() on the outer message - // may return true even if the inner message has missing required fields. - // This is necessary because otherwise the inner message would have to be - // parsed in order to perform the check, defeating the purpose of lazy - // parsing. An implementation which chooses not to check required fields - // must be consistent about it. That is, for any particular sub-message, the - // implementation must either *always* check its required fields, or *never* - // check its required fields, regardless of whether or not the message has - // been parsed. - // - // As of May 2022, lazy verifies the contents of the byte stream during - // parsing. An invalid byte stream will cause the overall parsing to fail. - optional bool lazy = 5 [default = false]; - - // unverified_lazy does no correctness checks on the byte stream. This should - // only be used where lazy with verification is prohibitive for performance - // reasons. - optional bool unverified_lazy = 15 [default = false]; - - // Is this field deprecated? - // Depending on the target platform, this can emit Deprecated annotations - // for accessors, or it will be completely ignored; in the very least, this - // is a formalization for deprecating fields. - optional bool deprecated = 3 [default = false]; - - // For Google-internal migration only. Do not use. - optional bool weak = 10 [default = false]; - - - // The parser stores options it doesn't recognize here. See above. - repeated UninterpretedOption uninterpreted_option = 999; - - // Clients can define custom options in extensions of this message. See above. - extensions 1000 to max; - - reserved 4; // removed jtype -} - -message OneofOptions { - // The parser stores options it doesn't recognize here. See above. - repeated UninterpretedOption uninterpreted_option = 999; - - // Clients can define custom options in extensions of this message. See above. - extensions 1000 to max; -} - -message EnumOptions { - - // Set this option to true to allow mapping different tag names to the same - // value. - optional bool allow_alias = 2; - - // Is this enum deprecated? - // Depending on the target platform, this can emit Deprecated annotations - // for the enum, or it will be completely ignored; in the very least, this - // is a formalization for deprecating enums. - optional bool deprecated = 3 [default = false]; - - reserved 5; // javanano_as_lite - - // The parser stores options it doesn't recognize here. See above. - repeated UninterpretedOption uninterpreted_option = 999; - - // Clients can define custom options in extensions of this message. See above. - extensions 1000 to max; -} - -message EnumValueOptions { - // Is this enum value deprecated? - // Depending on the target platform, this can emit Deprecated annotations - // for the enum value, or it will be completely ignored; in the very least, - // this is a formalization for deprecating enum values. - optional bool deprecated = 1 [default = false]; - - // The parser stores options it doesn't recognize here. See above. - repeated UninterpretedOption uninterpreted_option = 999; - - // Clients can define custom options in extensions of this message. See above. - extensions 1000 to max; -} - -message ServiceOptions { - - // Note: Field numbers 1 through 32 are reserved for Google's internal RPC - // framework. We apologize for hoarding these numbers to ourselves, but - // we were already using them long before we decided to release Protocol - // Buffers. - - // Is this service deprecated? - // Depending on the target platform, this can emit Deprecated annotations - // for the service, or it will be completely ignored; in the very least, - // this is a formalization for deprecating services. - optional bool deprecated = 33 [default = false]; - - // The parser stores options it doesn't recognize here. See above. - repeated UninterpretedOption uninterpreted_option = 999; - - // Clients can define custom options in extensions of this message. See above. - extensions 1000 to max; -} - -message MethodOptions { - - // Note: Field numbers 1 through 32 are reserved for Google's internal RPC - // framework. We apologize for hoarding these numbers to ourselves, but - // we were already using them long before we decided to release Protocol - // Buffers. - - // Is this method deprecated? - // Depending on the target platform, this can emit Deprecated annotations - // for the method, or it will be completely ignored; in the very least, - // this is a formalization for deprecating methods. - optional bool deprecated = 33 [default = false]; - - // Is this method side-effect-free (or safe in HTTP parlance), or idempotent, - // or neither? HTTP based RPC implementation may choose GET verb for safe - // methods, and PUT verb for idempotent methods instead of the default POST. - enum IdempotencyLevel { - IDEMPOTENCY_UNKNOWN = 0; - NO_SIDE_EFFECTS = 1; // implies idempotent - IDEMPOTENT = 2; // idempotent, but may have side effects - } - optional IdempotencyLevel idempotency_level = 34 - [default = IDEMPOTENCY_UNKNOWN]; - - // The parser stores options it doesn't recognize here. See above. - repeated UninterpretedOption uninterpreted_option = 999; - - // Clients can define custom options in extensions of this message. See above. - extensions 1000 to max; -} - - -// A message representing a option the parser does not recognize. This only -// appears in options protos created by the compiler::Parser class. -// DescriptorPool resolves these when building Descriptor objects. Therefore, -// options protos in descriptor objects (e.g. returned by Descriptor::options(), -// or produced by Descriptor::CopyTo()) will never have UninterpretedOptions -// in them. -message UninterpretedOption { - // The name of the uninterpreted option. Each string represents a segment in - // a dot-separated name. is_extension is true iff a segment represents an - // extension (denoted with parentheses in options specs in .proto files). - // E.g.,{ ["foo", false], ["bar.baz", true], ["moo", false] } represents - // "foo.(bar.baz).moo". - message NamePart { - required string name_part = 1; - required bool is_extension = 2; - } - repeated NamePart name = 2; - - // The value of the uninterpreted option, in whatever type the tokenizer - // identified it as during parsing. Exactly one of these should be set. - optional string identifier_value = 3; - optional uint64 positive_int_value = 4; - optional int64 negative_int_value = 5; - optional double double_value = 6; - optional bytes string_value = 7; - optional string aggregate_value = 8; -} - -// =================================================================== -// Optional source code info - -// Encapsulates information about the original source file from which a -// FileDescriptorProto was generated. -message SourceCodeInfo { - // A Location identifies a piece of source code in a .proto file which - // corresponds to a particular definition. This information is intended - // to be useful to IDEs, code indexers, documentation generators, and similar - // tools. - // - // For example, say we have a file like: - // message Foo { - // optional string foo = 1; - // } - // Let's look at just the field definition: - // optional string foo = 1; - // ^ ^^ ^^ ^ ^^^ - // a bc de f ghi - // We have the following locations: - // span path represents - // [a,i) [ 4, 0, 2, 0 ] The whole field definition. - // [a,b) [ 4, 0, 2, 0, 4 ] The label (optional). - // [c,d) [ 4, 0, 2, 0, 5 ] The type (string). - // [e,f) [ 4, 0, 2, 0, 1 ] The name (foo). - // [g,h) [ 4, 0, 2, 0, 3 ] The number (1). - // - // Notes: - // - A location may refer to a repeated field itself (i.e. not to any - // particular index within it). This is used whenever a set of elements are - // logically enclosed in a single code segment. For example, an entire - // extend block (possibly containing multiple extension definitions) will - // have an outer location whose path refers to the "extensions" repeated - // field without an index. - // - Multiple locations may have the same path. This happens when a single - // logical declaration is spread out across multiple places. The most - // obvious example is the "extend" block again -- there may be multiple - // extend blocks in the same scope, each of which will have the same path. - // - A location's span is not always a subset of its parent's span. For - // example, the "extendee" of an extension declaration appears at the - // beginning of the "extend" block and is shared by all extensions within - // the block. - // - Just because a location's span is a subset of some other location's span - // does not mean that it is a descendant. For example, a "group" defines - // both a type and a field in a single declaration. Thus, the locations - // corresponding to the type and field and their components will overlap. - // - Code which tries to interpret locations should probably be designed to - // ignore those that it doesn't understand, as more types of locations could - // be recorded in the future. - repeated Location location = 1; - message Location { - // Identifies which part of the FileDescriptorProto was defined at this - // location. - // - // Each element is a field number or an index. They form a path from - // the root FileDescriptorProto to the place where the definition occurs. - // For example, this path: - // [ 4, 3, 2, 7, 1 ] - // refers to: - // file.message_type(3) // 4, 3 - // .field(7) // 2, 7 - // .name() // 1 - // This is because FileDescriptorProto.message_type has field number 4: - // repeated DescriptorProto message_type = 4; - // and DescriptorProto.field has field number 2: - // repeated FieldDescriptorProto field = 2; - // and FieldDescriptorProto.name has field number 1: - // optional string name = 1; - // - // Thus, the above path gives the location of a field name. If we removed - // the last element: - // [ 4, 3, 2, 7 ] - // this path refers to the whole field declaration (from the beginning - // of the label to the terminating semicolon). - repeated int32 path = 1 [packed = true]; - - // Always has exactly three or four elements: start line, start column, - // end line (optional, otherwise assumed same as start line), end column. - // These are packed into a single field for efficiency. Note that line - // and column numbers are zero-based -- typically you will want to add - // 1 to each before displaying to a user. - repeated int32 span = 2 [packed = true]; - - // If this SourceCodeInfo represents a complete declaration, these are any - // comments appearing before and after the declaration which appear to be - // attached to the declaration. - // - // A series of line comments appearing on consecutive lines, with no other - // tokens appearing on those lines, will be treated as a single comment. - // - // leading_detached_comments will keep paragraphs of comments that appear - // before (but not connected to) the current element. Each paragraph, - // separated by empty lines, will be one comment element in the repeated - // field. - // - // Only the comment content is provided; comment markers (e.g. //) are - // stripped out. For block comments, leading whitespace and an asterisk - // will be stripped from the beginning of each line other than the first. - // Newlines are included in the output. - // - // Examples: - // - // optional int32 foo = 1; // Comment attached to foo. - // // Comment attached to bar. - // optional int32 bar = 2; - // - // optional string baz = 3; - // // Comment attached to baz. - // // Another line attached to baz. - // - // // Comment attached to moo. - // // - // // Another line attached to moo. - // optional double moo = 4; - // - // // Detached comment for corge. This is not leading or trailing comments - // // to moo or corge because there are blank lines separating it from - // // both. - // - // // Detached comment for corge paragraph 2. - // - // optional string corge = 5; - // /* Block comment attached - // * to corge. Leading asterisks - // * will be removed. */ - // /* Block comment attached to - // * grault. */ - // optional int32 grault = 6; - // - // // ignored detached comments. - optional string leading_comments = 3; - optional string trailing_comments = 4; - repeated string leading_detached_comments = 6; - } -} - -// Describes the relationship between generated code and its original source -// file. A GeneratedCodeInfo message is associated with only one generated -// source file, but may contain references to different source .proto files. -message GeneratedCodeInfo { - // An Annotation connects some span of text in generated code to an element - // of its generating .proto file. - repeated Annotation annotation = 1; - message Annotation { - // Identifies the element in the original source .proto file. This field - // is formatted the same as SourceCodeInfo.Location.path. - repeated int32 path = 1 [packed = true]; - - // Identifies the filesystem path to the original source .proto. - optional string source_file = 2; - - // Identifies the starting offset in bytes in the generated code - // that relates to the identified object. - optional int32 begin = 3; - - // Identifies the ending offset in bytes in the generated code that - // relates to the identified offset. The end offset should be one past - // the last relevant byte (so the length of the text = end - begin). - optional int32 end = 4; - } -} diff --git a/karapace/protobuf/google/protobuf/duration.proto b/karapace/protobuf/google/protobuf/duration.proto deleted file mode 100644 index 81c3e369f..000000000 --- a/karapace/protobuf/google/protobuf/duration.proto +++ /dev/null @@ -1,116 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2008 Google Inc. All rights reserved. -// https://developers.google.com/protocol-buffers/ -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -syntax = "proto3"; - -package google.protobuf; - -option csharp_namespace = "Google.Protobuf.WellKnownTypes"; -option cc_enable_arenas = true; -option go_package = "google.golang.org/protobuf/types/known/durationpb"; -option java_package = "com.google.protobuf"; -option java_outer_classname = "DurationProto"; -option java_multiple_files = true; -option objc_class_prefix = "GPB"; - -// A Duration represents a signed, fixed-length span of time represented -// as a count of seconds and fractions of seconds at nanosecond -// resolution. It is independent of any calendar and concepts like "day" -// or "month". It is related to Timestamp in that the difference between -// two Timestamp values is a Duration and it can be added or subtracted -// from a Timestamp. Range is approximately +-10,000 years. -// -// # Examples -// -// Example 1: Compute Duration from two Timestamps in pseudo code. -// -// Timestamp start = ...; -// Timestamp end = ...; -// Duration duration = ...; -// -// duration.seconds = end.seconds - start.seconds; -// duration.nanos = end.nanos - start.nanos; -// -// if (duration.seconds < 0 && duration.nanos > 0) { -// duration.seconds += 1; -// duration.nanos -= 1000000000; -// } else if (duration.seconds > 0 && duration.nanos < 0) { -// duration.seconds -= 1; -// duration.nanos += 1000000000; -// } -// -// Example 2: Compute Timestamp from Timestamp + Duration in pseudo code. -// -// Timestamp start = ...; -// Duration duration = ...; -// Timestamp end = ...; -// -// end.seconds = start.seconds + duration.seconds; -// end.nanos = start.nanos + duration.nanos; -// -// if (end.nanos < 0) { -// end.seconds -= 1; -// end.nanos += 1000000000; -// } else if (end.nanos >= 1000000000) { -// end.seconds += 1; -// end.nanos -= 1000000000; -// } -// -// Example 3: Compute Duration from datetime.timedelta in Python. -// -// td = datetime.timedelta(days=3, minutes=10) -// duration = Duration() -// duration.FromTimedelta(td) -// -// # JSON Mapping -// -// In JSON format, the Duration type is encoded as a string rather than an -// object, where the string ends in the suffix "s" (indicating seconds) and -// is preceded by the number of seconds, with nanoseconds expressed as -// fractional seconds. For example, 3 seconds with 0 nanoseconds should be -// encoded in JSON format as "3s", while 3 seconds and 1 nanosecond should -// be expressed in JSON format as "3.000000001s", and 3 seconds and 1 -// microsecond should be expressed in JSON format as "3.000001s". -// -// -message Duration { - // Signed seconds of the span of time. Must be from -315,576,000,000 - // to +315,576,000,000 inclusive. Note: these bounds are computed from: - // 60 sec/min * 60 min/hr * 24 hr/day * 365.25 days/year * 10000 years - int64 seconds = 1; - - // Signed fractions of a second at nanosecond resolution of the span - // of time. Durations less than one second are represented with a 0 - // `seconds` field and a positive or negative `nanos` field. For durations - // of one second or more, a non-zero value for the `nanos` field must be - // of the same sign as the `seconds` field. Must be from -999,999,999 - // to +999,999,999 inclusive. - int32 nanos = 2; -} diff --git a/karapace/protobuf/google/protobuf/empty.proto b/karapace/protobuf/google/protobuf/empty.proto deleted file mode 100644 index 222746219..000000000 --- a/karapace/protobuf/google/protobuf/empty.proto +++ /dev/null @@ -1,51 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2008 Google Inc. All rights reserved. -// https://developers.google.com/protocol-buffers/ -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -syntax = "proto3"; - -package google.protobuf; - -option csharp_namespace = "Google.Protobuf.WellKnownTypes"; -option go_package = "google.golang.org/protobuf/types/known/emptypb"; -option java_package = "com.google.protobuf"; -option java_outer_classname = "EmptyProto"; -option java_multiple_files = true; -option objc_class_prefix = "GPB"; -option cc_enable_arenas = true; - -// A generic empty message that you can re-use to avoid defining duplicated -// empty messages in your APIs. A typical example is to use it as the request -// or the response type of an API method. For instance: -// -// service Foo { -// rpc Bar(google.protobuf.Empty) returns (google.protobuf.Empty); -// } -// -message Empty {} diff --git a/karapace/protobuf/google/protobuf/field_mask.proto b/karapace/protobuf/google/protobuf/field_mask.proto deleted file mode 100644 index 6b5104f18..000000000 --- a/karapace/protobuf/google/protobuf/field_mask.proto +++ /dev/null @@ -1,245 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2008 Google Inc. All rights reserved. -// https://developers.google.com/protocol-buffers/ -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -syntax = "proto3"; - -package google.protobuf; - -option csharp_namespace = "Google.Protobuf.WellKnownTypes"; -option java_package = "com.google.protobuf"; -option java_outer_classname = "FieldMaskProto"; -option java_multiple_files = true; -option objc_class_prefix = "GPB"; -option go_package = "google.golang.org/protobuf/types/known/fieldmaskpb"; -option cc_enable_arenas = true; - -// `FieldMask` represents a set of symbolic field paths, for example: -// -// paths: "f.a" -// paths: "f.b.d" -// -// Here `f` represents a field in some root message, `a` and `b` -// fields in the message found in `f`, and `d` a field found in the -// message in `f.b`. -// -// Field masks are used to specify a subset of fields that should be -// returned by a get operation or modified by an update operation. -// Field masks also have a custom JSON encoding (see below). -// -// # Field Masks in Projections -// -// When used in the context of a projection, a response message or -// sub-message is filtered by the API to only contain those fields as -// specified in the mask. For example, if the mask in the previous -// example is applied to a response message as follows: -// -// f { -// a : 22 -// b { -// d : 1 -// x : 2 -// } -// y : 13 -// } -// z: 8 -// -// The result will not contain specific values for fields x,y and z -// (their value will be set to the default, and omitted in proto text -// output): -// -// -// f { -// a : 22 -// b { -// d : 1 -// } -// } -// -// A repeated field is not allowed except at the last position of a -// paths string. -// -// If a FieldMask object is not present in a get operation, the -// operation applies to all fields (as if a FieldMask of all fields -// had been specified). -// -// Note that a field mask does not necessarily apply to the -// top-level response message. In case of a REST get operation, the -// field mask applies directly to the response, but in case of a REST -// list operation, the mask instead applies to each individual message -// in the returned resource list. In case of a REST custom method, -// other definitions may be used. Where the mask applies will be -// clearly documented together with its declaration in the API. In -// any case, the effect on the returned resource/resources is required -// behavior for APIs. -// -// # Field Masks in Update Operations -// -// A field mask in update operations specifies which fields of the -// targeted resource are going to be updated. The API is required -// to only change the values of the fields as specified in the mask -// and leave the others untouched. If a resource is passed in to -// describe the updated values, the API ignores the values of all -// fields not covered by the mask. -// -// If a repeated field is specified for an update operation, new values will -// be appended to the existing repeated field in the target resource. Note that -// a repeated field is only allowed in the last position of a `paths` string. -// -// If a sub-message is specified in the last position of the field mask for an -// update operation, then new value will be merged into the existing sub-message -// in the target resource. -// -// For example, given the target message: -// -// f { -// b { -// d: 1 -// x: 2 -// } -// c: [1] -// } -// -// And an update message: -// -// f { -// b { -// d: 10 -// } -// c: [2] -// } -// -// then if the field mask is: -// -// paths: ["f.b", "f.c"] -// -// then the result will be: -// -// f { -// b { -// d: 10 -// x: 2 -// } -// c: [1, 2] -// } -// -// An implementation may provide options to override this default behavior for -// repeated and message fields. -// -// In order to reset a field's value to the default, the field must -// be in the mask and set to the default value in the provided resource. -// Hence, in order to reset all fields of a resource, provide a default -// instance of the resource and set all fields in the mask, or do -// not provide a mask as described below. -// -// If a field mask is not present on update, the operation applies to -// all fields (as if a field mask of all fields has been specified). -// Note that in the presence of schema evolution, this may mean that -// fields the client does not know and has therefore not filled into -// the request will be reset to their default. If this is unwanted -// behavior, a specific service may require a client to always specify -// a field mask, producing an error if not. -// -// As with get operations, the location of the resource which -// describes the updated values in the request message depends on the -// operation kind. In any case, the effect of the field mask is -// required to be honored by the API. -// -// ## Considerations for HTTP REST -// -// The HTTP kind of an update operation which uses a field mask must -// be set to PATCH instead of PUT in order to satisfy HTTP semantics -// (PUT must only be used for full updates). -// -// # JSON Encoding of Field Masks -// -// In JSON, a field mask is encoded as a single string where paths are -// separated by a comma. Fields name in each path are converted -// to/from lower-camel naming conventions. -// -// As an example, consider the following message declarations: -// -// message Profile { -// User user = 1; -// Photo photo = 2; -// } -// message User { -// string display_name = 1; -// string address = 2; -// } -// -// In proto a field mask for `Profile` may look as such: -// -// mask { -// paths: "user.display_name" -// paths: "photo" -// } -// -// In JSON, the same mask is represented as below: -// -// { -// mask: "user.displayName,photo" -// } -// -// # Field Masks and Oneof Fields -// -// Field masks treat fields in oneofs just as regular fields. Consider the -// following message: -// -// message SampleMessage { -// oneof test_oneof { -// string name = 4; -// SubMessage sub_message = 9; -// } -// } -// -// The field mask can be: -// -// mask { -// paths: "name" -// } -// -// Or: -// -// mask { -// paths: "sub_message" -// } -// -// Note that oneof type names ("test_oneof" in this case) cannot be used in -// paths. -// -// ## Field Mask Verification -// -// The implementation of any API method which has a FieldMask type field in the -// request should verify the included field paths, and return an -// `INVALID_ARGUMENT` error if any path is unmappable. -message FieldMask { - // The set of field mask paths. - repeated string paths = 1; -} diff --git a/karapace/protobuf/google/protobuf/map_lite_unittest.proto b/karapace/protobuf/google/protobuf/map_lite_unittest.proto deleted file mode 100644 index 7f104315c..000000000 --- a/karapace/protobuf/google/protobuf/map_lite_unittest.proto +++ /dev/null @@ -1,131 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2008 Google Inc. All rights reserved. -// https://developers.google.com/protocol-buffers/ -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -syntax = "proto2"; - -package protobuf_unittest; - -import "google/protobuf/unittest_lite.proto"; - -option cc_enable_arenas = true; -option optimize_for = LITE_RUNTIME; - -message TestMapLite { - map map_int32_int32 = 1; - map map_int64_int64 = 2; - map map_uint32_uint32 = 3; - map map_uint64_uint64 = 4; - map map_sint32_sint32 = 5; - map map_sint64_sint64 = 6; - map map_fixed32_fixed32 = 7; - map map_fixed64_fixed64 = 8; - map map_sfixed32_sfixed32 = 9; - map map_sfixed64_sfixed64 = 10; - map map_int32_float = 11; - map map_int32_double = 12; - map map_bool_bool = 13; - map map_string_string = 14; - map map_int32_bytes = 15; - map map_int32_enum = 16; - map map_int32_foreign_message = 17; - map teboring = 18; -} - -message TestArenaMapLite { - map map_int32_int32 = 1; - map map_int64_int64 = 2; - map map_uint32_uint32 = 3; - map map_uint64_uint64 = 4; - map map_sint32_sint32 = 5; - map map_sint64_sint64 = 6; - map map_fixed32_fixed32 = 7; - map map_fixed64_fixed64 = 8; - map map_sfixed32_sfixed32 = 9; - map map_sfixed64_sfixed64 = 10; - map map_int32_float = 11; - map map_int32_double = 12; - map map_bool_bool = 13; - map map_string_string = 14; - map map_int32_bytes = 15; - map map_int32_enum = 16; - map map_int32_foreign_message = 17; -} - -// Test embedded message with required fields -message TestRequiredMessageMapLite { - map map_field = 1; -} - -message TestEnumMapLite { - map known_map_field = 101; - map unknown_map_field = 102; -} - -message TestEnumMapPlusExtraLite { - map known_map_field = 101; - map unknown_map_field = 102; -} - -message TestMessageMapLite { - map map_int32_message = 1; -} - -enum Proto2MapEnumLite { - PROTO2_MAP_ENUM_FOO_LITE = 0; - PROTO2_MAP_ENUM_BAR_LITE = 1; - PROTO2_MAP_ENUM_BAZ_LITE = 2; -} - -enum Proto2MapEnumPlusExtraLite { - E_PROTO2_MAP_ENUM_FOO_LITE = 0; - E_PROTO2_MAP_ENUM_BAR_LITE = 1; - E_PROTO2_MAP_ENUM_BAZ_LITE = 2; - E_PROTO2_MAP_ENUM_EXTRA_LITE = 3; -} - -enum MapEnumLite { - MAP_ENUM_FOO_LITE = 0; - MAP_ENUM_BAR_LITE = 1; - MAP_ENUM_BAZ_LITE = 2; -} - -message TestRequiredLite { - required int32 a = 1; - required int32 b = 2; - required int32 c = 3; - - extend TestAllExtensionsLite { - optional TestRequiredLite single = 1000; - } -} - -message ForeignMessageArenaLite { - optional int32 c = 1; -} diff --git a/karapace/protobuf/google/protobuf/map_proto2_unittest.proto b/karapace/protobuf/google/protobuf/map_proto2_unittest.proto deleted file mode 100644 index 20d58f903..000000000 --- a/karapace/protobuf/google/protobuf/map_proto2_unittest.proto +++ /dev/null @@ -1,91 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2008 Google Inc. All rights reserved. -// https://developers.google.com/protocol-buffers/ -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -syntax = "proto2"; -option cc_enable_arenas = true; - -import "google/protobuf/unittest_import.proto"; - -// We don't put this in a package within proto2 because we need to make sure -// that the generated code doesn't depend on being in the proto2 namespace. -// In map_test_util.h we do "using namespace unittest = protobuf_unittest". -package protobuf_unittest; - -enum Proto2MapEnum { - PROTO2_MAP_ENUM_FOO = 0; - PROTO2_MAP_ENUM_BAR = 1; - PROTO2_MAP_ENUM_BAZ = 2; -} - -enum Proto2MapEnumPlusExtra { - E_PROTO2_MAP_ENUM_FOO = 0; - E_PROTO2_MAP_ENUM_BAR = 1; - E_PROTO2_MAP_ENUM_BAZ = 2; - E_PROTO2_MAP_ENUM_EXTRA = 3; -} - -message TestEnumMap { - map known_map_field = 101; - map unknown_map_field = 102; -} - -message TestEnumMapPlusExtra { - map known_map_field = 101; - map unknown_map_field = 102; -} - -message TestImportEnumMap { - map import_enum_amp = 1; -} - -message TestIntIntMap { - map m = 1; -} - -// Test all key types: string, plus the non-floating-point scalars. -message TestMaps { - map m_int32 = 1; - map m_int64 = 2; - map m_uint32 = 3; - map m_uint64 = 4; - map m_sint32 = 5; - map m_sint64 = 6; - map m_fixed32 = 7; - map m_fixed64 = 8; - map m_sfixed32 = 9; - map m_sfixed64 = 10; - map m_bool = 11; - map m_string = 12; -} - -// Test maps in submessages. -message TestSubmessageMaps { - optional TestMaps m = 1; -} diff --git a/karapace/protobuf/google/protobuf/map_unittest.proto b/karapace/protobuf/google/protobuf/map_unittest.proto deleted file mode 100644 index 263ef61f8..000000000 --- a/karapace/protobuf/google/protobuf/map_unittest.proto +++ /dev/null @@ -1,125 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2008 Google Inc. All rights reserved. -// https://developers.google.com/protocol-buffers/ -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -syntax = "proto3"; - -option cc_enable_arenas = true; - -import "google/protobuf/unittest.proto"; - -// We don't put this in a package within proto2 because we need to make sure -// that the generated code doesn't depend on being in the proto2 namespace. -// In map_test_util.h we do "using namespace unittest = protobuf_unittest". -package protobuf_unittest; - -// Tests maps. -message TestMap { - map map_int32_int32 = 1; - map map_int64_int64 = 2; - map map_uint32_uint32 = 3; - map map_uint64_uint64 = 4; - map map_sint32_sint32 = 5; - map map_sint64_sint64 = 6; - map map_fixed32_fixed32 = 7; - map map_fixed64_fixed64 = 8; - map map_sfixed32_sfixed32 = 9; - map map_sfixed64_sfixed64 = 10; - map map_int32_float = 11; - map map_int32_double = 12; - map map_bool_bool = 13; - map map_string_string = 14; - map map_int32_bytes = 15; - map map_int32_enum = 16; - map map_int32_foreign_message = 17; - map map_string_foreign_message = 18; - map map_int32_all_types = 19; -} - -message TestMapSubmessage { - TestMap test_map = 1; -} - -message TestMessageMap { - map map_int32_message = 1; -} - -// Two map fields share the same entry default instance. -message TestSameTypeMap { - map map1 = 1; - map map2 = 2; -} - - -enum MapEnum { - MAP_ENUM_FOO = 0; - MAP_ENUM_BAR = 1; - MAP_ENUM_BAZ = 2; -} - -// Test embedded message with required fields -message TestRequiredMessageMap { - map map_field = 1; -} - -message TestArenaMap { - map map_int32_int32 = 1; - map map_int64_int64 = 2; - map map_uint32_uint32 = 3; - map map_uint64_uint64 = 4; - map map_sint32_sint32 = 5; - map map_sint64_sint64 = 6; - map map_fixed32_fixed32 = 7; - map map_fixed64_fixed64 = 8; - map map_sfixed32_sfixed32 = 9; - map map_sfixed64_sfixed64 = 10; - map map_int32_float = 11; - map map_int32_double = 12; - map map_bool_bool = 13; - map map_string_string = 14; - map map_int32_bytes = 15; - map map_int32_enum = 16; - map map_int32_foreign_message = 17; -} - -// Previously, message containing enum called Type cannot be used as value of -// map field. -message MessageContainingEnumCalledType { - enum Type { TYPE_FOO = 0; } - map type = 1; -} - -// Previously, message cannot contain map field called "entry". -message MessageContainingMapCalledEntry { - map entry = 1; -} - -message TestRecursiveMapMessage { - map a = 1; -} diff --git a/karapace/protobuf/google/protobuf/source_context.proto b/karapace/protobuf/google/protobuf/source_context.proto deleted file mode 100644 index 06bfc43a7..000000000 --- a/karapace/protobuf/google/protobuf/source_context.proto +++ /dev/null @@ -1,48 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2008 Google Inc. All rights reserved. -// https://developers.google.com/protocol-buffers/ -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -syntax = "proto3"; - -package google.protobuf; - -option csharp_namespace = "Google.Protobuf.WellKnownTypes"; -option java_package = "com.google.protobuf"; -option java_outer_classname = "SourceContextProto"; -option java_multiple_files = true; -option objc_class_prefix = "GPB"; -option go_package = "google.golang.org/protobuf/types/known/sourcecontextpb"; - -// `SourceContext` represents information about the source of a -// protobuf element, like the file in which it is defined. -message SourceContext { - // The path-qualified name of the .proto file that contained the associated - // protobuf element. For example: `"google/protobuf/source_context.proto"`. - string file_name = 1; -} diff --git a/karapace/protobuf/google/protobuf/struct.proto b/karapace/protobuf/google/protobuf/struct.proto deleted file mode 100644 index 0ac843ca0..000000000 --- a/karapace/protobuf/google/protobuf/struct.proto +++ /dev/null @@ -1,95 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2008 Google Inc. All rights reserved. -// https://developers.google.com/protocol-buffers/ -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -syntax = "proto3"; - -package google.protobuf; - -option csharp_namespace = "Google.Protobuf.WellKnownTypes"; -option cc_enable_arenas = true; -option go_package = "google.golang.org/protobuf/types/known/structpb"; -option java_package = "com.google.protobuf"; -option java_outer_classname = "StructProto"; -option java_multiple_files = true; -option objc_class_prefix = "GPB"; - -// `Struct` represents a structured data value, consisting of fields -// which map to dynamically typed values. In some languages, `Struct` -// might be supported by a native representation. For example, in -// scripting languages like JS a struct is represented as an -// object. The details of that representation are described together -// with the proto support for the language. -// -// The JSON representation for `Struct` is JSON object. -message Struct { - // Unordered map of dynamically typed values. - map fields = 1; -} - -// `Value` represents a dynamically typed value which can be either -// null, a number, a string, a boolean, a recursive struct value, or a -// list of values. A producer of value is expected to set one of these -// variants. Absence of any variant indicates an error. -// -// The JSON representation for `Value` is JSON value. -message Value { - // The kind of value. - oneof kind { - // Represents a null value. - NullValue null_value = 1; - // Represents a double value. - double number_value = 2; - // Represents a string value. - string string_value = 3; - // Represents a boolean value. - bool bool_value = 4; - // Represents a structured value. - Struct struct_value = 5; - // Represents a repeated `Value`. - ListValue list_value = 6; - } -} - -// `NullValue` is a singleton enumeration to represent the null value for the -// `Value` type union. -// -// The JSON representation for `NullValue` is JSON `null`. -enum NullValue { - // Null value. - NULL_VALUE = 0; -} - -// `ListValue` is a wrapper around a repeated field of values. -// -// The JSON representation for `ListValue` is JSON array. -message ListValue { - // Repeated field of dynamically typed values. - repeated Value values = 1; -} diff --git a/karapace/protobuf/google/protobuf/timestamp.proto b/karapace/protobuf/google/protobuf/timestamp.proto deleted file mode 100644 index 3b2df6d91..000000000 --- a/karapace/protobuf/google/protobuf/timestamp.proto +++ /dev/null @@ -1,147 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2008 Google Inc. All rights reserved. -// https://developers.google.com/protocol-buffers/ -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -syntax = "proto3"; - -package google.protobuf; - -option csharp_namespace = "Google.Protobuf.WellKnownTypes"; -option cc_enable_arenas = true; -option go_package = "google.golang.org/protobuf/types/known/timestamppb"; -option java_package = "com.google.protobuf"; -option java_outer_classname = "TimestampProto"; -option java_multiple_files = true; -option objc_class_prefix = "GPB"; - -// A Timestamp represents a point in time independent of any time zone or local -// calendar, encoded as a count of seconds and fractions of seconds at -// nanosecond resolution. The count is relative to an epoch at UTC midnight on -// January 1, 1970, in the proleptic Gregorian calendar which extends the -// Gregorian calendar backwards to year one. -// -// All minutes are 60 seconds long. Leap seconds are "smeared" so that no leap -// second table is needed for interpretation, using a [24-hour linear -// smear](https://developers.google.com/time/smear). -// -// The range is from 0001-01-01T00:00:00Z to 9999-12-31T23:59:59.999999999Z. By -// restricting to that range, we ensure that we can convert to and from [RFC -// 3339](https://www.ietf.org/rfc/rfc3339.txt) date strings. -// -// # Examples -// -// Example 1: Compute Timestamp from POSIX `time()`. -// -// Timestamp timestamp; -// timestamp.set_seconds(time(NULL)); -// timestamp.set_nanos(0); -// -// Example 2: Compute Timestamp from POSIX `gettimeofday()`. -// -// struct timeval tv; -// gettimeofday(&tv, NULL); -// -// Timestamp timestamp; -// timestamp.set_seconds(tv.tv_sec); -// timestamp.set_nanos(tv.tv_usec * 1000); -// -// Example 3: Compute Timestamp from Win32 `GetSystemTimeAsFileTime()`. -// -// FILETIME ft; -// GetSystemTimeAsFileTime(&ft); -// UINT64 ticks = (((UINT64)ft.dwHighDateTime) << 32) | ft.dwLowDateTime; -// -// // A Windows tick is 100 nanoseconds. Windows epoch 1601-01-01T00:00:00Z -// // is 11644473600 seconds before Unix epoch 1970-01-01T00:00:00Z. -// Timestamp timestamp; -// timestamp.set_seconds((INT64) ((ticks / 10000000) - 11644473600LL)); -// timestamp.set_nanos((INT32) ((ticks % 10000000) * 100)); -// -// Example 4: Compute Timestamp from Java `System.currentTimeMillis()`. -// -// long millis = System.currentTimeMillis(); -// -// Timestamp timestamp = Timestamp.newBuilder().setSeconds(millis / 1000) -// .setNanos((int) ((millis % 1000) * 1000000)).build(); -// -// -// Example 5: Compute Timestamp from Java `Instant.now()`. -// -// Instant now = Instant.now(); -// -// Timestamp timestamp = -// Timestamp.newBuilder().setSeconds(now.getEpochSecond()) -// .setNanos(now.getNano()).build(); -// -// -// Example 6: Compute Timestamp from current time in Python. -// -// timestamp = Timestamp() -// timestamp.GetCurrentTime() -// -// # JSON Mapping -// -// In JSON format, the Timestamp type is encoded as a string in the -// [RFC 3339](https://www.ietf.org/rfc/rfc3339.txt) format. That is, the -// format is "{year}-{month}-{day}T{hour}:{min}:{sec}[.{frac_sec}]Z" -// where {year} is always expressed using four digits while {month}, {day}, -// {hour}, {min}, and {sec} are zero-padded to two digits each. The fractional -// seconds, which can go up to 9 digits (i.e. up to 1 nanosecond resolution), -// are optional. The "Z" suffix indicates the timezone ("UTC"); the timezone -// is required. A proto3 JSON serializer should always use UTC (as indicated by -// "Z") when printing the Timestamp type and a proto3 JSON parser should be -// able to accept both UTC and other timezones (as indicated by an offset). -// -// For example, "2017-01-15T01:30:15.01Z" encodes 15.01 seconds past -// 01:30 UTC on January 15, 2017. -// -// In JavaScript, one can convert a Date object to this format using the -// standard -// [toISOString()](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Date/toISOString) -// method. In Python, a standard `datetime.datetime` object can be converted -// to this format using -// [`strftime`](https://docs.python.org/2/library/time.html#time.strftime) with -// the time format spec '%Y-%m-%dT%H:%M:%S.%fZ'. Likewise, in Java, one can use -// the Joda Time's [`ISODateTimeFormat.dateTime()`]( -// http://www.joda.org/joda-time/apidocs/org/joda/time/format/ISODateTimeFormat.html#dateTime%2D%2D -// ) to obtain a formatter capable of generating timestamps in this format. -// -// -message Timestamp { - // Represents seconds of UTC time since Unix epoch - // 1970-01-01T00:00:00Z. Must be from 0001-01-01T00:00:00Z to - // 9999-12-31T23:59:59Z inclusive. - int64 seconds = 1; - - // Non-negative fractions of a second at nanosecond resolution. Negative - // second values with fractions must still have non-negative nanos values - // that count forward in time. Must be from 0 to 999,999,999 - // inclusive. - int32 nanos = 2; -} diff --git a/karapace/protobuf/google/protobuf/type.proto b/karapace/protobuf/google/protobuf/type.proto deleted file mode 100644 index d3f6a68b8..000000000 --- a/karapace/protobuf/google/protobuf/type.proto +++ /dev/null @@ -1,187 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2008 Google Inc. All rights reserved. -// https://developers.google.com/protocol-buffers/ -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -syntax = "proto3"; - -package google.protobuf; - -import "google/protobuf/any.proto"; -import "google/protobuf/source_context.proto"; - -option csharp_namespace = "Google.Protobuf.WellKnownTypes"; -option cc_enable_arenas = true; -option java_package = "com.google.protobuf"; -option java_outer_classname = "TypeProto"; -option java_multiple_files = true; -option objc_class_prefix = "GPB"; -option go_package = "google.golang.org/protobuf/types/known/typepb"; - -// A protocol buffer message type. -message Type { - // The fully qualified message name. - string name = 1; - // The list of fields. - repeated Field fields = 2; - // The list of types appearing in `oneof` definitions in this type. - repeated string oneofs = 3; - // The protocol buffer options. - repeated Option options = 4; - // The source context. - SourceContext source_context = 5; - // The source syntax. - Syntax syntax = 6; -} - -// A single field of a message type. -message Field { - // Basic field types. - enum Kind { - // Field type unknown. - TYPE_UNKNOWN = 0; - // Field type double. - TYPE_DOUBLE = 1; - // Field type float. - TYPE_FLOAT = 2; - // Field type int64. - TYPE_INT64 = 3; - // Field type uint64. - TYPE_UINT64 = 4; - // Field type int32. - TYPE_INT32 = 5; - // Field type fixed64. - TYPE_FIXED64 = 6; - // Field type fixed32. - TYPE_FIXED32 = 7; - // Field type bool. - TYPE_BOOL = 8; - // Field type string. - TYPE_STRING = 9; - // Field type group. Proto2 syntax only, and deprecated. - TYPE_GROUP = 10; - // Field type message. - TYPE_MESSAGE = 11; - // Field type bytes. - TYPE_BYTES = 12; - // Field type uint32. - TYPE_UINT32 = 13; - // Field type enum. - TYPE_ENUM = 14; - // Field type sfixed32. - TYPE_SFIXED32 = 15; - // Field type sfixed64. - TYPE_SFIXED64 = 16; - // Field type sint32. - TYPE_SINT32 = 17; - // Field type sint64. - TYPE_SINT64 = 18; - } - - // Whether a field is optional, required, or repeated. - enum Cardinality { - // For fields with unknown cardinality. - CARDINALITY_UNKNOWN = 0; - // For optional fields. - CARDINALITY_OPTIONAL = 1; - // For required fields. Proto2 syntax only. - CARDINALITY_REQUIRED = 2; - // For repeated fields. - CARDINALITY_REPEATED = 3; - } - - // The field type. - Kind kind = 1; - // The field cardinality. - Cardinality cardinality = 2; - // The field number. - int32 number = 3; - // The field name. - string name = 4; - // The field type URL, without the scheme, for message or enumeration - // types. Example: `"type.googleapis.com/google.protobuf.Timestamp"`. - string type_url = 6; - // The index of the field type in `Type.oneofs`, for message or enumeration - // types. The first type has index 1; zero means the type is not in the list. - int32 oneof_index = 7; - // Whether to use alternative packed wire representation. - bool packed = 8; - // The protocol buffer options. - repeated Option options = 9; - // The field JSON name. - string json_name = 10; - // The string value of the default value of this field. Proto2 syntax only. - string default_value = 11; -} - -// Enum type definition. -message Enum { - // Enum type name. - string name = 1; - // Enum value definitions. - repeated EnumValue enumvalue = 2; - // Protocol buffer options. - repeated Option options = 3; - // The source context. - SourceContext source_context = 4; - // The source syntax. - Syntax syntax = 5; -} - -// Enum value definition. -message EnumValue { - // Enum value name. - string name = 1; - // Enum value number. - int32 number = 2; - // Protocol buffer options. - repeated Option options = 3; -} - -// A protocol buffer option, which can be attached to a message, field, -// enumeration, etc. -message Option { - // The option's name. For protobuf built-in options (options defined in - // descriptor.proto), this is the short name. For example, `"map_entry"`. - // For custom options, it should be the fully-qualified name. For example, - // `"google.api.http"`. - string name = 1; - // The option's value packed in an Any message. If the value is a primitive, - // the corresponding wrapper type defined in google/protobuf/wrappers.proto - // should be used. If the value is an enum, it should be stored as an int32 - // value using the google.protobuf.Int32Value type. - Any value = 2; -} - -// The syntax in which a protocol buffer element is defined. -enum Syntax { - // Syntax `proto2`. - SYNTAX_PROTO2 = 0; - // Syntax `proto3`. - SYNTAX_PROTO3 = 1; -} diff --git a/karapace/protobuf/google/protobuf/wrappers.proto b/karapace/protobuf/google/protobuf/wrappers.proto deleted file mode 100644 index d49dd53c8..000000000 --- a/karapace/protobuf/google/protobuf/wrappers.proto +++ /dev/null @@ -1,123 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2008 Google Inc. All rights reserved. -// https://developers.google.com/protocol-buffers/ -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// Wrappers for primitive (non-message) types. These types are useful -// for embedding primitives in the `google.protobuf.Any` type and for places -// where we need to distinguish between the absence of a primitive -// typed field and its default value. -// -// These wrappers have no meaningful use within repeated fields as they lack -// the ability to detect presence on individual elements. -// These wrappers have no meaningful use within a map or a oneof since -// individual entries of a map or fields of a oneof can already detect presence. - -syntax = "proto3"; - -package google.protobuf; - -option csharp_namespace = "Google.Protobuf.WellKnownTypes"; -option cc_enable_arenas = true; -option go_package = "google.golang.org/protobuf/types/known/wrapperspb"; -option java_package = "com.google.protobuf"; -option java_outer_classname = "WrappersProto"; -option java_multiple_files = true; -option objc_class_prefix = "GPB"; - -// Wrapper message for `double`. -// -// The JSON representation for `DoubleValue` is JSON number. -message DoubleValue { - // The double value. - double value = 1; -} - -// Wrapper message for `float`. -// -// The JSON representation for `FloatValue` is JSON number. -message FloatValue { - // The float value. - float value = 1; -} - -// Wrapper message for `int64`. -// -// The JSON representation for `Int64Value` is JSON string. -message Int64Value { - // The int64 value. - int64 value = 1; -} - -// Wrapper message for `uint64`. -// -// The JSON representation for `UInt64Value` is JSON string. -message UInt64Value { - // The uint64 value. - uint64 value = 1; -} - -// Wrapper message for `int32`. -// -// The JSON representation for `Int32Value` is JSON number. -message Int32Value { - // The int32 value. - int32 value = 1; -} - -// Wrapper message for `uint32`. -// -// The JSON representation for `UInt32Value` is JSON number. -message UInt32Value { - // The uint32 value. - uint32 value = 1; -} - -// Wrapper message for `bool`. -// -// The JSON representation for `BoolValue` is JSON `true` and `false`. -message BoolValue { - // The bool value. - bool value = 1; -} - -// Wrapper message for `string`. -// -// The JSON representation for `StringValue` is JSON string. -message StringValue { - // The string value. - string value = 1; -} - -// Wrapper message for `bytes`. -// -// The JSON representation for `BytesValue` is JSON string. -message BytesValue { - // The bytes value. - bytes value = 1; -} diff --git a/karapace/protobuf/google/type/BUILD.bazel b/karapace/protobuf/google/type/BUILD.bazel deleted file mode 100644 index a7ad4d4d2..000000000 --- a/karapace/protobuf/google/type/BUILD.bazel +++ /dev/null @@ -1,536 +0,0 @@ -load("@rules_proto//proto:defs.bzl", "proto_library") - -# This is an API workspace, having public visibility by default makes perfect sense. -package(default_visibility = ["//visibility:public"]) - -############################################################################## -# Common -############################################################################## -proto_library( - name = "calendar_period_proto", - srcs = ["calendar_period.proto"], -) - -proto_library( - name = "color_proto", - srcs = ["color.proto"], - deps = [ - "@com_google_protobuf//:wrappers_proto", - ], -) - -proto_library( - name = "date_proto", - srcs = ["date.proto"], -) - -proto_library( - name = "datetime_proto", - srcs = ["datetime.proto"], - deps = [ - "@com_google_protobuf//:duration_proto", - ], -) - -proto_library( - name = "dayofweek_proto", - srcs = ["dayofweek.proto"], -) - -proto_library( - name = "decimal_proto", - srcs = ["decimal.proto"], -) - -proto_library( - name = "expr_proto", - srcs = ["expr.proto"], -) - -proto_library( - name = "fraction_proto", - srcs = ["fraction.proto"], -) - -proto_library( - name = "interval_proto", - srcs = ["interval.proto"], - deps = [ - "@com_google_protobuf//:timestamp_proto", - ], -) - -proto_library( - name = "latlng_proto", - srcs = ["latlng.proto"], -) - -proto_library( - name = "localized_text_proto", - srcs = ["localized_text.proto"], -) - -proto_library( - name = "money_proto", - srcs = ["money.proto"], -) - -proto_library( - name = "month_proto", - srcs = ["month.proto"], -) - -proto_library( - name = "phone_number_proto", - srcs = ["phone_number.proto"], -) - -proto_library( - name = "postal_address_proto", - srcs = ["postal_address.proto"], -) - -proto_library( - name = "quaternion_proto", - srcs = ["quaternion.proto"], -) - -proto_library( - name = "timeofday_proto", - srcs = ["timeofday.proto"], -) - -############################################################################## -# Java -############################################################################## -load("@com_google_googleapis_imports//:imports.bzl", - "java_gapic_assembly_gradle_pkg", - "java_proto_library") - -java_proto_library( - name = "type_java_proto", - deps = [ - ":calendar_period_proto", - ":color_proto", - ":date_proto", - ":datetime_proto", - ":dayofweek_proto", - ":decimal_proto", - ":expr_proto", - ":fraction_proto", - ":interval_proto", - ":latlng_proto", - ":localized_text_proto", - ":money_proto", - ":month_proto", - ":phone_number_proto", - ":postal_address_proto", - ":quaternion_proto", - ":timeofday_proto", - ], -) - -# Please DO-NOT-REMOVE this section. -# This is required to generate java files for these protos. -# Open Source Packages -java_gapic_assembly_gradle_pkg( - name = "google-type-java", - deps = [ - ":type_java_proto", - ":calendar_period_proto", - ":color_proto", - ":date_proto", - ":datetime_proto", - ":dayofweek_proto", - ":decimal_proto", - ":expr_proto", - ":fraction_proto", - ":interval_proto", - ":latlng_proto", - ":localized_text_proto", - ":money_proto", - ":month_proto", - ":phone_number_proto", - ":postal_address_proto", - ":quaternion_proto", - ":timeofday_proto", - ], -) - - -############################################################################## -# Go -############################################################################## -load("@com_google_googleapis_imports//:imports.bzl", "go_proto_library") - -go_proto_library( - name = "calendar_period_go_proto", - importpath = "google.golang.org/genproto/googleapis/type/calendarperiod", - protos = [":calendar_period_proto"], -) - -go_proto_library( - name = "color_go_proto", - importpath = "google.golang.org/genproto/googleapis/type/color", - protos = [":color_proto"], -) - -go_proto_library( - name = "date_go_proto", - importpath = "google.golang.org/genproto/googleapis/type/date", - protos = [":date_proto"], -) - -go_proto_library( - name = "datetime_go_proto", - importpath = "google.golang.org/genproto/googleapis/type/datetime", - protos = [":datetime_proto"], -) - -go_proto_library( - name = "dayofweek_go_proto", - importpath = "google.golang.org/genproto/googleapis/type/dayofweek", - protos = [":dayofweek_proto"], -) - -go_proto_library( - name = "decimal_go_proto", - importpath = "google.golang.org/genproto/googleapis/type/decimal", - protos = [":decimal_proto"], -) - -go_proto_library( - name = "expr_go_proto", - importpath = "google.golang.org/genproto/googleapis/type/expr", - protos = [":expr_proto"], -) - -go_proto_library( - name = "fraction_go_proto", - importpath = "google.golang.org/genproto/googleapis/type/fraction", - protos = [":fraction_proto"], -) - -go_proto_library( - name = "interval_go_proto", - importpath = "google.golang.org/genproto/googleapis/type/interval", - protos = [":interval_proto"], -) - -go_proto_library( - name = "latlng_go_proto", - importpath = "google.golang.org/genproto/googleapis/type/latlng", - protos = [":latlng_proto"], -) - -go_proto_library( - name = "localized_text_go_proto", - importpath = "google.golang.org/genproto/googleapis/type/localized_text", - protos = [":localized_text_proto"], -) - -go_proto_library( - name = "money_go_proto", - importpath = "google.golang.org/genproto/googleapis/type/money", - protos = [":money_proto"], -) - -go_proto_library( - name = "month_go_proto", - importpath = "google.golang.org/genproto/googleapis/type/month", - protos = [":month_proto"], -) - -go_proto_library( - name = "phone_number_go_proto", - importpath = "google.golang.org/genproto/googleapis/type/phone_number", - protos = [":phone_number_proto"], -) - -go_proto_library( - name = "postaladdress_go_proto", - importpath = "google.golang.org/genproto/googleapis/type/postaladdress", - protos = [":postal_address_proto"], -) - -go_proto_library( - name = "quaternion_go_proto", - importpath = "google.golang.org/genproto/googleapis/type/quaternion", - protos = [":quaternion_proto"], -) - -go_proto_library( - name = "timeofday_go_proto", - importpath = "google.golang.org/genproto/googleapis/type/timeofday", - protos = [":timeofday_proto"], -) - -############################################################################## -# C++ -############################################################################## -load( - "@com_google_googleapis_imports//:imports.bzl", - "cc_proto_library", -) - -cc_proto_library( - name = "calendar_period_cc_proto", - deps = [":calendar_period_proto"], -) - -cc_proto_library( - name = "color_cc_proto", - deps = [":color_proto"], -) - -cc_proto_library( - name = "date_cc_proto", - deps = [":date_proto"], -) - -cc_proto_library( - name = "datetime_cc_proto", - deps = [":datetime_proto"], -) - -cc_proto_library( - name = "dayofweek_cc_proto", - deps = [":dayofweek_proto"], -) - -cc_proto_library( - name = "decimal_cc_proto", - deps = [":decimal_proto"], -) - -cc_proto_library( - name = "expr_cc_proto", - deps = [":expr_proto"], -) - -cc_proto_library( - name = "fraction_cc_proto", - deps = [":fraction_proto"], -) - -cc_proto_library( - name = "interval_cc_proto", - deps = [":interval_proto"], -) - -cc_proto_library( - name = "latlng_cc_proto", - deps = [":latlng_proto"], -) - -cc_proto_library( - name = "money_cc_proto", - deps = [":money_proto"], -) - -cc_proto_library( - name = "month_cc_proto", - deps = [":month_proto"], -) - -cc_proto_library( - name = "phone_number_cc_proto", - deps = [":phone_number_proto"], -) - -cc_proto_library( - name = "postal_address_cc_proto", - deps = [":postal_address_proto"], -) - -cc_proto_library( - name = "quaternion_cc_proto", - deps = [":quaternion_proto"], -) - -cc_proto_library( - name = "timeofday_cc_proto", - deps = [":timeofday_proto"], -) - -############################################################################## -# Python -############################################################################## -load( - "@com_google_googleapis_imports//:imports.bzl", - "py_proto_library", -) - -py_proto_library( - name = "calendar_period_py_proto", - deps = [":calendar_period_proto"], -) - -py_proto_library( - name = "color_py_proto", - deps = [":color_proto"], -) - -py_proto_library( - name = "date_py_proto", - deps = [":date_proto"], -) - -py_proto_library( - name = "datetime_py_proto", - deps = [":datetime_proto"], -) - -py_proto_library( - name = "dayofweek_py_proto", - deps = [":dayofweek_proto"], -) - -py_proto_library( - name = "decimal_py_proto", - deps = [":decimal_proto"], -) - -py_proto_library( - name = "expr_py_proto", - deps = [":expr_proto"], -) - -py_proto_library( - name = "fraction_py_proto", - deps = [":fraction_proto"], -) - -py_proto_library( - name = "interval_py_proto", - deps = [":interval_proto"], -) - -py_proto_library( - name = "latlng_py_proto", - deps = [":latlng_proto"], -) - -py_proto_library( - name = "localized_text_py_proto", - deps = [":localized_text_proto"], -) - -py_proto_library( - name = "money_py_proto", - deps = [":money_proto"], -) - -py_proto_library( - name = "month_py_proto", - deps = [":month_proto"], -) - -py_proto_library( - name = "phone_number_py_proto", - deps = [":phone_number_proto"], -) - -py_proto_library( - name = "postal_address_py_proto", - deps = [":postal_address_proto"], -) - -py_proto_library( - name = "quaternion_py_proto", - deps = [":quaternion_proto"], -) - -py_proto_library( - name = "timeofday_py_proto", - deps = [":timeofday_proto"], -) - -############################################################################## -# C# -############################################################################## - -load( - "@com_google_googleapis_imports//:imports.bzl", - "csharp_proto_library", -) - -csharp_proto_library( - name = "calendar_period_csharp_proto", - deps = [":calendar_period_proto"], -) - -csharp_proto_library( - name = "color_csharp_proto", - deps = [":color_proto"], -) - -csharp_proto_library( - name = "date_csharp_proto", - deps = [":date_proto"], -) - -csharp_proto_library( - name = "datetime_csharp_proto", - deps = [":datetime_proto"], -) - -csharp_proto_library( - name = "dayofweek_csharp_proto", - deps = [":dayofweek_proto"], -) - -csharp_proto_library( - name = "decimal_csharp_proto", - deps = [":decimal_proto"], -) - -csharp_proto_library( - name = "expr_csharp_proto", - deps = [":expr_proto"], -) - -csharp_proto_library( - name = "fraction_csharp_proto", - deps = [":fraction_proto"], -) - -csharp_proto_library( - name = "interval_csharp_proto", - deps = [":interval_proto"], -) - -csharp_proto_library( - name = "latlng_csharp_proto", - deps = [":latlng_proto"], -) - -csharp_proto_library( - name = "localized_text_csharp_proto", - deps = [":localized_text_proto"], -) - -csharp_proto_library( - name = "money_csharp_proto", - deps = [":money_proto"], -) - -csharp_proto_library( - name = "month_csharp_proto", - deps = [":month_proto"], -) - -csharp_proto_library( - name = "phone_number_csharp_proto", - deps = [":phone_number_proto"], -) - -csharp_proto_library( - name = "postal_address_csharp_proto", - deps = [":postal_address_proto"], -) - -csharp_proto_library( - name = "quaternion_csharp_proto", - deps = [":quaternion_proto"], -) diff --git a/karapace/protobuf/google/type/README.md b/karapace/protobuf/google/type/README.md deleted file mode 100644 index de6a835d7..000000000 --- a/karapace/protobuf/google/type/README.md +++ /dev/null @@ -1,16 +0,0 @@ -# Google Common Types - -This package contains definitions of common types for Google APIs. -All types defined in this package are suitable for different APIs to -exchange data, and will never break binary compatibility. They should -have design quality comparable to major programming languages like -Java and C#. - -NOTE: Some common types are defined in the package `google.protobuf` -as they are directly supported by Protocol Buffers compiler and -runtime. Those types are called Well-Known Types. - -## Java Utilities - -A set of Java utilities for the Common Types are provided in the -`//java/com/google/type/util/` package. diff --git a/karapace/protobuf/google/type/calendar_period.proto b/karapace/protobuf/google/type/calendar_period.proto deleted file mode 100644 index 82f5690b7..000000000 --- a/karapace/protobuf/google/type/calendar_period.proto +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright 2021 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package google.type; - -option go_package = "google.golang.org/genproto/googleapis/type/calendarperiod;calendarperiod"; -option java_multiple_files = true; -option java_outer_classname = "CalendarPeriodProto"; -option java_package = "com.google.type"; -option objc_class_prefix = "GTP"; - -// A `CalendarPeriod` represents the abstract concept of a time period that has -// a canonical start. Grammatically, "the start of the current -// `CalendarPeriod`." All calendar times begin at midnight UTC. -enum CalendarPeriod { - // Undefined period, raises an error. - CALENDAR_PERIOD_UNSPECIFIED = 0; - - // A day. - DAY = 1; - - // A week. Weeks begin on Monday, following - // [ISO 8601](https://en.wikipedia.org/wiki/ISO_week_date). - WEEK = 2; - - // A fortnight. The first calendar fortnight of the year begins at the start - // of week 1 according to - // [ISO 8601](https://en.wikipedia.org/wiki/ISO_week_date). - FORTNIGHT = 3; - - // A month. - MONTH = 4; - - // A quarter. Quarters start on dates 1-Jan, 1-Apr, 1-Jul, and 1-Oct of each - // year. - QUARTER = 5; - - // A half-year. Half-years start on dates 1-Jan and 1-Jul. - HALF = 6; - - // A year. - YEAR = 7; -} diff --git a/karapace/protobuf/google/type/color.proto b/karapace/protobuf/google/type/color.proto deleted file mode 100644 index 5dc85a6a3..000000000 --- a/karapace/protobuf/google/type/color.proto +++ /dev/null @@ -1,174 +0,0 @@ -// Copyright 2021 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package google.type; - -import "google/protobuf/wrappers.proto"; - -option cc_enable_arenas = true; -option go_package = "google.golang.org/genproto/googleapis/type/color;color"; -option java_multiple_files = true; -option java_outer_classname = "ColorProto"; -option java_package = "com.google.type"; -option objc_class_prefix = "GTP"; - -// Represents a color in the RGBA color space. This representation is designed -// for simplicity of conversion to/from color representations in various -// languages over compactness. For example, the fields of this representation -// can be trivially provided to the constructor of `java.awt.Color` in Java; it -// can also be trivially provided to UIColor's `+colorWithRed:green:blue:alpha` -// method in iOS; and, with just a little work, it can be easily formatted into -// a CSS `rgba()` string in JavaScript. -// -// This reference page doesn't carry information about the absolute color -// space -// that should be used to interpret the RGB value (e.g. sRGB, Adobe RGB, -// DCI-P3, BT.2020, etc.). By default, applications should assume the sRGB color -// space. -// -// When color equality needs to be decided, implementations, unless -// documented otherwise, treat two colors as equal if all their red, -// green, blue, and alpha values each differ by at most 1e-5. -// -// Example (Java): -// -// import com.google.type.Color; -// -// // ... -// public static java.awt.Color fromProto(Color protocolor) { -// float alpha = protocolor.hasAlpha() -// ? protocolor.getAlpha().getValue() -// : 1.0; -// -// return new java.awt.Color( -// protocolor.getRed(), -// protocolor.getGreen(), -// protocolor.getBlue(), -// alpha); -// } -// -// public static Color toProto(java.awt.Color color) { -// float red = (float) color.getRed(); -// float green = (float) color.getGreen(); -// float blue = (float) color.getBlue(); -// float denominator = 255.0; -// Color.Builder resultBuilder = -// Color -// .newBuilder() -// .setRed(red / denominator) -// .setGreen(green / denominator) -// .setBlue(blue / denominator); -// int alpha = color.getAlpha(); -// if (alpha != 255) { -// result.setAlpha( -// FloatValue -// .newBuilder() -// .setValue(((float) alpha) / denominator) -// .build()); -// } -// return resultBuilder.build(); -// } -// // ... -// -// Example (iOS / Obj-C): -// -// // ... -// static UIColor* fromProto(Color* protocolor) { -// float red = [protocolor red]; -// float green = [protocolor green]; -// float blue = [protocolor blue]; -// FloatValue* alpha_wrapper = [protocolor alpha]; -// float alpha = 1.0; -// if (alpha_wrapper != nil) { -// alpha = [alpha_wrapper value]; -// } -// return [UIColor colorWithRed:red green:green blue:blue alpha:alpha]; -// } -// -// static Color* toProto(UIColor* color) { -// CGFloat red, green, blue, alpha; -// if (![color getRed:&red green:&green blue:&blue alpha:&alpha]) { -// return nil; -// } -// Color* result = [[Color alloc] init]; -// [result setRed:red]; -// [result setGreen:green]; -// [result setBlue:blue]; -// if (alpha <= 0.9999) { -// [result setAlpha:floatWrapperWithValue(alpha)]; -// } -// [result autorelease]; -// return result; -// } -// // ... -// -// Example (JavaScript): -// -// // ... -// -// var protoToCssColor = function(rgb_color) { -// var redFrac = rgb_color.red || 0.0; -// var greenFrac = rgb_color.green || 0.0; -// var blueFrac = rgb_color.blue || 0.0; -// var red = Math.floor(redFrac * 255); -// var green = Math.floor(greenFrac * 255); -// var blue = Math.floor(blueFrac * 255); -// -// if (!('alpha' in rgb_color)) { -// return rgbToCssColor(red, green, blue); -// } -// -// var alphaFrac = rgb_color.alpha.value || 0.0; -// var rgbParams = [red, green, blue].join(','); -// return ['rgba(', rgbParams, ',', alphaFrac, ')'].join(''); -// }; -// -// var rgbToCssColor = function(red, green, blue) { -// var rgbNumber = new Number((red << 16) | (green << 8) | blue); -// var hexString = rgbNumber.toString(16); -// var missingZeros = 6 - hexString.length; -// var resultBuilder = ['#']; -// for (var i = 0; i < missingZeros; i++) { -// resultBuilder.push('0'); -// } -// resultBuilder.push(hexString); -// return resultBuilder.join(''); -// }; -// -// // ... -message Color { - // The amount of red in the color as a value in the interval [0, 1]. - float red = 1; - - // The amount of green in the color as a value in the interval [0, 1]. - float green = 2; - - // The amount of blue in the color as a value in the interval [0, 1]. - float blue = 3; - - // The fraction of this color that should be applied to the pixel. That is, - // the final pixel color is defined by the equation: - // - // `pixel color = alpha * (this color) + (1.0 - alpha) * (background color)` - // - // This means that a value of 1.0 corresponds to a solid color, whereas - // a value of 0.0 corresponds to a completely transparent color. This - // uses a wrapper message rather than a simple float scalar so that it is - // possible to distinguish between a default value and the value being unset. - // If omitted, this color object is rendered as a solid color - // (as if the alpha value had been explicitly given a value of 1.0). - google.protobuf.FloatValue alpha = 4; -} diff --git a/karapace/protobuf/google/type/date.proto b/karapace/protobuf/google/type/date.proto deleted file mode 100644 index e4e730e6f..000000000 --- a/karapace/protobuf/google/type/date.proto +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright 2021 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package google.type; - -option cc_enable_arenas = true; -option go_package = "google.golang.org/genproto/googleapis/type/date;date"; -option java_multiple_files = true; -option java_outer_classname = "DateProto"; -option java_package = "com.google.type"; -option objc_class_prefix = "GTP"; - -// Represents a whole or partial calendar date, such as a birthday. The time of -// day and time zone are either specified elsewhere or are insignificant. The -// date is relative to the Gregorian Calendar. This can represent one of the -// following: -// -// * A full date, with non-zero year, month, and day values -// * A month and day value, with a zero year, such as an anniversary -// * A year on its own, with zero month and day values -// * A year and month value, with a zero day, such as a credit card expiration -// date -// -// Related types are [google.type.TimeOfDay][google.type.TimeOfDay] and -// `google.protobuf.Timestamp`. -message Date { - // Year of the date. Must be from 1 to 9999, or 0 to specify a date without - // a year. - int32 year = 1; - - // Month of a year. Must be from 1 to 12, or 0 to specify a year without a - // month and day. - int32 month = 2; - - // Day of a month. Must be from 1 to 31 and valid for the year and month, or 0 - // to specify a year by itself or a year and month where the day isn't - // significant. - int32 day = 3; -} diff --git a/karapace/protobuf/google/type/datetime.proto b/karapace/protobuf/google/type/datetime.proto deleted file mode 100644 index cfed85d70..000000000 --- a/karapace/protobuf/google/type/datetime.proto +++ /dev/null @@ -1,104 +0,0 @@ -// Copyright 2021 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package google.type; - -import "google/protobuf/duration.proto"; - -option cc_enable_arenas = true; -option go_package = "google.golang.org/genproto/googleapis/type/datetime;datetime"; -option java_multiple_files = true; -option java_outer_classname = "DateTimeProto"; -option java_package = "com.google.type"; -option objc_class_prefix = "GTP"; - -// Represents civil time (or occasionally physical time). -// -// This type can represent a civil time in one of a few possible ways: -// -// * When utc_offset is set and time_zone is unset: a civil time on a calendar -// day with a particular offset from UTC. -// * When time_zone is set and utc_offset is unset: a civil time on a calendar -// day in a particular time zone. -// * When neither time_zone nor utc_offset is set: a civil time on a calendar -// day in local time. -// -// The date is relative to the Proleptic Gregorian Calendar. -// -// If year is 0, the DateTime is considered not to have a specific year. month -// and day must have valid, non-zero values. -// -// This type may also be used to represent a physical time if all the date and -// time fields are set and either case of the `time_offset` oneof is set. -// Consider using `Timestamp` message for physical time instead. If your use -// case also would like to store the user's timezone, that can be done in -// another field. -// -// This type is more flexible than some applications may want. Make sure to -// document and validate your application's limitations. -message DateTime { - // Optional. Year of date. Must be from 1 to 9999, or 0 if specifying a - // datetime without a year. - int32 year = 1; - - // Required. Month of year. Must be from 1 to 12. - int32 month = 2; - - // Required. Day of month. Must be from 1 to 31 and valid for the year and - // month. - int32 day = 3; - - // Required. Hours of day in 24 hour format. Should be from 0 to 23. An API - // may choose to allow the value "24:00:00" for scenarios like business - // closing time. - int32 hours = 4; - - // Required. Minutes of hour of day. Must be from 0 to 59. - int32 minutes = 5; - - // Required. Seconds of minutes of the time. Must normally be from 0 to 59. An - // API may allow the value 60 if it allows leap-seconds. - int32 seconds = 6; - - // Required. Fractions of seconds in nanoseconds. Must be from 0 to - // 999,999,999. - int32 nanos = 7; - - // Optional. Specifies either the UTC offset or the time zone of the DateTime. - // Choose carefully between them, considering that time zone data may change - // in the future (for example, a country modifies their DST start/end dates, - // and future DateTimes in the affected range had already been stored). - // If omitted, the DateTime is considered to be in local time. - oneof time_offset { - // UTC offset. Must be whole seconds, between -18 hours and +18 hours. - // For example, a UTC offset of -4:00 would be represented as - // { seconds: -14400 }. - google.protobuf.Duration utc_offset = 8; - - // Time zone. - TimeZone time_zone = 9; - } -} - -// Represents a time zone from the -// [IANA Time Zone Database](https://www.iana.org/time-zones). -message TimeZone { - // IANA Time Zone Database time zone, e.g. "America/New_York". - string id = 1; - - // Optional. IANA Time Zone Database version number, e.g. "2019a". - string version = 2; -} diff --git a/karapace/protobuf/google/type/dayofweek.proto b/karapace/protobuf/google/type/dayofweek.proto deleted file mode 100644 index 4c80c62ec..000000000 --- a/karapace/protobuf/google/type/dayofweek.proto +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2021 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package google.type; - -option go_package = "google.golang.org/genproto/googleapis/type/dayofweek;dayofweek"; -option java_multiple_files = true; -option java_outer_classname = "DayOfWeekProto"; -option java_package = "com.google.type"; -option objc_class_prefix = "GTP"; - -// Represents a day of the week. -enum DayOfWeek { - // The day of the week is unspecified. - DAY_OF_WEEK_UNSPECIFIED = 0; - - // Monday - MONDAY = 1; - - // Tuesday - TUESDAY = 2; - - // Wednesday - WEDNESDAY = 3; - - // Thursday - THURSDAY = 4; - - // Friday - FRIDAY = 5; - - // Saturday - SATURDAY = 6; - - // Sunday - SUNDAY = 7; -} diff --git a/karapace/protobuf/google/type/decimal.proto b/karapace/protobuf/google/type/decimal.proto deleted file mode 100644 index beb18a5d8..000000000 --- a/karapace/protobuf/google/type/decimal.proto +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright 2021 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package google.type; - -option cc_enable_arenas = true; -option go_package = "google.golang.org/genproto/googleapis/type/decimal;decimal"; -option java_multiple_files = true; -option java_outer_classname = "DecimalProto"; -option java_package = "com.google.type"; -option objc_class_prefix = "GTP"; - -// A representation of a decimal value, such as 2.5. Clients may convert values -// into language-native decimal formats, such as Java's [BigDecimal][] or -// Python's [decimal.Decimal][]. -// -// [BigDecimal]: -// https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/math/BigDecimal.html -// [decimal.Decimal]: https://docs.python.org/3/library/decimal.html -message Decimal { - // The decimal value, as a string. - // - // The string representation consists of an optional sign, `+` (`U+002B`) - // or `-` (`U+002D`), followed by a sequence of zero or more decimal digits - // ("the integer"), optionally followed by a fraction, optionally followed - // by an exponent. - // - // The fraction consists of a decimal point followed by zero or more decimal - // digits. The string must contain at least one digit in either the integer - // or the fraction. The number formed by the sign, the integer and the - // fraction is referred to as the significand. - // - // The exponent consists of the character `e` (`U+0065`) or `E` (`U+0045`) - // followed by one or more decimal digits. - // - // Services **should** normalize decimal values before storing them by: - // - // - Removing an explicitly-provided `+` sign (`+2.5` -> `2.5`). - // - Replacing a zero-length integer value with `0` (`.5` -> `0.5`). - // - Coercing the exponent character to lower-case (`2.5E8` -> `2.5e8`). - // - Removing an explicitly-provided zero exponent (`2.5e0` -> `2.5`). - // - // Services **may** perform additional normalization based on its own needs - // and the internal decimal implementation selected, such as shifting the - // decimal point and exponent value together (example: `2.5e-1` <-> `0.25`). - // Additionally, services **may** preserve trailing zeroes in the fraction - // to indicate increased precision, but are not required to do so. - // - // Note that only the `.` character is supported to divide the integer - // and the fraction; `,` **should not** be supported regardless of locale. - // Additionally, thousand separators **should not** be supported. If a - // service does support them, values **must** be normalized. - // - // The ENBF grammar is: - // - // DecimalString = - // [Sign] Significand [Exponent]; - // - // Sign = '+' | '-'; - // - // Significand = - // Digits ['.'] [Digits] | [Digits] '.' Digits; - // - // Exponent = ('e' | 'E') [Sign] Digits; - // - // Digits = { '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' }; - // - // Services **should** clearly document the range of supported values, the - // maximum supported precision (total number of digits), and, if applicable, - // the scale (number of digits after the decimal point), as well as how it - // behaves when receiving out-of-bounds values. - // - // Services **may** choose to accept values passed as input even when the - // value has a higher precision or scale than the service supports, and - // **should** round the value to fit the supported scale. Alternatively, the - // service **may** error with `400 Bad Request` (`INVALID_ARGUMENT` in gRPC) - // if precision would be lost. - // - // Services **should** error with `400 Bad Request` (`INVALID_ARGUMENT` in - // gRPC) if the service receives a value outside of the supported range. - string value = 1; -} diff --git a/karapace/protobuf/google/type/expr.proto b/karapace/protobuf/google/type/expr.proto deleted file mode 100644 index af0778cf9..000000000 --- a/karapace/protobuf/google/type/expr.proto +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright 2021 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package google.type; - -option go_package = "google.golang.org/genproto/googleapis/type/expr;expr"; -option java_multiple_files = true; -option java_outer_classname = "ExprProto"; -option java_package = "com.google.type"; -option objc_class_prefix = "GTP"; - -// Represents a textual expression in the Common Expression Language (CEL) -// syntax. CEL is a C-like expression language. The syntax and semantics of CEL -// are documented at https://github.com/google/cel-spec. -// -// Example (Comparison): -// -// title: "Summary size limit" -// description: "Determines if a summary is less than 100 chars" -// expression: "document.summary.size() < 100" -// -// Example (Equality): -// -// title: "Requestor is owner" -// description: "Determines if requestor is the document owner" -// expression: "document.owner == request.auth.claims.email" -// -// Example (Logic): -// -// title: "Public documents" -// description: "Determine whether the document should be publicly visible" -// expression: "document.type != 'private' && document.type != 'internal'" -// -// Example (Data Manipulation): -// -// title: "Notification string" -// description: "Create a notification string with a timestamp." -// expression: "'New message received at ' + string(document.create_time)" -// -// The exact variables and functions that may be referenced within an expression -// are determined by the service that evaluates it. See the service -// documentation for additional information. -message Expr { - // Textual representation of an expression in Common Expression Language - // syntax. - string expression = 1; - - // Optional. Title for the expression, i.e. a short string describing - // its purpose. This can be used e.g. in UIs which allow to enter the - // expression. - string title = 2; - - // Optional. Description of the expression. This is a longer text which - // describes the expression, e.g. when hovered over it in a UI. - string description = 3; - - // Optional. String indicating the location of the expression for error - // reporting, e.g. a file name and a position in the file. - string location = 4; -} diff --git a/karapace/protobuf/google/type/fraction.proto b/karapace/protobuf/google/type/fraction.proto deleted file mode 100644 index 6c5ae6e2a..000000000 --- a/karapace/protobuf/google/type/fraction.proto +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2021 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package google.type; - -option go_package = "google.golang.org/genproto/googleapis/type/fraction;fraction"; -option java_multiple_files = true; -option java_outer_classname = "FractionProto"; -option java_package = "com.google.type"; -option objc_class_prefix = "GTP"; - -// Represents a fraction in terms of a numerator divided by a denominator. -message Fraction { - // The numerator in the fraction, e.g. 2 in 2/3. - int64 numerator = 1; - - // The value by which the numerator is divided, e.g. 3 in 2/3. Must be - // positive. - int64 denominator = 2; -} diff --git a/karapace/protobuf/google/type/interval.proto b/karapace/protobuf/google/type/interval.proto deleted file mode 100644 index 9702324cd..000000000 --- a/karapace/protobuf/google/type/interval.proto +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2021 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package google.type; - -import "google/protobuf/timestamp.proto"; - -option cc_enable_arenas = true; -option go_package = "google.golang.org/genproto/googleapis/type/interval;interval"; -option java_multiple_files = true; -option java_outer_classname = "IntervalProto"; -option java_package = "com.google.type"; -option objc_class_prefix = "GTP"; - -// Represents a time interval, encoded as a Timestamp start (inclusive) and a -// Timestamp end (exclusive). -// -// The start must be less than or equal to the end. -// When the start equals the end, the interval is empty (matches no time). -// When both start and end are unspecified, the interval matches any time. -message Interval { - // Optional. Inclusive start of the interval. - // - // If specified, a Timestamp matching this interval will have to be the same - // or after the start. - google.protobuf.Timestamp start_time = 1; - - // Optional. Exclusive end of the interval. - // - // If specified, a Timestamp matching this interval will have to be before the - // end. - google.protobuf.Timestamp end_time = 2; -} diff --git a/karapace/protobuf/google/type/latlng.proto b/karapace/protobuf/google/type/latlng.proto deleted file mode 100644 index 9231456e3..000000000 --- a/karapace/protobuf/google/type/latlng.proto +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2021 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package google.type; - -option cc_enable_arenas = true; -option go_package = "google.golang.org/genproto/googleapis/type/latlng;latlng"; -option java_multiple_files = true; -option java_outer_classname = "LatLngProto"; -option java_package = "com.google.type"; -option objc_class_prefix = "GTP"; - -// An object that represents a latitude/longitude pair. This is expressed as a -// pair of doubles to represent degrees latitude and degrees longitude. Unless -// specified otherwise, this must conform to the -// WGS84 -// standard. Values must be within normalized ranges. -message LatLng { - // The latitude in degrees. It must be in the range [-90.0, +90.0]. - double latitude = 1; - - // The longitude in degrees. It must be in the range [-180.0, +180.0]. - double longitude = 2; -} diff --git a/karapace/protobuf/google/type/localized_text.proto b/karapace/protobuf/google/type/localized_text.proto deleted file mode 100644 index 5c6922b8c..000000000 --- a/karapace/protobuf/google/type/localized_text.proto +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright 2021 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package google.type; - -option cc_enable_arenas = true; -option go_package = "google.golang.org/genproto/googleapis/type/localized_text;localized_text"; -option java_multiple_files = true; -option java_outer_classname = "LocalizedTextProto"; -option java_package = "com.google.type"; -option objc_class_prefix = "GTP"; - -// Localized variant of a text in a particular language. -message LocalizedText { - // Localized string in the language corresponding to `language_code' below. - string text = 1; - - // The text's BCP-47 language code, such as "en-US" or "sr-Latn". - // - // For more information, see - // http://www.unicode.org/reports/tr35/#Unicode_locale_identifier. - string language_code = 2; -} diff --git a/karapace/protobuf/google/type/money.proto b/karapace/protobuf/google/type/money.proto deleted file mode 100644 index 98d6494e4..000000000 --- a/karapace/protobuf/google/type/money.proto +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2021 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package google.type; - -option cc_enable_arenas = true; -option go_package = "google.golang.org/genproto/googleapis/type/money;money"; -option java_multiple_files = true; -option java_outer_classname = "MoneyProto"; -option java_package = "com.google.type"; -option objc_class_prefix = "GTP"; - -// Represents an amount of money with its currency type. -message Money { - // The three-letter currency code defined in ISO 4217. - string currency_code = 1; - - // The whole units of the amount. - // For example if `currencyCode` is `"USD"`, then 1 unit is one US dollar. - int64 units = 2; - - // Number of nano (10^-9) units of the amount. - // The value must be between -999,999,999 and +999,999,999 inclusive. - // If `units` is positive, `nanos` must be positive or zero. - // If `units` is zero, `nanos` can be positive, zero, or negative. - // If `units` is negative, `nanos` must be negative or zero. - // For example $-1.75 is represented as `units`=-1 and `nanos`=-750,000,000. - int32 nanos = 3; -} diff --git a/karapace/protobuf/google/type/month.proto b/karapace/protobuf/google/type/month.proto deleted file mode 100644 index 99e7551b1..000000000 --- a/karapace/protobuf/google/type/month.proto +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright 2021 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package google.type; - -option go_package = "google.golang.org/genproto/googleapis/type/month;month"; -option java_multiple_files = true; -option java_outer_classname = "MonthProto"; -option java_package = "com.google.type"; -option objc_class_prefix = "GTP"; - -// Represents a month in the Gregorian calendar. -enum Month { - // The unspecified month. - MONTH_UNSPECIFIED = 0; - - // The month of January. - JANUARY = 1; - - // The month of February. - FEBRUARY = 2; - - // The month of March. - MARCH = 3; - - // The month of April. - APRIL = 4; - - // The month of May. - MAY = 5; - - // The month of June. - JUNE = 6; - - // The month of July. - JULY = 7; - - // The month of August. - AUGUST = 8; - - // The month of September. - SEPTEMBER = 9; - - // The month of October. - OCTOBER = 10; - - // The month of November. - NOVEMBER = 11; - - // The month of December. - DECEMBER = 12; -} diff --git a/karapace/protobuf/google/type/phone_number.proto b/karapace/protobuf/google/type/phone_number.proto deleted file mode 100644 index 7bbb7d873..000000000 --- a/karapace/protobuf/google/type/phone_number.proto +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright 2021 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package google.type; - -option cc_enable_arenas = true; -option go_package = "google.golang.org/genproto/googleapis/type/phone_number;phone_number"; -option java_multiple_files = true; -option java_outer_classname = "PhoneNumberProto"; -option java_package = "com.google.type"; -option objc_class_prefix = "GTP"; - -// An object representing a phone number, suitable as an API wire format. -// -// This representation: -// -// - should not be used for locale-specific formatting of a phone number, such -// as "+1 (650) 253-0000 ext. 123" -// -// - is not designed for efficient storage -// - may not be suitable for dialing - specialized libraries (see references) -// should be used to parse the number for that purpose -// -// To do something meaningful with this number, such as format it for various -// use-cases, convert it to an `i18n.phonenumbers.PhoneNumber` object first. -// -// For instance, in Java this would be: -// -// com.google.type.PhoneNumber wireProto = -// com.google.type.PhoneNumber.newBuilder().build(); -// com.google.i18n.phonenumbers.Phonenumber.PhoneNumber phoneNumber = -// PhoneNumberUtil.getInstance().parse(wireProto.getE164Number(), "ZZ"); -// if (!wireProto.getExtension().isEmpty()) { -// phoneNumber.setExtension(wireProto.getExtension()); -// } -// -// Reference(s): -// - https://github.com/google/libphonenumber -message PhoneNumber { - // An object representing a short code, which is a phone number that is - // typically much shorter than regular phone numbers and can be used to - // address messages in MMS and SMS systems, as well as for abbreviated dialing - // (e.g. "Text 611 to see how many minutes you have remaining on your plan."). - // - // Short codes are restricted to a region and are not internationally - // dialable, which means the same short code can exist in different regions, - // with different usage and pricing, even if those regions share the same - // country calling code (e.g. US and CA). - message ShortCode { - // Required. The BCP-47 region code of the location where calls to this - // short code can be made, such as "US" and "BB". - // - // Reference(s): - // - http://www.unicode.org/reports/tr35/#unicode_region_subtag - string region_code = 1; - - // Required. The short code digits, without a leading plus ('+') or country - // calling code, e.g. "611". - string number = 2; - } - - // Required. Either a regular number, or a short code. New fields may be - // added to the oneof below in the future, so clients should ignore phone - // numbers for which none of the fields they coded against are set. - oneof kind { - // The phone number, represented as a leading plus sign ('+'), followed by a - // phone number that uses a relaxed ITU E.164 format consisting of the - // country calling code (1 to 3 digits) and the subscriber number, with no - // additional spaces or formatting, e.g.: - // - correct: "+15552220123" - // - incorrect: "+1 (555) 222-01234 x123". - // - // The ITU E.164 format limits the latter to 12 digits, but in practice not - // all countries respect that, so we relax that restriction here. - // National-only numbers are not allowed. - // - // References: - // - https://www.itu.int/rec/T-REC-E.164-201011-I - // - https://en.wikipedia.org/wiki/E.164. - // - https://en.wikipedia.org/wiki/List_of_country_calling_codes - string e164_number = 1; - - // A short code. - // - // Reference(s): - // - https://en.wikipedia.org/wiki/Short_code - ShortCode short_code = 2; - } - - // The phone number's extension. The extension is not standardized in ITU - // recommendations, except for being defined as a series of numbers with a - // maximum length of 40 digits. Other than digits, some other dialing - // characters such as ',' (indicating a wait) or '#' may be stored here. - // - // Note that no regions currently use extensions with short codes, so this - // field is normally only set in conjunction with an E.164 number. It is held - // separately from the E.164 number to allow for short code extensions in the - // future. - string extension = 3; -} diff --git a/karapace/protobuf/google/type/postal_address.proto b/karapace/protobuf/google/type/postal_address.proto deleted file mode 100644 index c57c7c31a..000000000 --- a/karapace/protobuf/google/type/postal_address.proto +++ /dev/null @@ -1,134 +0,0 @@ -// Copyright 2021 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package google.type; - -option cc_enable_arenas = true; -option go_package = "google.golang.org/genproto/googleapis/type/postaladdress;postaladdress"; -option java_multiple_files = true; -option java_outer_classname = "PostalAddressProto"; -option java_package = "com.google.type"; -option objc_class_prefix = "GTP"; - -// Represents a postal address, e.g. for postal delivery or payments addresses. -// Given a postal address, a postal service can deliver items to a premise, P.O. -// Box or similar. -// It is not intended to model geographical locations (roads, towns, -// mountains). -// -// In typical usage an address would be created via user input or from importing -// existing data, depending on the type of process. -// -// Advice on address input / editing: -// - Use an i18n-ready address widget such as -// https://github.com/google/libaddressinput) -// - Users should not be presented with UI elements for input or editing of -// fields outside countries where that field is used. -// -// For more guidance on how to use this schema, please see: -// https://support.google.com/business/answer/6397478 -message PostalAddress { - // The schema revision of the `PostalAddress`. This must be set to 0, which is - // the latest revision. - // - // All new revisions **must** be backward compatible with old revisions. - int32 revision = 1; - - // Required. CLDR region code of the country/region of the address. This - // is never inferred and it is up to the user to ensure the value is - // correct. See http://cldr.unicode.org/ and - // http://www.unicode.org/cldr/charts/30/supplemental/territory_information.html - // for details. Example: "CH" for Switzerland. - string region_code = 2; - - // Optional. BCP-47 language code of the contents of this address (if - // known). This is often the UI language of the input form or is expected - // to match one of the languages used in the address' country/region, or their - // transliterated equivalents. - // This can affect formatting in certain countries, but is not critical - // to the correctness of the data and will never affect any validation or - // other non-formatting related operations. - // - // If this value is not known, it should be omitted (rather than specifying a - // possibly incorrect default). - // - // Examples: "zh-Hant", "ja", "ja-Latn", "en". - string language_code = 3; - - // Optional. Postal code of the address. Not all countries use or require - // postal codes to be present, but where they are used, they may trigger - // additional validation with other parts of the address (e.g. state/zip - // validation in the U.S.A.). - string postal_code = 4; - - // Optional. Additional, country-specific, sorting code. This is not used - // in most regions. Where it is used, the value is either a string like - // "CEDEX", optionally followed by a number (e.g. "CEDEX 7"), or just a number - // alone, representing the "sector code" (Jamaica), "delivery area indicator" - // (Malawi) or "post office indicator" (e.g. Côte d'Ivoire). - string sorting_code = 5; - - // Optional. Highest administrative subdivision which is used for postal - // addresses of a country or region. - // For example, this can be a state, a province, an oblast, or a prefecture. - // Specifically, for Spain this is the province and not the autonomous - // community (e.g. "Barcelona" and not "Catalonia"). - // Many countries don't use an administrative area in postal addresses. E.g. - // in Switzerland this should be left unpopulated. - string administrative_area = 6; - - // Optional. Generally refers to the city/town portion of the address. - // Examples: US city, IT comune, UK post town. - // In regions of the world where localities are not well defined or do not fit - // into this structure well, leave locality empty and use address_lines. - string locality = 7; - - // Optional. Sublocality of the address. - // For example, this can be neighborhoods, boroughs, districts. - string sublocality = 8; - - // Unstructured address lines describing the lower levels of an address. - // - // Because values in address_lines do not have type information and may - // sometimes contain multiple values in a single field (e.g. - // "Austin, TX"), it is important that the line order is clear. The order of - // address lines should be "envelope order" for the country/region of the - // address. In places where this can vary (e.g. Japan), address_language is - // used to make it explicit (e.g. "ja" for large-to-small ordering and - // "ja-Latn" or "en" for small-to-large). This way, the most specific line of - // an address can be selected based on the language. - // - // The minimum permitted structural representation of an address consists - // of a region_code with all remaining information placed in the - // address_lines. It would be possible to format such an address very - // approximately without geocoding, but no semantic reasoning could be - // made about any of the address components until it was at least - // partially resolved. - // - // Creating an address only containing a region_code and address_lines, and - // then geocoding is the recommended way to handle completely unstructured - // addresses (as opposed to guessing which parts of the address should be - // localities or administrative areas). - repeated string address_lines = 9; - - // Optional. The recipient at the address. - // This field may, under certain circumstances, contain multiline information. - // For example, it might contain "care of" information. - repeated string recipients = 10; - - // Optional. The name of the organization at the address. - string organization = 11; -} diff --git a/karapace/protobuf/google/type/quaternion.proto b/karapace/protobuf/google/type/quaternion.proto deleted file mode 100644 index dfb822def..000000000 --- a/karapace/protobuf/google/type/quaternion.proto +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright 2021 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package google.type; - -option cc_enable_arenas = true; -option go_package = "google.golang.org/genproto/googleapis/type/quaternion;quaternion"; -option java_multiple_files = true; -option java_outer_classname = "QuaternionProto"; -option java_package = "com.google.type"; -option objc_class_prefix = "GTP"; - -// A quaternion is defined as the quotient of two directed lines in a -// three-dimensional space or equivalently as the quotient of two Euclidean -// vectors (https://en.wikipedia.org/wiki/Quaternion). -// -// Quaternions are often used in calculations involving three-dimensional -// rotations (https://en.wikipedia.org/wiki/Quaternions_and_spatial_rotation), -// as they provide greater mathematical robustness by avoiding the gimbal lock -// problems that can be encountered when using Euler angles -// (https://en.wikipedia.org/wiki/Gimbal_lock). -// -// Quaternions are generally represented in this form: -// -// w + xi + yj + zk -// -// where x, y, z, and w are real numbers, and i, j, and k are three imaginary -// numbers. -// -// Our naming choice `(x, y, z, w)` comes from the desire to avoid confusion for -// those interested in the geometric properties of the quaternion in the 3D -// Cartesian space. Other texts often use alternative names or subscripts, such -// as `(a, b, c, d)`, `(1, i, j, k)`, or `(0, 1, 2, 3)`, which are perhaps -// better suited for mathematical interpretations. -// -// To avoid any confusion, as well as to maintain compatibility with a large -// number of software libraries, the quaternions represented using the protocol -// buffer below *must* follow the Hamilton convention, which defines `ij = k` -// (i.e. a right-handed algebra), and therefore: -// -// i^2 = j^2 = k^2 = ijk = −1 -// ij = −ji = k -// jk = −kj = i -// ki = −ik = j -// -// Please DO NOT use this to represent quaternions that follow the JPL -// convention, or any of the other quaternion flavors out there. -// -// Definitions: -// -// - Quaternion norm (or magnitude): `sqrt(x^2 + y^2 + z^2 + w^2)`. -// - Unit (or normalized) quaternion: a quaternion whose norm is 1. -// - Pure quaternion: a quaternion whose scalar component (`w`) is 0. -// - Rotation quaternion: a unit quaternion used to represent rotation. -// - Orientation quaternion: a unit quaternion used to represent orientation. -// -// A quaternion can be normalized by dividing it by its norm. The resulting -// quaternion maintains the same direction, but has a norm of 1, i.e. it moves -// on the unit sphere. This is generally necessary for rotation and orientation -// quaternions, to avoid rounding errors: -// https://en.wikipedia.org/wiki/Rotation_formalisms_in_three_dimensions -// -// Note that `(x, y, z, w)` and `(-x, -y, -z, -w)` represent the same rotation, -// but normalization would be even more useful, e.g. for comparison purposes, if -// it would produce a unique representation. It is thus recommended that `w` be -// kept positive, which can be achieved by changing all the signs when `w` is -// negative. -// -message Quaternion { - // The x component. - double x = 1; - - // The y component. - double y = 2; - - // The z component. - double z = 3; - - // The scalar component. - double w = 4; -} diff --git a/karapace/protobuf/google/type/timeofday.proto b/karapace/protobuf/google/type/timeofday.proto deleted file mode 100644 index 5cb48aa93..000000000 --- a/karapace/protobuf/google/type/timeofday.proto +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright 2021 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package google.type; - -option cc_enable_arenas = true; -option go_package = "google.golang.org/genproto/googleapis/type/timeofday;timeofday"; -option java_multiple_files = true; -option java_outer_classname = "TimeOfDayProto"; -option java_package = "com.google.type"; -option objc_class_prefix = "GTP"; - -// Represents a time of day. The date and time zone are either not significant -// or are specified elsewhere. An API may choose to allow leap seconds. Related -// types are [google.type.Date][google.type.Date] and -// `google.protobuf.Timestamp`. -message TimeOfDay { - // Hours of day in 24 hour format. Should be from 0 to 23. An API may choose - // to allow the value "24:00:00" for scenarios like business closing time. - int32 hours = 1; - - // Minutes of hour of day. Must be from 0 to 59. - int32 minutes = 2; - - // Seconds of minutes of the time. Must normally be from 0 to 59. An API may - // allow the value 60 if it allows leap-seconds. - int32 seconds = 3; - - // Fractions of seconds in nanoseconds. Must be from 0 to 999,999,999. - int32 nanos = 4; -} diff --git a/karapace/protobuf/google/type/type.yaml b/karapace/protobuf/google/type/type.yaml deleted file mode 100644 index d5c71364d..000000000 --- a/karapace/protobuf/google/type/type.yaml +++ /dev/null @@ -1,40 +0,0 @@ -type: google.api.Service -config_version: 3 -name: type.googleapis.com -title: Common Types - -types: -- name: google.type.Color -- name: google.type.Date -- name: google.type.DateTime -- name: google.type.Decimal -- name: google.type.Expr -- name: google.type.Fraction -- name: google.type.Interval -- name: google.type.LatLng -- name: google.type.LocalizedText -- name: google.type.Money -- name: google.type.PhoneNumber -- name: google.type.PostalAddress -- name: google.type.Quaternion -- name: google.type.TimeOfDay - -enums: -- name: google.type.CalendarPeriod -- name: google.type.DayOfWeek -- name: google.type.Month - -documentation: - summary: Defines common types for Google APIs. - overview: |- - # Google Common Types - - This package contains definitions of common types for Google APIs. - All types defined in this package are suitable for different APIs to - exchange data, and will never break binary compatibility. They should - have design quality comparable to major programming languages like - Java and C#. - - NOTE: Some common types are defined in the package `google.protobuf` - as they are directly supported by Protocol Buffers compiler and - runtime. Those types are called Well-Known Types. diff --git a/karapace/schema_registry.py b/karapace/schema_registry.py index daf4af26b..b5d65979f 100644 --- a/karapace/schema_registry.py +++ b/karapace/schema_registry.py @@ -222,14 +222,12 @@ async def subject_delete_local(self, subject: str, permanent: bool) -> List[Reso return version_list - async def subject_version_delete_local(self, subject: Subject, version: Version, - permanent: bool) -> ResolvedVersion: + async def subject_version_delete_local(self, subject: Subject, version: Version, permanent: bool) -> ResolvedVersion: async with self.schema_lock: subject_data = self.subject_get(subject, include_deleted=True) if not permanent and isinstance(version, str) and version == "latest": subject_data["schemas"] = { - key: value for (key, value) in subject_data["schemas"].items() if - value.get("deleted", False) is False + key: value for (key, value) in subject_data["schemas"].items() if value.get("deleted", False) is False } resolved_version = _resolve_version(subject_data=subject_data, version=version) subject_schema_data = subject_data["schemas"].get(resolved_version, None) @@ -259,7 +257,7 @@ async def subject_version_delete_local(self, subject: Subject, version: Version, schema_id=schema_id, version=resolved_version, deleted=True, - references=references + references=references, ) if references_list and len(references_list) > 0: self.schema_reader.remove_referenced_by(schema_id, references_list) @@ -278,8 +276,7 @@ def subject_get(self, subject: Subject, include_deleted: bool = False) -> Subjec subject_data["schemas"] = schemas return subject_data - async def subject_version_get(self, subject: Subject, version: Version, *, - include_deleted: bool = False) -> SubjectData: + async def subject_version_get(self, subject: Subject, version: Version, *, include_deleted: bool = False) -> SubjectData: validate_version(version) subject_data = self.subject_get(subject, include_deleted=include_deleted) if not subject_data: @@ -310,10 +307,10 @@ async def subject_version_get(self, subject: Subject, version: Version, *, return ret async def write_new_schema_local( - self, - subject: Subject, - new_schema: ValidatedTypedSchema, - new_schema_references: Optional[References], + self, + subject: Subject, + new_schema: ValidatedTypedSchema, + new_schema_references: Optional[References], ) -> int: """Write new schema and return new id or return id of matching existing schema @@ -480,14 +477,14 @@ def send_kafka_message(self, key: Union[bytes, str], value: Union[bytes, str]) - return future def send_schema_message( - self, - *, - subject: Subject, - schema: Optional[TypedSchema], - schema_id: int, - version: int, - deleted: bool, - references: Optional[References], + self, + *, + subject: Subject, + schema: Optional[TypedSchema], + schema_id: int, + version: int, + deleted: bool, + references: Optional[References], ) -> FutureRecordMetadata: key = self.key_formatter.format_key( {"subject": subject, "version": version, "magic": 1, "keytype": "SCHEMA"}, @@ -510,7 +507,7 @@ def send_schema_message( return self.send_kafka_message(key, value) def send_config_message( - self, compatibility_level: CompatibilityModes, subject: Optional[Subject] = None + self, compatibility_level: CompatibilityModes, subject: Optional[Subject] = None ) -> FutureRecordMetadata: key = self.key_formatter.format_key( { @@ -547,6 +544,6 @@ def resolve_references(self, references: Optional["References"] = None) -> Optio return self.schema_reader.resolve_references(references) def resolve_schema_references( - self, schema_data: Optional[dict] + self, schema_data: Optional[dict] ) -> Tuple[Optional[References], Optional[Dict[str, Dependency]]]: return self.schema_reader.resolve_schema_references(schema_data) diff --git a/tests/integration/test_schema_protobuf.py b/tests/integration/test_schema_protobuf.py index af36f261c..7026146a8 100644 --- a/tests/integration/test_schema_protobuf.py +++ b/tests/integration/test_schema_protobuf.py @@ -271,6 +271,21 @@ async def test_protobuf_schema_references(registry_async_client: Client) -> None res = await registry_async_client.delete("subjects/test_schema/versions/1") assert res.status_code == 200 + res = await registry_async_client.delete("subjects/test_schema/versions/1") + myjson = res.json() + match_msg = "Subject 'test_schema' Version 1 was soft deleted.Set permanent=true to delete permanently" + assert res.status_code == 404 + assert myjson["error_code"] == 40406 and myjson["message"] == match_msg + + res = await registry_async_client.delete("subjects/customer/versions/1") + myjson = res.json() + match_msg = "One or more references exist to the schema {magic=1,keytype=SCHEMA,subject=customer,version=1}" + assert res.status_code == 404 + assert myjson["error_code"] == 42206 and myjson["message"] == match_msg + + res = await registry_async_client.delete("subjects/test_schema/versions/1?permanent=true") + assert res.status_code == 200 + res = await registry_async_client.delete("subjects/customer/versions/1") assert res.status_code == 200 @@ -460,5 +475,11 @@ async def test_protobuf_schema_verifier(registry_async_client: Client) -> None: res = await registry_async_client.delete("subjects/test_schema/versions/1") assert res.status_code == 200 + res = await registry_async_client.delete("subjects/customer/versions/1") + assert res.status_code == 404 + + res = await registry_async_client.delete("subjects/test_schema/versions/1?permanent=true") + assert res.status_code == 200 + res = await registry_async_client.delete("subjects/customer/versions/1") assert res.status_code == 200 From d524e0fbdd9db570f09fcab66e486ce784869232 Mon Sep 17 00:00:00 2001 From: Sergiy Zaschipas Date: Sat, 22 Oct 2022 00:14:10 +0300 Subject: [PATCH 12/25] remove head dots from known_dependency.py full type name --- karapace/protobuf/known_dependency.py | 131 +++++++++++++------------- 1 file changed, 66 insertions(+), 65 deletions(-) diff --git a/karapace/protobuf/known_dependency.py b/karapace/protobuf/known_dependency.py index fd862c98a..289d17df7 100644 --- a/karapace/protobuf/known_dependency.py +++ b/karapace/protobuf/known_dependency.py @@ -45,79 +45,79 @@ class KnownDependency: index: Dict = dict() index_simple: Dict = dict() map: Dict = { - "google/protobuf/any.proto": [".google.protobuf.Any"], - "google/protobuf/api.proto": [".google.protobuf.Api", ".google.protobuf.Method", ".google.protobuf.Mixin"], + "google/protobuf/any.proto": ["google.protobuf.Any"], + "google/protobuf/api.proto": ["google.protobuf.Api", "google.protobuf.Method", "google.protobuf.Mixin"], "google/protobuf/descriptor.proto": [ - ".google.protobuf.FileDescriptorSet", - ".google.protobuf.FileDescriptorProto", - ".google.protobuf.DescriptorProto", - ".google.protobuf.ExtensionRangeOptions", - ".google.protobuf.FieldDescriptorProto", - ".google.protobuf.OneofDescriptorProto", - ".google.protobuf.EnumDescriptorProto", - ".google.protobuf.EnumValueDescriptorProto", - ".google.protobuf.ServiceDescriptorProto", - ".google.protobuf.MethodDescriptorProto", - ".google.protobuf.FileOptions", - ".google.protobuf.MessageOptions", - ".google.protobuf.FieldOptions", - ".google.protobuf.OneofOptions", - ".google.protobuf.EnumOptions", - ".google.protobuf.EnumValueOptions", - ".google.protobuf.ServiceOptions", - ".google.protobuf.MethodOptions", - ".google.protobuf.UninterpretedOption", - ".google.protobuf.SourceCodeInfo", - ".google.protobuf.GeneratedCodeInfo", + "google.protobuf.FileDescriptorSet", + "google.protobuf.FileDescriptorProto", + "google.protobuf.DescriptorProto", + "google.protobuf.ExtensionRangeOptions", + "google.protobuf.FieldDescriptorProto", + "google.protobuf.OneofDescriptorProto", + "google.protobuf.EnumDescriptorProto", + "google.protobuf.EnumValueDescriptorProto", + "google.protobuf.ServiceDescriptorProto", + "google.protobuf.MethodDescriptorProto", + "google.protobuf.FileOptions", + "google.protobuf.MessageOptions", + "google.protobuf.FieldOptions", + "google.protobuf.OneofOptions", + "google.protobuf.EnumOptions", + "google.protobuf.EnumValueOptions", + "google.protobuf.ServiceOptions", + "google.protobuf.MethodOptions", + "google.protobuf.UninterpretedOption", + "google.protobuf.SourceCodeInfo", + "google.protobuf.GeneratedCodeInfo", ], - "google/protobuf/duration.proto": [".google.protobuf.Duration"], - "google/protobuf/empty.proto": [".google.protobuf.Empty"], - "google/protobuf/field_mask.proto": [".google.protobuf.FieldMask"], - "google/protobuf/source_context.proto": [".google.protobuf.SourceContext"], + "google/protobuf/duration.proto": ["google.protobuf.Duration"], + "google/protobuf/empty.proto": ["google.protobuf.Empty"], + "google/protobuf/field_mask.proto": ["google.protobuf.FieldMask"], + "google/protobuf/source_context.proto": ["google.protobuf.SourceContext"], "google/protobuf/struct.proto": [ - ".google.protobuf.Struct", - ".google.protobuf.Value", - ".google.protobuf.NullValue", - ".google.protobuf.ListValue", + "google.protobuf.Struct", + "google.protobuf.Value", + "google.protobuf.NullValue", + "google.protobuf.ListValue", ], - "google/protobuf/timestamp.proto": [".google.protobuf.Timestamp"], + "google/protobuf/timestamp.proto": ["google.protobuf.Timestamp"], "google/protobuf/type.proto": [ - ".google.protobuf.Type", - ".google.protobuf.Field", - ".google.protobuf.Enum", - ".google.protobuf.EnumValue", - ".google.protobuf.Option", - ".google.protobuf.Syntax", + "google.protobuf.Type", + "google.protobuf.Field", + "google.protobuf.Enum", + "google.protobuf.EnumValue", + "google.protobuf.Option", + "google.protobuf.Syntax", ], "google/protobuf/wrappers.proto": [ - ".google.protobuf.DoubleValue", - ".google.protobuf.FloatValue", - ".google.protobuf.Int64Value", - ".google.protobuf.UInt64Value", - ".google.protobuf.Int32Value", - ".google.protobuf.UInt32Value", - ".google.protobuf.BoolValue", - ".google.protobuf.StringValue", - ".google.protobuf.BytesValue", + "google.protobuf.DoubleValue", + "google.protobuf.FloatValue", + "google.protobuf.Int64Value", + "google.protobuf.UInt64Value", + "google.protobuf.Int32Value", + "google.protobuf.UInt32Value", + "google.protobuf.BoolValue", + "google.protobuf.StringValue", + "google.protobuf.BytesValue", ], - "google/type/calendar_period.proto": [".google.type.CalendarPeriod"], - "google/type/color.proto": [".google.type.Color"], - "google/type/date.proto": [".google.type.Date"], - "google/type/datetime.proto": [".google.type.DateTime", ".google.type.TimeZone"], - "google/type/dayofweek.proto": [".google.type.DayOfWeek"], - "google/type/decimal.proto": [".google.type.Decimal"], - "google/type/expr.proto": [".google.type.Expr"], - "google/type/fraction.proto": [".google.type.Fraction"], - "google/type/interval.proto": [".google.type.Interval"], - "google/type/latlng.proto": [".google.type.LatLng"], - "google/type/money.proto": [".google.type.Money"], - "google/type/month.proto": [".google.type.Month"], - "google/type/phone_number.proto": [".google.type.PhoneNumber"], - "google/type/postal_address.proto": [".google.type.PostalAddress"], - "google/type/quaternion.proto": [".google.type.Quaternion"], - "google/type/timeofday.proto": [".google.type.TimeOfDay"], - "confluent/meta.proto": [".confluent.Meta"], - "confluent/type/decimal.proto": [".confluent.type.Decimal"], + "google/type/calendar_period.proto": ["google.type.CalendarPeriod"], + "google/type/color.proto": ["google.type.Color"], + "google/type/date.proto": ["google.type.Date"], + "google/type/datetime.proto": ["google.type.DateTime", "google.type.TimeZone"], + "google/type/dayofweek.proto": ["google.type.DayOfWeek"], + "google/type/decimal.proto": ["google.type.Decimal"], + "google/type/expr.proto": ["google.type.Expr"], + "google/type/fraction.proto": ["google.type.Fraction"], + "google/type/interval.proto": ["google.type.Interval"], + "google/type/latlng.proto": ["google.type.LatLng"], + "google/type/money.proto": ["google.type.Money"], + "google/type/month.proto": ["google.type.Month"], + "google/type/phone_number.proto": ["google.type.PhoneNumber"], + "google/type/postal_address.proto": ["google.type.PostalAddress"], + "google/type/quaternion.proto": ["google.type.Quaternion"], + "google/type/timeofday.proto": ["google.type.TimeOfDay"], + "confluent/meta.proto": ["confluent.Meta"], + "confluent/type/decimal.proto": ["confluent.type.Decimal"], } @classmethod @@ -125,6 +125,7 @@ def static_init(cls) -> None: for key, value in cls.map.items(): for item in value: cls.index[item] = key + cls.index["." + item] = key dot = item.rfind(".") cls.index_simple[item[dot + 1 :]] = key cls.index_simple[item] = key From 6201bef3b3ed81a558e92ff96325a93bcf0600a4 Mon Sep 17 00:00:00 2001 From: Sergiy Zaschipas Date: Mon, 31 Oct 2022 23:10:02 +0200 Subject: [PATCH 13/25] sync with aiven references branch --- karapace/dependency.py | 41 +- karapace/errors.py | 9 +- karapace/protobuf/compare_type_storage.py | 30 +- karapace/protobuf/field_element.py | 11 +- karapace/protobuf/known_dependency.py | 7 +- karapace/protobuf/message_element.py | 5 +- karapace/protobuf/proto_file_element.py | 103 +++- karapace/protobuf/schema.py | 32 +- karapace/protobuf/type_element.py | 5 - karapace/schema_models.py | 86 ++- karapace/schema_reader.py | 149 ++--- karapace/schema_references.py | 39 +- karapace/schema_registry.py | 43 +- karapace/schema_registry_apis.py | 117 ++-- karapace/serialization.py | 26 +- tests/integration/test_schema.py | 17 +- tests/integration/test_schema_protobuf.py | 676 +++++++++++++++++----- 17 files changed, 939 insertions(+), 457 deletions(-) diff --git a/karapace/dependency.py b/karapace/dependency.py index ef0e62a4c..3cc2a6ff0 100644 --- a/karapace/dependency.py +++ b/karapace/dependency.py @@ -1,24 +1,47 @@ -from typing import Optional, TYPE_CHECKING +from karapace.schema_references import Reference +from karapace.typing import JsonData, Subject, Version +from typing import Any, Optional, TYPE_CHECKING if TYPE_CHECKING: from karapace.schema_models import ValidatedTypedSchema +class DependencyVerifierResult: + def __init__(self, result: bool, message: Optional[str] = "") -> None: + self.result = result + self.message = message + + class Dependency: - def __init__(self, name: str, subject: str, version: int, schema: "ValidatedTypedSchema") -> None: + def __init__(self, name: str, subject: Subject, version: Version, target_schema: "ValidatedTypedSchema") -> None: self.name = name self.subject = subject self.version = version - self.schema = schema + self.schema = target_schema - def get_schema(self) -> "ValidatedTypedSchema": - return self.schema + @staticmethod + def of(reference: Reference, target_schema: "ValidatedTypedSchema") -> "Dependency": + return Dependency(reference.name, reference.subject, reference.version, target_schema) + + def to_dict(self) -> JsonData: + return { + "name": self.name, + "subject": self.subject, + "version": self.version, + } def identifier(self) -> str: return self.name + "_" + self.subject + "_" + str(self.version) + def __hash__(self) -> int: + return hash((self.name, self.subject, self.version, self.schema)) -class DependencyVerifierResult: - def __init__(self, result: bool, message: Optional[str] = ""): - self.result = result - self.message = message + def __eq__(self, other: Any) -> bool: + if other is None or not isinstance(other, Dependency): + return False + return ( + self.name == other.name + and self.subject == other.subject + and self.version == other.version + and self.schema == other.schema + ) diff --git a/karapace/errors.py b/karapace/errors.py index 188da45d0..4a2eb138d 100644 --- a/karapace/errors.py +++ b/karapace/errors.py @@ -1,4 +1,5 @@ -from typing import List, Union +from karapace.typing import Version +from typing import List class VersionNotFoundException(Exception): @@ -25,10 +26,6 @@ class InvalidReferences(Exception): pass -class ReferencesNotSupportedException(Exception): - pass - - class SchemasNotFoundException(Exception): pass @@ -50,7 +47,7 @@ class SubjectNotSoftDeletedException(Exception): class ReferenceExistsException(Exception): - def __init__(self, referenced_by: List, version: Union[int, str]): + def __init__(self, referenced_by: List, version: Version): super().__init__() self.version = version self.referenced_by = referenced_by diff --git a/karapace/protobuf/compare_type_storage.py b/karapace/protobuf/compare_type_storage.py index 68ea6983d..3a8df52e5 100644 --- a/karapace/protobuf/compare_type_storage.py +++ b/karapace/protobuf/compare_type_storage.py @@ -2,12 +2,12 @@ from karapace.protobuf.compare_result import CompareResult from karapace.protobuf.exception import IllegalArgumentException from karapace.protobuf.proto_type import ProtoType +from karapace.protobuf.type_element import TypeElement from typing import Dict, List, Optional, TYPE_CHECKING, Union if TYPE_CHECKING: from karapace.protobuf.field_element import FieldElement from karapace.protobuf.message_element import MessageElement - from karapace.protobuf.type_element import TypeElement def compute_name(t: ProtoType, result_path: List[str], package_name: str, types: dict) -> Optional[str]: @@ -35,15 +35,15 @@ def compute_name(t: ProtoType, result_path: List[str], package_name: str, types: class CompareTypes: def __init__(self, self_package_name: str, other_package_name: str, result: CompareResult) -> None: - self.self_package_name = self_package_name or "" - self.other_package_name = other_package_name or "" + self.self_package_name = self_package_name + self.other_package_name = other_package_name self.self_types: Dict[str, Union[TypeRecord, TypeRecordMap]] = {} self.other_types: Dict[str, Union[TypeRecord, TypeRecordMap]] = {} self.locked_messages: List["MessageElement"] = [] self.environment: List["MessageElement"] = [] self.result = result - def add_a_type(self, prefix: str, package_name: str, type_element: "TypeElement", types: dict) -> None: + def add_a_type(self, prefix: str, package_name: str, type_element: TypeElement, types: dict) -> None: name: str if prefix: name = prefix + "." + type_element.name @@ -65,10 +65,10 @@ def add_a_type(self, prefix: str, package_name: str, type_element: "TypeElement" for t in type_element.nested_types: self.add_a_type(name, package_name, t, types) - def add_self_type(self, package_name: str, type_element: "TypeElement") -> None: + def add_self_type(self, package_name: str, type_element: TypeElement) -> None: self.add_a_type(package_name, package_name, type_element, self.self_types) - def add_other_type(self, package_name: str, type_element: "TypeElement") -> None: + def add_other_type(self, package_name: str, type_element: TypeElement) -> None: self.add_a_type(package_name, package_name, type_element, self.other_types) def get_self_type(self, t: ProtoType) -> Union[None, "TypeRecord", "TypeRecordMap"]: @@ -90,11 +90,8 @@ def self_type_short_name(self, t: ProtoType) -> Optional[str]: if name is None: raise IllegalArgumentException(f"Cannot determine message type {t}") type_record: TypeRecord = self.self_types.get(name) - package_name = type_record.package_name - if package_name is None: - package_name = "" - if name.startswith(package_name): - return name[(len(package_name) + 1) :] + if name.startswith(type_record.package_name): + return name[(len(type_record.package_name) + 1) :] return name def other_type_short_name(self, t: ProtoType) -> Optional[str]: @@ -102,11 +99,8 @@ def other_type_short_name(self, t: ProtoType) -> Optional[str]: if name is None: raise IllegalArgumentException(f"Cannot determine message type {t}") type_record: TypeRecord = self.other_types.get(name) - package_name = type_record.package_name - if package_name is None: - package_name = "" - if name.startswith(package_name): - return name[(len(package_name) + 1) :] + if name.startswith(type_record.package_name): + return name[(len(type_record.package_name) + 1) :] return name def lock_message(self, message: "MessageElement") -> bool: @@ -125,12 +119,12 @@ def unlock_message(self, message: "MessageElement") -> bool: @dataclass class TypeRecord: package_name: str - type_element: "TypeElement" + type_element: TypeElement class TypeRecordMap(TypeRecord): def __init__( - self, package_name: str, type_element: "TypeElement", key: Optional["FieldElement"], value: Optional["FieldElement"] + self, package_name: str, type_element: TypeElement, key: Optional["FieldElement"], value: Optional["FieldElement"] ) -> None: super().__init__(package_name, type_element) self.key = key diff --git a/karapace/protobuf/field_element.py b/karapace/protobuf/field_element.py index c6dd9b0a9..4816b2728 100644 --- a/karapace/protobuf/field_element.py +++ b/karapace/protobuf/field_element.py @@ -1,6 +1,5 @@ # Ported from square/wire: # wire-library/wire-schema/src/commonMain/kotlin/com/squareup/wire/schema/internal/parser/FieldElement.kt -from karapace.errors import InvalidSchema from karapace.protobuf.compare_result import CompareResult, Modification from karapace.protobuf.compare_type_storage import TypeRecordMap from karapace.protobuf.field import Field @@ -144,10 +143,6 @@ def compare_message( self_type_record = types.get_self_type(self_type) other_type_record = types.get_other_type(other_type) - if self_type_record is None: - raise InvalidSchema(f"Cannot resolve type {self_type}") - if other_type_record is None: - raise InvalidSchema(f"Cannot resolve type {other_type}") self_type_element: MessageElement = self_type_record.type_element other_type_element: MessageElement = other_type_record.type_element @@ -156,3 +151,9 @@ def compare_message( result.add_modification(Modification.FIELD_NAME_ALTER) else: self_type_element.compare(other_type_element, result, types) + + def __repr__(self): + return f"{self.element_type} {self.name} = {self.tag}" + + def __str__(self): + return f"{self.element_type} {self.name} = {self.tag}" diff --git a/karapace/protobuf/known_dependency.py b/karapace/protobuf/known_dependency.py index 289d17df7..0ba9559f7 100644 --- a/karapace/protobuf/known_dependency.py +++ b/karapace/protobuf/known_dependency.py @@ -116,8 +116,8 @@ class KnownDependency: "google/type/postal_address.proto": ["google.type.PostalAddress"], "google/type/quaternion.proto": ["google.type.Quaternion"], "google/type/timeofday.proto": ["google.type.TimeOfDay"], - "confluent/meta.proto": ["confluent.Meta"], - "confluent/type/decimal.proto": ["confluent.type.Decimal"], + "confluent/meta.proto": [".confluent.Meta"], + "confluent/type/decimal.proto": [".confluent.type.Decimal"], } @classmethod @@ -125,11 +125,8 @@ def static_init(cls) -> None: for key, value in cls.map.items(): for item in value: cls.index[item] = key - cls.index["." + item] = key dot = item.rfind(".") cls.index_simple[item[dot + 1 :]] = key - cls.index_simple[item] = key - cls.index_simple[item[1:]] = key @static_init diff --git a/karapace/protobuf/message_element.py b/karapace/protobuf/message_element.py index 054166816..d0acecd5c 100644 --- a/karapace/protobuf/message_element.py +++ b/karapace/protobuf/message_element.py @@ -22,7 +22,7 @@ def __init__( location: Location, name: str, documentation: str = "", - nested_types: List[TypeElement] = None, + nested_types: List[str] = None, options: List[OptionElement] = None, reserveds: List[ReservedElement] = None, fields: List[FieldElement] = None, @@ -80,7 +80,6 @@ def to_schema(self) -> str: return "".join(result) def compare(self, other: "MessageElement", result: CompareResult, types: CompareTypes) -> None: - from karapace.protobuf.compare_type_lists import compare_type_lists if types.lock_message(self): field: FieldElement @@ -139,5 +138,5 @@ def compare(self, other: "MessageElement", result: CompareResult, types: Compare self_one_ofs[name].compare(other_one_ofs[name], result, types) result.pop_path() - compare_type_lists(self.nested_types, other.nested_types, result, types) + types.unlock_message(self) diff --git a/karapace/protobuf/proto_file_element.py b/karapace/protobuf/proto_file_element.py index d27e01076..f227d63c0 100644 --- a/karapace/protobuf/proto_file_element.py +++ b/karapace/protobuf/proto_file_element.py @@ -1,31 +1,15 @@ # Ported from square/wire: # wire-library/wire-schema/src/commonMain/kotlin/com/squareup/wire/schema/internal/parser/ProtoFileElement.kt -from karapace.dependency import Dependency +from itertools import chain from karapace.protobuf.compare_result import CompareResult, Modification -from karapace.protobuf.compare_type_lists import compare_type_lists from karapace.protobuf.compare_type_storage import CompareTypes +from karapace.protobuf.enum_element import EnumElement +from karapace.protobuf.exception import IllegalStateException from karapace.protobuf.location import Location +from karapace.protobuf.message_element import MessageElement from karapace.protobuf.syntax import Syntax from karapace.protobuf.type_element import TypeElement -from typing import Dict, List, Optional - - -def collect_dependencies_types(compare_types: CompareTypes, dependencies: Optional[Dict[str, Dependency]], is_self: bool): - if dependencies is None: - return - for dep in dependencies.values(): - types: List[TypeElement] = dep.schema.schema.proto_file_element.types - sub_deps = dep.schema.schema.dependencies - package_name = dep.schema.schema.proto_file_element.package_name - type_: TypeElement - for type_ in types: - if is_self: - compare_types.add_self_type(package_name, type_) - else: - compare_types.add_other_type(package_name, type_) - if sub_deps is None: - return - collect_dependencies_types(compare_types, sub_deps, is_self) +from typing import List, Optional class ProtoFileElement: @@ -119,17 +103,82 @@ def compare( self, other: "ProtoFileElement", result: CompareResult, - self_dependencies: Optional[Dict[str, Dependency]] = None, - other_dependencies: Optional[Dict[str, Dependency]] = None, + self_dependency_types: Optional[List[TypeElement]] = None, + other_dependency_types: Optional[List[TypeElement]] = None, ) -> CompareResult: - if self.package_name != other.package_name: result.add_modification(Modification.PACKAGE_ALTER) # TODO: do we need syntax check? if self.syntax != other.syntax: result.add_modification(Modification.SYNTAX_ALTER) + self_types = {} + other_types = {} + self_indexes = {} + other_indexes = {} compare_types = CompareTypes(self.package_name, other.package_name, result) - collect_dependencies_types(compare_types, self_dependencies, True) - collect_dependencies_types(compare_types, other_dependencies, False) - return compare_type_lists(self.types, other.types, result, compare_types) + type_: TypeElement + for i, type_ in enumerate(self.types): + self_types[type_.name] = type_ + self_indexes[type_.name] = i + package_name = self.package_name or "" + compare_types.add_self_type(package_name, type_) + + for i, type_ in enumerate(other.types): + other_types[type_.name] = type_ + other_indexes[type_.name] = i + package_name = other.package_name or "" + compare_types.add_other_type(package_name, type_) + + # If there are dependencies declared, add the types for both. + if self_dependency_types: + for i, type_ in enumerate(self_dependency_types): + package_name = "" + + self_types[type_.name] = type_ + self_indexes[type_.name] = i + compare_types.add_self_type(package_name, type_) + + if other_dependency_types: + for i, type_ in enumerate(other_dependency_types): + package_name = "" + other_types[type_.name] = type_ + other_indexes[type_.name] = i + compare_types.add_other_type(package_name, type_) + + for name in chain(self_types.keys(), other_types.keys() - self_types.keys()): + + result.push_path(str(name), True) + + if self_types.get(name) is None and other_types.get(name) is not None: + if isinstance(other_types[name], MessageElement): + result.add_modification(Modification.MESSAGE_ADD) + elif isinstance(other_types[name], EnumElement): + result.add_modification(Modification.ENUM_ADD) + else: + raise IllegalStateException("Instance of element is not applicable") + elif self_types.get(name) is not None and other_types.get(name) is None: + if isinstance(self_types[name], MessageElement): + result.add_modification(Modification.MESSAGE_DROP) + elif isinstance(self_types[name], EnumElement): + result.add_modification(Modification.ENUM_DROP) + else: + raise IllegalStateException("Instance of element is not applicable") + else: + if other_indexes[name] != self_indexes[name]: + if isinstance(self_types[name], MessageElement): + # incompatible type + result.add_modification(Modification.MESSAGE_MOVE) + else: + raise IllegalStateException("Instance of element is not applicable") + else: + if isinstance(self_types[name], MessageElement) and isinstance(other_types[name], MessageElement): + self_types[name].compare(other_types[name], result, compare_types) + elif isinstance(self_types[name], EnumElement) and isinstance(other_types[name], EnumElement): + self_types[name].compare(other_types[name], result, compare_types) + else: + # incompatible type + result.add_modification(Modification.TYPE_ALTER) + result.pop_path(True) + + return result diff --git a/karapace/protobuf/schema.py b/karapace/protobuf/schema.py index acde54506..e85b3df35 100644 --- a/karapace/protobuf/schema.py +++ b/karapace/protobuf/schema.py @@ -1,4 +1,5 @@ # Ported from square/wire: +# Ported from square/wire: # wire-library/wire-schema/src/commonMain/kotlin/com/squareup/wire/schema/Schema.kt # Ported partially for required functionality. from karapace.dependency import Dependency, DependencyVerifierResult @@ -13,8 +14,7 @@ from karapace.protobuf.proto_parser import ProtoParser from karapace.protobuf.type_element import TypeElement from karapace.protobuf.utils import append_documentation, append_indented -from karapace.schema_references import References -from typing import Dict, Optional +from typing import List, Optional def add_slashes(text: str) -> str: @@ -105,15 +105,12 @@ def option_element_string(option: OptionElement) -> str: class ProtobufSchema: DEFAULT_LOCATION = Location.get("") - def __init__( - self, schema: str, references: Optional[References] = None, dependencies: Optional[Dict[str, Dependency]] = None - ) -> None: + def __init__(self, schema: str, dependencies: Optional[List[Dependency]] = None) -> None: if type(schema).__name__ != "str": raise IllegalArgumentException("Non str type of schema string") self.dirty = schema self.cache_string = "" self.proto_file_element = ProtoParser.parse(self.DEFAULT_LOCATION, schema) - self.references = references self.dependencies = dependencies def gather_deps(self) -> ProtobufDependencyVerifier: @@ -127,10 +124,9 @@ def verify_schema_dependencies(self) -> DependencyVerifierResult: return verifier.verify() def collect_dependencies(self, verifier: ProtobufDependencyVerifier): - if self.dependencies: - for key in self.dependencies: - self.dependencies[key].schema.schema.collect_dependencies(verifier) + for dependency in self.dependencies: + dependency.schema.schema.collect_dependencies(verifier) # verifier.add_import?? we have no access to own Kafka structure from this class... # but we need data to analyse imports to avoid ciclyc dependencies... @@ -222,9 +218,21 @@ def to_schema(self) -> str: return "".join(strings) def compare(self, other: "ProtobufSchema", result: CompareResult) -> CompareResult: - return self.proto_file_element.compare( + self_dependency_types: List[TypeElement] = [] + other_dependency_types: List[TypeElement] = [] + if self.dependencies: + for dependency in self.dependencies: + schema = dependency.schema.schema + if schema.proto_file_element.types: + self_dependency_types += schema.proto_file_element.types + if other.dependencies: + for dependency in other.dependencies: + schema = dependency.schema.schema + if schema.proto_file_element.types: + other_dependency_types += schema.proto_file_element.types + self.proto_file_element.compare( other.proto_file_element, result, - self.dependencies, - other.dependencies, + self_dependency_types=self_dependency_types, + other_dependency_types=other_dependency_types, ) diff --git a/karapace/protobuf/type_element.py b/karapace/protobuf/type_element.py index 58be0369b..568c6e4d5 100644 --- a/karapace/protobuf/type_element.py +++ b/karapace/protobuf/type_element.py @@ -1,8 +1,6 @@ # Ported from square/wire: # wire-library/wire-schema/src/commonMain/kotlin/com/squareup/wire/schema/internal/parser/TypeElement.kt from dataclasses import dataclass -from karapace.protobuf.compare_result import CompareResult -from karapace.protobuf.compare_type_storage import CompareTypes from karapace.protobuf.location import Location from typing import List, TYPE_CHECKING @@ -32,6 +30,3 @@ def __repr__(self) -> str: def __str__(self) -> str: mytype = type(self) return f"{mytype}({self.to_schema()})" - - def compare(self, other: "TypeElement", result: CompareResult, types: CompareTypes) -> None: - pass diff --git a/karapace/schema_models.py b/karapace/schema_models.py index 7448c4a8f..8ad8b43fc 100644 --- a/karapace/schema_models.py +++ b/karapace/schema_models.py @@ -2,6 +2,7 @@ from avro.schema import parse as avro_parse, Schema as AvroSchema from jsonschema import Draft7Validator from jsonschema.exceptions import SchemaError +from karapace.dependency import Dependency from karapace.errors import InvalidSchema from karapace.protobuf.exception import ( Error as ProtobufError, @@ -12,11 +13,10 @@ ProtobufUnresolvedDependencyException, SchemaParseException as ProtobufSchemaParseException, ) -from karapace.protobuf.schema import Dependency, ProtobufSchema -from karapace.schema_references import References +from karapace.protobuf.schema import ProtobufSchema from karapace.schema_type import SchemaType from karapace.utils import json_encode -from typing import Any, Dict, Optional, Union +from typing import Any, Dict, List, Optional, Union import json @@ -52,7 +52,9 @@ def parse_jsonschema_definition(schema_definition: str) -> Draft7Validator: def parse_protobuf_schema_definition( - schema_definition: str, references: Optional[References] = None, dependencies: Optional[Dict[str, Dependency]] = None + schema_definition: str, + references: Optional[List[Dependency]] = None, + validate_references: bool = True, ) -> ProtobufSchema: """Parses and validates `schema_definition`. @@ -60,34 +62,38 @@ def parse_protobuf_schema_definition( Nothing yet. """ - protobuf_schema = ProtobufSchema(schema_definition, references, dependencies) - result = protobuf_schema.verify_schema_dependencies() - if not result.result: - raise ProtobufUnresolvedDependencyException(f"{result.message}") + protobuf_schema = ProtobufSchema(schema_definition, references) + if validate_references: + result = protobuf_schema.verify_schema_dependencies() + if not result.result: + raise ProtobufUnresolvedDependencyException(f"{result.message}") return protobuf_schema class TypedSchema: def __init__( self, + *, schema_type: SchemaType, schema_str: str, - references: Optional[References] = None, - dependencies: Optional[Dict[str, Dependency]] = None, + schema: Optional[Union[Draft7Validator, AvroSchema, ProtobufSchema]] = None, + references: Optional[List[Dependency]] = None, ): """Schema with type information Args: schema_type (SchemaType): The type of the schema schema_str (str): The original schema string - references(References): The references of schema + schema (Optional[Union[Draft7Validator, AvroSchema, ProtobufSchema]]): The parsed and validated schema + references (Optional[List[Dependency]]): The references of schema """ self.schema_type = schema_type self.schema_str = schema_str self.references = references - self.dependencies = dependencies self.max_id: Optional[int] = None + self._str_cached: Optional[str] = None + self._schema_cached: Optional[Union[Draft7Validator, AvroSchema, ProtobufSchema]] = schema def to_dict(self) -> Dict[str, Any]: if self.schema_type is SchemaType.PROTOBUF: @@ -105,7 +111,43 @@ def __str__(self) -> str: def __repr__(self) -> str: return f"TypedSchema(type={self.schema_type}, schema={str(self)})" - def get_references(self) -> Optional["References"]: + @property + def schema(self) -> Union[Draft7Validator, AvroSchema, ProtobufSchema]: + if self._schema_cached is not None: + return self._schema_cached + if self.schema_type is SchemaType.AVRO: + try: + self._schema_cached = parse_avro_schema_definition(self.schema_str) + except (SchemaParseException, json.JSONDecodeError, TypeError) as e: + raise InvalidSchema from e + + elif self.schema_type is SchemaType.JSONSCHEMA: + try: + self._schema_cached = parse_jsonschema_definition(self.schema_str) + # TypeError - Raised when the user forgets to encode the schema as a string. + except (TypeError, json.JSONDecodeError, SchemaError, AssertionError) as e: + raise InvalidSchema from e + + elif self.schema_type is SchemaType.PROTOBUF: + try: + self._schema_cached = parse_protobuf_schema_definition(self.schema_str, self.references) + except ( + TypeError, + SchemaError, + AssertionError, + ProtobufParserRuntimeException, + IllegalStateException, + IllegalArgumentException, + ProtobufError, + ProtobufException, + ProtobufSchemaParseException, + ) as e: + raise InvalidSchema from e + else: + raise InvalidSchema(f"Unknown parser {self.schema_type} for {self.schema_str}") + return self._schema_cached + + def get_references(self) -> Optional[List[Dependency]]: return self.references def __eq__(self, other: Any) -> bool: @@ -114,9 +156,7 @@ def __eq__(self, other: Any) -> bool: ) if not schema_is_equal: return False - if self.references is not None: - return self.references == other.references - return other.references is None + return self.references == other.references class ValidatedTypedSchema(TypedSchema): @@ -125,18 +165,15 @@ def __init__( schema_type: SchemaType, schema_str: str, schema: Union[Draft7Validator, AvroSchema, ProtobufSchema], - references: Optional["References"] = None, - dependencies: Optional[Dict[str, Dependency]] = None, + references: Optional[List[Dependency]] = None, ): - super().__init__(schema_type=schema_type, schema_str=schema_str, references=references, dependencies=dependencies) - self.schema = schema + super().__init__(schema_type=schema_type, schema_str=schema_str, references=references, schema=schema) @staticmethod def parse( schema_type: SchemaType, schema_str: str, - references: Optional["References"] = None, - dependencies: Optional[Dict[str, Dependency]] = None, + references: Optional[List[Dependency]] = None, ) -> "ValidatedTypedSchema": if schema_type not in [SchemaType.AVRO, SchemaType.JSONSCHEMA, SchemaType.PROTOBUF]: raise InvalidSchema(f"Unknown parser {schema_type} for {schema_str}") @@ -157,14 +194,14 @@ def parse( elif schema_type is SchemaType.PROTOBUF: try: - parsed_schema = parse_protobuf_schema_definition(schema_str, references, dependencies) + parsed_schema = parse_protobuf_schema_definition(schema_str, references) except ( TypeError, SchemaError, AssertionError, ProtobufParserRuntimeException, - IllegalStateException, IllegalArgumentException, + IllegalStateException, ProtobufError, ProtobufException, ProtobufSchemaParseException, @@ -178,7 +215,6 @@ def parse( schema_str=schema_str, schema=parsed_schema, references=references, - dependencies=dependencies, ) def __str__(self) -> str: diff --git a/karapace/schema_reader.py b/karapace/schema_reader.py index 01953f3a7..21bfdf500 100644 --- a/karapace/schema_reader.py +++ b/karapace/schema_reader.py @@ -11,17 +11,18 @@ from karapace import constants from karapace.config import Config from karapace.dependency import Dependency -from karapace.errors import InvalidReferences, InvalidSchema, ReferencesNotSupportedException +from karapace.errors import InvalidReferences, InvalidSchema from karapace.key_format import is_key_in_canonical_format, KeyFormatter, KeyMode from karapace.master_coordinator import MasterCoordinator -from karapace.schema_models import SchemaType, TypedSchema, ValidatedTypedSchema -from karapace.schema_references import References +from karapace.schema_models import parse_protobuf_schema_definition, SchemaType, TypedSchema +from karapace.schema_references import Reference from karapace.statsd import StatsClient -from karapace.typing import SubjectData +from karapace.typing import JsonData, SubjectData from karapace.utils import KarapaceKafkaClient, reference_key from threading import Condition, Event, Lock, Thread -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Dict, List, Optional +import hashlib import json import logging @@ -314,6 +315,7 @@ def handle_messages(self) -> None: schema_records_processed_keymode_canonical, schema_records_processed_keymode_deprecated_karapace, ) + self.log_state() def _report_schema_metrics( self, @@ -431,6 +433,7 @@ def _handle_msg_schema(self, key: dict, value: Optional[dict]) -> None: schema_version = value["version"] schema_deleted = value.get("deleted", False) schema_references = value.get("references", None) + resolved_references: Optional[List[Dependency]] = None try: schema_type_parsed = SchemaType(schema_type) @@ -452,10 +455,13 @@ def _handle_msg_schema(self, key: dict, value: Optional[dict]) -> None: return elif schema_type_parsed == SchemaType.PROTOBUF: try: - resolved_references, resolved_dependencies = self.resolve_schema_references(value) - parsed_schema = ValidatedTypedSchema.parse( - SchemaType.PROTOBUF, schema_str, resolved_references, resolved_dependencies - ) + if schema_references: + references = [ + Reference(reference["name"], reference["subject"], reference["version"]) + for reference in schema_references + ] + resolved_references = self.resolve_references(references) + parsed_schema = parse_protobuf_schema_definition(schema_str, resolved_references, validate_references=False) schema_str = str(parsed_schema) except InvalidSchema: LOG.exception("Schema is not valid ProtoBuf definition") @@ -474,7 +480,7 @@ def _handle_msg_schema(self, key: dict, value: Optional[dict]) -> None: typed_schema = TypedSchema( schema_type=schema_type_parsed, schema_str=schema_str, - references=schema_references, + references=resolved_references, ) schema = { "schema": typed_schema, @@ -482,8 +488,8 @@ def _handle_msg_schema(self, key: dict, value: Optional[dict]) -> None: "id": schema_id, "deleted": schema_deleted, } - if schema_references: - schema["references"] = schema_references + if resolved_references: + schema["references"] = resolved_references if schema_version in subjects_schemas: LOG.info("Updating entry subject: %r version: %r id: %r", schema_subject, schema_version, schema_id) @@ -491,9 +497,9 @@ def _handle_msg_schema(self, key: dict, value: Optional[dict]) -> None: LOG.info("Adding entry subject: %r version: %r id: %r", schema_subject, schema_version, schema_id) subjects_schemas[schema_version] = schema - if schema_references: - for ref in schema_references: - ref_str = reference_key(ref["subject"], ref["version"]) + if resolved_references: + for ref in resolved_references: + ref_str = reference_key(ref.subject, ref.version) referents = self.referenced_by.get(ref_str, None) if referents: referents.append(schema_id) @@ -511,6 +517,7 @@ def handle_msg(self, key: dict, value: Optional[dict]) -> None: if key["keytype"] == "CONFIG": self._handle_msg_config(key, value) elif key["keytype"] == "SCHEMA": + LOG.error("HANDLING SCHEMA MESSAGE") self._handle_msg_schema(key, value) elif key["keytype"] == "DELETE_SUBJECT": self._handle_msg_delete_subject(key, value) @@ -552,61 +559,71 @@ def get_schemas_list(self, *, include_deleted: bool, latest_only: bool) -> Dict[ res_schemas[subject] = selected_schemas return res_schemas - def remove_referenced_by(self, schema_id: SchemaId, references: List): + def remove_referenced_by(self, schema_id: SchemaId, references: List[Reference]): for ref in references: - key = reference_key(ref["subject"], ref["version"]) + key = reference_key(ref.subject, ref.version) if self.referenced_by.get(key, None) and schema_id in self.referenced_by[key]: self.referenced_by[key].remove(schema_id) - def resolve_references(self, references: Optional["References"] = None) -> Optional[Dict[str, Dependency]]: - if references is None: - return None - dependencies = dict() - - for r in references.val(): - subject = r["subject"] - version = r["version"] - name = r["name"] - subject_data = self.subjects.get(subject) - if subject_data is not None: - schema_data = subject_data["schemas"][version] - schema_references, schema_dependencies = self.resolve_schema_references(schema_data) + def _resolve_reference(self, reference: Reference) -> List[Dependency]: + subject_data = self.subjects.get(reference.subject) + if not subject_data: + raise InvalidReferences(f"Subject not found {reference.subject}.") + schema = subject_data["schemas"].get(reference.version, {}).get("schema", None) + if not schema: + raise InvalidReferences(f"No schema in {reference.subject} with version {reference.version}.") + + resolved_references = [Dependency.of(reference, schema)] + if schema.references: + resolved_references += self.resolve_references(schema.references) + return resolved_references + + def resolve_references(self, references: List[Reference]) -> List[Dependency]: + resolved_references = [] + for reference in references: + resolved_references += self._resolve_reference(reference) + return resolved_references + + def _build_state_dict(self) -> JsonData: + state = {"schemas": [], "subjects": {}} + for schema_id, schema in self.schemas.items(): + if schema.schema_type == SchemaType.AVRO: + schema_str = json.dumps(json.loads(schema.schema_str), sort_keys=True) else: - raise InvalidReferences - - parsed_schema = ValidatedTypedSchema.parse( - schema_type=schema_data["schema"].schema_type, - schema_str=schema_data["schema"].schema_str, - references=schema_references, - dependencies=schema_dependencies, + schema_str = schema.schema_str + schema_hash = hashlib.sha1(schema_str.encode("utf8")).hexdigest() + state["schemas"].append( + { + "id": schema_id, + "schema_hash": schema_hash, + } ) - dependencies[name] = Dependency(name, subject, version, parsed_schema) - return dependencies - - def resolve_schema_references( - self, schema_data: Optional[dict] - ) -> Tuple[Optional[References], Optional[Dict[str, Dependency]]]: - - if schema_data is None: - raise InvalidSchema - - schema_references = schema_data.get("references") - if schema_references is None: - return None, None - - schema_type = schema_data.get("schemaType") - if schema_type is None: - schema = schema_data.get("schema") - if schema is None: - raise InvalidReferences - if isinstance(schema, TypedSchema): - schema_type = schema.schema_type - else: - schema_type = None - if schema_type != SchemaType.PROTOBUF: - raise ReferencesNotSupportedException - - schema_references = References(schema_type, schema_references) - schema_dependencies = self.resolve_references(schema_references) + for subject, subject_data in self.subjects.items(): + subject_state = [] + state["subjects"][subject] = subject_state + for _, schema in subject_data.get("schemas", {}).items(): + schema_id = schema.get("id") + version = schema.get("version") + if schema.get("schema").schema_type == SchemaType.AVRO: + schema_str = json.dumps(json.loads(schema.get("schema").schema_str), sort_keys=True) + else: + schema_str = schema.get("schema").schema_str + schema_hash = hashlib.sha1(schema_str.encode("utf8")).hexdigest() + references = schema.get("references", None) + references_list = None + if references: + references_list = [reference.to_dict() for reference in schema["references"]] + subject_state.append( + { + "id": schema_id, + "version": version, + "schema_hash": schema_hash, + "references": references_list, + "deleted": schema.get("deleted", False), + } + ) + return json.dumps(state, sort_keys=True, indent=2) - return schema_references, schema_dependencies + def log_state(self) -> None: + state_str = self._build_state_dict() + LOG.log(level=100, msg=state_str) diff --git a/karapace/schema_references.py b/karapace/schema_references.py index 6663a3d12..46823fb4c 100644 --- a/karapace/schema_references.py +++ b/karapace/schema_references.py @@ -1,27 +1,30 @@ -from karapace.schema_type import SchemaType -from karapace.typing import JsonData -from karapace.utils import json_encode +from karapace.typing import JsonData, Subject, Version from typing import Any -class References: - def __init__(self, schema_type: SchemaType, references: JsonData): - """Schema with type information +class Reference: + def __init__(self, name: str, subject: Subject, version: Version): + self.name = name + self.subject = subject + self.version = version - Args: - schema_type (SchemaType): The type of the schema - references (str): The original schema string - """ - self.schema_type = schema_type - self.references = references + def identifier(self) -> str: + return self.name + "_" + self.subject + "_" + str(self.version) - def val(self) -> JsonData: - return self.references + def to_dict(self) -> JsonData: + return { + "name": self.name, + "subject": self.subject, + "version": self.version, + } - def json(self) -> str: - return str(json_encode(self.references, sort_keys=True)) + def __repr__(self) -> str: + return f"{{name='{self.name}', subject='{self.subject}', version={self.version}}}" + + def __hash__(self) -> int: + return hash((self.name, self.subject, self.version)) def __eq__(self, other: Any) -> bool: - if other is None or not isinstance(other, References): + if other is None or not isinstance(other, Reference): return False - return self.json() == other.json() + return self.name == other.name and self.subject == other.subject and self.version == other.version diff --git a/karapace/schema_registry.py b/karapace/schema_registry.py index b5d65979f..1b618a481 100644 --- a/karapace/schema_registry.py +++ b/karapace/schema_registry.py @@ -22,7 +22,7 @@ from karapace.master_coordinator import MasterCoordinator from karapace.schema_models import SchemaType, TypedSchema, ValidatedTypedSchema from karapace.schema_reader import KafkaSchemaReader -from karapace.schema_references import References +from karapace.schema_references import Reference from karapace.typing import ResolvedVersion, Subject, SubjectData, Version from karapace.utils import json_encode, KarapaceKafkaClient, reference_key from typing import Dict, List, Optional, Tuple, Union @@ -247,10 +247,7 @@ async def subject_version_delete_local(self, subject: Subject, version: Version, schema_id = subject_schema_data["id"] schema = subject_schema_data["schema"] - references_list = subject_schema_data.get("references", None) - references = None - if references_list: - references = References(schema.schema_type, references_list) + references = subject_schema_data.get("references", None) self.send_schema_message( subject=subject, schema=None if permanent else schema, @@ -259,8 +256,8 @@ async def subject_version_delete_local(self, subject: Subject, version: Version, deleted=True, references=references, ) - if references_list and len(references_list) > 0: - self.schema_reader.remove_referenced_by(schema_id, references_list) + if references and len(references) > 0: + self.schema_reader.remove_referenced_by(schema_id, references) return resolved_version def subject_get(self, subject: Subject, include_deleted: bool = False) -> SubjectData: @@ -297,8 +294,7 @@ async def subject_version_get(self, subject: Subject, version: Version, *, inclu "id": schema_id, "schema": schema.schema_str, } - if schema.references is not None: - ret["references"] = schema.references + if schema.schema_type is not SchemaType.AVRO: ret["schemaType"] = schema.schema_type # Return also compatibility information to compatibility check @@ -310,7 +306,7 @@ async def write_new_schema_local( self, subject: Subject, new_schema: ValidatedTypedSchema, - new_schema_references: Optional[References], + new_schema_references: Optional[List[Dependency]], ) -> int: """Write new schema and return new id or return id of matching existing schema @@ -372,18 +368,8 @@ async def write_new_schema_local( for old_version in check_against: old_schema = subject_data["schemas"][old_version]["schema"] - old_schema_references, old_schema_dependencies = self.resolve_schema_references( - subject_data["schemas"][old_version], - ) - - validated_old_schema = ValidatedTypedSchema.parse( - schema_type=old_schema.schema_type, - schema_str=old_schema.schema_str, - references=old_schema_references, - dependencies=old_schema_dependencies, - ) result = check_compatibility( - old_schema=validated_old_schema, + old_schema=old_schema, new_schema=new_schema, compatibility_mode=compatibility_mode, ) @@ -484,7 +470,7 @@ def send_schema_message( schema_id: int, version: int, deleted: bool, - references: Optional[References], + references: Optional[List[Dependency]], ) -> FutureRecordMetadata: key = self.key_formatter.format_key( {"subject": subject, "version": version, "magic": 1, "keytype": "SCHEMA"}, @@ -498,7 +484,7 @@ def send_schema_message( "deleted": deleted, } if references: - valuedict["references"] = references.val() + valuedict["references"] = [reference.to_dict() for reference in references] if schema.schema_type is not SchemaType.AVRO: valuedict["schemaType"] = schema.schema_type value = json_encode(valuedict) @@ -540,10 +526,7 @@ def send_delete_subject_message(self, subject: Subject, version: Version) -> Fut value = '{{"subject":"{}","version":{}}}'.format(subject, version) return self.send_kafka_message(key, value) - def resolve_references(self, references: Optional["References"] = None) -> Optional[Dict[str, Dependency]]: - return self.schema_reader.resolve_references(references) - - def resolve_schema_references( - self, schema_data: Optional[dict] - ) -> Tuple[Optional[References], Optional[Dict[str, Dependency]]]: - return self.schema_reader.resolve_schema_references(schema_data) + def resolve_references(self, references: Optional[List[Reference]]) -> Optional[List[Dependency]]: + if references: + return self.schema_reader.resolve_references(references) + return None diff --git a/karapace/schema_registry_apis.py b/karapace/schema_registry_apis.py index 3be4ffd4b..216ed02e2 100644 --- a/karapace/schema_registry_apis.py +++ b/karapace/schema_registry_apis.py @@ -13,7 +13,6 @@ InvalidSchemaType, InvalidVersion, ReferenceExistsException, - ReferencesNotSupportedException, SchemasNotFoundException, SchemaTooLargeException, SchemaVersionNotSoftDeletedException, @@ -26,10 +25,11 @@ from karapace.karapace import KarapaceBase from karapace.rapu import HTTPRequest, JSON_CONTENT_TYPE, SERVER_NAME from karapace.schema_models import SchemaType, TypedSchema, ValidatedTypedSchema +from karapace.schema_references import Reference from karapace.schema_registry import KarapaceSchemaRegistry, validate_version from karapace.typing import JsonData from karapace.utils import reference_key -from typing import Any, Dict, Optional, Union +from typing import Any, Dict, List, Optional, Union import aiohttp import async_timeout @@ -55,7 +55,6 @@ class SchemaErrorCodes(Enum): INVALID_SCHEMA = 42201 INVALID_SUBJECT = 42208 SCHEMA_TOO_LARGE_ERROR_CODE = 42209 - INVALID_REFERENCES = 44301 REFERENCES_SUPPORT_NOT_IMPLEMENTED = 44302 REFERENCE_EXISTS = 42206 NO_MASTER_ERROR = 50003 @@ -309,13 +308,13 @@ async def compatibility_check( body = request.json schema_type = self._validate_schema_type(content_type=content_type, data=body) + references = self._validate_references(content_type, schema_type, body) try: - new_schema_references, new_schema_dependencies = self.schema_registry.resolve_schema_references(body) + new_schema_references = self.schema_registry.resolve_references(references) new_schema = ValidatedTypedSchema.parse( schema_type=schema_type, schema_str=body["schema"], references=new_schema_references, - dependencies=new_schema_dependencies, ) except InvalidSchema: self.r( @@ -327,7 +326,7 @@ async def compatibility_check( status=HTTPStatus.UNPROCESSABLE_ENTITY, ) try: - old = await self.schema_registry.subject_version_get(subject=subject, version=version) + old_subject_data = await self.schema_registry.subject_version_get(subject=subject, version=version) except InvalidVersion: self._invalid_version(content_type, version) except (VersionNotFoundException, SchemasNotFoundException, SubjectNotFoundException): @@ -340,27 +339,8 @@ async def compatibility_check( status=HTTPStatus.NOT_FOUND, ) - old_schema_type = self._validate_schema_type(content_type=content_type, data=old) try: - old_schema_references, old_schema_dependencies = self.schema_registry.resolve_schema_references(old) - old_schema = ValidatedTypedSchema.parse( - schema_type=old_schema_type, - schema_str=old["schema"], - references=old_schema_references, - dependencies=old_schema_dependencies, - ) - except InvalidSchema: - self.r( - body={ - "error_code": SchemaErrorCodes.INVALID_SCHEMA.value, - "message": f"Found an invalid {old_schema_type} schema registered", - }, - content_type=content_type, - status=HTTPStatus.UNPROCESSABLE_ENTITY, - ) - - try: - compatibility_mode = self.schema_registry.get_compatibility_mode(subject=old) + compatibility_mode = self.schema_registry.get_compatibility_mode(subject=old_subject_data) except ValueError as ex: # Using INTERNAL_SERVER_ERROR because the subject and configuration # should have been validated before. @@ -373,6 +353,7 @@ async def compatibility_check( status=HTTPStatus.INTERNAL_SERVER_ERROR, ) + old_schema = self.schema_registry.schemas_get(old_subject_data.get("id")) result = check_compatibility( old_schema=old_schema, new_schema=new_schema, @@ -672,11 +653,11 @@ async def subject_delete( "error_code": SchemaErrorCodes.REFERENCE_EXISTS.value, "message": ( f"One or more references exist to the schema " - f"{{magic=1,keytype=SCHEMA,subject={subject},version={arg.version}}}" + f"{{magic=1,keytype=SCHEMA,subject={subject},version={arg.version}}}." ), }, content_type=content_type, - status=HTTPStatus.NOT_FOUND, + status=HTTPStatus.UNPROCESSABLE_ENTITY, ) elif not master_url: self.no_master_error(content_type) @@ -775,11 +756,11 @@ async def subject_version_delete( "error_code": SchemaErrorCodes.REFERENCE_EXISTS.value, "message": ( f"One or more references exist to the schema " - f"{{magic=1,keytype=SCHEMA,subject={subject},version={arg.version}}}" + f"{{magic=1,keytype=SCHEMA,subject={subject},version={arg.version}}}." ), }, content_type=content_type, - status=HTTPStatus.NOT_FOUND, + status=HTTPStatus.UNPROCESSABLE_ENTITY, ) elif not master_url: self.no_master_error(content_type) @@ -926,6 +907,29 @@ def _validate_schema_key(self, content_type: str, body: dict) -> None: status=HTTPStatus.UNPROCESSABLE_ENTITY, ) + def _validate_references(self, content_type: str, schema_type: SchemaType, body: JsonData) -> List[Reference]: + references = body.get("references", []) + if references and schema_type != SchemaType.PROTOBUF: + self.r( + body={ + "error_code": SchemaErrorCodes.REFERENCES_SUPPORT_NOT_IMPLEMENTED.value, + "message": SchemaErrorMessages.REFERENCES_SUPPORT_NOT_IMPLEMENTED.value.format( + schema_type=schema_type.value + ), + }, + content_type=content_type, + status=HTTPStatus.UNPROCESSABLE_ENTITY, + ) + + validated_references = [] + for reference in references: + if ["name", "subject", "version"] != sorted(reference.keys()): + raise InvalidReferences() + validated_references.append( + Reference(name=reference["name"], subject=reference["subject"], version=reference["version"]) + ) + return validated_references + async def subjects_schema_post( self, content_type: str, *, subject: str, request: HTTPRequest, user: Optional[User] = None ) -> None: @@ -957,14 +961,14 @@ async def subjects_schema_post( ) schema_str = body["schema"] schema_type = self._validate_schema_type(content_type=content_type, data=body) + references = self._validate_references(content_type, schema_type, body) try: - new_schema_references, new_schema_dependencies = self.schema_registry.resolve_schema_references(body) + new_schema_references = self.schema_registry.resolve_references(references) new_schema = ValidatedTypedSchema.parse( schema_type=schema_type, schema_str=schema_str, references=new_schema_references, - dependencies=new_schema_dependencies, ) for schema in subject_data["schemas"].values(): validated_typed_schema = ValidatedTypedSchema.parse( @@ -1000,22 +1004,11 @@ async def subjects_schema_post( content_type=content_type, status=HTTPStatus.INTERNAL_SERVER_ERROR, ) - except ReferencesNotSupportedException: - self.r( - body={ - "error_code": SchemaErrorCodes.REFERENCES_SUPPORT_NOT_IMPLEMENTED.value, - "message": SchemaErrorMessages.REFERENCES_SUPPORT_NOT_IMPLEMENTED.value.format( - schema_type=schema_type.value - ), - }, - content_type=content_type, - status=HTTPStatus.UNPROCESSABLE_ENTITY, - ) except InvalidReferences: human_error = "Provided references is not valid" self.r( body={ - "error_code": SchemaErrorCodes.INVALID_REFERENCES.value, + "error_code": SchemaErrorCodes.INVALID_SCHEMA.value, "message": f"Invalid {schema_type} references. Error: {human_error}", }, content_type=content_type, @@ -1062,46 +1055,26 @@ async def subject_post( self._validate_schema_request_body(content_type, body) schema_type = self._validate_schema_type(content_type, body) self._validate_schema_key(content_type, body) + references = self._validate_references(content_type, schema_type, body) try: - new_schema_references, new_schema_dependencies = self.schema_registry.resolve_schema_references(body) + resolved_references = self.schema_registry.resolve_references(references) new_schema = ValidatedTypedSchema.parse( schema_type=schema_type, schema_str=body["schema"], - references=new_schema_references, - dependencies=new_schema_dependencies, + references=resolved_references, ) - except (InvalidSchema, InvalidSchemaType) as e: + except (InvalidReferences, InvalidSchema, InvalidSchemaType) as e: self.log.warning("Invalid schema: %r", body["schema"], exc_info=True) if isinstance(e.__cause__, (SchemaParseException, json.JSONDecodeError)): human_error = f"{e.__cause__.args[0]}" # pylint: disable=no-member else: - human_error = "Provided schema is not valid" + from_body_schema_str = body["schema"] + human_error = f"Invalid schema {from_body_schema_str} with refs {references} of type {schema_type}" self.r( body={ "error_code": SchemaErrorCodes.INVALID_SCHEMA.value, - "message": f"Invalid {schema_type} schema. Error: {human_error}", - }, - content_type=content_type, - status=HTTPStatus.UNPROCESSABLE_ENTITY, - ) - except ReferencesNotSupportedException: - self.r( - body={ - "error_code": SchemaErrorCodes.REFERENCES_SUPPORT_NOT_IMPLEMENTED.value, - "message": SchemaErrorMessages.REFERENCES_SUPPORT_NOT_IMPLEMENTED.value.format( - schema_type=schema_type.value - ), - }, - content_type=content_type, - status=HTTPStatus.UNPROCESSABLE_ENTITY, - ) - except InvalidReferences: - human_error = "Provided references is not valid" - self.r( - body={ - "error_code": SchemaErrorCodes.INVALID_REFERENCES.value, - "message": f"Invalid {schema_type} references. Error: {human_error}", + "message": human_error, }, content_type=content_type, status=HTTPStatus.UNPROCESSABLE_ENTITY, @@ -1122,7 +1095,7 @@ async def subject_post( are_we_master, master_url = await self.schema_registry.get_master() if are_we_master: try: - schema_id = await self.schema_registry.write_new_schema_local(subject, new_schema, new_schema_references) + schema_id = await self.schema_registry.write_new_schema_local(subject, new_schema, resolved_references) self.r( body={"id": schema_id}, content_type=content_type, diff --git a/karapace/serialization.py b/karapace/serialization.py index 4b272d9e4..9c0d988a5 100644 --- a/karapace/serialization.py +++ b/karapace/serialization.py @@ -3,9 +3,12 @@ from google.protobuf.message import DecodeError from jsonschema import ValidationError from karapace.client import Client +from karapace.dependency import Dependency +from karapace.errors import InvalidReferences from karapace.protobuf.exception import ProtobufTypeException from karapace.protobuf.io import ProtobufDatumReader, ProtobufDatumWriter -from karapace.schema_models import InvalidSchema, References, SchemaType, TypedSchema, ValidatedTypedSchema +from karapace.schema_models import InvalidSchema, SchemaType, TypedSchema, ValidatedTypedSchema +from karapace.schema_references import Reference from karapace.utils import json_encode from typing import Any, Dict, Optional, Tuple from urllib.parse import quote @@ -76,7 +79,7 @@ def __init__( self.base_url = schema_registry_url async def post_new_schema( - self, subject: str, schema: ValidatedTypedSchema, references: Optional[References] = None + self, subject: str, schema: ValidatedTypedSchema, references: Optional[Dependency] = None ) -> int: if schema.schema_type is SchemaType.PROTOBUF: if references: @@ -105,7 +108,7 @@ async def get_latest_schema(self, subject: str) -> Tuple[int, ValidatedTypedSche except InvalidSchema as e: raise SchemaRetrievalError(f"Failed to parse schema string from response: {json_result}") from e - async def get_schema_for_id(self, schema_id: int) -> Tuple[ValidatedTypedSchema, Optional[References]]: + async def get_schema_for_id(self, schema_id: int) -> ValidatedTypedSchema: result = await self.client.get(f"schemas/ids/{schema_id}") if not result.ok: raise SchemaRetrievalError(result.json()["message"]) @@ -114,13 +117,18 @@ async def get_schema_for_id(self, schema_id: int) -> Tuple[ValidatedTypedSchema, raise SchemaRetrievalError(f"Invalid result format: {json_result}") try: schema_type = SchemaType(json_result.get("schemaType", "AVRO")) - references_str = json_result.get("references") - if references_str: - references = References(schema_type, references_str) - else: - references = None + references = json_result.get("references") + validated_references = None if references: - return ValidatedTypedSchema.parse(schema_type, json_result["schema"]), references + validated_references = [] + for reference in references: + if ["name", "subject", "version"] != sorted(reference.keys()): + raise InvalidReferences() + validated_references.append( + Reference(name=reference["name"], subject=reference["subject"], version=reference["version"]) + ) + if validated_references: + return ValidatedTypedSchema.parse(schema_type, json_result["schema"], references=validated_references) return ValidatedTypedSchema.parse(schema_type, json_result["schema"]) except InvalidSchema as e: diff --git a/tests/integration/test_schema.py b/tests/integration/test_schema.py index 3b0ec3d5d..1b836ba88 100644 --- a/tests/integration/test_schema.py +++ b/tests/integration/test_schema.py @@ -2273,8 +2273,7 @@ async def test_invalid_namespace(registry_async_client: Client) -> None: json_res = res.json() assert json_res["error_code"] == 42201, json_res expected_message = ( - "Invalid AVRO schema. Error: foo-bar-baz is not a valid Avro name because it does not match the pattern " - "(?:^|\\.)[A-Za-z_][A-Za-z0-9_]*$" + "foo-bar-baz is not a valid Avro name because it does not match the pattern (?:^|\\.)[A-Za-z_][A-Za-z0-9_]*$" ) assert json_res["message"] == expected_message, json_res @@ -2883,7 +2882,7 @@ async def test_invalid_schema_should_provide_good_error_messages(registry_async_ f"subjects/{test_subject}/versions", json={"schema": schema_str[:-1]}, ) - assert res.json()["message"] == "Invalid AVRO schema. Error: Expecting ',' delimiter: line 1 column 18 (char 17)" + assert res.json()["message"] == "Expecting ',' delimiter: line 1 column 18 (char 17)" # Unfortunately the AVRO library doesn't provide a good error message, it just raises an TypeError schema_str = json.dumps({"type": "enum", "name": "error"}) @@ -2891,10 +2890,7 @@ async def test_invalid_schema_should_provide_good_error_messages(registry_async_ f"subjects/{test_subject}/versions", json={"schema": schema_str}, ) - assert ( - res.json()["message"] - == "Invalid AVRO schema. Error: Enum symbols must be a sequence of strings, but it is " - ) + assert res.json()["message"] == "Enum symbols must be a sequence of strings, but it is " # This is an upstream bug in the python AVRO library, until the bug is fixed we should at least have a nice error message schema_str = json.dumps({"type": "enum", "name": "error", "symbols": {}}) @@ -2902,10 +2898,7 @@ async def test_invalid_schema_should_provide_good_error_messages(registry_async_ f"subjects/{test_subject}/versions", json={"schema": schema_str}, ) - assert ( - res.json()["message"] - == "Invalid AVRO schema. Error: Enum symbols must be a sequence of strings, but it is " - ) + assert res.json()["message"] == "Enum symbols must be a sequence of strings, but it is " # This is an upstream bug in the python AVRO library, until the bug is fixed we should at least have a nice error message schema_str = json.dumps({"type": "enum", "name": "error", "symbols": ["A", "B"]}) @@ -2913,4 +2906,4 @@ async def test_invalid_schema_should_provide_good_error_messages(registry_async_ f"subjects/{test_subject}/versions", json={"schema": schema_str}, ) - assert res.json()["message"] == "Invalid AVRO schema. Error: error is a reserved type name." + assert res.json()["message"] == "error is a reserved type name." diff --git a/tests/integration/test_schema_protobuf.py b/tests/integration/test_schema_protobuf.py index 7026146a8..8ab438c80 100644 --- a/tests/integration/test_schema_protobuf.py +++ b/tests/integration/test_schema_protobuf.py @@ -4,9 +4,14 @@ Copyright (c) 2019 Aiven Ltd See LICENSE for details """ +from dataclasses import dataclass from karapace.client import Client from karapace.protobuf.kotlin_wrapper import trim_margin +from karapace.schema_type import SchemaType +from karapace.typing import JsonData +from tests.base_testcase import BaseTestCase from tests.utils import create_subject_name_factory +from typing import List, Optional, Union import logging import pytest @@ -170,6 +175,7 @@ async def test_protobuf_schema_normalization(registry_async_client: Client, trai async def test_protobuf_schema_references(registry_async_client: Client) -> None: + customer_schema = """ |syntax = "proto3"; |package a1; @@ -261,28 +267,17 @@ async def test_protobuf_schema_references(registry_async_client: Client) -> None res = await registry_async_client.get("subjects/place/versions/latest/referencedby", json={}) assert res.status_code == 200 + myjson = res.json() res = await registry_async_client.delete("subjects/customer/versions/1") - assert res.status_code == 404 + assert res.status_code == 422 - match_msg = "One or more references exist to the schema {magic=1,keytype=SCHEMA,subject=customer,version=1}" + match_msg = "One or more references exist to the schema {magic=1,keytype=SCHEMA,subject=customer,version=1}." myjson = res.json() assert myjson["error_code"] == 42206 and myjson["message"] == match_msg res = await registry_async_client.delete("subjects/test_schema/versions/1") assert res.status_code == 200 - res = await registry_async_client.delete("subjects/test_schema/versions/1") - myjson = res.json() - match_msg = "Subject 'test_schema' Version 1 was soft deleted.Set permanent=true to delete permanently" - assert res.status_code == 404 - assert myjson["error_code"] == 40406 and myjson["message"] == match_msg - - res = await registry_async_client.delete("subjects/customer/versions/1") - myjson = res.json() - match_msg = "One or more references exist to the schema {magic=1,keytype=SCHEMA,subject=customer,version=1}" - assert res.status_code == 404 - assert myjson["error_code"] == 42206 and myjson["message"] == match_msg - res = await registry_async_client.delete("subjects/test_schema/versions/1?permanent=true") assert res.status_code == 200 @@ -290,122 +285,6 @@ async def test_protobuf_schema_references(registry_async_client: Client) -> None assert res.status_code == 200 -async def test_protobuf_schema_jjaakola_one(registry_async_client: Client) -> None: - no_ref = """ - |syntax = "proto3"; - | - |message NoReference { - | string name = 1; - |} - |""" - - no_ref = trim_margin(no_ref) - res = await registry_async_client.post("subjects/sub1/versions", json={"schemaType": "PROTOBUF", "schema": no_ref}) - assert res.status_code == 200 - assert "id" in res.json() - - with_first_ref = """ - |syntax = "proto3"; - | - |import "NoReference.proto"; - | - |message WithReference { - | string name = 1; - | NoReference ref = 2; - |}""" - - with_first_ref = trim_margin(with_first_ref) - references = [{"name": "NoReference.proto", "subject": "sub1", "version": 1}] - - res = await registry_async_client.post( - "subjects/sub2/versions", - json={"schemaType": "PROTOBUF", "schema": with_first_ref, "references": references}, - ) - assert res.status_code == 200 - assert "id" in res.json() - - no_ref_second = """ - |syntax = "proto3"; - | - |message NoReferenceTwo { - | string name = 1; - |} - |""" - - no_ref_second = trim_margin(no_ref_second) - res = await registry_async_client.post( - "subjects/sub3/versions", json={"schemaType": "PROTOBUF", "schema": no_ref_second} - ) - assert res.status_code == 200 - assert "id" in res.json() - - add_new_ref_in_sub2 = """ - |syntax = "proto3"; - |import "NoReference.proto"; - |import "NoReferenceTwo.proto"; - |message WithReference { - | string name = 1; - | NoReference ref = 2; - | NoReferenceTwo refTwo = 3; - |} - |""" - - add_new_ref_in_sub2 = trim_margin(add_new_ref_in_sub2) - - references = [ - {"name": "NoReference.proto", "subject": "sub1", "version": 1}, - {"name": "NoReferenceTwo.proto", "subject": "sub3", "version": 1}, - ] - - res = await registry_async_client.post( - "subjects/sub2/versions", - json={"schemaType": "PROTOBUF", "schema": add_new_ref_in_sub2, "references": references}, - ) - assert res.status_code == 200 - assert "id" in res.json() - - -async def test_protobuf_schema_jjaakola_two(registry_async_client: Client) -> None: - no_ref = """ - |syntax = "proto3"; - | - |message NoReference { - | string name = 1; - |} - |""" - - no_ref = trim_margin(no_ref) - res = await registry_async_client.post("subjects/sub1/versions", json={"schemaType": "PROTOBUF", "schema": no_ref}) - assert res.status_code == 200 - assert "id" in res.json() - - with_first_ref = """ - |syntax = "proto3"; - | - |import "NoReference.proto"; - | - |message WithReference { - | string name = 1; - | NoReference ref = 2; - |}""" - - with_first_ref = trim_margin(with_first_ref) - references = [{"name": "NoReference.proto", "subject": "sub1", "version": 1}] - res = await registry_async_client.post( - "subjects/sub2/versions", - json={"schemaType": "PROTOBUF", "schema": with_first_ref, "references": references}, - ) - assert res.status_code == 200 - assert "id" in res.json() - - res = await registry_async_client.delete("subjects/sub2/versions/1") - assert res.status_code == 200 - res = await registry_async_client.get("subjects/sub2/versions/1") - assert res.status_code == 404 - res = await registry_async_client.delete("subjects/sub2/versions/1") - assert res.status_code == 404 - - async def test_protobuf_schema_verifier(registry_async_client: Client) -> None: customer_schema = """ |syntax = "proto3"; @@ -467,19 +346,546 @@ async def test_protobuf_schema_verifier(registry_async_client: Client) -> None: assert not any(x != y for x, y in zip(myjson, referents)) res = await registry_async_client.delete("subjects/customer/versions/1") - assert res.status_code == 404 - match_msg = "One or more references exist to the schema {magic=1,keytype=SCHEMA,subject=customer,version=1}" + assert res.status_code == 422 + match_msg = "One or more references exist to the schema {magic=1,keytype=SCHEMA,subject=customer,version=1}." myjson = res.json() assert myjson["error_code"] == 42206 and myjson["message"] == match_msg res = await registry_async_client.delete("subjects/test_schema/versions/1") assert res.status_code == 200 - res = await registry_async_client.delete("subjects/customer/versions/1") - assert res.status_code == 404 - res = await registry_async_client.delete("subjects/test_schema/versions/1?permanent=true") assert res.status_code == 200 res = await registry_async_client.delete("subjects/customer/versions/1") assert res.status_code == 200 + + +@dataclass +class TestCaseSchema: + schema_type: SchemaType + schema_str: str + subject: str + references: Optional[List[JsonData]] = None + expected: int = 200 + expected_msg: str = "" + expected_error_code: Optional[int] = None + + +TestCaseSchema.__test__ = False + + +@dataclass +class TestCaseDeleteSchema: + subject: str + version: int + schema_id: int + expected: int = 200 + expected_msg: str = "" + expected_error_code: Optional[int] = None + + +TestCaseDeleteSchema.__test__ = False + + +@dataclass +class TestCaseHardDeleteSchema(TestCaseDeleteSchema): + pass + + +@dataclass +class ReferenceTestCase(BaseTestCase): + schemas: List[Union[TestCaseSchema, TestCaseDeleteSchema]] + + +# Base case +SCHEMA_NO_REF = """\ +syntax = "proto3"; + +message NoReference { + string name = 1; +} +""" + +SCHEMA_NO_REF_TWO = """\ +syntax = "proto3"; + +message NoReferenceTwo { + string name = 1; +} +""" + +SCHEMA_WITH_REF = """\ +syntax = "proto3"; + +import "NoReference.proto"; + +message WithReference { + string name = 1; + NoReference ref = 2; +} +""" + +SCHEMA_WITH_2ND_LEVEL_REF = """\ +syntax = "proto3"; + +import "WithReference.proto"; + +message With2ndLevelReference { + string name = 1; + WithReference ref = 2; +} +""" + +SCHEMA_REMOVES_REFERENCED_FIELD_INCOMPATIBLE = """\ +syntax = "proto3"; + +message WithReference { + string name = 1; +} +""" + +SCHEMA_ADDS_NEW_REFERENCE = """\ +syntax = "proto3"; + +import "NoReference.proto"; +import "NoReferenceTwo.proto"; + +message WithReference { + string name = 1; + NoReference ref = 2; + NoReferenceTwo refTwo = 3; +} +""" + +# Invalid schema +SCHEMA_INVALID_MISSING_CLOSING_BRACE = """\ +syntax = "proto3"; + +import "NoReference.proto"; + +message SchemaMissingClosingBrace { + string name = 1; + NoReference ref = 2; + +""" + +# Schema having multiple messages +SCHEMA_NO_REF_TWO_MESSAGES = """\ +syntax = "proto3"; + +message NoReferenceOne { + string nameOne = 1; +} + +message NoReferenceTwo { + string nameTwo = 1; +} +""" + +SCHEMA_WITH_REF_TO_NO_REFERENCE_TWO = """\ +syntax = "proto3"; + +import "NoReferenceTwo.proto"; + +message WithReference { + string name = 1; + NoReferenceTwo ref = 2; +} +""" + + +# Nested references +SCHEMA_NO_REF_NESTED_MESSAGE = """\ +syntax = "proto3"; + +message NoReference { + message NoReferenceNested { + string nameNested = 1; + } + string name = 1; + NoReferenceNested ref = 2; +} +""" + +SCHEMA_WITH_REF_TO_NESTED = """\ +syntax = "proto3"; + +import "NoReferenceNested.proto"; + +message WithReference { + string name = 1; + NoReference.NoReferenceNested ref = 2; +} +""" + + +@pytest.mark.parametrize( + "testcase", + [ + ReferenceTestCase( + test_name="No references", + schemas=[ + TestCaseSchema( + schema_type=SchemaType.PROTOBUF, + schema_str=SCHEMA_NO_REF, + subject="nr_s1", + references=[], + expected=200, + ), + TestCaseDeleteSchema( + subject="nr_s1", + schema_id=1, + version=1, + expected=200, + ), + ], + ), + # Better error message should be given back, now it is only InvalidSchema + ReferenceTestCase( + test_name="With reference, ref schema does not exist", + schemas=[ + TestCaseSchema( + schema_type=SchemaType.PROTOBUF, + schema_str=SCHEMA_WITH_REF, + subject="wr_nonexisting_s1", + references=[{"name": "NoReference.proto", "subject": "wr_not_found", "version": 1}], + expected=422, + expected_msg=( + f"Invalid schema {SCHEMA_WITH_REF} " + "with refs [{name='NoReference.proto', subject='wr_not_found', version=1}] of type PROTOBUF" + ), + expected_error_code=42201, + ), + ], + ), + ReferenceTestCase( + test_name="With reference, references not given", + schemas=[ + TestCaseSchema( + schema_type=SchemaType.PROTOBUF, + schema_str=SCHEMA_WITH_REF, + subject="wr_nonexisting_s1_missing_references", + references=[], + expected=422, + expected_msg=f"Invalid schema {SCHEMA_WITH_REF} with refs [] of type PROTOBUF", + expected_error_code=42201, + ), + ], + ), + ReferenceTestCase( + test_name="With reference", + schemas=[ + TestCaseSchema( + schema_type=SchemaType.PROTOBUF, + schema_str=SCHEMA_NO_REF, + subject="wr_s1", + references=[], + expected=200, + ), + TestCaseSchema( + schema_type=SchemaType.PROTOBUF, + schema_str=SCHEMA_WITH_REF, + subject="wr_s2", + references=[{"name": "NoReference.proto", "subject": "wr_s1", "version": 1}], + expected=200, + ), + TestCaseDeleteSchema( + subject="wr_s1", + schema_id=1, + version=1, + expected=422, + expected_msg=( + "One or more references exist to the schema {magic=1,keytype=SCHEMA,subject=wr_s1,version=1}." + ), + expected_error_code=42206, + ), + TestCaseDeleteSchema( + subject="wr_s2", + schema_id=2, + version=1, + expected=200, + ), + TestCaseHardDeleteSchema( + subject="wr_s2", + schema_id=2, + version=1, + expected=200, + ), + TestCaseDeleteSchema( + subject="wr_s1", + schema_id=1, + version=1, + expected=200, + ), + ], + ), + ReferenceTestCase( + test_name="With reference, remove referenced field causes incompatible schema", + schemas=[ + TestCaseSchema( + schema_type=SchemaType.PROTOBUF, + schema_str=SCHEMA_NO_REF, + subject="wr_s1_test_incompatible_change", + references=[], + expected=200, + ), + TestCaseSchema( + schema_type=SchemaType.PROTOBUF, + schema_str=SCHEMA_WITH_REF, + subject="wr_s2_test_incompatible_change", + references=[{"name": "NoReference.proto", "subject": "wr_s1_test_incompatible_change", "version": 1}], + expected=200, + ), + TestCaseSchema( + schema_type=SchemaType.PROTOBUF, + schema_str=SCHEMA_REMOVES_REFERENCED_FIELD_INCOMPATIBLE, + subject="wr_s2_test_incompatible_change", + references=[], + expected=409, + expected_msg=( + "Incompatible schema, compatibility_mode=BACKWARD " + "Incompatible modification Modification.MESSAGE_DROP found" + ), + ), + ], + ), + ReferenceTestCase( + test_name="With reference, add new referenced field", + schemas=[ + TestCaseSchema( + schema_type=SchemaType.PROTOBUF, + schema_str=SCHEMA_NO_REF, + subject="wr_s1_add_new_reference", + references=[], + expected=200, + ), + TestCaseSchema( + schema_type=SchemaType.PROTOBUF, + schema_str=SCHEMA_WITH_REF, + subject="wr_s2_add_new_reference", + references=[{"name": "NoReference.proto", "subject": "wr_s1_add_new_reference", "version": 1}], + expected=200, + ), + TestCaseSchema( + schema_type=SchemaType.PROTOBUF, + schema_str=SCHEMA_NO_REF_TWO, + subject="wr_s3_the_new_reference", + references=[], + expected=200, + ), + TestCaseSchema( + schema_type=SchemaType.PROTOBUF, + schema_str=SCHEMA_ADDS_NEW_REFERENCE, + subject="wr_s2_add_new_reference", + references=[ + {"name": "NoReference.proto", "subject": "wr_s1_add_new_reference", "version": 1}, + {"name": "NoReferenceTwo.proto", "subject": "wr_s3_the_new_reference", "version": 1}, + ], + expected=200, + ), + ], + ), + ReferenceTestCase( + test_name="With reference chain, nonexisting schema", + schemas=[ + TestCaseSchema( + schema_type=SchemaType.PROTOBUF, + schema_str=SCHEMA_NO_REF, + subject="wr_chain_s1", + references=[], + expected=200, + ), + TestCaseSchema( + schema_type=SchemaType.PROTOBUF, + schema_str=SCHEMA_WITH_REF, + subject="wr_chain_s2", + references=[ + {"name": "NoReference.proto", "subject": "wr_chain_s1", "version": 1}, + {"name": "NotFoundReference.proto", "subject": "wr_chain_nonexisting", "version": 1}, + ], + expected=422, + expected_msg=( + f"Invalid schema {SCHEMA_WITH_REF} " + "with refs [{name='NoReference.proto', subject='wr_chain_s1', version=1}, " + "{name='NotFoundReference.proto', subject='wr_chain_nonexisting', version=1}] " + "of type PROTOBUF" + ), + expected_error_code=42201, + ), + ], + ), + ReferenceTestCase( + test_name="With reference chain", + schemas=[ + TestCaseSchema( + schema_type=SchemaType.PROTOBUF, + schema_str=SCHEMA_NO_REF, + subject="wr_chain_s1", + references=[], + expected=200, + ), + TestCaseSchema( + schema_type=SchemaType.PROTOBUF, + schema_str=SCHEMA_WITH_REF, + subject="wr_chain_s2", + references=[{"name": "NoReference.proto", "subject": "wr_chain_s1", "version": 1}], + expected=200, + ), + TestCaseSchema( + schema_type=SchemaType.PROTOBUF, + schema_str=SCHEMA_WITH_2ND_LEVEL_REF, + subject="wr_chain_s3", + references=[{"name": "WithReference.proto", "subject": "wr_chain_s2", "version": 1}], + expected=200, + ), + TestCaseDeleteSchema( + subject="wr_chain_s1", + schema_id=1, + version=1, + expected=422, + ), + TestCaseDeleteSchema( + subject="wr_chain_s2", + schema_id=2, + version=1, + expected=422, + ), + TestCaseDeleteSchema( + subject="wr_chain_s3", + schema_id=3, + version=1, + expected=200, + ), + TestCaseHardDeleteSchema( + subject="wr_chain_s3", + schema_id=3, + version=1, + expected=200, + ), + TestCaseDeleteSchema( + subject="wr_chain_s2", + schema_id=2, + version=1, + expected=200, + ), + TestCaseHardDeleteSchema( + subject="wr_chain_s2", + schema_id=2, + version=1, + expected=200, + ), + TestCaseDeleteSchema( + subject="wr_chain_s1", + schema_id=1, + version=1, + expected=200, + ), + ], + ), + ReferenceTestCase( + test_name="Invalid schema missing closing brace", + schemas=[ + TestCaseSchema( + schema_type=SchemaType.PROTOBUF, + schema_str=SCHEMA_NO_REF, + subject="wr_invalid_reference_ok_schema", + references=[], + expected=200, + ), + TestCaseSchema( + schema_type=SchemaType.PROTOBUF, + schema_str=SCHEMA_INVALID_MISSING_CLOSING_BRACE, + subject="wr_invalid_missing_closing_brace", + references=[{"name": "NoReference.proto", "subject": "wr_invalid_reference_ok_schema", "version": 1}], + expected=422, + expected_msg=( + f"Invalid schema {SCHEMA_INVALID_MISSING_CLOSING_BRACE} " + "with refs [{name='NoReference.proto', subject='wr_invalid_reference_ok_schema', version=1}] " + "of type PROTOBUF" + ), + expected_error_code=42201, + ), + ], + ), + ReferenceTestCase( + test_name="With reference to message from schema file defining two messages", + schemas=[ + TestCaseSchema( + schema_type=SchemaType.PROTOBUF, + schema_str=SCHEMA_NO_REF_TWO_MESSAGES, + subject="wr_s1_two_messages", + references=[], + expected=200, + ), + TestCaseSchema( + schema_type=SchemaType.PROTOBUF, + schema_str=SCHEMA_WITH_REF_TO_NO_REFERENCE_TWO, + subject="wr_s2_referencing_message_two", + references=[{"name": "NoReferenceTwo.proto", "subject": "wr_s1_two_messages", "version": 1}], + expected=200, + ), + ], + ), + ReferenceTestCase( + test_name="With reference to nested message", + schemas=[ + TestCaseSchema( + schema_type=SchemaType.PROTOBUF, + schema_str=SCHEMA_NO_REF_NESTED_MESSAGE, + subject="wr_s1_with_nested_message", + references=[], + expected=200, + ), + TestCaseSchema( + schema_type=SchemaType.PROTOBUF, + schema_str=SCHEMA_WITH_REF_TO_NESTED, + subject="wr_s2_referencing_nested_message", + references=[{"name": "NoReference.proto", "subject": "wr_s1_with_nested_message", "version": 1}], + expected=200, + ), + ], + ), + ], + ids=str, +) +async def test_references(testcase: ReferenceTestCase, registry_async_client: Client): + for testdata in testcase.schemas: + if isinstance(testdata, TestCaseSchema): + print(f"Adding new schema, subject: '{testdata.subject}'\n{testdata.schema_str}") + body = {"schemaType": testdata.schema_type, "schema": testdata.schema_str} + if testdata.references: + body["references"] = testdata.references + res = await registry_async_client.post(f"subjects/{testdata.subject}/versions", json=body) + elif isinstance(testdata, TestCaseHardDeleteSchema): + print( + f"Permanently deleting schema, subject: '{testdata.subject}, " + f"schema: {testdata.schema_id}, version: {testdata.version}' " + ) + res = await registry_async_client.delete( + f"subjects/{testdata.subject}/versions/{testdata.version}?permanent=true" + ) + elif isinstance(testdata, TestCaseDeleteSchema): + print( + f"Deleting schema, subject: '{testdata.subject}, schema: {testdata.schema_id}, version: {testdata.version}' " + ) + res = await registry_async_client.delete(f"subjects/{testdata.subject}/versions/{testdata.version}") + else: + raise Exception("Unknown test case.") + assert res.status_code == testdata.expected + if testdata.expected_msg: + assert res.json_result.get("message", None) == testdata.expected_msg + if testdata.expected_error_code: + assert res.json_result.get("error_code") == testdata.expected_error_code + if isinstance(testdata, TestCaseSchema): + if testdata.expected == 200: + schema_id = res.json().get("id") + fetch_schema_res = await registry_async_client.get(f"/schemas/ids/{schema_id}") + assert fetch_schema_res.status_code == 200 + if isinstance(testdata, TestCaseDeleteSchema): + if testdata.expected == 200: + fetch_res = await registry_async_client.get(f"/subjects/{testdata.subject}/versions/{testdata.version}") + assert fetch_res.status_code == 404 + else: + fetch_schema_res = await registry_async_client.get(f"/schemas/ids/{testdata.schema_id}") + assert fetch_schema_res.status_code == 200 From ecb22b91d2b3ca31fb5c0cd7ada99a469f7badd8 Mon Sep 17 00:00:00 2001 From: Sergiy Zaschipas Date: Tue, 1 Nov 2022 16:53:51 +0200 Subject: [PATCH 14/25] fixup bugs --- karapace/protobuf/compare_type_storage.py | 15 +++- karapace/protobuf/known_dependency.py | 7 +- karapace/protobuf/message_element.py | 4 +- karapace/protobuf/proto_file_element.py | 66 +--------------- karapace/protobuf/type_element.py | 6 ++ tests/integration/test_schema_protobuf.py | 92 ++++++++++++++++++++++- 6 files changed, 119 insertions(+), 71 deletions(-) diff --git a/karapace/protobuf/compare_type_storage.py b/karapace/protobuf/compare_type_storage.py index 3a8df52e5..6ea3e31c6 100644 --- a/karapace/protobuf/compare_type_storage.py +++ b/karapace/protobuf/compare_type_storage.py @@ -90,8 +90,12 @@ def self_type_short_name(self, t: ProtoType) -> Optional[str]: if name is None: raise IllegalArgumentException(f"Cannot determine message type {t}") type_record: TypeRecord = self.self_types.get(name) - if name.startswith(type_record.package_name): - return name[(len(type_record.package_name) + 1) :] + package_name = type_record.package_name + if package_name is None: + package_name = "" + if name.startswith(package_name): + return name[(len(package_name) + 1):] + return name def other_type_short_name(self, t: ProtoType) -> Optional[str]: @@ -99,8 +103,11 @@ def other_type_short_name(self, t: ProtoType) -> Optional[str]: if name is None: raise IllegalArgumentException(f"Cannot determine message type {t}") type_record: TypeRecord = self.other_types.get(name) - if name.startswith(type_record.package_name): - return name[(len(type_record.package_name) + 1) :] + package_name = type_record.package_name + if package_name is None: + package_name = "" + if name.startswith(package_name): + return name[(len(package_name) + 1):] return name def lock_message(self, message: "MessageElement") -> bool: diff --git a/karapace/protobuf/known_dependency.py b/karapace/protobuf/known_dependency.py index 0ba9559f7..05f3f96e6 100644 --- a/karapace/protobuf/known_dependency.py +++ b/karapace/protobuf/known_dependency.py @@ -116,8 +116,9 @@ class KnownDependency: "google/type/postal_address.proto": ["google.type.PostalAddress"], "google/type/quaternion.proto": ["google.type.Quaternion"], "google/type/timeofday.proto": ["google.type.TimeOfDay"], - "confluent/meta.proto": [".confluent.Meta"], - "confluent/type/decimal.proto": [".confluent.type.Decimal"], + "confluent/meta.proto": ["confluent.Meta"], + "confluent/type/decimal.proto": ["confluent.type.Decimal"], + } @classmethod @@ -125,8 +126,10 @@ def static_init(cls) -> None: for key, value in cls.map.items(): for item in value: cls.index[item] = key + cls.index["." + item] = key dot = item.rfind(".") cls.index_simple[item[dot + 1 :]] = key + cls.index_simple[item] = key @static_init diff --git a/karapace/protobuf/message_element.py b/karapace/protobuf/message_element.py index d0acecd5c..48921b7dc 100644 --- a/karapace/protobuf/message_element.py +++ b/karapace/protobuf/message_element.py @@ -80,7 +80,7 @@ def to_schema(self) -> str: return "".join(result) def compare(self, other: "MessageElement", result: CompareResult, types: CompareTypes) -> None: - + from karapace.protobuf.compare_type_lists import compare_type_lists if types.lock_message(self): field: FieldElement subfield: FieldElement @@ -138,5 +138,5 @@ def compare(self, other: "MessageElement", result: CompareResult, types: Compare self_one_ofs[name].compare(other_one_ofs[name], result, types) result.pop_path() - + compare_type_lists(self.nested_types, other.nested_types, result, types) types.unlock_message(self) diff --git a/karapace/protobuf/proto_file_element.py b/karapace/protobuf/proto_file_element.py index f227d63c0..4b29309f6 100644 --- a/karapace/protobuf/proto_file_element.py +++ b/karapace/protobuf/proto_file_element.py @@ -1,12 +1,8 @@ # Ported from square/wire: # wire-library/wire-schema/src/commonMain/kotlin/com/squareup/wire/schema/internal/parser/ProtoFileElement.kt -from itertools import chain from karapace.protobuf.compare_result import CompareResult, Modification from karapace.protobuf.compare_type_storage import CompareTypes -from karapace.protobuf.enum_element import EnumElement -from karapace.protobuf.exception import IllegalStateException from karapace.protobuf.location import Location -from karapace.protobuf.message_element import MessageElement from karapace.protobuf.syntax import Syntax from karapace.protobuf.type_element import TypeElement from typing import List, Optional @@ -106,79 +102,25 @@ def compare( self_dependency_types: Optional[List[TypeElement]] = None, other_dependency_types: Optional[List[TypeElement]] = None, ) -> CompareResult: + from karapace.protobuf.compare_type_lists import compare_type_lists if self.package_name != other.package_name: result.add_modification(Modification.PACKAGE_ALTER) # TODO: do we need syntax check? if self.syntax != other.syntax: result.add_modification(Modification.SYNTAX_ALTER) - self_types = {} - other_types = {} - self_indexes = {} - other_indexes = {} compare_types = CompareTypes(self.package_name, other.package_name, result) type_: TypeElement - for i, type_ in enumerate(self.types): - self_types[type_.name] = type_ - self_indexes[type_.name] = i - package_name = self.package_name or "" - compare_types.add_self_type(package_name, type_) - - for i, type_ in enumerate(other.types): - other_types[type_.name] = type_ - other_indexes[type_.name] = i - package_name = other.package_name or "" - compare_types.add_other_type(package_name, type_) # If there are dependencies declared, add the types for both. if self_dependency_types: - for i, type_ in enumerate(self_dependency_types): + for type_ in self_dependency_types: package_name = "" - - self_types[type_.name] = type_ - self_indexes[type_.name] = i compare_types.add_self_type(package_name, type_) if other_dependency_types: - for i, type_ in enumerate(other_dependency_types): + for type_ in other_dependency_types: package_name = "" - other_types[type_.name] = type_ - other_indexes[type_.name] = i compare_types.add_other_type(package_name, type_) - for name in chain(self_types.keys(), other_types.keys() - self_types.keys()): - - result.push_path(str(name), True) - - if self_types.get(name) is None and other_types.get(name) is not None: - if isinstance(other_types[name], MessageElement): - result.add_modification(Modification.MESSAGE_ADD) - elif isinstance(other_types[name], EnumElement): - result.add_modification(Modification.ENUM_ADD) - else: - raise IllegalStateException("Instance of element is not applicable") - elif self_types.get(name) is not None and other_types.get(name) is None: - if isinstance(self_types[name], MessageElement): - result.add_modification(Modification.MESSAGE_DROP) - elif isinstance(self_types[name], EnumElement): - result.add_modification(Modification.ENUM_DROP) - else: - raise IllegalStateException("Instance of element is not applicable") - else: - if other_indexes[name] != self_indexes[name]: - if isinstance(self_types[name], MessageElement): - # incompatible type - result.add_modification(Modification.MESSAGE_MOVE) - else: - raise IllegalStateException("Instance of element is not applicable") - else: - if isinstance(self_types[name], MessageElement) and isinstance(other_types[name], MessageElement): - self_types[name].compare(other_types[name], result, compare_types) - elif isinstance(self_types[name], EnumElement) and isinstance(other_types[name], EnumElement): - self_types[name].compare(other_types[name], result, compare_types) - else: - # incompatible type - result.add_modification(Modification.TYPE_ALTER) - result.pop_path(True) - - return result + return compare_type_lists(self.types, other.types, result, compare_types) diff --git a/karapace/protobuf/type_element.py b/karapace/protobuf/type_element.py index 568c6e4d5..6a0436896 100644 --- a/karapace/protobuf/type_element.py +++ b/karapace/protobuf/type_element.py @@ -1,11 +1,14 @@ # Ported from square/wire: # wire-library/wire-schema/src/commonMain/kotlin/com/squareup/wire/schema/internal/parser/TypeElement.kt from dataclasses import dataclass + from karapace.protobuf.location import Location from typing import List, TYPE_CHECKING if TYPE_CHECKING: from karapace.protobuf.option_element import OptionElement + from karapace.protobuf.compare_type_storage import CompareTypes + from karapace.protobuf.compare_result import CompareResult @dataclass @@ -30,3 +33,6 @@ def __repr__(self) -> str: def __str__(self) -> str: mytype = type(self) return f"{mytype}({self.to_schema()})" + + def compare(self, other: "TypeElement", result: "CompareResult", types: "CompareTypes") -> None: + pass diff --git a/tests/integration/test_schema_protobuf.py b/tests/integration/test_schema_protobuf.py index 8ab438c80..63f59cac5 100644 --- a/tests/integration/test_schema_protobuf.py +++ b/tests/integration/test_schema_protobuf.py @@ -267,7 +267,6 @@ async def test_protobuf_schema_references(registry_async_client: Client) -> None res = await registry_async_client.get("subjects/place/versions/latest/referencedby", json={}) assert res.status_code == 200 - myjson = res.json() res = await registry_async_client.delete("subjects/customer/versions/1") assert res.status_code == 422 @@ -277,6 +276,19 @@ async def test_protobuf_schema_references(registry_async_client: Client) -> None res = await registry_async_client.delete("subjects/test_schema/versions/1") assert res.status_code == 200 + res = await registry_async_client.delete("subjects/test_schema/versions/1") + myjson = res.json() + match_msg = "Subject 'test_schema' Version 1 was soft deleted.Set permanent=true to delete permanently" + + assert res.status_code == 422 + + assert myjson["error_code"] == 42206 and myjson["message"] == match_msg + res = await registry_async_client.delete("subjects/customer/versions/1") + myjson = res.json() + match_msg = "One or more references exist to the schema {magic=1,keytype=SCHEMA,subject=customer,version=1}." + + assert res.status_code == 422 + assert myjson["error_code"] == 42206 and myjson["message"] == match_msg res = await registry_async_client.delete("subjects/test_schema/versions/1?permanent=true") assert res.status_code == 200 @@ -285,6 +297,81 @@ async def test_protobuf_schema_references(registry_async_client: Client) -> None assert res.status_code == 200 +async def test_protobuf_schema_jjaakola_one(registry_async_client: Client) -> None: + no_ref = """ + |syntax = "proto3"; + | + |message NoReference { + | string name = 1; + |} + |""" + + no_ref = trim_margin(no_ref) + res = await registry_async_client.post("subjects/sub1/versions", json={"schemaType": "PROTOBUF", "schema": no_ref}) + assert res.status_code == 200 + assert "id" in res.json() + + with_first_ref = """ + |syntax = "proto3"; + | + |import "NoReference.proto"; + | + |message WithReference { + | string name = 1; + | NoReference ref = 2; + |}""" + + with_first_ref = trim_margin(with_first_ref) + references = [{"name": "NoReference.proto", "subject": "sub1", "version": 1}] + + res = await registry_async_client.post( + "subjects/sub2/versions", + json={"schemaType": "PROTOBUF", "schema": with_first_ref, "references": references}, + ) + assert res.status_code == 200 + assert "id" in res.json() + + no_ref_second = """ + |syntax = "proto3"; + | + |message NoReferenceTwo { + | string name = 1; + |} + |""" + + no_ref_second = trim_margin(no_ref_second) + res = await registry_async_client.post( + "subjects/sub3/versions", json={"schemaType": "PROTOBUF", "schema": no_ref_second} + ) + assert res.status_code == 200 + assert "id" in res.json() + + add_new_ref_in_sub2 = """ + |syntax = "proto3"; + |import "NoReference.proto"; + |import "NoReferenceTwo.proto"; + |message WithReference { + | string name = 1; + | NoReference ref = 2; + | NoReferenceTwo refTwo = 3; + |} + |""" + + add_new_ref_in_sub2 = trim_margin(add_new_ref_in_sub2) + + references = [ + {"name": "NoReference.proto", "subject": "sub1", "version": 1}, + {"name": "NoReferenceTwo.proto", "subject": "sub3", "version": 1}, + ] + + res = await registry_async_client.post( + "subjects/sub2/versions", + json={"schemaType": "PROTOBUF", "schema": add_new_ref_in_sub2, "references": references}, + ) + assert res.status_code == 200 + assert "id" in res.json() + + async def test_protobuf_schema_verifier(registry_async_client: Client) -> None: customer_schema = """ |syntax = "proto3"; @@ -354,6 +441,9 @@ async def test_protobuf_schema_verifier(registry_async_client: Client) -> None: res = await registry_async_client.delete("subjects/test_schema/versions/1") assert res.status_code == 200 + res = await registry_async_client.delete("subjects/customer/versions/1") + assert res.status_code == 404 + res = await registry_async_client.delete("subjects/test_schema/versions/1?permanent=true") assert res.status_code == 200 From b2a796d2d75b1e58f24f27c1e521b1769739babc Mon Sep 17 00:00:00 2001 From: Sergiy Zaschipas Date: Thu, 3 Nov 2022 13:49:22 +0200 Subject: [PATCH 15/25] fixup --- karapace/protobuf/compare_type_storage.py | 4 ++-- karapace/protobuf/known_dependency.py | 1 - karapace/protobuf/message_element.py | 1 + karapace/protobuf/proto_file_element.py | 1 + karapace/protobuf/type_element.py | 5 ++--- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/karapace/protobuf/compare_type_storage.py b/karapace/protobuf/compare_type_storage.py index 6ea3e31c6..1205623e2 100644 --- a/karapace/protobuf/compare_type_storage.py +++ b/karapace/protobuf/compare_type_storage.py @@ -94,7 +94,7 @@ def self_type_short_name(self, t: ProtoType) -> Optional[str]: if package_name is None: package_name = "" if name.startswith(package_name): - return name[(len(package_name) + 1):] + return name[(len(package_name) + 1) :] return name @@ -107,7 +107,7 @@ def other_type_short_name(self, t: ProtoType) -> Optional[str]: if package_name is None: package_name = "" if name.startswith(package_name): - return name[(len(package_name) + 1):] + return name[(len(package_name) + 1) :] return name def lock_message(self, message: "MessageElement") -> bool: diff --git a/karapace/protobuf/known_dependency.py b/karapace/protobuf/known_dependency.py index 05f3f96e6..e6b2366f7 100644 --- a/karapace/protobuf/known_dependency.py +++ b/karapace/protobuf/known_dependency.py @@ -118,7 +118,6 @@ class KnownDependency: "google/type/timeofday.proto": ["google.type.TimeOfDay"], "confluent/meta.proto": ["confluent.Meta"], "confluent/type/decimal.proto": ["confluent.type.Decimal"], - } @classmethod diff --git a/karapace/protobuf/message_element.py b/karapace/protobuf/message_element.py index 48921b7dc..fb8886245 100644 --- a/karapace/protobuf/message_element.py +++ b/karapace/protobuf/message_element.py @@ -81,6 +81,7 @@ def to_schema(self) -> str: def compare(self, other: "MessageElement", result: CompareResult, types: CompareTypes) -> None: from karapace.protobuf.compare_type_lists import compare_type_lists + if types.lock_message(self): field: FieldElement subfield: FieldElement diff --git a/karapace/protobuf/proto_file_element.py b/karapace/protobuf/proto_file_element.py index 4b29309f6..25cd4b4e3 100644 --- a/karapace/protobuf/proto_file_element.py +++ b/karapace/protobuf/proto_file_element.py @@ -103,6 +103,7 @@ def compare( other_dependency_types: Optional[List[TypeElement]] = None, ) -> CompareResult: from karapace.protobuf.compare_type_lists import compare_type_lists + if self.package_name != other.package_name: result.add_modification(Modification.PACKAGE_ALTER) # TODO: do we need syntax check? diff --git a/karapace/protobuf/type_element.py b/karapace/protobuf/type_element.py index 6a0436896..ddbf05356 100644 --- a/karapace/protobuf/type_element.py +++ b/karapace/protobuf/type_element.py @@ -1,14 +1,13 @@ # Ported from square/wire: # wire-library/wire-schema/src/commonMain/kotlin/com/squareup/wire/schema/internal/parser/TypeElement.kt from dataclasses import dataclass - from karapace.protobuf.location import Location from typing import List, TYPE_CHECKING if TYPE_CHECKING: - from karapace.protobuf.option_element import OptionElement - from karapace.protobuf.compare_type_storage import CompareTypes from karapace.protobuf.compare_result import CompareResult + from karapace.protobuf.compare_type_storage import CompareTypes + from karapace.protobuf.option_element import OptionElement @dataclass From d82c25b43066fd8629214b5e3cc74c54842f0b97 Mon Sep 17 00:00:00 2001 From: Sergiy Zaschipas Date: Thu, 10 Nov 2022 00:07:32 +0200 Subject: [PATCH 16/25] apply compatibility check with depependencies on top of jjaakola changes to our branch --- karapace/dependency.py | 3 + karapace/protobuf/compare_type_storage.py | 5 +- karapace/protobuf/known_dependency.py | 1 + karapace/protobuf/message_element.py | 2 +- karapace/protobuf/proto_file_element.py | 39 +++++---- karapace/protobuf/schema.py | 32 +++----- karapace/schema_models.py | 32 +++++--- karapace/schema_reader.py | 43 ++++++---- karapace/schema_registry.py | 24 ++++-- karapace/schema_registry_apis.py | 32 +++++--- karapace/serialization.py | 3 +- tests/integration/test_schema_protobuf.py | 99 +++++++++++++++++------ 12 files changed, 211 insertions(+), 104 deletions(-) diff --git a/karapace/dependency.py b/karapace/dependency.py index 3cc2a6ff0..ec4182a6c 100644 --- a/karapace/dependency.py +++ b/karapace/dependency.py @@ -19,6 +19,9 @@ def __init__(self, name: str, subject: Subject, version: Version, target_schema: self.version = version self.schema = target_schema + def get_schema(self) -> "ValidatedTypedSchema": + return self.schema + @staticmethod def of(reference: Reference, target_schema: "ValidatedTypedSchema") -> "Dependency": return Dependency(reference.name, reference.subject, reference.version, target_schema) diff --git a/karapace/protobuf/compare_type_storage.py b/karapace/protobuf/compare_type_storage.py index 1205623e2..8615c5da0 100644 --- a/karapace/protobuf/compare_type_storage.py +++ b/karapace/protobuf/compare_type_storage.py @@ -35,8 +35,9 @@ def compute_name(t: ProtoType, result_path: List[str], package_name: str, types: class CompareTypes: def __init__(self, self_package_name: str, other_package_name: str, result: CompareResult) -> None: - self.self_package_name = self_package_name - self.other_package_name = other_package_name + self.self_package_name = self_package_name or "" + self.other_package_name = other_package_name or "" + self.self_types: Dict[str, Union[TypeRecord, TypeRecordMap]] = {} self.other_types: Dict[str, Union[TypeRecord, TypeRecordMap]] = {} self.locked_messages: List["MessageElement"] = [] diff --git a/karapace/protobuf/known_dependency.py b/karapace/protobuf/known_dependency.py index f7c327e5a..e6b2366f7 100644 --- a/karapace/protobuf/known_dependency.py +++ b/karapace/protobuf/known_dependency.py @@ -130,6 +130,7 @@ def static_init(cls) -> None: cls.index_simple[item[dot + 1 :]] = key cls.index_simple[item] = key + @static_init class DependenciesHardcoded: index: Set = set() diff --git a/karapace/protobuf/message_element.py b/karapace/protobuf/message_element.py index fb8886245..054166816 100644 --- a/karapace/protobuf/message_element.py +++ b/karapace/protobuf/message_element.py @@ -22,7 +22,7 @@ def __init__( location: Location, name: str, documentation: str = "", - nested_types: List[str] = None, + nested_types: List[TypeElement] = None, options: List[OptionElement] = None, reserveds: List[ReservedElement] = None, fields: List[FieldElement] = None, diff --git a/karapace/protobuf/proto_file_element.py b/karapace/protobuf/proto_file_element.py index 6219af042..994bf9722 100644 --- a/karapace/protobuf/proto_file_element.py +++ b/karapace/protobuf/proto_file_element.py @@ -1,11 +1,28 @@ # Ported from square/wire: # wire-library/wire-schema/src/commonMain/kotlin/com/squareup/wire/schema/internal/parser/ProtoFileElement.kt +from karapace.dependency import Dependency from karapace.protobuf.compare_result import CompareResult, Modification from karapace.protobuf.compare_type_storage import CompareTypes from karapace.protobuf.location import Location from karapace.protobuf.syntax import Syntax from karapace.protobuf.type_element import TypeElement -from typing import List, Optional +from typing import Dict, List, Optional + + +def collect_dependencies_types(compare_types: CompareTypes, dependencies: Optional[Dict[str, Dependency]], is_self: bool): + for dep in dependencies.values(): + types: List[TypeElement] = dep.schema.schema.proto_file_element.types + sub_deps = dep.schema.schema.dependencies + package_name = dep.schema.schema.proto_file_element.package_name + type_: TypeElement + for type_ in types: + if is_self: + compare_types.add_self_type(package_name, type_) + else: + compare_types.add_other_type(package_name, type_) + if sub_deps is None: + return + collect_dependencies_types(compare_types, sub_deps, is_self) class ProtoFileElement: @@ -99,8 +116,8 @@ def compare( self, other: "ProtoFileElement", result: CompareResult, - self_dependency_types: Optional[List[TypeElement]] = None, - other_dependency_types: Optional[List[TypeElement]] = None, + self_dependencies: Optional[Dict[str, Dependency]] = None, + other_dependencies: Optional[Dict[str, Dependency]] = None, ) -> CompareResult: from karapace.protobuf.compare_type_lists import compare_type_lists @@ -112,17 +129,9 @@ def compare( result.add_modification(Modification.SYNTAX_ALTER) compare_types = CompareTypes(self.package_name, other.package_name, result) - type_: TypeElement - - # If there are dependencies declared, add the types for both. - if self_dependency_types: - for type_ in self_dependency_types: - package_name = "" - compare_types.add_self_type(package_name, type_) - - if other_dependency_types: - for type_ in other_dependency_types: - package_name = "" - compare_types.add_other_type(package_name, type_) + if self_dependencies: + collect_dependencies_types(compare_types, self_dependencies, True) + if other_dependencies: + collect_dependencies_types(compare_types, other_dependencies, False) return compare_type_lists(self.types, other.types, result, compare_types) diff --git a/karapace/protobuf/schema.py b/karapace/protobuf/schema.py index e85b3df35..3af2d2adb 100644 --- a/karapace/protobuf/schema.py +++ b/karapace/protobuf/schema.py @@ -1,5 +1,4 @@ # Ported from square/wire: -# Ported from square/wire: # wire-library/wire-schema/src/commonMain/kotlin/com/squareup/wire/schema/Schema.kt # Ported partially for required functionality. from karapace.dependency import Dependency, DependencyVerifierResult @@ -14,7 +13,8 @@ from karapace.protobuf.proto_parser import ProtoParser from karapace.protobuf.type_element import TypeElement from karapace.protobuf.utils import append_documentation, append_indented -from typing import List, Optional +from karapace.schema_references import Reference +from typing import Dict, List, Optional def add_slashes(text: str) -> str: @@ -105,12 +105,15 @@ def option_element_string(option: OptionElement) -> str: class ProtobufSchema: DEFAULT_LOCATION = Location.get("") - def __init__(self, schema: str, dependencies: Optional[List[Dependency]] = None) -> None: + def __init__( + self, schema: str, references: Optional[List[Reference]] = None, dependencies: Optional[Dict[str, Dependency]] = None + ) -> None: if type(schema).__name__ != "str": raise IllegalArgumentException("Non str type of schema string") self.dirty = schema self.cache_string = "" self.proto_file_element = ProtoParser.parse(self.DEFAULT_LOCATION, schema) + self.references = references self.dependencies = dependencies def gather_deps(self) -> ProtobufDependencyVerifier: @@ -125,8 +128,9 @@ def verify_schema_dependencies(self) -> DependencyVerifierResult: def collect_dependencies(self, verifier: ProtobufDependencyVerifier): if self.dependencies: - for dependency in self.dependencies: - dependency.schema.schema.collect_dependencies(verifier) + for key in self.dependencies: + self.dependencies[key].schema.schema.collect_dependencies(verifier) + # verifier.add_import?? we have no access to own Kafka structure from this class... # but we need data to analyse imports to avoid ciclyc dependencies... @@ -218,21 +222,9 @@ def to_schema(self) -> str: return "".join(strings) def compare(self, other: "ProtobufSchema", result: CompareResult) -> CompareResult: - self_dependency_types: List[TypeElement] = [] - other_dependency_types: List[TypeElement] = [] - if self.dependencies: - for dependency in self.dependencies: - schema = dependency.schema.schema - if schema.proto_file_element.types: - self_dependency_types += schema.proto_file_element.types - if other.dependencies: - for dependency in other.dependencies: - schema = dependency.schema.schema - if schema.proto_file_element.types: - other_dependency_types += schema.proto_file_element.types - self.proto_file_element.compare( + return self.proto_file_element.compare( other.proto_file_element, result, - self_dependency_types=self_dependency_types, - other_dependency_types=other_dependency_types, + self_dependencies=self.dependencies, + other_dependencies=other.dependencies, ) diff --git a/karapace/schema_models.py b/karapace/schema_models.py index 8ad8b43fc..485d184da 100644 --- a/karapace/schema_models.py +++ b/karapace/schema_models.py @@ -14,6 +14,7 @@ SchemaParseException as ProtobufSchemaParseException, ) from karapace.protobuf.schema import ProtobufSchema +from karapace.schema_references import Reference from karapace.schema_type import SchemaType from karapace.utils import json_encode from typing import Any, Dict, List, Optional, Union @@ -53,7 +54,8 @@ def parse_jsonschema_definition(schema_definition: str) -> Draft7Validator: def parse_protobuf_schema_definition( schema_definition: str, - references: Optional[List[Dependency]] = None, + references: Optional[List[Reference]] = None, + dependencies: Optional[Dict[str, Dependency]] = None, validate_references: bool = True, ) -> ProtobufSchema: """Parses and validates `schema_definition`. @@ -62,7 +64,7 @@ def parse_protobuf_schema_definition( Nothing yet. """ - protobuf_schema = ProtobufSchema(schema_definition, references) + protobuf_schema = ProtobufSchema(schema_definition, references, dependencies) if validate_references: result = protobuf_schema.verify_schema_dependencies() if not result.result: @@ -77,7 +79,8 @@ def __init__( schema_type: SchemaType, schema_str: str, schema: Optional[Union[Draft7Validator, AvroSchema, ProtobufSchema]] = None, - references: Optional[List[Dependency]] = None, + references: Optional[List[Reference]] = None, + dependencies: Optional[Dict[str, Dependency]] = None, ): """Schema with type information @@ -90,6 +93,7 @@ def __init__( self.schema_type = schema_type self.schema_str = schema_str self.references = references + self.dependencies = dependencies self.max_id: Optional[int] = None self._str_cached: Optional[str] = None @@ -130,7 +134,7 @@ def schema(self) -> Union[Draft7Validator, AvroSchema, ProtobufSchema]: elif self.schema_type is SchemaType.PROTOBUF: try: - self._schema_cached = parse_protobuf_schema_definition(self.schema_str, self.references) + self._schema_cached = parse_protobuf_schema_definition(self.schema_str, self.references, self.dependencies) except ( TypeError, SchemaError, @@ -147,12 +151,14 @@ def schema(self) -> Union[Draft7Validator, AvroSchema, ProtobufSchema]: raise InvalidSchema(f"Unknown parser {self.schema_type} for {self.schema_str}") return self._schema_cached - def get_references(self) -> Optional[List[Dependency]]: + def get_references(self) -> Optional[List[Reference]]: return self.references def __eq__(self, other: Any) -> bool: schema_is_equal = ( - isinstance(other, TypedSchema) and self.schema_type is other.schema_type and self.__str__() == other.__str__() + isinstance(other, (TypedSchema, ValidatedTypedSchema)) + and self.schema_type is other.schema_type + and self.__str__() == other.__str__() ) if not schema_is_equal: return False @@ -165,15 +171,20 @@ def __init__( schema_type: SchemaType, schema_str: str, schema: Union[Draft7Validator, AvroSchema, ProtobufSchema], - references: Optional[List[Dependency]] = None, + references: Optional[List[Reference]] = None, + dependencies: Optional[Dict[str, Dependency]] = None, ): - super().__init__(schema_type=schema_type, schema_str=schema_str, references=references, schema=schema) + + super().__init__( + schema_type=schema_type, schema_str=schema_str, references=references, dependencies=dependencies, schema=schema + ) @staticmethod def parse( schema_type: SchemaType, schema_str: str, - references: Optional[List[Dependency]] = None, + references: Optional[List[Reference]] = None, + dependencies: Optional[Dict[str, Dependency]] = None, ) -> "ValidatedTypedSchema": if schema_type not in [SchemaType.AVRO, SchemaType.JSONSCHEMA, SchemaType.PROTOBUF]: raise InvalidSchema(f"Unknown parser {schema_type} for {schema_str}") @@ -194,7 +205,7 @@ def parse( elif schema_type is SchemaType.PROTOBUF: try: - parsed_schema = parse_protobuf_schema_definition(schema_str, references) + parsed_schema = parse_protobuf_schema_definition(schema_str, references, dependencies) except ( TypeError, SchemaError, @@ -215,6 +226,7 @@ def parse( schema_str=schema_str, schema=parsed_schema, references=references, + dependencies=dependencies, ) def __str__(self) -> str: diff --git a/karapace/schema_reader.py b/karapace/schema_reader.py index 21bfdf500..121c13f7e 100644 --- a/karapace/schema_reader.py +++ b/karapace/schema_reader.py @@ -4,7 +4,9 @@ Copyright (c) 2019 Aiven Ltd See LICENSE for details """ +from avro.schema import Schema as AvroSchema from contextlib import closing, ExitStack +from jsonschema.validators import Draft7Validator from kafka import KafkaConsumer from kafka.admin import KafkaAdminClient, NewTopic from kafka.errors import KafkaConfigurationError, NoBrokersAvailable, NodeNotReadyError, TopicAlreadyExistsError @@ -14,13 +16,14 @@ from karapace.errors import InvalidReferences, InvalidSchema from karapace.key_format import is_key_in_canonical_format, KeyFormatter, KeyMode from karapace.master_coordinator import MasterCoordinator +from karapace.protobuf.schema import ProtobufSchema from karapace.schema_models import parse_protobuf_schema_definition, SchemaType, TypedSchema from karapace.schema_references import Reference from karapace.statsd import StatsClient from karapace.typing import JsonData, SubjectData from karapace.utils import KarapaceKafkaClient, reference_key from threading import Condition, Event, Lock, Thread -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Union import hashlib import json @@ -433,7 +436,8 @@ def _handle_msg_schema(self, key: dict, value: Optional[dict]) -> None: schema_version = value["version"] schema_deleted = value.get("deleted", False) schema_references = value.get("references", None) - resolved_references: Optional[List[Dependency]] = None + resolved_references: Optional[List[Reference]] = None + resolved_dependencies: Optional[Dict[str, Dependency]] = None try: schema_type_parsed = SchemaType(schema_type) @@ -447,6 +451,8 @@ def _handle_msg_schema(self, key: dict, value: Optional[dict]) -> None: # won't interfere with the equality. Note: This means it is possible # for the REST API to return data that is formatted differently from # what is available in the topic. + parsed_schema: Optional[Union[Draft7Validator, AvroSchema, ProtobufSchema]] = None + resolved_dependencies: Dict[str, Dependency] = None if schema_type_parsed in [SchemaType.AVRO, SchemaType.JSONSCHEMA]: try: schema_str = json.dumps(json.loads(schema_str), sort_keys=True) @@ -456,12 +462,17 @@ def _handle_msg_schema(self, key: dict, value: Optional[dict]) -> None: elif schema_type_parsed == SchemaType.PROTOBUF: try: if schema_references: - references = [ + resolved_references = [ Reference(reference["name"], reference["subject"], reference["version"]) for reference in schema_references ] - resolved_references = self.resolve_references(references) - parsed_schema = parse_protobuf_schema_definition(schema_str, resolved_references, validate_references=False) + resolved_dependencies = self.resolve_references(resolved_references) + parsed_schema = parse_protobuf_schema_definition( + schema_str, + resolved_references, + resolved_dependencies, + validate_references=False, + ) schema_str = str(parsed_schema) except InvalidSchema: LOG.exception("Schema is not valid ProtoBuf definition") @@ -481,6 +492,8 @@ def _handle_msg_schema(self, key: dict, value: Optional[dict]) -> None: schema_type=schema_type_parsed, schema_str=schema_str, references=resolved_references, + dependencies=resolved_dependencies, + schema=parsed_schema, ) schema = { "schema": typed_schema, @@ -565,24 +578,24 @@ def remove_referenced_by(self, schema_id: SchemaId, references: List[Reference]) if self.referenced_by.get(key, None) and schema_id in self.referenced_by[key]: self.referenced_by[key].remove(schema_id) - def _resolve_reference(self, reference: Reference) -> List[Dependency]: + def _resolve_reference(self, reference: Reference) -> Dependency: subject_data = self.subjects.get(reference.subject) if not subject_data: raise InvalidReferences(f"Subject not found {reference.subject}.") - schema = subject_data["schemas"].get(reference.version, {}).get("schema", None) + schema: TypedSchema = subject_data["schemas"].get(reference.version, {}).get("schema", None) if not schema: raise InvalidReferences(f"No schema in {reference.subject} with version {reference.version}.") - - resolved_references = [Dependency.of(reference, schema)] if schema.references: - resolved_references += self.resolve_references(schema.references) - return resolved_references + schema_dependencies = self.resolve_references(schema.references) + if schema.dependencies is None: + schema.dependencies = schema_dependencies + return Dependency.of(reference, schema) - def resolve_references(self, references: List[Reference]) -> List[Dependency]: - resolved_references = [] + def resolve_references(self, references: List[Reference]) -> Dict[str, Dependency]: + dependencies: Dict[str, Dependency] = dict() for reference in references: - resolved_references += self._resolve_reference(reference) - return resolved_references + dependencies[reference.name] = self._resolve_reference(reference) + return dependencies def _build_state_dict(self) -> JsonData: state = {"schemas": [], "subjects": {}} diff --git a/karapace/schema_registry.py b/karapace/schema_registry.py index 1b618a481..a5fab855d 100644 --- a/karapace/schema_registry.py +++ b/karapace/schema_registry.py @@ -294,7 +294,8 @@ async def subject_version_get(self, subject: Subject, version: Version, *, inclu "id": schema_id, "schema": schema.schema_str, } - + if schema.references is not None: + ret["references"] = schema.references if schema.schema_type is not SchemaType.AVRO: ret["schemaType"] = schema.schema_type # Return also compatibility information to compatibility check @@ -306,7 +307,7 @@ async def write_new_schema_local( self, subject: Subject, new_schema: ValidatedTypedSchema, - new_schema_references: Optional[List[Dependency]], + new_schema_references: Optional[List[Reference]], ) -> int: """Write new schema and return new id or return id of matching existing schema @@ -368,8 +369,21 @@ async def write_new_schema_local( for old_version in check_against: old_schema = subject_data["schemas"][old_version]["schema"] + old_schema_references: Optional[List[Reference]] = subject_data["schemas"][old_version][ + "schema" + ].references + old_schema_dependencies: Optional[Dict[str, Dependency]] = None + + if old_schema_references: + old_schema_dependencies = self.resolve_references(old_schema_references) + validated_old_schema = ValidatedTypedSchema.parse( + schema_type=old_schema.schema_type, + schema_str=old_schema.schema_str, + references=old_schema_references, + dependencies=old_schema_dependencies, + ) result = check_compatibility( - old_schema=old_schema, + old_schema=validated_old_schema, new_schema=new_schema, compatibility_mode=compatibility_mode, ) @@ -470,7 +484,7 @@ def send_schema_message( schema_id: int, version: int, deleted: bool, - references: Optional[List[Dependency]], + references: Optional[List[Reference]], ) -> FutureRecordMetadata: key = self.key_formatter.format_key( {"subject": subject, "version": version, "magic": 1, "keytype": "SCHEMA"}, @@ -526,7 +540,7 @@ def send_delete_subject_message(self, subject: Subject, version: Version) -> Fut value = '{{"subject":"{}","version":{}}}'.format(subject, version) return self.send_kafka_message(key, value) - def resolve_references(self, references: Optional[List[Reference]]) -> Optional[List[Dependency]]: + def resolve_references(self, references: Optional[List[Reference]]) -> Optional[Dict[str, Dependency]]: if references: return self.schema_reader.resolve_references(references) return None diff --git a/karapace/schema_registry_apis.py b/karapace/schema_registry_apis.py index 216ed02e2..741c7d07d 100644 --- a/karapace/schema_registry_apis.py +++ b/karapace/schema_registry_apis.py @@ -72,6 +72,13 @@ class SchemaErrorMessages(Enum): REFERENCES_SUPPORT_NOT_IMPLEMENTED = "Schema references are not supported for '{schema_type}' schema type" +def references_list(references: Optional[Dict]) -> Optional[List[Reference]]: + _references: Optional[List[Reference]] = None + if references: + _references = [Reference(reference["name"], reference["subject"], reference["version"]) for reference in references] + return _references + + class KarapaceSchemaRegistryController(KarapaceBase): def __init__(self, config: Config) -> None: super().__init__(config=config) @@ -310,11 +317,12 @@ async def compatibility_check( schema_type = self._validate_schema_type(content_type=content_type, data=body) references = self._validate_references(content_type, schema_type, body) try: - new_schema_references = self.schema_registry.resolve_references(references) + new_schema_dependencies = self.schema_registry.resolve_references(references) new_schema = ValidatedTypedSchema.parse( schema_type=schema_type, schema_str=body["schema"], - references=new_schema_references, + references=references, + dependencies=new_schema_dependencies, ) except InvalidSchema: self.r( @@ -907,7 +915,7 @@ def _validate_schema_key(self, content_type: str, body: dict) -> None: status=HTTPStatus.UNPROCESSABLE_ENTITY, ) - def _validate_references(self, content_type: str, schema_type: SchemaType, body: JsonData) -> List[Reference]: + def _validate_references(self, content_type: str, schema_type: SchemaType, body: JsonData) -> Optional[List[Reference]]: references = body.get("references", []) if references and schema_type != SchemaType.PROTOBUF: self.r( @@ -928,7 +936,9 @@ def _validate_references(self, content_type: str, schema_type: SchemaType, body: validated_references.append( Reference(name=reference["name"], subject=reference["subject"], version=reference["version"]) ) - return validated_references + if validated_references: + return validated_references + return None async def subjects_schema_post( self, content_type: str, *, subject: str, request: HTTPRequest, user: Optional[User] = None @@ -964,11 +974,12 @@ async def subjects_schema_post( references = self._validate_references(content_type, schema_type, body) try: - new_schema_references = self.schema_registry.resolve_references(references) + new_schema_dependencies = self.schema_registry.resolve_references(references) new_schema = ValidatedTypedSchema.parse( schema_type=schema_type, schema_str=schema_str, - references=new_schema_references, + references=references, + dependencies=new_schema_dependencies, ) for schema in subject_data["schemas"].values(): validated_typed_schema = ValidatedTypedSchema.parse( @@ -981,7 +992,7 @@ async def subjects_schema_post( if ( validated_typed_schema.schema_type == new_schema.schema_type and schema_valid - and schema.get("references", None) == new_schema_references + and references_list(schema.get("references", None)) == references ): ret = { "subject": subject, @@ -1058,11 +1069,12 @@ async def subject_post( references = self._validate_references(content_type, schema_type, body) try: - resolved_references = self.schema_registry.resolve_references(references) + resolved_dependencies = self.schema_registry.resolve_references(references) new_schema = ValidatedTypedSchema.parse( schema_type=schema_type, schema_str=body["schema"], - references=resolved_references, + references=references, + dependencies=resolved_dependencies, ) except (InvalidReferences, InvalidSchema, InvalidSchemaType) as e: self.log.warning("Invalid schema: %r", body["schema"], exc_info=True) @@ -1095,7 +1107,7 @@ async def subject_post( are_we_master, master_url = await self.schema_registry.get_master() if are_we_master: try: - schema_id = await self.schema_registry.write_new_schema_local(subject, new_schema, resolved_references) + schema_id = await self.schema_registry.write_new_schema_local(subject, new_schema, references) self.r( body={"id": schema_id}, content_type=content_type, diff --git a/karapace/serialization.py b/karapace/serialization.py index 9c0d988a5..52f80829b 100644 --- a/karapace/serialization.py +++ b/karapace/serialization.py @@ -3,7 +3,6 @@ from google.protobuf.message import DecodeError from jsonschema import ValidationError from karapace.client import Client -from karapace.dependency import Dependency from karapace.errors import InvalidReferences from karapace.protobuf.exception import ProtobufTypeException from karapace.protobuf.io import ProtobufDatumReader, ProtobufDatumWriter @@ -79,7 +78,7 @@ def __init__( self.base_url = schema_registry_url async def post_new_schema( - self, subject: str, schema: ValidatedTypedSchema, references: Optional[Dependency] = None + self, subject: str, schema: ValidatedTypedSchema, references: Optional[Reference] = None ) -> int: if schema.schema_type is SchemaType.PROTOBUF: if references: diff --git a/tests/integration/test_schema_protobuf.py b/tests/integration/test_schema_protobuf.py index 1275b071c..7cf8de44f 100644 --- a/tests/integration/test_schema_protobuf.py +++ b/tests/integration/test_schema_protobuf.py @@ -175,7 +175,6 @@ async def test_protobuf_schema_normalization(registry_async_client: Client, trai async def test_protobuf_schema_references(registry_async_client: Client) -> None: - customer_schema = """ |syntax = "proto3"; |package a1; @@ -280,9 +279,9 @@ async def test_protobuf_schema_references(registry_async_client: Client) -> None myjson = res.json() match_msg = "Subject 'test_schema' Version 1 was soft deleted.Set permanent=true to delete permanently" - assert res.status_code == 422 + assert res.status_code == 404 - assert myjson["error_code"] == 42206 and myjson["message"] == match_msg + assert myjson["error_code"] == 40406 and myjson["message"] == match_msg res = await registry_async_client.delete("subjects/customer/versions/1") myjson = res.json() match_msg = "One or more references exist to the schema {magic=1,keytype=SCHEMA,subject=customer,version=1}." @@ -297,7 +296,6 @@ async def test_protobuf_schema_references(registry_async_client: Client) -> None assert res.status_code == 200 - async def test_protobuf_schema_jjaakola_one(registry_async_client: Client) -> None: no_ref = """ |syntax = "proto3"; @@ -443,7 +441,7 @@ async def test_protobuf_schema_verifier(registry_async_client: Client) -> None: assert res.status_code == 200 res = await registry_async_client.delete("subjects/customer/versions/1") - assert res.status_code == 404 + assert res.status_code == 422 res = await registry_async_client.delete("subjects/test_schema/versions/1?permanent=true") assert res.status_code == 200 @@ -585,7 +583,6 @@ class ReferenceTestCase(BaseTestCase): } """ - # Nested references SCHEMA_NO_REF_NESTED_MESSAGE = """\ syntax = "proto3"; @@ -621,7 +618,7 @@ class ReferenceTestCase(BaseTestCase): schema_type=SchemaType.PROTOBUF, schema_str=SCHEMA_NO_REF, subject="nr_s1", - references=[], + references=None, expected=200, ), TestCaseDeleteSchema( @@ -657,9 +654,9 @@ class ReferenceTestCase(BaseTestCase): schema_type=SchemaType.PROTOBUF, schema_str=SCHEMA_WITH_REF, subject="wr_nonexisting_s1_missing_references", - references=[], + references=None, expected=422, - expected_msg=f"Invalid schema {SCHEMA_WITH_REF} with refs [] of type PROTOBUF", + expected_msg=f"Invalid schema {SCHEMA_WITH_REF} with refs None of type PROTOBUF", expected_error_code=42201, ), ], @@ -671,7 +668,7 @@ class ReferenceTestCase(BaseTestCase): schema_type=SchemaType.PROTOBUF, schema_str=SCHEMA_NO_REF, subject="wr_s1", - references=[], + references=None, expected=200, ), TestCaseSchema( @@ -718,7 +715,7 @@ class ReferenceTestCase(BaseTestCase): schema_type=SchemaType.PROTOBUF, schema_str=SCHEMA_NO_REF, subject="wr_s1_test_incompatible_change", - references=[], + references=None, expected=200, ), TestCaseSchema( @@ -732,12 +729,14 @@ class ReferenceTestCase(BaseTestCase): schema_type=SchemaType.PROTOBUF, schema_str=SCHEMA_REMOVES_REFERENCED_FIELD_INCOMPATIBLE, subject="wr_s2_test_incompatible_change", - references=[], - expected=409, - expected_msg=( - "Incompatible schema, compatibility_mode=BACKWARD " - "Incompatible modification Modification.MESSAGE_DROP found" - ), + references=None, + expected=200, + # It is erroneous assumption, there FIELD_DROP only, and it is compatible. + # expected = 200 + # expected_msg=( + # "Incompatible schema, compatibility_mode=BACKWARD " + # "Incompatible modification Modification.MESSAGE_DROP found" + # ), ), ], ), @@ -748,7 +747,7 @@ class ReferenceTestCase(BaseTestCase): schema_type=SchemaType.PROTOBUF, schema_str=SCHEMA_NO_REF, subject="wr_s1_add_new_reference", - references=[], + references=None, expected=200, ), TestCaseSchema( @@ -762,7 +761,7 @@ class ReferenceTestCase(BaseTestCase): schema_type=SchemaType.PROTOBUF, schema_str=SCHEMA_NO_REF_TWO, subject="wr_s3_the_new_reference", - references=[], + references=None, expected=200, ), TestCaseSchema( @@ -784,7 +783,7 @@ class ReferenceTestCase(BaseTestCase): schema_type=SchemaType.PROTOBUF, schema_str=SCHEMA_NO_REF, subject="wr_chain_s1", - references=[], + references=None, expected=200, ), TestCaseSchema( @@ -813,7 +812,7 @@ class ReferenceTestCase(BaseTestCase): schema_type=SchemaType.PROTOBUF, schema_str=SCHEMA_NO_REF, subject="wr_chain_s1", - references=[], + references=None, expected=200, ), TestCaseSchema( @@ -881,7 +880,7 @@ class ReferenceTestCase(BaseTestCase): schema_type=SchemaType.PROTOBUF, schema_str=SCHEMA_NO_REF, subject="wr_invalid_reference_ok_schema", - references=[], + references=None, expected=200, ), TestCaseSchema( @@ -906,7 +905,7 @@ class ReferenceTestCase(BaseTestCase): schema_type=SchemaType.PROTOBUF, schema_str=SCHEMA_NO_REF_TWO_MESSAGES, subject="wr_s1_two_messages", - references=[], + references=None, expected=200, ), TestCaseSchema( @@ -925,7 +924,7 @@ class ReferenceTestCase(BaseTestCase): schema_type=SchemaType.PROTOBUF, schema_str=SCHEMA_NO_REF_NESTED_MESSAGE, subject="wr_s1_with_nested_message", - references=[], + references=None, expected=200, ), TestCaseSchema( @@ -942,6 +941,7 @@ class ReferenceTestCase(BaseTestCase): ) async def test_references(testcase: ReferenceTestCase, registry_async_client: Client): for testdata in testcase.schemas: + if isinstance(testdata, TestCaseSchema): print(f"Adding new schema, subject: '{testdata.subject}'\n{testdata.schema_str}") body = {"schemaType": testdata.schema_type, "schema": testdata.schema_str} @@ -963,6 +963,7 @@ async def test_references(testcase: ReferenceTestCase, registry_async_client: Cl res = await registry_async_client.delete(f"subjects/{testdata.subject}/versions/{testdata.version}") else: raise Exception("Unknown test case.") + assert res.status_code == testdata.expected if testdata.expected_msg: assert res.json_result.get("message", None) == testdata.expected_msg @@ -980,3 +981,53 @@ async def test_references(testcase: ReferenceTestCase, registry_async_client: Cl else: fetch_schema_res = await registry_async_client.get(f"/schemas/ids/{testdata.schema_id}") assert fetch_schema_res.status_code == 200 + + +async def test_protobuf_error(registry_async_client: Client) -> None: + testdata = TestCaseSchema( + schema_type=SchemaType.PROTOBUF, + schema_str=SCHEMA_NO_REF, + subject="wr_s1_test_incompatible_change", + references=None, + expected=200, + ) + print(f"Adding new schema, subject: '{testdata.subject}'\n{testdata.schema_str}") + body = {"schemaType": testdata.schema_type, "schema": testdata.schema_str} + if testdata.references: + body["references"] = testdata.references + res = await registry_async_client.post(f"subjects/{testdata.subject}/versions", json=body) + + assert res.status_code == 200 + + testdata = TestCaseSchema( + schema_type=SchemaType.PROTOBUF, + schema_str=SCHEMA_WITH_REF, + subject="wr_s2_test_incompatible_change", + references=[{"name": "NoReference.proto", "subject": "wr_s1_test_incompatible_change", "version": 1}], + expected=200, + ) + print(f"Adding new schema, subject: '{testdata.subject}'\n{testdata.schema_str}") + body = {"schemaType": testdata.schema_type, "schema": testdata.schema_str} + if testdata.references: + body["references"] = testdata.references + res = await registry_async_client.post(f"subjects/{testdata.subject}/versions", json=body) + assert res.status_code == 200 + testdata = TestCaseSchema( + schema_type=SchemaType.PROTOBUF, + schema_str=SCHEMA_REMOVES_REFERENCED_FIELD_INCOMPATIBLE, + subject="wr_s2_test_incompatible_change", + references=None, + expected=409, + expected_msg=( + # ACTUALLY THERE NO MESSAGE_DROP!!! + "Incompatible schema, compatibility_mode=BACKWARD " + "Incompatible modification Modification.MESSAGE_DROP found" + ), + ) + print(f"Adding new schema, subject: '{testdata.subject}'\n{testdata.schema_str}") + body = {"schemaType": testdata.schema_type, "schema": testdata.schema_str} + if testdata.references: + body["references"] = testdata.references + res = await registry_async_client.post(f"subjects/{testdata.subject}/versions", json=body) + + assert res.status_code == 200 From 2977a2d45e4f573ba12b29b408370fb8eb61e642 Mon Sep 17 00:00:00 2001 From: Sergiy Zaschipas Date: Mon, 5 Dec 2022 13:54:20 +0200 Subject: [PATCH 17/25] add references support to serialization/deserialization with tests --- karapace/protobuf/io.py | 51 +++++-- tests/unit/test_protobuf_serialization.py | 157 ++++++++++++++++++++++ 2 files changed, 197 insertions(+), 11 deletions(-) diff --git a/karapace/protobuf/io.py b/karapace/protobuf/io.py index ba8d8a223..12dccb66a 100644 --- a/karapace/protobuf/io.py +++ b/karapace/protobuf/io.py @@ -12,6 +12,7 @@ import importlib.util import os import subprocess +import sys def calculate_class_name(name: str) -> str: @@ -45,29 +46,57 @@ def find_message_name(schema: ProtobufSchema, indexes: List[int]) -> str: return ".".join(result) +def crawl_dependencies_(schema: ProtobufSchema, deps_list: Dict[str, str]): + if schema.dependencies: + for name, dependency in schema.dependencies.items(): + crawl_dependencies_(dependency.schema, deps_list) + deps_list[name] = str(dependency.schema) + + +def crawl_dependencies(schema: ProtobufSchema) -> Dict[str, str]: + deps_list: Dict[str, str] = {} + crawl_dependencies_(schema, deps_list) + return deps_list + + def get_protobuf_class_instance(schema: ProtobufSchema, class_name: str, cfg: Dict) -> Any: directory = cfg["protobuf_runtime_directory"] proto_name = calculate_class_name(str(schema)) proto_path = f"{proto_name}.proto" - class_path = f"{directory}/{proto_name}_pb2.py" - if not os.path.isfile(proto_path): - with open(f"{directory}/{proto_name}.proto", mode="w", encoding="utf8") as proto_text: - proto_text.write(str(schema)) + work_dir = f"{directory}/{proto_name}" + + if not os.path.isdir(work_dir): + os.mkdir(work_dir) + class_path = f"{directory}/{proto_name}/{proto_name}_pb2.py" + with open(f"{directory}/{proto_name}/{proto_name}.proto", mode="w", encoding="utf8") as proto_text: + proto_text.write(str(schema)) + deps_list = crawl_dependencies(schema) + protoc_arguments = [ + "protoc", + "--python_out=./", + proto_path, + ] + for name, dependency_str in deps_list.items(): + dependency_path = f"{directory}/{proto_name}/{name}" + protoc_arguments.append(name) + with open(dependency_path, mode="w", encoding="utf8") as proto_text: + proto_text.write(dependency_str) if not os.path.isfile(class_path): subprocess.run( - [ - "protoc", - "--python_out=./", - proto_path, - ], + protoc_arguments, check=True, - cwd=directory, + cwd=work_dir, ) - spec = importlib.util.spec_from_file_location(f"{proto_name}_pb2", class_path) + sys.path.append(f"./runtime/{proto_name}") + spec = importlib.util.spec_from_file_location( + f"{proto_name}_pb2", + class_path, + ) tmp_module = importlib.util.module_from_spec(spec) spec.loader.exec_module(tmp_module) + sys.path.pop() class_to_call = getattr(tmp_module, class_name) return class_to_call() diff --git a/tests/unit/test_protobuf_serialization.py b/tests/unit/test_protobuf_serialization.py index 964803f38..a017ee85a 100644 --- a/tests/unit/test_protobuf_serialization.py +++ b/tests/unit/test_protobuf_serialization.py @@ -1,6 +1,8 @@ from karapace.config import read_config +from karapace.dependency import Dependency from karapace.protobuf.kotlin_wrapper import trim_margin from karapace.schema_models import SchemaType, ValidatedTypedSchema +from karapace.schema_references import Reference from karapace.serialization import ( InvalidMessageHeader, InvalidMessageSchema, @@ -50,6 +52,161 @@ async def test_happy_flow(default_config_path): assert mock_protobuf_registry_client.method_calls == [call.get_latest_schema("top")] +async def test_happy_flow_references(default_config_path): + no_ref_schema_str = """ + |syntax = "proto3"; + | + |option java_package = "com.codingharbour.protobuf"; + |option java_outer_classname = "TestEnumOrder"; + | + |message Speed { + | Enum speed = 1; + |} + | + |enum Enum { + | HIGH = 0; + | MIDDLE = 1; + | LOW = 2; + |} + | + """ + + ref_schema_str = """ + |syntax = "proto3"; + | + |option java_package = "com.codingharbour.protobuf"; + |option java_outer_classname = "TestEnumOrder"; + |import "Speed.proto"; + | + |message Message { + | int32 query = 1; + | Speed speed = 2; + |} + | + | + """ + no_ref_schema_str = trim_margin(no_ref_schema_str) + ref_schema_str = trim_margin(ref_schema_str) + + test_objects = [ + {"query": 5, "speed": {"speed": "HIGH"}}, + {"query": 10, "speed": {"speed": "MIDDLE"}}, + ] + + references = [Reference("Speed.proto", "speed", 1)] + + no_ref_schema = ValidatedTypedSchema.parse(SchemaType.PROTOBUF, no_ref_schema_str) + dep = Dependency("Speed.proto", "speed", 1, no_ref_schema) + ref_schema = ValidatedTypedSchema.parse(SchemaType.PROTOBUF, ref_schema_str, references, {"Speed.proto": dep}) + + mock_protobuf_registry_client = Mock() + schema_for_id_one_future = asyncio.Future() + schema_for_id_one_future.set_result(ref_schema) + mock_protobuf_registry_client.get_schema_for_id.return_value = schema_for_id_one_future + get_latest_schema_future = asyncio.Future() + get_latest_schema_future.set_result((1, ref_schema)) + mock_protobuf_registry_client.get_latest_schema.return_value = get_latest_schema_future + + serializer = await make_ser_deser(default_config_path, mock_protobuf_registry_client) + assert len(serializer.ids_to_schemas) == 0 + schema = await serializer.get_schema_for_subject("top") + for o in test_objects: + a = await serializer.serialize(schema, o) + u = await serializer.deserialize(a) + assert o == u + assert len(serializer.ids_to_schemas) == 1 + assert 1 in serializer.ids_to_schemas + + assert mock_protobuf_registry_client.method_calls == [call.get_latest_schema("top")] + + +async def test_happy_flow_references_two(default_config_path): + no_ref_schema_str = """ + |syntax = "proto3"; + | + |option java_package = "com.serge.protobuf"; + |option java_outer_classname = "TestSpeed"; + | + |message Speed { + | Enum speed = 1; + |} + | + |enum Enum { + | HIGH = 0; + | MIDDLE = 1; + | LOW = 2; + |} + | + """ + + ref_schema_str = """ + |syntax = "proto3"; + | + |option java_package = "com.serge.protobuf"; + |option java_outer_classname = "TestQuery"; + |import "Speed.proto"; + | + |message Query { + | int32 query = 1; + | Speed speed = 2; + |} + | + """ + + ref_schema_str_two = """ + |syntax = "proto3"; + | + |option java_package = "com.serge.protobuf"; + |option java_outer_classname = "TestMessage"; + |import "Query.proto"; + | + |message Message { + | int32 index = 1; + | Query qry = 2; + |} + | + """ + + no_ref_schema_str = trim_margin(no_ref_schema_str) + ref_schema_str = trim_margin(ref_schema_str) + ref_schema_str_two = trim_margin(ref_schema_str_two) + test_objects = [ + {"index": 1, "qry": {"query": 5, "speed": {"speed": "HIGH"}}}, + {"index": 2, "qry": {"query": 10, "speed": {"speed": "HIGH"}}}, + ] + + references = [Reference("Speed.proto", "speed", 1)] + references_two = [Reference("Query.proto", "msg", 1)] + + no_ref_schema = ValidatedTypedSchema.parse(SchemaType.PROTOBUF, no_ref_schema_str) + dep = Dependency("Speed.proto", "speed", 1, no_ref_schema) + ref_schema = ValidatedTypedSchema.parse(SchemaType.PROTOBUF, ref_schema_str, references, {"Speed.proto": dep}) + dep_two = Dependency("Query.proto", "qry", 1, ref_schema) + ref_schema_two = ValidatedTypedSchema.parse( + SchemaType.PROTOBUF, ref_schema_str_two, references_two, {"Query.proto": dep_two} + ) + + mock_protobuf_registry_client = Mock() + schema_for_id_one_future = asyncio.Future() + schema_for_id_one_future.set_result(ref_schema_two) + mock_protobuf_registry_client.get_schema_for_id.return_value = schema_for_id_one_future + get_latest_schema_future = asyncio.Future() + get_latest_schema_future.set_result((1, ref_schema_two)) + mock_protobuf_registry_client.get_latest_schema.return_value = get_latest_schema_future + + serializer = await make_ser_deser(default_config_path, mock_protobuf_registry_client) + assert len(serializer.ids_to_schemas) == 0 + schema = await serializer.get_schema_for_subject("top") + for o in test_objects: + a = await serializer.serialize(schema, o) + u = await serializer.deserialize(a) + assert o == u + assert len(serializer.ids_to_schemas) == 1 + assert 1 in serializer.ids_to_schemas + + assert mock_protobuf_registry_client.method_calls == [call.get_latest_schema("top")] + + async def test_serialization_fails(default_config_path): mock_protobuf_registry_client = Mock() get_latest_schema_future = asyncio.Future() From a9c8fec2fdfed0d91d21aaef8f98d1a1a3a5d38f Mon Sep 17 00:00:00 2001 From: Sergiy Zaschipas Date: Mon, 5 Dec 2022 23:47:35 +0200 Subject: [PATCH 18/25] fixup repetetive generation of .proto files for serialization and add unique paths for all of them --- karapace/protobuf/io.py | 72 ++++++++++++++++++++++++++--------------- 1 file changed, 46 insertions(+), 26 deletions(-) diff --git a/karapace/protobuf/io.py b/karapace/protobuf/io.py index 12dccb66a..b7810a4b1 100644 --- a/karapace/protobuf/io.py +++ b/karapace/protobuf/io.py @@ -5,7 +5,7 @@ from karapace.protobuf.protobuf_to_dict import dict_to_protobuf, protobuf_to_dict from karapace.protobuf.schema import ProtobufSchema from karapace.protobuf.type_element import TypeElement -from typing import Any, Dict, List +from typing import Any, Dict, List, Optional import hashlib import importlib @@ -46,48 +46,68 @@ def find_message_name(schema: ProtobufSchema, indexes: List[int]) -> str: return ".".join(result) -def crawl_dependencies_(schema: ProtobufSchema, deps_list: Dict[str, str]): +def crawl_dependencies_(schema: ProtobufSchema, deps_list: Dict[str, Dict[str, str]]): if schema.dependencies: for name, dependency in schema.dependencies.items(): crawl_dependencies_(dependency.schema, deps_list) - deps_list[name] = str(dependency.schema) + deps_list[name] = { + "schema": str(dependency.schema), + "unique_class_name": calculate_class_name(f"{dependency.version}_{dependency.name}"), + } -def crawl_dependencies(schema: ProtobufSchema) -> Dict[str, str]: - deps_list: Dict[str, str] = {} +def crawl_dependencies(schema: ProtobufSchema) -> Dict[str, Dict[str, str]]: + deps_list: Dict[str, Dict[str, str]] = {} crawl_dependencies_(schema, deps_list) return deps_list +def replace_imports(string: str, deps_list: Optional[Dict[str, Dict[str, str]]]) -> str: + if deps_list is None: + return string + for key, value in deps_list.items(): + unique_class_name = value["unique_class_name"] + ".proto" + string = string.replace('"' + key + '"', f'"{unique_class_name}"') + return string + + def get_protobuf_class_instance(schema: ProtobufSchema, class_name: str, cfg: Dict) -> Any: directory = cfg["protobuf_runtime_directory"] - proto_name = calculate_class_name(str(schema)) + deps_list = crawl_dependencies(schema) + root_class_name = "" + for value in deps_list.values(): + root_class_name = root_class_name + value["unique_class_name"] + root_class_name = root_class_name + str(schema) + proto_name = calculate_class_name(root_class_name) + proto_path = f"{proto_name}.proto" work_dir = f"{directory}/{proto_name}" if not os.path.isdir(work_dir): os.mkdir(work_dir) class_path = f"{directory}/{proto_name}/{proto_name}_pb2.py" - with open(f"{directory}/{proto_name}/{proto_name}.proto", mode="w", encoding="utf8") as proto_text: - proto_text.write(str(schema)) - deps_list = crawl_dependencies(schema) - protoc_arguments = [ - "protoc", - "--python_out=./", - proto_path, - ] - for name, dependency_str in deps_list.items(): - dependency_path = f"{directory}/{proto_name}/{name}" - protoc_arguments.append(name) - with open(dependency_path, mode="w", encoding="utf8") as proto_text: - proto_text.write(dependency_str) - - if not os.path.isfile(class_path): - subprocess.run( - protoc_arguments, - check=True, - cwd=work_dir, - ) + if not os.path.exists(class_path): + with open(f"{directory}/{proto_name}/{proto_name}.proto", mode="w", encoding="utf8") as proto_text: + proto_text.write(replace_imports(str(schema), deps_list)) + + protoc_arguments = [ + "protoc", + "--python_out=./", + proto_path, + ] + for value in deps_list.values(): + proto_file_name = value["unique_class_name"] + ".proto" + dependency_path = f"{directory}/{proto_name}/{proto_file_name}" + protoc_arguments.append(proto_file_name) + with open(dependency_path, mode="w", encoding="utf8") as proto_text: + proto_text.write(replace_imports(value["schema"], deps_list)) + + if not os.path.isfile(class_path): + subprocess.run( + protoc_arguments, + check=True, + cwd=work_dir, + ) sys.path.append(f"./runtime/{proto_name}") spec = importlib.util.spec_from_file_location( From 7aebbe0d4689d3a11f328637ad2d71472d8c3a18 Mon Sep 17 00:00:00 2001 From: Sergiy Zaschipas Date: Thu, 2 Mar 2023 15:50:25 +0200 Subject: [PATCH 19/25] merge workaround --- karapace/in_memory_database.py | 19 ++++---- karapace/schema_models.py | 80 ++++++++++++++++---------------- karapace/schema_reader.py | 18 +++---- karapace/schema_registry.py | 6 +-- karapace/schema_registry_apis.py | 12 ++--- karapace/serialization.py | 2 +- karapace/utils.py | 1 - tests/integration/test_schema.py | 8 ---- 8 files changed, 65 insertions(+), 81 deletions(-) diff --git a/karapace/in_memory_database.py b/karapace/in_memory_database.py index c770d3e16..518dd7f18 100644 --- a/karapace/in_memory_database.py +++ b/karapace/in_memory_database.py @@ -9,13 +9,12 @@ from karapace.schema_reader import Referents from karapace.schema_references import Reference from karapace.typing import ResolvedVersion, SchemaId, Subject +from karapace.utils import reference_key from threading import Lock, RLock from typing import Dict, List, Optional, Tuple import logging -from karapace.utils import reference_key - LOG = logging.getLogger(__name__) @@ -73,7 +72,7 @@ def get_schema_id(self, new_schema: TypedSchema) -> SchemaId: return self.global_schema_id def get_schema_id_if_exists( - self, *, subject: Subject, schema: TypedSchema, include_deleted: bool # pylint: disable=unused-argument + self, *, subject: Subject, schema: TypedSchema, include_deleted: bool # pylint: disable=unused-argument ) -> Optional[SchemaId]: subject_fingerprints = self._hash_to_schema_id_on_subject.get(subject) if subject_fingerprints: @@ -101,8 +100,14 @@ def get_next_version(self, *, subject: Subject) -> ResolvedVersion: return max(self.subjects[subject].schemas) + 1 def insert_schema_version( - self, *, subject: Subject, schema_id: SchemaId, version: ResolvedVersion, deleted: bool, schema: TypedSchema, - references: List[Reference] + self, + *, + subject: Subject, + schema_id: SchemaId, + version: ResolvedVersion, + deleted: bool, + schema: TypedSchema, + references: List[Reference], ) -> None: with self.schema_lock_thread: self.global_schema_id = max(self.global_schema_id, schema_id) @@ -133,7 +138,6 @@ def insert_schema_version( subject=subject, schema=schema, schema_id=schema_id, - ) else: self._delete_from_schema_id_on_subject( @@ -195,8 +199,7 @@ def find_subjects(self, *, include_deleted: bool) -> List[Subject]: return list(self.subjects.keys()) with self.schema_lock_thread: return [ - subject for subject in self.subjects if - self.find_subject_schemas(subject=subject, include_deleted=False) + subject for subject in self.subjects if self.find_subject_schemas(subject=subject, include_deleted=False) ] def find_subject_schemas(self, *, subject: Subject, include_deleted: bool) -> Dict[ResolvedVersion, SchemaVersion]: diff --git a/karapace/schema_models.py b/karapace/schema_models.py index a204f5613..daf92ad7a 100644 --- a/karapace/schema_models.py +++ b/karapace/schema_models.py @@ -2,18 +2,11 @@ Copyright (c) 2023 Aiven Ltd See LICENSE for details """ -import hashlib -import json -import logging -from dataclasses import dataclass -from typing import Any, cast, Dict, NoReturn, Optional, Union -from typing import List - from avro.errors import SchemaParseException from avro.schema import parse as avro_parse, Schema as AvroSchema +from dataclasses import dataclass from jsonschema import Draft7Validator from jsonschema.exceptions import SchemaError - from karapace.dependency import Dependency from karapace.errors import InvalidSchema from karapace.protobuf.exception import ( @@ -30,6 +23,11 @@ from karapace.schema_type import SchemaType from karapace.typing import ResolvedVersion, SchemaId, Subject from karapace.utils import json_decode, json_encode, JSONDecodeError +from typing import Any, cast, Dict, List, NoReturn, Optional, Union + +import hashlib +import json +import logging LOG = logging.getLogger(__name__) @@ -183,10 +181,12 @@ def get_references(self) -> Optional[List[Reference]]: return self.references def __eq__(self, other: Any) -> bool: - return isinstance(other, (TypedSchema, ValidatedTypedSchema)) and \ - self.schema_type is other.schema_type and\ - str(self) == str(other) and\ - self.references == other.references + return ( + isinstance(other, (TypedSchema, ValidatedTypedSchema)) + and self.schema_type is other.schema_type + and str(self) == str(other) + and self.references == other.references + ) def parse( @@ -196,7 +196,6 @@ def parse( validate_avro_names: bool, references: Optional[List[Reference]] = None, dependencies: Optional[Dict[str, Dependency]] = None, - ) -> "ParsedTypedSchema": if schema_type not in [SchemaType.AVRO, SchemaType.JSONSCHEMA, SchemaType.PROTOBUF]: raise InvalidSchema(f"Unknown parser {schema_type} for {schema_str}") @@ -259,24 +258,24 @@ class ParsedTypedSchema(TypedSchema): are considered by the current version of the SDK invalid. """ - def __init__(self, schema_type: SchemaType, - schema_str: str, - schema: Union[Draft7Validator, AvroSchema, ProtobufSchema], - references: Optional[List[Reference]] = None, - dependencies: Optional[Dict[str, Dependency]] = None, - ): - super().__init__(schema_type=schema_type, - schema_str=schema_str, - references=references, - dependencies=dependencies) + def __init__( + self, + schema_type: SchemaType, + schema_str: str, + schema: Union[Draft7Validator, AvroSchema, ProtobufSchema], + references: Optional[List[Reference]] = None, + dependencies: Optional[Dict[str, Dependency]] = None, + ): + super().__init__(schema_type=schema_type, schema_str=schema_str, references=references, dependencies=dependencies) self.schema = schema @staticmethod - def parse(schema_type: SchemaType, - schema_str: str, - references: Optional[List[Reference]] = None, - dependencies: Optional[Dict[str, Dependency]] = None, - ) -> "ParsedTypedSchema": + def parse( + schema_type: SchemaType, + schema_str: str, + references: Optional[List[Reference]] = None, + dependencies: Optional[Dict[str, Dependency]] = None, + ) -> "ParsedTypedSchema": return parse( schema_type=schema_type, schema_str=schema_str, @@ -307,23 +306,24 @@ class ValidatedTypedSchema(ParsedTypedSchema): """ def __init__( - self, - schema_type: SchemaType, - schema_str: str, - schema: Union[Draft7Validator, AvroSchema, ProtobufSchema], - references: Optional[List[Reference]] = None, - dependencies: Optional[Dict[str, Dependency]] = None, + self, + schema_type: SchemaType, + schema_str: str, + schema: Union[Draft7Validator, AvroSchema, ProtobufSchema], + references: Optional[List[Reference]] = None, + dependencies: Optional[Dict[str, Dependency]] = None, ): super().__init__( - schema_type=schema_type, schema_str=schema_str, references=references, dependencies=dependencies, - schema=schema + schema_type=schema_type, schema_str=schema_str, references=references, dependencies=dependencies, schema=schema ) @staticmethod - def parse(schema_type: SchemaType, schema_str: str, - references: Optional[List[Reference]] = None, - dependencies: Optional[Dict[str, Dependency]] = None, - ) -> "ValidatedTypedSchema": + def parse( + schema_type: SchemaType, + schema_str: str, + references: Optional[List[Reference]] = None, + dependencies: Optional[Dict[str, Dependency]] = None, + ) -> "ValidatedTypedSchema": parsed_schema = parse( schema_type=schema_type, schema_str=schema_str, diff --git a/karapace/schema_reader.py b/karapace/schema_reader.py index ea7964030..fa359b9ca 100644 --- a/karapace/schema_reader.py +++ b/karapace/schema_reader.py @@ -6,7 +6,6 @@ """ from avro.schema import Schema as AvroSchema from contextlib import closing, ExitStack - from jsonschema.validators import Draft7Validator from kafka import KafkaConsumer, TopicPartition from kafka.admin import KafkaAdminClient, NewTopic @@ -20,25 +19,22 @@ ) from karapace import constants from karapace.config import Config - - +from karapace.dependency import Dependency from karapace.errors import InvalidReferences, InvalidSchema from karapace.in_memory_database import InMemoryDatabase from karapace.key_format import is_key_in_canonical_format, KeyFormatter, KeyMode from karapace.master_coordinator import MasterCoordinator -from karapace.schema_models import parse_protobuf_schema_definition,SchemaType, TypedSchema +from karapace.protobuf.schema import ProtobufSchema +from karapace.schema_models import parse_protobuf_schema_definition, SchemaType, TypedSchema +from karapace.schema_references import Reference from karapace.statsd import StatsClient from karapace.utils import json_decode, JSONDecodeError, KarapaceKafkaClient from threading import Condition, Event, Thread from typing import Any, Dict, List, Optional, Union -from karapace.dependency import Dependency -from karapace.protobuf.schema import ProtobufSchema -from karapace.schema_references import Reference - +import json import logging import time -import json Offset = int Subject = str @@ -530,9 +526,7 @@ def _handle_msg_schema(self, key: dict, value: Optional[dict]) -> None: if resolved_references: for ref in resolved_references: - self.database.insert_referenced_by(subject=ref.subject, - version=ref.version, - schema_id=schema_id) + self.database.insert_referenced_by(subject=ref.subject, version=ref.version, schema_id=schema_id) def handle_msg(self, key: dict, value: Optional[dict]) -> None: if key["keytype"] == "CONFIG": diff --git a/karapace/schema_registry.py b/karapace/schema_registry.py index 895efbd14..538bd1dea 100644 --- a/karapace/schema_registry.py +++ b/karapace/schema_registry.py @@ -27,11 +27,11 @@ from karapace.master_coordinator import MasterCoordinator from karapace.schema_models import ParsedTypedSchema, SchemaType, SchemaVersion, TypedSchema, ValidatedTypedSchema from karapace.schema_reader import KafkaSchemaReader +from karapace.schema_references import Reference from karapace.typing import JsonData, ResolvedVersion, Subject, Version from karapace.utils import json_encode, KarapaceKafkaClient, reference_key from karapace.version import __version__ from typing import cast, Dict, List, Optional, Tuple, Union -from karapace.schema_references import Reference import asyncio import logging @@ -397,8 +397,8 @@ async def write_new_schema_local( parsed_old_schema = ParsedTypedSchema.parse( schema_type=old_schema.schema_type, schema_str=old_schema.schema_str, - references = old_schema_references, - dependencies = old_schema_dependencies, + references=old_schema_references, + dependencies=old_schema_dependencies, ) result = check_compatibility( old_schema=parsed_old_schema, diff --git a/karapace/schema_registry_apis.py b/karapace/schema_registry_apis.py index c30d9e47f..7d9037d7f 100644 --- a/karapace/schema_registry_apis.py +++ b/karapace/schema_registry_apis.py @@ -29,12 +29,11 @@ from karapace.karapace import KarapaceBase from karapace.rapu import HTTPRequest, JSON_CONTENT_TYPE, SERVER_NAME from karapace.schema_models import ParsedTypedSchema, SchemaType, SchemaVersion, TypedSchema, ValidatedTypedSchema +from karapace.schema_references import Reference from karapace.schema_registry import KarapaceSchemaRegistry, validate_version from karapace.typing import JsonData, ResolvedVersion, SchemaId -from karapace.utils import JSONDecodeError -from typing import Any, Dict, Optional, Union, List -from karapace.schema_references import Reference -from karapace.utils import reference_key +from karapace.utils import JSONDecodeError, reference_key +from typing import Any, Dict, List, Optional, Union import aiohttp import async_timeout @@ -1023,10 +1022,7 @@ async def subjects_schema_post( # When checking if schema is already registered, allow unvalidated schema in as # there might be stored schemas that are non-compliant from the past. new_schema = ParsedTypedSchema.parse( - schema_type=schema_type, - schema_str=schema_str, - references=references, - dependencies=new_schema_dependencies + schema_type=schema_type, schema_str=schema_str, references=references, dependencies=new_schema_dependencies ) except InvalidSchema: self.log.exception("No proper parser found") diff --git a/karapace/serialization.py b/karapace/serialization.py index b3adb8401..266bf3c2a 100644 --- a/karapace/serialization.py +++ b/karapace/serialization.py @@ -11,8 +11,8 @@ from karapace.protobuf.exception import ProtobufTypeException from karapace.protobuf.io import ProtobufDatumReader, ProtobufDatumWriter from karapace.schema_models import InvalidSchema, ParsedTypedSchema, SchemaType, TypedSchema, ValidatedTypedSchema -from karapace.utils import json_decode, json_encode from karapace.schema_references import Reference +from karapace.utils import json_decode, json_encode from typing import Any, Dict, Optional, Tuple from urllib.parse import quote diff --git a/karapace/utils.py b/karapace/utils.py index 32a019e07..43c9becb5 100644 --- a/karapace/utils.py +++ b/karapace/utils.py @@ -14,7 +14,6 @@ from kafka.client_async import BrokerConnection, KafkaClient from karapace.typing import JsonData from pathlib import Path - from types import MappingProxyType from typing import Any, AnyStr, IO, NoReturn, Optional, overload, Union diff --git a/tests/integration/test_schema.py b/tests/integration/test_schema.py index bc5c94232..3f3656aec 100644 --- a/tests/integration/test_schema.py +++ b/tests/integration/test_schema.py @@ -2907,11 +2907,7 @@ async def test_invalid_schema_should_provide_good_error_messages(registry_async_ f"subjects/{test_subject}/versions", json={"schema": schema_str[:-1]}, ) -<<<<<<< HEAD - assert res.json()["message"] == "Expecting ',' delimiter: line 1 column 18 (char 17)" -======= assert res.json()["message"].startswith("Invalid AVRO schema. Error: ") ->>>>>>> master # Unfortunately the AVRO library doesn't provide a good error message, it just raises an TypeError schema_str = json.dumps({"type": "enum", "name": "error"}) @@ -2993,9 +2989,6 @@ async def test_schema_non_compliant_namespace_in_existing( f"compatibility/subjects/{subject}/versions/latest", json={"schema": json.dumps(evolved_schema)}, ) -<<<<<<< HEAD - assert res.json()["message"] == "error is a reserved type name." -======= assert res.status_code == 200 # Post evolved new schema @@ -3141,4 +3134,3 @@ async def test_schema_non_compliant_name_in_existing( assert "id" in res.json() schema_id = res.json()["id"] assert schema_id == 2 ->>>>>>> master From d0ded167f09b20272f8a1985242228c4d575c2ac Mon Sep 17 00:00:00 2001 From: Sergiy Zaschipas Date: Thu, 2 Mar 2023 18:07:06 +0200 Subject: [PATCH 20/25] fixup merge issues --- karapace/errors.py | 2 ++ karapace/in_memory_database.py | 6 +++--- karapace/schema_models.py | 2 +- karapace/schema_reader.py | 4 ++++ karapace/schema_references.py | 4 ++-- karapace/schema_registry.py | 32 ++++++++++++++++++++++---------- karapace/schema_registry_apis.py | 9 +++------ karapace/utils.py | 4 ++-- 8 files changed, 39 insertions(+), 24 deletions(-) diff --git a/karapace/errors.py b/karapace/errors.py index e49508286..953e74b15 100644 --- a/karapace/errors.py +++ b/karapace/errors.py @@ -2,6 +2,8 @@ Copyright (c) 2023 Aiven Ltd See LICENSE for details """ +from karapace.typing import Version +from typing import List class VersionNotFoundException(Exception): diff --git a/karapace/in_memory_database.py b/karapace/in_memory_database.py index 518dd7f18..82c25377a 100644 --- a/karapace/in_memory_database.py +++ b/karapace/in_memory_database.py @@ -248,7 +248,7 @@ def num_schema_versions(self) -> Tuple[int, int]: soft_deleted_versions += 1 return (live_versions, soft_deleted_versions) - def insert_referenced_by(self, *, subject, version, schema_id: SchemaId): + def insert_referenced_by(self, *, subject: Subject, version: ResolvedVersion, schema_id: SchemaId) -> None: with self.schema_lock_thread: ref_str = reference_key(subject, version) referents = self.referenced_by.get(ref_str, None) @@ -257,12 +257,12 @@ def insert_referenced_by(self, *, subject, version, schema_id: SchemaId): else: self.referenced_by[ref_str] = [schema_id] - def get_referenced_by(self, subject, version) -> Optional[SchemaId]: + def get_referenced_by(self, subject: Subject, version: ResolvedVersion) -> Optional[Referents]: with self.schema_lock_thread: ref_str = reference_key(subject, version) return self.referenced_by.get(ref_str, None) - def remove_referenced_by(self, schema_id: SchemaId, references: List[Reference]): + def remove_referenced_by(self, schema_id: SchemaId, references: List[Reference]) -> None: with self.schema_lock_thread: for ref in references: key = reference_key(ref.subject, ref.version) diff --git a/karapace/schema_models.py b/karapace/schema_models.py index daf92ad7a..ced2a1211 100644 --- a/karapace/schema_models.py +++ b/karapace/schema_models.py @@ -343,4 +343,4 @@ class SchemaVersion: deleted: bool schema_id: SchemaId schema: TypedSchema - references: List[Reference] + references: Optional[List[Reference]] diff --git a/karapace/schema_reader.py b/karapace/schema_reader.py index fa359b9ca..5316cdd59 100644 --- a/karapace/schema_reader.py +++ b/karapace/schema_reader.py @@ -28,6 +28,7 @@ from karapace.schema_models import parse_protobuf_schema_definition, SchemaType, TypedSchema from karapace.schema_references import Reference from karapace.statsd import StatsClient +from karapace.typing import ResolvedVersion from karapace.utils import json_decode, JSONDecodeError, KarapaceKafkaClient from threading import Condition, Event, Thread from typing import Any, Dict, List, Optional, Union @@ -542,6 +543,9 @@ def handle_msg(self, key: dict, value: Optional[dict]) -> None: def remove_referenced_by(self, schema_id: SchemaId, references: List[Reference]): self.database.remove_referenced_by(schema_id, references) + def get_referenced_by(self, subject: Subject, version: ResolvedVersion) -> Optional[Referents]: + return self.database.get_referenced_by(subject, version) + def _resolve_reference(self, reference: Reference) -> Dependency: subject_data = self.database.find_subject_schemas(subject=reference.subject, include_deleted=False) if not subject_data: diff --git a/karapace/schema_references.py b/karapace/schema_references.py index 46823fb4c..7639d448d 100644 --- a/karapace/schema_references.py +++ b/karapace/schema_references.py @@ -1,9 +1,9 @@ -from karapace.typing import JsonData, Subject, Version +from karapace.typing import JsonData, ResolvedVersion, Subject from typing import Any class Reference: - def __init__(self, name: str, subject: Subject, version: Version): + def __init__(self, name: str, subject: Subject, version: ResolvedVersion): self.name = name self.subject = subject self.version = version diff --git a/karapace/schema_registry.py b/karapace/schema_registry.py index 538bd1dea..2d837a093 100644 --- a/karapace/schema_registry.py +++ b/karapace/schema_registry.py @@ -29,7 +29,7 @@ from karapace.schema_reader import KafkaSchemaReader from karapace.schema_references import Reference from karapace.typing import JsonData, ResolvedVersion, Subject, Version -from karapace.utils import json_encode, KarapaceKafkaClient, reference_key +from karapace.utils import json_encode, KarapaceKafkaClient from karapace.version import __version__ from typing import cast, Dict, List, Optional, Tuple, Union @@ -199,10 +199,8 @@ async def subject_delete_local(self, subject: str, permanent: bool) -> List[Reso version_list = [] if permanent: version_list = list(schema_versions) - latest_version_id = version_list[-1] - for version_id, schema_version in list(schema_versions.items()): - referenced_by = self.schema_reader.referenced_by.get(reference_key(subject, version_id), None) + referenced_by = self.schema_reader.get_referenced_by(subject, schema_version.version) if referenced_by and len(referenced_by) > 0: raise ReferenceExistsException(referenced_by, version_id) @@ -213,17 +211,16 @@ async def subject_delete_local(self, subject: str, permanent: bool) -> List[Reso version_id, schema_version.schema_id, ) - references = schema_version.get("references", None) self.send_schema_message( subject=subject, schema=None, schema_id=schema_version.schema_id, version=version_id, deleted=True, - references=references, + references=schema_version.references, ) - if references and len(references) > 0: - self.schema_reader.remove_referenced_by(schema_id, references) + if schema_version.references and len(schema_version.references) > 0: + self.schema_reader.remove_referenced_by(schema_version.schema_id, schema_version.references) else: try: schema_versions_live = self.subject_get(subject, include_deleted=False) @@ -233,7 +230,7 @@ async def subject_delete_local(self, subject: str, permanent: bool) -> List[Reso except SchemasNotFoundException: pass - referenced_by = self.schema_reader.referenced_by.get(reference_key(subject, latest_version_id), None) + referenced_by = self.schema_reader.get_referenced_by(subject, latest_version_id) if referenced_by and len(referenced_by) > 0: raise ReferenceExistsException(referenced_by, latest_version_id) self.send_delete_subject_message(subject, latest_version_id) @@ -261,7 +258,7 @@ async def subject_version_delete_local(self, subject: Subject, version: Version, if permanent and not schema_version.deleted: raise SchemaVersionNotSoftDeletedException() - referenced_by = self.schema_reader.referenced_by.get(reference_key(subject, int(resolved_version)), None) + referenced_by = self.schema_reader.get_referenced_by(subject, resolved_version) if referenced_by and len(referenced_by) > 0: raise ReferenceExistsException(referenced_by, version) @@ -316,6 +313,21 @@ def subject_version_get(self, subject: Subject, version: Version, *, include_del ret["compatibility"] = compatibility return ret + def subject_version_referencedby_get(self, subject: Subject, version: Version, *, include_deleted: bool = False) -> List: + validate_version(version) + schema_versions = self.subject_get(subject, include_deleted=include_deleted) + if not schema_versions: + raise SubjectNotFoundException() + resolved_version = _resolve_version(schema_versions=schema_versions, version=version) + schema_data: Optional[SchemaVersion] = schema_versions.get(resolved_version, None) + if not schema_data: + raise VersionNotFoundException() + referenced_by = self.schema_reader.get_referenced_by(schema_data.subject, schema_data.version) + + if referenced_by and len(referenced_by) > 0: + return list(referenced_by) + return [] + async def write_new_schema_local( self, subject: Subject, diff --git a/karapace/schema_registry_apis.py b/karapace/schema_registry_apis.py index 7d9037d7f..b22d99e10 100644 --- a/karapace/schema_registry_apis.py +++ b/karapace/schema_registry_apis.py @@ -32,7 +32,7 @@ from karapace.schema_references import Reference from karapace.schema_registry import KarapaceSchemaRegistry, validate_version from karapace.typing import JsonData, ResolvedVersion, SchemaId -from karapace.utils import JSONDecodeError, reference_key +from karapace.utils import JSONDecodeError from typing import Any, Dict, List, Optional, Union import aiohttp @@ -855,7 +855,7 @@ async def subject_version_referencedby_get(self, content_type, *, subject, versi self._check_authorization(user, Operation.Read, f"Subject:{subject}") try: - subject_data = await self.schema_registry.subject_version_get(subject, version) + referenced_by = await self.schema_registry.subject_version_referencedby_get(subject, version) except (SubjectNotFoundException, SchemasNotFoundException): self.r( body={ @@ -877,10 +877,7 @@ async def subject_version_referencedby_get(self, content_type, *, subject, versi except InvalidVersion: self._invalid_version(content_type, version) - referenced_by = self.schema_registry.schema_reader.referenced_by.get( - reference_key(subject_data["subject"], subject_data["version"]), [] - ) - self.r(list(referenced_by), content_type, status=HTTPStatus.OK) + self.r(referenced_by, content_type, status=HTTPStatus.OK) async def subject_versions_list( self, content_type: str, *, subject: str, request: HTTPRequest, user: Optional[User] = None diff --git a/karapace/utils.py b/karapace/utils.py index 43c9becb5..e618d2dce 100644 --- a/karapace/utils.py +++ b/karapace/utils.py @@ -12,7 +12,7 @@ from decimal import Decimal from http import HTTPStatus from kafka.client_async import BrokerConnection, KafkaClient -from karapace.typing import JsonData +from karapace.typing import JsonData, ResolvedVersion from pathlib import Path from types import MappingProxyType from typing import Any, AnyStr, IO, NoReturn, Optional, overload, Union @@ -66,7 +66,7 @@ def default_json_serialization(obj: MappingProxyType) -> dict: ... -def reference_key(subject: str, version: int) -> str: +def reference_key(subject: str, version: ResolvedVersion) -> str: return hash((subject, version)) From 3d5a21549cc227d7c96b8520d81fb3556ef7e6c5 Mon Sep 17 00:00:00 2001 From: Sergiy Zaschipas Date: Wed, 8 Mar 2023 16:51:22 +0200 Subject: [PATCH 21/25] debugg workaround --- karapace/in_memory_database.py | 3 +-- karapace/protobuf/io.py | 29 +++++++++++++++++++---- karapace/schema_models.py | 16 +++++++++---- karapace/schema_reader.py | 3 +-- karapace/schema_references.py | 4 +++- tests/unit/test_protobuf_serialization.py | 28 +++++++++++----------- 6 files changed, 56 insertions(+), 27 deletions(-) diff --git a/karapace/in_memory_database.py b/karapace/in_memory_database.py index 82c25377a..8beae55b3 100644 --- a/karapace/in_memory_database.py +++ b/karapace/in_memory_database.py @@ -6,8 +6,7 @@ """ from dataclasses import dataclass, field from karapace.schema_models import SchemaVersion, TypedSchema -from karapace.schema_reader import Referents -from karapace.schema_references import Reference +from karapace.schema_references import Reference, Referents from karapace.typing import ResolvedVersion, SchemaId, Subject from karapace.utils import reference_key from threading import Lock, RLock diff --git a/karapace/protobuf/io.py b/karapace/protobuf/io.py index 7da6ba736..ba7f4d6cf 100644 --- a/karapace/protobuf/io.py +++ b/karapace/protobuf/io.py @@ -4,7 +4,8 @@ """ from io import BytesIO from karapace.protobuf.encoding_variants import read_indexes, write_indexes -from karapace.protobuf.exception import IllegalArgumentException, ProtobufSchemaResolutionException, ProtobufTypeException +from karapace.protobuf.exception import IllegalArgumentException, ProtobufSchemaResolutionException, \ + ProtobufTypeException from karapace.protobuf.message_element import MessageElement from karapace.protobuf.protobuf_to_dict import dict_to_protobuf, protobuf_to_dict from karapace.protobuf.schema import ProtobufSchema @@ -18,6 +19,8 @@ import subprocess import sys +_module_cache: Dict[str, Any] = dict() + def calculate_class_name(name: str) -> str: return "c_" + hashlib.md5(name.encode("utf-8")).hexdigest() @@ -86,9 +89,17 @@ def get_protobuf_class_instance(schema: ProtobufSchema, class_name: str, cfg: Di proto_path = f"{proto_name}.proto" work_dir = f"{directory}/{proto_name}" + if not os.path.isdir(directory): + try: + os.mkdir(directory) + except Exception as e: + pass if not os.path.isdir(work_dir): - os.mkdir(work_dir) + try: + os.mkdir(work_dir) + except Exception as e: + pass class_path = f"{directory}/{proto_name}/{proto_name}_pb2.py" if not os.path.exists(class_path): with open(f"{directory}/{proto_name}/{proto_name}.proto", mode="w", encoding="utf8") as proto_text: @@ -119,9 +130,18 @@ def get_protobuf_class_instance(schema: ProtobufSchema, class_name: str, cfg: Di class_path, ) tmp_module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(tmp_module) + class_to_call = _module_cache.get(tmp_module, None) + if class_to_call is not None: + sys.path.pop() + return class_to_call() + try: + spec.loader.exec_module(tmp_module) + except Exception as e: + pass + sys.path.pop() class_to_call = getattr(tmp_module, class_name) + _module_cache[tmp_module] = class_to_call return class_to_call() @@ -142,7 +162,8 @@ def read_data(config: dict, writer_schema: ProtobufSchema, reader_schema: Protob class ProtobufDatumReader: """Deserialize Protobuf-encoded data into a Python data structure.""" - def __init__(self, config: dict, writer_schema: ProtobufSchema = None, reader_schema: ProtobufSchema = None) -> None: + def __init__(self, config: dict, writer_schema: ProtobufSchema = None, + reader_schema: ProtobufSchema = None) -> None: """As defined in the Protobuf specification, we call the schema encoded in the data the "writer's schema", and the schema expected by the reader the "reader's schema". diff --git a/karapace/schema_models.py b/karapace/schema_models.py index ced2a1211..125e68dc9 100644 --- a/karapace/schema_models.py +++ b/karapace/schema_models.py @@ -100,7 +100,7 @@ def __init__( self.schema_type = schema_type self.references = references self.dependencies = dependencies - self.schema_str = TypedSchema.normalize_schema_str(schema_str, schema_type) + self.schema_str = TypedSchema.normalize_schema_str(schema_str, schema_type, references, dependencies) self.max_id: Optional[SchemaId] = None self._fingerprint_cached: Optional[str] = None @@ -118,7 +118,11 @@ def fingerprint(self) -> str: return self._fingerprint_cached @staticmethod - def normalize_schema_str(schema_str: str, schema_type: SchemaType) -> str: + def normalize_schema_str(schema_str: str, + schema_type: SchemaType, + references: Optional[List[Reference]] = None, + dependencies: Optional[Dict[str, Dependency]] = None, + ) -> str: if schema_type is SchemaType.AVRO or schema_type is SchemaType.JSONSCHEMA: try: schema_str = json_encode(json_decode(schema_str), compact=True, sort_keys=True) @@ -127,7 +131,7 @@ def normalize_schema_str(schema_str: str, schema_type: SchemaType) -> str: raise e elif schema_type == SchemaType.PROTOBUF: try: - schema_str = str(parse_protobuf_schema_definition(schema_str)) + schema_str = str(parse_protobuf_schema_definition(schema_str,references,dependencies,None)) except InvalidSchema as e: LOG.exception("Schema is not valid ProtoBuf definition") raise e @@ -236,7 +240,11 @@ def parse( else: raise InvalidSchema(f"Unknown parser {schema_type} for {schema_str}") - return ParsedTypedSchema(schema_type=schema_type, schema_str=schema_str, schema=parsed_schema) + return ParsedTypedSchema(schema_type=schema_type, + schema_str=schema_str, + schema=parsed_schema, + references=references, + dependencies=dependencies) class ParsedTypedSchema(TypedSchema): diff --git a/karapace/schema_reader.py b/karapace/schema_reader.py index 5316cdd59..7805ce413 100644 --- a/karapace/schema_reader.py +++ b/karapace/schema_reader.py @@ -26,7 +26,7 @@ from karapace.master_coordinator import MasterCoordinator from karapace.protobuf.schema import ProtobufSchema from karapace.schema_models import parse_protobuf_schema_definition, SchemaType, TypedSchema -from karapace.schema_references import Reference +from karapace.schema_references import Reference, Referents from karapace.statsd import StatsClient from karapace.typing import ResolvedVersion from karapace.utils import json_decode, JSONDecodeError, KarapaceKafkaClient @@ -42,7 +42,6 @@ Version = int Schema = Dict[str, Any] -Referents = List SchemaId = int # The value `0` is a valid offset and it represents the first message produced diff --git a/karapace/schema_references.py b/karapace/schema_references.py index 7639d448d..074b6a58c 100644 --- a/karapace/schema_references.py +++ b/karapace/schema_references.py @@ -1,5 +1,7 @@ from karapace.typing import JsonData, ResolvedVersion, Subject -from typing import Any +from typing import Any, List + +Referents = List class Reference: diff --git a/tests/unit/test_protobuf_serialization.py b/tests/unit/test_protobuf_serialization.py index 36ced56fb..1716cb1d4 100644 --- a/tests/unit/test_protobuf_serialization.py +++ b/tests/unit/test_protobuf_serialization.py @@ -5,7 +5,7 @@ from karapace.config import read_config from karapace.dependency import Dependency from karapace.protobuf.kotlin_wrapper import trim_margin -from karapace.schema_models import SchemaType, ValidatedTypedSchema +from karapace.schema_models import SchemaType, ParsedTypedSchema from karapace.schema_references import Reference from karapace.serialization import ( InvalidMessageHeader, @@ -37,10 +37,10 @@ async def make_ser_deser(config_path: str, mock_client) -> SchemaRegistrySeriali async def test_happy_flow(default_config_path): mock_protobuf_registry_client = Mock() schema_for_id_one_future = asyncio.Future() - schema_for_id_one_future.set_result(ValidatedTypedSchema.parse(SchemaType.PROTOBUF, trim_margin(schema_protobuf))) + schema_for_id_one_future.set_result(ParsedTypedSchema.parse(SchemaType.PROTOBUF, trim_margin(schema_protobuf))) mock_protobuf_registry_client.get_schema_for_id.return_value = schema_for_id_one_future get_latest_schema_future = asyncio.Future() - get_latest_schema_future.set_result((1, ValidatedTypedSchema.parse(SchemaType.PROTOBUF, trim_margin(schema_protobuf)))) + get_latest_schema_future.set_result((1, ParsedTypedSchema.parse(SchemaType.PROTOBUF, trim_margin(schema_protobuf)))) mock_protobuf_registry_client.get_latest_schema.return_value = get_latest_schema_future serializer = await make_ser_deser(default_config_path, mock_protobuf_registry_client) @@ -59,7 +59,7 @@ async def test_happy_flow(default_config_path): async def test_happy_flow_references(default_config_path): no_ref_schema_str = """ |syntax = "proto3"; - | + |package aaa; |option java_package = "com.codingharbour.protobuf"; |option java_outer_classname = "TestEnumOrder"; | @@ -77,7 +77,7 @@ async def test_happy_flow_references(default_config_path): ref_schema_str = """ |syntax = "proto3"; - | + |package aaa; |option java_package = "com.codingharbour.protobuf"; |option java_outer_classname = "TestEnumOrder"; |import "Speed.proto"; @@ -99,9 +99,9 @@ async def test_happy_flow_references(default_config_path): references = [Reference("Speed.proto", "speed", 1)] - no_ref_schema = ValidatedTypedSchema.parse(SchemaType.PROTOBUF, no_ref_schema_str) + no_ref_schema = ParsedTypedSchema.parse(SchemaType.PROTOBUF, no_ref_schema_str) dep = Dependency("Speed.proto", "speed", 1, no_ref_schema) - ref_schema = ValidatedTypedSchema.parse(SchemaType.PROTOBUF, ref_schema_str, references, {"Speed.proto": dep}) + ref_schema = ParsedTypedSchema.parse(SchemaType.PROTOBUF, ref_schema_str, references, {"Speed.proto": dep}) mock_protobuf_registry_client = Mock() schema_for_id_one_future = asyncio.Future() @@ -127,7 +127,7 @@ async def test_happy_flow_references(default_config_path): async def test_happy_flow_references_two(default_config_path): no_ref_schema_str = """ |syntax = "proto3"; - | + |package bbb; |option java_package = "com.serge.protobuf"; |option java_outer_classname = "TestSpeed"; | @@ -145,7 +145,7 @@ async def test_happy_flow_references_two(default_config_path): ref_schema_str = """ |syntax = "proto3"; - | + |package bbb; |option java_package = "com.serge.protobuf"; |option java_outer_classname = "TestQuery"; |import "Speed.proto"; @@ -159,7 +159,7 @@ async def test_happy_flow_references_two(default_config_path): ref_schema_str_two = """ |syntax = "proto3"; - | + |package bbb; |option java_package = "com.serge.protobuf"; |option java_outer_classname = "TestMessage"; |import "Query.proto"; @@ -182,11 +182,11 @@ async def test_happy_flow_references_two(default_config_path): references = [Reference("Speed.proto", "speed", 1)] references_two = [Reference("Query.proto", "msg", 1)] - no_ref_schema = ValidatedTypedSchema.parse(SchemaType.PROTOBUF, no_ref_schema_str) + no_ref_schema = ParsedTypedSchema.parse(SchemaType.PROTOBUF, no_ref_schema_str) dep = Dependency("Speed.proto", "speed", 1, no_ref_schema) - ref_schema = ValidatedTypedSchema.parse(SchemaType.PROTOBUF, ref_schema_str, references, {"Speed.proto": dep}) + ref_schema = ParsedTypedSchema.parse(SchemaType.PROTOBUF, ref_schema_str, references, {"Speed.proto": dep}) dep_two = Dependency("Query.proto", "qry", 1, ref_schema) - ref_schema_two = ValidatedTypedSchema.parse( + ref_schema_two = ParsedTypedSchema.parse( SchemaType.PROTOBUF, ref_schema_str_two, references_two, {"Query.proto": dep_two} ) @@ -214,7 +214,7 @@ async def test_happy_flow_references_two(default_config_path): async def test_serialization_fails(default_config_path): mock_protobuf_registry_client = Mock() get_latest_schema_future = asyncio.Future() - get_latest_schema_future.set_result((1, ValidatedTypedSchema.parse(SchemaType.PROTOBUF, trim_margin(schema_protobuf)))) + get_latest_schema_future.set_result((1, ParsedTypedSchema.parse(SchemaType.PROTOBUF, trim_margin(schema_protobuf)))) mock_protobuf_registry_client.get_latest_schema.return_value = get_latest_schema_future serializer = await make_ser_deser(default_config_path, mock_protobuf_registry_client) From afbff90874c3d43abe0899c78f28e378cd109802 Mon Sep 17 00:00:00 2001 From: Sergiy Zaschipas Date: Mon, 13 Mar 2023 22:46:19 +0200 Subject: [PATCH 22/25] debuggina --- karapace/protobuf/io.py | 107 +++++++++++++++------- karapace/schema_models.py | 39 ++++---- karapace/schema_reader.py | 7 +- karapace/schema_registry.py | 4 +- karapace/schema_registry_apis.py | 10 +- tests/integration/test_schema_protobuf.py | 9 +- tests/unit/test_protobuf_serialization.py | 12 +-- 7 files changed, 123 insertions(+), 65 deletions(-) diff --git a/karapace/protobuf/io.py b/karapace/protobuf/io.py index ba7f4d6cf..5d9fb113c 100644 --- a/karapace/protobuf/io.py +++ b/karapace/protobuf/io.py @@ -4,12 +4,12 @@ """ from io import BytesIO from karapace.protobuf.encoding_variants import read_indexes, write_indexes -from karapace.protobuf.exception import IllegalArgumentException, ProtobufSchemaResolutionException, \ - ProtobufTypeException +from karapace.protobuf.exception import IllegalArgumentException, ProtobufSchemaResolutionException, ProtobufTypeException from karapace.protobuf.message_element import MessageElement from karapace.protobuf.protobuf_to_dict import dict_to_protobuf, protobuf_to_dict from karapace.protobuf.schema import ProtobufSchema from karapace.protobuf.type_element import TypeElement +from multiprocessing import Process, Queue from typing import Any, Dict, List, Optional import hashlib @@ -90,16 +90,9 @@ def get_protobuf_class_instance(schema: ProtobufSchema, class_name: str, cfg: Di proto_path = f"{proto_name}.proto" work_dir = f"{directory}/{proto_name}" if not os.path.isdir(directory): - try: - os.mkdir(directory) - except Exception as e: - pass - + os.mkdir(directory) if not os.path.isdir(work_dir): - try: - os.mkdir(work_dir) - except Exception as e: - pass + os.mkdir(work_dir) class_path = f"{directory}/{proto_name}/{proto_name}_pb2.py" if not os.path.exists(class_path): with open(f"{directory}/{proto_name}/{proto_name}.proto", mode="w", encoding="utf8") as proto_text: @@ -130,18 +123,10 @@ def get_protobuf_class_instance(schema: ProtobufSchema, class_name: str, cfg: Di class_path, ) tmp_module = importlib.util.module_from_spec(spec) - class_to_call = _module_cache.get(tmp_module, None) - if class_to_call is not None: - sys.path.pop() - return class_to_call() - try: - spec.loader.exec_module(tmp_module) - except Exception as e: - pass - + spec.loader.exec_module(tmp_module) sys.path.pop() class_to_call = getattr(tmp_module, class_name) - _module_cache[tmp_module] = class_to_call + return class_to_call() @@ -159,11 +144,39 @@ def read_data(config: dict, writer_schema: ProtobufSchema, reader_schema: Protob return class_instance +def reader_process(queue: Queue, config: dict, writer_schema: ProtobufSchema, reader_schema: ProtobufSchema, bio: BytesIO): + try: + queue.put(protobuf_to_dict(read_data(config, writer_schema, reader_schema, bio), True)) + except Exception as e: + queue.put(e) + + +def reader_mp(config: dict, writer_schema: ProtobufSchema, reader_schema: ProtobufSchema, bio: BytesIO) -> Dict: + # Note Protobuf enum values use C++ scoping rules, + # meaning that enum values are siblings of their type, not children of it. + # Therefore, if we have two proto files with Enums which elements have the same name we will have error. + # There we use simple way of Serialization/Deserialization (SerDe) which use python Protobuf library and + # protoc compiler. + # To avoid problem with enum values for basic SerDe support we + # will isolate work with call protobuf libraries in child process. + if __name__ == "karapace.protobuf.io": + queue = Queue() + p = Process(target=reader_process, args=(queue, config, writer_schema, reader_schema, bio)) + p.start() + result = queue.get() + p.join() + if isinstance(result, Dict): + return result + if isinstance(result, Exception): + raise result + raise IllegalArgumentException() + return {"Error": "This never must be returned"} + + class ProtobufDatumReader: """Deserialize Protobuf-encoded data into a Python data structure.""" - def __init__(self, config: dict, writer_schema: ProtobufSchema = None, - reader_schema: ProtobufSchema = None) -> None: + def __init__(self, config: dict, writer_schema: ProtobufSchema = None, reader_schema: ProtobufSchema = None) -> None: """As defined in the Protobuf specification, we call the schema encoded in the data the "writer's schema", and the schema expected by the reader the "reader's schema". @@ -172,10 +185,44 @@ def __init__(self, config: dict, writer_schema: ProtobufSchema = None, self._writer_schema = writer_schema self._reader_schema = reader_schema - def read(self, bio: BytesIO) -> None: + def read(self, bio: BytesIO) -> Dict: if self._reader_schema is None: self._reader_schema = self._writer_schema - return protobuf_to_dict(read_data(self.config, self._writer_schema, self._reader_schema, bio), True) + return reader_mp(self.config, self._writer_schema, self._reader_schema, bio) + + +def writer_process(queue: Queue, config: Dict, writer_schema: ProtobufSchema, message_name: str, datum: dict): + class_instance = get_protobuf_class_instance(writer_schema, message_name, config) + try: + dict_to_protobuf(class_instance, datum) + except Exception: + # pylint: disable=raise-missing-from + e = ProtobufTypeException(writer_schema, datum) + queue.put(e) + raise e + queue.put(class_instance.SerializeToString()) + + +def writer_mp(config: Dict, writer_schema: ProtobufSchema, message_name: str, datum: Dict) -> str: + # Note Protobuf enum values use C++ scoping rules, + # meaning that enum values are siblings of their type, not children of it. + # Therefore, if we have two proto files with Enums which elements have the same name we will have error. + # There we use simple way of Serialization/Deserialization (SerDe) which use python Protobuf library and + # protoc compiler. + # To avoid problem with enum values for basic SerDe support we + # will isolate work with call protobuf libraries in child process. + if __name__ == "karapace.protobuf.io": + queue = Queue() + p = Process(target=writer_process, args=(queue, config, writer_schema, message_name, datum)) + p.start() + result = queue.get() + p.join() + if isinstance(result, bytes): + return result + if isinstance(result, Exception): + raise result + raise IllegalArgumentException() + return "Error :This never must be returned" class ProtobufDatumWriter: @@ -201,12 +248,4 @@ def write_index(self, writer: BytesIO) -> None: def write(self, datum: dict, writer: BytesIO) -> None: - class_instance = get_protobuf_class_instance(self._writer_schema, self._message_name, self.config) - - try: - dict_to_protobuf(class_instance, datum) - except Exception: - # pylint: disable=raise-missing-from - raise ProtobufTypeException(self._writer_schema, datum) - - writer.write(class_instance.SerializeToString()) + writer.write(writer_mp(self.config, self._writer_schema, self._message_name, datum)) diff --git a/karapace/schema_models.py b/karapace/schema_models.py index 125e68dc9..8bc52f93d 100644 --- a/karapace/schema_models.py +++ b/karapace/schema_models.py @@ -95,7 +95,7 @@ def __init__( schema_type (SchemaType): The type of the schema schema_str (str): The original schema string schema (Optional[Union[Draft7Validator, AvroSchema, ProtobufSchema]]): The parsed and validated schema - references (Optional[List[Dependency]]): The references of schema + references (Optional[List[Dependency]]): The references of schema """ self.schema_type = schema_type self.references = references @@ -103,7 +103,6 @@ def __init__( self.schema_str = TypedSchema.normalize_schema_str(schema_str, schema_type, references, dependencies) self.max_id: Optional[SchemaId] = None self._fingerprint_cached: Optional[str] = None - self._str_cached: Optional[str] = None self._schema_cached: Optional[Union[Draft7Validator, AvroSchema, ProtobufSchema]] = schema @@ -118,11 +117,12 @@ def fingerprint(self) -> str: return self._fingerprint_cached @staticmethod - def normalize_schema_str(schema_str: str, - schema_type: SchemaType, - references: Optional[List[Reference]] = None, - dependencies: Optional[Dict[str, Dependency]] = None, - ) -> str: + def normalize_schema_str( + schema_str: str, + schema_type: SchemaType, + references: Optional[List[Reference]] = None, + dependencies: Optional[Dict[str, Dependency]] = None, + ) -> str: if schema_type is SchemaType.AVRO or schema_type is SchemaType.JSONSCHEMA: try: schema_str = json_encode(json_decode(schema_str), compact=True, sort_keys=True) @@ -131,7 +131,7 @@ def normalize_schema_str(schema_str: str, raise e elif schema_type == SchemaType.PROTOBUF: try: - schema_str = str(parse_protobuf_schema_definition(schema_str,references,dependencies,None)) + schema_str = str(parse_protobuf_schema_definition(schema_str, references, dependencies, False)) except InvalidSchema as e: LOG.exception("Schema is not valid ProtoBuf definition") raise e @@ -140,13 +140,18 @@ def normalize_schema_str(schema_str: str, return schema_str def __str__(self) -> str: - return self.schema_str + if self.schema_type == SchemaType.PROTOBUF: + return self.schema_str + + if self._str_cached is None: + self._str_cached = json_encode(self.to_dict()) + return self._str_cached def __repr__(self) -> str: return f"TypedSchema(type={self.schema_type}, schema={str(self)})" @property - def schema_cached(self) -> Union[Draft7Validator, AvroSchema, ProtobufSchema]: + def schema(self) -> Union[Draft7Validator, AvroSchema, ProtobufSchema]: if self._schema_cached is not None: return self._schema_cached if self.schema_type is SchemaType.AVRO: @@ -240,11 +245,13 @@ def parse( else: raise InvalidSchema(f"Unknown parser {schema_type} for {schema_str}") - return ParsedTypedSchema(schema_type=schema_type, - schema_str=schema_str, - schema=parsed_schema, - references=references, - dependencies=dependencies) + return ParsedTypedSchema( + schema_type=schema_type, + schema_str=schema_str, + schema=parsed_schema, + references=references, + dependencies=dependencies, + ) class ParsedTypedSchema(TypedSchema): @@ -275,7 +282,7 @@ def __init__( dependencies: Optional[Dict[str, Dependency]] = None, ): super().__init__(schema_type=schema_type, schema_str=schema_str, references=references, dependencies=dependencies) - self.schema = schema + self._schema_cached = schema @staticmethod def parse( diff --git a/karapace/schema_reader.py b/karapace/schema_reader.py index 7805ce413..46315afb1 100644 --- a/karapace/schema_reader.py +++ b/karapace/schema_reader.py @@ -25,7 +25,7 @@ from karapace.key_format import is_key_in_canonical_format, KeyFormatter, KeyMode from karapace.master_coordinator import MasterCoordinator from karapace.protobuf.schema import ProtobufSchema -from karapace.schema_models import parse_protobuf_schema_definition, SchemaType, TypedSchema +from karapace.schema_models import parse_protobuf_schema_definition, SchemaType, SchemaVersion, TypedSchema from karapace.schema_references import Reference, Referents from karapace.statsd import StatsClient from karapace.typing import ResolvedVersion @@ -549,7 +549,10 @@ def _resolve_reference(self, reference: Reference) -> Dependency: subject_data = self.database.find_subject_schemas(subject=reference.subject, include_deleted=False) if not subject_data: raise InvalidReferences(f"Subject not found {reference.subject}.") - schema: TypedSchema = subject_data.get(reference.version, {}).get("schema", None) + schema_version: SchemaVersion = subject_data.get(reference.version, {}) + if schema_version is None: + raise InvalidReferences(f"Subject {reference.subject} has no such schema version") + schema: TypedSchema = schema_version.schema if not schema: raise InvalidReferences(f"No schema in {reference.subject} with version {reference.version}.") if schema.references: diff --git a/karapace/schema_registry.py b/karapace/schema_registry.py index 2d837a093..560ef62e0 100644 --- a/karapace/schema_registry.py +++ b/karapace/schema_registry.py @@ -313,7 +313,9 @@ def subject_version_get(self, subject: Subject, version: Version, *, include_del ret["compatibility"] = compatibility return ret - def subject_version_referencedby_get(self, subject: Subject, version: Version, *, include_deleted: bool = False) -> List: + async def subject_version_referencedby_get( + self, subject: Subject, version: Version, *, include_deleted: bool = False + ) -> List: validate_version(version) schema_versions = self.subject_get(subject, include_deleted=include_deleted) if not schema_versions: diff --git a/karapace/schema_registry_apis.py b/karapace/schema_registry_apis.py index b22d99e10..8d62c3ba1 100644 --- a/karapace/schema_registry_apis.py +++ b/karapace/schema_registry_apis.py @@ -375,7 +375,11 @@ async def compatibility_check( ) old_schema_type = self._validate_schema_type(content_type=content_type, data=old) try: - old_schema = ParsedTypedSchema.parse(old_schema_type, old["schema"]) + old_references = old.get("references", None) + old_dependencies = None + if old_references: + old_dependencies = self.schema_registry.resolve_references(old_references) + old_schema = ParsedTypedSchema.parse(old_schema_type, old["schema"], old_references, old_dependencies) except InvalidSchema: self.r( body={ @@ -876,6 +880,8 @@ async def subject_version_referencedby_get(self, content_type, *, subject, versi ) except InvalidVersion: self._invalid_version(content_type, version) + except Exception as a: + pass self.r(referenced_by, content_type, status=HTTPStatus.OK) @@ -1165,6 +1171,8 @@ async def subject_post( content_type=content_type, status=HTTPStatus.UNPROCESSABLE_ENTITY, ) + except Exception as xx: + raise xx elif not master_url: self.no_master_error(content_type) diff --git a/tests/integration/test_schema_protobuf.py b/tests/integration/test_schema_protobuf.py index 5be358a50..63e914ff3 100644 --- a/tests/integration/test_schema_protobuf.py +++ b/tests/integration/test_schema_protobuf.py @@ -322,7 +322,6 @@ async def test_protobuf_schema_jjaakola_one(registry_async_client: Client) -> No with_first_ref = trim_margin(with_first_ref) references = [{"name": "NoReference.proto", "subject": "sub1", "version": 1}] - res = await registry_async_client.post( "subjects/sub2/versions", json={"schemaType": "PROTOBUF", "schema": with_first_ref, "references": references}, @@ -640,7 +639,7 @@ class ReferenceTestCase(BaseTestCase): references=[{"name": "NoReference.proto", "subject": "wr_not_found", "version": 1}], expected=422, expected_msg=( - f"Invalid schema {SCHEMA_WITH_REF} " + f"Invalid PROTOBUF schema. Error: Invalid schema {SCHEMA_WITH_REF} " "with refs [{name='NoReference.proto', subject='wr_not_found', version=1}] of type PROTOBUF" ), expected_error_code=42201, @@ -656,7 +655,7 @@ class ReferenceTestCase(BaseTestCase): subject="wr_nonexisting_s1_missing_references", references=None, expected=422, - expected_msg=f"Invalid schema {SCHEMA_WITH_REF} with refs None of type PROTOBUF", + expected_msg=f"Invalid PROTOBUF schema. Error: Invalid schema {SCHEMA_WITH_REF} with refs None of type PROTOBUF", expected_error_code=42201, ), ], @@ -796,7 +795,7 @@ class ReferenceTestCase(BaseTestCase): ], expected=422, expected_msg=( - f"Invalid schema {SCHEMA_WITH_REF} " + f"Invalid PROTOBUF schema. Error: Invalid schema {SCHEMA_WITH_REF} " "with refs [{name='NoReference.proto', subject='wr_chain_s1', version=1}, " "{name='NotFoundReference.proto', subject='wr_chain_nonexisting', version=1}] " "of type PROTOBUF" @@ -890,7 +889,7 @@ class ReferenceTestCase(BaseTestCase): references=[{"name": "NoReference.proto", "subject": "wr_invalid_reference_ok_schema", "version": 1}], expected=422, expected_msg=( - f"Invalid schema {SCHEMA_INVALID_MISSING_CLOSING_BRACE} " + f"Invalid PROTOBUF schema. Error: Invalid schema {SCHEMA_INVALID_MISSING_CLOSING_BRACE} " "with refs [{name='NoReference.proto', subject='wr_invalid_reference_ok_schema', version=1}] " "of type PROTOBUF" ), diff --git a/tests/unit/test_protobuf_serialization.py b/tests/unit/test_protobuf_serialization.py index 1716cb1d4..29d249f3e 100644 --- a/tests/unit/test_protobuf_serialization.py +++ b/tests/unit/test_protobuf_serialization.py @@ -5,7 +5,7 @@ from karapace.config import read_config from karapace.dependency import Dependency from karapace.protobuf.kotlin_wrapper import trim_margin -from karapace.schema_models import SchemaType, ParsedTypedSchema +from karapace.schema_models import ParsedTypedSchema, SchemaType from karapace.schema_references import Reference from karapace.serialization import ( InvalidMessageHeader, @@ -59,7 +59,7 @@ async def test_happy_flow(default_config_path): async def test_happy_flow_references(default_config_path): no_ref_schema_str = """ |syntax = "proto3"; - |package aaa; + | |option java_package = "com.codingharbour.protobuf"; |option java_outer_classname = "TestEnumOrder"; | @@ -77,7 +77,7 @@ async def test_happy_flow_references(default_config_path): ref_schema_str = """ |syntax = "proto3"; - |package aaa; + | |option java_package = "com.codingharbour.protobuf"; |option java_outer_classname = "TestEnumOrder"; |import "Speed.proto"; @@ -127,7 +127,7 @@ async def test_happy_flow_references(default_config_path): async def test_happy_flow_references_two(default_config_path): no_ref_schema_str = """ |syntax = "proto3"; - |package bbb; + | |option java_package = "com.serge.protobuf"; |option java_outer_classname = "TestSpeed"; | @@ -145,7 +145,7 @@ async def test_happy_flow_references_two(default_config_path): ref_schema_str = """ |syntax = "proto3"; - |package bbb; + | |option java_package = "com.serge.protobuf"; |option java_outer_classname = "TestQuery"; |import "Speed.proto"; @@ -159,7 +159,7 @@ async def test_happy_flow_references_two(default_config_path): ref_schema_str_two = """ |syntax = "proto3"; - |package bbb; + | |option java_package = "com.serge.protobuf"; |option java_outer_classname = "TestMessage"; |import "Query.proto"; From a307d26e4315e3f87fe49d4b47c3146bbd100349 Mon Sep 17 00:00:00 2001 From: Sergiy Zaschipas Date: Tue, 14 Mar 2023 12:37:26 +0200 Subject: [PATCH 23/25] debug --- karapace/protobuf/io.py | 2 +- karapace/schema_registry_apis.py | 2 -- tests/integration/test_schema.py | 2 ++ tests/integration/test_schema_protobuf.py | 3 ++- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/karapace/protobuf/io.py b/karapace/protobuf/io.py index 5d9fb113c..f429a713c 100644 --- a/karapace/protobuf/io.py +++ b/karapace/protobuf/io.py @@ -147,7 +147,7 @@ def read_data(config: dict, writer_schema: ProtobufSchema, reader_schema: Protob def reader_process(queue: Queue, config: dict, writer_schema: ProtobufSchema, reader_schema: ProtobufSchema, bio: BytesIO): try: queue.put(protobuf_to_dict(read_data(config, writer_schema, reader_schema, bio), True)) - except Exception as e: + except Exception as e: # pylint: disable=broad-except queue.put(e) diff --git a/karapace/schema_registry_apis.py b/karapace/schema_registry_apis.py index e32983fc8..be495611c 100644 --- a/karapace/schema_registry_apis.py +++ b/karapace/schema_registry_apis.py @@ -881,8 +881,6 @@ async def subject_version_referencedby_get(self, content_type, *, subject, versi ) except InvalidVersion: self._invalid_version(content_type, version) - except Exception as a: - pass self.r(referenced_by, content_type, status=HTTPStatus.OK) diff --git a/tests/integration/test_schema.py b/tests/integration/test_schema.py index 7be45c879..ba77e9b70 100644 --- a/tests/integration/test_schema.py +++ b/tests/integration/test_schema.py @@ -2921,6 +2921,8 @@ async def test_invalid_schema_should_provide_good_error_messages(registry_async_ f"subjects/{test_subject}/versions", json={"schema": schema_str}, ) + if res.json()["message"] != "Enum symbols must be a sequence of strings, but it is ": + breakpoint() assert res.json()["message"] == "Enum symbols must be a sequence of strings, but it is " # This is an upstream bug in the python AVRO library, until the bug is fixed we should at least have a nice error message diff --git a/tests/integration/test_schema_protobuf.py b/tests/integration/test_schema_protobuf.py index 63e914ff3..aaf33ae0a 100644 --- a/tests/integration/test_schema_protobuf.py +++ b/tests/integration/test_schema_protobuf.py @@ -655,7 +655,8 @@ class ReferenceTestCase(BaseTestCase): subject="wr_nonexisting_s1_missing_references", references=None, expected=422, - expected_msg=f"Invalid PROTOBUF schema. Error: Invalid schema {SCHEMA_WITH_REF} with refs None of type PROTOBUF", + expected_msg=f"Invalid PROTOBUF schema. Error: Invalid schema {SCHEMA_WITH_REF} " + "with refs None of type PROTOBUF", expected_error_code=42201, ), ], From 73122dfc6ce849d6677d68b82623196904de7ce9 Mon Sep 17 00:00:00 2001 From: Sergiy Zaschipas Date: Tue, 14 Mar 2023 12:42:48 +0200 Subject: [PATCH 24/25] fixup merge issue --- tests/__init__.py | 0 tests/integration/test_schema.py | 26 ++++++++++---------------- 2 files changed, 10 insertions(+), 16 deletions(-) delete mode 100644 tests/__init__.py diff --git a/tests/__init__.py b/tests/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/integration/test_schema.py b/tests/integration/test_schema.py index ba77e9b70..f39b4c929 100644 --- a/tests/integration/test_schema.py +++ b/tests/integration/test_schema.py @@ -1450,17 +1450,6 @@ async def test_schema_subject_post_invalid(registry_async_client: Client) -> Non assert res.json()["error_code"] == 40401 assert res.json()["message"] == f"Subject '{subject_3}' not found." - schema_str = json.dumps({"type": "string"}) - # Create the subject - subject_1 = subject_name_factory() - res = await registry_async_client.post( - f"subjects/{subject_1}/versions", - json={"schema": schema_str, "references": [{"name": "Customer.avro", "subject": "customer", "version": 1}]}, - ) - assert res.status_code == 422 - assert res.json()["error_code"] == 44302 - assert res.json()["message"] == "Schema references are not supported for 'AVRO' schema type" - @pytest.mark.parametrize("trail", ["", "/"]) async def test_schema_lifecycle(registry_async_client: Client, trail: str) -> None: @@ -2303,7 +2292,8 @@ async def test_invalid_namespace(registry_async_client: Client) -> None: json_res = res.json() assert json_res["error_code"] == 42201, json_res expected_message = ( - "foo-bar-baz is not a valid Avro name because it does not match the pattern (?:^|\\.)[A-Za-z_][A-Za-z0-9_]*$" + "Invalid AVRO schema. Error: foo-bar-baz is not a valid Avro name because it does not match the pattern " + "(?:^|\\.)[A-Za-z_][A-Za-z0-9_]*$" ) assert json_res["message"] == expected_message, json_res @@ -2921,9 +2911,10 @@ async def test_invalid_schema_should_provide_good_error_messages(registry_async_ f"subjects/{test_subject}/versions", json={"schema": schema_str}, ) - if res.json()["message"] != "Enum symbols must be a sequence of strings, but it is ": - breakpoint() - assert res.json()["message"] == "Enum symbols must be a sequence of strings, but it is " + assert ( + res.json()["message"] + == "Invalid AVRO schema. Error: Enum symbols must be a sequence of strings, but it is " + ) # This is an upstream bug in the python AVRO library, until the bug is fixed we should at least have a nice error message schema_str = json.dumps({"type": "enum", "name": "error", "symbols": {}}) @@ -2931,7 +2922,10 @@ async def test_invalid_schema_should_provide_good_error_messages(registry_async_ f"subjects/{test_subject}/versions", json={"schema": schema_str}, ) - assert res.json()["message"] == "Enum symbols must be a sequence of strings, but it is " + assert ( + res.json()["message"] + == "Invalid AVRO schema. Error: Enum symbols must be a sequence of strings, but it is " + ) async def test_schema_non_compliant_namespace_in_existing( From 7d6bdf88723345f75a6ff9e475c7b1e83747c85d Mon Sep 17 00:00:00 2001 From: Sergiy Zaschipas Date: Tue, 14 Mar 2023 12:51:07 +0200 Subject: [PATCH 25/25] python 3.9 module requirement --- tests/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/__init__.py diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 000000000..e69de29bb