From efb456de65e3a6628c422a839bdee58e951b0d95 Mon Sep 17 00:00:00 2001 From: Sergiy Zaschipas Date: Tue, 19 Apr 2022 21:04:50 +0300 Subject: [PATCH 01/44] add basic support of references to POST/GET subjects --- karapace/schema_models.py | 45 +++++++++--- karapace/schema_reader.py | 4 + karapace/schema_registry_apis.py | 89 +++++++++++++++++++---- karapace/serialization.py | 41 ++++++++--- tests/integration/test_schema_protobuf.py | 56 ++++++++++++++ 5 files changed, 197 insertions(+), 38 deletions(-) diff --git a/karapace/schema_models.py b/karapace/schema_models.py index e8e111440..6e6a41bae 100644 --- a/karapace/schema_models.py +++ b/karapace/schema_models.py @@ -13,7 +13,7 @@ ) from karapace.protobuf.schema import ProtobufSchema from karapace.utils import json_encode -from typing import Any, Dict, Union +from typing import Any, Dict, Union, List import json import logging @@ -71,6 +71,10 @@ class InvalidSchema(Exception): pass +class InvalidReferences(Exception): + pass + + @unique class SchemaType(str, Enum): AVRO = "AVRO" @@ -105,11 +109,13 @@ def __repr__(self) -> str: return f"TypedSchema(type={self.schema_type}, schema={json_encode(self.to_dict())})" def __eq__(self, other: Any) -> bool: - return isinstance(other, TypedSchema) and self.__str__() == other.__str__() and self.schema_type is other.schema_type + return isinstance(other, + TypedSchema) and self.__str__() == other.__str__() and self.schema_type is other.schema_type class ValidatedTypedSchema(TypedSchema): - def __init__(self, schema_type: SchemaType, schema_str: str, schema: Union[Draft7Validator, AvroSchema, ProtobufSchema]): + def __init__(self, schema_type: SchemaType, schema_str: str, + schema: Union[Draft7Validator, AvroSchema, ProtobufSchema]): super().__init__(schema_type=schema_type, schema_str=schema_str) self.schema = schema @@ -136,15 +142,15 @@ def parse(schema_type: SchemaType, schema_str: str) -> "ValidatedTypedSchema": try: parsed_schema = parse_protobuf_schema_definition(schema_str) except ( - TypeError, - SchemaError, - AssertionError, - ProtobufParserRuntimeException, - IllegalStateException, - IllegalArgumentException, - ProtobufError, - ProtobufException, - ProtobufSchemaParseException, + TypeError, + SchemaError, + AssertionError, + ProtobufParserRuntimeException, + IllegalStateException, + IllegalArgumentException, + ProtobufError, + ProtobufException, + ProtobufSchemaParseException, ) as e: log.exception("Unexpected error: %s \n schema:[%s]", e, schema_str) raise InvalidSchema from e @@ -157,3 +163,18 @@ def __str__(self) -> str: if self.schema_type == SchemaType.PROTOBUF: return str(self.schema) return super().__str__() + + +class References: + def __init__(self, schema_type: SchemaType, references: List): + """Schema with type information + + Args: + schema_type (SchemaType): The type of the schema + references (str): The original schema string + """ + self.schema_type = schema_type + self.references = references + + def json(self) -> str: + return json_encode(self.references) diff --git a/karapace/schema_reader.py b/karapace/schema_reader.py index 335166560..3eaa181b2 100644 --- a/karapace/schema_reader.py +++ b/karapace/schema_reader.py @@ -326,6 +326,7 @@ def _handle_msg_schema(self, key: dict, value: Optional[dict]) -> None: schema_id = value["id"] schema_version = value["version"] schema_deleted = value.get("deleted", False) + schema_references = value.get("references", None) try: schema_type_parsed = SchemaType(schema_type) @@ -361,6 +362,9 @@ def _handle_msg_schema(self, key: dict, value: Optional[dict]) -> None: "id": schema_id, "deleted": schema_deleted, } + if schema_references: + subjects_schemas[schema_version]["references"] = schema_references + with self.id_lock: self.schemas[schema_id] = typed_schema self.global_schema_id = max(self.global_schema_id, schema_id) diff --git a/karapace/schema_registry_apis.py b/karapace/schema_registry_apis.py index 5079c4976..28bbbadbd 100644 --- a/karapace/schema_registry_apis.py +++ b/karapace/schema_registry_apis.py @@ -8,7 +8,7 @@ from karapace.karapace import KarapaceBase from karapace.master_coordinator import MasterCoordinator from karapace.rapu import HTTPRequest, JSON_CONTENT_TYPE, SERVER_NAME -from karapace.schema_models import InvalidSchema, InvalidSchemaType, ValidatedTypedSchema +from karapace.schema_models import InvalidSchema, InvalidSchemaType, ValidatedTypedSchema, References, InvalidReferences from karapace.schema_reader import KafkaSchemaReader, SchemaType, TypedSchema from karapace.utils import json_encode, KarapaceKafkaClient from typing import Any, Dict, Optional, Tuple @@ -36,6 +36,8 @@ class SchemaErrorCodes(Enum): INVALID_VERSION_ID = 42202 INVALID_COMPATIBILITY_LEVEL = 42203 INVALID_AVRO_SCHEMA = 44201 + INVALID_PROTOBUF_SCHEMA = 44202 + INVALID_REFERECES = 442203 NO_MASTER_ERROR = 50003 @@ -113,7 +115,8 @@ def _add_schema_registry_routes(self) -> None: self.route("/config", callback=self.config_get, method="GET", schema_request=True) self.route("/config", callback=self.config_set, method="PUT", schema_request=True) self.route( - "/schemas/ids//versions", callback=self.schemas_get_versions, method="GET", schema_request=True + "/schemas/ids//versions", callback=self.schemas_get_versions, method="GET", + schema_request=True ) self.route("/schemas/ids/", callback=self.schemas_get, method="GET", schema_request=True) self.route("/schemas/types", callback=self.schemas_types, method="GET", schema_request=True) @@ -143,6 +146,12 @@ def _add_schema_registry_routes(self) -> None: method="GET", schema_request=True, ) + self.route( + "/subjects//versions//referencedby", + callback=self.subject_version_referencedby_get, + method="GET", + schema_request=True, + ) self.route( "/subjects/", callback=self.subject_delete, @@ -262,13 +271,14 @@ def send_kafka_message(self, key, value): return future def send_schema_message( - self, - *, - subject: str, - schema: Optional[TypedSchema], - schema_id: int, - version: int, - deleted: bool, + self, + *, + subject: str, + schema: Optional[TypedSchema], + schema_id: int, + version: int, + deleted: bool, + references: Optional[References], ): key = '{{"subject":"{}","version":{},"magic":1,"keytype":"SCHEMA"}}'.format(subject, version) if schema: @@ -279,6 +289,8 @@ def send_schema_message( "schema": schema.schema_str, "deleted": deleted, } + if references: + valuedict["references"] = references.json() if schema.schema_type is not SchemaType.AVRO: valuedict["schemaType"] = schema.schema_type value = json_encode(valuedict) @@ -303,7 +315,8 @@ async def compatibility_check(self, content_type, *, subject, version, request): """Check for schema compatibility""" body = request.json self.log.info("Got request to check subject: %r, version_id: %r compatibility", subject, version) - old = await self.subject_version_get(content_type=content_type, subject=subject, version=version, return_dict=True) + old = await self.subject_version_get(content_type=content_type, subject=subject, version=version, + return_dict=True) self.log.info("Existing schema: %r, new_schema: %r", old["schema"], body["schema"]) try: schema_type = SchemaType(body.get("schemaType", "AVRO")) @@ -491,7 +504,8 @@ async def subjects_list(self, content_type): async def _subject_delete_local(self, content_type: str, subject: str, permanent: bool): subject_data = self._subject_get(subject, content_type, include_deleted=permanent) - if permanent and [version for version, value in subject_data["schemas"].items() if not value.get("deleted", False)]: + if permanent and [version for version, value in subject_data["schemas"].items() if + not value.get("deleted", False)]: self.r( body={ "error_code": SchemaErrorCodes.SUBJECT_NOT_SOFT_DELETED.value, @@ -510,8 +524,10 @@ async def _subject_delete_local(self, content_type: str, subject: str, permanent if permanent: for version, value in list(subject_data["schemas"].items()): schema_id = value.get("id") - self.log.info("Permanently deleting subject '%s' version %s (schema id=%s)", subject, version, schema_id) - self.send_schema_message(subject=subject, schema=None, schema_id=schema_id, version=version, deleted=True) + self.log.info("Permanently deleting subject '%s' version %s (schema id=%s)", subject, version, + schema_id) + self.send_schema_message(subject=subject, schema=None, schema_id=schema_id, version=version, + deleted=True) else: self.send_delete_subject_message(subject, latest_schema_id) self.r(version_list, content_type, status=HTTPStatus.OK) @@ -566,6 +582,9 @@ async def subject_version_get(self, content_type, *, subject, version, return_di "id": schema_id, "schema": schema.schema_str, } + references = schema_data.get("references") + if references : + ret["references"] = references if schema.schema_type is not SchemaType.AVRO: ret["schemaType"] = schema.schema_type if return_dict: @@ -637,6 +656,26 @@ async def subject_version_schema_get(self, content_type, *, subject, version): subject_data = self._subject_get(subject, content_type) max_version = max(subject_data["schemas"]) + if version == "latest": + schema_data = subject_data["schemas"][max_version] + elif int(version) <= max_version: + schema_data = subject_data["schemas"].get(int(version)) + else: + self.r( + body={ + "error_code": SchemaErrorCodes.VERSION_NOT_FOUND.value, + "message": f"Version {version} not found.", + }, + content_type=content_type, + status=HTTPStatus.NOT_FOUND, + ) + self.r(self.get_referencedby(subject, version), content_type) + + async def subject_version_referencedby_get(self, content_type, *, subject, version): + self._validate_version(content_type, version) + subject_data = self._subject_get(subject, content_type) + max_version = max(subject_data["schemas"]) + if version == "latest": schema_data = subject_data["schemas"][max_version] elif int(version) <= max_version: @@ -679,7 +718,7 @@ def _validate_schema_request_body(self, content_type, body) -> None: status=HTTPStatus.INTERNAL_SERVER_ERROR, ) for field in body: - if field not in {"schema", "schemaType"}: + if field not in {"schema", "schemaType", "references"}: self.r( body={ "error_code": SchemaErrorCodes.HTTP_UNPROCESSABLE_ENTITY.value, @@ -799,6 +838,22 @@ def write_new_schema_local(self, subject, body, content_type): content_type=content_type, status=HTTPStatus.UNPROCESSABLE_ENTITY, ) + new_schema_references = None + if body.get("references"): + b = body.get("references") + try: + new_schema_references = References(schema_type, body["references"]) + except InvalidReferences: + human_error = "Provided references is not valid" + self.r( + body={ + "error_code": SchemaErrorCodes.INVALID_REFERECES.value, + "message": f"Invalid {schema_type} references. Error: {human_error}", + }, + content_type=content_type, + status=HTTPStatus.UNPROCESSABLE_ENTITY, + ) + if subject not in self.ksr.subjects or not self.ksr.subjects.get(subject)["schemas"]: schema_id = self.ksr.get_schema_id(new_schema) version = 1 @@ -830,6 +885,7 @@ def write_new_schema_local(self, subject, body, content_type): schema_id=schema_id, version=version, deleted=False, + references=new_schema_references ) self.r({"id": schema_id}, content_type) @@ -902,6 +958,7 @@ def write_new_schema_local(self, subject, body, content_type): schema_id=schema_id, version=version, deleted=False, + references=new_schema_references, ) self.r({"id": schema_id}, content_type) @@ -929,3 +986,7 @@ def no_master_error(self, content_type): content_type=content_type, status=HTTPStatus.INTERNAL_SERVER_ERROR, ) + + def get_referencedby(self, subject, version): + + pass diff --git a/karapace/serialization.py b/karapace/serialization.py index 4e95f5b5c..9976defd0 100644 --- a/karapace/serialization.py +++ b/karapace/serialization.py @@ -4,7 +4,7 @@ from karapace.client import Client from karapace.protobuf.exception import ProtobufTypeException from karapace.protobuf.io import ProtobufDatumReader, ProtobufDatumWriter -from karapace.schema_models import InvalidSchema, SchemaType, TypedSchema, ValidatedTypedSchema +from karapace.schema_models import InvalidSchema, SchemaType, TypedSchema, ValidatedTypedSchema, References from karapace.utils import json_encode from typing import Any, Dict, Optional, Tuple from urllib.parse import quote @@ -72,9 +72,13 @@ def __init__(self, schema_registry_url: str = "http://localhost:8081", server_ca self.client = Client(server_uri=schema_registry_url, server_ca=server_ca) self.base_url = schema_registry_url - async def post_new_schema(self, subject: str, schema: ValidatedTypedSchema) -> int: + async def post_new_schema(self, subject: str, schema: ValidatedTypedSchema, references: Optional[References]) -> int: if schema.schema_type is SchemaType.PROTOBUF: - payload = {"schema": str(schema), "schemaType": schema.schema_type.value} + if references : + payload = {"schema": str(schema), "schemaType": schema.schema_type.value, + "references": references.json()} + else: + payload = {"schema": str(schema), "schemaType": schema.schema_type.value} else: payload = {"schema": json_encode(schema.to_dict()), "schemaType": schema.schema_type.value} result = await self.client.post(f"subjects/{quote(subject)}/versions", json=payload) @@ -82,7 +86,7 @@ async def post_new_schema(self, subject: str, schema: ValidatedTypedSchema) -> i raise SchemaRetrievalError(result.json()) return result.json()["id"] - async def get_latest_schema(self, subject: str) -> Tuple[int, ValidatedTypedSchema]: + async def get_latest_schema(self, subject: str) -> Tuple[int, ValidatedTypedSchema, Optional[References]]: result = await self.client.get(f"subjects/{quote(subject)}/versions/latest") if not result.ok: raise SchemaRetrievalError(result.json()) @@ -91,11 +95,18 @@ async def get_latest_schema(self, subject: str) -> Tuple[int, ValidatedTypedSche raise SchemaRetrievalError(f"Invalid result format: {json_result}") try: schema_type = SchemaType(json_result.get("schemaType", "AVRO")) - return json_result["id"], ValidatedTypedSchema.parse(schema_type, json_result["schema"]) + + if json_result["references"]: + references = References(json_result["references"]) + else: + references = None + + return json_result["id"], ValidatedTypedSchema.parse(schema_type, json_result["schema"]), references + except InvalidSchema as e: raise SchemaRetrievalError(f"Failed to parse schema string from response: {json_result}") from e - async def get_schema_for_id(self, schema_id: int) -> ValidatedTypedSchema: + async def get_schema_for_id(self, schema_id: int) -> Tuple[ValidatedTypedSchema, Optional[References]] : result = await self.client.get(f"schemas/ids/{schema_id}") if not result.ok: raise SchemaRetrievalError(result.json()["message"]) @@ -104,7 +115,13 @@ async def get_schema_for_id(self, schema_id: int) -> ValidatedTypedSchema: raise SchemaRetrievalError(f"Invalid result format: {json_result}") try: schema_type = SchemaType(json_result.get("schemaType", "AVRO")) - return ValidatedTypedSchema.parse(schema_type, json_result["schema"]) + if json_result["references"]: + references = References(json_result["references"]) + else: + references = None + + return ValidatedTypedSchema.parse(schema_type, json_result["schema"]), references + except InvalidSchema as e: raise SchemaRetrievalError(f"Failed to parse schema string from response: {json_result}") from e @@ -114,10 +131,10 @@ async def close(self): class SchemaRegistrySerializerDeserializer: def __init__( - self, - config: dict, - name_strategy: str = "topic_name", - **cfg, # pylint: disable=unused-argument + self, + config: dict, + name_strategy: str = "topic_name", + **cfg, # pylint: disable=unused-argument ) -> None: self.config = config self.state_lock = asyncio.Lock() @@ -151,7 +168,7 @@ def get_subject_name(self, topic_name: str, schema: str, subject_type: str, sche async def get_schema_for_subject(self, subject: str) -> TypedSchema: assert self.registry_client, "must not call this method after the object is closed." - schema_id, schema = await self.registry_client.get_latest_schema(subject) + schema_id, schema, references = await self.registry_client.get_latest_schema(subject) async with self.state_lock: schema_ser = schema.__str__() self.schemas_to_ids[schema_ser] = schema_id diff --git a/tests/integration/test_schema_protobuf.py b/tests/integration/test_schema_protobuf.py index 09be5f739..a57eb67e1 100644 --- a/tests/integration/test_schema_protobuf.py +++ b/tests/integration/test_schema_protobuf.py @@ -10,6 +10,7 @@ import logging import pytest +import json baseurl = "http://localhost:8081" @@ -104,3 +105,58 @@ async def test_protobuf_schema_compatibility(registry_async_client: Client, trai ) assert res.status_code == 200 assert "id" in res.json() + + +async def test_protobuf_schema_references(registry_async_client: Client) -> None: + + customer_schema = """ + |syntax = "proto3"; + |package a1; + |message Customer { + | string name = 1; + | int32 code = 2; + |} + |""" + customer_schema = trim_margin(customer_schema) + res = await registry_async_client.post( + f"subjects/customer/versions", json={"schemaType": "PROTOBUF", "schema": customer_schema} + ) + assert res.status_code == 200 + assert "id" in res.json() + + original_schema = """ + |syntax = "proto3"; + |package a1; + |import "Customer.proto"; + |message TestMessage { + | message Value { + | Customer customer = 1; + | int32 x = 2; + | } + | string test = 1; + | .a1.TestMessage.Value val = 2; + |} + |""" + + original_schema = trim_margin(original_schema) + references = [{"name": "Customer.proto", + "subject": "customer", + "version": 1}] + res = await registry_async_client.post( + f"subjects/test_schema/versions", + json={"schemaType": "PROTOBUF", + "schema": original_schema, + "references": references} + ) + assert res.status_code == 200 + assert "id" in res.json() + res = await registry_async_client.get( + f"subjects/test_schema/versions/latest", json={} ) + assert res.status_code == 200 + myjson = res.json() + assert "id" in myjson + references = [{"name": "Customer.proto", + "subject": "customer", + "version": 1}] + refs2 = json.loads(myjson["references"]) + assert not any(x != y for x, y in zip(refs2, references)) From 843a1da4a5c9538ad13060d392f85a67ce0053ad Mon Sep 17 00:00:00 2001 From: Sergiy Zaschipas Date: Tue, 19 Apr 2022 22:51:09 +0300 Subject: [PATCH 02/44] fixups --- karapace/schema_registry_apis.py | 68 +++++++++-------------- karapace/serialization.py | 28 +++++----- tests/integration/test_client.py | 4 +- tests/integration/test_schema_protobuf.py | 25 +++------ 4 files changed, 51 insertions(+), 74 deletions(-) diff --git a/karapace/schema_registry_apis.py b/karapace/schema_registry_apis.py index 28bbbadbd..4d8b2c060 100644 --- a/karapace/schema_registry_apis.py +++ b/karapace/schema_registry_apis.py @@ -8,7 +8,7 @@ from karapace.karapace import KarapaceBase from karapace.master_coordinator import MasterCoordinator from karapace.rapu import HTTPRequest, JSON_CONTENT_TYPE, SERVER_NAME -from karapace.schema_models import InvalidSchema, InvalidSchemaType, ValidatedTypedSchema, References, InvalidReferences +from karapace.schema_models import InvalidReferences, InvalidSchema, InvalidSchemaType, References, ValidatedTypedSchema from karapace.schema_reader import KafkaSchemaReader, SchemaType, TypedSchema from karapace.utils import json_encode, KarapaceKafkaClient from typing import Any, Dict, Optional, Tuple @@ -115,8 +115,7 @@ def _add_schema_registry_routes(self) -> None: self.route("/config", callback=self.config_get, method="GET", schema_request=True) self.route("/config", callback=self.config_set, method="PUT", schema_request=True) self.route( - "/schemas/ids//versions", callback=self.schemas_get_versions, method="GET", - schema_request=True + "/schemas/ids//versions", callback=self.schemas_get_versions, method="GET", schema_request=True ) self.route("/schemas/ids/", callback=self.schemas_get, method="GET", schema_request=True) self.route("/schemas/types", callback=self.schemas_types, method="GET", schema_request=True) @@ -271,14 +270,14 @@ def send_kafka_message(self, key, value): return future def send_schema_message( - self, - *, - subject: str, - schema: Optional[TypedSchema], - schema_id: int, - version: int, - deleted: bool, - references: Optional[References], + self, + *, + subject: str, + schema: Optional[TypedSchema], + schema_id: int, + version: int, + deleted: bool, + references: Optional[References], ): key = '{{"subject":"{}","version":{},"magic":1,"keytype":"SCHEMA"}}'.format(subject, version) if schema: @@ -315,8 +314,7 @@ async def compatibility_check(self, content_type, *, subject, version, request): """Check for schema compatibility""" body = request.json self.log.info("Got request to check subject: %r, version_id: %r compatibility", subject, version) - old = await self.subject_version_get(content_type=content_type, subject=subject, version=version, - return_dict=True) + old = await self.subject_version_get(content_type=content_type, subject=subject, version=version, return_dict=True) self.log.info("Existing schema: %r, new_schema: %r", old["schema"], body["schema"]) try: schema_type = SchemaType(body.get("schemaType", "AVRO")) @@ -504,8 +502,7 @@ async def subjects_list(self, content_type): async def _subject_delete_local(self, content_type: str, subject: str, permanent: bool): subject_data = self._subject_get(subject, content_type, include_deleted=permanent) - if permanent and [version for version, value in subject_data["schemas"].items() if - not value.get("deleted", False)]: + if permanent and [version for version, value in subject_data["schemas"].items() if not value.get("deleted", False)]: self.r( body={ "error_code": SchemaErrorCodes.SUBJECT_NOT_SOFT_DELETED.value, @@ -524,10 +521,10 @@ async def _subject_delete_local(self, content_type: str, subject: str, permanent if permanent: for version, value in list(subject_data["schemas"].items()): schema_id = value.get("id") - self.log.info("Permanently deleting subject '%s' version %s (schema id=%s)", subject, version, - schema_id) - self.send_schema_message(subject=subject, schema=None, schema_id=schema_id, version=version, - deleted=True) + self.log.info("Permanently deleting subject '%s' version %s (schema id=%s)", subject, version, schema_id) + self.send_schema_message( + subject=subject, schema=None, schema_id=schema_id, version=version, deleted=True, references=None + ) else: self.send_delete_subject_message(subject, latest_schema_id) self.r(version_list, content_type, status=HTTPStatus.OK) @@ -583,7 +580,7 @@ async def subject_version_get(self, content_type, *, subject, version, return_di "schema": schema.schema_str, } references = schema_data.get("references") - if references : + if references: ret["references"] = references if schema.schema_type is not SchemaType.AVRO: ret["schemaType"] = schema.schema_type @@ -633,7 +630,12 @@ async def _subject_version_delete_local(self, content_type: str, subject: str, v schema_id = subject_schema_data["id"] schema = subject_schema_data["schema"] self.send_schema_message( - subject=subject, schema=None if permanent else schema, schema_id=schema_id, version=version, deleted=True + subject=subject, + schema=None if permanent else schema, + schema_id=schema_id, + version=version, + deleted=True, + references=None, ) self.r(str(version), content_type, status=HTTPStatus.OK) @@ -669,27 +671,10 @@ async def subject_version_schema_get(self, content_type, *, subject, version): content_type=content_type, status=HTTPStatus.NOT_FOUND, ) - self.r(self.get_referencedby(subject, version), content_type) + self.r(schema_data["schema"].schema_str, content_type) async def subject_version_referencedby_get(self, content_type, *, subject, version): - self._validate_version(content_type, version) - subject_data = self._subject_get(subject, content_type) - max_version = max(subject_data["schemas"]) - - if version == "latest": - schema_data = subject_data["schemas"][max_version] - elif int(version) <= max_version: - schema_data = subject_data["schemas"].get(int(version)) - else: - self.r( - body={ - "error_code": SchemaErrorCodes.VERSION_NOT_FOUND.value, - "message": f"Version {version} not found.", - }, - content_type=content_type, - status=HTTPStatus.NOT_FOUND, - ) - self.r(schema_data["schema"].schema_str, content_type) + pass async def subject_versions_list(self, content_type, *, subject): subject_data = self._subject_get(subject, content_type) @@ -840,7 +825,6 @@ def write_new_schema_local(self, subject, body, content_type): ) new_schema_references = None if body.get("references"): - b = body.get("references") try: new_schema_references = References(schema_type, body["references"]) except InvalidReferences: @@ -885,7 +869,7 @@ def write_new_schema_local(self, subject, body, content_type): schema_id=schema_id, version=version, deleted=False, - references=new_schema_references + references=new_schema_references, ) self.r({"id": schema_id}, content_type) diff --git a/karapace/serialization.py b/karapace/serialization.py index 9976defd0..2fe4359e3 100644 --- a/karapace/serialization.py +++ b/karapace/serialization.py @@ -4,7 +4,7 @@ from karapace.client import Client from karapace.protobuf.exception import ProtobufTypeException from karapace.protobuf.io import ProtobufDatumReader, ProtobufDatumWriter -from karapace.schema_models import InvalidSchema, SchemaType, TypedSchema, ValidatedTypedSchema, References +from karapace.schema_models import InvalidSchema, References, SchemaType, TypedSchema, ValidatedTypedSchema from karapace.utils import json_encode from typing import Any, Dict, Optional, Tuple from urllib.parse import quote @@ -74,9 +74,8 @@ def __init__(self, schema_registry_url: str = "http://localhost:8081", server_ca async def post_new_schema(self, subject: str, schema: ValidatedTypedSchema, references: Optional[References]) -> int: if schema.schema_type is SchemaType.PROTOBUF: - if references : - payload = {"schema": str(schema), "schemaType": schema.schema_type.value, - "references": references.json()} + if references: + payload = {"schema": str(schema), "schemaType": schema.schema_type.value, "references": references.json()} else: payload = {"schema": str(schema), "schemaType": schema.schema_type.value} else: @@ -86,7 +85,7 @@ async def post_new_schema(self, subject: str, schema: ValidatedTypedSchema, refe raise SchemaRetrievalError(result.json()) return result.json()["id"] - async def get_latest_schema(self, subject: str) -> Tuple[int, ValidatedTypedSchema, Optional[References]]: + async def get_latest_schema(self, subject: str) -> Tuple[int, ValidatedTypedSchema, References]: result = await self.client.get(f"subjects/{quote(subject)}/versions/latest") if not result.ok: raise SchemaRetrievalError(result.json()) @@ -95,9 +94,8 @@ async def get_latest_schema(self, subject: str) -> Tuple[int, ValidatedTypedSche raise SchemaRetrievalError(f"Invalid result format: {json_result}") try: schema_type = SchemaType(json_result.get("schemaType", "AVRO")) - if json_result["references"]: - references = References(json_result["references"]) + references = References(schema_type, json_result["references"]) else: references = None @@ -106,7 +104,7 @@ async def get_latest_schema(self, subject: str) -> Tuple[int, ValidatedTypedSche except InvalidSchema as e: raise SchemaRetrievalError(f"Failed to parse schema string from response: {json_result}") from e - async def get_schema_for_id(self, schema_id: int) -> Tuple[ValidatedTypedSchema, Optional[References]] : + async def get_schema_for_id(self, schema_id: int) -> Tuple[ValidatedTypedSchema, Optional[References]]: result = await self.client.get(f"schemas/ids/{schema_id}") if not result.ok: raise SchemaRetrievalError(result.json()["message"]) @@ -116,7 +114,7 @@ async def get_schema_for_id(self, schema_id: int) -> Tuple[ValidatedTypedSchema, try: schema_type = SchemaType(json_result.get("schemaType", "AVRO")) if json_result["references"]: - references = References(json_result["references"]) + references = References(schema_type, json_result["references"]) else: references = None @@ -131,10 +129,10 @@ async def close(self): class SchemaRegistrySerializerDeserializer: def __init__( - self, - config: dict, - name_strategy: str = "topic_name", - **cfg, # pylint: disable=unused-argument + self, + config: dict, + name_strategy: str = "topic_name", + **cfg, # pylint: disable=unused-argument ) -> None: self.config = config self.state_lock = asyncio.Lock() @@ -168,6 +166,7 @@ def get_subject_name(self, topic_name: str, schema: str, subject_type: str, sche async def get_schema_for_subject(self, subject: str) -> TypedSchema: assert self.registry_client, "must not call this method after the object is closed." + # pylint: disable=unused-variable schema_id, schema, references = await self.registry_client.get_latest_schema(subject) async with self.state_lock: schema_ser = schema.__str__() @@ -184,7 +183,8 @@ async def get_id_for_schema(self, schema: str, subject: str, schema_type: Schema schema_ser = schema_typed.__str__() if schema_ser in self.schemas_to_ids: return self.schemas_to_ids[schema_ser] - schema_id = await self.registry_client.post_new_schema(subject, schema_typed) + schema_id = await self.registry_client.post_new_schema(subject, schema_typed) # pylint: disable=E1120 + async with self.state_lock: self.schemas_to_ids[schema_ser] = schema_id self.ids_to_schemas[schema_id] = schema_typed diff --git a/tests/integration/test_client.py b/tests/integration/test_client.py index 1467b5ccd..df9b7fb1e 100644 --- a/tests/integration/test_client.py +++ b/tests/integration/test_client.py @@ -9,7 +9,7 @@ async def test_remote_client(registry_async_client: Client) -> None: reg_cli = SchemaRegistryClient() reg_cli.client = registry_async_client subject = new_random_name("subject") - sc_id = await reg_cli.post_new_schema(subject, schema_avro) + sc_id = await reg_cli.post_new_schema(subject, schema_avro) # pylint: disable=E1120 assert sc_id >= 0 stored_schema = await reg_cli.get_schema_for_id(sc_id) assert stored_schema == schema_avro, f"stored schema {stored_schema.to_dict()} is not {schema_avro.to_dict()}" @@ -23,7 +23,7 @@ async def test_remote_client_tls(registry_async_client_tls: Client) -> None: reg_cli = SchemaRegistryClient() reg_cli.client = registry_async_client_tls subject = new_random_name("subject") - sc_id = await reg_cli.post_new_schema(subject, schema_avro) + sc_id = await reg_cli.post_new_schema(subject, schema_avro) # pylint: disable=E1120 assert sc_id >= 0 stored_schema = await reg_cli.get_schema_for_id(sc_id) assert stored_schema == schema_avro, f"stored schema {stored_schema.to_dict()} is not {schema_avro.to_dict()}" diff --git a/tests/integration/test_schema_protobuf.py b/tests/integration/test_schema_protobuf.py index a57eb67e1..c83ca39b9 100644 --- a/tests/integration/test_schema_protobuf.py +++ b/tests/integration/test_schema_protobuf.py @@ -8,9 +8,9 @@ from karapace.protobuf.kotlin_wrapper import trim_margin from tests.utils import create_subject_name_factory +import json import logging import pytest -import json baseurl = "http://localhost:8081" @@ -109,7 +109,7 @@ async def test_protobuf_schema_compatibility(registry_async_client: Client, trai async def test_protobuf_schema_references(registry_async_client: Client) -> None: - customer_schema = """ + customer_schema = """ |syntax = "proto3"; |package a1; |message Customer { @@ -117,13 +117,13 @@ async def test_protobuf_schema_references(registry_async_client: Client) -> None | int32 code = 2; |} |""" + customer_schema = trim_margin(customer_schema) res = await registry_async_client.post( - f"subjects/customer/versions", json={"schemaType": "PROTOBUF", "schema": customer_schema} + "subjects/customer/versions", json={"schemaType": "PROTOBUF", "schema": customer_schema} ) assert res.status_code == 200 assert "id" in res.json() - original_schema = """ |syntax = "proto3"; |package a1; @@ -139,24 +139,17 @@ async def test_protobuf_schema_references(registry_async_client: Client) -> None |""" original_schema = trim_margin(original_schema) - references = [{"name": "Customer.proto", - "subject": "customer", - "version": 1}] + references = [{"name": "Customer.proto", "subject": "customer", "version": 1}] res = await registry_async_client.post( - f"subjects/test_schema/versions", - json={"schemaType": "PROTOBUF", - "schema": original_schema, - "references": references} + "subjects/test_schema/versions", + json={"schemaType": "PROTOBUF", "schema": original_schema, "references": references}, ) assert res.status_code == 200 assert "id" in res.json() - res = await registry_async_client.get( - f"subjects/test_schema/versions/latest", json={} ) + res = await registry_async_client.get("subjects/test_schema/versions/latest", json={}) assert res.status_code == 200 myjson = res.json() assert "id" in myjson - references = [{"name": "Customer.proto", - "subject": "customer", - "version": 1}] + references = [{"name": "Customer.proto", "subject": "customer", "version": 1}] refs2 = json.loads(myjson["references"]) assert not any(x != y for x, y in zip(refs2, references)) From 44f853a4d04d423c0f91f0cd3acaee747c51982f Mon Sep 17 00:00:00 2001 From: Sergiy Zaschipas Date: Tue, 19 Apr 2022 23:41:07 +0300 Subject: [PATCH 03/44] fixup --- tests/integration/test_client_protobuf.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/integration/test_client_protobuf.py b/tests/integration/test_client_protobuf.py index 476dbd2de..6a1e186d4 100644 --- a/tests/integration/test_client_protobuf.py +++ b/tests/integration/test_client_protobuf.py @@ -10,11 +10,11 @@ async def test_remote_client_protobuf(registry_async_client): reg_cli = SchemaRegistryClient() reg_cli.client = registry_async_client subject = new_random_name("subject") - sc_id = await reg_cli.post_new_schema(subject, schema_protobuf) + sc_id = await reg_cli.post_new_schema(subject, schema_protobuf, None) assert sc_id >= 0 stored_schema = await reg_cli.get_schema_for_id(sc_id) assert stored_schema == schema_protobuf, f"stored schema {stored_schema} is not {schema_protobuf}" - stored_id, stored_schema = await reg_cli.get_latest_schema(subject) + stored_id, stored_schema, references = await reg_cli.get_latest_schema(subject) # pylint: disable=unused-variable assert stored_id == sc_id assert stored_schema == schema_protobuf @@ -25,10 +25,10 @@ async def test_remote_client_protobuf2(registry_async_client): reg_cli = SchemaRegistryClient() reg_cli.client = registry_async_client subject = new_random_name("subject") - sc_id = await reg_cli.post_new_schema(subject, schema_protobuf) + sc_id = await reg_cli.post_new_schema(subject, schema_protobuf, None) assert sc_id >= 0 stored_schema = await reg_cli.get_schema_for_id(sc_id) assert stored_schema == schema_protobuf, f"stored schema {stored_schema} is not {schema_protobuf}" - stored_id, stored_schema = await reg_cli.get_latest_schema(subject) + stored_id, stored_schema, references = await reg_cli.get_latest_schema(subject) # pylint: disable=unused-variable assert stored_id == sc_id assert stored_schema == schema_protobuf_after From 3498962c7386fd8f072cc0964e4e15b3458a57e9 Mon Sep 17 00:00:00 2001 From: Sergiy Zaschipas Date: Tue, 19 Apr 2022 23:47:31 +0300 Subject: [PATCH 04/44] fixup --- karapace/schema_models.py | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/karapace/schema_models.py b/karapace/schema_models.py index 6e6a41bae..b198bda41 100644 --- a/karapace/schema_models.py +++ b/karapace/schema_models.py @@ -13,7 +13,7 @@ ) from karapace.protobuf.schema import ProtobufSchema from karapace.utils import json_encode -from typing import Any, Dict, Union, List +from typing import Any, Dict, List, Union import json import logging @@ -109,13 +109,11 @@ def __repr__(self) -> str: return f"TypedSchema(type={self.schema_type}, schema={json_encode(self.to_dict())})" def __eq__(self, other: Any) -> bool: - return isinstance(other, - TypedSchema) and self.__str__() == other.__str__() and self.schema_type is other.schema_type + return isinstance(other, TypedSchema) and self.__str__() == other.__str__() and self.schema_type is other.schema_type class ValidatedTypedSchema(TypedSchema): - def __init__(self, schema_type: SchemaType, schema_str: str, - schema: Union[Draft7Validator, AvroSchema, ProtobufSchema]): + def __init__(self, schema_type: SchemaType, schema_str: str, schema: Union[Draft7Validator, AvroSchema, ProtobufSchema]): super().__init__(schema_type=schema_type, schema_str=schema_str) self.schema = schema @@ -142,15 +140,15 @@ def parse(schema_type: SchemaType, schema_str: str) -> "ValidatedTypedSchema": try: parsed_schema = parse_protobuf_schema_definition(schema_str) except ( - TypeError, - SchemaError, - AssertionError, - ProtobufParserRuntimeException, - IllegalStateException, - IllegalArgumentException, - ProtobufError, - ProtobufException, - ProtobufSchemaParseException, + TypeError, + SchemaError, + AssertionError, + ProtobufParserRuntimeException, + IllegalStateException, + IllegalArgumentException, + ProtobufError, + ProtobufException, + ProtobufSchemaParseException, ) as e: log.exception("Unexpected error: %s \n schema:[%s]", e, schema_str) raise InvalidSchema from e From 9e9b16dde4ecf0377bb776e6197c572ecaf897b9 Mon Sep 17 00:00:00 2001 From: Sergiy Zaschipas Date: Wed, 20 Apr 2022 09:43:54 +0300 Subject: [PATCH 05/44] fixup --- .gitignore | 1 + karapace/serialization.py | 8 ++------ tests/integration/test_client_protobuf.py | 2 +- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index 67b386bbf..4a162fa59 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,4 @@ __pycache__/ /kafka_*/ venv /karapace/version.py +.run diff --git a/karapace/serialization.py b/karapace/serialization.py index 2fe4359e3..1716690d7 100644 --- a/karapace/serialization.py +++ b/karapace/serialization.py @@ -85,7 +85,7 @@ async def post_new_schema(self, subject: str, schema: ValidatedTypedSchema, refe raise SchemaRetrievalError(result.json()) return result.json()["id"] - async def get_latest_schema(self, subject: str) -> Tuple[int, ValidatedTypedSchema, References]: + async def get_latest_schema(self, subject: str) -> Tuple[int, ValidatedTypedSchema]: result = await self.client.get(f"subjects/{quote(subject)}/versions/latest") if not result.ok: raise SchemaRetrievalError(result.json()) @@ -94,12 +94,8 @@ async def get_latest_schema(self, subject: str) -> Tuple[int, ValidatedTypedSche raise SchemaRetrievalError(f"Invalid result format: {json_result}") try: schema_type = SchemaType(json_result.get("schemaType", "AVRO")) - if json_result["references"]: - references = References(schema_type, json_result["references"]) - else: - references = None - return json_result["id"], ValidatedTypedSchema.parse(schema_type, json_result["schema"]), references + return json_result["id"], ValidatedTypedSchema.parse(schema_type, json_result["schema"]) except InvalidSchema as e: raise SchemaRetrievalError(f"Failed to parse schema string from response: {json_result}") from e diff --git a/tests/integration/test_client_protobuf.py b/tests/integration/test_client_protobuf.py index 6a1e186d4..d73e7b9c8 100644 --- a/tests/integration/test_client_protobuf.py +++ b/tests/integration/test_client_protobuf.py @@ -29,6 +29,6 @@ async def test_remote_client_protobuf2(registry_async_client): assert sc_id >= 0 stored_schema = await reg_cli.get_schema_for_id(sc_id) assert stored_schema == schema_protobuf, f"stored schema {stored_schema} is not {schema_protobuf}" - stored_id, stored_schema, references = await reg_cli.get_latest_schema(subject) # pylint: disable=unused-variable + stored_id, stored_schema = await reg_cli.get_latest_schema(subject) # pylint: disable=unused-variable assert stored_id == sc_id assert stored_schema == schema_protobuf_after From 944cd3953fa1733173d759d7bc606f835d71a5fe Mon Sep 17 00:00:00 2001 From: Sergiy Zaschipas Date: Wed, 20 Apr 2022 09:51:42 +0300 Subject: [PATCH 06/44] fixup --- karapace/serialization.py | 2 +- tests/integration/test_client_protobuf.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/karapace/serialization.py b/karapace/serialization.py index 1716690d7..90f2b8200 100644 --- a/karapace/serialization.py +++ b/karapace/serialization.py @@ -163,7 +163,7 @@ def get_subject_name(self, topic_name: str, schema: str, subject_type: str, sche async def get_schema_for_subject(self, subject: str) -> TypedSchema: assert self.registry_client, "must not call this method after the object is closed." # pylint: disable=unused-variable - schema_id, schema, references = await self.registry_client.get_latest_schema(subject) + schema_id, schema = await self.registry_client.get_latest_schema(subject) async with self.state_lock: schema_ser = schema.__str__() self.schemas_to_ids[schema_ser] = schema_id diff --git a/tests/integration/test_client_protobuf.py b/tests/integration/test_client_protobuf.py index d73e7b9c8..8d5f43b9c 100644 --- a/tests/integration/test_client_protobuf.py +++ b/tests/integration/test_client_protobuf.py @@ -14,7 +14,7 @@ async def test_remote_client_protobuf(registry_async_client): assert sc_id >= 0 stored_schema = await reg_cli.get_schema_for_id(sc_id) assert stored_schema == schema_protobuf, f"stored schema {stored_schema} is not {schema_protobuf}" - stored_id, stored_schema, references = await reg_cli.get_latest_schema(subject) # pylint: disable=unused-variable + stored_id, stored_schema = await reg_cli.get_latest_schema(subject) # pylint: disable=unused-variable assert stored_id == sc_id assert stored_schema == schema_protobuf From 956b68ef3e2cba8b9287b19722453fbab51fbafe Mon Sep 17 00:00:00 2001 From: Sergiy Zaschipas Date: Tue, 26 Apr 2022 21:35:45 +0300 Subject: [PATCH 07/44] fixup --- karapace/serialization.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/karapace/serialization.py b/karapace/serialization.py index 90f2b8200..df7d3fb53 100644 --- a/karapace/serialization.py +++ b/karapace/serialization.py @@ -72,7 +72,9 @@ def __init__(self, schema_registry_url: str = "http://localhost:8081", server_ca self.client = Client(server_uri=schema_registry_url, server_ca=server_ca) self.base_url = schema_registry_url - async def post_new_schema(self, subject: str, schema: ValidatedTypedSchema, references: Optional[References]) -> int: + async def post_new_schema( + self, subject: str, schema: ValidatedTypedSchema, references: Optional[References] = None + ) -> int: if schema.schema_type is SchemaType.PROTOBUF: if references: payload = {"schema": str(schema), "schemaType": schema.schema_type.value, "references": references.json()} @@ -109,8 +111,9 @@ async def get_schema_for_id(self, schema_id: int) -> Tuple[ValidatedTypedSchema, raise SchemaRetrievalError(f"Invalid result format: {json_result}") try: schema_type = SchemaType(json_result.get("schemaType", "AVRO")) - if json_result["references"]: - references = References(schema_type, json_result["references"]) + references_str = json_result.get("references") + if references_str: + references = References(schema_type, references_str) else: references = None From 764facfb763105eeea261b1ff57efa974bc6ece8 Mon Sep 17 00:00:00 2001 From: Sergiy Zaschipas Date: Mon, 2 May 2022 01:25:09 +0300 Subject: [PATCH 08/44] debugging --- karapace/schema_models.py | 53 +++++++++++++++++++---- karapace/schema_reader.py | 8 ++-- karapace/schema_registry_apis.py | 34 ++++++++------- karapace/serialization.py | 5 ++- tests/integration/test_schema_protobuf.py | 3 +- 5 files changed, 69 insertions(+), 34 deletions(-) diff --git a/karapace/schema_models.py b/karapace/schema_models.py index 07d09bae9..24bbb1718 100644 --- a/karapace/schema_models.py +++ b/karapace/schema_models.py @@ -12,8 +12,9 @@ SchemaParseException as ProtobufSchemaParseException, ) from karapace.protobuf.schema import ProtobufSchema +from karapace.typing import JsonData from karapace.utils import json_encode -from typing import Any, Dict, List, Union +from typing import Any, Dict, Optional, Union import json import ujson @@ -80,7 +81,7 @@ class SchemaType(str, Enum): class TypedSchema: - def __init__(self, schema_type: SchemaType, schema_str: str): + def __init__(self, schema_type: SchemaType, schema_str: str, references: Optional["References"] = None): """Schema with type information Args: @@ -89,6 +90,7 @@ def __init__(self, schema_type: SchemaType, schema_str: str): """ self.schema_type = schema_type self.schema_str = schema_str + self.references = references def to_dict(self) -> Dict[str, Any]: if self.schema_type is SchemaType.PROTOBUF: @@ -105,17 +107,42 @@ def __repr__(self) -> str: return f"TypedSchema(type={self.schema_type}, schema={str(self)})" return f"TypedSchema(type={self.schema_type}, schema={json_encode(self.to_dict())})" + def get_references(self) -> Optional["References"]: + return self.references + def __eq__(self, other: Any) -> bool: - return isinstance(other, TypedSchema) and self.__str__() == other.__str__() and self.schema_type is other.schema_type + a = isinstance(other, TypedSchema) and self.schema_type is other.schema_type and self.__str__() == other.__str__() + + if not a: + return False + x = self.get_references() + y = other.get_references() + if x: + if y: + if x == y: + return True + else: + return False + else: + if y: + return False + + return True class ValidatedTypedSchema(TypedSchema): - def __init__(self, schema_type: SchemaType, schema_str: str, schema: Union[Draft7Validator, AvroSchema, ProtobufSchema]): - super().__init__(schema_type=schema_type, schema_str=schema_str) + def __init__( + self, + schema_type: SchemaType, + schema_str: str, + schema: Union[Draft7Validator, AvroSchema, ProtobufSchema], + references: Optional["References"] = None, + ): + super().__init__(schema_type=schema_type, schema_str=schema_str, references=references) self.schema = schema @staticmethod - def parse(schema_type: SchemaType, schema_str: str) -> "ValidatedTypedSchema": + def parse(schema_type: SchemaType, schema_str: str, references: Optional["References"] = None) -> "ValidatedTypedSchema": if schema_type not in [SchemaType.AVRO, SchemaType.JSONSCHEMA, SchemaType.PROTOBUF]: raise InvalidSchema(f"Unknown parser {schema_type} for {schema_str}") @@ -151,7 +178,9 @@ def parse(schema_type: SchemaType, schema_str: str) -> "ValidatedTypedSchema": else: raise InvalidSchema(f"Unknown parser {schema_type} for {schema_str}") - return ValidatedTypedSchema(schema_type=schema_type, schema_str=schema_str, schema=parsed_schema) + return ValidatedTypedSchema( + schema_type=schema_type, schema_str=schema_str, schema=parsed_schema, references=references + ) def __str__(self) -> str: if self.schema_type == SchemaType.PROTOBUF: @@ -160,7 +189,7 @@ def __str__(self) -> str: class References: - def __init__(self, schema_type: SchemaType, references: List): + def __init__(self, schema_type: SchemaType, references: JsonData): """Schema with type information Args: @@ -170,5 +199,11 @@ def __init__(self, schema_type: SchemaType, references: List): self.schema_type = schema_type self.references = references + def val(self) -> JsonData: + return self.references + def json(self) -> str: - return json_encode(self.references) + return str(json_encode(self.references, sort_keys=True)) + + def __eq__(self, other: Any) -> bool: + return self.json() == other.json() diff --git a/karapace/schema_reader.py b/karapace/schema_reader.py index b76cd172e..11e78778d 100644 --- a/karapace/schema_reader.py +++ b/karapace/schema_reader.py @@ -348,10 +348,7 @@ def _handle_msg_schema(self, key: dict, value: Optional[dict]) -> None: subjects_schemas = self.subjects[schema_subject]["schemas"] - typed_schema = TypedSchema( - schema_type=schema_type_parsed, - schema_str=schema_str, - ) + typed_schema = TypedSchema(schema_type=schema_type_parsed, schema_str=schema_str, references=schema_references) schema = { "schema": typed_schema, "version": schema_version, @@ -359,7 +356,8 @@ def _handle_msg_schema(self, key: dict, value: Optional[dict]) -> None: "deleted": schema_deleted, } if schema_references: - subjects_schemas[schema_version]["references"] = schema_references + schema["references"] = schema_references + if schema_version in subjects_schemas: LOG.info("Updating entry subject: %r version: %r id: %r", schema_subject, schema_version, schema_id) else: diff --git a/karapace/schema_registry_apis.py b/karapace/schema_registry_apis.py index 42864b326..c75672dbd 100644 --- a/karapace/schema_registry_apis.py +++ b/karapace/schema_registry_apis.py @@ -283,7 +283,7 @@ def send_schema_message( "deleted": deleted, } if references: - valuedict["references"] = references.json() + valuedict["references"] = references.val() if schema.schema_type is not SchemaType.AVRO: valuedict["schemaType"] = schema.schema_type value = json_encode(valuedict) @@ -804,8 +804,24 @@ def write_new_schema_local( ) -> NoReturn: """Since we're the master we get to write the new schema""" self.log.info("Writing new schema locally since we're the master") + new_schema_references = None + if body.get("references"): + try: + new_schema_references = References(schema_type, body["references"]) + except InvalidReferences: + human_error = "Provided references is not valid" + self.r( + body={ + "error_code": SchemaErrorCodes.INVALID_REFERECES.value, + "message": f"Invalid {schema_type} references. Error: {human_error}", + }, + content_type=content_type, + status=HTTPStatus.UNPROCESSABLE_ENTITY, + ) try: - new_schema = ValidatedTypedSchema.parse(schema_type=schema_type, schema_str=body["schema"]) + new_schema = ValidatedTypedSchema.parse( + schema_type=schema_type, schema_str=body["schema"], references=new_schema_references + ) except (InvalidSchema, InvalidSchemaType) as e: self.log.warning("Invalid schema: %r", body["schema"], exc_info=True) if isinstance(e.__cause__, (SchemaParseException, ValueError)): @@ -820,20 +836,6 @@ def write_new_schema_local( content_type=content_type, status=HTTPStatus.UNPROCESSABLE_ENTITY, ) - new_schema_references = None - if body.get("references"): - try: - new_schema_references = References(schema_type, body["references"]) - except InvalidReferences: - human_error = "Provided references is not valid" - self.r( - body={ - "error_code": SchemaErrorCodes.INVALID_REFERECES.value, - "message": f"Invalid {schema_type} references. Error: {human_error}", - }, - content_type=content_type, - status=HTTPStatus.UNPROCESSABLE_ENTITY, - ) if subject not in self.ksr.subjects or not self.ksr.subjects.get(subject)["schemas"]: schema_id = self.ksr.get_schema_id(new_schema) diff --git a/karapace/serialization.py b/karapace/serialization.py index 3fbd839f1..22b5b8b7c 100644 --- a/karapace/serialization.py +++ b/karapace/serialization.py @@ -113,8 +113,9 @@ async def get_schema_for_id(self, schema_id: int) -> Tuple[ValidatedTypedSchema, references = References(schema_type, references_str) else: references = None - - return ValidatedTypedSchema.parse(schema_type, json_result["schema"]), references + if references: + return ValidatedTypedSchema.parse(schema_type, json_result["schema"]), references + return ValidatedTypedSchema.parse(schema_type, json_result["schema"]) except InvalidSchema as e: raise SchemaRetrievalError(f"Failed to parse schema string from response: {json_result}") from e diff --git a/tests/integration/test_schema_protobuf.py b/tests/integration/test_schema_protobuf.py index c83ca39b9..f5e7fafa1 100644 --- a/tests/integration/test_schema_protobuf.py +++ b/tests/integration/test_schema_protobuf.py @@ -8,7 +8,6 @@ from karapace.protobuf.kotlin_wrapper import trim_margin from tests.utils import create_subject_name_factory -import json import logging import pytest @@ -151,5 +150,5 @@ async def test_protobuf_schema_references(registry_async_client: Client) -> None myjson = res.json() assert "id" in myjson references = [{"name": "Customer.proto", "subject": "customer", "version": 1}] - refs2 = json.loads(myjson["references"]) + refs2 = myjson["references"] assert not any(x != y for x, y in zip(refs2, references)) From c1c4c50d43cfaefb80d65bbdef94560b1c9bcb20 Mon Sep 17 00:00:00 2001 From: Sergiy Zaschipas Date: Wed, 11 May 2022 09:56:53 +0300 Subject: [PATCH 09/44] referencedby/delete workarounds --- karapace/schema_reader.py | 17 ++++- karapace/schema_registry_apis.py | 86 ++++++++++++++++++++--- tests/__init__.py | 0 tests/integration/test_schema_protobuf.py | 12 ++++ 4 files changed, 105 insertions(+), 10 deletions(-) create mode 100644 tests/__init__.py diff --git a/karapace/schema_reader.py b/karapace/schema_reader.py index 11e78778d..ad283ab0b 100644 --- a/karapace/schema_reader.py +++ b/karapace/schema_reader.py @@ -15,7 +15,7 @@ from karapace.statsd import StatsClient from karapace.utils import KarapaceKafkaClient from threading import Event, Lock, Thread -from typing import Any, Dict, Optional +from typing import Any, Dict, List, Optional import logging import ujson @@ -26,6 +26,7 @@ Schema = Dict[str, Any] # Container type for a subject, with configuration settings and all the schemas SubjectData = Dict[str, Any] +Referents = List # The value `0` is a valid offset and it represents the first message produced # to a topic, therefore it can not be used. @@ -139,6 +140,7 @@ def __init__( self.config = config self.subjects: Dict[Subject, SubjectData] = {} self.schemas: Dict[int, TypedSchema] = {} + self.referenced_by: Dict[str, Referents] = {} self.global_schema_id = 0 self.admin_client: Optional[KafkaAdminClient] = None self.topic_replication_factor = self.config["replication_factor"] @@ -284,6 +286,7 @@ def _handle_msg_config(self, key: dict, value: Optional[dict]) -> None: else: LOG.info("Setting subject: %r config to: %r, value: %r", subject, value["compatibilityLevel"], value) self.subjects[subject]["compatibility"] = value["compatibilityLevel"] + elif value is not None: LOG.info("Setting global config to: %r, value: %r", value["compatibilityLevel"], value) self.config["compatibility"] = value["compatibilityLevel"] @@ -326,8 +329,8 @@ def _handle_msg_schema(self, key: dict, value: Optional[dict]) -> None: schema_id = value["id"] schema_version = value["version"] schema_deleted = value.get("deleted", False) - schema_references = value.get("references", None) + schema_references = value.get("references", None) try: schema_type_parsed = SchemaType(schema_type) except ValueError: @@ -364,6 +367,16 @@ def _handle_msg_schema(self, key: dict, value: Optional[dict]) -> None: LOG.info("Adding entry subject: %r version: %r id: %r", schema_subject, schema_version, schema_id) subjects_schemas[schema_version] = schema + if schema_references: + for ref in schema_references: + ref_str = str(ref["subject"]) + "_" + str(ref["version"]) + referents = self.referenced_by.get(ref_str, None) + if referents: + LOG.info("Adding entry subject referenced_by : %r", ref_str) + referents.append(schema_id) + else: + LOG.info("Adding entry subject referenced_by : %r", ref_str) + self.referenced_by[ref_str] = [schema_id] with self.id_lock: self.schemas[schema_id] = typed_schema diff --git a/karapace/schema_registry_apis.py b/karapace/schema_registry_apis.py index c75672dbd..7d59cc7ff 100644 --- a/karapace/schema_registry_apis.py +++ b/karapace/schema_registry_apis.py @@ -24,6 +24,7 @@ @unique class SchemaErrorCodes(Enum): EMPTY_SCHEMA = 42201 + REFERENCE_EXISTS_ERROR_CODE = 42206 HTTP_NOT_FOUND = HTTPStatus.NOT_FOUND.value HTTP_CONFLICT = HTTPStatus.CONFLICT.value HTTP_UNPROCESSABLE_ENTITY = HTTPStatus.UNPROCESSABLE_ENTITY.value @@ -39,7 +40,9 @@ class SchemaErrorCodes(Enum): INVALID_COMPATIBILITY_LEVEL = 42203 INVALID_AVRO_SCHEMA = 44201 INVALID_PROTOBUF_SCHEMA = 44202 - INVALID_REFERECES = 442203 + INVALID_REFERECES = 44203 + REFERENCES_SUPPORT_NOT_IMPLEMENTED = 44501 + SCHEMAVERSION_HAS_REFERENCES = 44503 NO_MASTER_ERROR = 50003 @@ -51,6 +54,7 @@ class SchemaErrorMessages(Enum): "forward, full, backward_transitive, forward_transitive, and " "full_transitive" ) + REFERENCES_SUPPORT_NOT_IMPLEMENTED = "Schema references are not supported for '{schema_type}' schema type yet" class KarapaceSchemaRegistry(KarapaceBase): @@ -613,6 +617,20 @@ async def _subject_version_delete_local(self, content_type: str, subject: str, v status=HTTPStatus.NOT_FOUND, ) + referenced_by = self.ksr.referenced_by.get(str(subject) + "_" + str(version), None) + if referenced_by and len(referenced_by) > 0: + self.r( + body={ + "error_code": SchemaErrorCodes.SCHEMAVERSION_HAS_REFERENCES.value, + "message": ( + f"Subject '{subject}' Version {version} was not deleted " + "because it is referenced by schemas with ids:[" + ", ".join(map(str, referenced_by)) + "]" + ), + }, + content_type=content_type, + status=HTTPStatus.NOT_FOUND, + ) + schema_id = subject_schema_data["id"] schema = subject_schema_data["schema"] self.send_schema_message( @@ -660,7 +678,50 @@ async def subject_version_schema_get(self, content_type, *, subject, version): self.r(schema_data["schema"].schema_str, content_type) async def subject_version_referencedby_get(self, content_type, *, subject, version): - pass + self._validate_version(content_type, version) + subject_data = self._subject_get(subject, content_type) + schema_data = None + max_version = max(subject_data["schemas"]) + if version == "latest": + version = max(subject_data["schemas"]) + schema_data = subject_data["schemas"][version] + elif int(version) <= max_version: + schema_data = subject_data["schemas"].get(int(version)) + else: + self.r( + body={ + "error_code": SchemaErrorCodes.VERSION_NOT_FOUND.value, + "message": f"Version {version} not found.", + }, + content_type=content_type, + status=HTTPStatus.NOT_FOUND, + ) + if not schema_data: + self.r( + body={ + "error_code": SchemaErrorCodes.VERSION_NOT_FOUND.value, + "message": f"Version {version} not found.", + }, + content_type=content_type, + status=HTTPStatus.NOT_FOUND, + ) + + if schema_data["schema"].schema_type != SchemaType.PROTOBUF: + self.r( + body={ + "error_code": SchemaErrorCodes.REFERENCES_SUPPORT_NOT_IMPLEMENTED.value, + "message": SchemaErrorMessages.REFERENCES_SUPPORT_NOT_IMPLEMENTED.value.format( + schema_type=schema_data["schema"].schema_type + ), + }, + content_type=content_type, + status=HTTPStatus.UNPROCESSABLE_ENTITY, + ) + + referenced_by = self.ksr.referenced_by.get(str(subject) + "_" + str(version), None) + if not referenced_by: + referenced_by = list() + self.r(list(referenced_by), content_type, status=HTTPStatus.OK) async def subject_versions_list(self, content_type, *, subject): subject_data = self._subject_get(subject, content_type) @@ -805,9 +866,22 @@ def write_new_schema_local( """Since we're the master we get to write the new schema""" self.log.info("Writing new schema locally since we're the master") new_schema_references = None - if body.get("references"): + references = body.get("references") + if references: + if schema_type != SchemaType.PROTOBUF: + self.r( + body={ + "error_code": SchemaErrorCodes.REFERENCES_SUPPORT_NOT_IMPLEMENTED.value, + "message": SchemaErrorMessages.REFERENCES_SUPPORT_NOT_IMPLEMENTED.value.format( + schema_type=schema_type.value + ), + }, + content_type=content_type, + status=HTTPStatus.UNPROCESSABLE_ENTITY, + ) + try: - new_schema_references = References(schema_type, body["references"]) + new_schema_references = References(schema_type, references) except InvalidReferences: human_error = "Provided references is not valid" self.r( @@ -969,7 +1043,3 @@ def no_master_error(self, content_type): content_type=content_type, status=HTTPStatus.INTERNAL_SERVER_ERROR, ) - - def get_referencedby(self, subject, version): - - pass diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integration/test_schema_protobuf.py b/tests/integration/test_schema_protobuf.py index f5e7fafa1..1edff88fc 100644 --- a/tests/integration/test_schema_protobuf.py +++ b/tests/integration/test_schema_protobuf.py @@ -152,3 +152,15 @@ async def test_protobuf_schema_references(registry_async_client: Client) -> None references = [{"name": "Customer.proto", "subject": "customer", "version": 1}] refs2 = myjson["references"] assert not any(x != y for x, y in zip(refs2, references)) + res = await registry_async_client.get("subjects/customer/versions/latest/referencedby", json={}) + assert res.status_code == 200 + myjson = res.json() + referents = [2] + assert not any(x != y for x, y in zip(myjson, referents)) +# res = await registry_async_client.delete("subjects/customer/versions/latest") +# assert res.status_code == 200 + + +#TODO +#AVRO references error +#JSONSCHEMA references error \ No newline at end of file From 732a9ed4d7b67dd1944ed7c12460e3ba9d8cdb9e Mon Sep 17 00:00:00 2001 From: Sergiy Zaschipas Date: Wed, 11 May 2022 10:12:54 +0300 Subject: [PATCH 10/44] referencedby/delete workarounds --- tests/integration/test_schema_protobuf.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_schema_protobuf.py b/tests/integration/test_schema_protobuf.py index 1edff88fc..e37eac84a 100644 --- a/tests/integration/test_schema_protobuf.py +++ b/tests/integration/test_schema_protobuf.py @@ -157,10 +157,12 @@ async def test_protobuf_schema_references(registry_async_client: Client) -> None myjson = res.json() referents = [2] assert not any(x != y for x, y in zip(myjson, referents)) + + # res = await registry_async_client.delete("subjects/customer/versions/latest") # assert res.status_code == 200 -#TODO -#AVRO references error -#JSONSCHEMA references error \ No newline at end of file +# TODO +# AVRO references error +# JSONSCHEMA references error From 1f33b7dae5c516680554eb18e7cc1028f5851dad Mon Sep 17 00:00:00 2001 From: Sergiy Zaschipas Date: Tue, 17 May 2022 00:38:39 +0300 Subject: [PATCH 11/44] Add basic support of references with basic tests --- karapace/schema_registry_apis.py | 66 +++++++++++++++++++++-- tests/integration/test_schema.py | 11 ++++ tests/integration/test_schema_protobuf.py | 20 ++----- 3 files changed, 78 insertions(+), 19 deletions(-) diff --git a/karapace/schema_registry_apis.py b/karapace/schema_registry_apis.py index 7d59cc7ff..8fe81b106 100644 --- a/karapace/schema_registry_apis.py +++ b/karapace/schema_registry_apis.py @@ -509,6 +509,21 @@ async def _subject_delete_local(self, content_type: str, subject: str, permanent latest_schema_id = 0 if permanent: + for version, value in list(subject_data["schemas"].items()): + referenced_by = self.ksr.referenced_by.get(str(subject) + "_" + str(version), None) + if referenced_by and len(referenced_by) > 0: + self.r( + body={ + "error_code": SchemaErrorCodes.SCHEMAVERSION_HAS_REFERENCES.value, + "message": ( + f"Subject '{subject}' Version {version} cannot be not be deleted " + "because it is referenced by schemas with ids:[" + ", ".join(map(str, referenced_by)) + "]" + ), + }, + content_type=content_type, + status=HTTPStatus.NOT_FOUND, + ) + for version, value in list(subject_data["schemas"].items()): schema_id = value.get("id") self.log.info("Permanently deleting subject '%s' version %s (schema id=%s)", subject, version, schema_id) @@ -516,6 +531,19 @@ async def _subject_delete_local(self, content_type: str, subject: str, permanent subject=subject, schema=None, schema_id=schema_id, version=version, deleted=True, references=None ) else: + referenced_by = self.ksr.referenced_by.get(str(subject) + "_" + str(latest_schema_id), None) + if referenced_by and len(referenced_by) > 0: + self.r( + body={ + "error_code": SchemaErrorCodes.SCHEMAVERSION_HAS_REFERENCES.value, + "message": ( + f"Subject '{subject}' Version {latest_schema_id} cannot be not be deleted " + "because it is referenced by schemas with ids:[" + ", ".join(map(str, referenced_by)) + "]" + ), + }, + content_type=content_type, + status=HTTPStatus.NOT_FOUND, + ) self.send_delete_subject_message(subject, latest_schema_id) self.r(version_list, content_type, status=HTTPStatus.OK) @@ -569,9 +597,6 @@ async def subject_version_get(self, content_type, *, subject, version, return_di "id": schema_id, "schema": schema.schema_str, } - references = schema_data.get("references") - if references: - ret["references"] = references if schema.schema_type is not SchemaType.AVRO: ret["schemaType"] = schema.schema_type if return_dict: @@ -802,8 +827,39 @@ async def subjects_schema_post(self, content_type, *, subject, request): ) schema_str = body["schema"] schema_type = self._validate_schema_type(content_type=content_type, data=body) + + new_schema_references = None + references = body.get("references") + if references: + if schema_type != SchemaType.PROTOBUF: + self.r( + body={ + "error_code": SchemaErrorCodes.REFERENCES_SUPPORT_NOT_IMPLEMENTED.value, + "message": SchemaErrorMessages.REFERENCES_SUPPORT_NOT_IMPLEMENTED.value.format( + schema_type=schema_type.value + ), + }, + content_type=content_type, + status=HTTPStatus.UNPROCESSABLE_ENTITY, + ) + + try: + new_schema_references = References(schema_type, references) + except InvalidReferences: + human_error = "Provided references is not valid" + self.r( + body={ + "error_code": SchemaErrorCodes.INVALID_REFERECES.value, + "message": f"Invalid {schema_type} references. Error: {human_error}", + }, + content_type=content_type, + status=HTTPStatus.UNPROCESSABLE_ENTITY, + ) + try: - new_schema = ValidatedTypedSchema.parse(schema_type, schema_str) + new_schema = ValidatedTypedSchema.parse( + schema_type=schema_type, schema_str=schema_str, references=new_schema_references + ) except InvalidSchema: self.log.exception("No proper parser found") self.r( @@ -814,11 +870,13 @@ async def subjects_schema_post(self, content_type, *, subject, request): content_type=content_type, status=HTTPStatus.INTERNAL_SERVER_ERROR, ) + for schema in subject_data["schemas"].values(): validated_typed_schema = ValidatedTypedSchema.parse(schema["schema"].schema_type, schema["schema"].schema_str) if ( validated_typed_schema.schema_type == new_schema.schema_type and validated_typed_schema.schema == new_schema.schema + and schema.get("references", None) == new_schema_references ): ret = { "subject": subject, diff --git a/tests/integration/test_schema.py b/tests/integration/test_schema.py index 9f816500f..a6c6764e1 100644 --- a/tests/integration/test_schema.py +++ b/tests/integration/test_schema.py @@ -1240,6 +1240,17 @@ async def test_schema_subject_post_invalid(registry_async_client: Client) -> Non assert res.json()["error_code"] == 40401 assert res.json()["message"] == f"Subject '{subject_3}' not found." + schema_str = ujson.dumps({"type": "string"}) + # Create the subject + subject_1 = subject_name_factory() + res = await registry_async_client.post( + f"subjects/{subject_1}/versions", + json={"schema": schema_str, "references": [{"name": "Customer.avro", "subject": "customer", "version": 1}]}, + ) + assert res.status_code == 422 + assert res.json()["error_code"] == 44501 + assert res.json()["message"] == "Schema references are not supported for 'AVRO' schema type yet" + @pytest.mark.parametrize("trail", ["", "/"]) async def test_schema_lifecycle(registry_async_client: Client, trail: str) -> None: diff --git a/tests/integration/test_schema_protobuf.py b/tests/integration/test_schema_protobuf.py index e37eac84a..456f8872b 100644 --- a/tests/integration/test_schema_protobuf.py +++ b/tests/integration/test_schema_protobuf.py @@ -145,24 +145,14 @@ async def test_protobuf_schema_references(registry_async_client: Client) -> None ) assert res.status_code == 200 assert "id" in res.json() - res = await registry_async_client.get("subjects/test_schema/versions/latest", json={}) - assert res.status_code == 200 - myjson = res.json() - assert "id" in myjson - references = [{"name": "Customer.proto", "subject": "customer", "version": 1}] - refs2 = myjson["references"] - assert not any(x != y for x, y in zip(refs2, references)) res = await registry_async_client.get("subjects/customer/versions/latest/referencedby", json={}) assert res.status_code == 200 myjson = res.json() referents = [2] assert not any(x != y for x, y in zip(myjson, referents)) - -# res = await registry_async_client.delete("subjects/customer/versions/latest") -# assert res.status_code == 200 - - -# TODO -# AVRO references error -# JSONSCHEMA references error + res = await registry_async_client.delete("subjects/customer/versions/1") + assert res.status_code == 404 + match_msg = "Subject 'customer' Version 1 was not deleted because it is referenced by schemas with ids:[2]" + myjson = res.json() + assert myjson["error_code"] == 44503 and myjson["message"] == match_msg From 93cd2419392c5cdae8bfbf1467be8b6b7c1131e4 Mon Sep 17 00:00:00 2001 From: Sujay Bhowmick Date: Mon, 23 May 2022 18:11:42 +1000 Subject: [PATCH 12/44] removed reference for ujson --- tests/integration/test_schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_schema.py b/tests/integration/test_schema.py index 27fed7457..81fc2f6ed 100644 --- a/tests/integration/test_schema.py +++ b/tests/integration/test_schema.py @@ -1235,7 +1235,7 @@ async def test_schema_subject_post_invalid(registry_async_client: Client) -> Non assert res.json()["error_code"] == 40401 assert res.json()["message"] == f"Subject '{subject_3}' not found." - schema_str = ujson.dumps({"type": "string"}) + schema_str = json.dumps({"type": "string"}) # Create the subject subject_1 = subject_name_factory() res = await registry_async_client.post( From 2e9a8c6cc783344a150b3217da5e7f3048512bf7 Mon Sep 17 00:00:00 2001 From: Sujay Bhowmick Date: Mon, 23 May 2022 18:51:15 +1000 Subject: [PATCH 13/44] added comma at the end --- karapace/schema_reader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/karapace/schema_reader.py b/karapace/schema_reader.py index b8d9f16e5..30a30c18f 100644 --- a/karapace/schema_reader.py +++ b/karapace/schema_reader.py @@ -370,7 +370,7 @@ def _handle_msg_schema(self, key: dict, value: Optional[dict]) -> None: typed_schema = TypedSchema( schema_type=schema_type_parsed, schema_str=schema_str, - references=schema_references + references=schema_references, ) schema = { "schema": typed_schema, From 516d2809836f76cdfadf181c1f89fc439e5f538f Mon Sep 17 00:00:00 2001 From: Sergiy Zaschipas Date: Wed, 15 Jun 2022 00:53:07 +0300 Subject: [PATCH 14/44] dependencies verification without known deps --- karapace/protobuf/exception.py | 4 + karapace/protobuf/known_dependency.py | 35 +++++++ karapace/protobuf/proto_file_element.py | 19 ++-- karapace/protobuf/proto_parser.py | 9 +- karapace/protobuf/schema.py | 108 +++++++++++++++++++++- karapace/schema_models.py | 22 ++++- karapace/schema_registry_apis.py | 4 +- tests/integration/test_schema_protobuf.py | 15 ++- 8 files changed, 193 insertions(+), 23 deletions(-) create mode 100644 karapace/protobuf/known_dependency.py diff --git a/karapace/protobuf/exception.py b/karapace/protobuf/exception.py index b42d6e141..7f286c343 100644 --- a/karapace/protobuf/exception.py +++ b/karapace/protobuf/exception.py @@ -25,6 +25,10 @@ class ProtobufTypeException(Error): """Generic Protobuf type error.""" +class ProtobufUnresolvedDependencyException(ProtobufException): + """a Protobuf schema has unresolved dependency""" + + class SchemaParseException(ProtobufException): """Error while parsing a Protobuf schema descriptor.""" diff --git a/karapace/protobuf/known_dependency.py b/karapace/protobuf/known_dependency.py new file mode 100644 index 000000000..c8bbe6133 --- /dev/null +++ b/karapace/protobuf/known_dependency.py @@ -0,0 +1,35 @@ +from enum import Enum + + +class KnownDependencyLocation(Enum): + ANY_LOCATION = "google/protobuf/any.proto" + API_LOCATION = "google/protobuf/api.proto" + DESCRIPTOR_LOCATION = "google/protobuf/descriptor.proto" + DURATION_LOCATION = "google/protobuf/duration.proto" + EMPTY_LOCATION = "google/protobuf/empty.proto" + FIELD_MASK_LOCATION = "google/protobuf/field_mask.proto" + SOURCE_CONTEXT_LOCATION = "google/protobuf/source_context.proto" + STRUCT_LOCATION = "google/protobuf/struct.proto" + TIMESTAMP_LOCATION = "google/protobuf/timestamp.proto" + TYPE_LOCATION = "google/protobuf/type.proto" + WRAPPER_LOCATION = "google/protobuf/wrappers.proto" + CALENDAR_PERIOD_LOCATION = "google/type/calendar_period.proto" + COLOR_LOCATION = "google/type/color.proto" + DATE_LOCATION = "google/type/date.proto" + DATETIME_LOCATION = "google/type/datetime.proto" + DAY_OF_WEEK_LOCATION = "google/type/dayofweek.proto" + DECIMAL_LOCATION = "google/type/decimal.proto" + EXPR_LOCATION = "google/type/expr.proto" + FRACTION_LOCATION = "google/type/fraction.proto" + INTERVAL_LOCATION = "google/type/interval.proto" + LATLNG_LOCATION = "google/type/latlng.proto" + MONEY_LOCATION = "google/type/money.proto" + MONTH_LOCATION = "google/type/month.proto" + PHONE_NUMBER_LOCATION = "google/type/phone_number.proto" + POSTAL_ADDRESS_LOCATION = "google/type/postal_address.proto" + QUATERNION_LOCATION = "google/type/quaternion.proto" + TIME_OF_DAY_LOCATION = "google/type/timeofday.proto" + + +class KnownDependency(Enum): + pass diff --git a/karapace/protobuf/proto_file_element.py b/karapace/protobuf/proto_file_element.py index 4774a8c67..fa43dd9e8 100644 --- a/karapace/protobuf/proto_file_element.py +++ b/karapace/protobuf/proto_file_element.py @@ -9,23 +9,24 @@ from karapace.protobuf.message_element import MessageElement from karapace.protobuf.syntax import Syntax from karapace.protobuf.type_element import TypeElement +from typing import List, Optional class ProtoFileElement: def __init__( self, location: Location, - package_name: str = None, - syntax: Syntax = None, - imports: list = None, - public_imports: list = None, - types=None, - services: list = None, - extend_declarations: list = None, - options: list = None, + package_name: Optional[str] = None, + syntax: Optional[Syntax] = None, + imports: Optional[list] = None, + public_imports: Optional[list] = None, + types: Optional[List[TypeElement]] = None, + services: Optional[list] = None, + extend_declarations: Optional[list] = None, + options: Optional[list] = None, ) -> None: if types is None: - types = [] + types = list() self.location = location self.package_name = package_name self.syntax = syntax diff --git a/karapace/protobuf/proto_parser.py b/karapace/protobuf/proto_parser.py index 81c333af4..27cc59f11 100644 --- a/karapace/protobuf/proto_parser.py +++ b/karapace/protobuf/proto_parser.py @@ -25,7 +25,7 @@ from karapace.protobuf.syntax_reader import SyntaxReader from karapace.protobuf.type_element import TypeElement from karapace.protobuf.utils import MAX_TAG_VALUE -from typing import List, Union +from typing import List, Optional, Union class Context(Enum): @@ -71,13 +71,13 @@ class ProtoParser: def __init__(self, location: Location, data: str) -> None: self.location = location self.imports: List[str] = [] - self.nested_types: List[str] = [] + self.nested_types: List[TypeElement] = [] self.services: List[str] = [] self.extends_list: List[str] = [] self.options: List[str] = [] self.declaration_count = 0 - self.syntax: Union[Syntax, None] = None - self.package_name: Union[str, None] = None + self.syntax: Optional[Syntax] = None + self.package_name: Optional[str] = None self.prefix = "" self.data = data self.public_imports: List[str] = [] @@ -176,7 +176,6 @@ def read_declaration( import_string = self.reader.read_string() if import_string == "public": self.public_imports.append(self.reader.read_string()) - else: self.imports.append(import_string) self.reader.require(";") diff --git a/karapace/protobuf/schema.py b/karapace/protobuf/schema.py index 0ec5d6d6e..c041963ee 100644 --- a/karapace/protobuf/schema.py +++ b/karapace/protobuf/schema.py @@ -6,10 +6,17 @@ from karapace.protobuf.exception import IllegalArgumentException from karapace.protobuf.location import Location from karapace.protobuf.message_element import MessageElement +from karapace.protobuf.one_of_element import OneOfElement from karapace.protobuf.option_element import OptionElement from karapace.protobuf.proto_file_element import ProtoFileElement from karapace.protobuf.proto_parser import ProtoParser +from karapace.protobuf.type_element import TypeElement from karapace.protobuf.utils import append_documentation, append_indented +from karapace.schema_models import References +from karapace.schema_reader import KafkaSchemaReader +from typing import Dict, Optional + +import re def add_slashes(text: str) -> str: @@ -97,15 +104,97 @@ def option_element_string(option: OptionElement) -> str: return f"option {result};\n" +class Dependency: + def __init__(self, name: str, subject: str, version: int, schema: "ProtobufSchema") -> None: + self.name = name + self.subject = subject + self.version = version + self.schema = schema + + def identifier(self) -> str: + return self.name + "_" + self.subject + "_" + str(self.version) + + +class DependencyVerifierResult: + def __init__(self, result: bool, message: Optional[str] = ""): + self.result = result + self.message = message + + +class DependencyVerifier: + def __init__(self): + self.declared_types = list() + self.used_types = list() + + def add_declared_type(self, full_name: str): + self.declared_types.append(full_name) + + def add_used_type(self, element_type: str): + self.used_types.append(element_type) + + def verify(self) -> DependencyVerifierResult: + for used_type in self.used_types: + t = True + r = re.compile(re.escape(used_type) + "$") + for delcared_type in self.declared_types: + if r.match(delcared_type): + t = True + break + if not t: + return DependencyVerifierResult(False, f"type {used_type} is not defined") + return DependencyVerifierResult(True) + + +def _process_one_of(verifier: DependencyVerifier, one_of: OneOfElement): + for field in one_of.fields: + verifier.add_used_type(field.element_type) + + class ProtobufSchema: DEFAULT_LOCATION = Location.get("") - def __init__(self, schema: str) -> None: + def __init__( + self, schema: str, references: Optional[References] = None, ksr: Optional[KafkaSchemaReader] = None + ) -> None: if type(schema).__name__ != "str": raise IllegalArgumentException("Non str type of schema string") self.dirty = schema self.cache_string = "" + self.ksr = ksr self.proto_file_element = ProtoParser.parse(self.DEFAULT_LOCATION, schema) + self.references = references + self.dependencies: Dict[str, Dependency] = dict() + self.reslove_dependencies() + + def verify_schema_dependencies(self) -> DependencyVerifierResult: + verifier = DependencyVerifier() + self._verify_schema_dependencies(verifier) + return verifier.verify() + + def _verify_schema_dependencies(self, verifier: DependencyVerifier) -> bool: + package_name = self.proto_file_element.package_name + for element_type in self.proto_file_element.types: + type_name = element_type.name + full_name = "." + package_name + type_name + verifier.add_declared_type(full_name) + if isinstance(element_type, MessageElement): + for one_of in element_type.one_ofs: + _process_one_of(verifier, one_of) + for field in element_type.fields: + verifier.add_used_type(field.element_type) + + for nested_type in element_type.nested_types: + self._process_nested_type(verifier, full_name, nested_type) + + def _process_nested_type(self, verifier: DependencyVerifier, full_name: str, element_type: TypeElement): + full_name = full_name + element_type.name + if isinstance(element_type, MessageElement): + for one_of in element_type.one_ofs: + _process_one_of(verifier, one_of) + for field in element_type.fields: + verifier.add_used_type(field.element_type) + for nested_type in element_type.nested_types: + self._process_nested_type(verifier, full_name, nested_type) def __str__(self) -> str: if not self.cache_string: @@ -159,3 +248,20 @@ def to_schema(self) -> str: def compare(self, other: "ProtobufSchema", result: CompareResult) -> CompareResult: self.proto_file_element.compare(other.proto_file_element, result) + + def reslove_dependencies(self): + try: + if self.references is not None: + for r in self.references.val(): + subject = r["subject"] + version = r["version"] + name = r["name"] + subject_data = self.ksr.subjects.get(subject) + schema_data = subject_data["schemas"][version] + schema = schema_data["schema"].schema_str + references = schema_data.get("references") + parsed_schema = ProtobufSchema(schema, references) + self.dependencies[name] = Dependency(name, subject, version, parsed_schema) + except Exception as e: + # TODO: need the exception? + raise e diff --git a/karapace/schema_models.py b/karapace/schema_models.py index 85c8bc965..cae20e88e 100644 --- a/karapace/schema_models.py +++ b/karapace/schema_models.py @@ -9,9 +9,11 @@ IllegalStateException, ProtobufException, ProtobufParserRuntimeException, + ProtobufUnresolvedDependencyException, SchemaParseException as ProtobufSchemaParseException, ) from karapace.protobuf.schema import ProtobufSchema +from karapace.schema_reader import KafkaSchemaReader from karapace.typing import JsonData from karapace.utils import json_encode from typing import Any, Dict, Optional, Union @@ -49,15 +51,20 @@ def parse_jsonschema_definition(schema_definition: str) -> Draft7Validator: return Draft7Validator(schema) -def parse_protobuf_schema_definition(schema_definition: str) -> ProtobufSchema: +def parse_protobuf_schema_definition( + schema_definition: str, references: Optional["References"] = None, ksr: Optional[KafkaSchemaReader] = None +) -> ProtobufSchema: """Parses and validates `schema_definition`. Raises: Nothing yet. """ - - return ProtobufSchema(schema_definition) + protobuf_schema = ProtobufSchema(schema_definition, references, ksr) + result = protobuf_schema.verify_schema_dependencies() + if not result.result: + raise ProtobufUnresolvedDependencyException(f"{result.message}") + return protobuf_schema class InvalidSchemaType(Exception): @@ -141,7 +148,12 @@ def __init__( self.schema = schema @staticmethod - def parse(schema_type: SchemaType, schema_str: str, references: Optional["References"] = None) -> "ValidatedTypedSchema": + def parse( + schema_type: SchemaType, + schema_str: str, + references: Optional["References"] = None, + ksr: Optional["KafkaSchemaReader"] = None, + ) -> "ValidatedTypedSchema": if schema_type not in [SchemaType.AVRO, SchemaType.JSONSCHEMA, SchemaType.PROTOBUF]: raise InvalidSchema(f"Unknown parser {schema_type} for {schema_str}") @@ -161,7 +173,7 @@ def parse(schema_type: SchemaType, schema_str: str, references: Optional["Refere elif schema_type is SchemaType.PROTOBUF: try: - parsed_schema = parse_protobuf_schema_definition(schema_str) + parsed_schema = parse_protobuf_schema_definition(schema_str, references, ksr) except ( TypeError, SchemaError, diff --git a/karapace/schema_registry_apis.py b/karapace/schema_registry_apis.py index ff4a75be6..baed34b13 100644 --- a/karapace/schema_registry_apis.py +++ b/karapace/schema_registry_apis.py @@ -859,7 +859,7 @@ async def subjects_schema_post(self, content_type, *, subject, request): try: new_schema = ValidatedTypedSchema.parse( - schema_type=schema_type, schema_str=schema_str, references=new_schema_references + schema_type=schema_type, schema_str=schema_str, references=new_schema_references, ksr=self.ksr ) except InvalidSchema: self.log.exception("No proper parser found") @@ -953,7 +953,7 @@ def write_new_schema_local( ) try: new_schema = ValidatedTypedSchema.parse( - schema_type=schema_type, schema_str=body["schema"], references=new_schema_references + schema_type=schema_type, schema_str=body["schema"], references=new_schema_references, ksr=self.ksr ) except (InvalidSchema, InvalidSchemaType) as e: self.log.warning("Invalid schema: %r", body["schema"], exc_info=True) diff --git a/tests/integration/test_schema_protobuf.py b/tests/integration/test_schema_protobuf.py index 456f8872b..2dc349f73 100644 --- a/tests/integration/test_schema_protobuf.py +++ b/tests/integration/test_schema_protobuf.py @@ -107,7 +107,6 @@ async def test_protobuf_schema_compatibility(registry_async_client: Client, trai async def test_protobuf_schema_references(registry_async_client: Client) -> None: - customer_schema = """ |syntax = "proto3"; |package a1; @@ -128,12 +127,26 @@ async def test_protobuf_schema_references(registry_async_client: Client) -> None |package a1; |import "Customer.proto"; |message TestMessage { + | enum Enum { + | HIGH = 0; + | MIDDLE = 1; + | LOW = 2; + | } | message Value { + | message Label{ + | int32 Id = 1; + | str name = 2; + | } | Customer customer = 1; | int32 x = 2; | } | string test = 1; | .a1.TestMessage.Value val = 2; + | oneof page_info { + | option (my_option) = true; + | int32 page_number = 3; + | int32 result_per_page = 4; + | } |} |""" From cc1a691a79c32fcde47f7f3af44133f68533e9d6 Mon Sep 17 00:00:00 2001 From: Sergiy Zaschipas Date: Fri, 17 Jun 2022 12:14:11 +0300 Subject: [PATCH 15/44] fixup cyclic dependencies --- karapace/protobuf/schema.py | 10 +++++---- karapace/schema_models.py | 42 +++++++---------------------------- karapace/schema_reader.py | 3 ++- karapace/schema_references.py | 25 +++++++++++++++++++++ karapace/schema_type.py | 8 +++++++ 5 files changed, 49 insertions(+), 39 deletions(-) create mode 100644 karapace/schema_references.py create mode 100644 karapace/schema_type.py diff --git a/karapace/protobuf/schema.py b/karapace/protobuf/schema.py index c041963ee..37076a297 100644 --- a/karapace/protobuf/schema.py +++ b/karapace/protobuf/schema.py @@ -12,12 +12,14 @@ from karapace.protobuf.proto_parser import ProtoParser from karapace.protobuf.type_element import TypeElement from karapace.protobuf.utils import append_documentation, append_indented -from karapace.schema_models import References -from karapace.schema_reader import KafkaSchemaReader -from typing import Dict, Optional +from karapace.schema_references import References +from typing import Dict, Optional, TYPE_CHECKING import re +if TYPE_CHECKING: + from karapace.schema_reader import KafkaSchemaReader + def add_slashes(text: str) -> str: escape_dict = { @@ -154,7 +156,7 @@ class ProtobufSchema: DEFAULT_LOCATION = Location.get("") def __init__( - self, schema: str, references: Optional[References] = None, ksr: Optional[KafkaSchemaReader] = None + self, schema: str, references: Optional[References] = None, ksr: Optional["KafkaSchemaReader"] = None ) -> None: if type(schema).__name__ != "str": raise IllegalArgumentException("Non str type of schema string") diff --git a/karapace/schema_models.py b/karapace/schema_models.py index cae20e88e..ecb7379c9 100644 --- a/karapace/schema_models.py +++ b/karapace/schema_models.py @@ -1,6 +1,5 @@ from avro.errors import SchemaParseException from avro.schema import parse as avro_parse, Schema as AvroSchema -from enum import Enum, unique from jsonschema import Draft7Validator from jsonschema.exceptions import SchemaError from karapace.protobuf.exception import ( @@ -13,13 +12,16 @@ SchemaParseException as ProtobufSchemaParseException, ) from karapace.protobuf.schema import ProtobufSchema -from karapace.schema_reader import KafkaSchemaReader -from karapace.typing import JsonData +from karapace.schema_references import References +from karapace.schema_type import SchemaType from karapace.utils import json_encode -from typing import Any, Dict, Optional, Union +from typing import Any, Dict, Optional, TYPE_CHECKING, Union import json +if TYPE_CHECKING: + from karapace.schema_reader import KafkaSchemaReader + def parse_avro_schema_definition(s: str) -> AvroSchema: """Compatibility function with Avro which ignores trailing data in JSON @@ -52,7 +54,7 @@ def parse_jsonschema_definition(schema_definition: str) -> Draft7Validator: def parse_protobuf_schema_definition( - schema_definition: str, references: Optional["References"] = None, ksr: Optional[KafkaSchemaReader] = None + schema_definition: str, references: Optional[References] = None, ksr: Optional["KafkaSchemaReader"] = None ) -> ProtobufSchema: """Parses and validates `schema_definition`. @@ -79,15 +81,8 @@ class InvalidReferences(Exception): pass -@unique -class SchemaType(str, Enum): - AVRO = "AVRO" - JSONSCHEMA = "JSON" - PROTOBUF = "PROTOBUF" - - class TypedSchema: - def __init__(self, schema_type: SchemaType, schema_str: str, references: Optional["References"] = None): + def __init__(self, schema_type: SchemaType, schema_str: str, references: Optional[References] = None): """Schema with type information Args: @@ -197,24 +192,3 @@ def __str__(self) -> str: if self.schema_type == SchemaType.PROTOBUF: return str(self.schema) return super().__str__() - - -class References: - def __init__(self, schema_type: SchemaType, references: JsonData): - """Schema with type information - - Args: - schema_type (SchemaType): The type of the schema - references (str): The original schema string - """ - self.schema_type = schema_type - self.references = references - - def val(self) -> JsonData: - return self.references - - def json(self) -> str: - return str(json_encode(self.references, sort_keys=True)) - - def __eq__(self, other: Any) -> bool: - return self.json() == other.json() diff --git a/karapace/schema_reader.py b/karapace/schema_reader.py index 30a30c18f..5d98e1b2c 100644 --- a/karapace/schema_reader.py +++ b/karapace/schema_reader.py @@ -11,7 +11,8 @@ from karapace import constants from karapace.config import Config from karapace.master_coordinator import MasterCoordinator -from karapace.schema_models import SchemaType, TypedSchema +from karapace.schema_models import TypedSchema +from karapace.schema_type import SchemaType from karapace.statsd import StatsClient from karapace.utils import KarapaceKafkaClient from threading import Event, Lock, Thread diff --git a/karapace/schema_references.py b/karapace/schema_references.py new file mode 100644 index 000000000..ca4e7feac --- /dev/null +++ b/karapace/schema_references.py @@ -0,0 +1,25 @@ +from karapace.schema_type import SchemaType +from karapace.typing import JsonData +from karapace.utils import json_encode +from typing import Any + + +class References: + def __init__(self, schema_type: SchemaType, references: JsonData): + """Schema with type information + + Args: + schema_type (SchemaType): The type of the schema + references (str): The original schema string + """ + self.schema_type = schema_type + self.references = references + + def val(self) -> JsonData: + return self.references + + def json(self) -> str: + return str(json_encode(self.references, sort_keys=True)) + + def __eq__(self, other: Any) -> bool: + return self.json() == other.json() diff --git a/karapace/schema_type.py b/karapace/schema_type.py new file mode 100644 index 000000000..2881c4f3f --- /dev/null +++ b/karapace/schema_type.py @@ -0,0 +1,8 @@ +from enum import Enum, unique + + +@unique +class SchemaType(str, Enum): + AVRO = "AVRO" + JSONSCHEMA = "JSON" + PROTOBUF = "PROTOBUF" From 7f2cf0175887e89317bb975a63f3d814dd6cb1e1 Mon Sep 17 00:00:00 2001 From: Sergiy Zaschipas Date: Sun, 19 Jun 2022 21:00:59 +0300 Subject: [PATCH 16/44] Update schema_models.py fixed docstring --- karapace/schema_models.py | 1 + 1 file changed, 1 insertion(+) diff --git a/karapace/schema_models.py b/karapace/schema_models.py index 85c8bc965..acd211e52 100644 --- a/karapace/schema_models.py +++ b/karapace/schema_models.py @@ -86,6 +86,7 @@ def __init__(self, schema_type: SchemaType, schema_str: str, references: Optiona Args: schema_type (SchemaType): The type of the schema schema_str (str): The original schema string + references(References): The references which schema reference. """ self.schema_type = schema_type self.schema_str = schema_str From 0f2fab68ddce74ee17dd3b85075e1297c9443412 Mon Sep 17 00:00:00 2001 From: Sergiy Zaschipas Date: Sun, 19 Jun 2022 22:12:30 +0300 Subject: [PATCH 17/44] Update schema_models.py fiexed equation functions --- karapace/schema_models.py | 24 ++++++++---------------- 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/karapace/schema_models.py b/karapace/schema_models.py index acd211e52..f3265f987 100644 --- a/karapace/schema_models.py +++ b/karapace/schema_models.py @@ -111,24 +111,14 @@ def get_references(self) -> Optional["References"]: return self.references def __eq__(self, other: Any) -> bool: - a = isinstance(other, TypedSchema) and self.schema_type is other.schema_type and self.__str__() == other.__str__() - - if not a: + schema_is_equal = isinstance(other, TypedSchema) and \ + self.schema_type is other.schema_type and self.__str__() == other.__str__() + if not schema_is_equal: return False - x = self.get_references() - y = other.get_references() - if x: - if y: - if x == y: - return True - else: - return False + if self.references is not None: + return self.references == other.references else: - if y: - return False - - return True - + return other.references is None class ValidatedTypedSchema(TypedSchema): def __init__( @@ -206,4 +196,6 @@ def json(self) -> str: return str(json_encode(self.references, sort_keys=True)) def __eq__(self, other: Any) -> bool: + if other is None or not isinstance(other, References): + return False return self.json() == other.json() From bbbb5a30ded8af0b3aa9afe823cf95b981e4f0a0 Mon Sep 17 00:00:00 2001 From: Sergiy Zaschipas Date: Sun, 19 Jun 2022 22:20:04 +0300 Subject: [PATCH 18/44] Update schema_models.py --- karapace/schema_models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/karapace/schema_models.py b/karapace/schema_models.py index f3265f987..5567e06ae 100644 --- a/karapace/schema_models.py +++ b/karapace/schema_models.py @@ -86,7 +86,7 @@ def __init__(self, schema_type: SchemaType, schema_str: str, references: Optiona Args: schema_type (SchemaType): The type of the schema schema_str (str): The original schema string - references(References): The references which schema reference. + references(References): The references of schema """ self.schema_type = schema_type self.schema_str = schema_str @@ -184,7 +184,7 @@ def __init__(self, schema_type: SchemaType, references: JsonData): Args: schema_type (SchemaType): The type of the schema - references (str): The original schema string + references (str): The references of schema in Kafka/Json representation """ self.schema_type = schema_type self.references = references From 1d2b7c455aee47ed82775d89aa0e4a7f1aa472c8 Mon Sep 17 00:00:00 2001 From: Sergiy Zaschipas Date: Sun, 19 Jun 2022 22:22:03 +0300 Subject: [PATCH 19/44] Update schema_reader.py change line feed --- karapace/schema_reader.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/karapace/schema_reader.py b/karapace/schema_reader.py index 30a30c18f..7034d8a9d 100644 --- a/karapace/schema_reader.py +++ b/karapace/schema_reader.py @@ -4,6 +4,8 @@ Copyright (c) 2019 Aiven Ltd See LICENSE for details """ +import json +import logging from contextlib import closing, ExitStack from kafka import KafkaConsumer from kafka.admin import KafkaAdminClient, NewTopic @@ -17,9 +19,6 @@ from threading import Event, Lock, Thread from typing import Any, Dict, List, Optional -import json -import logging - Offset = int Subject = str Version = int From 4634373eb3e9d31b98413c2db501da3dc30ea30f Mon Sep 17 00:00:00 2001 From: Sergiy Zaschipas Date: Sun, 19 Jun 2022 23:00:32 +0300 Subject: [PATCH 20/44] Update karapace/schema_registry_apis.py Co-authored-by: Jarkko Jaakola <91882676+jjaakola-aiven@users.noreply.github.com> --- karapace/schema_registry_apis.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/karapace/schema_registry_apis.py b/karapace/schema_registry_apis.py index ff4a75be6..b3b7b0a3a 100644 --- a/karapace/schema_registry_apis.py +++ b/karapace/schema_registry_apis.py @@ -744,9 +744,7 @@ async def subject_version_referencedby_get(self, content_type, *, subject, versi status=HTTPStatus.UNPROCESSABLE_ENTITY, ) - referenced_by = self.ksr.referenced_by.get(str(subject) + "_" + str(version), None) - if not referenced_by: - referenced_by = list() + referenced_by = self.ksr.referenced_by.get(str(subject) + "_" + str(version), []) self.r(list(referenced_by), content_type, status=HTTPStatus.OK) async def subject_versions_list(self, content_type, *, subject): From 4b99fc8928d4886ac1137cf4d6317c9e4c990367 Mon Sep 17 00:00:00 2001 From: Sergiy Zaschipas Date: Sun, 19 Jun 2022 23:15:30 +0300 Subject: [PATCH 21/44] fixup style --- karapace/schema_models.py | 59 +++++++++++++++-------------------- karapace/schema_references.py | 2 ++ 2 files changed, 27 insertions(+), 34 deletions(-) diff --git a/karapace/schema_models.py b/karapace/schema_models.py index ecb7379c9..c10e0db72 100644 --- a/karapace/schema_models.py +++ b/karapace/schema_models.py @@ -54,7 +54,7 @@ def parse_jsonschema_definition(schema_definition: str) -> Draft7Validator: def parse_protobuf_schema_definition( - schema_definition: str, references: Optional[References] = None, ksr: Optional["KafkaSchemaReader"] = None + schema_definition: str, references: Optional[References] = None, ksr: Optional["KafkaSchemaReader"] = None ) -> ProtobufSchema: """Parses and validates `schema_definition`. @@ -112,42 +112,33 @@ def get_references(self) -> Optional["References"]: return self.references def __eq__(self, other: Any) -> bool: - a = isinstance(other, TypedSchema) and self.schema_type is other.schema_type and self.__str__() == other.__str__() - - if not a: + schema_is_equal = isinstance(other, TypedSchema) and \ + self.schema_type is other.schema_type and self.__str__() == other.__str__() + if not schema_is_equal: return False - x = self.get_references() - y = other.get_references() - if x: - if y: - if x == y: - return True - else: - return False + if self.references is not None: + return self.references == other.references else: - if y: - return False - - return True + return other.references is None class ValidatedTypedSchema(TypedSchema): def __init__( - self, - schema_type: SchemaType, - schema_str: str, - schema: Union[Draft7Validator, AvroSchema, ProtobufSchema], - references: Optional["References"] = None, + self, + schema_type: SchemaType, + schema_str: str, + schema: Union[Draft7Validator, AvroSchema, ProtobufSchema], + references: Optional["References"] = None, ): super().__init__(schema_type=schema_type, schema_str=schema_str, references=references) self.schema = schema @staticmethod def parse( - schema_type: SchemaType, - schema_str: str, - references: Optional["References"] = None, - ksr: Optional["KafkaSchemaReader"] = None, + schema_type: SchemaType, + schema_str: str, + references: Optional["References"] = None, + ksr: Optional["KafkaSchemaReader"] = None, ) -> "ValidatedTypedSchema": if schema_type not in [SchemaType.AVRO, SchemaType.JSONSCHEMA, SchemaType.PROTOBUF]: raise InvalidSchema(f"Unknown parser {schema_type} for {schema_str}") @@ -170,15 +161,15 @@ def parse( try: parsed_schema = parse_protobuf_schema_definition(schema_str, references, ksr) except ( - TypeError, - SchemaError, - AssertionError, - ProtobufParserRuntimeException, - IllegalStateException, - IllegalArgumentException, - ProtobufError, - ProtobufException, - ProtobufSchemaParseException, + TypeError, + SchemaError, + AssertionError, + ProtobufParserRuntimeException, + IllegalStateException, + IllegalArgumentException, + ProtobufError, + ProtobufException, + ProtobufSchemaParseException, ) as e: raise InvalidSchema from e else: diff --git a/karapace/schema_references.py b/karapace/schema_references.py index ca4e7feac..6663a3d12 100644 --- a/karapace/schema_references.py +++ b/karapace/schema_references.py @@ -22,4 +22,6 @@ def json(self) -> str: return str(json_encode(self.references, sort_keys=True)) def __eq__(self, other: Any) -> bool: + if other is None or not isinstance(other, References): + return False return self.json() == other.json() From efe29f4e3759bb27fc0d0c4b3e70a0bcc1986fcb Mon Sep 17 00:00:00 2001 From: Sergiy Zaschipas Date: Sun, 19 Jun 2022 23:27:19 +0300 Subject: [PATCH 22/44] update code by PR review --- karapace/schema_models.py | 11 ++++++----- karapace/schema_reader.py | 5 +++-- karapace/serialization.py | 4 ++-- tests/integration/test_client.py | 4 ++-- tests/integration/test_client_protobuf.py | 4 ++-- 5 files changed, 15 insertions(+), 13 deletions(-) diff --git a/karapace/schema_models.py b/karapace/schema_models.py index 5567e06ae..fe2961592 100644 --- a/karapace/schema_models.py +++ b/karapace/schema_models.py @@ -86,7 +86,7 @@ def __init__(self, schema_type: SchemaType, schema_str: str, references: Optiona Args: schema_type (SchemaType): The type of the schema schema_str (str): The original schema string - references(References): The references of schema + references(References): The references of schema """ self.schema_type = schema_type self.schema_str = schema_str @@ -111,14 +111,15 @@ def get_references(self) -> Optional["References"]: return self.references def __eq__(self, other: Any) -> bool: - schema_is_equal = isinstance(other, TypedSchema) and \ - self.schema_type is other.schema_type and self.__str__() == other.__str__() + schema_is_equal = ( + isinstance(other, TypedSchema) and self.schema_type is other.schema_type and self.__str__() == other.__str__() + ) if not schema_is_equal: return False if self.references is not None: return self.references == other.references - else: - return other.references is None + return other.references is None + class ValidatedTypedSchema(TypedSchema): def __init__( diff --git a/karapace/schema_reader.py b/karapace/schema_reader.py index 7034d8a9d..30a30c18f 100644 --- a/karapace/schema_reader.py +++ b/karapace/schema_reader.py @@ -4,8 +4,6 @@ Copyright (c) 2019 Aiven Ltd See LICENSE for details """ -import json -import logging from contextlib import closing, ExitStack from kafka import KafkaConsumer from kafka.admin import KafkaAdminClient, NewTopic @@ -19,6 +17,9 @@ from threading import Event, Lock, Thread from typing import Any, Dict, List, Optional +import json +import logging + Offset = int Subject = str Version = int diff --git a/karapace/serialization.py b/karapace/serialization.py index 1b7536e93..86331b15d 100644 --- a/karapace/serialization.py +++ b/karapace/serialization.py @@ -163,7 +163,7 @@ def get_subject_name(self, topic_name: str, schema: str, subject_type: str, sche async def get_schema_for_subject(self, subject: str) -> TypedSchema: assert self.registry_client, "must not call this method after the object is closed." - # pylint: disable=unused-variable + schema_id, schema = await self.registry_client.get_latest_schema(subject) async with self.state_lock: schema_ser = schema.__str__() @@ -180,7 +180,7 @@ async def get_id_for_schema(self, schema: str, subject: str, schema_type: Schema schema_ser = schema_typed.__str__() if schema_ser in self.schemas_to_ids: return self.schemas_to_ids[schema_ser] - schema_id = await self.registry_client.post_new_schema(subject, schema_typed) # pylint: disable=E1120 + schema_id = await self.registry_client.post_new_schema(subject, schema_typed) async with self.state_lock: self.schemas_to_ids[schema_ser] = schema_id diff --git a/tests/integration/test_client.py b/tests/integration/test_client.py index df9b7fb1e..1467b5ccd 100644 --- a/tests/integration/test_client.py +++ b/tests/integration/test_client.py @@ -9,7 +9,7 @@ async def test_remote_client(registry_async_client: Client) -> None: reg_cli = SchemaRegistryClient() reg_cli.client = registry_async_client subject = new_random_name("subject") - sc_id = await reg_cli.post_new_schema(subject, schema_avro) # pylint: disable=E1120 + sc_id = await reg_cli.post_new_schema(subject, schema_avro) assert sc_id >= 0 stored_schema = await reg_cli.get_schema_for_id(sc_id) assert stored_schema == schema_avro, f"stored schema {stored_schema.to_dict()} is not {schema_avro.to_dict()}" @@ -23,7 +23,7 @@ async def test_remote_client_tls(registry_async_client_tls: Client) -> None: reg_cli = SchemaRegistryClient() reg_cli.client = registry_async_client_tls subject = new_random_name("subject") - sc_id = await reg_cli.post_new_schema(subject, schema_avro) # pylint: disable=E1120 + sc_id = await reg_cli.post_new_schema(subject, schema_avro) assert sc_id >= 0 stored_schema = await reg_cli.get_schema_for_id(sc_id) assert stored_schema == schema_avro, f"stored schema {stored_schema.to_dict()} is not {schema_avro.to_dict()}" diff --git a/tests/integration/test_client_protobuf.py b/tests/integration/test_client_protobuf.py index 8d5f43b9c..f924b536f 100644 --- a/tests/integration/test_client_protobuf.py +++ b/tests/integration/test_client_protobuf.py @@ -14,7 +14,7 @@ async def test_remote_client_protobuf(registry_async_client): assert sc_id >= 0 stored_schema = await reg_cli.get_schema_for_id(sc_id) assert stored_schema == schema_protobuf, f"stored schema {stored_schema} is not {schema_protobuf}" - stored_id, stored_schema = await reg_cli.get_latest_schema(subject) # pylint: disable=unused-variable + stored_id, stored_schema = await reg_cli.get_latest_schema(subject) assert stored_id == sc_id assert stored_schema == schema_protobuf @@ -29,6 +29,6 @@ async def test_remote_client_protobuf2(registry_async_client): assert sc_id >= 0 stored_schema = await reg_cli.get_schema_for_id(sc_id) assert stored_schema == schema_protobuf, f"stored schema {stored_schema} is not {schema_protobuf}" - stored_id, stored_schema = await reg_cli.get_latest_schema(subject) # pylint: disable=unused-variable + stored_id, stored_schema = await reg_cli.get_latest_schema(subject) assert stored_id == sc_id assert stored_schema == schema_protobuf_after From 0854296ed1a117f53d7299af1f87511e476a330e Mon Sep 17 00:00:00 2001 From: Sergiy Zaschipas Date: Wed, 22 Jun 2022 00:02:01 +0300 Subject: [PATCH 23/44] add reference_key() --- karapace/schema_reader.py | 5 +++-- karapace/schema_registry_apis.py | 10 +++++----- karapace/utils.py | 4 ++++ 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/karapace/schema_reader.py b/karapace/schema_reader.py index 30a30c18f..ab942d6c9 100644 --- a/karapace/schema_reader.py +++ b/karapace/schema_reader.py @@ -13,7 +13,7 @@ from karapace.master_coordinator import MasterCoordinator from karapace.schema_models import SchemaType, TypedSchema from karapace.statsd import StatsClient -from karapace.utils import KarapaceKafkaClient +from karapace.utils import KarapaceKafkaClient, reference_key from threading import Event, Lock, Thread from typing import Any, Dict, List, Optional @@ -389,7 +389,8 @@ def _handle_msg_schema(self, key: dict, value: Optional[dict]) -> None: subjects_schemas[schema_version] = schema if schema_references: for ref in schema_references: - ref_str = str(ref["subject"]) + "_" + str(ref["version"]) + + ref_str = reference_key(ref["subject"], ref["version"]) referents = self.referenced_by.get(ref_str, None) if referents: LOG.info("Adding entry subject referenced_by : %r", ref_str) diff --git a/karapace/schema_registry_apis.py b/karapace/schema_registry_apis.py index b3b7b0a3a..16c652389 100644 --- a/karapace/schema_registry_apis.py +++ b/karapace/schema_registry_apis.py @@ -12,7 +12,7 @@ from karapace.schema_models import InvalidReferences, InvalidSchema, InvalidSchemaType, References, ValidatedTypedSchema from karapace.schema_reader import KafkaSchemaReader, SchemaType, TypedSchema from karapace.typing import JsonData -from karapace.utils import json_encode, KarapaceKafkaClient +from karapace.utils import json_encode, KarapaceKafkaClient, reference_key from typing import Any, Dict, NoReturn, Optional, Tuple import aiohttp @@ -511,7 +511,7 @@ async def _subject_delete_local(self, content_type: str, subject: str, permanent if permanent: for version, value in list(subject_data["schemas"].items()): - referenced_by = self.ksr.referenced_by.get(str(subject) + "_" + str(version), None) + referenced_by = self.ksr.referenced_by.get(reference_key(subject, version), None) if referenced_by and len(referenced_by) > 0: self.r( body={ @@ -532,7 +532,7 @@ async def _subject_delete_local(self, content_type: str, subject: str, permanent subject=subject, schema=None, schema_id=schema_id, version=version, deleted=True, references=None ) else: - referenced_by = self.ksr.referenced_by.get(str(subject) + "_" + str(latest_schema_id), None) + referenced_by = self.ksr.referenced_by.get(reference_key(subject, latest_schema_id), None) if referenced_by and len(referenced_by) > 0: self.r( body={ @@ -643,7 +643,7 @@ async def _subject_version_delete_local(self, content_type: str, subject: str, v status=HTTPStatus.NOT_FOUND, ) - referenced_by = self.ksr.referenced_by.get(str(subject) + "_" + str(version), None) + referenced_by = self.ksr.referenced_by.get(reference_key(subject, version), None) if referenced_by and len(referenced_by) > 0: self.r( body={ @@ -744,7 +744,7 @@ async def subject_version_referencedby_get(self, content_type, *, subject, versi status=HTTPStatus.UNPROCESSABLE_ENTITY, ) - referenced_by = self.ksr.referenced_by.get(str(subject) + "_" + str(version), []) + referenced_by = self.ksr.referenced_by.get(reference_key(subject, version), []) self.r(list(referenced_by), content_type, status=HTTPStatus.OK) async def subject_versions_list(self, content_type, *, subject): diff --git a/karapace/utils.py b/karapace/utils.py index 95302da04..9433d6523 100644 --- a/karapace/utils.py +++ b/karapace/utils.py @@ -55,6 +55,10 @@ def default_json_serialization(obj: MappingProxyType) -> dict: ... +def reference_key(subject: str, version: int) -> str: + return hash((subject, version)) + + def default_json_serialization( # pylint: disable=inconsistent-return-statements obj: Union[datetime, timedelta, Decimal, MappingProxyType], ) -> Union[str, float, dict]: From 46146dd74b2e6558d9ea082903fa4e56de4c2076 Mon Sep 17 00:00:00 2001 From: Sujay Bhowmick Date: Thu, 23 Jun 2022 16:39:47 +1000 Subject: [PATCH 24/44] fixed undefined variable missing due to merge --- karapace/schema_registry_apis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/karapace/schema_registry_apis.py b/karapace/schema_registry_apis.py index ff0109ce6..6979217b9 100644 --- a/karapace/schema_registry_apis.py +++ b/karapace/schema_registry_apis.py @@ -925,7 +925,7 @@ def _validate_schema_request_body(self, content_type: str, body: Union[dict, Any self.r( body={ "error_code": SchemaErrorCodes.HTTP_UNPROCESSABLE_ENTITY.value, - "message": f"Unrecognized field: {attr}", + "message": f"Unrecognized field: {field}", }, content_type=content_type, status=HTTPStatus.UNPROCESSABLE_ENTITY, From 967c8223676f6b493b7342113ed6b7baca973439 Mon Sep 17 00:00:00 2001 From: Sujay Bhowmick Date: Wed, 29 Jun 2022 16:36:12 +1000 Subject: [PATCH 25/44] removed extra line feeds --- karapace/schema_reader.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/karapace/schema_reader.py b/karapace/schema_reader.py index ab942d6c9..0acd38596 100644 --- a/karapace/schema_reader.py +++ b/karapace/schema_reader.py @@ -286,14 +286,12 @@ def _handle_msg_config(self, key: dict, value: Optional[dict]) -> None: if subject not in self.subjects: LOG.info("Adding first version of subject: %r with no schemas", subject) self.subjects[subject] = {"schemas": {}} - if not value: LOG.info("Deleting compatibility config completely for subject: %r", subject) self.subjects[subject].pop("compatibility", None) else: LOG.info("Setting subject: %r config to: %r, value: %r", subject, value["compatibilityLevel"], value) self.subjects[subject]["compatibility"] = value["compatibilityLevel"] - elif value is not None: LOG.info("Setting global config to: %r, value: %r", value["compatibilityLevel"], value) self.config["compatibility"] = value["compatibilityLevel"] From 73ff0f3c2664a6f0d03c07347a66c3e408fb88f4 Mon Sep 17 00:00:00 2001 From: Sergiy Zaschipas Date: Thu, 7 Jul 2022 13:09:05 +0300 Subject: [PATCH 26/44] implementation of protobuf dependency verifications beta --- karapace/protobuf/google/protobuf/any.proto | 158 +++ .../protobuf/google/protobuf/any_test.proto | 44 + karapace/protobuf/google/protobuf/api.proto | 208 ++++ .../protobuf/google/protobuf/descriptor.proto | 918 ++++++++++++++++++ .../protobuf/google/protobuf/duration.proto | 116 +++ karapace/protobuf/google/protobuf/empty.proto | 51 + .../protobuf/google/protobuf/field_mask.proto | 245 +++++ .../google/protobuf/map_lite_unittest.proto | 131 +++ .../google/protobuf/map_proto2_unittest.proto | 91 ++ .../google/protobuf/map_unittest.proto | 125 +++ .../google/protobuf/source_context.proto | 48 + .../protobuf/google/protobuf/struct.proto | 95 ++ .../protobuf/google/protobuf/timestamp.proto | 147 +++ karapace/protobuf/google/protobuf/type.proto | 187 ++++ .../protobuf/google/protobuf/wrappers.proto | 123 +++ karapace/protobuf/google/type/BUILD.bazel | 536 ++++++++++ karapace/protobuf/google/type/README.md | 16 + .../google/type/calendar_period.proto | 56 ++ karapace/protobuf/google/type/color.proto | 174 ++++ karapace/protobuf/google/type/date.proto | 52 + karapace/protobuf/google/type/datetime.proto | 104 ++ karapace/protobuf/google/type/dayofweek.proto | 50 + karapace/protobuf/google/type/decimal.proto | 95 ++ karapace/protobuf/google/type/expr.proto | 73 ++ karapace/protobuf/google/type/fraction.proto | 33 + karapace/protobuf/google/type/interval.proto | 46 + karapace/protobuf/google/type/latlng.proto | 37 + .../protobuf/google/type/localized_text.proto | 36 + karapace/protobuf/google/type/money.proto | 42 + karapace/protobuf/google/type/month.proto | 65 ++ .../protobuf/google/type/phone_number.proto | 113 +++ .../protobuf/google/type/postal_address.proto | 134 +++ .../protobuf/google/type/quaternion.proto | 94 ++ karapace/protobuf/google/type/timeofday.proto | 44 + karapace/protobuf/google/type/type.yaml | 40 + karapace/protobuf/known_dependency.py | 121 ++- karapace/protobuf/schema.py | 88 +- karapace/schema_models.py | 46 +- karapace/utils.py | 5 + tests/integration/test_schema_protobuf.py | 69 +- .../test_protobuf_schema_workaround.py | 0 41 files changed, 4802 insertions(+), 54 deletions(-) create mode 100644 karapace/protobuf/google/protobuf/any.proto create mode 100644 karapace/protobuf/google/protobuf/any_test.proto create mode 100644 karapace/protobuf/google/protobuf/api.proto create mode 100644 karapace/protobuf/google/protobuf/descriptor.proto create mode 100644 karapace/protobuf/google/protobuf/duration.proto create mode 100644 karapace/protobuf/google/protobuf/empty.proto create mode 100644 karapace/protobuf/google/protobuf/field_mask.proto create mode 100644 karapace/protobuf/google/protobuf/map_lite_unittest.proto create mode 100644 karapace/protobuf/google/protobuf/map_proto2_unittest.proto create mode 100644 karapace/protobuf/google/protobuf/map_unittest.proto create mode 100644 karapace/protobuf/google/protobuf/source_context.proto create mode 100644 karapace/protobuf/google/protobuf/struct.proto create mode 100644 karapace/protobuf/google/protobuf/timestamp.proto create mode 100644 karapace/protobuf/google/protobuf/type.proto create mode 100644 karapace/protobuf/google/protobuf/wrappers.proto create mode 100644 karapace/protobuf/google/type/BUILD.bazel create mode 100644 karapace/protobuf/google/type/README.md create mode 100644 karapace/protobuf/google/type/calendar_period.proto create mode 100644 karapace/protobuf/google/type/color.proto create mode 100644 karapace/protobuf/google/type/date.proto create mode 100644 karapace/protobuf/google/type/datetime.proto create mode 100644 karapace/protobuf/google/type/dayofweek.proto create mode 100644 karapace/protobuf/google/type/decimal.proto create mode 100644 karapace/protobuf/google/type/expr.proto create mode 100644 karapace/protobuf/google/type/fraction.proto create mode 100644 karapace/protobuf/google/type/interval.proto create mode 100644 karapace/protobuf/google/type/latlng.proto create mode 100644 karapace/protobuf/google/type/localized_text.proto create mode 100644 karapace/protobuf/google/type/money.proto create mode 100644 karapace/protobuf/google/type/month.proto create mode 100644 karapace/protobuf/google/type/phone_number.proto create mode 100644 karapace/protobuf/google/type/postal_address.proto create mode 100644 karapace/protobuf/google/type/quaternion.proto create mode 100644 karapace/protobuf/google/type/timeofday.proto create mode 100644 karapace/protobuf/google/type/type.yaml create mode 100644 tests/unit/protobuf/test_protobuf_schema_workaround.py diff --git a/karapace/protobuf/google/protobuf/any.proto b/karapace/protobuf/google/protobuf/any.proto new file mode 100644 index 000000000..e2c2042fd --- /dev/null +++ b/karapace/protobuf/google/protobuf/any.proto @@ -0,0 +1,158 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package google.protobuf; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option go_package = "google.golang.org/protobuf/types/known/anypb"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "AnyProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; + +// `Any` contains an arbitrary serialized protocol buffer message along with a +// URL that describes the type of the serialized message. +// +// Protobuf library provides support to pack/unpack Any values in the form +// of utility functions or additional generated methods of the Any type. +// +// Example 1: Pack and unpack a message in C++. +// +// Foo foo = ...; +// Any any; +// any.PackFrom(foo); +// ... +// if (any.UnpackTo(&foo)) { +// ... +// } +// +// Example 2: Pack and unpack a message in Java. +// +// Foo foo = ...; +// Any any = Any.pack(foo); +// ... +// if (any.is(Foo.class)) { +// foo = any.unpack(Foo.class); +// } +// +// Example 3: Pack and unpack a message in Python. +// +// foo = Foo(...) +// any = Any() +// any.Pack(foo) +// ... +// if any.Is(Foo.DESCRIPTOR): +// any.Unpack(foo) +// ... +// +// Example 4: Pack and unpack a message in Go +// +// foo := &pb.Foo{...} +// any, err := anypb.New(foo) +// if err != nil { +// ... +// } +// ... +// foo := &pb.Foo{} +// if err := any.UnmarshalTo(foo); err != nil { +// ... +// } +// +// The pack methods provided by protobuf library will by default use +// 'type.googleapis.com/full.type.name' as the type URL and the unpack +// methods only use the fully qualified type name after the last '/' +// in the type URL, for example "foo.bar.com/x/y.z" will yield type +// name "y.z". +// +// +// JSON +// +// The JSON representation of an `Any` value uses the regular +// representation of the deserialized, embedded message, with an +// additional field `@type` which contains the type URL. Example: +// +// package google.profile; +// message Person { +// string first_name = 1; +// string last_name = 2; +// } +// +// { +// "@type": "type.googleapis.com/google.profile.Person", +// "firstName": , +// "lastName": +// } +// +// If the embedded message type is well-known and has a custom JSON +// representation, that representation will be embedded adding a field +// `value` which holds the custom JSON in addition to the `@type` +// field. Example (for message [google.protobuf.Duration][]): +// +// { +// "@type": "type.googleapis.com/google.protobuf.Duration", +// "value": "1.212s" +// } +// +message Any { + // A URL/resource name that uniquely identifies the type of the serialized + // protocol buffer message. This string must contain at least + // one "/" character. The last segment of the URL's path must represent + // the fully qualified name of the type (as in + // `path/google.protobuf.Duration`). The name should be in a canonical form + // (e.g., leading "." is not accepted). + // + // In practice, teams usually precompile into the binary all types that they + // expect it to use in the context of Any. However, for URLs which use the + // scheme `http`, `https`, or no scheme, one can optionally set up a type + // server that maps type URLs to message definitions as follows: + // + // * If no scheme is provided, `https` is assumed. + // * An HTTP GET on the URL must yield a [google.protobuf.Type][] + // value in binary format, or produce an error. + // * Applications are allowed to cache lookup results based on the + // URL, or have them precompiled into a binary to avoid any + // lookup. Therefore, binary compatibility needs to be preserved + // on changes to types. (Use versioned type names to manage + // breaking changes.) + // + // Note: this functionality is not currently available in the official + // protobuf release, and it is not used for type URLs beginning with + // type.googleapis.com. + // + // Schemes other than `http`, `https` (or the empty scheme) might be + // used with implementation specific semantics. + // + string type_url = 1; + + // Must be a valid serialized protocol buffer of the above specified type. + bytes value = 2; +} diff --git a/karapace/protobuf/google/protobuf/any_test.proto b/karapace/protobuf/google/protobuf/any_test.proto new file mode 100644 index 000000000..256035b44 --- /dev/null +++ b/karapace/protobuf/google/protobuf/any_test.proto @@ -0,0 +1,44 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package protobuf_unittest; + +import "google/protobuf/any.proto"; + +option java_outer_classname = "TestAnyProto"; + +message TestAny { + int32 int32_value = 1; + google.protobuf.Any any_value = 2; + repeated google.protobuf.Any repeated_any_value = 3; + string text = 4; +} diff --git a/karapace/protobuf/google/protobuf/api.proto b/karapace/protobuf/google/protobuf/api.proto new file mode 100644 index 000000000..3d598fc85 --- /dev/null +++ b/karapace/protobuf/google/protobuf/api.proto @@ -0,0 +1,208 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package google.protobuf; + +import "google/protobuf/source_context.proto"; +import "google/protobuf/type.proto"; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "ApiProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; +option go_package = "google.golang.org/protobuf/types/known/apipb"; + +// Api is a light-weight descriptor for an API Interface. +// +// Interfaces are also described as "protocol buffer services" in some contexts, +// such as by the "service" keyword in a .proto file, but they are different +// from API Services, which represent a concrete implementation of an interface +// as opposed to simply a description of methods and bindings. They are also +// sometimes simply referred to as "APIs" in other contexts, such as the name of +// this message itself. See https://cloud.google.com/apis/design/glossary for +// detailed terminology. +message Api { + // The fully qualified name of this interface, including package name + // followed by the interface's simple name. + string name = 1; + + // The methods of this interface, in unspecified order. + repeated Method methods = 2; + + // Any metadata attached to the interface. + repeated Option options = 3; + + // A version string for this interface. If specified, must have the form + // `major-version.minor-version`, as in `1.10`. If the minor version is + // omitted, it defaults to zero. If the entire version field is empty, the + // major version is derived from the package name, as outlined below. If the + // field is not empty, the version in the package name will be verified to be + // consistent with what is provided here. + // + // The versioning schema uses [semantic + // versioning](http://semver.org) where the major version number + // indicates a breaking change and the minor version an additive, + // non-breaking change. Both version numbers are signals to users + // what to expect from different versions, and should be carefully + // chosen based on the product plan. + // + // The major version is also reflected in the package name of the + // interface, which must end in `v`, as in + // `google.feature.v1`. For major versions 0 and 1, the suffix can + // be omitted. Zero major versions must only be used for + // experimental, non-GA interfaces. + // + // + string version = 4; + + // Source context for the protocol buffer service represented by this + // message. + SourceContext source_context = 5; + + // Included interfaces. See [Mixin][]. + repeated Mixin mixins = 6; + + // The source syntax of the service. + Syntax syntax = 7; +} + +// Method represents a method of an API interface. +message Method { + // The simple name of this method. + string name = 1; + + // A URL of the input message type. + string request_type_url = 2; + + // If true, the request is streamed. + bool request_streaming = 3; + + // The URL of the output message type. + string response_type_url = 4; + + // If true, the response is streamed. + bool response_streaming = 5; + + // Any metadata attached to the method. + repeated Option options = 6; + + // The source syntax of this method. + Syntax syntax = 7; +} + +// Declares an API Interface to be included in this interface. The including +// interface must redeclare all the methods from the included interface, but +// documentation and options are inherited as follows: +// +// - If after comment and whitespace stripping, the documentation +// string of the redeclared method is empty, it will be inherited +// from the original method. +// +// - Each annotation belonging to the service config (http, +// visibility) which is not set in the redeclared method will be +// inherited. +// +// - If an http annotation is inherited, the path pattern will be +// modified as follows. Any version prefix will be replaced by the +// version of the including interface plus the [root][] path if +// specified. +// +// Example of a simple mixin: +// +// package google.acl.v1; +// service AccessControl { +// // Get the underlying ACL object. +// rpc GetAcl(GetAclRequest) returns (Acl) { +// option (google.api.http).get = "/v1/{resource=**}:getAcl"; +// } +// } +// +// package google.storage.v2; +// service Storage { +// rpc GetAcl(GetAclRequest) returns (Acl); +// +// // Get a data record. +// rpc GetData(GetDataRequest) returns (Data) { +// option (google.api.http).get = "/v2/{resource=**}"; +// } +// } +// +// Example of a mixin configuration: +// +// apis: +// - name: google.storage.v2.Storage +// mixins: +// - name: google.acl.v1.AccessControl +// +// The mixin construct implies that all methods in `AccessControl` are +// also declared with same name and request/response types in +// `Storage`. A documentation generator or annotation processor will +// see the effective `Storage.GetAcl` method after inheriting +// documentation and annotations as follows: +// +// service Storage { +// // Get the underlying ACL object. +// rpc GetAcl(GetAclRequest) returns (Acl) { +// option (google.api.http).get = "/v2/{resource=**}:getAcl"; +// } +// ... +// } +// +// Note how the version in the path pattern changed from `v1` to `v2`. +// +// If the `root` field in the mixin is specified, it should be a +// relative path under which inherited HTTP paths are placed. Example: +// +// apis: +// - name: google.storage.v2.Storage +// mixins: +// - name: google.acl.v1.AccessControl +// root: acls +// +// This implies the following inherited HTTP annotation: +// +// service Storage { +// // Get the underlying ACL object. +// rpc GetAcl(GetAclRequest) returns (Acl) { +// option (google.api.http).get = "/v2/acls/{resource=**}:getAcl"; +// } +// ... +// } +message Mixin { + // The fully qualified name of the interface which is included. + string name = 1; + + // If non-empty specifies a path under which inherited HTTP paths + // are rooted. + string root = 2; +} diff --git a/karapace/protobuf/google/protobuf/descriptor.proto b/karapace/protobuf/google/protobuf/descriptor.proto new file mode 100644 index 000000000..5b4d06b41 --- /dev/null +++ b/karapace/protobuf/google/protobuf/descriptor.proto @@ -0,0 +1,918 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Author: kenton@google.com (Kenton Varda) +// Based on original Protocol Buffers design by +// Sanjay Ghemawat, Jeff Dean, and others. +// +// The messages in this file describe the definitions found in .proto files. +// A valid .proto file can be translated directly to a FileDescriptorProto +// without any other information (e.g. without reading its imports). + + +syntax = "proto2"; + +package google.protobuf; + +option go_package = "google.golang.org/protobuf/types/descriptorpb"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "DescriptorProtos"; +option csharp_namespace = "Google.Protobuf.Reflection"; +option objc_class_prefix = "GPB"; +option cc_enable_arenas = true; + +// descriptor.proto must be optimized for speed because reflection-based +// algorithms don't work during bootstrapping. +option optimize_for = SPEED; + +// The protocol compiler can output a FileDescriptorSet containing the .proto +// files it parses. +message FileDescriptorSet { + repeated FileDescriptorProto file = 1; +} + +// Describes a complete .proto file. +message FileDescriptorProto { + optional string name = 1; // file name, relative to root of source tree + optional string package = 2; // e.g. "foo", "foo.bar", etc. + + // Names of files imported by this file. + repeated string dependency = 3; + // Indexes of the public imported files in the dependency list above. + repeated int32 public_dependency = 10; + // Indexes of the weak imported files in the dependency list. + // For Google-internal migration only. Do not use. + repeated int32 weak_dependency = 11; + + // All top-level definitions in this file. + repeated DescriptorProto message_type = 4; + repeated EnumDescriptorProto enum_type = 5; + repeated ServiceDescriptorProto service = 6; + repeated FieldDescriptorProto extension = 7; + + optional FileOptions options = 8; + + // This field contains optional information about the original source code. + // You may safely remove this entire field without harming runtime + // functionality of the descriptors -- the information is needed only by + // development tools. + optional SourceCodeInfo source_code_info = 9; + + // The syntax of the proto file. + // The supported values are "proto2" and "proto3". + optional string syntax = 12; +} + +// Describes a message type. +message DescriptorProto { + optional string name = 1; + + repeated FieldDescriptorProto field = 2; + repeated FieldDescriptorProto extension = 6; + + repeated DescriptorProto nested_type = 3; + repeated EnumDescriptorProto enum_type = 4; + + message ExtensionRange { + optional int32 start = 1; // Inclusive. + optional int32 end = 2; // Exclusive. + + optional ExtensionRangeOptions options = 3; + } + repeated ExtensionRange extension_range = 5; + + repeated OneofDescriptorProto oneof_decl = 8; + + optional MessageOptions options = 7; + + // Range of reserved tag numbers. Reserved tag numbers may not be used by + // fields or extension ranges in the same message. Reserved ranges may + // not overlap. + message ReservedRange { + optional int32 start = 1; // Inclusive. + optional int32 end = 2; // Exclusive. + } + repeated ReservedRange reserved_range = 9; + // Reserved field names, which may not be used by fields in the same message. + // A given name may only be reserved once. + repeated string reserved_name = 10; +} + +message ExtensionRangeOptions { + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; +} + +// Describes a field within a message. +message FieldDescriptorProto { + enum Type { + // 0 is reserved for errors. + // Order is weird for historical reasons. + TYPE_DOUBLE = 1; + TYPE_FLOAT = 2; + // Not ZigZag encoded. Negative numbers take 10 bytes. Use TYPE_SINT64 if + // negative values are likely. + TYPE_INT64 = 3; + TYPE_UINT64 = 4; + // Not ZigZag encoded. Negative numbers take 10 bytes. Use TYPE_SINT32 if + // negative values are likely. + TYPE_INT32 = 5; + TYPE_FIXED64 = 6; + TYPE_FIXED32 = 7; + TYPE_BOOL = 8; + TYPE_STRING = 9; + // Tag-delimited aggregate. + // Group type is deprecated and not supported in proto3. However, Proto3 + // implementations should still be able to parse the group wire format and + // treat group fields as unknown fields. + TYPE_GROUP = 10; + TYPE_MESSAGE = 11; // Length-delimited aggregate. + + // New in version 2. + TYPE_BYTES = 12; + TYPE_UINT32 = 13; + TYPE_ENUM = 14; + TYPE_SFIXED32 = 15; + TYPE_SFIXED64 = 16; + TYPE_SINT32 = 17; // Uses ZigZag encoding. + TYPE_SINT64 = 18; // Uses ZigZag encoding. + } + + enum Label { + // 0 is reserved for errors + LABEL_OPTIONAL = 1; + LABEL_REQUIRED = 2; + LABEL_REPEATED = 3; + } + + optional string name = 1; + optional int32 number = 3; + optional Label label = 4; + + // If type_name is set, this need not be set. If both this and type_name + // are set, this must be one of TYPE_ENUM, TYPE_MESSAGE or TYPE_GROUP. + optional Type type = 5; + + // For message and enum types, this is the name of the type. If the name + // starts with a '.', it is fully-qualified. Otherwise, C++-like scoping + // rules are used to find the type (i.e. first the nested types within this + // message are searched, then within the parent, on up to the root + // namespace). + optional string type_name = 6; + + // For extensions, this is the name of the type being extended. It is + // resolved in the same manner as type_name. + optional string extendee = 2; + + // For numeric types, contains the original text representation of the value. + // For booleans, "true" or "false". + // For strings, contains the default text contents (not escaped in any way). + // For bytes, contains the C escaped value. All bytes >= 128 are escaped. + optional string default_value = 7; + + // If set, gives the index of a oneof in the containing type's oneof_decl + // list. This field is a member of that oneof. + optional int32 oneof_index = 9; + + // JSON name of this field. The value is set by protocol compiler. If the + // user has set a "json_name" option on this field, that option's value + // will be used. Otherwise, it's deduced from the field's name by converting + // it to camelCase. + optional string json_name = 10; + + optional FieldOptions options = 8; + + // If true, this is a proto3 "optional". When a proto3 field is optional, it + // tracks presence regardless of field type. + // + // When proto3_optional is true, this field must be belong to a oneof to + // signal to old proto3 clients that presence is tracked for this field. This + // oneof is known as a "synthetic" oneof, and this field must be its sole + // member (each proto3 optional field gets its own synthetic oneof). Synthetic + // oneofs exist in the descriptor only, and do not generate any API. Synthetic + // oneofs must be ordered after all "real" oneofs. + // + // For message fields, proto3_optional doesn't create any semantic change, + // since non-repeated message fields always track presence. However it still + // indicates the semantic detail of whether the user wrote "optional" or not. + // This can be useful for round-tripping the .proto file. For consistency we + // give message fields a synthetic oneof also, even though it is not required + // to track presence. This is especially important because the parser can't + // tell if a field is a message or an enum, so it must always create a + // synthetic oneof. + // + // Proto2 optional fields do not set this flag, because they already indicate + // optional with `LABEL_OPTIONAL`. + optional bool proto3_optional = 17; +} + +// Describes a oneof. +message OneofDescriptorProto { + optional string name = 1; + optional OneofOptions options = 2; +} + +// Describes an enum type. +message EnumDescriptorProto { + optional string name = 1; + + repeated EnumValueDescriptorProto value = 2; + + optional EnumOptions options = 3; + + // Range of reserved numeric values. Reserved values may not be used by + // entries in the same enum. Reserved ranges may not overlap. + // + // Note that this is distinct from DescriptorProto.ReservedRange in that it + // is inclusive such that it can appropriately represent the entire int32 + // domain. + message EnumReservedRange { + optional int32 start = 1; // Inclusive. + optional int32 end = 2; // Inclusive. + } + + // Range of reserved numeric values. Reserved numeric values may not be used + // by enum values in the same enum declaration. Reserved ranges may not + // overlap. + repeated EnumReservedRange reserved_range = 4; + + // Reserved enum value names, which may not be reused. A given name may only + // be reserved once. + repeated string reserved_name = 5; +} + +// Describes a value within an enum. +message EnumValueDescriptorProto { + optional string name = 1; + optional int32 number = 2; + + optional EnumValueOptions options = 3; +} + +// Describes a service. +message ServiceDescriptorProto { + optional string name = 1; + repeated MethodDescriptorProto method = 2; + + optional ServiceOptions options = 3; +} + +// Describes a method of a service. +message MethodDescriptorProto { + optional string name = 1; + + // Input and output type names. These are resolved in the same way as + // FieldDescriptorProto.type_name, but must refer to a message type. + optional string input_type = 2; + optional string output_type = 3; + + optional MethodOptions options = 4; + + // Identifies if client streams multiple client messages + optional bool client_streaming = 5 [default = false]; + // Identifies if server streams multiple server messages + optional bool server_streaming = 6 [default = false]; +} + + +// =================================================================== +// Options + +// Each of the definitions above may have "options" attached. These are +// just annotations which may cause code to be generated slightly differently +// or may contain hints for code that manipulates protocol messages. +// +// Clients may define custom options as extensions of the *Options messages. +// These extensions may not yet be known at parsing time, so the parser cannot +// store the values in them. Instead it stores them in a field in the *Options +// message called uninterpreted_option. This field must have the same name +// across all *Options messages. We then use this field to populate the +// extensions when we build a descriptor, at which point all protos have been +// parsed and so all extensions are known. +// +// Extension numbers for custom options may be chosen as follows: +// * For options which will only be used within a single application or +// organization, or for experimental options, use field numbers 50000 +// through 99999. It is up to you to ensure that you do not use the +// same number for multiple options. +// * For options which will be published and used publicly by multiple +// independent entities, e-mail protobuf-global-extension-registry@google.com +// to reserve extension numbers. Simply provide your project name (e.g. +// Objective-C plugin) and your project website (if available) -- there's no +// need to explain how you intend to use them. Usually you only need one +// extension number. You can declare multiple options with only one extension +// number by putting them in a sub-message. See the Custom Options section of +// the docs for examples: +// https://developers.google.com/protocol-buffers/docs/proto#options +// If this turns out to be popular, a web service will be set up +// to automatically assign option numbers. + +message FileOptions { + + // Sets the Java package where classes generated from this .proto will be + // placed. By default, the proto package is used, but this is often + // inappropriate because proto packages do not normally start with backwards + // domain names. + optional string java_package = 1; + + + // Controls the name of the wrapper Java class generated for the .proto file. + // That class will always contain the .proto file's getDescriptor() method as + // well as any top-level extensions defined in the .proto file. + // If java_multiple_files is disabled, then all the other classes from the + // .proto file will be nested inside the single wrapper outer class. + optional string java_outer_classname = 8; + + // If enabled, then the Java code generator will generate a separate .java + // file for each top-level message, enum, and service defined in the .proto + // file. Thus, these types will *not* be nested inside the wrapper class + // named by java_outer_classname. However, the wrapper class will still be + // generated to contain the file's getDescriptor() method as well as any + // top-level extensions defined in the file. + optional bool java_multiple_files = 10 [default = false]; + + // This option does nothing. + optional bool java_generate_equals_and_hash = 20 [deprecated=true]; + + // If set true, then the Java2 code generator will generate code that + // throws an exception whenever an attempt is made to assign a non-UTF-8 + // byte sequence to a string field. + // Message reflection will do the same. + // However, an extension field still accepts non-UTF-8 byte sequences. + // This option has no effect on when used with the lite runtime. + optional bool java_string_check_utf8 = 27 [default = false]; + + + // Generated classes can be optimized for speed or code size. + enum OptimizeMode { + SPEED = 1; // Generate complete code for parsing, serialization, + // etc. + CODE_SIZE = 2; // Use ReflectionOps to implement these methods. + LITE_RUNTIME = 3; // Generate code using MessageLite and the lite runtime. + } + optional OptimizeMode optimize_for = 9 [default = SPEED]; + + // Sets the Go package where structs generated from this .proto will be + // placed. If omitted, the Go package will be derived from the following: + // - The basename of the package import path, if provided. + // - Otherwise, the package statement in the .proto file, if present. + // - Otherwise, the basename of the .proto file, without extension. + optional string go_package = 11; + + + + + // Should generic services be generated in each language? "Generic" services + // are not specific to any particular RPC system. They are generated by the + // main code generators in each language (without additional plugins). + // Generic services were the only kind of service generation supported by + // early versions of google.protobuf. + // + // Generic services are now considered deprecated in favor of using plugins + // that generate code specific to your particular RPC system. Therefore, + // these default to false. Old code which depends on generic services should + // explicitly set them to true. + optional bool cc_generic_services = 16 [default = false]; + optional bool java_generic_services = 17 [default = false]; + optional bool py_generic_services = 18 [default = false]; + optional bool php_generic_services = 42 [default = false]; + + // Is this file deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for everything in the file, or it will be completely ignored; in the very + // least, this is a formalization for deprecating files. + optional bool deprecated = 23 [default = false]; + + // Enables the use of arenas for the proto messages in this file. This applies + // only to generated classes for C++. + optional bool cc_enable_arenas = 31 [default = true]; + + + // Sets the objective c class prefix which is prepended to all objective c + // generated classes from this .proto. There is no default. + optional string objc_class_prefix = 36; + + // Namespace for generated classes; defaults to the package. + optional string csharp_namespace = 37; + + // By default Swift generators will take the proto package and CamelCase it + // replacing '.' with underscore and use that to prefix the types/symbols + // defined. When this options is provided, they will use this value instead + // to prefix the types/symbols defined. + optional string swift_prefix = 39; + + // Sets the php class prefix which is prepended to all php generated classes + // from this .proto. Default is empty. + optional string php_class_prefix = 40; + + // Use this option to change the namespace of php generated classes. Default + // is empty. When this option is empty, the package name will be used for + // determining the namespace. + optional string php_namespace = 41; + + // Use this option to change the namespace of php generated metadata classes. + // Default is empty. When this option is empty, the proto file name will be + // used for determining the namespace. + optional string php_metadata_namespace = 44; + + // Use this option to change the package of ruby generated classes. Default + // is empty. When this option is not set, the package name will be used for + // determining the ruby package. + optional string ruby_package = 45; + + + // The parser stores options it doesn't recognize here. + // See the documentation for the "Options" section above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. + // See the documentation for the "Options" section above. + extensions 1000 to max; + + reserved 38; +} + +message MessageOptions { + // Set true to use the old proto1 MessageSet wire format for extensions. + // This is provided for backwards-compatibility with the MessageSet wire + // format. You should not use this for any other reason: It's less + // efficient, has fewer features, and is more complicated. + // + // The message must be defined exactly as follows: + // message Foo { + // option message_set_wire_format = true; + // extensions 4 to max; + // } + // Note that the message cannot have any defined fields; MessageSets only + // have extensions. + // + // All extensions of your type must be singular messages; e.g. they cannot + // be int32s, enums, or repeated messages. + // + // Because this is an option, the above two restrictions are not enforced by + // the protocol compiler. + optional bool message_set_wire_format = 1 [default = false]; + + // Disables the generation of the standard "descriptor()" accessor, which can + // conflict with a field of the same name. This is meant to make migration + // from proto1 easier; new code should avoid fields named "descriptor". + optional bool no_standard_descriptor_accessor = 2 [default = false]; + + // Is this message deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for the message, or it will be completely ignored; in the very least, + // this is a formalization for deprecating messages. + optional bool deprecated = 3 [default = false]; + + reserved 4, 5, 6; + + // Whether the message is an automatically generated map entry type for the + // maps field. + // + // For maps fields: + // map map_field = 1; + // The parsed descriptor looks like: + // message MapFieldEntry { + // option map_entry = true; + // optional KeyType key = 1; + // optional ValueType value = 2; + // } + // repeated MapFieldEntry map_field = 1; + // + // Implementations may choose not to generate the map_entry=true message, but + // use a native map in the target language to hold the keys and values. + // The reflection APIs in such implementations still need to work as + // if the field is a repeated message field. + // + // NOTE: Do not set the option in .proto files. Always use the maps syntax + // instead. The option should only be implicitly set by the proto compiler + // parser. + optional bool map_entry = 7; + + reserved 8; // javalite_serializable + reserved 9; // javanano_as_lite + + + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; +} + +message FieldOptions { + // The ctype option instructs the C++ code generator to use a different + // representation of the field than it normally would. See the specific + // options below. This option is not yet implemented in the open source + // release -- sorry, we'll try to include it in a future version! + optional CType ctype = 1 [default = STRING]; + enum CType { + // Default mode. + STRING = 0; + + CORD = 1; + + STRING_PIECE = 2; + } + // The packed option can be enabled for repeated primitive fields to enable + // a more efficient representation on the wire. Rather than repeatedly + // writing the tag and type for each element, the entire array is encoded as + // a single length-delimited blob. In proto3, only explicit setting it to + // false will avoid using packed encoding. + optional bool packed = 2; + + // The jstype option determines the JavaScript type used for values of the + // field. The option is permitted only for 64 bit integral and fixed types + // (int64, uint64, sint64, fixed64, sfixed64). A field with jstype JS_STRING + // is represented as JavaScript string, which avoids loss of precision that + // can happen when a large value is converted to a floating point JavaScript. + // Specifying JS_NUMBER for the jstype causes the generated JavaScript code to + // use the JavaScript "number" type. The behavior of the default option + // JS_NORMAL is implementation dependent. + // + // This option is an enum to permit additional types to be added, e.g. + // goog.math.Integer. + optional JSType jstype = 6 [default = JS_NORMAL]; + enum JSType { + // Use the default type. + JS_NORMAL = 0; + + // Use JavaScript strings. + JS_STRING = 1; + + // Use JavaScript numbers. + JS_NUMBER = 2; + } + + // Should this field be parsed lazily? Lazy applies only to message-type + // fields. It means that when the outer message is initially parsed, the + // inner message's contents will not be parsed but instead stored in encoded + // form. The inner message will actually be parsed when it is first accessed. + // + // This is only a hint. Implementations are free to choose whether to use + // eager or lazy parsing regardless of the value of this option. However, + // setting this option true suggests that the protocol author believes that + // using lazy parsing on this field is worth the additional bookkeeping + // overhead typically needed to implement it. + // + // This option does not affect the public interface of any generated code; + // all method signatures remain the same. Furthermore, thread-safety of the + // interface is not affected by this option; const methods remain safe to + // call from multiple threads concurrently, while non-const methods continue + // to require exclusive access. + // + // + // Note that implementations may choose not to check required fields within + // a lazy sub-message. That is, calling IsInitialized() on the outer message + // may return true even if the inner message has missing required fields. + // This is necessary because otherwise the inner message would have to be + // parsed in order to perform the check, defeating the purpose of lazy + // parsing. An implementation which chooses not to check required fields + // must be consistent about it. That is, for any particular sub-message, the + // implementation must either *always* check its required fields, or *never* + // check its required fields, regardless of whether or not the message has + // been parsed. + // + // As of May 2022, lazy verifies the contents of the byte stream during + // parsing. An invalid byte stream will cause the overall parsing to fail. + optional bool lazy = 5 [default = false]; + + // unverified_lazy does no correctness checks on the byte stream. This should + // only be used where lazy with verification is prohibitive for performance + // reasons. + optional bool unverified_lazy = 15 [default = false]; + + // Is this field deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for accessors, or it will be completely ignored; in the very least, this + // is a formalization for deprecating fields. + optional bool deprecated = 3 [default = false]; + + // For Google-internal migration only. Do not use. + optional bool weak = 10 [default = false]; + + + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; + + reserved 4; // removed jtype +} + +message OneofOptions { + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; +} + +message EnumOptions { + + // Set this option to true to allow mapping different tag names to the same + // value. + optional bool allow_alias = 2; + + // Is this enum deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for the enum, or it will be completely ignored; in the very least, this + // is a formalization for deprecating enums. + optional bool deprecated = 3 [default = false]; + + reserved 5; // javanano_as_lite + + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; +} + +message EnumValueOptions { + // Is this enum value deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for the enum value, or it will be completely ignored; in the very least, + // this is a formalization for deprecating enum values. + optional bool deprecated = 1 [default = false]; + + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; +} + +message ServiceOptions { + + // Note: Field numbers 1 through 32 are reserved for Google's internal RPC + // framework. We apologize for hoarding these numbers to ourselves, but + // we were already using them long before we decided to release Protocol + // Buffers. + + // Is this service deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for the service, or it will be completely ignored; in the very least, + // this is a formalization for deprecating services. + optional bool deprecated = 33 [default = false]; + + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; +} + +message MethodOptions { + + // Note: Field numbers 1 through 32 are reserved for Google's internal RPC + // framework. We apologize for hoarding these numbers to ourselves, but + // we were already using them long before we decided to release Protocol + // Buffers. + + // Is this method deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for the method, or it will be completely ignored; in the very least, + // this is a formalization for deprecating methods. + optional bool deprecated = 33 [default = false]; + + // Is this method side-effect-free (or safe in HTTP parlance), or idempotent, + // or neither? HTTP based RPC implementation may choose GET verb for safe + // methods, and PUT verb for idempotent methods instead of the default POST. + enum IdempotencyLevel { + IDEMPOTENCY_UNKNOWN = 0; + NO_SIDE_EFFECTS = 1; // implies idempotent + IDEMPOTENT = 2; // idempotent, but may have side effects + } + optional IdempotencyLevel idempotency_level = 34 + [default = IDEMPOTENCY_UNKNOWN]; + + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; +} + + +// A message representing a option the parser does not recognize. This only +// appears in options protos created by the compiler::Parser class. +// DescriptorPool resolves these when building Descriptor objects. Therefore, +// options protos in descriptor objects (e.g. returned by Descriptor::options(), +// or produced by Descriptor::CopyTo()) will never have UninterpretedOptions +// in them. +message UninterpretedOption { + // The name of the uninterpreted option. Each string represents a segment in + // a dot-separated name. is_extension is true iff a segment represents an + // extension (denoted with parentheses in options specs in .proto files). + // E.g.,{ ["foo", false], ["bar.baz", true], ["moo", false] } represents + // "foo.(bar.baz).moo". + message NamePart { + required string name_part = 1; + required bool is_extension = 2; + } + repeated NamePart name = 2; + + // The value of the uninterpreted option, in whatever type the tokenizer + // identified it as during parsing. Exactly one of these should be set. + optional string identifier_value = 3; + optional uint64 positive_int_value = 4; + optional int64 negative_int_value = 5; + optional double double_value = 6; + optional bytes string_value = 7; + optional string aggregate_value = 8; +} + +// =================================================================== +// Optional source code info + +// Encapsulates information about the original source file from which a +// FileDescriptorProto was generated. +message SourceCodeInfo { + // A Location identifies a piece of source code in a .proto file which + // corresponds to a particular definition. This information is intended + // to be useful to IDEs, code indexers, documentation generators, and similar + // tools. + // + // For example, say we have a file like: + // message Foo { + // optional string foo = 1; + // } + // Let's look at just the field definition: + // optional string foo = 1; + // ^ ^^ ^^ ^ ^^^ + // a bc de f ghi + // We have the following locations: + // span path represents + // [a,i) [ 4, 0, 2, 0 ] The whole field definition. + // [a,b) [ 4, 0, 2, 0, 4 ] The label (optional). + // [c,d) [ 4, 0, 2, 0, 5 ] The type (string). + // [e,f) [ 4, 0, 2, 0, 1 ] The name (foo). + // [g,h) [ 4, 0, 2, 0, 3 ] The number (1). + // + // Notes: + // - A location may refer to a repeated field itself (i.e. not to any + // particular index within it). This is used whenever a set of elements are + // logically enclosed in a single code segment. For example, an entire + // extend block (possibly containing multiple extension definitions) will + // have an outer location whose path refers to the "extensions" repeated + // field without an index. + // - Multiple locations may have the same path. This happens when a single + // logical declaration is spread out across multiple places. The most + // obvious example is the "extend" block again -- there may be multiple + // extend blocks in the same scope, each of which will have the same path. + // - A location's span is not always a subset of its parent's span. For + // example, the "extendee" of an extension declaration appears at the + // beginning of the "extend" block and is shared by all extensions within + // the block. + // - Just because a location's span is a subset of some other location's span + // does not mean that it is a descendant. For example, a "group" defines + // both a type and a field in a single declaration. Thus, the locations + // corresponding to the type and field and their components will overlap. + // - Code which tries to interpret locations should probably be designed to + // ignore those that it doesn't understand, as more types of locations could + // be recorded in the future. + repeated Location location = 1; + message Location { + // Identifies which part of the FileDescriptorProto was defined at this + // location. + // + // Each element is a field number or an index. They form a path from + // the root FileDescriptorProto to the place where the definition occurs. + // For example, this path: + // [ 4, 3, 2, 7, 1 ] + // refers to: + // file.message_type(3) // 4, 3 + // .field(7) // 2, 7 + // .name() // 1 + // This is because FileDescriptorProto.message_type has field number 4: + // repeated DescriptorProto message_type = 4; + // and DescriptorProto.field has field number 2: + // repeated FieldDescriptorProto field = 2; + // and FieldDescriptorProto.name has field number 1: + // optional string name = 1; + // + // Thus, the above path gives the location of a field name. If we removed + // the last element: + // [ 4, 3, 2, 7 ] + // this path refers to the whole field declaration (from the beginning + // of the label to the terminating semicolon). + repeated int32 path = 1 [packed = true]; + + // Always has exactly three or four elements: start line, start column, + // end line (optional, otherwise assumed same as start line), end column. + // These are packed into a single field for efficiency. Note that line + // and column numbers are zero-based -- typically you will want to add + // 1 to each before displaying to a user. + repeated int32 span = 2 [packed = true]; + + // If this SourceCodeInfo represents a complete declaration, these are any + // comments appearing before and after the declaration which appear to be + // attached to the declaration. + // + // A series of line comments appearing on consecutive lines, with no other + // tokens appearing on those lines, will be treated as a single comment. + // + // leading_detached_comments will keep paragraphs of comments that appear + // before (but not connected to) the current element. Each paragraph, + // separated by empty lines, will be one comment element in the repeated + // field. + // + // Only the comment content is provided; comment markers (e.g. //) are + // stripped out. For block comments, leading whitespace and an asterisk + // will be stripped from the beginning of each line other than the first. + // Newlines are included in the output. + // + // Examples: + // + // optional int32 foo = 1; // Comment attached to foo. + // // Comment attached to bar. + // optional int32 bar = 2; + // + // optional string baz = 3; + // // Comment attached to baz. + // // Another line attached to baz. + // + // // Comment attached to moo. + // // + // // Another line attached to moo. + // optional double moo = 4; + // + // // Detached comment for corge. This is not leading or trailing comments + // // to moo or corge because there are blank lines separating it from + // // both. + // + // // Detached comment for corge paragraph 2. + // + // optional string corge = 5; + // /* Block comment attached + // * to corge. Leading asterisks + // * will be removed. */ + // /* Block comment attached to + // * grault. */ + // optional int32 grault = 6; + // + // // ignored detached comments. + optional string leading_comments = 3; + optional string trailing_comments = 4; + repeated string leading_detached_comments = 6; + } +} + +// Describes the relationship between generated code and its original source +// file. A GeneratedCodeInfo message is associated with only one generated +// source file, but may contain references to different source .proto files. +message GeneratedCodeInfo { + // An Annotation connects some span of text in generated code to an element + // of its generating .proto file. + repeated Annotation annotation = 1; + message Annotation { + // Identifies the element in the original source .proto file. This field + // is formatted the same as SourceCodeInfo.Location.path. + repeated int32 path = 1 [packed = true]; + + // Identifies the filesystem path to the original source .proto. + optional string source_file = 2; + + // Identifies the starting offset in bytes in the generated code + // that relates to the identified object. + optional int32 begin = 3; + + // Identifies the ending offset in bytes in the generated code that + // relates to the identified offset. The end offset should be one past + // the last relevant byte (so the length of the text = end - begin). + optional int32 end = 4; + } +} diff --git a/karapace/protobuf/google/protobuf/duration.proto b/karapace/protobuf/google/protobuf/duration.proto new file mode 100644 index 000000000..81c3e369f --- /dev/null +++ b/karapace/protobuf/google/protobuf/duration.proto @@ -0,0 +1,116 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package google.protobuf; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option cc_enable_arenas = true; +option go_package = "google.golang.org/protobuf/types/known/durationpb"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "DurationProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; + +// A Duration represents a signed, fixed-length span of time represented +// as a count of seconds and fractions of seconds at nanosecond +// resolution. It is independent of any calendar and concepts like "day" +// or "month". It is related to Timestamp in that the difference between +// two Timestamp values is a Duration and it can be added or subtracted +// from a Timestamp. Range is approximately +-10,000 years. +// +// # Examples +// +// Example 1: Compute Duration from two Timestamps in pseudo code. +// +// Timestamp start = ...; +// Timestamp end = ...; +// Duration duration = ...; +// +// duration.seconds = end.seconds - start.seconds; +// duration.nanos = end.nanos - start.nanos; +// +// if (duration.seconds < 0 && duration.nanos > 0) { +// duration.seconds += 1; +// duration.nanos -= 1000000000; +// } else if (duration.seconds > 0 && duration.nanos < 0) { +// duration.seconds -= 1; +// duration.nanos += 1000000000; +// } +// +// Example 2: Compute Timestamp from Timestamp + Duration in pseudo code. +// +// Timestamp start = ...; +// Duration duration = ...; +// Timestamp end = ...; +// +// end.seconds = start.seconds + duration.seconds; +// end.nanos = start.nanos + duration.nanos; +// +// if (end.nanos < 0) { +// end.seconds -= 1; +// end.nanos += 1000000000; +// } else if (end.nanos >= 1000000000) { +// end.seconds += 1; +// end.nanos -= 1000000000; +// } +// +// Example 3: Compute Duration from datetime.timedelta in Python. +// +// td = datetime.timedelta(days=3, minutes=10) +// duration = Duration() +// duration.FromTimedelta(td) +// +// # JSON Mapping +// +// In JSON format, the Duration type is encoded as a string rather than an +// object, where the string ends in the suffix "s" (indicating seconds) and +// is preceded by the number of seconds, with nanoseconds expressed as +// fractional seconds. For example, 3 seconds with 0 nanoseconds should be +// encoded in JSON format as "3s", while 3 seconds and 1 nanosecond should +// be expressed in JSON format as "3.000000001s", and 3 seconds and 1 +// microsecond should be expressed in JSON format as "3.000001s". +// +// +message Duration { + // Signed seconds of the span of time. Must be from -315,576,000,000 + // to +315,576,000,000 inclusive. Note: these bounds are computed from: + // 60 sec/min * 60 min/hr * 24 hr/day * 365.25 days/year * 10000 years + int64 seconds = 1; + + // Signed fractions of a second at nanosecond resolution of the span + // of time. Durations less than one second are represented with a 0 + // `seconds` field and a positive or negative `nanos` field. For durations + // of one second or more, a non-zero value for the `nanos` field must be + // of the same sign as the `seconds` field. Must be from -999,999,999 + // to +999,999,999 inclusive. + int32 nanos = 2; +} diff --git a/karapace/protobuf/google/protobuf/empty.proto b/karapace/protobuf/google/protobuf/empty.proto new file mode 100644 index 000000000..222746219 --- /dev/null +++ b/karapace/protobuf/google/protobuf/empty.proto @@ -0,0 +1,51 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package google.protobuf; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option go_package = "google.golang.org/protobuf/types/known/emptypb"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "EmptyProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; +option cc_enable_arenas = true; + +// A generic empty message that you can re-use to avoid defining duplicated +// empty messages in your APIs. A typical example is to use it as the request +// or the response type of an API method. For instance: +// +// service Foo { +// rpc Bar(google.protobuf.Empty) returns (google.protobuf.Empty); +// } +// +message Empty {} diff --git a/karapace/protobuf/google/protobuf/field_mask.proto b/karapace/protobuf/google/protobuf/field_mask.proto new file mode 100644 index 000000000..6b5104f18 --- /dev/null +++ b/karapace/protobuf/google/protobuf/field_mask.proto @@ -0,0 +1,245 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package google.protobuf; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "FieldMaskProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; +option go_package = "google.golang.org/protobuf/types/known/fieldmaskpb"; +option cc_enable_arenas = true; + +// `FieldMask` represents a set of symbolic field paths, for example: +// +// paths: "f.a" +// paths: "f.b.d" +// +// Here `f` represents a field in some root message, `a` and `b` +// fields in the message found in `f`, and `d` a field found in the +// message in `f.b`. +// +// Field masks are used to specify a subset of fields that should be +// returned by a get operation or modified by an update operation. +// Field masks also have a custom JSON encoding (see below). +// +// # Field Masks in Projections +// +// When used in the context of a projection, a response message or +// sub-message is filtered by the API to only contain those fields as +// specified in the mask. For example, if the mask in the previous +// example is applied to a response message as follows: +// +// f { +// a : 22 +// b { +// d : 1 +// x : 2 +// } +// y : 13 +// } +// z: 8 +// +// The result will not contain specific values for fields x,y and z +// (their value will be set to the default, and omitted in proto text +// output): +// +// +// f { +// a : 22 +// b { +// d : 1 +// } +// } +// +// A repeated field is not allowed except at the last position of a +// paths string. +// +// If a FieldMask object is not present in a get operation, the +// operation applies to all fields (as if a FieldMask of all fields +// had been specified). +// +// Note that a field mask does not necessarily apply to the +// top-level response message. In case of a REST get operation, the +// field mask applies directly to the response, but in case of a REST +// list operation, the mask instead applies to each individual message +// in the returned resource list. In case of a REST custom method, +// other definitions may be used. Where the mask applies will be +// clearly documented together with its declaration in the API. In +// any case, the effect on the returned resource/resources is required +// behavior for APIs. +// +// # Field Masks in Update Operations +// +// A field mask in update operations specifies which fields of the +// targeted resource are going to be updated. The API is required +// to only change the values of the fields as specified in the mask +// and leave the others untouched. If a resource is passed in to +// describe the updated values, the API ignores the values of all +// fields not covered by the mask. +// +// If a repeated field is specified for an update operation, new values will +// be appended to the existing repeated field in the target resource. Note that +// a repeated field is only allowed in the last position of a `paths` string. +// +// If a sub-message is specified in the last position of the field mask for an +// update operation, then new value will be merged into the existing sub-message +// in the target resource. +// +// For example, given the target message: +// +// f { +// b { +// d: 1 +// x: 2 +// } +// c: [1] +// } +// +// And an update message: +// +// f { +// b { +// d: 10 +// } +// c: [2] +// } +// +// then if the field mask is: +// +// paths: ["f.b", "f.c"] +// +// then the result will be: +// +// f { +// b { +// d: 10 +// x: 2 +// } +// c: [1, 2] +// } +// +// An implementation may provide options to override this default behavior for +// repeated and message fields. +// +// In order to reset a field's value to the default, the field must +// be in the mask and set to the default value in the provided resource. +// Hence, in order to reset all fields of a resource, provide a default +// instance of the resource and set all fields in the mask, or do +// not provide a mask as described below. +// +// If a field mask is not present on update, the operation applies to +// all fields (as if a field mask of all fields has been specified). +// Note that in the presence of schema evolution, this may mean that +// fields the client does not know and has therefore not filled into +// the request will be reset to their default. If this is unwanted +// behavior, a specific service may require a client to always specify +// a field mask, producing an error if not. +// +// As with get operations, the location of the resource which +// describes the updated values in the request message depends on the +// operation kind. In any case, the effect of the field mask is +// required to be honored by the API. +// +// ## Considerations for HTTP REST +// +// The HTTP kind of an update operation which uses a field mask must +// be set to PATCH instead of PUT in order to satisfy HTTP semantics +// (PUT must only be used for full updates). +// +// # JSON Encoding of Field Masks +// +// In JSON, a field mask is encoded as a single string where paths are +// separated by a comma. Fields name in each path are converted +// to/from lower-camel naming conventions. +// +// As an example, consider the following message declarations: +// +// message Profile { +// User user = 1; +// Photo photo = 2; +// } +// message User { +// string display_name = 1; +// string address = 2; +// } +// +// In proto a field mask for `Profile` may look as such: +// +// mask { +// paths: "user.display_name" +// paths: "photo" +// } +// +// In JSON, the same mask is represented as below: +// +// { +// mask: "user.displayName,photo" +// } +// +// # Field Masks and Oneof Fields +// +// Field masks treat fields in oneofs just as regular fields. Consider the +// following message: +// +// message SampleMessage { +// oneof test_oneof { +// string name = 4; +// SubMessage sub_message = 9; +// } +// } +// +// The field mask can be: +// +// mask { +// paths: "name" +// } +// +// Or: +// +// mask { +// paths: "sub_message" +// } +// +// Note that oneof type names ("test_oneof" in this case) cannot be used in +// paths. +// +// ## Field Mask Verification +// +// The implementation of any API method which has a FieldMask type field in the +// request should verify the included field paths, and return an +// `INVALID_ARGUMENT` error if any path is unmappable. +message FieldMask { + // The set of field mask paths. + repeated string paths = 1; +} diff --git a/karapace/protobuf/google/protobuf/map_lite_unittest.proto b/karapace/protobuf/google/protobuf/map_lite_unittest.proto new file mode 100644 index 000000000..7f104315c --- /dev/null +++ b/karapace/protobuf/google/protobuf/map_lite_unittest.proto @@ -0,0 +1,131 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto2"; + +package protobuf_unittest; + +import "google/protobuf/unittest_lite.proto"; + +option cc_enable_arenas = true; +option optimize_for = LITE_RUNTIME; + +message TestMapLite { + map map_int32_int32 = 1; + map map_int64_int64 = 2; + map map_uint32_uint32 = 3; + map map_uint64_uint64 = 4; + map map_sint32_sint32 = 5; + map map_sint64_sint64 = 6; + map map_fixed32_fixed32 = 7; + map map_fixed64_fixed64 = 8; + map map_sfixed32_sfixed32 = 9; + map map_sfixed64_sfixed64 = 10; + map map_int32_float = 11; + map map_int32_double = 12; + map map_bool_bool = 13; + map map_string_string = 14; + map map_int32_bytes = 15; + map map_int32_enum = 16; + map map_int32_foreign_message = 17; + map teboring = 18; +} + +message TestArenaMapLite { + map map_int32_int32 = 1; + map map_int64_int64 = 2; + map map_uint32_uint32 = 3; + map map_uint64_uint64 = 4; + map map_sint32_sint32 = 5; + map map_sint64_sint64 = 6; + map map_fixed32_fixed32 = 7; + map map_fixed64_fixed64 = 8; + map map_sfixed32_sfixed32 = 9; + map map_sfixed64_sfixed64 = 10; + map map_int32_float = 11; + map map_int32_double = 12; + map map_bool_bool = 13; + map map_string_string = 14; + map map_int32_bytes = 15; + map map_int32_enum = 16; + map map_int32_foreign_message = 17; +} + +// Test embedded message with required fields +message TestRequiredMessageMapLite { + map map_field = 1; +} + +message TestEnumMapLite { + map known_map_field = 101; + map unknown_map_field = 102; +} + +message TestEnumMapPlusExtraLite { + map known_map_field = 101; + map unknown_map_field = 102; +} + +message TestMessageMapLite { + map map_int32_message = 1; +} + +enum Proto2MapEnumLite { + PROTO2_MAP_ENUM_FOO_LITE = 0; + PROTO2_MAP_ENUM_BAR_LITE = 1; + PROTO2_MAP_ENUM_BAZ_LITE = 2; +} + +enum Proto2MapEnumPlusExtraLite { + E_PROTO2_MAP_ENUM_FOO_LITE = 0; + E_PROTO2_MAP_ENUM_BAR_LITE = 1; + E_PROTO2_MAP_ENUM_BAZ_LITE = 2; + E_PROTO2_MAP_ENUM_EXTRA_LITE = 3; +} + +enum MapEnumLite { + MAP_ENUM_FOO_LITE = 0; + MAP_ENUM_BAR_LITE = 1; + MAP_ENUM_BAZ_LITE = 2; +} + +message TestRequiredLite { + required int32 a = 1; + required int32 b = 2; + required int32 c = 3; + + extend TestAllExtensionsLite { + optional TestRequiredLite single = 1000; + } +} + +message ForeignMessageArenaLite { + optional int32 c = 1; +} diff --git a/karapace/protobuf/google/protobuf/map_proto2_unittest.proto b/karapace/protobuf/google/protobuf/map_proto2_unittest.proto new file mode 100644 index 000000000..20d58f903 --- /dev/null +++ b/karapace/protobuf/google/protobuf/map_proto2_unittest.proto @@ -0,0 +1,91 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto2"; +option cc_enable_arenas = true; + +import "google/protobuf/unittest_import.proto"; + +// We don't put this in a package within proto2 because we need to make sure +// that the generated code doesn't depend on being in the proto2 namespace. +// In map_test_util.h we do "using namespace unittest = protobuf_unittest". +package protobuf_unittest; + +enum Proto2MapEnum { + PROTO2_MAP_ENUM_FOO = 0; + PROTO2_MAP_ENUM_BAR = 1; + PROTO2_MAP_ENUM_BAZ = 2; +} + +enum Proto2MapEnumPlusExtra { + E_PROTO2_MAP_ENUM_FOO = 0; + E_PROTO2_MAP_ENUM_BAR = 1; + E_PROTO2_MAP_ENUM_BAZ = 2; + E_PROTO2_MAP_ENUM_EXTRA = 3; +} + +message TestEnumMap { + map known_map_field = 101; + map unknown_map_field = 102; +} + +message TestEnumMapPlusExtra { + map known_map_field = 101; + map unknown_map_field = 102; +} + +message TestImportEnumMap { + map import_enum_amp = 1; +} + +message TestIntIntMap { + map m = 1; +} + +// Test all key types: string, plus the non-floating-point scalars. +message TestMaps { + map m_int32 = 1; + map m_int64 = 2; + map m_uint32 = 3; + map m_uint64 = 4; + map m_sint32 = 5; + map m_sint64 = 6; + map m_fixed32 = 7; + map m_fixed64 = 8; + map m_sfixed32 = 9; + map m_sfixed64 = 10; + map m_bool = 11; + map m_string = 12; +} + +// Test maps in submessages. +message TestSubmessageMaps { + optional TestMaps m = 1; +} diff --git a/karapace/protobuf/google/protobuf/map_unittest.proto b/karapace/protobuf/google/protobuf/map_unittest.proto new file mode 100644 index 000000000..263ef61f8 --- /dev/null +++ b/karapace/protobuf/google/protobuf/map_unittest.proto @@ -0,0 +1,125 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +option cc_enable_arenas = true; + +import "google/protobuf/unittest.proto"; + +// We don't put this in a package within proto2 because we need to make sure +// that the generated code doesn't depend on being in the proto2 namespace. +// In map_test_util.h we do "using namespace unittest = protobuf_unittest". +package protobuf_unittest; + +// Tests maps. +message TestMap { + map map_int32_int32 = 1; + map map_int64_int64 = 2; + map map_uint32_uint32 = 3; + map map_uint64_uint64 = 4; + map map_sint32_sint32 = 5; + map map_sint64_sint64 = 6; + map map_fixed32_fixed32 = 7; + map map_fixed64_fixed64 = 8; + map map_sfixed32_sfixed32 = 9; + map map_sfixed64_sfixed64 = 10; + map map_int32_float = 11; + map map_int32_double = 12; + map map_bool_bool = 13; + map map_string_string = 14; + map map_int32_bytes = 15; + map map_int32_enum = 16; + map map_int32_foreign_message = 17; + map map_string_foreign_message = 18; + map map_int32_all_types = 19; +} + +message TestMapSubmessage { + TestMap test_map = 1; +} + +message TestMessageMap { + map map_int32_message = 1; +} + +// Two map fields share the same entry default instance. +message TestSameTypeMap { + map map1 = 1; + map map2 = 2; +} + + +enum MapEnum { + MAP_ENUM_FOO = 0; + MAP_ENUM_BAR = 1; + MAP_ENUM_BAZ = 2; +} + +// Test embedded message with required fields +message TestRequiredMessageMap { + map map_field = 1; +} + +message TestArenaMap { + map map_int32_int32 = 1; + map map_int64_int64 = 2; + map map_uint32_uint32 = 3; + map map_uint64_uint64 = 4; + map map_sint32_sint32 = 5; + map map_sint64_sint64 = 6; + map map_fixed32_fixed32 = 7; + map map_fixed64_fixed64 = 8; + map map_sfixed32_sfixed32 = 9; + map map_sfixed64_sfixed64 = 10; + map map_int32_float = 11; + map map_int32_double = 12; + map map_bool_bool = 13; + map map_string_string = 14; + map map_int32_bytes = 15; + map map_int32_enum = 16; + map map_int32_foreign_message = 17; +} + +// Previously, message containing enum called Type cannot be used as value of +// map field. +message MessageContainingEnumCalledType { + enum Type { TYPE_FOO = 0; } + map type = 1; +} + +// Previously, message cannot contain map field called "entry". +message MessageContainingMapCalledEntry { + map entry = 1; +} + +message TestRecursiveMapMessage { + map a = 1; +} diff --git a/karapace/protobuf/google/protobuf/source_context.proto b/karapace/protobuf/google/protobuf/source_context.proto new file mode 100644 index 000000000..06bfc43a7 --- /dev/null +++ b/karapace/protobuf/google/protobuf/source_context.proto @@ -0,0 +1,48 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package google.protobuf; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "SourceContextProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; +option go_package = "google.golang.org/protobuf/types/known/sourcecontextpb"; + +// `SourceContext` represents information about the source of a +// protobuf element, like the file in which it is defined. +message SourceContext { + // The path-qualified name of the .proto file that contained the associated + // protobuf element. For example: `"google/protobuf/source_context.proto"`. + string file_name = 1; +} diff --git a/karapace/protobuf/google/protobuf/struct.proto b/karapace/protobuf/google/protobuf/struct.proto new file mode 100644 index 000000000..0ac843ca0 --- /dev/null +++ b/karapace/protobuf/google/protobuf/struct.proto @@ -0,0 +1,95 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package google.protobuf; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option cc_enable_arenas = true; +option go_package = "google.golang.org/protobuf/types/known/structpb"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "StructProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; + +// `Struct` represents a structured data value, consisting of fields +// which map to dynamically typed values. In some languages, `Struct` +// might be supported by a native representation. For example, in +// scripting languages like JS a struct is represented as an +// object. The details of that representation are described together +// with the proto support for the language. +// +// The JSON representation for `Struct` is JSON object. +message Struct { + // Unordered map of dynamically typed values. + map fields = 1; +} + +// `Value` represents a dynamically typed value which can be either +// null, a number, a string, a boolean, a recursive struct value, or a +// list of values. A producer of value is expected to set one of these +// variants. Absence of any variant indicates an error. +// +// The JSON representation for `Value` is JSON value. +message Value { + // The kind of value. + oneof kind { + // Represents a null value. + NullValue null_value = 1; + // Represents a double value. + double number_value = 2; + // Represents a string value. + string string_value = 3; + // Represents a boolean value. + bool bool_value = 4; + // Represents a structured value. + Struct struct_value = 5; + // Represents a repeated `Value`. + ListValue list_value = 6; + } +} + +// `NullValue` is a singleton enumeration to represent the null value for the +// `Value` type union. +// +// The JSON representation for `NullValue` is JSON `null`. +enum NullValue { + // Null value. + NULL_VALUE = 0; +} + +// `ListValue` is a wrapper around a repeated field of values. +// +// The JSON representation for `ListValue` is JSON array. +message ListValue { + // Repeated field of dynamically typed values. + repeated Value values = 1; +} diff --git a/karapace/protobuf/google/protobuf/timestamp.proto b/karapace/protobuf/google/protobuf/timestamp.proto new file mode 100644 index 000000000..3b2df6d91 --- /dev/null +++ b/karapace/protobuf/google/protobuf/timestamp.proto @@ -0,0 +1,147 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package google.protobuf; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option cc_enable_arenas = true; +option go_package = "google.golang.org/protobuf/types/known/timestamppb"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "TimestampProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; + +// A Timestamp represents a point in time independent of any time zone or local +// calendar, encoded as a count of seconds and fractions of seconds at +// nanosecond resolution. The count is relative to an epoch at UTC midnight on +// January 1, 1970, in the proleptic Gregorian calendar which extends the +// Gregorian calendar backwards to year one. +// +// All minutes are 60 seconds long. Leap seconds are "smeared" so that no leap +// second table is needed for interpretation, using a [24-hour linear +// smear](https://developers.google.com/time/smear). +// +// The range is from 0001-01-01T00:00:00Z to 9999-12-31T23:59:59.999999999Z. By +// restricting to that range, we ensure that we can convert to and from [RFC +// 3339](https://www.ietf.org/rfc/rfc3339.txt) date strings. +// +// # Examples +// +// Example 1: Compute Timestamp from POSIX `time()`. +// +// Timestamp timestamp; +// timestamp.set_seconds(time(NULL)); +// timestamp.set_nanos(0); +// +// Example 2: Compute Timestamp from POSIX `gettimeofday()`. +// +// struct timeval tv; +// gettimeofday(&tv, NULL); +// +// Timestamp timestamp; +// timestamp.set_seconds(tv.tv_sec); +// timestamp.set_nanos(tv.tv_usec * 1000); +// +// Example 3: Compute Timestamp from Win32 `GetSystemTimeAsFileTime()`. +// +// FILETIME ft; +// GetSystemTimeAsFileTime(&ft); +// UINT64 ticks = (((UINT64)ft.dwHighDateTime) << 32) | ft.dwLowDateTime; +// +// // A Windows tick is 100 nanoseconds. Windows epoch 1601-01-01T00:00:00Z +// // is 11644473600 seconds before Unix epoch 1970-01-01T00:00:00Z. +// Timestamp timestamp; +// timestamp.set_seconds((INT64) ((ticks / 10000000) - 11644473600LL)); +// timestamp.set_nanos((INT32) ((ticks % 10000000) * 100)); +// +// Example 4: Compute Timestamp from Java `System.currentTimeMillis()`. +// +// long millis = System.currentTimeMillis(); +// +// Timestamp timestamp = Timestamp.newBuilder().setSeconds(millis / 1000) +// .setNanos((int) ((millis % 1000) * 1000000)).build(); +// +// +// Example 5: Compute Timestamp from Java `Instant.now()`. +// +// Instant now = Instant.now(); +// +// Timestamp timestamp = +// Timestamp.newBuilder().setSeconds(now.getEpochSecond()) +// .setNanos(now.getNano()).build(); +// +// +// Example 6: Compute Timestamp from current time in Python. +// +// timestamp = Timestamp() +// timestamp.GetCurrentTime() +// +// # JSON Mapping +// +// In JSON format, the Timestamp type is encoded as a string in the +// [RFC 3339](https://www.ietf.org/rfc/rfc3339.txt) format. That is, the +// format is "{year}-{month}-{day}T{hour}:{min}:{sec}[.{frac_sec}]Z" +// where {year} is always expressed using four digits while {month}, {day}, +// {hour}, {min}, and {sec} are zero-padded to two digits each. The fractional +// seconds, which can go up to 9 digits (i.e. up to 1 nanosecond resolution), +// are optional. The "Z" suffix indicates the timezone ("UTC"); the timezone +// is required. A proto3 JSON serializer should always use UTC (as indicated by +// "Z") when printing the Timestamp type and a proto3 JSON parser should be +// able to accept both UTC and other timezones (as indicated by an offset). +// +// For example, "2017-01-15T01:30:15.01Z" encodes 15.01 seconds past +// 01:30 UTC on January 15, 2017. +// +// In JavaScript, one can convert a Date object to this format using the +// standard +// [toISOString()](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Date/toISOString) +// method. In Python, a standard `datetime.datetime` object can be converted +// to this format using +// [`strftime`](https://docs.python.org/2/library/time.html#time.strftime) with +// the time format spec '%Y-%m-%dT%H:%M:%S.%fZ'. Likewise, in Java, one can use +// the Joda Time's [`ISODateTimeFormat.dateTime()`]( +// http://www.joda.org/joda-time/apidocs/org/joda/time/format/ISODateTimeFormat.html#dateTime%2D%2D +// ) to obtain a formatter capable of generating timestamps in this format. +// +// +message Timestamp { + // Represents seconds of UTC time since Unix epoch + // 1970-01-01T00:00:00Z. Must be from 0001-01-01T00:00:00Z to + // 9999-12-31T23:59:59Z inclusive. + int64 seconds = 1; + + // Non-negative fractions of a second at nanosecond resolution. Negative + // second values with fractions must still have non-negative nanos values + // that count forward in time. Must be from 0 to 999,999,999 + // inclusive. + int32 nanos = 2; +} diff --git a/karapace/protobuf/google/protobuf/type.proto b/karapace/protobuf/google/protobuf/type.proto new file mode 100644 index 000000000..d3f6a68b8 --- /dev/null +++ b/karapace/protobuf/google/protobuf/type.proto @@ -0,0 +1,187 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package google.protobuf; + +import "google/protobuf/any.proto"; +import "google/protobuf/source_context.proto"; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option cc_enable_arenas = true; +option java_package = "com.google.protobuf"; +option java_outer_classname = "TypeProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; +option go_package = "google.golang.org/protobuf/types/known/typepb"; + +// A protocol buffer message type. +message Type { + // The fully qualified message name. + string name = 1; + // The list of fields. + repeated Field fields = 2; + // The list of types appearing in `oneof` definitions in this type. + repeated string oneofs = 3; + // The protocol buffer options. + repeated Option options = 4; + // The source context. + SourceContext source_context = 5; + // The source syntax. + Syntax syntax = 6; +} + +// A single field of a message type. +message Field { + // Basic field types. + enum Kind { + // Field type unknown. + TYPE_UNKNOWN = 0; + // Field type double. + TYPE_DOUBLE = 1; + // Field type float. + TYPE_FLOAT = 2; + // Field type int64. + TYPE_INT64 = 3; + // Field type uint64. + TYPE_UINT64 = 4; + // Field type int32. + TYPE_INT32 = 5; + // Field type fixed64. + TYPE_FIXED64 = 6; + // Field type fixed32. + TYPE_FIXED32 = 7; + // Field type bool. + TYPE_BOOL = 8; + // Field type string. + TYPE_STRING = 9; + // Field type group. Proto2 syntax only, and deprecated. + TYPE_GROUP = 10; + // Field type message. + TYPE_MESSAGE = 11; + // Field type bytes. + TYPE_BYTES = 12; + // Field type uint32. + TYPE_UINT32 = 13; + // Field type enum. + TYPE_ENUM = 14; + // Field type sfixed32. + TYPE_SFIXED32 = 15; + // Field type sfixed64. + TYPE_SFIXED64 = 16; + // Field type sint32. + TYPE_SINT32 = 17; + // Field type sint64. + TYPE_SINT64 = 18; + } + + // Whether a field is optional, required, or repeated. + enum Cardinality { + // For fields with unknown cardinality. + CARDINALITY_UNKNOWN = 0; + // For optional fields. + CARDINALITY_OPTIONAL = 1; + // For required fields. Proto2 syntax only. + CARDINALITY_REQUIRED = 2; + // For repeated fields. + CARDINALITY_REPEATED = 3; + } + + // The field type. + Kind kind = 1; + // The field cardinality. + Cardinality cardinality = 2; + // The field number. + int32 number = 3; + // The field name. + string name = 4; + // The field type URL, without the scheme, for message or enumeration + // types. Example: `"type.googleapis.com/google.protobuf.Timestamp"`. + string type_url = 6; + // The index of the field type in `Type.oneofs`, for message or enumeration + // types. The first type has index 1; zero means the type is not in the list. + int32 oneof_index = 7; + // Whether to use alternative packed wire representation. + bool packed = 8; + // The protocol buffer options. + repeated Option options = 9; + // The field JSON name. + string json_name = 10; + // The string value of the default value of this field. Proto2 syntax only. + string default_value = 11; +} + +// Enum type definition. +message Enum { + // Enum type name. + string name = 1; + // Enum value definitions. + repeated EnumValue enumvalue = 2; + // Protocol buffer options. + repeated Option options = 3; + // The source context. + SourceContext source_context = 4; + // The source syntax. + Syntax syntax = 5; +} + +// Enum value definition. +message EnumValue { + // Enum value name. + string name = 1; + // Enum value number. + int32 number = 2; + // Protocol buffer options. + repeated Option options = 3; +} + +// A protocol buffer option, which can be attached to a message, field, +// enumeration, etc. +message Option { + // The option's name. For protobuf built-in options (options defined in + // descriptor.proto), this is the short name. For example, `"map_entry"`. + // For custom options, it should be the fully-qualified name. For example, + // `"google.api.http"`. + string name = 1; + // The option's value packed in an Any message. If the value is a primitive, + // the corresponding wrapper type defined in google/protobuf/wrappers.proto + // should be used. If the value is an enum, it should be stored as an int32 + // value using the google.protobuf.Int32Value type. + Any value = 2; +} + +// The syntax in which a protocol buffer element is defined. +enum Syntax { + // Syntax `proto2`. + SYNTAX_PROTO2 = 0; + // Syntax `proto3`. + SYNTAX_PROTO3 = 1; +} diff --git a/karapace/protobuf/google/protobuf/wrappers.proto b/karapace/protobuf/google/protobuf/wrappers.proto new file mode 100644 index 000000000..d49dd53c8 --- /dev/null +++ b/karapace/protobuf/google/protobuf/wrappers.proto @@ -0,0 +1,123 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Wrappers for primitive (non-message) types. These types are useful +// for embedding primitives in the `google.protobuf.Any` type and for places +// where we need to distinguish between the absence of a primitive +// typed field and its default value. +// +// These wrappers have no meaningful use within repeated fields as they lack +// the ability to detect presence on individual elements. +// These wrappers have no meaningful use within a map or a oneof since +// individual entries of a map or fields of a oneof can already detect presence. + +syntax = "proto3"; + +package google.protobuf; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option cc_enable_arenas = true; +option go_package = "google.golang.org/protobuf/types/known/wrapperspb"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "WrappersProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; + +// Wrapper message for `double`. +// +// The JSON representation for `DoubleValue` is JSON number. +message DoubleValue { + // The double value. + double value = 1; +} + +// Wrapper message for `float`. +// +// The JSON representation for `FloatValue` is JSON number. +message FloatValue { + // The float value. + float value = 1; +} + +// Wrapper message for `int64`. +// +// The JSON representation for `Int64Value` is JSON string. +message Int64Value { + // The int64 value. + int64 value = 1; +} + +// Wrapper message for `uint64`. +// +// The JSON representation for `UInt64Value` is JSON string. +message UInt64Value { + // The uint64 value. + uint64 value = 1; +} + +// Wrapper message for `int32`. +// +// The JSON representation for `Int32Value` is JSON number. +message Int32Value { + // The int32 value. + int32 value = 1; +} + +// Wrapper message for `uint32`. +// +// The JSON representation for `UInt32Value` is JSON number. +message UInt32Value { + // The uint32 value. + uint32 value = 1; +} + +// Wrapper message for `bool`. +// +// The JSON representation for `BoolValue` is JSON `true` and `false`. +message BoolValue { + // The bool value. + bool value = 1; +} + +// Wrapper message for `string`. +// +// The JSON representation for `StringValue` is JSON string. +message StringValue { + // The string value. + string value = 1; +} + +// Wrapper message for `bytes`. +// +// The JSON representation for `BytesValue` is JSON string. +message BytesValue { + // The bytes value. + bytes value = 1; +} diff --git a/karapace/protobuf/google/type/BUILD.bazel b/karapace/protobuf/google/type/BUILD.bazel new file mode 100644 index 000000000..a7ad4d4d2 --- /dev/null +++ b/karapace/protobuf/google/type/BUILD.bazel @@ -0,0 +1,536 @@ +load("@rules_proto//proto:defs.bzl", "proto_library") + +# This is an API workspace, having public visibility by default makes perfect sense. +package(default_visibility = ["//visibility:public"]) + +############################################################################## +# Common +############################################################################## +proto_library( + name = "calendar_period_proto", + srcs = ["calendar_period.proto"], +) + +proto_library( + name = "color_proto", + srcs = ["color.proto"], + deps = [ + "@com_google_protobuf//:wrappers_proto", + ], +) + +proto_library( + name = "date_proto", + srcs = ["date.proto"], +) + +proto_library( + name = "datetime_proto", + srcs = ["datetime.proto"], + deps = [ + "@com_google_protobuf//:duration_proto", + ], +) + +proto_library( + name = "dayofweek_proto", + srcs = ["dayofweek.proto"], +) + +proto_library( + name = "decimal_proto", + srcs = ["decimal.proto"], +) + +proto_library( + name = "expr_proto", + srcs = ["expr.proto"], +) + +proto_library( + name = "fraction_proto", + srcs = ["fraction.proto"], +) + +proto_library( + name = "interval_proto", + srcs = ["interval.proto"], + deps = [ + "@com_google_protobuf//:timestamp_proto", + ], +) + +proto_library( + name = "latlng_proto", + srcs = ["latlng.proto"], +) + +proto_library( + name = "localized_text_proto", + srcs = ["localized_text.proto"], +) + +proto_library( + name = "money_proto", + srcs = ["money.proto"], +) + +proto_library( + name = "month_proto", + srcs = ["month.proto"], +) + +proto_library( + name = "phone_number_proto", + srcs = ["phone_number.proto"], +) + +proto_library( + name = "postal_address_proto", + srcs = ["postal_address.proto"], +) + +proto_library( + name = "quaternion_proto", + srcs = ["quaternion.proto"], +) + +proto_library( + name = "timeofday_proto", + srcs = ["timeofday.proto"], +) + +############################################################################## +# Java +############################################################################## +load("@com_google_googleapis_imports//:imports.bzl", + "java_gapic_assembly_gradle_pkg", + "java_proto_library") + +java_proto_library( + name = "type_java_proto", + deps = [ + ":calendar_period_proto", + ":color_proto", + ":date_proto", + ":datetime_proto", + ":dayofweek_proto", + ":decimal_proto", + ":expr_proto", + ":fraction_proto", + ":interval_proto", + ":latlng_proto", + ":localized_text_proto", + ":money_proto", + ":month_proto", + ":phone_number_proto", + ":postal_address_proto", + ":quaternion_proto", + ":timeofday_proto", + ], +) + +# Please DO-NOT-REMOVE this section. +# This is required to generate java files for these protos. +# Open Source Packages +java_gapic_assembly_gradle_pkg( + name = "google-type-java", + deps = [ + ":type_java_proto", + ":calendar_period_proto", + ":color_proto", + ":date_proto", + ":datetime_proto", + ":dayofweek_proto", + ":decimal_proto", + ":expr_proto", + ":fraction_proto", + ":interval_proto", + ":latlng_proto", + ":localized_text_proto", + ":money_proto", + ":month_proto", + ":phone_number_proto", + ":postal_address_proto", + ":quaternion_proto", + ":timeofday_proto", + ], +) + + +############################################################################## +# Go +############################################################################## +load("@com_google_googleapis_imports//:imports.bzl", "go_proto_library") + +go_proto_library( + name = "calendar_period_go_proto", + importpath = "google.golang.org/genproto/googleapis/type/calendarperiod", + protos = [":calendar_period_proto"], +) + +go_proto_library( + name = "color_go_proto", + importpath = "google.golang.org/genproto/googleapis/type/color", + protos = [":color_proto"], +) + +go_proto_library( + name = "date_go_proto", + importpath = "google.golang.org/genproto/googleapis/type/date", + protos = [":date_proto"], +) + +go_proto_library( + name = "datetime_go_proto", + importpath = "google.golang.org/genproto/googleapis/type/datetime", + protos = [":datetime_proto"], +) + +go_proto_library( + name = "dayofweek_go_proto", + importpath = "google.golang.org/genproto/googleapis/type/dayofweek", + protos = [":dayofweek_proto"], +) + +go_proto_library( + name = "decimal_go_proto", + importpath = "google.golang.org/genproto/googleapis/type/decimal", + protos = [":decimal_proto"], +) + +go_proto_library( + name = "expr_go_proto", + importpath = "google.golang.org/genproto/googleapis/type/expr", + protos = [":expr_proto"], +) + +go_proto_library( + name = "fraction_go_proto", + importpath = "google.golang.org/genproto/googleapis/type/fraction", + protos = [":fraction_proto"], +) + +go_proto_library( + name = "interval_go_proto", + importpath = "google.golang.org/genproto/googleapis/type/interval", + protos = [":interval_proto"], +) + +go_proto_library( + name = "latlng_go_proto", + importpath = "google.golang.org/genproto/googleapis/type/latlng", + protos = [":latlng_proto"], +) + +go_proto_library( + name = "localized_text_go_proto", + importpath = "google.golang.org/genproto/googleapis/type/localized_text", + protos = [":localized_text_proto"], +) + +go_proto_library( + name = "money_go_proto", + importpath = "google.golang.org/genproto/googleapis/type/money", + protos = [":money_proto"], +) + +go_proto_library( + name = "month_go_proto", + importpath = "google.golang.org/genproto/googleapis/type/month", + protos = [":month_proto"], +) + +go_proto_library( + name = "phone_number_go_proto", + importpath = "google.golang.org/genproto/googleapis/type/phone_number", + protos = [":phone_number_proto"], +) + +go_proto_library( + name = "postaladdress_go_proto", + importpath = "google.golang.org/genproto/googleapis/type/postaladdress", + protos = [":postal_address_proto"], +) + +go_proto_library( + name = "quaternion_go_proto", + importpath = "google.golang.org/genproto/googleapis/type/quaternion", + protos = [":quaternion_proto"], +) + +go_proto_library( + name = "timeofday_go_proto", + importpath = "google.golang.org/genproto/googleapis/type/timeofday", + protos = [":timeofday_proto"], +) + +############################################################################## +# C++ +############################################################################## +load( + "@com_google_googleapis_imports//:imports.bzl", + "cc_proto_library", +) + +cc_proto_library( + name = "calendar_period_cc_proto", + deps = [":calendar_period_proto"], +) + +cc_proto_library( + name = "color_cc_proto", + deps = [":color_proto"], +) + +cc_proto_library( + name = "date_cc_proto", + deps = [":date_proto"], +) + +cc_proto_library( + name = "datetime_cc_proto", + deps = [":datetime_proto"], +) + +cc_proto_library( + name = "dayofweek_cc_proto", + deps = [":dayofweek_proto"], +) + +cc_proto_library( + name = "decimal_cc_proto", + deps = [":decimal_proto"], +) + +cc_proto_library( + name = "expr_cc_proto", + deps = [":expr_proto"], +) + +cc_proto_library( + name = "fraction_cc_proto", + deps = [":fraction_proto"], +) + +cc_proto_library( + name = "interval_cc_proto", + deps = [":interval_proto"], +) + +cc_proto_library( + name = "latlng_cc_proto", + deps = [":latlng_proto"], +) + +cc_proto_library( + name = "money_cc_proto", + deps = [":money_proto"], +) + +cc_proto_library( + name = "month_cc_proto", + deps = [":month_proto"], +) + +cc_proto_library( + name = "phone_number_cc_proto", + deps = [":phone_number_proto"], +) + +cc_proto_library( + name = "postal_address_cc_proto", + deps = [":postal_address_proto"], +) + +cc_proto_library( + name = "quaternion_cc_proto", + deps = [":quaternion_proto"], +) + +cc_proto_library( + name = "timeofday_cc_proto", + deps = [":timeofday_proto"], +) + +############################################################################## +# Python +############################################################################## +load( + "@com_google_googleapis_imports//:imports.bzl", + "py_proto_library", +) + +py_proto_library( + name = "calendar_period_py_proto", + deps = [":calendar_period_proto"], +) + +py_proto_library( + name = "color_py_proto", + deps = [":color_proto"], +) + +py_proto_library( + name = "date_py_proto", + deps = [":date_proto"], +) + +py_proto_library( + name = "datetime_py_proto", + deps = [":datetime_proto"], +) + +py_proto_library( + name = "dayofweek_py_proto", + deps = [":dayofweek_proto"], +) + +py_proto_library( + name = "decimal_py_proto", + deps = [":decimal_proto"], +) + +py_proto_library( + name = "expr_py_proto", + deps = [":expr_proto"], +) + +py_proto_library( + name = "fraction_py_proto", + deps = [":fraction_proto"], +) + +py_proto_library( + name = "interval_py_proto", + deps = [":interval_proto"], +) + +py_proto_library( + name = "latlng_py_proto", + deps = [":latlng_proto"], +) + +py_proto_library( + name = "localized_text_py_proto", + deps = [":localized_text_proto"], +) + +py_proto_library( + name = "money_py_proto", + deps = [":money_proto"], +) + +py_proto_library( + name = "month_py_proto", + deps = [":month_proto"], +) + +py_proto_library( + name = "phone_number_py_proto", + deps = [":phone_number_proto"], +) + +py_proto_library( + name = "postal_address_py_proto", + deps = [":postal_address_proto"], +) + +py_proto_library( + name = "quaternion_py_proto", + deps = [":quaternion_proto"], +) + +py_proto_library( + name = "timeofday_py_proto", + deps = [":timeofday_proto"], +) + +############################################################################## +# C# +############################################################################## + +load( + "@com_google_googleapis_imports//:imports.bzl", + "csharp_proto_library", +) + +csharp_proto_library( + name = "calendar_period_csharp_proto", + deps = [":calendar_period_proto"], +) + +csharp_proto_library( + name = "color_csharp_proto", + deps = [":color_proto"], +) + +csharp_proto_library( + name = "date_csharp_proto", + deps = [":date_proto"], +) + +csharp_proto_library( + name = "datetime_csharp_proto", + deps = [":datetime_proto"], +) + +csharp_proto_library( + name = "dayofweek_csharp_proto", + deps = [":dayofweek_proto"], +) + +csharp_proto_library( + name = "decimal_csharp_proto", + deps = [":decimal_proto"], +) + +csharp_proto_library( + name = "expr_csharp_proto", + deps = [":expr_proto"], +) + +csharp_proto_library( + name = "fraction_csharp_proto", + deps = [":fraction_proto"], +) + +csharp_proto_library( + name = "interval_csharp_proto", + deps = [":interval_proto"], +) + +csharp_proto_library( + name = "latlng_csharp_proto", + deps = [":latlng_proto"], +) + +csharp_proto_library( + name = "localized_text_csharp_proto", + deps = [":localized_text_proto"], +) + +csharp_proto_library( + name = "money_csharp_proto", + deps = [":money_proto"], +) + +csharp_proto_library( + name = "month_csharp_proto", + deps = [":month_proto"], +) + +csharp_proto_library( + name = "phone_number_csharp_proto", + deps = [":phone_number_proto"], +) + +csharp_proto_library( + name = "postal_address_csharp_proto", + deps = [":postal_address_proto"], +) + +csharp_proto_library( + name = "quaternion_csharp_proto", + deps = [":quaternion_proto"], +) diff --git a/karapace/protobuf/google/type/README.md b/karapace/protobuf/google/type/README.md new file mode 100644 index 000000000..6caf02cf1 --- /dev/null +++ b/karapace/protobuf/google/type/README.md @@ -0,0 +1,16 @@ +# Google Common Types + +This package contains definitions of common types for Google APIs. +All types defined in this package are suitable for different APIs to +exchange data, and will never break binary compatibility. They should +have design quality comparable to major programming languages like +Java and C#. + +NOTE: Some common types are defined in the package `google.protobuf` +as they are directly supported by Protocol Buffers compiler and +runtime. Those types are called Well-Known Types. + +## Java Utilities + +A set of Java utilities for the Common Types are provided in the +`//java/com/google/type/util/` package. \ No newline at end of file diff --git a/karapace/protobuf/google/type/calendar_period.proto b/karapace/protobuf/google/type/calendar_period.proto new file mode 100644 index 000000000..82f5690b7 --- /dev/null +++ b/karapace/protobuf/google/type/calendar_period.proto @@ -0,0 +1,56 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option go_package = "google.golang.org/genproto/googleapis/type/calendarperiod;calendarperiod"; +option java_multiple_files = true; +option java_outer_classname = "CalendarPeriodProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// A `CalendarPeriod` represents the abstract concept of a time period that has +// a canonical start. Grammatically, "the start of the current +// `CalendarPeriod`." All calendar times begin at midnight UTC. +enum CalendarPeriod { + // Undefined period, raises an error. + CALENDAR_PERIOD_UNSPECIFIED = 0; + + // A day. + DAY = 1; + + // A week. Weeks begin on Monday, following + // [ISO 8601](https://en.wikipedia.org/wiki/ISO_week_date). + WEEK = 2; + + // A fortnight. The first calendar fortnight of the year begins at the start + // of week 1 according to + // [ISO 8601](https://en.wikipedia.org/wiki/ISO_week_date). + FORTNIGHT = 3; + + // A month. + MONTH = 4; + + // A quarter. Quarters start on dates 1-Jan, 1-Apr, 1-Jul, and 1-Oct of each + // year. + QUARTER = 5; + + // A half-year. Half-years start on dates 1-Jan and 1-Jul. + HALF = 6; + + // A year. + YEAR = 7; +} diff --git a/karapace/protobuf/google/type/color.proto b/karapace/protobuf/google/type/color.proto new file mode 100644 index 000000000..5dc85a6a3 --- /dev/null +++ b/karapace/protobuf/google/type/color.proto @@ -0,0 +1,174 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +import "google/protobuf/wrappers.proto"; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/color;color"; +option java_multiple_files = true; +option java_outer_classname = "ColorProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a color in the RGBA color space. This representation is designed +// for simplicity of conversion to/from color representations in various +// languages over compactness. For example, the fields of this representation +// can be trivially provided to the constructor of `java.awt.Color` in Java; it +// can also be trivially provided to UIColor's `+colorWithRed:green:blue:alpha` +// method in iOS; and, with just a little work, it can be easily formatted into +// a CSS `rgba()` string in JavaScript. +// +// This reference page doesn't carry information about the absolute color +// space +// that should be used to interpret the RGB value (e.g. sRGB, Adobe RGB, +// DCI-P3, BT.2020, etc.). By default, applications should assume the sRGB color +// space. +// +// When color equality needs to be decided, implementations, unless +// documented otherwise, treat two colors as equal if all their red, +// green, blue, and alpha values each differ by at most 1e-5. +// +// Example (Java): +// +// import com.google.type.Color; +// +// // ... +// public static java.awt.Color fromProto(Color protocolor) { +// float alpha = protocolor.hasAlpha() +// ? protocolor.getAlpha().getValue() +// : 1.0; +// +// return new java.awt.Color( +// protocolor.getRed(), +// protocolor.getGreen(), +// protocolor.getBlue(), +// alpha); +// } +// +// public static Color toProto(java.awt.Color color) { +// float red = (float) color.getRed(); +// float green = (float) color.getGreen(); +// float blue = (float) color.getBlue(); +// float denominator = 255.0; +// Color.Builder resultBuilder = +// Color +// .newBuilder() +// .setRed(red / denominator) +// .setGreen(green / denominator) +// .setBlue(blue / denominator); +// int alpha = color.getAlpha(); +// if (alpha != 255) { +// result.setAlpha( +// FloatValue +// .newBuilder() +// .setValue(((float) alpha) / denominator) +// .build()); +// } +// return resultBuilder.build(); +// } +// // ... +// +// Example (iOS / Obj-C): +// +// // ... +// static UIColor* fromProto(Color* protocolor) { +// float red = [protocolor red]; +// float green = [protocolor green]; +// float blue = [protocolor blue]; +// FloatValue* alpha_wrapper = [protocolor alpha]; +// float alpha = 1.0; +// if (alpha_wrapper != nil) { +// alpha = [alpha_wrapper value]; +// } +// return [UIColor colorWithRed:red green:green blue:blue alpha:alpha]; +// } +// +// static Color* toProto(UIColor* color) { +// CGFloat red, green, blue, alpha; +// if (![color getRed:&red green:&green blue:&blue alpha:&alpha]) { +// return nil; +// } +// Color* result = [[Color alloc] init]; +// [result setRed:red]; +// [result setGreen:green]; +// [result setBlue:blue]; +// if (alpha <= 0.9999) { +// [result setAlpha:floatWrapperWithValue(alpha)]; +// } +// [result autorelease]; +// return result; +// } +// // ... +// +// Example (JavaScript): +// +// // ... +// +// var protoToCssColor = function(rgb_color) { +// var redFrac = rgb_color.red || 0.0; +// var greenFrac = rgb_color.green || 0.0; +// var blueFrac = rgb_color.blue || 0.0; +// var red = Math.floor(redFrac * 255); +// var green = Math.floor(greenFrac * 255); +// var blue = Math.floor(blueFrac * 255); +// +// if (!('alpha' in rgb_color)) { +// return rgbToCssColor(red, green, blue); +// } +// +// var alphaFrac = rgb_color.alpha.value || 0.0; +// var rgbParams = [red, green, blue].join(','); +// return ['rgba(', rgbParams, ',', alphaFrac, ')'].join(''); +// }; +// +// var rgbToCssColor = function(red, green, blue) { +// var rgbNumber = new Number((red << 16) | (green << 8) | blue); +// var hexString = rgbNumber.toString(16); +// var missingZeros = 6 - hexString.length; +// var resultBuilder = ['#']; +// for (var i = 0; i < missingZeros; i++) { +// resultBuilder.push('0'); +// } +// resultBuilder.push(hexString); +// return resultBuilder.join(''); +// }; +// +// // ... +message Color { + // The amount of red in the color as a value in the interval [0, 1]. + float red = 1; + + // The amount of green in the color as a value in the interval [0, 1]. + float green = 2; + + // The amount of blue in the color as a value in the interval [0, 1]. + float blue = 3; + + // The fraction of this color that should be applied to the pixel. That is, + // the final pixel color is defined by the equation: + // + // `pixel color = alpha * (this color) + (1.0 - alpha) * (background color)` + // + // This means that a value of 1.0 corresponds to a solid color, whereas + // a value of 0.0 corresponds to a completely transparent color. This + // uses a wrapper message rather than a simple float scalar so that it is + // possible to distinguish between a default value and the value being unset. + // If omitted, this color object is rendered as a solid color + // (as if the alpha value had been explicitly given a value of 1.0). + google.protobuf.FloatValue alpha = 4; +} diff --git a/karapace/protobuf/google/type/date.proto b/karapace/protobuf/google/type/date.proto new file mode 100644 index 000000000..e4e730e6f --- /dev/null +++ b/karapace/protobuf/google/type/date.proto @@ -0,0 +1,52 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/date;date"; +option java_multiple_files = true; +option java_outer_classname = "DateProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a whole or partial calendar date, such as a birthday. The time of +// day and time zone are either specified elsewhere or are insignificant. The +// date is relative to the Gregorian Calendar. This can represent one of the +// following: +// +// * A full date, with non-zero year, month, and day values +// * A month and day value, with a zero year, such as an anniversary +// * A year on its own, with zero month and day values +// * A year and month value, with a zero day, such as a credit card expiration +// date +// +// Related types are [google.type.TimeOfDay][google.type.TimeOfDay] and +// `google.protobuf.Timestamp`. +message Date { + // Year of the date. Must be from 1 to 9999, or 0 to specify a date without + // a year. + int32 year = 1; + + // Month of a year. Must be from 1 to 12, or 0 to specify a year without a + // month and day. + int32 month = 2; + + // Day of a month. Must be from 1 to 31 and valid for the year and month, or 0 + // to specify a year by itself or a year and month where the day isn't + // significant. + int32 day = 3; +} diff --git a/karapace/protobuf/google/type/datetime.proto b/karapace/protobuf/google/type/datetime.proto new file mode 100644 index 000000000..cfed85d70 --- /dev/null +++ b/karapace/protobuf/google/type/datetime.proto @@ -0,0 +1,104 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +import "google/protobuf/duration.proto"; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/datetime;datetime"; +option java_multiple_files = true; +option java_outer_classname = "DateTimeProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents civil time (or occasionally physical time). +// +// This type can represent a civil time in one of a few possible ways: +// +// * When utc_offset is set and time_zone is unset: a civil time on a calendar +// day with a particular offset from UTC. +// * When time_zone is set and utc_offset is unset: a civil time on a calendar +// day in a particular time zone. +// * When neither time_zone nor utc_offset is set: a civil time on a calendar +// day in local time. +// +// The date is relative to the Proleptic Gregorian Calendar. +// +// If year is 0, the DateTime is considered not to have a specific year. month +// and day must have valid, non-zero values. +// +// This type may also be used to represent a physical time if all the date and +// time fields are set and either case of the `time_offset` oneof is set. +// Consider using `Timestamp` message for physical time instead. If your use +// case also would like to store the user's timezone, that can be done in +// another field. +// +// This type is more flexible than some applications may want. Make sure to +// document and validate your application's limitations. +message DateTime { + // Optional. Year of date. Must be from 1 to 9999, or 0 if specifying a + // datetime without a year. + int32 year = 1; + + // Required. Month of year. Must be from 1 to 12. + int32 month = 2; + + // Required. Day of month. Must be from 1 to 31 and valid for the year and + // month. + int32 day = 3; + + // Required. Hours of day in 24 hour format. Should be from 0 to 23. An API + // may choose to allow the value "24:00:00" for scenarios like business + // closing time. + int32 hours = 4; + + // Required. Minutes of hour of day. Must be from 0 to 59. + int32 minutes = 5; + + // Required. Seconds of minutes of the time. Must normally be from 0 to 59. An + // API may allow the value 60 if it allows leap-seconds. + int32 seconds = 6; + + // Required. Fractions of seconds in nanoseconds. Must be from 0 to + // 999,999,999. + int32 nanos = 7; + + // Optional. Specifies either the UTC offset or the time zone of the DateTime. + // Choose carefully between them, considering that time zone data may change + // in the future (for example, a country modifies their DST start/end dates, + // and future DateTimes in the affected range had already been stored). + // If omitted, the DateTime is considered to be in local time. + oneof time_offset { + // UTC offset. Must be whole seconds, between -18 hours and +18 hours. + // For example, a UTC offset of -4:00 would be represented as + // { seconds: -14400 }. + google.protobuf.Duration utc_offset = 8; + + // Time zone. + TimeZone time_zone = 9; + } +} + +// Represents a time zone from the +// [IANA Time Zone Database](https://www.iana.org/time-zones). +message TimeZone { + // IANA Time Zone Database time zone, e.g. "America/New_York". + string id = 1; + + // Optional. IANA Time Zone Database version number, e.g. "2019a". + string version = 2; +} diff --git a/karapace/protobuf/google/type/dayofweek.proto b/karapace/protobuf/google/type/dayofweek.proto new file mode 100644 index 000000000..4c80c62ec --- /dev/null +++ b/karapace/protobuf/google/type/dayofweek.proto @@ -0,0 +1,50 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option go_package = "google.golang.org/genproto/googleapis/type/dayofweek;dayofweek"; +option java_multiple_files = true; +option java_outer_classname = "DayOfWeekProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a day of the week. +enum DayOfWeek { + // The day of the week is unspecified. + DAY_OF_WEEK_UNSPECIFIED = 0; + + // Monday + MONDAY = 1; + + // Tuesday + TUESDAY = 2; + + // Wednesday + WEDNESDAY = 3; + + // Thursday + THURSDAY = 4; + + // Friday + FRIDAY = 5; + + // Saturday + SATURDAY = 6; + + // Sunday + SUNDAY = 7; +} diff --git a/karapace/protobuf/google/type/decimal.proto b/karapace/protobuf/google/type/decimal.proto new file mode 100644 index 000000000..beb18a5d8 --- /dev/null +++ b/karapace/protobuf/google/type/decimal.proto @@ -0,0 +1,95 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/decimal;decimal"; +option java_multiple_files = true; +option java_outer_classname = "DecimalProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// A representation of a decimal value, such as 2.5. Clients may convert values +// into language-native decimal formats, such as Java's [BigDecimal][] or +// Python's [decimal.Decimal][]. +// +// [BigDecimal]: +// https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/math/BigDecimal.html +// [decimal.Decimal]: https://docs.python.org/3/library/decimal.html +message Decimal { + // The decimal value, as a string. + // + // The string representation consists of an optional sign, `+` (`U+002B`) + // or `-` (`U+002D`), followed by a sequence of zero or more decimal digits + // ("the integer"), optionally followed by a fraction, optionally followed + // by an exponent. + // + // The fraction consists of a decimal point followed by zero or more decimal + // digits. The string must contain at least one digit in either the integer + // or the fraction. The number formed by the sign, the integer and the + // fraction is referred to as the significand. + // + // The exponent consists of the character `e` (`U+0065`) or `E` (`U+0045`) + // followed by one or more decimal digits. + // + // Services **should** normalize decimal values before storing them by: + // + // - Removing an explicitly-provided `+` sign (`+2.5` -> `2.5`). + // - Replacing a zero-length integer value with `0` (`.5` -> `0.5`). + // - Coercing the exponent character to lower-case (`2.5E8` -> `2.5e8`). + // - Removing an explicitly-provided zero exponent (`2.5e0` -> `2.5`). + // + // Services **may** perform additional normalization based on its own needs + // and the internal decimal implementation selected, such as shifting the + // decimal point and exponent value together (example: `2.5e-1` <-> `0.25`). + // Additionally, services **may** preserve trailing zeroes in the fraction + // to indicate increased precision, but are not required to do so. + // + // Note that only the `.` character is supported to divide the integer + // and the fraction; `,` **should not** be supported regardless of locale. + // Additionally, thousand separators **should not** be supported. If a + // service does support them, values **must** be normalized. + // + // The ENBF grammar is: + // + // DecimalString = + // [Sign] Significand [Exponent]; + // + // Sign = '+' | '-'; + // + // Significand = + // Digits ['.'] [Digits] | [Digits] '.' Digits; + // + // Exponent = ('e' | 'E') [Sign] Digits; + // + // Digits = { '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' }; + // + // Services **should** clearly document the range of supported values, the + // maximum supported precision (total number of digits), and, if applicable, + // the scale (number of digits after the decimal point), as well as how it + // behaves when receiving out-of-bounds values. + // + // Services **may** choose to accept values passed as input even when the + // value has a higher precision or scale than the service supports, and + // **should** round the value to fit the supported scale. Alternatively, the + // service **may** error with `400 Bad Request` (`INVALID_ARGUMENT` in gRPC) + // if precision would be lost. + // + // Services **should** error with `400 Bad Request` (`INVALID_ARGUMENT` in + // gRPC) if the service receives a value outside of the supported range. + string value = 1; +} diff --git a/karapace/protobuf/google/type/expr.proto b/karapace/protobuf/google/type/expr.proto new file mode 100644 index 000000000..af0778cf9 --- /dev/null +++ b/karapace/protobuf/google/type/expr.proto @@ -0,0 +1,73 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option go_package = "google.golang.org/genproto/googleapis/type/expr;expr"; +option java_multiple_files = true; +option java_outer_classname = "ExprProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a textual expression in the Common Expression Language (CEL) +// syntax. CEL is a C-like expression language. The syntax and semantics of CEL +// are documented at https://github.com/google/cel-spec. +// +// Example (Comparison): +// +// title: "Summary size limit" +// description: "Determines if a summary is less than 100 chars" +// expression: "document.summary.size() < 100" +// +// Example (Equality): +// +// title: "Requestor is owner" +// description: "Determines if requestor is the document owner" +// expression: "document.owner == request.auth.claims.email" +// +// Example (Logic): +// +// title: "Public documents" +// description: "Determine whether the document should be publicly visible" +// expression: "document.type != 'private' && document.type != 'internal'" +// +// Example (Data Manipulation): +// +// title: "Notification string" +// description: "Create a notification string with a timestamp." +// expression: "'New message received at ' + string(document.create_time)" +// +// The exact variables and functions that may be referenced within an expression +// are determined by the service that evaluates it. See the service +// documentation for additional information. +message Expr { + // Textual representation of an expression in Common Expression Language + // syntax. + string expression = 1; + + // Optional. Title for the expression, i.e. a short string describing + // its purpose. This can be used e.g. in UIs which allow to enter the + // expression. + string title = 2; + + // Optional. Description of the expression. This is a longer text which + // describes the expression, e.g. when hovered over it in a UI. + string description = 3; + + // Optional. String indicating the location of the expression for error + // reporting, e.g. a file name and a position in the file. + string location = 4; +} diff --git a/karapace/protobuf/google/type/fraction.proto b/karapace/protobuf/google/type/fraction.proto new file mode 100644 index 000000000..6c5ae6e2a --- /dev/null +++ b/karapace/protobuf/google/type/fraction.proto @@ -0,0 +1,33 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option go_package = "google.golang.org/genproto/googleapis/type/fraction;fraction"; +option java_multiple_files = true; +option java_outer_classname = "FractionProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a fraction in terms of a numerator divided by a denominator. +message Fraction { + // The numerator in the fraction, e.g. 2 in 2/3. + int64 numerator = 1; + + // The value by which the numerator is divided, e.g. 3 in 2/3. Must be + // positive. + int64 denominator = 2; +} diff --git a/karapace/protobuf/google/type/interval.proto b/karapace/protobuf/google/type/interval.proto new file mode 100644 index 000000000..9702324cd --- /dev/null +++ b/karapace/protobuf/google/type/interval.proto @@ -0,0 +1,46 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +import "google/protobuf/timestamp.proto"; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/interval;interval"; +option java_multiple_files = true; +option java_outer_classname = "IntervalProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a time interval, encoded as a Timestamp start (inclusive) and a +// Timestamp end (exclusive). +// +// The start must be less than or equal to the end. +// When the start equals the end, the interval is empty (matches no time). +// When both start and end are unspecified, the interval matches any time. +message Interval { + // Optional. Inclusive start of the interval. + // + // If specified, a Timestamp matching this interval will have to be the same + // or after the start. + google.protobuf.Timestamp start_time = 1; + + // Optional. Exclusive end of the interval. + // + // If specified, a Timestamp matching this interval will have to be before the + // end. + google.protobuf.Timestamp end_time = 2; +} diff --git a/karapace/protobuf/google/type/latlng.proto b/karapace/protobuf/google/type/latlng.proto new file mode 100644 index 000000000..9231456e3 --- /dev/null +++ b/karapace/protobuf/google/type/latlng.proto @@ -0,0 +1,37 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/latlng;latlng"; +option java_multiple_files = true; +option java_outer_classname = "LatLngProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// An object that represents a latitude/longitude pair. This is expressed as a +// pair of doubles to represent degrees latitude and degrees longitude. Unless +// specified otherwise, this must conform to the +// WGS84 +// standard. Values must be within normalized ranges. +message LatLng { + // The latitude in degrees. It must be in the range [-90.0, +90.0]. + double latitude = 1; + + // The longitude in degrees. It must be in the range [-180.0, +180.0]. + double longitude = 2; +} diff --git a/karapace/protobuf/google/type/localized_text.proto b/karapace/protobuf/google/type/localized_text.proto new file mode 100644 index 000000000..5c6922b8c --- /dev/null +++ b/karapace/protobuf/google/type/localized_text.proto @@ -0,0 +1,36 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/localized_text;localized_text"; +option java_multiple_files = true; +option java_outer_classname = "LocalizedTextProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Localized variant of a text in a particular language. +message LocalizedText { + // Localized string in the language corresponding to `language_code' below. + string text = 1; + + // The text's BCP-47 language code, such as "en-US" or "sr-Latn". + // + // For more information, see + // http://www.unicode.org/reports/tr35/#Unicode_locale_identifier. + string language_code = 2; +} diff --git a/karapace/protobuf/google/type/money.proto b/karapace/protobuf/google/type/money.proto new file mode 100644 index 000000000..98d6494e4 --- /dev/null +++ b/karapace/protobuf/google/type/money.proto @@ -0,0 +1,42 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/money;money"; +option java_multiple_files = true; +option java_outer_classname = "MoneyProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents an amount of money with its currency type. +message Money { + // The three-letter currency code defined in ISO 4217. + string currency_code = 1; + + // The whole units of the amount. + // For example if `currencyCode` is `"USD"`, then 1 unit is one US dollar. + int64 units = 2; + + // Number of nano (10^-9) units of the amount. + // The value must be between -999,999,999 and +999,999,999 inclusive. + // If `units` is positive, `nanos` must be positive or zero. + // If `units` is zero, `nanos` can be positive, zero, or negative. + // If `units` is negative, `nanos` must be negative or zero. + // For example $-1.75 is represented as `units`=-1 and `nanos`=-750,000,000. + int32 nanos = 3; +} diff --git a/karapace/protobuf/google/type/month.proto b/karapace/protobuf/google/type/month.proto new file mode 100644 index 000000000..99e7551b1 --- /dev/null +++ b/karapace/protobuf/google/type/month.proto @@ -0,0 +1,65 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option go_package = "google.golang.org/genproto/googleapis/type/month;month"; +option java_multiple_files = true; +option java_outer_classname = "MonthProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a month in the Gregorian calendar. +enum Month { + // The unspecified month. + MONTH_UNSPECIFIED = 0; + + // The month of January. + JANUARY = 1; + + // The month of February. + FEBRUARY = 2; + + // The month of March. + MARCH = 3; + + // The month of April. + APRIL = 4; + + // The month of May. + MAY = 5; + + // The month of June. + JUNE = 6; + + // The month of July. + JULY = 7; + + // The month of August. + AUGUST = 8; + + // The month of September. + SEPTEMBER = 9; + + // The month of October. + OCTOBER = 10; + + // The month of November. + NOVEMBER = 11; + + // The month of December. + DECEMBER = 12; +} diff --git a/karapace/protobuf/google/type/phone_number.proto b/karapace/protobuf/google/type/phone_number.proto new file mode 100644 index 000000000..7bbb7d873 --- /dev/null +++ b/karapace/protobuf/google/type/phone_number.proto @@ -0,0 +1,113 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/phone_number;phone_number"; +option java_multiple_files = true; +option java_outer_classname = "PhoneNumberProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// An object representing a phone number, suitable as an API wire format. +// +// This representation: +// +// - should not be used for locale-specific formatting of a phone number, such +// as "+1 (650) 253-0000 ext. 123" +// +// - is not designed for efficient storage +// - may not be suitable for dialing - specialized libraries (see references) +// should be used to parse the number for that purpose +// +// To do something meaningful with this number, such as format it for various +// use-cases, convert it to an `i18n.phonenumbers.PhoneNumber` object first. +// +// For instance, in Java this would be: +// +// com.google.type.PhoneNumber wireProto = +// com.google.type.PhoneNumber.newBuilder().build(); +// com.google.i18n.phonenumbers.Phonenumber.PhoneNumber phoneNumber = +// PhoneNumberUtil.getInstance().parse(wireProto.getE164Number(), "ZZ"); +// if (!wireProto.getExtension().isEmpty()) { +// phoneNumber.setExtension(wireProto.getExtension()); +// } +// +// Reference(s): +// - https://github.com/google/libphonenumber +message PhoneNumber { + // An object representing a short code, which is a phone number that is + // typically much shorter than regular phone numbers and can be used to + // address messages in MMS and SMS systems, as well as for abbreviated dialing + // (e.g. "Text 611 to see how many minutes you have remaining on your plan."). + // + // Short codes are restricted to a region and are not internationally + // dialable, which means the same short code can exist in different regions, + // with different usage and pricing, even if those regions share the same + // country calling code (e.g. US and CA). + message ShortCode { + // Required. The BCP-47 region code of the location where calls to this + // short code can be made, such as "US" and "BB". + // + // Reference(s): + // - http://www.unicode.org/reports/tr35/#unicode_region_subtag + string region_code = 1; + + // Required. The short code digits, without a leading plus ('+') or country + // calling code, e.g. "611". + string number = 2; + } + + // Required. Either a regular number, or a short code. New fields may be + // added to the oneof below in the future, so clients should ignore phone + // numbers for which none of the fields they coded against are set. + oneof kind { + // The phone number, represented as a leading plus sign ('+'), followed by a + // phone number that uses a relaxed ITU E.164 format consisting of the + // country calling code (1 to 3 digits) and the subscriber number, with no + // additional spaces or formatting, e.g.: + // - correct: "+15552220123" + // - incorrect: "+1 (555) 222-01234 x123". + // + // The ITU E.164 format limits the latter to 12 digits, but in practice not + // all countries respect that, so we relax that restriction here. + // National-only numbers are not allowed. + // + // References: + // - https://www.itu.int/rec/T-REC-E.164-201011-I + // - https://en.wikipedia.org/wiki/E.164. + // - https://en.wikipedia.org/wiki/List_of_country_calling_codes + string e164_number = 1; + + // A short code. + // + // Reference(s): + // - https://en.wikipedia.org/wiki/Short_code + ShortCode short_code = 2; + } + + // The phone number's extension. The extension is not standardized in ITU + // recommendations, except for being defined as a series of numbers with a + // maximum length of 40 digits. Other than digits, some other dialing + // characters such as ',' (indicating a wait) or '#' may be stored here. + // + // Note that no regions currently use extensions with short codes, so this + // field is normally only set in conjunction with an E.164 number. It is held + // separately from the E.164 number to allow for short code extensions in the + // future. + string extension = 3; +} diff --git a/karapace/protobuf/google/type/postal_address.proto b/karapace/protobuf/google/type/postal_address.proto new file mode 100644 index 000000000..c57c7c31a --- /dev/null +++ b/karapace/protobuf/google/type/postal_address.proto @@ -0,0 +1,134 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/postaladdress;postaladdress"; +option java_multiple_files = true; +option java_outer_classname = "PostalAddressProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a postal address, e.g. for postal delivery or payments addresses. +// Given a postal address, a postal service can deliver items to a premise, P.O. +// Box or similar. +// It is not intended to model geographical locations (roads, towns, +// mountains). +// +// In typical usage an address would be created via user input or from importing +// existing data, depending on the type of process. +// +// Advice on address input / editing: +// - Use an i18n-ready address widget such as +// https://github.com/google/libaddressinput) +// - Users should not be presented with UI elements for input or editing of +// fields outside countries where that field is used. +// +// For more guidance on how to use this schema, please see: +// https://support.google.com/business/answer/6397478 +message PostalAddress { + // The schema revision of the `PostalAddress`. This must be set to 0, which is + // the latest revision. + // + // All new revisions **must** be backward compatible with old revisions. + int32 revision = 1; + + // Required. CLDR region code of the country/region of the address. This + // is never inferred and it is up to the user to ensure the value is + // correct. See http://cldr.unicode.org/ and + // http://www.unicode.org/cldr/charts/30/supplemental/territory_information.html + // for details. Example: "CH" for Switzerland. + string region_code = 2; + + // Optional. BCP-47 language code of the contents of this address (if + // known). This is often the UI language of the input form or is expected + // to match one of the languages used in the address' country/region, or their + // transliterated equivalents. + // This can affect formatting in certain countries, but is not critical + // to the correctness of the data and will never affect any validation or + // other non-formatting related operations. + // + // If this value is not known, it should be omitted (rather than specifying a + // possibly incorrect default). + // + // Examples: "zh-Hant", "ja", "ja-Latn", "en". + string language_code = 3; + + // Optional. Postal code of the address. Not all countries use or require + // postal codes to be present, but where they are used, they may trigger + // additional validation with other parts of the address (e.g. state/zip + // validation in the U.S.A.). + string postal_code = 4; + + // Optional. Additional, country-specific, sorting code. This is not used + // in most regions. Where it is used, the value is either a string like + // "CEDEX", optionally followed by a number (e.g. "CEDEX 7"), or just a number + // alone, representing the "sector code" (Jamaica), "delivery area indicator" + // (Malawi) or "post office indicator" (e.g. Côte d'Ivoire). + string sorting_code = 5; + + // Optional. Highest administrative subdivision which is used for postal + // addresses of a country or region. + // For example, this can be a state, a province, an oblast, or a prefecture. + // Specifically, for Spain this is the province and not the autonomous + // community (e.g. "Barcelona" and not "Catalonia"). + // Many countries don't use an administrative area in postal addresses. E.g. + // in Switzerland this should be left unpopulated. + string administrative_area = 6; + + // Optional. Generally refers to the city/town portion of the address. + // Examples: US city, IT comune, UK post town. + // In regions of the world where localities are not well defined or do not fit + // into this structure well, leave locality empty and use address_lines. + string locality = 7; + + // Optional. Sublocality of the address. + // For example, this can be neighborhoods, boroughs, districts. + string sublocality = 8; + + // Unstructured address lines describing the lower levels of an address. + // + // Because values in address_lines do not have type information and may + // sometimes contain multiple values in a single field (e.g. + // "Austin, TX"), it is important that the line order is clear. The order of + // address lines should be "envelope order" for the country/region of the + // address. In places where this can vary (e.g. Japan), address_language is + // used to make it explicit (e.g. "ja" for large-to-small ordering and + // "ja-Latn" or "en" for small-to-large). This way, the most specific line of + // an address can be selected based on the language. + // + // The minimum permitted structural representation of an address consists + // of a region_code with all remaining information placed in the + // address_lines. It would be possible to format such an address very + // approximately without geocoding, but no semantic reasoning could be + // made about any of the address components until it was at least + // partially resolved. + // + // Creating an address only containing a region_code and address_lines, and + // then geocoding is the recommended way to handle completely unstructured + // addresses (as opposed to guessing which parts of the address should be + // localities or administrative areas). + repeated string address_lines = 9; + + // Optional. The recipient at the address. + // This field may, under certain circumstances, contain multiline information. + // For example, it might contain "care of" information. + repeated string recipients = 10; + + // Optional. The name of the organization at the address. + string organization = 11; +} diff --git a/karapace/protobuf/google/type/quaternion.proto b/karapace/protobuf/google/type/quaternion.proto new file mode 100644 index 000000000..dfb822def --- /dev/null +++ b/karapace/protobuf/google/type/quaternion.proto @@ -0,0 +1,94 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/quaternion;quaternion"; +option java_multiple_files = true; +option java_outer_classname = "QuaternionProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// A quaternion is defined as the quotient of two directed lines in a +// three-dimensional space or equivalently as the quotient of two Euclidean +// vectors (https://en.wikipedia.org/wiki/Quaternion). +// +// Quaternions are often used in calculations involving three-dimensional +// rotations (https://en.wikipedia.org/wiki/Quaternions_and_spatial_rotation), +// as they provide greater mathematical robustness by avoiding the gimbal lock +// problems that can be encountered when using Euler angles +// (https://en.wikipedia.org/wiki/Gimbal_lock). +// +// Quaternions are generally represented in this form: +// +// w + xi + yj + zk +// +// where x, y, z, and w are real numbers, and i, j, and k are three imaginary +// numbers. +// +// Our naming choice `(x, y, z, w)` comes from the desire to avoid confusion for +// those interested in the geometric properties of the quaternion in the 3D +// Cartesian space. Other texts often use alternative names or subscripts, such +// as `(a, b, c, d)`, `(1, i, j, k)`, or `(0, 1, 2, 3)`, which are perhaps +// better suited for mathematical interpretations. +// +// To avoid any confusion, as well as to maintain compatibility with a large +// number of software libraries, the quaternions represented using the protocol +// buffer below *must* follow the Hamilton convention, which defines `ij = k` +// (i.e. a right-handed algebra), and therefore: +// +// i^2 = j^2 = k^2 = ijk = −1 +// ij = −ji = k +// jk = −kj = i +// ki = −ik = j +// +// Please DO NOT use this to represent quaternions that follow the JPL +// convention, or any of the other quaternion flavors out there. +// +// Definitions: +// +// - Quaternion norm (or magnitude): `sqrt(x^2 + y^2 + z^2 + w^2)`. +// - Unit (or normalized) quaternion: a quaternion whose norm is 1. +// - Pure quaternion: a quaternion whose scalar component (`w`) is 0. +// - Rotation quaternion: a unit quaternion used to represent rotation. +// - Orientation quaternion: a unit quaternion used to represent orientation. +// +// A quaternion can be normalized by dividing it by its norm. The resulting +// quaternion maintains the same direction, but has a norm of 1, i.e. it moves +// on the unit sphere. This is generally necessary for rotation and orientation +// quaternions, to avoid rounding errors: +// https://en.wikipedia.org/wiki/Rotation_formalisms_in_three_dimensions +// +// Note that `(x, y, z, w)` and `(-x, -y, -z, -w)` represent the same rotation, +// but normalization would be even more useful, e.g. for comparison purposes, if +// it would produce a unique representation. It is thus recommended that `w` be +// kept positive, which can be achieved by changing all the signs when `w` is +// negative. +// +message Quaternion { + // The x component. + double x = 1; + + // The y component. + double y = 2; + + // The z component. + double z = 3; + + // The scalar component. + double w = 4; +} diff --git a/karapace/protobuf/google/type/timeofday.proto b/karapace/protobuf/google/type/timeofday.proto new file mode 100644 index 000000000..5cb48aa93 --- /dev/null +++ b/karapace/protobuf/google/type/timeofday.proto @@ -0,0 +1,44 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.type; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/type/timeofday;timeofday"; +option java_multiple_files = true; +option java_outer_classname = "TimeOfDayProto"; +option java_package = "com.google.type"; +option objc_class_prefix = "GTP"; + +// Represents a time of day. The date and time zone are either not significant +// or are specified elsewhere. An API may choose to allow leap seconds. Related +// types are [google.type.Date][google.type.Date] and +// `google.protobuf.Timestamp`. +message TimeOfDay { + // Hours of day in 24 hour format. Should be from 0 to 23. An API may choose + // to allow the value "24:00:00" for scenarios like business closing time. + int32 hours = 1; + + // Minutes of hour of day. Must be from 0 to 59. + int32 minutes = 2; + + // Seconds of minutes of the time. Must normally be from 0 to 59. An API may + // allow the value 60 if it allows leap-seconds. + int32 seconds = 3; + + // Fractions of seconds in nanoseconds. Must be from 0 to 999,999,999. + int32 nanos = 4; +} diff --git a/karapace/protobuf/google/type/type.yaml b/karapace/protobuf/google/type/type.yaml new file mode 100644 index 000000000..d5c71364d --- /dev/null +++ b/karapace/protobuf/google/type/type.yaml @@ -0,0 +1,40 @@ +type: google.api.Service +config_version: 3 +name: type.googleapis.com +title: Common Types + +types: +- name: google.type.Color +- name: google.type.Date +- name: google.type.DateTime +- name: google.type.Decimal +- name: google.type.Expr +- name: google.type.Fraction +- name: google.type.Interval +- name: google.type.LatLng +- name: google.type.LocalizedText +- name: google.type.Money +- name: google.type.PhoneNumber +- name: google.type.PostalAddress +- name: google.type.Quaternion +- name: google.type.TimeOfDay + +enums: +- name: google.type.CalendarPeriod +- name: google.type.DayOfWeek +- name: google.type.Month + +documentation: + summary: Defines common types for Google APIs. + overview: |- + # Google Common Types + + This package contains definitions of common types for Google APIs. + All types defined in this package are suitable for different APIs to + exchange data, and will never break binary compatibility. They should + have design quality comparable to major programming languages like + Java and C#. + + NOTE: Some common types are defined in the package `google.protobuf` + as they are directly supported by Protocol Buffers compiler and + runtime. Those types are called Well-Known Types. diff --git a/karapace/protobuf/known_dependency.py b/karapace/protobuf/known_dependency.py index c8bbe6133..7265d02c4 100644 --- a/karapace/protobuf/known_dependency.py +++ b/karapace/protobuf/known_dependency.py @@ -1,4 +1,13 @@ +# Support of known dependencies + from enum import Enum +from typing import Any, Dict, Set + + +def static_init(cls: Any) -> object: + if getattr(cls, "static_init", None): + cls.static_init() + return cls class KnownDependencyLocation(Enum): @@ -31,5 +40,113 @@ class KnownDependencyLocation(Enum): TIME_OF_DAY_LOCATION = "google/type/timeofday.proto" -class KnownDependency(Enum): - pass +@static_init +class KnownDependency: + index: Dict = dict() + index_simple: Dict = dict() + map: Dict = { + "google/protobuf/any.proto": [".google.protobuf.Any"], + "google/protobuf/api.proto": [".google.protobuf.Api", ".google.protobuf.Method", ".google.protobuf.Mixin"], + "google/protobuf/descriptor.proto": [ + ".google.protobuf.FileDescriptorSet", + ".google.protobuf.FileDescriptorProto", + ".google.protobuf.DescriptorProto", + ".google.protobuf.ExtensionRangeOptions", + ".google.protobuf.FieldDescriptorProto", + ".google.protobuf.OneofDescriptorProto", + ".google.protobuf.EnumDescriptorProto", + ".google.protobuf.EnumValueDescriptorProto", + ".google.protobuf.ServiceDescriptorProto", + ".google.protobuf.MethodDescriptorProto", + ".google.protobuf.FileOptions", + ".google.protobuf.MessageOptions", + ".google.protobuf.FieldOptions", + ".google.protobuf.OneofOptions", + ".google.protobuf.EnumOptions", + ".google.protobuf.EnumValueOptions", + ".google.protobuf.ServiceOptions", + ".google.protobuf.MethodOptions", + ".google.protobuf.UninterpretedOption", + ".google.protobuf.SourceCodeInfo", + ".google.protobuf.GeneratedCodeInfo", + ], + "google/protobuf/duration.proto": [".google.protobuf.Duration"], + "google/protobuf/empty.proto": [".google.protobuf.Empty"], + "google/protobuf/field_mask.proto": [".google.protobuf.FieldMask"], + "google/protobuf/source_context.proto": [".google.protobuf.SourceContext"], + "google/protobuf/struct.proto": [ + ".google.protobuf.Struct", + ".google.protobuf.Value", + ".google.protobuf.NullValue", + ".google.protobuf.ListValue", + ], + "google/protobuf/timestamp.proto": [".google.protobuf.Timestamp"], + "google/protobuf/type.proto": [ + ".google.protobuf.Type", + ".google.protobuf.Field", + ".google.protobuf.Enum", + ".google.protobuf.EnumValue", + ".google.protobuf.Option", + ".google.protobuf.Syntax", + ], + "google/protobuf/wrappers.proto": [ + ".google.protobuf.DoubleValue", + ".google.protobuf.FloatValue", + ".google.protobuf.Int64Value", + ".google.protobuf.UInt64Value", + ".google.protobuf.Int32Value", + ".google.protobuf.UInt32Value", + ".google.protobuf.BoolValue", + ".google.protobuf.StringValue", + ".google.protobuf.BytesValue", + ], + "google/type/calendar_period.proto": [".google.type.CalendarPeriod"], + "google/type/color.proto": [".google.type.Color"], + "google/type/date.proto": [".google.type.Date"], + "google/type/datetime.proto": [".google.type.DateTime", ".google.type.TimeZone"], + "google/type/dayofweek.proto": [".google.type.DayOfWeek"], + "google/type/decimal.proto": [".google.type.Decimal"], + "google/type/expr.proto": [".google.type.Expr"], + "google/type/fraction.proto": [".google.type.Fraction"], + "google/type/interval.proto": [".google.type.Interval"], + "google/type/latlng.proto": [".google.type.LatLng"], + "google/type/money.proto": [".google.type.Money"], + "google/type/month.proto": [".google.type.Month"], + "google/type/phone_number.proto": [".google.type.PhoneNumber"], + "google/type/postal_address.proto": [".google.type.PostalAddress"], + "google/type/quaternion.proto": [".google.type.Quaternion"], + "google/type/timeofday.proto": [".google.type.TimeOfDay"], + } + + @classmethod + def static_init(cls) -> None: + for key, value in cls.map.items(): + for item in value: + cls.index[item] = key + dot = item.rfind(".") + cls.index_simple[item[dot + 1 :]] = key + + +@static_init +class DependenciesHardcoded: + index: Set = set() + + @classmethod + def static_init(cls) -> None: + cls.index = { + "bool", + "bytes", + "double", + "float", + "fixed32", + "fixed64", + "int32", + "int64", + "sfixed32", + "sfixed64", + "sint32", + "sint64", + "string", + "uint32", + "uint64", + } diff --git a/karapace/protobuf/schema.py b/karapace/protobuf/schema.py index 37076a297..1ef09bccc 100644 --- a/karapace/protobuf/schema.py +++ b/karapace/protobuf/schema.py @@ -4,6 +4,7 @@ from karapace.protobuf.compare_result import CompareResult from karapace.protobuf.enum_element import EnumElement from karapace.protobuf.exception import IllegalArgumentException +from karapace.protobuf.known_dependency import DependenciesHardcoded, KnownDependency from karapace.protobuf.location import Location from karapace.protobuf.message_element import MessageElement from karapace.protobuf.one_of_element import OneOfElement @@ -15,8 +16,6 @@ from karapace.schema_references import References from typing import Dict, Optional, TYPE_CHECKING -import re - if TYPE_CHECKING: from karapace.schema_reader import KafkaSchemaReader @@ -127,6 +126,7 @@ class DependencyVerifier: def __init__(self): self.declared_types = list() self.used_types = list() + self.import_path = list() def add_declared_type(self, full_name: str): self.declared_types.append(full_name) @@ -134,18 +134,33 @@ def add_declared_type(self, full_name: str): def add_used_type(self, element_type: str): self.used_types.append(element_type) + def add_import(self, import_name: str): + self.import_path.append(import_name) + def verify(self) -> DependencyVerifierResult: + declared_index = set(self.declared_types) for used_type in self.used_types: - t = True - r = re.compile(re.escape(used_type) + "$") - for delcared_type in self.declared_types: - if r.match(delcared_type): - t = True - break - if not t: + + # TODO: it must be improved !!! + if not ( + used_type in DependenciesHardcoded.index + or KnownDependency.index_simple.get(used_type) is not None + or KnownDependency.index.get(used_type) is not None + or used_type in declared_index + or "." + used_type in declared_index + ): return DependencyVerifierResult(False, f"type {used_type} is not defined") + + # result: DependencyVerifierResult = self.verify_ciclyc_dependencies() + # if not result.result: + # return result return DependencyVerifierResult(True) + # def verify_ciclyc_dependencies(self) -> DependencyVerifierResult: + + # TODO: add recursion detection + # return DependencyVerifierResult(True) + def _process_one_of(verifier: DependencyVerifier, one_of: OneOfElement): for field in one_of.fields: @@ -168,35 +183,51 @@ def __init__( self.dependencies: Dict[str, Dependency] = dict() self.reslove_dependencies() + def gather_deps(self) -> DependencyVerifier: + verifier = DependencyVerifier() + self.collect_dependencies(verifier) + return verifier + def verify_schema_dependencies(self) -> DependencyVerifierResult: verifier = DependencyVerifier() - self._verify_schema_dependencies(verifier) + self.collect_dependencies(verifier) return verifier.verify() - def _verify_schema_dependencies(self, verifier: DependencyVerifier) -> bool: + def collect_dependencies(self, verifier: DependencyVerifier): + + for key in self.dependencies: + self.dependencies[key].schema.collect_dependencies(verifier) + # verifier.add_import?? we have no access to own Kafka structure from this class... + # but we need data to analyse imports to avoid ciclyc dependencies... + package_name = self.proto_file_element.package_name + if package_name is None: + package_name = "" for element_type in self.proto_file_element.types: type_name = element_type.name - full_name = "." + package_name + type_name + full_name = "." + package_name + "." + type_name verifier.add_declared_type(full_name) + verifier.add_declared_type(type_name) if isinstance(element_type, MessageElement): for one_of in element_type.one_ofs: _process_one_of(verifier, one_of) for field in element_type.fields: verifier.add_used_type(field.element_type) - for nested_type in element_type.nested_types: - self._process_nested_type(verifier, full_name, nested_type) + self._process_nested_type(verifier, package_name, type_name, nested_type) + + def _process_nested_type(self, verifier: DependencyVerifier, package_name: str, parent_name, element_type: TypeElement): + + verifier.add_declared_type("." + package_name + "." + parent_name + "." + element_type.name) + verifier.add_declared_type(parent_name + "." + element_type.name) - def _process_nested_type(self, verifier: DependencyVerifier, full_name: str, element_type: TypeElement): - full_name = full_name + element_type.name if isinstance(element_type, MessageElement): for one_of in element_type.one_ofs: _process_one_of(verifier, one_of) for field in element_type.fields: verifier.add_used_type(field.element_type) for nested_type in element_type.nested_types: - self._process_nested_type(verifier, full_name, nested_type) + self._process_nested_type(verifier, package_name, parent_name + "." + element_type.name, nested_type) def __str__(self) -> str: if not self.cache_string: @@ -252,18 +283,19 @@ def compare(self, other: "ProtobufSchema", result: CompareResult) -> CompareResu self.proto_file_element.compare(other.proto_file_element, result) def reslove_dependencies(self): + if self.references is None: + return try: - if self.references is not None: - for r in self.references.val(): - subject = r["subject"] - version = r["version"] - name = r["name"] - subject_data = self.ksr.subjects.get(subject) - schema_data = subject_data["schemas"][version] - schema = schema_data["schema"].schema_str - references = schema_data.get("references") - parsed_schema = ProtobufSchema(schema, references) - self.dependencies[name] = Dependency(name, subject, version, parsed_schema) + for r in self.references.val(): + subject = r["subject"] + version = r["version"] + name = r["name"] + subject_data = self.ksr.subjects.get(subject) + schema_data = subject_data["schemas"][version] + schema = schema_data["schema"].schema_str + references = schema_data.get("references") + parsed_schema = ProtobufSchema(schema, references) + self.dependencies[name] = Dependency(name, subject, version, parsed_schema) except Exception as e: # TODO: need the exception? raise e diff --git a/karapace/schema_models.py b/karapace/schema_models.py index c10e0db72..0900e36d2 100644 --- a/karapace/schema_models.py +++ b/karapace/schema_models.py @@ -54,7 +54,7 @@ def parse_jsonschema_definition(schema_definition: str) -> Draft7Validator: def parse_protobuf_schema_definition( - schema_definition: str, references: Optional[References] = None, ksr: Optional["KafkaSchemaReader"] = None + schema_definition: str, references: Optional[References] = None, ksr: Optional["KafkaSchemaReader"] = None ) -> ProtobufSchema: """Parses and validates `schema_definition`. @@ -112,33 +112,33 @@ def get_references(self) -> Optional["References"]: return self.references def __eq__(self, other: Any) -> bool: - schema_is_equal = isinstance(other, TypedSchema) and \ - self.schema_type is other.schema_type and self.__str__() == other.__str__() + schema_is_equal = ( + isinstance(other, TypedSchema) and self.schema_type is other.schema_type and self.__str__() == other.__str__() + ) if not schema_is_equal: return False if self.references is not None: return self.references == other.references - else: - return other.references is None + return other.references is None class ValidatedTypedSchema(TypedSchema): def __init__( - self, - schema_type: SchemaType, - schema_str: str, - schema: Union[Draft7Validator, AvroSchema, ProtobufSchema], - references: Optional["References"] = None, + self, + schema_type: SchemaType, + schema_str: str, + schema: Union[Draft7Validator, AvroSchema, ProtobufSchema], + references: Optional["References"] = None, ): super().__init__(schema_type=schema_type, schema_str=schema_str, references=references) self.schema = schema @staticmethod def parse( - schema_type: SchemaType, - schema_str: str, - references: Optional["References"] = None, - ksr: Optional["KafkaSchemaReader"] = None, + schema_type: SchemaType, + schema_str: str, + references: Optional["References"] = None, + ksr: Optional["KafkaSchemaReader"] = None, ) -> "ValidatedTypedSchema": if schema_type not in [SchemaType.AVRO, SchemaType.JSONSCHEMA, SchemaType.PROTOBUF]: raise InvalidSchema(f"Unknown parser {schema_type} for {schema_str}") @@ -161,15 +161,15 @@ def parse( try: parsed_schema = parse_protobuf_schema_definition(schema_str, references, ksr) except ( - TypeError, - SchemaError, - AssertionError, - ProtobufParserRuntimeException, - IllegalStateException, - IllegalArgumentException, - ProtobufError, - ProtobufException, - ProtobufSchemaParseException, + TypeError, + SchemaError, + AssertionError, + ProtobufParserRuntimeException, + IllegalStateException, + IllegalArgumentException, + ProtobufError, + ProtobufException, + ProtobufSchemaParseException, ) as e: raise InvalidSchema from e else: diff --git a/karapace/utils.py b/karapace/utils.py index 95302da04..711bff5d8 100644 --- a/karapace/utils.py +++ b/karapace/utils.py @@ -12,6 +12,7 @@ from decimal import Decimal from http import HTTPStatus from kafka.client_async import BrokerConnection, KafkaClient, MetadataRequest +from pathlib import Path from types import MappingProxyType from typing import NoReturn, overload, Union @@ -83,6 +84,10 @@ def assert_never(value: NoReturn) -> NoReturn: raise RuntimeError(f"This code should never be reached, got: {value}") +def get_project_root() -> Path: + return Path(__file__).parent.parent + + class Timeout(Exception): pass diff --git a/tests/integration/test_schema_protobuf.py b/tests/integration/test_schema_protobuf.py index 2dc349f73..ca62f87ed 100644 --- a/tests/integration/test_schema_protobuf.py +++ b/tests/integration/test_schema_protobuf.py @@ -135,7 +135,7 @@ async def test_protobuf_schema_references(registry_async_client: Client) -> None | message Value { | message Label{ | int32 Id = 1; - | str name = 2; + | string name = 2; | } | Customer customer = 1; | int32 x = 2; @@ -169,3 +169,70 @@ async def test_protobuf_schema_references(registry_async_client: Client) -> None match_msg = "Subject 'customer' Version 1 was not deleted because it is referenced by schemas with ids:[2]" myjson = res.json() assert myjson["error_code"] == 44503 and myjson["message"] == match_msg + + +async def test_protobuf_schema_verifier(registry_async_client: Client) -> None: + customer_schema = """ + |syntax = "proto3"; + |package a1; + |message Customer { + | string name = 1; + | int32 code = 2; + |} + |""" + + customer_schema = trim_margin(customer_schema) + res = await registry_async_client.post( + "subjects/customer/versions", json={"schemaType": "PROTOBUF", "schema": customer_schema} + ) + assert res.status_code == 200 + assert "id" in res.json() + original_schema = """ + |syntax = "proto3"; + |package a1; + |import "Customer.proto"; + |message TestMessage { + | enum Enum { + | HIGH = 0; + | MIDDLE = 1; + | LOW = 2; + | } + | message Value { + | message Label{ + | int32 Id = 1; + | string name = 2; + | } + | Customer customer = 1; + | int32 x = 2; + | } + | string test = 1; + | .a1.TestMessage.Value val = 2; + | TestMessage.Value valx = 3; + | + | oneof page_info { + | option (my_option) = true; + | int32 page_number = 5; + | int32 result_per_page = 6; + | } + |} + |""" + + original_schema = trim_margin(original_schema) + references = [{"name": "Customer.proto", "subject": "customer", "version": 1}] + res = await registry_async_client.post( + "subjects/test_schema/versions", + json={"schemaType": "PROTOBUF", "schema": original_schema, "references": references}, + ) + assert res.status_code == 200 + assert "id" in res.json() + res = await registry_async_client.get("subjects/customer/versions/latest/referencedby", json={}) + assert res.status_code == 200 + myjson = res.json() + referents = [2] + assert not any(x != y for x, y in zip(myjson, referents)) + + res = await registry_async_client.delete("subjects/customer/versions/1") + assert res.status_code == 404 + match_msg = "Subject 'customer' Version 1 was not deleted because it is referenced by schemas with ids:[2]" + myjson = res.json() + assert myjson["error_code"] == 44503 and myjson["message"] == match_msg diff --git a/tests/unit/protobuf/test_protobuf_schema_workaround.py b/tests/unit/protobuf/test_protobuf_schema_workaround.py new file mode 100644 index 000000000..e69de29bb From c38c9529df892d8e59202f9a774c1dfb82bf0bcb Mon Sep 17 00:00:00 2001 From: Sergiy Zaschipas Date: Tue, 12 Jul 2022 23:08:19 +0300 Subject: [PATCH 27/44] fixup --- karapace/protobuf/google/type/README.md | 2 +- karapace/protobuf/schema.py | 39 ++++++++++++++++++------- 2 files changed, 30 insertions(+), 11 deletions(-) diff --git a/karapace/protobuf/google/type/README.md b/karapace/protobuf/google/type/README.md index 6caf02cf1..de6a835d7 100644 --- a/karapace/protobuf/google/type/README.md +++ b/karapace/protobuf/google/type/README.md @@ -13,4 +13,4 @@ runtime. Those types are called Well-Known Types. ## Java Utilities A set of Java utilities for the Common Types are provided in the -`//java/com/google/type/util/` package. \ No newline at end of file +`//java/com/google/type/util/` package. diff --git a/karapace/protobuf/schema.py b/karapace/protobuf/schema.py index 1ef09bccc..e6b6a9932 100644 --- a/karapace/protobuf/schema.py +++ b/karapace/protobuf/schema.py @@ -131,8 +131,18 @@ def __init__(self): def add_declared_type(self, full_name: str): self.declared_types.append(full_name) - def add_used_type(self, element_type: str): - self.used_types.append(element_type) + def add_used_type(self, parent: str, element_type: str): + if element_type.find("map<") == 0: + end = element_type.find(">") + virgule = element_type.find(",") + key = element_type[4:virgule] + value = element_type[virgule + 1 : end] + value = value.strip() + self.used_types.append(parent + ";" + key) + self.used_types.append(parent + ";" + value) + # raise ProtobufUnresolvedDependencyException(f"Error in parsing string: {parent+'.'+element_type}") + else: + self.used_types.append(parent + ";" + element_type) def add_import(self, import_name: str): self.import_path.append(import_name) @@ -140,6 +150,11 @@ def add_import(self, import_name: str): def verify(self) -> DependencyVerifierResult: declared_index = set(self.declared_types) for used_type in self.used_types: + delimiter = used_type.rfind(";") + used_type_with_scope = "" + if delimiter != -1: + used_type_with_scope = used_type[:delimiter] + "." + used_type[delimiter + 1 :] + used_type = used_type[delimiter + 1 :] # TODO: it must be improved !!! if not ( @@ -147,6 +162,7 @@ def verify(self) -> DependencyVerifierResult: or KnownDependency.index_simple.get(used_type) is not None or KnownDependency.index.get(used_type) is not None or used_type in declared_index + or (delimiter != -1 and used_type_with_scope in declared_index) or "." + used_type in declared_index ): return DependencyVerifierResult(False, f"type {used_type} is not defined") @@ -162,9 +178,10 @@ def verify(self) -> DependencyVerifierResult: # return DependencyVerifierResult(True) -def _process_one_of(verifier: DependencyVerifier, one_of: OneOfElement): +def _process_one_of(verifier: DependencyVerifier, package_name: str, parent_name: str, one_of: OneOfElement): + parent = package_name + "." + parent_name for field in one_of.fields: - verifier.add_used_type(field.element_type) + verifier.add_used_type(parent, field.element_type) class ProtobufSchema: @@ -203,29 +220,31 @@ def collect_dependencies(self, verifier: DependencyVerifier): package_name = self.proto_file_element.package_name if package_name is None: package_name = "" + else: + package_name = "." + package_name for element_type in self.proto_file_element.types: type_name = element_type.name - full_name = "." + package_name + "." + type_name + full_name = package_name + "." + type_name verifier.add_declared_type(full_name) verifier.add_declared_type(type_name) if isinstance(element_type, MessageElement): for one_of in element_type.one_ofs: - _process_one_of(verifier, one_of) + _process_one_of(verifier, package_name, type_name, one_of) for field in element_type.fields: - verifier.add_used_type(field.element_type) + verifier.add_used_type(full_name, field.element_type) for nested_type in element_type.nested_types: self._process_nested_type(verifier, package_name, type_name, nested_type) def _process_nested_type(self, verifier: DependencyVerifier, package_name: str, parent_name, element_type: TypeElement): - verifier.add_declared_type("." + package_name + "." + parent_name + "." + element_type.name) + verifier.add_declared_type(package_name + "." + parent_name + "." + element_type.name) verifier.add_declared_type(parent_name + "." + element_type.name) if isinstance(element_type, MessageElement): for one_of in element_type.one_ofs: - _process_one_of(verifier, one_of) + _process_one_of(verifier, package_name, parent_name, one_of) for field in element_type.fields: - verifier.add_used_type(field.element_type) + verifier.add_used_type(parent_name, field.element_type) for nested_type in element_type.nested_types: self._process_nested_type(verifier, package_name, parent_name + "." + element_type.name, nested_type) From 1f198af0626ac5af208f725274c41f02ed1defc9 Mon Sep 17 00:00:00 2001 From: Sergiy Zaschipas Date: Wed, 13 Jul 2022 10:52:05 +0300 Subject: [PATCH 28/44] delete workaround file --- tests/unit/protobuf/test_protobuf_schema_workaround.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 tests/unit/protobuf/test_protobuf_schema_workaround.py diff --git a/tests/unit/protobuf/test_protobuf_schema_workaround.py b/tests/unit/protobuf/test_protobuf_schema_workaround.py deleted file mode 100644 index e69de29bb..000000000 From b0d2532a6617366dc68b5cf6a42ad1e5879d40c6 Mon Sep 17 00:00:00 2001 From: Sergiy Zaschipas Date: Mon, 18 Jul 2022 23:05:00 +0300 Subject: [PATCH 29/44] fixup issue --- karapace/schema_models.py | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/karapace/schema_models.py b/karapace/schema_models.py index bc5d4aeb0..cf0dac34a 100644 --- a/karapace/schema_models.py +++ b/karapace/schema_models.py @@ -184,25 +184,3 @@ def __str__(self) -> str: if self.schema_type == SchemaType.PROTOBUF: return str(self.schema) return super().__str__() - -class References: - def __init__(self, schema_type: SchemaType, references: JsonData): - """Schema with type information - - Args: - schema_type (SchemaType): The type of the schema - references (str): The references of schema in Kafka/Json representation - """ - self.schema_type = schema_type - self.references = references - - def val(self) -> JsonData: - return self.references - - def json(self) -> str: - return str(json_encode(self.references, sort_keys=True)) - - def __eq__(self, other: Any) -> bool: - if other is None or not isinstance(other, References): - return False - return self.json() == other.json() From 69c5293168c57dea1b7b3afdfe18137b7e62a66f Mon Sep 17 00:00:00 2001 From: Sergiy Zaschipas Date: Mon, 18 Jul 2022 23:38:42 +0300 Subject: [PATCH 30/44] add info about confluent known dependencies --- karapace/protobuf/known_dependency.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/karapace/protobuf/known_dependency.py b/karapace/protobuf/known_dependency.py index 7265d02c4..5c3e74a00 100644 --- a/karapace/protobuf/known_dependency.py +++ b/karapace/protobuf/known_dependency.py @@ -116,6 +116,8 @@ class KnownDependency: "google/type/postal_address.proto": [".google.type.PostalAddress"], "google/type/quaternion.proto": [".google.type.Quaternion"], "google/type/timeofday.proto": [".google.type.TimeOfDay"], + "confluent/meta.proto": [".confluent.Meta"], + "confluent/type/decimal.proto": [".confluent.type.Decimal"], } @classmethod From 70e8077f5fd6455f87d9032893f6cac4ab9a9977 Mon Sep 17 00:00:00 2001 From: Sergiy Zaschipas Date: Tue, 26 Jul 2022 12:01:21 +0300 Subject: [PATCH 31/44] Update karapace/schema_registry_apis.py Co-authored-by: Jarkko Jaakola <91882676+jjaakola-aiven@users.noreply.github.com> --- karapace/schema_registry_apis.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/karapace/schema_registry_apis.py b/karapace/schema_registry_apis.py index 2f49d884c..01621b0e8 100644 --- a/karapace/schema_registry_apis.py +++ b/karapace/schema_registry_apis.py @@ -45,10 +45,9 @@ class SchemaErrorCodes(Enum): INVALID_VERSION_ID = 42202 INVALID_COMPATIBILITY_LEVEL = 42203 INVALID_AVRO_SCHEMA = 44201 - INVALID_PROTOBUF_SCHEMA = 44202 - INVALID_REFERECES = 44203 - REFERENCES_SUPPORT_NOT_IMPLEMENTED = 44501 - SCHEMAVERSION_HAS_REFERENCES = 44503 + INVALID_REFERENCES = 44301 + REFERENCES_SUPPORT_NOT_IMPLEMENTED = 44302 + SCHEMAVERSION_HAS_REFERENCES = 44303 NO_MASTER_ERROR = 50003 From ef05bd4da00db2e9a7073c126f982f7e747fcc56 Mon Sep 17 00:00:00 2001 From: Sergiy Zaschipas Date: Fri, 29 Jul 2022 15:23:29 +0300 Subject: [PATCH 32/44] improve PR1 code --- karapace/schema_models.py | 9 +++++++++ karapace/schema_reader.py | 6 ++++++ karapace/schema_registry_apis.py | 22 ++++++++++++++-------- tests/integration/test_schema.py | 4 ++-- tests/integration/test_schema_protobuf.py | 8 +++++++- 5 files changed, 38 insertions(+), 11 deletions(-) diff --git a/karapace/schema_models.py b/karapace/schema_models.py index fe2961592..327229ec0 100644 --- a/karapace/schema_models.py +++ b/karapace/schema_models.py @@ -188,6 +188,15 @@ def __init__(self, schema_type: SchemaType, references: JsonData): references (str): The references of schema in Kafka/Json representation """ self.schema_type = schema_type + if self.schema_type != "PROTOBUF" or not isinstance(references, list): + raise InvalidReferences + + for ref in references: + if not isinstance(ref, dict): + raise InvalidReferences + if ref.get("name") is None or ref.get("subject") is None or ref.get("version") is None: + raise InvalidReferences + self.references = references def val(self) -> JsonData: diff --git a/karapace/schema_reader.py b/karapace/schema_reader.py index 4b13a9197..efc81f086 100644 --- a/karapace/schema_reader.py +++ b/karapace/schema_reader.py @@ -445,3 +445,9 @@ def get_schemas( key: val for key, val in self.subjects[subject]["schemas"].items() if val.get("deleted", False) is False } return non_deleted_schemas + + def remove_referenced_by(self, schema_id: SchemaId, references: List): + for ref in references: + key = reference_key(ref["subject"], ref["version"]) + if self.referenced_by.get(key, None) and schema_id in self.referenced_by[key]: + self.referenced_by[key].remove(schema_id) diff --git a/karapace/schema_registry_apis.py b/karapace/schema_registry_apis.py index 01621b0e8..b9a1c0744 100644 --- a/karapace/schema_registry_apis.py +++ b/karapace/schema_registry_apis.py @@ -30,7 +30,6 @@ @unique class SchemaErrorCodes(Enum): EMPTY_SCHEMA = 42201 - REFERENCE_EXISTS_ERROR_CODE = 42206 HTTP_NOT_FOUND = HTTPStatus.NOT_FOUND.value HTTP_CONFLICT = HTTPStatus.CONFLICT.value HTTP_UNPROCESSABLE_ENTITY = HTTPStatus.UNPROCESSABLE_ENTITY.value @@ -47,7 +46,7 @@ class SchemaErrorCodes(Enum): INVALID_AVRO_SCHEMA = 44201 INVALID_REFERENCES = 44301 REFERENCES_SUPPORT_NOT_IMPLEMENTED = 44302 - SCHEMAVERSION_HAS_REFERENCES = 44303 + REFERENCE_EXISTS = 42206 NO_MASTER_ERROR = 50003 @@ -59,7 +58,7 @@ class SchemaErrorMessages(Enum): "forward, full, backward_transitive, forward_transitive, and " "full_transitive" ) - REFERENCES_SUPPORT_NOT_IMPLEMENTED = "Schema references are not supported for '{schema_type}' schema type yet" + REFERENCES_SUPPORT_NOT_IMPLEMENTED = "Schema references are not supported for '{schema_type}' schema type" class KarapaceSchemaRegistry(KarapaceBase): @@ -642,7 +641,7 @@ async def _subject_delete_local(self, content_type: str, subject: str, permanent if referenced_by and len(referenced_by) > 0: self.r( body={ - "error_code": SchemaErrorCodes.SCHEMAVERSION_HAS_REFERENCES.value, + "error_code": SchemaErrorCodes.REFERENCE_EXISTS.value, "message": ( f"Subject '{subject}' Version {version} cannot be not be deleted " "because it is referenced by schemas with ids:[" + ", ".join(map(str, referenced_by)) + "]" @@ -654,16 +653,20 @@ async def _subject_delete_local(self, content_type: str, subject: str, permanent for version, value in list(subject_data["schemas"].items()): schema_id = value.get("id") + references = value.get("references", None) self.log.info("Permanently deleting subject '%s' version %s (schema id=%s)", subject, version, schema_id) self.send_schema_message( subject=subject, schema=None, schema_id=schema_id, version=version, deleted=True, references=None ) + if references and len(references) > 0: + self.ksr.remove_referenced_by(schema_id, references) + else: referenced_by = self.ksr.referenced_by.get(reference_key(subject, latest_schema_id), None) if referenced_by and len(referenced_by) > 0: self.r( body={ - "error_code": SchemaErrorCodes.SCHEMAVERSION_HAS_REFERENCES.value, + "error_code": SchemaErrorCodes.REFERENCE_EXISTS.value, "message": ( f"Subject '{subject}' Version {latest_schema_id} cannot be not be deleted " "because it is referenced by schemas with ids:[" + ", ".join(map(str, referenced_by)) + "]" @@ -784,7 +787,7 @@ async def _subject_version_delete_local(self, content_type: str, subject: str, v if referenced_by and len(referenced_by) > 0: self.r( body={ - "error_code": SchemaErrorCodes.SCHEMAVERSION_HAS_REFERENCES.value, + "error_code": SchemaErrorCodes.REFERENCE_EXISTS.value, "message": ( f"Subject '{subject}' Version {version} was not deleted " "because it is referenced by schemas with ids:[" + ", ".join(map(str, referenced_by)) + "]" @@ -796,6 +799,7 @@ async def _subject_version_delete_local(self, content_type: str, subject: str, v schema_id = subject_schema_data["id"] schema = subject_schema_data["schema"] + references = schema.references self.send_schema_message( subject=subject, schema=None if permanent else schema, @@ -804,6 +808,8 @@ async def _subject_version_delete_local(self, content_type: str, subject: str, v deleted=True, references=None, ) + if references and len(references) > 0: + self.ksr.remove_referenced_by(schema_id, references) self.r(str(version), content_type, status=HTTPStatus.OK) async def subject_version_delete( @@ -998,7 +1004,7 @@ async def subjects_schema_post( human_error = "Provided references is not valid" self.r( body={ - "error_code": SchemaErrorCodes.INVALID_REFERECES.value, + "error_code": SchemaErrorCodes.INVALID_REFERENCES.value, "message": f"Invalid {schema_type} references. Error: {human_error}", }, content_type=content_type, @@ -1078,7 +1084,7 @@ async def subject_post( human_error = "Provided references is not valid" self.r( body={ - "error_code": SchemaErrorCodes.INVALID_REFERECES.value, + "error_code": SchemaErrorCodes.INVALID_REFERENCES.value, "message": f"Invalid {schema_type} references. Error: {human_error}", }, content_type=content_type, diff --git a/tests/integration/test_schema.py b/tests/integration/test_schema.py index a1052a372..6e3689046 100644 --- a/tests/integration/test_schema.py +++ b/tests/integration/test_schema.py @@ -1254,8 +1254,8 @@ async def test_schema_subject_post_invalid(registry_async_client: Client) -> Non json={"schema": schema_str, "references": [{"name": "Customer.avro", "subject": "customer", "version": 1}]}, ) assert res.status_code == 422 - assert res.json()["error_code"] == 44501 - assert res.json()["message"] == "Schema references are not supported for 'AVRO' schema type yet" + assert res.json()["error_code"] == 44302 + assert res.json()["message"] == "Schema references are not supported for 'AVRO' schema type" @pytest.mark.parametrize("trail", ["", "/"]) diff --git a/tests/integration/test_schema_protobuf.py b/tests/integration/test_schema_protobuf.py index 456f8872b..139bc889c 100644 --- a/tests/integration/test_schema_protobuf.py +++ b/tests/integration/test_schema_protobuf.py @@ -155,4 +155,10 @@ async def test_protobuf_schema_references(registry_async_client: Client) -> None assert res.status_code == 404 match_msg = "Subject 'customer' Version 1 was not deleted because it is referenced by schemas with ids:[2]" myjson = res.json() - assert myjson["error_code"] == 44503 and myjson["message"] == match_msg + assert myjson["error_code"] == 42206 and myjson["message"] == match_msg + + res = await registry_async_client.delete("subjects/test_schema/versions/1") + assert res.status_code == 200 + + res = await registry_async_client.delete("subjects/customer/versions/1") + assert res.status_code == 200 From 1d4be0b83a15517e5229f269e5f9c94f123fe4db Mon Sep 17 00:00:00 2001 From: Sergiy Zaschipas Date: Mon, 8 Aug 2022 23:51:24 +0300 Subject: [PATCH 33/44] fixup merge issues --- karapace/schema_registry.py | 3 +++ tests/integration/test_schema_protobuf.py | 7 ++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/karapace/schema_registry.py b/karapace/schema_registry.py index 3b2c56f1b..e589d1453 100644 --- a/karapace/schema_registry.py +++ b/karapace/schema_registry.py @@ -205,6 +205,7 @@ async def subject_version_delete_local(self, subject: Subject, version: Version, schema_id = subject_schema_data["id"] schema = subject_schema_data["schema"] + references = subject_schema_data.get("references", None) self.send_schema_message( subject=subject, schema=None if permanent else schema, @@ -213,6 +214,8 @@ async def subject_version_delete_local(self, subject: Subject, version: Version, deleted=True, references=None, ) + if references and len(references) > 0: + self.schema_reader.remove_referenced_by(schema_id, references) return resolved_version def subject_get(self, subject: Subject, include_deleted: bool = False) -> SubjectData: diff --git a/tests/integration/test_schema_protobuf.py b/tests/integration/test_schema_protobuf.py index d7fc252f1..b96f8c8bc 100644 --- a/tests/integration/test_schema_protobuf.py +++ b/tests/integration/test_schema_protobuf.py @@ -166,9 +166,10 @@ async def test_protobuf_schema_references(registry_async_client: Client) -> None res = await registry_async_client.delete("subjects/customer/versions/1") assert res.status_code == 404 - match_msg = "Subject 'customer' Version 1 was not deleted because it is referenced by schemas with ids:[2]" + match_msg = "Subject 'customer' Version 1 cannot be not be deleted because it is referenced by schemas with ids:[2]" + myjson = res.json() - assert myjson["error_code"] == 44503 and myjson["message"] == match_msg + assert myjson["error_code"] == 42206 and myjson["message"] == match_msg async def test_protobuf_schema_verifier(registry_async_client: Client) -> None: @@ -233,7 +234,7 @@ async def test_protobuf_schema_verifier(registry_async_client: Client) -> None: res = await registry_async_client.delete("subjects/customer/versions/1") assert res.status_code == 404 - match_msg = "Subject 'customer' Version 1 was not deleted because it is referenced by schemas with ids:[2]" + match_msg = "Subject 'customer' Version 1 cannot be not be deleted because it is referenced by schemas with ids:[2]" myjson = res.json() assert myjson["error_code"] == 42206 and myjson["message"] == match_msg From 45c6c54aaaef0c1f9dd95a84b706abdb467a883b Mon Sep 17 00:00:00 2001 From: Sergiy Zaschipas Date: Tue, 9 Aug 2022 00:28:36 +0300 Subject: [PATCH 34/44] lint --- karapace/schema_reader.py | 1 - 1 file changed, 1 deletion(-) diff --git a/karapace/schema_reader.py b/karapace/schema_reader.py index e520b6f73..b2e98b159 100644 --- a/karapace/schema_reader.py +++ b/karapace/schema_reader.py @@ -14,7 +14,6 @@ from karapace.schema_models import TypedSchema from karapace.schema_type import SchemaType from karapace.statsd import StatsClient - from karapace.utils import KarapaceKafkaClient, reference_key from threading import Condition, Event, Lock, Thread from typing import Any, Dict, List, Optional From 4bd7ff784174744bd8313845d3f57801b70c61ec Mon Sep 17 00:00:00 2001 From: Sergiy Zaschipas Date: Tue, 6 Sep 2022 10:44:00 +0300 Subject: [PATCH 35/44] move dependency classes into separate files --- karapace/dependency.py | 21 ++++++++++++ karapace/protobuf/dependency.py | 57 +++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+) create mode 100644 karapace/dependency.py create mode 100644 karapace/protobuf/dependency.py diff --git a/karapace/dependency.py b/karapace/dependency.py new file mode 100644 index 000000000..19736c02f --- /dev/null +++ b/karapace/dependency.py @@ -0,0 +1,21 @@ +from typing import Optional, TYPE_CHECKING + +if TYPE_CHECKING: + from karapace.schema_models import ValidatedTypedSchema + + +class Dependency: + def __init__(self, name: str, subject: str, version: int, schema: "ValidatedTypedSchema") -> None: + self.name = name + self.subject = subject + self.version = version + self.schema = schema + + def identifier(self) -> str: + return self.name + "_" + self.subject + "_" + str(self.version) + + +class DependencyVerifierResult: + def __init__(self, result: bool, message: Optional[str] = ""): + self.result = result + self.message = message diff --git a/karapace/protobuf/dependency.py b/karapace/protobuf/dependency.py new file mode 100644 index 000000000..006a48187 --- /dev/null +++ b/karapace/protobuf/dependency.py @@ -0,0 +1,57 @@ +from karapace.dependency import DependencyVerifierResult +from karapace.protobuf.known_dependency import DependenciesHardcoded, KnownDependency +from karapace.protobuf.one_of_element import OneOfElement +from typing import List + + +class ProtobufDependencyVerifier: + def __init__(self) -> None: + self.declared_types: List[str] = [] + self.used_types: List[str] = [] + self.import_path: List[str] = [] + + def add_declared_type(self, full_name: str) -> None: + self.declared_types.append(full_name) + + def add_used_type(self, parent: str, element_type: str) -> None: + if element_type.find("map<") == 0: + end = element_type.find(">") + virgule = element_type.find(",") + key = element_type[4:virgule] + value = element_type[virgule + 1 : end] + value = value.strip() + self.used_types.append(parent + ";" + key) + self.used_types.append(parent + ";" + value) + else: + self.used_types.append(parent + ";" + element_type) + + def add_import(self, import_name: str) -> None: + self.import_path.append(import_name) + + def verify(self) -> DependencyVerifierResult: + declared_index = set(self.declared_types) + for used_type in self.used_types: + delimiter = used_type.rfind(";") + used_type_with_scope = "" + if delimiter != -1: + used_type_with_scope = used_type[:delimiter] + "." + used_type[delimiter + 1 :] + used_type = used_type[delimiter + 1 :] + + # TODO: it must be improved !!! + if not ( + used_type in DependenciesHardcoded.index + or KnownDependency.index_simple.get(used_type) is not None + or KnownDependency.index.get(used_type) is not None + or used_type in declared_index + or (delimiter != -1 and used_type_with_scope in declared_index) + or "." + used_type in declared_index + ): + return DependencyVerifierResult(False, f"type {used_type} is not defined") + + return DependencyVerifierResult(True) + + +def _process_one_of(verifier: ProtobufDependencyVerifier, package_name: str, parent_name: str, one_of: OneOfElement) -> None: + parent = package_name + "." + parent_name + for field in one_of.fields: + verifier.add_used_type(parent, field.element_type) From d8c54e18146e1f1469191a7585fdc545a05b799d Mon Sep 17 00:00:00 2001 From: Sergiy Zaschipas Date: Wed, 7 Sep 2022 17:48:10 +0300 Subject: [PATCH 36/44] change dependencies class --- karapace/protobuf/schema.py | 130 ++++--------------------------- karapace/schema_models.py | 27 ++++--- karapace/schema_registry_apis.py | 24 +++++- tests/integration/test_schema.py | 1 + 4 files changed, 56 insertions(+), 126 deletions(-) diff --git a/karapace/protobuf/schema.py b/karapace/protobuf/schema.py index 9d72e9aef..6bf30c2d2 100644 --- a/karapace/protobuf/schema.py +++ b/karapace/protobuf/schema.py @@ -1,23 +1,21 @@ # Ported from square/wire: +# Ported from square/wire: # wire-library/wire-schema/src/commonMain/kotlin/com/squareup/wire/schema/Schema.kt # Ported partially for required functionality. +from karapace.dependency import Dependency, DependencyVerifierResult from karapace.protobuf.compare_result import CompareResult +from karapace.protobuf.dependency import _process_one_of, ProtobufDependencyVerifier from karapace.protobuf.enum_element import EnumElement from karapace.protobuf.exception import IllegalArgumentException -from karapace.protobuf.known_dependency import DependenciesHardcoded, KnownDependency from karapace.protobuf.location import Location from karapace.protobuf.message_element import MessageElement -from karapace.protobuf.one_of_element import OneOfElement from karapace.protobuf.option_element import OptionElement from karapace.protobuf.proto_file_element import ProtoFileElement from karapace.protobuf.proto_parser import ProtoParser from karapace.protobuf.type_element import TypeElement from karapace.protobuf.utils import append_documentation, append_indented from karapace.schema_references import References -from typing import Dict, Optional, TYPE_CHECKING - -if TYPE_CHECKING: - from karapace.schema_reader import KafkaSchemaReader +from typing import Dict, Optional def add_slashes(text: str) -> str: @@ -105,115 +103,37 @@ def option_element_string(option: OptionElement) -> str: return f"option {result};\n" -class Dependency: - def __init__(self, name: str, subject: str, version: int, schema: "ProtobufSchema") -> None: - self.name = name - self.subject = subject - self.version = version - self.schema = schema - - def identifier(self) -> str: - return self.name + "_" + self.subject + "_" + str(self.version) - - -class DependencyVerifierResult: - def __init__(self, result: bool, message: Optional[str] = ""): - self.result = result - self.message = message - - -class DependencyVerifier: - def __init__(self): - self.declared_types = list() - self.used_types = list() - self.import_path = list() - - def add_declared_type(self, full_name: str): - self.declared_types.append(full_name) - - def add_used_type(self, parent: str, element_type: str): - if element_type.find("map<") == 0: - end = element_type.find(">") - virgule = element_type.find(",") - key = element_type[4:virgule] - value = element_type[virgule + 1 : end] - value = value.strip() - self.used_types.append(parent + ";" + key) - self.used_types.append(parent + ";" + value) - # raise ProtobufUnresolvedDependencyException(f"Error in parsing string: {parent+'.'+element_type}") - else: - self.used_types.append(parent + ";" + element_type) - - def add_import(self, import_name: str): - self.import_path.append(import_name) - - def verify(self) -> DependencyVerifierResult: - declared_index = set(self.declared_types) - for used_type in self.used_types: - delimiter = used_type.rfind(";") - used_type_with_scope = "" - if delimiter != -1: - used_type_with_scope = used_type[:delimiter] + "." + used_type[delimiter + 1 :] - used_type = used_type[delimiter + 1 :] - - # TODO: it must be improved !!! - if not ( - used_type in DependenciesHardcoded.index - or KnownDependency.index_simple.get(used_type) is not None - or KnownDependency.index.get(used_type) is not None - or used_type in declared_index - or (delimiter != -1 and used_type_with_scope in declared_index) - or "." + used_type in declared_index - ): - return DependencyVerifierResult(False, f"type {used_type} is not defined") - - # result: DependencyVerifierResult = self.verify_ciclyc_dependencies() - # if not result.result: - # return result - return DependencyVerifierResult(True) - - # def verify_ciclyc_dependencies(self) -> DependencyVerifierResult: - - # TODO: add recursion detection - # return DependencyVerifierResult(True) - - -def _process_one_of(verifier: DependencyVerifier, package_name: str, parent_name: str, one_of: OneOfElement): - parent = package_name + "." + parent_name - for field in one_of.fields: - verifier.add_used_type(parent, field.element_type) - - class ProtobufSchema: DEFAULT_LOCATION = Location.get("") def __init__( - self, schema: str, references: Optional[References] = None, schema_reader: Optional["KafkaSchemaReader"] = None + self, schema: str, references: Optional[References] = None, dependencies: Optional[Dict[str, Dependency]] = None ) -> None: if type(schema).__name__ != "str": raise IllegalArgumentException("Non str type of schema string") self.dirty = schema self.cache_string = "" - self.schema_reader = schema_reader self.proto_file_element = ProtoParser.parse(self.DEFAULT_LOCATION, schema) self.references = references - self.dependencies: Dict[str, Dependency] = dict() - self.reslove_dependencies() + self.dependencies = dependencies - def gather_deps(self) -> DependencyVerifier: - verifier = DependencyVerifier() + def gather_deps(self) -> ProtobufDependencyVerifier: + verifier = ProtobufDependencyVerifier() self.collect_dependencies(verifier) return verifier def verify_schema_dependencies(self) -> DependencyVerifierResult: - verifier = DependencyVerifier() + verifier = ProtobufDependencyVerifier() self.collect_dependencies(verifier) return verifier.verify() - def collect_dependencies(self, verifier: DependencyVerifier): + def collect_dependencies(self, verifier: ProtobufDependencyVerifier): + + if self.dependencies is None: + return for key in self.dependencies: - self.dependencies[key].schema.collect_dependencies(verifier) + self.dependencies[key].schema.schema.collect_dependencies(verifier) # verifier.add_import?? we have no access to own Kafka structure from this class... # but we need data to analyse imports to avoid ciclyc dependencies... @@ -235,7 +155,9 @@ def collect_dependencies(self, verifier: DependencyVerifier): for nested_type in element_type.nested_types: self._process_nested_type(verifier, package_name, type_name, nested_type) - def _process_nested_type(self, verifier: DependencyVerifier, package_name: str, parent_name, element_type: TypeElement): + def _process_nested_type( + self, verifier: ProtobufDependencyVerifier, package_name: str, parent_name, element_type: TypeElement + ): verifier.add_declared_type(package_name + "." + parent_name + "." + element_type.name) verifier.add_declared_type(parent_name + "." + element_type.name) @@ -300,21 +222,3 @@ def to_schema(self) -> str: def compare(self, other: "ProtobufSchema", result: CompareResult) -> CompareResult: self.proto_file_element.compare(other.proto_file_element, result) - - def reslove_dependencies(self): - if self.references is None: - return - try: - for r in self.references.val(): - subject = r["subject"] - version = r["version"] - name = r["name"] - subject_data = self.schema_reader.subjects.get(subject) - schema_data = subject_data["schemas"][version] - schema = schema_data["schema"].schema_str - references = schema_data.get("references") - parsed_schema = ProtobufSchema(schema, references) - self.dependencies[name] = Dependency(name, subject, version, parsed_schema) - except Exception as e: - # TODO: need the exception? - raise e diff --git a/karapace/schema_models.py b/karapace/schema_models.py index f2e4e6b64..3a29b557a 100644 --- a/karapace/schema_models.py +++ b/karapace/schema_models.py @@ -12,17 +12,14 @@ ProtobufUnresolvedDependencyException, SchemaParseException as ProtobufSchemaParseException, ) -from karapace.protobuf.schema import ProtobufSchema +from karapace.protobuf.schema import Dependency, ProtobufSchema from karapace.schema_references import References from karapace.schema_type import SchemaType from karapace.utils import json_encode -from typing import Any, Dict, Optional, TYPE_CHECKING, Union +from typing import Any, Dict, Optional, Union import json -if TYPE_CHECKING: - from karapace.schema_reader import KafkaSchemaReader - def parse_avro_schema_definition(s: str) -> AvroSchema: """Compatibility function with Avro which ignores trailing data in JSON @@ -55,7 +52,7 @@ def parse_jsonschema_definition(schema_definition: str) -> Draft7Validator: def parse_protobuf_schema_definition( - schema_definition: str, references: Optional[References] = None, schema_reader: Optional["KafkaSchemaReader"] = None + schema_definition: str, references: Optional[References] = None, dependencies: Optional[Dict[str, Dependency]] = None ) -> ProtobufSchema: """Parses and validates `schema_definition`. @@ -63,7 +60,7 @@ def parse_protobuf_schema_definition( Nothing yet. """ - protobuf_schema = ProtobufSchema(schema_definition, references, schema_reader) + protobuf_schema = ProtobufSchema(schema_definition, references, dependencies) result = protobuf_schema.verify_schema_dependencies() if not result.result: raise ProtobufUnresolvedDependencyException(f"{result.message}") @@ -71,7 +68,13 @@ def parse_protobuf_schema_definition( class TypedSchema: - def __init__(self, schema_type: SchemaType, schema_str: str, references: Optional[References] = None): + def __init__( + self, + schema_type: SchemaType, + schema_str: str, + references: Optional[References] = None, + dependencies: Optional[Dict[str, Dependency]] = None, + ): """Schema with type information Args: @@ -82,6 +85,7 @@ def __init__(self, schema_type: SchemaType, schema_str: str, references: Optiona self.schema_type = schema_type self.schema_str = schema_str self.references = references + self.dependencies = dependencies def to_dict(self) -> Dict[str, Any]: if self.schema_type is SchemaType.PROTOBUF: @@ -119,8 +123,9 @@ def __init__( schema_str: str, schema: Union[Draft7Validator, AvroSchema, ProtobufSchema], references: Optional["References"] = None, + dependencies: Optional[Dict[str, Dependency]] = None, ): - super().__init__(schema_type=schema_type, schema_str=schema_str, references=references) + super().__init__(schema_type=schema_type, schema_str=schema_str, references=references, dependencies=dependencies) self.schema = schema @staticmethod @@ -128,7 +133,7 @@ def parse( schema_type: SchemaType, schema_str: str, references: Optional["References"] = None, - schema_reader: Optional["KafkaSchemaReader"] = None, + dependencies: Optional[Dict[str, Dependency]] = None, ) -> "ValidatedTypedSchema": if schema_type not in [SchemaType.AVRO, SchemaType.JSONSCHEMA, SchemaType.PROTOBUF]: raise InvalidSchema(f"Unknown parser {schema_type} for {schema_str}") @@ -149,7 +154,7 @@ def parse( elif schema_type is SchemaType.PROTOBUF: try: - parsed_schema = parse_protobuf_schema_definition(schema_str, references, schema_reader) + parsed_schema = parse_protobuf_schema_definition(schema_str, references, dependencies) except ( TypeError, SchemaError, diff --git a/karapace/schema_registry_apis.py b/karapace/schema_registry_apis.py index 1b71c2644..a6bf7e638 100644 --- a/karapace/schema_registry_apis.py +++ b/karapace/schema_registry_apis.py @@ -21,6 +21,7 @@ VersionNotFoundException, ) from karapace.karapace import KarapaceBase +from karapace.protobuf.schema import Dependency from karapace.rapu import HTTPRequest, JSON_CONTENT_TYPE, SERVER_NAME from karapace.schema_models import SchemaType, TypedSchema, ValidatedTypedSchema from karapace.schema_references import References @@ -844,7 +845,7 @@ async def subjects_schema_post( schema_type=schema_type, schema_str=schema_str, references=new_schema_references, - schema_reader=self.schema_registry.schema_reader, + dependencies=self.resolve_references(references), ) except InvalidSchema: self.log.exception("No proper parser found") @@ -929,7 +930,7 @@ async def subject_post( schema_type=schema_type, schema_str=body["schema"], references=new_schema_references, - schema_reader=self.schema_registry.schema_reader, + dependencies=self.resolve_references(new_schema_references), ) except (InvalidSchema, InvalidSchemaType) as e: self.log.warning("Invalid schema: %r", body["schema"], exc_info=True) @@ -1017,3 +1018,22 @@ def no_master_error(self, content_type: str) -> None: content_type=content_type, status=HTTPStatus.INTERNAL_SERVER_ERROR, ) + + def resolve_references(self, references: Optional["References"] = None) -> Optional[Dict[str, Dependency]]: + + if references is None: + return + dependencies = dict() + + for r in references.val(): + subject = r["subject"] + version = r["version"] + name = r["name"] + subject_data = self.schema_registry.schema_reader.subjects.get(subject) + schema_data = subject_data["schemas"][version] + parsed_schema = ValidatedTypedSchema.parse( + schema_type=schema_data["schema"].schema_type, + schema_str=schema_data["schema"].schema_str, + dependencies=self.resolve_references(schema_data.get("references")), + ) + dependencies[name] = Dependency(name, subject, version, parsed_schema) diff --git a/tests/integration/test_schema.py b/tests/integration/test_schema.py index 0951017d7..20ead2af0 100644 --- a/tests/integration/test_schema.py +++ b/tests/integration/test_schema.py @@ -266,6 +266,7 @@ async def test_record_schema_subject_compatibility(registry_async_client: Client res = await registry_async_client.get(f"subjects/{subject}/versions/1") assert res.status_code == 200 + r = res.json() assert res.json() == result result = {"id": 2, "schema": json.dumps(evolved_schema), "subject": subject, "version": 2} From d1fa06ec77146c1db99a38c936a38c75ba48c63e Mon Sep 17 00:00:00 2001 From: Sergiy Zaschipas Date: Tue, 20 Sep 2022 22:10:59 +0300 Subject: [PATCH 37/44] fixup references issues --- karapace/schema_reader.py | 2 - karapace/schema_registry_apis.py | 15 ++++--- tests/integration/test_schema.py | 1 - tests/integration/test_schema_protobuf.py | 48 ++++++++++++++++++----- 4 files changed, 49 insertions(+), 17 deletions(-) diff --git a/karapace/schema_reader.py b/karapace/schema_reader.py index f335c28ab..067791263 100644 --- a/karapace/schema_reader.py +++ b/karapace/schema_reader.py @@ -402,10 +402,8 @@ def _handle_msg_schema(self, key: dict, value: Optional[dict]) -> None: ref_str = reference_key(ref["subject"], ref["version"]) referents = self.referenced_by.get(ref_str, None) if referents: - LOG.info("Adding entry subject referenced_by : %r", ref_str) referents.append(schema_id) else: - LOG.info("Adding entry subject referenced_by : %r", ref_str) self.referenced_by[ref_str] = [schema_id] self.schemas[schema_id] = typed_schema diff --git a/karapace/schema_registry_apis.py b/karapace/schema_registry_apis.py index a6bf7e638..c01deec13 100644 --- a/karapace/schema_registry_apis.py +++ b/karapace/schema_registry_apis.py @@ -551,8 +551,8 @@ async def subject_delete( body={ "error_code": SchemaErrorCodes.REFERENCE_EXISTS.value, "message": ( - f"Subject '{subject}' Version {arg.version} cannot be not be deleted " - "because it is referenced by schemas with ids:[" + ", ".join(map(str, arg.referenced_by)) + "]" + f"One or more references exist to the schema " + f"{{magic=1,keytype=SCHEMA,subject={subject},version={arg.version}}}" ), }, content_type=content_type, @@ -645,8 +645,8 @@ async def subject_version_delete( body={ "error_code": SchemaErrorCodes.REFERENCE_EXISTS.value, "message": ( - f"Subject '{subject}' Version {arg.version} cannot be not be deleted " - "because it is referenced by schemas with ids:[" + ", ".join(map(str, arg.referenced_by)) + "]" + f"One or more references exist to the schema " + f"{{magic=1,keytype=SCHEMA,subject={subject},version={arg.version}}}" ), }, content_type=content_type, @@ -1031,9 +1031,14 @@ def resolve_references(self, references: Optional["References"] = None) -> Optio name = r["name"] subject_data = self.schema_registry.schema_reader.subjects.get(subject) schema_data = subject_data["schemas"][version] + sub_refs = schema_data.get("references") + sub_references = None + if sub_refs is not None: + sub_references = References(references.schema_type, sub_refs) + parsed_schema = ValidatedTypedSchema.parse( schema_type=schema_data["schema"].schema_type, schema_str=schema_data["schema"].schema_str, - dependencies=self.resolve_references(schema_data.get("references")), + dependencies=self.resolve_references(sub_references), ) dependencies[name] = Dependency(name, subject, version, parsed_schema) diff --git a/tests/integration/test_schema.py b/tests/integration/test_schema.py index 20ead2af0..0951017d7 100644 --- a/tests/integration/test_schema.py +++ b/tests/integration/test_schema.py @@ -266,7 +266,6 @@ async def test_record_schema_subject_compatibility(registry_async_client: Client res = await registry_async_client.get(f"subjects/{subject}/versions/1") assert res.status_code == 200 - r = res.json() assert res.json() == result result = {"id": 2, "schema": json.dumps(evolved_schema), "subject": subject, "version": 2} diff --git a/tests/integration/test_schema_protobuf.py b/tests/integration/test_schema_protobuf.py index b96f8c8bc..760b5e4d1 100644 --- a/tests/integration/test_schema_protobuf.py +++ b/tests/integration/test_schema_protobuf.py @@ -107,18 +107,40 @@ async def test_protobuf_schema_compatibility(registry_async_client: Client, trai async def test_protobuf_schema_references(registry_async_client: Client) -> None: + customer_schema = """ + |syntax = "proto3"; + |package a1; + |import "Place.proto"; + |message Customer { + | string name = 1; + | int32 code = 2; + | Place place = 3; + |} + |""" + + customer_schema = trim_margin(customer_schema) + + place_schema = """ |syntax = "proto3"; |package a1; - |message Customer { - | string name = 1; - | int32 code = 2; + |message Place { + | string city = 1; + | int32 zone = 2; |} |""" - customer_schema = trim_margin(customer_schema) + place_schema = trim_margin(place_schema) res = await registry_async_client.post( - "subjects/customer/versions", json={"schemaType": "PROTOBUF", "schema": customer_schema} + "subjects/place/versions", json={"schemaType": "PROTOBUF", "schema": place_schema} + ) + assert res.status_code == 200 + assert "id" in res.json() + + customer_references = [{"name": "Place.proto", "subject": "place", "version": 1}] + res = await registry_async_client.post( + "subjects/customer/versions", + json={"schemaType": "PROTOBUF", "schema": customer_schema, "references": customer_references}, ) assert res.status_code == 200 assert "id" in res.json() @@ -161,15 +183,23 @@ async def test_protobuf_schema_references(registry_async_client: Client) -> None res = await registry_async_client.get("subjects/customer/versions/latest/referencedby", json={}) assert res.status_code == 200 myjson = res.json() - referents = [2] + referents = [3] assert not any(x != y for x, y in zip(myjson, referents)) + res = await registry_async_client.get("subjects/place/versions/latest/referencedby", json={}) + assert res.status_code == 200 + myjson = res.json() + res = await registry_async_client.delete("subjects/customer/versions/1") assert res.status_code == 404 - match_msg = "Subject 'customer' Version 1 cannot be not be deleted because it is referenced by schemas with ids:[2]" - + match_msg = "One or more references exist to the schema {magic=1,keytype=SCHEMA,subject=customer,version=1}" myjson = res.json() assert myjson["error_code"] == 42206 and myjson["message"] == match_msg + res = await registry_async_client.delete("subjects/test_schema/versions/1") + assert res.status_code == 200 + + res = await registry_async_client.delete("subjects/customer/versions/1") + assert res.status_code == 200 async def test_protobuf_schema_verifier(registry_async_client: Client) -> None: @@ -234,7 +264,7 @@ async def test_protobuf_schema_verifier(registry_async_client: Client) -> None: res = await registry_async_client.delete("subjects/customer/versions/1") assert res.status_code == 404 - match_msg = "Subject 'customer' Version 1 cannot be not be deleted because it is referenced by schemas with ids:[2]" + match_msg = "One or more references exist to the schema {magic=1,keytype=SCHEMA,subject=customer,version=1}" myjson = res.json() assert myjson["error_code"] == 42206 and myjson["message"] == match_msg From f72f8559d3488ab12999981d6591f6da282f12ba Mon Sep 17 00:00:00 2001 From: Sergiy Zaschipas Date: Mon, 26 Sep 2022 22:52:02 +0300 Subject: [PATCH 38/44] fixup issues in code --- karapace/errors.py | 4 + karapace/protobuf/dependency.py | 1 - karapace/protobuf/schema.py | 13 +- karapace/schema_models.py | 6 +- karapace/schema_registry.py | 66 ++++++- karapace/schema_registry_apis.py | 202 ++++++++++------------ tests/integration/test_schema_protobuf.py | 10 +- 7 files changed, 180 insertions(+), 122 deletions(-) diff --git a/karapace/errors.py b/karapace/errors.py index 6c5075498..6bfda80b3 100644 --- a/karapace/errors.py +++ b/karapace/errors.py @@ -25,6 +25,10 @@ class InvalidReferences(Exception): pass +class ReferencesNotSupportedException(Exception): + pass + + class SchemasNotFoundException(Exception): pass diff --git a/karapace/protobuf/dependency.py b/karapace/protobuf/dependency.py index 006a48187..126b7c887 100644 --- a/karapace/protobuf/dependency.py +++ b/karapace/protobuf/dependency.py @@ -37,7 +37,6 @@ def verify(self) -> DependencyVerifierResult: used_type_with_scope = used_type[:delimiter] + "." + used_type[delimiter + 1 :] used_type = used_type[delimiter + 1 :] - # TODO: it must be improved !!! if not ( used_type in DependenciesHardcoded.index or KnownDependency.index_simple.get(used_type) is not None diff --git a/karapace/protobuf/schema.py b/karapace/protobuf/schema.py index 6bf30c2d2..17f586e35 100644 --- a/karapace/protobuf/schema.py +++ b/karapace/protobuf/schema.py @@ -107,7 +107,8 @@ class ProtobufSchema: DEFAULT_LOCATION = Location.get("") def __init__( - self, schema: str, references: Optional[References] = None, dependencies: Optional[Dict[str, Dependency]] = None + self, schema: str, references: Optional[References] = None, + dependencies: Optional[Dict[str, Dependency]] = None ) -> None: if type(schema).__name__ != "str": raise IllegalArgumentException("Non str type of schema string") @@ -129,11 +130,9 @@ def verify_schema_dependencies(self) -> DependencyVerifierResult: def collect_dependencies(self, verifier: ProtobufDependencyVerifier): - if self.dependencies is None: - return - - for key in self.dependencies: - self.dependencies[key].schema.schema.collect_dependencies(verifier) + if self.dependencies: + for key in self.dependencies: + self.dependencies[key].schema.schema.collect_dependencies(verifier) # verifier.add_import?? we have no access to own Kafka structure from this class... # but we need data to analyse imports to avoid ciclyc dependencies... @@ -156,7 +155,7 @@ def collect_dependencies(self, verifier: ProtobufDependencyVerifier): self._process_nested_type(verifier, package_name, type_name, nested_type) def _process_nested_type( - self, verifier: ProtobufDependencyVerifier, package_name: str, parent_name, element_type: TypeElement + self, verifier: ProtobufDependencyVerifier, package_name: str, parent_name, element_type: TypeElement ): verifier.add_declared_type(package_name + "." + parent_name + "." + element_type.name) diff --git a/karapace/schema_models.py b/karapace/schema_models.py index 3a29b557a..cfd4f8ea2 100644 --- a/karapace/schema_models.py +++ b/karapace/schema_models.py @@ -171,7 +171,11 @@ def parse( raise InvalidSchema(f"Unknown parser {schema_type} for {schema_str}") return ValidatedTypedSchema( - schema_type=schema_type, schema_str=schema_str, schema=parsed_schema, references=references + schema_type=schema_type, + schema_str=schema_str, + schema=parsed_schema, + references=references, + dependencies=dependencies, ) def __str__(self) -> str: diff --git a/karapace/schema_registry.py b/karapace/schema_registry.py index 0b5a5adfd..2fb802050 100644 --- a/karapace/schema_registry.py +++ b/karapace/schema_registry.py @@ -4,10 +4,14 @@ from karapace.compatibility import check_compatibility, CompatibilityModes from karapace.compatibility.jsonschema.checks import is_incompatible from karapace.config import Config +from karapace.dependency import Dependency from karapace.errors import ( IncompatibleSchema, + InvalidReferences, + InvalidSchema, InvalidVersion, ReferenceExistsException, + ReferencesNotSupportedException, SchemasNotFoundException, SchemaVersionNotSoftDeletedException, SchemaVersionSoftDeletedException, @@ -330,8 +334,15 @@ async def write_new_schema_local( for old_version in check_against: old_schema = subject_data["schemas"][old_version]["schema"] + old_schema_references, old_schema_dependencies = self.resolve_schema_references( + subject_data["schemas"][old_version], + ) + validated_old_schema = ValidatedTypedSchema.parse( - schema_type=old_schema.schema_type, schema_str=old_schema.schema_str + schema_type=old_schema.schema_type, + schema_str=old_schema.schema_str, + references=old_schema_references, + dependencies=old_schema_dependencies, ) result = check_compatibility( old_schema=validated_old_schema, @@ -477,3 +488,56 @@ def send_delete_subject_message(self, subject: Subject, version: Version) -> Fut ) value = '{{"subject":"{}","version":{}}}'.format(subject, version) return self.send_kafka_message(key, value) + + def resolve_references(self, references: Optional["References"] = None) -> Optional[Dict[str, Dependency]]: + if references is None: + return None + dependencies = dict() + + for r in references.val(): + subject = r["subject"] + version = r["version"] + name = r["name"] + subject_data = self.schema_reader.subjects.get(subject) + if subject_data is not None: + schema_data = subject_data["schemas"][version] + schema_references, schema_dependencies = self.resolve_schema_references(schema_data) + else: + raise InvalidReferences + + parsed_schema = ValidatedTypedSchema.parse( + schema_type=schema_data["schema"].schema_type, + schema_str=schema_data["schema"].schema_str, + references=schema_references, + dependencies=schema_dependencies, + ) + dependencies[name] = Dependency(name, subject, version, parsed_schema) + return dependencies + + def resolve_schema_references( + self, schema_data: Optional[dict] + ) -> Tuple[Optional[References], Optional[Dict[str, Dependency]]]: + + if schema_data is None: + raise InvalidSchema + + schema_references = schema_data.get("references") + if schema_references is None: + return None, None + + schema_type = schema_data.get("schemaType") + if schema_type is None: + schema = schema_data.get("schema") + if schema is None: + raise InvalidReferences + if isinstance(schema,TypedSchema): + schema_type = schema.schema_type + else: + schema_type = None + if schema_type != SchemaType.PROTOBUF: + raise ReferencesNotSupportedException + + schema_references = References(schema_type, schema_references) + schema_dependencies = self.resolve_references(schema_references) + + return schema_references, schema_dependencies diff --git a/karapace/schema_registry_apis.py b/karapace/schema_registry_apis.py index c01deec13..16944d76b 100644 --- a/karapace/schema_registry_apis.py +++ b/karapace/schema_registry_apis.py @@ -13,6 +13,7 @@ InvalidSchemaType, InvalidVersion, ReferenceExistsException, + ReferencesNotSupportedException, SchemasNotFoundException, SchemaVersionNotSoftDeletedException, SchemaVersionSoftDeletedException, @@ -21,10 +22,8 @@ VersionNotFoundException, ) from karapace.karapace import KarapaceBase -from karapace.protobuf.schema import Dependency from karapace.rapu import HTTPRequest, JSON_CONTENT_TYPE, SERVER_NAME from karapace.schema_models import SchemaType, TypedSchema, ValidatedTypedSchema -from karapace.schema_references import References from karapace.schema_registry import KarapaceSchemaRegistry from karapace.typing import JsonData from karapace.utils import reference_key @@ -278,7 +277,13 @@ async def compatibility_check( body = request.json schema_type = self._validate_schema_type(content_type=content_type, data=body) try: - new_schema = ValidatedTypedSchema.parse(schema_type, body["schema"]) + new_schema_references, new_schema_dependencies = self.schema_registry.resolve_schema_references(body) + new_schema = ValidatedTypedSchema.parse( + schema_type=schema_type, + schema_str=body["schema"], + references=new_schema_references, + dependencies=new_schema_dependencies, + ) except InvalidSchema: self.r( body={ @@ -313,7 +318,13 @@ async def compatibility_check( old_schema_type = self._validate_schema_type(content_type=content_type, data=old) try: - old_schema = ValidatedTypedSchema.parse(old_schema_type, old["schema"]) + old_schema_references, old_schema_dependencies = self.schema_registry.resolve_schema_references(old) + old_schema = ValidatedTypedSchema.parse( + schema_type=old_schema_type, + schema_str=old["schema"], + references=old_schema_references, + dependencies=old_schema_dependencies, + ) except InvalidSchema: self.r( body={ @@ -799,7 +810,7 @@ async def subjects_schema_post( content_type=content_type, status=HTTPStatus.NOT_FOUND, ) - new_schema = None + if "schema" not in body: self.r( body={ @@ -812,41 +823,38 @@ async def subjects_schema_post( schema_str = body["schema"] schema_type = self._validate_schema_type(content_type=content_type, data=body) - new_schema_references = None - references = body.get("references") - if references: - if schema_type != SchemaType.PROTOBUF: - self.r( - body={ - "error_code": SchemaErrorCodes.REFERENCES_SUPPORT_NOT_IMPLEMENTED.value, - "message": SchemaErrorMessages.REFERENCES_SUPPORT_NOT_IMPLEMENTED.value.format( - schema_type=schema_type.value - ), - }, - content_type=content_type, - status=HTTPStatus.UNPROCESSABLE_ENTITY, - ) - - try: - new_schema_references = References(schema_type, references) - except InvalidReferences: - human_error = "Provided references is not valid" - self.r( - body={ - "error_code": SchemaErrorCodes.INVALID_REFERENCES.value, - "message": f"Invalid {schema_type} references. Error: {human_error}", - }, - content_type=content_type, - status=HTTPStatus.UNPROCESSABLE_ENTITY, - ) - try: + new_schema_references, new_schema_dependencies = self.schema_registry.resolve_schema_references(body) new_schema = ValidatedTypedSchema.parse( schema_type=schema_type, schema_str=schema_str, references=new_schema_references, - dependencies=self.resolve_references(references), + dependencies=new_schema_dependencies, ) + for schema in subject_data["schemas"].values(): + validated_typed_schema = ValidatedTypedSchema.parse( + schema["schema"].schema_type, schema["schema"].schema_str + ) + if schema_type is SchemaType.JSONSCHEMA: + schema_valid = validated_typed_schema.to_dict() == new_schema.to_dict() + else: + schema_valid = validated_typed_schema.schema == new_schema.schema + if ( + validated_typed_schema.schema_type == new_schema.schema_type + and schema_valid + and schema.get("references", None) == new_schema_references + ): + ret = { + "subject": subject, + "version": schema["version"], + "id": schema["id"], + "schema": validated_typed_schema.schema_str, + } + if schema_type is not SchemaType.AVRO: + ret["schemaType"] = schema_type + self.r(ret, content_type) + else: + self.log.debug("Schema %r did not match %r", schema, validated_typed_schema) except InvalidSchema: self.log.exception("No proper parser found") self.r( @@ -857,28 +865,28 @@ async def subjects_schema_post( content_type=content_type, status=HTTPStatus.INTERNAL_SERVER_ERROR, ) - for schema in subject_data["schemas"].values(): - validated_typed_schema = ValidatedTypedSchema.parse(schema["schema"].schema_type, schema["schema"].schema_str) - if schema_type is SchemaType.JSONSCHEMA: - schema_valid = validated_typed_schema.to_dict() == new_schema.to_dict() - else: - schema_valid = validated_typed_schema.schema == new_schema.schema - if ( - validated_typed_schema.schema_type == new_schema.schema_type - and schema_valid - and schema.get("references", None) == new_schema_references - ): - ret = { - "subject": subject, - "version": schema["version"], - "id": schema["id"], - "schema": validated_typed_schema.schema_str, - } - if schema_type is not SchemaType.AVRO: - ret["schemaType"] = schema_type - self.r(ret, content_type) - else: - self.log.debug("Schema %r did not match %r", schema, validated_typed_schema) + except ReferencesNotSupportedException: + self.r( + body={ + "error_code": SchemaErrorCodes.REFERENCES_SUPPORT_NOT_IMPLEMENTED.value, + "message": SchemaErrorMessages.REFERENCES_SUPPORT_NOT_IMPLEMENTED.value.format( + schema_type=schema_type.value + ), + }, + content_type=content_type, + status=HTTPStatus.UNPROCESSABLE_ENTITY, + ) + except InvalidReferences: + human_error = "Provided references is not valid" + self.r( + body={ + "error_code": SchemaErrorCodes.INVALID_REFERENCES.value, + "message": f"Invalid {schema_type} references. Error: {human_error}", + }, + content_type=content_type, + status=HTTPStatus.UNPROCESSABLE_ENTITY, + ) + self.r( body={ "error_code": SchemaErrorCodes.SCHEMA_NOT_FOUND.value, @@ -898,40 +906,19 @@ async def subject_post( self._validate_schema_request_body(content_type, body) schema_type = self._validate_schema_type(content_type, body) self._validate_schema_key(content_type, body) - new_schema_references = None - references = body.get("references") - if references: - if schema_type != SchemaType.PROTOBUF: - self.r( - body={ - "error_code": SchemaErrorCodes.REFERENCES_SUPPORT_NOT_IMPLEMENTED.value, - "message": SchemaErrorMessages.REFERENCES_SUPPORT_NOT_IMPLEMENTED.value.format( - schema_type=schema_type.value - ), - }, - content_type=content_type, - status=HTTPStatus.UNPROCESSABLE_ENTITY, - ) - try: - new_schema_references = References(schema_type, references) - except InvalidReferences: - human_error = "Provided references is not valid" - self.r( - body={ - "error_code": SchemaErrorCodes.INVALID_REFERENCES.value, - "message": f"Invalid {schema_type} references. Error: {human_error}", - }, - content_type=content_type, - status=HTTPStatus.UNPROCESSABLE_ENTITY, - ) try: + new_schema_references, new_schema_dependencies = self.schema_registry.resolve_schema_references(body) new_schema = ValidatedTypedSchema.parse( schema_type=schema_type, schema_str=body["schema"], references=new_schema_references, - dependencies=self.resolve_references(new_schema_references), + dependencies=new_schema_dependencies, ) + schema_id = self.get_schema_id_if_exists(subject=subject, schema=new_schema) + if schema_id is not None: + self.r({"id": schema_id}, content_type) + except (InvalidSchema, InvalidSchemaType) as e: self.log.warning("Invalid schema: %r", body["schema"], exc_info=True) if isinstance(e.__cause__, (SchemaParseException, json.JSONDecodeError)): @@ -947,10 +934,27 @@ async def subject_post( content_type=content_type, status=HTTPStatus.UNPROCESSABLE_ENTITY, ) - - schema_id = self.get_schema_id_if_exists(subject=subject, schema=new_schema) - if schema_id is not None: - self.r({"id": schema_id}, content_type) + except ReferencesNotSupportedException: + self.r( + body={ + "error_code": SchemaErrorCodes.REFERENCES_SUPPORT_NOT_IMPLEMENTED.value, + "message": SchemaErrorMessages.REFERENCES_SUPPORT_NOT_IMPLEMENTED.value.format( + schema_type=schema_type.value + ), + }, + content_type=content_type, + status=HTTPStatus.UNPROCESSABLE_ENTITY, + ) + except InvalidReferences: + human_error = "Provided references is not valid" + self.r( + body={ + "error_code": SchemaErrorCodes.INVALID_REFERENCES.value, + "message": f"Invalid {schema_type} references. Error: {human_error}", + }, + content_type=content_type, + status=HTTPStatus.UNPROCESSABLE_ENTITY, + ) are_we_master, master_url = await self.schema_registry.get_master() if are_we_master: @@ -1018,27 +1022,3 @@ def no_master_error(self, content_type: str) -> None: content_type=content_type, status=HTTPStatus.INTERNAL_SERVER_ERROR, ) - - def resolve_references(self, references: Optional["References"] = None) -> Optional[Dict[str, Dependency]]: - - if references is None: - return - dependencies = dict() - - for r in references.val(): - subject = r["subject"] - version = r["version"] - name = r["name"] - subject_data = self.schema_registry.schema_reader.subjects.get(subject) - schema_data = subject_data["schemas"][version] - sub_refs = schema_data.get("references") - sub_references = None - if sub_refs is not None: - sub_references = References(references.schema_type, sub_refs) - - parsed_schema = ValidatedTypedSchema.parse( - schema_type=schema_data["schema"].schema_type, - schema_str=schema_data["schema"].schema_str, - dependencies=self.resolve_references(sub_references), - ) - dependencies[name] = Dependency(name, subject, version, parsed_schema) diff --git a/tests/integration/test_schema_protobuf.py b/tests/integration/test_schema_protobuf.py index 760b5e4d1..4e96e5e77 100644 --- a/tests/integration/test_schema_protobuf.py +++ b/tests/integration/test_schema_protobuf.py @@ -135,6 +135,7 @@ async def test_protobuf_schema_references(registry_async_client: Client) -> None "subjects/place/versions", json={"schemaType": "PROTOBUF", "schema": place_schema} ) assert res.status_code == 200 + assert "id" in res.json() customer_references = [{"name": "Place.proto", "subject": "place", "version": 1}] @@ -143,7 +144,9 @@ async def test_protobuf_schema_references(registry_async_client: Client) -> None json={"schemaType": "PROTOBUF", "schema": customer_schema, "references": customer_references}, ) assert res.status_code == 200 + assert "id" in res.json() + original_schema = """ |syntax = "proto3"; |package a1; @@ -179,22 +182,27 @@ async def test_protobuf_schema_references(registry_async_client: Client) -> None json={"schemaType": "PROTOBUF", "schema": original_schema, "references": references}, ) assert res.status_code == 200 + assert "id" in res.json() + res = await registry_async_client.get("subjects/customer/versions/latest/referencedby", json={}) assert res.status_code == 200 + myjson = res.json() referents = [3] assert not any(x != y for x, y in zip(myjson, referents)) res = await registry_async_client.get("subjects/place/versions/latest/referencedby", json={}) assert res.status_code == 200 - myjson = res.json() + myjson = res.json() res = await registry_async_client.delete("subjects/customer/versions/1") assert res.status_code == 404 + match_msg = "One or more references exist to the schema {magic=1,keytype=SCHEMA,subject=customer,version=1}" myjson = res.json() assert myjson["error_code"] == 42206 and myjson["message"] == match_msg + res = await registry_async_client.delete("subjects/test_schema/versions/1") assert res.status_code == 200 From 4d14ff0648b600f1d81e609fba253a9dd6b8b696 Mon Sep 17 00:00:00 2001 From: Sergiy Zaschipas Date: Tue, 27 Sep 2022 00:48:33 +0300 Subject: [PATCH 39/44] fixup lint --- karapace/protobuf/schema.py | 5 ++--- karapace/schema_registry.py | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/karapace/protobuf/schema.py b/karapace/protobuf/schema.py index 17f586e35..9451a9c8a 100644 --- a/karapace/protobuf/schema.py +++ b/karapace/protobuf/schema.py @@ -107,8 +107,7 @@ class ProtobufSchema: DEFAULT_LOCATION = Location.get("") def __init__( - self, schema: str, references: Optional[References] = None, - dependencies: Optional[Dict[str, Dependency]] = None + self, schema: str, references: Optional[References] = None, dependencies: Optional[Dict[str, Dependency]] = None ) -> None: if type(schema).__name__ != "str": raise IllegalArgumentException("Non str type of schema string") @@ -155,7 +154,7 @@ def collect_dependencies(self, verifier: ProtobufDependencyVerifier): self._process_nested_type(verifier, package_name, type_name, nested_type) def _process_nested_type( - self, verifier: ProtobufDependencyVerifier, package_name: str, parent_name, element_type: TypeElement + self, verifier: ProtobufDependencyVerifier, package_name: str, parent_name, element_type: TypeElement ): verifier.add_declared_type(package_name + "." + parent_name + "." + element_type.name) diff --git a/karapace/schema_registry.py b/karapace/schema_registry.py index 2fb802050..380903c4f 100644 --- a/karapace/schema_registry.py +++ b/karapace/schema_registry.py @@ -530,7 +530,7 @@ def resolve_schema_references( schema = schema_data.get("schema") if schema is None: raise InvalidReferences - if isinstance(schema,TypedSchema): + if isinstance(schema, TypedSchema): schema_type = schema.schema_type else: schema_type = None From e1f39b9f104d41ce5b7872444308180157180fe6 Mon Sep 17 00:00:00 2001 From: Sergiy Zaschipas Date: Wed, 28 Sep 2022 13:06:04 +0300 Subject: [PATCH 40/44] add dependency verifier unit test --- tests/unit/test_dependency_verifier.py | 67 ++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 tests/unit/test_dependency_verifier.py diff --git a/tests/unit/test_dependency_verifier.py b/tests/unit/test_dependency_verifier.py new file mode 100644 index 000000000..9a1923a65 --- /dev/null +++ b/tests/unit/test_dependency_verifier.py @@ -0,0 +1,67 @@ +from karapace.config import read_config +from karapace.protobuf.dependency import ProtobufDependencyVerifier +from karapace.serialization import ( + InvalidMessageHeader, + InvalidMessageSchema, + InvalidPayload, + SchemaRegistryDeserializer, + SchemaRegistrySerializer, + START_BYTE, +) +from tests.utils import test_fail_objects_protobuf, test_objects_protobuf + +import logging +import pytest +import struct + +log = logging.getLogger(__name__) + + +async def test_protobuf_dependency_verifier(): + declared_types = [ + ".a1.Place", + "Place", + ".a1.Customer", + "Customer", + ".a1.TestMessage", + "TestMessage", + ".a1", + ".TestMessage", + ".Enum", + "TestMessage.Enum", + ".a1.TestMessage.Value", + "TestMessage.Value", + ".a1.TestMessage.Value.Label", + "TestMessage", + ".Value.Label", + ] + + used_types = [ + ".a1.Place;string", + ".a1.Place;int32", + ".a1.Customer;string", + ".a1.Customer;int32", + ".a1.Customer;Place", + ".a1.TestMessage;int32", + ".a1.TestMessage;int32", + ".a1.TestMessage;string", + ".a1.TestMessage;.a1.TestMessage.Value", + "TestMessage;Customer", + "TestMessage;int32", + "TestMessage.Value;int32", + "TestMessage.Value;string", + ] + + verifier = ProtobufDependencyVerifier() + for declared in declared_types: + verifier.add_declared_type(declared) + for used in used_types: + x = used.split(";") + verifier.add_used_type(x[0], x[1]) + + result = verifier.verify() + assert result.result, True + + verifier.add_used_type("TestMessage.Delta", "Tag") + assert result.result, False + From 90ef33cc2594678d51d087350039488e9dc6e675 Mon Sep 17 00:00:00 2001 From: Sergiy Zaschipas Date: Wed, 28 Sep 2022 13:09:25 +0300 Subject: [PATCH 41/44] add dependency verifier unit test --- tests/unit/test_dependency_verifier.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/tests/unit/test_dependency_verifier.py b/tests/unit/test_dependency_verifier.py index 9a1923a65..49a5ddf6b 100644 --- a/tests/unit/test_dependency_verifier.py +++ b/tests/unit/test_dependency_verifier.py @@ -1,18 +1,6 @@ -from karapace.config import read_config from karapace.protobuf.dependency import ProtobufDependencyVerifier -from karapace.serialization import ( - InvalidMessageHeader, - InvalidMessageSchema, - InvalidPayload, - SchemaRegistryDeserializer, - SchemaRegistrySerializer, - START_BYTE, -) -from tests.utils import test_fail_objects_protobuf, test_objects_protobuf import logging -import pytest -import struct log = logging.getLogger(__name__) @@ -64,4 +52,3 @@ async def test_protobuf_dependency_verifier(): verifier.add_used_type("TestMessage.Delta", "Tag") assert result.result, False - From 7bd1ab648e10d69483b6249652baa98462e4281c Mon Sep 17 00:00:00 2001 From: Sergiy Zaschipas Date: Tue, 4 Oct 2022 21:55:01 +0300 Subject: [PATCH 42/44] fixup test bug with protobuf --- karapace/schema_reader.py | 65 ++++++++++++++++++++++++++++++++++--- karapace/schema_registry.py | 52 ++--------------------------- 2 files changed, 63 insertions(+), 54 deletions(-) diff --git a/karapace/schema_reader.py b/karapace/schema_reader.py index 0eb7aa11c..f7fe60f08 100644 --- a/karapace/schema_reader.py +++ b/karapace/schema_reader.py @@ -10,15 +10,17 @@ from kafka.errors import KafkaConfigurationError, NoBrokersAvailable, NodeNotReadyError, TopicAlreadyExistsError from karapace import constants from karapace.config import Config -from karapace.errors import InvalidSchema +from karapace.dependency import Dependency +from karapace.errors import InvalidReferences, InvalidSchema, ReferencesNotSupportedException from karapace.key_format import is_key_in_canonical_format, KeyFormatter, KeyMode from karapace.master_coordinator import MasterCoordinator from karapace.schema_models import SchemaType, TypedSchema, ValidatedTypedSchema +from karapace.schema_references import References from karapace.statsd import StatsClient from karapace.typing import SubjectData from karapace.utils import KarapaceKafkaClient, reference_key from threading import Condition, Event, Lock, Thread -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Tuple import json import logging @@ -38,7 +40,6 @@ KAFKA_CLIENT_CREATION_TIMEOUT_SECONDS = 2.0 SCHEMA_TOPIC_CREATION_TIMEOUT_SECONDS = 5.0 - # Metric names METRIC_SCHEMA_TOPIC_RECORDS_PROCESSED_COUNT = "karapace_schema_reader_records_processed" METRIC_SCHEMA_TOPIC_RECORDS_PER_KEYMODE_GAUGE = "karapace_schema_reader_records_per_keymode" @@ -450,7 +451,10 @@ def _handle_msg_schema(self, key: dict, value: Optional[dict]) -> None: return elif schema_type_parsed == SchemaType.PROTOBUF: try: - parsed_schema = ValidatedTypedSchema.parse(SchemaType.PROTOBUF, schema_str) + resolved_references, resolved_dependencies = self.resolve_schema_references(value) + parsed_schema = ValidatedTypedSchema.parse( + SchemaType.PROTOBUF, schema_str, resolved_references, resolved_dependencies + ) schema_str = str(parsed_schema) except InvalidSchema: LOG.exception("Schema is not valid ProtoBuf definition") @@ -552,3 +556,56 @@ def remove_referenced_by(self, schema_id: SchemaId, references: List): key = reference_key(ref["subject"], ref["version"]) if self.referenced_by.get(key, None) and schema_id in self.referenced_by[key]: self.referenced_by[key].remove(schema_id) + + def resolve_references(self, references: Optional["References"] = None) -> Optional[Dict[str, Dependency]]: + if references is None: + return None + dependencies = dict() + + for r in references.val(): + subject = r["subject"] + version = r["version"] + name = r["name"] + subject_data = self.subjects.get(subject) + if subject_data is not None: + schema_data = subject_data["schemas"][version] + schema_references, schema_dependencies = self.resolve_schema_references(schema_data) + else: + raise InvalidReferences + + parsed_schema = ValidatedTypedSchema.parse( + schema_type=schema_data["schema"].schema_type, + schema_str=schema_data["schema"].schema_str, + references=schema_references, + dependencies=schema_dependencies, + ) + dependencies[name] = Dependency(name, subject, version, parsed_schema) + return dependencies + + def resolve_schema_references( + self, schema_data: Optional[dict] + ) -> Tuple[Optional[References], Optional[Dict[str, Dependency]]]: + + if schema_data is None: + raise InvalidSchema + + schema_references = schema_data.get("references") + if schema_references is None: + return None, None + + schema_type = schema_data.get("schemaType") + if schema_type is None: + schema = schema_data.get("schema") + if schema is None: + raise InvalidReferences + if isinstance(schema, TypedSchema): + schema_type = schema.schema_type + else: + schema_type = None + if schema_type != SchemaType.PROTOBUF: + raise ReferencesNotSupportedException + + schema_references = References(schema_type, schema_references) + schema_dependencies = self.resolve_references(schema_references) + + return schema_references, schema_dependencies diff --git a/karapace/schema_registry.py b/karapace/schema_registry.py index 523f84317..c94d2dcbc 100644 --- a/karapace/schema_registry.py +++ b/karapace/schema_registry.py @@ -7,11 +7,8 @@ from karapace.dependency import Dependency from karapace.errors import ( IncompatibleSchema, - InvalidReferences, - InvalidSchema, InvalidVersion, ReferenceExistsException, - ReferencesNotSupportedException, SchemasNotFoundException, SchemaTooLargeException, SchemaVersionNotSoftDeletedException, @@ -540,54 +537,9 @@ def send_delete_subject_message(self, subject: Subject, version: Version) -> Fut return self.send_kafka_message(key, value) def resolve_references(self, references: Optional["References"] = None) -> Optional[Dict[str, Dependency]]: - if references is None: - return None - dependencies = dict() - - for r in references.val(): - subject = r["subject"] - version = r["version"] - name = r["name"] - subject_data = self.schema_reader.subjects.get(subject) - if subject_data is not None: - schema_data = subject_data["schemas"][version] - schema_references, schema_dependencies = self.resolve_schema_references(schema_data) - else: - raise InvalidReferences - - parsed_schema = ValidatedTypedSchema.parse( - schema_type=schema_data["schema"].schema_type, - schema_str=schema_data["schema"].schema_str, - references=schema_references, - dependencies=schema_dependencies, - ) - dependencies[name] = Dependency(name, subject, version, parsed_schema) - return dependencies + return self.schema_reader.resolve_references(references) def resolve_schema_references( self, schema_data: Optional[dict] ) -> Tuple[Optional[References], Optional[Dict[str, Dependency]]]: - - if schema_data is None: - raise InvalidSchema - - schema_references = schema_data.get("references") - if schema_references is None: - return None, None - - schema_type = schema_data.get("schemaType") - if schema_type is None: - schema = schema_data.get("schema") - if schema is None: - raise InvalidReferences - if isinstance(schema, TypedSchema): - schema_type = schema.schema_type - else: - schema_type = None - if schema_type != SchemaType.PROTOBUF: - raise ReferencesNotSupportedException - - schema_references = References(schema_type, schema_references) - schema_dependencies = self.resolve_references(schema_references) - - return schema_references, schema_dependencies + return self.schema_reader.resolve_schema_references(schema_data) From d5fdf5970bd42d4ac7bc7b33d34c7fdf0290def9 Mon Sep 17 00:00:00 2001 From: Sergiy Zaschipas Date: Wed, 5 Oct 2022 08:51:41 +0300 Subject: [PATCH 43/44] fixup bug2 --- karapace/schema_registry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/karapace/schema_registry.py b/karapace/schema_registry.py index c94d2dcbc..6317b36c2 100644 --- a/karapace/schema_registry.py +++ b/karapace/schema_registry.py @@ -241,7 +241,7 @@ async def subject_version_delete_local(self, subject: Subject, version: Version, if permanent and not subject_schema_data.get("deleted", False): raise SchemaVersionNotSoftDeletedException() - referenced_by = self.schema_reader.referenced_by.get(reference_key(subject, int(version)), None) + referenced_by = self.schema_reader.referenced_by.get(reference_key(subject, int(resolved_version)), None) if referenced_by and len(referenced_by) > 0: raise ReferenceExistsException(referenced_by, version) From 37e8adbb3cf75c4f5248ed616f5775bda12e0af9 Mon Sep 17 00:00:00 2001 From: Sergiy Zaschipas Date: Wed, 5 Oct 2022 11:06:26 +0300 Subject: [PATCH 44/44] fixup bug2 --- karapace/schema_registry_apis.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/karapace/schema_registry_apis.py b/karapace/schema_registry_apis.py index 1a28560c5..3200338bd 100644 --- a/karapace/schema_registry_apis.py +++ b/karapace/schema_registry_apis.py @@ -1063,10 +1063,6 @@ async def subject_post( references=new_schema_references, dependencies=new_schema_dependencies, ) - schema_id = self.get_schema_id_if_exists(subject=subject, schema=new_schema) - if schema_id is not None: - self.r({"id": schema_id}, content_type) - except (InvalidSchema, InvalidSchemaType) as e: self.log.warning("Invalid schema: %r", body["schema"], exc_info=True) if isinstance(e.__cause__, (SchemaParseException, json.JSONDecodeError)):