From 4d707a8c159ad4fea863867c5edd12ff95bcb572 Mon Sep 17 00:00:00 2001 From: sgayangi Date: Wed, 11 Sep 2024 13:56:52 +0530 Subject: [PATCH] Update parser for proto files --- .../ballerina/ConfigGenreatorClient.bal | 24 ++- .../RuntimeAPICommonUtil.bal | 15 +- .../wso2/apk/config/RuntimeAPICommonUtil.java | 8 +- .../apk/config/definitions/ProtoParser.java | 166 ++---------------- 4 files changed, 43 insertions(+), 170 deletions(-) diff --git a/runtime/config-deployer-service/ballerina/ConfigGenreatorClient.bal b/runtime/config-deployer-service/ballerina/ConfigGenreatorClient.bal index d56bd2ac3..108e05829 100644 --- a/runtime/config-deployer-service/ballerina/ConfigGenreatorClient.bal +++ b/runtime/config-deployer-service/ballerina/ConfigGenreatorClient.bal @@ -50,7 +50,14 @@ public class ConfigGeneratorClient { record {|byte[] fileContent; string fileName; anydata...;|} definition = definitionBody.definition; fileName = definition.fileName; } - apiFromDefinition = check runtimeUtil:RuntimeAPICommonUtil_getGRPCAPIFromProtoDefinition(check validateAndRetrieveDefinitionResult.getProtoContent(), fileName); + do { + apiFromDefinition = check runtimeUtil:RuntimeAPICommonUtil_getGRPCAPIFromProtoDefinition(check validateAndRetrieveDefinitionResult.getProtoContent(), fileName); + } + on fail var e { + if e is error { + return e909022("Error occurred while validating the .proto definition", ()); + } + } } else { apiFromDefinition = check runtimeUtil:RuntimeAPICommonUtil_getAPIFromDefinition(validateAndRetrieveDefinitionResult.getContent(), apiType); } @@ -72,17 +79,21 @@ public class ConfigGeneratorClient { BadRequestError badRequest = {body: {code: 90091, message: "Invalid API Definition", 'error: errorItems}}; return badRequest; } - } else if validateAndRetrieveDefinitionResult is runtimeapi:APIManagementException { + } + else if validateAndRetrieveDefinitionResult is runtimeapi:APIManagementException { return e909022("Error occured while validating the definition", validateAndRetrieveDefinitionResult.cause()); } else { return e909022("Error occured while validating the definition", ()); } - } on fail var e { - if e is commons:APKError { + } +on fail var e { + if e + is commons:APKError { return e; } return e909022("Internal error occured while creating APK conf", e); } + } private isolated function prepareDefinitionBodyFromRequest(http:Request request) returns DefinitionBody|error { DefinitionBody definitionBody = {}; @@ -102,6 +113,7 @@ public class ConfigGeneratorClient { } return definitionBody; } + private isolated function validateAndRetrieveDefinition(string 'type, string? url, byte[]? content, string? fileName) returns runtimeapi:APIDefinitionValidationResponse|runtimeapi:APIManagementException|error|commons:APKError { runtimeapi:APIDefinitionValidationResponse|runtimeapi:APIManagementException|error validationResponse; boolean typeAvailable = 'type.length() > 0; @@ -122,6 +134,7 @@ public class ConfigGeneratorClient { } return validationResponse; } + private isolated function retrieveDefinitionFromUrl(string url) returns string|error { string domain = getDomain(url); string path = getPath(url); @@ -136,6 +149,7 @@ public class ConfigGeneratorClient { } return e909044(); } + public isolated function getGeneratedK8sResources(http:Request request, commons:Organization organization) returns http:Response|BadRequestError|InternalServerErrorError|commons:APKError { GenerateK8sResourcesBody body = {}; do { @@ -167,6 +181,7 @@ public class ConfigGeneratorClient { return e909052(e); } } + private isolated function zipAPIArtifact(string apiId, model:APIArtifact apiArtifact) returns [string, string]|error { string zipDir = check file:createTempDir(uuid:createType1AsString()); model:API? k8sAPI = apiArtifact.api; @@ -245,6 +260,7 @@ public class ConfigGeneratorClient { } return e909022("Error while converting json to yaml", convertedYaml); } + private isolated function storeFile(string jsonString, string fileName, string? directroy = ()) returns error? { string fullPath = directroy ?: ""; fullPath = fullPath + file:pathSeparator + fileName + ".yaml"; diff --git a/runtime/config-deployer-service/ballerina/modules/org.wso2.apk.config/RuntimeAPICommonUtil.bal b/runtime/config-deployer-service/ballerina/modules/org.wso2.apk.config/RuntimeAPICommonUtil.bal index 321953223..5de894a46 100644 --- a/runtime/config-deployer-service/ballerina/modules/org.wso2.apk.config/RuntimeAPICommonUtil.bal +++ b/runtime/config-deployer-service/ballerina/modules/org.wso2.apk.config/RuntimeAPICommonUtil.bal @@ -167,10 +167,15 @@ public isolated function RuntimeAPICommonUtil_getAPIFromDefinition(string arg0, # + arg0 - The `byte[]` value required to map with the Java method parameter. # + arg1 - The `string` value required to map with the Java method parameter. # + return - The `orgwso2apkconfigmodel:API` value returning from the Java mapping. -public isolated function RuntimeAPICommonUtil_getGRPCAPIFromProtoDefinition(byte[] arg0, string arg1) returns orgwso2apkconfigmodel:API|error { - handle externalObj = org_wso2_apk_config_RuntimeAPICommonUtil_getGRPCAPIFromProtoDefinition(check jarrays:toHandle(arg0, "byte"), java:fromString(arg1)); - orgwso2apkconfigmodel:API newObj = new (externalObj); - return newObj; +public isolated function RuntimeAPICommonUtil_getGRPCAPIFromProtoDefinition(byte[] arg0, string arg1) returns orgwso2apkconfigmodel:API|orgwso2apkconfigapi:APIManagementException|error { + handle|error externalObj = org_wso2_apk_config_RuntimeAPICommonUtil_getGRPCAPIFromProtoDefinition(check jarrays:toHandle(arg0, "byte"), java:fromString(arg1)); + if (externalObj is error) { + orgwso2apkconfigapi:APIManagementException e = error orgwso2apkconfigapi:APIManagementException(orgwso2apkconfigapi:APIMANAGEMENTEXCEPTION, externalObj, message = externalObj.message()); + return e; + } else { + orgwso2apkconfigmodel:API newObj = new (externalObj); + return newObj; + } } # The function that maps to the `validateOpenAPIDefinition` method of `org.wso2.apk.config.RuntimeAPICommonUtil`. @@ -228,7 +233,7 @@ isolated function org_wso2_apk_config_RuntimeAPICommonUtil_getClass(handle recei paramTypes: [] } external; -isolated function org_wso2_apk_config_RuntimeAPICommonUtil_getGRPCAPIFromProtoDefinition(handle arg0, handle arg1) returns handle = @java:Method { +isolated function org_wso2_apk_config_RuntimeAPICommonUtil_getGRPCAPIFromProtoDefinition(handle arg0, handle arg1) returns handle|error = @java:Method { name: "getGRPCAPIFromProtoDefinition", 'class: "org.wso2.apk.config.RuntimeAPICommonUtil", paramTypes: ["[B", "java.lang.String"] diff --git a/runtime/config-deployer-service/java/src/main/java/org/wso2/apk/config/RuntimeAPICommonUtil.java b/runtime/config-deployer-service/java/src/main/java/org/wso2/apk/config/RuntimeAPICommonUtil.java index 463064ecc..b03e0f3a4 100644 --- a/runtime/config-deployer-service/java/src/main/java/org/wso2/apk/config/RuntimeAPICommonUtil.java +++ b/runtime/config-deployer-service/java/src/main/java/org/wso2/apk/config/RuntimeAPICommonUtil.java @@ -80,13 +80,9 @@ public static APIDefinitionValidationResponse validateOpenAPIDefinition(String t return validationResponse; } - public static API getGRPCAPIFromProtoDefinition(byte[] definition, String fileName) { + public static API getGRPCAPIFromProtoDefinition(byte[] definition, String fileName) throws APIManagementException { ProtoParser protoParser = new ProtoParser(); - try { - return protoParser.getAPIFromProtoFile(definition, fileName); - } catch (APIManagementException e) { - throw new RuntimeException(e); - } + return protoParser.getAPIFromProtoFile(definition, fileName); } public static Set generateUriTemplatesFromAPIDefinition(String apiType, String content) diff --git a/runtime/config-deployer-service/java/src/main/java/org/wso2/apk/config/definitions/ProtoParser.java b/runtime/config-deployer-service/java/src/main/java/org/wso2/apk/config/definitions/ProtoParser.java index 15745ca3e..1e846bc96 100644 --- a/runtime/config-deployer-service/java/src/main/java/org/wso2/apk/config/definitions/ProtoParser.java +++ b/runtime/config-deployer-service/java/src/main/java/org/wso2/apk/config/definitions/ProtoParser.java @@ -9,7 +9,6 @@ import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; -import com.google.protobuf.*; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.wso2.apk.config.api.*; @@ -31,28 +30,6 @@ public class ProtoParser extends APIDefinition { public ProtoParser() { } - private static Descriptors.FileDescriptor resolveWellKnownType(String descriptorName) - throws ClassNotFoundException, NoSuchMethodException, IllegalAccessException, InvocationTargetException { - // Extract the proto file base name (e.g., "timestamp.proto" -> "Timestamp") - String baseName = descriptorName.substring(descriptorName.lastIndexOf('/') + 1, - descriptorName.lastIndexOf('.')); - // Convert to CamelCase (e.g., "timestamp" -> "Timestamp") - String className = baseName.substring(0, 1).toUpperCase() + baseName.substring(1); - // Find the corresponding class in the com.google.protobuf package - Class clazz = Class.forName("com.google.protobuf." + className); - // Use reflection to get the descriptor - Method getDescriptorMethod = clazz.getMethod("getDescriptor"); - Descriptors.Descriptor descriptor = (Descriptors.Descriptor) getDescriptorMethod.invoke(null); - return descriptor.getFile(); - } - - private static void populateProtoMap(Map protoMap, - DescriptorProtos.FileDescriptorSet fileDescriptorSet) { - for (DescriptorProtos.FileDescriptorProto fileDescriptorProto : fileDescriptorSet.getFileList()) { - protoMap.put(fileDescriptorProto.getName(), fileDescriptorProto); - } - } - @Override public Set getURITemplates(String resourceConfigsJSON) throws APIManagementException { // TODO Auto-generated method stub @@ -176,7 +153,13 @@ private List processProtoFile(byte[] definition, ProtoFile protoFil String packageString = getPackageString(content); List uriTemplates = new ArrayList<>(); StringBuilder apiName = new StringBuilder().append(protoFile.getApiName()); + if (packageString == null && protoFile.getPackageName() != null) { + throw new APIManagementException("Package string has not been defined in proto file"); + } String packageName = getPackageName(packageString); + if (packageName == null) { + packageName = packageString; + } List services = new ArrayList<>(); protoFile.setVersion(getVersion(packageString)); protoFile.setBasePath(getBasePath(packageString)); @@ -240,121 +223,6 @@ private byte[] readProtoFileBytesFromZip(ZipInputStream zis) throws IOException return byteArrayOutputStream.toByteArray(); } - ProtoFile getProtoFileFromDefinition(byte[] fileContent, String fileName) { - Map protoMap = new HashMap<>(); - Map descriptorMap = new HashMap<>(); - ArrayList services = new ArrayList<>(); - String packageName = ""; - ProtoFile tempProtoFile = new ProtoFile(); - Map wellKnownTypesMap = new HashMap<>(); - try { - DescriptorProtos.FileDescriptorSet fileDescriptorSet = DescriptorProtos.FileDescriptorSet.parseFrom( - fileContent); - - populateProtoMap(protoMap, fileDescriptorSet); - - for (DescriptorProtos.FileDescriptorProto fileDescriptorProto : fileDescriptorSet.getFileList()) { - packageName = processFileDescriptor(fileName, descriptorMap, protoMap, services, wellKnownTypesMap, - fileDescriptorProto); - } - - tempProtoFile.setServices(convertServiceDescriptorsToServices(services, packageName)); - String[] info = packageName.split("\\."); - if (info.length < 3) { - throw new APIManagementException( - "Invalid package name: specify in the format of basepath.version.packageName"); - } - tempProtoFile.setVersion(info[info.length - 2]); - tempProtoFile.setPackageName(info[info.length - 1]); - StringBuilder basePath = new StringBuilder("/").append(info[0]); - for (int i = 1; i < info.length - 2; i++) { - basePath.append(".").append(info[i]); - } - tempProtoFile.setBasePath(basePath.toString()); - return tempProtoFile; - } catch (Exception e) { - e.printStackTrace(); - log.error("Proto definition validation failed for " + fileName + ": " + e.getMessage()); - return null; - } - } - - /** - * @param fileName - The name of the .desc file provided as input for - * the config generator - * @param descriptorMap - * @param protoMap - * @param services - * @param wellKnownTypesMap - * @param fileDescriptorProto - * @return - * @throws Descriptors.DescriptorValidationException - */ - private String processFileDescriptor(String fileName, Map descriptorMap, - Map protoMap, - ArrayList services, - Map wellKnownTypesMap, - DescriptorProtos.FileDescriptorProto fileDescriptorProto) throws Descriptors.DescriptorValidationException { - - String packageName = fileDescriptorProto.getPackage(); - - // Process and resolve dependencies for a given file descriptor - Descriptors.FileDescriptor[] dependencies = fileDescriptorProto.getDependencyList().stream() - .map(descriptorMap::get).toArray(Descriptors.FileDescriptor[]::new); - - // Build the file descriptor based on the proto and its dependencies - Descriptors.FileDescriptor fileDescriptor = Descriptors.FileDescriptor.buildFrom(fileDescriptorProto, - dependencies); - services.addAll(fileDescriptor.getServices()); - descriptorMap.put(fileDescriptor.getName(), fileDescriptor); - return packageName; - } - - private Descriptors.FileDescriptor resolveDependency(Map descriptorMap, - Map protoMap, - Map wellKnownTypesMap, String descriptorName) { - Descriptors.FileDescriptor dependency = descriptorMap.get(descriptorName); - // Dependency has not been resolved yet - if (dependency == null) { - try { - // if the dependency is a well known type - if (descriptorName.startsWith("com.google.protobuf.")) { - dependency = resolveWellKnownType(descriptorName); - wellKnownTypesMap.put(descriptorName, dependency); - } else { - // if the dependency is on another file that was imported, we resolve it and add - // it to the - // descriptor map - dependency = buildAndCacheDescriptor(descriptorName, protoMap, descriptorMap, wellKnownTypesMap); - } - } catch (Exception e) { - System.err.println("Error loading well-known type: " + descriptorName + " - " + e.getMessage()); - } - } - if (dependency == null) { - System.err.println("Missing dependency for " + descriptorName); - } - return dependency; - } - - private Descriptors.FileDescriptor buildAndCacheDescriptor(String descriptorName, - Map protoMap, - Map descriptorMap, - Map wellKnownTypesMap) { - // this scenario is when you have an import in your proto file but that file - // hasnt been built yet - // in that scenario, it needs to have its dependencies resolved as well - DescriptorProtos.FileDescriptorProto dependencyProto = protoMap.get(descriptorName); - if (dependencyProto != null) { - // Descriptors.FileDescriptor dependency = resolveDependency(descriptorMap, - // protoMap, wellKnownTypesMap, - // descriptorName); - // descriptorMap.put(dependency.getName(), dependency); - // return dependency; - } - return null; - } - boolean validateProtoContent(byte[] fileContent, String fileName) { try { // ProtoFile protoFile = getProtoFileFromDefinition(fileContent, fileName); @@ -365,20 +233,6 @@ boolean validateProtoContent(byte[] fileContent, String fileName) { } } - public ArrayList convertServiceDescriptorsToServices( - ArrayList serviceDescriptors, String packageName) { - ArrayList services = new ArrayList<>(); - for (Descriptors.ServiceDescriptor serviceDescriptor : serviceDescriptors) { - List methodDescriptors = serviceDescriptor.getMethods(); - ArrayList methods = new ArrayList<>(); - for (Descriptors.MethodDescriptor methodDescriptor : methodDescriptors) { - methods.add(methodDescriptor.getName()); - } - services.add(new Service(serviceDescriptor.getName(), methods)); - } - return services; - } - public void validateGRPCAPIDefinition(byte[] inputByteArray, String fileName, APIDefinitionValidationResponse validationResponse, ArrayList errors) { try { @@ -391,8 +245,7 @@ public void validateGRPCAPIDefinition(byte[] inputByteArray, String fileName, boolean validated = validateProtoContent(protoFileContentBytes, fileName); if (!validated) { throw new APIManagementException( - "Invalid definition file provided. " - + "Please provide a valid .zip or .proto file."); + "Invalid definition file provided. " + "Please provide a valid .zip or .proto file."); } } } @@ -466,10 +319,13 @@ public String getServiceName(String serviceBlock) { } public String getPackageString(String content) { + // package string has the format "package something" Pattern packagePattern = Pattern.compile("package\\s+([\\w\\.]+);"); Matcher packageMatcher = packagePattern.matcher(content); if (packageMatcher.find()) { - return packageMatcher.group(1); + if (packageMatcher.group().length() > 1) { + return packageMatcher.group(1); + } } log.error("Package has not been defined in the proto file"); return null;