Skip to content

Commit

Permalink
Update parser for proto files
Browse files Browse the repository at this point in the history
  • Loading branch information
@sgayangi
sgayangi committed Sep 18, 2024
1 parent 31b06b4 commit 4d707a8
Show file tree
Hide file tree
Showing 4 changed files with 43 additions and 170 deletions.
24 changes: 20 additions & 4 deletions runtime/config-deployer-service/ballerina/ConfigGenreatorClient.bal
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,14 @@ public class ConfigGeneratorClient {
record {|byte[] fileContent; string fileName; anydata...;|} definition = <record {|byte[] fileContent; string fileName; anydata...;|}>definitionBody.definition;
fileName = <string>definition.fileName;
}
apiFromDefinition = check runtimeUtil:RuntimeAPICommonUtil_getGRPCAPIFromProtoDefinition(check validateAndRetrieveDefinitionResult.getProtoContent(), fileName);
do {
apiFromDefinition = check runtimeUtil:RuntimeAPICommonUtil_getGRPCAPIFromProtoDefinition(check validateAndRetrieveDefinitionResult.getProtoContent(), fileName);
}
on fail var e {
if e is error {
return e909022("Error occurred while validating the .proto definition", ());
}
}
} else {
apiFromDefinition = check runtimeUtil:RuntimeAPICommonUtil_getAPIFromDefinition(validateAndRetrieveDefinitionResult.getContent(), apiType);
}
Expand All @@ -72,17 +79,21 @@ public class ConfigGeneratorClient {
BadRequestError badRequest = {body: {code: 90091, message: "Invalid API Definition", 'error: errorItems}};
return badRequest;
}
} else if validateAndRetrieveDefinitionResult is runtimeapi:APIManagementException {
}
else if validateAndRetrieveDefinitionResult is runtimeapi:APIManagementException {
return e909022("Error occured while validating the definition", validateAndRetrieveDefinitionResult.cause());
} else {
return e909022("Error occured while validating the definition", ());
}
} on fail var e {
if e is commons:APKError {
}
on fail var e {
if e
is commons:APKError {
return e;
}
return e909022("Internal error occured while creating APK conf", e);
}

}
private isolated function prepareDefinitionBodyFromRequest(http:Request request) returns DefinitionBody|error {
DefinitionBody definitionBody = {};
Expand All @@ -102,6 +113,7 @@ public class ConfigGeneratorClient {
}
return definitionBody;
}

private isolated function validateAndRetrieveDefinition(string 'type, string? url, byte[]? content, string? fileName) returns runtimeapi:APIDefinitionValidationResponse|runtimeapi:APIManagementException|error|commons:APKError {
runtimeapi:APIDefinitionValidationResponse|runtimeapi:APIManagementException|error validationResponse;
boolean typeAvailable = 'type.length() > 0;
Expand All @@ -122,6 +134,7 @@ public class ConfigGeneratorClient {
}
return validationResponse;
}

private isolated function retrieveDefinitionFromUrl(string url) returns string|error {
string domain = getDomain(url);
string path = getPath(url);
Expand All @@ -136,6 +149,7 @@ public class ConfigGeneratorClient {
}
return e909044();
}

public isolated function getGeneratedK8sResources(http:Request request, commons:Organization organization) returns http:Response|BadRequestError|InternalServerErrorError|commons:APKError {
GenerateK8sResourcesBody body = {};
do {
Expand Down Expand Up @@ -167,6 +181,7 @@ public class ConfigGeneratorClient {
return e909052(e);
}
}

private isolated function zipAPIArtifact(string apiId, model:APIArtifact apiArtifact) returns [string, string]|error {
string zipDir = check file:createTempDir(uuid:createType1AsString());
model:API? k8sAPI = apiArtifact.api;
Expand Down Expand Up @@ -245,6 +260,7 @@ public class ConfigGeneratorClient {
}
return e909022("Error while converting json to yaml", convertedYaml);
}

private isolated function storeFile(string jsonString, string fileName, string? directroy = ()) returns error? {
string fullPath = directroy ?: "";
fullPath = fullPath + file:pathSeparator + fileName + ".yaml";
Expand Down
15 changes: 10 additions & 5 deletions ...me/config-deployer-service/ballerina/modules/org.wso2.apk.config/RuntimeAPICommonUtil.bal
View file
Original file line number Diff line number Diff line change
Expand Up @@ -167,10 +167,15 @@ public isolated function RuntimeAPICommonUtil_getAPIFromDefinition(string arg0,
# + arg0 - The `byte[]` value required to map with the Java method parameter.
# + arg1 - The `string` value required to map with the Java method parameter.
# + return - The `orgwso2apkconfigmodel:API` value returning from the Java mapping.
public isolated function RuntimeAPICommonUtil_getGRPCAPIFromProtoDefinition(byte[] arg0, string arg1) returns orgwso2apkconfigmodel:API|error {
handle externalObj = org_wso2_apk_config_RuntimeAPICommonUtil_getGRPCAPIFromProtoDefinition(check jarrays:toHandle(arg0, "byte"), java:fromString(arg1));
orgwso2apkconfigmodel:API newObj = new (externalObj);
return newObj;
public isolated function RuntimeAPICommonUtil_getGRPCAPIFromProtoDefinition(byte[] arg0, string arg1) returns orgwso2apkconfigmodel:API|orgwso2apkconfigapi:APIManagementException|error {
handle|error externalObj = org_wso2_apk_config_RuntimeAPICommonUtil_getGRPCAPIFromProtoDefinition(check jarrays:toHandle(arg0, "byte"), java:fromString(arg1));
if (externalObj is error) {
orgwso2apkconfigapi:APIManagementException e = error orgwso2apkconfigapi:APIManagementException(orgwso2apkconfigapi:APIMANAGEMENTEXCEPTION, externalObj, message = externalObj.message());
return e;
} else {
orgwso2apkconfigmodel:API newObj = new (externalObj);
return newObj;
}
}

# The function that maps to the `validateOpenAPIDefinition` method of `org.wso2.apk.config.RuntimeAPICommonUtil`.
Expand Down Expand Up @@ -228,7 +233,7 @@ isolated function org_wso2_apk_config_RuntimeAPICommonUtil_getClass(handle recei
paramTypes: []
} external;

isolated function org_wso2_apk_config_RuntimeAPICommonUtil_getGRPCAPIFromProtoDefinition(handle arg0, handle arg1) returns handle = @java:Method {
isolated function org_wso2_apk_config_RuntimeAPICommonUtil_getGRPCAPIFromProtoDefinition(handle arg0, handle arg1) returns handle|error = @java:Method {
name: "getGRPCAPIFromProtoDefinition",
'class: "org.wso2.apk.config.RuntimeAPICommonUtil",
paramTypes: ["[B", "java.lang.String"]
Expand Down
8 changes: 2 additions & 6 deletions .../config-deployer-service/java/src/main/java/org/wso2/apk/config/RuntimeAPICommonUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,13 +80,9 @@ public static APIDefinitionValidationResponse validateOpenAPIDefinition(String t
return validationResponse;
}

public static API getGRPCAPIFromProtoDefinition(byte[] definition, String fileName) {
public static API getGRPCAPIFromProtoDefinition(byte[] definition, String fileName) throws APIManagementException {
ProtoParser protoParser = new ProtoParser();
try {
return protoParser.getAPIFromProtoFile(definition, fileName);
} catch (APIManagementException e) {
throw new RuntimeException(e);
}
return protoParser.getAPIFromProtoFile(definition, fileName);
}

public static Set<URITemplate> generateUriTemplatesFromAPIDefinition(String apiType, String content)
Expand Down
166 changes: 11 additions & 155 deletions ...nfig-deployer-service/java/src/main/java/org/wso2/apk/config/definitions/ProtoParser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;

import com.google.protobuf.*;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.wso2.apk.config.api.*;
Expand All @@ -31,28 +30,6 @@ public class ProtoParser extends APIDefinition {
public ProtoParser() {
}

private static Descriptors.FileDescriptor resolveWellKnownType(String descriptorName)
throws ClassNotFoundException, NoSuchMethodException, IllegalAccessException, InvocationTargetException {
// Extract the proto file base name (e.g., "timestamp.proto" -> "Timestamp")
String baseName = descriptorName.substring(descriptorName.lastIndexOf('/') + 1,
descriptorName.lastIndexOf('.'));
// Convert to CamelCase (e.g., "timestamp" -> "Timestamp")
String className = baseName.substring(0, 1).toUpperCase() + baseName.substring(1);
// Find the corresponding class in the com.google.protobuf package
Class<?> clazz = Class.forName("com.google.protobuf." + className);
// Use reflection to get the descriptor
Method getDescriptorMethod = clazz.getMethod("getDescriptor");
Descriptors.Descriptor descriptor = (Descriptors.Descriptor) getDescriptorMethod.invoke(null);
return descriptor.getFile();
}

private static void populateProtoMap(Map<String, DescriptorProtos.FileDescriptorProto> protoMap,
DescriptorProtos.FileDescriptorSet fileDescriptorSet) {
for (DescriptorProtos.FileDescriptorProto fileDescriptorProto : fileDescriptorSet.getFileList()) {
protoMap.put(fileDescriptorProto.getName(), fileDescriptorProto);
}
}

@Override
public Set<URITemplate> getURITemplates(String resourceConfigsJSON) throws APIManagementException {
// TODO Auto-generated method stub
Expand Down Expand Up @@ -176,7 +153,13 @@ private List<URITemplate> processProtoFile(byte[] definition, ProtoFile protoFil
String packageString = getPackageString(content);
List<URITemplate> uriTemplates = new ArrayList<>();
StringBuilder apiName = new StringBuilder().append(protoFile.getApiName());
if (packageString == null && protoFile.getPackageName() != null) {
throw new APIManagementException("Package string has not been defined in proto file");
}
String packageName = getPackageName(packageString);
if (packageName == null) {
packageName = packageString;
}
List<Service> services = new ArrayList<>();
protoFile.setVersion(getVersion(packageString));
protoFile.setBasePath(getBasePath(packageString));
Expand Down Expand Up @@ -240,121 +223,6 @@ private byte[] readProtoFileBytesFromZip(ZipInputStream zis) throws IOException
return byteArrayOutputStream.toByteArray();
}

ProtoFile getProtoFileFromDefinition(byte[] fileContent, String fileName) {
Map<String, DescriptorProtos.FileDescriptorProto> protoMap = new HashMap<>();
Map<String, Descriptors.FileDescriptor> descriptorMap = new HashMap<>();
ArrayList<Descriptors.ServiceDescriptor> services = new ArrayList<>();
String packageName = "";
ProtoFile tempProtoFile = new ProtoFile();
Map<String, Descriptors.FileDescriptor> wellKnownTypesMap = new HashMap<>();
try {
DescriptorProtos.FileDescriptorSet fileDescriptorSet = DescriptorProtos.FileDescriptorSet.parseFrom(
fileContent);

populateProtoMap(protoMap, fileDescriptorSet);

for (DescriptorProtos.FileDescriptorProto fileDescriptorProto : fileDescriptorSet.getFileList()) {
packageName = processFileDescriptor(fileName, descriptorMap, protoMap, services, wellKnownTypesMap,
fileDescriptorProto);
}

tempProtoFile.setServices(convertServiceDescriptorsToServices(services, packageName));
String[] info = packageName.split("\\.");
if (info.length < 3) {
throw new APIManagementException(
"Invalid package name: specify in the format of basepath.version.packageName");
}
tempProtoFile.setVersion(info[info.length - 2]);
tempProtoFile.setPackageName(info[info.length - 1]);
StringBuilder basePath = new StringBuilder("/").append(info[0]);
for (int i = 1; i < info.length - 2; i++) {
basePath.append(".").append(info[i]);
}
tempProtoFile.setBasePath(basePath.toString());
return tempProtoFile;
} catch (Exception e) {
e.printStackTrace();
log.error("Proto definition validation failed for " + fileName + ": " + e.getMessage());
return null;
}
}

/**
* @param fileName - The name of the .desc file provided as input for
* the config generator
* @param descriptorMap
* @param protoMap
* @param services
* @param wellKnownTypesMap
* @param fileDescriptorProto
* @return
* @throws Descriptors.DescriptorValidationException
*/
private String processFileDescriptor(String fileName, Map<String, Descriptors.FileDescriptor> descriptorMap,
Map<String, DescriptorProtos.FileDescriptorProto> protoMap,
ArrayList<Descriptors.ServiceDescriptor> services,
Map<String, Descriptors.FileDescriptor> wellKnownTypesMap,
DescriptorProtos.FileDescriptorProto fileDescriptorProto) throws Descriptors.DescriptorValidationException {

String packageName = fileDescriptorProto.getPackage();

// Process and resolve dependencies for a given file descriptor
Descriptors.FileDescriptor[] dependencies = fileDescriptorProto.getDependencyList().stream()
.map(descriptorMap::get).toArray(Descriptors.FileDescriptor[]::new);

// Build the file descriptor based on the proto and its dependencies
Descriptors.FileDescriptor fileDescriptor = Descriptors.FileDescriptor.buildFrom(fileDescriptorProto,
dependencies);
services.addAll(fileDescriptor.getServices());
descriptorMap.put(fileDescriptor.getName(), fileDescriptor);
return packageName;
}

private Descriptors.FileDescriptor resolveDependency(Map<String, Descriptors.FileDescriptor> descriptorMap,
Map<String, DescriptorProtos.FileDescriptorProto> protoMap,
Map<String, Descriptors.FileDescriptor> wellKnownTypesMap, String descriptorName) {
Descriptors.FileDescriptor dependency = descriptorMap.get(descriptorName);
// Dependency has not been resolved yet
if (dependency == null) {
try {
// if the dependency is a well known type
if (descriptorName.startsWith("com.google.protobuf.")) {
dependency = resolveWellKnownType(descriptorName);
wellKnownTypesMap.put(descriptorName, dependency);
} else {
// if the dependency is on another file that was imported, we resolve it and add
// it to the
// descriptor map
dependency = buildAndCacheDescriptor(descriptorName, protoMap, descriptorMap, wellKnownTypesMap);
}
} catch (Exception e) {
System.err.println("Error loading well-known type: " + descriptorName + " - " + e.getMessage());
}
}
if (dependency == null) {
System.err.println("Missing dependency for " + descriptorName);
}
return dependency;
}

private Descriptors.FileDescriptor buildAndCacheDescriptor(String descriptorName,
Map<String, DescriptorProtos.FileDescriptorProto> protoMap,
Map<String, Descriptors.FileDescriptor> descriptorMap,
Map<String, Descriptors.FileDescriptor> wellKnownTypesMap) {
// this scenario is when you have an import in your proto file but that file
// hasnt been built yet
// in that scenario, it needs to have its dependencies resolved as well
DescriptorProtos.FileDescriptorProto dependencyProto = protoMap.get(descriptorName);
if (dependencyProto != null) {
// Descriptors.FileDescriptor dependency = resolveDependency(descriptorMap,
// protoMap, wellKnownTypesMap,
// descriptorName);
// descriptorMap.put(dependency.getName(), dependency);
// return dependency;
}
return null;
}

boolean validateProtoContent(byte[] fileContent, String fileName) {
try {
// ProtoFile protoFile = getProtoFileFromDefinition(fileContent, fileName);
Expand All @@ -365,20 +233,6 @@ boolean validateProtoContent(byte[] fileContent, String fileName) {
}
}

public ArrayList<Service> convertServiceDescriptorsToServices(
ArrayList<Descriptors.ServiceDescriptor> serviceDescriptors, String packageName) {
ArrayList<Service> services = new ArrayList<>();
for (Descriptors.ServiceDescriptor serviceDescriptor : serviceDescriptors) {
List<Descriptors.MethodDescriptor> methodDescriptors = serviceDescriptor.getMethods();
ArrayList<String> methods = new ArrayList<>();
for (Descriptors.MethodDescriptor methodDescriptor : methodDescriptors) {
methods.add(methodDescriptor.getName());
}
services.add(new Service(serviceDescriptor.getName(), methods));
}
return services;
}

public void validateGRPCAPIDefinition(byte[] inputByteArray, String fileName,
APIDefinitionValidationResponse validationResponse, ArrayList<ErrorHandler> errors) {
try {
Expand All @@ -391,8 +245,7 @@ public void validateGRPCAPIDefinition(byte[] inputByteArray, String fileName,
boolean validated = validateProtoContent(protoFileContentBytes, fileName);
if (!validated) {
throw new APIManagementException(
"Invalid definition file provided. "
+ "Please provide a valid .zip or .proto file.");
"Invalid definition file provided. " + "Please provide a valid .zip or .proto file.");
}
}
}
Expand Down Expand Up @@ -466,10 +319,13 @@ public String getServiceName(String serviceBlock) {
}

public String getPackageString(String content) {
// package string has the format "package something"
Pattern packagePattern = Pattern.compile("package\\s+([\\w\\.]+);");
Matcher packageMatcher = packagePattern.matcher(content);
if (packageMatcher.find()) {
return packageMatcher.group(1);
if (packageMatcher.group().length() > 1) {
return packageMatcher.group(1);
}
}
log.error("Package has not been defined in the proto file");
return null;
Expand Down

0 comments on commit 4d707a8

Please sign in to comment.