Skip to content

Commit

Permalink
Improve API definition content search code
Browse files Browse the repository at this point in the history
  • Loading branch information
@Avishka-Shamendra
Avishka-Shamendra committed Sep 10, 2024
1 parent f3bc13b commit bb03db3
Show file tree
Hide file tree
Showing 6 changed files with 211 additions and 21 deletions.
6 changes: 6 additions & 0 deletions ...c/main/java/org/wso2/carbon/apimgt/impl/indexing/indexer/GraphQLAPIDefinitionIndexer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import org.wso2.carbon.registry.core.RegistryConstants;
import org.wso2.carbon.registry.core.Resource;
import org.wso2.carbon.registry.core.exceptions.RegistryException;
import org.wso2.carbon.registry.core.utils.RegistryUtils;
import org.wso2.carbon.registry.indexing.AsyncIndexer;
import org.wso2.carbon.registry.indexing.IndexingManager;
import org.wso2.carbon.registry.indexing.solr.IndexDocument;
Expand Down Expand Up @@ -56,6 +57,11 @@ public IndexDocument getIndexedDocument(AsyncIndexer.File2Index fileData) throws
return null;
}

// Extract only required info (types) from graphql definition to index
String schemaString = RegistryUtils.decodeBytes(fileData.data);
String definitionString = IndexerUtil.getTypesFromGraphQLSchemaString(schemaString);
fileData.data = definitionString.getBytes();

IndexDocument indexDocument = super.getIndexedDocument(fileData);
IndexDocument newIndexDocument = indexDocument;

Expand Down
11 changes: 8 additions & 3 deletions ...main/java/org/wso2/carbon/apimgt/impl/indexing/indexer/RESTAsyncAPIDefinitionIndexer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,9 @@
import org.wso2.carbon.registry.core.RegistryConstants;
import org.wso2.carbon.registry.core.Resource;
import org.wso2.carbon.registry.core.exceptions.RegistryException;
import org.wso2.carbon.registry.core.utils.RegistryUtils;
import org.wso2.carbon.registry.indexing.AsyncIndexer.File2Index;
import org.wso2.carbon.registry.indexing.IndexingManager;
import org.wso2.carbon.registry.indexing.indexer.JSONIndexer;
import org.wso2.carbon.registry.indexing.solr.IndexDocument;

import java.util.Arrays;
Expand All @@ -41,7 +41,7 @@
/**
* This is indexer introduced to index swagger,async api artifacts for unified content search.
*/
public class RESTAsyncAPIDefinitionIndexer extends JSONIndexer {
public class RESTAsyncAPIDefinitionIndexer extends PlainTextIndexer {
public static final Log log = LogFactory.getLog(RESTAsyncAPIDefinitionIndexer.class);

@Override
Expand All @@ -58,6 +58,12 @@ public IndexDocument getIndexedDocument(File2Index fileData) throws SolrExceptio
return null;
}

// Filter out only values from the swagger, async json files for indexing
String jsonAsString = RegistryUtils.decodeBytes(fileData.data);
String valuesString = IndexerUtil.getValuesFromJsonString(jsonAsString);
fileData.data = valuesString.getBytes();
fileData.mediaType = "application/json";

IndexDocument indexDocument = super.getIndexedDocument(fileData);
IndexDocument newIndexDocument = indexDocument;

Expand Down Expand Up @@ -89,5 +95,4 @@ public IndexDocument getIndexedDocument(File2Index fileData) throws SolrExceptio
return newIndexDocument;
}


}
6 changes: 5 additions & 1 deletion .../src/main/java/org/wso2/carbon/apimgt/impl/indexing/indexer/SOAPAPIDefinitionIndexer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@
/**
* This indexer is introduced to index .wsdl files for unified content search.
*/
public class SOAPAPIDefinitionIndexer extends XMLIndexer {
public class SOAPAPIDefinitionIndexer extends PlainTextIndexer {

public static final Log log = LogFactory.getLog(SOAPAPIDefinitionIndexer.class);

Expand All @@ -58,6 +58,10 @@ public IndexDocument getIndexedDocument(AsyncIndexer.File2Index fileData) throws
return null;
}

// Extract only required data before indexing
String contentString = IndexerUtil.getContentFromXMLData(fileData.data);
fileData.data = contentString.getBytes();

IndexDocument indexDocument = super.getIndexedDocument(fileData);
IndexDocument newIndexDocument = indexDocument;

Expand Down
122 changes: 122 additions & 0 deletions ....wso2.carbon.apimgt.impl/src/main/java/org/wso2/carbon/apimgt/impl/utils/IndexerUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,15 +20,28 @@

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.json.JSONArray;
import org.json.JSONObject;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.NamedNodeMap;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.wso2.carbon.apimgt.api.APIManagementException;
import org.wso2.carbon.apimgt.api.model.graphql.queryanalysis.GraphqlSchemaType;
import org.wso2.carbon.apimgt.impl.APIConstants;
import org.wso2.carbon.apimgt.impl.definitions.GraphQLSchemaDefinition;
import org.wso2.carbon.governance.api.generic.GenericArtifactManager;
import org.wso2.carbon.governance.api.generic.dataobjects.GenericArtifact;
import org.wso2.carbon.registry.core.Registry;
import org.wso2.carbon.registry.core.Resource;
import org.wso2.carbon.registry.core.exceptions.RegistryException;

import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import java.io.ByteArrayInputStream;
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import java.util.Map;

Expand Down Expand Up @@ -65,4 +78,113 @@ public static void fetchRequiredDetailsFromAssociatedAPI(Registry registry, Reso
log.warn("API does not exist at " + apiPath);
}
}


/**
* This method can be used to filter out the json values, excluding keys from the json files
*
* @param jsonAsString JSON string
* @return values string
*/
public static String getValuesFromJsonString(String jsonAsString) {
JSONObject jsonObject = new JSONObject(jsonAsString);
StringBuilder values = new StringBuilder();
extractJsonValues(jsonObject, values);
return values.toString().trim();
}

/**
* Extract json values as a string from JSON object recursively
*
* @param json json object
* @param values StringBuilder values
*/
private static void extractJsonValues(Object json, StringBuilder values) {
if (json instanceof JSONObject) {
JSONObject jsonObject = (JSONObject) json;
Iterator<String> keys = jsonObject.keys();
while (keys.hasNext()) {
String key = keys.next();
extractJsonValues(jsonObject.get(key), values);
}
} else if (json instanceof JSONArray) {
JSONArray jsonArray = (JSONArray) json;
for (int i = 0; i < jsonArray.length(); i++) {
extractJsonValues(jsonArray.get(i), values);
}
} else {
values.append(json.toString()).append(" ");
}
}

/**
* This method is used to extract types from graphql schema
*
* @param schemaString Schema String
* @return definition string
*/
public static String getTypesFromGraphQLSchemaString(String schemaString) {
List<GraphqlSchemaType> types = new GraphQLSchemaDefinition()
.extractGraphQLTypeList(schemaString);
StringBuilder definitionString = new StringBuilder();

for (GraphqlSchemaType type : types) {
definitionString.append(type.getType()).append(" ");
definitionString.append(String.join(" ", type.getFieldList()));
}
return definitionString.toString();
}

/**
* This method returns string content extracted from XML data
*
* @param xmlData xml data bytes
* @return string content
*/
public static String getContentFromXMLData(byte[] xmlData) {
try {
DocumentBuilderFactory factory = APIUtil.getSecuredDocumentBuilder();
DocumentBuilder builder = factory.newDocumentBuilder();
Document doc = builder.parse(new ByteArrayInputStream(xmlData));
doc.getDocumentElement().normalize();
return extractTextAndAttributesFromElement(doc.getDocumentElement());
} catch (Exception e) {
log.error("Error while parsing XML data", e);
return "";
}
}

/**
* This method is used to recursively extract text and attributes as a string
*
* @param element Doc element
* @return text and attributes string
*/

private static String extractTextAndAttributesFromElement(Element element) {
StringBuilder text = new StringBuilder();

// Extract attributes from the element
NamedNodeMap attributes = element.getAttributes();
for (int i = 0; i < attributes.getLength(); i++) {
Node attr = attributes.item(i);
text.append(attr.getNodeName()).append("=").append(attr.getNodeValue()).append(" ");
}

// Extract text from child nodes
NodeList nodeList = element.getChildNodes();
for (int i = 0; i < nodeList.getLength(); i++) {
Node node = nodeList.item(i);
if (node.getNodeType() == Node.ELEMENT_NODE) {
// Recursively extract text and attributes
text.append(extractTextAndAttributesFromElement((Element) node)).append(" ");
} else if (node.getNodeType() == Node.TEXT_NODE) {
// Append text directly
text.append(node.getTextContent().trim()).append(" ");
}
}
return text.toString().trim();
}

}

84 changes: 67 additions & 17 deletions ...persistence/src/main/java/org/wso2/carbon/apimgt/persistence/RegistryPersistenceImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1712,7 +1712,8 @@ public DevPortalContentSearchResult searchContentForDevPortal(Organization org,
.getRegistry(CarbonConstants.REGISTRY_SYSTEM_USERNAME, tenantId);
ContentBasedSearchService contentBasedSearchService = new ContentBasedSearchService();

SearchResultsBean resultsBean = contentBasedSearchService.searchByAttribute(attributes, systemUserRegistry);
SearchResultsBean resultsBean = contentBasedSearchService.searchByAttribute(attributes,
systemUserRegistry);
String errorMsg = resultsBean.getErrorMessage();
if (errorMsg != null) {
throw new APIPersistenceException("Error while searching " + errorMsg);
Expand Down Expand Up @@ -4112,7 +4113,8 @@ private void addAPIDefinitionSearchContent(String resourcePath, Registry registr
content.setApiType(APIDefSearchContent.ApiType.SOAP);
} else {
index = resourcePath.indexOf(APIConstants.API_OAS_DEFINITION_RESOURCE_NAME);
content.setApiType(APIDefSearchContent.ApiType.REST);
// swagger.json is included in all types of APIs, hence we need to properly set the API type
setAPITypeForSwagger(resourcePath, index, content, registry);
}

String apiPath = resourcePath.substring(0, index) + APIConstants.API_KEY;
Expand All @@ -4123,26 +4125,74 @@ private void addAPIDefinitionSearchContent(String resourcePath, Registry registr
String defResourceName = defResource.getId().substring(defResource.getId().lastIndexOf('/') + 1);
DevPortalAPI devAPI;

/* Ignore internal swagger.json content search results of non REST APIs
as most of the data is duplicated in WSDL, GraphQL, AsyncAPI def. */
boolean ignoreDuplicateSwaggerContent =
(!APIDefSearchContent.ApiType.REST.toString().equals(content.getApiType())
&& defResourceName.contains("swagger"));

if (apiArtifactId != null) {
GenericArtifact apiArtifact = apiArtifactManager.getGenericArtifact(apiArtifactId);
devAPI = RegistryPersistenceUtil.getDevPortalAPIForSearch(apiArtifact);
content.setId(defResourceId);
content.setName(defResourceName);
content.setApiUUID(devAPI.getId());
content.setApiName(devAPI.getApiName());
content.setApiContext(devAPI.getContext());
content.setApiProvider(devAPI.getProviderName());
content.setApiVersion(devAPI.getVersion());
if (apiArtifact.getAttribute(APIConstants.API_OVERVIEW_TYPE)
.equals(APIConstants.AuditLogConstants.API_PRODUCT)) {
content.setAssociatedType(APIConstants.API_PRODUCT);
} else {
content.setAssociatedType(APIConstants.API);
if (!ignoreDuplicateSwaggerContent) {
GenericArtifact apiArtifact = apiArtifactManager.getGenericArtifact(apiArtifactId);
devAPI = RegistryPersistenceUtil.getDevPortalAPIForSearch(apiArtifact);
content.setId(defResourceId);
content.setName(defResourceName);
content.setApiUUID(devAPI.getId());
content.setApiName(devAPI.getApiName());
content.setApiContext(devAPI.getContext());
content.setApiProvider(devAPI.getProviderName());
content.setApiVersion(devAPI.getVersion());
if (apiArtifact.getAttribute(APIConstants.API_OVERVIEW_TYPE)
.equals(APIConstants.AuditLogConstants.API_PRODUCT)) {
content.setAssociatedType(APIConstants.API_PRODUCT);
} else {
content.setAssociatedType(APIConstants.API);
}
contentData.add(content);
}
contentData.add(content);

} else {
throw new GovernanceException("artifact id is null of " + apiPath);
}
}

/**
* This method is used to set the correct API Type for swagger.json as all API types have a swagger.json
* file in registry
*
* @param resourcePath registry resourcePath
* @param index int index
* @param content APIDefSearchContent obj.
* @param registry registry obj.
* @throws RegistryException on failure
*/
private void setAPITypeForSwagger(String resourcePath, int index,
APIDefSearchContent content, Registry registry) throws RegistryException {
String directoryPath = resourcePath.substring(0, index);
Resource directoryResource = registry.get(directoryPath);

if (directoryResource instanceof Collection) {
Collection collection = (Collection) directoryResource;
int childrenCount = collection.getChildCount();
String[] childPaths = collection.getChildren();
if (childrenCount > 2) {
for (String childPath : childPaths) {
if (childPath.contains(APIConstants.API_ASYNC_API_DEFINITION_RESOURCE_NAME)) {
content.setApiType(APIDefSearchContent.ApiType.ASYNC);
break;
} else if (childPath.contains(APIConstants.GRAPHQL_SCHEMA_FILE_EXTENSION)) {
content.setApiType(APIDefSearchContent.ApiType.GRAPHQL);
break;
} else if (childPath.contains(APIConstants.WSDL_FILE_EXTENSION)) {
content.setApiType(APIDefSearchContent.ApiType.SOAP);
break;
}
}
}
if (content.getApiType() == null) {
content.setApiType(APIDefSearchContent.ApiType.REST);
}
}

}
}
3 changes: 3 additions & 0 deletions ...persistence/src/main/java/org/wso2/carbon/apimgt/persistence/dto/APIDefSearchContent.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,9 @@ public void setAssociatedType(String associatedType) {
}

public String getApiType() {
if (apiType == null) {
return null;
}
return apiType.toString();
}

Expand Down

0 comments on commit bb03db3

Please sign in to comment.