Merge branch 'master' into Fix-incorrrect-Oauth-behavior-3

.github/workflows/maven.yml

@@ -99,7 +99,7 @@ jobs:
     needs: build
     runs-on: 'ubuntu-latest'
     steps:
-      - uses: actions/download-artifact@v2
+      - uses: actions/download-artifact@v3
         with:
           path: reports
       - name: Publish Test Report

modules/distribution/product/src/main/conf/log4j2.properties

@@ -1,6 +1,6 @@
 # list of all appenders
 #add entry "syslog" to use the syslog appender
-appenders=CARBON_CONSOLE, APIM_METRICS_APPENDER, CARBON_LOGFILE, AUDIT_LOGFILE, ATOMIKOS_LOGFILE, CARBON_TRACE_LOGFILE, ERROR_LOGFILE, OPEN_TRACING,SERVICE_APPENDER, TRACE_APPENDER, osgi, CORRELATION, BOTDATA_APPENDER, API_LOGFILE
+appenders=CARBON_CONSOLE, APIM_METRICS_APPENDER, CARBON_LOGFILE, AUDIT_LOGFILE, ATOMIKOS_LOGFILE, CARBON_TRACE_LOGFILE, ERROR_LOGFILE, OPEN_TRACING,SERVICE_APPENDER, TRACE_APPENDER, osgi, CORRELATION, BOTDATA_APPENDER, API_LOGFILE, APIM_GATEWAY_ACCESS_APPENDER
 #, syslog
 
 # CARBON_CONSOLE is set to be a ConsoleAppender using a PatternLayout.
@@ -196,11 +196,28 @@ appender.APIM_METRICS_APPENDER.policies.size.size=1000MB
 appender.APIM_METRICS_APPENDER.strategy.type = DefaultRolloverStrategy
 appender.APIM_METRICS_APPENDER.strategy.max = 10
 
+appender.APIM_GATEWAY_ACCESS_APPENDER.type = RollingFile
+appender.APIM_GATEWAY_ACCESS_APPENDER.name = APIM_GATEWAY_ACCESS_APPENDER
+appender.APIM_GATEWAY_ACCESS_APPENDER.fileName = ${sys:carbon.home}/repository/logs/http_access.log
+appender.APIM_GATEWAY_ACCESS_APPENDER.filePattern = ${sys:carbon.home}/repository/logs/http_access_%d{MM-dd-yyyy}-%i.log
+appender.APIM_GATEWAY_ACCESS_APPENDER.layout.type = PatternLayout
+appender.APIM_GATEWAY_ACCESS_APPENDER.layout.pattern = %msg%n
+appender.APIM_GATEWAY_ACCESS_APPENDER.policies.type = Policies
+appender.APIM_GATEWAY_ACCESS_APPENDER.policies.time.type = TimeBasedTriggeringPolicy
+appender.APIM_GATEWAY_ACCESS_APPENDER.policies.time.interval = 1
+appender.APIM_GATEWAY_ACCESS_APPENDER.policies.time.modulate = true
+appender.APIM_GATEWAY_ACCESS_APPENDER.policies.size.type = SizeBasedTriggeringPolicy
+appender.APIM_GATEWAY_ACCESS_APPENDER.policies.size.size=1000MB
+appender.APIM_GATEWAY_ACCESS_APPENDER.strategy.type = DefaultRolloverStrategy
+appender.APIM_GATEWAY_ACCESS_APPENDER.strategy.max = 10
+appender.APIM_GATEWAY_ACCESS_APPENDER.filter.threshold.type = ThresholdFilter
+appender.APIM_GATEWAY_ACCESS_APPENDER.filter.threshold.level = DEBUG
+
 appender.osgi.type = PaxOsgi
 appender.osgi.name = PaxOsgi
 appender.osgi.filter = *
 
-loggers = AUDIT_LOG, reporter, trace-messages, org-apache-coyote, com-hazelcast, Owasp-CsrfGuard, org-apache-axis2-wsdl-codegen-writer-PrettyPrinter, org-apache-axis2-clustering, org-apache-catalina, org-apache-tomcat, org-wso2-carbon-apacheds, org-apache-directory-server-ldap, org-apache-directory-server-core-event, com-atomikos, org-quartz, org-apache-jackrabbit-webdav, org-apache-juddi, org-apache-commons-digester-Digester, org-apache-jasper-compiler-TldLocationsCache, org-apache-qpid, org-apache-qpid-server-Main, qpid-message, qpid-message-broker-listening, org-apache-tiles, org-apache-commons-httpclient, org-apache-solr, me-prettyprint-cassandra-hector-TimingLogger, org-apache-axis-enterprise, org-apache-directory-shared-ldap, org-apache-directory-server-ldap-handlers, org-apache-directory-shared-ldap-entry-DefaultServerAttribute, org-apache-directory-server-core-DefaultDirectoryService, org-apache-directory-shared-ldap-ldif-LdifReader, org-apache-directory-server-ldap-LdapProtocolHandler, org-apache-directory-server-core, org-apache-directory-server-ldap-LdapSession, DataNucleus, Datastore, Datastore-Schema, JPOX-Datastore, JPOX-Plugin, JPOX-MetaData, JPOX-Query, JPOX-General, JPOX-Enhancer, org-apache-hadoop-hive, hive, ExecMapper, ExecReducer, net-sf-ehcache-config-ConfigurationFactory, axis2Deployment, equinox, tomcat2, StAXDialectDetector, trace, synapse, synapse_transport, axis2, axis2_transport, org-wso2-carbon, hunsicker, thrift-publisher, service_logger, trace_logger, org-wso2-carbon-apimgt-gateway-mediators-BotDetectionMediator, correlation, API_LOG
+loggers = AUDIT_LOG, reporter, trace-messages, org-apache-coyote, com-hazelcast, Owasp-CsrfGuard, org-apache-axis2-wsdl-codegen-writer-PrettyPrinter, org-apache-axis2-clustering, org-apache-catalina, org-apache-tomcat, org-wso2-carbon-apacheds, org-apache-directory-server-ldap, org-apache-directory-server-core-event, com-atomikos, org-quartz, org-apache-jackrabbit-webdav, org-apache-juddi, org-apache-commons-digester-Digester, org-apache-jasper-compiler-TldLocationsCache, org-apache-qpid, org-apache-qpid-server-Main, qpid-message, qpid-message-broker-listening, org-apache-tiles, org-apache-commons-httpclient, org-apache-solr, me-prettyprint-cassandra-hector-TimingLogger, org-apache-axis-enterprise, org-apache-directory-shared-ldap, org-apache-directory-server-ldap-handlers, org-apache-directory-shared-ldap-entry-DefaultServerAttribute, org-apache-directory-server-core-DefaultDirectoryService, org-apache-directory-shared-ldap-ldif-LdifReader, org-apache-directory-server-ldap-LdapProtocolHandler, org-apache-directory-server-core, org-apache-directory-server-ldap-LdapSession, DataNucleus, Datastore, Datastore-Schema, JPOX-Datastore, JPOX-Plugin, JPOX-MetaData, JPOX-Query, JPOX-General, JPOX-Enhancer, org-apache-hadoop-hive, hive, ExecMapper, ExecReducer, net-sf-ehcache-config-ConfigurationFactory, axis2Deployment, equinox, tomcat2, StAXDialectDetector, trace, synapse, synapse_transport, axis2, axis2_transport, org-wso2-carbon, hunsicker, thrift-publisher, service_logger, trace_logger, org-wso2-carbon-apimgt-gateway-mediators-BotDetectionMediator, correlation, API_LOG, GatewayAccessLogger
 
 logger.API_LOG.name = API_LOG
 logger.API_LOG.level = INFO
@@ -412,6 +429,11 @@ logger.trace.appenderRef.OPEN_TRACING.ref = OPEN_TRACING
 logger.synapse.name = org.apache.synapse
 logger.synapse.level = INFO
 
+logger.GatewayAccessLogger.name = org.apache.synapse.transport.http.access.logs
+logger.GatewayAccessLogger.level = DEBUG
+logger.GatewayAccessLogger.appenderRef.APIM_GATEWAY_ACCESS_APPENDER.ref = APIM_GATEWAY_ACCESS_APPENDER
+logger.GatewayAccessLogger.additivity = false
+
 logger.synapse_transport.name = org.apache.synapse.transport
 logger.synapse_transport.level = INFO
 

modules/distribution/product/src/main/resources/conf/default.json

@@ -175,6 +175,7 @@
   "synapse_properties.'synapse.main.xml.location'": "repository/resources/apim-synapse-config/main.xml",
   "synapse_properties.'synapse.fault.xml.location'": "repository/resources/apim-synapse-config/fault.xml",
   "synapse_properties.'synapse.registry.xml.location'": "repository/resources/apim-synapse-config/registry.xml",
+  "synapse_properties.'synapse.script.mediator.default.engine'": "rhinoJs",
   "passthru_http.'http.socket.timeout'": "180000",
   "passthru_http.worker_pool_size_core": "400",
   "passthru_http.worker_pool_size_max": "500",

modules/distribution/product/src/main/resources/conf/templates/repository/conf/registry.xml.j2

@@ -330,6 +330,9 @@
             <indexer class="org.wso2.carbon.governance.registry.extensions.indexers.RXTIndexer" mediaTypeRegEx="application/policy\+xml" profiles ="default,control-plane"/-->
             <indexer class="org.wso2.carbon.apimgt.impl.indexing.indexer.DocumentIndexer" mediaTypeRegEx="application/vnd.wso2-document\+xml" profiles ="default,control-plane"/>
             <indexer class="org.wso2.carbon.apimgt.impl.indexing.indexer.CustomAPIIndexer" mediaTypeRegEx="application/vnd.(.)+\+xml" profiles ="default,control-plane"/>
+            <indexer class="org.wso2.carbon.apimgt.impl.indexing.indexer.RESTAsyncAPIDefinitionIndexer" mediaTypeRegEx="application/json" profiles ="default,control-plane"/>
+            <indexer class="org.wso2.carbon.apimgt.impl.indexing.indexer.GraphQLAPIDefinitionIndexer" mediaTypeRegEx="text/plain(.)+charset=ISO-8859-1" profiles ="default,control-plane"/>
+            <indexer class="org.wso2.carbon.apimgt.impl.indexing.indexer.SOAPAPIDefinitionIndexer" mediaTypeRegEx="application/wsdl\+xml|application/octet-stream" profiles ="default,control-plane"/>
             <!--indexer class="org.wso2.carbon.registry.indexing.indexer.XMLIndexer" mediaTypeRegEx="application/(.)+\+xml"/>
             <indexer class="org.wso2.carbon.registry.indexing.indexer.PlainTextIndexer" mediaTypeRegEx="text/(.)+"/>
             <indexer class="org.wso2.carbon.registry.indexing.indexer.PlainTextIndexer" mediaTypeRegEx="application/x-javascript"/-->

modules/distribution/product/src/main/resources/conf/templates/repository/conf/synapse-handlers.xml.j2

@@ -1,6 +1,6 @@
 <handlers xmlns:svns="http://org.wso2.securevault/configuration">
-    <handler name="external_call_logger" class="org.wso2.carbon.apimgt.gateway.handlers.LogsHandler"/>
     <handler name="external_call_logger" class="org.wso2.carbon.apimgt.gateway.handlers.DefaultAPIHandler"/>
+    <handler name="external_call_logger" class="org.wso2.carbon.apimgt.gateway.handlers.LogsHandler"/>
     <handler name="open_tracing" class="org.wso2.carbon.apimgt.gateway.handlers.common.APIMgtLatencySynapseHandler"/>
 {% if apim.transport_headers is defined %}
 {% if apim.transport_headers.enable %}

modules/integration/tests-common/integration-test-utils/src/main/java/org/wso2/am/integration/test/utils/base/APIMIntegrationBaseTest.java

@@ -852,6 +852,7 @@ protected String createAPIRevisionAndDeployUsingRest(String apiId, RestAPIPublis
             throws ApiException, JSONException, APIManagerIntegrationTestException {
         int HTTP_RESPONSE_CODE_OK = Response.Status.OK.getStatusCode();
         int HTTP_RESPONSE_CODE_CREATED = Response.Status.CREATED.getStatusCode();
+        int HTTP_RESPONSE_CODE_BAD_REQUEST = Response.Status.BAD_REQUEST.getStatusCode();
         int HTTP_RESPONSE_CODE_INTERNAL_SERVER_ERROR = Response.Status.INTERNAL_SERVER_ERROR.getStatusCode();
         String revisionUUID = null;
         //Add the API Revision using the API publisher.
@@ -860,8 +861,9 @@ protected String createAPIRevisionAndDeployUsingRest(String apiId, RestAPIPublis
         apiRevisionRequest.setDescription("Test Revision 1");
 
         HttpResponse apiRevisionResponse = restAPIPublisher.addAPIRevision(apiRevisionRequest);
-
-        if (apiRevisionResponse.getResponseCode() != HTTP_RESPONSE_CODE_INTERNAL_SERVER_ERROR) {
+        int responseCode = apiRevisionResponse.getResponseCode();
+        if (responseCode != HTTP_RESPONSE_CODE_INTERNAL_SERVER_ERROR && responseCode
+                != HTTP_RESPONSE_CODE_BAD_REQUEST) {
             assertEquals(apiRevisionResponse.getResponseCode(), HTTP_RESPONSE_CODE_CREATED,
                     "Create API Response Code is invalid." + apiRevisionResponse.getData());
             JSONObject jsonObject = new JSONObject(apiRevisionResponse.getData());

modules/integration/tests-integration/tests-backend/src/test/java/org/wso2/am/integration/tests/restapi/ContentSearchTestCase.java

@@ -111,23 +111,11 @@ public void setEnvironment() throws Exception {
             userManagementClient1.updateRolesOfUser(user.getUserNameWithoutDomain(), newRoleList);
         }
 
-        userManagementClient1
-                .addUser(user1, password, new String[] { role1, "Internal/publisher", "Internal/subscriber" }, user1);
-        userManagementClient1
-                .addUser(user2, password, new String[] { role2, "Internal/publisher", "Internal/subscriber" }, user2);
-
         APIRequest apiRequest = createAPIRequest(contentSearchTestAPI, contentSearchTestAPI, endpointURL, version,
                 user.getUserName(), description);
 
         apiId = createAndPublishAPIUsingRest(apiRequest, restAPIPublisher, false);
 
-        //Login to API Publisher adn Store with CarbonSuper normal user1
-        restAPIPublisherFirstUser = new RestAPIPublisherImpl(user1, password, user.getUserDomain(), publisherURLHttps);
-        restAPIPublisherSecondUser = new RestAPIPublisherImpl(user2, password, user.getUserDomain(), publisherURLHttps);
-
-        restAPIStoreFirstUser = new RestAPIStoreImpl(user1, password, user.getUserDomain(), storeURLHttps);
-        restAPIStoreSecondUser = new RestAPIStoreImpl(user2, password, user.getUserDomain(), storeURLHttps);
-
     }
 
     @Test(groups = { "wso2.am" }, description = "Test basic content Search")
@@ -137,7 +125,7 @@ public void testBasicContentSearch() throws Exception {
         //check in publisher
         for (int i = 0; i <= retries; i++) {
             SearchResultListDTO searchResultListDTO = restAPIPublisher.searchAPIs(description);
-            if (searchResultListDTO.getCount() == 1) {
+            if (searchResultListDTO.getCount() >= 1) {
                 Assert.assertTrue(true);
                 break;
             } else {
@@ -156,7 +144,7 @@ public void testBasicContentSearch() throws Exception {
             //search term : UnifiedSearchFeature, created api has this in description filed
             org.wso2.am.integration.clients.store.api.v1.dto.SearchResultListDTO searchResultListDTO = restAPIStore
                     .searchAPIs(description);
-            if (searchResultListDTO.getCount() == 1) {
+            if (searchResultListDTO.getCount() >= 1) { // API and API Definitions
                 Assert.assertTrue(true);
                 break;
             } else {
@@ -252,35 +240,40 @@ public void testDocumentContentSearch() throws Exception {
     @Test(groups = {
             "wso2.am" }, description = "Test content Search with access control", dependsOnMethods = "testBasicContentSearch")
     public void testContentSearchWithAccessControl() throws Exception {
-
+        // Set API publisher access control to role1
         HttpResponse httpResponse = restAPIPublisher.getAPI(apiId);
-        Gson g = new Gson();
-        APIDTO apiDto = g.fromJson(httpResponse.getData(), APIDTO.class);
+        APIDTO apiDto = new Gson().fromJson(httpResponse.getData(), APIDTO.class);
 
         apiDto.setAccessControl(APIDTO.AccessControlEnum.RESTRICTED);
         List<String> roles = new ArrayList<>();
         roles.add(role1);
         apiDto.setAccessControlRoles(roles);
 
-        apiDto.setVisibility(APIDTO.VisibilityEnum.RESTRICTED);
-        apiDto.setVisibleRoles(roles);
-
         restAPIPublisher.updateAPI(apiDto);
-
         restAPIPublisher.changeAPILifeCycleStatus(apiId, APILifeCycleAction.PUBLISH.getAction());
 
+        // Add users for test
+        userManagementClient1
+                .addUser(user1, password, new String[] { role1, "Internal/publisher"}, user1);
+        userManagementClient1
+                .addUser(user2, password, new String[] { role2, "Internal/publisher"}, user2);
+
+        //Login to API Publisher adn Store with CarbonSuper normal user1
+        restAPIPublisherFirstUser = new RestAPIPublisherImpl(user1, password, user.getUserDomain(), publisherURLHttps);
+        restAPIPublisherSecondUser = new RestAPIPublisherImpl(user2, password, user.getUserDomain(), publisherURLHttps);
+
         //check with user1
         for (int i = 0; i <= retries; i++) {
             SearchResultListDTO searchResultListDTO = restAPIPublisherFirstUser.searchAPIs(description);
-            if (searchResultListDTO.getCount() == 1) {
+            if (searchResultListDTO.getCount() == 2) { // API and API Definition
                 Assert.assertTrue(true);
                 break;
             } else {
                 if (i == retries) {
-                    Assert.fail("Content search with access control failed. 1 result expected. Received response : "
+                    Assert.fail("Content search with access control failed. 2 result expected. Received response : "
                             + searchResultListDTO.getCount());
                 } else {
-                    log.warn("Content search with access control failed. 1 results expected. Received response : "
+                    log.warn("Content search with access control failed. 2 results expected. Received response : "
                             + searchResultListDTO.getCount() + " Retrying...");
                     Thread.sleep(3000);
                 }
@@ -314,19 +307,38 @@ public void testContentSearchWithAccessControl() throws Exception {
             "wso2.am" }, description = "Test content Search with store visibility", dependsOnMethods = "testContentSearchWithAccessControl")
     public void testContentSearchWithStoreVisibility() throws Exception {
 
+        // Set store visibility to role1
+        HttpResponse httpResponse = restAPIPublisher.getAPI(apiId);
+        APIDTO apiDto = new Gson().fromJson(httpResponse.getData(), APIDTO.class);
+        apiDto.setAccessControl(APIDTO.AccessControlEnum.NONE);
+
+        List<String> roles = new ArrayList<>();
+        roles.add(role1);
+        apiDto.setVisibility(APIDTO.VisibilityEnum.RESTRICTED);
+        apiDto.setVisibleRoles(roles);
+
+        restAPIPublisher.updateAPI(apiDto);
+
+        // Update user for tests
+        userManagementClient1.updateRolesOfUser(user1, new String[] { role1, "Internal/subscriber" });
+        userManagementClient1.updateRolesOfUser(user2, new String[] { role2, "Internal/subscriber" });
+
+        restAPIStoreFirstUser = new RestAPIStoreImpl(user1, password, user.getUserDomain(), storeURLHttps);
+        restAPIStoreSecondUser = new RestAPIStoreImpl(user2, password, user.getUserDomain(), storeURLHttps);
+
         //check with user1
         for (int i = 0; i <= retries; i++) {
             org.wso2.am.integration.clients.store.api.v1.dto.SearchResultListDTO searchResultListDTO = restAPIStoreFirstUser
                     .searchAPIs(description);
-            if (searchResultListDTO.getCount() == 1) {
+            if (searchResultListDTO.getCount() == 2) { // API and API Definition
                 Assert.assertTrue(true);
                 break;
             } else {
                 if (i == retries) {
-                    Assert.fail("Content search with visibility failed. 1 result expected. Received response : "
+                    Assert.fail("Content search with visibility failed. 2 result expected. Received response : "
                             + searchResultListDTO.getCount());
                 } else {
-                    log.warn("Content search with visibility failed. 1 results expected. Received response : "
+                    log.warn("Content search with visibility failed. 2 results expected. Received response : "
                             + searchResultListDTO.getCount() + " Retrying...");
                     Thread.sleep(5000);
                 }

modules/integration/tests-integration/tests-backend/src/test/java/org/wso2/am/integration/tests/streamingapis/async/AsyncAPITestCase.java

@@ -157,7 +157,8 @@ public void testPublishAsyncApi() throws Exception {
         try {
             revisionUUID = createAPIRevisionAndDeployUsingRest(apiId, restAPIPublisher);
         } catch (ApiException e) {
-            Assert.assertTrue(e.getMessage().contains("Error while adding new API Revision for API : "));
+            Assert.assertTrue(e.getMessage().contains("Error while adding new API Revision for API : ")
+                    || e.getMessage().contains("Creating API Revisions is not supported"));
         }
         if (revisionUUID != null) {
             Assert.fail();