Merge branch 'wso2:master' into changeapiprovider

wso2 · Dec 11, 2023 · efe6f3c · efe6f3c
2 parents bf8d504 + 39cf70e
commit efe6f3c
Show file tree

Hide file tree

Showing 81 changed files with 24,334 additions and 96 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -101,7 +101,7 @@ jobs:
           fetch-depth: '10'
           path: product-apim
       - name: Build product-apim with tests
-        run: mvn clean install -Dcarbon.apimgt.version=$(cat ../CARBON_APIMGT_VERSION_FILE) -fae --file product-apim/pom.xml -DskipBenchMarkTest=true
+        run: mvn clean install -Dcarbon.apimgt.version=$(cat ../CARBON_APIMGT_VERSION_FILE) -fae --file product-apim/pom.xml -DskipBenchMarkTest=true -DskipRestartTests=true
         env:
           PRODUCT_APIM_TEST_GROUPS: ${{ matrix.segment }}
       - name: Publish Test Report

diff --git a/.gitignore b/.gitignore
@@ -36,9 +36,6 @@ features/apimgt/org.wso2.carbon.apimgt.rest.api.admin.feature/src/main/resources
 features/apimgt/org.wso2.carbon.apimgt.rest.api.admin.feature/src/main/resources/api-docs/package.json
 features/apimgt/org.wso2.carbon.apimgt.core.feature/src/main/resources/tenant/tenant-conf.json
 components/apimgt/org.wso2.carbon.apimgt.hybrid.gateway/org.wso2.carbon.apimgt.hybrid.gateway.usage.publisher/src/test/resources/carbon-home/api-usage-data/api-usage-data.dat
-components/apimgt/org.wso2.carbon.apimgt.rest.api.common/src/main/resources/publisher-api.yaml
-components/apimgt/org.wso2.carbon.apimgt.rest.api.common/src/main/resources/devportal-api.yaml
-components/apimgt/org.wso2.carbon.apimgt.rest.api.common/src/main/resources/admin-api.yaml
 components/apimgt/org.wso2.carbon.apimgt.rest.api.store.v1/swagger.json
 components/apimgt/org.wso2.carbon.apimgt.rest.api.publisher.v1/swagger.json
 components/apimgt/org.wso2.carbon.apimgt.rest.api.admin.v1/swagger.json

diff --git a/components/apimgt/org.wso2.carbon.apimgt.api/pom.xml b/components/apimgt/org.wso2.carbon.apimgt.api/pom.xml
@@ -11,7 +11,7 @@
     <parent>
         <groupId>org.wso2.carbon.apimgt</groupId>
         <artifactId>apimgt</artifactId>
-        <version>9.28.190-SNAPSHOT</version>
+        <version>9.28.200-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

diff --git a/components/apimgt/org.wso2.carbon.apimgt.broker.lifecycle/pom.xml b/components/apimgt/org.wso2.carbon.apimgt.broker.lifecycle/pom.xml
@@ -4,7 +4,7 @@
     <parent>
         <artifactId>apimgt</artifactId>
         <groupId>org.wso2.carbon.apimgt</groupId>
-        <version>9.28.190-SNAPSHOT</version>
+        <version>9.28.200-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

diff --git a/components/apimgt/org.wso2.carbon.apimgt.cache.invalidation/pom.xml b/components/apimgt/org.wso2.carbon.apimgt.cache.invalidation/pom.xml
@@ -19,7 +19,7 @@
     <parent>
         <artifactId>apimgt</artifactId>
         <groupId>org.wso2.carbon.apimgt</groupId>
-        <version>9.28.190-SNAPSHOT</version>
+        <version>9.28.200-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

diff --git a/components/apimgt/org.wso2.carbon.apimgt.cleanup.service/pom.xml b/components/apimgt/org.wso2.carbon.apimgt.cleanup.service/pom.xml
@@ -19,7 +19,7 @@
     <parent>
         <groupId>org.wso2.carbon.apimgt</groupId>
         <artifactId>apimgt</artifactId>
-        <version>9.28.190-SNAPSHOT</version>
+        <version>9.28.200-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

diff --git a/components/apimgt/org.wso2.carbon.apimgt.common.analytics/pom.xml b/components/apimgt/org.wso2.carbon.apimgt.common.analytics/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <groupId>org.wso2.carbon.apimgt</groupId>
         <artifactId>apimgt</artifactId>
-        <version>9.28.190-SNAPSHOT</version>
+        <version>9.28.200-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>

diff --git a/components/apimgt/org.wso2.carbon.apimgt.common.gateway/pom.xml b/components/apimgt/org.wso2.carbon.apimgt.common.gateway/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <groupId>org.wso2.carbon.apimgt</groupId>
         <artifactId>apimgt</artifactId>
-        <version>9.28.190-SNAPSHOT</version>
+        <version>9.28.200-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>

diff --git a/components/apimgt/org.wso2.carbon.apimgt.common.jms/pom.xml b/components/apimgt/org.wso2.carbon.apimgt.common.jms/pom.xml
@@ -4,7 +4,7 @@
     <parent>
         <groupId>org.wso2.carbon.apimgt</groupId>
         <artifactId>apimgt</artifactId>
-        <version>9.28.190-SNAPSHOT</version>
+        <version>9.28.200-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>

diff --git a/components/apimgt/org.wso2.carbon.apimgt.core/pom.xml b/components/apimgt/org.wso2.carbon.apimgt.core/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <groupId>org.wso2.carbon.apimgt</groupId>
         <artifactId>apimgt</artifactId>
-        <version>9.28.190-SNAPSHOT</version>
+        <version>9.28.200-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

diff --git a/components/apimgt/org.wso2.carbon.apimgt.devops.impl/pom.xml b/components/apimgt/org.wso2.carbon.apimgt.devops.impl/pom.xml
@@ -21,7 +21,7 @@
     <parent>
         <groupId>org.wso2.carbon.apimgt</groupId>
         <artifactId>apimgt</artifactId>
-        <version>9.28.190-SNAPSHOT</version>
+        <version>9.28.200-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

diff --git a/components/apimgt/org.wso2.carbon.apimgt.eventing.hub/pom.xml b/components/apimgt/org.wso2.carbon.apimgt.eventing.hub/pom.xml
@@ -20,7 +20,7 @@
     <parent>
         <artifactId>apimgt</artifactId>
         <groupId>org.wso2.carbon.apimgt</groupId>
-        <version>9.28.190-SNAPSHOT</version>
+        <version>9.28.200-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

diff --git a/components/apimgt/org.wso2.carbon.apimgt.eventing/pom.xml b/components/apimgt/org.wso2.carbon.apimgt.eventing/pom.xml
@@ -20,7 +20,7 @@
     <parent>
         <artifactId>apimgt</artifactId>
         <groupId>org.wso2.carbon.apimgt</groupId>
-        <version>9.28.190-SNAPSHOT</version>
+        <version>9.28.200-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

diff --git a/components/apimgt/org.wso2.carbon.apimgt.gateway/pom.xml b/components/apimgt/org.wso2.carbon.apimgt.gateway/pom.xml
@@ -19,7 +19,7 @@
     <parent>
         <groupId>org.wso2.carbon.apimgt</groupId>
         <artifactId>apimgt</artifactId>
-        <version>9.28.190-SNAPSHOT</version>
+        <version>9.28.200-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 
@@ -359,6 +359,21 @@
                     <argLine>
                         --add-opens=java.xml/jdk.xml.internal=ALL-UNNAMED
                         --add-opens=java.base/jdk.internal.loader=ALL-UNNAMED
+                        --add-opens java.base/java.lang=ALL-UNNAMED 
+                        --add-opens java.base/sun.net.www.protocol.jar=ALL-UNNAMED 
+                        --add-opens java.base/java.util=ALL-UNNAMED 
+                        --add-opens java.base/java.io=ALL-UNNAMED 
+                        --add-opens java.base/java.net=ALL-UNNAMED
+                        --add-opens java.base/sun.security.x509=ALL-UNNAMED
+                        --add-opens java.base/java.security.cert=ALL-UNNAMED
+                        --add-exports java.base/sun.security.x509=ALL-UNNAMED     
+                        --add-opens java.base/java.util.concurrent=ALL-UNNAMED
+                        --add-opens java.base/java.nio.charset=ALL-UNNAMED
+                        --add-opens java.base/java.security=ALL-UNNAMED  
+                        --add-opens java.base/java.math=ALL-UNNAMED
+                        --add-opens java.base/sun.nio.cs=ALL-UNNAMED
+                        --add-opens java.base/java.nio=ALL-UNNAMED
+                        --add-exports java.base/sun.nio.cs=ALL-UNNAMED                 
                     </argLine>
                     <classpathDependencyExcludes>
                         <classpathDependencyExclude>org.ops4j.pax.logging:pax-logging-api</classpathDependencyExclude>

diff --git a/...eway/src/main/java/org/wso2/carbon/apimgt/gateway/mediators/oauth/client/OAuthClient.java b/...eway/src/main/java/org/wso2/carbon/apimgt/gateway/mediators/oauth/client/OAuthClient.java
@@ -82,11 +82,13 @@ public static TokenResponse generateToken(String url, String clientId, String cl
             // Set authorization header
             httpPost.setHeader(APIConstants.OAuthConstants.AUTHORIZATION_HEADER, "Basic " + credentials);
             httpPost.setHeader(APIConstants.HEADER_CONTENT_TYPE, APIConstants.OAuthConstants.APPLICATION_X_WWW_FORM_URLENCODED);
-            if (refreshToken != null) {
+            if (APIConstants.OAuthConstants.CLIENT_CREDENTIALS.equals(grantType)) {
+                // As per the RFC 6749, a refresh token should not be included in token response for client credentials grant type.
+                refreshToken = null;
+                payload.append(APIConstants.OAuthConstants.CLIENT_CRED_GRANT_TYPE);
+            } else if (refreshToken != null) {
                 payload.append(APIConstants.OAuthConstants.REFRESH_TOKEN_GRANT_TYPE)
                         .append("&refresh_token=").append(refreshToken);
-            } else if (APIConstants.OAuthConstants.CLIENT_CREDENTIALS.equals(grantType)) {
-                payload.append(APIConstants.OAuthConstants.CLIENT_CRED_GRANT_TYPE);
             } else if (APIConstants.OAuthConstants.PASSWORD.equals(grantType)) {
                 payload.append(APIConstants.OAuthConstants.PASSWORD_GRANT_TYPE + "&username=")
                         .append(URLEncoder.encode(username, APIConstants.DigestAuthConstants.CHARSET)).append("&password=")

diff --git a/components/apimgt/org.wso2.carbon.apimgt.impl/pom.xml b/components/apimgt/org.wso2.carbon.apimgt.impl/pom.xml
@@ -12,7 +12,7 @@
     <parent>
         <groupId>org.wso2.carbon.apimgt</groupId>
         <artifactId>apimgt</artifactId>
-        <version>9.28.190-SNAPSHOT</version>
+        <version>9.28.200-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 
@@ -487,6 +487,22 @@
                     <argLine>
                         --add-opens=java.xml/jdk.xml.internal=ALL-UNNAMED
                         --add-opens=java.base/jdk.internal.loader=ALL-UNNAMED
+                        --add-opens java.base/java.lang=ALL-UNNAMED 
+                        --add-opens java.base/java.io=ALL-UNNAMED 
+                        --add-opens java.base/java.util=ALL-UNNAMED 
+                        --add-opens java.base/java.time=ALL-UNNAMED
+                        --add-opens java.base/java.net=ALL-UNNAMED
+                        --add-opens java.base/sun.reflect.misc=ALL-UNNAMED 
+                        --add-opens java.base/java.util.stream=ALL-UNNAMED 
+                        --add-opens java.base/java.lang.reflect=ALL-UNNAMED
+                        --add-opens java.base/java.math=ALL-UNNAMED
+                        --add-opens java.base/java.security=ALL-UNNAMED 
+                        --add-opens java.base/java.nio.charset=ALL-UNNAMED
+                        --add-opens java.base/java.util.concurrent=ALL-UNNAMED
+                        --add-opens java.base/java.text=ALL-UNNAMED
+                        --add-opens java.base/sun.security.x509=ALL-UNNAMED
+                        --add-opens java.base/java.security.cert=ALL-UNNAMED
+                        --add-exports java.base/sun.security.x509=ALL-UNNAMED
                     </argLine>
                     <systemProperties>
                         <property>

diff --git a/...carbon.apimgt.impl/src/main/java/org/wso2/carbon/apimgt/impl/APIManagerConfiguration.java b/...carbon.apimgt.impl/src/main/java/org/wso2/carbon/apimgt/impl/APIManagerConfiguration.java
@@ -1668,6 +1668,14 @@ private void setJWTConfiguration(OMElement omElement) {
                             configurationElement.getFirstChildWithName(new QName(APIConstants.ENABLE_USER_CLAIMS_RETRIEVAL_FROM_KEY_MANAGER));
                     if (claimRetrievalElement != null) {
                         jwtConfigurationDto.setEnableUserClaimRetrievalFromUserStore(Boolean.parseBoolean(claimRetrievalElement.getText()));
+                        OMElement isBindFederatedUserClaims =
+                                omElement.getFirstChildWithName(new QName(APIConstants.BINDING_FEDERATED_USER_CLAIMS));
+                        if (isBindFederatedUserClaims != null) {
+                            jwtConfigurationDto.setBindFederatedUserClaims(
+                                    Boolean.parseBoolean(isBindFederatedUserClaims.getText()));
+                        } else {
+                            jwtConfigurationDto.setBindFederatedUserClaims(true);
+                        }
                     }
                 }
             }

diff --git a/...n.apimgt.impl/src/main/java/org/wso2/carbon/apimgt/impl/config/APIMConfigServiceImpl.java b/...n.apimgt.impl/src/main/java/org/wso2/carbon/apimgt/impl/config/APIMConfigServiceImpl.java
@@ -46,6 +46,8 @@
 import java.io.IOException;
 import java.io.StringReader;
 import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Iterator;
 
 /**
@@ -176,7 +178,62 @@ public String getTenantConfig(String organization) throws APIManagementException
         if (organization == null) {
             organization = MultitenantConstants.SUPER_TENANT_DOMAIN_NAME;
         }
-        return systemConfigurationsDAO.getSystemConfig(organization, ConfigType.TENANT.toString());
+        return addMissingScopes(systemConfigurationsDAO.getSystemConfig(organization, ConfigType.TENANT.toString()));
+    }
+
+    /*
+     *  This method facilitates the on-the-fly migration of the scope section in the tenant-config.json. This
+     *  checks whether RESTAPIScopes section has newly introduced scopes and add them to the json String if it
+     *  is not available. 
+     */
+    private String addMissingScopes(String systemConfig) {
+        if (systemConfig == null) {
+            return null;
+        }
+        // List of newly introduced scopes
+        String[] scopesToCheck = {
+                "apim:admin_tier_view",
+                "apim:admin_tier_manage",
+                "apim:keymanagers_manage",
+                "apim:api_category"
+            };
+
+        ArrayList<String> missingScopesList = new ArrayList<>(Arrays.asList(scopesToCheck));
+
+        JsonParser jsonParser = new JsonParser();
+        JsonObject jsonObject = jsonParser.parse(systemConfig).getAsJsonObject();
+
+        // Get the existing rest api scopes
+        if (jsonObject.has("RESTAPIScopes")) {
+            JsonObject restApiScopes = jsonObject.getAsJsonObject("RESTAPIScopes");
+            if (restApiScopes.has("Scope")) {
+                JsonArray scopeArray = restApiScopes.getAsJsonArray("Scope");                
+                for (int i = 0; i < scopeArray.size(); i++) {
+                    String existingScope = scopeArray.get(i).getAsJsonObject().get("Name").getAsString();
+                    if (missingScopesList.contains(existingScope)) {
+                        missingScopesList.remove(existingScope);
+                    }
+                }
+            }
+        }
+
+        // Check if there is no missing scopes in the tenant-conf.json and return the original file
+        if (missingScopesList.isEmpty()) {
+            return systemConfig;
+        }
+
+        JsonArray scopeArray = jsonObject.getAsJsonObject("RESTAPIScopes").getAsJsonArray("Scope"); 
+        // Add the missing scopes to the tenant-conf 
+        for (String missingScope : missingScopesList) {
+            JsonObject newScope = new JsonObject();
+            newScope.addProperty("Name", missingScope);
+            newScope.addProperty("Roles", "admin");
+            scopeArray.add(newScope);
+        }
+
+        // Convert the modified JSON back to a string
+        String modifiedJson = jsonObject.toString();
+        return modifiedJson;
     }
 
     @Override
@@ -188,6 +245,9 @@ public void updateTenantConfig(String organization, String tenantConfig) throws
         Cache tenantConfigCache = CacheProvider.getTenantConfigCache();
         String cacheName = organization + "_" + APIConstants.TENANT_CONFIG_CACHE_NAME;
         tenantConfigCache.remove(cacheName);
+
+        // Clear restapi scope cache
+        CacheProvider.getRESTAPIScopeCache().remove(organization);
         systemConfigurationsDAO.updateSystemConfig(organization, ConfigType.TENANT.toString(), tenantConfig);
     }
 

diff --git a/.../org.wso2.carbon.apimgt.impl/src/main/java/org/wso2/carbon/apimgt/impl/utils/APIUtil.java b/.../org.wso2.carbon.apimgt.impl/src/main/java/org/wso2/carbon/apimgt/impl/utils/APIUtil.java
@@ -505,6 +505,8 @@ public static CloseableHttpResponse executeHTTPRequestWithRetries(HttpRequestBas
                     throw new DataLoadingException("Error while retrieving "
                             + path + ". Received response with status code "
                             + httpResponse.getStatusLine().getStatusCode());
+                } else if (retryCount > 0) {
+                    log.info("Successfully retrieved " + path);
                 }
                 retry = false;
             } catch (IOException | DataLoadingException ex) {

diff --git a/components/apimgt/org.wso2.carbon.apimgt.impl/src/main/resources/tenant/tenant-conf.json b/components/apimgt/org.wso2.carbon.apimgt.impl/src/main/resources/tenant/tenant-conf.json
@@ -348,6 +348,14 @@
       {
         "Name": "apim:policies_import_export",
         "Roles": "admin,Internal/devops"
+      },
+      {
+        "Name": "apim:admin_tier_manage",
+        "Roles": "admin"
+      },
+      {
+        "Name": "apim:admin_tier_view",
+        "Roles": "admin"
       }
     ]
   },

diff --git a/...so2.carbon.apimgt.impl/src/test/java/org/wso2/carbon/apimgt/impl/APIProviderImplTest.java b/...so2.carbon.apimgt.impl/src/test/java/org/wso2/carbon/apimgt/impl/APIProviderImplTest.java
@@ -156,7 +156,7 @@
         Caching.class, PaginationContext.class, MultitenantUtils.class, AbstractAPIManager.class, OASParserUtil.class,
         KeyManagerHolder.class, CertificateManagerImpl.class , PublisherAPI.class, Organization.class,
         APIPersistence.class, GatewayArtifactsMgtDAO.class, RegistryPersistenceUtil.class})
-@PowerMockIgnore("org.mockito.*")
+
 public class APIProviderImplTest {
 
     private ApiMgtDAO apimgtDAO;

diff --git a/.../src/test/java/org/wso2/carbon/apimgt/impl/certificatemgt/CertificateManagerImplTest.java b/.../src/test/java/org/wso2/carbon/apimgt/impl/certificatemgt/CertificateManagerImplTest.java
@@ -157,7 +157,7 @@ public void testAddToPublisherWithExistingAlias()
         Assert.assertEquals(ResponseCode.ALIAS_EXISTS_IN_TRUST_STORE, responseCode);
     }
 
-    @Test
+    //@Test
     public void testAddToPublisherWhenDBError() {
 
         PowerMockito.stub(PowerMockito.method(CertificateMgtDAO.class, "addCertificate")).toReturn(false);
@@ -166,7 +166,7 @@ public void testAddToPublisherWhenDBError() {
         Assert.assertEquals(ResponseCode.INTERNAL_SERVER_ERROR, responseCode);
     }
 
-    @Test
+    //@Test
     public void testAddToPublisherWithExistingAliasInDB()
             throws CertificateManagementException, CertificateAliasExistsException {
 

diff --git a/components/apimgt/org.wso2.carbon.apimgt.internal.service/pom.xml b/components/apimgt/org.wso2.carbon.apimgt.internal.service/pom.xml
@@ -20,7 +20,7 @@
     <parent>
         <groupId>org.wso2.carbon.apimgt</groupId>
         <artifactId>apimgt</artifactId>
-        <version>9.28.190-SNAPSHOT</version>
+        <version>9.28.200-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

diff --git a/components/apimgt/org.wso2.carbon.apimgt.jms.listener/pom.xml b/components/apimgt/org.wso2.carbon.apimgt.jms.listener/pom.xml
@@ -4,7 +4,7 @@
     <parent>
         <groupId>org.wso2.carbon.apimgt</groupId>
         <artifactId>apimgt</artifactId>
-        <version>9.28.190-SNAPSHOT</version>
+        <version>9.28.200-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>

diff --git a/components/apimgt/org.wso2.carbon.apimgt.keymgt.client/pom.xml b/components/apimgt/org.wso2.carbon.apimgt.keymgt.client/pom.xml
@@ -16,7 +16,7 @@
     <parent>
         <groupId>org.wso2.carbon.apimgt</groupId>
         <artifactId>apimgt</artifactId>
-        <version>9.28.190-SNAPSHOT</version>
+        <version>9.28.200-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

diff --git a/components/apimgt/org.wso2.carbon.apimgt.keymgt/pom.xml b/components/apimgt/org.wso2.carbon.apimgt.keymgt/pom.xml
@@ -16,7 +16,7 @@
     <parent>
         <groupId>org.wso2.carbon.apimgt</groupId>
         <artifactId>apimgt</artifactId>
-        <version>9.28.190-SNAPSHOT</version>
+        <version>9.28.200-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 
@@ -205,6 +205,13 @@
                 <configuration>
                     <argLine>
                         --add-opens=java.xml/jdk.xml.internal=ALL-UNNAMED
+                        --add-opens java.base/java.lang=ALL-UNNAMED 
+                        --add-opens java.base/java.io=ALL-UNNAMED 
+                        --add-opens java.base/sun.net.www.protocol.jar=ALL-UNNAMED    
+                        --add-opens java.base/java.util=ALL-UNNAMED 
+                        --add-opens java.base/java.security=ALL-UNNAMED 
+                        --add-opens java.base/sun.security.x509=ALL-UNNAMED
+                        --add-opens java.base/java.security.cert=ALL-UNNAMED                   
                     </argLine>
                     <excludes>
                         <exclude>**/Abstract*</exclude>