Merge pull request cdapio#15449 from cdapio/CDAP-20893

[CDAP-20893] Add metrics for credential provider resource and namespaced workload identity counts

cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/CredentialIdentityManager.java

@@ -16,8 +16,10 @@
 
 package io.cdap.cdap.internal.credential;
 
+import io.cdap.cdap.api.metrics.MetricsCollectionService;
 import io.cdap.cdap.common.AlreadyExistsException;
 import io.cdap.cdap.common.NotFoundException;
+import io.cdap.cdap.common.conf.Constants.Metrics.Credential;
 import io.cdap.cdap.proto.credential.CredentialIdentity;
 import io.cdap.cdap.proto.id.CredentialIdentityId;
 import io.cdap.cdap.proto.id.CredentialProfileId;
@@ -27,6 +29,7 @@
 import io.cdap.cdap.spi.data.transaction.TransactionRunners;
 import java.io.IOException;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.Optional;
 import javax.inject.Inject;
 
@@ -38,13 +41,16 @@ public class CredentialIdentityManager {
   private final CredentialIdentityStore identityStore;
   private final CredentialProfileStore profileStore;
   private final TransactionRunner transactionRunner;
+  private final MetricsCollectionService metricsCollectionService;
 
   @Inject
   CredentialIdentityManager(CredentialIdentityStore identityStore,
-      CredentialProfileStore profileStore, TransactionRunner transactionRunner) {
+      CredentialProfileStore profileStore, TransactionRunner transactionRunner,
+      MetricsCollectionService metricsCollectionService) {
     this.identityStore = identityStore;
     this.profileStore = profileStore;
     this.transactionRunner = transactionRunner;
+    this.metricsCollectionService = metricsCollectionService;
   }
 
   /**
@@ -94,6 +100,7 @@ public void create(CredentialIdentityId id, CredentialIdentity identity)
             id.getNamespace(), id.getName()));
       }
       validateAndWriteIdentity(context, id, identity);
+      emitCredentialIdentityCountMetric(context);
     }, AlreadyExistsException.class, IOException.class, NotFoundException.class);
   }
 
@@ -131,6 +138,7 @@ public void delete(CredentialIdentityId id) throws IOException, NotFoundExceptio
             id.getNamespace(), id.getName()));
       }
       identityStore.delete(context, id);
+      emitCredentialIdentityCountMetric(context);
     }, IOException.class, NotFoundException.class);
   }
 
@@ -149,4 +157,10 @@ private void validateAndWriteIdentity(StructuredTableContext context, Credential
       throw new IOException("Failed to encrypt identity", e);
     }
   }
+
+  private void emitCredentialIdentityCountMetric(StructuredTableContext context)
+      throws IOException {
+    metricsCollectionService.getContext(Collections.emptyMap())
+        .gauge(Credential.CREDENTIAL_IDENTITY_COUNT, identityStore.getIdentityCount(context));
+  }
 }

cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/CredentialIdentityStore.java

@@ -177,6 +177,18 @@ public void delete(StructuredTableContext context, CredentialIdentityId id)
     table.delete(key);
   }
 
+  /**
+   * Returns the count of all credential identities.
+   *
+   * @param context The transaction context to use.
+   * @return The number of credential identities.
+   * @throws IOException If any failure reading from storage occurs.
+   */
+  public long getIdentityCount(StructuredTableContext context) throws IOException {
+    StructuredTable table = context.getTable(CredentialProviderStore.CREDENTIAL_IDENTITIES);
+    return table.count(Collections.singletonList(Range.all()));
+  }
+
   private static Collection<CredentialIdentityId> identitiesFromRowIterator(
       Iterator<StructuredRow> iterator) {
     return StreamSupport.stream(Spliterators.spliteratorUnknownSize(iterator,

cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/CredentialProfileManager.java

@@ -17,20 +17,24 @@
 package io.cdap.cdap.internal.credential;
 
 import com.google.gson.Gson;
+import io.cdap.cdap.api.metrics.MetricsCollectionService;
 import io.cdap.cdap.common.AlreadyExistsException;
 import io.cdap.cdap.common.BadRequestException;
 import io.cdap.cdap.common.ConflictException;
 import io.cdap.cdap.common.NotFoundException;
+import io.cdap.cdap.common.conf.Constants.Metrics.Credential;
 import io.cdap.cdap.proto.credential.CredentialProfile;
 import io.cdap.cdap.proto.id.CredentialIdentityId;
 import io.cdap.cdap.proto.id.CredentialProfileId;
 import io.cdap.cdap.security.spi.credential.CredentialProvider;
 import io.cdap.cdap.security.spi.credential.ProfileValidationException;
 import io.cdap.cdap.security.spi.encryption.CipherException;
+import io.cdap.cdap.spi.data.StructuredTableContext;
 import io.cdap.cdap.spi.data.transaction.TransactionRunner;
 import io.cdap.cdap.spi.data.transaction.TransactionRunners;
 import java.io.IOException;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.Map;
 import java.util.Optional;
 import javax.inject.Inject;
@@ -46,15 +50,18 @@ public class CredentialProfileManager {
   private final CredentialProfileStore profileStore;
   private final TransactionRunner transactionRunner;
   private final Map<String, CredentialProvider> credentialProviders;
+  private final MetricsCollectionService metricsCollectionService;
 
   @Inject
   CredentialProfileManager(CredentialIdentityStore identityStore,
       CredentialProfileStore profileStore, TransactionRunner transactionRunner,
-      CredentialProviderLoader credentialProviderLoader) {
+      CredentialProviderLoader credentialProviderLoader,
+      MetricsCollectionService metricsCollectionService) {
     this.identityStore = identityStore;
     this.profileStore = profileStore;
     this.transactionRunner = transactionRunner;
     this.credentialProviders = credentialProviderLoader.loadCredentialProviders();
+    this.metricsCollectionService = metricsCollectionService;
   }
 
   /**
@@ -106,6 +113,7 @@ public void create(CredentialProfileId id, CredentialProfile profile)
       }
       try {
         profileStore.write(context, id, profile);
+        emitCredentialProfileCountMetric(context);
       } catch (CipherException e) {
         throw new IOException("Failed to encrypt profile", e);
       }
@@ -160,6 +168,7 @@ public void delete(CredentialProfileId id)
             + "has the following attached identities: %s", GSON.toJson(profileIdentities)));
       }
       profileStore.delete(context, id);
+      emitCredentialProfileCountMetric(context);
     }, ConflictException.class, IOException.class, NotFoundException.class);
   }
 
@@ -180,4 +189,10 @@ private void validateProfile(CredentialProfile profile) throws BadRequestExcepti
           e.getMessage()), e);
     }
   }
+
+  private void emitCredentialProfileCountMetric(StructuredTableContext context)
+      throws IOException {
+    metricsCollectionService.getContext(Collections.emptyMap())
+        .gauge(Credential.CREDENTIAL_PROFILE_COUNT, profileStore.getProfileCount(context));
+  }
 }

cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/CredentialProfileStore.java

@@ -153,6 +153,18 @@ public void delete(StructuredTableContext context, CredentialProfileId id)
     table.delete(key);
   }
 
+  /**
+   * Returns the count of all credential profiles.
+   *
+   * @param context The transaction context to use.
+   * @return The number of credential profiles.
+   * @throws IOException If any failure reading from storage occurs.
+   */
+  public long getProfileCount(StructuredTableContext context) throws IOException {
+    StructuredTable table = context.getTable(CredentialProviderStore.CREDENTIAL_PROFILES);
+    return table.count(Collections.singletonList(Range.all()));
+  }
+
   private static Collection<CredentialProfileId> profilesFromRowIterator(
       Iterator<StructuredRow> iterator) {
     return StreamSupport.stream(Spliterators.spliteratorUnknownSize(iterator,

cdap-app-fabric/src/main/java/io/cdap/cdap/internal/namespace/credential/GcpWorkloadIdentityUtil.java

@@ -30,7 +30,7 @@
  */
 public final class GcpWorkloadIdentityUtil {
 
-  private static final String NAMESPACE_IDENTITY_NAME_PREFIX = "ns-gcp-wi";
+  public static final String NAMESPACE_IDENTITY_NAME_PREFIX = "ns-gcp-wi";
 
   public static final String SYSTEM_PROFILE_NAME = "ns-gcp-wi";
 

cdap-app-fabric/src/main/java/io/cdap/cdap/internal/namespace/credential/handler/GcpWorkloadIdentityHttpHandler.java

@@ -20,11 +20,13 @@
 import com.google.gson.JsonSyntaxException;
 import com.google.inject.Inject;
 import com.google.inject.Singleton;
+import io.cdap.cdap.api.metrics.MetricsCollectionService;
 import io.cdap.cdap.common.AlreadyExistsException;
 import io.cdap.cdap.common.BadRequestException;
 import io.cdap.cdap.common.NamespaceNotFoundException;
 import io.cdap.cdap.common.NotFoundException;
 import io.cdap.cdap.common.conf.Constants.Gateway;
+import io.cdap.cdap.common.conf.Constants.Metrics.WorkloadIdentity;
 import io.cdap.cdap.common.namespace.NamespaceQueryAdmin;
 import io.cdap.cdap.internal.credential.CredentialIdentityManager;
 import io.cdap.cdap.internal.namespace.credential.GcpWorkloadIdentityUtil;
@@ -50,6 +52,7 @@
 import java.io.InputStreamReader;
 import java.io.Reader;
 import java.nio.charset.StandardCharsets;
+import java.util.Collections;
 import java.util.Optional;
 import javax.ws.rs.DELETE;
 import javax.ws.rs.GET;
@@ -71,16 +74,19 @@ public class GcpWorkloadIdentityHttpHandler extends AbstractHttpHandler {
   private final NamespaceQueryAdmin namespaceQueryAdmin;
   private final CredentialIdentityManager credentialIdentityManager;
   private final NamespaceCredentialProvider credentialProvider;
+  private final MetricsCollectionService metricsCollectionService;
 
   @Inject
   GcpWorkloadIdentityHttpHandler(ContextAccessEnforcer accessEnforcer,
       NamespaceQueryAdmin namespaceQueryAdmin,
       CredentialIdentityManager credentialIdentityManager,
-      NamespaceCredentialProvider credentialProvider) {
+      NamespaceCredentialProvider credentialProvider,
+      MetricsCollectionService metricsCollectionService) {
     this.accessEnforcer = accessEnforcer;
     this.namespaceQueryAdmin = namespaceQueryAdmin;
     this.credentialIdentityManager = credentialIdentityManager;
     this.credentialProvider = credentialProvider;
+    this.metricsCollectionService = metricsCollectionService;
   }
 
   /**
@@ -171,6 +177,7 @@ public void createIdentity(FullHttpRequest request, HttpResponder responder,
       credentialIdentityManager.create(credentialIdentityId, credentialIdentity);
     }
     responder.sendStatus(HttpResponseStatus.OK);
+    emitNamespaceWorkloadIdentityCountMetric();
   }
 
   /**
@@ -195,6 +202,7 @@ public void deleteIdentity(HttpRequest request, HttpResponder responder,
     switchToInternalUser();
     credentialIdentityManager.delete(credentialIdentityId);
     responder.sendStatus(HttpResponseStatus.OK);
+    emitNamespaceWorkloadIdentityCountMetric();
   }
 
   private NamespaceMeta getNamespaceMeta(String namespace) throws Exception {
@@ -240,5 +248,13 @@ private <T> T deserializeRequestContent(FullHttpRequest request, Class<T> clazz)
       throw new BadRequestException("Unable to parse request: " + e.getMessage(), e);
     }
   }
+
+  private void emitNamespaceWorkloadIdentityCountMetric() throws IOException {
+    long namespaceIdentitiesCount = credentialIdentityManager
+        .list(NamespaceId.SYSTEM.getNamespace()).stream().filter(identityId -> identityId.getName()
+            .startsWith(GcpWorkloadIdentityUtil.NAMESPACE_IDENTITY_NAME_PREFIX)).count();
+    metricsCollectionService.getContext(Collections.emptyMap())
+        .gauge(WorkloadIdentity.NAMESPACE_WORKLOAD_IDENTITY_COUNT, namespaceIdentitiesCount);
+  }
 }
 

cdap-app-fabric/src/test/java/io/cdap/cdap/internal/credential/CredentialProviderTestBase.java

@@ -123,9 +123,9 @@ protected void configure() {
     CredentialProfileStore profileStore = new CredentialProfileStore(new NoOpAeadCipher());
     CredentialIdentityStore identityStore = new CredentialIdentityStore(new NoOpAeadCipher());
     credentialProfileManager = new CredentialProfileManager(identityStore, profileStore,
-        runner, mockCredentialProviderLoader);
+        runner, mockCredentialProviderLoader, new NoOpMetricsCollectionService());
     credentialIdentityManager = new CredentialIdentityManager(identityStore, profileStore,
-        runner);
+        runner, new NoOpMetricsCollectionService());
   }
 
   @AfterClass

cdap-common/src/main/java/io/cdap/cdap/common/conf/Constants.java

@@ -1315,6 +1315,23 @@ public static final class SourceControlManagement {
       public static final String COMMIT_PUSH_LATENCY_MILLIS =
           "source.control.git.commit.push.duration.ms";
     }
+
+    /**
+     * Credential Provider Metrics.
+     */
+    public static final class Credential {
+      public static final String CREDENTIAL_IDENTITY_COUNT = "credential.identity.count";
+      public static final String CREDENTIAL_PROFILE_COUNT = "credential.profile.count";
+    }
+
+    /**
+     * Workload Identity Metrics.
+     */
+    public static final class WorkloadIdentity {
+
+      public static final String NAMESPACE_WORKLOAD_IDENTITY_COUNT =
+          "namespace.workload.identity.count";
+    }
   }
 
   /**
@@ -1858,6 +1875,7 @@ public static final class AuthenticationServer {
      * Security configurations for encryption.
      */
     public static final class Encryption {
+
       /**
        * Directory for encryption extensions.
        */