From c5b8a8848eb392569221d16d851a04dc5c346c5b Mon Sep 17 00:00:00 2001 From: karthik-tarento Date: Fri, 1 Sep 2023 18:13:34 +0530 Subject: [PATCH 1/7] Using Async process for uploading zip file content --- .../sunbird/mimetype/mgr/BaseMimeTypeManager.scala | 14 ++++++++++++++ .../mimetype/mgr/impl/HtmlMimeTypeMgrImpl.scala | 8 +++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/platform-modules/mimetype-manager/src/main/scala/org/sunbird/mimetype/mgr/BaseMimeTypeManager.scala b/platform-modules/mimetype-manager/src/main/scala/org/sunbird/mimetype/mgr/BaseMimeTypeManager.scala index c357443c5..416d75925 100644 --- a/platform-modules/mimetype-manager/src/main/scala/org/sunbird/mimetype/mgr/BaseMimeTypeManager.scala +++ b/platform-modules/mimetype-manager/src/main/scala/org/sunbird/mimetype/mgr/BaseMimeTypeManager.scala @@ -214,6 +214,20 @@ class BaseMimeTypeManager(implicit ss: StorageService) { } } + def extractPackageInCloudAsync(objectId: String, uploadFile: File, node: Node, extractionType: String, slugFile: Boolean)(implicit ec: ExecutionContext): Future[List[String]] = { + val file = Slug.createSlugFile(uploadFile) + val mimeType = node.getMetadata.get("mimeType").asInstanceOf[String] + validationForCloudExtraction(file, extractionType, mimeType) + if(extractableMimeTypes.contains(mimeType)){ + val extractionBasePath = getBasePath(objectId) + extractPackage(file, extractionBasePath) + ss.uploadDirectoryAsync(getExtractionPath(objectId, node, extractionType, mimeType), new File(extractionBasePath), Option(slugFile)) + } else { + val emptyFuture: Future[List[String]] = Future.successful(List.empty[String]) + emptyFuture + } + } + def extractH5PPackageInCloud(objectId: String, extractionBasePath: String, node: Node, extractionType: String, slugFile: Boolean)(implicit ec: ExecutionContext): Future[List[String]] = { val mimeType = node.getMetadata.get("mimeType").asInstanceOf[String] if(null == extractionType) diff --git a/platform-modules/mimetype-manager/src/main/scala/org/sunbird/mimetype/mgr/impl/HtmlMimeTypeMgrImpl.scala b/platform-modules/mimetype-manager/src/main/scala/org/sunbird/mimetype/mgr/impl/HtmlMimeTypeMgrImpl.scala index 8418d2180..b4dfb323f 100644 --- a/platform-modules/mimetype-manager/src/main/scala/org/sunbird/mimetype/mgr/impl/HtmlMimeTypeMgrImpl.scala +++ b/platform-modules/mimetype-manager/src/main/scala/org/sunbird/mimetype/mgr/impl/HtmlMimeTypeMgrImpl.scala @@ -24,7 +24,13 @@ class HtmlMimeTypeMgrImpl(implicit ss: StorageService) extends BaseMimeTypeManag val urls = uploadArtifactToCloud(uploadFile, objectId, filePath) node.getMetadata.put("s3Key", urls(IDX_S3_KEY)) node.getMetadata.put("artifactUrl", urls(IDX_S3_URL)) - extractPackageInCloud(objectId, uploadFile, node, "snapshot", false) + Future { + extractPackageInCloudAsync(objectId, uploadFile, node, "snapshot", false).map(resp => + TelemetryManager.info("H5P content snapshot folder upload success for " + objectId) + ) onFailure { case e: Throwable => + TelemetryManager.error("H5P content snapshot folder upload failed for " + objectId, e.getCause) + } + } Future(Map[String, AnyRef]("identifier" -> objectId, "artifactUrl" -> urls(IDX_S3_URL), "s3Key" -> urls(IDX_S3_KEY), "size" -> getFileSize(uploadFile).asInstanceOf[AnyRef])) } else { TelemetryManager.error("ERR_INVALID_FILE" + "Please Provide Valid File! with file name: " + uploadFile.getName) From c81c3d9f3a85003627f867a6e137eebb5878049a Mon Sep 17 00:00:00 2001 From: karthik-tarento Date: Fri, 1 Sep 2023 18:52:30 +0530 Subject: [PATCH 2/7] Not waiting to unzip the file --- .../sunbird/mimetype/mgr/impl/HtmlMimeTypeMgrImpl.scala | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/platform-modules/mimetype-manager/src/main/scala/org/sunbird/mimetype/mgr/impl/HtmlMimeTypeMgrImpl.scala b/platform-modules/mimetype-manager/src/main/scala/org/sunbird/mimetype/mgr/impl/HtmlMimeTypeMgrImpl.scala index b4dfb323f..1ef8f2125 100644 --- a/platform-modules/mimetype-manager/src/main/scala/org/sunbird/mimetype/mgr/impl/HtmlMimeTypeMgrImpl.scala +++ b/platform-modules/mimetype-manager/src/main/scala/org/sunbird/mimetype/mgr/impl/HtmlMimeTypeMgrImpl.scala @@ -24,19 +24,12 @@ class HtmlMimeTypeMgrImpl(implicit ss: StorageService) extends BaseMimeTypeManag val urls = uploadArtifactToCloud(uploadFile, objectId, filePath) node.getMetadata.put("s3Key", urls(IDX_S3_KEY)) node.getMetadata.put("artifactUrl", urls(IDX_S3_URL)) - Future { - extractPackageInCloudAsync(objectId, uploadFile, node, "snapshot", false).map(resp => - TelemetryManager.info("H5P content snapshot folder upload success for " + objectId) - ) onFailure { case e: Throwable => - TelemetryManager.error("H5P content snapshot folder upload failed for " + objectId, e.getCause) - } - } + extractPackageInCloudAsync(objectId, uploadFile, node, "snapshot", false) Future(Map[String, AnyRef]("identifier" -> objectId, "artifactUrl" -> urls(IDX_S3_URL), "s3Key" -> urls(IDX_S3_KEY), "size" -> getFileSize(uploadFile).asInstanceOf[AnyRef])) } else { TelemetryManager.error("ERR_INVALID_FILE" + "Please Provide Valid File! with file name: " + uploadFile.getName) throw new ClientException("ERR_INVALID_FILE", "Please Provide Valid File!") } - } override def upload(objectId: String, node: Node, fileUrl: String, filePath: Option[String], params: UploadParams)(implicit ec: ExecutionContext): Future[Map[String, AnyRef]] = { From e3becdd94e5f4b0d1c6c6bd5f9246e81a186233a Mon Sep 17 00:00:00 2001 From: karthik-tarento Date: Fri, 1 Sep 2023 19:30:12 +0530 Subject: [PATCH 3/7] Added debug logs --- .../org/sunbird/mimetype/mgr/BaseMimeTypeManager.scala | 5 ++++- .../org/sunbird/mimetype/mgr/impl/HtmlMimeTypeMgrImpl.scala | 6 ++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/platform-modules/mimetype-manager/src/main/scala/org/sunbird/mimetype/mgr/BaseMimeTypeManager.scala b/platform-modules/mimetype-manager/src/main/scala/org/sunbird/mimetype/mgr/BaseMimeTypeManager.scala index 416d75925..d710a986d 100644 --- a/platform-modules/mimetype-manager/src/main/scala/org/sunbird/mimetype/mgr/BaseMimeTypeManager.scala +++ b/platform-modules/mimetype-manager/src/main/scala/org/sunbird/mimetype/mgr/BaseMimeTypeManager.scala @@ -221,7 +221,10 @@ class BaseMimeTypeManager(implicit ss: StorageService) { if(extractableMimeTypes.contains(mimeType)){ val extractionBasePath = getBasePath(objectId) extractPackage(file, extractionBasePath) - ss.uploadDirectoryAsync(getExtractionPath(objectId, node, extractionType, mimeType), new File(extractionBasePath), Option(slugFile)) + TelemetryManager.info("***** Calling ss.uploadDirectoryAsync ****") + var result: Future[List[String]] = ss.uploadDirectoryAsync(getExtractionPath(objectId, node, extractionType, mimeType), new File(extractionBasePath), Option(slugFile)) + TelemetryManager.info("***** returning from extractPackageInCloudSync ****") + result } else { val emptyFuture: Future[List[String]] = Future.successful(List.empty[String]) emptyFuture diff --git a/platform-modules/mimetype-manager/src/main/scala/org/sunbird/mimetype/mgr/impl/HtmlMimeTypeMgrImpl.scala b/platform-modules/mimetype-manager/src/main/scala/org/sunbird/mimetype/mgr/impl/HtmlMimeTypeMgrImpl.scala index 1ef8f2125..0c910f52a 100644 --- a/platform-modules/mimetype-manager/src/main/scala/org/sunbird/mimetype/mgr/impl/HtmlMimeTypeMgrImpl.scala +++ b/platform-modules/mimetype-manager/src/main/scala/org/sunbird/mimetype/mgr/impl/HtmlMimeTypeMgrImpl.scala @@ -18,13 +18,15 @@ class HtmlMimeTypeMgrImpl(implicit ss: StorageService) extends BaseMimeTypeManag override def upload(objectId: String, node: Node, uploadFile: File, filePath: Option[String], params: UploadParams)(implicit ec: ExecutionContext): Future[Map[String, AnyRef]] = { validateUploadRequest(objectId, node, uploadFile) val indexHtmlValidation: Boolean = if (Platform.config.hasPath("indexHtmlValidation.env")) Platform.config.getBoolean("indexHtmlValidation.env") else true - TelemetryManager.error("Value of indexHtmlValidation: " + indexHtmlValidation) + TelemetryManager.info("Value of indexHtmlValidation: " + indexHtmlValidation) val flag: Boolean = if (indexHtmlValidation) isValidPackageStructure(uploadFile, List[String]("index.html")) else true if (flag) { val urls = uploadArtifactToCloud(uploadFile, objectId, filePath) node.getMetadata.put("s3Key", urls(IDX_S3_KEY)) node.getMetadata.put("artifactUrl", urls(IDX_S3_URL)) - extractPackageInCloudAsync(objectId, uploadFile, node, "snapshot", false) + TelemetryManager.info("***** Calling extractPackageInCloudSync ****") + Future { extractPackageInCloudAsync(objectId, uploadFile, node, "snapshot", false) } + TelemetryManager.info("***** constructing return value from upload ****") Future(Map[String, AnyRef]("identifier" -> objectId, "artifactUrl" -> urls(IDX_S3_URL), "s3Key" -> urls(IDX_S3_KEY), "size" -> getFileSize(uploadFile).asInstanceOf[AnyRef])) } else { TelemetryManager.error("ERR_INVALID_FILE" + "Please Provide Valid File! with file name: " + uploadFile.getName) From cd1d011c7de2e938addc896610e5154572e5f08c Mon Sep 17 00:00:00 2001 From: karthik-tarento Date: Mon, 4 Sep 2023 11:24:33 +0530 Subject: [PATCH 4/7] added log to print duration --- .../org/sunbird/mimetype/mgr/impl/HtmlMimeTypeMgrImpl.scala | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/platform-modules/mimetype-manager/src/main/scala/org/sunbird/mimetype/mgr/impl/HtmlMimeTypeMgrImpl.scala b/platform-modules/mimetype-manager/src/main/scala/org/sunbird/mimetype/mgr/impl/HtmlMimeTypeMgrImpl.scala index 0c910f52a..f7305a207 100644 --- a/platform-modules/mimetype-manager/src/main/scala/org/sunbird/mimetype/mgr/impl/HtmlMimeTypeMgrImpl.scala +++ b/platform-modules/mimetype-manager/src/main/scala/org/sunbird/mimetype/mgr/impl/HtmlMimeTypeMgrImpl.scala @@ -21,11 +21,16 @@ class HtmlMimeTypeMgrImpl(implicit ss: StorageService) extends BaseMimeTypeManag TelemetryManager.info("Value of indexHtmlValidation: " + indexHtmlValidation) val flag: Boolean = if (indexHtmlValidation) isValidPackageStructure(uploadFile, List[String]("index.html")) else true if (flag) { + TelemetryManager.info("**** HtmlMimeTypeMgrImpl::upload... started... ") + var startTime = System.currentTimeMillis() val urls = uploadArtifactToCloud(uploadFile, objectId, filePath) + TelemetryManager.info("**** HtmlMimeTypeMgrImpl::upload... timeTaken : " + (System.currentTimeMillis() - startTime)) + startTime = System.currentTimeMillis() node.getMetadata.put("s3Key", urls(IDX_S3_KEY)) node.getMetadata.put("artifactUrl", urls(IDX_S3_URL)) TelemetryManager.info("***** Calling extractPackageInCloudSync ****") Future { extractPackageInCloudAsync(objectId, uploadFile, node, "snapshot", false) } + TelemetryManager.info("**** HtmlMimeTypeMgrImpl::upload... timeTaken for extractPackageInCloudAsync **** : " + (System.currentTimeMillis() - startTime)) TelemetryManager.info("***** constructing return value from upload ****") Future(Map[String, AnyRef]("identifier" -> objectId, "artifactUrl" -> urls(IDX_S3_URL), "s3Key" -> urls(IDX_S3_KEY), "size" -> getFileSize(uploadFile).asInstanceOf[AnyRef])) } else { From f2f6b5f633b6d9b65fd0411cc949eaf07fa9e45d Mon Sep 17 00:00:00 2001 From: karthik-tarento Date: Mon, 4 Sep 2023 17:07:45 +0530 Subject: [PATCH 5/7] Added logs in content controller --- .../content-service/app/controllers/v3/ContentController.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/content-api/content-service/app/controllers/v3/ContentController.scala b/content-api/content-service/app/controllers/v3/ContentController.scala index a5c73c51a..4b02f233e 100644 --- a/content-api/content-service/app/controllers/v3/ContentController.scala +++ b/content-api/content-service/app/controllers/v3/ContentController.scala @@ -12,6 +12,7 @@ import utils.{ActorNames, ApiId, JavaJsonUtils} import scala.collection.JavaConverters._ import scala.concurrent.{ExecutionContext, Future} +import org.sunbird.telemetry.logger.TelemetryManager @Singleton class ContentController @Inject()(@Named(ActorNames.CONTENT_ACTOR) contentActor: ActorRef, @Named(ActorNames.COLLECTION_ACTOR) collectionActor: ActorRef, cc: ControllerComponents, actorSystem: ActorSystem)(implicit exec: ExecutionContext) extends BaseController(cc) { @@ -224,6 +225,7 @@ class ContentController @Inject()(@Named(ActorNames.CONTENT_ACTOR) contentActor: } def upload(identifier: String, fileFormat: Option[String], validation: Option[String]) = Action.async { implicit request => + TelemetryManager.info("**** ContentController::upload... started... ") val headers = commonHeaders() val content = requestFormData(identifier) content.putAll(headers) From c6fd2d797275bef792874aff851bb664459f560e Mon Sep 17 00:00:00 2001 From: karthik-tarento Date: Tue, 5 Sep 2023 18:51:18 +0530 Subject: [PATCH 6/7] Removed debug logs --- .../app/controllers/v3/ContentController.scala | 1 - .../org/sunbird/mimetype/mgr/BaseMimeTypeManager.scala | 5 +---- .../sunbird/mimetype/mgr/impl/HtmlMimeTypeMgrImpl.scala | 9 +-------- 3 files changed, 2 insertions(+), 13 deletions(-) diff --git a/content-api/content-service/app/controllers/v3/ContentController.scala b/content-api/content-service/app/controllers/v3/ContentController.scala index 4b02f233e..e449d4c79 100644 --- a/content-api/content-service/app/controllers/v3/ContentController.scala +++ b/content-api/content-service/app/controllers/v3/ContentController.scala @@ -225,7 +225,6 @@ class ContentController @Inject()(@Named(ActorNames.CONTENT_ACTOR) contentActor: } def upload(identifier: String, fileFormat: Option[String], validation: Option[String]) = Action.async { implicit request => - TelemetryManager.info("**** ContentController::upload... started... ") val headers = commonHeaders() val content = requestFormData(identifier) content.putAll(headers) diff --git a/platform-modules/mimetype-manager/src/main/scala/org/sunbird/mimetype/mgr/BaseMimeTypeManager.scala b/platform-modules/mimetype-manager/src/main/scala/org/sunbird/mimetype/mgr/BaseMimeTypeManager.scala index d710a986d..416d75925 100644 --- a/platform-modules/mimetype-manager/src/main/scala/org/sunbird/mimetype/mgr/BaseMimeTypeManager.scala +++ b/platform-modules/mimetype-manager/src/main/scala/org/sunbird/mimetype/mgr/BaseMimeTypeManager.scala @@ -221,10 +221,7 @@ class BaseMimeTypeManager(implicit ss: StorageService) { if(extractableMimeTypes.contains(mimeType)){ val extractionBasePath = getBasePath(objectId) extractPackage(file, extractionBasePath) - TelemetryManager.info("***** Calling ss.uploadDirectoryAsync ****") - var result: Future[List[String]] = ss.uploadDirectoryAsync(getExtractionPath(objectId, node, extractionType, mimeType), new File(extractionBasePath), Option(slugFile)) - TelemetryManager.info("***** returning from extractPackageInCloudSync ****") - result + ss.uploadDirectoryAsync(getExtractionPath(objectId, node, extractionType, mimeType), new File(extractionBasePath), Option(slugFile)) } else { val emptyFuture: Future[List[String]] = Future.successful(List.empty[String]) emptyFuture diff --git a/platform-modules/mimetype-manager/src/main/scala/org/sunbird/mimetype/mgr/impl/HtmlMimeTypeMgrImpl.scala b/platform-modules/mimetype-manager/src/main/scala/org/sunbird/mimetype/mgr/impl/HtmlMimeTypeMgrImpl.scala index f7305a207..833bc27ff 100644 --- a/platform-modules/mimetype-manager/src/main/scala/org/sunbird/mimetype/mgr/impl/HtmlMimeTypeMgrImpl.scala +++ b/platform-modules/mimetype-manager/src/main/scala/org/sunbird/mimetype/mgr/impl/HtmlMimeTypeMgrImpl.scala @@ -18,20 +18,13 @@ class HtmlMimeTypeMgrImpl(implicit ss: StorageService) extends BaseMimeTypeManag override def upload(objectId: String, node: Node, uploadFile: File, filePath: Option[String], params: UploadParams)(implicit ec: ExecutionContext): Future[Map[String, AnyRef]] = { validateUploadRequest(objectId, node, uploadFile) val indexHtmlValidation: Boolean = if (Platform.config.hasPath("indexHtmlValidation.env")) Platform.config.getBoolean("indexHtmlValidation.env") else true - TelemetryManager.info("Value of indexHtmlValidation: " + indexHtmlValidation) + TelemetryManager.log("Value of indexHtmlValidation: " + indexHtmlValidation) val flag: Boolean = if (indexHtmlValidation) isValidPackageStructure(uploadFile, List[String]("index.html")) else true if (flag) { - TelemetryManager.info("**** HtmlMimeTypeMgrImpl::upload... started... ") - var startTime = System.currentTimeMillis() val urls = uploadArtifactToCloud(uploadFile, objectId, filePath) - TelemetryManager.info("**** HtmlMimeTypeMgrImpl::upload... timeTaken : " + (System.currentTimeMillis() - startTime)) - startTime = System.currentTimeMillis() node.getMetadata.put("s3Key", urls(IDX_S3_KEY)) node.getMetadata.put("artifactUrl", urls(IDX_S3_URL)) - TelemetryManager.info("***** Calling extractPackageInCloudSync ****") Future { extractPackageInCloudAsync(objectId, uploadFile, node, "snapshot", false) } - TelemetryManager.info("**** HtmlMimeTypeMgrImpl::upload... timeTaken for extractPackageInCloudAsync **** : " + (System.currentTimeMillis() - startTime)) - TelemetryManager.info("***** constructing return value from upload ****") Future(Map[String, AnyRef]("identifier" -> objectId, "artifactUrl" -> urls(IDX_S3_URL), "s3Key" -> urls(IDX_S3_KEY), "size" -> getFileSize(uploadFile).asInstanceOf[AnyRef])) } else { TelemetryManager.error("ERR_INVALID_FILE" + "Please Provide Valid File! with file name: " + uploadFile.getName) From 0e5f394422f536876d2cbd4c18f71d3cce17f116 Mon Sep 17 00:00:00 2001 From: karthik-tarento Date: Tue, 5 Sep 2023 19:08:53 +0530 Subject: [PATCH 7/7] Removed unnecessary imports --- .../content-service/app/controllers/v3/ContentController.scala | 1 - .../org/sunbird/mimetype/mgr/impl/HtmlMimeTypeMgrImpl.scala | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/content-api/content-service/app/controllers/v3/ContentController.scala b/content-api/content-service/app/controllers/v3/ContentController.scala index e449d4c79..a5c73c51a 100644 --- a/content-api/content-service/app/controllers/v3/ContentController.scala +++ b/content-api/content-service/app/controllers/v3/ContentController.scala @@ -12,7 +12,6 @@ import utils.{ActorNames, ApiId, JavaJsonUtils} import scala.collection.JavaConverters._ import scala.concurrent.{ExecutionContext, Future} -import org.sunbird.telemetry.logger.TelemetryManager @Singleton class ContentController @Inject()(@Named(ActorNames.CONTENT_ACTOR) contentActor: ActorRef, @Named(ActorNames.COLLECTION_ACTOR) collectionActor: ActorRef, cc: ControllerComponents, actorSystem: ActorSystem)(implicit exec: ExecutionContext) extends BaseController(cc) { diff --git a/platform-modules/mimetype-manager/src/main/scala/org/sunbird/mimetype/mgr/impl/HtmlMimeTypeMgrImpl.scala b/platform-modules/mimetype-manager/src/main/scala/org/sunbird/mimetype/mgr/impl/HtmlMimeTypeMgrImpl.scala index 833bc27ff..1ba192dec 100644 --- a/platform-modules/mimetype-manager/src/main/scala/org/sunbird/mimetype/mgr/impl/HtmlMimeTypeMgrImpl.scala +++ b/platform-modules/mimetype-manager/src/main/scala/org/sunbird/mimetype/mgr/impl/HtmlMimeTypeMgrImpl.scala @@ -30,6 +30,7 @@ class HtmlMimeTypeMgrImpl(implicit ss: StorageService) extends BaseMimeTypeManag TelemetryManager.error("ERR_INVALID_FILE" + "Please Provide Valid File! with file name: " + uploadFile.getName) throw new ClientException("ERR_INVALID_FILE", "Please Provide Valid File!") } + } override def upload(objectId: String, node: Node, fileUrl: String, filePath: Option[String], params: UploadParams)(implicit ec: ExecutionContext): Future[Map[String, AnyRef]] = {