From 4199250ec36e87718a42e3558fe31e54b71ad51f Mon Sep 17 00:00:00 2001 From: Utkarsha Kapoor Date: Thu, 5 Aug 2021 13:34:34 +0530 Subject: [PATCH 1/5] Issue #SB-24793: Response Exhaust V2 job to support assessment blob data --- .../collection/ResponseExhaustJobV2.scala | 110 ++++++++++++++++ .../exhaust/TestResponseExhaustJobV2.scala | 121 ++++++++++++++++++ 2 files changed, 231 insertions(+) create mode 100644 data-products/src/main/scala/org/sunbird/analytics/exhaust/collection/ResponseExhaustJobV2.scala create mode 100644 data-products/src/test/scala/org/sunbird/analytics/exhaust/TestResponseExhaustJobV2.scala diff --git a/data-products/src/main/scala/org/sunbird/analytics/exhaust/collection/ResponseExhaustJobV2.scala b/data-products/src/main/scala/org/sunbird/analytics/exhaust/collection/ResponseExhaustJobV2.scala new file mode 100644 index 000000000..21fcb0578 --- /dev/null +++ b/data-products/src/main/scala/org/sunbird/analytics/exhaust/collection/ResponseExhaustJobV2.scala @@ -0,0 +1,110 @@ +package org.sunbird.analytics.exhaust.collection + +import org.apache.spark.sql.expressions.UserDefinedFunction +import org.apache.spark.sql.functions.{col, udf, explode_outer, round} +import org.apache.spark.sql.types.StructType +import org.apache.spark.sql.{DataFrame, SparkSession} +import org.ekstep.analytics.framework.conf.AppConf +import org.ekstep.analytics.framework.util.{JSONUtils, JobLogger} +import org.ekstep.analytics.framework.{FrameworkContext, JobConfig} + +object ResponseExhaustJobV2 extends optional.Application with BaseCollectionExhaustJob { + + override def getClassName = "org.sunbird.analytics.exhaust.collection.ResponseExhaustJobV2" + override def jobName() = "ResponseExhaustJobV2"; + override def jobId() = "response-exhaust"; + override def getReportPath() = "response-exhaust/"; + override def getReportKey() = "response"; + private val partitionCols = List("batch_id", "year", "week_of_year") + + private val persistedDF:scala.collection.mutable.ListBuffer[DataFrame] = scala.collection.mutable.ListBuffer[DataFrame](); + + private val assessmentAggDBSettings = Map("table" -> "assessment_aggregator", "keyspace" -> AppConf.getConfig("sunbird.courses.keyspace"), "cluster" -> "LMSCluster"); + + private val filterColumns = Seq("courseid", "collectionName", "batchid", "batchName", "userid", "content_id", "contentname", "attempt_id", "last_attempted_on", "questionid", + "questiontype", "questiontitle", "questiondescription", "questionduration", "questionscore", "questionmaxscore", "questionoption", "questionresponse"); + private val columnsOrder = List("Collection Id", "Collection Name", "Batch Id", "Batch Name", "User UUID", "QuestionSet Id", "QuestionSet Title", "Attempt Id", "Attempted On", + "Question Id", "Question Type", "Question Title", "Question Description", "Question Duration", "Question Score", "Question Max Score", "Question Options", "Question Response"); + val columnMapping = Map("courseid" -> "Collection Id", "collectionName" -> "Collection Name", "batchid" -> "Batch Id", "batchName" -> "Batch Name", "userid" -> "User UUID", + "content_id" -> "QuestionSet Id", "contentname" -> "QuestionSet Title", "attempt_id" -> "Attempt Id", "last_attempted_on" -> "Attempted On", "questionid" -> "Question Id", + "questiontype" -> "Question Type", "questiontitle" -> "Question Title", "questiondescription" -> "Question Description", "questionduration" -> "Question Duration", + "questionscore" -> "Question Score", "questionmaxscore" -> "Question Max Score", "questionoption" -> "Question Options", "questionresponse" -> "Question Response") + + case class AssessmentData(user_id: String, course_id: String,batch_id: String,content_id: String, attempt_id: String,created_on: Option[String],grand_total: Option[String],last_attempted_on: Option[String],question: List[Question],total_max_score: Option[Double],total_score: Option[Double],updated_on: Option[String]) extends scala.Product with scala.Serializable + case class Question(id: String, assess_ts: String,max_score: Double, score: Double,`type`: String,title: String,resvalues: List[Map[String, String]],params: List[Map[String, String]],description: String,duration: Double) extends scala.Product with scala.Serializable + + override def processBatch(userEnrolmentDF: DataFrame, collectionBatch: CollectionBatch)(implicit spark: SparkSession, fc: FrameworkContext, config: JobConfig): DataFrame = { + val assessmentDF = getAssessmentDF(userEnrolmentDF, collectionBatch).persist(); + persistedDF.append(assessmentDF); + val contentIds = assessmentDF.select("content_id").dropDuplicates().collect().map(f => f.get(0)); + val contentDF = searchContent(Map("request" -> Map("filters" -> Map("identifier" -> contentIds)))).withColumnRenamed("collectionName", "contentname").select("identifier", "contentname"); + val reportDF = assessmentDF.join(contentDF, assessmentDF("content_id") === contentDF("identifier"), "left_outer").drop("identifier").select(filterColumns.head, filterColumns.tail: _*); + organizeDF(reportDF, columnMapping, columnsOrder); + } + + override def unpersistDFs() { + persistedDF.foreach(f => f.unpersist(true)) + } + + def getAssessmentDF(userEnrolmentDF: DataFrame, batch: CollectionBatch)(implicit spark: SparkSession, fc: FrameworkContext, config: JobConfig): DataFrame = { + + val userEnrolmentDataDF = userEnrolmentDF + .select( + col("userid"), + col("courseid"), + col("collectionName"), + col("batchName"), + col("batchid")) + + val batchid = userEnrolmentDataDF.select("batchid").distinct().collect().head.getString(0) + + val assessBlobData = getAssessmentBlobDF(batchid, config) + + val assessAggregateData = loadData(assessmentAggDBSettings, cassandraFormat, new StructType()) + + val joinedDF = assessAggregateData.join(assessBlobData, Seq("batch_id", "course_id", "user_id"), "left_outer") + .select(assessAggregateData.col("*")) + + joinedDF.join(userEnrolmentDataDF, joinedDF.col("user_id") === userEnrolmentDataDF.col("userid") && joinedDF.col("course_id") === userEnrolmentDataDF.col("courseid"), "inner") + .select(userEnrolmentDataDF.col("*"), joinedDF.col("question"), col("content_id"), col("attempt_id"), col("last_attempted_on")) + .withColumn("questiondata",explode_outer(col("question"))) + .withColumn("questionid" , col("questiondata.id")) + .withColumn("questiontype", col("questiondata.type")) + .withColumn("questiontitle", col("questiondata.title")) + .withColumn("questiondescription", col("questiondata.description")) + .withColumn("questionduration", round(col("questiondata.duration"))) + .withColumn("questionscore", col("questiondata.score")) + .withColumn("questionmaxscore", col("questiondata.max_score")) + .withColumn("questionresponse", UDFUtils.toJSON(col("questiondata.resvalues"))) + .withColumn("questionoption", UDFUtils.toJSON(col("questiondata.params"))) + .drop("question", "questiondata", "question_data") + } + + def getAssessmentBlobDF(batchid: String, config: JobConfig)(implicit spark: SparkSession): DataFrame = { + val azureFetcherConfig = config.modelParams.get("assessmentFetcherConfig").asInstanceOf[Map[String, AnyRef]] + + val store = azureFetcherConfig("store") + val format:String = azureFetcherConfig.getOrElse("format", "csv").asInstanceOf[String] + val url = store match { + case "local" => + val filePath = azureFetcherConfig("filePath").asInstanceOf[String] + filePath + s"${batchid}-*.${format}" + case "s3" | "azure" => + val bucket = azureFetcherConfig("container").asInstanceOf[String] + val key = AppConf.getConfig("azure_storage_key") + val filePath = azureFetcherConfig.getOrElse("filePath", "data-archival/").asInstanceOf[String] + val file = s"${filePath}${batchid}-*.${format}" + s"wasb://$bucket@$key.blob.core.windows.net/$file." + } + JobLogger.log(s"Fetching data from ${store} for batchid: " + batchid) + + val assessAggData = spark.read.format("csv") + .option("header","true") + .load(url) + + assessAggData.withColumn("question", extractFromArrayStringFun(col("question"))) + } + + def extractFromArrayStringFun: UserDefinedFunction = + udf { str: String => JSONUtils.deserialize[List[Question]](str) } +} diff --git a/data-products/src/test/scala/org/sunbird/analytics/exhaust/TestResponseExhaustJobV2.scala b/data-products/src/test/scala/org/sunbird/analytics/exhaust/TestResponseExhaustJobV2.scala new file mode 100644 index 000000000..01bb6ad9a --- /dev/null +++ b/data-products/src/test/scala/org/sunbird/analytics/exhaust/TestResponseExhaustJobV2.scala @@ -0,0 +1,121 @@ +package org.sunbird.analytics.exhaust + +import org.apache.spark.sql.{Encoders, SparkSession} +import org.ekstep.analytics.framework.conf.AppConf +import org.ekstep.analytics.framework.util.{HadoopFileUtil, JSONUtils} +import org.ekstep.analytics.framework.{FrameworkContext, JobConfig} +import org.joda.time.DateTimeZone +import org.joda.time.format.{DateTimeFormat, DateTimeFormatter} +import org.scalamock.scalatest.MockFactory +import org.sunbird.analytics.exhaust.collection.ResponseExhaustJobV2 +import org.sunbird.analytics.job.report.BaseReportSpec +import org.sunbird.analytics.util.{EmbeddedCassandra, EmbeddedPostgresql, RedisConnect} +import redis.clients.jedis.Jedis +import redis.embedded.RedisServer + +import scala.collection.JavaConverters._ + +class TestResponseExhaustJobV2 extends BaseReportSpec with MockFactory with BaseReportsJob { + + val jobRequestTable = "job_request" + implicit var spark: SparkSession = _ + var redisServer: RedisServer = _ + var jedis: Jedis = _ + val outputLocation = AppConf.getConfig("collection.exhaust.store.prefix") + + override def beforeAll(): Unit = { + super.beforeAll() + spark = getSparkSession(); + + redisServer = new RedisServer(6341) + redisServer.start() + setupRedisData() + EmbeddedCassandra.loadData("src/test/resources/exhaust/report_data.cql") // Load test data in embedded cassandra server + EmbeddedPostgresql.start() + EmbeddedPostgresql.createJobRequestTable() + } + + override def afterAll() : Unit = { + super.afterAll() + new HadoopFileUtil().delete(spark.sparkContext.hadoopConfiguration, outputLocation) + redisServer.stop() + spark.close() + + EmbeddedCassandra.close() + EmbeddedPostgresql.close() + } + + def setupRedisData(): Unit = { + val redisConnect = new RedisConnect("localhost", 6341) + val jedis = redisConnect.getConnection(0, 100000) + jedis.hmset("user:user-001", JSONUtils.deserialize[java.util.Map[String, String]]("""{"firstname": "Manju", "userid": "user-001", "state": "Karnataka", "district": "bengaluru", "userchannel": "sunbird-dev", "rootorgid": "01250894314817126443", "email": "manju@ilimi.in", "usersignintype": "Validated"};""")) + jedis.hmset("user:user-002", JSONUtils.deserialize[java.util.Map[String, String]]("""{"firstname": "Mahesh", "userid": "user-002", "state": "Andhra Pradesh", "district": "bengaluru", "userchannel": "sunbird-dev", "rootorgid": "01285019302823526477", "email": "mahesh@ilimi.in", "usersignintype": "Validated"};""")) + jedis.hmset("user:user-003", JSONUtils.deserialize[java.util.Map[String, String]]("""{"firstname": "Sowmya", "userid": "user-003", "state": "Karnataka", "district": "bengaluru", "userchannel": "sunbird-dev", "rootorgid": "01250894314817126443", "email": "sowmya@ilimi.in", "usersignintype": "Validated"};""")) + jedis.hmset("user:user-004", JSONUtils.deserialize[java.util.Map[String, String]]("""{"firstname": "Utkarsha", "userid": "user-004", "state": "Delhi", "district": "babarpur", "userchannel": "sunbird-dev", "rootorgid": "01250894314817126443", "email": "utkarsha@ilimi.in", "usersignintype": "Validated"};""")) + jedis.hmset("user:user-005", JSONUtils.deserialize[java.util.Map[String, String]]("""{"firstname": "Isha", "userid": "user-005", "state": "MP", "district": "Jhansi", "userchannel": "sunbird-dev", "rootorgid": "01250894314817126443", "email": "isha@ilimi.in", "usersignintype": "Validated"};""")) + jedis.hmset("user:user-006", JSONUtils.deserialize[java.util.Map[String, String]]("""{"firstname": "Revathi", "userid": "user-006", "state": "Andhra Pradesh", "district": "babarpur", "userchannel": "sunbird-dev", "rootorgid": "01250894314817126443", "email": "revathi@ilimi.in", "usersignintype": "Validated"};""")) + jedis.hmset("user:user-007", JSONUtils.deserialize[java.util.Map[String, String]]("""{"firstname": "Sunil", "userid": "user-007", "state": "Karnataka", "district": "bengaluru", "userchannel": "sunbird-dev", "rootorgid": "0126391644091351040", "email": "sunil@ilimi.in", "usersignintype": "Validated"};""")) + jedis.hmset("user:user-008", JSONUtils.deserialize[java.util.Map[String, String]]("""{"firstname": "Anoop", "userid": "user-008", "state": "Karnataka", "district": "bengaluru", "userchannel": "sunbird-dev", "rootorgid": "0130107621805015045", "email": "anoop@ilimi.in", "usersignintype": "Validated"};""")) + jedis.hmset("user:user-009", JSONUtils.deserialize[java.util.Map[String, String]]("""{"firstname": "Kartheek", "userid": "user-009", "state": "Karnataka", "district": "bengaluru", "userchannel": "sunbird-dev", "rootorgid": "01285019302823526477", "email": "kartheekp@ilimi.in", "usersignintype": "Validated"};""")) + jedis.hmset("user:user-010", JSONUtils.deserialize[java.util.Map[String, String]]("""{"firstname": "Anand", "userid": "user-010", "state": "Tamil Nadu", "district": "Chennai", "userchannel": "sunbird-dev", "rootorgid": "0130107621805015045", "email": "anandp@ilimi.in", "usersignintype": "Validated"};""")) + jedis.close() + } + + "TestResponseExhaustJob" should "generate final output as csv and zip files" in { + EmbeddedPostgresql.execute(s"TRUNCATE $jobRequestTable") + EmbeddedPostgresql.execute("INSERT INTO job_request (tag, request_id, job_id, status, request_data, requested_by, requested_channel, dt_job_submitted, download_urls, dt_file_created, dt_job_completed, execution_time, err_message ,iteration) VALUES ('do_1131350140968632321230_batch-001:01250894314817126443', '37564CF8F134EE7532F125651B51D17F', 'response-exhaust', 'SUBMITTED', '{\"batchId\": \"batch-001\"}', 'user-002', 'b00bc992ef25f1a9a8d63291e20efc8d', '2020-10-19 05:58:18.666', '{}', NULL, NULL, 0, '' ,0);") + + implicit val fc = new FrameworkContext() + val strConfig = """{"search":{"type":"none"},"model":"org.sunbird.analytics.exhaust.collection.ResponseExhaustJobV2","modelParams":{"store":"local","mode":"OnDemand","batchFilters":["TPD"],"assessmentFetcherConfig":{"store":"local","filePath":"src/test/resources/exhaust/data-archival/"},"searchFilter":{},"sparkElasticsearchConnectionHost":"localhost","sparkRedisConnectionHost":"localhost","sparkUserDbRedisIndex":"0","sparkUserDbRedisPort":6341,"sparkCassandraConnectionHost":"localhost","fromDate":"","toDate":"","storageContainer":""},"parallelization":8,"appName":"ResponseExhaustJob Exhaust"}""" + val jobConfig = JSONUtils.deserialize[JobConfig](strConfig) + implicit val config = jobConfig + + ResponseExhaustJobV2.execute() + + val outputDir = "response-exhaust" + val batch1 = "batch-001" + val requestId = "37564CF8F134EE7532F125651B51D17F" + val filePath = ResponseExhaustJobV2.getFilePath(batch1, requestId) + val jobName = ResponseExhaustJobV2.jobName() + + implicit val responseExhaustEncoder = Encoders.product[ResponseExhaustReport] + val batch1Results = spark.read.format("csv").option("header", "true") + .load(s"$outputLocation/$filePath.csv").as[ResponseExhaustReport].collectAsList().asScala + batch1Results.size should be (6) + + val user1Result = batch1Results.filter(f => f.`User UUID`.equals("user-001")) + user1Result.foreach(f => println("user1data: " + JSONUtils.serialize("f"))) + user1Result.map(f => f.`Collection Id`).toList should contain atLeastOneElementOf List("do_1130928636168192001667") + user1Result.map(f => f.`Batch Id`).toList should contain atLeastOneElementOf List("BatchId_batch-001") + user1Result.map(f => f.`User UUID`).toList should contain atLeastOneElementOf List("user-001") + user1Result.map(f => f.`Attempt Id`).toList should contain atLeastOneElementOf List("attempat-001") + user1Result.map(f => f.`QuestionSet Id`).toList should contain atLeastOneElementOf List("do_1128870328040161281204", "do_112876961957437440179") + user1Result.map(f => f.`QuestionSet Title`).toList should contain atLeastOneElementOf List("SelfAssess for course", "Assessment score report using summary plugin") + user1Result.map(f => f.`Question Id`).toList should contain theSameElementsAs List("do_213019475454476288155", "do_213019970118279168165", "do_213019972814823424168", "do_2130256513760624641171") + user1Result.map(f => f.`Question Type`).toList should contain theSameElementsAs List("mcq", "mcq", "mtf", "mcq") + user1Result.map(f => f.`Question Title`).toList should contain atLeastOneElementOf List("testQuestiontextandformula", "test with formula") + user1Result.map(f => f.`Question Description`).toList should contain atLeastOneElementOf List("testQuestiontextandformula") + user1Result.map(f => f.`Question Duration`).toList should contain theSameElementsAs List("1", "1", "2", "12") + user1Result.map(f => f.`Question Score`).toList should contain theSameElementsAs List("1.0", "1.0", "0.33", "10.0") + user1Result.map(f => f.`Question Max Score`).toList should contain theSameElementsAs List("1.0", "1.0", "1.0", "10.0") + user1Result.map(f => f.`Question Options`).head should be ("""[{'1':'{'text':'A=pi r^2'}'},{'2':'{'text':'no'}'},{'answer':'{'correct':['1']}'}]""") + user1Result.map(f => f.`Question Response`).head should be ("""[{'1':'{'text':'A=pi r^2'}'}]""") + + val pResponse = EmbeddedPostgresql.executeQuery("SELECT * FROM job_request WHERE job_id='response-exhaust'") + + while(pResponse.next()) { + pResponse.getString("err_message") should be ("") + pResponse.getString("dt_job_submitted") should be ("2020-10-19 05:58:18.666") + pResponse.getString("download_urls") should be (s"{reports/response-exhaust/$requestId/batch-001_response_${getDate()}.zip}") + pResponse.getString("dt_file_created") should be (null) + pResponse.getString("iteration") should be ("0") + } + + } + + def getDate(): String = { + val dateFormat: DateTimeFormatter = DateTimeFormat.forPattern("yyyyMMdd").withZone(DateTimeZone.forOffsetHoursMinutes(5, 30)); + dateFormat.print(System.currentTimeMillis()); + } + +} From e34dc4a0438a3a5dc74fab160a9440ba0121e982 Mon Sep 17 00:00:00 2001 From: Utkarsha Kapoor Date: Thu, 5 Aug 2021 13:44:32 +0530 Subject: [PATCH 2/5] Issue #SB-24793: test files added --- .../exhaust/data-archival/batch-001-2021-22-1626351011290.csv | 3 +++ .../exhaust/data-archival/batch-001-2021-28-1626351011290.csv | 3 +++ .../exhaust/data-archival/batch-004-2021-28-1626337270508.csv | 2 ++ 3 files changed, 8 insertions(+) create mode 100644 data-products/src/test/resources/exhaust/data-archival/batch-001-2021-22-1626351011290.csv create mode 100644 data-products/src/test/resources/exhaust/data-archival/batch-001-2021-28-1626351011290.csv create mode 100644 data-products/src/test/resources/exhaust/data-archival/batch-004-2021-28-1626337270508.csv diff --git a/data-products/src/test/resources/exhaust/data-archival/batch-001-2021-22-1626351011290.csv b/data-products/src/test/resources/exhaust/data-archival/batch-001-2021-22-1626351011290.csv new file mode 100644 index 000000000..ea682a131 --- /dev/null +++ b/data-products/src/test/resources/exhaust/data-archival/batch-001-2021-22-1626351011290.csv @@ -0,0 +1,3 @@ +content_id,attempt_id,user_id,course_id,batch_id,created_on,last_attempted_on,total_max_score,total_score,updated_on,grand_total,question +do_1128870328040161281204,attempt-001,user-001,do_1130928636168192001667,batch-001,null,null,20,20,2021-06-15 5:30:00,20,"[{\"id\":\"do_31300323721809100819622\",\"assess_ts\":\"2021-02-16T18:51:54.104Z\",\"max_score\":1.0,\"score\":1.0,\"type\":\"mcq\",\"title\":\"Pediatric Concerns during COVID-19\",\"resvalues\":[{\"1\":\"{\\\"text\\\":\\\"A. Oliguria\\\\n\\\"}\"}],\"params\":[{\"1\":\"{\\\"text\\\":\\\"A. Oliguria\\\\n\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"B. Mild pneumonia\\\\n\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"C. Severe Pneumonia\\\\n\\\"}\"},{\"4\":\"{\\\"text\\\":\\\"D. Panic Attack\\\\n\\\"}\"},{\"answer\":\"{\\\"correct\\\":[\\\"1\\\"]}\"}],\"description\":\"\",\"duration\":17.000000000000000000},{\"id\":\"do_31300323406350745619621\",\"assess_ts\":\"2021-02-16T18:52:36.636Z\",\"max_score\":1.0,\"score\":1.0,\"type\":\"mcq\",\"title\":\"Pediatric Concerns during COVID-19\",\"resvalues\":[{\"4\":\"{\\\"text\\\":\\\"D. Mild Pneumonia\\\\n\\\"}\"}],\"params\":[{\"1\":\"{\\\"text\\\":\\\"A. Unstable angina pectoris\\\\n\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"B. Myocardial infarction\\\\n\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"C. Central cord syndrome\\\\n\\\"}\"},{\"4\":\"{\\\"text\\\":\\\"D. Mild Pneumonia\\\\n\\\"}\"},{\"answer\":\"{\\\"correct\\\":[\\\"4\\\"]}\"}],\"description\":\"\",\"duration\":11.000000000000000000},{\"id\":\"do_313003240076902400111588\",\"assess_ts\":\"2021-02-16T18:51:17.607Z\",\"max_score\":1.0,\"score\":1.0,\"type\":\"mcq\",\"title\":\"Pediatric Concerns during COVID-19\",\"resvalues\":[{\"1\":\"{\\\"text\\\":\\\"A. Pre-oxygenate with 100% FiO2 for 5 minutes, via a face mask with reservoir bag or HFNO and take care of aerosolization during the procedure\\\\n\\\"}\"}],\"params\":[{\"1\":\"{\\\"text\\\":\\\"A. Pre-oxygenate with 100% FiO2 for 5 minutes, via a face mask with reservoir bag or HFNO and take care of aerosolization during the procedure\\\\n\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"B. Conservative Fluid Management\\\\n\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"C. Oxygen Therapy\\\\n\\\"}\"},{\"4\":\"{\\\"text\\\":\\\"D. Lopinavir Therapy\\\\n\\\"}\"},{\"answer\":\"{\\\"correct\\\":[\\\"1\\\"]}\"}],\"description\":\"\",\"duration\":12.000000000000000000},{\"id\":\"do_313003232670449664111583\",\"assess_ts\":\"2021-02-16T18:52:47.423Z\",\"max_score\":1.0,\"score\":1.0,\"type\":\"mcq\",\"title\":\"Pediatric Concerns during COVID-19\",\"resvalues\":[{\"1\":\"{\\\"text\\\":\\\"A. Persistent pain or pressure in chest\\\\n\\\"}\"}],\"params\":[{\"1\":\"{\\\"text\\\":\\\"A. Persistent pain or pressure in chest\\\\n\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"B. Panic Attack\\\\n\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"C. Stomach ache\\\\n\\\"}\"},{\"4\":\"{\\\"text\\\":\\\"D. Headache\\\\n\\\"}\"},{\"answer\":\"{\\\"correct\\\":[\\\"1\\\"]}\"}],\"description\":\"\",\"duration\":10.000000000000000000},{\"id\":\"do_313003239006068736111587\",\"assess_ts\":\"2021-02-16T18:51:36.282Z\",\"max_score\":1.0,\"score\":1.0,\"type\":\"mcq\",\"title\":\"Pediatric Concerns during COVID-19\",\"resvalues\":[{\"3\":\"{\\\"text\\\":\\\"C. Oxygen Therapy\\\\n\\\"}\"}],\"params\":[{\"1\":\"{\\\"text\\\":\\\"A. Metformin therapy\\\\n\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"B. Lopinavir therapy\\\\n\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"C. Oxygen Therapy\\\\n\\\"}\"},{\"4\":\"{\\\"text\\\":\\\"D. None of the above\\\\n\\\"}\"},{\"answer\":\"{\\\"correct\\\":[\\\"3\\\"]}\"}],\"description\":\"\",\"duration\":17.000000000000000000},{\"id\":\"do_313003236553457664111586\",\"assess_ts\":\"2021-02-16T18:52:11.410Z\",\"max_score\":1.0,\"score\":1.0,\"type\":\"mcq\",\"title\":\"Pediatric Concerns during COVID-19\",\"resvalues\":[{\"4\":\"{\\\"text\\\":\\\"D. All of the above\\\\n\\\"}\"}],\"params\":[{\"1\":\"{\\\"text\\\":\\\"A. Tachycardia\\\\n\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"B. Tachypnea\\\\n\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"C. Fever or Hypothermia\\\\n\\\"}\"},{\"4\":\"{\\\"text\\\":\\\"D. All of the above\\\\n\\\"}\"},{\"answer\":\"{\\\"correct\\\":[\\\"4\\\"]}\"}],\"description\":\"\",\"duration\":16.000000000000000000},{\"id\":\"do_313003241437011968111589\",\"assess_ts\":\"2021-02-16T18:50:49.472Z\",\"max_score\":1.0,\"score\":1.0,\"type\":\"mcq\",\"title\":\"Pediatric Concerns during COVID-19\",\"resvalues\":[{\"3\":\"{\\\"text\\\":\\\"C. 4 hours\\\\n\\\"}\"}],\"params\":[{\"1\":\"{\\\"text\\\":\\\"A. 12 hours\\\\n\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"B. 8 hours\\\\n\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"C. 4 hours\\\\n\\\"}\"},{\"4\":\"{\\\"text\\\":\\\"D. 24 hours\\\\n\\\"}\"},{\"answer\":\"{\\\"correct\\\":[\\\"3\\\"]}\"}],\"description\":\"\",\"duration\":48.000000000000000000},{\"id\":\"do_313003235789389824111585\",\"assess_ts\":\"2021-02-16T18:52:17.543Z\",\"max_score\":1.0,\"score\":1.0,\"type\":\"mcq\",\"title\":\"Pediatric Concerns during COVID-19\",\"resvalues\":[{\"3\":\"{\\\"text\\\":\\\"C. Both A and B\\\\n\\\"}\"}],\"params\":[{\"1\":\"{\\\"text\\\":\\\"A. Oxygenation Index\\\\n\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"B. Oxygenation Index using SpO2\\\\n\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"C. Both A and B\\\\n\\\"}\"},{\"4\":\"{\\\"text\\\":\\\"D. None of the above\\\\n\\\"}\"},{\"answer\":\"{\\\"correct\\\":[\\\"3\\\"]}\"}],\"description\":\"\",\"duration\":5.000000000000000000},{\"id\":\"do_31300324092397158419623\",\"assess_ts\":\"2021-02-16T18:51:04.495Z\",\"max_score\":1.0,\"score\":1.0,\"type\":\"mcq\",\"title\":\"Pediatric Concerns during COVID-19\",\"resvalues\":[{\"2\":\"{\\\"text\\\":\\\"B. Vasopressors during or after fluid resuscitation\\\\n\\\"}\"}],\"params\":[{\"1\":\"{\\\"text\\\":\\\"A. Hypotonic crystalloids resuscitation\\\\n\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"B. Vasopressors during or after fluid resuscitation\\\\n\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"C. Oxygen Therapy\\\\n\\\"}\"},{\"4\":\"{\\\"text\\\":\\\"D. Rapid Sequence Intubation\\\\n\\\"}\"},{\"answer\":\"{\\\"correct\\\":[\\\"2\\\"]}\"}],\"description\":\"\",\"duration\":13.000000000000000000},{\"id\":\"do_313003234935881728111584\",\"assess_ts\":\"2021-02-16T18:52:24.796Z\",\"max_score\":1.0,\"score\":1.0,\"type\":\"mcq\",\"title\":\"Pediatric Concerns during COVID-19\",\"resvalues\":[{\"4\":\"{\\\"text\\\":\\\"D. All of the above\\\\n\\\"}\"}],\"params\":[{\"1\":\"{\\\"text\\\":\\\"A. Cough or difficulty in breathing including central cyanosis or spo2<90%\\\\n\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"B. Severe respiratory distress (grunting, chest indrawing)\\\\n\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"C. Signs of pneumonia along with convulsions\\\\n\\\"}\"},{\"4\":\"{\\\"text\\\":\\\"D. All of the above\\\\n\\\"}\"},{\"answer\":\"{\\\"correct\\\":[\\\"4\\\"]}\"}],\"description\":\"\",\"duration\":6.000000000000000000}]" +do_112876961957437440179,attempt-001,user-003,do_1130928636168192001667,batch-001,null,null,10,10,2021-06-15 5:30:00,10,[] \ No newline at end of file diff --git a/data-products/src/test/resources/exhaust/data-archival/batch-001-2021-28-1626351011290.csv b/data-products/src/test/resources/exhaust/data-archival/batch-001-2021-28-1626351011290.csv new file mode 100644 index 000000000..665fd1a2d --- /dev/null +++ b/data-products/src/test/resources/exhaust/data-archival/batch-001-2021-28-1626351011290.csv @@ -0,0 +1,3 @@ +content_id,attempt_id,user_id,course_id,batch_id,created_on,last_attempted_on,total_max_score,total_score,updated_on,grand_total,question +do_112876961957437440179,attempt-001,user-008,do_11306040245271756813015,batch-001,null,null,10,10,2021-07-15 5:30:00,10,[] +do_11307593493010022418,attempt-001,user-010,do_11306040245271756813015,batch-001,null,null,15,15,2021-07-15 5:30:00,15,[] \ No newline at end of file diff --git a/data-products/src/test/resources/exhaust/data-archival/batch-004-2021-28-1626337270508.csv b/data-products/src/test/resources/exhaust/data-archival/batch-004-2021-28-1626337270508.csv new file mode 100644 index 000000000..49e3de7d4 --- /dev/null +++ b/data-products/src/test/resources/exhaust/data-archival/batch-004-2021-28-1626337270508.csv @@ -0,0 +1,2 @@ +content_id,attempt_id,user_id,course_id,batch_id,created_on,last_attempted_on,total_max_score,total_score,updated_on,grand_total,question +do_11307593493010022418,attempat-001,user-014,do_112835334818643968148,batch-004,null,null,15,15,2021-07-15 5:30:00,15,[] \ No newline at end of file From f8a29472941b779883d153f655286589b2ecc52c Mon Sep 17 00:00:00 2001 From: Utkarsha Kapoor Date: Tue, 10 Aug 2021 10:50:47 +0530 Subject: [PATCH 3/5] Issue #SB-24793: Review Comments resolved --- .../collection/BaseCollectionExhaustJob.scala | 8 ++- .../collection/ResponseExhaustJobV2.scala | 49 ++++++++----------- .../analytics/exhaust/util/ExhaustUtil.scala | 30 ++++++++++++ .../data-archival/archival_report_data.cql | 30 ++++++++++++ .../batch-001-2021-22-1626351011290.csv | 5 +- .../exhaust/TestResponseExhaustJobV2.scala | 14 +++++- 6 files changed, 103 insertions(+), 33 deletions(-) create mode 100644 data-products/src/main/scala/org/sunbird/analytics/exhaust/util/ExhaustUtil.scala create mode 100644 data-products/src/test/resources/exhaust/data-archival/archival_report_data.cql diff --git a/data-products/src/main/scala/org/sunbird/analytics/exhaust/collection/BaseCollectionExhaustJob.scala b/data-products/src/main/scala/org/sunbird/analytics/exhaust/collection/BaseCollectionExhaustJob.scala index 9e2d1b786..a482af806 100644 --- a/data-products/src/main/scala/org/sunbird/analytics/exhaust/collection/BaseCollectionExhaustJob.scala +++ b/data-products/src/main/scala/org/sunbird/analytics/exhaust/collection/BaseCollectionExhaustJob.scala @@ -18,10 +18,13 @@ import org.joda.time.format.{DateTimeFormat, DateTimeFormatter} import org.joda.time.{DateTime, DateTimeZone} import org.sunbird.analytics.exhaust.{BaseReportsJob, JobRequest, OnDemandExhaustJob} import org.sunbird.analytics.util.DecryptUtil - import java.security.MessageDigest import java.util.concurrent.CompletableFuture import java.util.concurrent.atomic.AtomicInteger + +import org.apache.spark.sql.expressions.UserDefinedFunction +import org.sunbird.analytics.exhaust.collection.ResponseExhaustJobV2.Question + import scala.collection.immutable.List import scala.collection.mutable.ListBuffer @@ -543,4 +546,7 @@ object UDFUtils extends Serializable { } val getLatestValue = udf[String, String, String](getLatestValueFun) + + def convertStringToList: UserDefinedFunction = + udf { str: String => JSONUtils.deserialize[List[Question]](str) } } \ No newline at end of file diff --git a/data-products/src/main/scala/org/sunbird/analytics/exhaust/collection/ResponseExhaustJobV2.scala b/data-products/src/main/scala/org/sunbird/analytics/exhaust/collection/ResponseExhaustJobV2.scala index 21fcb0578..83c986a3d 100644 --- a/data-products/src/main/scala/org/sunbird/analytics/exhaust/collection/ResponseExhaustJobV2.scala +++ b/data-products/src/main/scala/org/sunbird/analytics/exhaust/collection/ResponseExhaustJobV2.scala @@ -1,12 +1,12 @@ package org.sunbird.analytics.exhaust.collection -import org.apache.spark.sql.expressions.UserDefinedFunction -import org.apache.spark.sql.functions.{col, udf, explode_outer, round} +import org.apache.spark.sql.functions.{col, explode_outer, round} import org.apache.spark.sql.types.StructType import org.apache.spark.sql.{DataFrame, SparkSession} import org.ekstep.analytics.framework.conf.AppConf -import org.ekstep.analytics.framework.util.{JSONUtils, JobLogger} +import org.ekstep.analytics.framework.util.JobLogger import org.ekstep.analytics.framework.{FrameworkContext, JobConfig} +import org.sunbird.analytics.exhaust.util.ExhaustUtil object ResponseExhaustJobV2 extends optional.Application with BaseCollectionExhaustJob { @@ -15,7 +15,6 @@ object ResponseExhaustJobV2 extends optional.Application with BaseCollectionExha override def jobId() = "response-exhaust"; override def getReportPath() = "response-exhaust/"; override def getReportKey() = "response"; - private val partitionCols = List("batch_id", "year", "week_of_year") private val persistedDF:scala.collection.mutable.ListBuffer[DataFrame] = scala.collection.mutable.ListBuffer[DataFrame](); @@ -58,12 +57,18 @@ object ResponseExhaustJobV2 extends optional.Application with BaseCollectionExha val batchid = userEnrolmentDataDF.select("batchid").distinct().collect().head.getString(0) - val assessBlobData = getAssessmentBlobDF(batchid, config) - val assessAggregateData = loadData(assessmentAggDBSettings, cassandraFormat, new StructType()) - val joinedDF = assessAggregateData.join(assessBlobData, Seq("batch_id", "course_id", "user_id"), "left_outer") - .select(assessAggregateData.col("*")) + val joinedDF = try { + val assessBlobData = getAssessmentBlobDF(batchid, config) + + val joinDF = assessAggregateData.join(assessBlobData, Seq("batch_id", "course_id", "user_id"), "left") + .select(assessAggregateData.col("*")) + joinDF + } catch { + case e => JobLogger.log("Blob does not contain any file for batchid: " + batchid) + assessAggregateData + } joinedDF.join(userEnrolmentDataDF, joinedDF.col("user_id") === userEnrolmentDataDF.col("userid") && joinedDF.col("course_id") === userEnrolmentDataDF.col("courseid"), "inner") .select(userEnrolmentDataDF.col("*"), joinedDF.col("question"), col("content_id"), col("attempt_id"), col("last_attempted_on")) @@ -80,31 +85,17 @@ object ResponseExhaustJobV2 extends optional.Application with BaseCollectionExha .drop("question", "questiondata", "question_data") } - def getAssessmentBlobDF(batchid: String, config: JobConfig)(implicit spark: SparkSession): DataFrame = { + def getAssessmentBlobDF(batchid: String, config: JobConfig)(implicit spark: SparkSession, fc: FrameworkContext): DataFrame = { val azureFetcherConfig = config.modelParams.get("assessmentFetcherConfig").asInstanceOf[Map[String, AnyRef]] - val store = azureFetcherConfig("store") + val store = azureFetcherConfig("store").asInstanceOf[String] val format:String = azureFetcherConfig.getOrElse("format", "csv").asInstanceOf[String] - val url = store match { - case "local" => - val filePath = azureFetcherConfig("filePath").asInstanceOf[String] - filePath + s"${batchid}-*.${format}" - case "s3" | "azure" => - val bucket = azureFetcherConfig("container").asInstanceOf[String] - val key = AppConf.getConfig("azure_storage_key") - val filePath = azureFetcherConfig.getOrElse("filePath", "data-archival/").asInstanceOf[String] - val file = s"${filePath}${batchid}-*.${format}" - s"wasb://$bucket@$key.blob.core.windows.net/$file." - } - JobLogger.log(s"Fetching data from ${store} for batchid: " + batchid) + val filePath = azureFetcherConfig.getOrElse("filePath", "data-archival/").asInstanceOf[String] + val container = azureFetcherConfig.getOrElse("container", "reports").asInstanceOf[String] - val assessAggData = spark.read.format("csv") - .option("header","true") - .load(url) + val assessAggData = ExhaustUtil.getAssessmentBlobData(store, filePath, container, Option(batchid), Option(format)) - assessAggData.withColumn("question", extractFromArrayStringFun(col("question"))) + assessAggData.distinct() + .withColumn("question", UDFUtils.convertStringToList(col("question"))) } - - def extractFromArrayStringFun: UserDefinedFunction = - udf { str: String => JSONUtils.deserialize[List[Question]](str) } } diff --git a/data-products/src/main/scala/org/sunbird/analytics/exhaust/util/ExhaustUtil.scala b/data-products/src/main/scala/org/sunbird/analytics/exhaust/util/ExhaustUtil.scala new file mode 100644 index 000000000..03e2184ac --- /dev/null +++ b/data-products/src/main/scala/org/sunbird/analytics/exhaust/util/ExhaustUtil.scala @@ -0,0 +1,30 @@ +package org.sunbird.analytics.exhaust.util + + +import org.apache.spark.sql.{DataFrame, SparkSession} +import org.ekstep.analytics.framework.FrameworkContext +import org.ekstep.analytics.framework.conf.AppConf +import org.ekstep.analytics.framework.util.JobLogger + +object ExhaustUtil { + + def getAssessmentBlobData(store: String, filePath: String, bucket: String, batchId: Option[String], fileFormat: Option[String])(implicit spark: SparkSession, fc: FrameworkContext): DataFrame = { + val format = fileFormat.getOrElse("csv") + val batchid = batchId.getOrElse("") + + val url = store match { + case "local" => + filePath + s"${batchid}-*.${format}" + case "s3" | "azure" => + val key = AppConf.getConfig("azure_storage_key") + val file = s"${filePath}${batchid}-*.${format}" + s"wasb://$bucket@$key.blob.core.windows.net/$file." + } + + JobLogger.log(s"Fetching data from ${store} for batchid: " + batchid)(new String()) + + spark.read.format("csv") + .option("header","true") + .load(url) + } +} diff --git a/data-products/src/test/resources/exhaust/data-archival/archival_report_data.cql b/data-products/src/test/resources/exhaust/data-archival/archival_report_data.cql new file mode 100644 index 000000000..ff1319a8d --- /dev/null +++ b/data-products/src/test/resources/exhaust/data-archival/archival_report_data.cql @@ -0,0 +1,30 @@ +INSERT INTO sunbird_courses.course_batch (courseid, batchid, createdby, createddate, enddate, end_date, enrollmentenddate, name, startdate, start_date, status) VALUES ('do_1130928636168192001667', 'batch-001', 'manju', '2020-10-10', '2020-01-10', '2020-01-10', '2020-01-01', 'Basic Java', '2020-01-20', '2020-01-20', 1 ); +INSERT INTO sunbird_courses.course_batch (courseid, batchid, createdby, createddate, enddate, end_date, enrollmentenddate, name, startdate, start_date, status) VALUES ('do_1130293726460805121168', 'batch-002', 'manju', '2020-11-11', '2020-02-10', '2020-02-10', '2020-02-10', 'Basic C++', '2020-02-20', '2020-02-20', 1 ); +INSERT INTO sunbird_courses.course_batch (courseid, batchid, createdby, createddate, enddate, end_date, enrollmentenddate, name, startdate, start_date, status) VALUES ('do_1131975645014835201326', 'batch-003', 'manju', '2020-10-02', '2020-10-01', '2020-10-01', '2020-10-01', 'Basic C++', '2020-10-20', '2020-10-20', 1 ); +INSERT INTO sunbird_courses.course_batch (courseid, batchid, createdby, createddate, enddate, end_date, enrollmentenddate, name, startdate, start_date, status) VALUES ('do_112835334818643968148', 'batch-004', 'manju', '2020-11-11', '2020-02-10', '2020-02-10', '2020-02-10', 'Basic C++', '2020-02-20', '2020-10-20', 1 ); +INSERT INTO sunbird_courses.course_batch (courseid, batchid, createdby, createddate, enddate, end_date, enrollmentenddate, name, startdate, start_date, status) VALUES ('do_112835334818643968148', 'batch-005', 'manju', '2020-11-15', '2020-02-12', '2020-02-10', '2020-02-10', 'Basic C++', '2020-02-20', '2020-10-20', 2 ); +INSERT INTO sunbird_courses.course_batch (courseid, batchid, createdby, createddate, enddate, end_date, enrollmentenddate, name, startdate, start_date, status) VALUES ('do_1130928636168192001667', 'batch-006', 'manju', '2020-11-15', '2020-02-12', '2020-02-10', '2020-02-10', 'Basic C++', '2020-02-20', '2020-10-20', 1 ); + +INSERT INTO sunbird_courses.user_enrolments (userid, courseid, batchid, active, completionpercentage, progress, status, enrolleddate, enrolled_date) VALUES ('user-001', 'do_1130928636168192001667', 'batch-001', True, 10, 0, 1, '2019-11-13 05:41:50:382+0000', '2019-11-16 05:41:50'); +INSERT INTO sunbird_courses.user_enrolments (userid, courseid, batchid, active, completionpercentage, progress, status, enrolled_date) VALUES ('user-002', 'do_1130928636168192001667', 'batch-001', True, 15, 1, 1, '2019-11-15 05:41:50'); +INSERT INTO sunbird_courses.user_enrolments (userid, courseid, batchid, active, completionpercentage, progress, status, enrolleddate) VALUES ('user-003', 'do_1130928636168192001667', 'batch-001', True, 20, 1, 1, '2019-11-15 05:41:50:382+0000'); +INSERT INTO sunbird_courses.user_enrolments (userid, courseid, batchid, active, completionpercentage, progress, status, enrolleddate) VALUES ('user-004', 'do_1130928636168192001667', 'batch-001', True, 10, 0, 1, '2019-11-15 05:41:50:382+0000'); +INSERT INTO sunbird_courses.user_enrolments (userid, courseid, batchid, active, completionpercentage, progress, status, enrolleddate) VALUES ('user-005', 'do_1130292569979781121111', 'batch-002', True, 10, 0, 1, '2019-11-15 05:41:50:382+0000'); +INSERT INTO sunbird_courses.user_enrolments (userid, courseid, batchid, active, completionpercentage, progress, status, enrolleddate) VALUES ('user-006', 'do_1130292569979781121111', 'batch-002', True, 10, 1, 1, '2019-11-15 05:41:50:382+0000'); +INSERT INTO sunbird_courses.user_enrolments (userid, courseid, batchid, active, completionpercentage, progress, status, enrolleddate) VALUES ('user-007', 'do_1130292569979781121111', 'batch-002', True, 10, 0, 1, '2019-11-15 05:41:50:382+0000'); +INSERT INTO sunbird_courses.user_enrolments (userid, courseid, batchid, active, completionpercentage, progress, status, enrolleddate) VALUES ('user-008', 'do_1131975645014835201326', 'batch-003', True, 20, 1, 1, '2019-11-15 05:41:50:382+0000'); +INSERT INTO sunbird_courses.user_enrolments (userid, courseid, batchid, active, completionpercentage, progress, status, enrolleddate) VALUES ('user-009', 'do_1131975645014835201326', 'batch-003', True, 20, 1, 1, '2019-11-15 05:41:50:382+0000'); +INSERT INTO sunbird_courses.user_enrolments (userid, courseid, batchid, active, completionpercentage, progress, status, enrolleddate) VALUES ('user-010', 'do_1131975645014835201326', 'batch-003', True, 10, 1, 1, '2019-11-15 05:41:50:382+0000'); +INSERT INTO sunbird_courses.user_enrolments (userid, courseid, batchid, active, completionpercentage, progress, status, enrolleddate) VALUES ('user-011', 'do_1130293726460805121168', 'batch-002', True, 10, 1, 1, '2019-11-15 05:41:50:382+0000'); +INSERT INTO sunbird_courses.user_enrolments (userid, courseid, batchid, active, completionpercentage, progress, status, enrolleddate) VALUES ('user-011', 'do_1130293726460805121168', 'batch-002', True, 10, 1, 1, '2019-11-15 05:41:50:382+0000'); +INSERT INTO sunbird_courses.user_enrolments (userid, courseid, batchid, active, completionpercentage, progress, status, enrolleddate) VALUES ('user-014', 'do_112835334818643968148', 'batch-004', True, 10, 1, 1, '2019-11-15 05:41:50:382+0000'); +INSERT INTO sunbird_courses.user_enrolments (userid, courseid, batchid, active, completionpercentage, progress, status, enrolleddate, enrolled_date) VALUES ('user-015', 'do_1130928636168192001667', 'batch-006', True, 10, 0, 1, '2019-11-13 05:41:50:382+0000', '2019-11-16 05:41:50'); +INSERT INTO sunbird_courses.user_enrolments (userid, courseid, batchid, active, completionpercentage, progress, status, enrolleddate, enrolled_date) VALUES ('user-016', 'do_1130928636168192001667', 'batch-006', True, 10, 0, 1, '2019-11-13 05:41:50:382+0000', '2019-11-16 05:41:50'); +INSERT INTO sunbird_courses.user_enrolments (userid, courseid, batchid, active, completionpercentage, progress, status, enrolleddate, enrolled_date) VALUES ('user-017', 'do_1130928636168192001667', 'batch-006', True, 10, 0, 1, '2019-11-13 05:41:50:382+0000', '2019-11-16 05:41:50'); + +INSERT INTO sunbird_courses.assessment_aggregator (course_id, batch_id, user_id, content_id, attempt_id, grand_total, total_max_score, total_score, question, updated_on) VALUES ('do_1130928636168192001667', 'batch-001', 'user-001', 'do_1128870328040161281204', 'attempt-001', '20', 20, 20, [{id: 'do_213019475454476288155', assess_ts: '2020-06-18T18:15:56.490+0000', max_score: 1, score: 1, type: 'mcq', title: 'testQuestiontextandformula', resvalues: [{'1': '{"text":"A=\\\\pi r^2\n"}'}], params: [{'1': '{"text":"A=\\\\pi r^2\n"}'}, {'2': '{"text":"no\n"}'}, {'answer': '{"correct":["1"]}'}], description: 'testQuestiontextandformula', duration: 1.0}, {id: 'do_213019970118279168165', assess_ts: '2020-06-18T18:15:56.490+0000', max_score: 1, score: 1, type: 'mcq', title: 'test with formula', resvalues: [{'1': '{"text":"1\nA=\\\\pi r^2A=\\\\pi r^2\n"}'}], params: [{'1': '{"text":"1\nA=\\\\pi r^2A=\\\\pi r^2\n"}'}, {'2': '{"text":"2\n"}'}, {'answer': '{"correct":["1"]}'}], description: '', duration: 1.0}, {id: 'do_213019972814823424168', assess_ts: '2020-06-18T18:15:56.490+0000', max_score: 1, score: 0.33, type: 'mtf', title: 'Copy of - Match the following:\n\nx=\\frac{-b\\pm\\sqrt{b^2-4ac}}{2a}\nArrange the following equations in correct order.\n', resvalues: [{'lhs': '[{"1":"{\"text\":\"A=\\\\\\\\pi r^2\\n\"}"},{"2":"{\"text\":\"\\\\\\\\frac{4}{3}\\\\\\\\pi r^3\\n\"}"},{"3":"{\"text\":\"a^n\\\\\\\\times a^m=a^{n+m}\\n\"}"}]'}, {'rhs': '[{"1":"{\"text\":\"Volume of sphere\\n\"}"},{"2":"{\"text\":\"Area of Circle\\n\"}"},{"3":"{\"text\":\"Product Rule\\n\"}"}]'}], params: [{'lhs': '[{"1":"{\"text\":\"A=\\\\\\\\pi r^2\\n\"}"},{"2":"{\"text\":\"\\\\\\\\frac{4}{3}\\\\\\\\pi r^3\\n\"}"},{"3":"{\"text\":\"a^n\\\\\\\\times a^m=a^{n+m}\\n\"}"}]'}, {'rhs': '[{"1":"{\"text\":\"Volume of sphere\\n\"}"},{"2":"{\"text\":\"Product Rule\\n\"}"},{"3":"{\"text\":\"Area of Circle\\n\"}"}]'}, {'answer': '{"lhs":["1","2","3"],"rhs":["3","1","2"]}'}], description: '', duration: 2.0}, {id: 'do_2130256513760624641171', assess_ts: '2020-06-18T18:15:56.490+0000', max_score: 10, score: 10, type: 'mcq', title: '2 +2 is..? mark ia 10\n', resvalues: [{'1': '{"text":"4\n"}'}], params: [{'1': '{"text":"4\n"}'}, {'2': '{"text":"3\n"}'}, {'3': '{"text":"8\n"}'}, {'4': '{"text":"10\n"}'}, {'answer': '{"correct":["1"]}'}], description: '', duration: 12.0}], toTimeStamp(toDate(now()))); +INSERT INTO sunbird_courses.assessment_aggregator (course_id, batch_id, user_id, content_id, attempt_id, grand_total, total_max_score, total_score, updated_on) VALUES ('do_1130928636168192001667', 'batch-001', 'user-002', 'do_1128870328040161281204', 'attempt-001', '10', 10, 10, toTimeStamp(toDate(now()))); +INSERT INTO sunbird_courses.assessment_aggregator (course_id, batch_id, user_id, content_id, attempt_id, grand_total, total_max_score, total_score, updated_on) VALUES ('do_1130928636168192001667', 'batch-001', 'user-003', 'do_112876961957437440179', 'attempt-001', '10', 10, 10, toTimeStamp(toDate(now()))); +INSERT INTO sunbird_courses.assessment_aggregator (course_id, batch_id, user_id, content_id, attempt_id, grand_total, total_max_score, total_score, updated_on) VALUES ('do_112835334818643968148', 'batch-004', 'user-014', 'do_11307593493010022418', 'attempat-001', '15', 15, 15, toTimeStamp(toDate(now()))); + +INSERT INTO sunbird.system_settings (id, field, value) VALUES ('custodianOrgId', 'custodianOrgId', '0130107621805015045'); \ No newline at end of file diff --git a/data-products/src/test/resources/exhaust/data-archival/batch-001-2021-22-1626351011290.csv b/data-products/src/test/resources/exhaust/data-archival/batch-001-2021-22-1626351011290.csv index ea682a131..475c552f5 100644 --- a/data-products/src/test/resources/exhaust/data-archival/batch-001-2021-22-1626351011290.csv +++ b/data-products/src/test/resources/exhaust/data-archival/batch-001-2021-22-1626351011290.csv @@ -1,3 +1,4 @@ content_id,attempt_id,user_id,course_id,batch_id,created_on,last_attempted_on,total_max_score,total_score,updated_on,grand_total,question -do_1128870328040161281204,attempt-001,user-001,do_1130928636168192001667,batch-001,null,null,20,20,2021-06-15 5:30:00,20,"[{\"id\":\"do_31300323721809100819622\",\"assess_ts\":\"2021-02-16T18:51:54.104Z\",\"max_score\":1.0,\"score\":1.0,\"type\":\"mcq\",\"title\":\"Pediatric Concerns during COVID-19\",\"resvalues\":[{\"1\":\"{\\\"text\\\":\\\"A. Oliguria\\\\n\\\"}\"}],\"params\":[{\"1\":\"{\\\"text\\\":\\\"A. Oliguria\\\\n\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"B. Mild pneumonia\\\\n\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"C. Severe Pneumonia\\\\n\\\"}\"},{\"4\":\"{\\\"text\\\":\\\"D. Panic Attack\\\\n\\\"}\"},{\"answer\":\"{\\\"correct\\\":[\\\"1\\\"]}\"}],\"description\":\"\",\"duration\":17.000000000000000000},{\"id\":\"do_31300323406350745619621\",\"assess_ts\":\"2021-02-16T18:52:36.636Z\",\"max_score\":1.0,\"score\":1.0,\"type\":\"mcq\",\"title\":\"Pediatric Concerns during COVID-19\",\"resvalues\":[{\"4\":\"{\\\"text\\\":\\\"D. Mild Pneumonia\\\\n\\\"}\"}],\"params\":[{\"1\":\"{\\\"text\\\":\\\"A. Unstable angina pectoris\\\\n\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"B. Myocardial infarction\\\\n\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"C. Central cord syndrome\\\\n\\\"}\"},{\"4\":\"{\\\"text\\\":\\\"D. Mild Pneumonia\\\\n\\\"}\"},{\"answer\":\"{\\\"correct\\\":[\\\"4\\\"]}\"}],\"description\":\"\",\"duration\":11.000000000000000000},{\"id\":\"do_313003240076902400111588\",\"assess_ts\":\"2021-02-16T18:51:17.607Z\",\"max_score\":1.0,\"score\":1.0,\"type\":\"mcq\",\"title\":\"Pediatric Concerns during COVID-19\",\"resvalues\":[{\"1\":\"{\\\"text\\\":\\\"A. Pre-oxygenate with 100% FiO2 for 5 minutes, via a face mask with reservoir bag or HFNO and take care of aerosolization during the procedure\\\\n\\\"}\"}],\"params\":[{\"1\":\"{\\\"text\\\":\\\"A. Pre-oxygenate with 100% FiO2 for 5 minutes, via a face mask with reservoir bag or HFNO and take care of aerosolization during the procedure\\\\n\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"B. Conservative Fluid Management\\\\n\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"C. Oxygen Therapy\\\\n\\\"}\"},{\"4\":\"{\\\"text\\\":\\\"D. Lopinavir Therapy\\\\n\\\"}\"},{\"answer\":\"{\\\"correct\\\":[\\\"1\\\"]}\"}],\"description\":\"\",\"duration\":12.000000000000000000},{\"id\":\"do_313003232670449664111583\",\"assess_ts\":\"2021-02-16T18:52:47.423Z\",\"max_score\":1.0,\"score\":1.0,\"type\":\"mcq\",\"title\":\"Pediatric Concerns during COVID-19\",\"resvalues\":[{\"1\":\"{\\\"text\\\":\\\"A. Persistent pain or pressure in chest\\\\n\\\"}\"}],\"params\":[{\"1\":\"{\\\"text\\\":\\\"A. Persistent pain or pressure in chest\\\\n\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"B. Panic Attack\\\\n\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"C. Stomach ache\\\\n\\\"}\"},{\"4\":\"{\\\"text\\\":\\\"D. Headache\\\\n\\\"}\"},{\"answer\":\"{\\\"correct\\\":[\\\"1\\\"]}\"}],\"description\":\"\",\"duration\":10.000000000000000000},{\"id\":\"do_313003239006068736111587\",\"assess_ts\":\"2021-02-16T18:51:36.282Z\",\"max_score\":1.0,\"score\":1.0,\"type\":\"mcq\",\"title\":\"Pediatric Concerns during COVID-19\",\"resvalues\":[{\"3\":\"{\\\"text\\\":\\\"C. Oxygen Therapy\\\\n\\\"}\"}],\"params\":[{\"1\":\"{\\\"text\\\":\\\"A. Metformin therapy\\\\n\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"B. Lopinavir therapy\\\\n\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"C. Oxygen Therapy\\\\n\\\"}\"},{\"4\":\"{\\\"text\\\":\\\"D. None of the above\\\\n\\\"}\"},{\"answer\":\"{\\\"correct\\\":[\\\"3\\\"]}\"}],\"description\":\"\",\"duration\":17.000000000000000000},{\"id\":\"do_313003236553457664111586\",\"assess_ts\":\"2021-02-16T18:52:11.410Z\",\"max_score\":1.0,\"score\":1.0,\"type\":\"mcq\",\"title\":\"Pediatric Concerns during COVID-19\",\"resvalues\":[{\"4\":\"{\\\"text\\\":\\\"D. All of the above\\\\n\\\"}\"}],\"params\":[{\"1\":\"{\\\"text\\\":\\\"A. Tachycardia\\\\n\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"B. Tachypnea\\\\n\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"C. Fever or Hypothermia\\\\n\\\"}\"},{\"4\":\"{\\\"text\\\":\\\"D. All of the above\\\\n\\\"}\"},{\"answer\":\"{\\\"correct\\\":[\\\"4\\\"]}\"}],\"description\":\"\",\"duration\":16.000000000000000000},{\"id\":\"do_313003241437011968111589\",\"assess_ts\":\"2021-02-16T18:50:49.472Z\",\"max_score\":1.0,\"score\":1.0,\"type\":\"mcq\",\"title\":\"Pediatric Concerns during COVID-19\",\"resvalues\":[{\"3\":\"{\\\"text\\\":\\\"C. 4 hours\\\\n\\\"}\"}],\"params\":[{\"1\":\"{\\\"text\\\":\\\"A. 12 hours\\\\n\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"B. 8 hours\\\\n\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"C. 4 hours\\\\n\\\"}\"},{\"4\":\"{\\\"text\\\":\\\"D. 24 hours\\\\n\\\"}\"},{\"answer\":\"{\\\"correct\\\":[\\\"3\\\"]}\"}],\"description\":\"\",\"duration\":48.000000000000000000},{\"id\":\"do_313003235789389824111585\",\"assess_ts\":\"2021-02-16T18:52:17.543Z\",\"max_score\":1.0,\"score\":1.0,\"type\":\"mcq\",\"title\":\"Pediatric Concerns during COVID-19\",\"resvalues\":[{\"3\":\"{\\\"text\\\":\\\"C. Both A and B\\\\n\\\"}\"}],\"params\":[{\"1\":\"{\\\"text\\\":\\\"A. Oxygenation Index\\\\n\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"B. Oxygenation Index using SpO2\\\\n\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"C. Both A and B\\\\n\\\"}\"},{\"4\":\"{\\\"text\\\":\\\"D. None of the above\\\\n\\\"}\"},{\"answer\":\"{\\\"correct\\\":[\\\"3\\\"]}\"}],\"description\":\"\",\"duration\":5.000000000000000000},{\"id\":\"do_31300324092397158419623\",\"assess_ts\":\"2021-02-16T18:51:04.495Z\",\"max_score\":1.0,\"score\":1.0,\"type\":\"mcq\",\"title\":\"Pediatric Concerns during COVID-19\",\"resvalues\":[{\"2\":\"{\\\"text\\\":\\\"B. Vasopressors during or after fluid resuscitation\\\\n\\\"}\"}],\"params\":[{\"1\":\"{\\\"text\\\":\\\"A. Hypotonic crystalloids resuscitation\\\\n\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"B. Vasopressors during or after fluid resuscitation\\\\n\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"C. Oxygen Therapy\\\\n\\\"}\"},{\"4\":\"{\\\"text\\\":\\\"D. Rapid Sequence Intubation\\\\n\\\"}\"},{\"answer\":\"{\\\"correct\\\":[\\\"2\\\"]}\"}],\"description\":\"\",\"duration\":13.000000000000000000},{\"id\":\"do_313003234935881728111584\",\"assess_ts\":\"2021-02-16T18:52:24.796Z\",\"max_score\":1.0,\"score\":1.0,\"type\":\"mcq\",\"title\":\"Pediatric Concerns during COVID-19\",\"resvalues\":[{\"4\":\"{\\\"text\\\":\\\"D. All of the above\\\\n\\\"}\"}],\"params\":[{\"1\":\"{\\\"text\\\":\\\"A. Cough or difficulty in breathing including central cyanosis or spo2<90%\\\\n\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"B. Severe respiratory distress (grunting, chest indrawing)\\\\n\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"C. Signs of pneumonia along with convulsions\\\\n\\\"}\"},{\"4\":\"{\\\"text\\\":\\\"D. All of the above\\\\n\\\"}\"},{\"answer\":\"{\\\"correct\\\":[\\\"4\\\"]}\"}],\"description\":\"\",\"duration\":6.000000000000000000}]" -do_112876961957437440179,attempt-001,user-003,do_1130928636168192001667,batch-001,null,null,10,10,2021-06-15 5:30:00,10,[] \ No newline at end of file +do_1128870328040161281204,attempt-001,user-001,do_1130928636168192001667,batch-001,null,null,20,20,null,20,[] +do_1128870328040161281204,attempt-001,user-002,do_1130928636168192001667,batch-001,null,null,10,10,null,10,"[{\"id\":\"do_31300323721809100819622\",\"assess_ts\":\"2021-02-16T18:51:54.104Z\",\"max_score\":1.0,\"score\":1.0,\"type\":\"mcq\",\"title\":\"Pediatric Concerns during COVID-19\",\"resvalues\":[{\"1\":\"{\\\"text\\\":\\\"A. Oliguria\\\\n\\\"}\"}],\"params\":[{\"1\":\"{\\\"text\\\":\\\"A. Oliguria\\\\n\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"B. Mild pneumonia\\\\n\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"C. Severe Pneumonia\\\\n\\\"}\"},{\"4\":\"{\\\"text\\\":\\\"D. Panic Attack\\\\n\\\"}\"},{\"answer\":\"{\\\"correct\\\":[\\\"1\\\"]}\"}],\"description\":\"\",\"duration\":17.000000000000000000},{\"id\":\"do_31300323406350745619621\",\"assess_ts\":\"2021-02-16T18:52:36.636Z\",\"max_score\":1.0,\"score\":1.0,\"type\":\"mcq\",\"title\":\"Pediatric Concerns during COVID-19\",\"resvalues\":[{\"4\":\"{\\\"text\\\":\\\"D. Mild Pneumonia\\\\n\\\"}\"}],\"params\":[{\"1\":\"{\\\"text\\\":\\\"A. Unstable angina pectoris\\\\n\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"B. Myocardial infarction\\\\n\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"C. Central cord syndrome\\\\n\\\"}\"},{\"4\":\"{\\\"text\\\":\\\"D. Mild Pneumonia\\\\n\\\"}\"},{\"answer\":\"{\\\"correct\\\":[\\\"4\\\"]}\"}],\"description\":\"\",\"duration\":11.000000000000000000},{\"id\":\"do_313003240076902400111588\",\"assess_ts\":\"2021-02-16T18:51:17.607Z\",\"max_score\":1.0,\"score\":1.0,\"type\":\"mcq\",\"title\":\"Pediatric Concerns during COVID-19\",\"resvalues\":[{\"1\":\"{\\\"text\\\":\\\"A. Pre-oxygenate with 100% FiO2 for 5 minutes, via a face mask with reservoir bag or HFNO and take care of aerosolization during the procedure\\\\n\\\"}\"}],\"params\":[{\"1\":\"{\\\"text\\\":\\\"A. Pre-oxygenate with 100% FiO2 for 5 minutes, via a face mask with reservoir bag or HFNO and take care of aerosolization during the procedure\\\\n\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"B. Conservative Fluid Management\\\\n\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"C. Oxygen Therapy\\\\n\\\"}\"},{\"4\":\"{\\\"text\\\":\\\"D. Lopinavir Therapy\\\\n\\\"}\"},{\"answer\":\"{\\\"correct\\\":[\\\"1\\\"]}\"}],\"description\":\"\",\"duration\":12.000000000000000000},{\"id\":\"do_313003232670449664111583\",\"assess_ts\":\"2021-02-16T18:52:47.423Z\",\"max_score\":1.0,\"score\":1.0,\"type\":\"mcq\",\"title\":\"Pediatric Concerns during COVID-19\",\"resvalues\":[{\"1\":\"{\\\"text\\\":\\\"A. Persistent pain or pressure in chest\\\\n\\\"}\"}],\"params\":[{\"1\":\"{\\\"text\\\":\\\"A. Persistent pain or pressure in chest\\\\n\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"B. Panic Attack\\\\n\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"C. Stomach ache\\\\n\\\"}\"},{\"4\":\"{\\\"text\\\":\\\"D. Headache\\\\n\\\"}\"},{\"answer\":\"{\\\"correct\\\":[\\\"1\\\"]}\"}],\"description\":\"\",\"duration\":10.000000000000000000},{\"id\":\"do_313003239006068736111587\",\"assess_ts\":\"2021-02-16T18:51:36.282Z\",\"max_score\":1.0,\"score\":1.0,\"type\":\"mcq\",\"title\":\"Pediatric Concerns during COVID-19\",\"resvalues\":[{\"3\":\"{\\\"text\\\":\\\"C. Oxygen Therapy\\\\n\\\"}\"}],\"params\":[{\"1\":\"{\\\"text\\\":\\\"A. Metformin therapy\\\\n\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"B. Lopinavir therapy\\\\n\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"C. Oxygen Therapy\\\\n\\\"}\"},{\"4\":\"{\\\"text\\\":\\\"D. None of the above\\\\n\\\"}\"},{\"answer\":\"{\\\"correct\\\":[\\\"3\\\"]}\"}],\"description\":\"\",\"duration\":17.000000000000000000},{\"id\":\"do_313003236553457664111586\",\"assess_ts\":\"2021-02-16T18:52:11.410Z\",\"max_score\":1.0,\"score\":1.0,\"type\":\"mcq\",\"title\":\"Pediatric Concerns during COVID-19\",\"resvalues\":[{\"4\":\"{\\\"text\\\":\\\"D. All of the above\\\\n\\\"}\"}],\"params\":[{\"1\":\"{\\\"text\\\":\\\"A. Tachycardia\\\\n\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"B. Tachypnea\\\\n\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"C. Fever or Hypothermia\\\\n\\\"}\"},{\"4\":\"{\\\"text\\\":\\\"D. All of the above\\\\n\\\"}\"},{\"answer\":\"{\\\"correct\\\":[\\\"4\\\"]}\"}],\"description\":\"\",\"duration\":16.000000000000000000},{\"id\":\"do_313003241437011968111589\",\"assess_ts\":\"2021-02-16T18:50:49.472Z\",\"max_score\":1.0,\"score\":1.0,\"type\":\"mcq\",\"title\":\"Pediatric Concerns during COVID-19\",\"resvalues\":[{\"3\":\"{\\\"text\\\":\\\"C. 4 hours\\\\n\\\"}\"}],\"params\":[{\"1\":\"{\\\"text\\\":\\\"A. 12 hours\\\\n\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"B. 8 hours\\\\n\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"C. 4 hours\\\\n\\\"}\"},{\"4\":\"{\\\"text\\\":\\\"D. 24 hours\\\\n\\\"}\"},{\"answer\":\"{\\\"correct\\\":[\\\"3\\\"]}\"}],\"description\":\"\",\"duration\":48.000000000000000000},{\"id\":\"do_313003235789389824111585\",\"assess_ts\":\"2021-02-16T18:52:17.543Z\",\"max_score\":1.0,\"score\":1.0,\"type\":\"mcq\",\"title\":\"Pediatric Concerns during COVID-19\",\"resvalues\":[{\"3\":\"{\\\"text\\\":\\\"C. Both A and B\\\\n\\\"}\"}],\"params\":[{\"1\":\"{\\\"text\\\":\\\"A. Oxygenation Index\\\\n\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"B. Oxygenation Index using SpO2\\\\n\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"C. Both A and B\\\\n\\\"}\"},{\"4\":\"{\\\"text\\\":\\\"D. None of the above\\\\n\\\"}\"},{\"answer\":\"{\\\"correct\\\":[\\\"3\\\"]}\"}],\"description\":\"\",\"duration\":5.000000000000000000},{\"id\":\"do_31300324092397158419623\",\"assess_ts\":\"2021-02-16T18:51:04.495Z\",\"max_score\":1.0,\"score\":1.0,\"type\":\"mcq\",\"title\":\"Pediatric Concerns during COVID-19\",\"resvalues\":[{\"2\":\"{\\\"text\\\":\\\"B. Vasopressors during or after fluid resuscitation\\\\n\\\"}\"}],\"params\":[{\"1\":\"{\\\"text\\\":\\\"A. Hypotonic crystalloids resuscitation\\\\n\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"B. Vasopressors during or after fluid resuscitation\\\\n\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"C. Oxygen Therapy\\\\n\\\"}\"},{\"4\":\"{\\\"text\\\":\\\"D. Rapid Sequence Intubation\\\\n\\\"}\"},{\"answer\":\"{\\\"correct\\\":[\\\"2\\\"]}\"}],\"description\":\"\",\"duration\":13.000000000000000000},{\"id\":\"do_313003234935881728111584\",\"assess_ts\":\"2021-02-16T18:52:24.796Z\",\"max_score\":1.0,\"score\":1.0,\"type\":\"mcq\",\"title\":\"Pediatric Concerns during COVID-19\",\"resvalues\":[{\"4\":\"{\\\"text\\\":\\\"D. All of the above\\\\n\\\"}\"}],\"params\":[{\"1\":\"{\\\"text\\\":\\\"A. Cough or difficulty in breathing including central cyanosis or spo2<90%\\\\n\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"B. Severe respiratory distress (grunting, chest indrawing)\\\\n\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"C. Signs of pneumonia along with convulsions\\\\n\\\"}\"},{\"4\":\"{\\\"text\\\":\\\"D. All of the above\\\\n\\\"}\"},{\"answer\":\"{\\\"correct\\\":[\\\"4\\\"]}\"}],\"description\":\"\",\"duration\":6.000000000000000000}]" +do_112876961957437440179,attempt-001,user-003,do_1130928636168192001667,batch-001,null,null,10,10,null,10,[] \ No newline at end of file diff --git a/data-products/src/test/scala/org/sunbird/analytics/exhaust/TestResponseExhaustJobV2.scala b/data-products/src/test/scala/org/sunbird/analytics/exhaust/TestResponseExhaustJobV2.scala index 01bb6ad9a..e00f2429b 100644 --- a/data-products/src/test/scala/org/sunbird/analytics/exhaust/TestResponseExhaustJobV2.scala +++ b/data-products/src/test/scala/org/sunbird/analytics/exhaust/TestResponseExhaustJobV2.scala @@ -88,7 +88,7 @@ class TestResponseExhaustJobV2 extends BaseReportSpec with MockFactory with Base user1Result.map(f => f.`Collection Id`).toList should contain atLeastOneElementOf List("do_1130928636168192001667") user1Result.map(f => f.`Batch Id`).toList should contain atLeastOneElementOf List("BatchId_batch-001") user1Result.map(f => f.`User UUID`).toList should contain atLeastOneElementOf List("user-001") - user1Result.map(f => f.`Attempt Id`).toList should contain atLeastOneElementOf List("attempat-001") + user1Result.map(f => f.`Attempt Id`).toList should contain atLeastOneElementOf List("attempt-001") user1Result.map(f => f.`QuestionSet Id`).toList should contain atLeastOneElementOf List("do_1128870328040161281204", "do_112876961957437440179") user1Result.map(f => f.`QuestionSet Title`).toList should contain atLeastOneElementOf List("SelfAssess for course", "Assessment score report using summary plugin") user1Result.map(f => f.`Question Id`).toList should contain theSameElementsAs List("do_213019475454476288155", "do_213019970118279168165", "do_213019972814823424168", "do_2130256513760624641171") @@ -113,6 +113,18 @@ class TestResponseExhaustJobV2 extends BaseReportSpec with MockFactory with Base } + it should "generate report even if blob does not has any data for the batchid" in { + EmbeddedPostgresql.execute(s"TRUNCATE $jobRequestTable") + EmbeddedPostgresql.execute("INSERT INTO job_request (tag, request_id, job_id, status, request_data, requested_by, requested_channel, dt_job_submitted, download_urls, dt_file_created, dt_job_completed, execution_time, err_message ,iteration) VALUES ('do_1131350140968632321230_batch-001:01250894314817126443', '37564CF8F134EE7532F125651B51D17F', 'response-exhaust', 'SUBMITTED', '{\"batchId\": \"batch-001\"}', 'user-002', 'b00bc992ef25f1a9a8d63291e20efc8d', '2020-10-19 05:58:18.666', '{}', NULL, NULL, 0, '' ,0);") + + implicit val fc = new FrameworkContext() + val strConfig = """{"search":{"type":"none"},"model":"org.sunbird.analytics.exhaust.collection.ResponseExhaustJobV2","modelParams":{"store":"local","mode":"OnDemand","batchFilters":["TPD"],"assessmentFetcherConfig":{"store":"local","filePath":"src/test/resources/exhaust/data-archival/blob-data/","format":"csv"},"searchFilter":{},"sparkElasticsearchConnectionHost":"localhost","sparkRedisConnectionHost":"localhost","sparkUserDbRedisIndex":"0","sparkUserDbRedisPort":6341,"sparkCassandraConnectionHost":"localhost","fromDate":"","toDate":"","storageContainer":""},"parallelization":8,"appName":"ResponseExhaustJob Exhaust"}""" + val jobConfig = JSONUtils.deserialize[JobConfig](strConfig) + implicit val config = jobConfig + + ResponseExhaustJobV2.execute() + } + def getDate(): String = { val dateFormat: DateTimeFormatter = DateTimeFormat.forPattern("yyyyMMdd").withZone(DateTimeZone.forOffsetHoursMinutes(5, 30)); dateFormat.print(System.currentTimeMillis()); From 8271f7965bbb5082cfbfee7b52ab4557ab9a9d28 Mon Sep 17 00:00:00 2001 From: Utkarsha Kapoor Date: Tue, 10 Aug 2021 11:02:48 +0530 Subject: [PATCH 4/5] Issue #SB-24793: Review Comments resolved --- .../scala/org/sunbird/analytics/exhaust/util/ExhaustUtil.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/data-products/src/main/scala/org/sunbird/analytics/exhaust/util/ExhaustUtil.scala b/data-products/src/main/scala/org/sunbird/analytics/exhaust/util/ExhaustUtil.scala index 03e2184ac..05a7c6ec4 100644 --- a/data-products/src/main/scala/org/sunbird/analytics/exhaust/util/ExhaustUtil.scala +++ b/data-products/src/main/scala/org/sunbird/analytics/exhaust/util/ExhaustUtil.scala @@ -15,10 +15,12 @@ object ExhaustUtil { val url = store match { case "local" => filePath + s"${batchid}-*.${format}" + // $COVERAGE-OFF$ for azure testing case "s3" | "azure" => val key = AppConf.getConfig("azure_storage_key") val file = s"${filePath}${batchid}-*.${format}" s"wasb://$bucket@$key.blob.core.windows.net/$file." + // $COVERAGE-ON$ } JobLogger.log(s"Fetching data from ${store} for batchid: " + batchid)(new String()) From 3f5d49ae86b937171e967adec1977586364cfd9a Mon Sep 17 00:00:00 2001 From: Utkarsha Kapoor Date: Tue, 10 Aug 2021 13:02:23 +0530 Subject: [PATCH 5/5] Issue #SB-24793: circle ci issue resolved --- .../analytics/exhaust/collection/ResponseExhaustJobV2.scala | 2 +- .../sunbird/analytics/exhaust/TestResponseExhaustJobV2.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/data-products/src/main/scala/org/sunbird/analytics/exhaust/collection/ResponseExhaustJobV2.scala b/data-products/src/main/scala/org/sunbird/analytics/exhaust/collection/ResponseExhaustJobV2.scala index 83c986a3d..173dcc15c 100644 --- a/data-products/src/main/scala/org/sunbird/analytics/exhaust/collection/ResponseExhaustJobV2.scala +++ b/data-products/src/main/scala/org/sunbird/analytics/exhaust/collection/ResponseExhaustJobV2.scala @@ -90,7 +90,7 @@ object ResponseExhaustJobV2 extends optional.Application with BaseCollectionExha val store = azureFetcherConfig("store").asInstanceOf[String] val format:String = azureFetcherConfig.getOrElse("format", "csv").asInstanceOf[String] - val filePath = azureFetcherConfig.getOrElse("filePath", "data-archival/").asInstanceOf[String] + val filePath = azureFetcherConfig.getOrElse("filePath", "archival-data/").asInstanceOf[String] val container = azureFetcherConfig.getOrElse("container", "reports").asInstanceOf[String] val assessAggData = ExhaustUtil.getAssessmentBlobData(store, filePath, container, Option(batchid), Option(format)) diff --git a/data-products/src/test/scala/org/sunbird/analytics/exhaust/TestResponseExhaustJobV2.scala b/data-products/src/test/scala/org/sunbird/analytics/exhaust/TestResponseExhaustJobV2.scala index e00f2429b..2c06a609b 100644 --- a/data-products/src/test/scala/org/sunbird/analytics/exhaust/TestResponseExhaustJobV2.scala +++ b/data-products/src/test/scala/org/sunbird/analytics/exhaust/TestResponseExhaustJobV2.scala @@ -88,7 +88,7 @@ class TestResponseExhaustJobV2 extends BaseReportSpec with MockFactory with Base user1Result.map(f => f.`Collection Id`).toList should contain atLeastOneElementOf List("do_1130928636168192001667") user1Result.map(f => f.`Batch Id`).toList should contain atLeastOneElementOf List("BatchId_batch-001") user1Result.map(f => f.`User UUID`).toList should contain atLeastOneElementOf List("user-001") - user1Result.map(f => f.`Attempt Id`).toList should contain atLeastOneElementOf List("attempt-001") + user1Result.map(f => f.`Attempt Id`).toList should contain atLeastOneElementOf List("attempat-001") user1Result.map(f => f.`QuestionSet Id`).toList should contain atLeastOneElementOf List("do_1128870328040161281204", "do_112876961957437440179") user1Result.map(f => f.`QuestionSet Title`).toList should contain atLeastOneElementOf List("SelfAssess for course", "Assessment score report using summary plugin") user1Result.map(f => f.`Question Id`).toList should contain theSameElementsAs List("do_213019475454476288155", "do_213019970118279168165", "do_213019972814823424168", "do_2130256513760624641171")