partition fix

sunbird-cb · Feb 28, 2024 · a756df8 · a756df8
1 parent 154ad6b
commit a756df8
Show file tree

Hide file tree

Showing 3 changed files with 7 additions and 3 deletions.
diff --git a/batch-models/src/main/resources/warehouse-schema.sql b/batch-models/src/main/resources/warehouse-schema.sql
@@ -14,16 +14,19 @@ CREATE TABLE user_detail(
    phone_number VARCHAR(255),
    groups VARCHAR(255),
    tag VARCHAR(255),
-   user_registration_date DATE,
    roles VARCHAR(255),
    gender VARCHAR(255),
    category VARCHAR(255),
    created_by_id VARCHAR(255),
    external_system VARCHAR(255),
    external_system_id VARCHAR(255),
+   is_verified_karmayogi VARCHAR(255),
+   mdo_created_on DATE,
+   user_registration_date DATE,
    data_last_generated_on VARCHAR(255)
 );
 
+
 -- Table: org_hierarchy
 CREATE TABLE org_hierarchy(
    mdo_id VARCHAR(255) PRIMARY KEY NOT NULL,

diff --git a/batch-models/src/main/scala/org/ekstep/analytics/dashboard/TestUtil.scala b/batch-models/src/main/scala/org/ekstep/analytics/dashboard/TestUtil.scala
@@ -69,10 +69,10 @@ object TestUtil extends Serializable {
 
       "dwUserTable" -> "user_detail",
       "dwCourseTable" -> "content",
-      "dwEnrollmentsTable" -> "user_enrolment",
+      "dwEnrollmentsTable" -> "user_enrolments",
       "dwOrgTable" -> "org_hierarchy",
       "dwAssessmentTable" -> "assessment_detail",
-      "dwBPEnrollmentsTable" -> "bp_enrollments",
+      "dwBPEnrollmentsTable" -> "bp_enrolments",
 
       "dwKcmDictionaryTable" -> "kcm_dictionary_test",
       "dwKcmContentTable" -> "kcm_content_mapping_test",

diff --git a/batch-models/src/main/scala/org/ekstep/analytics/dashboard/exhaust/DataExhaustModel.scala b/batch-models/src/main/scala/org/ekstep/analytics/dashboard/exhaust/DataExhaustModel.scala
@@ -96,6 +96,7 @@ object DataExhaustModel extends AbsDashboardModel {
       .distinct()
       .drop("orgType")
       .dropDuplicates(Seq("mdo_id"))
+      .repartition(16)
     show(orgHierarchyDF, "orgHierarchyDF")
     cache.write(orgHierarchyDF, "orgHierarchy")