diff --git a/batch-models/src/main/resources/warehouse-schema.sql b/batch-models/src/main/resources/warehouse-schema.sql index 8f29bf18..222a26d8 100644 --- a/batch-models/src/main/resources/warehouse-schema.sql +++ b/batch-models/src/main/resources/warehouse-schema.sql @@ -14,16 +14,19 @@ CREATE TABLE user_detail( phone_number VARCHAR(255), groups VARCHAR(255), tag VARCHAR(255), - user_registration_date DATE, roles VARCHAR(255), gender VARCHAR(255), category VARCHAR(255), created_by_id VARCHAR(255), external_system VARCHAR(255), external_system_id VARCHAR(255), + is_verified_karmayogi VARCHAR(255), + mdo_created_on DATE, + user_registration_date DATE, data_last_generated_on VARCHAR(255) ); + -- Table: org_hierarchy CREATE TABLE org_hierarchy( mdo_id VARCHAR(255) PRIMARY KEY NOT NULL, diff --git a/batch-models/src/main/scala/org/ekstep/analytics/dashboard/TestUtil.scala b/batch-models/src/main/scala/org/ekstep/analytics/dashboard/TestUtil.scala index affd48a2..ebdb4912 100644 --- a/batch-models/src/main/scala/org/ekstep/analytics/dashboard/TestUtil.scala +++ b/batch-models/src/main/scala/org/ekstep/analytics/dashboard/TestUtil.scala @@ -69,10 +69,10 @@ object TestUtil extends Serializable { "dwUserTable" -> "user_detail", "dwCourseTable" -> "content", - "dwEnrollmentsTable" -> "user_enrolment", + "dwEnrollmentsTable" -> "user_enrolments", "dwOrgTable" -> "org_hierarchy", "dwAssessmentTable" -> "assessment_detail", - "dwBPEnrollmentsTable" -> "bp_enrollments", + "dwBPEnrollmentsTable" -> "bp_enrolments", "dwKcmDictionaryTable" -> "kcm_dictionary_test", "dwKcmContentTable" -> "kcm_content_mapping_test", diff --git a/batch-models/src/main/scala/org/ekstep/analytics/dashboard/exhaust/DataExhaustModel.scala b/batch-models/src/main/scala/org/ekstep/analytics/dashboard/exhaust/DataExhaustModel.scala index 2a33e581..32fe11ce 100644 --- a/batch-models/src/main/scala/org/ekstep/analytics/dashboard/exhaust/DataExhaustModel.scala +++ b/batch-models/src/main/scala/org/ekstep/analytics/dashboard/exhaust/DataExhaustModel.scala @@ -96,6 +96,7 @@ object DataExhaustModel extends AbsDashboardModel { .distinct() .drop("orgType") .dropDuplicates(Seq("mdo_id")) + .repartition(16) show(orgHierarchyDF, "orgHierarchyDF") cache.write(orgHierarchyDF, "orgHierarchy")