Release 3.9.0 copy

.circleci/config.yml

@@ -24,9 +24,8 @@ jobs:
       - run:
           name: sonar
           command: |
-            srcBranch=$(curl -s https://api.github.com/repos/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME/pulls/$CIRCLE_PR_NUMBER | jq .head.ref -r)
-            tarBranch=$(curl -s https://api.github.com/repos/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME/pulls/$CIRCLE_PR_NUMBER | jq .base.ref -r)
-            mvn verify -DskipTests=true sonar:sonar -Dsonar.projectKey=project-sunbird_sunbird-course-service -Dsonar.organization=project-sunbird -Dsonar.host.url=https://sonarcloud.io -Dsonar.pullrequest.branch=$srcbranch -Dsonar.pullrequest.key=$CIRCLE_PR_NUMBER -Dsonar.pullrequest.base=$tarbranch -Dsonar.scala.coverage.reportPaths=/home/circleci/project/course-mw/enrolment-actor/target/scoverage.xml,/home/circleci/project/course-mw/sunbird-util/cache-utils/target/scoverage.xml -Dsonar.scanner.force-deprecated-java-version-grace-period=true
+            JAVA_REPORT_PATHS=`find /home/circleci/project  -iname jacoco.xml | awk 'BEGIN { RS = "" ; FS = "\n"; OFS = ","}{$1=$1; print $0}'`
+            mvn verify -DskipTests=true sonar:sonar -Dsonar.projectKey=project-sunbird_sunbird-course-service -Dsonar.organization=project-sunbird -Dsonar.host.url=https://sonarcloud.io -Dsonar.pullrequest.branch=$srcbranch -Dsonar.pullrequest.key=$CIRCLE_PR_NUMBER -Dsonar.pullrequest.base=$tarbranch -Dsonar.scala.coverage.reportPaths=/home/circleci/project/course-mw/enrolment-actor/target/scoverage.xml,/home/circleci/project/course-mw/sunbird-util/cache-utils/target/scoverage.xml -Dsonar.coverage.jacoco.xmlReportPaths=${JAVA_REPORT_PATHS}
 
 workflows:
   version: 2.1

.github/pull_request_template.md

@@ -0,0 +1,32 @@
+Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. List any dependencies that are required for this change.
+
+### Type of change
+
+Please choose appropriate options.
+
+- [ ] Bug fix (non-breaking change which fixes an issue)
+- [ ] New feature (non-breaking change which adds functionality)
+- [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected)
+- [ ] This change requires a documentation update
+
+### How Has This Been Tested?
+
+Please describe the tests that you ran to verify your changes in the below checkboxes. Provide instructions so we can reproduce. Please also list any relevant details for your test configuration
+
+- [ ] Ran Test A
+- [ ] Ran Test B
+
+**Test Configuration**:
+* Software versions: Java-11, play2-2.7.2, scala-2.11, redis-5.0.3
+* Hardware versions: 2 CPU/ 4GB RAM
+
+### Checklist:
+
+- [ ] My code follows the style guidelines of this project
+- [ ] I have performed a self-review of my own code
+- [ ] I have commented my code, particularly in hard-to-understand areas
+- [ ] I have made corresponding changes to the documentation
+- [ ] My changes generate no new warnings
+- [ ] I have added tests that prove my fix is effective or that my feature works
+- [ ] New and existing unit tests pass locally with my changes
+- [ ] Any dependent changes have been merged and published in downstream modules

.github/workflows/jira-description-action.yml

@@ -0,0 +1,16 @@
+name: jira-description-action
+on:
+  pull_request_target:
+    types: [opened, labeled]
+jobs:
+  add-jira-description:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: project-sunbird/[email protected]
+        name: jira-description-action
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          jira-token: ${{ secrets.JIRA_TOKEN }}
+          jira-base-url: ${{ secrets.JIRA_BASE_URL }}
+          fail-when-jira-issue-not-found: ${{ secrets.FAIL_WHEN_JIRA_ISSUE_NOT_FOUND }}
+          use: both

course-mw/course-actors-common/pom.xml

@@ -16,6 +16,16 @@
 		<logback.version>1.0.7</logback.version>
 	</properties>
 	<dependencies>
+		<dependency>
+			<groupId>net.logstash.logback</groupId>
+			<artifactId>logstash-logback-encoder</artifactId>
+			<version>6.6</version>
+		</dependency>
+		<dependency>
+			<groupId>ch.qos.logback</groupId>
+			<artifactId>logback-classic</artifactId>
+			<version>1.2.3</version>
+		</dependency>
 		<dependency>
 			<groupId>joda-time</groupId>
 			<artifactId>joda-time</artifactId>
@@ -68,11 +78,6 @@
 				</exclusion>
 			</exclusions>
 		</dependency>
-		<dependency>
-			<groupId>log4j</groupId>
-			<artifactId>log4j</artifactId>
-			<version>1.2.17</version>
-		</dependency>
 		<dependency>
 			<groupId>com.typesafe.akka</groupId>
 			<artifactId>akka-testkit_2.11</artifactId>
@@ -168,9 +173,11 @@
 				<artifactId>maven-surefire-plugin</artifactId>
 				<version>3.0.0-M4</version>
 				<configuration>
-					<argLine>
-						--illegal-access=warn
-					</argLine>
+					<!-- <skipTests>true</skipTests> -->
+					<includes>
+						<include>**/*Spec.java</include>
+						<include>**/*Test.java</include>
+					</includes>
 				</configuration>
 			</plugin>
 		</plugins>

course-mw/course-actors-common/src/main/java/org/sunbird/actor/base/BaseActor.java

@@ -3,6 +3,7 @@
 import akka.actor.UntypedAbstractActor;
 import org.sunbird.common.exception.ProjectCommonException;
 import org.sunbird.common.models.response.Response;
+import org.sunbird.common.models.response.ResponseParams;
 import org.sunbird.common.models.util.LoggerUtil;
 import org.sunbird.common.request.Request;
 import org.sunbird.common.responsecode.ResponseCode;
@@ -51,4 +52,16 @@ public Response successResponse() {
     response.put("response", "SUCCESS");
     return response;
   }
+
+  public Response clientError(String message) {
+    Response response = new Response();
+    response.setResponseCode(ResponseCode.CLIENT_ERROR);
+    ResponseParams params = new ResponseParams();
+    params.setStatus(ResponseParams.StatusType.FAILED.name());
+    params.setErr(ResponseCode.invalidRequestData.getErrorCode());
+    params.setErrmsg(ResponseCode.invalidRequestData.getErrorMessage());
+    response.setParams(params);
+    response.put("response", message);
+    return response;
+  }
 }

course-mw/course-actors-common/src/main/java/org/sunbird/learner/actors/BackgroundJobManager.java

@@ -161,7 +161,7 @@ private boolean insertDataToElastic(
     logger.info(requestContext, "BackgroundJobManager:insertDataToElastic: type = " + type + " identifier = " + identifier);
     Future<String> responseF = esService.save(requestContext, type, identifier, data);
     String response = (String) ElasticSearchHelper.getResponseFromFuture(responseF);
-    logger.debug(requestContext, "ES save response for identifier :" + identifier + " : ", response);
+    logger.debug(requestContext, "ES save response for identifier :" + identifier, null, new HashMap<String, Object>() {{put("response", response);}});
     if (!StringUtils.isBlank(response)) {
       return true;
     }

course-mw/course-actors-common/src/main/java/org/sunbird/learner/actors/PageManagementActor.java

@@ -11,6 +11,7 @@
 import org.apache.commons.lang3.StringUtils;
 import org.sunbird.actor.base.BaseActor;
 import org.sunbird.cassandra.CassandraOperation;
+import org.sunbird.common.CassandraUtil;
 import org.sunbird.common.ElasticSearchHelper;
 import org.sunbird.common.cacheloader.PageCacheLoaderService;
 import org.sunbird.common.exception.ProjectCommonException;
@@ -25,6 +26,7 @@
 import org.sunbird.common.request.Request;
 import org.sunbird.common.request.RequestContext;
 import org.sunbird.common.responsecode.ResponseCode;
+import org.sunbird.common.util.JsonUtil;
 import org.sunbird.dto.SearchDTO;
 import org.sunbird.helper.ServiceFactory;
 import org.sunbird.learner.util.ContentSearchUtil;
@@ -37,8 +39,11 @@
 import scala.concurrent.Promise;
 
 import java.io.IOException;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Date;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -47,6 +52,7 @@
 import java.util.Objects;
 import java.util.Set;
 import java.util.stream.Collectors;
+import java.util.TimeZone;
 
 import static org.sunbird.common.models.util.JsonKey.ID;
 
@@ -67,10 +73,15 @@ public class PageManagementActor extends BaseActor {
   private static final String DYNAMIC_FILTERS = "dynamicFilters";
   private static List<String> userProfilePropList = Arrays.asList("board");
   private LoggerUtil logger = new LoggerUtil(PageManagementActor.class);
+  private static final SimpleDateFormat DATE_FORMAT = ProjectUtil.getDateFormatter();
 
+  static {
+    DATE_FORMAT.setTimeZone(
+            TimeZone.getTimeZone(ProjectUtil.getConfigValue(JsonKey.SUNBIRD_TIMEZONE)));
+  }
   @Override
   public void onReceive(Request request) throws Throwable {
-    Util.initializeContext(request, TelemetryEnvKey.PAGE);
+    Util.initializeContext(request, TelemetryEnvKey.PAGE, this.getClass().getName());
 
     if(request.getOperation().equalsIgnoreCase(ActorOperations.GET_DIAL_PAGE_DATA.getValue())) {
       getDIALPageData(request);
@@ -114,7 +125,7 @@ private void getAllSections(RequestContext requestContext) {
   }
 
   @SuppressWarnings("unchecked")
-  private void getSection(Request actorMessage) {
+  private void getSection(Request actorMessage) throws Exception {
     Response response = null;
     Map<String, Object> req = actorMessage.getRequest();
     String sectionId = (String) req.get(JsonKey.ID);
@@ -132,7 +143,7 @@ private void getSection(Request actorMessage) {
         Map<String, Object> map = result.get(0);
         removeUnwantedData(map, "");
         Response section = new Response();
-        section.put(JsonKey.SECTION, response.get(JsonKey.RESPONSE));
+        section.put(JsonKey.SECTION, JsonUtil.convertWithDateFormat(response.get(JsonKey.RESPONSE), Map.class, DATE_FORMAT));
         PageCacheLoaderService.putDataIntoCache(
             ActorOperations.GET_SECTION.getValue(), sectionId, response.get(JsonKey.RESPONSE));
         sender().tell(section, self());
@@ -142,7 +153,7 @@ private void getSection(Request actorMessage) {
       }
     } else {
       response = new Response();
-      response.put(JsonKey.SECTION, sectionMap);
+      response.put(JsonKey.SECTION, JsonUtil.convertWithDateFormat(sectionMap, Map.class, DATE_FORMAT));
     }
     sender().tell(response, self());
   }
@@ -171,7 +182,22 @@ private void updatePageSection(Request actorMessage) {
         logger.error(actorMessage.getRequestContext(), "Exception occurred while processing display " + e.getMessage(), e);
       }
     }
-    sectionMap.put(JsonKey.UPDATED_DATE, ProjectUtil.getFormattedDate());
+    sectionMap.put(JsonKey.UPDATED_DATE, ProjectUtil.getTimeStamp());
+    sectionMap = CassandraUtil.changeCassandraColumnMapping(sectionMap);
+
+    if (!StringUtils.isBlank((String) sectionMap.get(JsonKey.ID))) {
+      Map<String, Object> map = new HashMap<>();
+      map.put(JsonKey.ID, (String) sectionMap.get(JsonKey.ID));
+      Response res =
+              cassandraOperation.getRecordsByProperties(
+                      actorMessage.getRequestContext(), sectionDbInfo.getKeySpace(), sectionDbInfo.getTableName(), map);
+      if (!((List<Map<String, Object>>) res.get(JsonKey.RESPONSE)).isEmpty()) {
+        Map<String, Object> pageSection = ((List<Map<String, Object>>) res.get(JsonKey.RESPONSE)).get(0);
+        pageSection.put(JsonKey.CREATED_DATE, createdDateCheck(pageSection));
+      }
+    }
+
+    sectionMap = CassandraUtil.changeCassandraColumnMapping(sectionMap);
     Response response =
         cassandraOperation.updateRecord(
                 actorMessage.getRequestContext(), sectionDbInfo.getKeySpace(), sectionDbInfo.getTableName(), sectionMap);
@@ -215,7 +241,8 @@ private void createPageSection(Request actorMessage) {
     }
     sectionMap.put(JsonKey.ID, uniqueId);
     sectionMap.put(JsonKey.STATUS, ProjectUtil.Status.ACTIVE.getValue());
-    sectionMap.put(JsonKey.CREATED_DATE, ProjectUtil.getFormattedDate());
+    sectionMap.put(JsonKey.CREATED_DATE, ProjectUtil.getTimeStamp());
+    sectionMap = CassandraUtil.changeCassandraColumnMapping(sectionMap);
     Response response =
         cassandraOperation.insertRecord(
                 actorMessage.getRequestContext(), sectionDbInfo.getKeySpace(), sectionDbInfo.getTableName(), sectionMap);
@@ -442,6 +469,7 @@ private void updatePage(Request actorMessage) {
                   actorMessage.getRequestContext(), pageDbInfo.getKeySpace(), pageDbInfo.getTableName(), map);
       if (!((List<Map<String, Object>>) res.get(JsonKey.RESPONSE)).isEmpty()) {
         Map<String, Object> page = ((List<Map<String, Object>>) res.get(JsonKey.RESPONSE)).get(0);
+        pageMap.put(JsonKey.CREATED_DATE, createdDateCheck(page));
         if (!(((String) page.get(JsonKey.ID)).equals(pageMap.get(JsonKey.ID)))) {
           ProjectCommonException exception =
               new ProjectCommonException(
@@ -453,7 +481,7 @@ private void updatePage(Request actorMessage) {
         }
       }
     }
-    pageMap.put(JsonKey.UPDATED_DATE, ProjectUtil.getFormattedDate());
+    pageMap.put(JsonKey.UPDATED_DATE, ProjectUtil.getTimeStamp());
     if (null != pageMap.get(JsonKey.PORTAL_MAP)) {
       try {
         pageMap.put(JsonKey.PORTAL_MAP, mapper.writeValueAsString(pageMap.get(JsonKey.PORTAL_MAP)));
@@ -468,6 +496,7 @@ private void updatePage(Request actorMessage) {
         logger.error(actorMessage.getRequestContext(), "Exception occurred while updating app map data " + e.getMessage(), e);
       }
     }
+    pageMap = CassandraUtil.changeCassandraColumnMapping(pageMap);
     Response response =
         cassandraOperation.updateRecord(
                 actorMessage.getRequestContext(), pageDbInfo.getKeySpace(), pageDbInfo.getTableName(), pageMap);
@@ -516,7 +545,7 @@ private void createPage(Request actorMessage) {
       }
     }
     pageMap.put(JsonKey.ID, uniqueId);
-    pageMap.put(JsonKey.CREATED_DATE, ProjectUtil.getFormattedDate());
+    pageMap.put(JsonKey.CREATED_DATE, ProjectUtil.getTimeStamp());
     if (null != pageMap.get(JsonKey.PORTAL_MAP)) {
       try {
         pageMap.put(JsonKey.PORTAL_MAP, mapper.writeValueAsString(pageMap.get(JsonKey.PORTAL_MAP)));
@@ -531,6 +560,7 @@ private void createPage(Request actorMessage) {
         logger.error(actorMessage.getRequestContext(), "createPage: " + e.getMessage(), e);
       }
     }
+    pageMap = CassandraUtil.changeCassandraColumnMapping(pageMap);
     Response response =
         cassandraOperation.insertRecord(
                 actorMessage.getRequestContext(), pageDbInfo.getKeySpace(), pageDbInfo.getTableName(), pageMap);
@@ -932,4 +962,16 @@ private List<String> getStringListFromObj(Object obj) {
       return Arrays.asList((String)obj);
     }
   }
+
+  // Remove this implementation after deprecating text date columns
+  private Date createdDateCheck(Map<String, Object> page) {
+    try {
+      if (page.containsKey(JsonKey.CREATED_DATE) && page.get(JsonKey.CREATED_DATE) == null) {
+        return DATE_FORMAT.parse((String) page.get(JsonKey.OLD_CREATED_DATE));
+      }
+    } catch (ParseException e) {
+      logger.error(null, "PageManagementActor:createdDateCheck: Exception occurred with error message = " + e.getMessage(), e);
+    }
+    return (Date) page.get(JsonKey.CREATED_DATE);
+  }
 }

course-mw/course-actors-common/src/main/java/org/sunbird/learner/actors/certificate/service/CertificateActor.java

@@ -41,7 +41,7 @@ public String getValue() {
 
   @Override
   public void onReceive(Request request) throws Throwable {
-    Util.initializeContext(request, TelemetryEnvKey.USER);
+    Util.initializeContext(request, TelemetryEnvKey.USER, this.getClass().getName());
 
     String requestedOperation = request.getOperation();
     switch (requestedOperation) {

course-mw/course-actors-common/src/main/java/org/sunbird/learner/actors/certificate/service/CourseBatchCertificateActor.java

@@ -33,7 +33,7 @@ public class CourseBatchCertificateActor extends BaseActor {
 
   @Override
   public void onReceive(Request request) throws Throwable {
-    Util.initializeContext(request, TelemetryEnvKey.USER);
+    Util.initializeContext(request, TelemetryEnvKey.USER, this.getClass().getName());
 
     String requestedOperation = request.getOperation();
     switch (requestedOperation) {

course-mw/course-actors-common/src/main/java/org/sunbird/learner/actors/course/CourseManagementActor.java

@@ -35,7 +35,8 @@ public class CourseManagementActor extends BaseActor {
 
     @Override
     public void onReceive(Request request) throws Throwable {
-        Util.initializeContext(request, TelemetryEnvKey.COURSE_CREATE);
+        Util.initializeContext(request, TelemetryEnvKey.COURSE_CREATE, this.getClass().getName());
+
         String requestedOperation = request.getOperation();
         switch (requestedOperation) {
             case "createCourse":

course-mw/course-actors-common/src/main/java/org/sunbird/learner/actors/coursebatch/dao/impl/CourseBatchDaoImpl.java

@@ -4,9 +4,13 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.stream.Collectors;
+
 import org.sunbird.cassandra.CassandraOperation;
+import org.sunbird.common.CassandraUtil;
 import org.sunbird.common.exception.ProjectCommonException;
 import org.sunbird.common.models.response.Response;
+import org.sunbird.common.models.util.CassandraPropertyReader;
 import org.sunbird.common.models.util.JsonKey;
 import org.sunbird.common.request.RequestContext;
 import org.sunbird.common.responsecode.ResponseCode;
@@ -19,12 +23,14 @@
 public class CourseBatchDaoImpl implements CourseBatchDao {
   private CassandraOperation cassandraOperation = ServiceFactory.getInstance();
   private Util.DbInfo courseBatchDb = Util.dbInfoMap.get(JsonKey.COURSE_BATCH_DB);
-
+  private static final CassandraPropertyReader propertiesCache =
+          CassandraPropertyReader.getInstance();
   private ObjectMapper mapper = new ObjectMapper();
 
   @Override
   public Response create(RequestContext requestContext, CourseBatch courseBatch) {
     Map<String, Object> map = mapper.convertValue(courseBatch, Map.class);
+    map = CassandraUtil.changeCassandraColumnMapping(map);
     return cassandraOperation.insertRecord(
             requestContext, courseBatchDb.getKeySpace(), courseBatchDb.getTableName(), map);
   }
@@ -38,6 +44,7 @@ public Response update(RequestContext requestContext, String courseId, String ba
     attributeMap.putAll(map);
     attributeMap.remove(JsonKey.COURSE_ID);
     attributeMap.remove(JsonKey.BATCH_ID);
+    attributeMap = CassandraUtil.changeCassandraColumnMapping(attributeMap);
     return cassandraOperation.updateRecord(
             requestContext, courseBatchDb.getKeySpace(), courseBatchDb.getTableName(), attributeMap, primaryKey);
   }