Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

4.8.20 dev v3 #69

Open
wants to merge 3 commits into
base: cbrelease-4.8.20
Choose a base branch
from
Open

4.8.20 dev v3 #69

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
f772e39
added logs to check issue. (#65)
ArpithaSureshappa Dec 20, 2024
aec39dc
removed interface (#66)
ArpithaSureshappa Dec 20, 2024
974fc0c
interface deleted and renamed methods name (#67)
ArpithaSureshappa Dec 20, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 14 additions & 5 deletions src/main/java/com/igot/cios/plugins/DataTransformUtility.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -358,6 +358,14 @@ public void updateProcessedDataInDb(JsonNode processedData, String partnerCode,
processedData.forEach(eachContentData -> {
JsonNode transformData = transformData(eachContentData, contentJson);
Timestamp currentTime = new Timestamp(System.currentTimeMillis());
if (transformData != null &&
transformData.path(Constants.CONTENT) != null &&
transformData.path(Constants.CONTENT).get(Constants.DURATION) != null) {
String durationString = transformData.path(Constants.CONTENT).get(Constants.DURATION).textValue();
String[] parts = durationString.split(" ");
String duration = String.valueOf(Integer.parseInt(parts[0]));
((ObjectNode) transformData.path(Constants.CONTENT)).put(Constants.DURATION, duration).asText();
}
((ObjectNode) transformData.path(Constants.CONTENT)).put(Constants.FILE_ID, fileId).asText();
((ObjectNode) transformData.path(Constants.CONTENT)).put(Constants.SOURCE, fileName).asText();
((ObjectNode) transformData.path(Constants.CONTENT)).put(Constants.PARTNER_CODE, partnerCode).asText();
Expand Down Expand Up @@ -389,7 +397,7 @@ public CornellContentEntity saveOrUpdateCornellContent(String externalId, JsonNo
Optional<CornellContentEntity> optExternalContent = cornellContentRepository.findByExternalIdAndPartnerId(externalId,partnerId);
if (optExternalContent.isPresent()) {
CornellContentEntity externalContent = optExternalContent.get();
if(!externalContent.getCiosData().get("content").get("status").equals("live")||externalContent.getCiosData().get("content").get("status").equals("draft")) {
if(!(externalContent.getCiosData().get("content").get("status").equals("live")||externalContent.getCiosData().get("content").get("status").equals("draft"))){
externalContent.setExternalId(externalId);
externalContent.setCiosData(transformData);
externalContent.setIsActive(externalContent.getIsActive());
Expand All @@ -399,6 +407,8 @@ public CornellContentEntity saveOrUpdateCornellContent(String externalId, JsonNo
externalContent.setFileId(fileId);
externalContent.setPartnerId(partnerId);
externalContent.setPartnerCode((partnerCode));
}else{
//kafka changes need to add
}
return externalContent;
} else {
Expand Down Expand Up @@ -440,10 +450,9 @@ private void cornellBulkSave(List<CornellContentEntity> cornellContentEntityList
Long totalCourseCount = cornellContentRepository.countByPartnerCode(partnerCode);
log.info("Total courses onboarded {} for partner {}",totalCourseCount,partnerCode);
JsonNode response = fetchPartnerInfoUsingApi(partnerCode);
JsonNode resultData = response.path(Constants.RESULT);
JsonNode data = resultData.path(Constants.DATA);
((ObjectNode) data).put(Constants.TOTAL_COURSES_COUNT, totalCourseCount);
updatingPartnerInfo(resultData);
JsonNode resultData = response.path(Constants.DATA);
((ObjectNode) resultData).put(Constants.TOTAL_COURSES_COUNT, totalCourseCount);
updatingPartnerInfo(response);
}

public void flattenContentData(Map<String, Object> entityMap) {
Expand Down
34 changes: 18 additions & 16 deletions src/main/java/com/igot/cios/scheduler/CornellSchedulerService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@

@Slf4j
@Service
public class CornellSchedulerService implements SchedulerInterface {
public class CornellSchedulerService{

@Autowired
private ObjectMapper objectMapper;
Expand All @@ -48,14 +48,10 @@ public class CornellSchedulerService implements SchedulerInterface {
@Autowired
private DataTransformUtility dataTransformUtility;

public void callEnrollmentAPI(String partnerCode, JsonNode rawContentData) {
private void callEnrollmentAPI(String partnerCode, String partnerId, JsonNode transformData) {
try {
log.info("CornellSchedulerService::callEnrollmentAPI");
JsonNode entity = dataTransformUtility.fetchPartnerInfoUsingApi(partnerCode);
List<Object> contentJson = objectMapper.convertValue(entity.get("transformProgressViaApi"), new TypeReference<List<Object>>() {});
JsonNode transformData = dataTransformUtility.transformData(rawContentData, contentJson);
String extCourseId = transformData.get("courseid").asText();
String partnerId = entity.get("id").asText();
JsonNode result = dataTransformUtility.callCiosReadApi(extCourseId,partnerId);
String courseId = result.path("content").get("contentId").asText();
String[] parts = transformData.get("userid").asText().split("@");
Expand All @@ -72,7 +68,7 @@ public void callEnrollmentAPI(String partnerCode, JsonNode rawContentData) {
String formatedDate = updateDateFormatFromTimestamp(date);
((ObjectNode) transformData).put("completedon", formatedDate);
((ObjectNode) transformData).put("partnerCode", partnerCode);
((ObjectNode) transformData).put("partnerId", entity.get("id").asText());
((ObjectNode) transformData).put("partnerId", partnerId);
payloadValidation.validatePayload(Constants.PROGRESS_DATA_VALIDATION_FILE, transformData);
kafkaProducer.push(cbServerProperties.getTopic(), transformData);
} else {
Expand All @@ -91,7 +87,8 @@ private String updateDateFormatFromTimestamp(Long completedon) {
return sdf.format(date);
}

public JsonNode loadEnrollment() {
public JsonNode loadCornellEnrollment() {
log.info("CornellSchedulerService :: loadEnrollment()");
RequestBodyDTO requestBodyDTO = new RequestBodyDTO();
requestBodyDTO.setServiceCode(cbServerProperties.getCornellEnrollmentServiceCode());
requestBodyDTO.setUrlMap(formUrlMapForEnrollment());
Expand All @@ -118,8 +115,8 @@ private Map<String, String> formUrlMapForEnrollment() {
return urlMap;
}

public JsonNode performEnrollmentCall(String partnerCode, String requestBody) {
log.info("calling service locator for getting {} enrollment list", partnerCode);
private JsonNode performEnrollmentCall(String partnerCode, String requestBody) {
log.info("CornellSchedulerService :: performEnrollmentCall partnerCode {} and requestBody {}", partnerCode,requestBody);
String url = cbServerProperties.getServiceLocatorHost() + cbServerProperties.getServiceLocatorFixedUrl();
HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.APPLICATION_JSON);
Expand All @@ -131,12 +128,17 @@ public JsonNode performEnrollmentCall(String partnerCode, String requestBody) {
Object.class
);
if (response.getStatusCode().is2xxSuccessful()) {
JsonNode jsonNode = objectMapper.valueToTree(response.getBody());
JsonNode jsonData = jsonNode.path("responseData").get("enrollments");
jsonData.forEach(
eachContentData -> {
callEnrollmentAPI(partnerCode, eachContentData);
});
JsonNode jsonData = objectMapper.valueToTree(response.getBody());
if(!jsonData.isMissingNode()){
JsonNode contentPartnerResponse = dataTransformUtility.fetchPartnerInfoUsingApi(partnerCode);
String partnerId = contentPartnerResponse.get("id").asText();
jsonData.forEach(
eachContentData -> {
callEnrollmentAPI(partnerCode, partnerId, eachContentData);
});
}else{
log.error("Failed to retrieve response data: for partner code {}", partnerCode);
}
return jsonData;
} else {
throw new RuntimeException("Failed to retrieve externalId. Status code: " + response.getStatusCodeValue());
Expand Down
4 changes: 2 additions & 2 deletions src/main/java/com/igot/cios/scheduler/CourseScheduler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,15 +28,15 @@ public class CourseScheduler {
private void callCornellEnrollmentApi() throws JsonProcessingException{
if (schedulerEnabled) {
log.info("CourseScheduler :: callCornellEnrollmentApi");
cornellSchedulerService.loadEnrollment();
cornellSchedulerService.loadCornellEnrollment();
}
}

@Scheduled(cron = "${coursera.scheduler.cron}")
private void callCourseraEnrollmentApi() throws JsonProcessingException {
if (courseraSchedulerEnabled) {
log.info("CourseScheduler :: callCourseraEnrollmentApi");
courseraSchedulerService.loadEnrollment();
courseraSchedulerService.loadCourseraEnrollment();
}
}
}
129 changes: 70 additions & 59 deletions src/main/java/com/igot/cios/scheduler/CourseraSchedulerService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.igot.cios.dto.RequestBodyDTO;
import com.igot.cios.exception.CiosContentException;
Expand All @@ -28,7 +29,7 @@

@Slf4j
@Service
public class CourseraSchedulerService implements SchedulerInterface {
public class CourseraSchedulerService {
@Autowired
private ObjectMapper objectMapper;
@Autowired
Expand All @@ -44,39 +45,58 @@ public class CourseraSchedulerService implements SchedulerInterface {
@Autowired
private DataTransformUtility dataTransformUtility;

@Override
public JsonNode loadEnrollment() {
RequestBodyDTO requestBodyDTO = new RequestBodyDTO();
requestBodyDTO.setServiceCode(cbServerProperties.getCourseraEnrollmentServiceCode());
requestBodyDTO.setUrlMap(formUrlMapForEnrollment());
public JsonNode loadCourseraEnrollment() {
log.info("Coursera Scheduler Service::loadEnrollment()");
int start = 0;
int limit = cbServerProperties.getCourseraEnrollmentListLimit();
String payload = null;
try {
payload = objectMapper.writeValueAsString(requestBodyDTO);
} catch (JsonProcessingException e) {
throw new RuntimeException(e);
ArrayNode allEnrollmentData = objectMapper.createArrayNode();
int total = 0;
while (start == 0 || start < total) {
RequestBodyDTO requestBodyDTO = new RequestBodyDTO();
requestBodyDTO.setServiceCode(cbServerProperties.getCourseraEnrollmentServiceCode());
requestBodyDTO.setUrlMap(formUrlMapForEnrollment(start, limit));
try {
payload = objectMapper.writeValueAsString(requestBodyDTO);
} catch (JsonProcessingException e) {
throw new RuntimeException(e);
}
JsonNode response = performEnrollmentCall(cbServerProperties.courseraPartnerCode, payload);
total = response.get("count").asInt();
JsonNode enrollmentData = response.path("data");
if (enrollmentData != null && !enrollmentData.isMissingNode() && enrollmentData.isArray()) {
allEnrollmentData.addAll((ArrayNode) enrollmentData);
}
start += limit;
}
return performEnrollmentCall(cbServerProperties.courseraPartnerCode, payload);
allEnrollmentData.forEach(eachContentData -> {
JsonNode contentPartnerInfo = dataTransformUtility.fetchPartnerInfoUsingApi(cbServerProperties.courseraPartnerCode);
String partnerId = contentPartnerInfo.get("id").asText();
callEnrollmentAPI(cbServerProperties.courseraPartnerCode, partnerId, eachContentData);
});
return allEnrollmentData;
}

private Map<String, String> formUrlMapForEnrollment() {
private Map<String, String> formUrlMapForEnrollment(int start, int limit) {
log.info("Coursera Scheduler Service::formUrlMapForEnrollment");
LocalDateTime currentDateTime = LocalDateTime.now();
LocalDateTime previousDate = currentDateTime.minusDays(cbServerProperties.getCourseraDateRange());
ZonedDateTime zonedDateTime = previousDate.atZone(ZoneId.of("UTC"));
long timestamp = zonedDateTime.toInstant().getEpochSecond();
long currentMillis = System.currentTimeMillis();
long timestampWithoutMillis = (currentMillis / 1000) * 1000;
Map<String, String> urlMap = new HashMap<>();
urlMap.put("limit", cbServerProperties.getCourseraEnrollmentListLimit());
urlMap.put("start", "0");
urlMap.put("completedAtBefore", String.valueOf(timestampWithoutMillis));
urlMap.put("completedAtAfter", String.valueOf(timestamp));
urlMap.put("limit", String.valueOf(cbServerProperties.getCourseraEnrollmentListLimit()));
urlMap.put("start", String.valueOf(start));
urlMap.put(cbServerProperties.getCourseraDateBefore(), String.valueOf(timestampWithoutMillis));
urlMap.put(cbServerProperties.getCourseraDateAfter(), String.valueOf(timestamp));
return urlMap;
}

@Override
public JsonNode performEnrollmentCall(String partnerCode, String requestBody) {
log.info("calling service locator for getting {} enrollment list", partnerCode);
private JsonNode performEnrollmentCall(String partnerCode, String requestBody) {
log.info("coursera scheduler service: performEnrollmentCall for partner code {}, request body {}", partnerCode,requestBody);
String url = cbServerProperties.getServiceLocatorHost() + cbServerProperties.getServiceLocatorFixedUrl();
log.info("CourseraSchedulerService :: performEnrollmentCall url {}", url);
HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.APPLICATION_JSON);
HttpEntity<Object> entity = new HttpEntity<>(requestBody, headers);
Expand All @@ -87,54 +107,45 @@ public JsonNode performEnrollmentCall(String partnerCode, String requestBody) {
Object.class
);
if (response.getStatusCode().is2xxSuccessful()) {
JsonNode jsonNode = objectMapper.valueToTree(response.getBody());
JsonNode jsonData = jsonNode.path("responseData").get("elements");
jsonData.forEach(
eachContentData -> {
callEnrollmentAPI(partnerCode, eachContentData);
});
return jsonData;
log.info("CourseraSchedulerService :: performEnrollmentCall response {}");
JsonNode jsonData = objectMapper.valueToTree(response.getBody());
if(!jsonData.isMissingNode()&&jsonData != null){
return jsonData;
}else{
log.error("Failed to retrieve response data: for partner code {}", partnerCode);
}
} else {
throw new RuntimeException("Failed to retrieve externalId. Status code: " + response.getStatusCodeValue());
}
return null;
}

@Override
public void callEnrollmentAPI(String partnerCode, JsonNode rawContentData) {
private void callEnrollmentAPI(String partnerCode, String partnerId, JsonNode transformData) {
try {
log.info("CourseSchedulerService::callCourseraEnrollmentAPI");
if (rawContentData.get("contentType").asText().equalsIgnoreCase("Specialization") && rawContentData.get("isCompleted").asBoolean()) {
JsonNode entity = dataTransformUtility.fetchPartnerInfoUsingApi(partnerCode);
if (!entity.path("transformProgressViaApi").isMissingNode()) {
List<Object> contentJson = objectMapper.convertValue(entity.get("transformProgressViaApi"), new TypeReference<List<Object>>() {
});
JsonNode transformData = dataTransformUtility.transformData(rawContentData, contentJson);
String extCourseId = transformData.get("courseid").asText();
String partnerId = entity.get("id").asText();
JsonNode result = dataTransformUtility.callCiosReadApi(extCourseId, partnerId);
String courseId = result.path("content").get("contentId").asText();
String[] parts = transformData.get("userid").asText().split("@");
((ObjectNode) transformData).put("userid", parts[0]);
String userId = transformData.get("userid").asText();
log.info("courseId and userid {} {}", courseId, userId);
Map<String, Object> propertyMap = new HashMap<>();
propertyMap.put("userid", userId);
propertyMap.put("courseid", courseId);
propertyMap.put("progress", 100);
List<Map<String, Object>> listOfMasterData = cassandraOperation.getRecordsByProperties(Constants.KEYSPACE_SUNBIRD_COURSES, Constants.TABLE_USER_EXTERNAL_ENROLMENTS, propertyMap, null);
if (CollectionUtils.isEmpty(listOfMasterData)) {
Long date = Long.valueOf(transformData.get("completedon").asText());
String formatedDate = updateDateFormatFromTimestamp(date);
((ObjectNode) transformData).put("completedon", formatedDate);
((ObjectNode) transformData).put("partnerCode", partnerCode);
((ObjectNode) transformData).put("partnerId", entity.get("id").asText());
payloadValidation.validatePayload(Constants.PROGRESS_DATA_VALIDATION_FILE, transformData);
kafkaProducer.push(cbServerProperties.getTopic(), transformData);
} else {
log.info("Progress updated 100 for user {}", userId);
}
if (transformData.get("contentType").asText().equalsIgnoreCase(cbServerProperties.getCourseraEnrollmentListCourseType()) && transformData.get("isCompleted").asBoolean()) {
String extCourseId = transformData.get("courseid").asText();
JsonNode result = dataTransformUtility.callCiosReadApi(extCourseId, partnerId);
String courseId = result.path("content").get("contentId").asText();
String[] parts = transformData.get("userid").asText().split("@");
((ObjectNode) transformData).put("userid", parts[0]);
String userId = transformData.get("userid").asText();
log.info("courseId and userid {} {}", courseId, userId);
Map<String, Object> propertyMap = new HashMap<>();
propertyMap.put("userid", userId);
propertyMap.put("courseid", courseId);
propertyMap.put("progress", 100);
List<Map<String, Object>> listOfMasterData = cassandraOperation.getRecordsByProperties(Constants.KEYSPACE_SUNBIRD_COURSES, Constants.TABLE_USER_EXTERNAL_ENROLMENTS, propertyMap, null);
if (CollectionUtils.isEmpty(listOfMasterData)) {
Long date = Long.valueOf(transformData.get("completedon").asText());
String formatedDate = updateDateFormatFromTimestamp(date);
((ObjectNode) transformData).put("completedon", formatedDate);
((ObjectNode) transformData).put("partnerCode", partnerCode);
((ObjectNode) transformData).put("partnerId", partnerId);
payloadValidation.validatePayload(Constants.PROGRESS_DATA_VALIDATION_FILE, transformData);
kafkaProducer.push(cbServerProperties.getTopic(), transformData);
} else {
throw new CiosContentException(Constants.ERROR, "please update transformProgressJson in content partner db for partnerCode " + partnerCode, HttpStatus.INTERNAL_SERVER_ERROR);
log.info("Progress updated 100 for user {}", userId);
}
}
} catch (Exception e) {
Expand Down
Loading