From 42250c4a912e4792d7b8fc9b706c1293e1281fa9 Mon Sep 17 00:00:00 2001 From: shankaragoudab <140387294+shankaragoudab@users.noreply.github.com> Date: Mon, 6 Jan 2025 16:55:17 +0530 Subject: [PATCH 1/5] bug fix for api metrics --- .../cb/discussion/service/impl/DiscussionServiceImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/igot/cb/discussion/service/impl/DiscussionServiceImpl.java b/src/main/java/com/igot/cb/discussion/service/impl/DiscussionServiceImpl.java index cdb422c..c64d069 100644 --- a/src/main/java/com/igot/cb/discussion/service/impl/DiscussionServiceImpl.java +++ b/src/main/java/com/igot/cb/discussion/service/impl/DiscussionServiceImpl.java @@ -629,7 +629,7 @@ public ApiResponse createAnswerPost(JsonNode answerPostData, String token) { updateMetricsApiCall(Constants.DISCUSSION_ANSWER_POST); long redisTimer = System.currentTimeMillis(); DiscussionEntity discussionEntity = discussionRepository.findById(answerPostData.get(Constants.PARENT_DISCUSSION_ID).asText()).orElse(null); - updateMetricsDbOperation(Constants.DISCUSSION_ANSWER_POST, Constants.REDIS, Constants.READ, redisTimer); + updateMetricsDbOperation(Constants.DISCUSSION_ANSWER_POST, Constants.POSTGRES, Constants.READ, redisTimer); if (discussionEntity == null || !discussionEntity.getIsActive()) { returnErrorMsg(Constants.INVALID_PARENT_DISCUSSION_ID, HttpStatus.BAD_REQUEST, response, Constants.FAILED); return response; @@ -713,7 +713,7 @@ private void updateAnswerPostToDiscussion(DiscussionEntity discussionEntity, Str Map map = objectMapper.convertValue(jsonNode, Map.class); long esTime = System.currentTimeMillis(); esUtilService.addDocument(cbServerProperties.getDiscussionEntity(), Constants.INDEX_TYPE, discussionEntity.getDiscussionId(), map, cbServerProperties.getElasticDiscussionJsonPath()); - updateMetricsDbOperation(Constants.DISCUSSION_ANSWER_POST, Constants.ELASTICSEARCH, Constants.INSERT, esTime); + updateMetricsDbOperation(Constants.DISCUSSION_ANSWER_POST, Constants.ELASTICSEARCH, Constants.UPDATE_KEY, esTime); long redisTime = System.currentTimeMillis(); cacheService.putCache(Constants.DISCUSSION_CACHE_PREFIX + discussionEntity.getDiscussionId(), jsonNode); updateMetricsDbOperation(Constants.DISCUSSION_ANSWER_POST, Constants.REDIS, Constants.INSERT, redisTime); From 49853c24296667eaea5571c6ea0f4a16a07c5c6c Mon Sep 17 00:00:00 2001 From: shankaragoudab <140387294+shankaragoudab@users.noreply.github.com> Date: Tue, 7 Jan 2025 11:46:09 +0530 Subject: [PATCH 2/5] handling create post api using async --- .../service/impl/DiscussionServiceImpl.java | 30 +++++++++++++++---- .../com/igot/cb/pores/config/AsyncConfig.java | 24 +++++++++++++++ 2 files changed, 48 insertions(+), 6 deletions(-) create mode 100644 src/main/java/com/igot/cb/pores/config/AsyncConfig.java diff --git a/src/main/java/com/igot/cb/discussion/service/impl/DiscussionServiceImpl.java b/src/main/java/com/igot/cb/discussion/service/impl/DiscussionServiceImpl.java index c64d069..af27774 100644 --- a/src/main/java/com/igot/cb/discussion/service/impl/DiscussionServiceImpl.java +++ b/src/main/java/com/igot/cb/discussion/service/impl/DiscussionServiceImpl.java @@ -27,6 +27,8 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.redis.core.RedisTemplate; import org.springframework.http.HttpStatus; +import org.springframework.scheduling.annotation.Async; +import org.springframework.scheduling.annotation.EnableAsync; import org.springframework.stereotype.Service; import org.springframework.web.multipart.MultipartFile; import org.sunbird.cloud.storage.BaseStorageService; @@ -44,6 +46,7 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +@EnableAsync @Service @Slf4j public class DiscussionServiceImpl implements DiscussionService { @@ -119,16 +122,12 @@ public ApiResponse createDiscussion(JsonNode discussionDetails, String token) { ObjectNode jsonNode = objectMapper.createObjectNode(); jsonNode.setAll((ObjectNode) saveJsonEntity.getData()); Map map = objectMapper.convertValue(jsonNode, Map.class); - long esTime = System.currentTimeMillis(); - esUtilService.addDocument(cbServerProperties.getDiscussionEntity(), Constants.INDEX_TYPE, String.valueOf(id), map, cbServerProperties.getElasticDiscussionJsonPath()); - updateMetricsDbOperation(Constants.DISCUSSION_CREATE, Constants.ELASTICSEARCH, Constants.INSERT, esTime); - long redisTime = System.currentTimeMillis(); - cacheService.putCache("discussion_" + String.valueOf(id), jsonNode); - updateMetricsDbOperation(Constants.DISCUSSION_CREATE, Constants.REDIS, Constants.INSERT, redisTime); + map.put(Constants.CREATED_ON, currentTime); response.setResponseCode(HttpStatus.CREATED); response.getParams().setStatus(Constants.SUCCESS); response.setResult(map); + updateElasticsearchAndRedis(saveJsonEntity); } catch (Exception e) { log.error("Failed to create discussion: {}", e.getMessage(), e); createErrorResponse(response, Constants.FAILED_TO_CREATE_DISCUSSION, HttpStatus.INTERNAL_SERVER_ERROR, Constants.FAILED); @@ -897,4 +896,23 @@ private void updateMetricsApiCall(String apiName) { ApiMetricsTracker.recordApiCall(apiName); } } + + @Async + public void updateElasticsearchAndRedis(DiscussionEntity saveJsonEntity) { + try { + // Elasticsearch update + ObjectMapper objectMapper = new ObjectMapper(); + ObjectNode jsonNode = objectMapper.createObjectNode(); + jsonNode.setAll((ObjectNode) saveJsonEntity.getData()); + Map map = objectMapper.convertValue(jsonNode, Map.class); + esUtilService.addDocument(cbServerProperties.getDiscussionEntity(), Constants.INDEX_TYPE, saveJsonEntity.getDiscussionId(), map, cbServerProperties.getElasticDiscussionJsonPath()); + log.info("Updated Elasticsearch for discussion ID: {}", saveJsonEntity.getDiscussionId()); + + // Redis update + cacheService.putCache("discussion_" + saveJsonEntity.getDiscussionId(), jsonNode); + log.info("Updated Redis cache for discussion ID: {}", saveJsonEntity.getDiscussionId()); + } catch (Exception e) { + log.error("Failed to update Elasticsearch or Redis for discussion ID: {}", saveJsonEntity.getDiscussionId(), e); + } + } } diff --git a/src/main/java/com/igot/cb/pores/config/AsyncConfig.java b/src/main/java/com/igot/cb/pores/config/AsyncConfig.java new file mode 100644 index 0000000..dc791ca --- /dev/null +++ b/src/main/java/com/igot/cb/pores/config/AsyncConfig.java @@ -0,0 +1,24 @@ +package com.igot.cb.pores.config; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.scheduling.annotation.EnableAsync; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; + +@Configuration +@EnableAsync +public class AsyncConfig { + + @Bean + + public ThreadPoolTaskExecutor taskExecutor() { + + ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); + executor.setCorePoolSize(10); + executor.setMaxPoolSize(20); + executor.setQueueCapacity(500); + executor.setThreadNamePrefix("async-"); + executor.initialize(); + return executor; + } +} \ No newline at end of file From 009eb2157910890b93a800a4b5e200bc4cbfb4d0 Mon Sep 17 00:00:00 2001 From: shankaragoudab <140387294+shankaragoudab@users.noreply.github.com> Date: Tue, 7 Jan 2025 11:54:22 +0530 Subject: [PATCH 3/5] added dbmetrics to async method --- .../cb/discussion/service/impl/DiscussionServiceImpl.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/com/igot/cb/discussion/service/impl/DiscussionServiceImpl.java b/src/main/java/com/igot/cb/discussion/service/impl/DiscussionServiceImpl.java index af27774..e81dc7b 100644 --- a/src/main/java/com/igot/cb/discussion/service/impl/DiscussionServiceImpl.java +++ b/src/main/java/com/igot/cb/discussion/service/impl/DiscussionServiceImpl.java @@ -905,11 +905,15 @@ public void updateElasticsearchAndRedis(DiscussionEntity saveJsonEntity) { ObjectNode jsonNode = objectMapper.createObjectNode(); jsonNode.setAll((ObjectNode) saveJsonEntity.getData()); Map map = objectMapper.convertValue(jsonNode, Map.class); + long esTime = System.currentTimeMillis(); esUtilService.addDocument(cbServerProperties.getDiscussionEntity(), Constants.INDEX_TYPE, saveJsonEntity.getDiscussionId(), map, cbServerProperties.getElasticDiscussionJsonPath()); + updateMetricsDbOperation(Constants.DISCUSSION_CREATE, Constants.ELASTICSEARCH, Constants.INSERT, esTime); log.info("Updated Elasticsearch for discussion ID: {}", saveJsonEntity.getDiscussionId()); // Redis update + long redisTime = System.currentTimeMillis(); cacheService.putCache("discussion_" + saveJsonEntity.getDiscussionId(), jsonNode); + updateMetricsDbOperation(Constants.DISCUSSION_CREATE, Constants.REDIS, Constants.INSERT, redisTime); log.info("Updated Redis cache for discussion ID: {}", saveJsonEntity.getDiscussionId()); } catch (Exception e) { log.error("Failed to update Elasticsearch or Redis for discussion ID: {}", saveJsonEntity.getDiscussionId(), e); From f009c19ec27ecbbf42d845f3de0523aa0283666f Mon Sep 17 00:00:00 2001 From: shankaragoudab <140387294+shankaragoudab@users.noreply.github.com> Date: Tue, 7 Jan 2025 12:09:08 +0530 Subject: [PATCH 4/5] removed the @EnableAsync annotation in service class --- .../igot/cb/discussion/service/impl/DiscussionServiceImpl.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/com/igot/cb/discussion/service/impl/DiscussionServiceImpl.java b/src/main/java/com/igot/cb/discussion/service/impl/DiscussionServiceImpl.java index e81dc7b..d888fea 100644 --- a/src/main/java/com/igot/cb/discussion/service/impl/DiscussionServiceImpl.java +++ b/src/main/java/com/igot/cb/discussion/service/impl/DiscussionServiceImpl.java @@ -28,7 +28,6 @@ import org.springframework.data.redis.core.RedisTemplate; import org.springframework.http.HttpStatus; import org.springframework.scheduling.annotation.Async; -import org.springframework.scheduling.annotation.EnableAsync; import org.springframework.stereotype.Service; import org.springframework.web.multipart.MultipartFile; import org.sunbird.cloud.storage.BaseStorageService; @@ -46,7 +45,6 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; -@EnableAsync @Service @Slf4j public class DiscussionServiceImpl implements DiscussionService { From 2c37ab1ebbb0a7a7c04b924e291665a5defc9088 Mon Sep 17 00:00:00 2001 From: shankaragoudab <140387294+shankaragoudab@users.noreply.github.com> Date: Tue, 7 Jan 2025 15:04:29 +0530 Subject: [PATCH 5/5] handled async process in different class --- .../service/impl/DiscussionAsyncProcess.java | 47 +++++++++++++++++++ .../service/impl/DiscussionServiceImpl.java | 28 ++--------- 2 files changed, 50 insertions(+), 25 deletions(-) create mode 100644 src/main/java/com/igot/cb/discussion/service/impl/DiscussionAsyncProcess.java diff --git a/src/main/java/com/igot/cb/discussion/service/impl/DiscussionAsyncProcess.java b/src/main/java/com/igot/cb/discussion/service/impl/DiscussionAsyncProcess.java new file mode 100644 index 0000000..eaf973d --- /dev/null +++ b/src/main/java/com/igot/cb/discussion/service/impl/DiscussionAsyncProcess.java @@ -0,0 +1,47 @@ +package com.igot.cb.discussion.service.impl; + +import com.igot.cb.discussion.entity.DiscussionEntity; +import com.igot.cb.pores.cache.CacheService; +import com.igot.cb.pores.elasticsearch.service.EsUtilService; +import com.igot.cb.pores.util.CbServerProperties; +import com.igot.cb.pores.util.Constants; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.extern.slf4j.Slf4j; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.scheduling.annotation.Async; +import org.springframework.stereotype.Service; + +import java.util.Map; + +@Service +@Slf4j +public class DiscussionAsyncProcess { + @Autowired + private CacheService cacheService; + @Autowired + private EsUtilService esUtilService; + @Autowired + private CbServerProperties cbServerProperties; + + @Async + public void updateElasticsearchAndRedis(DiscussionEntity saveJsonEntity) { + try { + // Elasticsearch update + ObjectMapper objectMapper = new ObjectMapper(); + ObjectNode jsonNode = objectMapper.createObjectNode(); + jsonNode.setAll((ObjectNode) saveJsonEntity.getData()); + Map map = objectMapper.convertValue(jsonNode, Map.class); + esUtilService.addDocument(cbServerProperties.getDiscussionEntity(), Constants.INDEX_TYPE, saveJsonEntity.getDiscussionId(), map, cbServerProperties.getElasticDiscussionJsonPath()); + log.info("Updated Elasticsearch for discussion ID: {}", saveJsonEntity.getDiscussionId()); + + // Redis update + cacheService.putCache("discussion_" + saveJsonEntity.getDiscussionId(), jsonNode); + log.info("Updated Redis cache for discussion ID: {}", saveJsonEntity.getDiscussionId()); + } catch (Exception e) { + log.error("Failed to update Elasticsearch or Redis for discussion ID: {}", saveJsonEntity.getDiscussionId(), e); + } + } + +} diff --git a/src/main/java/com/igot/cb/discussion/service/impl/DiscussionServiceImpl.java b/src/main/java/com/igot/cb/discussion/service/impl/DiscussionServiceImpl.java index d888fea..ecc1575 100644 --- a/src/main/java/com/igot/cb/discussion/service/impl/DiscussionServiceImpl.java +++ b/src/main/java/com/igot/cb/discussion/service/impl/DiscussionServiceImpl.java @@ -27,7 +27,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.redis.core.RedisTemplate; import org.springframework.http.HttpStatus; -import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; import org.springframework.web.multipart.MultipartFile; import org.sunbird.cloud.storage.BaseStorageService; @@ -70,6 +69,8 @@ public class DiscussionServiceImpl implements DiscussionService { private AccessTokenValidator accessTokenValidator; @Autowired private RedisTemplate redisTemp; + @Autowired + private DiscussionAsyncProcess asyncService; @PostConstruct public void init() { @@ -125,7 +126,7 @@ public ApiResponse createDiscussion(JsonNode discussionDetails, String token) { response.setResponseCode(HttpStatus.CREATED); response.getParams().setStatus(Constants.SUCCESS); response.setResult(map); - updateElasticsearchAndRedis(saveJsonEntity); + asyncService.updateElasticsearchAndRedis(saveJsonEntity); } catch (Exception e) { log.error("Failed to create discussion: {}", e.getMessage(), e); createErrorResponse(response, Constants.FAILED_TO_CREATE_DISCUSSION, HttpStatus.INTERNAL_SERVER_ERROR, Constants.FAILED); @@ -894,27 +895,4 @@ private void updateMetricsApiCall(String apiName) { ApiMetricsTracker.recordApiCall(apiName); } } - - @Async - public void updateElasticsearchAndRedis(DiscussionEntity saveJsonEntity) { - try { - // Elasticsearch update - ObjectMapper objectMapper = new ObjectMapper(); - ObjectNode jsonNode = objectMapper.createObjectNode(); - jsonNode.setAll((ObjectNode) saveJsonEntity.getData()); - Map map = objectMapper.convertValue(jsonNode, Map.class); - long esTime = System.currentTimeMillis(); - esUtilService.addDocument(cbServerProperties.getDiscussionEntity(), Constants.INDEX_TYPE, saveJsonEntity.getDiscussionId(), map, cbServerProperties.getElasticDiscussionJsonPath()); - updateMetricsDbOperation(Constants.DISCUSSION_CREATE, Constants.ELASTICSEARCH, Constants.INSERT, esTime); - log.info("Updated Elasticsearch for discussion ID: {}", saveJsonEntity.getDiscussionId()); - - // Redis update - long redisTime = System.currentTimeMillis(); - cacheService.putCache("discussion_" + saveJsonEntity.getDiscussionId(), jsonNode); - updateMetricsDbOperation(Constants.DISCUSSION_CREATE, Constants.REDIS, Constants.INSERT, redisTime); - log.info("Updated Redis cache for discussion ID: {}", saveJsonEntity.getDiscussionId()); - } catch (Exception e) { - log.error("Failed to update Elasticsearch or Redis for discussion ID: {}", saveJsonEntity.getDiscussionId(), e); - } - } }