Skip to content

Commit

Permalink
Merge pull request #2587 from julien-louis/notifications-cleanup
Browse files Browse the repository at this point in the history
auto clean of stale events
julien-louis authored Jan 30, 2025
2 parents aed4de5 + 0f3df37 commit 8128bd3
Showing 3 changed files with 50 additions and 30 deletions.
Original file line number Diff line number Diff line change
@@ -3,22 +3,27 @@
import java.util.Date;
import java.util.List;

import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.apache.commons.lang3.time.DateUtils;
import org.springframework.data.repository.CrudRepository;
import org.springframework.stereotype.Repository;

@Repository
public interface ShanoirEventRepository extends CrudRepository<ShanoirEvent, Long> {

public static final int TIMEOUT_DAYS = 7;

/**
* Find event by userId and eventType
*
* @param userId
* @param eventType
* @return a list of ShanoirEvents with given userID and event type
*/
List<ShanoirEvent> findByUserIdAndEventTypeIn(Long userId, List<String> eventType);
List<ShanoirEvent> findByUserIdAndEventTypeInAndLastUpdateGreaterThan(Long userId, List<String> eventType, Date earlyDateThreshold);

default List<ShanoirEvent> findByUserIdAndEventTypeInAndLastUpdateYoungerThan7Days(Long userId, List<String> eventType) {
return findByUserIdAndEventTypeInAndLastUpdateGreaterThan(userId, eventType, DateUtils.addDays(new Date(), -1 * TIMEOUT_DAYS));
}

/**
* Deletes all events older than a date.
Original file line number Diff line number Diff line change
@@ -1,16 +1,15 @@
package org.shanoir.ng.events;

import java.io.IOException;
import java.util.*;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.stream.Collectors;

import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateUtils;
import org.shanoir.ng.shared.event.ShanoirEventType;
import org.shanoir.ng.tasks.AsyncTaskApiController;
import org.shanoir.ng.tasks.UserSseEmitter;
import org.shanoir.ng.user.model.User;
import org.shanoir.ng.user.model.dto.UserDTO;
import org.shanoir.ng.user.repository.UserRepository;
import org.shanoir.ng.utils.KeycloakUtil;
import org.shanoir.ng.utils.Utils;
@@ -19,13 +18,8 @@
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.data.util.Pair;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.security.access.prepost.PostAuthorize;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.stereotype.Service;
import org.springframework.web.servlet.mvc.method.annotation.SseEmitter;

@@ -47,6 +41,8 @@ public class ShanoirEventsService {

private static final Logger LOG = LoggerFactory.getLogger(ShanoirEventsService.class);

public static final long INACTIVE_TIMEOUT = 5 * DateUtils.MILLIS_PER_MINUTE;

public void addEvent(ShanoirEvent event) {
// Call repository
repository.save(event);
@@ -64,26 +60,44 @@ public void addEvent(ShanoirEvent event) {
}
}

public List<ShanoirEvent> getEventsByUserIdAndTypeIn(Long userId, List<String> eventType) {
return Utils.toList(repository.findByUserIdAndEventTypeIn(userId, eventType));
}

public List<ShanoirEvent> getEventsByObjectIdAndTypeIn(String objectId, String eventType) {
return Utils.toList(repository.findByObjectIdAndEventType(objectId, eventType));
}
}

/**
* Get events younger than 7 days
*/
public List<ShanoirEventLight> getEventsByUserAndType(Long userId, String... eventType) {
List<String> list = new ArrayList<String>();
for (String type : eventType) {
list.add(type);
}
List<ShanoirEvent> dbEvents = Utils.toList(repository.findByUserIdAndEventTypeInAndLastUpdateYoungerThan7Days(userId, list));
List<ShanoirEventLight> events = new ArrayList<>();
for (ShanoirEvent event : Utils.toList(repository.findByUserIdAndEventTypeIn(userId, list))) {
cleanEvents(dbEvents);
for (ShanoirEvent event : dbEvents) {
events.add(event.toLightEvent());
}
return events;
}

/**
* Set inactive event that are still in a running status to error status
*/
private void cleanEvents(List<ShanoirEvent> events) {
Long now = new Date().getTime();
// set inactive tasks since > 5 min with a running status
List<ShanoirEvent> updatedEvents = events.stream().filter(event -> {
return (event.getStatus() == 2 || event.getStatus() == 5)
&& now - event.getLastUpdate().getTime() > INACTIVE_TIMEOUT;
}).map(event -> {
event.setStatus(-1);
event.setMessage("inactivity timeout, there must has been");
return event;
}).collect(Collectors.toList());
if (!updatedEvents.isEmpty()) repository.saveAll(updatedEvents);
}

/**
* Deletes everyday events older than 1 year.
*/
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
package org.shanoir.ng.tasks;

import java.io.IOException;
import java.util.*;
import java.util.stream.Collectors;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;

import org.apache.commons.lang3.time.DateUtils;
import org.shanoir.ng.events.ShanoirEvent;
import org.shanoir.ng.events.ShanoirEventLight;
import org.shanoir.ng.events.ShanoirEventsService;
@@ -36,14 +37,15 @@ public class AsyncTaskApiController implements AsyncTaskApi {
@Override
public ResponseEntity<List<ShanoirEventLight>> findTasks() {
Long userId = KeycloakUtil.getTokenUserId();

List<ShanoirEventLight> taskList = taskService.getEventsByUserAndType(userId, ShanoirEventType.IMPORT_DATASET_EVENT, ShanoirEventType.COPY_DATASET_EVENT, ShanoirEventType.EXECUTION_MONITORING_EVENT, ShanoirEventType.CHECK_QUALITY_EVENT, ShanoirEventType.SOLR_INDEX_ALL_EVENT, ShanoirEventType.DOWNLOAD_STATISTICS_EVENT, ShanoirEventType.DELETE_EXAMINATION_EVENT);

// Get only event with last updates < 7 days
Date now = new Date();
Long nowMinusSevenDays = now.getTime() - 7 * DateUtils.MILLIS_PER_DAY;
taskList = taskList.stream().filter(event -> event.getLastUpdate().getTime() > nowMinusSevenDays).collect(Collectors.toList());

List<ShanoirEventLight> taskList = taskService.getEventsByUserAndType(
userId,
ShanoirEventType.IMPORT_DATASET_EVENT,
ShanoirEventType.COPY_DATASET_EVENT,
ShanoirEventType.EXECUTION_MONITORING_EVENT,
ShanoirEventType.CHECK_QUALITY_EVENT,
ShanoirEventType.SOLR_INDEX_ALL_EVENT,
ShanoirEventType.DOWNLOAD_STATISTICS_EVENT,
ShanoirEventType.DELETE_EXAMINATION_EVENT);
// Order by last update date
Comparator<ShanoirEventLight> comparator = new Comparator<ShanoirEventLight>() {
@Override
@@ -52,7 +54,6 @@ public int compare(ShanoirEventLight event1, ShanoirEventLight event2) {
}
};
taskList.sort(comparator);

return new ResponseEntity<>(taskList, HttpStatus.OK);
}

0 comments on commit 8128bd3

Please sign in to comment.