Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactor/37: 설문 및 댓글 이벤트 관련 API 수정 #45

Merged
merged 12 commits into from
Aug 18, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ build/
!**/src/test/**/build/
/src/main/resources/firebase/
/src/main/resources/firebase/suppin-a5657-firebase-adminsdk-s75m9-d65cc88029.json
/src/main/resources/drivers/


### STS ###
.apt_generated
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import com.cmc.suppin.event.crawl.controller.dto.CommentResponseDTO;
import com.cmc.suppin.event.crawl.service.CommentService;
import com.cmc.suppin.global.response.ApiResponse;
import com.cmc.suppin.global.response.ResponseCode;
import com.cmc.suppin.global.security.reslover.Account;
import com.cmc.suppin.global.security.reslover.CurrentAccount;
import io.swagger.v3.oas.annotations.Operation;
Expand Down Expand Up @@ -44,7 +45,7 @@ public ResponseEntity<ApiResponse<CommentResponseDTO.CrawledCommentListDTO>> get
}

@PostMapping("/draft-winners")
@Operation(summary = "당첨자 랜덤 추첨 결과 리스트 조회 API(댓글 이벤트)", description = "주어진 조건에 따라 이벤트의 당첨자를 추첨합니다.")
@Operation(summary = "당첨자 랜덤 추첨 결과 조회 API(댓글 이벤트)", description = "주어진 조건에 따라 이벤트의 당첨자를 추첨합니다.")
public ResponseEntity<ApiResponse<CommentResponseDTO.WinnerResponseDTO>> drawWinners(
@RequestBody @Valid CommentRequestDTO.WinnerRequestDTO request,
@CurrentAccount Account account) {
Expand All @@ -61,4 +62,11 @@ public ResponseEntity<ApiResponse<List<CommentResponseDTO.CommentDetailDTO>>> ge
List<CommentResponseDTO.CommentDetailDTO> filteredWinners = commentService.getCommentsByKeyword(eventId, keyword, account.userId());
return ResponseEntity.ok(ApiResponse.of(filteredWinners));
}

@DeleteMapping("/")
@Operation(summary = "댓글 이벤트 당첨자 리스트 삭제 API(당첨자 재추첨 시, 기존 당첨자 리스트를 삭제한 후 진행해야 합니다.", description = "모든 당첨자들의 isWinner 값을 false로 변경합니다.")
public ResponseEntity<ApiResponse<Void>> deleteWinners(@RequestParam("eventId") Long eventId, @CurrentAccount Account account) {
commentService.deleteWinners(eventId);
return ResponseEntity.ok(ApiResponse.of(ResponseCode.SUCCESS));
}
}
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
package com.cmc.suppin.event.crawl.controller.dto;

import jakarta.validation.constraints.Pattern;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Getter;
Expand All @@ -27,14 +26,10 @@ public static class CommentListRequestDTO {
@Builder
public static class WinnerRequestDTO {
private Long eventId;
private int winnerCount;
private int minLength;

@Pattern(regexp = "\\d{4}\\. \\d{2}\\. \\d{2} \\d{2}:\\d{2}", message = "날짜 형식은 yyyy. MM. dd HH:mm 이어야 합니다.")
private Integer winnerCount;
private Integer minLength;
private String startDate;
@Pattern(regexp = "\\d{4}\\. \\d{2}\\. \\d{2} \\d{2}:\\d{2}", message = "날짜 형식은 yyyy. MM. dd HH:mm 이어야 합니다.")
private String endDate;

private List<String> keywords;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,19 @@ public static class WinnerResponseDTO {
private List<CommentDetailDTO> winners;
}

@Getter
@NoArgsConstructor
@AllArgsConstructor
@Builder
public static class CommentEventWinnersWithCriteria {
private Integer winnerCount;
private Integer minLength;
private String startDate;
private String endDate;
private List<String> keywords;
private List<CommentEventWinners> winners;
}

@Getter
@NoArgsConstructor
@AllArgsConstructor
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,4 +26,7 @@ public interface CommentRepository extends JpaRepository<Comment, Long> {
List<Comment> findByEventIdAndCommentTextContaining(Long eventId, String keyword);

List<Comment> findByEventIdAndIsWinnerTrue(Long eventId);


boolean existsByUrlAndEventId(String url, Long eventId);
}
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ public CommentResponseDTO.CrawledCommentListDTO getComments(Long eventId, String

int totalComments = commentRepository.countByEventIdAndUrl(eventId, url);

String crawlTime = comments.isEmpty() ? "" : comments.getContent().get(0).getCrawlTime().format(DateTimeFormatter.ofPattern("yyyy. MM. dd HH:mm"));
String crawlTime = comments.isEmpty() ? "" : comments.getContent().get(0).getCrawlTime().format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm"));

return CommentConverter.toCommentListDTO(comments.getContent(), crawlTime, totalComments);
}
Expand All @@ -62,15 +62,35 @@ public CommentResponseDTO.WinnerResponseDTO drawWinners(CommentRequestDTO.Winner
Event event = eventRepository.findByIdAndMemberId(request.getEventId(), member.getId())
.orElseThrow(() -> new IllegalArgumentException("Event not found"));

DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy. MM. dd HH:mm");
LocalDateTime startDateTime = LocalDateTime.parse(request.getStartDate(), dateTimeFormatter);
LocalDateTime endDateTime = LocalDateTime.parse(request.getEndDate(), dateTimeFormatter);
// 당첨자 선별 조건 Event 엔티티에 저장
event.setSelectionCriteria(request);
eventRepository.save(event);

List<Comment> comments = commentRepository.findByEventIdAndCommentDateBetween(event.getId(), startDateTime, endDateTime);
DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm");

LocalDateTime startDateTime = null;
LocalDateTime endDateTime = null;

// 날짜 필터링을 위한 조건 설정
if (request.getStartDate() != null && !request.getStartDate().isEmpty()) {
startDateTime = LocalDateTime.parse(request.getStartDate(), dateTimeFormatter);
}
if (request.getEndDate() != null && !request.getEndDate().isEmpty()) {
endDateTime = LocalDateTime.parse(request.getEndDate(), dateTimeFormatter);
}

List<Comment> comments;

// 날짜 조건이 있을 경우에만 필터링, 없으면 전체 댓글을 조회
if (startDateTime != null && endDateTime != null) {
comments = commentRepository.findByEventIdAndCommentDateBetween(event.getId(), startDateTime, endDateTime);
} else {
comments = commentRepository.findByEventId(event.getId());
}

// 키워드 필터링(OR 로직) 및 minLength 필터링 추가
List<Comment> filteredComments = comments.stream()
.filter(comment -> request.getKeywords().stream().anyMatch(keyword -> comment.getCommentText().contains(keyword)))
.filter(comment -> request.getKeywords().isEmpty() || request.getKeywords().stream().anyMatch(keyword -> comment.getCommentText().contains(keyword)))
.filter(comment -> comment.getCommentText().length() >= request.getMinLength())
.collect(Collectors.toList());

Expand Down Expand Up @@ -109,15 +129,37 @@ public List<CommentResponseDTO.CommentDetailDTO> filterWinnersByKeyword(List<Com
.collect(Collectors.toList());
}

public List<CommentResponseDTO.CommentEventWinners> getCommentEventWinners(Long eventId, String userId) {
public CommentResponseDTO.CommentEventWinnersWithCriteria getCommentEventWinnersWithCriteria(Long eventId, String userId) {
Member member = memberRepository.findByUserIdAndStatusNot(userId, UserStatus.DELETED)
.orElseThrow(() -> new IllegalArgumentException("Member not found"));

Event event = eventRepository.findByIdAndMemberId(eventId, member.getId())
.orElseThrow(() -> new IllegalArgumentException("Event not found or does not belong to the user"));

List<Comment> winners = commentRepository.findByEventIdAndIsWinnerTrue(eventId);

return winners.stream()
List<CommentResponseDTO.CommentEventWinners> winnerList = winners.stream()
.map(CommentConverter::toCommentEventWinners)
.collect(Collectors.toList());

return CommentResponseDTO.CommentEventWinnersWithCriteria.builder()
.winners(winnerList)
.winnerCount(event.getWinnerCount())
.minLength(event.getMinLength())
.startDate(event.getSelectionStartDate() != null ? event.getSelectionStartDate().format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm")) : null)
.endDate(event.getSelectionEndDate() != null ? event.getSelectionEndDate().format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm")) : null)
.keywords(event.getKeywords())
.build();
}


public void deleteWinners(Long eventId) {
List<Comment> comments = commentRepository.findByEventIdAndIsWinnerTrue(eventId);

for (Comment comment : comments) {
comment.setIsWinner(false);
commentRepository.save(comment);
}
}

}
78 changes: 47 additions & 31 deletions src/main/java/com/cmc/suppin/event/crawl/service/CrawlService.java
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,10 @@
import org.springframework.transaction.annotation.Transactional;

import java.time.LocalDateTime;
import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.*;

@Service
@Slf4j
Expand All @@ -48,7 +48,7 @@ public String checkExistingComments(String url, String userId) {

List<Comment> existingComments = commentRepository.findByUrl(url);
if (!existingComments.isEmpty()) {
LocalDateTime firstCommentDate = existingComments.get(0).getCrawlTime();
ZonedDateTime firstCommentDate = existingComments.get(0).getCrawlTime().atZone(ZoneId.of("Asia/Seoul"));
DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm");
return "동일한 URL의 댓글을 " + firstCommentDate.format(formatter) + " 일자에 수집한 이력이 있습니다.";
}
Expand All @@ -64,23 +64,17 @@ public CrawlResponseDTO.CrawlResultDTO crawlYoutubeComments(String url, Long eve
.orElseThrow(() -> new IllegalArgumentException("Event not found"));

if (forceUpdate) {
// 기존 댓글 삭제
commentRepository.deleteByUrlAndEventId(url, eventId);

// 삭제 후, 확인을 위한 로그 출력 또는 추가 검증
List<Comment> deletedComments = commentRepository.findByUrlAndEventId(url, eventId);
if (!deletedComments.isEmpty()) {
if (commentRepository.existsByUrlAndEventId(url, eventId)) {
throw new RuntimeException("기존 댓글 삭제에 실패했습니다.");
}
} else {
// 기존 댓글이 존재하는 경우: 크롤링을 중지하고 예외를 던집니다.
List<Comment> existingComments = commentRepository.findByUrlAndEventId(url, eventId);
if (!existingComments.isEmpty()) {
if (commentRepository.existsByUrlAndEventId(url, eventId)) {
throw new CrawlException(CrawlErrorCode.DUPLICATE_URL);
}
}

// 크롤링 코드 실행
String chromeDriverPath = System.getenv("CHROME_DRIVER_PATH");
if (chromeDriverPath != null && !chromeDriverPath.isEmpty()) {
System.setProperty("webdriver.chrome.driver", chromeDriverPath);
Expand All @@ -99,36 +93,52 @@ public CrawlResponseDTO.CrawlResultDTO crawlYoutubeComments(String url, Long eve
options.addArguments("--disable-infobars");
options.addArguments("--disable-browser-side-navigation");
options.addArguments("--disable-software-rasterizer");
options.addArguments("--blink-settings=imagesEnabled=false"); // 이미지 로딩 비활성화
options.setPageLoadStrategy(PageLoadStrategy.NORMAL); // 페이지 로드 전략 설정
options.addArguments("--blink-settings=imagesEnabled=false");
options.setPageLoadStrategy(PageLoadStrategy.NORMAL);

WebDriver driver = new ChromeDriver(options);
driver.get(url);

Set<String> uniqueComments = new HashSet<>();

try {
Thread.sleep(5000); // 초기 로딩 대기
Thread.sleep(5000);

long endTime = System.currentTimeMillis() + 600000; // 스크롤 시간을 10분으로 설정 (600,000ms)
JavascriptExecutor jsExecutor = (JavascriptExecutor) driver;

int previousCommentCount = 0;
int currentCommentCount;
Queue<Long> heightQueue = new LinkedList<>();
int maxQueueSize = 50;
int enqueueCount = 0;
int retryCount = 0;

while (System.currentTimeMillis() < endTime) {
while (true) {
jsExecutor.executeScript("window.scrollTo(0, document.documentElement.scrollHeight);");
Thread.sleep(100);

Thread.sleep(3000); // 3초 대기
long newPageHeight = (long) jsExecutor.executeScript("return document.documentElement.scrollHeight");

if (enqueueCount > maxQueueSize) {
break;
}

if (heightQueue.isEmpty()) {
heightQueue.offer(newPageHeight);
enqueueCount++;
} else {
if (heightQueue.peek().equals(newPageHeight)) {
heightQueue.offer(newPageHeight);
enqueueCount++;
} else {
heightQueue.clear();
heightQueue.offer(newPageHeight);
enqueueCount = 1;
}
}

String pageSource = driver.getPageSource();
Document doc = Jsoup.parse(pageSource);
Elements comments = doc.select("ytd-comment-thread-renderer");

currentCommentCount = comments.size();

LocalDateTime crawlTime = LocalDateTime.now();

for (Element commentElement : comments) {
String author = commentElement.select("#author-text span").text();
String text = commentElement.select("#content yt-attributed-string#content-text").text();
Expand All @@ -137,27 +147,33 @@ public CrawlResponseDTO.CrawlResultDTO crawlYoutubeComments(String url, Long eve
if (!uniqueComments.contains(text)) {
uniqueComments.add(text);

// 엔티티 저장
LocalDateTime actualCommentDate = DateConverter.convertRelativeTime(time);
Comment comment = CommentConverter.toCommentEntity(author, text, actualCommentDate, url, event);
comment.setCrawlTime(crawlTime);

// Set crawl time with timezone
comment.setCrawlTime(ZonedDateTime.now(ZoneId.of("Asia/Seoul")).toLocalDateTime());
commentRepository.save(comment);
}
}

// 더 이상 새로운 댓글이 없을 때, 크롤링 종료
if (currentCommentCount == previousCommentCount) {
break; // 새로운 댓글이 로드되지 않으면 루프를 종료합니다.
if (comments.size() == uniqueComments.size()) {
if (retryCount >= 3) {
break;
}
retryCount++;
} else {
retryCount = 0;
}
previousCommentCount = currentCommentCount;
}
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
e.printStackTrace();
} finally {
driver.quit();
}

return CommentConverter.toCrawlResultDTO(LocalDateTime.now(), uniqueComments.size());
}

}


Original file line number Diff line number Diff line change
Expand Up @@ -74,22 +74,23 @@ public ResponseEntity<ApiResponse<Void>> deleteEvent(@PathVariable("eventId") Lo
}

@GetMapping("/comment-winners")
@Operation(summary = "댓글 이벤트 당첨자 조회 API", description = "댓글 이벤트의 당첨자 리스트를 조회합니다.")
public ResponseEntity<ApiResponse<List<CommentResponseDTO.CommentEventWinners>>> getCommentEventWinners(
@Operation(summary = "댓글 이벤트 당첨자 조회 API", description = "댓글 이벤트의 당첨자 리스트와 선별 조건을 조회합니다.")
public ResponseEntity<ApiResponse<CommentResponseDTO.CommentEventWinnersWithCriteria>> getCommentEventWinners(
@RequestParam("eventId") Long eventId,
@CurrentAccount Account account) {

List<CommentResponseDTO.CommentEventWinners> winners = commentService.getCommentEventWinners(eventId, account.userId());
return ResponseEntity.ok(ApiResponse.of(winners));
CommentResponseDTO.CommentEventWinnersWithCriteria winnersWithCriteria = commentService.getCommentEventWinnersWithCriteria(eventId, account.userId());
return ResponseEntity.ok(ApiResponse.of(winnersWithCriteria));
}


@GetMapping("/survey-winners")
@Operation(summary = "설문 이벤트 당첨자 조회 API", description = "설문 이벤트의 당첨자 리스트를 조회합니다.")
public ResponseEntity<ApiResponse<List<SurveyResponseDTO.SurveyEventWinners>>> getSurveyEventWinners(
@Operation(summary = "설문 이벤트 당첨자 조회 API", description = "설문 이벤트의 당첨자 리스트 및 선별 조건을 조회합니다.")
public ResponseEntity<ApiResponse<SurveyResponseDTO.SurveyEventWinnersResponse>> getSurveyEventWinners(
@RequestParam("surveyId") Long surveyId,
@CurrentAccount Account account) {

List<SurveyResponseDTO.SurveyEventWinners> winners = surveyService.getSurveyEventWinners(surveyId, account.userId());
return ResponseEntity.ok(ApiResponse.of(winners));
SurveyResponseDTO.SurveyEventWinnersResponse response = surveyService.getSurveyEventWinners(surveyId, account.userId());
return ResponseEntity.ok(ApiResponse.of(response));
}
}
Loading
Loading