From 6389876ce6883ed10f39bd18b05fa1003541223e Mon Sep 17 00:00:00 2001 From: pcm Date: Thu, 23 Jan 2025 17:04:13 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9EFastExcel=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=E7=9A=84=E5=B7=A5=E7=A8=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 6 +- springboot-excel/pom.xml | 26 ++- .../src/main/java/com/pancm/Application.java | 2 +- .../com/pancm/controller/UserController.java | 29 ++- .../src/main/java/com/pancm/dao/UserDao.java | 4 + .../com/pancm/excel/UserDataListener.java | 107 +++++++++++ .../java/com/pancm/service/IUserService.java | 20 +- .../pancm/service/impl/UserServiceImpl.java | 177 ++++++++++++++---- .../java/com/pancm/vo/excel/UserExcelVO.java | 2 + .../src/main/resources/application.yml | 4 + .../resources/pancm/mapper/UserMapper.xml | 8 +- 11 files changed, 329 insertions(+), 56 deletions(-) create mode 100644 springboot-excel/src/main/java/com/pancm/excel/UserDataListener.java diff --git a/README.md b/README.md index a713f44..41afa72 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,11 @@ springBoot-study 是本人在学习SpringBoot的一些工程！ - [springboot-guacamole](https://github.com/xuwujing/springBoot-study/tree/master/springboot-guacamole):SpringBoot结合guacamole实现堡垒机远程桌面控制功能。 -- [springboot-nacos](https://github.com/xuwujing/springBoot-study/tree/master/springboot-guacamole):SpringBoot结合Nacos实现配置中心的示例。 +- [springboot-nacos](https://github.com/xuwujing/springBoot-study/tree/master/springboot-nacos):SpringBoot结合Nacos实现配置中心的示例。 + +- [springboot-saToken](https://github.com/xuwujing/springBoot-study/tree/master/springboot-saToken):SpringBoot结合saToken实现用户登录，权限控制等功能，并提供sql。 + +- [springboot-excel](https://github.com/xuwujing/springBoot-study/tree/master/springboot-excel):SpringBoot结合FastExcel(EasyExcel的升级版)实现大量数据导入导出示例。 ## 文章列表 [SpringBoot系列博客:](https://www.cnblogs.com/xuwujing/category/1145997.html) diff --git a/springboot-excel/pom.xml b/springboot-excel/pom.xml index fe37afe..28b0ccd 100644 --- a/springboot-excel/pom.xml +++ b/springboot-excel/pom.xml @@ -67,6 +67,12 @@ org.springframework.boot spring-boot-starter-web + + + log4j-api + org.apache.logging.log4j + + org.springframework.boot @@ -107,6 +113,12 @@ tk.mybatis mapper-spring-boot-starter ${tk.mybatis.boot.version} + + + mybatis-spring-boot-starter + org.mybatis.spring.boot + + com.baomidou @@ -208,9 +220,21 @@ cn.idev.excel fastexcel - 1.1.0 + 1.0.0 + + + + log4j-api + org.apache.logging.log4j + + + + org.apache.logging.log4j + log4j-api + 2.17.1 + diff --git a/springboot-excel/src/main/java/com/pancm/Application.java b/springboot-excel/src/main/java/com/pancm/Application.java index 3e83b9d..0d9e730 100644 --- a/springboot-excel/src/main/java/com/pancm/Application.java +++ b/springboot-excel/src/main/java/com/pancm/Application.java @@ -29,7 +29,7 @@ public static void main(String[] args) throws UnknownHostException { String ip = InetAddress.getLocalHost().getHostAddress(); String port = env.getProperty("server.port"); String path = env.getProperty("server.servlet.context-path"); - String name = env.getProperty("server.application.name"); + String name = env.getProperty("spring.application.name"); log.info("\n----------------------------------------------------------\n\t" + "Application " + name + " is running! Access URLs:\n\t" + "Local: \t\thttp://localhost:" + port + path + "/\n\t" + diff --git a/springboot-excel/src/main/java/com/pancm/controller/UserController.java b/springboot-excel/src/main/java/com/pancm/controller/UserController.java index a37d69e..19cbd1b 100644 --- a/springboot-excel/src/main/java/com/pancm/controller/UserController.java +++ b/springboot-excel/src/main/java/com/pancm/controller/UserController.java @@ -13,6 +13,7 @@ import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.multipart.MultipartFile; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; @@ -102,17 +103,33 @@ public ApiResult view( @RequestParam("id") Long id) { } - @ApiOperation(value = "/export", notes = "用户导出成Excel") + @ApiOperation(value = "用户表生成数据",notes = "用户表生成数据") + @RequestMapping(value = "generatedData", method = RequestMethod.GET) + public ApiResult generatedData( @RequestParam("size") Integer size) { + return userService.generatedData(size); + } + + + + @ApiOperation(value = "用户导出成Excel", notes = "用户导出成Excel") @RequestMapping(value = "/export", method = {RequestMethod.GET}) - public ApiResult export(@RequestBody UserVO userVO, HttpServletRequest request, HttpServletResponse response){ - return userService.export(userVO, request, response); + public void export(@RequestBody UserVO userVO, HttpServletRequest request, HttpServletResponse response){ + userService.export(userVO, request, response); } - @ApiOperation(value = "/exportBatch", notes = "用户导出成Excel,量大的时候") + @ApiOperation(value = "用户导出成Excel,量大的时候", notes = "用户导出成Excel,量大的时候") @RequestMapping(value = "/exportBatch", method = {RequestMethod.GET}) - public ApiResult exportBatch(@RequestBody UserVO userVO, HttpServletRequest request, HttpServletResponse response){ - return userService.exportBatch(userVO, request, response); + public void exportBatch(@RequestBody UserVO userVO, HttpServletRequest request, HttpServletResponse response){ + userService.exportBatch(userVO, request, response); } + @ApiOperation(value = "用户导入Excel", notes = "用户导入Excel") + @PostMapping("/import") + public ApiResult importExcel(@RequestParam("file") MultipartFile file, + HttpServletRequest request) { + return userService.importExcel(file, request); + } + + } diff --git a/springboot-excel/src/main/java/com/pancm/dao/UserDao.java b/springboot-excel/src/main/java/com/pancm/dao/UserDao.java index a4813ff..d2b817e 100644 --- a/springboot-excel/src/main/java/com/pancm/dao/UserDao.java +++ b/springboot-excel/src/main/java/com/pancm/dao/UserDao.java @@ -44,6 +44,10 @@ public interface UserDao { */ List queryAll(UserVO userVO); + /** + * 查询总数 + * @return + */ int queryAllCount(UserVO userVO); /** * 新增数据 diff --git a/springboot-excel/src/main/java/com/pancm/excel/UserDataListener.java b/springboot-excel/src/main/java/com/pancm/excel/UserDataListener.java new file mode 100644 index 0000000..fd1b2b5 --- /dev/null +++ b/springboot-excel/src/main/java/com/pancm/excel/UserDataListener.java @@ -0,0 +1,107 @@ +package com.pancm.excel; + +import cn.idev.excel.context.AnalysisContext; +import cn.idev.excel.read.listener.ReadListener; + +import com.pancm.dao.UserDao; +import com.pancm.model.User; +import com.pancm.vo.excel.UserExcelVO; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.BeanUtils; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; +import org.springframework.stereotype.Component; + +import javax.annotation.Resource; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; + + +/** + * @author pancm + * @Title: Application + * @Description: 实现 ReadListener 接口，设置读取数据的操作 + * @Version:1.0.0 + * @Since:jdk1.8 + * @Date 2025/01/23 + **/ +@Slf4j +@Component +public class UserDataListener implements ReadListener { + + @Resource + private UserDao userDao; + + @Resource(name = "pcmTaskExecutor") + private ThreadPoolTaskExecutor pcmTaskExecutor; + + private static final int BATCH_COUNT = 1000; + + /** + * 缓存的数据 + */ + private List cachedDataList = Collections.synchronizedList(new ArrayList<>()); + private AtomicInteger count = new AtomicInteger(0); // 仅用于统计总记录数 + + /** + * 记录开始时间 + */ + private long startTime; + + /** + * @param userExcelVO + * @param analysisContext + */ + @Override + public void invoke(UserExcelVO userExcelVO, AnalysisContext analysisContext) { + if (count.get() == 0) { + // 记录开始时间 + startTime = System.currentTimeMillis(); + } + + cachedDataList.add(userExcelVO); + if (cachedDataList.size() >= BATCH_COUNT) { + flushData(); + } + } + + private synchronized void flushData() { + if (!cachedDataList.isEmpty()) { + List dataToFlush = new ArrayList<>(cachedDataList); + cachedDataList.clear(); + pcmTaskExecutor.submit(() -> { + try { + saveData(dataToFlush); + } catch (Exception e) { + log.error("数据保存出现异常：", e); + } + }); + } + } + + private void saveData(List dataList) { + log.info("{}条数据，开始存储数据库！", dataList.size()); + List entities = dataList.stream() + .map(userVO -> { + User user = new User(); + BeanUtils.copyProperties(userVO, user); + user.setSex(Objects.equals(userVO.getSex(), "男") ? 1 : 2); + return user; + }) + .collect(Collectors.toList()); + userDao.insertBatch(entities); + count.addAndGet(dataList.size()); + } + + @Override + public void doAfterAllAnalysed(AnalysisContext context) { + // 处理剩余未提交的数据 + flushData(); + long endTime = System.currentTimeMillis(); + log.info("所有数据解析完成！共解析:{}条数据! 耗时:{}毫秒", count.get(), (endTime - startTime)); + count.set(0); + } +} \ No newline at end of file diff --git a/springboot-excel/src/main/java/com/pancm/service/IUserService.java b/springboot-excel/src/main/java/com/pancm/service/IUserService.java index 489c032..6c93fa4 100644 --- a/springboot-excel/src/main/java/com/pancm/service/IUserService.java +++ b/springboot-excel/src/main/java/com/pancm/service/IUserService.java @@ -2,6 +2,7 @@ import com.pancm.vo.UserVO; import com.pancm.vo.ApiResult; +import org.springframework.web.multipart.MultipartFile; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; @@ -67,7 +68,7 @@ public interface IUserService { * @param response * @return */ - ApiResult export(UserVO userVO, HttpServletRequest request, HttpServletResponse response); + void export(UserVO userVO, HttpServletRequest request, HttpServletResponse response); /** @@ -78,5 +79,20 @@ public interface IUserService { * @param response * @return */ - ApiResult exportBatch(UserVO userVO, HttpServletRequest request, HttpServletResponse response); + void exportBatch(UserVO userVO, HttpServletRequest request, HttpServletResponse response); + + /** + * 生成数据 + * @param size + * @return + */ + ApiResult generatedData(Integer size); + + /** + * 导入数据 + * @param file + * @param request + * @return + */ + ApiResult importExcel(MultipartFile file, HttpServletRequest request); } diff --git a/springboot-excel/src/main/java/com/pancm/service/impl/UserServiceImpl.java b/springboot-excel/src/main/java/com/pancm/service/impl/UserServiceImpl.java index 77d72c5..1bd9c8d 100644 --- a/springboot-excel/src/main/java/com/pancm/service/impl/UserServiceImpl.java +++ b/springboot-excel/src/main/java/com/pancm/service/impl/UserServiceImpl.java @@ -9,6 +9,7 @@ import cn.idev.excel.write.metadata.WriteSheet; import cn.idev.excel.write.style.column.LongestMatchColumnWidthStyleStrategy; import com.alibaba.fastjson.JSONObject; +import com.pancm.excel.UserDataListener; import com.pancm.vo.UserVO; import com.pancm.model.User; import com.pancm.dao.UserDao; @@ -21,16 +22,22 @@ import com.pancm.vo.excel.UserExcelVO; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; import org.springframework.stereotype.Service; import org.springframework.beans.BeanUtils; +import org.springframework.web.multipart.MultipartFile; + import javax.annotation.Resource; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.io.InputStream; import java.io.OutputStream; import java.net.URLEncoder; import java.util.ArrayList; import java.util.List; +import java.util.Objects; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @@ -38,13 +45,13 @@ /** -* @Title: 用户表(User)表服务实现类 -* @Description: -* @Version:1.0.0 -* @Since:jdk1.8 -* @author pancm -* @date 2024-01-15 15:27:04 -*/ + * @author pancm + * @Title: 用户表(User)表服务实现类 + * @Description: + * @Version:1.0.0 + * @Since:jdk1.8 + * @date 2024-01-15 15:27:04 + */ @Service("userService") @Slf4j public class UserServiceImpl implements IUserService { @@ -53,6 +60,11 @@ public class UserServiceImpl implements IUserService { @Resource(name = "pcmTaskExecutor") private ThreadPoolTaskExecutor pcmTaskExecutor; + + @Resource + private UserDataListener userDataListener; + + /** * 通过ID查询单条数据 * @@ -64,22 +76,22 @@ public UserVO queryById(Long id) { return this.userDao.queryById(id); } - - /** + + /** * 根据条件查询 * * @return 实例对象的集合 */ @Override public ApiResult list(UserVO user) { - int pageNum = user.getPageNum(); - int pageSize = user.getPageSize(); - Page page = PageHelper.startPage(pageNum, pageSize); - List result = userDao.queryAll(user); - return ApiResult.success(new PageResult<>(page.getTotal(), result, pageSize, pageNum)); - + int pageNum = user.getPageNum(); + int pageSize = user.getPageSize(); + Page page = PageHelper.startPage(pageNum, pageSize); + List result = userDao.queryAll(user); + return ApiResult.success(new PageResult<>(page.getTotal(), result, pageSize, pageNum)); + } - + /** * 新增数据 * @@ -88,8 +100,8 @@ public ApiResult list(UserVO user) { */ @Override public int insert(UserVO userVO) { - User user = new User(); - BeanUtils.copyProperties(userVO,user); + User user = new User(); + BeanUtils.copyProperties(userVO, user); return userDao.insert(user); } @@ -101,8 +113,8 @@ public int insert(UserVO userVO) { */ @Override public int update(UserVO userVO) { - User user = new User(); - BeanUtils.copyProperties(userVO,user); + User user = new User(); + BeanUtils.copyProperties(userVO, user); return userDao.update(user); } @@ -119,7 +131,8 @@ public boolean deleteById(Long id) { /** * 导出数据 - * 注:单线程导出,如果数据量过大，会耗时较长 + * 注:单线程导出,如果数据量过大，会耗时较长 + * * @param userVO * @param request * @param response @@ -127,10 +140,11 @@ public boolean deleteById(Long id) { */ @SneakyThrows @Override - public ApiResult export(UserVO userVO, HttpServletRequest request, HttpServletResponse response) { + public void export(UserVO userVO, HttpServletRequest request, HttpServletResponse response) { + userVO.setPageSize(10000); List result = userDao.queryAll(userVO); if (result.isEmpty()) { - return ApiResult.error("没有数据可导出"); + return; } String fileName = "用户表_" + DateUtils.DATE_FORMAT_14 + ".xlsx"; @@ -151,10 +165,9 @@ public ApiResult export(UserVO userVO, HttpServletRequest request, HttpServletRe excelWriter.write(excelData, writeSheet); } catch (Exception e) { log.error("导出失败", e); - return ApiResult.error("导出失败"); } - return ApiResult.success("导出成功"); + } private List getExcelData(List userVOList) { @@ -162,17 +175,17 @@ private List getExcelData(List userVOList) { .map(userVO -> { UserExcelVO userExcelVO = new UserExcelVO(); BeanUtils.copyProperties(userVO, userExcelVO); + userExcelVO.setSex(Objects.equals(userVO.getSex(), 1) ? "男" : "女"); return userExcelVO; }) .collect(Collectors.toList()); } - - /** * 导出数据 - * 注:使用多线程，可以提高导出效率 + * 注:使用多线程，可以提高导出效率,此方案是导出到一个sheet，如果有多个sheet，按需进行处理即可 + * * @param userVO * @param request * @param response @@ -180,10 +193,13 @@ private List getExcelData(List userVOList) { */ @SneakyThrows @Override - public ApiResult exportBatch(UserVO userVO, HttpServletRequest request, HttpServletResponse response) { - int listCount = userDao.queryAllCount(userVO); + public void exportBatch(UserVO userVO, HttpServletRequest request, HttpServletResponse response) { + userVO.setPageNum(null); + userVO.setPageSize(null); + long startTime = System.currentTimeMillis(); + int listCount = userDao.queryAllCount(userVO); if (listCount == 0) { - return ApiResult.error("没有数据可导出"); + return; } String fileName = "用户表大量数据_" + DateUtils.DATE_FORMAT_14 + ".xlsx"; response.setContentType("application/vnd.ms-excel"); @@ -213,7 +229,7 @@ public ApiResult exportBatch(UserVO userVO, HttpServletRequest request, HttpServ pcmTaskExecutor.setThreadPriority(priority); pcmTaskExecutor.submit(() -> { try { - handlerExport(finalPageNum, pageSize, userVO, excelWriter, writeSheet, count); + handlerExport(finalPageNum, pageSize, userVO, excelWriter, writeSheet, count); } catch (Exception e) { log.error("导出失败！ 请求参数:{}", userVO, e); } finally { @@ -230,29 +246,108 @@ public ApiResult exportBatch(UserVO userVO, HttpServletRequest request, HttpServ log.error("导出任务未能在超时时间内完成!超过了:{}秒!", timeout); } } catch (InterruptedException e) { - log.error("报导出等待失败！ 请求参数:{}", userVO, e); + log.error("导出等待失败！ 请求参数:{}", userVO, e); } - + log.info("导出完成！总数:{},耗时：{}ms", count.get(), (System.currentTimeMillis() - startTime)); } catch (Exception e) { log.error("导出失败", e); - return ApiResult.error("导出失败"); } + } + + /** + * 生成数据 + * + * @param size + * @return + */ + @Override + public ApiResult generatedData(Integer size) { + int pageSize = 2000; + int k = ceilDivision(size, pageSize); + CountDownLatch sheetThreadSignal = new CountDownLatch(k); + log.info("开始写入！总数:{}，创建线程数量:{}", size, k); + // 为每个处理页创建一个异步任务 + for (int pageNum = 1; pageNum <= k; pageNum++) { + int finalPageNum = pageNum; + pcmTaskExecutor.submit(() -> { + try { + generatedData(finalPageNum,pageSize); + } catch (Exception e) { + log.error("写入失败！ 请求参数:{}", size, e); + } finally { + sheetThreadSignal.countDown(); + } + }); + } + try { + // 等待所有异步任务完成， + int timeout = 600; + boolean completed = sheetThreadSignal.await(timeout, TimeUnit.SECONDS); + if (!completed) { + log.error("写入任务未能在超时时间内完成!超过了:{}秒!", timeout); + } + } catch (InterruptedException e) { + log.error("写入等待失败！ 请求参数:{}", size, e); + } + + return ApiResult.success(); + } + + /** + * 导入数据 + * + * @param file + * @param request + * @return + */ + @Override + public ApiResult importExcel(MultipartFile file, HttpServletRequest request) { + if (file == null || file.isEmpty()) { + return ApiResult.error("上传文件不能为空"); + } + + try (InputStream inputStream = file.getInputStream()) { + FastExcel.read(inputStream, UserExcelVO.class, userDataListener) + .sheet().doRead(); + } catch (IOException e) { + log.error("文件处理失败", e); + return ApiResult.error("文件处理失败: " + e.getMessage()); + } catch (Exception e) { + log.error("发生未知错误", e); + return ApiResult.error("发生未知错误: " + e.getMessage()); + } + + return ApiResult.success(); + } + - return ApiResult.success("导出成功"); + private void generatedData(int pageNum,int finalPageSize) { + List entities = new ArrayList<>(); + for (int i = 0; i < finalPageSize; i++) { + User user = new User(); + user.setAge(i); + user.setName("张三_"+pageNum+"_" + i); + user.setSex(i % 2 == 0 ? 1 : 2); + entities.add(user); + } + userDao.insertBatch(entities); } + private void handlerExport(int finalPageNum, int finalPageSize, UserVO userVO, ExcelWriter excelWriter, WriteSheet writeSheet, AtomicInteger count) { - PageHelper.startPage(finalPageNum, finalPageSize); - List result = userDao.queryAll(userVO); - if(result.isEmpty()){ +// PageHelper.startPage(finalPageNum, finalPageSize); + userVO.setPageNum(finalPageNum); + userVO.setPageSize(finalPageSize); + List result = userDao.queryAll(userVO); + if (result.isEmpty()) { log.info("第{}页没有数据", finalPageNum); return; } List excelData = getExcelData(result); - synchronized (excelWriter){ + synchronized (excelWriter) { excelWriter.write(excelData, writeSheet); count.addAndGet(excelData.size()); - log.info("第{}页写入完成，共写入{}条数据!当前已写入数量:{}", finalPageNum, excelData.size(),count.get()); + log.info("第{}页写入完成，共写入{}条数据!当前已写入数量:{}", finalPageNum, excelData.size(), count.get()); } } diff --git a/springboot-excel/src/main/java/com/pancm/vo/excel/UserExcelVO.java b/springboot-excel/src/main/java/com/pancm/vo/excel/UserExcelVO.java index 0ac1891..360104d 100644 --- a/springboot-excel/src/main/java/com/pancm/vo/excel/UserExcelVO.java +++ b/springboot-excel/src/main/java/com/pancm/vo/excel/UserExcelVO.java @@ -2,6 +2,7 @@ import cn.idev.excel.annotation.ExcelProperty; import com.alibaba.fastjson.JSONObject; +import com.fasterxml.jackson.annotation.JsonValue; import com.pancm.vo.BasePage; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; @@ -9,6 +10,7 @@ import java.io.Serializable; import java.sql.Date; +import java.util.Objects; /** * @Title: 用户表(User)请求响应对象 diff --git a/springboot-excel/src/main/resources/application.yml b/springboot-excel/src/main/resources/application.yml index da2df08..7621b4f 100644 --- a/springboot-excel/src/main/resources/application.yml +++ b/springboot-excel/src/main/resources/application.yml @@ -3,3 +3,7 @@ spring: active: local application: name: springboot-excel + servlet: + multipart: + max-file-size: 20MB + max-request-size: 20MB \ No newline at end of file diff --git a/springboot-excel/src/main/resources/pancm/mapper/UserMapper.xml b/springboot-excel/src/main/resources/pancm/mapper/UserMapper.xml index 22f2f91..928c9cb 100644 --- a/springboot-excel/src/main/resources/pancm/mapper/UserMapper.xml +++ b/springboot-excel/src/main/resources/pancm/mapper/UserMapper.xml @@ -87,10 +87,10 @@ id, name, sex, age, create_time, create_by, update_time, update_by - + SELECT + count(1) + FROM user and id = #{id}