新增FastExcel项目的工程

xuwujing · Jan 23, 2025 · 6389876 · 6389876
1 parent 52f70d8
commit 6389876
Show file tree

Hide file tree

Showing 11 changed files with 329 additions and 56 deletions.
diff --git a/README.md b/README.md
@@ -76,7 +76,11 @@ springBoot-study 是本人在学习SpringBoot的一些工程！
 
 - [springboot-guacamole](https://github.com/xuwujing/springBoot-study/tree/master/springboot-guacamole):SpringBoot结合guacamole实现堡垒机远程桌面控制功能。
 
-- [springboot-nacos](https://github.com/xuwujing/springBoot-study/tree/master/springboot-guacamole):SpringBoot结合Nacos实现配置中心的示例。
+- [springboot-nacos](https://github.com/xuwujing/springBoot-study/tree/master/springboot-nacos):SpringBoot结合Nacos实现配置中心的示例。
+
+- [springboot-saToken](https://github.com/xuwujing/springBoot-study/tree/master/springboot-saToken):SpringBoot结合saToken实现用户登录，权限控制等功能，并提供sql。
+
+- [springboot-excel](https://github.com/xuwujing/springBoot-study/tree/master/springboot-excel):SpringBoot结合FastExcel(EasyExcel的升级版)实现大量数据导入导出示例。
 ## 文章列表
 
 [SpringBoot系列博客:](https://www.cnblogs.com/xuwujing/category/1145997.html)

diff --git a/springboot-excel/pom.xml b/springboot-excel/pom.xml
@@ -67,6 +67,12 @@
         <dependency>
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-web</artifactId>
+            <exclusions>
+                <exclusion>
+                    <artifactId>log4j-api</artifactId>
+                    <groupId>org.apache.logging.log4j</groupId>
+                </exclusion>
+            </exclusions>
         </dependency>
         <dependency>
             <groupId>org.springframework.boot</groupId>
@@ -107,6 +113,12 @@
             <groupId>tk.mybatis</groupId>
             <artifactId>mapper-spring-boot-starter</artifactId>
             <version>${tk.mybatis.boot.version}</version>
+            <exclusions>
+                <exclusion>
+                    <artifactId>mybatis-spring-boot-starter</artifactId>
+                    <groupId>org.mybatis.spring.boot</groupId>
+                </exclusion>
+            </exclusions>
         </dependency>
         <dependency>
             <groupId>com.baomidou</groupId>
@@ -208,9 +220,21 @@
         <dependency>
             <groupId>cn.idev.excel</groupId>
             <artifactId>fastexcel</artifactId>
-            <version>1.1.0</version>
+            <version>1.0.0</version>
+            <exclusions>
+
+                <exclusion>
+                    <artifactId>log4j-api</artifactId>
+                    <groupId>org.apache.logging.log4j</groupId>
+                </exclusion>
+            </exclusions>
         </dependency>
 
+        <dependency>
+            <groupId>org.apache.logging.log4j</groupId>
+            <artifactId>log4j-api</artifactId>
+            <version>2.17.1</version>
+        </dependency>
 
     </dependencies>
 

diff --git a/springboot-excel/src/main/java/com/pancm/Application.java b/springboot-excel/src/main/java/com/pancm/Application.java
@@ -29,7 +29,7 @@ public static void main(String[] args) throws UnknownHostException {
         String ip = InetAddress.getLocalHost().getHostAddress();
         String port = env.getProperty("server.port");
         String path = env.getProperty("server.servlet.context-path");
-        String name = env.getProperty("server.application.name");
+        String name = env.getProperty("spring.application.name");
         log.info("\n----------------------------------------------------------\n\t" +
                 "Application " + name + " is running! Access URLs:\n\t" +
                 "Local: \t\thttp://localhost:" + port + path + "/\n\t" +

diff --git a/springboot-excel/src/main/java/com/pancm/controller/UserController.java b/springboot-excel/src/main/java/com/pancm/controller/UserController.java
@@ -13,6 +13,7 @@
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RequestMethod;
 import org.springframework.web.bind.annotation.RestController;
+import org.springframework.web.multipart.MultipartFile;
 
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
@@ -102,17 +103,33 @@ public ApiResult view( @RequestParam("id") Long id) {
     }
 
 
-    @ApiOperation(value = "/export", notes = "用户导出成Excel")
+    @ApiOperation(value = "用户表生成数据",notes = "用户表生成数据")
+    @RequestMapping(value = "generatedData", method = RequestMethod.GET)
+    public ApiResult generatedData( @RequestParam("size") Integer size) {
+        return userService.generatedData(size);
+    }
+
+
+
+    @ApiOperation(value = "用户导出成Excel", notes = "用户导出成Excel")
     @RequestMapping(value = "/export", method = {RequestMethod.GET})
-    public ApiResult export(@RequestBody UserVO userVO, HttpServletRequest request, HttpServletResponse response){
-        return userService.export(userVO, request, response);
+    public void export(@RequestBody UserVO userVO, HttpServletRequest request, HttpServletResponse response){
+         userService.export(userVO, request, response);
     }
 
 
-    @ApiOperation(value = "/exportBatch", notes = "用户导出成Excel,量大的时候")
+    @ApiOperation(value = "用户导出成Excel,量大的时候", notes = "用户导出成Excel,量大的时候")
     @RequestMapping(value = "/exportBatch", method = {RequestMethod.GET})
-    public ApiResult exportBatch(@RequestBody UserVO userVO, HttpServletRequest request, HttpServletResponse response){
-        return userService.exportBatch(userVO, request, response);
+    public void exportBatch(@RequestBody UserVO userVO, HttpServletRequest request, HttpServletResponse response){
+         userService.exportBatch(userVO, request, response);
     }
 
+    @ApiOperation(value = "用户导入Excel", notes = "用户导入Excel")
+    @PostMapping("/import")
+    public ApiResult importExcel(@RequestParam("file") MultipartFile file,
+                                          HttpServletRequest request) {
+        return userService.importExcel(file, request);
+    }
+
+
 }
diff --git a/springboot-excel/src/main/java/com/pancm/dao/UserDao.java b/springboot-excel/src/main/java/com/pancm/dao/UserDao.java
@@ -44,6 +44,10 @@ public interface UserDao  {
      */
     List<UserVO> queryAll(UserVO userVO);
 
+     /**
+      * 查询总数
+      * @return
+      */
      int queryAllCount(UserVO userVO);
     /**
      * 新增数据

diff --git a/springboot-excel/src/main/java/com/pancm/excel/UserDataListener.java b/springboot-excel/src/main/java/com/pancm/excel/UserDataListener.java
@@ -0,0 +1,107 @@
+package com.pancm.excel;
+
+import cn.idev.excel.context.AnalysisContext;
+import cn.idev.excel.read.listener.ReadListener;
+
+import com.pancm.dao.UserDao;
+import com.pancm.model.User;
+import com.pancm.vo.excel.UserExcelVO;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.BeanUtils;
+import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
+import org.springframework.stereotype.Component;
+
+import javax.annotation.Resource;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+
+
+/**
+ * @author pancm
+ * @Title: Application
+ * @Description: 实现 ReadListener 接口，设置读取数据的操作
+ * @Version:1.0.0
+ * @Since:jdk1.8
+ * @Date 2025/01/23
+ **/
+@Slf4j
+@Component
+public class UserDataListener implements ReadListener<UserExcelVO> {
+
+    @Resource
+    private UserDao userDao;
+
+    @Resource(name = "pcmTaskExecutor")
+    private ThreadPoolTaskExecutor pcmTaskExecutor;
+
+    private static final int BATCH_COUNT = 1000;
+
+    /**
+     * 缓存的数据
+     */
+    private List<UserExcelVO> cachedDataList = Collections.synchronizedList(new ArrayList<>());
+    private AtomicInteger count = new AtomicInteger(0); // 仅用于统计总记录数
+
+    /**
+     * 记录开始时间
+     */
+    private long startTime;
+
+    /**
+     * @param userExcelVO
+     * @param analysisContext
+     */
+    @Override
+    public void invoke(UserExcelVO userExcelVO, AnalysisContext analysisContext) {
+        if (count.get() == 0) {
+            // 记录开始时间
+            startTime = System.currentTimeMillis();
+        }
+
+        cachedDataList.add(userExcelVO);
+        if (cachedDataList.size() >= BATCH_COUNT) {
+            flushData();
+        }
+    }
+
+    private synchronized void flushData() {
+        if (!cachedDataList.isEmpty()) {
+            List<UserExcelVO> dataToFlush = new ArrayList<>(cachedDataList);
+            cachedDataList.clear();
+            pcmTaskExecutor.submit(() -> {
+                try {
+                    saveData(dataToFlush);
+                } catch (Exception e) {
+                    log.error("数据保存出现异常：", e);
+                }
+            });
+        }
+    }
+
+    private void saveData(List<UserExcelVO> dataList) {
+        log.info("{}条数据，开始存储数据库！", dataList.size());
+        List<User> entities = dataList.stream()
+                .map(userVO -> {
+                    User user = new User();
+                    BeanUtils.copyProperties(userVO, user);
+                    user.setSex(Objects.equals(userVO.getSex(), "男") ? 1 : 2);
+                    return user;
+                })
+                .collect(Collectors.toList());
+        userDao.insertBatch(entities);
+        count.addAndGet(dataList.size());
+    }
+
+    @Override
+    public void doAfterAllAnalysed(AnalysisContext context) {
+        // 处理剩余未提交的数据
+        flushData();
+        long endTime = System.currentTimeMillis();
+        log.info("所有数据解析完成！共解析:{}条数据! 耗时:{}毫秒", count.get(), (endTime - startTime));
+        count.set(0);
+    }
+}
diff --git a/springboot-excel/src/main/java/com/pancm/service/IUserService.java b/springboot-excel/src/main/java/com/pancm/service/IUserService.java
@@ -2,6 +2,7 @@
 
 import com.pancm.vo.UserVO;
 import com.pancm.vo.ApiResult;
+import org.springframework.web.multipart.MultipartFile;
 
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
@@ -67,7 +68,7 @@ public interface IUserService {
       * @param response
       * @return
       */
-     ApiResult export(UserVO userVO, HttpServletRequest request, HttpServletResponse response);
+     void export(UserVO userVO, HttpServletRequest request, HttpServletResponse response);
 
 
      /**
@@ -78,5 +79,20 @@ public interface IUserService {
       * @param response
       * @return
       */
-     ApiResult exportBatch(UserVO userVO, HttpServletRequest request, HttpServletResponse response);
+     void exportBatch(UserVO userVO, HttpServletRequest request, HttpServletResponse response);
+
+     /**
+      * 生成数据
+      * @param size
+      * @return
+      */
+     ApiResult generatedData(Integer size);
+
+     /**
+      * 导入数据
+      * @param file
+      * @param request
+      * @return
+      */
+     ApiResult importExcel(MultipartFile file, HttpServletRequest request);
  }