石家庄做淘宝网站,中国有名的网站建设公司,我的百度账号,wordpress下载系统插件说明#xff1a;在一些场景#xff0c;如导入数据#xff0c;批量插入数据库#xff0c;使用常规方法#xff0c;需要等待较长时间#xff0c;而使用线程池可以提高效率。本文介绍如何在Spring Boot中使用线程池来批量插入数据。
搭建环境
首先#xff0c;创建一个Spr…说明在一些场景如导入数据批量插入数据库使用常规方法需要等待较长时间而使用线程池可以提高效率。本文介绍如何在Spring Boot中使用线程池来批量插入数据。
搭建环境
首先创建一个Spring Boot项目pom文件如下
?xml version1.0 encodingUTF-8?
project xmlnshttp://maven.apache.org/POM/4.0.0xmlns:xsihttp://www.w3.org/2001/XMLSchema-instancexsi:schemaLocationhttp://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsdmodelVersion4.0.0/modelVersionparentgroupIdorg.springframework.boot/groupIdartifactIdspring-boot-starter-parent/artifactIdversion2.7.12/versionrelativePath//parentgroupIdcom.hezy/groupIdartifactIdthread_pool_demo/artifactIdversion1.0-SNAPSHOT/versionpropertiesmaven.compiler.source11/maven.compiler.sourcemaven.compiler.target11/maven.compiler.targetproject.build.sourceEncodingUTF-8/project.build.sourceEncoding/propertiesdependenciesdependencygroupIdorg.springframework.boot/groupIdartifactIdspring-boot-starter-web/artifactId/dependencydependencygroupIdorg.projectlombok/groupIdartifactIdlombok/artifactId/dependencydependencygroupIdcom.alibaba/groupIdartifactIddruid-spring-boot-starter/artifactIdversion1.2.8/version/dependencydependencygroupIdcom.mysql/groupIdartifactIdmysql-connector-j/artifactIdscoperuntime/scope/dependencydependencygroupIdorg.mybatis.spring.boot/groupIdartifactIdmybatis-spring-boot-starter/artifactIdversion2.2.2/version/dependencydependencygroupIdorg.springframework.boot/groupIdartifactIdspring-boot-starter-test/artifactIdscopetest/scope/dependencydependencygroupIdcn.hutool/groupIdartifactIdhutool-all/artifactIdversion5.8.6/version/dependency/dependencies
/project写一个插入数据的Mapper方法
import com.hezy.pojo.User;
import org.apache.ibatis.annotations.Insert;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Param;Mapper
public interface UserMapper {Insert(insert into i_users (username, password) values (#{user.username}, #{user.password}))void insert(Param(user) User user);
}写一个接口用来插入20万条记录如下
import com.hezy.pojo.User;
import com.hezy.service.AsyncService;
import com.hezy.service.UserService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;import java.util.ArrayList;RestController
RequestMapping(user)
public class UserController {/*** 总记录数*/private final static int SIZE 40 * 10000;Autowiredprivate UserService userService;Autowiredprivate AsyncService asyncService;GetMapping(insert1)public void insert1() {ArrayListUser list new ArrayList(SIZE);for (int i 0; i SIZE; i) {User user new User();user.setUsername(user i);user.setPassword(password i);list.add(user);}long startTime System.currentTimeMillis();// 批量插入for (User user : list) {userService.insert(user);}long endTime System.currentTimeMillis();System.out.println(不用线程池插入40万条记录耗时: ((endTime - startTime) / 1000) s);}
}启动项目测试一下看要多长时间……11分钟 使用线程池
Spring Boot有自动注入的线程池threadPoolTaskExecutor可以手动设置一些属性为我们所用。
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.scheduling.annotation.EnableAsync;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;import java.util.concurrent.Executor;
import java.util.concurrent.ThreadPoolExecutor;Configuration
EnableAsync
public class ThreadPoolConfig {Bean(name threadPoolTaskExecutor)public Executor threadPoolTaskExecutor() {ThreadPoolTaskExecutor executor new ThreadPoolTaskExecutor();executor.setCorePoolSize(20);executor.setMaxPoolSize(40);executor.setQueueCapacity(500);executor.setKeepAliveSeconds(60);executor.setThreadNamePrefix(hezy-);executor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy());executor.initialize();return executor;}
}使用线程池来完成上面插入数据的操作如下 GetMapping(insert2)public void insert2() {ArrayListUser list new ArrayList(SIZE);for (int i 0; i SIZE; i) {User user new User();user.setUsername(user i);user.setPassword(password i);list.add(user);}// 将数据分成4000批每批插入100条ListListUser batchList new ArrayList();for (int i 0; i list.size(); i 100) {batchList.add(list.subList(i, i 100));}long startTime System.currentTimeMillis();CountDownLatch countDownLatch new CountDownLatch(batchList.size());// 线程池分批插入for (ListUser batch : batchList) {asyncService.executeAsync(batch, userService, countDownLatch);}try {countDownLatch.await();} catch (InterruptedException e) {throw new RuntimeException(e);}long endTime System.currentTimeMillis();System.out.println(使用线程池插入40万条记录耗时: ((endTime - startTime) / 1000) s);}AsyncService实现类
import com.hezy.pojo.User;
import com.hezy.service.AsyncService;
import com.hezy.service.UserService;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;import java.util.List;
import java.util.concurrent.CountDownLatch;Service
public class AsyncServiceImpl implements AsyncService {Async(threadPoolTaskExecutor)Overridepublic void executeAsync(ListUser batch, UserService userService, CountDownLatch countDownLatch) {try {for (User user : batch) {userService.insert(user);}} finally {countDownLatch.countDown();}}
}启动测试速度提升很明显。如果再改造一下insert()方法一次插入多条数据肯定还能更快。 总结
本文介绍如何使用Spring Boot装配的线程池Bean完成大数据量的批量插入操作提高程序执行效率。
实例完整代码https://github.com/HeZhongYing/thread_pool_demo
参考B站UP主孟哥说Java视频https://www.bilibili.com/video/BV18r421F7CQ
