接口并发能力优化提升

标签：redis com springframework 接口并发 user org import 优化

写了一个插入接口，进行了并发处理的优化，优化过程如下：

初始接口代码

yml配置文件

server:
    port: 8081
spring:
    datasource:
        url: jdbc:mysql://ip:3306/my_test?characterEncoding=utf-8&useSSL=false&serverTimezone=UTC
        username: lin
        password: 123456
    redis:
        host: ip
        port: 6379

实体类代码

package com.example.demo.entity;

import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import lombok.Data;

@Data
public class User {

    @TableId(type = IdType.AUTO)
    private Integer id;
    private String name;
    private String sex;
    private String phone;

    public User() {
    }

    public User(String name, String sex, String phone) {
        this.name = name;
        this.sex = sex;
        this.phone = phone;
    }
}

接口代码

package com.example.demo.service;

import com.baomidou.mybatisplus.extension.service.IService;
import com.example.demo.entity.User;

import java.util.List;


public interface UserService extends IService<User> {

        //单条数据插入
        int insertUser(User user);

        //获取所有数据
        List<User> getAll();

        //获取所有数据数量
        Integer getCount();
}

接口方法实现代码

package com.example.demo.service.Impl;

import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.example.demo.entity.User;
import com.example.demo.mapper.UserMapper;
import com.example.demo.service.UserService;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;

import java.util.List;

@Service
@RequiredArgsConstructor
public class UserServicepImpl extends ServiceImpl<UserMapper, User> implements UserService {

    @Override
    public int insertUser(User user) {
        return this.baseMapper.insertUser(user);
    }

    @Override
    public List<User> getAll() {
        LambdaQueryWrapper<User> wrapper =new LambdaQueryWrapper<>();
        List<User> list = this.baseMapper.selectList(wrapper);
        return list;
    }

    @Override
    public Integer getCount() {
        LambdaQueryWrapper<User> wrapper =new LambdaQueryWrapper<>();
        return this.baseMapper.selectCount(wrapper);
    }
}

Controller层接口代码

package com.example.demo.controller;


import com.alibaba.fastjson.JSONObject;
import com.example.demo.entity.User;
import com.example.demo.service.UserService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import org.springframework.validation.annotation.Validated;
import org.springframework.web.bind.annotation.*;

import java.util.List;

@Slf4j
@Validated
@RestController
@RequestMapping("/user")
@RequiredArgsConstructor
public class UserController {

    private final UserService userService;

    /**
     * 获取全部数据接口
     * **/
    @GetMapping("getAll")
    public JSONObject getAll(){
        JSONObject object = new JSONObject();
        object.put("code","200");
        object.put("meg","success");
        List<User> list = userService.getAll();
        object.put("data",list);
        return object;
    }

    /**
     * 直接实现接口
     * */
    @PostMapping("/insert")
    public String insertUser(@RequestBody JSONObject jsonObject){
        User user =new User();
        user.setName(jsonObject.getString("name"));
        user.setSex(jsonObject.getString("sex"));
        user.setPhone(jsonObject.getString("phone"));
        int result = userService.insertUser(user);
        JSONObject resp = new JSONObject();
        if(result != 0){
            log.info("插入成功");
            resp.put("code","200");
            resp.put("msg","成功");
        }else {
            log.info("插入失败");
            resp.put("code","400");
            resp.put("msg","失败");
        }
        return resp.toString();
    }

}

多线程+redis进行优化

初始版本的接口，接口承受的并发量不高，于是优化第一步是增加redis缓存和多线程异步处理，即接口请求时，对于数据处理使用多线程进行异步处理，并且将数据缓存到redis中，然后定时将缓存数据从redis中的数据同步到数据库中，代码如下：

package com.example.demo.controller;


import com.alibaba.fastjson.JSONObject;
import com.example.demo.entity.User;
import com.example.demo.service.UserService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import org.springframework.validation.annotation.Validated;
import org.springframework.web.bind.annotation.*;

import java.util.List;

@Slf4j
@Validated
@RestController
@RequestMapping("/user")
@RequiredArgsConstructor
public class UserController {

    @Autowired
    private StringRedisTemplate redisTemplate;  //redis工具类

    @Autowired
    private ThreadPoolTaskExecutor threadPoolTaskExecutor;

    private final UserService userService;

    Integer number = 0;

    /**
     * 获取全部数据接口
     * **/
    @GetMapping("getAll")
    public JSONObject getAll(){
        JSONObject object = new JSONObject();
        object.put("code","200");
        object.put("meg","success");
        List<User> list = userService.getAll();
        object.put("data",list);
        return object;
    }

    /**
     * redis异步缓存实现
     * */
    @PostMapping("/redisAsyInsert")
    public String redisAsyInsert(@RequestBody JSONObject jsonObject){
        threadPoolTaskExecutor.execute(()->redisAdd(jsonObject));
        return "缓存成功";
    }

    public void redisAdd(JSONObject jsonObject){
        User user =new User();
        user.setName(jsonObject.getString("name"));
        user.setSex(jsonObject.getString("sex"));
        user.setPhone(jsonObject.getString("phone"));
        redisTemplate.opsForList().rightPush("userList",String.valueOf(user)); //存放进缓存列表
        log.info("缓存完毕");
    }

}

redis定时任务同步数据代码：

package com.example.demo.utils;

import com.alibaba.fastjson.JSONObject;
import com.example.demo.entity.UserEntity;
import com.example.demo.service.UserService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;

import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

@Slf4j
@Component
public class RedisService {

    @Autowired
    private StringRedisTemplate redisTemplate;

    @Autowired
    UserService userService;

    //定时同步至数据库中
    @Scheduled(cron = "0/30 * * * * ?")
    private void addData(){
        List<String> userList = redisTemplate.opsForList().range("userList",0,-1);
        log.info("开始同步到数据库");
        int number =0;
        for(String str:userList){
            Pattern pattern = Pattern.compile("User\\(id=(\\w*), name=([a-zA-Z]*), sex=([a-zA-Z?]*), phone=(\\d*)\\)");
            Matcher matcher = pattern.matcher(str);
            if (matcher.find()) {
                String id = matcher.group(1);
                String name = matcher.group(2);
                String sex = matcher.group(3);
                String phone = matcher.group(4);
                UserEntity user = new UserEntity(name, sex, phone);
                userService.insertUser(user);
                log.info(String.valueOf(++number));
            } else {
                System.out.println("无法解析字符串：" + str);
            }

        }
        redisTemplate.delete("userList");
        log.info("同步完毕，删除了缓存");
    }

}

修改tomcat和redis配置

为了提高数接口的并发，修改tomcat线程池的最大线程数，提高接口数据处理的速度

server:
    port: 8081
    # 修改tomcat的配置
    tomcat:
        threads:
            max: 300
spring:
    datasource:
        url: jdbc:mysql://ip:3306/my_test?characterEncoding=utf-8&useSSL=false&serverTimezone=UTC
        username: lin
        password: 123456
    redis:
        host: ip
        port: 6379
        # 通过jedis，修改redis的配置
        jedis:
            pool:
                max-active: 12
                max-idle: 12
                min-idle: 2

1、server.tomcat.max-threads：指定tomcat线程池的最大线程数，一般的默认值是200；
2、spring.redis.jedis.pool.max-active：指定Jedis连接池的最大活动连接数。默认值为8。
3、spring.redis.jedis.pool.max-idle：指定Jedis连接池的最大空闲连接数。默认值为8。
4、spring.redis.jedis.pool.min-idle：指定Jedis连接池的最小空闲连接数。默认值为0。

通过修改tomcat和redis的线程池设置，可以进一步提高接口的并发处理能力；

ThreadPoolTaskExecutor优化

在进行并发测试时，发现进行redis缓存时，线程id数量很少，所以进行threadPoolTaskExecutor线程池的配置，设置线程池的核心线程数（一般为电脑内核的两倍）和最大线程数，实现代码如下：

package com.example.demo.controller;


import com.alibaba.fastjson.JSONObject;
import com.example.demo.entity.User;
import com.example.demo.service.UserService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import org.springframework.validation.annotation.Validated;
import org.springframework.web.bind.annotation.*;

import java.util.List;

@Slf4j
@Validated
@RestController
@RequestMapping("/user")
@RequiredArgsConstructor
public class UserController {

    @Autowired
    private StringRedisTemplate redisTemplate;  //redis工具类

    @Autowired
    private ThreadPoolTaskExecutor threadPoolTaskExecutor;

    private final UserService userService;

    Integer number = 0;

    /**
     * 获取全部数据接口
     * **/
    @GetMapping("getAll")
    public JSONObject getAll(){
        JSONObject object = new JSONObject();
        object.put("code","200");
        object.put("meg","success");
        List<User> list = userService.getAll();
        object.put("data",list);
        return object;
    }


    /**
     * redis异步缓存实现
     * */
    @PostMapping("/redisAsyInsert")
    public String redisAsyInsert(@RequestBody JSONObject jsonObject){
        threadPoolTaskExecutor.setCorePoolSize(4);
        threadPoolTaskExecutor.setMaxPoolSize(12);
        threadPoolTaskExecutor.execute(()->redisAdd(jsonObject));
        return "缓存成功";
    }

    public void redisAdd(JSONObject jsonObject){
        User user =new User();
        user.setName(jsonObject.getString("name"));
        user.setSex(jsonObject.getString("sex"));
        user.setPhone(jsonObject.getString("phone"));
        redisTemplate.opsForList().rightPush("userList",String.valueOf(user)); //存放进缓存列表
        log.info("缓存完毕");
    }

}

数据同步优化

在进行数据同步时，经过一定时间的缓存，redis中缓存了大量数据，代码中进行数据库同步时，是逐条数据进行同步到数据库中，效率很低，使用批量插入的方式进行一个优化，代码如下：

package com.example.demo.utils;

import com.example.demo.entity.User;
import com.example.demo.service.UserService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;

import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

@Slf4j
@Component
public class RedisService {

    @Autowired
    private StringRedisTemplate redisTemplate;

    @Autowired
    UserService userService;

    //定时同步至数据库中
    @Scheduled(cron = "0 */1 * * * ?")
    private void addData(){
        List<String> userList = redisTemplate.opsForList().range("userList",0,-1);
        List<User> users = new ArrayList<>();
        log.info("开始同步到数据库");
        int cout = userService.getCount();
        log.info("数据库中有："+cout);
        int number =0;
        for(String str:userList){
            Pattern pattern = Pattern.compile("User\\(id=(\\w*), name=([a-zA-Z]*), sex=([a-zA-Z?]*), phone=(\\d*)\\)");
            Matcher matcher = pattern.matcher(str);
            if (matcher.find()) {
                String name = matcher.group(2);
                String sex = matcher.group(3);
                String phone = matcher.group(4);
                User user = new User(name, sex, phone);
                users.add(user);
                ++number;
                if(number%8000 == 0){ //8000时进行一次批量存储
                    userService.saveBatch(users);
                    log.info("同步数据成功"+number);
                    number = 0;
                    users.clear();
                }
            } else {
                System.out.println("无法解析字符串：" + str);
            }

        }
        if(number != 0) userService.saveBatch(users);
        redisTemplate.delete("userList");
        int count2 = userService.getCount();
        log.info("数据库中有："+count2);
        int all = count2-cout;
        log.info("同步了："+all+"条数据");
        log.info("同步完毕，删除了缓存");
    }
}

mybatis-plus中提供了一个批量插入数据的方法saveBatch（），可以一次性插入到数据库中，从而提高数据的效率。实现分批次进行数据同步，提高数据的的同步效率。

后语

优化到这里，我的接口在相同的配置下，从并发量2000优化到了16000（以我的电脑配置为准，更高的服务器配置还没经济能力测试），目前还在继续学习，下一步会通过docker容器和nginx的负载均衡，进一步实现实现更高的并发量处理，后续会继续更新，欢迎大佬们指正里面的错误。

标签：redis,com,springframework,接口,并发,user,org,import,优化
From： https://www.cnblogs.com/caijilin/p/17463756.html