Spring Data Redis Lettuce Connection Issue

问题

I have spring boot application with MYSQL DB and i am caching the data in Redis in server layer with @Cacheable Annotation

@Cacheable(value= "employeeCache", key= "#customerId")
@Override
public Customer getEmployee(String customerId) {
    Optional<Customer> cust = customerRepository.findById(Long.parseLong(customerId));
    if(cust.isPresent()) {
        return cust.get();
    }
    return null;
}

I am using 1 Master 2 Slave and 2 sentinel node, i have deployed the application in docker containers, in AWS ec2 ubuntu instance.

Application and Redis Master/Slave Set up works fine.

When Redis master container goes down, one of the slave becomes master, that also works fine

But once one of the slave becomes master, then Spring boot not able to connect to Redis, i am getting connection time out, it only work if i restart the container.

Please check if any mistake in the configuration.

Java Logs

2020-05-21 18:04:50.016 ERROR 1 --- [io-8080-exec-10] o.a.c.c.C.[.[.[/].[dispatcherServlet]    : Servlet.service() for servlet [dispatcherServlet] in context with path [] threw exception [Request processing failed; nested exception is org.springframework.dao.QueryTimeoutException: Redis command timed out; nested exception is io.lettuce.core.RedisCommandTimeoutException: io.lettuce.core.RedisCommandTimeoutException: Command timed out after 1 minute(s)] with root cause

io.lettuce.core.RedisCommandTimeoutException: Command timed out after 1 minute(s)
        at io.lettuce.core.ExceptionFactory.createTimeoutException(ExceptionFactory.java:51) ~[lettuce-core-5.3.0.RELEASE.jar!/:5.3.0.RELEASE]
        at io.lettuce.core.protocol.CommandExpiryWriter.lambda$potentiallyExpire$0(CommandExpiryWriter.java:167) ~[lettuce-core-5.3.0.RELEASE.jar!/:5.3.0.RELEASE]
        at io.netty.util.concurrent.PromiseTask.runTask(PromiseTask.java:98) ~[netty-common-4.1.49.Final.jar!/:4.1.49.Final]
        at io.netty.util.concurrent.ScheduledFutureTask.run(ScheduledFutureTask.java:170) ~[netty-common-4.1.49.Final.jar!/:4.1.49.Final]
        at io.netty.util.concurrent.DefaultEventExecutor.run(DefaultEventExecutor.java:66) ~[netty-common-4.1.49.Final.jar!/:4.1.49.Final]
        at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[netty-common-4.1.49.Final.jar!/:4.1.49.Final]
        at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.49.Final.jar!/:4.1.49.Final]
        at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[netty-common-4.1.49.Final.jar!/:4.1.49.Final]
        at java.lang.Thread.run(Thread.java:745) [na:1.8.0_111]

Spring Boot Files

application.yaml

spring:
  datasource:
    url: jdbc:mysql://docker-mysql:3306/customerdb
    username: root
    password: root

  jpa:
    show-sql: true

    hibernate:
      naming:
        physical-strategy: org.hibernate.boot.model.naming.PhysicalNamingStrategyStandardImpl


  cache:
    type: redis  

  redis:
    sentinel:
      master: mymaster
      nodes:
        - redis-sentinel:26379
        - redis-sentinel-1:26380

management:
  endpoints:
    web: 
      exposure:
        include:
        - "*"
server:
  port: 8080

logging:
  file:
    name: customer.log
    path: ../logs

Files for Redis and Spring Boot deployment

Docker-compose.yaml

version: '3.5'

services:

  redis-master:
    container_name: redis-master
    image: redis
    volumes:
      - "./data/redis-master:/data"
    networks:
      - backend

  redis-slave:
    container_name: redis-slave
    image: redis
    command: redis-server --slaveof redis-master 6379
    volumes:
      - "./data/redis-slave:/data"
    depends_on:
      - redis-master
    networks:
      - backend

  redis-slave-1:
    container_name: redis-slave-1
    image: redis
    command: redis-server --slaveof redis-master 6379
    volumes:
      - "./data/redis-slave:/data"
    depends_on:
      - redis-master
    networks:
      - backend    

  redis-sentinel:
    container_name: redis-sentinel
    ports:
      - "26379:26379"
    build: redis-sentinel
    depends_on:
      - redis-master
      - redis-slave
    networks:
      - backend

  redis-sentinel-1:
    container_name: redis-sentinel-1
    build: redis-sentinel
    ports:
      - "26380:26379"
    depends_on:
      - redis-master
      - redis-slave
    networks:
      - backend

  docker-mysql:
    restart: always
    container_name: docker-mysql
    image: mysql:5.7
    environment:
      MYSQL_DATABASE: customerdb
      MYSQL_ROOT_PASSWORD: root
      MYSQL_ROOT_HOST: "%"
    volumes:
      - db-data-new:/var/lib/mysql
    ports:
      - "3306:3306"
    networks:
      - backend

  customer-app:
    container_name: java-sentinel
    restart: on-failure
    image: kuldeep99/customer-sentinel:v10
    expose:
      - "8080"
    volumes:
      - /tmp:/logs
    ports:
      - 8080:8080
    depends_on:
      - docker-mysql
      - redis-master
      - redis-slave
      - redis-sentinel
    networks:
      - backend

volumes:
  db-data-new:

networks:
  backend:

Dockerfile

FROM redis

EXPOSE 26379
ENV SENTINEL_QUORUM 2
ENV SENTINEL_DOWN_AFTER 3000
ENV SENTINEL_FAILOVER 3000
ADD sentinel.conf /etc/redis/sentinel.conf
RUN chown redis:redis /etc/redis/sentinel.conf
ADD sentinel-entrypoint.sh /usr/local/bin/
RUN chmod +x /usr/local/bin/sentinel-entrypoint.sh
ENTRYPOINT ["sentinel-entrypoint.sh"]

sentinel-entrypoint.sh

#!/bin/sh

sed -i "s/\$SENTINEL_QUORUM/$SENTINEL_QUORUM/g" /etc/redis/sentinel.conf
sed -i "s/\$SENTINEL_DOWN_AFTER/$SENTINEL_DOWN_AFTER/g" /etc/redis/sentinel.conf
sed -i "s/\$SENTINEL_FAILOVER/$SENTINEL_FAILOVER/g" /etc/redis/sentinel.conf

redis-server /etc/redis/sentinel.conf --sentinel

sentinel.conf

port 26379
protected-mode no
dir /tmp
sentinel monitor mymaster redis-master 6379 $SENTINEL_QUORUM
sentinel down-after-milliseconds mymaster $SENTINEL_DOWN_AFTER
sentinel parallel-syncs mymaster 1
sentinel failover-timeout mymaster $SENTINEL_FAILOVER

来源：https://stackoverflow.com/questions/61941396/spring-data-redis-lettuce-connection-issue