diff --git a/build.gradle b/build.gradle index 31c7b550..167f0ecb 100644 --- a/build.gradle +++ b/build.gradle @@ -37,7 +37,11 @@ dependencies { implementation 'org.springframework.session:spring-session-core' implementation 'org.springdoc:springdoc-openapi-starter-webmvc-ui:2.8.9' implementation 'org.springframework.boot:spring-boot-starter-data-mongodb' - implementation 'org.springframework.boot:spring-boot-starter-data-redis' + implementation('org.springframework.boot:spring-boot-starter-data-redis') { + exclude group: 'io.lettuce', module: 'lettuce-core' + } + implementation 'redis.clients:jedis' + implementation 'org.redisson:redisson:3.51.0' implementation 'org.springframework.boot:spring-boot-starter-security' implementation('org.springframework.ai:spring-ai-bedrock-converse-spring-boot-starter') { exclude group: 'io.swagger.core.v3', module: 'swagger-annotations' @@ -49,7 +53,7 @@ dependencies { implementation 'com.sksamuel.scrimage:scrimage-webp:4.3.5' implementation 'com.bucket4j:bucket4j_jdk17-core:8.15.0' implementation 'com.bucket4j:bucket4j_jdk17-redis-common:8.15.0' - implementation 'com.bucket4j:bucket4j_jdk17-lettuce:8.15.0' + implementation 'com.bucket4j:bucket4j_jdk17-redisson:8.15.0' implementation 'org.jsoup:jsoup:1.17.2' implementation 'com.rometools:rome:2.1.0' compileOnly 'org.projectlombok:lombok' diff --git a/src/main/java/com/linglevel/api/common/config/RedisConfig.java b/src/main/java/com/linglevel/api/common/config/RedisConfig.java index acc9f70f..90bf0106 100644 --- a/src/main/java/com/linglevel/api/common/config/RedisConfig.java +++ b/src/main/java/com/linglevel/api/common/config/RedisConfig.java @@ -1,13 +1,17 @@ package com.linglevel.api.common.config; +import org.redisson.Redisson; +import org.redisson.api.RedissonClient; +import org.redisson.config.Config; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.data.redis.connection.RedisConnectionFactory; import org.springframework.data.redis.connection.RedisStandaloneConfiguration; -import org.springframework.data.redis.connection.lettuce.LettuceClientConfiguration; -import org.springframework.data.redis.connection.lettuce.LettuceConnectionFactory; +import org.springframework.data.redis.connection.jedis.JedisClientConfiguration; +import org.springframework.data.redis.connection.jedis.JedisConnectionFactory; import org.springframework.data.redis.core.RedisTemplate; +import org.springframework.data.redis.listener.RedisMessageListenerContainer; import org.springframework.data.redis.serializer.GenericJackson2JsonRedisSerializer; import org.springframework.data.redis.serializer.StringRedisSerializer; @@ -27,16 +31,16 @@ public class RedisConfig { public RedisConnectionFactory redisConnectionFactory() { RedisStandaloneConfiguration config = new RedisStandaloneConfiguration(host, port); - LettuceClientConfiguration clientConfig; + JedisClientConfiguration clientConfig; if (ssl) { - clientConfig = LettuceClientConfiguration.builder() + clientConfig = JedisClientConfiguration.builder() .useSsl() .build(); } else { - clientConfig = LettuceClientConfiguration.builder().build(); + clientConfig = JedisClientConfiguration.builder().build(); } - return new LettuceConnectionFactory(config, clientConfig); + return new JedisConnectionFactory(config, clientConfig); } @Bean @@ -52,4 +56,20 @@ public RedisTemplate redisTemplate() { template.afterPropertiesSet(); return template; } -} \ No newline at end of file + + @Bean + public RedisMessageListenerContainer redisMessageListenerContainer(RedisConnectionFactory redisConnectionFactory) { + RedisMessageListenerContainer container = new RedisMessageListenerContainer(); + container.setConnectionFactory(redisConnectionFactory); + return container; + } + + @Bean(destroyMethod = "shutdown") + public RedissonClient redissonClient() { + Config config = new Config(); + String scheme = ssl ? "rediss://" : "redis://"; + config.useSingleServer() + .setAddress(scheme + host + ":" + port); + return Redisson.create(config); + } +} diff --git a/src/main/java/com/linglevel/api/common/ratelimit/bucket4j/Bucket4jConfig.java b/src/main/java/com/linglevel/api/common/ratelimit/bucket4j/Bucket4jConfig.java index 7b1a324f..7a344a37 100644 --- a/src/main/java/com/linglevel/api/common/ratelimit/bucket4j/Bucket4jConfig.java +++ b/src/main/java/com/linglevel/api/common/ratelimit/bucket4j/Bucket4jConfig.java @@ -1,51 +1,22 @@ package com.linglevel.api.common.ratelimit.bucket4j; import io.github.bucket4j.distributed.proxy.ProxyManager; -import io.github.bucket4j.redis.lettuce.cas.LettuceBasedProxyManager; -import io.lettuce.core.RedisClient; -import io.lettuce.core.RedisURI; -import io.lettuce.core.api.StatefulRedisConnection; -import io.lettuce.core.codec.ByteArrayCodec; -import io.lettuce.core.codec.RedisCodec; -import io.lettuce.core.codec.StringCodec; -import org.springframework.beans.factory.annotation.Value; +import io.github.bucket4j.redis.redisson.Bucket4jRedisson; +import org.redisson.Redisson; +import org.redisson.api.RedissonClient; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import java.time.Duration; - /** * Bucket4j configuration for rate limiting with Redis backend. */ @Configuration public class Bucket4jConfig { - @Value("${spring.data.redis.host}") - private String host; - - @Value("${spring.data.redis.port}") - private int port; - - @Value("${spring.data.redis.ssl.enabled}") - private boolean ssl; - @Bean - public ProxyManager proxyManager() { - RedisURI.Builder uriBuilder = RedisURI.builder() - .withHost(host) - .withPort(port) - .withTimeout(Duration.ofSeconds(10)); - - if (ssl) { - uriBuilder.withSsl(true); - } - - RedisClient redisClient = RedisClient.create(uriBuilder.build()); - StatefulRedisConnection connection = redisClient.connect( - RedisCodec.of(StringCodec.UTF8, ByteArrayCodec.INSTANCE) - ); - - return LettuceBasedProxyManager.builderFor(connection) + public ProxyManager proxyManager(RedissonClient redissonClient) { + Redisson redisson = (Redisson) redissonClient; + return Bucket4jRedisson.casBasedBuilder(redisson.getCommandExecutor()) .build(); } } diff --git a/src/main/java/com/linglevel/api/word/exception/WordsErrorCode.java b/src/main/java/com/linglevel/api/word/exception/WordsErrorCode.java index a8b6229b..774a5061 100644 --- a/src/main/java/com/linglevel/api/word/exception/WordsErrorCode.java +++ b/src/main/java/com/linglevel/api/word/exception/WordsErrorCode.java @@ -11,10 +11,11 @@ public enum WordsErrorCode { WORD_IS_MEANINGLESS(HttpStatus.BAD_REQUEST, "The word is meaningless."), WORD_ALREADY_EXISTS(HttpStatus.BAD_REQUEST, "Word already exists."), WORD_NOT_FOUND_BY_ID(HttpStatus.NOT_FOUND, "Word not found with id."), + WORD_ANALYSIS_TIMEOUT(HttpStatus.SERVICE_UNAVAILABLE, "Word analysis is temporarily delayed. Please try again."), INVALID_WORD_FORMAT(HttpStatus.BAD_REQUEST, "Word contains invalid characters (spaces, tabs, newlines, or special characters are not allowed)."), WORD_TOO_LONG(HttpStatus.BAD_REQUEST, "Word is too long (maximum 50 characters)."), SAME_SOURCE_TARGET_LANGUAGE(HttpStatus.BAD_REQUEST, "Source and target languages cannot be the same."); private final HttpStatus status; private final String message; -} \ No newline at end of file +} diff --git a/src/main/java/com/linglevel/api/word/service/WordService.java b/src/main/java/com/linglevel/api/word/service/WordService.java index 4a2c7a4b..25aaf2e3 100644 --- a/src/main/java/com/linglevel/api/word/service/WordService.java +++ b/src/main/java/com/linglevel/api/word/service/WordService.java @@ -32,6 +32,7 @@ public class WordService { private final WordVariantRepository wordVariantRepository; private final InvalidWordRepository invalidWordRepository; private final WordAiService wordAiService; + private final WordSingleFlightRedisCoordinator singleFlightCoordinator; public WordSearchResponse getOrCreateWords(String userId, String word, LanguageCode targetLanguage) { List wordVariants = getOrCreateWordEntities(word, targetLanguage); @@ -49,10 +50,21 @@ public WordSearchResponse getOrCreateWords(String userId, String word, LanguageC log.info("Word '{}' not found for targetLanguage {}, creating new one...", wordVariant.getOriginalForm(), targetLanguage); - List analysisResults = wordAiService.analyzeWord( - wordVariant.getOriginalForm(), - targetLanguage.getCode() - ); + List analysisResults; + try { + analysisResults = singleFlightCoordinator.execute( + wordVariant.getOriginalForm(), + targetLanguage, + () -> wordAiService.analyzeWord( + wordVariant.getOriginalForm(), + targetLanguage.getCode() + ) + ); + } catch (WordSingleFlightTimeoutException | WordSingleFlightLeaderFailureException e) { + log.warn("Single-flight temporary failure for originalForm '{}'. Returning timeout error.", + wordVariant.getOriginalForm(), e); + throw new WordsException(WordsErrorCode.WORD_ANALYSIS_TIMEOUT); + } // Word 생성 및 저장 (빈 결과는 WordAiService에서 예외 발생) Word newWord = convertAnalysisResultToWord(analysisResults.get(0)); @@ -102,7 +114,11 @@ public List getOrCreateWordEntities(String word, LanguageCode targe log.info("Word '{}' not found in database. Calling AI to analyze...", word); List analysisResults; try { - analysisResults = wordAiService.analyzeWord(word, targetLanguage.getCode()); + analysisResults = singleFlightCoordinator.execute( + word, + targetLanguage, + () -> wordAiService.analyzeWord(word, targetLanguage.getCode()) + ); // AI 호출 성공 시 InvalidWord 캐시에서 제거 (일시적 오류였던 경우 복구) cachedInvalidWord.ifPresent(invalidWord -> { @@ -111,6 +127,9 @@ public List getOrCreateWordEntities(String word, LanguageCode targe word, invalidWord.getAttemptCount()); }); + } catch (WordSingleFlightTimeoutException | WordSingleFlightLeaderFailureException e) { + log.warn("Single-flight temporary failure for word '{}'. Keeping invalid-word cache untouched.", word, e); + throw new WordsException(WordsErrorCode.WORD_ANALYSIS_TIMEOUT); } catch (Exception e) { // AI 호출 실패 또는 무의미한 단어인 경우 InvalidWord로 캐싱 log.warn("AI call failed for word '{}'. Caching as invalid word to prevent retries.", word, e); @@ -417,4 +436,4 @@ public WordSearchResponse forceReanalyzeWord(String word, LanguageCode targetLan .results(results) .build(); } -} \ No newline at end of file +} diff --git a/src/main/java/com/linglevel/api/word/service/WordSingleFlightLeaderFailureException.java b/src/main/java/com/linglevel/api/word/service/WordSingleFlightLeaderFailureException.java new file mode 100644 index 00000000..96da1553 --- /dev/null +++ b/src/main/java/com/linglevel/api/word/service/WordSingleFlightLeaderFailureException.java @@ -0,0 +1,8 @@ +package com.linglevel.api.word.service; + +public class WordSingleFlightLeaderFailureException extends RuntimeException { + + public WordSingleFlightLeaderFailureException(String message) { + super(message); + } +} diff --git a/src/main/java/com/linglevel/api/word/service/WordSingleFlightProperties.java b/src/main/java/com/linglevel/api/word/service/WordSingleFlightProperties.java new file mode 100644 index 00000000..2ec11423 --- /dev/null +++ b/src/main/java/com/linglevel/api/word/service/WordSingleFlightProperties.java @@ -0,0 +1,27 @@ +package com.linglevel.api.word.service; + +import lombok.Getter; +import lombok.Setter; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +@Getter +@Setter +@Component +@ConfigurationProperties(prefix = "word.single-flight") +public class WordSingleFlightProperties { + + private boolean enabled = true; + + private long lockTtlMs = 20_000; + + private long waitTimeoutMs = 5_000; + + private long resultTtlMs = 60_000; + + private String promptVersion = "v1"; + + private String model = "default"; + + private String schemaVersion = "v2"; +} diff --git a/src/main/java/com/linglevel/api/word/service/WordSingleFlightRedisCoordinator.java b/src/main/java/com/linglevel/api/word/service/WordSingleFlightRedisCoordinator.java new file mode 100644 index 00000000..552fbbd8 --- /dev/null +++ b/src/main/java/com/linglevel/api/word/service/WordSingleFlightRedisCoordinator.java @@ -0,0 +1,275 @@ +package com.linglevel.api.word.service; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.linglevel.api.i18n.LanguageCode; +import com.linglevel.api.word.dto.WordAnalysisResult; +import jakarta.annotation.PostConstruct; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.redisson.api.RLock; +import org.redisson.api.RedissonClient; +import org.springframework.data.redis.connection.Message; +import org.springframework.data.redis.connection.MessageListener; +import org.springframework.data.redis.core.StringRedisTemplate; +import org.springframework.data.redis.listener.PatternTopic; +import org.springframework.data.redis.listener.RedisMessageListenerContainer; +import org.springframework.stereotype.Service; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.time.Duration; +import java.util.HexFormat; +import java.util.List; +import java.util.Locale; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.function.Supplier; + +@Service +@RequiredArgsConstructor +@Slf4j +public class WordSingleFlightRedisCoordinator { + + private static final String LOCK_PREFIX = "sf:word:lock"; + private static final String RESULT_PREFIX = "sf:word:result"; + private static final String DONE_PREFIX = "sf:word:done"; + private static final String DONE_PATTERN = DONE_PREFIX + ":*"; + + private final StringRedisTemplate stringRedisTemplate; + private final RedisMessageListenerContainer redisMessageListenerContainer; + private final RedissonClient redissonClient; + private final WordSingleFlightProperties properties; + private final ObjectMapper objectMapper; + + private final ConcurrentHashMap>> channelWaiters = new ConcurrentHashMap<>(); + + private final MessageListener doneListener = this::onDoneMessage; + + @PostConstruct + void subscribeDonePattern() { + redisMessageListenerContainer.addMessageListener(doneListener, new PatternTopic(DONE_PATTERN)); + } + + public List execute( + String word, + LanguageCode targetLanguage, + Supplier> leaderAction + ) { + if (!properties.isEnabled()) { + return leaderAction.get(); + } + + KeySet keys = buildKeySet(word, targetLanguage); + ResultEnvelope cached = readResult(keys.resultKey()); + if (cached != null) { + return unwrap(cached, keys.digest()); + } + + RLock lock = redissonClient.getLock(keys.lockKey()); + boolean lockAcquired = tryAcquireLeaderLock(lock); + if (lockAcquired) { + return executeAsLeader(keys, lock, leaderAction); + } + + return waitAsFollower(keys); + } + + private List executeAsLeader( + KeySet keys, + RLock lock, + Supplier> leaderAction + ) { + try { + List result = leaderAction.get(); + writeResult(keys.resultKey(), ResultEnvelope.success(result)); + publishDone(keys.channel()); + return result; + } catch (RuntimeException e) { + writeResult(keys.resultKey(), ResultEnvelope.failed(e.getMessage())); + publishDone(keys.channel()); + throw e; + } finally { + releaseLock(lock, keys.lockKey()); + } + } + + private boolean tryAcquireLeaderLock(RLock lock) { + try { + return lock.tryLock(0, properties.getLockTtlMs(), TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException("Interrupted while acquiring single-flight lock", e); + } + } + + private List waitAsFollower(KeySet keys) { + ResultEnvelope current = readResult(keys.resultKey()); + if (current != null) { + return unwrap(current, keys.digest()); + } + + CompletableFuture signal = new CompletableFuture<>(); + registerWaiter(keys.channel(), signal); + + try { + ResultEnvelope afterRegister = readResult(keys.resultKey()); + if (afterRegister != null) { + return unwrap(afterRegister, keys.digest()); + } + + signal.get(properties.getWaitTimeoutMs(), TimeUnit.MILLISECONDS); + } catch (TimeoutException e) { + log.warn("Single-flight wait timed out for key digest={}", keys.digest()); + } catch (Exception e) { + throw new RuntimeException("Single-flight wait interrupted for key digest=" + keys.digest(), e); + } finally { + unregisterWaiter(keys.channel(), signal); + } + + ResultEnvelope finalResult = readResult(keys.resultKey()); + if (finalResult != null) { + return unwrap(finalResult, keys.digest()); + } + + throw new WordSingleFlightTimeoutException( + "Timed out waiting single-flight result for key digest=" + keys.digest() + ); + } + + private void publishDone(String channel) { + stringRedisTemplate.convertAndSend(channel, "done"); + } + + private void releaseLock(RLock lock, String lockKey) { + try { + if (lock.isHeldByCurrentThread()) { + lock.unlock(); + } + } catch (Exception e) { + log.warn("Failed to release single-flight lock key={}", lockKey, e); + } + } + + private ResultEnvelope readResult(String resultKey) { + String raw = stringRedisTemplate.opsForValue().get(resultKey); + if (raw == null) { + return null; + } + + try { + return objectMapper.readValue(raw, ResultEnvelope.class); + } catch (JsonProcessingException e) { + log.warn("Failed to deserialize single-flight result key={}", resultKey, e); + return null; + } + } + + private void writeResult(String resultKey, ResultEnvelope envelope) { + try { + String raw = objectMapper.writeValueAsString(envelope); + stringRedisTemplate.opsForValue().set( + resultKey, + raw, + Duration.ofMillis(properties.getResultTtlMs()) + ); + } catch (JsonProcessingException e) { + throw new RuntimeException("Failed to serialize single-flight result", e); + } + } + + private List unwrap(ResultEnvelope envelope, String digest) { + if (envelope.success()) { + return envelope.results(); + } + + throw new WordSingleFlightLeaderFailureException( + "Single-flight leader failed for key digest=" + digest + ": " + envelope.errorMessage() + ); + } + + private void registerWaiter(String channel, CompletableFuture signal) { + channelWaiters.compute(channel, (key, waiters) -> { + CopyOnWriteArrayList> values = waiters == null + ? new CopyOnWriteArrayList<>() + : waiters; + values.add(signal); + return values; + }); + } + + private void unregisterWaiter(String channel, CompletableFuture signal) { + channelWaiters.computeIfPresent(channel, (key, waiters) -> { + waiters.remove(signal); + return waiters.isEmpty() ? null : waiters; + }); + } + + private void onDoneMessage(Message message, byte[] pattern) { + String channel = new String(message.getChannel(), StandardCharsets.UTF_8); + List> waiters = channelWaiters.remove(channel); + if (waiters == null || waiters.isEmpty()) { + return; + } + + for (CompletableFuture waiter : waiters) { + waiter.complete(null); + } + } + + private KeySet buildKeySet(String word, LanguageCode targetLanguage) { + String normalizedWord = word.trim().toLowerCase(Locale.ROOT); + String canonicalKey = String.join("|", + "word=" + normalizedWord, + "lang=" + targetLanguage.getCode(), + "prompt=" + properties.getPromptVersion(), + "model=" + properties.getModel(), + "schema=" + properties.getSchemaVersion() + ); + + String digest = sha256(canonicalKey); + String suffix = properties.getSchemaVersion() + ":" + digest; + + return new KeySet( + LOCK_PREFIX + ":" + suffix, + RESULT_PREFIX + ":" + suffix, + DONE_PREFIX + ":" + suffix, + digest + ); + } + + private String sha256(String value) { + try { + MessageDigest md = MessageDigest.getInstance("SHA-256"); + byte[] digest = md.digest(value.getBytes(StandardCharsets.UTF_8)); + return HexFormat.of().formatHex(digest); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException("SHA-256 is not available", e); + } + } + + private record KeySet( + String lockKey, + String resultKey, + String channel, + String digest + ) { } + + private record ResultEnvelope( + boolean success, + List results, + String errorMessage + ) { + static ResultEnvelope success(List results) { + return new ResultEnvelope(true, results, null); + } + + static ResultEnvelope failed(String errorMessage) { + return new ResultEnvelope(false, List.of(), errorMessage); + } + } +} diff --git a/src/main/java/com/linglevel/api/word/service/WordSingleFlightTimeoutException.java b/src/main/java/com/linglevel/api/word/service/WordSingleFlightTimeoutException.java new file mode 100644 index 00000000..cdf98a12 --- /dev/null +++ b/src/main/java/com/linglevel/api/word/service/WordSingleFlightTimeoutException.java @@ -0,0 +1,8 @@ +package com.linglevel.api.word.service; + +public class WordSingleFlightTimeoutException extends RuntimeException { + + public WordSingleFlightTimeoutException(String message) { + super(message); + } +} diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index b3f13192..e9a48dc4 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -51,6 +51,15 @@ firebase.config=${FIREBASE_CONFIG_BASE64} # Redis spring.data.redis.ssl.enabled=false +# Word single-flight (Redis lock + Pub/Sub + result key fallback) +word.single-flight.enabled=true +word.single-flight.lock-ttl-ms=13000 +word.single-flight.wait-timeout-ms=11000 +word.single-flight.result-ttl-ms=25000 +word.single-flight.prompt-version=v1 +word.single-flight.model=${spring.ai.bedrock.converse.chat.options.model:default} +word.single-flight.schema-version=v2 + # AWS S3 (AI Input/Output buckets) aws.s3.region=${S3_REGION} aws.s3.ai.input.bucket=${S3_AI_INPUT_NAME} @@ -74,4 +83,4 @@ sentry.environment=${spring.profiles.active} # Rate Limiting rate.limit.capacity=5000 -rate.limit.refill.duration.minutes=1 \ No newline at end of file +rate.limit.refill.duration.minutes=1 diff --git a/src/test/java/com/linglevel/api/common/filter/RateLimitFilterTest.java b/src/test/java/com/linglevel/api/common/filter/RateLimitFilterTest.java index 2922df41..144f8025 100644 --- a/src/test/java/com/linglevel/api/common/filter/RateLimitFilterTest.java +++ b/src/test/java/com/linglevel/api/common/filter/RateLimitFilterTest.java @@ -8,19 +8,16 @@ import com.linglevel.api.common.ratelimit.filter.RateLimitResolver; import com.linglevel.api.user.entity.UserRole; import io.github.bucket4j.distributed.proxy.ProxyManager; -import io.github.bucket4j.redis.lettuce.cas.LettuceBasedProxyManager; -import io.lettuce.core.RedisClient; -import io.lettuce.core.RedisURI; -import io.lettuce.core.api.StatefulRedisConnection; -import io.lettuce.core.codec.ByteArrayCodec; -import io.lettuce.core.codec.RedisCodec; -import io.lettuce.core.codec.StringCodec; +import io.github.bucket4j.redis.redisson.Bucket4jRedisson; import jakarta.servlet.FilterChain; import jakarta.servlet.http.HttpServletRequest; import jakarta.servlet.http.HttpServletResponse; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.redisson.Redisson; +import org.redisson.api.RedissonClient; +import org.redisson.config.Config; import org.springframework.security.authentication.UsernamePasswordAuthenticationToken; import org.springframework.security.core.authority.SimpleGrantedAuthority; import org.springframework.security.core.context.SecurityContextHolder; @@ -39,8 +36,7 @@ class RateLimitFilterTest extends AbstractRedisTest { private RateLimitFilter rateLimitFilter; private ProxyManager proxyManager; - private RedisClient redisClient; - private StatefulRedisConnection redisConnection; + private RedissonClient redissonClient; private RateLimitResolver rateLimitResolver; private HttpServletRequest request; @@ -57,19 +53,15 @@ void setUp() throws Exception { String host = redis.getHost(); Integer port = redis.getMappedPort(6379); - RedisURI redisUri = RedisURI.builder() - .withHost(host) - .withPort(port) - .withTimeout(Duration.ofSeconds(10)) - .build(); - - redisClient = RedisClient.create(redisUri); - redisConnection = redisClient.connect( - RedisCodec.of(StringCodec.UTF8, ByteArrayCodec.INSTANCE) - ); + Config redissonConfig = new Config(); + redissonConfig.useSingleServer() + .setAddress("redis://" + host + ":" + port) + .setTimeout((int) Duration.ofSeconds(10).toMillis()); + Redisson redisson = (Redisson) Redisson.create(redissonConfig); + redissonClient = redisson; // ProxyManager 생성 - proxyManager = LettuceBasedProxyManager.builderFor(redisConnection).build(); + proxyManager = Bucket4jRedisson.casBasedBuilder(redisson.getCommandExecutor()).build(); // RateLimitProperties 설정 RateLimitProperties properties = new RateLimitProperties(); @@ -88,7 +80,7 @@ void setUp() throws Exception { rateLimitFilter = new RateLimitFilter(proxyManager, properties, rateLimitResolver); // Redis 플러시 - redisConnection.sync().flushall(); + redissonClient.getKeys().flushall(); // Clear SecurityContext before each test SecurityContextHolder.clearContext(); @@ -106,11 +98,8 @@ void setUp() throws Exception { @AfterEach void tearDown() { - if (redisConnection != null) { - redisConnection.close(); - } - if (redisClient != null) { - redisClient.shutdown(); + if (redissonClient != null) { + redissonClient.shutdown(); } SecurityContextHolder.clearContext(); } diff --git a/src/test/java/com/linglevel/api/streak/service/ReadingSessionServiceTest.java b/src/test/java/com/linglevel/api/streak/service/ReadingSessionServiceTest.java index de32ff7b..6258c53c 100644 --- a/src/test/java/com/linglevel/api/streak/service/ReadingSessionServiceTest.java +++ b/src/test/java/com/linglevel/api/streak/service/ReadingSessionServiceTest.java @@ -8,7 +8,7 @@ import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import org.springframework.data.redis.connection.RedisStandaloneConfiguration; -import org.springframework.data.redis.connection.lettuce.LettuceConnectionFactory; +import org.springframework.data.redis.connection.jedis.JedisConnectionFactory; import org.springframework.data.redis.core.RedisTemplate; import org.springframework.data.redis.serializer.GenericJackson2JsonRedisSerializer; import org.springframework.data.redis.serializer.StringRedisSerializer; @@ -31,7 +31,7 @@ void setup() { config.setHostName(getRedisContainer().getHost()); config.setPort(getRedisContainer().getMappedPort(6379)); - LettuceConnectionFactory connectionFactory = new LettuceConnectionFactory(config); + JedisConnectionFactory connectionFactory = new JedisConnectionFactory(config); connectionFactory.afterPropertiesSet(); // RedisTemplate 설정 diff --git a/src/test/java/com/linglevel/api/word/service/WordServiceTest.java b/src/test/java/com/linglevel/api/word/service/WordServiceTest.java index 06f2a86c..f36910db 100644 --- a/src/test/java/com/linglevel/api/word/service/WordServiceTest.java +++ b/src/test/java/com/linglevel/api/word/service/WordServiceTest.java @@ -5,6 +5,7 @@ import com.linglevel.api.word.dto.*; import com.linglevel.api.word.entity.Word; import com.linglevel.api.word.entity.WordVariant; +import com.linglevel.api.word.exception.WordsException; import com.linglevel.api.word.repository.InvalidWordRepository; import com.linglevel.api.word.repository.WordRepository; import com.linglevel.api.word.repository.WordVariantRepository; @@ -18,8 +19,10 @@ import java.util.List; import java.util.Optional; +import java.util.function.Supplier; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.*; @@ -46,6 +49,9 @@ class WordServiceTest { @Mock private InvalidWordRepository invalidWordRepository; + @Mock + private WordSingleFlightRedisCoordinator singleFlightCoordinator; + @InjectMocks private WordService wordService; @@ -54,6 +60,13 @@ class WordServiceTest { @BeforeEach void setUp() { + lenient().when(singleFlightCoordinator.execute(anyString(), any(LanguageCode.class), any())) + .thenAnswer(invocation -> { + @SuppressWarnings("unchecked") + Supplier> supplier = invocation.getArgument(2); + return supplier.get(); + }); + // 샘플 Word 데이터 생성 sampleWord = Word.builder() .id("word-123") @@ -246,5 +259,84 @@ void setUp() { // then assertThat(response.getResults().get(0).getBookmarked()).isTrue(); } -} + @Test + @DisplayName("single-flight timeout은 무의미 단어로 캐시하지 않고 별도 에러를 반환") + void getOrCreateWords_singleFlightTimeout_doesNotCacheInvalidWord() { + String word = "resilience"; + + when(wordVariantRepository.findAllByWord(word)).thenReturn(List.of()); + when(invalidWordRepository.findByWord(word)).thenReturn(Optional.empty()); + when(singleFlightCoordinator.execute(eq(word), eq(LanguageCode.KO), any())) + .thenThrow(new WordSingleFlightTimeoutException("Timed out waiting single-flight result")); + + assertThatThrownBy(() -> wordService.getOrCreateWords(userId, word, LanguageCode.KO)) + .isInstanceOf(WordsException.class) + .hasMessageContaining("temporarily delayed"); + + verify(invalidWordRepository, never()).save(any()); + } + + @Test + @DisplayName("translation-miss 경로의 single-flight timeout도 WORD_ANALYSIS_TIMEOUT으로 변환") + void getOrCreateWords_translationMissTimeout_returnsDomainTimeoutError() { + String inputWord = "ran"; + String originalForm = "run"; + + WordVariant wordVariant = WordVariant.builder() + .word(inputWord) + .originalForm(originalForm) + .variantTypes(List.of(VariantType.PAST_TENSE)) + .build(); + + when(wordVariantRepository.findAllByWord(inputWord)).thenReturn(List.of(wordVariant)); + when(wordRepository.findByWordAndTargetLanguageCode(originalForm, LanguageCode.KO)) + .thenReturn(Optional.empty()); + when(singleFlightCoordinator.execute(eq(originalForm), eq(LanguageCode.KO), any())) + .thenThrow(new WordSingleFlightTimeoutException("Timed out waiting single-flight result")); + + assertThatThrownBy(() -> wordService.getOrCreateWords(userId, inputWord, LanguageCode.KO)) + .isInstanceOf(WordsException.class) + .hasMessageContaining("temporarily delayed"); + } + + @Test + @DisplayName("single-flight leader 실패 재생은 invalid 캐시를 증가시키지 않고 timeout 에러를 반환") + void getOrCreateWords_singleFlightLeaderFailure_doesNotCacheInvalidWord() { + String word = "resilience"; + + when(wordVariantRepository.findAllByWord(word)).thenReturn(List.of()); + when(invalidWordRepository.findByWord(word)).thenReturn(Optional.empty()); + when(singleFlightCoordinator.execute(eq(word), eq(LanguageCode.KO), any())) + .thenThrow(new WordSingleFlightLeaderFailureException("Single-flight leader failed")); + + assertThatThrownBy(() -> wordService.getOrCreateWords(userId, word, LanguageCode.KO)) + .isInstanceOf(WordsException.class) + .hasMessageContaining("temporarily delayed"); + + verify(invalidWordRepository, never()).save(any()); + } + + @Test + @DisplayName("translation-miss 경로의 leader 실패 재생도 WORD_ANALYSIS_TIMEOUT으로 변환") + void getOrCreateWords_translationMissLeaderFailure_returnsDomainTimeoutError() { + String inputWord = "ran"; + String originalForm = "run"; + + WordVariant wordVariant = WordVariant.builder() + .word(inputWord) + .originalForm(originalForm) + .variantTypes(List.of(VariantType.PAST_TENSE)) + .build(); + + when(wordVariantRepository.findAllByWord(inputWord)).thenReturn(List.of(wordVariant)); + when(wordRepository.findByWordAndTargetLanguageCode(originalForm, LanguageCode.KO)) + .thenReturn(Optional.empty()); + when(singleFlightCoordinator.execute(eq(originalForm), eq(LanguageCode.KO), any())) + .thenThrow(new WordSingleFlightLeaderFailureException("Single-flight leader failed")); + + assertThatThrownBy(() -> wordService.getOrCreateWords(userId, inputWord, LanguageCode.KO)) + .isInstanceOf(WordsException.class) + .hasMessageContaining("temporarily delayed"); + } +} diff --git a/src/test/java/com/linglevel/api/word/service/WordSingleFlightRedisCoordinatorIntegrationTest.java b/src/test/java/com/linglevel/api/word/service/WordSingleFlightRedisCoordinatorIntegrationTest.java new file mode 100644 index 00000000..9470bba5 --- /dev/null +++ b/src/test/java/com/linglevel/api/word/service/WordSingleFlightRedisCoordinatorIntegrationTest.java @@ -0,0 +1,210 @@ +package com.linglevel.api.word.service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.linglevel.api.common.AbstractRedisTest; +import com.linglevel.api.i18n.LanguageCode; +import com.linglevel.api.word.dto.WordAnalysisResult; +import org.redisson.Redisson; +import org.redisson.api.RedissonClient; +import org.redisson.config.Config; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.data.redis.connection.RedisStandaloneConfiguration; +import org.springframework.data.redis.connection.RedisConnection; +import org.springframework.data.redis.connection.jedis.JedisConnectionFactory; +import org.springframework.data.redis.core.StringRedisTemplate; +import org.springframework.data.redis.listener.RedisMessageListenerContainer; +import org.springframework.test.util.ReflectionTestUtils; +import org.testcontainers.containers.GenericContainer; + +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class WordSingleFlightRedisCoordinatorIntegrationTest extends AbstractRedisTest { + + private CoordinatorFixture nodeA; + private CoordinatorFixture nodeB; + + @BeforeEach + void setUp() { + nodeA = createNode("test-model-a", 3_000); + nodeB = createNode("test-model-a", 3_000); + flushAll(nodeA.template); + } + + @AfterEach + void tearDown() { + if (nodeA != null) { + nodeA.close(); + } + if (nodeB != null) { + nodeB.close(); + } + } + + @Test + @DisplayName("실제 Redis에서 두 인스턴스 동시 요청 시 AI 호출은 1회만 수행된다") + void deduplicatesAcrossTwoCoordinatorsUsingRealRedis() throws Exception { + AtomicInteger aiCalls = new AtomicInteger(); + ExecutorService executor = Executors.newFixedThreadPool(2); + CountDownLatch start = new CountDownLatch(1); + + try { + Future> f1 = executor.submit(() -> { + start.await(1, TimeUnit.SECONDS); + return nodeA.coordinator.execute("run", LanguageCode.KO, () -> { + aiCalls.incrementAndGet(); + sleep(250); + return List.of(sample("run")); + }); + }); + + Future> f2 = executor.submit(() -> { + start.await(1, TimeUnit.SECONDS); + return nodeB.coordinator.execute("run", LanguageCode.KO, () -> { + aiCalls.incrementAndGet(); + return List.of(sample("run")); + }); + }); + + start.countDown(); + + List r1 = f1.get(5, TimeUnit.SECONDS); + List r2 = f2.get(5, TimeUnit.SECONDS); + + assertThat(r1).hasSize(1); + assertThat(r2).hasSize(1); + assertThat(r1.get(0).getOriginalForm()).isEqualTo("run"); + assertThat(r2.get(0).getOriginalForm()).isEqualTo("run"); + assertThat(aiCalls.get()).isEqualTo(1); + } finally { + executor.shutdownNow(); + } + } + + @Test + @DisplayName("leader 실패는 실제 Redis resultKey를 통해 follower에도 동일 전파된다") + void propagatesLeaderFailureAcrossTwoCoordinatorsUsingRealRedis() { + RuntimeException leaderFailure = new RuntimeException("bedrock unavailable"); + AtomicInteger aiCalls = new AtomicInteger(); + + assertThatThrownBy(() -> + nodeA.coordinator.execute("left", LanguageCode.KO, () -> { + aiCalls.incrementAndGet(); + throw leaderFailure; + }) + ).isInstanceOf(RuntimeException.class) + .hasMessageContaining("bedrock unavailable"); + + assertThatThrownBy(() -> + nodeB.coordinator.execute("left", LanguageCode.KO, () -> { + aiCalls.incrementAndGet(); + return List.of(sample("left")); + }) + ).isInstanceOf(RuntimeException.class) + .hasMessageContaining("Single-flight leader failed"); + + assertThat(aiCalls.get()).isEqualTo(1); + } + + private CoordinatorFixture createNode(String model, long waitTimeoutMs) { + GenericContainer redis = getRedisContainer(); + RedisStandaloneConfiguration config = new RedisStandaloneConfiguration(redis.getHost(), redis.getMappedPort(6379)); + + JedisConnectionFactory connectionFactory = new JedisConnectionFactory(config); + connectionFactory.afterPropertiesSet(); + + StringRedisTemplate template = new StringRedisTemplate(); + template.setConnectionFactory(connectionFactory); + template.afterPropertiesSet(); + + RedisMessageListenerContainer listenerContainer = new RedisMessageListenerContainer(); + listenerContainer.setConnectionFactory(connectionFactory); + listenerContainer.afterPropertiesSet(); + listenerContainer.start(); + + Config redissonConfig = new Config(); + redissonConfig.useSingleServer() + .setAddress("redis://" + redis.getHost() + ":" + redis.getMappedPort(6379)); + RedissonClient redissonClient = Redisson.create(redissonConfig); + + WordSingleFlightProperties properties = new WordSingleFlightProperties(); + properties.setEnabled(true); + properties.setLockTtlMs(5_000); + properties.setWaitTimeoutMs(waitTimeoutMs); + properties.setResultTtlMs(30_000); + properties.setPromptVersion("v1"); + properties.setModel(model); + properties.setSchemaVersion("v2"); + + WordSingleFlightRedisCoordinator coordinator = new WordSingleFlightRedisCoordinator( + template, + listenerContainer, + redissonClient, + properties, + new ObjectMapper() + ); + ReflectionTestUtils.invokeMethod(coordinator, "subscribeDonePattern"); + + return new CoordinatorFixture(connectionFactory, template, listenerContainer, redissonClient, coordinator); + } + + private void flushAll(StringRedisTemplate template) { + RedisConnection connection = template.getConnectionFactory().getConnection(); + try { + connection.serverCommands().flushAll(); + } finally { + connection.close(); + } + } + + private WordAnalysisResult sample(String originalForm) { + return WordAnalysisResult.builder() + .originalForm(originalForm) + .sourceLanguageCode(LanguageCode.EN) + .targetLanguageCode(LanguageCode.KO) + .build(); + } + + private void sleep(long millis) { + try { + Thread.sleep(millis); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + } + + private record CoordinatorFixture( + JedisConnectionFactory connectionFactory, + StringRedisTemplate template, + RedisMessageListenerContainer listenerContainer, + RedissonClient redissonClient, + WordSingleFlightRedisCoordinator coordinator + ) { + void close() { + try { + listenerContainer.stop(); + } catch (Exception ignored) { + } + try { + redissonClient.shutdown(); + } catch (Exception ignored) { + } + try { + connectionFactory.destroy(); + } catch (Exception ignored) { + } + } + } +} diff --git a/src/test/java/com/linglevel/api/word/service/WordSingleFlightRedisCoordinatorTest.java b/src/test/java/com/linglevel/api/word/service/WordSingleFlightRedisCoordinatorTest.java new file mode 100644 index 00000000..ebc4c0fe --- /dev/null +++ b/src/test/java/com/linglevel/api/word/service/WordSingleFlightRedisCoordinatorTest.java @@ -0,0 +1,244 @@ +package com.linglevel.api.word.service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.linglevel.api.i18n.LanguageCode; +import com.linglevel.api.word.dto.WordAnalysisResult; +import org.redisson.api.RLock; +import org.redisson.api.RedissonClient; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.mockito.junit.jupiter.MockitoSettings; +import org.mockito.quality.Strictness; +import org.springframework.data.redis.core.StringRedisTemplate; +import org.springframework.data.redis.core.ValueOperations; +import org.springframework.data.redis.listener.RedisMessageListenerContainer; +import org.springframework.test.util.ReflectionTestUtils; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +@MockitoSettings(strictness = Strictness.LENIENT) +class WordSingleFlightRedisCoordinatorTest { + + @Mock + private StringRedisTemplate stringRedisTemplate; + + @Mock + private RedisMessageListenerContainer redisMessageListenerContainer; + + @Mock + private RedissonClient redissonClient; + + @Mock + private RLock redissonLock; + + @Mock + private ValueOperations valueOperations; + + private final ObjectMapper objectMapper = new ObjectMapper(); + private final Map redisStore = new ConcurrentHashMap<>(); + + private WordSingleFlightProperties properties; + private WordSingleFlightRedisCoordinator coordinator; + + @BeforeEach + void setUp() { + properties = new WordSingleFlightProperties(); + properties.setEnabled(true); + properties.setLockTtlMs(1_000); + properties.setWaitTimeoutMs(120); + properties.setResultTtlMs(2_000); + properties.setPromptVersion("v1"); + properties.setModel("test-model"); + properties.setSchemaVersion("v2"); + + when(stringRedisTemplate.opsForValue()).thenReturn(valueOperations); + when(redissonClient.getLock(anyString())).thenReturn(redissonLock); + when(redissonLock.isHeldByCurrentThread()).thenReturn(true); + + doAnswer(invocation -> redisStore.get(invocation.getArgument(0))) + .when(valueOperations).get(anyString()); + + doAnswer(invocation -> { + String key = invocation.getArgument(0); + String value = invocation.getArgument(1); + redisStore.put(key, value); + return null; + }).when(valueOperations).set(anyString(), anyString(), any(Duration.class)); + + coordinator = new WordSingleFlightRedisCoordinator( + stringRedisTemplate, + redisMessageListenerContainer, + redissonClient, + properties, + objectMapper + ); + ReflectionTestUtils.invokeMethod(coordinator, "subscribeDonePattern"); + } + + @Test + @DisplayName("동일 키 동시 요청은 leader action을 한 번만 실행한다") + void execute_deduplicatesConcurrentRequests() throws Exception { + stubTryLock(true, false); + + AtomicInteger aiCalls = new AtomicInteger(); + WordAnalysisResult sample = WordAnalysisResult.builder() + .originalForm("run") + .targetLanguageCode(LanguageCode.KO) + .sourceLanguageCode(LanguageCode.EN) + .build(); + + CountDownLatch start = new CountDownLatch(1); + ExecutorService executor = Executors.newFixedThreadPool(2); + + try { + Future> f1 = executor.submit(() -> { + start.await(1, TimeUnit.SECONDS); + return coordinator.execute("run", LanguageCode.KO, () -> { + aiCalls.incrementAndGet(); + sleep(50); + return List.of(sample); + }); + }); + + Future> f2 = executor.submit(() -> { + start.await(1, TimeUnit.SECONDS); + return coordinator.execute("run", LanguageCode.KO, () -> { + aiCalls.incrementAndGet(); + return List.of(sample); + }); + }); + + start.countDown(); + + List r1 = f1.get(2, TimeUnit.SECONDS); + List r2 = f2.get(2, TimeUnit.SECONDS); + + assertThat(r1).hasSize(1); + assertThat(r2).hasSize(1); + assertThat(aiCalls.get()).isEqualTo(1); + } finally { + executor.shutdownNow(); + } + } + + @Test + @DisplayName("알림 유실 상황에서도 timeout 이후 resultKey 재조회로 결과를 반환한다") + void execute_fallbacksToResultKeyAfterTimeout() { + stubTryLock(false); + + redisStore.clear(); + + WordAnalysisResult sample = WordAnalysisResult.builder() + .originalForm("book") + .targetLanguageCode(LanguageCode.KO) + .sourceLanguageCode(LanguageCode.EN) + .build(); + + String serialized = toSuccessEnvelopeJson(sample); + AtomicInteger getCalls = new AtomicInteger(); + + doAnswer(invocation -> { + // execute() 내부 readResult 호출 순서: + // 1) 캐시 확인 -> null + // 2) follower 진입 후 pre-check -> null + // 3) register 후 post-check -> null + // 4) timeout 후 final-check -> success + int n = getCalls.incrementAndGet(); + if (n < 4) { + return null; + } + return serialized; + }).when(valueOperations).get(anyString()); + + List result = coordinator.execute( + "book", + LanguageCode.KO, + () -> { + throw new IllegalStateException("follower path should not run leader action"); + } + ); + + assertThat(result).hasSize(1); + assertThat(result.get(0).getOriginalForm()).isEqualTo("book"); + assertThat(getCalls.get()).isGreaterThanOrEqualTo(4); + } + + @Test + @DisplayName("leader 실패 결과는 같은 키 요청에 동일하게 전파된다") + void execute_propagatesLeaderFailure() { + stubTryLock(true, false); + + RuntimeException failure = new RuntimeException("bedrock failure"); + + assertThatThrownBy(() -> + coordinator.execute("left", LanguageCode.KO, () -> { + throw failure; + }) + ).isInstanceOf(RuntimeException.class) + .hasMessageContaining("bedrock failure"); + + assertThatThrownBy(() -> + coordinator.execute("left", LanguageCode.KO, ArrayList::new) + ).isInstanceOf(RuntimeException.class) + .hasMessageContaining("Single-flight leader failed"); + } + + private void sleep(long millis) { + try { + Thread.sleep(millis); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + } + + private String toSuccessEnvelopeJson(WordAnalysisResult result) { + try { + Map payload = new HashMap<>(); + payload.put("success", true); + payload.put("results", List.of(result)); + payload.put("errorMessage", null); + return objectMapper.writeValueAsString(payload); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private void stubTryLock(boolean first, boolean... others) { + Boolean[] sequence = new Boolean[others.length + 1]; + sequence[0] = first; + for (int i = 0; i < others.length; i++) { + sequence[i + 1] = others[i]; + } + + try { + when(redissonLock.tryLock(0, properties.getLockTtlMs(), TimeUnit.MILLISECONDS)) + .thenReturn(sequence[0], java.util.Arrays.copyOfRange(sequence, 1, sequence.length)); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } +}