新增:redis工具类重构,配置主从,哨兵模式。hpc大文件上传优化。

This commit is contained in:
2026-03-18 18:29:09 +08:00
parent ecb515a0b8
commit c9c1ee9b06
19 changed files with 840 additions and 306 deletions

View File

@@ -0,0 +1,32 @@
package com.sdm.common.common;
import lombok.AllArgsConstructor;
import lombok.Getter;
@Getter
@AllArgsConstructor
public enum SpdmRedisKeyEnum {
/**
* 场景测试1有效期2小时
*/
YANG_TEST1("yang:test1:%s", 2 * 60 * 60),
/**
* 场景测试2永久有效
*/
YANG_TEST2("yang:test2:%s", -1);
/**
* key 模板
*/
private final String key;
/**
* 过期时间(秒) -1 表示永久有效
*/
private final int expire;
}

View File

@@ -0,0 +1,70 @@
package com.sdm.common.config;
import com.alibaba.fastjson2.support.spring6.data.redis.GenericFastJsonRedisSerializer;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.redis.connection.RedisConnectionFactory;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.data.redis.serializer.RedisSerializer;
import org.springframework.data.redis.serializer.StringRedisSerializer;
@Configuration
public class RedisConfig {
/**
* 自定义RedisTemplate使用FASTJSON序列化
*/
@Bean(name = "spdmRedisTemplate")
public RedisTemplate<String, Object> redisTemplate(RedisConnectionFactory factory) {
RedisTemplate<String, Object> redisTemplate = new RedisTemplate<>();
redisTemplate.setConnectionFactory(factory);
// String类型key序列化
StringRedisSerializer stringRedisSerializer = new StringRedisSerializer();
// fastJSON类型value序列化
GenericFastJsonRedisSerializer fastJsonRedisSerializer = new GenericFastJsonRedisSerializer();
// key采用String序列化
redisTemplate.setKeySerializer(stringRedisSerializer);
// hash的key也采用String序列化
redisTemplate.setHashKeySerializer(stringRedisSerializer);
// value序列化采用JSON
redisTemplate.setValueSerializer(fastJsonRedisSerializer);
redisTemplate.setHashValueSerializer(fastJsonRedisSerializer);
redisTemplate.afterPropertiesSet();
return redisTemplate;
}
// @Bean(name = "redisTemplate")
// public RedisTemplate<Object, Object> redisTemplate(RedisConnectionFactory factory) {
// RedisTemplate<Object, Object> template = new RedisTemplate<>();
// RedisSerializer<String> redisSerializer = new StringRedisSerializer();
//
// template.setConnectionFactory(factory);
// // key序列化方式
// template.setKeySerializer(redisSerializer);
// // value序列化
// template.setValueSerializer(redisSerializer);
// // value hashmap序列化
// template.setHashValueSerializer(redisSerializer);
// // key haspmap序列化
// template.setHashKeySerializer(redisSerializer);
//
// return template;
// }
@Bean(name = "bytesRedisTemplate")
public RedisTemplate<String, byte[]> bytesRedisTemplate(RedisConnectionFactory connectionFactory) {
RedisTemplate<String, byte[]> redisTemplate = new RedisTemplate<>();
redisTemplate.setConnectionFactory(connectionFactory);
// 设置key和value的序列化规则
redisTemplate.setValueSerializer(RedisSerializer.byteArray());
redisTemplate.setKeySerializer(new StringRedisSerializer());
redisTemplate.afterPropertiesSet();
return redisTemplate;
}
}

View File

@@ -10,12 +10,16 @@ import com.sdm.common.entity.resp.pbs.hpc.FileNodeInfo;
import com.sdm.common.log.CoreLogger;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.http.HttpEntity;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.core.ParameterizedTypeReference;
import org.springframework.core.io.ByteArrayResource;
import org.springframework.core.io.buffer.DataBuffer;
import org.springframework.core.io.buffer.DataBufferUtils;
import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
@@ -30,7 +34,6 @@ import java.io.*;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLEncoder;
import java.nio.channels.Channels;
import java.nio.charset.StandardCharsets;
import java.util.*;
@@ -157,33 +160,48 @@ public class HpcCommandExcuteUtil {
return nodeInfos;
}
public ResponseEntity<StreamingResponseBody> hpcDownloadFile(String path, Long fileSize,WebClient pbsWebClient) {
public ResponseEntity<StreamingResponseBody> hpcDownloadFile(String path) {
String fileName = extractFileName(path);
String encodedFileName = URLEncoder.encode(fileName, StandardCharsets.UTF_8);
StreamingResponseBody body = outputStream -> {
// 构建完整 URL并安全编码 path
String url = remoteDownLoadFileUrl + "?path=" + URLEncoder.encode(path, StandardCharsets.UTF_8);
// 调用 B 服务并流式写出
DataBufferUtils.write(
pbsWebClient.get()
.uri(url)
.retrieve()
.bodyToFlux(DataBuffer.class),
Channels.newChannel(outputStream)
).blockLast();
// 超时 24 小时
RequestConfig config = RequestConfig.custom()
.setSocketTimeout(86400000)
.setConnectTimeout(30000)
.build();
HttpGet httpGet = new HttpGet(url);
httpGet.setConfig(config);
try (CloseableHttpClient client = HttpClientBuilder.create().build();
CloseableHttpResponse resp = client.execute(httpGet)) {
HttpEntity entity = resp.getEntity();
if (entity == null) return;
try (BufferedInputStream in = new BufferedInputStream(entity.getContent())) {
byte[] buffer = new byte[8192];
int len;
while ((len = in.read(buffer)) != -1) {
outputStream.write(buffer, 0, len);
outputStream.flush();
}
}
} catch (Exception ignored) {
log.error("hpcDownloadFile error,url:{},errMsg:{}", url, ignored.getMessage());
}
};
ResponseEntity.BodyBuilder builder = ResponseEntity.ok()
return ResponseEntity.ok()
.contentType(MediaType.APPLICATION_OCTET_STREAM)
.header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename*=UTF-8''" + encodedFileName)
.contentType(MediaType.APPLICATION_OCTET_STREAM);
if (fileSize != null && fileSize > 0) {
builder.contentLength(fileSize);
}
return builder.body(body);
.header("Connection", "keep-alive")
// .header("Transfer-Encoding", "chunked")
.header("Cache-Control", "no-cache")
.body(body);
}
public SdmResponse<Boolean> callHpcUploadToTarget(String jobId, String workDir,String minioBucket,

View File

@@ -0,0 +1,377 @@
package com.sdm.common.utils;
import com.sdm.common.common.SpdmRedisKeyEnum;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.tuple.Pair;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.stereotype.Component;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
/**
* 基于spring和redis的redis工具类
* 针对所有的hash 都是以h开头的方法
* 针对所有的Set 都是以s开头的方法 不含通用方法
* 针对所有的List 都是以l开头的方法
*/
@Component
@Slf4j
public class SpdmRedisUtil {
public static final String REDIS_KEY_SEPARATOR = ":";
@Autowired
@Qualifier("spdmRedisTemplate")
private RedisTemplate<String, Object> redisTemplate;
// redis key 的前缀
@Value("${spring.profiles.active:}")
private String keyPrefixEnv;
// ==================== 核心:根据枚举生成真实 KEY ====================
private String getRealKey(SpdmRedisKeyEnum keyEnum, Object... params) {
String format = String.format(keyEnum.getKey(), params);
return keyPrefixEnv + REDIS_KEY_SEPARATOR+format;
}
// ==================== 1. 公共通用方法 ====================
/**
* 指定缓存失效时间(使用枚举自带过期时间)
*/
public boolean expire(SpdmRedisKeyEnum keyEnum, Object... params) {
try {
String realKey = getRealKey(keyEnum, params);
int expire = keyEnum.getExpire();
if (expire > 0) {
redisTemplate.expire(realKey, expire, TimeUnit.SECONDS);
}
return true;
} catch (Exception e) {
log.error("redis设置过期时间异常key:{}", getRealKey(keyEnum, params), e);
return false;
}
}
/**
* 根据key获取过期时间
* 0 = 永久有效
*/
public long getExpire(SpdmRedisKeyEnum keyEnum, Object... params) {
String realKey = getRealKey(keyEnum, params);
Long expire = redisTemplate.getExpire(realKey, TimeUnit.SECONDS);
return expire == null ? -2 : expire;
}
/**
* 判断key是否存在
*/
public boolean hasKey(SpdmRedisKeyEnum keyEnum, Object... params) {
try {
String realKey = getRealKey(keyEnum, params);
return Boolean.TRUE.equals(redisTemplate.hasKey(realKey));
} catch (Exception e) {
log.error("redis判断key是否存在异常key:{}", getRealKey(keyEnum, params), e);
return false;
}
}
/**
* 删除缓存(支持单个/多个枚举key
*/
public void del(SpdmRedisKeyEnum... keyEnums) {
for (SpdmRedisKeyEnum keyEnum : keyEnums) {
del(keyEnum, new Object[0]);
}
}
public void del(SpdmRedisKeyEnum keyEnum, Object... params) {
String realKey = getRealKey(keyEnum, params);
try {
redisTemplate.delete(realKey);
} catch (Exception e) {
log.error("redis删除key异常key:{}", realKey, e);
}
}
// ==================== 2. String 类型 ====================
/**
* 普通缓存放入(自动使用枚举过期时间)
* Pair<是否成功,真实key>
*/
public Pair<Boolean,String> set(SpdmRedisKeyEnum keyEnum, Object value, Object... params) {
try {
String realKey = getRealKey(keyEnum, params);
int expire = keyEnum.getExpire();
if (expire > 0) {
redisTemplate.opsForValue().set(realKey, value, expire, TimeUnit.SECONDS);
} else {
redisTemplate.opsForValue().set(realKey, value);
}
return Pair.of(true, realKey);
} catch (Exception e) {
log.error("redis字符串存入异常key:{}", getRealKey(keyEnum, params), e);
return Pair.of(false, "");
}
}
/**
* 普通缓存获取
*/
public Object get(SpdmRedisKeyEnum keyEnum, Object... params) {
String realKey = getRealKey(keyEnum, params);
return redisTemplate.opsForValue().get(realKey);
}
/**
* 获取Java对象自动反序列化
*/
public <T> T getBean(SpdmRedisKeyEnum keyEnum, Class<T> clazz, Object... params) {
String realKey = getRealKey(keyEnum, params);
Object obj = redisTemplate.opsForValue().get(realKey);
if (obj == null) return null;
// 校验对象类型是否匹配
if (!clazz.isInstance(obj)) {
log.warn("Redis缓存对象类型不匹配期望{},实际:{},redis key:{}",clazz.getName(),obj.getClass().getName(),realKey);
return null;
}
return clazz.cast(obj);
}
/**
* 递增
*/
public long incr(SpdmRedisKeyEnum keyEnum, long delta, Object... params) {
if (delta < 0) {
throw new RuntimeException("递增因子必须大于0");
}
String realKey = getRealKey(keyEnum, params);
Long increment = redisTemplate.opsForValue().increment(realKey, delta);
return increment == null ? 0 : increment;
}
/**
* 递减
*/
public long decr(SpdmRedisKeyEnum keyEnum, long delta, Object... params) {
if (delta < 0) {
throw new RuntimeException("递减因子必须大于0");
}
String realKey = getRealKey(keyEnum, params);
Long increment = redisTemplate.opsForValue().increment(realKey, -delta);
return increment == null ? 0 : increment;
}
// ==================== 3. Hash 类型 ====================
public Object hget(SpdmRedisKeyEnum keyEnum, String item, Object... params) {
return redisTemplate.opsForHash().get(getRealKey(keyEnum, params), item);
}
public Map<Object, Object> hmget(SpdmRedisKeyEnum keyEnum, Object... params) {
return redisTemplate.opsForHash().entries(getRealKey(keyEnum, params));
}
public boolean hmset(SpdmRedisKeyEnum keyEnum, Map<String, Object> map, Object... params) {
try {
String realKey = getRealKey(keyEnum, params);
redisTemplate.opsForHash().putAll(realKey, map);
expire(keyEnum, params);
return true;
} catch (Exception e) {
log.error("redis hash批量存入异常key:{}", getRealKey(keyEnum, params), e);
return false;
}
}
public boolean hset(SpdmRedisKeyEnum keyEnum, String item, Object value, Object... params) {
try {
String realKey = getRealKey(keyEnum, params);
redisTemplate.opsForHash().put(realKey, item, value);
expire(keyEnum, params);
return true;
} catch (Exception e) {
log.error("redis hash存入异常key:{}", getRealKey(keyEnum, params), e);
return false;
}
}
public void hdel(SpdmRedisKeyEnum keyEnum, Object... items) {
hdel(keyEnum, null, items);
}
public void hdel(SpdmRedisKeyEnum keyEnum, Object[] params, Object... items) {
redisTemplate.opsForHash().delete(getRealKey(keyEnum, params), items);
}
public boolean hHasKey(SpdmRedisKeyEnum keyEnum, String item, Object... params) {
return redisTemplate.opsForHash().hasKey(getRealKey(keyEnum, params), item);
}
public double hincr(SpdmRedisKeyEnum keyEnum, String item, double by, Object... params) {
return redisTemplate.opsForHash().increment(getRealKey(keyEnum, params), item, by);
}
public double hdecr(SpdmRedisKeyEnum keyEnum, String item, double by, Object... params) {
return redisTemplate.opsForHash().increment(getRealKey(keyEnum, params), item, -by);
}
// ==================== 4. Set 类型 ====================
public Set<Object> sGet(SpdmRedisKeyEnum keyEnum, Object... params) {
return redisTemplate.opsForSet().members(getRealKey(keyEnum, params));
}
public boolean sHasKey(SpdmRedisKeyEnum keyEnum, Object value, Object... params) {
return Boolean.TRUE.equals(redisTemplate.opsForSet().isMember(getRealKey(keyEnum, params), value));
}
public long sSet(SpdmRedisKeyEnum keyEnum, Object... values) {
return sSet(keyEnum, new Object[0], values);
}
public long sSet(SpdmRedisKeyEnum keyEnum, Object[] params, Object... values) {
try {
String realKey = getRealKey(keyEnum, params);
Long count = redisTemplate.opsForSet().add(realKey, values);
expire(keyEnum, params);
return count == null ? 0 : count;
} catch (Exception e) {
log.error("redis set存入异常key:{}", getRealKey(keyEnum, params), e);
return 0;
}
}
public long sGetSetSize(SpdmRedisKeyEnum keyEnum, Object... params) {
Long size = redisTemplate.opsForSet().size(getRealKey(keyEnum, params));
return size == null ? 0 : size;
}
public long setRemove(SpdmRedisKeyEnum keyEnum, Object... values) {
return setRemove(keyEnum, new Object[0], values);
}
public long setRemove(SpdmRedisKeyEnum keyEnum, Object[] params, Object... values) {
Long count = redisTemplate.opsForSet().remove(getRealKey(keyEnum, params), values);
return count == null ? 0 : count;
}
// ==================== 5. ZSet 类型 ====================
// ==================== 5. ZSet 类型(修正版)====================
/**
* ZSet 添加元素(带分数)
*/
public Boolean zAdd(SpdmRedisKeyEnum keyEnum, Object value, double score, Object... params) {
try {
String realKey = getRealKey(keyEnum, params);
return redisTemplate.opsForZSet().add(realKey, value, score);
} catch (Exception e) {
log.error("redis zset添加异常key:{}", getRealKey(keyEnum, params), e);
return false;
}
}
/**
* ZSet 移除指定元素(修复完成)
*/
public long zRemove(SpdmRedisKeyEnum keyEnum, Object... values) {
return zRemove(keyEnum, new Object[0], values);
}
public long zRemove(SpdmRedisKeyEnum keyEnum, Object[] params, Object... values) {
try {
String realKey = getRealKey(keyEnum, params);
Long count = redisTemplate.opsForZSet().remove(realKey, values);
return count == null ? 0 : count;
} catch (Exception e) {
log.error("redis zset移除异常key:{}", getRealKey(keyEnum, params), e);
return 0;
}
}
/**
* 获取ZSet元素数量
*/
public long zSize(SpdmRedisKeyEnum keyEnum, Object... params) {
Long size = redisTemplate.opsForZSet().size(getRealKey(keyEnum, params));
return size == null ? 0 : size;
}
// ==================== 6. List 类型 ====================
public List<Object> lGet(SpdmRedisKeyEnum keyEnum, long start, long end, Object... params) {
return redisTemplate.opsForList().range(getRealKey(keyEnum, params), start, end);
}
public List<Object> lGetAll(SpdmRedisKeyEnum keyEnum, Object... params) {
return lGet(keyEnum, 0, -1, params);
}
public long lGetListSize(SpdmRedisKeyEnum keyEnum, Object... params) {
Long size = redisTemplate.opsForList().size(getRealKey(keyEnum, params));
return size == null ? 0 : size;
}
public Object lGetIndex(SpdmRedisKeyEnum keyEnum, long index, Object... params) {
return redisTemplate.opsForList().index(getRealKey(keyEnum, params), index);
}
public boolean lSet(SpdmRedisKeyEnum keyEnum, Object value, Object... params) {
try {
redisTemplate.opsForList().rightPush(getRealKey(keyEnum, params), value);
expire(keyEnum, params);
return true;
} catch (Exception e) {
log.error("redis list存入异常key:{}", getRealKey(keyEnum, params), e);
return false;
}
}
public boolean lSet(SpdmRedisKeyEnum keyEnum, List<Object> value, Object... params) {
try {
redisTemplate.opsForList().rightPushAll(getRealKey(keyEnum, params), value);
expire(keyEnum, params);
return true;
} catch (Exception e) {
log.error("redis list批量存入异常key:{}", getRealKey(keyEnum, params), e);
return false;
}
}
public Object lLeftPop(SpdmRedisKeyEnum keyEnum, Object... params) {
return redisTemplate.opsForList().leftPop(getRealKey(keyEnum, params));
}
public boolean lUpdateIndex(SpdmRedisKeyEnum keyEnum, long index, Object value, Object... params) {
try {
redisTemplate.opsForList().set(getRealKey(keyEnum, params), index, value);
return true;
} catch (Exception e) {
log.error("redis list更新异常key:{}", getRealKey(keyEnum, params), e);
return false;
}
}
public long lRemove(SpdmRedisKeyEnum keyEnum, long count, Object value, Object... params) {
Long remove = redisTemplate.opsForList().remove(getRealKey(keyEnum, params), count, value);
return remove == null ? 0 : remove;
}
public void lRemoveAll(SpdmRedisKeyEnum keyEnum, Object... params) {
redisTemplate.opsForList().trim(getRealKey(keyEnum, params), -1, 0);
}
}