新增:redis工具类重构,配置主从,哨兵模式。hpc大文件上传优化。

This commit is contained in:
2026-03-18 18:29:09 +08:00
parent ecb515a0b8
commit c9c1ee9b06
19 changed files with 840 additions and 306 deletions

View File

@@ -1,45 +1,45 @@
package com.sdm.capability.config;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.redis.connection.RedisConnectionFactory;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.data.redis.serializer.RedisSerializer;
import org.springframework.data.redis.serializer.StringRedisSerializer;
@Configuration
public class RedisConfig {
@Bean(name = "redisTemplate")
public RedisTemplate<Object, Object> redisTemplate(RedisConnectionFactory factory) {
RedisTemplate<Object, Object> template = new RedisTemplate<>();
RedisSerializer<String> redisSerializer = new StringRedisSerializer();
template.setConnectionFactory(factory);
// key序列化方式
template.setKeySerializer(redisSerializer);
// value序列化
template.setValueSerializer(redisSerializer);
// value hashmap序列化
template.setHashValueSerializer(redisSerializer);
// key haspmap序列化
template.setHashKeySerializer(redisSerializer);
return template;
}
@Bean(name = "bytesRedisTemplate")
public RedisTemplate<String, byte[]> bytesRedisTemplate(RedisConnectionFactory connectionFactory) {
RedisTemplate<String, byte[]> redisTemplate = new RedisTemplate<>();
redisTemplate.setConnectionFactory(connectionFactory);
// 设置key和value的序列化规则
redisTemplate.setValueSerializer(RedisSerializer.byteArray());
redisTemplate.setKeySerializer(new StringRedisSerializer());
redisTemplate.afterPropertiesSet();
return redisTemplate;
}
}
//package com.sdm.capability.config;
//
//import org.springframework.context.annotation.Bean;
//import org.springframework.context.annotation.Configuration;
//import org.springframework.data.redis.connection.RedisConnectionFactory;
//import org.springframework.data.redis.core.RedisTemplate;
//import org.springframework.data.redis.serializer.RedisSerializer;
//import org.springframework.data.redis.serializer.StringRedisSerializer;
//
//@Configuration
//public class RedisConfig {
//
// @Bean(name = "redisTemplate")
// public RedisTemplate<Object, Object> redisTemplate(RedisConnectionFactory factory) {
// RedisTemplate<Object, Object> template = new RedisTemplate<>();
// RedisSerializer<String> redisSerializer = new StringRedisSerializer();
//
// template.setConnectionFactory(factory);
// // key序列化方式
// template.setKeySerializer(redisSerializer);
// // value序列化
// template.setValueSerializer(redisSerializer);
// // value hashmap序列化
// template.setHashValueSerializer(redisSerializer);
// // key haspmap序列化
// template.setHashKeySerializer(redisSerializer);
//
// return template;
// }
//
// @Bean(name = "bytesRedisTemplate")
// public RedisTemplate<String, byte[]> bytesRedisTemplate(RedisConnectionFactory connectionFactory) {
// RedisTemplate<String, byte[]> redisTemplate = new RedisTemplate<>();
// redisTemplate.setConnectionFactory(connectionFactory);
// // 设置key和value的序列化规则
// redisTemplate.setValueSerializer(RedisSerializer.byteArray());
// redisTemplate.setKeySerializer(new StringRedisSerializer());
// redisTemplate.afterPropertiesSet();
//
// return redisTemplate;
// }
//}
//
//
//

View File

@@ -77,6 +77,12 @@
<scope>provided</scope> <!-- 关键:运行时由 web 模块的容器(如 Tomcat提供实现 -->
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.5.14</version>
</dependency>
</dependencies>
<dependencyManagement>
<dependencies>

View File

@@ -0,0 +1,32 @@
package com.sdm.common.common;
import lombok.AllArgsConstructor;
import lombok.Getter;
@Getter
@AllArgsConstructor
public enum SpdmRedisKeyEnum {
/**
* 场景测试1有效期2小时
*/
YANG_TEST1("yang:test1:%s", 2 * 60 * 60),
/**
* 场景测试2永久有效
*/
YANG_TEST2("yang:test2:%s", -1);
/**
* key 模板
*/
private final String key;
/**
* 过期时间(秒) -1 表示永久有效
*/
private final int expire;
}

View File

@@ -0,0 +1,70 @@
package com.sdm.common.config;
import com.alibaba.fastjson2.support.spring6.data.redis.GenericFastJsonRedisSerializer;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.redis.connection.RedisConnectionFactory;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.data.redis.serializer.RedisSerializer;
import org.springframework.data.redis.serializer.StringRedisSerializer;
@Configuration
public class RedisConfig {
/**
* 自定义RedisTemplate使用FASTJSON序列化
*/
@Bean(name = "spdmRedisTemplate")
public RedisTemplate<String, Object> redisTemplate(RedisConnectionFactory factory) {
RedisTemplate<String, Object> redisTemplate = new RedisTemplate<>();
redisTemplate.setConnectionFactory(factory);
// String类型key序列化
StringRedisSerializer stringRedisSerializer = new StringRedisSerializer();
// fastJSON类型value序列化
GenericFastJsonRedisSerializer fastJsonRedisSerializer = new GenericFastJsonRedisSerializer();
// key采用String序列化
redisTemplate.setKeySerializer(stringRedisSerializer);
// hash的key也采用String序列化
redisTemplate.setHashKeySerializer(stringRedisSerializer);
// value序列化采用JSON
redisTemplate.setValueSerializer(fastJsonRedisSerializer);
redisTemplate.setHashValueSerializer(fastJsonRedisSerializer);
redisTemplate.afterPropertiesSet();
return redisTemplate;
}
// @Bean(name = "redisTemplate")
// public RedisTemplate<Object, Object> redisTemplate(RedisConnectionFactory factory) {
// RedisTemplate<Object, Object> template = new RedisTemplate<>();
// RedisSerializer<String> redisSerializer = new StringRedisSerializer();
//
// template.setConnectionFactory(factory);
// // key序列化方式
// template.setKeySerializer(redisSerializer);
// // value序列化
// template.setValueSerializer(redisSerializer);
// // value hashmap序列化
// template.setHashValueSerializer(redisSerializer);
// // key haspmap序列化
// template.setHashKeySerializer(redisSerializer);
//
// return template;
// }
@Bean(name = "bytesRedisTemplate")
public RedisTemplate<String, byte[]> bytesRedisTemplate(RedisConnectionFactory connectionFactory) {
RedisTemplate<String, byte[]> redisTemplate = new RedisTemplate<>();
redisTemplate.setConnectionFactory(connectionFactory);
// 设置key和value的序列化规则
redisTemplate.setValueSerializer(RedisSerializer.byteArray());
redisTemplate.setKeySerializer(new StringRedisSerializer());
redisTemplate.afterPropertiesSet();
return redisTemplate;
}
}

View File

@@ -10,12 +10,16 @@ import com.sdm.common.entity.resp.pbs.hpc.FileNodeInfo;
import com.sdm.common.log.CoreLogger;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.http.HttpEntity;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.core.ParameterizedTypeReference;
import org.springframework.core.io.ByteArrayResource;
import org.springframework.core.io.buffer.DataBuffer;
import org.springframework.core.io.buffer.DataBufferUtils;
import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
@@ -30,7 +34,6 @@ import java.io.*;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLEncoder;
import java.nio.channels.Channels;
import java.nio.charset.StandardCharsets;
import java.util.*;
@@ -157,33 +160,48 @@ public class HpcCommandExcuteUtil {
return nodeInfos;
}
public ResponseEntity<StreamingResponseBody> hpcDownloadFile(String path, Long fileSize,WebClient pbsWebClient) {
public ResponseEntity<StreamingResponseBody> hpcDownloadFile(String path) {
String fileName = extractFileName(path);
String encodedFileName = URLEncoder.encode(fileName, StandardCharsets.UTF_8);
StreamingResponseBody body = outputStream -> {
// 构建完整 URL并安全编码 path
String url = remoteDownLoadFileUrl + "?path=" + URLEncoder.encode(path, StandardCharsets.UTF_8);
// 调用 B 服务并流式写出
DataBufferUtils.write(
pbsWebClient.get()
.uri(url)
.retrieve()
.bodyToFlux(DataBuffer.class),
Channels.newChannel(outputStream)
).blockLast();
// 超时 24 小时
RequestConfig config = RequestConfig.custom()
.setSocketTimeout(86400000)
.setConnectTimeout(30000)
.build();
HttpGet httpGet = new HttpGet(url);
httpGet.setConfig(config);
try (CloseableHttpClient client = HttpClientBuilder.create().build();
CloseableHttpResponse resp = client.execute(httpGet)) {
HttpEntity entity = resp.getEntity();
if (entity == null) return;
try (BufferedInputStream in = new BufferedInputStream(entity.getContent())) {
byte[] buffer = new byte[8192];
int len;
while ((len = in.read(buffer)) != -1) {
outputStream.write(buffer, 0, len);
outputStream.flush();
}
}
} catch (Exception ignored) {
log.error("hpcDownloadFile error,url:{},errMsg:{}", url, ignored.getMessage());
}
};
ResponseEntity.BodyBuilder builder = ResponseEntity.ok()
return ResponseEntity.ok()
.contentType(MediaType.APPLICATION_OCTET_STREAM)
.header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename*=UTF-8''" + encodedFileName)
.contentType(MediaType.APPLICATION_OCTET_STREAM);
if (fileSize != null && fileSize > 0) {
builder.contentLength(fileSize);
}
return builder.body(body);
.header("Connection", "keep-alive")
// .header("Transfer-Encoding", "chunked")
.header("Cache-Control", "no-cache")
.body(body);
}
public SdmResponse<Boolean> callHpcUploadToTarget(String jobId, String workDir,String minioBucket,

View File

@@ -0,0 +1,377 @@
package com.sdm.common.utils;
import com.sdm.common.common.SpdmRedisKeyEnum;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.tuple.Pair;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.stereotype.Component;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
/**
* 基于spring和redis的redis工具类
* 针对所有的hash 都是以h开头的方法
* 针对所有的Set 都是以s开头的方法 不含通用方法
* 针对所有的List 都是以l开头的方法
*/
@Component
@Slf4j
public class SpdmRedisUtil {
public static final String REDIS_KEY_SEPARATOR = ":";
@Autowired
@Qualifier("spdmRedisTemplate")
private RedisTemplate<String, Object> redisTemplate;
// redis key 的前缀
@Value("${spring.profiles.active:}")
private String keyPrefixEnv;
// ==================== 核心:根据枚举生成真实 KEY ====================
private String getRealKey(SpdmRedisKeyEnum keyEnum, Object... params) {
String format = String.format(keyEnum.getKey(), params);
return keyPrefixEnv + REDIS_KEY_SEPARATOR+format;
}
// ==================== 1. 公共通用方法 ====================
/**
* 指定缓存失效时间(使用枚举自带过期时间)
*/
public boolean expire(SpdmRedisKeyEnum keyEnum, Object... params) {
try {
String realKey = getRealKey(keyEnum, params);
int expire = keyEnum.getExpire();
if (expire > 0) {
redisTemplate.expire(realKey, expire, TimeUnit.SECONDS);
}
return true;
} catch (Exception e) {
log.error("redis设置过期时间异常key:{}", getRealKey(keyEnum, params), e);
return false;
}
}
/**
* 根据key获取过期时间
* 0 = 永久有效
*/
public long getExpire(SpdmRedisKeyEnum keyEnum, Object... params) {
String realKey = getRealKey(keyEnum, params);
Long expire = redisTemplate.getExpire(realKey, TimeUnit.SECONDS);
return expire == null ? -2 : expire;
}
/**
* 判断key是否存在
*/
public boolean hasKey(SpdmRedisKeyEnum keyEnum, Object... params) {
try {
String realKey = getRealKey(keyEnum, params);
return Boolean.TRUE.equals(redisTemplate.hasKey(realKey));
} catch (Exception e) {
log.error("redis判断key是否存在异常key:{}", getRealKey(keyEnum, params), e);
return false;
}
}
/**
* 删除缓存(支持单个/多个枚举key
*/
public void del(SpdmRedisKeyEnum... keyEnums) {
for (SpdmRedisKeyEnum keyEnum : keyEnums) {
del(keyEnum, new Object[0]);
}
}
public void del(SpdmRedisKeyEnum keyEnum, Object... params) {
String realKey = getRealKey(keyEnum, params);
try {
redisTemplate.delete(realKey);
} catch (Exception e) {
log.error("redis删除key异常key:{}", realKey, e);
}
}
// ==================== 2. String 类型 ====================
/**
* 普通缓存放入(自动使用枚举过期时间)
* Pair<是否成功,真实key>
*/
public Pair<Boolean,String> set(SpdmRedisKeyEnum keyEnum, Object value, Object... params) {
try {
String realKey = getRealKey(keyEnum, params);
int expire = keyEnum.getExpire();
if (expire > 0) {
redisTemplate.opsForValue().set(realKey, value, expire, TimeUnit.SECONDS);
} else {
redisTemplate.opsForValue().set(realKey, value);
}
return Pair.of(true, realKey);
} catch (Exception e) {
log.error("redis字符串存入异常key:{}", getRealKey(keyEnum, params), e);
return Pair.of(false, "");
}
}
/**
* 普通缓存获取
*/
public Object get(SpdmRedisKeyEnum keyEnum, Object... params) {
String realKey = getRealKey(keyEnum, params);
return redisTemplate.opsForValue().get(realKey);
}
/**
* 获取Java对象自动反序列化
*/
public <T> T getBean(SpdmRedisKeyEnum keyEnum, Class<T> clazz, Object... params) {
String realKey = getRealKey(keyEnum, params);
Object obj = redisTemplate.opsForValue().get(realKey);
if (obj == null) return null;
// 校验对象类型是否匹配
if (!clazz.isInstance(obj)) {
log.warn("Redis缓存对象类型不匹配期望{},实际:{},redis key:{}",clazz.getName(),obj.getClass().getName(),realKey);
return null;
}
return clazz.cast(obj);
}
/**
* 递增
*/
public long incr(SpdmRedisKeyEnum keyEnum, long delta, Object... params) {
if (delta < 0) {
throw new RuntimeException("递增因子必须大于0");
}
String realKey = getRealKey(keyEnum, params);
Long increment = redisTemplate.opsForValue().increment(realKey, delta);
return increment == null ? 0 : increment;
}
/**
* 递减
*/
public long decr(SpdmRedisKeyEnum keyEnum, long delta, Object... params) {
if (delta < 0) {
throw new RuntimeException("递减因子必须大于0");
}
String realKey = getRealKey(keyEnum, params);
Long increment = redisTemplate.opsForValue().increment(realKey, -delta);
return increment == null ? 0 : increment;
}
// ==================== 3. Hash 类型 ====================
public Object hget(SpdmRedisKeyEnum keyEnum, String item, Object... params) {
return redisTemplate.opsForHash().get(getRealKey(keyEnum, params), item);
}
public Map<Object, Object> hmget(SpdmRedisKeyEnum keyEnum, Object... params) {
return redisTemplate.opsForHash().entries(getRealKey(keyEnum, params));
}
public boolean hmset(SpdmRedisKeyEnum keyEnum, Map<String, Object> map, Object... params) {
try {
String realKey = getRealKey(keyEnum, params);
redisTemplate.opsForHash().putAll(realKey, map);
expire(keyEnum, params);
return true;
} catch (Exception e) {
log.error("redis hash批量存入异常key:{}", getRealKey(keyEnum, params), e);
return false;
}
}
public boolean hset(SpdmRedisKeyEnum keyEnum, String item, Object value, Object... params) {
try {
String realKey = getRealKey(keyEnum, params);
redisTemplate.opsForHash().put(realKey, item, value);
expire(keyEnum, params);
return true;
} catch (Exception e) {
log.error("redis hash存入异常key:{}", getRealKey(keyEnum, params), e);
return false;
}
}
public void hdel(SpdmRedisKeyEnum keyEnum, Object... items) {
hdel(keyEnum, null, items);
}
public void hdel(SpdmRedisKeyEnum keyEnum, Object[] params, Object... items) {
redisTemplate.opsForHash().delete(getRealKey(keyEnum, params), items);
}
public boolean hHasKey(SpdmRedisKeyEnum keyEnum, String item, Object... params) {
return redisTemplate.opsForHash().hasKey(getRealKey(keyEnum, params), item);
}
public double hincr(SpdmRedisKeyEnum keyEnum, String item, double by, Object... params) {
return redisTemplate.opsForHash().increment(getRealKey(keyEnum, params), item, by);
}
public double hdecr(SpdmRedisKeyEnum keyEnum, String item, double by, Object... params) {
return redisTemplate.opsForHash().increment(getRealKey(keyEnum, params), item, -by);
}
// ==================== 4. Set 类型 ====================
public Set<Object> sGet(SpdmRedisKeyEnum keyEnum, Object... params) {
return redisTemplate.opsForSet().members(getRealKey(keyEnum, params));
}
public boolean sHasKey(SpdmRedisKeyEnum keyEnum, Object value, Object... params) {
return Boolean.TRUE.equals(redisTemplate.opsForSet().isMember(getRealKey(keyEnum, params), value));
}
public long sSet(SpdmRedisKeyEnum keyEnum, Object... values) {
return sSet(keyEnum, new Object[0], values);
}
public long sSet(SpdmRedisKeyEnum keyEnum, Object[] params, Object... values) {
try {
String realKey = getRealKey(keyEnum, params);
Long count = redisTemplate.opsForSet().add(realKey, values);
expire(keyEnum, params);
return count == null ? 0 : count;
} catch (Exception e) {
log.error("redis set存入异常key:{}", getRealKey(keyEnum, params), e);
return 0;
}
}
public long sGetSetSize(SpdmRedisKeyEnum keyEnum, Object... params) {
Long size = redisTemplate.opsForSet().size(getRealKey(keyEnum, params));
return size == null ? 0 : size;
}
public long setRemove(SpdmRedisKeyEnum keyEnum, Object... values) {
return setRemove(keyEnum, new Object[0], values);
}
public long setRemove(SpdmRedisKeyEnum keyEnum, Object[] params, Object... values) {
Long count = redisTemplate.opsForSet().remove(getRealKey(keyEnum, params), values);
return count == null ? 0 : count;
}
// ==================== 5. ZSet 类型 ====================
// ==================== 5. ZSet 类型(修正版)====================
/**
* ZSet 添加元素(带分数)
*/
public Boolean zAdd(SpdmRedisKeyEnum keyEnum, Object value, double score, Object... params) {
try {
String realKey = getRealKey(keyEnum, params);
return redisTemplate.opsForZSet().add(realKey, value, score);
} catch (Exception e) {
log.error("redis zset添加异常key:{}", getRealKey(keyEnum, params), e);
return false;
}
}
/**
* ZSet 移除指定元素(修复完成)
*/
public long zRemove(SpdmRedisKeyEnum keyEnum, Object... values) {
return zRemove(keyEnum, new Object[0], values);
}
public long zRemove(SpdmRedisKeyEnum keyEnum, Object[] params, Object... values) {
try {
String realKey = getRealKey(keyEnum, params);
Long count = redisTemplate.opsForZSet().remove(realKey, values);
return count == null ? 0 : count;
} catch (Exception e) {
log.error("redis zset移除异常key:{}", getRealKey(keyEnum, params), e);
return 0;
}
}
/**
* 获取ZSet元素数量
*/
public long zSize(SpdmRedisKeyEnum keyEnum, Object... params) {
Long size = redisTemplate.opsForZSet().size(getRealKey(keyEnum, params));
return size == null ? 0 : size;
}
// ==================== 6. List 类型 ====================
public List<Object> lGet(SpdmRedisKeyEnum keyEnum, long start, long end, Object... params) {
return redisTemplate.opsForList().range(getRealKey(keyEnum, params), start, end);
}
public List<Object> lGetAll(SpdmRedisKeyEnum keyEnum, Object... params) {
return lGet(keyEnum, 0, -1, params);
}
public long lGetListSize(SpdmRedisKeyEnum keyEnum, Object... params) {
Long size = redisTemplate.opsForList().size(getRealKey(keyEnum, params));
return size == null ? 0 : size;
}
public Object lGetIndex(SpdmRedisKeyEnum keyEnum, long index, Object... params) {
return redisTemplate.opsForList().index(getRealKey(keyEnum, params), index);
}
public boolean lSet(SpdmRedisKeyEnum keyEnum, Object value, Object... params) {
try {
redisTemplate.opsForList().rightPush(getRealKey(keyEnum, params), value);
expire(keyEnum, params);
return true;
} catch (Exception e) {
log.error("redis list存入异常key:{}", getRealKey(keyEnum, params), e);
return false;
}
}
public boolean lSet(SpdmRedisKeyEnum keyEnum, List<Object> value, Object... params) {
try {
redisTemplate.opsForList().rightPushAll(getRealKey(keyEnum, params), value);
expire(keyEnum, params);
return true;
} catch (Exception e) {
log.error("redis list批量存入异常key:{}", getRealKey(keyEnum, params), e);
return false;
}
}
public Object lLeftPop(SpdmRedisKeyEnum keyEnum, Object... params) {
return redisTemplate.opsForList().leftPop(getRealKey(keyEnum, params));
}
public boolean lUpdateIndex(SpdmRedisKeyEnum keyEnum, long index, Object value, Object... params) {
try {
redisTemplate.opsForList().set(getRealKey(keyEnum, params), index, value);
return true;
} catch (Exception e) {
log.error("redis list更新异常key:{}", getRealKey(keyEnum, params), e);
return false;
}
}
public long lRemove(SpdmRedisKeyEnum keyEnum, long count, Object value, Object... params) {
Long remove = redisTemplate.opsForList().remove(getRealKey(keyEnum, params), count, value);
return remove == null ? 0 : remove;
}
public void lRemoveAll(SpdmRedisKeyEnum keyEnum, Object... params) {
redisTemplate.opsForList().trim(getRealKey(keyEnum, params), -1, 0);
}
}

View File

@@ -1,45 +1,45 @@
package com.sdm.data.config;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.redis.connection.RedisConnectionFactory;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.data.redis.serializer.RedisSerializer;
import org.springframework.data.redis.serializer.StringRedisSerializer;
@Configuration
public class RedisConfig {
@Bean(name = "redisTemplate")
public RedisTemplate<Object, Object> redisTemplate(RedisConnectionFactory factory) {
RedisTemplate<Object, Object> template = new RedisTemplate<>();
RedisSerializer<String> redisSerializer = new StringRedisSerializer();
template.setConnectionFactory(factory);
// key序列化方式
template.setKeySerializer(redisSerializer);
// value序列化
template.setValueSerializer(redisSerializer);
// value hashmap序列化
template.setHashValueSerializer(redisSerializer);
// key haspmap序列化
template.setHashKeySerializer(redisSerializer);
return template;
}
@Bean(name = "bytesRedisTemplate")
public RedisTemplate<String, byte[]> bytesRedisTemplate(RedisConnectionFactory connectionFactory) {
RedisTemplate<String, byte[]> redisTemplate = new RedisTemplate<>();
redisTemplate.setConnectionFactory(connectionFactory);
// 设置key和value的序列化规则
redisTemplate.setValueSerializer(RedisSerializer.byteArray());
redisTemplate.setKeySerializer(new StringRedisSerializer());
redisTemplate.afterPropertiesSet();
return redisTemplate;
}
}
//package com.sdm.data.config;
//
//import org.springframework.context.annotation.Bean;
//import org.springframework.context.annotation.Configuration;
//import org.springframework.data.redis.connection.RedisConnectionFactory;
//import org.springframework.data.redis.core.RedisTemplate;
//import org.springframework.data.redis.serializer.RedisSerializer;
//import org.springframework.data.redis.serializer.StringRedisSerializer;
//
//@Configuration
//public class RedisConfig {
//
// @Bean(name = "redisTemplate")
// public RedisTemplate<Object, Object> redisTemplate(RedisConnectionFactory factory) {
// RedisTemplate<Object, Object> template = new RedisTemplate<>();
// RedisSerializer<String> redisSerializer = new StringRedisSerializer();
//
// template.setConnectionFactory(factory);
// // key序列化方式
// template.setKeySerializer(redisSerializer);
// // value序列化
// template.setValueSerializer(redisSerializer);
// // value hashmap序列化
// template.setHashValueSerializer(redisSerializer);
// // key haspmap序列化
// template.setHashKeySerializer(redisSerializer);
//
// return template;
// }
//
// @Bean(name = "bytesRedisTemplate")
// public RedisTemplate<String, byte[]> bytesRedisTemplate(RedisConnectionFactory connectionFactory) {
// RedisTemplate<String, byte[]> redisTemplate = new RedisTemplate<>();
// redisTemplate.setConnectionFactory(connectionFactory);
// // 设置key和value的序列化规则
// redisTemplate.setValueSerializer(RedisSerializer.byteArray());
// redisTemplate.setKeySerializer(new StringRedisSerializer());
// redisTemplate.afterPropertiesSet();
//
// return redisTemplate;
// }
//}
//
//
//

View File

@@ -1,45 +0,0 @@
package com.sdm.pbs.config;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.redis.connection.RedisConnectionFactory;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.data.redis.serializer.RedisSerializer;
import org.springframework.data.redis.serializer.StringRedisSerializer;
@Configuration
public class RedisConfig {
@Bean(name = "redisTemplate")
public RedisTemplate<Object, Object> redisTemplate(RedisConnectionFactory factory) {
RedisTemplate<Object, Object> template = new RedisTemplate<>();
RedisSerializer<String> redisSerializer = new StringRedisSerializer();
template.setConnectionFactory(factory);
// key序列化方式
template.setKeySerializer(redisSerializer);
// value序列化
template.setValueSerializer(redisSerializer);
// value hashmap序列化
template.setHashValueSerializer(redisSerializer);
// key haspmap序列化
template.setHashKeySerializer(redisSerializer);
return template;
}
@Bean(name = "bytesRedisTemplate")
public RedisTemplate<String, byte[]> bytesRedisTemplate(RedisConnectionFactory connectionFactory) {
RedisTemplate<String, byte[]> redisTemplate = new RedisTemplate<>();
redisTemplate.setConnectionFactory(connectionFactory);
// 设置key和value的序列化规则
redisTemplate.setValueSerializer(RedisSerializer.byteArray());
redisTemplate.setKeySerializer(new StringRedisSerializer());
redisTemplate.afterPropertiesSet();
return redisTemplate;
}
}

View File

@@ -16,7 +16,9 @@ import com.sdm.pbs.model.bo.HpcJobStatusInfo;
import com.sdm.pbs.model.bo.HpcResouceInfo;
import com.sdm.pbs.model.entity.SimulationHpcCommand;
import com.sdm.pbs.model.entity.SimulationJob;
import com.sdm.pbs.model.req.*;
import com.sdm.pbs.model.req.JobFileCallBackReq;
import com.sdm.pbs.model.req.QueryJobReq;
import com.sdm.pbs.model.req.SubmitHpcTaskReq;
import com.sdm.pbs.service.HpcInstructionService;
import com.sdm.pbs.service.IPbsService;
import com.sdm.pbs.service.impl.PbsServiceDecorator;
@@ -101,7 +103,7 @@ public class TaskController {
@PostMapping("/hpcDownloadFile")
@Operation(summary = "作业下文件下载")
ResponseEntity<StreamingResponseBody> hpcDownloadFile(@RequestBody HpcTaskFileDownReq req) {
return pbsService.downloadFile(req.getJobId(),req.getFileName(),req.getFileSize());
return pbsService.downloadFile(req.getJobId(),req.getFileName());
}
@PostMapping("/queryJobs")
@@ -280,4 +282,13 @@ public class TaskController {
Long.valueOf(paramMap.get("userId").toString()),Long.valueOf(paramMap.get("tenantId").toString()),paramMap.get("outputFormat").toString());
}
@GetMapping("/mockHpcDownloadFile")
@Operation(summary = "模拟作业下文件下载")
ResponseEntity<StreamingResponseBody> mockHpcDownloadFile() {
HpcTaskFileDownReq req = new HpcTaskFileDownReq();
req.setFileName("\\\\HPC-COMPUTE-01\\share\\testbigfile\\test.zip");
// req.setFileName("D:\\0000-测试\\test.txt");
return pbsService.downloadFile(req.getJobId(),req.getFileName());
}
}

View File

@@ -0,0 +1,69 @@
package com.sdm.pbs.controller;
import com.sdm.common.common.SdmResponse;
import com.sdm.common.common.SpdmRedisKeyEnum;
import com.sdm.common.utils.SpdmRedisUtil;
import com.sdm.pbs.model.entity.SimulationJob;
import io.swagger.v3.oas.annotations.tags.Tag;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.tuple.Pair;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import java.util.HashMap;
import java.util.Map;
@Slf4j
@RestController
@RequestMapping
@Tag(name = "测试redis", description = "测试redis")
public class TestRedisController {
@Autowired
private SpdmRedisUtil spdmRedisUtil;
/**
* 测试设置缓存POST请求Map接收参数
* 请求体示例:{"key":"testKey", "value":"testValue", "timeout":1, "timeUnit":"HOURS"}
*/
@PostMapping("/redis/set")
public SdmResponse setRedis(@RequestBody Map<String, Object> paramMap) {
// 1. 从Map中获取参数做非空校验
String id1 = (String) paramMap.get("id1");
String id2 = (String) paramMap.get("id2");
SimulationJob job = new SimulationJob();
job.setDelFlag("1");
job.setInputFormat("11111");
job.setOutputFormat("22222");
job.setJobId("4444");
// 3. 设置缓存
Pair<Boolean, String> pair1 = spdmRedisUtil.set(SpdmRedisKeyEnum.YANG_TEST1, job, id1);
Pair<Boolean, String> pair2 = spdmRedisUtil.set(SpdmRedisKeyEnum.YANG_TEST2, "value222", id2);
Map<String, Object> map = new HashMap<>();
map.put(pair1.getRight(), pair1.getLeft());
map.put(pair2.getRight(), pair2.getLeft());
return SdmResponse.success(map);
}
/**
* 测试获取缓存POST请求Map接收参数
* 请求体示例:{"key":"testKey"}
*/
@PostMapping("/redis/get")
public SdmResponse getRedis(@RequestBody Map<String, Object> paramMap) {
String id1 = (String) paramMap.get("id1");
String id2 = (String) paramMap.get("id2");
// 获取缓存
SimulationJob job = spdmRedisUtil.getBean(SpdmRedisKeyEnum.YANG_TEST1, SimulationJob.class, id1);
String value2 = (String) spdmRedisUtil.get(SpdmRedisKeyEnum.YANG_TEST2,id2);
Map<String, Object> map = new HashMap<>();
map.put("value1", job);
map.put("value2", value2);
return SdmResponse.success(map);
}
}

View File

@@ -53,7 +53,7 @@ public interface HpcInstructionService {
SdmResponse<List<FileNodeInfo>> scanDir(String targetDir);
ResponseEntity<StreamingResponseBody> hpcDownloadFile(String fileName,Long fileSize);
ResponseEntity<StreamingResponseBody> hpcDownloadFile(String fileName);
// 通知hpc回传文件
SdmResponse<Boolean> callHpcUploadToTarget(String jobId,String workDir,String minioBucket,

View File

@@ -54,7 +54,7 @@ public interface IPbsService {
* @param fileName
* @return
*/
ResponseEntity<StreamingResponseBody> downloadFile(String jobId, String fileName,Long fileSize);
ResponseEntity<StreamingResponseBody> downloadFile(String jobId, String fileName);
void streamTaskLog(String filePath,String clientToken, OutputStream outputStream);

View File

@@ -57,10 +57,6 @@ public class HpcInstructionServiceImpl implements HpcInstructionService {
@Autowired
private HpcCommandExcuteUtil hpcCommandExcuteUtil;
@Autowired
@Qualifier("pbsWebClient")
private WebClient pbsWebClient;
@Value("#{'${hpc.newJob.stringFields:jobname,license,projectname}'.split(',')}")
private List<String> newJobStringFields;
@Value("#{'${hpc.addJob.stringFields:name,license,projectname}'.split(',')}")
@@ -494,8 +490,8 @@ public class HpcInstructionServiceImpl implements HpcInstructionService {
}
@Override
public ResponseEntity<StreamingResponseBody> hpcDownloadFile(String fileName,Long fileSize) {
return hpcCommandExcuteUtil.hpcDownloadFile(fileName,fileSize,pbsWebClient);
public ResponseEntity<StreamingResponseBody> hpcDownloadFile(String fileName) {
return hpcCommandExcuteUtil.hpcDownloadFile(fileName);
}
@Override

View File

@@ -131,8 +131,8 @@ public class IPbsHpcServiceImpl implements IPbsService {
}
@Override
public ResponseEntity<StreamingResponseBody> downloadFile(String jobId, String fileName,Long fileSize) {
return hpcInstructionService.hpcDownloadFile(fileName,fileSize);
public ResponseEntity<StreamingResponseBody> downloadFile(String jobId, String fileName) {
return hpcInstructionService.hpcDownloadFile(fileName);
}
@Override

View File

@@ -590,12 +590,12 @@ public class PbsServiceDecorator implements IPbsServiceDecorator {
}
@Override
public ResponseEntity<StreamingResponseBody> downloadFile(String jobId, String fileName,Long fileSize) {
public ResponseEntity<StreamingResponseBody> downloadFile(String jobId, String fileName) {
// todo 预留根据jobId 获取工作目录,做权限控制,越权
if(StringUtils.isBlank(fileName)){
throw new RuntimeException("下载文件路径为空");
}
return pbsService.downloadFile("", fileName,fileSize);
return pbsService.downloadFile("", fileName);
}
@Override

View File

@@ -7,9 +7,9 @@ import com.paratera.lpcs.core.invoker.ApiClient;
import com.paratera.lpcs.core.invoker.ApiException;
import com.paratera.lpcs.core.invoker.ApiResponse;
import com.paratera.lpcs.core.model.*;
import com.sdm.common.entity.constants.RedisConstant;
import com.sdm.common.common.SdmResponse;
import com.sdm.common.common.ThreadLocalContext;
import com.sdm.common.entity.constants.RedisConstant;
import com.sdm.common.entity.pojo.pbs.*;
import com.sdm.common.entity.pojo.project.ProjectNodeBase;
import com.sdm.common.entity.pojo.task.TaskBase;

View File

@@ -1,45 +1,45 @@
package com.sdm.project.config;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.redis.connection.RedisConnectionFactory;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.data.redis.serializer.RedisSerializer;
import org.springframework.data.redis.serializer.StringRedisSerializer;
@Configuration
public class RedisConfig {
@Bean(name = "redisTemplate")
public RedisTemplate<Object, Object> redisTemplate(RedisConnectionFactory factory) {
RedisTemplate<Object, Object> template = new RedisTemplate<>();
RedisSerializer<String> redisSerializer = new StringRedisSerializer();
template.setConnectionFactory(factory);
// key序列化方式
template.setKeySerializer(redisSerializer);
// value序列化
template.setValueSerializer(redisSerializer);
// value hashmap序列化
template.setHashValueSerializer(redisSerializer);
// key haspmap序列化
template.setHashKeySerializer(redisSerializer);
return template;
}
@Bean(name = "bytesRedisTemplate")
public RedisTemplate<String, byte[]> bytesRedisTemplate(RedisConnectionFactory connectionFactory) {
RedisTemplate<String, byte[]> redisTemplate = new RedisTemplate<>();
redisTemplate.setConnectionFactory(connectionFactory);
// 设置key和value的序列化规则
redisTemplate.setValueSerializer(RedisSerializer.byteArray());
redisTemplate.setKeySerializer(new StringRedisSerializer());
redisTemplate.afterPropertiesSet();
return redisTemplate;
}
}
//package com.sdm.project.config;
//
//import org.springframework.context.annotation.Bean;
//import org.springframework.context.annotation.Configuration;
//import org.springframework.data.redis.connection.RedisConnectionFactory;
//import org.springframework.data.redis.core.RedisTemplate;
//import org.springframework.data.redis.serializer.RedisSerializer;
//import org.springframework.data.redis.serializer.StringRedisSerializer;
//
//@Configuration
//public class RedisConfig {
//
// @Bean(name = "redisTemplate")
// public RedisTemplate<Object, Object> redisTemplate(RedisConnectionFactory factory) {
// RedisTemplate<Object, Object> template = new RedisTemplate<>();
// RedisSerializer<String> redisSerializer = new StringRedisSerializer();
//
// template.setConnectionFactory(factory);
// // key序列化方式
// template.setKeySerializer(redisSerializer);
// // value序列化
// template.setValueSerializer(redisSerializer);
// // value hashmap序列化
// template.setHashValueSerializer(redisSerializer);
// // key haspmap序列化
// template.setHashKeySerializer(redisSerializer);
//
// return template;
// }
//
// @Bean(name = "bytesRedisTemplate")
// public RedisTemplate<String, byte[]> bytesRedisTemplate(RedisConnectionFactory connectionFactory) {
// RedisTemplate<String, byte[]> redisTemplate = new RedisTemplate<>();
// redisTemplate.setConnectionFactory(connectionFactory);
// // 设置key和value的序列化规则
// redisTemplate.setValueSerializer(RedisSerializer.byteArray());
// redisTemplate.setKeySerializer(new StringRedisSerializer());
// redisTemplate.afterPropertiesSet();
//
// return redisTemplate;
// }
//}
//
//
//

View File

@@ -1,45 +1,45 @@
package com.sdm.system.config;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.redis.connection.RedisConnectionFactory;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.data.redis.serializer.RedisSerializer;
import org.springframework.data.redis.serializer.StringRedisSerializer;
@Configuration
public class RedisConfig {
@Bean(name = "redisTemplate")
public RedisTemplate<Object, Object> redisTemplate(RedisConnectionFactory factory) {
RedisTemplate<Object, Object> template = new RedisTemplate<>();
RedisSerializer<String> redisSerializer = new StringRedisSerializer();
template.setConnectionFactory(factory);
// key序列化方式
template.setKeySerializer(redisSerializer);
// value序列化
template.setValueSerializer(redisSerializer);
// value hashmap序列化
template.setHashValueSerializer(redisSerializer);
// key haspmap序列化
template.setHashKeySerializer(redisSerializer);
return template;
}
@Bean(name = "bytesRedisTemplate")
public RedisTemplate<String, byte[]> bytesRedisTemplate(RedisConnectionFactory connectionFactory) {
RedisTemplate<String, byte[]> redisTemplate = new RedisTemplate<>();
redisTemplate.setConnectionFactory(connectionFactory);
// 设置key和value的序列化规则
redisTemplate.setValueSerializer(RedisSerializer.byteArray());
redisTemplate.setKeySerializer(new StringRedisSerializer());
redisTemplate.afterPropertiesSet();
return redisTemplate;
}
}
//package com.sdm.system.config;
//
//import org.springframework.context.annotation.Bean;
//import org.springframework.context.annotation.Configuration;
//import org.springframework.data.redis.connection.RedisConnectionFactory;
//import org.springframework.data.redis.core.RedisTemplate;
//import org.springframework.data.redis.serializer.RedisSerializer;
//import org.springframework.data.redis.serializer.StringRedisSerializer;
//
//@Configuration
//public class RedisConfig {
//
// @Bean(name = "redisTemplate")
// public RedisTemplate<Object, Object> redisTemplate(RedisConnectionFactory factory) {
// RedisTemplate<Object, Object> template = new RedisTemplate<>();
// RedisSerializer<String> redisSerializer = new StringRedisSerializer();
//
// template.setConnectionFactory(factory);
// // key序列化方式
// template.setKeySerializer(redisSerializer);
// // value序列化
// template.setValueSerializer(redisSerializer);
// // value hashmap序列化
// template.setHashValueSerializer(redisSerializer);
// // key haspmap序列化
// template.setHashKeySerializer(redisSerializer);
//
// return template;
// }
//
// @Bean(name = "bytesRedisTemplate")
// public RedisTemplate<String, byte[]> bytesRedisTemplate(RedisConnectionFactory connectionFactory) {
// RedisTemplate<String, byte[]> redisTemplate = new RedisTemplate<>();
// redisTemplate.setConnectionFactory(connectionFactory);
// // 设置key和value的序列化规则
// redisTemplate.setValueSerializer(RedisSerializer.byteArray());
// redisTemplate.setKeySerializer(new StringRedisSerializer());
// redisTemplate.afterPropertiesSet();
//
// return redisTemplate;
// }
//}
//
//
//

View File

@@ -1,45 +1,45 @@
package com.sdm.task.config;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.redis.connection.RedisConnectionFactory;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.data.redis.serializer.RedisSerializer;
import org.springframework.data.redis.serializer.StringRedisSerializer;
@Configuration
public class RedisConfig {
@Bean(name = "redisTemplate")
public RedisTemplate<Object, Object> redisTemplate(RedisConnectionFactory factory) {
RedisTemplate<Object, Object> template = new RedisTemplate<>();
RedisSerializer<String> redisSerializer = new StringRedisSerializer();
template.setConnectionFactory(factory);
// key序列化方式
template.setKeySerializer(redisSerializer);
// value序列化
template.setValueSerializer(redisSerializer);
// value hashmap序列化
template.setHashValueSerializer(redisSerializer);
// key haspmap序列化
template.setHashKeySerializer(redisSerializer);
return template;
}
@Bean(name = "bytesRedisTemplate")
public RedisTemplate<String, byte[]> bytesRedisTemplate(RedisConnectionFactory connectionFactory) {
RedisTemplate<String, byte[]> redisTemplate = new RedisTemplate<>();
redisTemplate.setConnectionFactory(connectionFactory);
// 设置key和value的序列化规则
redisTemplate.setValueSerializer(RedisSerializer.byteArray());
redisTemplate.setKeySerializer(new StringRedisSerializer());
redisTemplate.afterPropertiesSet();
return redisTemplate;
}
}
//package com.sdm.task.config;
//
//import org.springframework.context.annotation.Bean;
//import org.springframework.context.annotation.Configuration;
//import org.springframework.data.redis.connection.RedisConnectionFactory;
//import org.springframework.data.redis.core.RedisTemplate;
//import org.springframework.data.redis.serializer.RedisSerializer;
//import org.springframework.data.redis.serializer.StringRedisSerializer;
//
//@Configuration
//public class RedisConfig {
//
// @Bean(name = "redisTemplate")
// public RedisTemplate<Object, Object> redisTemplate(RedisConnectionFactory factory) {
// RedisTemplate<Object, Object> template = new RedisTemplate<>();
// RedisSerializer<String> redisSerializer = new StringRedisSerializer();
//
// template.setConnectionFactory(factory);
// // key序列化方式
// template.setKeySerializer(redisSerializer);
// // value序列化
// template.setValueSerializer(redisSerializer);
// // value hashmap序列化
// template.setHashValueSerializer(redisSerializer);
// // key haspmap序列化
// template.setHashKeySerializer(redisSerializer);
//
// return template;
// }
//
// @Bean(name = "bytesRedisTemplate")
// public RedisTemplate<String, byte[]> bytesRedisTemplate(RedisConnectionFactory connectionFactory) {
// RedisTemplate<String, byte[]> redisTemplate = new RedisTemplate<>();
// redisTemplate.setConnectionFactory(connectionFactory);
// // 设置key和value的序列化规则
// redisTemplate.setValueSerializer(RedisSerializer.byteArray());
// redisTemplate.setKeySerializer(new StringRedisSerializer());
// redisTemplate.afterPropertiesSet();
//
// return redisTemplate;
// }
//}
//
//
//