timi/TimiServerAPI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 57 Wiki Activity

refactor TempFileService, use Attachment and mongodb

Browse Source
This commit is contained in:
Timi
2026-01-05 14:35:38 +08:00
parent b6eb7980e4
commit fa11b388db
13 changed files with 222 additions and 215 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/java/com/imyeyu/api/modules/common/service/AttachmentService.java
+3 -1
View File
@@ -56,7 +56,9 @@ public interface AttachmentService extends BaseService<Attachment, Long> {
*/
long countByBizId(Attachment.BizType bizType, Long bizId, Enum<?> ...attachTypes);
List<Attachment> listByMd5s(Attachment.BizType bizType, Long bizId, List<Enum<?>> attachTypeList, List<String> md5s) throws TimiException;
List<Attachment> listByMd5s(Attachment.BizType bizType, Long bizId, List<Enum<?>> attachTypeList, List<String> md5s);
void deleteByBizId(Attachment.BizType bizType, long bizId, Enum<?> ...attachTypes);
List<Attachment> listNeedDestroy();
}
src/main/java/com/imyeyu/api/modules/common/service/TempFileService.java
+1 -11
View File
@@ -1,12 +1,8 @@
package com.imyeyu.api.modules.common.service;
import com.imyeyu.api.modules.common.bean.TempFileMetaData;
import com.imyeyu.api.modules.common.vo.TempFileResponse;
import com.imyeyu.java.bean.timi.TimiException;
import org.springframework.web.multipart.MultipartFile;
import java.io.File;
import java.io.InputStream;
import java.util.List;
/**
@@ -15,11 +11,5 @@ import java.util.List;
*/
public interface TempFileService {
List<TempFileResponse> store(List<MultipartFile> files, Long ttl) throws TimiException;
File get(String id) throws TimiException;
InputStream getInputStream(String id) throws TimiException;
TempFileMetaData metadata(String id) throws TimiException;
List<TempFileResponse> store(List<MultipartFile> files, Long ttl);
}
src/main/java/com/imyeyu/api/modules/common/service/implement/AttachmentServiceImplement.java
+25 -18
View File
@@ -15,6 +15,7 @@ import com.imyeyu.java.TimiJava;
import com.imyeyu.java.bean.timi.TimiCode;
import com.imyeyu.java.bean.timi.TimiException;
import com.imyeyu.network.Network;
import com.imyeyu.spring.TimiSpring;
import com.imyeyu.spring.mapper.BaseMapper;
import com.imyeyu.spring.service.AbstractEntityService;
import com.imyeyu.utils.Time;
@@ -67,13 +68,11 @@ public class AttachmentServiceImplement extends AbstractEntityService<Attachment
@Override
public void create(Attachment attachment) {
TimiException.required(attachment.getBizType(), "not found attachment.bizType");
TimiException.required(attachment.getBizId(), "not found attachment.bizId");
TimiException.required(attachment.getName(), "not found attachment.name");
String mongoId = null;
try {
InputStream is = attachment.getInputStream();
TimiException.required(is, "not found attachment.inputStream");
TimiException.requiredTrue(is.available() != 0, "empty attachment.inputStream");
StringBuilder mongoName = new StringBuilder(attachment.getBizType().toString());
if (TimiJava.isNotEmpty(attachment.getAttachType())) {
@@ -87,6 +86,8 @@ public class AttachmentServiceImplement extends AbstractEntityService<Attachment
attachment.setSize(gridFSFile.getLength());
attachment.setMd5(IO.md5(gridFSBucket.openDownloadStream(gridFSFile.getObjectId())));
attachment.setMimeType(new Tika().detect(gridFSBucket.openDownloadStream(gridFSFile.getObjectId())));
attachment.setUploaderIp(TimiSpring.getRequestIP());
attachment.setIsDestroyed(false);
if (attachment.getMimeType().startsWith("image")) {
BufferedImage image = ImageIO.read(gridFSBucket.openDownloadStream(gridFSFile.getObjectId()));
attachment.setMetadata(TimiJava.defaultIfNull(attachment.getMetadata(), new JsonObject()));
@@ -108,14 +109,17 @@ public class AttachmentServiceImplement extends AbstractEntityService<Attachment
@Override
public void destroy(Long id) {
try {
Attachment attachment = get(id);
gridFsTemplate.delete(Query.query(Criteria.where("_id").is(attachment.getMongoId())));
if (!attachment.isDeleted()) {
attachment.setDeletedAt(Time.now());
Attachment attach = mapper.selectRaw(id);
GridFSFile file = readByMongoId(attach.getMongoId());
if (file != null) {
gridFsTemplate.delete(Query.query(Criteria.where("_id").is(attach.getMongoId())));
}
attachment.setDestroyAt(Time.now());
mapper.update(attachment);
if (!attach.isDeleted()) {
attach.setDeletedAt(Time.now());
}
attach.setIsDestroyed(true);
attach.setDestroyAt(Time.now());
mapper.update(attach);
} catch (Exception e) {
log.error("delete mongo file error", e);
throw new TimiException(TimiCode.ERROR).msgKey("TODO delete mongo file error");
@@ -131,15 +135,13 @@ public class AttachmentServiceImplement extends AbstractEntityService<Attachment
create(attachment);
// 生成缩略图
InputStream mimeStream = getInputStreamByMongoId(attachment.getMongoId());
InputStream mimeStream = attachment.openInputStream();
String mimeType = new Tika().detect(mimeStream);
InputStream sourceStream = getInputStreamByMongoId(attachment.getMongoId());
InputStream sourceStream = attachment.openInputStream();
ByteArrayOutputStream thumbStream = new ByteArrayOutputStream();
switch (mimeType) {
case "image/png", "image/bmp", "image/jpeg" -> {
Thumbnails.of(sourceStream).width(256).keepAspectRatio(true).toOutputStream(thumbStream);
}
case "image/png", "image/bmp", "image/jpeg" -> Thumbnails.of(sourceStream).width(256).keepAspectRatio(true).toOutputStream(thumbStream);
case "video/mp4", "video/quicktime" -> {
log.info("capturing thumbnail: {}", attachment.getName());
long start = Time.now();
@@ -201,9 +203,9 @@ public class AttachmentServiceImplement extends AbstractEntityService<Attachment
@Override
public GridFSFile readByMongoId(String mongoId) {
Attachment view = mapper.selectByMongoId(mongoId);
TimiException.required(view, "not found attachment: %s".formatted(mongoId));
return gridFsTemplate.findOne(new Query(Criteria.where("_id").is(view.getMongoId())));
Attachment attach = mapper.selectByMongoId(mongoId);
TimiException.required(attach, "not found attachment: %s".formatted(mongoId));
return gridFsTemplate.findOne(new Query(Criteria.where("_id").is(attach.getMongoId())));
}
@Override
@@ -214,7 +216,7 @@ public class AttachmentServiceImplement extends AbstractEntityService<Attachment
@Override
public byte[] readAllByMongoId(String mongoId) {
try {
return IO.toBytes(getInputStreamByMongoId(mongoId));
return IO.toBytes(getByMongoId(mongoId).openInputStream());
} catch (IOException e) {
throw new TimiException(TimiCode.ERROR, "TODO 读取失败");
}
@@ -249,4 +251,9 @@ public class AttachmentServiceImplement extends AbstractEntityService<Attachment
}
}
}
@Override
public List<Attachment> listNeedDestroy() {
return mapper.selectNeedDestroy();
}
}
src/main/java/com/imyeyu/api/modules/common/service/implement/TempFileServiceImplement.java
+54 -90
View File
@@ -1,38 +1,27 @@
package com.imyeyu.api.modules.common.service.implement;
import com.google.gson.Gson;
import com.imyeyu.api.modules.common.bean.SettingKey;
import com.imyeyu.api.modules.common.bean.TempFileMetaData;
import com.imyeyu.api.modules.common.entity.Attachment;
import com.imyeyu.api.modules.common.service.AttachmentService;
import com.imyeyu.api.modules.common.service.SettingService;
import com.imyeyu.api.modules.common.service.TempFileService;
import com.imyeyu.api.modules.common.vo.TempFileResponse;
import com.imyeyu.io.IO;
import com.imyeyu.io.IOSize;
import com.imyeyu.java.TimiJava;
import com.imyeyu.java.bean.timi.TimiCode;
import com.imyeyu.java.bean.timi.TimiException;
import com.imyeyu.network.Network;
import com.imyeyu.spring.TimiSpring;
import com.imyeyu.spring.bean.Page;
import com.imyeyu.utils.Time;
import jakarta.annotation.PostConstruct;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import org.springframework.web.multipart.MultipartFile;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
/**
* @author 夜雨
@@ -43,61 +32,58 @@ import java.util.concurrent.ConcurrentHashMap;
@RequiredArgsConstructor
public class TempFileServiceImplement implements TempFileService {
private static final Long DEFAULT_TTL = Time.H * 6;
private static final Long MAX_TTL = Time.D * 3;
private static final Long LIMIT = IOSize.GB * 10;
private final Gson gson;
private final SettingService settingService;
private final AttachmentService attachmentService;
private Path storagePath;
private final Map<String, TempFileMetaData> metadataMap = new ConcurrentHashMap<>();
private final Map<String, Long> usageMap = new ConcurrentHashMap<>();
@PostConstruct
public void init() throws IOException {
storagePath = Paths.get(settingService.getAsString(SettingKey.TEMP_FILE_PATH));
IO.destroy(storagePath.toFile());
Files.createDirectories(storagePath);
}
public List<TempFileResponse> store(List<MultipartFile> files, Long ttl) throws TimiException {
ttl = TimiJava.defaultIfNull(ttl, DEFAULT_TTL);
TimiException.requiredTrue(0 < ttl && ttl <= MAX_TTL, "ttl must be between 1ms and 3 days(259200000ms)");
String ttlMinStr = settingService.getAsString(SettingKey.TEMP_FILE_TTL_MIN);
String ttlMaxStr = settingService.getAsString(SettingKey.TEMP_FILE_TTL_MAX);
String limitStr = settingService.getAsString(SettingKey.TEMP_FILE_LIMIT);
long minTTL = Time.parseToMS(ttlMinStr);
long maxTTL = Time.parseToMS(ttlMaxStr);
long defaultTTL = Time.parseToMS(settingService.getAsString(SettingKey.TEMP_FILE_TTL_DEFAULT));
long limit = IOSize.parse(limitStr);
long residueTime = Time.parseToMS(settingService.getAsString(SettingKey.TEMP_FILE_RESIDUE_TIME));
ttl = TimiJava.defaultIfNull(ttl, defaultTTL);
TimiException.requiredTrue(minTTL < ttl && ttl <= maxTTL, String.format("ttl must be between %s and %s", ttlMinStr, ttlMaxStr));
long newFileSize = files.stream().mapToLong(MultipartFile::getSize).sum();
long currentUsage = usageMap.getOrDefault(TimiSpring.getRequestIP(), 0L);
TimiException.requiredTrue(currentUsage + newFileSize < LIMIT, "out of storage limit(10 GB)");
{
// 缓存池大小限制
Page<Attachment> page = new Page<>();
page.setIndex(0);
page.setSize(Long.MAX_VALUE);
{
Attachment example = new Attachment();
example.setUploaderIp(TimiSpring.getRequestIP());
page.setEqualsExample(example);
}
long currentUsage = attachmentService.page(page).getList().stream().mapToLong(Attachment::getSize).sum();
TimiException.requiredTrue(currentUsage + newFileSize < limit, "out of storage limit(%s)".formatted(limitStr));
}
try {
List<TempFileResponse> result = new ArrayList<>();
long deletedAt = Time.now() + ttl;
long destroyAt = deletedAt + residueTime;
for (int i = 0; i < files.size(); i++) {
MultipartFile file = files.get(i);
String fileId = UUID.randomUUID().toString();
String fileName = fileId;
if (TimiJava.isNotEmpty(file.getOriginalFilename())) {
fileName = fileId + "." + Network.uriFileExtension(file.getOriginalFilename());
}
Path filePath = storagePath.resolve(fileName);
Files.copy(file.getInputStream(), filePath);
// 创建元数据
TempFileMetaData metadata = new TempFileMetaData();
metadata.setId(fileId);
metadata.setPath(filePath);
metadata.setName(fileName);
metadata.setOriginalName(file.getOriginalFilename());
metadata.setLastAccessAt(Time.now());
metadata.setTtl(ttl);
metadataMap.put(metadata.getId(), metadata);
Attachment attach = new Attachment();
attach.setBizType(Attachment.BizType.TEMP_FILE);
attach.setName(file.getOriginalFilename());
attach.setDeletedAt(deletedAt);
attach.setDestroyAt(destroyAt);
attach.setInputStream(file.getInputStream());
attachmentService.create(attach);
TempFileResponse resp = new TempFileResponse();
resp.setId(metadata.getId());
resp.setExpireAt(metadata.getLastAccessAt() + ttl);
usageMap.put(TimiSpring.getRequestIP(), currentUsage + newFileSize);
resp.setId(attach.getMongoId());
resp.setExpireAt(deletedAt);
result.add(resp);
}
return result;
@@ -107,42 +93,20 @@ public class TempFileServiceImplement implements TempFileService {
}
}
@Override
public File get(String id) throws TimiException {
TempFileMetaData metaData = metadataMap.get(id);
TimiException.required(metaData, "not found temp file");
metaData.setLastAccessAt(Time.now());
return metaData.getPath().toFile();
}
@Override
public InputStream getInputStream(String id) throws TimiException {
try {
return IO.getInputStream(get(id));
} catch (FileNotFoundException e) {
throw new TimiException(TimiCode.RESULT_NULL, "not found temp file");
}
}
@Override
public TempFileMetaData metadata(String id) throws TimiException {
return metadataMap.get(id);
}
@Scheduled(fixedRate = 3600000)
public void cleanup() {
List<String> expiredIds = metadataMap.values()
.stream()
.filter(metadata -> metadata.getLastAccessAt() + metadata.getTtl() < Time.now())
.map(TempFileMetaData::getId)
.toList();
for (int i = 0; i < expiredIds.size(); i++) {
TempFileMetaData removed = metadataMap.remove(expiredIds.get(i));
if (TimiJava.isNotEmpty(removed)) {
File file = removed.getPath().toFile();
IO.destroy(file);
usageMap.computeIfPresent(TimiSpring.getRequestIP(), (ip, usage) -> usage - file.length());
}
}
// List<String> expiredIds = metadataMap.values()
// .stream()
// .filter(metadata -> metadata.getLastAccessAt() + metadata.getTtl() < Time.now())
// .map(TempFileMetaData::getId)
// .toList();
// for (int i = 0; i < expiredIds.size(); i++) {
// TempFileMetaData removed = metadataMap.remove(expiredIds.get(i));
// if (TimiJava.isNotEmpty(removed)) {
// File file = removed.getPath().toFile();
// IO.destroy(file);
// usageMap.computeIfPresent(TimiSpring.getRequestIP(), (ip, usage) -> usage - file.length());
// }
// }
}
}