wzp/FileManager
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: add chunk function

Browse Source
This commit is contained in:
Wzp-2008 2025-04-20 23:24:42 +08:00
parent a0c75059ba
commit 105e1443d9
12 changed files with 380 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
src/main/java/cn/wzpmc/filemanager/controller/FileController.java
View File

@ -4,6 +4,8 @@ import cn.wzpmc.filemanager.annotation.Address;
import cn.wzpmc.filemanager.annotation.AuthorizationRequired;
import cn.wzpmc.filemanager.entities.PageResult;
import cn.wzpmc.filemanager.entities.Result;
import cn.wzpmc.filemanager.entities.chunk.CheckChunkResult;
import cn.wzpmc.filemanager.entities.chunk.SaveChunksRequest;
import cn.wzpmc.filemanager.entities.files.FolderCreateRequest;
import cn.wzpmc.filemanager.entities.files.FullRawFileObject;
import cn.wzpmc.filemanager.entities.files.enums.FileType;
@ -16,9 +18,11 @@ import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
import lombok.RequiredArgsConstructor;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;
import org.springframework.web.multipart.MultipartHttpServletRequest;
import java.util.Date;
import java.util.List;
/**
* 文件操作相关接口
@ -129,6 +133,7 @@ public class FileController {
* @deprecated
*/
@GetMapping("/share")
@Deprecated
public Result<String> shareFile(@RequestParam Long id, @RequestParam(defaultValue = "9999-12-31") Date lastCouldDownloadTime, @RequestParam(defaultValue = "-1") int maxDownloadCount) {
return fileService.shareFile(id, lastCouldDownloadTime, maxDownloadCount);
}
@ -153,4 +158,19 @@ public class FileController {
public Result<String> findFilePathById(@PathVariable("id") long id, @RequestParam(value = "type", defaultValue = "FILE") FileType type) {
return type.equals(FileType.FILE) ? fileService.findFilePathById(id) : fileService.findFolderPathById(id);
}
@PostMapping("/chunk/check")
public Result<List<CheckChunkResult>> checkChunkUploaded(@RequestBody List<String> hash) {
return fileService.checkChunkUploaded(hash);
}
@PostMapping("/chunk/upload")
public Result<Long> uploadChunk(@RequestBody MultipartFile block) {
return fileService.uploadChunk(block);
}
@PutMapping("/chunk/save")
public Result<FileVo> saveFile(@RequestBody SaveChunksRequest request) {
return fileService.saveFile(request);
}
}

11
src/main/java/cn/wzpmc/filemanager/entities/chunk/CheckChunkResult.java Normal file
View File

@ -0,0 +1,11 @@
package cn.wzpmc.filemanager.entities.chunk;
import lombok.AllArgsConstructor;
import lombok.Data;
@Data
@AllArgsConstructor
public class CheckChunkResult {
private String hash;
private Long chunkId;
}

12
src/main/java/cn/wzpmc/filemanager/entities/chunk/SaveChunksRequest.java Normal file
View File

@ -0,0 +1,12 @@
package cn.wzpmc.filemanager.entities.chunk;
import lombok.Data;
import java.util.List;
@Data
public class SaveChunksRequest {
private String filename;
private List<Long> chunks;
private Long folderId;
}

17
src/main/java/cn/wzpmc/filemanager/entities/vo/ChunkFileVo.java Normal file
View File

@ -0,0 +1,17 @@
package cn.wzpmc.filemanager.entities.vo;
import com.mybatisflex.annotation.Column;
import com.mybatisflex.annotation.Table;
import lombok.AllArgsConstructor;
import lombok.Data;
@Table("chunk_file")
@Data
@AllArgsConstructor
public class ChunkFileVo {
@Column("chunk_id")
private long chunkId;
@Column("file_id")
private long fileId;
private long index;
}

15
src/main/java/cn/wzpmc/filemanager/entities/vo/ChunksVo.java Normal file
View File

@ -0,0 +1,15 @@
package cn.wzpmc.filemanager.entities.vo;
import com.mybatisflex.annotation.Id;
import com.mybatisflex.annotation.KeyType;
import com.mybatisflex.annotation.Table;
import lombok.Data;
@Table("chunks")
@Data
public class ChunksVo {
@Id(keyType = KeyType.Auto)
private long id;
private String hash;
private long size;
}

9
src/main/java/cn/wzpmc/filemanager/mapper/ChunkFileMapper.java Normal file
View File

@ -0,0 +1,9 @@
package cn.wzpmc.filemanager.mapper;
import cn.wzpmc.filemanager.entities.vo.ChunkFileVo;
import com.mybatisflex.core.BaseMapper;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface ChunkFileMapper extends BaseMapper<ChunkFileVo> {
}

9
src/main/java/cn/wzpmc/filemanager/mapper/ChunksMapper.java Normal file
View File

@ -0,0 +1,9 @@
package cn.wzpmc.filemanager.mapper;
import cn.wzpmc.filemanager.entities.vo.ChunksVo;
import com.mybatisflex.core.BaseMapper;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface ChunksMapper extends BaseMapper<ChunksVo> {
}

183
src/main/java/cn/wzpmc/filemanager/service/FileService.java
View File

@ -3,6 +3,8 @@ package cn.wzpmc.filemanager.service;
import cn.wzpmc.filemanager.config.FileManagerProperties;
import cn.wzpmc.filemanager.entities.PageResult;
import cn.wzpmc.filemanager.entities.Result;
import cn.wzpmc.filemanager.entities.chunk.CheckChunkResult;
import cn.wzpmc.filemanager.entities.chunk.SaveChunksRequest;
import cn.wzpmc.filemanager.entities.files.FolderCreateRequest;
import cn.wzpmc.filemanager.entities.files.FullRawFileObject;
import cn.wzpmc.filemanager.entities.files.RawFileObject;
@ -10,16 +12,13 @@ import cn.wzpmc.filemanager.entities.files.enums.FileType;
import cn.wzpmc.filemanager.entities.files.enums.SortField;
import cn.wzpmc.filemanager.entities.statistics.enums.Actions;
import cn.wzpmc.filemanager.entities.user.enums.Auth;
import cn.wzpmc.filemanager.entities.vo.FileVo;
import cn.wzpmc.filemanager.entities.vo.FolderVo;
import cn.wzpmc.filemanager.entities.vo.UserVo;
import cn.wzpmc.filemanager.entities.vo.*;
import cn.wzpmc.filemanager.interfaces.FilePathService;
import cn.wzpmc.filemanager.mapper.FileMapper;
import cn.wzpmc.filemanager.mapper.FolderMapper;
import cn.wzpmc.filemanager.mapper.RawFileMapper;
import cn.wzpmc.filemanager.mapper.*;
import cn.wzpmc.filemanager.utils.JwtUtils;
import cn.wzpmc.filemanager.utils.RandomUtils;
import cn.wzpmc.filemanager.utils.SizeStatisticsDigestInputStream;
import cn.wzpmc.filemanager.utils.stream.SerialFileInputStream;
import cn.wzpmc.filemanager.utils.stream.SizeStatisticsDigestInputStream;
import com.alibaba.fastjson2.JSONObject;
import com.mybatisflex.core.audit.http.HashUtil;
import com.mybatisflex.core.paginate.Page;
@ -32,6 +31,7 @@ import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.tika.Tika;
import org.apache.tomcat.util.buf.HexUtils;
import org.apache.tomcat.util.http.fileupload.FileItemStream;
import org.apache.tomcat.util.http.fileupload.FileUpload;
import org.apache.tomcat.util.http.fileupload.impl.FileItemIteratorImpl;
@ -43,17 +43,22 @@ import org.springframework.http.HttpStatus;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.util.StreamUtils;
import org.springframework.web.multipart.MultipartFile;
import org.springframework.web.multipart.MultipartHttpServletRequest;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.io.*;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import static cn.wzpmc.filemanager.entities.files.table.FullRawFileObjectTableDef.FULL_RAW_FILE_OBJECT;
import static cn.wzpmc.filemanager.entities.vo.table.ChunkFileVoTableDef.CHUNK_FILE_VO;
import static cn.wzpmc.filemanager.entities.vo.table.ChunksVoTableDef.CHUNKS_VO;
import static cn.wzpmc.filemanager.entities.vo.table.FileVoTableDef.FILE_VO;
import static cn.wzpmc.filemanager.entities.vo.table.FolderVoTableDef.FOLDER_VO;
import static cn.wzpmc.filemanager.entities.vo.table.UserVoTableDef.USER_VO;
@ -70,6 +75,8 @@ public class FileService {
private final FileManagerProperties properties;
private final StatisticsService statisticsService;
private final RedisTemplate<String, FileVo> linkMapper;
private final ChunkFileMapper chunkFileMapper;
private final ChunksMapper chunksMapper;
/*private final RedisTemplate<String, Long> linkCountMapper;*/
private final StringRedisTemplate idAddrLinkMapper;
private final JwtUtils jwtUtils;
@ -114,6 +121,19 @@ public class FileService {
}
}
private FilenameDescription getFilename(String name) {
int i = name.lastIndexOf(".");
String extName = null;
String start = name;
if (i != -1) {
start = name.substring(0, i);
}
if (!(i == -1 || i == name.length() - 1)) {
extName = name.substring(i + 1);
}
return new FilenameDescription(start, extName);
}
@SneakyThrows
@Transactional
public Result<FileVo> simpleUpload(MultipartHttpServletRequest request, UserVo user, String address) {
@ -127,15 +147,9 @@ public class FileService {
String fieldName = next.getFieldName();
if (fieldName.equals("file")) {
String name = next.getName();
int i = name.lastIndexOf(".");
String extName = null;
String start = name;
if (i != -1) {
start = name.substring(0, i);
}
if (!(i == -1 || i == name.length() - 1)) {
extName = name.substring(i + 1);
}
FilenameDescription filename = getFilename(name);
String start = filename.name();
String extName = filename.ext();
if (fileMapper.selectCountByCondition(FILE_VO.NAME.eq(start).and(FILE_VO.EXT.eq(extName)).and(FILE_VO.FOLDER.eq(folderParams))) > 0) {
return Result.failed(HttpStatus.CONFLICT, "存在同名文件，请改名或删除后重试！");
}
@ -280,7 +294,6 @@ public class FileService {
return Result.success();
}
@SneakyThrows
public void downloadFile(String id, String range, HttpServletResponse response) {
FileVo fileVo = linkMapper.opsForValue().get(id);
if (fileVo == null) {
@ -304,22 +317,43 @@ public class FileService {
} else {
response.setStatus(200);
}
// log.info("-------Prepare-Response-{}-{}-------", min, max);
String hash = fileVo.getHash();
File file = new File(properties.getSavePath(), hash);
String fullName = fileVo.getName();
String ext = fileVo.getExt();
if (ext != null) {
fullName += '.' + ext;
}
response.addHeader("Content-Length", String.valueOf(max - min));
ContentDisposition disposition = ContentDisposition.attachment().filename(fullName, StandardCharsets.UTF_8).build();
response.addHeader("Content-Disposition", disposition.toString());
ServletOutputStream outputStream = response.getOutputStream();
try (FileInputStream fis = new FileInputStream(file)) {
StreamUtils.copyRange(fis, outputStream, min, max);
File file = new File(properties.getSavePath(), hash);
// log.info("-------Copy-{}-{}-------", min, max);
try (ServletOutputStream outputStream = response.getOutputStream()) {
InputStream stream = null;
try {
if (file.exists()) {
stream = new FileInputStream(file);
} else {
List<ChunksVo> chunksVos = chunksMapper.selectListByQuery(select(CHUNKS_VO.ALL_COLUMNS).from(CHUNK_FILE_VO).rightJoin(CHUNKS_VO).on(CHUNK_FILE_VO.CHUNK_ID.eq(CHUNKS_VO.ID)).where(CHUNK_FILE_VO.FILE_ID.eq(fileVo.getId())).orderBy(CHUNK_FILE_VO.INDEX.asc()));
if (chunksVos.isEmpty()) {
Result.failed(HttpStatus.NOT_FOUND, "未知文件").writeToResponse(response);
return;
}
stream = openSerialFileInputStreamByChunks(chunksVos);
}
StreamUtils.copyRange(stream, outputStream, min, max);
} finally {
if (stream != null) {
stream.close();
}
}
} catch (IOException e) {
if (!response.isCommitted()) {
response.reset();
}
}
outputStream.flush();
// log.info("-------flush-{}-{}-------", min, max);
}
public Result<String> getFileLink(long id, String address, HttpServletRequest request) {
@ -400,4 +434,99 @@ public class FileService {
}
return Result.success(fileVo);
}
public Result<List<CheckChunkResult>> checkChunkUploaded(List<String> hash) {
List<ChunksVo> chunksVos = chunksMapper.selectListByCondition(CHUNKS_VO.HASH.in(hash));
Map<String, Long> hashResult = new HashMap<>();
chunksVos.forEach(e -> hashResult.put(e.getHash(), e.getId()));
List<CheckChunkResult> list1 = hash.stream().map(e -> new CheckChunkResult(e, hashResult.get(e))).toList();
return Result.success(list1);
}
@SneakyThrows
public Result<Long> uploadChunk(MultipartFile block) {
File savePath = properties.getSavePath();
File blobDir = new File(savePath, "blobs");
SizeStatisticsDigestInputStream sizeStatisticsDigestInputStream = new SizeStatisticsDigestInputStream(block.getInputStream(), DigestUtils.getSha1Digest());
byte[] bytes = sizeStatisticsDigestInputStream.readAllBytes();
long size = sizeStatisticsDigestInputStream.getSize();
sizeStatisticsDigestInputStream.close();
MessageDigest messageDigest = sizeStatisticsDigestInputStream.getMessageDigest();
String hex = HashUtil.toHex(messageDigest.digest());
ChunksVo chunksVo = chunksMapper.selectOneByCondition(CHUNKS_VO.HASH.eq(hex));
if (chunksVo != null) {
return Result.success(chunksVo.getId());
}
String start = hex.substring(0, 2);
File chunkBlockDir = new File(blobDir, start);
if (!chunkBlockDir.exists()) {
if (!chunkBlockDir.mkdirs()) {
return Result.failed(HttpStatus.INTERNAL_SERVER_ERROR, "无法创建分区文件夹");
}
}
File file = new File(chunkBlockDir, hex);
try (FileOutputStream fos = new FileOutputStream(file)) {
fos.write(bytes);
}
chunksVo = new ChunksVo();
chunksVo.setHash(hex);
chunksVo.setSize(size);
chunksMapper.insert(chunksVo);
return Result.success(chunksVo.getId());
}
@SneakyThrows
@Transactional
public Result<FileVo> saveFile(SaveChunksRequest chunks) {
FilenameDescription filename = getFilename(chunks.getFilename());
String name = filename.name();
String ext = filename.ext();
Long folderId = chunks.getFolderId();
if (fileMapper.selectCountByCondition(FILE_VO.FOLDER.eq(folderId).and(FILE_VO.NAME.eq(name)).and(FILE_VO.EXT.eq(ext))) > 0) {
return Result.failed(HttpStatus.CONFLICT, "文件已存在！");
}
FileVo fileVo = new FileVo();
List<Long> chunkIds = chunks.getChunks();
List<ChunksVo> chunksVos = chunksMapper.selectListByIds(chunkIds);
//noinspection OptionalGetWithoutIsPresent
List<ChunksVo> sortedChunks = chunkIds.stream().map(e -> chunksVos.stream().filter(a -> a.getId() == e).findFirst().get()).toList();
String mime;
String sha512;
long size;
FileOutputStream fileOutputStream = new FileOutputStream("test.bin");
try (SerialFileInputStream serialFileInputStream = openSerialFileInputStreamByChunks(sortedChunks)) {
Tika tika = new Tika();
mime = tika.detect(serialFileInputStream);
serialFileInputStream.reset();
try (SizeStatisticsDigestInputStream sizeStatisticsDigestInputStream = new SizeStatisticsDigestInputStream(serialFileInputStream, DigestUtils.getSha512Digest())) {
sizeStatisticsDigestInputStream.transferTo(fileOutputStream);
sizeStatisticsDigestInputStream.close();
sha512 = HexUtils.toHexString(sizeStatisticsDigestInputStream.getMessageDigest().digest());
size = sizeStatisticsDigestInputStream.getSize();
}
}
fileOutputStream.close();
fileVo.setName(name);
fileVo.setExt(ext);
fileVo.setUploader(-2);
fileVo.setFolder(folderId);
fileVo.setMime(mime);
fileVo.setHash(sha512);
fileVo.setSize(size);
fileMapper.insert(fileVo);
long fileId = fileVo.getId();
AtomicLong currentIndex = new AtomicLong();
List<ChunkFileVo> mapper = chunkIds.stream().map(e -> new ChunkFileVo(e, fileId, currentIndex.getAndIncrement())).toList();
chunkFileMapper.insertBatch(mapper);
return Result.success(fileVo);
}
private SerialFileInputStream openSerialFileInputStreamByChunks(List<ChunksVo> chunks) {
List<File> list = chunks.stream().map(ChunksVo::getHash).map(e -> new File(new File(new File(properties.getSavePath(), "blobs"), e.substring(0, 2)), e)).toList();
return new SerialFileInputStream(list);
}
private record FilenameDescription(String name, String ext) {
}
}

2
src/main/java/cn/wzpmc/filemanager/service/UserService.java
View File

@ -21,6 +21,8 @@ import org.springframework.data.redis.core.ValueOperations;
import org.springframework.http.HttpStatus;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.concurrent.TimeUnit;
import static cn.wzpmc.filemanager.entities.vo.table.UserVoTableDef.USER_VO;

109
src/main/java/cn/wzpmc/filemanager/utils/stream/SerialFileInputStream.java Normal file
View File

@ -0,0 +1,109 @@
package cn.wzpmc.filemanager.utils.stream;
import lombok.NonNull;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.util.List;
@Slf4j
@RequiredArgsConstructor
public class SerialFileInputStream extends InputStream {
private final List<File> files;
private FileInputStream currentFileStream;
private int filePointer = 0;
@Override
public int read() throws IOException {
if (files.isEmpty()) {
return -1;
}
if (currentFileStream == null) {
File file = files.get(0);
currentFileStream = new FileInputStream(file);
}
int read = currentFileStream.read();
if (read == -1) {
filePointer++;
File file = files.get(filePointer);
if (file == null) {
return -1;
}
currentFileStream.close();
currentFileStream = new FileInputStream(file);
read = currentFileStream.read();
}
return read;
}
@Override
public int read(byte @NonNull [] b, int off, int len) throws IOException {
if (len == 0) return 0;
if (files.isEmpty()) return -1;
if (currentFileStream == null) {
File file = files.get(0);
currentFileStream = new FileInputStream(file);
}
int readBytes = 0;
while (true) {
int currentReadBytes = currentFileStream.read(b, off + readBytes, len - readBytes);
if (currentReadBytes != -1) {
readBytes += currentReadBytes;
if (readBytes >= len) {
return readBytes;
}
}
filePointer++;
log.debug("POINTER: {}", filePointer);
if (filePointer >= files.size()) {
if (readBytes == 0) return -1;
return readBytes;
}
File file = files.get(filePointer);
currentFileStream.close();
currentFileStream = new FileInputStream(file);
}
}
@Override
public void close() throws IOException {
if (this.currentFileStream != null) {
this.currentFileStream.close();
}
super.close();
}
@Override
public synchronized void reset() throws IOException {
this.filePointer = 0;
currentFileStream = null;
}
@Override
public long skip(long n) throws IOException {
long originalN = n;
if (currentFileStream != null) {
currentFileStream.close();
currentFileStream = null;
}
for (File file : files) {
long usableSpace = Files.size(file.toPath());
if (usableSpace > n) break;
n -= usableSpace;
filePointer++;
if (filePointer >= files.size()) {
return originalN - n;
}
}
currentFileStream = new FileInputStream(files.get(filePointer));
if (n > 0) {
return currentFileStream.skip(n) + (originalN - n);
}
return originalN;
}
}

2
src/main/java/cn/wzpmc/filemanager/utils/SizeStatisticsDigestInputStream.java → src/main/java/cn/wzpmc/filemanager/utils/stream/SizeStatisticsDigestInputStream.java
View File

@ -1,4 +1,4 @@
package cn.wzpmc.filemanager.utils;
package cn.wzpmc.filemanager.utils.stream;
import lombok.Getter;

22
src/test/java/cn/wzpmc/filemanager/FileManagerApplicationTests.java
View File

@ -1,13 +1,29 @@
package cn.wzpmc.filemanager;
import cn.wzpmc.filemanager.utils.stream.SerialFileInputStream;
import org.junit.jupiter.api.Test;
import org.springframework.boot.test.context.SpringBootTest;
@SpringBootTest
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.List;
class FileManagerApplicationTests {
private static final File baseFolder = new File("run");
@Test
void contextLoads() {
void serialFileStreamTest() throws IOException {
File outputFile = new File(baseFolder, "out.txt");
if (outputFile.exists()) {
outputFile.delete();
}
try (SerialFileInputStream serialFileInputStream = new SerialFileInputStream(List.of(new File(baseFolder, "a.txt"), new File(baseFolder, "b.txt"), new File(baseFolder, "c.txt"), new File(baseFolder, "d.txt"))); FileOutputStream fos = new FileOutputStream(outputFile)) {
byte[] buf = new byte[1026];
int read;
while ((read = serialFileInputStream.read(buf)) != -1) {
fos.write(buf, 0, read);
}
}
}
}