From e3508ad65d69fad2cd3b4e8fd9a0fa1cbd56a1c6 Mon Sep 17 00:00:00 2001 From: wzp Date: Wed, 26 Apr 2023 18:53:17 +0800 Subject: [PATCH] feat: adding get files --- .idea/uiDesigner.xml | 124 +++++++++++++++++ .idea/workspace.xml | 88 ++++++------ FFMpegJ/build.gradle.kts | 4 +- .../filemanager/ffmpeg/FFMpegRuntime.java | 2 +- .../ffmpeg/enums/VideoEncoder.java | 20 +++ .../ffmpeg/threads/TranscodingFileThread.java | 128 ++++++++++++++++++ application.properties | 8 ++ .../filemanager/FileManagerApplication.java | 1 - .../controller/FileController.java | 27 +++- .../controller/UserController.java | 10 +- .../entities/EncodingThreadInfo.java | 16 +++ .../filemanager/enums/EncodingStatus.java | 5 + .../filemanager/service/FileService.java | 89 +++++++++++- 13 files changed, 453 insertions(+), 69 deletions(-) create mode 100644 .idea/uiDesigner.xml create mode 100644 FFMpegJ/src/main/java/cn/wzpmc/filemanager/ffmpeg/enums/VideoEncoder.java create mode 100644 FFMpegJ/src/main/java/cn/wzpmc/filemanager/ffmpeg/threads/TranscodingFileThread.java create mode 100644 application.properties create mode 100644 src/main/java/cn/wzpmc/filemanager/entities/EncodingThreadInfo.java create mode 100644 src/main/java/cn/wzpmc/filemanager/enums/EncodingStatus.java diff --git a/.idea/uiDesigner.xml b/.idea/uiDesigner.xml new file mode 100644 index 0000000..2b63946 --- /dev/null +++ b/.idea/uiDesigner.xml @@ -0,0 +1,124 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/workspace.xml b/.idea/workspace.xml index 2713ca6..8ff5046 100644 --- a/.idea/workspace.xml +++ b/.idea/workspace.xml @@ -5,44 +5,19 @@ - - - - + + + - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + - - - - - - + + @@ -151,28 +128,29 @@ - + - - + + - - - - - + + + + + + - + @@ -346,8 +324,8 @@ - + @@ -378,7 +356,9 @@ - + + + 1679928730544 @@ -387,7 +367,21 @@ - diff --git a/FFMpegJ/build.gradle.kts b/FFMpegJ/build.gradle.kts index 6971fb9..a0f8a81 100644 --- a/FFMpegJ/build.gradle.kts +++ b/FFMpegJ/build.gradle.kts @@ -21,7 +21,9 @@ dependencies { annotationProcessor("org.projectlombok:lombok:1.18.26") // https://mvnrepository.com/artifact/com.alibaba.fastjson2/fastjson2 implementation("com.alibaba.fastjson2:fastjson2:2.0.26") - implementation("org.apache.tika:tika-core:2.7.0") + implementation( + + "org.apache.tika:tika-core:2.7.0") } tasks.withType { diff --git a/FFMpegJ/src/main/java/cn/wzpmc/filemanager/ffmpeg/FFMpegRuntime.java b/FFMpegJ/src/main/java/cn/wzpmc/filemanager/ffmpeg/FFMpegRuntime.java index 1ce1e12..f7564cc 100644 --- a/FFMpegJ/src/main/java/cn/wzpmc/filemanager/ffmpeg/FFMpegRuntime.java +++ b/FFMpegJ/src/main/java/cn/wzpmc/filemanager/ffmpeg/FFMpegRuntime.java @@ -40,7 +40,7 @@ public class FFMpegRuntime { if (contentType == null || !contentType.startsWith("video")) { return null; } - ProcessBuilder processBuilder = new ProcessBuilder("ffprobe", "-v", "quiet", "-print_format", "json", "-show_streams", file.getAbsolutePath()); + ProcessBuilder processBuilder = new ProcessBuilder(this.ffprobePath, "-v", "quiet", "-print_format", "json", "-show_streams", file.getAbsolutePath()); Process process = processBuilder.start(); BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream())); StringBuilder stringBuilder = new StringBuilder(); diff --git a/FFMpegJ/src/main/java/cn/wzpmc/filemanager/ffmpeg/enums/VideoEncoder.java b/FFMpegJ/src/main/java/cn/wzpmc/filemanager/ffmpeg/enums/VideoEncoder.java new file mode 100644 index 0000000..8426ff8 --- /dev/null +++ b/FFMpegJ/src/main/java/cn/wzpmc/filemanager/ffmpeg/enums/VideoEncoder.java @@ -0,0 +1,20 @@ +package cn.wzpmc.filemanager.ffmpeg.enums; + +public enum VideoEncoder { + H264("h264", "h264_nvenc", "h264_cuvid"), + HEVC("h265", "hevc_nvenc", "hevc_cuvid"); + public final String name; + public final String encoderName; + public final String decoderName; + VideoEncoder(String name, String encoderName, String decoderName){ + this.name = name; + this.encoderName = encoderName; + this.decoderName = decoderName; + } + public static VideoEncoder getEncoder(String name){ + if (name.equals("h264")){ + return H264; + } + return HEVC; + } +} diff --git a/FFMpegJ/src/main/java/cn/wzpmc/filemanager/ffmpeg/threads/TranscodingFileThread.java b/FFMpegJ/src/main/java/cn/wzpmc/filemanager/ffmpeg/threads/TranscodingFileThread.java new file mode 100644 index 0000000..4376f02 --- /dev/null +++ b/FFMpegJ/src/main/java/cn/wzpmc/filemanager/ffmpeg/threads/TranscodingFileThread.java @@ -0,0 +1,128 @@ +package cn.wzpmc.filemanager.ffmpeg.threads; + +import cn.wzpmc.filemanager.ffmpeg.FFMpegRuntime; +import cn.wzpmc.filemanager.ffmpeg.enums.VideoEncoder; +import com.alibaba.fastjson2.JSONArray; +import com.alibaba.fastjson2.JSONObject; +import lombok.Getter; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; + +import java.io.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +@Slf4j +public class TranscodingFileThread extends Thread{ + private final static Pattern getDataPattern = Pattern.compile("frame=\\s*(\\d+)\\s+fps=([\\d\\.]+)\\s+q=([\\d\\.]+)\\s+size=\\s*(\\d+kB)\\s+time=([\\d:.]+)\\s+bitrate=([\\d\\.]+kbits/s)\\s+speed=([\\d\\.]+x)\\s*"); + private final File input; + @Getter + private final VideoEncoder outputEncoder; + @Getter + private final File output; + private final Callback callback; + private final FFMpegRuntime ffMpegRuntime; + private VideoEncoder inputEncoder; + @Getter + private long totalFrames; + @Getter + private float progress; + @Getter + private long frames; + @Getter + private float fps; + @SneakyThrows + public TranscodingFileThread(File input, VideoEncoder outputEncoder, File output, Callback callback, FFMpegRuntime ffMpegRuntime){ + this.input = input; + this.outputEncoder = outputEncoder; + this.output = output; + this.callback = callback; + this.ffMpegRuntime = ffMpegRuntime; + ProcessBuilder probeProcessBuilder = new ProcessBuilder(); + probeProcessBuilder.command( + this.ffMpegRuntime.getFfprobePath(), + "-v", + "quiet", + "-print_format", + "json", + "-show_streams", + this.input.getAbsolutePath() + ); + Process probeProcess = probeProcessBuilder.start(); + probeProcess.waitFor(); + String fileProbeResult = new String(probeProcess.getInputStream().readAllBytes()); + JSONObject probeResultObject = JSONObject.parseObject(fileProbeResult); + JSONArray streams = probeResultObject.getJSONArray("streams"); + for (int i = 0; i < streams.size(); i++) { + JSONObject stream = streams.getJSONObject(i); + if (stream.getString("codec_type").equals("video")) { + this.inputEncoder = VideoEncoder.getEncoder(stream.getString("codec_name")); + this.totalFrames = Long.parseLong(stream.getString("nb_frames")); + } + } + if (this.inputEncoder == null){ + throw new RuntimeException(new IOException("The input file isn't video file!")); + } + super.setName("Transcoding-File-Thread-" + input.getName()); + } + + @Override + public void run() { + try{ + if (this.output.exists()){ + boolean delete = this.output.delete(); + if (!delete){ + return; + } + } + ProcessBuilder processBuilder = new ProcessBuilder(); + processBuilder.command( + this.ffMpegRuntime.getFfmpegPath(), + "-hide_banner", + "-vsync", + "0", + "-hwaccel", + "cuvid", + "-hwaccel_output_format", + "cuda", + "-c:v", + this.inputEncoder.decoderName, + "-i", + this.input.getAbsolutePath(), + "-c:v", + this.outputEncoder.encoderName, + "-c:a", + "copy", + this.output.getAbsolutePath() + ); + System.out.println(String.join(" ", processBuilder.command())); + Process start = processBuilder.start(); + BufferedInputStream in = new BufferedInputStream(start.getErrorStream()); + BufferedReader inBr = new BufferedReader(new InputStreamReader(in)); + String lineStr; + while (start.isAlive()) { + lineStr = inBr.readLine(); + if (lineStr == null){ + this.frames = totalFrames; + progress = 1.0f; + return; + } + Matcher matcher = getDataPattern.matcher(lineStr); + if (matcher.find()) { + this.frames = Long.parseLong(matcher.group(1)); + this.fps = Float.parseFloat(matcher.group(2)); + progress = (float) this.frames / totalFrames; + } + } + }catch (Throwable throwable){ + throwable.printStackTrace(); + }finally { + this.callback.onDone(this); + } + } + + @FunctionalInterface + public interface Callback{ + void onDone(TranscodingFileThread thread); + } +} diff --git a/application.properties b/application.properties new file mode 100644 index 0000000..079d89a --- /dev/null +++ b/application.properties @@ -0,0 +1,8 @@ +server.port=8080 +spring.datasource.url=jdbc:mysql://192.168.1.2:3306/fs +spring.datasource.username=FS +spring.datasource.password=JiGSnWrwpPFnrthM +save-path=/main/files +hmac-key=rsnIkOKV,8#5Ex>B +spring.servlet.multipart.max-request-size=20GB +spring.servlet.multipart.max-file-size=20GB \ No newline at end of file diff --git a/src/main/java/cn/wzpmc/filemanager/FileManagerApplication.java b/src/main/java/cn/wzpmc/filemanager/FileManagerApplication.java index c64dc7e..2dcfa0c 100644 --- a/src/main/java/cn/wzpmc/filemanager/FileManagerApplication.java +++ b/src/main/java/cn/wzpmc/filemanager/FileManagerApplication.java @@ -10,5 +10,4 @@ public class FileManagerApplication { public static void main(String[] args) { SpringApplication.run(FileManagerApplication.class, args); } - } diff --git a/src/main/java/cn/wzpmc/filemanager/controller/FileController.java b/src/main/java/cn/wzpmc/filemanager/controller/FileController.java index 7b25a84..2200774 100644 --- a/src/main/java/cn/wzpmc/filemanager/controller/FileController.java +++ b/src/main/java/cn/wzpmc/filemanager/controller/FileController.java @@ -1,10 +1,14 @@ package cn.wzpmc.filemanager.controller; import cn.wzpmc.filemanager.entities.CountableList; +import cn.wzpmc.filemanager.entities.EncodingThreadInfo; import cn.wzpmc.filemanager.entities.FileObject; import cn.wzpmc.filemanager.enums.SearchType; +import cn.wzpmc.filemanager.ffmpeg.enums.VideoEncoder; import cn.wzpmc.filemanager.service.FileService; import jakarta.servlet.http.HttpServletResponse; +import lombok.Data; +import lombok.EqualsAndHashCode; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.*; import org.springframework.web.multipart.MultipartFile; @@ -14,21 +18,22 @@ import java.util.Map; @RestController @CrossOrigin +@RequestMapping("/api/file") public class FileController { private final FileService service; @Autowired public FileController(FileService service){ this.service = service; } - @GetMapping("/api/file/count") + @GetMapping("/count") public Long getFileCount(){ return service.getFileCount(); } - @GetMapping("/api/file/get") + @GetMapping("/get") public List getFiles(@RequestParam("page") int page){ return service.getFiles(page); } - @GetMapping("/api/file/search") + @GetMapping("/search") public CountableList searchFiles(@RequestParam("type") SearchType type, @RequestParam("keywords") String keywords, @RequestParam("page") int page){ return switch (type){ case NAME -> service.searchFilesByName(keywords, page); @@ -37,20 +42,28 @@ public class FileController { case FORMAT -> service.searchFilesByFormat(keywords, page); }; } - @GetMapping("/api/file") + @GetMapping("/") public void downloadFile(@RequestParam("id") int id, HttpServletResponse response){ service.downloadFile(id, response); } - @PostMapping("/api/file/upload") + @PostMapping("/upload") public FileObject uploadFile(@RequestHeader("Authorization") String token, @RequestBody MultipartFile file){ return service.uploadFile(token, file); } - @PostMapping("/api/file/remove") + @PostMapping("/remove") public boolean removeFile(@RequestHeader("Authorization") String token, @RequestBody FileObject file){ return service.removeFile(token, file); } - @GetMapping("/api/file/details") + @GetMapping("/details") public Map getFileDetails(@RequestParam("id") int id){ return service.getFileDetails(id); } + @PostMapping("/encoding") + public Object transEncoding(@RequestBody FileObject object, @RequestParam("enc") String encoder, @RequestHeader("Authorization") String token){ + return service.encoding(object, VideoEncoder.getEncoder(encoder), token); + } + @GetMapping("/getEncodingInfo") + public List getEncodingInfo(){ + return service.getEncodingInfo(); + } } diff --git a/src/main/java/cn/wzpmc/filemanager/controller/UserController.java b/src/main/java/cn/wzpmc/filemanager/controller/UserController.java index 20b2721..f879132 100644 --- a/src/main/java/cn/wzpmc/filemanager/controller/UserController.java +++ b/src/main/java/cn/wzpmc/filemanager/controller/UserController.java @@ -6,24 +6,22 @@ import cn.wzpmc.filemanager.service.UserService; import jakarta.servlet.http.HttpServletRequest; import jakarta.servlet.http.HttpServletResponse; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.web.bind.annotation.CrossOrigin; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.bind.annotation.*; @RestController @CrossOrigin +@RequestMapping("/api/user/") public class UserController { private final UserService service; @Autowired public UserController(UserService service){ this.service = service; } - @PostMapping("/api/user/login") + @PostMapping("/login") public ResponseResult login(@RequestBody User user, HttpServletRequest request, HttpServletResponse response){ return new ResponseResult<>(this.service.login(user, request.getRemoteAddr(), response)); } - @PostMapping("/api/user/register") + @PostMapping("/register") public User register(@RequestBody User user, HttpServletRequest request, HttpServletResponse response){ return this.service.register(user, request.getRemoteAddr(), response); } diff --git a/src/main/java/cn/wzpmc/filemanager/entities/EncodingThreadInfo.java b/src/main/java/cn/wzpmc/filemanager/entities/EncodingThreadInfo.java new file mode 100644 index 0000000..86a50d4 --- /dev/null +++ b/src/main/java/cn/wzpmc/filemanager/entities/EncodingThreadInfo.java @@ -0,0 +1,16 @@ +package cn.wzpmc.filemanager.entities; + +import cn.wzpmc.filemanager.enums.EncodingStatus; +import lombok.AllArgsConstructor; +import lombok.Data; + +@Data +@AllArgsConstructor +public class EncodingThreadInfo { + private EncodingStatus status; + private FileObject file; + private float progress; + private long totalFrame; + private long nowFrame; + private float fps; +} diff --git a/src/main/java/cn/wzpmc/filemanager/enums/EncodingStatus.java b/src/main/java/cn/wzpmc/filemanager/enums/EncodingStatus.java new file mode 100644 index 0000000..509b793 --- /dev/null +++ b/src/main/java/cn/wzpmc/filemanager/enums/EncodingStatus.java @@ -0,0 +1,5 @@ +package cn.wzpmc.filemanager.enums; + +public enum EncodingStatus { + WAITING, RUNNING, END +} diff --git a/src/main/java/cn/wzpmc/filemanager/service/FileService.java b/src/main/java/cn/wzpmc/filemanager/service/FileService.java index 64c7d71..bdcd5cc 100644 --- a/src/main/java/cn/wzpmc/filemanager/service/FileService.java +++ b/src/main/java/cn/wzpmc/filemanager/service/FileService.java @@ -2,9 +2,13 @@ package cn.wzpmc.filemanager.service; import cn.wzpmc.filemanager.dao.FileDao; import cn.wzpmc.filemanager.entities.CountableList; +import cn.wzpmc.filemanager.entities.EncodingThreadInfo; import cn.wzpmc.filemanager.entities.FileObject; import cn.wzpmc.filemanager.entities.User; +import cn.wzpmc.filemanager.enums.EncodingStatus; import cn.wzpmc.filemanager.ffmpeg.FFMpegRuntime; +import cn.wzpmc.filemanager.ffmpeg.enums.VideoEncoder; +import cn.wzpmc.filemanager.ffmpeg.threads.TranscodingFileThread; import cn.wzpmc.filemanager.utils.JwtUtils; import jakarta.servlet.http.HttpServletResponse; import lombok.SneakyThrows; @@ -19,6 +23,8 @@ import org.springframework.web.multipart.MultipartFile; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.file.Files; import java.security.MessageDigest; import java.text.DateFormat; import java.text.SimpleDateFormat; @@ -44,6 +50,8 @@ public class FileService { log.info("成功创建保存文件夹！"); } this.runtime = new FFMpegRuntime(); + DaemonThread daemonThread = new DaemonThread(); + daemonThread.start(); } public Long getFileCount() { @@ -88,12 +96,21 @@ public class FileService { } @SneakyThrows public FileObject uploadFile(String token, MultipartFile file) { + String fullName = file.getOriginalFilename(); + assert fullName != null; + File tempFile = File.createTempFile(fullName, ""); + FileOutputStream fileOutputStream = new FileOutputStream(tempFile); + StreamUtils.copy(file.getInputStream(), fileOutputStream); + fileOutputStream.close(); + return uploadFile(token, tempFile, fullName); + } + public FileObject uploadFile(String token, File file, String fullName){ Optional user = jwtUtils.getUser(token); if (user.isEmpty()) { return null; } User requestUser = user.get(); - String fullName = file.getOriginalFilename(); + assert fullName != null; int i = fullName.lastIndexOf('.'); String name = fullName.substring(0, i); @@ -102,20 +119,26 @@ public class FileService { FileObject fileObject = new FileObject(); fileObject.setFileName(name); fileObject.setFileFormat(format); - fileObject.setFileSize(file.getSize()); + fileObject.setFileSize(file.length()); fileObject.setUploader(requestUser.getUsername()); fileObject.setMd5(md5); File saveFile = new File(this.savePath, md5); - if (!saveFile.exists()) { - StreamUtils.copy(file.getInputStream(), new FileOutputStream(saveFile)); + try { + Files.move(file.toPath(), saveFile.toPath()); + } catch (IOException e) { + e.printStackTrace(); + log.warn("移动文件失败！, File={}", fileObject); + return null; } dao.uploadFile(fileObject); return fileObject; } @SneakyThrows - private static String getFileMd5(MultipartFile file){ + private static String getFileMd5(File file){ MessageDigest MD5 = MessageDigest.getInstance("MD5"); - MD5.digest(file.getBytes()); + FileInputStream fileInputStream = new FileInputStream(file); + MD5.digest(fileInputStream.readAllBytes()); + fileInputStream.close(); return new String(Hex.encodeHex(MD5.digest())); } @@ -166,4 +189,58 @@ public class FileService { } return result; } + private final Map transcodingFileThreadMap = new Hashtable<>(); + private final Map statusMap = new Hashtable<>(); + @SneakyThrows + public Object encoding(FileObject fileObject, VideoEncoder outputEnc, String token){ + if (!runtime.check()) { + log.warn("服务端缺少FFMpeg环境（FFMpeg和FFProbe），会影响某些功能使用！"); + } + if (transcodingFileThreadMap.containsKey(fileObject.getId())){ + return "文件存在，无法转码！"; + } + File outputTempFile = File.createTempFile("video-encoding-" + fileObject.getMd5(), ".mp4"); + File inputFile = new File(savePath, fileObject.getMd5()); + TranscodingFileThread transcodingFileThread = new TranscodingFileThread(inputFile, outputEnc, outputTempFile, (thread) -> { + statusMap.put(thread.getName(), EncodingStatus.END); + this.uploadFile(token, thread.getOutput(), fileObject.getFileName() + "-" + thread.getOutputEncoder().name + ".mp4"); + }, runtime); + statusMap.put(transcodingFileThread.getName(), EncodingStatus.WAITING); + transcodingFileThreadMap.put(fileObject.getId(), transcodingFileThread); + return transcodingFileThreadMap.hashCode(); + } + + public List getEncodingInfo() { + List result = new ArrayList<>(); + for (Map.Entry entry : transcodingFileThreadMap.entrySet()) { + Integer fileId = entry.getKey(); + TranscodingFileThread value = entry.getValue(); + FileObject fullFileInfo = dao.getFullFileInfo(fileId); + result.add(new EncodingThreadInfo(statusMap.get(value.getName()), fullFileInfo, value.getProgress(), value.getTotalFrames(), value.getFrames(), value.getFps())); + } + return result; + } + private class DaemonThread extends Thread { + @SneakyThrows + @Override + public void run() { + while (true){ + for (TranscodingFileThread value : FileService.this.transcodingFileThreadMap.values()) { + String name = value.getName(); + EncodingStatus status = FileService.this.statusMap.get(name); + if (status.equals(EncodingStatus.RUNNING)){ + break; + } + if (status.equals(EncodingStatus.END)){ + continue; + } + value.start(); + FileService.this.statusMap.put(name, EncodingStatus.RUNNING); + break; + } + Thread.sleep(1000); + } + + } + } }