优化:文件补录功能查询数据调整为在后台创建单独线程定时校验

master
guoxin 1 year ago
parent 9218a5c787
commit 2a33fc9d8a
  1. 7
      shandan-control/src/main/java/com/keyware/shandan/ControlApplication.java
  2. 180
      shandan-control/src/main/java/com/keyware/shandan/control/InvalidFileHandler.java
  3. 85
      shandan-control/src/main/java/com/keyware/shandan/control/controller/FileRepairController.java
  4. 3
      shandan-system/src/main/java/com/keyware/shandan/bianmu/service/impl/DirectoryServiceImpl.java
  5. 1
      shandan-system/src/main/resources/static/js/sys/file/repair.js
  6. 4
      shandan-system/src/main/resources/view/sys/file/fileRepair.html

@ -1,6 +1,7 @@
package com.keyware.shandan;
import com.keyware.shandan.common.enums.SystemTypes;
import com.keyware.shandan.control.InvalidFileHandler;
import com.keyware.shandan.frame.config.security.SecurityUtil;
import com.keyware.shandan.system.entity.SysMenu;
import com.keyware.shandan.system.entity.SysRole;
@ -12,6 +13,7 @@ import com.keyware.shandan.system.utils.SysSettingUtil;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.ApplicationContext;
import org.springframework.context.annotation.Bean;
import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories;
import org.springframework.scheduling.annotation.EnableAsync;
@ -37,7 +39,10 @@ import java.util.Set;
@EnableElasticsearchRepositories
public class ControlApplication {
public static void main(String[] args) {
SpringApplication.run(ControlApplication.class, args);
ApplicationContext context = SpringApplication.run(ControlApplication.class, args);
InvalidFileHandler invalidFileHandler = context.getBean(InvalidFileHandler.class);
invalidFileHandler.startValid();
}
/**

@ -0,0 +1,180 @@
package com.keyware.shandan.control;
import cn.hutool.core.date.DateUtil;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.keyware.shandan.bianmu.entity.DirectoryVo;
import com.keyware.shandan.bianmu.service.DirectoryService;
import com.keyware.shandan.frame.properties.CustomProperties;
import com.keyware.shandan.system.entity.SysFile;
import com.keyware.shandan.system.entity.SysUser;
import com.keyware.shandan.system.service.SysFileService;
import com.keyware.shandan.system.service.SysUserService;
import lombok.Builder;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import org.springframework.util.Assert;
import org.springframework.util.StringUtils;
import java.io.File;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
/**
* 用于对文件进行失效验证
*/
@Slf4j
@Component
public class InvalidFileHandler {
private final String storagePath;
private final SysUserService userService;
private final DirectoryService directoryService;
private final SysFileService fileService;
private final ConcurrentHashMap<String, InvalidItem> cache = new ConcurrentHashMap<>();
public InvalidFileHandler(CustomProperties customProperties,
SysUserService userService,
DirectoryService directoryService,
SysFileService fileService) {
this.userService = userService;
this.directoryService = directoryService;
this.fileService = fileService;
this.storagePath = customProperties.getFileStorage().getPath();
}
/**
* 开始验证
*/
public void startValid() {
log.info("文件校验处理器启动成功");
new Thread(() -> {
while (true) {
refreshInvalidCache("ROOT");
try {
Thread.sleep(1000 * 10);// 1小时执行一次
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
}
}).start();
}
/**
* 刷新失效文件的缓存
*
* @param dirId 指定目录ID
*/
public void refreshInvalidCache(String dirId) {
if (!StringUtils.hasText(dirId)) {
dirId = "ROOT";
}
Set<String> idSet = new HashSet<>();
directoryService.handlerAllChildrenListById(dirId, (context) -> {
DirectoryVo dir = context.getResultObject();
if (StringUtils.hasText(dir.getResourceId())) {
// 查询对应的文件资源
SysFile sf = fileService.getById(dir.getResourceId());
if (sf != null) {
File file = new File(storagePath + File.separator + sf.getPath());
if (!file.exists()) {
SysUser user = userService.getById(sf.getModifyUser());
InvalidItem item = InvalidItem.builder()
.id(sf.getId())
.dirId(dir.getId())
.directoryPath(dir.getDirectoryPath())
.fileName(sf.getFileName())
.fileType(sf.getFileType())
.fileSize(sf.getFileSize())
.createTime(DateUtil.format(sf.getModifyTime(), "yyyy-MM-dd HH:mm"))
.createUser(user == null ? sf.getModifyUser() : user.getUserName())
.build();
cache.put(sf.getId(), item);
idSet.add(sf.getId());
}
}
}
});
// 与新的结果进行比对,将结果中不存在的缓存清除掉
List<String> keys = new ArrayList<>();
cache.forEachKey(1, keys::add);
keys.forEach(key -> {
if (!idSet.contains(key)) {
cache.remove(key);
}
});
}
/**
* 检索指定目录下的失效文件
*
* @param dirId 目录ID
* @param refresh 是否重新刷新
* @return
*/
public List<InvalidItem> listByDir(String dirId, boolean refresh) {
DirectoryVo dir = directoryService.getById(dirId);
Assert.notNull(dir, "查询的目录不存在");
if (refresh) {
refreshInvalidCache(dirId);
}
String path = dir.getDirectoryPath();
return cache.values().stream().filter(item -> item.getDirectoryPath().startsWith(path + "/")).collect(Collectors.toList());
}
/**
* 清理文件数据
*
* @param fileId 文件ID
*/
public void clean(String fileId) {
fileService.deleteById(fileId);
cleanDirectory(fileId);
cache.remove(fileId);
}
/**
* 移除指定文件的缓存
*
* @param fileId 文件ID
*/
public void removeCache(String fileId) {
cache.remove(fileId);
}
/**
* 递归清理文件管理的目录资源以及引用的资源
*
* @param resourceId
*/
private void cleanDirectory(String resourceId) {
LambdaQueryWrapper<DirectoryVo> query = new LambdaQueryWrapper<>();
query.eq(DirectoryVo::getResourceId, resourceId);
List<DirectoryVo> dirs = directoryService.list(query);
dirs.forEach(dir -> {
cleanDirectory(dir.getId());
directoryService.deleteById(dir.getId());
});
}
/**
* 失效的缓存项
*/
@Data
@Builder
public static class InvalidItem {
private String id;
private String dirId;
private String directoryPath;
private String fileName;
private String fileType;
private Double fileSize;
private String createTime;
private String createUser;
}
}

@ -1,50 +1,34 @@
package com.keyware.shandan.control.controller;
import cn.hutool.core.date.DateUtil;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.keyware.shandan.bianmu.entity.DirectoryVo;
import com.keyware.shandan.bianmu.service.DirectoryService;
import com.keyware.shandan.common.entity.Result;
import com.keyware.shandan.common.enums.SystemTypes;
import com.keyware.shandan.frame.properties.CustomProperties;
import com.keyware.shandan.control.InvalidFileHandler;
import com.keyware.shandan.system.entity.SysFile;
import com.keyware.shandan.system.entity.SysFileChunk;
import com.keyware.shandan.system.entity.SysSetting;
import com.keyware.shandan.system.entity.SysUser;
import com.keyware.shandan.system.service.SysFileService;
import com.keyware.shandan.system.service.SysUserService;
import com.keyware.shandan.system.utils.FileChunkUploadUtil;
import com.keyware.shandan.system.utils.SysSettingUtil;
import lombok.extern.slf4j.Slf4j;
import org.springframework.util.StringUtils;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;
import org.springframework.web.servlet.ModelAndView;
import java.io.File;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@Slf4j
@RestController
@RequestMapping("/sys/file/repair")
public class FileRepairController {
private final String storagePath;
private final SysUserService userService;
private final DirectoryService directoryService;
private final SysFileService fileService;
private final InvalidFileHandler invalidFileHandler;
public FileRepairController(CustomProperties customProperties,
SysUserService userService,
DirectoryService directoryService,
SysFileService fileService) {
storagePath = customProperties.getFileStorage().getPath();
this.userService = userService;
this.directoryService = directoryService;
public FileRepairController(SysFileService fileService,
InvalidFileHandler invalidFileHandler) {
this.fileService = fileService;
this.invalidFileHandler = invalidFileHandler;
}
/**
@ -61,29 +45,10 @@ public class FileRepairController {
}
@GetMapping("/list")
public Result<Object> invalidFile(@RequestParam String dirId) {
Map<String, Object> fileMap = new HashMap<>();
directoryService.handlerAllChildrenListById(dirId, (context) -> {
DirectoryVo dir = context.getResultObject();
if (StringUtils.hasText(dir.getResourceId())) {
// 查询对应的文件资源
SysFile file = fileService.getById(dir.getResourceId());
if (file != null && !validate(file)) {
Map<String, Object> data = new HashMap<>();
data.put("id", file.getId());
data.put("dirId", dir.getId());
data.put("directoryPath", dir.getDirectoryPath());
data.put("fileName", file.getFileName());
data.put("fileType", file.getFileType());
data.put("fileSize", file.getFileSize());
data.put("createTime", DateUtil.format(file.getModifyTime(), "yyyy-MM-dd HH:mm"));
SysUser user = userService.getById(file.getModifyUser());
data.put("createUser", user == null ? file.getModifyUser() : user.getUserName());
fileMap.put(file.getId(), data);
}
}
});
return Result.of(fileMap.values());
public Result<Object> fileList(@RequestParam String dirId,
@RequestParam(required = false, defaultValue = "false") boolean refresh) {
return Result.of(invalidFileHandler.listByDir(dirId, refresh));
}
/**
@ -95,42 +60,13 @@ public class FileRepairController {
@GetMapping("/clean/{fileId}")
public Result<Object> remove(@PathVariable String fileId) {
try {
fileService.deleteById(fileId);
cleanDirectory(fileId);
invalidFileHandler.clean(fileId);
} catch (Exception e) {
log.error("清理文件异常", e);
}
return Result.of(null);
}
/**
* 递归清理文件管理的目录资源以及引用的资源
*
* @param resourceId
*/
private void cleanDirectory(String resourceId) {
LambdaQueryWrapper<DirectoryVo> query = new LambdaQueryWrapper<>();
query.eq(DirectoryVo::getResourceId, resourceId);
List<DirectoryVo> dirs = directoryService.list(query);
dirs.forEach(dir -> {
cleanDirectory(dir.getId());
directoryService.deleteById(dir.getId());
});
}
/**
* 验证文件是否存在
*
* @param sysFile
* @return
*/
private boolean validate(SysFile sysFile) {
if (sysFile == null) {
return false;
}
File file = new File(storagePath + File.separator + sysFile.getPath());
return file.exists();
}
@PostMapping("/upload/check")
public Result<Object> uploadFileCheck() {
@ -170,6 +106,7 @@ public class FileRepairController {
List<SysFile> files = fileService.list(query);
files.forEach(f -> f.setPath(sysFile.getPath()));
fileService.updateBatchById(files);
invalidFileHandler.removeCache(file.getId());
});
return Result.of(true);
}

@ -23,11 +23,11 @@ import com.keyware.shandan.system.entity.SysOrg;
import com.keyware.shandan.system.entity.SysUser;
import com.keyware.shandan.system.service.SysFileService;
import com.keyware.shandan.system.service.SysOrgService;
import org.apache.ibatis.cursor.Cursor;
import org.apache.ibatis.session.ResultHandler;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.util.Assert;
import java.io.Serializable;
@ -379,6 +379,7 @@ public class DirectoryServiceImpl extends BaseServiceImpl<DirectoryMapper, Direc
}
@Override
@Transactional(readOnly = true)
public void handlerAllChildrenListById(String dirId, ResultHandler<DirectoryVo> handler) {
directoryMapper.handlerAllChildrenListById(dirId, handler);
}

@ -123,7 +123,6 @@ function startRender() {
fileTable = listPage.init({
table: {
id: 'dirMetadataTable',
toolbar: '#tableToolBar',
searchFieldNames: 'metadataName',
height: 'full-110',
data: data,

@ -65,10 +65,10 @@
<script type="text/html" id="tableToolBar">
<div class="layui-btn-container">
<div class="layui-layout-left" style="top:10px; left: 20px">
<input type="text" id="searchKeyInput" name="searchKeyInput"
<!--<input type="text" id="searchKeyInput" name="searchKeyInput"
autocomplete="off"
placeholder="请输入关键字查询" class="layui-input layui-btn-sm">
<button class="layui-btn layui-btn-sm" lay-event="query">查询</button>
<button class="layui-btn layui-btn-sm" lay-event="query">查询</button>-->
</div>
<div class="layui-layout-right" style="padding-right: 20px; top:10px">
</div>