修复:优化目录导出时因数据过大打包时间太长导致请求超时的问题

master
guoxin 1 year ago
parent 57c42e70bb
commit a00d201763
  1. 36
      shandan-browser/src/main/java/com/keyware/shandan/browser/controller/SearchController.java
  2. 130
      shandan-browser/src/main/java/com/keyware/shandan/browser/service/ExportComponent.java
  3. 59
      shandan-browser/src/main/resources/static/js/browser.js

@ -11,7 +11,6 @@ import com.keyware.shandan.bianmu.service.MetadataService;
import com.keyware.shandan.browser.entity.FullSearchParam; import com.keyware.shandan.browser.entity.FullSearchParam;
import com.keyware.shandan.browser.entity.PageVo; import com.keyware.shandan.browser.entity.PageVo;
import com.keyware.shandan.browser.entity.SearchConditionVo; import com.keyware.shandan.browser.entity.SearchConditionVo;
import com.keyware.shandan.browser.entity.SearchResultSort;
import com.keyware.shandan.browser.service.*; import com.keyware.shandan.browser.service.*;
import com.keyware.shandan.common.entity.Result; import com.keyware.shandan.common.entity.Result;
import com.keyware.shandan.common.util.FileDownload; import com.keyware.shandan.common.util.FileDownload;
@ -21,7 +20,8 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.*; import org.springframework.web.bind.annotation.*;
import javax.servlet.http.HttpServletResponse; import javax.servlet.http.HttpServletResponse;
import java.io.*; import java.io.File;
import java.io.IOException;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
@ -199,22 +199,22 @@ public class SearchController {
* 数据导出-查询数据 * 数据导出-查询数据
* *
* @param directoryId 目录ID * @param directoryId 目录ID
* @param metadataName 资源模糊查询
* @param condition 复杂查询条件 * @param condition 复杂查询条件
* @return 导出ID * @return 导出ID
*/ */
@PostMapping("/export/query/{directoryId}") @PostMapping("/export/directory/{directoryId}")
public Result<Object> exportQuery(@PathVariable String directoryId, String metadataName, SearchConditionVo condition) { public Result<Object> exportQuery(@PathVariable String directoryId, SearchConditionVo condition) {
String exportId = UUIDUtil.getUUID(); String exportId = UUIDUtil.getUUID();
ExportComponent exportComponent = new ExportComponent(exportId); ExportComponent export = new ExportComponent(exportId);
// 查询数据
List<DirectoryResource> list = metadataSearchService.searchAllListByCondition(directoryId, condition); List<DirectoryResource> list = metadataSearchService.searchAllListByCondition(directoryId, condition);
exportComponent.setData(list); export.setData(list);
exportCache.put(exportId, exportComponent); // 打包数据
export.start();
exportCache.put(exportId, export);
Map<String, String> result = new HashMap<>(); return Result.of(export.getProgress());
result.put("exportId", exportId);
return Result.of(result);
} }
/** /**
@ -223,14 +223,10 @@ public class SearchController {
* @param exportId 导出ID * @param exportId 导出ID
* @return 导出ID * @return 导出ID
*/ */
@GetMapping("/export/package/{exportId}") @GetMapping("/export/status/{exportId}")
public Result<Object> exportPackage(@PathVariable String exportId) { public Result<Object> exportStatus(@PathVariable String exportId) {
ExportComponent exportComponent = exportCache.get(exportId); ExportComponent export = exportCache.get(exportId);
boolean flag = exportComponent.startPackage(); return Result.of(export.getProgress());
Map<String, String> result = new HashMap<>();
result.put("exportId", exportId);
return Result.of(result, flag);
} }
/** /**
@ -245,7 +241,7 @@ public class SearchController {
String path = exportCache.remove(exportId).getExportPath() + ".zip"; String path = exportCache.remove(exportId).getExportPath() + ".zip";
String downloadName = FileUtil.getName(path); String downloadName = FileUtil.getName(path);
File file = new File(path); File file = new File(path);
return FileDownload.download(response, file, downloadName); return FileDownload.download(response, file, downloadName, true);
} }
} }

@ -19,31 +19,62 @@ import com.keyware.shandan.frame.properties.CustomProperties;
import com.keyware.shandan.system.entity.SysFile; import com.keyware.shandan.system.entity.SysFile;
import com.keyware.shandan.system.service.SysFileService; import com.keyware.shandan.system.service.SysFileService;
import com.keyware.shandan.system.utils.SysSettingUtil; import com.keyware.shandan.system.utils.SysSettingUtil;
import lombok.AllArgsConstructor; import lombok.Data;
import lombok.Getter;
import java.io.File; import java.io.File;
import java.util.ArrayList; import java.io.Serializable;
import java.util.Date; import java.util.*;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors; import java.util.stream.Collectors;
public class ExportComponent { public class ExportComponent extends Thread {
@Getter
private final String exportId; private final String exportId;
private Status status;
private final String fileName; private final String fileName;
private List<DirectoryResource> dataList; private List<DirectoryResource> dataList;
private final CustomProperties customProperties; private final CustomProperties customProperties;
private final SysFileService fileService; private final SysFileService fileService;
private final MetadataService metadataService; private final MetadataService metadataService;
@Getter
private final Progress progress;
// Progress
public ExportComponent(String exportId) { public ExportComponent(String exportId) {
this.exportId = exportId;
this.customProperties = AppContext.getContext().getBean(CustomProperties.class); this.customProperties = AppContext.getContext().getBean(CustomProperties.class);
this.fileService = AppContext.getContext().getBean(SysFileService.class); this.fileService = AppContext.getContext().getBean(SysFileService.class);
this.metadataService = AppContext.getContext().getBean(MetadataService.class); this.metadataService = AppContext.getContext().getBean(MetadataService.class);
this.exportId = exportId; fileName = "目录导出_" + DateUtil.format(new Date(), "yyyyMMddHHmmss");
this.status = Status.querying; this.progress = new Progress(this.exportId);
fileName = "export_" + DateUtil.format(new Date(), "yyyyMMddHHmmss"); }
/**
* 开始打包
*/
@Override
public void run() {
String path = getExportPath();
try {
this.progress.setTitle("正在生成临时数据文件");
dataList.forEach(resource -> {
if ("file".equals(resource.getResourceType())) {
exportFile(resource);
} else {
exportMetadata(resource);
}
this.progress.offsetIncrease();
});
this.progress.setTitle("正在打包数据文件");
ZipUtil.zip(path, path + ".zip", true);
this.progress.setTitle("数据包准备完毕,开始下载");
this.progress.offsetIncrease();
} catch (Exception e) {
throw new RuntimeException("导出数据在打包时出错", e);
} finally {
delete(path);
}
} }
/** /**
@ -60,7 +91,6 @@ public class ExportComponent {
HttpUtil.downloadFile(fileDownloadUrl, serverFile); HttpUtil.downloadFile(fileDownloadUrl, serverFile);
} catch (Exception e) { } catch (Exception e) {
List<String> lines = new ArrayList<>(); List<String> lines = new ArrayList<>();
lines.add("文件读取异常,请检查文件是否存在!");
String dirPath = serverFile.getParentFile().getPath(); String dirPath = serverFile.getParentFile().getPath();
String name = "(文件读取错误)" + serverFile.getName(); String name = "(文件读取错误)" + serverFile.getName();
FileUtil.writeLines(lines, new File(dirPath, name), "utf-8"); FileUtil.writeLines(lines, new File(dirPath, name), "utf-8");
@ -134,29 +164,6 @@ public class ExportComponent {
writer.close(); writer.close();
} }
/**
* 开始打包
*/
public boolean startPackage() {
status = Status.packaging;
String path = getExportPath();
try {
dataList.forEach(resource -> {
if ("file".equals(resource.getResourceType())) {
exportFile(resource);
} else {
exportMetadata(resource);
}
});
ZipUtil.zip(path, path + ".zip", true);
} catch (Exception e) {
delete(path);
throw new RuntimeException("导出数据在打包时出错", e);
}
return true;
}
/** /**
* 设置需要导出的数据 * 设置需要导出的数据
* *
@ -164,10 +171,8 @@ public class ExportComponent {
*/ */
public void setData(List<DirectoryResource> dataList) { public void setData(List<DirectoryResource> dataList) {
this.dataList = dataList; this.dataList = dataList;
} this.progress.setFileTotal(dataList.size());
this.progress.offsetIncrease();
public void setStatus(String status) {
this.status = Status.valueOf(status);
} }
public String getExportPath() { public String getExportPath() {
@ -197,12 +202,49 @@ public class ExportComponent {
} }
} }
@AllArgsConstructor @Data
private enum Status { static class Progress implements Serializable {
querying("正在查询数据"),
packaging("正在打包数据"), private String exportId;
downloading("正在下载数据"); private String title;
private Integer fileTotal;
private Integer offset = 0;
public Progress(String exportId) {
this.exportId = exportId;
this.title = "正在查询需要导出的数据";
}
public void setFileTotal(Integer fileTotal) {
// 加2是因为需要增加一个查询步骤和一个打包的步骤
this.fileTotal = fileTotal + 2;
}
public void offsetIncrease() {
this.offset++;
}
/**
* 获取百分比
*
* @return
*/
public int getPercentage() {
return Math.round(offset / Float.valueOf(fileTotal) * 100);
}
/**
* 判断是否完成
*
* @return
*/
public boolean getIsDone() {
return Objects.equals(offset, fileTotal);
}
private final String text; public String getMsg() {
return "(" + getPercentage() + "%) " + this.title;
}
} }
} }

@ -177,27 +177,7 @@ layui.use(['layer', 'listPage', 'globalTree', 'gtable', 'form', 'element', 'drop
showErrorMsg('不允许导出根目录数据'); showErrorMsg('不允许导出根目录数据');
return false; return false;
} }
layer.confirm('该导出操作比较耗时,导出时请勿刷新页面', {title: '提示信息'}, function () { layer.confirm('该导出操作比较耗时,导出时请勿刷新页面', {title: '提示信息'}, () => exportDirectory(id, metaListTable.where));
let params = metaListTable.where;
let exportLayer = showExportMsgLayer('正在查询需要导出的数据');
Util.post(`/search/export/query/${id}`, params, false).then(res => {
layer.close(exportLayer);
if (res.flag) {
let exportId = res.data.exportId;
exportLayer = showExportMsgLayer('正在打包需要导出的数据');
Util.get(`/search/export/package/${exportId}`, {}, false).then(res => {
layer.close(exportLayer);
if (res.flag) {
parent.window.open(`${ctx}/search/export/download/${exportId}`, '_blank')
} else {
showExportMsgLayer('数据打包失败,' + res.msg);
}
})
} else {
showExportMsgLayer('数据查询失败,' + res.msg);
}
})
});
}) })
// 查看按钮监听 // 查看按钮监听
metaListTable.addTableRowEvent('details', function (obj) { metaListTable.addTableRowEvent('details', function (obj) {
@ -433,14 +413,49 @@ layui.use(['layer', 'listPage', 'globalTree', 'gtable', 'form', 'element', 'drop
return params; return params;
} }
function exportDirectory(id, params) {
let layerIndex;
showExportMsgLayer('正在查询需要导出的数据');
Util.post(`/search/export/directory/${id}`, params, false).then(({flag, data, msg}) => {
if (flag) {
showExportMsgLayer(data.msg);
queryProgress(data.exportId);
} else {
showExportMsgLayer('数据查询失败,' + msg);
}
})
function queryProgress(exportId) {
setTimeout(function () {
Util.get(`/search/export/status/${exportId}`, {}, false).then(({flag, data, msg}) => {
if (flag) {
showExportMsgLayer(data.msg);
if (data.isDone) {
layer.close(layerIndex);
parent.window.open(`${ctx}/search/export/download/${exportId}`, '_blank')
} else {
queryProgress(exportId);
}
} else {
showExportMsgLayer('数据查询失败,' + msg);
}
});
}, 1000);
}
function showExportMsgLayer(msg) { function showExportMsgLayer(msg) {
return layer.msg(msg, { layerIndex && layer.close(layerIndex);
layerIndex = layer.msg(msg, {
icon: 16, icon: 16,
shade: 0.01, shade: 0.01,
anim: -1,
isOutAnim: false,
time: 1000 * 60 * 30, time: 1000 * 60 * 30,
closeBtn: 1 closeBtn: 1
}); });
} }
}
/** /**
* 初始化文件全文检索查询列表 * 初始化文件全文检索查询列表