修复:优化目录导出时因数据过大打包时间太长导致请求超时的问题

master
guoxin 1 year ago
parent 57c42e70bb
commit a00d201763
  1. 40
      shandan-browser/src/main/java/com/keyware/shandan/browser/controller/SearchController.java
  2. 130
      shandan-browser/src/main/java/com/keyware/shandan/browser/service/ExportComponent.java
  3. 71
      shandan-browser/src/main/resources/static/js/browser.js

@ -11,7 +11,6 @@ import com.keyware.shandan.bianmu.service.MetadataService;
import com.keyware.shandan.browser.entity.FullSearchParam;
import com.keyware.shandan.browser.entity.PageVo;
import com.keyware.shandan.browser.entity.SearchConditionVo;
import com.keyware.shandan.browser.entity.SearchResultSort;
import com.keyware.shandan.browser.service.*;
import com.keyware.shandan.common.entity.Result;
import com.keyware.shandan.common.util.FileDownload;
@ -21,7 +20,8 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.*;
import javax.servlet.http.HttpServletResponse;
import java.io.*;
import java.io.File;
import java.io.IOException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -198,23 +198,23 @@ public class SearchController {
/**
* 数据导出-查询数据
*
* @param directoryId 目录ID
* @param metadataName 资源模糊查询
* @param condition 复杂查询条件
* @param directoryId 目录ID
* @param condition 复杂查询条件
* @return 导出ID
*/
@PostMapping("/export/query/{directoryId}")
public Result<Object> exportQuery(@PathVariable String directoryId, String metadataName, SearchConditionVo condition) {
@PostMapping("/export/directory/{directoryId}")
public Result<Object> exportQuery(@PathVariable String directoryId, SearchConditionVo condition) {
String exportId = UUIDUtil.getUUID();
ExportComponent exportComponent = new ExportComponent(exportId);
ExportComponent export = new ExportComponent(exportId);
// 查询数据
List<DirectoryResource> list = metadataSearchService.searchAllListByCondition(directoryId, condition);
exportComponent.setData(list);
exportCache.put(exportId, exportComponent);
export.setData(list);
// 打包数据
export.start();
exportCache.put(exportId, export);
Map<String, String> result = new HashMap<>();
result.put("exportId", exportId);
return Result.of(result);
return Result.of(export.getProgress());
}
/**
@ -223,14 +223,10 @@ public class SearchController {
* @param exportId 导出ID
* @return 导出ID
*/
@GetMapping("/export/package/{exportId}")
public Result<Object> exportPackage(@PathVariable String exportId) {
ExportComponent exportComponent = exportCache.get(exportId);
boolean flag = exportComponent.startPackage();
Map<String, String> result = new HashMap<>();
result.put("exportId", exportId);
return Result.of(result, flag);
@GetMapping("/export/status/{exportId}")
public Result<Object> exportStatus(@PathVariable String exportId) {
ExportComponent export = exportCache.get(exportId);
return Result.of(export.getProgress());
}
/**
@ -245,7 +241,7 @@ public class SearchController {
String path = exportCache.remove(exportId).getExportPath() + ".zip";
String downloadName = FileUtil.getName(path);
File file = new File(path);
return FileDownload.download(response, file, downloadName);
return FileDownload.download(response, file, downloadName, true);
}
}

@ -19,31 +19,62 @@ import com.keyware.shandan.frame.properties.CustomProperties;
import com.keyware.shandan.system.entity.SysFile;
import com.keyware.shandan.system.service.SysFileService;
import com.keyware.shandan.system.utils.SysSettingUtil;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.Getter;
import java.io.File;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.io.Serializable;
import java.util.*;
import java.util.stream.Collectors;
public class ExportComponent {
public class ExportComponent extends Thread {
@Getter
private final String exportId;
private Status status;
private final String fileName;
private List<DirectoryResource> dataList;
private final CustomProperties customProperties;
private final SysFileService fileService;
private final MetadataService metadataService;
@Getter
private final Progress progress;
// Progress
public ExportComponent(String exportId) {
this.exportId = exportId;
this.customProperties = AppContext.getContext().getBean(CustomProperties.class);
this.fileService = AppContext.getContext().getBean(SysFileService.class);
this.metadataService = AppContext.getContext().getBean(MetadataService.class);
this.exportId = exportId;
this.status = Status.querying;
fileName = "export_" + DateUtil.format(new Date(), "yyyyMMddHHmmss");
fileName = "目录导出_" + DateUtil.format(new Date(), "yyyyMMddHHmmss");
this.progress = new Progress(this.exportId);
}
/**
* 开始打包
*/
@Override
public void run() {
String path = getExportPath();
try {
this.progress.setTitle("正在生成临时数据文件");
dataList.forEach(resource -> {
if ("file".equals(resource.getResourceType())) {
exportFile(resource);
} else {
exportMetadata(resource);
}
this.progress.offsetIncrease();
});
this.progress.setTitle("正在打包数据文件");
ZipUtil.zip(path, path + ".zip", true);
this.progress.setTitle("数据包准备完毕,开始下载");
this.progress.offsetIncrease();
} catch (Exception e) {
throw new RuntimeException("导出数据在打包时出错", e);
} finally {
delete(path);
}
}
/**
@ -60,7 +91,6 @@ public class ExportComponent {
HttpUtil.downloadFile(fileDownloadUrl, serverFile);
} catch (Exception e) {
List<String> lines = new ArrayList<>();
lines.add("文件读取异常,请检查文件是否存在!");
String dirPath = serverFile.getParentFile().getPath();
String name = "(文件读取错误)" + serverFile.getName();
FileUtil.writeLines(lines, new File(dirPath, name), "utf-8");
@ -134,29 +164,6 @@ public class ExportComponent {
writer.close();
}
/**
* 开始打包
*/
public boolean startPackage() {
status = Status.packaging;
String path = getExportPath();
try {
dataList.forEach(resource -> {
if ("file".equals(resource.getResourceType())) {
exportFile(resource);
} else {
exportMetadata(resource);
}
});
ZipUtil.zip(path, path + ".zip", true);
} catch (Exception e) {
delete(path);
throw new RuntimeException("导出数据在打包时出错", e);
}
return true;
}
/**
* 设置需要导出的数据
*
@ -164,10 +171,8 @@ public class ExportComponent {
*/
public void setData(List<DirectoryResource> dataList) {
this.dataList = dataList;
}
public void setStatus(String status) {
this.status = Status.valueOf(status);
this.progress.setFileTotal(dataList.size());
this.progress.offsetIncrease();
}
public String getExportPath() {
@ -197,12 +202,49 @@ public class ExportComponent {
}
}
@AllArgsConstructor
private enum Status {
querying("正在查询数据"),
packaging("正在打包数据"),
downloading("正在下载数据");
@Data
static class Progress implements Serializable {
private String exportId;
private String title;
private Integer fileTotal;
private Integer offset = 0;
public Progress(String exportId) {
this.exportId = exportId;
this.title = "正在查询需要导出的数据";
}
public void setFileTotal(Integer fileTotal) {
// 加2是因为需要增加一个查询步骤和一个打包的步骤
this.fileTotal = fileTotal + 2;
}
public void offsetIncrease() {
this.offset++;
}
/**
* 获取百分比
*
* @return
*/
public int getPercentage() {
return Math.round(offset / Float.valueOf(fileTotal) * 100);
}
private final String text;
/**
* 判断是否完成
*
* @return
*/
public boolean getIsDone() {
return Objects.equals(offset, fileTotal);
}
public String getMsg() {
return "(" + getPercentage() + "%) " + this.title;
}
}
}

@ -177,27 +177,7 @@ layui.use(['layer', 'listPage', 'globalTree', 'gtable', 'form', 'element', 'drop
showErrorMsg('不允许导出根目录数据');
return false;
}
layer.confirm('该导出操作比较耗时,导出时请勿刷新页面', {title: '提示信息'}, function () {
let params = metaListTable.where;
let exportLayer = showExportMsgLayer('正在查询需要导出的数据');
Util.post(`/search/export/query/${id}`, params, false).then(res => {
layer.close(exportLayer);
if (res.flag) {
let exportId = res.data.exportId;
exportLayer = showExportMsgLayer('正在打包需要导出的数据');
Util.get(`/search/export/package/${exportId}`, {}, false).then(res => {
layer.close(exportLayer);
if (res.flag) {
parent.window.open(`${ctx}/search/export/download/${exportId}`, '_blank')
} else {
showExportMsgLayer('数据打包失败,' + res.msg);
}
})
} else {
showExportMsgLayer('数据查询失败,' + res.msg);
}
})
});
layer.confirm('该导出操作比较耗时,导出时请勿刷新页面', {title: '提示信息'}, () => exportDirectory(id, metaListTable.where));
})
// 查看按钮监听
metaListTable.addTableRowEvent('details', function (obj) {
@ -433,13 +413,48 @@ layui.use(['layer', 'listPage', 'globalTree', 'gtable', 'form', 'element', 'drop
return params;
}
function showExportMsgLayer(msg) {
return layer.msg(msg, {
icon: 16,
shade: 0.01,
time: 1000 * 60 * 30,
closeBtn: 1
});
function exportDirectory(id, params) {
let layerIndex;
showExportMsgLayer('正在查询需要导出的数据');
Util.post(`/search/export/directory/${id}`, params, false).then(({flag, data, msg}) => {
if (flag) {
showExportMsgLayer(data.msg);
queryProgress(data.exportId);
} else {
showExportMsgLayer('数据查询失败,' + msg);
}
})
function queryProgress(exportId) {
setTimeout(function () {
Util.get(`/search/export/status/${exportId}`, {}, false).then(({flag, data, msg}) => {
if (flag) {
showExportMsgLayer(data.msg);
if (data.isDone) {
layer.close(layerIndex);
parent.window.open(`${ctx}/search/export/download/${exportId}`, '_blank')
} else {
queryProgress(exportId);
}
} else {
showExportMsgLayer('数据查询失败,' + msg);
}
});
}, 1000);
}
function showExportMsgLayer(msg) {
layerIndex && layer.close(layerIndex);
layerIndex = layer.msg(msg, {
icon: 16,
shade: 0.01,
anim: -1,
isOutAnim: false,
time: 1000 * 60 * 30,
closeBtn: 1
});
}
}
/**