guoxin
/
shandan
Archived
2
0
Fork 0
Code Issues Pull Requests Packages Projects 1 Releases Wiki Activity

修复：优化目录导出时因数据过大打包时间太长导致请求超时的问题

Browse Source
master
guoxin 1 year ago
parent 57c42e70bb
commit a00d201763
3 changed files with 147 additions and 94 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 36
      shandan-browser/src/main/java/com/keyware/shandan/browser/controller/SearchController.java
  2. 130
      shandan-browser/src/main/java/com/keyware/shandan/browser/service/ExportComponent.java
  3. 59
      shandan-browser/src/main/resources/static/js/browser.js

36
shandan-browser/src/main/java/com/keyware/shandan/browser/controller/SearchController.java
Unescape View File

@ -11,7 +11,6 @@ import com.keyware.shandan.bianmu.service.MetadataService;
import com.keyware.shandan.browser.entity.FullSearchParam;
import com.keyware.shandan.browser.entity.PageVo;
import com.keyware.shandan.browser.entity.SearchConditionVo;
import com.keyware.shandan.browser.entity.SearchResultSort;
import com.keyware.shandan.browser.service.*;
import com.keyware.shandan.common.entity.Result;
import com.keyware.shandan.common.util.FileDownload;
@ -21,7 +20,8 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.*;
import javax.servlet.http.HttpServletResponse;
import java.io.*;
import java.io.File;
import java.io.IOException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -199,22 +199,22 @@ public class SearchController {
* 数据导出-查询数据
*
* @param directoryId 目录ID
* @param metadataName 资源模糊查询
* @param condition 复杂查询条件
* @return 导出ID
*/
@PostMapping("/export/query/{directoryId}")
public Result<Object> exportQuery(@PathVariable String directoryId, String metadataName, SearchConditionVo condition) {
@PostMapping("/export/directory/{directoryId}")
public Result<Object> exportQuery(@PathVariable String directoryId, SearchConditionVo condition) {
String exportId = UUIDUtil.getUUID();
ExportComponent exportComponent = new ExportComponent(exportId);
ExportComponent export = new ExportComponent(exportId);
// 查询数据
List<DirectoryResource> list = metadataSearchService.searchAllListByCondition(directoryId, condition);
exportComponent.setData(list);
exportCache.put(exportId, exportComponent);
export.setData(list);
// 打包数据
export.start();
exportCache.put(exportId, export);
Map<String, String> result = new HashMap<>();
result.put("exportId", exportId);
return Result.of(result);
return Result.of(export.getProgress());
}
/**
@ -223,14 +223,10 @@ public class SearchController {
* @param exportId 导出ID
* @return 导出ID
*/
@GetMapping("/export/package/{exportId}")
public Result<Object> exportPackage(@PathVariable String exportId) {
ExportComponent exportComponent = exportCache.get(exportId);
boolean flag = exportComponent.startPackage();
Map<String, String> result = new HashMap<>();
result.put("exportId", exportId);
return Result.of(result, flag);
@GetMapping("/export/status/{exportId}")
public Result<Object> exportStatus(@PathVariable String exportId) {
ExportComponent export = exportCache.get(exportId);
return Result.of(export.getProgress());
}
/**
@ -245,7 +241,7 @@ public class SearchController {
String path = exportCache.remove(exportId).getExportPath() + ".zip";
String downloadName = FileUtil.getName(path);
File file = new File(path);
return FileDownload.download(response, file, downloadName);
return FileDownload.download(response, file, downloadName, true);
}
}

130
shandan-browser/src/main/java/com/keyware/shandan/browser/service/ExportComponent.java
Unescape View File

@ -19,31 +19,62 @@ import com.keyware.shandan.frame.properties.CustomProperties;
import com.keyware.shandan.system.entity.SysFile;
import com.keyware.shandan.system.service.SysFileService;
import com.keyware.shandan.system.utils.SysSettingUtil;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.Getter;
import java.io.File;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.io.Serializable;
import java.util.*;
import java.util.stream.Collectors;
public class ExportComponent {
public class ExportComponent extends Thread {
@Getter
private final String exportId;
private Status status;
private final String fileName;
private List<DirectoryResource> dataList;
private final CustomProperties customProperties;
private final SysFileService fileService;
private final MetadataService metadataService;
@Getter
private final Progress progress;
// Progress
public ExportComponent(String exportId) {
this.exportId = exportId;
this.customProperties = AppContext.getContext().getBean(CustomProperties.class);
this.fileService = AppContext.getContext().getBean(SysFileService.class);
this.metadataService = AppContext.getContext().getBean(MetadataService.class);
this.exportId = exportId;
this.status = Status.querying;
fileName = "export_" + DateUtil.format(new Date(), "yyyyMMddHHmmss");
fileName = "目录导出_" + DateUtil.format(new Date(), "yyyyMMddHHmmss");
this.progress = new Progress(this.exportId);
}
/**
* 开始打包
*/
@Override
public void run() {
String path = getExportPath();
try {
this.progress.setTitle("正在生成临时数据文件");
dataList.forEach(resource -> {
if ("file".equals(resource.getResourceType())) {
exportFile(resource);
} else {
exportMetadata(resource);
}
this.progress.offsetIncrease();
});
this.progress.setTitle("正在打包数据文件");
ZipUtil.zip(path, path + ".zip", true);
this.progress.setTitle("数据包准备完毕，开始下载");
this.progress.offsetIncrease();
} catch (Exception e) {
throw new RuntimeException("导出数据在打包时出错", e);
} finally {
delete(path);
}
}
/**
@ -60,7 +91,6 @@ public class ExportComponent {
HttpUtil.downloadFile(fileDownloadUrl, serverFile);
} catch (Exception e) {
List<String> lines = new ArrayList<>();
lines.add("文件读取异常，请检查文件是否存在！");
String dirPath = serverFile.getParentFile().getPath();
String name = "(文件读取错误)" + serverFile.getName();
FileUtil.writeLines(lines, new File(dirPath, name), "utf-8");
@ -134,29 +164,6 @@ public class ExportComponent {
writer.close();
}
/**
* 开始打包
*/
public boolean startPackage() {
status = Status.packaging;
String path = getExportPath();
try {
dataList.forEach(resource -> {
if ("file".equals(resource.getResourceType())) {
exportFile(resource);
} else {
exportMetadata(resource);
}
});
ZipUtil.zip(path, path + ".zip", true);
} catch (Exception e) {
delete(path);
throw new RuntimeException("导出数据在打包时出错", e);
}
return true;
}
/**
* 设置需要导出的数据
*
@ -164,10 +171,8 @@ public class ExportComponent {
*/
public void setData(List<DirectoryResource> dataList) {
this.dataList = dataList;
}
public void setStatus(String status) {
this.status = Status.valueOf(status);
this.progress.setFileTotal(dataList.size());
this.progress.offsetIncrease();
}
public String getExportPath() {
@ -197,12 +202,49 @@ public class ExportComponent {
}
}
@AllArgsConstructor
private enum Status {
querying("正在查询数据"),
packaging("正在打包数据"),
downloading("正在下载数据");
@Data
static class Progress implements Serializable {
private String exportId;
private String title;
private Integer fileTotal;
private Integer offset = 0;
public Progress(String exportId) {
this.exportId = exportId;
this.title = "正在查询需要导出的数据";
}
public void setFileTotal(Integer fileTotal) {
// 加2是因为需要增加一个查询步骤和一个打包的步骤
this.fileTotal = fileTotal + 2;
}
public void offsetIncrease() {
this.offset++;
}
/**
* 获取百分比
*
* @return
*/
public int getPercentage() {
return Math.round(offset / Float.valueOf(fileTotal) * 100);
}
/**
* 判断是否完成
*
* @return
*/
public boolean getIsDone() {
return Objects.equals(offset, fileTotal);
}
private final String text;
public String getMsg() {
return "(" + getPercentage() + "%) " + this.title;
}
}
}

59
shandan-browser/src/main/resources/static/js/browser.js
Unescape View File

@ -177,27 +177,7 @@ layui.use(['layer', 'listPage', 'globalTree', 'gtable', 'form', 'element', 'drop
showErrorMsg('不允许导出根目录数据');
return false;
}
layer.confirm('该导出操作比较耗时，导出时请勿刷新页面', {title: '提示信息'}, function () {
let params = metaListTable.where;
let exportLayer = showExportMsgLayer('正在查询需要导出的数据');
Util.post(`/search/export/query/${id}`, params, false).then(res => {
layer.close(exportLayer);
if (res.flag) {
let exportId = res.data.exportId;
exportLayer = showExportMsgLayer('正在打包需要导出的数据');
Util.get(`/search/export/package/${exportId}`, {}, false).then(res => {
layer.close(exportLayer);
if (res.flag) {
parent.window.open(`${ctx}/search/export/download/${exportId}`, '_blank')
} else {
showExportMsgLayer('数据打包失败，' + res.msg);
}
})
} else {
showExportMsgLayer('数据查询失败，' + res.msg);
}
})
});
layer.confirm('该导出操作比较耗时，导出时请勿刷新页面', {title: '提示信息'}, () => exportDirectory(id, metaListTable.where));
})
// 查看按钮监听
metaListTable.addTableRowEvent('details', function (obj) {
@ -433,14 +413,49 @@ layui.use(['layer', 'listPage', 'globalTree', 'gtable', 'form', 'element', 'drop
return params;
}
function exportDirectory(id, params) {
let layerIndex;
showExportMsgLayer('正在查询需要导出的数据');
Util.post(`/search/export/directory/${id}`, params, false).then(({flag, data, msg}) => {
if (flag) {
showExportMsgLayer(data.msg);
queryProgress(data.exportId);
} else {
showExportMsgLayer('数据查询失败，' + msg);
}
})
function queryProgress(exportId) {
setTimeout(function () {
Util.get(`/search/export/status/${exportId}`, {}, false).then(({flag, data, msg}) => {
if (flag) {
showExportMsgLayer(data.msg);
if (data.isDone) {
layer.close(layerIndex);
parent.window.open(`${ctx}/search/export/download/${exportId}`, '_blank')
} else {
queryProgress(exportId);
}
} else {
showExportMsgLayer('数据查询失败，' + msg);
}
});
}, 1000);
}
function showExportMsgLayer(msg) {
return layer.msg(msg, {
layerIndex && layer.close(layerIndex);
layerIndex = layer.msg(msg, {
icon: 16,
shade: 0.01,
anim: -1,
isOutAnim: false,
time: 1000 * 60 * 30,
closeBtn: 1
});
}
}
/**
* 初始化文件全文检索查询列表