forked from liuzongren/compose-analysis
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
321 lines
13 KiB
321 lines
13 KiB
package com.keyware.composeanalysis.util;
|
|
|
|
import com.keyware.composeanalysis.constant.MongoDBConst;
|
|
import com.keyware.composeanalysis.solr.VersionTree;
|
|
import lombok.Data;
|
|
import lombok.extern.log4j.Log4j2;
|
|
import org.apache.commons.collections.CollectionUtils;
|
|
import org.apache.commons.lang3.StringUtils;
|
|
import org.apache.solr.client.solrj.SolrClient;
|
|
import org.apache.solr.client.solrj.SolrRequest;
|
|
import org.apache.solr.client.solrj.SolrServerException;
|
|
import org.apache.solr.client.solrj.impl.HttpSolrClient;
|
|
import org.apache.solr.client.solrj.response.QueryResponse;
|
|
import org.apache.solr.common.SolrDocument;
|
|
import org.apache.solr.common.SolrDocumentList;
|
|
import org.apache.solr.common.params.*;
|
|
import org.springframework.beans.factory.annotation.Value;
|
|
import org.springframework.stereotype.Component;
|
|
|
|
import java.io.IOException;
|
|
import java.util.*;
|
|
import java.util.function.Function;
|
|
import java.util.stream.Collectors;
|
|
|
|
|
|
/**
|
|
* 注意： 注意： 注意
|
|
* solr同步更新专用 solr地址以solr
|
|
*
|
|
* @author liuzongren
|
|
*/
|
|
@Log4j2
|
|
@Component
|
|
@Data
|
|
public class SolrUtils {
|
|
|
|
@Value("${solr.solrUrl}")
|
|
private String clientUrl;
|
|
|
|
@Value("${solr.row}")
|
|
private String ROWS;
|
|
|
|
//源码上传和解压的地址
|
|
@Value("${codeResourcePath}")
|
|
private String codeResourcePath;
|
|
|
|
private String fileAndFunSolrUrl;
|
|
|
|
//client 连接池
|
|
private Map<String, HttpSolrClient> coreClientMap = new HashMap<>();
|
|
|
|
|
|
/**
|
|
* @param coreName 表名
|
|
* @return todo 这里的client 不知道是否支持并发，后续需要测试优化
|
|
* @describe 获取solr连接
|
|
*/
|
|
public HttpSolrClient getClient(String coreName) {
|
|
if (coreClientMap.containsKey(coreName)) {
|
|
return coreClientMap.get(coreName);
|
|
} else {
|
|
HttpSolrClient solr = new HttpSolrClient.Builder(clientUrl + "" + coreName)
|
|
.withConnectionTimeout(6000000)
|
|
.withSocketTimeout(6000000)
|
|
.allowCompression(true)
|
|
.build();
|
|
coreClientMap.put(coreName, solr);
|
|
return solr;
|
|
}
|
|
}
|
|
|
|
|
|
/**
|
|
* 简单查询,指定返回字段
|
|
*
|
|
* @param searchContent 检索内容
|
|
* @param returneFields 返回字段
|
|
* @return
|
|
* @throws Exception
|
|
*/
|
|
public SolrDocumentList query(String coreName, String searchContent, String returneFields) {
|
|
SolrDocumentList docsList = null;
|
|
try {
|
|
HttpSolrClient client = getClient(coreName);
|
|
Map<String, String> map = new HashMap<String, String>();
|
|
map.put(CommonParams.Q, searchContent);
|
|
map.put(CommonParams.FL, returneFields);
|
|
map.put(CommonParams.START, "0");
|
|
map.put(CommonParams.ROWS, ROWS);
|
|
SolrParams params = new MapSolrParams(map);
|
|
QueryResponse query = client.query(params, SolrRequest.METHOD.POST);
|
|
if (!query.getResults().isEmpty()){
|
|
docsList = query.getResults();
|
|
}
|
|
} catch (SolrServerException | IOException e) {
|
|
log.error("solr查询失败,coreName:{},queryStr:{}", coreName, searchContent, e);
|
|
}
|
|
return docsList;
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
* 根据文件的MD5 去*_SourceFileBase获取当前文件的版本ID
|
|
* @param coreName solrCoreName
|
|
* @param originalFileMd5s 需要检索的文件md5
|
|
* todo 1.这里有一个极端的情况：如果查询的文件数量过多，返回值不知道会不会过大
|
|
* todo 2.这里没有查询出dirTreeId, 下一步并没有从VersionTree中查询出当前文件的具体信息,只是从versionTree查询出版本信息
|
|
* @return
|
|
*/
|
|
public Map<String,SolrDocument> batchQueryVersionIdFromSourceFileBaseBySourceMd5(String coreName, Set<String> originalFileMd5s) {
|
|
String queryStr = "sourceFileMd5:(" + StringUtils.join(originalFileMd5s, " OR ") + ")";
|
|
Map<String,SolrDocument> openFileMd5VersionIdMap = new HashMap<>();
|
|
long strtTime = System.currentTimeMillis();
|
|
log.info("batchQueryVersionIdFromSourceFileBaseBySourceMd5 queryStr:{},size:{}", queryStr, originalFileMd5s.size());
|
|
try {
|
|
HttpSolrClient client = getClient(coreName);
|
|
Map<String, String> map = new HashMap<>();
|
|
map.put(CommonParams.Q, queryStr);
|
|
map.put(CommonParams.FL, "sourceFileMd5,versionId,fullPath");
|
|
map.put(CommonParams.START, "0");
|
|
map.put(CommonParams.ROWS, String.valueOf(originalFileMd5s.size()));
|
|
//分组查询,某一个开源文件匹配一次即可
|
|
//todo 这里把匹配次数也查询出来了，貌似还是扫描了很多文档，看是否还有方法只匹配一次的
|
|
map.put(GroupParams.GROUP,"true");
|
|
map.put(GroupParams.GROUP_FIELD, "sourceFileMd5");
|
|
map.put(GroupParams.GROUP_LIMIT,"1");
|
|
map.put(GroupParams.GROUP_FORMAT,"simple");
|
|
SolrParams params = new MapSolrParams(map);
|
|
QueryResponse query = client.query(params, SolrRequest.METHOD.POST);
|
|
if (query.getGroupResponse().getValues().size() > 0){
|
|
//拿到sourceFileMd5分组数据
|
|
SolrDocumentList result = query.getGroupResponse().getValues().get(0).getValues().get(0).getResult();
|
|
openFileMd5VersionIdMap = result.stream().collect(Collectors.toMap(doc -> (String) doc.get("sourceFileMd5"), Function.identity()));
|
|
}
|
|
} catch (Exception e) {
|
|
log.error("solr查询失败,coreName:{},queryStr:{}", coreName, queryStr, e);
|
|
}
|
|
log.info("batchQueryVersionIdFromSourceFileBaseBySourceMd5 cost:{}s", (System.currentTimeMillis()-strtTime) / 1000);
|
|
return openFileMd5VersionIdMap;
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
/**
|
|
* 简单查询,指定返回字段
|
|
*
|
|
* @param searchContent 检索内容
|
|
* @param returneFields 返回字段
|
|
* @return
|
|
* @throws Exception
|
|
*/
|
|
public SolrDocument queryOne(String coreName, String searchContent, String returneFields) {
|
|
SolrDocument result = null;
|
|
try {
|
|
HttpSolrClient client = getClient(coreName);
|
|
Map<String, String> map = new HashMap<String, String>();
|
|
map.put(CommonParams.Q, searchContent);
|
|
map.put(CommonParams.FL, returneFields);
|
|
map.put(CommonParams.START, "0");
|
|
map.put(CommonParams.ROWS, "1");
|
|
SolrParams params = new MapSolrParams(map);
|
|
QueryResponse query = client.query(params, SolrRequest.METHOD.POST);
|
|
SolrDocumentList resp = query.getResults();
|
|
if (CollectionUtils.isNotEmpty(resp)) {
|
|
return resp.get(0);
|
|
}
|
|
} catch (SolrServerException | IOException e) {
|
|
log.error("查询solr失败!,coreName:{},queryStr:{}",coreName , searchContent, e);
|
|
}
|
|
return result;
|
|
}
|
|
|
|
|
|
/**
|
|
* 查询 versionTree
|
|
*
|
|
* @param searchContent 检索内容
|
|
* @return
|
|
* @throws Exception
|
|
*/
|
|
public VersionTree queryVersionTree(String searchContent) {
|
|
String returneFields = "proId,proName,versionName,downUrl,licenseType,dirTree";
|
|
VersionTree results = null;
|
|
try {
|
|
HttpSolrClient client = getClient(MongoDBConst.VERSION_TREE);
|
|
Map<String, String> map = new HashMap<String, String>();
|
|
map.put(CommonParams.Q, searchContent);
|
|
map.put(CommonParams.FL, returneFields);
|
|
map.put(CommonParams.START, "0");
|
|
map.put(CommonParams.ROWS, "1");
|
|
SolrParams params = new MapSolrParams(map);
|
|
QueryResponse query = client.query(params, SolrRequest.METHOD.POST);
|
|
SolrDocumentList response = query.getResults();
|
|
if (!response.isEmpty()) {
|
|
//转化对象
|
|
results = BeanUtil.domToVersionTree(response.get(0));
|
|
}
|
|
} catch (SolrServerException | IOException e) {
|
|
log.error("查询solr失败!,queryStr:{}" , searchContent, e);
|
|
}
|
|
return results;
|
|
}
|
|
|
|
|
|
/**
|
|
* 查询 versionTree
|
|
*
|
|
* @param versionId 版本ID
|
|
* @return
|
|
*/
|
|
public VersionTree queryVersionTreeByVersionId(String versionId) {
|
|
String returneFields = "proId,proName,versionName,downUrl,licenseType,dirTree";
|
|
String queryStr = "versionId:"+ versionId;
|
|
VersionTree results = null;
|
|
try {
|
|
HttpSolrClient client = getClient(MongoDBConst.VERSION_TREE);
|
|
Map<String, String> map = new HashMap<String, String>();
|
|
map.put(CommonParams.Q, queryStr);
|
|
map.put(CommonParams.FL, returneFields);
|
|
map.put(CommonParams.START, "0");
|
|
map.put(CommonParams.ROWS, "1");
|
|
SolrParams params = new MapSolrParams(map);
|
|
QueryResponse query = client.query(params, SolrRequest.METHOD.POST);
|
|
SolrDocumentList response = query.getResults();
|
|
//转化对象
|
|
if (!response.isEmpty()){
|
|
results = BeanUtil.domToVersionTree(response.get(0));
|
|
}else {
|
|
log.error("根据版本ID查询VersionTree失败,versionId:{}" , versionId);
|
|
}
|
|
} catch (SolrServerException | IOException e) {
|
|
log.error("查询solr失败!,queryStr:{}" , queryStr, e);
|
|
}
|
|
return results;
|
|
}
|
|
|
|
|
|
/**
|
|
* 查询 version 的具体信息
|
|
*
|
|
* @param versionId versionId
|
|
* @return
|
|
* @throws Exception
|
|
*/
|
|
public VersionTree queryVersionInfoByVersionId(Object versionId) {
|
|
String returneFields = "proId,proName,versionName,downUrl,licenseType";
|
|
VersionTree result = new VersionTree();
|
|
try {
|
|
HttpSolrClient client = getClient(MongoDBConst.VERSION_TREE);
|
|
Map<String, String> map = new HashMap<String, String>();
|
|
map.put(CommonParams.Q, "versionId:" + versionId);
|
|
map.put(CommonParams.FL, returneFields);
|
|
map.put(CommonParams.START, "0");
|
|
map.put(CommonParams.ROWS, "1");
|
|
SolrParams params = new MapSolrParams(map);
|
|
QueryResponse query = client.query(params, SolrRequest.METHOD.POST);
|
|
SolrDocumentList response = query.getResults();
|
|
//转化对象
|
|
if (CollectionUtils.isNotEmpty(response)) {
|
|
cn.hutool.core.bean.BeanUtil.copyProperties(response.get(0), result);
|
|
result.setLicenseType(response.get(0).get("licenseType") == null ? "" : response.get(0).get("licenseType").toString());
|
|
}else {
|
|
log.error("根据版本ID查询版本信息失败,versionId:{}" , versionId);
|
|
}
|
|
} catch (SolrServerException | IOException e) {
|
|
log.error("查询solr失败!,queryStr:{}" , versionId, e);
|
|
}
|
|
return result;
|
|
}
|
|
|
|
|
|
/**
|
|
* 批量查询 version 的具体信息
|
|
*
|
|
* @param versionIds versionIds
|
|
* @return
|
|
* @throws Exception
|
|
*/
|
|
public List<VersionTree> queryBatchVersionInfoByVersionIds(Collection<String> versionIds) {
|
|
List<VersionTree> results = new ArrayList<>();
|
|
if (CollectionUtils.isEmpty(versionIds)) {
|
|
return results;
|
|
}
|
|
//去一波重
|
|
versionIds = versionIds.stream().collect(Collectors.toSet());
|
|
String queryStr = "versionId:(" + StringUtils.join(versionIds, " OR ") + ")";
|
|
String returneFields = "versionId,proId,proName,versionName,downUrl,licenseType";
|
|
try {
|
|
HttpSolrClient client = getClient(MongoDBConst.VERSION_TREE);
|
|
Map<String, String> map = new HashMap<String, String>();
|
|
map.put(CommonParams.Q, queryStr);
|
|
map.put(CommonParams.FL, returneFields);
|
|
map.put(CommonParams.START, "0");
|
|
map.put(CommonParams.ROWS,String.valueOf(versionIds.size()));
|
|
SolrParams params = new MapSolrParams(map);
|
|
QueryResponse query = client.query(params, SolrRequest.METHOD.POST);
|
|
SolrDocumentList response = query.getResults();
|
|
//转化对象
|
|
if (!response.isEmpty()) {
|
|
for (int i = 0; i < response.size(); i++) {
|
|
VersionTree versionTree = new VersionTree();
|
|
try {
|
|
cn.hutool.core.bean.BeanUtil.copyProperties(response.get(i), versionTree);
|
|
versionTree.setLicenseType(response.get(i).get("licenseType") == null ? "" : response.get(i).get("licenseType").toString());
|
|
results.add(versionTree);
|
|
} catch (Exception e) {
|
|
e.printStackTrace();
|
|
}
|
|
}
|
|
}
|
|
} catch (SolrServerException | IOException e) {
|
|
log.error("查询solr失败!,queryStr:{}" , queryStr, e);
|
|
}
|
|
return results;
|
|
}
|
|
|
|
}
|
|
|