package com.keyware.composeanalysis.util; import com.keyware.composeanalysis.constant.MongoDBConst; import com.keyware.composeanalysis.solr.VersionTree; import lombok.Data; import lombok.extern.log4j.Log4j2; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrDocumentList; import org.apache.solr.common.params.*; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import java.io.IOException; import java.util.*; import java.util.function.Function; import java.util.stream.Collectors; /** * 注意: 注意: 注意 * solr同步更新专用 solr地址以solr * * @author liuzongren */ @Log4j2 @Component @Data public class SolrUtils { @Value("${solr.solrUrl}") private String clientUrl; @Value("${solr.row}") private String ROWS; //源码上传和解压的地址 @Value("${codeResourcePath}") private String codeResourcePath; private String fileAndFunSolrUrl; //client 连接池 private Map coreClientMap = new HashMap<>(); /** * @param coreName 表名 * @return todo 这里的client 不知道是否支持并发,后续需要测试优化 * @describe 获取solr连接 */ public HttpSolrClient getClient(String coreName) { if (coreClientMap.containsKey(coreName)) { return coreClientMap.get(coreName); } else { HttpSolrClient solr = new HttpSolrClient.Builder(clientUrl + "" + coreName) .withConnectionTimeout(6000000) .withSocketTimeout(6000000) .allowCompression(true) .build(); coreClientMap.put(coreName, solr); return solr; } } /** * 简单查询,指定返回字段 * * @param searchContent 检索内容 * @param returneFields 返回字段 * @return * @throws Exception */ public SolrDocumentList query(String coreName, String searchContent, String returneFields) { SolrDocumentList docsList = null; try { HttpSolrClient client = getClient(coreName); Map map = new HashMap(); map.put(CommonParams.Q, searchContent); map.put(CommonParams.FL, returneFields); map.put(CommonParams.START, "0"); map.put(CommonParams.ROWS, ROWS); SolrParams params = new MapSolrParams(map); QueryResponse query = client.query(params, SolrRequest.METHOD.POST); if (!query.getResults().isEmpty()){ docsList = query.getResults(); } } catch (SolrServerException | IOException e) { log.error("solr查询失败,coreName:{},queryStr:{}", coreName, searchContent, e); } return docsList; } /** * 根据文件的MD5 去*_SourceFileBase获取当前文件的版本ID * @param coreName solrCoreName * @param originalFileMd5s 需要检索的文件md5 * todo 1.这里有一个极端的情况:如果查询的文件数量过多,返回值不知道会不会过大 * todo 2.这里没有查询出dirTreeId, 下一步并没有从VersionTree中查询出当前文件的具体信息,只是从versionTree查询出版本信息 * @return */ public Map batchQueryVersionIdFromSourceFileBaseBySourceMd5(String coreName, Set originalFileMd5s) { String queryStr = "sourceFileMd5:(" + StringUtils.join(originalFileMd5s, " OR ") + ")"; Map openFileMd5VersionIdMap = new HashMap<>(); long strtTime = System.currentTimeMillis(); log.info("batchQueryVersionIdFromSourceFileBaseBySourceMd5 queryStr:{},size:{}", queryStr, originalFileMd5s.size()); try { HttpSolrClient client = getClient(coreName); Map map = new HashMap<>(); map.put(CommonParams.Q, queryStr); map.put(CommonParams.FL, "sourceFileMd5,versionId,fullPath"); map.put(CommonParams.START, "0"); map.put(CommonParams.ROWS, String.valueOf(originalFileMd5s.size())); //分组查询,某一个开源文件匹配一次即可 //todo 这里把匹配次数也查询出来了,貌似还是扫描了很多文档,看是否还有方法只匹配一次的 map.put(GroupParams.GROUP,"true"); map.put(GroupParams.GROUP_FIELD, "sourceFileMd5"); map.put(GroupParams.GROUP_LIMIT,"1"); map.put(GroupParams.GROUP_FORMAT,"simple"); SolrParams params = new MapSolrParams(map); QueryResponse query = client.query(params, SolrRequest.METHOD.POST); if (query.getGroupResponse().getValues().size() > 0){ //拿到sourceFileMd5分组数据 SolrDocumentList result = query.getGroupResponse().getValues().get(0).getValues().get(0).getResult(); openFileMd5VersionIdMap = result.stream().collect(Collectors.toMap(doc -> (String) doc.get("sourceFileMd5"), Function.identity())); } } catch (Exception e) { log.error("solr查询失败,coreName:{},queryStr:{}", coreName, queryStr, e); } log.info("batchQueryVersionIdFromSourceFileBaseBySourceMd5 cost:{}s", (System.currentTimeMillis()-strtTime) / 1000); return openFileMd5VersionIdMap; } /** * 简单查询,指定返回字段 * * @param searchContent 检索内容 * @param returneFields 返回字段 * @return * @throws Exception */ public SolrDocument queryOne(String coreName, String searchContent, String returneFields) { SolrDocument result = null; try { HttpSolrClient client = getClient(coreName); Map map = new HashMap(); map.put(CommonParams.Q, searchContent); map.put(CommonParams.FL, returneFields); map.put(CommonParams.START, "0"); map.put(CommonParams.ROWS, "1"); SolrParams params = new MapSolrParams(map); QueryResponse query = client.query(params, SolrRequest.METHOD.POST); SolrDocumentList resp = query.getResults(); if (CollectionUtils.isNotEmpty(resp)) { return resp.get(0); } } catch (SolrServerException | IOException e) { log.error("查询solr失败!,coreName:{},queryStr:{}",coreName , searchContent, e); } return result; } /** * 查询 versionTree * * @param searchContent 检索内容 * @return * @throws Exception */ public VersionTree queryVersionTree(String searchContent) { String returneFields = "proId,proName,versionName,downUrl,licenseType,dirTree"; VersionTree results = null; try { HttpSolrClient client = getClient(MongoDBConst.VERSION_TREE); Map map = new HashMap(); map.put(CommonParams.Q, searchContent); map.put(CommonParams.FL, returneFields); map.put(CommonParams.START, "0"); map.put(CommonParams.ROWS, "1"); SolrParams params = new MapSolrParams(map); QueryResponse query = client.query(params, SolrRequest.METHOD.POST); SolrDocumentList response = query.getResults(); if (!response.isEmpty()) { //转化对象 results = BeanUtil.domToVersionTree(response.get(0)); } } catch (SolrServerException | IOException e) { log.error("查询solr失败!,queryStr:{}" , searchContent, e); } return results; } /** * 查询 versionTree * * @param versionId 版本ID * @return */ public VersionTree queryVersionTreeByVersionId(String versionId) { String returneFields = "proId,proName,versionName,downUrl,licenseType,dirTree"; String queryStr = "versionId:"+ versionId; VersionTree results = null; try { HttpSolrClient client = getClient(MongoDBConst.VERSION_TREE); Map map = new HashMap(); map.put(CommonParams.Q, queryStr); map.put(CommonParams.FL, returneFields); map.put(CommonParams.START, "0"); map.put(CommonParams.ROWS, "1"); SolrParams params = new MapSolrParams(map); QueryResponse query = client.query(params, SolrRequest.METHOD.POST); SolrDocumentList response = query.getResults(); //转化对象 if (!response.isEmpty()){ results = BeanUtil.domToVersionTree(response.get(0)); }else { log.error("根据版本ID查询VersionTree失败,versionId:{}" , versionId); } } catch (SolrServerException | IOException e) { log.error("查询solr失败!,queryStr:{}" , queryStr, e); } return results; } /** * 查询 version 的具体信息 * * @param versionId versionId * @return * @throws Exception */ public VersionTree queryVersionInfoByVersionId(Object versionId) { String returneFields = "proId,proName,versionName,downUrl,licenseType"; VersionTree result = new VersionTree(); try { HttpSolrClient client = getClient(MongoDBConst.VERSION_TREE); Map map = new HashMap(); map.put(CommonParams.Q, "versionId:" + versionId); map.put(CommonParams.FL, returneFields); map.put(CommonParams.START, "0"); map.put(CommonParams.ROWS, "1"); SolrParams params = new MapSolrParams(map); QueryResponse query = client.query(params, SolrRequest.METHOD.POST); SolrDocumentList response = query.getResults(); //转化对象 if (CollectionUtils.isNotEmpty(response)) { cn.hutool.core.bean.BeanUtil.copyProperties(response.get(0), result); result.setLicenseType(response.get(0).get("licenseType") == null ? "" : response.get(0).get("licenseType").toString()); }else { log.error("根据版本ID查询版本信息失败,versionId:{}" , versionId); } } catch (SolrServerException | IOException e) { log.error("查询solr失败!,queryStr:{}" , versionId, e); } return result; } /** * 批量查询 version 的具体信息 * * @param versionIds versionIds * @return * @throws Exception */ public List queryBatchVersionInfoByVersionIds(Collection versionIds) { List results = new ArrayList<>(); if (CollectionUtils.isEmpty(versionIds)) { return results; } //去一波重 versionIds = versionIds.stream().collect(Collectors.toSet()); String queryStr = "versionId:(" + StringUtils.join(versionIds, " OR ") + ")"; String returneFields = "versionId,proId,proName,versionName,downUrl,licenseType"; try { HttpSolrClient client = getClient(MongoDBConst.VERSION_TREE); Map map = new HashMap(); map.put(CommonParams.Q, queryStr); map.put(CommonParams.FL, returneFields); map.put(CommonParams.START, "0"); map.put(CommonParams.ROWS,String.valueOf(versionIds.size())); SolrParams params = new MapSolrParams(map); QueryResponse query = client.query(params, SolrRequest.METHOD.POST); SolrDocumentList response = query.getResults(); //转化对象 if (!response.isEmpty()) { for (int i = 0; i < response.size(); i++) { VersionTree versionTree = new VersionTree(); try { cn.hutool.core.bean.BeanUtil.copyProperties(response.get(i), versionTree); versionTree.setLicenseType(response.get(i).get("licenseType") == null ? "" : response.get(i).get("licenseType").toString()); results.add(versionTree); } catch (Exception e) { e.printStackTrace(); } } } } catch (SolrServerException | IOException e) { log.error("查询solr失败!,queryStr:{}" , queryStr, e); } return results; } }