feat: 自定义es及接口完成

This commit is contained in:
2024-03-06 17:18:23 +08:00
parent 964c4beb11
commit ce4387c7e3
20 changed files with 866 additions and 112 deletions

View File

@@ -10,6 +10,7 @@ import com.landaiqing.subject.common.entity.Result;
import com.landaiqing.subject.domain.entity.SubjectAnswerBO;
import com.landaiqing.subject.domain.entity.SubjectInfoBO;
import com.landaiqing.subject.domain.service.SubjectInfoDomainService;
import com.landaiqing.subject.infra.basic.entity.SubjectInfoEs;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.util.CollectionUtils;
@@ -116,4 +117,32 @@ public class SubjectController {
return Result.fail("查询题目详情失败!");
}
}
/**
* @description: 全文检索
* @param: [subjectInfoDTO]
* @return: com.landaiqing.subject.common.entity.Result<com.landaiqing.subject.common.entity.PageResult < com.landaiqing.subject.infra.basic.entity.SubjectInfoEs>>
* @author landaiqing
* @date: 2024/3/6 17:05
*/
@PostMapping("/getSubjectPageBySearch")
public Result<PageResult<SubjectInfoEs>> getSubjectPageBySearch(@RequestBody SubjectInfoDTO subjectInfoDTO) {
try {
if (log.isInfoEnabled()) {
log.info("SubjectController.getSubjectPageBySearch.dto:{}", JSON.toJSONString(subjectInfoDTO));
}
Preconditions.checkArgument(StringUtils.isNotBlank(subjectInfoDTO.getKeyWord()), "关键词不能为空!");
SubjectInfoBO subjectInfoBO = SubjectInfoDTOConverter.INSTANCE.convertDtoToBO(subjectInfoDTO);
subjectInfoBO.setPageNo(subjectInfoDTO.getPageNo());
subjectInfoBO.setPageSize(subjectInfoDTO.getPageSize());
PageResult<SubjectInfoEs> boPageResult = subjectInfoDomainService.getSubjectPageBySearch(subjectInfoBO);
return Result.ok(boPageResult);
} catch (Exception e) {
log.error("SubjectController.getSubjectPageBySearch.error:{}", e.getMessage(), e);
return Result.fail("全文检索失败!");
}
}
}

View File

@@ -1,10 +1,14 @@
package com.landaiqing.subject.application.controller;
import com.alibaba.fastjson.JSON;
import com.landaiqing.subject.common.entity.PageResult;
import com.landaiqing.subject.infra.basic.entity.SubjectInfoEs;
import com.landaiqing.subject.infra.basic.service.SubjectEsService;
import com.landaiqing.subject.infra.entity.UserInfo;
import com.landaiqing.subject.infra.rpc.UserRpc;
import lombok.extern.slf4j.Slf4j;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
@@ -24,31 +28,12 @@ public class TestFeignController {
@Resource
private UserRpc userRpc;
@Resource
private SubjectEsService subjectEsService;
@GetMapping("testFeign")
public void testFeign() {
UserInfo userInfo = userRpc.getUserInfo("jichi");
log.info("testFeign.userInfo:{}", userInfo);
}
@GetMapping("testCreateIndex")
public void testCreateIndex() {
subjectEsService.createIndex();
}
@GetMapping("addDoc")
public void addDoc() {
subjectEsService.addDocs();
}
@GetMapping("find")
public void find() {
subjectEsService.find();
}
@GetMapping("search")
public void search() {
subjectEsService.search();
}
}

View File

@@ -73,6 +73,8 @@ public class SubjectInfoDTO extends PageInfo implements Serializable {
*/
private List<String> labelName;
private String keyWord;
}

View File

@@ -0,0 +1,99 @@
package com.landaiqing.subject.common.util;
public class IdWorkerUtil {
private long workerId;
private long datacenterId;
private long sequence;
private long twepoch = 1585644268888L;
private long workerIdBits = 5L;
private long datacenterIdBits = 5L;
private long sequenceBits = 12L;
private long maxWorkerId = -1L ^ (-1L << workerIdBits);
private long maxDatacenterId = -1L ^ (-1L << datacenterIdBits);
private long workerIdShift = sequenceBits;
private long datacenterIdShift = sequenceBits + workerIdBits;
private long timestampLeftShift = sequenceBits + workerIdBits + datacenterIdBits;
private long sequenceMask = -1L ^ (-1L << sequenceBits);
private long lastTimestamp = -1L;
public long getWorkerId() {
return workerId;
}
public long getDatacenterId() {
return datacenterId;
}
public long getTimestamp() {
return System.currentTimeMillis();
}
public IdWorkerUtil(long workerId, long datacenterId, long sequence) {
if (workerId > maxWorkerId || workerId < 0) {
throw new IllegalArgumentException(
String.format("worker Id can't be greater than %d or less than 0", maxWorkerId));
}
if (datacenterId > maxDatacenterId || datacenterId < 0) {
throw new IllegalArgumentException(
String.format("datacenter Id can't be greater than %d or less than 0", maxDatacenterId));
}
this.workerId = workerId;
this.datacenterId = datacenterId;
this.sequence = sequence;
}
public synchronized long nextId() {
long timestamp = timeGen();
if (timestamp < lastTimestamp) {
System.err.printf(
"clock is moving backwards. Rejecting requests until %d.", lastTimestamp);
throw new RuntimeException(
String.format("Clock moved backwards. Refusing to generate id for %d milliseconds",
lastTimestamp - timestamp));
}
if (lastTimestamp == timestamp) {
sequence = (sequence + 1) & sequenceMask;
if (sequence == 0) {
timestamp = tilNextMillis(lastTimestamp);
}
} else {
sequence = 0;
}
lastTimestamp = timestamp;
return ((timestamp - twepoch) << timestampLeftShift) |
(datacenterId << datacenterIdShift) |
(workerId << workerIdShift) | sequence;
}
private long tilNextMillis(long lastTimestamp) {
long timestamp = timeGen();
while (timestamp <= lastTimestamp) {
timestamp = timeGen();
}
return timestamp;
}
//获取当前时间戳
private long timeGen() {
return System.currentTimeMillis();
}
}

View File

@@ -74,6 +74,7 @@ public class SubjectInfoBO extends PageInfo implements Serializable {
private Long categoryId;
private Long labelId;
private String keyWord;

View File

@@ -3,6 +3,7 @@ package com.landaiqing.subject.domain.service;
import com.landaiqing.subject.common.entity.PageResult;
import com.landaiqing.subject.domain.entity.SubjectInfoBO;
import com.landaiqing.subject.domain.entity.SubjectLabelBO;
import com.landaiqing.subject.infra.basic.entity.SubjectInfoEs;
import java.util.List;
@@ -34,4 +35,12 @@ public interface SubjectInfoDomainService {
* @date: 2024/2/16 13:54
*/
SubjectInfoBO querySubjectInfo(SubjectInfoBO subjectInfoBO);
/**
* @description: 全文检索
* @param: [subjectInfoBO]
* @return: com.landaiqing.subject.common.entity.PageResult<com.landaiqing.subject.infra.basic.entity.SubjectInfoEs>
* @author landaiqing
* @date: 2024/3/6 17:06
*/
PageResult<SubjectInfoEs> getSubjectPageBySearch(SubjectInfoBO subjectInfoBO);
}

View File

@@ -3,6 +3,7 @@ package com.landaiqing.subject.domain.service.impl;
import com.alibaba.fastjson.JSON;
import com.landaiqing.subject.common.entity.PageResult;
import com.landaiqing.subject.common.enums.IsDeletedFlagEnum;
import com.landaiqing.subject.common.util.IdWorkerUtil;
import com.landaiqing.subject.domain.convert.SubjectInfoConverter;
import com.landaiqing.subject.domain.entity.SubjectInfoBO;
import com.landaiqing.subject.domain.entity.SubjectOptionBO;
@@ -10,8 +11,10 @@ import com.landaiqing.subject.domain.handler.subject.SubjectTypeHandler;
import com.landaiqing.subject.domain.handler.subject.SubjectTypeHandlerFactory;
import com.landaiqing.subject.domain.service.SubjectInfoDomainService;
import com.landaiqing.subject.infra.basic.entity.SubjectInfo;
import com.landaiqing.subject.infra.basic.entity.SubjectInfoEs;
import com.landaiqing.subject.infra.basic.entity.SubjectLabel;
import com.landaiqing.subject.infra.basic.entity.SubjectMapping;
import com.landaiqing.subject.infra.basic.service.SubjectEsService;
import com.landaiqing.subject.infra.basic.service.SubjectInfoService;
import com.landaiqing.subject.infra.basic.service.SubjectLabelService;
import com.landaiqing.subject.infra.basic.service.SubjectMappingService;
@@ -20,6 +23,7 @@ import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import javax.annotation.Resource;
import java.util.Date;
import java.util.LinkedList;
import java.util.List;
import java.util.stream.Collectors;
@@ -37,6 +41,9 @@ public class SubjectInfoDomainServiceImpl implements SubjectInfoDomainService {
@Resource
private SubjectLabelService subjectLabelService;
@Resource
private SubjectEsService subjectEsService;
/**
* @description: 新增标签
@@ -71,6 +78,17 @@ public class SubjectInfoDomainServiceImpl implements SubjectInfoDomainService {
});
});
subjectMappingService.batchInsert(mappingList);
//同步到es
SubjectInfoEs subjectInfoEs = new SubjectInfoEs();
subjectInfoEs.setDocId(new IdWorkerUtil(1, 1, 1).nextId());
subjectInfoEs.setSubjectId(subjectInfo.getId());
subjectInfoEs.setSubjectAnswer(subjectInfoBO.getSubjectAnswer());
subjectInfoEs.setCreateTime(new Date().getTime());
subjectInfoEs.setCreateUser("landaiqing");
subjectInfoEs.setSubjectName(subjectInfo.getSubjectName());
subjectInfoEs.setSubjectType(subjectInfo.getSubjectType());
subjectEsService.insert(subjectInfoEs);
subjectEsService.insert(subjectInfoEs);
}
@@ -119,5 +137,15 @@ public class SubjectInfoDomainServiceImpl implements SubjectInfoDomainService {
return bo;
}
@Override
public PageResult<SubjectInfoEs> getSubjectPageBySearch(SubjectInfoBO subjectInfoBO) {
SubjectInfoEs subjectInfoEs = new SubjectInfoEs();
subjectInfoEs.setPageNo(subjectInfoBO.getPageNo());
subjectInfoEs.setPageSize(subjectInfoBO.getPageSize());
subjectInfoEs.setKeyWord(subjectInfoBO.getKeyWord());
return subjectEsService.querySubjectList(subjectInfoEs);
}
}

View File

@@ -57,9 +57,24 @@
<version>1.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-elasticsearch</artifactId>
<version>2.4.2</version>
<groupId>org.elasticsearch</groupId>
<artifactId>elasticsearch</artifactId>
<version>7.5.2</version>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-client</artifactId>
<version>7.5.2</version>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-high-level-client</artifactId>
<version>7.5.2</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-collections4</artifactId>
<version>4.4</version>
</dependency>
</dependencies>
</project>

View File

@@ -0,0 +1,23 @@
package com.landaiqing.subject.infra.basic.entity;
public class EsSubjectFields {
public static final String DOC_ID = "doc_id";
public static final String SUBJECT_ID = "subject_id";
public static final String SUBJECT_NAME = "subject_name";
public static final String SUBJECT_ANSWER = "subject_answer";
public static final String SUBJECT_TYPE = "subject_type";
public static final String CREATE_USER = "create_user";
public static final String CREATE_TIME = "create_time";
public static final String[] FIELD_QUERY = {
SUBJECT_ID, SUBJECT_NAME, SUBJECT_ANSWER, SUBJECT_TYPE, DOC_ID, CREATE_USER, CREATE_TIME
};
}

View File

@@ -1,14 +1,10 @@
package com.landaiqing.subject.infra.basic.entity;
import lombok.AllArgsConstructor;
import com.landaiqing.subject.common.entity.PageInfo;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.springframework.data.annotation.Id;
import org.springframework.data.elasticsearch.annotations.Document;
import org.springframework.data.elasticsearch.annotations.Field;
import org.springframework.data.elasticsearch.annotations.FieldType;
import java.util.Date;
import java.io.Serializable;
import java.math.BigDecimal;
/**
* @Classname SubjectInfoEs
@@ -16,27 +12,26 @@ import java.util.Date;
* @BelongsPackage: com.landaiqing.subject.infra.basic.entity
* @Author: landaiqing
* @CreateTime: 2024-03-04 18:14
* @Description: TODO
* @Description: Es实体类
* @Version: 1.0
*/
@Data
@AllArgsConstructor
@NoArgsConstructor
@Document(indexName = "subject_index",createIndex = false)
public class SubjectInfoEs {
@Field(type = FieldType.Long)
@Id
private Long id;
public class SubjectInfoEs extends PageInfo implements Serializable {
private Long subjectId;
private Long docId;
@Field(type = FieldType.Text, analyzer = "ik_smart")
private String subjectName;
@Field(type = FieldType.Text, analyzer = "ik_smart")
private String subjectAnswer;
@Field(type = FieldType.Keyword)
private String createUser;
@Field(type = FieldType.Date,index = false)
private Date createTime;
private Long createTime;
private Integer subjectType;
private String keyWord;
private BigDecimal score;
}

View File

@@ -1,14 +1,28 @@
package com.landaiqing.subject.infra.basic.es;
import lombok.Data;
import java.io.Serializable;
/**
* @Classname EsClusterConfig
* @BelongsProject: qing-yu-club
* @BelongsPackage: com.landaiqing.subject.infra.basic.es
* @Author: landaiqing
* @CreateTime: 2024-03-05 21:15
* @Description: TODO
* @Description: Es集群类
* @Version: 1.0
*/
public class EsClusterConfig {
@Data
public class EsClusterConfig implements Serializable {
/**
* @description: 集群名称
*/
private String name;
/**
* @description: 集群节点
*/
private String nodes;
}

View File

@@ -0,0 +1,30 @@
package com.landaiqing.subject.infra.basic.es;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.List;
/**
* @Classname EsConfigProperties
* @BelongsProject: qing-yu-club
* @BelongsPackage: com.landaiqing.subject.infra.basic.es
* @Author: landaiqing
* @CreateTime: 2024-03-05 21:18
* @Description: TODO
* @Version: 1.0
*/
@Component
@ConfigurationProperties(prefix = "es.cluster")
public class EsConfigProperties {
private List<EsClusterConfig> esClusterConfigs=new ArrayList<>();
public List<EsClusterConfig> getEsClusterConfigs() {
return esClusterConfigs;
}
public void setEsClusterConfigs(List<EsClusterConfig> esClusterConfigs) {
this.esClusterConfigs = esClusterConfigs;
}
}

View File

@@ -0,0 +1,28 @@
package com.landaiqing.subject.infra.basic.es;
import lombok.Data;
import java.io.Serializable;
/**
* @Classname EsIndexInfo
* @BelongsProject: qing-yu-club
* @BelongsPackage: com.landaiqing.subject.infra.basic.es
* @Author: landaiqing
* @CreateTime: 2024-03-05 21:23
* @Description: 索引类
* @Version: 1.0
*/
@Data
public class EsIndexInfo implements Serializable {
/**
* @description: 集群名称
*/
private String clusterName;
/**
* @description: 索引名称
*/
private String indexName;
}

View File

@@ -0,0 +1,332 @@
package com.landaiqing.subject.infra.basic.es;
import com.alibaba.fastjson.JSON;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.math.NumberUtils;
import org.apache.http.HttpHost;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.delete.DeleteResponse;
import org.elasticsearch.action.get.GetRequest;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchType;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestClientBuilder;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.reindex.BulkByScrollResponse;
import org.elasticsearch.index.reindex.DeleteByQueryRequest;
import org.elasticsearch.search.Scroll;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
import org.elasticsearch.search.sort.ScoreSortBuilder;
import org.springframework.stereotype.Component;
import javax.annotation.PostConstruct;
import javax.annotation.Resource;
import javax.swing.*;
import java.util.*;
/**
* @Classname EsRestClient
* @BelongsProject: qing-yu-club
* @BelongsPackage: com.landaiqing.subject.infra.basic.es
* @Author: landaiqing
* @CreateTime: 2024-03-05 21:32
* @Description: TODO
* @Version: 1.0
*/
@Component
@Slf4j
public class EsRestClient {
private static Map<String, RestHighLevelClient> clientMap = new HashMap<>();
@Resource
private EsConfigProperties esConfigProperties;
private static final RequestOptions COMMON_OPTIONS;
static {
RequestOptions.Builder builder = RequestOptions.DEFAULT.toBuilder();
COMMON_OPTIONS = builder.build();
}
@PostConstruct
public void initialize() {
List<EsClusterConfig> esClusterConfigs = esConfigProperties.getEsClusterConfigs();
for (EsClusterConfig esClusterConfig : esClusterConfigs) {
log.info("initialize.config.name: {},node:{}", esClusterConfig.getName(), esClusterConfig.getNodes());
RestHighLevelClient restHighLevelClient = initRestClient(esClusterConfig);
if (restHighLevelClient != null) {
clientMap.put(esClusterConfig.getName(), restHighLevelClient);
} else {
log.error("config.name: {},node:{}.initError", esClusterConfig.getName(), esClusterConfig.getNodes());
}
}
}
private RestHighLevelClient initRestClient(EsClusterConfig esClusterConfig) {
String[] ipPortArr = esClusterConfig.getNodes().split(",");
List<HttpHost> httpHostList = new ArrayList<>(ipPortArr.length);
for (String ipPort : ipPortArr) {
String[] ipPortInfo = ipPort.split(":");
if (ipPortInfo.length == 2) {
HttpHost httpHost = new HttpHost(ipPortInfo[0], NumberUtils.toInt(ipPortInfo[1]));
httpHostList.add(httpHost);
}
}
HttpHost[] httpHosts = new HttpHost[httpHostList.size()];
httpHostList.toArray(httpHosts);
RestClientBuilder builder = RestClient.builder(httpHosts);
RestHighLevelClient restHighLevelClient = new RestHighLevelClient(builder);
return restHighLevelClient;
}
private static RestHighLevelClient getClient(String clusterName) {
return clientMap.get(clusterName);
}
/**
* @description: 创建文档
* @param: [esIndexInfo, esSourceData]
* @return: boolean
* @author landaiqing
* @date: 2024/3/5 22:03
*/
public static boolean insertDoc(EsIndexInfo esIndexInfo, EsSourceData esSourceData) {
try {
IndexRequest indexRequest = new IndexRequest(esIndexInfo.getIndexName());
indexRequest.source(esSourceData.getData());
indexRequest.id(esSourceData.getDocId());
getClient(esIndexInfo.getClusterName()).index(indexRequest, COMMON_OPTIONS);
return true;
} catch (Exception e) {
log.error("insertDoc.exception:{}", e.getMessage(), e);
}
return false;
}
/**
* @description: 更新文档
* @param: [esIndexInfo, esSourceData]
* @return: boolean
* @author landaiqing
* @date: 2024/3/5 22:30
*/
public static boolean updateDoc(EsIndexInfo esIndexInfo, EsSourceData esSourceData) {
try {
UpdateRequest updateRequest = new UpdateRequest();
updateRequest.index(esIndexInfo.getIndexName());
updateRequest.id(esSourceData.getDocId());
updateRequest.doc(esSourceData.getData());
getClient(esIndexInfo.getClusterName()).update(updateRequest, COMMON_OPTIONS);
return true;
} catch (Exception e) {
log.error("updateDoc.exception:{}", e.getMessage(), e);
}
return false;
}
/**
* @description: 批量更新文档
* @param: [esIndexInfo, esSourceDataList]
* @return: boolean
* @author landaiqing
* @date: 2024/3/5 22:31
*/
public static boolean batchUpdateDoc(EsIndexInfo esIndexInfo,
List<EsSourceData> esSourceDataList) {
try {
boolean flag = false;
BulkRequest bulkRequest = new BulkRequest();
for (EsSourceData esSourceData : esSourceDataList) {
String docId = esSourceData.getDocId();
if (StringUtils.isNotBlank(docId)) {
UpdateRequest updateRequest = new UpdateRequest();
updateRequest.index(esIndexInfo.getIndexName());
updateRequest.id(esSourceData.getDocId());
updateRequest.doc(esSourceData.getData());
bulkRequest.add(updateRequest);
flag = true;
}
}
if (flag) {
BulkResponse bulk = getClient(esIndexInfo.getClusterName()).bulk(bulkRequest, COMMON_OPTIONS);
if (bulk.hasFailures()) {
return false;
}
}
return true;
} catch (Exception e) {
log.error("batchUpdateDoc.exception:{}", e.getMessage(), e);
}
return false;
}
/**
* @description: 删除文档
* @param: [esIndexInfo]
* @return: boolean
* @author landaiqing
* @date: 2024/3/5 22:03
*/
public static boolean delete(EsIndexInfo esIndexInfo) {
try {
DeleteByQueryRequest deleteByQueryRequest =
new DeleteByQueryRequest(esIndexInfo.getClusterName());
deleteByQueryRequest.setQuery(QueryBuilders.matchAllQuery());
BulkByScrollResponse response = getClient(esIndexInfo.getClusterName()).deleteByQuery(
deleteByQueryRequest, COMMON_OPTIONS
);
long deleted = response.getDeleted();
log.info("deleted.size:{}", deleted);
return true;
} catch (Exception e) {
log.error("delete.exception:{}", e.getMessage(), e);
}
return false;
}
/**
* @description: 删除单个文档
* @param: [esIndexInfo, docId]
* @return: boolean
* @author landaiqing
* @date: 2024/3/5 22:06
*/
public static boolean deleteDoc(EsIndexInfo esIndexInfo, String docId) {
try {
DeleteRequest deleteRequest = new DeleteRequest(esIndexInfo.getIndexName());
deleteRequest.id(docId);
DeleteResponse response = getClient(esIndexInfo.getClusterName()).delete(deleteRequest, COMMON_OPTIONS);
log.info("deleteDoc.response:{}", JSON.toJSONString(response));
return true;
} catch (Exception e) {
log.error("deleteDoc.exception:{}", e.getMessage(), e);
}
return false;
}
/**
* @description: 文档是否存在
* @param: [esIndexInfo, docId]
* @return: boolean
* @author landaiqing
* @date: 2024/3/5 22:08
*/
public static boolean isExistDocById(EsIndexInfo esIndexInfo, String docId) {
try {
GetRequest getRequest = new GetRequest(esIndexInfo.getIndexName());
getRequest.id(docId);
return getClient(esIndexInfo.getClusterName()).exists(getRequest, COMMON_OPTIONS);
} catch (Exception e) {
log.error("isExistDocById.exception:{}", e.getMessage(), e);
}
return false;
}
/**
* @description: 通过id查找文档
* @param: [esIndexInfo, docId]
* @return: java.util.Map<java.lang.String, java.lang.Object>
* @author landaiqing
* @date: 2024/3/5 22:10
*/
public static Map<String, Object> getDocById(EsIndexInfo esIndexInfo, String docId) {
try {
GetRequest getRequest = new GetRequest(esIndexInfo.getIndexName());
getRequest.id(docId);
GetResponse response = getClient(esIndexInfo.getClusterName()).get(getRequest, COMMON_OPTIONS);
Map<String, Object> source = response.getSource();
return source;
} catch (Exception e) {
log.error("getDocById.exception:{}", e.getMessage(), e);
}
return null;
}
/**
* @description: 通过指定字段查找
* @param: [esIndexInfo, docId, fields]
* @return: java.util.Map<java.lang.String, java.lang.Object>
* @author landaiqing
* @date: 2024/3/5 22:13
*/
public static Map<String, Object> getDocById(EsIndexInfo esIndexInfo, String docId,
String[] fields) {
try {
GetRequest getRequest = new GetRequest(esIndexInfo.getIndexName());
getRequest.id(docId);
FetchSourceContext fetchSourceContext = new FetchSourceContext(true, fields, null);
getRequest.fetchSourceContext(fetchSourceContext);
GetResponse response = getClient(esIndexInfo.getClusterName()).get(getRequest, COMMON_OPTIONS);
Map<String, Object> source = response.getSource();
return source;
} catch (Exception e) {
log.error("getDocById.exception:{}", e.getMessage(), e);
}
return null;
}
/**
* @description: 查询文档
* @param: [esIndexInfo, esSearchRequest]
* @return: org.elasticsearch.action.search.SearchResponse
* @author landaiqing
* @date: 2024/3/5 22:27
*/
public static SearchResponse searchWithTermQuery(EsIndexInfo esIndexInfo, EsSearchRequest esSearchRequest) {
try {
BoolQueryBuilder bq = esSearchRequest.getBq();
String[] fields = esSearchRequest.getFields();
int from = esSearchRequest.getFrom();
int size = esSearchRequest.getSize();
Long minutes = esSearchRequest.getMinutes();
Boolean needScroll = esSearchRequest.getNeedScroll();
String sortName = esSearchRequest.getSortName();
SortOrder sortOrder = esSearchRequest.getSortOrder();
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(bq);
searchSourceBuilder.fetchSource(fields, null).from(from).size(size);
if (Objects.nonNull(esSearchRequest.getHighlightBuilder())) {
searchSourceBuilder.highlighter(esSearchRequest.getHighlightBuilder());
}
if (StringUtils.isNotBlank(sortName)) {
searchSourceBuilder.sort(sortName);
}
searchSourceBuilder.sort(new ScoreSortBuilder().order(org.elasticsearch.search.sort.SortOrder.DESC));
SearchRequest searchRequest = new SearchRequest();
searchRequest.searchType(SearchType.DEFAULT);
searchRequest.indices(esIndexInfo.getIndexName());
searchRequest.source(searchSourceBuilder);
if (needScroll) {
Scroll scroll = new Scroll(TimeValue.timeValueMinutes(minutes));
searchRequest.scroll(scroll);
}
SearchResponse search = getClient(esIndexInfo.getClusterName()).search(searchRequest, COMMON_OPTIONS);
return search;
} catch (Exception e) {
log.error("searchWithTermQuery.exception:{}", e.getMessage(), e);
}
return null;
}
}

View File

@@ -0,0 +1,56 @@
package com.landaiqing.subject.infra.basic.es;
import lombok.Data;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
import javax.swing.*;
/**
* @Classname EsSearchRequest
* @BelongsProject: qing-yu-club
* @BelongsPackage: com.landaiqing.subject.infra.basic.es
* @Author: landaiqing
* @CreateTime: 2024-03-05 21:25
* @Description: TODO
* @Version: 1.0
*/
@Data
public class EsSearchRequest {
/**
* @description: 查询条件
*/
private BoolQueryBuilder bq;
/**
* @description: 查询字段
*/
private String[] fields;
/**
* @description: 页数
*/
private int from;
/**
* @description: 条数
*/
private int size;
/**
* @description: 需要快照
*/
private Boolean needScroll;
/**
* @description: 快照缓存时间
*/
private Long minutes;
/**
* @description: 排序字段
*/
private String sortName;
/**
* @description: 排序类型
*/
private SortOrder sortOrder;
/**
* @description: 高亮builder
*/
private HighlightBuilder highlightBuilder;
}

View File

@@ -0,0 +1,24 @@
package com.landaiqing.subject.infra.basic.es;
import lombok.Data;
import java.io.Serializable;
import java.util.Map;
/**
* @Classname EsSourceData
* @BelongsProject: qing-yu-club
* @BelongsPackage: com.landaiqing.subject.infra.basic.es
* @Author: landaiqing
* @CreateTime: 2024-03-05 21:30
* @Description: TODO
* @Version: 1.0
*/
@Data
public class EsSourceData implements Serializable {
private String docId;
private Map<String,Object> data;
}

View File

@@ -1,16 +0,0 @@
package com.landaiqing.subject.infra.basic.esRepo;
import com.landaiqing.subject.infra.basic.entity.SubjectInfoEs;
import org.springframework.data.elasticsearch.repository.ElasticsearchRepository;
/**
* @Classname SubjectEsRepository
* @BelongsProject: qing-yu-club
* @BelongsPackage: com.landaiqing.subject.infra.basic.esRepo
* @Author: landaiqing
* @CreateTime: 2024-03-04 18:19
* @Description: TODO
* @Version: 1.0
*/
public interface SubjectEsRepository extends ElasticsearchRepository<SubjectInfoEs,Long> {
}

View File

@@ -1,5 +1,8 @@
package com.landaiqing.subject.infra.basic.service;
import com.landaiqing.subject.common.entity.PageResult;
import com.landaiqing.subject.infra.basic.entity.SubjectInfoEs;
/**
* @Classname SubjectEsService
* @BelongsProject: qing-yu-club
@@ -10,9 +13,7 @@ package com.landaiqing.subject.infra.basic.service;
* @Version: 1.0
*/
public interface SubjectEsService {
void createIndex();
void addDocs();
boolean insert(SubjectInfoEs subjectInfoEs);
void search();
void find();
PageResult<SubjectInfoEs> querySubjectList(SubjectInfoEs subjectInfoEs);
}

View File

@@ -1,24 +1,31 @@
package com.landaiqing.subject.infra.basic.service.impl;
import com.alibaba.fastjson.JSON;
import com.landaiqing.subject.common.entity.PageResult;
import com.landaiqing.subject.common.enums.SubjectInfoTypeEnum;
import com.landaiqing.subject.infra.basic.entity.EsSubjectFields;
import com.landaiqing.subject.infra.basic.entity.SubjectInfoEs;
import com.landaiqing.subject.infra.basic.esRepo.SubjectEsRepository;
import com.landaiqing.subject.infra.basic.es.EsIndexInfo;
import com.landaiqing.subject.infra.basic.es.EsRestClient;
import com.landaiqing.subject.infra.basic.es.EsSearchRequest;
import com.landaiqing.subject.infra.basic.es.EsSourceData;
import com.landaiqing.subject.infra.basic.service.SubjectEsService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.MapUtils;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.common.text.Text;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.MatchQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.springframework.data.elasticsearch.core.ElasticsearchRestTemplate;
import org.springframework.data.elasticsearch.core.IndexOperations;
import org.springframework.data.elasticsearch.core.SearchHit;
import org.springframework.data.elasticsearch.core.SearchHits;
import org.springframework.data.elasticsearch.core.document.Document;
import org.springframework.data.elasticsearch.core.query.NativeSearchQuery;
import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightField;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import javax.annotation.Resource;
import java.util.Date;
import java.util.LinkedList;
import java.util.List;
import java.math.BigDecimal;
import java.math.RoundingMode;
import java.util.*;
/**
* @Classname SubjectEsServiceImpl
@@ -32,42 +39,133 @@ import java.util.List;
@Service
@Slf4j
public class SubjectEsServiceImpl implements SubjectEsService {
@Resource
private ElasticsearchRestTemplate elasticsearchRestTemplate;
@Resource
private SubjectEsRepository subjectEsRepository;
@Override
public void createIndex() {
IndexOperations indexOperations =
elasticsearchRestTemplate.indexOps(SubjectInfoEs.class);
indexOperations.create();
Document mapping = indexOperations.createMapping(SubjectInfoEs.class);
indexOperations.putMapping(mapping);
public boolean insert(SubjectInfoEs subjectInfoEs) {
EsSourceData esSourceData = new EsSourceData();
Map<String, Object> data = convert2EsSourceData(subjectInfoEs);
esSourceData.setDocId(subjectInfoEs.getDocId().toString());
esSourceData.setData(data);
return EsRestClient.insertDoc(getEsIndexInfo(), esSourceData);
}
private Map<String, Object> convert2EsSourceData(SubjectInfoEs subjectInfoEs) {
Map<String, Object> data = new HashMap<>();
data.put(EsSubjectFields.SUBJECT_ID, subjectInfoEs.getSubjectId());
data.put(EsSubjectFields.DOC_ID, subjectInfoEs.getDocId());
data.put(EsSubjectFields.SUBJECT_NAME, subjectInfoEs.getSubjectName());
data.put(EsSubjectFields.SUBJECT_ANSWER, subjectInfoEs.getSubjectAnswer());
data.put(EsSubjectFields.SUBJECT_TYPE, subjectInfoEs.getSubjectType());
data.put(EsSubjectFields.CREATE_USER, subjectInfoEs.getCreateUser());
data.put(EsSubjectFields.CREATE_TIME, subjectInfoEs.getCreateTime());
return data;
}
@Override
public void addDocs() {
List<SubjectInfoEs> list = new LinkedList<>();
list.add(new SubjectInfoEs(1L, "redis 是什么", "redis是一个缓存", "landaiqing", new Date()));
list.add(new SubjectInfoEs(2L, "mysql 是什么", "mysql是一个数据库", "landaiqing", new Date()));
subjectEsRepository.saveAll(list);
}
public PageResult<SubjectInfoEs> querySubjectList(SubjectInfoEs req) {
PageResult<SubjectInfoEs> pageResult = new PageResult<>();
EsSearchRequest esSearchRequest = createSearchListQuery(req);
SearchResponse searchResponse = EsRestClient.searchWithTermQuery(getEsIndexInfo(), esSearchRequest);
@Override
public void search() {
NativeSearchQuery nativeSearchQuery = new NativeSearchQueryBuilder()
.withQuery(QueryBuilders.matchQuery("subjectName", "redis")).build();
SearchHits<SubjectInfoEs> search = elasticsearchRestTemplate.search(nativeSearchQuery, SubjectInfoEs.class);
List<SearchHit<SubjectInfoEs>> searchHits = search.getSearchHits();
log.info("searchHits:{}", JSON.toJSONString(searchHits));
}
@Override
public void find() {
Iterable<SubjectInfoEs> all = subjectEsRepository.findAll();
for (SubjectInfoEs subjectInfoEs: all){
log.info("subjectInfoEs{}",JSON.toJSONString(subjectInfoEs));
List<SubjectInfoEs> subjectInfoEsList = new LinkedList<>();
SearchHits searchHits = searchResponse.getHits();
if (searchHits == null || searchHits.getHits() == null) {
pageResult.setPageNo(req.getPageNo());
pageResult.setPageSize(req.getPageSize());
pageResult.setRecords(subjectInfoEsList);
pageResult.setTotal(0);
return pageResult;
}
SearchHit[] hits = searchHits.getHits();
for (SearchHit hit : hits) {
SubjectInfoEs subjectInfoEs = convertResult(hit);
if (Objects.nonNull(subjectInfoEs)) {
subjectInfoEsList.add(subjectInfoEs);
}
}
pageResult.setPageNo(req.getPageNo());
pageResult.setPageSize(req.getPageSize());
pageResult.setRecords(subjectInfoEsList);
pageResult.setTotal(Long.valueOf(searchHits.getTotalHits().value).intValue());
return pageResult;
}
private SubjectInfoEs convertResult(SearchHit hit) {
Map<String, Object> sourceAsMap = hit.getSourceAsMap();
if (CollectionUtils.isEmpty(sourceAsMap)) {
return null;
}
SubjectInfoEs result = new SubjectInfoEs();
result.setSubjectId(MapUtils.getLong(sourceAsMap, EsSubjectFields.SUBJECT_ID));
result.setSubjectName(MapUtils.getString(sourceAsMap, EsSubjectFields.SUBJECT_NAME));
result.setSubjectAnswer(MapUtils.getString(sourceAsMap, EsSubjectFields.SUBJECT_ANSWER));
result.setDocId(MapUtils.getLong(sourceAsMap, EsSubjectFields.DOC_ID));
result.setSubjectType(MapUtils.getInteger(sourceAsMap, EsSubjectFields.SUBJECT_TYPE));
result.setScore(new BigDecimal(String.valueOf(hit.getScore())).multiply(new BigDecimal("100.00")
.setScale(2, RoundingMode.HALF_UP)));
//处理name的高亮
Map<String, HighlightField> highlightFields = hit.getHighlightFields();
HighlightField subjectNameField = highlightFields.get(EsSubjectFields.SUBJECT_NAME);
if (Objects.nonNull(subjectNameField)) {
Text[] fragments = subjectNameField.getFragments();
StringBuilder subjectNameBuilder = new StringBuilder();
for (Text fragment : fragments) {
subjectNameBuilder.append(fragment);
}
result.setSubjectName(subjectNameBuilder.toString());
}
//处理答案高亮
HighlightField subjectAnswerField = highlightFields.get(EsSubjectFields.SUBJECT_ANSWER);
if (Objects.nonNull(subjectAnswerField)) {
Text[] fragments = subjectAnswerField.getFragments();
StringBuilder subjectAnswerBuilder = new StringBuilder();
for (Text fragment : fragments) {
subjectAnswerBuilder.append(fragment);
}
result.setSubjectAnswer(subjectAnswerBuilder.toString());
}
return result;
}
private EsSearchRequest createSearchListQuery(SubjectInfoEs req) {
EsSearchRequest esSearchRequest = new EsSearchRequest();
BoolQueryBuilder bq = new BoolQueryBuilder();
MatchQueryBuilder subjectNameQueryBuilder =
QueryBuilders.matchQuery(EsSubjectFields.SUBJECT_NAME, req.getKeyWord());
bq.should(subjectNameQueryBuilder);
subjectNameQueryBuilder.boost(2);
MatchQueryBuilder subjectAnswerQueryBuilder =
QueryBuilders.matchQuery(EsSubjectFields.SUBJECT_ANSWER, req.getKeyWord());
bq.should(subjectAnswerQueryBuilder);
MatchQueryBuilder subjectTypeQueryBuilder =
QueryBuilders.matchQuery(EsSubjectFields.SUBJECT_TYPE, SubjectInfoTypeEnum.BRIEF.getCode());
bq.must(subjectTypeQueryBuilder);
bq.minimumShouldMatch(1);
HighlightBuilder highlightBuilder = new HighlightBuilder().field("*").requireFieldMatch(false);
highlightBuilder.preTags("<span style = \"color:red\">");
highlightBuilder.postTags("</span>");
esSearchRequest.setBq(bq);
esSearchRequest.setHighlightBuilder(highlightBuilder);
esSearchRequest.setFields(EsSubjectFields.FIELD_QUERY);
esSearchRequest.setFrom((req.getPageNo() - 1) * req.getPageSize());
esSearchRequest.setSize(req.getPageSize());
esSearchRequest.setNeedScroll(false);
return esSearchRequest;
}
private EsIndexInfo getEsIndexInfo() {
EsIndexInfo esIndexInfo = new EsIndexInfo();
esIndexInfo.setClusterName("5fdd534ef33c");
esIndexInfo.setIndexName("subject_index");
return esIndexInfo;
}
}

View File

@@ -27,12 +27,13 @@ spring:
enabled: true
config:
enabled: true
elasticsearch:
rest:
uris: http://116.196.80.239:9200
publicKey: MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAJ9zUefu5PFeiy4nNNRCIaNT5IY3IxRrlHMiotffSPstMensKg4PoSWJsRRrp/zQEzWegxz2Bkv3F5vfGqqM9N0CAwEAAQ==
logging:
config: classpath:log4j2-spring.xml
es:
cluster:
es-cluster-configs[0]:
name: 5fdd534ef33c
nodes: 116.196.80.239:9200