04-Elasticsearch的 JestClient
1. 介绍
任何使用过Elasticsearch的人都知道，使用基于rest的搜索API构建查询可能是单调乏味且容易出错的。
在本教程中，我们将研究Jest，一个用于Elasticsearch的HTTP Java客户端。Elasticsearch提供了自己原生的Java客户端，然而 Jest提供了更流畅的API和更容易使用的接口
2. Maven依赖
<dependency>
<groupId>io.searchbox</groupId>
<artifactId>jest</artifactId>
<version>6.3.1</version>
</dependency>
3. Jest配置
@Configuration
public class JestClientConfiguration {
private static final Logger LOGGER = LoggerFactory.getLogger(JestClientConfiguration.class);
@Value("${elasticsearch.cluster-name:my-application}")
private String esClusterName;
@Value("#{'${elasticsearch.cluster-nodes:127.0.0.1:9200}'.split(',')}")
private List<String> serversList;
public @Bean
HttpClientConfig httpClientConfig() {
HttpClientConfig httpClientConfig = new HttpClientConfig
.Builder(serversList.get(0))
.gson(new GsonBuilder().setDateFormat("yyyy-MM-dd'T'HH:mm:ss").create())
.multiThreaded(true)
.readTimeout(1000000)
.build();
return httpClientConfig;
}
public @Bean
JestClient jestClient() {
JestClientFactory factory = new JestClientFactory();
factory.setHttpClientConfig(httpClientConfig());
LOGGER.info(">>[获得ES客户端连接：{}]", factory.getObject());
return factory.getObject();
}
}
4. 使用Jest
4.1 Jest常量
public final class JestConstant {
public static final String INDEX_TYPRE = "_doc";
}
4.2 索引枚举
@Slf4j
public enum JestEnum {
JEST_CUST_INFO_INDEX("jest_cust_info","jest_cust", "机构", JestCustInfoItem.class),
;
private String index; //索引名称
private String alias; //索引别名
private String text; //备注
private Class<?> dtoClass; // 索引类Class
JestEnum(String index, String alias, String text, Class dtoClass) {
this.index = index;
this.alias = alias;
this.text = text;
this.dtoClass = dtoClass;
}
public String getIndex() {
return index;
}
//如果未设置别名则使用索引名称
public String getAlias() {
String aliasStr = alias;
if (StringUtils.isBlank(aliasStr)) {
aliasStr = index;
}
return aliasStr;
}
public Class<?> getDtoClass() {
return dtoClass;
}
// 通过class反射获取索引的setting设置
public String getSettings() {
try {
Object obj = dtoClass.newInstance();
Method method = this.dtoClass.getMethod("getSettings");
return (String) method.invoke(obj);
} catch (NoSuchMethodException | IllegalAccessException
| InvocationTargetException | InstantiationException e) {
log.error("[反射调用异常, method: {}. class: {}]", "getSettings()", this.dtoClass);
}
return "";
}
// 通过class反射获取索引的Mappings设置
public String getMappings() {
try {
Object obj = dtoClass.newInstance();
Method method = this.dtoClass.getMethod("getMappings");
return (String) method.invoke(obj);
} catch (NoSuchMethodException | IllegalAccessException
| InvocationTargetException | InstantiationException e) {
log.error("[反射调用异常, method: {}. class: {}]", "getMappings()", this.dtoClass);
e.printStackTrace();
}
return "";
}
// 通过class反射获取索引类型
public String getIndexType() {
try {
Object obj = dtoClass.newInstance();
Method method = this.dtoClass.getMethod("getIndexType");
return (String) method.invoke(obj);
} catch (NoSuchMethodException | IllegalAccessException
| InvocationTargetException | InstantiationException e) {
log.error("[反射调用异常, method: {}. class: {}]", "getMappings()", this.dtoClass);
e.printStackTrace();
}
return "";
}
public String getText() {
return text;
}
}
4.3 创建索引类
先创建父类
@Slf4j
public abstract class AbstractJestItem {
@ApiModelProperty("总记录数")
private Long totalHis;
@ApiModelProperty("得分")
private Double itemScore;
public Long getTotalHis() {
return totalHis;
}
public void setTotalHis(Long totalHis) {
this.totalHis = totalHis;
}
public Double getItemScore() {
return itemScore;
}
public void setItemScore(Double itemScore) {
this.itemScore = itemScore;
}
@JsonIgnore
public abstract String getMappings();
@JsonIgnore
public String getIndexType() {
return JestConstant.INDEX_TYPRE;
}
@JsonIgnore
public String getSettings() {
try {
XContentBuilder builder = jsonBuilder()
.startObject()
.field("number_of_shards", 3)
.field("number_of_replicas", 1)
.endObject();
return Strings.toString(builder);
} catch (Exception e) {
log.error("[JSON 转化异常, method:{}. class:{}]", "getSettings()", this.getClass());
}
return "";
}
}
创建索引类
@Slf4j
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class JestCustInfoItem extends AbstractJestItem {
@ApiModelProperty("客户ID")
private String custId;
@ApiModelProperty("客户名称")
private String custName;
@Override
@JsonIgnore
public String getMappings() {
try {
XContentBuilder builder = jsonBuilder()
.startObject()
.startObject(getIndexType())
.startObject("properties")
.startObject("custId")
.field("type", "keyword")
.field("index", true)
.endObject()
.startObject("custName")
.field("type", "text")
.field("index", true)
.field("analyzer", "ik_max_word")
.field("search_analyzer", "ik_smart")
.endObject()
.endObject()
.endObject()
.endObject();
return Strings.toString(builder);
} catch (Exception e) {
log.error("[JSON 转化异常, method:{}. class:{}]", "getMappings()", this.getClass());
}
return "";
}
}
4.4 索引工具类
/**
* Elasticsearch处理工具类
* 采用Jest客户端
*/
public class JestUtils {
private static final Logger LOGGER = LoggerFactory.getLogger(JestUtils.class);
private static final JestClient jestClient = SpringContextUtils.getBean(JestClient.class);
private static final Integer BATCH_NUM = 1000;
/**
* 验证索引是否存在
* @param jestEnum 索引枚举
* @return true: 索引存在， false:索引不存在
*/
public static Boolean existsIndex(JestEnum jestEnum) {
try {
JestResult jr = jestClient.execute(new IndicesExists.Builder(jestEnum.getIndex()).build());
return null != jr && jr.isSucceeded();
} catch (IOException e) {
LOGGER.error("==> 验证索引失败，indexName:{}, msg :{}", jestEnum.getIndex(), e);
return false;
}
}
/**
* 创建ES索引和mapping
* @param jestEnum 索引枚举
* @return true： 成功，false: 失败
*/
public static Boolean createIndex(JestEnum jestEnum) {
try {
JestResult jr = jestClient.execute(new CreateIndex.Builder(jestEnum.getIndex())
.settings(jestEnum.getSettings())
.mappings(jestEnum.getMappings())
.build());
return null != jr && jr.isSucceeded();
} catch (IOException e) {
LOGGER.error("==> 创建索引失败，indexName:{}, msg :{}", jestEnum.getIndex(), e);
e.printStackTrace();
}
return false;
}
/**
* 删除索引
*/
public static Boolean deleteIndex(JestEnum jestEnum) {
try {
JestResult jr = jestClient.execute(new DeleteIndex.Builder(jestEnum.getIndex()).build());
return null != jr && jr.isSucceeded();
} catch (IOException e) {
LOGGER.error("==> 删除索引失败，indexName:{}, msg :{}", jestEnum.getIndex(), e);
return false;
}
}
/**
* 索引不存在则创建索引
*/
public static boolean createIndexAndMapping(JestEnum jestEnum) {
return !existsIndex(jestEnum) && createIndex(jestEnum);
}
/**
* 索引存在则删除索引
*/
public static boolean deleteIndexAndMapping(JestEnum jestEnum) {
return existsIndex(jestEnum) && deleteIndex(jestEnum);
}
/**
* 清理cache
*/
public static void clearCache() {
try {
ClearCache clearCache = new ClearCache.Builder().build();
jestClient.execute(clearCache);
} catch (IOException e) {
e.printStackTrace();
}
}
/**
* 给索引添加别名
*/
public static Boolean addAlias(JestEnum jestEnum) {
try {
AddAliasMapping build = new AddAliasMapping.Builder(jestEnum.getIndex(), jestEnum.getAlias()).build();
JestResult jr = jestClient.execute(new ModifyAliases.Builder(build).build());
return null != jr && jr.isSucceeded();
} catch (IOException e) {
LOGGER.error("==> 添加索引别名失败，indexName:{}, msg :{}", jestEnum.getIndex(), e);
return false;
}
}
/**
* 删除索引别名
*/
public static Boolean deleteAlias(JestEnum jestEnum) {
try {
RemoveAliasMapping build = new RemoveAliasMapping.Builder(jestEnum.getIndex(), jestEnum.getAlias()).build();
JestResult jr = jestClient.execute(new ModifyAliases.Builder(build).build());
return null != jr && jr.isSucceeded();
} catch (IOException e) {
LOGGER.error("==> 添加索引别名失败，indexName:{}, msg :{}", jestEnum.getIndex(), e);
return false;
}
}
public static void optimizeIndex() {
Optimize optimize = new Optimize.Builder().build();
jestClient.executeAsync(optimize, new JestResultHandler<JestResult>() {
public void completed(JestResult jestResult) {
LOGGER.error("==> optimizeIndex result:{}", jestResult.isSucceeded());
}
public void failed(Exception e) {
e.printStackTrace();
}
});
}
/**
* ES 单条插入书库并带主键
*/
public static <T> Boolean insert(JestEnum jestEnum, T data, String uniqueId) {
try {
JestResult jr = jestClient.execute(new Index.Builder(data)
.index(jestEnum.getIndex())
.type(jestEnum.getIndexType())
.id(uniqueId)
.refresh(true)
.build());
return null != jr && jr.isSucceeded();
} catch (IOException e) {
LOGGER.error("==> [保存文档异常]，indexName:{}, msg :{}", jestEnum.getIndex(), e);
return Boolean.FALSE;
}
}
/**
*单条插入数据
*/
public static <T> Boolean insert(JestEnum jestEnum, T data) {
try {
JestResult jr = jestClient.execute(new Index.Builder(data)
.index(jestEnum.getIndex())
.type(jestEnum.getIndexType())
.refresh(true)
.build());
return null != jr && jr.isSucceeded();
} catch (IOException e) {
LOGGER.error("==> [保存文档异常]，indexName:{}, msg :{}", jestEnum.getIndex(), e);
return Boolean.FALSE;
}
}
/**
* 批量插入数据
*/
public static <T> void insert(JestEnum jestEnum, List<T> dataList) {
if(CollectionUtils.isEmpty(dataList)) {
throw new BizServiceException("批量保存"+jestEnum.getText()+"数据不能为空");
}
List<List<T>> itemLists = Lists.partition(dataList, BATCH_NUM);
List<Index> actions = null;
for (List<T> itemList : itemLists) {
actions = new ArrayList<>();
for(T item : itemList) {
actions.add(new Index.Builder(item)
.id(UUIDUtil.create())
.build());
}
Bulk bulk = new Bulk.Builder()
.defaultIndex(jestEnum.getIndex())
.defaultType(jestEnum.getIndexType())
.addAction(actions)
.build();
try {
BulkResult result = jestClient.execute(bulk);
if (null != result && result.isSucceeded()) {
try {
LOGGER.info("==>[批量保存" + jestEnum.getText() + "成功]，条数 :{}", actions.size());
jestClient.execute(new Flush.Builder().build());
} catch (IOException e) {
LOGGER.info("==>[刷新索引失败]，MSG :{}", e);
}
}
} catch (IOException e) {
LOGGER.info("==>[批量保存" + jestEnum.getText() + "失败]，条数 :{}", actions.size());
}
}
}
private static Map<String, Object> buildAggregation(JestAggreReqDto jestSearchDto, MetricAggregation aggregation) {
// 处理统计
Map<String, Object> aggreMap = new HashedMap();
for (Map.Entry<String, String> entry : jestSearchDto.getAggreMap().entrySet()) {
if (StringUtils.isNotBlank(entry.getKey())) {
List<TermsAggregation.Entry> entryList = aggregation.getTermsAggregation(entry.getKey() + "_count").getBuckets();
Map<String, Long> longMap = new HashedMap();
for (TermsAggregation.Entry jobEntry : entryList) {
longMap.put(jobEntry.getKeyAsString(), jobEntry.getCount());
}
aggreMap.put(entry.getKey(), longMap);
}
}
return aggreMap;
}
/**
* ES 分页查询并做统计
*/
public static JestAggreRespDto queryRecordAggre(JestAggreReqDto jestSearchDto, QueryBuilder queryBuilder,
HighlightBuilder highlightBuilder) {
JestAggreRespDto result = new JestAggreRespDto();
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
BoolQueryBuilder boolQuery = (BoolQueryBuilder) queryBuilder;
for (Map.Entry<String, String> entry : jestSearchDto.getAggreMap().entrySet()) {
if (StringUtils.isNotBlank(entry.getKey())) {
searchSourceBuilder.aggregation(AggregationBuilders.terms(entry.getKey() + "_count").field(entry.getKey()));
}
if (StringUtils.isNotBlank(entry.getValue()) && !"ALL".equalsIgnoreCase(entry.getValue())) {
boolQuery.must(QueryBuilders.matchPhraseQuery(entry.getKey(), entry.getValue()));
}
}
Long formLong = (jestSearchDto.getPageNum() - 1) * jestSearchDto.getPageSize();
searchSourceBuilder.query(boolQuery)
.highlighter(highlightBuilder)
.from(formLong.intValue())
.size(jestSearchDto.getPageSize().intValue());
Search search = new Search.Builder(searchSourceBuilder.toString())
.addIndex(jestSearchDto.getJestEnum().getIndex())
.addType(jestSearchDto.getJestEnum().getIndexType())
.addSort(new Sort("_score", Sort.Sorting.DESC))
.build();
List<AbstractJestItem> dataList = new ArrayList<>();
try {
SearchResult searchResult = jestClient.execute(search);
if (searchResult != null && searchResult.isSucceeded()) {
Long totalHis = searchResult.getTotal();
// 封装数据
List<? extends SearchResult.Hit<?, Void>> hits = searchResult.getHits(jestSearchDto.getJestEnum().getDtoClass());
for (SearchResult.Hit<?, Void> hit : hits) {
AbstractJestItem source = (AbstractJestItem) hit.source;
// 处理高亮
Map<String, List<String>> highlight = hit.highlight;
for (HighlightBuilder.Field f : highlightBuilder.fields()) {
if ( f.name() != null) {
Field field = jestSearchDto.getJestEnum().getDtoClass().getDeclaredField(f.name());
field.setAccessible(true);
field.set(source, highlight.get(f.name()).get(0).toString());
}
}
source.setTotalHis(totalHis);
source.setItemScore(hit.score);
dataList.add(source);
}
//处理统计
if ( null != searchResult.getAggregations()) {
result.setAggrMap(buildAggregation(jestSearchDto, searchResult.getAggregations()));
}
result.setDataList(dataList);
result.setTotalHis(totalHis);
} else {
result.setDataList(dataList);
result.setAggrMap(null);
result.setTotalHis(0L);
}
} catch (IOException | NoSuchFieldException | IllegalAccessException e) {
LOGGER.error("[反射调用异常, method: {}. class: {}]", "getSettings()");
e.printStackTrace();
}
return result;
}
/**
* ES 分页查询关键字
*/
public static PageInfo<AbstractJestItem> queryRecords(AbstractSearchDto jestSearchDto, QueryBuilder queryBuilder,
HighlightBuilder highlightBuilder) {
PageInfo<AbstractJestItem> pageInfo = new PageInfo<AbstractJestItem>();
Long formLong = (jestSearchDto.getPageNum() - 1) * jestSearchDto.getPageSize();
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder()
.query(queryBuilder)
.highlighter(highlightBuilder)
.from(formLong.intValue())
.size(jestSearchDto.getPageSize().intValue());
Search search = new Search.Builder(searchSourceBuilder.toString())
.addIndex(jestSearchDto.getJestEnum().getIndex())
.addType(jestSearchDto.getJestEnum().getIndexType())
.addSort(new Sort("_score", Sort.Sorting.DESC))
.build();
List<AbstractJestItem> custInfoDtoList = new ArrayList<>();
try {
SearchResult searchResult = jestClient.execute(search);
if (searchResult != null && searchResult.isSucceeded()) {
Long totalHis = searchResult.getTotal();
List<? extends SearchResult.Hit<?, Void>> hits = searchResult.getHits(jestSearchDto.getJestEnum().getDtoClass());
for (SearchResult.Hit<?, Void> hit : hits) {
AbstractJestItem source = (AbstractJestItem) hit.source;
Map<String, List<String>> highlight = hit.highlight;
for (HighlightBuilder.Field f : highlightBuilder.fields()) {
if ( f.name() != null) {
Field field = jestSearchDto.getJestEnum().getDtoClass().getDeclaredField(f.name());
field.setAccessible(true);
field.set(source, highlight.get(f.name()).get(0).toString());
}
}
source.setTotalHis(totalHis);
source.setItemScore(hit.score);
custInfoDtoList.add(source);
}
pageInfo.setDataList(custInfoDtoList);
pageInfo.setPageNum(jestSearchDto.getPageNum());
pageInfo.setPageSize(jestSearchDto.getPageSize());
pageInfo.setTotalNum(totalHis);
pageInfo.setTotalSize((pageInfo.getTotalNum() / pageInfo.getPageSize()) + 1);
}else {
pageInfo.setDataList(custInfoDtoList);
pageInfo.setPageNum(0L);
pageInfo.setPageSize(jestSearchDto.getPageSize());
pageInfo.setTotalNum(0L);
pageInfo.setTotalSize((pageInfo.getTotalNum() / pageInfo.getPageSize()) + 1);
}
} catch (IOException | NoSuchFieldException | IllegalAccessException e) {
LOGGER.error("[反射调用异常, method: {}. class: {}]", "getSettings()");
e.printStackTrace();
}
return pageInfo;
}
private static String upperFirstLatter(String letter){
char[] chars = letter.toCharArray();
if(chars[0]>='a' && chars[0]<='z'){
chars[0] = (char) (chars[0]-32);
}
return new String(chars);
}
}
4.5 测试
public class JestTest extends BaseTest {
private static final Logger LOGGER = LoggerFactory.getLogger(JestTest.class);
private static final ObjectMapper MAPPER = new ObjectMapper();
@Test
public void create() throws Exception {
Boolean flag = JestUtils.createIndexAndMapping(JestEnum.JEST_CUST_INFO_INDEX);
LOGGER.info("[创建索引，flag: {}]", flag);
}
@Test
public void delete() throws Exception {
Boolean flag = JestUtils.deleteIndexAndMapping(JestEnum.JEST_CUST_INFO_INDEX);
LOGGER.info("[删除索引，flag: {}]", flag);
}
@Test
public void addAlias() throws Exception {
Boolean flag = JestUtils.addAlias(JestEnum.JEST_CUST_INFO_INDEX);
LOGGER.info("[添加索引别名，flag: {}]", flag);
}
@Test
public void deleteAlias() throws Exception {
Boolean flag = JestUtils.deleteAlias(JestEnum.JEST_CUST_INFO_INDEX);
LOGGER.info("[删除索引别名，flag: {}]", flag);
}
@Test
public void optimizeIndex() throws Exception {
JestUtils.optimizeIndex();
}
@Test
public void saveBatchCustInfo() {
List<CustInfoDbDto> custDbList = custInfoMapper.getCustInfo();
if(CollectionUtils.isEmpty(custDbList)) {
LOGGER.info("==>[批量保存企业机构信息，无数据]");
return ;
}
List<CustInfoItem> custInfoItemList = new ArrayList<CustInfoItem>();
JestUtils.insert(JestEnum.JEST_CUST_INFO_INDEX, custInfoItemList);
}
@Test
public void queryRecords() throws Exception {
AbstractSearchDto jestSearchDto = new AbstractSearchDto();
jestSearchDto.setJestEnum(JestEnum.JEST_CUST_INFO_INDEX);
jestSearchDto.setKeyword("汽车");
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
boolQuery.must(QueryBuilders.matchQuery("custName", jestSearchDto.getKeyword()));
HighlightBuilder highlightBuilder = new HighlightBuilder();
highlightBuilder.preTags("<em>");
highlightBuilder.postTags("</em>");
highlightBuilder.field("custName");
PageInfo<AbstractJestItem> pageInfo = JestUtils.queryRecords(jestSearchDto, boolQuery, highlightBuilder);
LOGGER.info("-> {}", MAPPER.writeValueAsString(pageInfo));
}
@Test
public void queryRecordAggre() throws Exception {
JestAggreReqDto jestSearchDto = new JestAggreReqDto();
jestSearchDto.setJestEnum(JestEnum.JEST_CUST_INFO_INDEX);
jestSearchDto.setKeyword("汽车");
Map<String, String> aggreMap = new HashedMap();
aggreMap.put("province", "广东省");
aggreMap.put("industryTypeName", null);
aggreMap.put("levelName", null);
jestSearchDto.setAggreMap(aggreMap);
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
boolQuery.must(QueryBuilders.matchQuery("custName", jestSearchDto.getKeyword()));
HighlightBuilder highlightBuilder = new HighlightBuilder();
highlightBuilder.preTags("<em>");
highlightBuilder.postTags("</em>");
highlightBuilder.field("custName");
JestAggreRespDto jestAggreCustInfo = JestUtils.queryRecordAggre(jestSearchDto, boolQuery, highlightBuilder);
LOGGER.info("-> {}", MAPPER.writeValueAsString(jestAggreCustInfo));
}
}
源码下载：https://download.csdn.net/download/yk10010/12094309
标签：return,String,04,jestSearchDto,JestClient,Elasticsearch,jestEnum,new,public From： https://www.cnblogs.com/hm201402/p/17127806.html