Spring Boot 引入Elasticsearch7

Spring Boot 引入Elasticsearch7

前置条件

这里使用的Elasticsearch版本为7.26，之前有在ELK安装中有介绍过了具体安装方法。有想法的小伙伴可以去考古一下！

在这里的使用场景需要将多个数据库的数据增量同步到ES中，使用到了Logstash导入数据库数据到ES,

而到了这一步是为了去查询ES中存在的数据，并对数据进行处理统计。

引入依赖

<!-- 这里使用的Spring boot 2.3.5.RELEASE 兼容 Elasticsearch7.* 其他版本请自行百度了解 -->
<parent>
    <groupId>org.springframework.boot</groupId>
    <artifactId>spring-boot-starter-parent</artifactId>
    <version>2.3.5.RELEASE</version>
    <relativePath/> <!-- lookup parent from repository -->
</parent>
<!-- elasticsearch -->
<dependency>
    <groupId>org.springframework.boot</groupId>
    <artifactId>spring-boot-starter-data-elasticsearch</artifactId>
    <!-- 防止与引入的fastjson冲突 -->
    <exclusions>
        <exclusion>
            <groupId>com.fasterxml.jackson.dataformat</groupId>
            <artifactId>jackson-dataformat-smile</artifactId>
        </exclusion>
        <exclusion>
            <groupId>com.fasterxml.jackson.dataformat</groupId>
            <artifactId>jackson-dataformat-cbor</artifactId>
        </exclusion>
        <exclusion>
            <groupId>net.sf.jopt-simple</groupId>
            <artifactId>jopt-simple</artifactId>
        </exclusion>
    </exclusions>
</dependency>

实现CURD的方法（一）

通过继承 ElasticsearchRepository来完成基本的CURD及分页操作和普通的JPA没有什么区别。。

ElasticsearchRepository extends PagingAndSortingRepository extends CrudRepository

数据操作层

@Repository
public interface PersonRepository extends ElasticsearchRepository<Person,Long> {
    
    @Query("{\"bool\" : {\"must\" : {\"field\" : {\"name.keyword\" : \"?\"}}}}")
    Page<Person> findByName(String name, Pageable pageable);
    
    @Query("{\"bool\" : {\"must\" : {\"field\" : {\"age.Long\" : \"?\"}}}}")
    Page<Person> findByAge(Integer age, Pageable pageable);
    
    @Query("{\"bool\" : {\"must\" : {\"field\" : {\"id.Long\" : \"?\"}}}}")
    Page<Person> query(String key, Pageable pageable);
}

实体类

@Document(indexName = "person")
public class Person {
    @Id
    private Integer id;
    @Field(type = FieldType.Text)
    private String name;
    @Field(type = FieldType.Long)
    private Integer age;
    @Field(type = FieldType.Keyword)
    private String demo;
    @Field(type = FieldType.Keyword)
    private String des;
}

业务逻辑层

@Service
public class PersonServiceImpl implements IPersonService {
    @Autowired
    private PersonRepository personRepository;

    private Pageable pageable = PageRequest.of(0,10);
    @Override
    public Iterator<Person> findAll() {
        return personRepository.findAll().iterator();
    }

    @Override
    public Page<Person> findByName(String name) {
        return personRepository.findByName(name,pageable);
    }

    @Override
    public Page<Person> findByAge(Integer age) {
        return personRepository.findByAge(age,pageable);
    }

    @Override
    public Page<Person> query(String key) {
        return personRepository.query(key,pageable);
    }
}

到这里对使用ElasticsearchRepository对ES基本CRUD已经有了大概的了解

实现 CURD的方法（二）

使用ElasticsearchRestTemplate

公共类

@Slf4j
@Component
public class CommonESRepository {
    /**
     * 索引的setting
     */

    private String esSettingFile = "json/es-setting.json";
    private final ElasticsearchRestTemplate elasticsearchRestTemplate;

    public CommonESRepository(ElasticsearchRestTemplate elasticsearchRestTemplate) {
        this.elasticsearchRestTemplate = elasticsearchRestTemplate;
    }

    /**
     * 判断索引是否存在
     * @param indexName 索引名称
     * @return boolean
     */
    public boolean indexExist(String indexName){
        if(StringUtils.isBlank(indexName)){
            return false;
        }
        IndexCoordinates indexCoordinates = IndexCoordinates.of(indexName);
        return elasticsearchRestTemplate.indexOps(indexCoordinates).exists();
    }

    /**
     * 判断index是否存在 不存在创建index
     * @param index 索引实体
     * @param indexName 创建索引的名称
     */
    public void indexCreate(Class index,String indexName){
        if(null == index || StringUtils.isBlank(indexName)){
            return;
        }
        IndexCoordinates indexCoordinates = IndexCoordinates.of(indexName);
        if(!elasticsearchRestTemplate.indexOps(indexCoordinates).exists()){
            // 根据索引实体，获取mapping字段
            Document mapping = elasticsearchRestTemplate.indexOps(indexCoordinates).createMapping(index);
            // 创建索引
            String esSettingStr = null;
            try {
                // 读取setting配置文件
                esSettingStr = IOUtils.toString(Thread.currentThread().getContextClassLoader().getResourceAsStream(esSettingFile), Charset.forName("utf-8"));
            } catch (IOException e) {
                log.error("读取setting配置文件错误", e);
            }
            // setting
            Document setting = Document.parse(esSettingStr);
            elasticsearchRestTemplate.indexOps(indexCoordinates).create(setting);
            // 创建索引mapping
            elasticsearchRestTemplate.indexOps(indexCoordinates).putMapping(mapping);
        }
    }

    /**
     * 根据索引名称，删除索引
     * @param index 索引类
     */
    public void indexDelete(String index){
        elasticsearchRestTemplate.indexOps(IndexCoordinates.of(index)).delete();
    }


    /**
     * 索引添加别名
     * @param indexName 索引名
     * @param aliasName 别名
     */
    public boolean indexAddAlias(String indexName,String aliasName){
        if(StringUtils.isBlank(indexName) || StringUtils.isBlank(aliasName)){
            return false;
        }
        // 索引封装类
        IndexCoordinates indexCoordinates = IndexCoordinates.of(indexName);
        // 判断索引是否存在
        if(elasticsearchRestTemplate.indexOps(indexCoordinates).exists()){
            // 索引别名
            AliasQuery query = new AliasQuery(aliasName);
            // 添加索引别名
            boolean bool = elasticsearchRestTemplate.indexOps(indexCoordinates).addAlias(query);
            return bool;
        }
        return false;
    }

    /**
     * 索引别名删除
     * @param indexName 索引名
     * @param aliasName 别名
     */
    public boolean indexRemoveAlias(String indexName,String aliasName){
        if(StringUtils.isBlank(indexName) || StringUtils.isBlank(aliasName)){
            return false;
        }
        // 索引封装类
        IndexCoordinates indexCoordinates = IndexCoordinates.of(indexName);
        // 判断索引是否存在
        if(elasticsearchRestTemplate.indexOps(indexCoordinates).exists()){
            // 索引别名
            AliasQuery query = new AliasQuery(aliasName);
            // 删除索引别名
            boolean bool = elasticsearchRestTemplate.indexOps(indexCoordinates).removeAlias(query);
            return bool;
        }
        return false;
    }

    /**
     * 索引新增数据
     * @param t 索引类
     * @param <T> 索引类
     */
    public <T> void save(T t){
        // 根据索引实体名新增数据
        elasticsearchRestTemplate.save(t);
    }

    /**
     * 批量插入数据
     * @param queries 数据
     * @param index 索引名称
     */
    public void bulkIndex(List<IndexQuery> queries, String index){
        // 索引封装类
        IndexCoordinates indexCoordinates = IndexCoordinates.of(index);
        // 批量新增数据，此处数据，不要超过100m，100m是es批量新增的筏值，修改可能会影响性能
        elasticsearchRestTemplate.bulkIndex(queries,indexCoordinates);
    }

    /**
     * 根据条件删除对应索引名称的数据
     * @param c 索引类对象
     * @param filedName 索引中字段
     * @param val 删除条件
     * @param index 索引名
     */
    public void delete(Class c,String filedName,Object val,String index){
        // 匹配文件查询
        TermQueryBuilder termQueryBuilder = QueryBuilders.termQuery(filedName, val);
        NativeSearchQuery nativeSearchQuery = new NativeSearchQuery(termQueryBuilder);
        // 删除索引数据
        elasticsearchRestTemplate.delete(nativeSearchQuery,c, IndexCoordinates.of(index));
    }

    /**
     * 根据数据id删除索引
     * @param id 索引id
     * @param index
     */
    public void deleteById(Object id,String index){
        if(null != id && StringUtils.isNotBlank(index)){
            // 根据索引删除索引id数据
            elasticsearchRestTemplate.delete(id.toString(),IndexCoordinates.of(index));
        }
    }

    /**
     * 根据id更新索引数据,不存在则创建索引
     * @param t 索引实体
     * @param id 主键
     * @param index 索引名称
     * @param <T> 索引实体
     */
    public <T> void update(T t,Integer id,String index){
        // 查询索引中数据是否存在
        Object data = elasticsearchRestTemplate.get(id.toString(), t.getClass(), IndexCoordinates.of(index));
        if(data != null){
            // 存在则更新
            UpdateQuery build = UpdateQuery.builder(id.toString()).withDocument(Document.parse(JSON.toJSONString(t))).build();
            elasticsearchRestTemplate.update(build,IndexCoordinates.of(index));
        }else {
            // 不存在则创建
            elasticsearchRestTemplate.save(t);
        }
    }

    /**
     * 查询数据根据实际情况自行修改
     * @param c
     * @param search
     * @param index
     * @param <T>
     * @return
     */
    public <T> List<T> getList(Class<T> c,String search,String index){
        //TermQueryBuilder termQueryBuilder = QueryBuilders.termQuery("title", search);
        NativeSearchQuery nativeSearchQuery = new NativeSearchQuery(QueryBuilders.matchAllQuery());
        nativeSearchQuery.addSort(Sort.by(Sort.Direction.DESC,"_score"));
        nativeSearchQuery.setTrackTotalHits(true);
        SearchHits<T> tax_knowledge_matter = elasticsearchRestTemplate.search(nativeSearchQuery, c, IndexCoordinates.of(index));
        List<SearchHit<T>> searchHits = tax_knowledge_matter.getSearchHits();
        List<T> returnResult = new ArrayList<>();
        searchHits.forEach(item -> {
            T content = item.getContent();
            returnResult.add(content);
        });
        return returnResult;
    }

    /**
     * 测试聚合（需要改进）
     * @param c
     * @param search
     * @param index
     * @return
     */
    public List<ESCdr> getListGroupBy(Class c, String search, String index){
        //TermQueryBuilder termQueryBuilder = QueryBuilders.termQuery("title", search);
        TermsAggregationBuilder accountGroupTerm = AggregationBuilders.terms("customeraccount").field("customeraccount.keyword")
                        .subAggregation(AggregationBuilders.terms("areacode").field("areacode.keyword")
                                .subAggregation(AggregationBuilders.sum("fee").field("fee.keyword")));
        FieldSortBuilder sortBuilder = new FieldSortBuilder("_score").order(SortOrder.DESC);//排序
        NativeSearchQuery nativeSearchQuery = new NativeSearchQueryBuilder().withQuery(QueryBuilders.matchAllQuery())
                .addAggregation(accountGroupTerm)
                .withSort(sortBuilder)
                .build();


        //nativeSearchQuery.addSort(Sort.by(Sort.Direction.DESC,"_score"));
        //nativeSearchQuery.setTrackTotalHits(true);
        SearchHits<ESCdr> tax_knowledge_matter = elasticsearchRestTemplate.search(nativeSearchQuery, c, IndexCoordinates.of(index));
        //List<SearchHit<T>> searchHits = tax_knowledge_matter.getSearchHits();
        Aggregations aggregationsList  = tax_knowledge_matter.getAggregations();
        List<ESCdr> returnResult = new ArrayList<>();
        Terms terms = (Terms) aggregationsList.getAsMap().get("customeraccount");
        if(terms.getBuckets().size() > 0) {
            for (Terms.Bucket bucket : terms.getBuckets()) {
                ESCdr esCdr = new ESCdr();
                esCdr.setCustomername(bucket.getKeyAsString());

                Terms terms1 = (Terms) bucket.getAggregations().get("areacode");
                for (Terms.Bucket bucket1 : terms1.getBuckets()) {
                    esCdr.setCustomeraccount(bucket1.getKeyAsString());
                    Map<String, Aggregation> map = bucket1.getAggregations().asMap();
                    esCdr.setFee(((InternalSum)map.get("fee")).getValue());
                }
                returnResult.add(esCdr);
            }
        }

        //terms.getBuckets().forEach(item -> {
        //    T content = item.;
        //    returnResult.add(content);
        //});
        return returnResult;
    }

    /**
     * scroll 分页查询数据
     * @param c
     * @param search
     * @param index
     * @param <T>
     * @return
     */
    public <T> List<T>  getListScroll(Class<T> c,String search,String index){
        int pageSize = 100000;
        MatchAllQueryBuilder queryBuilder = QueryBuilders.matchAllQuery();//条件
        //TermsAggregationBuilder accountGroupTerm = AggregationBuilders.terms("customername").field("customername.keyword")
        //        .subAggregation(AggregationBuilders.terms("customeraccount").field("customeraccount.keyword")
        //                .subAggregation(AggregationBuilders.terms("areacode").field("areacode.keyword")
        //                        .subAggregation(AggregationBuilders.sum("fee").field("fee.keyword"))));

        FieldSortBuilder sortBuilder = new FieldSortBuilder("_score").order(SortOrder.DESC);//排序
        // .addAggregation(accountGroupTerm)
        NativeSearchQuery nativeSearchQuery = new NativeSearchQueryBuilder().withQuery(queryBuilder).withSort(sortBuilder).build();
        // 设置每页数据量
        nativeSearchQuery.setMaxResults(pageSize);
        //设置缓存内数据的保留时间，不要把缓存时时间设置太长，否则占用内存。
        long scrollTimeInMillis = 60 * 1000;
        // 1、缓存第一页符合搜索条件的数据
        SearchScrollHits<T> searchScrollHits = elasticsearchRestTemplate.searchScrollStart(scrollTimeInMillis, nativeSearchQuery, c , IndexCoordinates.of(index));
        String scrollId = searchScrollHits.getScrollId();
        System.out.println("getListScroll-------------search:" + search+",index:"+index);
        int scrollTime=1;
        List<T> returnResult = new ArrayList<>();
        //2、判断searchScrollHits中是否有命中数据，如果为空，则表示已将符合查询条件的数据全部遍历完毕
        while (searchScrollHits.hasSearchHits()) {
            System.out.println("第"+scrollTime+"页数据，数据总数：" + searchScrollHits.getSearchHits().size());

            for (SearchHit<T> searchHit : searchScrollHits.getSearchHits()) {//3、 从缓存中读取数据
                T content = searchHit.getContent();
                returnResult.add(content);
            }
            //4、根据上次搜索结果scroll_id进入下一页数据搜索
            //该方法执行后将重新刷新快照保留时间
            searchScrollHits = elasticsearchRestTemplate.searchScrollContinue(scrollId, scrollTimeInMillis, c , IndexCoordinates.of(index));
            scrollId = searchScrollHits.getScrollId();
            scrollTime = scrollTime + 1;
        }

        List<String> scrollIds = new ArrayList<>();
        scrollIds.add(scrollId);
        // 清除 scroll
        elasticsearchRestTemplate.searchScrollClear(scrollIds);
        return returnResult;
    }

    public <T> List<T> getList(Class<T> c, String search, int page, int size, Integer siteId, String index){
        BoolQueryBuilder title = QueryBuilders.boolQuery().
                must(QueryBuilders.matchQuery("title", search)).
                must(QueryBuilders.termQuery("siteId", siteId));
        NativeSearchQuery nativeSearchQuery = new NativeSearchQuery(title);
//        nativeSearchQuery.addSort(Sort.by(Sort.Direction.DESC,"_score"));
        nativeSearchQuery.setPageable(PageRequest.of(page,size, Sort.by(Sort.Direction.DESC,"_score")));
        nativeSearchQuery.setTrackTotalHits(true);
        SearchHits<T> tax_knowledge_matter = elasticsearchRestTemplate.search(nativeSearchQuery, c, IndexCoordinates.of(index));
        List<SearchHit<T>> searchHits = tax_knowledge_matter.getSearchHits();
        List<T> returnResult = new ArrayList<>();
        searchHits.forEach(item -> {
            T content = item.getContent();
            returnResult.add(content);
        });
        return returnResult;
    }
}

es-setting.json

{
  "index": {
    "max_result_window": "5000000"
  }
}

简单测试测试~

  @Autowired
  private ElasticsearchRestTemplate elasticsearchRestTemplate;
  
  public List<Test> getTestList() {
      CommonESRepository commonESRepository = new CommonESRepository(elasticsearchRestTemplate);
      List<test> tests = commonESRepository.getListScroll(Test.class,null,"test");
      return tests;
  }

好了,本人很菜，大概就这样！上面两种方法，简单查询的话用第一种比较好，复杂查询的话用第二种，欢迎大佬指点迷津！