Skip to content

D
device-back
提交 bce9059c authored 作者: LJJ's avatar LJJ
浏览文件
操作

新增es模块

上级 f89d4c41
隐藏空白字符变更
内嵌 并排
正在显示 包含 491 行增加 和 0 行删除
dev-es/pom.xml 0 → 100644
浏览文件 @ bce9059c
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>equip</artifactId>
<groupId>com.tykj</groupId>
<version>1.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>dev-es</artifactId>
<dependencies>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-high-level-client</artifactId>
<version>7.10.1</version>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-client</artifactId>
<version>7.10.1</version>
</dependency>
<dependency>
<groupId>org.elasticsearch</groupId>
<artifactId>elasticsearch</artifactId>
<version>7.10.1</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
<version>2.11.1</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
<version>2.11.1</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
</dependency>
</dependencies>
</project>
\ No newline at end of file
dev-es/src/main/java/com/zjty/dev/device/es/EsApplication.java 0 → 100644
浏览文件 @ bce9059c
package com.zjty.dev.device.es;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
/**
* @author LJJ cnljj1995@gmail.com
* on 1/22/21
*/
@SpringBootApplication(scanBasePackages = {
"com.zjty.dev.device.es"
})
public class EsApplication {
public static void main(String[] args) {
SpringApplication.run(EsApplication.class, args);
}
}
dev-es/src/main/java/com/zjty/dev/device/es/subject/entity/CollectionRs.java 0 → 100755
浏览文件 @ bce9059c
package com.zjty.dev.device.es.subject.entity;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.util.List;
import java.util.Map;
/**
* @author ljj
*/
@Data
@AllArgsConstructor
@NoArgsConstructor
public class CollectionRs {
/**
* 搜索结果的数量
*/
private Long total;
/**
* id集合
*/
private List<String> ids;
/**
* 搜素的数据集合
*/
private List<Map<String, Object>> source;
}
dev-es/src/main/java/com/zjty/dev/device/es/subject/entity/EsSource.java 0 → 100755
浏览文件 @ bce9059c
package com.zjty.dev.device.es.subject.entity;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.util.Collections;
import java.util.Map;
/**
* @author LJJ
* on 2019-04-24
*/
@Data
@AllArgsConstructor
@NoArgsConstructor
@Builder
public class EsSource {
/**
* es id
*/
private String id = "";
/**
* es data source
*/
private Map<String, Object> jsonMap = Collections.emptyMap();
}
dev-es/src/main/java/com/zjty/dev/device/es/subject/service/EsUtil.java 0 → 100644
浏览文件 @ bce9059c
package com.zjty.dev.device.es.subject.service;
import com.sun.istack.internal.NotNull;
import com.zjty.dev.device.es.subject.entity.CollectionRs;
import com.zjty.dev.device.es.subject.entity.EsSource;
import java.io.IOException;
import java.util.List;
import java.util.Map;
/**
* @author LJJ cnljj1995@gmail.com
* on 1/23/21
*/
public interface EsUtil {
/**
* 创建索引
* @param indexName 索引名称
* @param mapping mapping
* @return t or f
* @throws Exception e
*/
boolean createIndex(String indexName, Map<String, Object> mapping);
/**
* 增量添加数据
* @param indexName 索引名称
* @param data
* @throws IOException
*/
void bulkAdd(String indexName, @NotNull List<EsSource> data);
/**
* 批量删除数据
* @param indexName 索引名称
* @param ids id集合
*/
void bulkDel(String indexName, List<String> ids);
/**
* 分页搜索
* @param indexName 索引名称
* @param key 搜索关键字
* @param page 0开始
* @param size 每页大小
* @return c
*/
CollectionRs search(String indexName, String key, int page, int size);
}
dev-es/src/main/java/com/zjty/dev/device/es/subject/service/impl/EsUtilImpl.java 0 → 100644
浏览文件 @ bce9059c
package com.zjty.dev.device.es.subject.service.impl;
import com.zjty.dev.device.es.subject.entity.CollectionRs;
import com.zjty.dev.device.es.subject.entity.EsSource;
import com.zjty.dev.device.es.subject.service.EsUtil;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.HttpHost;
import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.sort.SortOrder;
import org.springframework.stereotype.Service;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
/**
* @author LJJ cnljj1995@gmail.com
* on 1/23/21
*/
@Service
@Slf4j
public class EsUtilImpl implements EsUtil {
@Override
public boolean createIndex(String indexName, Map<String, Object> mapping) {
RestHighLevelClient client = new RestHighLevelClient(
RestClient.builder(
new HttpHost("localhost", 9200, "http")));
CreateIndexRequest request = new CreateIndexRequest(indexName);
request.settings(Settings.builder()
.put("index.number_of_shards", 3)
.put("index.number_of_replicas", 2)
.build());
request.mapping("doc", mapping);
try {
client.indices().create(request, RequestOptions.DEFAULT);
client.close();
log.info("[es] 创建索引成功,被创建的索引为:{}", indexName);
return true;
} catch (Exception e) {
log.error("[es] 创建索引出错，错误信息:{}", e.getMessage());
try {
client.close();
} catch (IOException ex) {
log.error("[es] es client关闭异常，错误信息:{}", ex.getMessage());
}
return false;
}
}
@Override
public void bulkAdd(String indexName, List<EsSource> data) {
// es的bulk方法不适合无限制的保存，此处预设为每次保存50条，具体需根据服务器性能进行测试
int limit = 50;
int size = data.size();
int page = size / limit;
boolean tag = size > page * limit;
for (int i = 0; i < page; i++) {
addByBulk(indexName, data.subList(limit * i, limit * (i + 1)));
log.info("[es] 批量保存进度:{}/{}", i + 1, tag ? page + 1 : page);
}
if (size > page * limit) {
addByBulk(indexName, data.subList(page * limit, size));
log.info("[es] 批量保存进度:{}/{}", page + 1, page + 1);
}
}
@Override
public void bulkDel(String indexName, List<String> ids) {
// es的bulk方法不适合无限制的保存，此处预设为每次保存50条，具体需根据服务器性能进行测试
int limit = 50;
int size = ids.size();
int page = size / limit;
boolean tag = size > page * limit;
for (int i = 0; i < page; i++) {
delByBulk(indexName, ids.subList(limit * i, limit * (i + 1)));
log.info("[es] 批量删除进度:{}/{}", i + 1, tag ? page + 1 : page);
}
if (size > page * limit) {
delByBulk(indexName, ids.subList(page * limit, size));
log.info("[es] 批量删除进度:{}/{}", page + 1, page + 1);
}
}
@Override
public CollectionRs search(String indexName, String key, int page, int size) {
RestHighLevelClient client = new RestHighLevelClient(
RestClient.builder(
new HttpHost("localhost", 9200, "http")));
SearchRequest searchRequest = new SearchRequest(indexName);
SearchSourceBuilder builder = new SearchSourceBuilder().query(
QueryBuilders.queryStringQuery(key));
builder.size(size)
.from(size * page)
.sort("updateTime", SortOrder.DESC);
searchRequest.source(builder);
SearchResponse search = null;
try {
search = client.search(searchRequest, RequestOptions.DEFAULT);
} catch (IOException e) {
log.error("[es] 关键字搜索出错:{}", e.getMessage());
}
try {
client.close();
} catch (IOException e) {
log.error("[es] 关闭客户端出错:{}", e.getMessage());
}
List<Map<String, Object>> rs = new ArrayList<>();
List<String> ids = new ArrayList<>();
for (SearchHit hit : search.getHits().getHits()) {
Map<String, Object> sourceAsMap = hit.getSourceAsMap();
sourceAsMap.put("_id",hit.getId());
rs.add(sourceAsMap);
ids.add(hit.getId());
}
CollectionRs collectionRs = new CollectionRs(search.getHits().getTotalHits().value, ids, rs);
return collectionRs;
}
private void delByBulk(String indexName, List<String> ids) {
RestHighLevelClient client = new RestHighLevelClient(
RestClient.builder(
new HttpHost("localhost", 9200, "http")));
BulkRequest bulkRequest = new BulkRequest();
ids.forEach(o -> bulkRequest.add(new DeleteRequest(indexName, o)));
try {
client.bulk(bulkRequest, RequestOptions.DEFAULT);
client.close();
} catch (Exception e) {
log.error("[es] 批量删除数据出错: {},出错信息: {}", indexName, e.getMessage());
try {
client.close();
} catch (IOException ex) {
log.error("[es] es client关闭异常，错误信息:{}", ex.getMessage());
}
}
}
private void addByBulk(String indexName, List<EsSource> data) {
RestHighLevelClient client = new RestHighLevelClient(
RestClient.builder(
new HttpHost("localhost", 9200, "http")));
BulkRequest bulkRequest = new BulkRequest();
data.forEach(o -> bulkRequest.add(new IndexRequest(indexName, "doc", o.getId()).source(o.getJsonMap())));
try {
client.bulk(bulkRequest, RequestOptions.DEFAULT);
client.close();
} catch (Exception e) {
log.error("[es] 批量插入数据出错: {},出错信息: {}", indexName, e.getMessage());
try {
client.close();
} catch (IOException ex) {
log.error("[es] es client关闭异常，错误信息:{}", ex.getMessage());
}
}
}
}
dev-es/src/main/resources/application.properties 0 → 100644
浏览文件 @ bce9059c
spring.datasource.url=jdbc:mysql://localhost:3306/test?serverTimezone=Asia/Shanghai&characterEncoding=UTF-8&zeroDateTimeBehavior=convertToNull&useSSL=false
spring.datasource.driver-class-name=com.mysql.cj.jdbc.Driver
spring.datasource.username=root
spring.datasource.password=ljj123456
\ No newline at end of file
dev-es/src/test/java/com/zjty/dev/device/es/Test.java 0 → 100644
浏览文件 @ bce9059c
package com.zjty.dev.device.es;
import com.zjty.dev.device.es.subject.entity.CollectionRs;
import com.zjty.dev.device.es.subject.entity.EsSource;
import com.zjty.dev.device.es.subject.service.EsUtil;
import com.zjty.dev.device.es.subject.service.impl.EsUtilImpl;
import lombok.extern.slf4j.Slf4j;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.junit4.SpringRunner;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* @author LJJ cnljj1995@gmail.com
* on 1/23/21
*/
@SpringBootTest
@Slf4j
public class Test {
@Autowired
private EsUtil esUtil;
/**
* 创建索引测试
* @throws Exception e
*/
@org.junit.jupiter.api.Test
public void createIndexTest() throws Exception {
// 需要分词的字段
Map<String, Object> text = new HashMap<>();
text.put("type", "text");
text.put("analyzer", "ik_max_word");
text.put("search_analyzer", "ik_max_word");
// 不需要分词的字段
Map<String, Object> keyword = new HashMap<>();
keyword.put("type", "keyword");
Map<String, Object> properties = new HashMap<>();
properties.put("title", text);
properties.put("updateTime", keyword);
properties.put("content", text);
Map<String, Object> mapping = new HashMap<>();
mapping.put("properties", properties);
esUtil.createIndex("dev_test", mapping);
}
/**
* 模拟添加数据测试
* @throws Exception e
*/
@org.junit.jupiter.api.Test
public void addData() throws Exception {
List<EsSource> list = new ArrayList<>();
for (int i = 0; i < 100; i++) {
HashMap<String, Object> map = new HashMap<>();
map.put("content", "中华人民共和国成立20周年，阿斯顿马丁是辆豪车"+i);
map.put("updateTime", 123333);
map.put("title", "这是一个正常的标题");
list.add(EsSource
.builder()
.id(i + "")
.jsonMap(map)
.build());
}
esUtil.bulkAdd("dev_test", list);
}
/**
* 搜索测试
*/
@org.junit.jupiter.api.Test
public void searchTest() {
CollectionRs search = esUtil.search("dev_test", "中华", 0, 50);
log.info("数量:{}，内容:{}",search.getIds().size(), search.toString() );
}
/**
* 删除测试
*/
@org.junit.jupiter.api.Test
public void delByBulkTest() {
List<String> ids = new ArrayList<>();
for (int i = 0; i < 20; i++) {
ids.add(i+"");
}
esUtil.bulkDel("dev_test",ids);
}
}
pom.xml
浏览文件 @ bce9059c
...@@ -43,6 +43,7 @@ ...@@ -43,6 +43,7 @@
<module>dev-user</module> <module>dev-user</module>
<module>dev-usereport</module> <module>dev-usereport</module>
<module>dev-statistical</module> <module>dev-statistical</module>
<module>dev-es</module>
</modules> </modules>
<properties> <properties>
......
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论