JeecgElasticsearchTemplate.java 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481
  1. package org.jeecg.common.es;
  2. import com.alibaba.fastjson.JSONArray;
  3. import com.alibaba.fastjson.JSONObject;
  4. import lombok.extern.slf4j.Slf4j;
  5. import org.apache.commons.lang.StringUtils;
  6. import org.jeecg.common.util.RestUtil;
  7. import org.jeecg.common.util.oConvertUtils;
  8. import org.springframework.beans.factory.annotation.Value;
  9. import org.springframework.http.HttpMethod;
  10. import org.springframework.http.HttpStatus;
  11. import org.springframework.http.ResponseEntity;
  12. import org.springframework.stereotype.Component;
  13. import java.util.*;
  14. /**
  15. * 关于 ElasticSearch 的一些方法(创建索引、添加数据、查询等)
  16. *
  17. * @author sunjianlei
  18. */
  19. @Slf4j
  20. @Component
  21. public class JeecgElasticsearchTemplate {
  22. /** es服务地址 */
  23. private String baseUrl;
  24. private final String FORMAT_JSON = "format=json";
  25. // ElasticSearch 最大可返回条目数
  26. public static final int ES_MAX_SIZE = 10000;
  27. public JeecgElasticsearchTemplate(@Value("${jeecg.elasticsearch.cluster-nodes}") String baseUrl, @Value("${jeecg.elasticsearch.check-enabled}") boolean checkEnabled) {
  28. log.debug("JeecgElasticsearchTemplate BaseURL:" + baseUrl);
  29. if (StringUtils.isNotEmpty(baseUrl)) {
  30. this.baseUrl = baseUrl;
  31. // 验证配置的ES地址是否有效
  32. if (checkEnabled) {
  33. try {
  34. RestUtil.get(this.getBaseUrl().toString());
  35. log.info("ElasticSearch 服务连接成功");
  36. } catch (Exception e) {
  37. log.warn("ElasticSearch 服务连接失败,原因:配置未通过。可能是BaseURL未配置或配置有误,也可能是Elasticsearch服务未启动。接下来将会拒绝执行任何方法!");
  38. }
  39. }
  40. }
  41. }
  42. public StringBuilder getBaseUrl(String indexName, String typeName) {
  43. typeName = typeName.trim().toLowerCase();
  44. return this.getBaseUrl(indexName).append("/").append(typeName);
  45. }
  46. public StringBuilder getBaseUrl(String indexName) {
  47. indexName = indexName.trim().toLowerCase();
  48. return this.getBaseUrl().append("/").append(indexName);
  49. }
  50. public StringBuilder getBaseUrl() {
  51. return new StringBuilder("http://").append(this.baseUrl);
  52. }
  53. /**
  54. * cat 查询ElasticSearch系统数据,返回json
  55. */
  56. public <T> ResponseEntity<T> _cat(String urlAfter, Class<T> responseType) {
  57. String url = this.getBaseUrl().append("/_cat").append(urlAfter).append("?").append(FORMAT_JSON).toString();
  58. return RestUtil.request(url, HttpMethod.GET, null, null, null, responseType);
  59. }
  60. /**
  61. * 查询所有索引
  62. * <p>
  63. * 查询地址:GET http://{baseUrl}/_cat/indices
  64. */
  65. public JSONArray getIndices() {
  66. return getIndices(null);
  67. }
  68. /**
  69. * 查询单个索引
  70. * <p>
  71. * 查询地址:GET http://{baseUrl}/_cat/indices/{indexName}
  72. */
  73. public JSONArray getIndices(String indexName) {
  74. StringBuilder urlAfter = new StringBuilder("/indices");
  75. if (!StringUtils.isEmpty(indexName)) {
  76. urlAfter.append("/").append(indexName.trim().toLowerCase());
  77. }
  78. return _cat(urlAfter.toString(), JSONArray.class).getBody();
  79. }
  80. /**
  81. * 索引是否存在
  82. */
  83. public boolean indexExists(String indexName) {
  84. try {
  85. JSONArray array = getIndices(indexName);
  86. return array != null;
  87. } catch (org.springframework.web.client.HttpClientErrorException ex) {
  88. if (HttpStatus.NOT_FOUND == ex.getStatusCode()) {
  89. return false;
  90. } else {
  91. throw ex;
  92. }
  93. }
  94. }
  95. /**
  96. * 根据ID获取索引数据,未查询到返回null
  97. * <p>
  98. * 查询地址:GET http://{baseUrl}/{indexName}/{typeName}/{dataId}
  99. *
  100. * @param indexName 索引名称
  101. * @param typeName type,一个任意字符串,用于分类
  102. * @param dataId 数据id
  103. * @return
  104. */
  105. public JSONObject getDataById(String indexName, String typeName, String dataId) {
  106. String url = this.getBaseUrl(indexName, typeName).append("/").append(dataId).toString();
  107. log.info("url:" + url);
  108. JSONObject result = RestUtil.get(url);
  109. boolean found = result.getBoolean("found");
  110. if (found) {
  111. return result.getJSONObject("_source");
  112. } else {
  113. return null;
  114. }
  115. }
  116. /**
  117. * 创建索引
  118. * <p>
  119. * 查询地址:PUT http://{baseUrl}/{indexName}
  120. */
  121. public boolean createIndex(String indexName) {
  122. String url = this.getBaseUrl(indexName).toString();
  123. /* 返回结果 (仅供参考)
  124. "createIndex": {
  125. "shards_acknowledged": true,
  126. "acknowledged": true,
  127. "index": "hello_world"
  128. }
  129. */
  130. try {
  131. return RestUtil.put(url).getBoolean("acknowledged");
  132. } catch (org.springframework.web.client.HttpClientErrorException ex) {
  133. if (HttpStatus.BAD_REQUEST == ex.getStatusCode()) {
  134. log.warn("索引创建失败:" + indexName + " 已存在,无需再创建");
  135. } else {
  136. ex.printStackTrace();
  137. }
  138. }
  139. return false;
  140. }
  141. /**
  142. * 删除索引
  143. * <p>
  144. * 查询地址:DELETE http://{baseUrl}/{indexName}
  145. */
  146. public boolean removeIndex(String indexName) {
  147. String url = this.getBaseUrl(indexName).toString();
  148. try {
  149. return RestUtil.delete(url).getBoolean("acknowledged");
  150. } catch (org.springframework.web.client.HttpClientErrorException ex) {
  151. if (HttpStatus.NOT_FOUND == ex.getStatusCode()) {
  152. log.warn("索引删除失败:" + indexName + " 不存在,无需删除");
  153. } else {
  154. ex.printStackTrace();
  155. }
  156. }
  157. return false;
  158. }
  159. /**
  160. * 获取索引字段映射(可获取字段类型)
  161. * <p>
  162. *
  163. * @param indexName 索引名称
  164. * @param typeName 分类名称
  165. * @return
  166. */
  167. public JSONObject getIndexMapping(String indexName, String typeName) {
  168. String url = this.getBaseUrl(indexName, typeName).append("/_mapping?").append(FORMAT_JSON).toString();
  169. log.info("getIndexMapping-url:" + url);
  170. /*
  171. * 参考返回JSON结构:
  172. *
  173. *{
  174. * // 索引名称
  175. * "[indexName]": {
  176. * "mappings": {
  177. * // 分类名称
  178. * "[typeName]": {
  179. * "properties": {
  180. * // 字段名
  181. * "input_number": {
  182. * // 字段类型
  183. * "type": "long"
  184. * },
  185. * "input_string": {
  186. * "type": "text",
  187. * "fields": {
  188. * "keyword": {
  189. * "type": "keyword",
  190. * "ignore_above": 256
  191. * }
  192. * }
  193. * }
  194. * }
  195. * }
  196. * }
  197. * }
  198. * }
  199. */
  200. try {
  201. return RestUtil.get(url);
  202. } catch (org.springframework.web.client.HttpClientErrorException e) {
  203. String message = e.getMessage();
  204. if (message != null && message.contains("404 Not Found")) {
  205. return null;
  206. }
  207. throw e;
  208. }
  209. }
  210. /**
  211. * 获取索引字段映射,返回Java实体类
  212. *
  213. * @param indexName
  214. * @param typeName
  215. * @return
  216. */
  217. public <T> Map<String, T> getIndexMappingFormat(String indexName, String typeName, Class<T> clazz) {
  218. JSONObject mapping = this.getIndexMapping(indexName, typeName);
  219. Map<String, T> map = new HashMap<>();
  220. if (mapping == null) {
  221. return map;
  222. }
  223. // 获取字段属性
  224. JSONObject properties = mapping.getJSONObject(indexName)
  225. .getJSONObject("mappings")
  226. .getJSONObject(typeName)
  227. .getJSONObject("properties");
  228. // 封装成 java类型
  229. for (String key : properties.keySet()) {
  230. T entity = properties.getJSONObject(key).toJavaObject(clazz);
  231. map.put(key, entity);
  232. }
  233. return map;
  234. }
  235. /**
  236. * 保存数据,详见:saveOrUpdate
  237. */
  238. public boolean save(String indexName, String typeName, String dataId, JSONObject data) {
  239. return this.saveOrUpdate(indexName, typeName, dataId, data);
  240. }
  241. /**
  242. * 更新数据,详见:saveOrUpdate
  243. */
  244. public boolean update(String indexName, String typeName, String dataId, JSONObject data) {
  245. return this.saveOrUpdate(indexName, typeName, dataId, data);
  246. }
  247. /**
  248. * 保存或修改索引数据
  249. * <p>
  250. * 查询地址:PUT http://{baseUrl}/{indexName}/{typeName}/{dataId}
  251. *
  252. * @param indexName 索引名称
  253. * @param typeName type,一个任意字符串,用于分类
  254. * @param dataId 数据id
  255. * @param data 要存储的数据
  256. * @return
  257. */
  258. public boolean saveOrUpdate(String indexName, String typeName, String dataId, JSONObject data) {
  259. String url = this.getBaseUrl(indexName, typeName).append("/").append(dataId).append("?refresh=wait_for").toString();
  260. /* 返回结果(仅供参考)
  261. "createIndexA2": {
  262. "result": "created",
  263. "_shards": {
  264. "total": 2,
  265. "successful": 1,
  266. "failed": 0
  267. },
  268. "_seq_no": 0,
  269. "_index": "test_index_1",
  270. "_type": "test_type_1",
  271. "_id": "a2",
  272. "_version": 1,
  273. "_primary_term": 1
  274. }
  275. */
  276. try {
  277. // 去掉 data 中为空的值
  278. Set<String> keys = data.keySet();
  279. List<String> emptyKeys = new ArrayList<>(keys.size());
  280. for (String key : keys) {
  281. String value = data.getString(key);
  282. //1、剔除空值
  283. if (oConvertUtils.isEmpty(value) || "[]".equals(value)) {
  284. emptyKeys.add(key);
  285. }
  286. //2、剔除上传控件值(会导致ES同步失败,报异常failed to parse field [ge_pic] of type [text] )
  287. if (oConvertUtils.isNotEmpty(value) && value.indexOf("[{")!=-1) {
  288. emptyKeys.add(key);
  289. log.info("-------剔除上传控件字段------------key: "+ key);
  290. }
  291. }
  292. for (String key : emptyKeys) {
  293. data.remove(key);
  294. }
  295. } catch (Exception e) {
  296. e.printStackTrace();
  297. }
  298. try {
  299. String result = RestUtil.put(url, data).getString("result");
  300. return "created".equals(result) || "updated".equals(result);
  301. } catch (Exception e) {
  302. log.error(e.getMessage() + "\n-- url: " + url + "\n-- data: " + data.toJSONString());
  303. //TODO 打印接口返回异常json
  304. return false;
  305. }
  306. }
  307. /**
  308. * 删除索引数据
  309. * <p>
  310. * 请求地址:DELETE http://{baseUrl}/{indexName}/{typeName}/{dataId}
  311. */
  312. public boolean delete(String indexName, String typeName, String dataId) {
  313. String url = this.getBaseUrl(indexName, typeName).append("/").append(dataId).toString();
  314. /* 返回结果(仅供参考)
  315. {
  316. "_index": "es_demo",
  317. "_type": "docs",
  318. "_id": "001",
  319. "_version": 3,
  320. "result": "deleted",
  321. "_shards": {
  322. "total": 1,
  323. "successful": 1,
  324. "failed": 0
  325. },
  326. "_seq_no": 28,
  327. "_primary_term": 18
  328. }
  329. */
  330. try {
  331. return "deleted".equals(RestUtil.delete(url).getString("result"));
  332. } catch (org.springframework.web.client.HttpClientErrorException ex) {
  333. if (HttpStatus.NOT_FOUND == ex.getStatusCode()) {
  334. return false;
  335. } else {
  336. throw ex;
  337. }
  338. }
  339. }
  340. /* = = = 以下关于查询和查询条件的方法 = = =*/
  341. /**
  342. * 查询数据
  343. * <p>
  344. * 请求地址:POST http://{baseUrl}/{indexName}/{typeName}/_search
  345. */
  346. public JSONObject search(String indexName, String typeName, JSONObject queryObject) {
  347. String url = this.getBaseUrl(indexName, typeName).append("/_search").toString();
  348. log.info("url:" + url + " ,search: " + queryObject.toJSONString());
  349. JSONObject res = RestUtil.post(url, queryObject);
  350. log.info("url:" + url + " ,return res: \n" + res.toJSONString());
  351. return res;
  352. }
  353. /**
  354. * @param _source (源滤波器)指定返回的字段,传null返回所有字段
  355. * @param query
  356. * @param from 从第几条数据开始
  357. * @param size 返回条目数
  358. * @return { "query": query }
  359. */
  360. public JSONObject buildQuery(List<String> _source, JSONObject query, int from, int size) {
  361. JSONObject json = new JSONObject();
  362. if (_source != null) {
  363. json.put("_source", _source);
  364. }
  365. json.put("query", query);
  366. json.put("from", from);
  367. json.put("size", size);
  368. return json;
  369. }
  370. /**
  371. * @return { "bool" : { "must": must, "must_not": mustNot, "should": should } }
  372. */
  373. public JSONObject buildBoolQuery(JSONArray must, JSONArray mustNot, JSONArray should) {
  374. JSONObject bool = new JSONObject();
  375. if (must != null) {
  376. bool.put("must", must);
  377. }
  378. if (mustNot != null) {
  379. bool.put("must_not", mustNot);
  380. }
  381. if (should != null) {
  382. bool.put("should", should);
  383. }
  384. JSONObject json = new JSONObject();
  385. json.put("bool", bool);
  386. return json;
  387. }
  388. /**
  389. * @param field 要查询的字段
  390. * @param args 查询参数,参考: *哈哈* OR *哒* NOT *呵* OR *啊*
  391. * @return
  392. */
  393. public JSONObject buildQueryString(String field, String... args) {
  394. if (field == null) {
  395. return null;
  396. }
  397. StringBuilder sb = new StringBuilder(field).append(":(");
  398. if (args != null) {
  399. for (String arg : args) {
  400. sb.append(arg).append(" ");
  401. }
  402. }
  403. sb.append(")");
  404. return this.buildQueryString(sb.toString());
  405. }
  406. /**
  407. * @return { "query_string": { "query": query } }
  408. */
  409. public JSONObject buildQueryString(String query) {
  410. JSONObject queryString = new JSONObject();
  411. queryString.put("query", query);
  412. JSONObject json = new JSONObject();
  413. json.put("query_string", queryString);
  414. return json;
  415. }
  416. /**
  417. * @param field 查询字段
  418. * @param min 最小值
  419. * @param max 最大值
  420. * @param containMin 范围内是否包含最小值
  421. * @param containMax 范围内是否包含最大值
  422. * @return { "range" : { field : { 『 "gt『e』?containMin" : min 』?min!=null , 『 "lt『e』?containMax" : max 』}} }
  423. */
  424. public JSONObject buildRangeQuery(String field, Object min, Object max, boolean containMin, boolean containMax) {
  425. JSONObject inner = new JSONObject();
  426. if (min != null) {
  427. if (containMin) {
  428. inner.put("gte", min);
  429. } else {
  430. inner.put("gt", min);
  431. }
  432. }
  433. if (max != null) {
  434. if (containMax) {
  435. inner.put("lte", max);
  436. } else {
  437. inner.put("lt", max);
  438. }
  439. }
  440. JSONObject range = new JSONObject();
  441. range.put(field, inner);
  442. JSONObject json = new JSONObject();
  443. json.put("range", range);
  444. return json;
  445. }
  446. }