add manual load data

elastic-publish-service/req/main-controller.http

@@ -21,7 +21,7 @@ Content-Type: application/json
 
 {
   "agencyPath": "/1",
-  "keyword": "b楼",
+  "keyword": "科苑一区",
   "page": 1,
   "size": 10,
   "regulatoryLevelId": "",
@@ -38,6 +38,9 @@ GET http://{{host}}/load
 
 ### load data
 
+GET http://{{host}}/set-load
+
+### load data
 
 
 GET http://{{host}}/loadByAgencyIds?agencyIds=190084,196581

elastic-publish-service/src/main/java/com/vcarecity/publish/constants/ElasticConstant.java

@@ -9,9 +9,6 @@ import java.time.format.DateTimeFormatter;
 public class ElasticConstant {
     public static final long CHECK_TIME_INTERVAL = 1000 * 3 * 60;
 
-    public static final String UNIT_AGENCY_INDEX = "unit-agency";
-
-
     public static final int QUERY_PAGE_COUNT = 2000;
 
 
@@ -22,5 +19,4 @@ public class ElasticConstant {
     public static final DateTimeFormatter ES_DT_FORMATTER = DateTimeFormatter.ofPattern(ES_DATETIME_PATTERN);
 
 
-
 }

elastic-publish-service/src/main/java/com/vcarecity/publish/controller/MainController.java

@@ -3,7 +3,6 @@ package com.vcarecity.publish.controller;
 import com.vcarecity.publish.annotation.SkipMethod;
 import com.vcarecity.publish.api.ApiResult;
 import com.vcarecity.publish.api.ApiStatus;
-import com.vcarecity.publish.constants.ElasticConstant;
 import com.vcarecity.publish.elastic.service.ElasticLoadDataService;
 import com.vcarecity.publish.elastic.service.ElasticUpdateService;
 import com.vcarecity.publish.entity.UnitAgencyMergeEntity;
@@ -38,6 +37,9 @@ public class MainController implements PrepareController {
     private final ElasticUpdateService elasticUpdateService;
     private final LoadDataStatusService loadDataStatusService;
 
+    @Value("${app.index.name}")
+    private String indexName;
+
     public MainController(UnitService unitService,
                           ElasticLoadDataService elasticLoadDataService,
                           LoadDataStatusService loadDataStatusService,
@@ -119,7 +121,7 @@ public class MainController implements PrepareController {
     @PostMapping("/query-unit-agency")
     public ApiResult queryUnitAgency(@RequestBody @Valid UnitAgencyQuery unitAgencyQuery) throws IOException {
 
-        TotalResultDTO<List<UnitIdDTO>> totalResult = unitService.getUnitAgencyByQuery(unitAgencyQuery, UnitIdDTO.class, ElasticConstant.UNIT_AGENCY_INDEX);
+        TotalResultDTO<List<UnitIdDTO>> totalResult = unitService.getUnitAgencyByQuery(unitAgencyQuery, UnitIdDTO.class, indexName);
 
         return ApiResult.builder()
                 .status(ApiStatus.SUCCESS_CODE)
@@ -130,7 +132,7 @@ public class MainController implements PrepareController {
 
     @PostMapping("/test/query-unit-agency")
     public ApiResult testQueryUnitAgency(@RequestBody @Valid UnitAgencyQuery unitAgencyQuery) throws IOException {
-        TotalResultDTO<List<UnitAgencyDTO>> totalResult = unitService.getUnitAgencyByQuery(unitAgencyQuery, UnitAgencyDTO.class, ElasticConstant.UNIT_AGENCY_INDEX);
+        TotalResultDTO<List<UnitAgencyDTO>> totalResult = unitService.getUnitAgencyByQuery(unitAgencyQuery, UnitAgencyDTO.class, indexName);
         return ApiResult.builder()
                 .status(ApiStatus.SUCCESS_CODE)
                 .data(totalResult.getData())

elastic-publish-service/src/main/java/com/vcarecity/publish/elastic/service/impl/ElasticLoadDataServiceImpl.java

@@ -7,6 +7,7 @@ import com.vcarecity.publish.elastic.service.ElasticUpdateService;
 import com.vcarecity.publish.entity.UnitAgencyMergeEntity;
 import com.vcarecity.publish.mapper.UnitMapper;
 import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 
 import java.io.IOException;
@@ -25,7 +26,10 @@ public class ElasticLoadDataServiceImpl implements ElasticLoadDataService {
     private final ElasticUpdateService elasticUpdateService;
     private final ElasticIndexService elasticIndexService;
     private final int pageCount;
-    private final String index;
+
+
+    @Value("${app.index.name}")
+    private String index;
 
     public ElasticLoadDataServiceImpl(UnitMapper unitMapper,
                                       ElasticUpdateService elasticUpdateService,
@@ -34,7 +38,6 @@ public class ElasticLoadDataServiceImpl implements ElasticLoadDataService {
         this.elasticUpdateService = elasticUpdateService;
         this.elasticIndexService = elasticIndexService;
         this.pageCount = ElasticConstant.QUERY_PAGE_COUNT;
-        this.index = ElasticConstant.UNIT_AGENCY_INDEX;
     }
 
 

elastic-publish-service/src/main/java/com/vcarecity/publish/elastic/service/impl/ElasticQueryServiceImpl.java

@@ -1,6 +1,5 @@
 package com.vcarecity.publish.elastic.service.impl;
 
-import com.vcarecity.publish.constants.ElasticConstant;
 import com.vcarecity.publish.elastic.service.ElasticQueryService;
 import lombok.extern.slf4j.Slf4j;
 import org.elasticsearch.action.search.SearchRequest;
@@ -11,6 +10,7 @@ import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.TermQueryBuilder;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 
 import java.io.IOException;
@@ -25,7 +25,12 @@ public class ElasticQueryServiceImpl implements ElasticQueryService {
 
     private final RestHighLevelClient restHighLevelClient;
 
+
+    @Value("${app.index.name}")
+    private String indexName;
+
     public ElasticQueryServiceImpl(RestHighLevelClient restHighLevelClient) {
+
         this.restHighLevelClient = restHighLevelClient;
     }
 
@@ -39,7 +44,7 @@ public class ElasticQueryServiceImpl implements ElasticQueryService {
         searchSourceBuilder.query(unitId);
 
 
-        SearchRequest request = new SearchRequest(ElasticConstant.UNIT_AGENCY_INDEX);
+        SearchRequest request = new SearchRequest(indexName);
         request.source(searchSourceBuilder);
 
         SearchResponse response = restHighLevelClient.search(request, RequestOptions.DEFAULT);

elastic-publish-service/src/main/java/com/vcarecity/publish/elastic/service/impl/ElasticUpdateServiceImpl.java

@@ -5,8 +5,6 @@ import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.DeserializationFeature;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.vcarecity.elastic.util.AgencyPathUtil;
-import com.vcarecity.elastic.util.SnowFlake;
-import com.vcarecity.publish.constants.ElasticConstant;
 import com.vcarecity.publish.elastic.entity.UnitAgencyDetailEntity;
 import com.vcarecity.publish.elastic.service.ElasticUpdateService;
 import com.vcarecity.publish.entity.UnitAgencyMergeEntity;
@@ -24,6 +22,7 @@ import org.elasticsearch.index.reindex.DeleteByQueryRequest;
 import org.elasticsearch.index.reindex.UpdateByQueryRequest;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptType;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.cglib.beans.BeanCopier;
 import org.springframework.stereotype.Service;
 
@@ -42,7 +41,8 @@ import java.util.Map;
 @Service
 public class ElasticUpdateServiceImpl implements ElasticUpdateService {
 
-    public final String index;
+    @Value("${app.index.name}")
+    public String index;
 
     private final RestHighLevelClient restHighLevelClient;
     private final ObjectMapper objectMapper;
@@ -58,7 +58,7 @@ public class ElasticUpdateServiceImpl implements ElasticUpdateService {
         objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
 
         this.objectMapper = objectMapper;
-        this.index = ElasticConstant.UNIT_AGENCY_INDEX;
+
     }
 
 
@@ -132,7 +132,10 @@ public class ElasticUpdateServiceImpl implements ElasticUpdateService {
     public List<UnitAgencyMergeEntity> updateDataByQuery(String key, List<UnitAgencyMergeEntity> data) throws IOException {
         List<UnitAgencyMergeEntity> addList = new ArrayList<>();
         final BeanCopier beanCopier = BeanCopier.create(UnitAgencyMergeEntity.class, UnitAgencyDetailEntity.class, false);
-        for (UnitAgencyMergeEntity item : data) {
+
+
+        for (int i = 0; i < data.size(); i++) {
+            UnitAgencyMergeEntity item = data.get(i);
 
             Map<String, Object> params = createElasticDataMap(beanCopier, item);
             Object value = params.get(key);
@@ -144,7 +147,7 @@ public class ElasticUpdateServiceImpl implements ElasticUpdateService {
 
             BulkByScrollResponse response = restHighLevelClient.updateByQuery(request, RequestOptions.DEFAULT);
 
-            logger.info("response:{}", response);
+            logger.info("updateDataByQuery.index: {} resp:{}", i, response);
 
             if (response.getUpdated() == 0L) {
                 addList.add(item);
@@ -205,7 +208,7 @@ public class ElasticUpdateServiceImpl implements ElasticUpdateService {
 
     private IndexRequest createIndexRequest(String index, Map<String, Object> value) {
         IndexRequest indexRequest = new IndexRequest(index);
-        indexRequest.id(SnowFlake.nextId() + "");
+        // indexRequest.id(SnowFlake.nextId() + "");
         indexRequest.source(value, XContentType.JSON);
         return indexRequest;
     }

elastic-publish-service/src/main/java/com/vcarecity/publish/task/CheckUpdateTask.java

@@ -9,6 +9,7 @@ import com.vcarecity.publish.service.CheckTimeService;
 import com.vcarecity.publish.service.LoadDataStatusService;
 import com.vcarecity.publish.util.DateUtil;
 import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.stereotype.Component;
 
@@ -32,6 +33,14 @@ public class CheckUpdateTask {
     private final UnitMapper unitMapper;
 
 
+    @Value("${app.index.update-unit}")
+    private Boolean updateUnit;
+
+
+    @Value("${app.index.update-agency}")
+    private Boolean updateAgency;
+
+
     public CheckUpdateTask(CheckTimeService checkTimeService,
                            ElasticUpdateService elasticUpdateService,
                            LoadDataStatusService loadDataStatusService, AgencyMapper agencyMapper,
@@ -60,41 +69,47 @@ public class CheckUpdateTask {
         List<UnitAgencyMergeEntity> updateUaList;
         String fromDateTime;
 
-        // 获取上一次的时间
-        fromDateTime = checkTimeService.getLastDateTime("T_UNIT");
+        if (updateUnit) {
+
+
+            // 获取上一次的时间
+            fromDateTime = checkTimeService.getLastDateTime("T_UNIT");
 
-        logger.info("start checkUpdateTask on T_UNIT from:{}. to:{}", fromDateTime, toDateTime);
+            logger.info("start checkUpdateTask on T_UNIT from:{}. to:{}", fromDateTime, toDateTime);
 
-        try {
-            updateUaList = unitMapper.findUpdateUaList(fromDateTime, toDateTime);
+            try {
+                updateUaList = unitMapper.findUpdateUaList(fromDateTime, toDateTime);
 
-            logger.debug("T_UNIT update list size = {}", updateUaList.size());
+                logger.debug("T_UNIT update list size = {}", updateUaList.size());
 
-            if (!updateUaList.isEmpty()) {
-                elasticUpdateService.handlerDataChange("unitId", updateUaList);
+                if (!updateUaList.isEmpty()) {
+                    elasticUpdateService.handlerDataChange("unitId", updateUaList);
+                }
+                checkTimeService.setLastDateTime("T_UNIT", curCheckTime);
+                updateUaList.clear();
+            } catch (Exception e) {
+                e.printStackTrace();
             }
-            checkTimeService.setLastDateTime("T_UNIT", curCheckTime);
-            updateUaList.clear();
-        } catch (Exception e) {
-            e.printStackTrace();
         }
 
-        fromDateTime = checkTimeService.getLastDateTime("T_AGENCY");
+        if (updateAgency) {
+            fromDateTime = checkTimeService.getLastDateTime("T_AGENCY");
 
-        logger.info("start checkUpdateTask on T_AGENCY from:{}. to:{}", fromDateTime, toDateTime);
+            logger.info("start checkUpdateTask on T_AGENCY from:{}. to:{}", fromDateTime, toDateTime);
 
-        try {
-            updateUaList = agencyMapper.findUpdateUaList(fromDateTime, toDateTime);
+            try {
+                updateUaList = agencyMapper.findUpdateUaList(fromDateTime, toDateTime);
 
-            logger.debug("T_AGENCY update list size = {}", updateUaList.size());
+                logger.debug("T_AGENCY update list size = {}", updateUaList.size());
 
-            if (!updateUaList.isEmpty()) {
-                elasticUpdateService.handlerDataChange("agencyId", updateUaList);
+                if (!updateUaList.isEmpty()) {
+                    elasticUpdateService.handlerDataChange("agencyId", updateUaList);
+                }
+                checkTimeService.setLastDateTime("T_AGENCY", curCheckTime);
+                updateUaList.clear();
+            } catch (Exception e) {
+                e.printStackTrace();
             }
-            checkTimeService.setLastDateTime("T_AGENCY", curCheckTime);
-            updateUaList.clear();
-        } catch (Exception e) {
-            e.printStackTrace();
         }
     }
 

elastic-publish-service/src/main/resources/application-beta.yml

@@ -25,7 +25,10 @@ mybatis:
 debug: true
 app:
   index:
+    name: unit-agency
     config: index-text
     reload: true
+    update-unit: true
+    update-agency: false
   scheduling:
     enable: true

elastic-publish-service/src/main/resources/application-dev.yml

@@ -26,7 +26,10 @@ mybatis:
 debug: true
 app:
   index:
+    name: unit-agency
     config: index-text
     reload: true
+    update-unit: true
+    update-agency: false
   scheduling:
     enable: false

elastic-publish-service/src/main/resources/application-production.yml

@@ -27,7 +27,10 @@ mybatis:
     map-underscore-to-camel-case: true
 app:
   index:
+    name: unit-agency
     config: index-text
     reload: false
+    update-unit: true
+    update-agency: false
   scheduling:
     enable: true

elastic-publish-service/src/test/java/com/vcarecity/publish/elastic/service/impl/ElasticIndexServiceImplTest.java

@@ -1,9 +1,9 @@
 package com.vcarecity.publish.elastic.service.impl;
 
-import com.vcarecity.publish.constants.ElasticConstant;
 import com.vcarecity.publish.elastic.service.ElasticIndexService;
 import org.junit.jupiter.api.Test;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.test.context.TestPropertySource;
 
@@ -16,7 +16,8 @@ class ElasticIndexServiceImplTest {
     @Autowired
     private ElasticIndexService elasticIndexService;
 
-    public static final String INDEX = ElasticConstant.UNIT_AGENCY_INDEX;
+    @Value("${app.index.name}")
+    public String INDEX;
 
     @Test
     void indexExist() throws Exception {