add Scheduled

common/src/main/java/com/vcarecity/elastic/constants/ElasticConstant.java

     public static final DateTimeFormatter ES_DATE_FORMATTER = DateTimeFormatter.ofPattern(ES_DATE);
 
     public static final String ES_AGENCY_ID_PREFIX = "aId";
+
+
+    public static final String UNIT_AGENCY_INDEX = "unit-agency";
 }

elastic-load-data/src/main/java/com/vcarecity/test/service/elastic/ElasticSaveServiceImpl.java

 
         List<Map> saveResult = new ArrayList<>(result.size());
 
-        for (UnitAgencyEntity uae : result) {
+
+            for (UnitAgencyEntity uae : result) {
             final List<Integer> agencyIdList = AgencyPathUtil.splitAgencyId(uae.getAgencyPath());
             List<String> agencyPathDetail = agencyService.getAgencyPathDetail(agencyIdList);
 
             UnitAgencyDetailEntity unitAgencyDetailEntity = new UnitAgencyDetailEntity();
 
-
             beanCopier.copy(uae, unitAgencyDetailEntity, null);
 
             unitAgencyDetailEntity.setAgencyDetail(String.join("", agencyPathDetail));
                 bulkRequest.add(indexRequest);
             }
             if (hasData) {
+                long time = System.currentTimeMillis();
                 BulkResponse bulkResponse = restHighLevelClient.bulk(bulkRequest, RequestOptions.DEFAULT);
-                logger.info("bulk insert status {}", bulkResponse.status());
+                logger.info("bulk insert status {}. usage: {} /ms", bulkResponse.status(), (System.currentTimeMillis() - time));
             }
         } catch (IOException e) {
             e.printStackTrace();

elastic-load-data/src/main/java/com/vcarecity/test/service/elastic/QueryDataServiceImpl.java

 package com.vcarecity.test.service.elastic;
 
 import com.google.inject.Inject;
+import com.vcarecity.elastic.constants.ElasticConstant;
 import com.vcarecity.elastic.util.AgencyPathUtil;
 import com.vcarecity.test.service.QueryDataService;
 import org.elasticsearch.action.search.SearchRequest;
 
     private final RestHighLevelClient restHighLevelClient;
 
+    private final String index;
+
     @Inject
     public QueryDataServiceImpl(RestHighLevelClient restHighLevelClient) {
         this.restHighLevelClient = restHighLevelClient;
+        this.index = ElasticConstant.UNIT_AGENCY_INDEX;
     }
 
 
         List<Integer> agencyIds = AgencyPathUtil.splitAgencyId(agencyPath);
         BoolQueryBuilder boolQueryBuilder = agencyCondition(agencyIds);
 
-        String index = "unit-agency";
+
 
 
         SearchRequest searchRequest = new SearchRequest();

elastic-load-data/src/main/java/com/vcarecity/test/service/mysql/MySQLObtainDataServiceImpl.java

 @Slf4j
 public class MySQLObtainDataServiceImpl implements ObtainDataService {
 
-    //language=mysql
+    //language=sql
     private static final String OB_SQL = "SELECT TU.UNIT_ID,\n" +
             "       TU.DTU_NO,\n" +
             "       TU.PUSER_CODE,\n" +
             "FROM T_UNIT TU\n" +
             "         INNER JOIN T_AGENCY TAG ON TU.AGENCY_ID = TAG.AGENCY_ID AND TU.ISDELETED = 0\n" +
             "         INNER JOIN T_BUILDING TB ON TB.BUILDING_ID = TU.BUILDING_ID\n" +
-            "         INNER JOIN T_UNIT_TYPE TUT ON TUT.UNIT_TYPE_ID = TU.UNIT_TYPE_ID  limit ?,?";
+            "         INNER JOIN T_UNIT_TYPE TUT ON TUT.UNIT_TYPE_ID = TU.UNIT_TYPE_ID limit ?,?";
 
     private final DataSource dataSource;
     private final TypeHandler typeHandler;

elastic-publish-service/src/main/java/com/vcarecity/publish/ElasticPublishApplication.java

 import org.mybatis.spring.annotation.MapperScan;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.scheduling.annotation.EnableScheduling;
 
 /**
  * @author Kerry on 19/12/11
  */
 
+
+@EnableScheduling
 @MapperScan("com.vcarecity.publish.mapper")
 @SpringBootApplication
 public class ElasticPublishApplication {

elastic-publish-service/src/main/java/com/vcarecity/publish/api/ApiResult.java

+package com.vcarecity.publish.api;
+
+import lombok.Builder;
+
+/**
+ * @author VcKerry on 12/18/19
+ */
+
+@Builder
+public class ApiResult {
+    private int status;
+    private String message;
+    private int errorCode;
+    private Object data;
+}

elastic-publish-service/src/main/java/com/vcarecity/publish/api/ApiStatus.java

+package com.vcarecity.publish.api;
+
+/**
+ * @author VcKerry on 12/18/19
+ */
+
+public class ApiStatus {
+    public static final int SUCCESS_CODE = 20000;
+    public static final int FAILED_CODE = 40000;
+}

elastic-publish-service/src/main/java/com/vcarecity/publish/constants/CheckConstant.java

-package com.vcarecity.publish.constants;
-
-/**
- * @author VcKerry on 12/18/19
- */
-
-public class CheckConstant {
-    public static final long CHECK_TIME_INTERVAL = 1000 * 3 * 60;
-}

elastic-publish-service/src/main/java/com/vcarecity/publish/constants/ElasticConstant.java

+package com.vcarecity.publish.constants;
+
+/**
+ * @author VcKerry on 12/18/19
+ */
+
+public class ElasticConstant {
+    public static final long CHECK_TIME_INTERVAL = 1000 * 3 * 60;
+
+    public static final String UNIT_AGENCY_INDEX = "unit-agency";
+
+
+    public static final int QUERY_PAGE_COUNT = 2000;
+}

elastic-publish-service/src/main/java/com/vcarecity/publish/controller/MainController.java

 package com.vcarecity.publish.controller;
 
+import com.vcarecity.publish.api.ApiResult;
+import com.vcarecity.publish.api.ApiStatus;
+import com.vcarecity.publish.constants.ElasticConstant;
 import com.vcarecity.publish.pojo.dto.UnitAgencyDTO;
 import com.vcarecity.publish.pojo.query.UnitAgencyQuery;
 import com.vcarecity.publish.sql.service.UnitService;
     }
 
     @PostMapping("/query-unit-agency")
-    public Object queryUnitAgency(@RequestBody @Valid UnitAgencyQuery unitAgencyQuery) throws IOException {
+    public ApiResult queryUnitAgency(@RequestBody @Valid UnitAgencyQuery unitAgencyQuery) throws IOException {
         final List<UnitAgencyDTO> unitAgencyByQuery = unitService.getUnitAgencyByQuery(unitAgencyQuery.getAgencyPath(), unitAgencyQuery.getKeyword(),
-                unitAgencyQuery.getPage(), unitAgencyQuery.getSize(), "unit-agency");
-        return unitAgencyByQuery.stream().map(UnitAgencyDTO::getUnitId).collect(Collectors.toList());
+                unitAgencyQuery.getPage(), unitAgencyQuery.getSize(), ElasticConstant.UNIT_AGENCY_INDEX);
+        List<Long> data = unitAgencyByQuery.stream().map(UnitAgencyDTO::getUnitId).collect(Collectors.toList());
+        return ApiResult.builder().status(ApiStatus.SUCCESS_CODE).data(data).build();
     }
 
     @PostMapping("/test/query-unit-agency")
-    public Object testQueryUnitAgency(@RequestBody @Valid UnitAgencyQuery unitAgencyQuery) throws IOException {
-        return unitService.getUnitAgencyByQuery(unitAgencyQuery.getAgencyPath(), unitAgencyQuery.getKeyword(),
-                unitAgencyQuery.getPage(), unitAgencyQuery.getSize(), "unit-agency");
+    public ApiResult testQueryUnitAgency(@RequestBody @Valid UnitAgencyQuery unitAgencyQuery) throws IOException {
+        List<UnitAgencyDTO> data = unitService.getUnitAgencyByQuery(unitAgencyQuery.getAgencyPath(), unitAgencyQuery.getKeyword(),
+                unitAgencyQuery.getPage(), unitAgencyQuery.getSize(), ElasticConstant.UNIT_AGENCY_INDEX);
+        return ApiResult.builder().status(ApiStatus.SUCCESS_CODE).data(data).build();
     }
 
 

elastic-publish-service/src/main/java/com/vcarecity/publish/elastic/service/ElasticIndexService.java

 package com.vcarecity.publish.elastic.service;
 
+import java.util.List;
+
 /**
  * @author kerryzhang on 2019/12/17
  */
 
 public interface ElasticIndexService {
 
-    void createIndex(String ...indices);
+    /**
+     * index 是否存在
+     *
+     * @param index
+     * @return
+     */
+    boolean indexExist(String index);
+
+    /**
+     * create index
+     *
+     * @param indices
+     */
+    void createIndex(String... indices);
 
+    /**
+     * delete index
+     *
+     * @param indices
+     */
     void deleteIndex(String... indices);
 
 
+    /**
+     * list index
+     *
+     * @return
+     */
+    List<String> indices();
+
 }

elastic-publish-service/src/main/java/com/vcarecity/publish/elastic/service/ElasticLoadDataService.java

 package com.vcarecity.publish.elastic.service;
 
+import java.io.IOException;
+
 /**
  * @author kerryzhang on 2019/12/17
  */
 
 public interface ElasticLoadDataService {
+    /**
+     * load data
+     *
+     * @throws IOException
+     */
+    void loadData() throws IOException;
 }

elastic-publish-service/src/main/java/com/vcarecity/publish/elastic/service/impl/ElasticIndexServiceImpl.java

 package com.vcarecity.publish.elastic.service.impl;
 
 import com.vcarecity.publish.elastic.service.ElasticIndexService;
+import org.elasticsearch.client.RestHighLevelClient;
 import org.springframework.stereotype.Service;
 
+import java.util.List;
+
 /**
  * @author kerryzhang on 2019/12/17
  */
 @Service
 public class ElasticIndexServiceImpl implements ElasticIndexService {
 
+    private final RestHighLevelClient restHighLevelClient;
+
+    public ElasticIndexServiceImpl(RestHighLevelClient restHighLevelClient) {
+        this.restHighLevelClient = restHighLevelClient;
+    }
+
+    @Override
+    public boolean indexExist(String index) {
+
+        return false;
+    }
 
     @Override
     public void createIndex(String... indices) {
     public void deleteIndex(String... indices) {
 
     }
+
+    @Override
+    public List<String> indices() {
+        return null;
+    }
 }

elastic-publish-service/src/main/java/com/vcarecity/publish/elastic/service/impl/ElasticLoadDataServiceImpl.java

 package com.vcarecity.publish.elastic.service.impl;
 
+import com.vcarecity.publish.constants.ElasticConstant;
 import com.vcarecity.publish.elastic.service.ElasticLoadDataService;
+import com.vcarecity.publish.elastic.service.ElasticUpdateService;
+import com.vcarecity.publish.entity.UnitAgencyMergeEntity;
+import com.vcarecity.publish.mapper.UnitMapper;
+import lombok.extern.slf4j.Slf4j;
 import org.springframework.stereotype.Service;
 
+import java.io.IOException;
+import java.util.List;
+
 /**
  * @author kerryzhang on 2019/12/17
  */
 
+@Slf4j
 @Service
 public class ElasticLoadDataServiceImpl implements ElasticLoadDataService {
+
+
+    private final UnitMapper unitMapper;
+    private final ElasticUpdateService elasticUpdateService;
+    private final int pageCount;
+
+    public ElasticLoadDataServiceImpl(UnitMapper unitMapper,
+                                      ElasticUpdateService elasticUpdateService) {
+        this.unitMapper = unitMapper;
+        this.elasticUpdateService = elasticUpdateService;
+        this.pageCount = ElasticConstant.QUERY_PAGE_COUNT;
+    }
+
+
+    @Override
+    public void loadData() throws IOException {
+
+        long allTime = System.currentTimeMillis();
+
+        // calc handler count
+        int count = 0;
+        // page
+        int start = 0;
+
+        List<UnitAgencyMergeEntity> result;
+        do {
+
+            long startTime = System.currentTimeMillis();
+
+            result = unitMapper.findAllWithPage(start * pageCount, pageCount);
+            start++;
+            count += result.size();
+
+            logger.debug("query database usageTime: {} ms. count = {}", (System.currentTimeMillis() - startTime), count);
+
+            startTime = System.currentTimeMillis();
+
+            elasticUpdateService.addElasticData(result);
+
+            logger.debug("write to Elasticsearch usageTime: {} ms.", (System.currentTimeMillis() - startTime));
+
+        } while (result.size() == pageCount);
+
+        logger.info("loadData ok. All usageTime = {} ms.", (System.currentTimeMillis() - allTime));
+
+    }
+
 }

elastic-publish-service/src/main/java/com/vcarecity/publish/elastic/service/impl/ElasticUpdateServiceImpl.java

 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.vcarecity.elastic.util.AgencyPathUtil;
 import com.vcarecity.elastic.util.SnowFlake;
+import com.vcarecity.publish.constants.ElasticConstant;
 import com.vcarecity.publish.elastic.entity.UnitAgencyDetailEntity;
 import com.vcarecity.publish.elastic.service.ElasticUpdateService;
 import com.vcarecity.publish.entity.UnitAgencyMergeEntity;
 @Service
 public class ElasticUpdateServiceImpl implements ElasticUpdateService {
 
-    public static final String INDEX = "unit-agency";
+    public final String index;
 
     private final RestHighLevelClient restHighLevelClient;
     private final ObjectMapper objectMapper;
         this.objectMapper = objectMapper;
         this.agencyService = agencyService;
 
+        this.index = ElasticConstant.UNIT_AGENCY_INDEX;
     }
 
 
         // 更新数据
         List<UnitAgencyMergeEntity> addData = updateDataByQuery(key, updateList);
 
+        logger.info("add data size = {}", addData.size());
+
         // 添加数据
         this.addElasticData(addData);
     }
             return;
         }
         for (Long id : ids) {
-            DeleteByQueryRequest request = new DeleteByQueryRequest(INDEX);
+            DeleteByQueryRequest request = new DeleteByQueryRequest(index);
             request.setQuery(QueryBuilders.termQuery(key, id));
             restHighLevelClient.deleteByQuery(request, RequestOptions.DEFAULT);
         }
             Object value = params.get(key);
             params.remove(key);
 
-            UpdateByQueryRequest request = new UpdateByQueryRequest(INDEX);
+            UpdateByQueryRequest request = new UpdateByQueryRequest(index);
             request.setQuery(QueryBuilders.termQuery(key, value));
             request.setScript(new Script(ScriptType.INLINE, "painless", getPainlessCode(params), params));
 
 
             logger.info("response:{}", response);
 
-            if (response.getUpdated() != 0L) {
+            if (response.getUpdated() == 0L) {
                 addList.add(item);
             }
         }
         try {
             BulkRequest bulkRequest = new BulkRequest();
             for (Map<String, Object> val : saveResult) {
-                IndexRequest indexRequest = createIndexRequest(INDEX, val);
+                IndexRequest indexRequest = createIndexRequest(index, val);
                 bulkRequest.add(indexRequest);
             }
             BulkResponse bulkResponse = restHighLevelClient.bulk(bulkRequest, RequestOptions.DEFAULT);
-            logger.info("bulk insert status {}", bulkResponse.status());
+
+            logger.debug("bulk addElasticData status {}, size = {}", bulkResponse.status(), saveResult.size());
+
         } catch (IOException e) {
             e.printStackTrace();
         }

elastic-publish-service/src/main/java/com/vcarecity/publish/mapper/AgencyMapper.java

      * @param agencyPathIds
      * @return
      */
-    List<String> findAgencyNameByAgencyIdList(@Param("agencyPathIds") List<Integer> agencyPathIds);
+    List<AgencyEntity> findAgencyNameByAgencyIdList(@Param("agencyPathIds") List<Integer> agencyPathIds);
 }

elastic-publish-service/src/main/java/com/vcarecity/publish/mapper/UnitMapper.java

     List<UnitAgencyMergeEntity> findUpdateUaList(@Param("fromDate") String fromDate, @Param("toDate") String toDate);
 
 
+    /**
+     * find all by page
+     *
+     * @param page
+     * @param size
+     * @return
+     */
+    List<UnitAgencyMergeEntity> findAllWithPage(@Param("page") int page, @Param("size") int size);
+
+
 }

elastic-publish-service/src/main/java/com/vcarecity/publish/service/impl/CheckTimeServiceImpl.java

 package com.vcarecity.publish.service.impl;
 
-import com.vcarecity.publish.constants.CheckConstant;
+import com.vcarecity.publish.constants.ElasticConstant;
 import com.vcarecity.publish.service.CheckTimeService;
 import com.vcarecity.publish.util.DateUtil;
 import org.springframework.data.redis.core.RedisTemplate;
             LocalDateTime.ofInstant(new Date(Long.parseLong(value.toString())).toInstant(), ZoneId.systemDefault()).format(DateUtil.DATE_TIME_FORMATTER);
         }
 
-        long last = System.currentTimeMillis() - CheckConstant.CHECK_TIME_INTERVAL;
+        long last = System.currentTimeMillis() - ElasticConstant.CHECK_TIME_INTERVAL;
         redisTemplate.opsForValue().set(redisKey, last);
 
         return LocalDateTime.ofInstant(new Date(last).toInstant(), ZoneId.systemDefault()).format(DateUtil.DATE_TIME_FORMATTER);

elastic-publish-service/src/main/java/com/vcarecity/publish/sql/service/impl/AgencyServiceImpl.java

 package com.vcarecity.publish.sql.service.impl;
 
+import com.vcarecity.publish.entity.AgencyEntity;
 import com.vcarecity.publish.mapper.AgencyMapper;
 import com.vcarecity.publish.sql.service.AgencyService;
 import org.springframework.stereotype.Service;
 
 import java.util.List;
+import java.util.stream.Collectors;
 
 /**
  * @author kerryzhang on 2019/12/17
     public List<String> getAgencyPathDetail(List<Integer> agencyPath) {
 
         //TODO 可以优化的空间
-        return agencyMapper.findAgencyNameByAgencyIdList(agencyPath);
+        List<AgencyEntity> agencyNameByAgencyIdList = agencyMapper.findAgencyNameByAgencyIdList(agencyPath);
+        return agencyNameByAgencyIdList.stream().map(AgencyEntity::getAgencyName).collect(Collectors.toList());
     }
 }

elastic-publish-service/src/main/java/com/vcarecity/publish/sql/service/impl/UnitServiceImpl.java

     private final RestHighLevelClient restHighLevelClient;
     private final ObjectMapper objectMapper;
 
+    static final String[] keyNames = new String[]{
+            "agencyName.keyword",
+            "agencyDetail.keyword",
+            "unitType.keyword",
+            "nameOfBuilding.keyword",
+    };
+
+    static String[] queryColumns = new String[]{"unitId", "nameOfBuilding", "agencyDetail", "agencyName"};
+
     public UnitServiceImpl(RestHighLevelClient restHighLevelClient,
                            ObjectMapper objectMapper) {
         this.restHighLevelClient = restHighLevelClient;
             queryBuilder.must(keywordBoolQuery);
         }
 
-        String[] queryColumns = new String[]{"unitId", "nameOfBuilding", "agencyDetail", "agencyName"};
 
         SearchRequest searchRequest = new SearchRequest();
         SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
             return null;
         }
 
-        final String[] keyNames = new String[]{
-                "agencyName.keyword",
-                "agencyDetail.keyword",
-                "nameOfBuilding.keyword",
-        };
 
         final BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
 

elastic-publish-service/src/main/java/com/vcarecity/publish/task/CheckUpdateTask.java

 package com.vcarecity.publish.task;
 
+import com.vcarecity.publish.constants.ElasticConstant;
+import com.vcarecity.publish.elastic.service.ElasticUpdateService;
 import com.vcarecity.publish.entity.UnitAgencyMergeEntity;
 import com.vcarecity.publish.mapper.AgencyMapper;
 import com.vcarecity.publish.mapper.UnitMapper;
 import com.vcarecity.publish.service.CheckTimeService;
-import com.vcarecity.publish.elastic.service.ElasticUpdateService;
 import com.vcarecity.publish.util.DateUtil;
 import lombok.extern.slf4j.Slf4j;
+import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.stereotype.Component;
 
 import java.time.LocalDateTime;
         this.unitMapper = unitMapper;
     }
 
+
+    @Scheduled(initialDelay = 1000L, fixedDelay = ElasticConstant.CHECK_TIME_INTERVAL)
     public void checkUpdateTask() {
 
         // 当前更新时间

elastic-publish-service/src/main/resources/mapper/AgencyMapper.xml

                  INNER JOIN T_UNIT_TYPE TUT ON TUT.UNIT_TYPE_ID = TU.UNIT_TYPE_ID
     </select>
 
-    <select id="findAgencyNameByAgencyIdList" resultType="string" parameterType="list">
-        select AGENCY_NAME from T_AGENCY
+    <select id="findAgencyNameByAgencyIdList" resultType="agencyEntity" parameterType="list">
+        select AGENCY_NAME,AGENCY_ID from T_AGENCY
         <where>
             AGENCY_ID in
             <foreach item="item" collection="agencyPathIds" separator="," open="(" close=")" index="">

elastic-publish-service/src/main/resources/mapper/UnitMapper.xml

 <?xml version="1.0" encoding="UTF-8" ?>
 <!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd" >
 <mapper namespace="com.vcarecity.publish.mapper.UnitMapper">
-
-    <select id="findUpdateList" resultType="unitEntity">
-        select UNIT_ID, DTU_NO, PUSER_CODE, ISDELETED
-        from T_UNIT
-        where UPDATESTAMP between to_date('${fromDate}', 'YYYY-MM-DD HH24:MI:SS')
-                  and to_date('${toDate}', 'YYYY-MM-DD HH24:MI:SS')
-    </select>
-
     <resultMap id="unitAgencyMergeEntityMap" type="unitAgencyMergeEntity">
         <result column="UNIT_ID" property="unitId"/>
         <result column="DTU_NO" property="dtuNo"/>
         <result column="UNIT_TYPE" property="unitType"/>
     </resultMap>
 
+    <select id="findUpdateList" resultType="unitEntity">
+        select UNIT_ID, DTU_NO, PUSER_CODE, ISDELETED
+        from T_UNIT
+        where UPDATESTAMP between '${fromDate}' and '${toDate}'
+    </select>
+
 
     <select id="findUpdateUaList" resultMap="unitAgencyMergeEntityMap">
         SELECT TU.UNIT_ID,
                  INNER JOIN T_BUILDING TB ON TB.BUILDING_ID = TU.BUILDING_ID
                  INNER JOIN T_UNIT_TYPE TUT ON TUT.UNIT_TYPE_ID = TU.UNIT_TYPE_ID
     </select>
+
+    <select id="findAllWithPage" resultType="unitAgencyMergeEntity">
+        SELECT TU.UNIT_ID,
+               TU.DTU_NO,
+               TU.PUSER_CODE,
+               TAG.AGENCY_ID,
+               TAG.AGENCY_PATH,
+               TAG.AGENCY_NAME,
+               TB.BUILDING_ID,
+               TB.NAME_OF_BUILDING,
+               TUT.UNIT_TYPE_ID,
+               TUT.UNIT_TYPE
+        FROM T_UNIT TU
+                 INNER JOIN T_AGENCY TAG ON TU.AGENCY_ID = TAG.AGENCY_ID AND TU.ISDELETED = 0
+                 INNER JOIN T_BUILDING TB ON TB.BUILDING_ID = TU.BUILDING_ID
+                 INNER JOIN T_UNIT_TYPE TUT ON TUT.UNIT_TYPE_ID = TU.UNIT_TYPE_ID
+        limit ?,?
+    </select>
 </mapper>