大小写问题

elastic-publish-service/config-index/unit-agency.mapping.json

@@ -0,0 +1,59 @@
+{
+  "properties": {
+    "agencyDetail": {
+      "type": "text",
+      "analyzer": "ik_max_word",
+      "search_analyzer": "ik_smart",
+      "fields": {
+        "keyword": {
+          "type": "keyword",
+          "ignore_above": 256
+        }
+      }
+    },
+    "agencyName": {
+      "type": "text",
+      "analyzer": "ik_max_word",
+      "search_analyzer": "ik_smart",
+      "fields": {
+        "keyword": {
+          "type": "keyword",
+          "ignore_above": 256
+        }
+      }
+    },
+    "nameOfBuilding": {
+      "type": "text",
+      "analyzer": "ik_max_word",
+      "search_analyzer": "ik_smart",
+      "fields": {
+        "keyword": {
+          "type": "keyword",
+          "ignore_above": 256
+        }
+      }
+    },
+    "unitType": {
+      "type": "text",
+      "analyzer": "ik_max_word",
+      "search_analyzer": "ik_smart",
+      "fields": {
+        "keyword": {
+          "type": "keyword",
+          "ignore_above": 256
+        }
+      }
+    },
+    "dtuNo": {
+      "type": "text",
+      "analyzer": "caseSensitive",
+      "search_analyzer": "caseSensitive",
+      "fields": {
+        "keyword": {
+          "type": "keyword",
+          "ignore_above": 256
+        }
+      }
+    }
+  }
+}

elastic-publish-service/config-index/unit-agency.setting.json

@@ -0,0 +1,11 @@
+{
+  "analysis": {
+    "analyzer": {
+      "caseSensitive": {
+        "filter": "lowercase",
+        "type": "custom",
+        "tokenizer": "keyword"
+      }
+    }
+  }
+}

elastic-publish-service/index-text/unit-agency.mapping.json

@@ -0,0 +1,59 @@
+{
+  "properties": {
+    "agencyDetail": {
+      "type": "text",
+      "analyzer": "case_sensitive",
+      "search_analyzer": "case_sensitive",
+      "fields": {
+        "keyword": {
+          "type": "keyword",
+          "ignore_above": 256
+        }
+      }
+    },
+    "agencyName": {
+      "type": "text",
+      "analyzer": "case_sensitive",
+      "search_analyzer": "case_sensitive",
+      "fields": {
+        "keyword": {
+          "type": "keyword",
+          "ignore_above": 256
+        }
+      }
+    },
+    "nameOfBuilding": {
+      "type": "text",
+      "analyzer": "case_sensitive",
+      "search_analyzer": "case_sensitive",
+      "fields": {
+        "keyword": {
+          "type": "keyword",
+          "ignore_above": 256
+        }
+      }
+    },
+    "unitType": {
+      "type": "text",
+      "analyzer": "case_sensitive",
+      "search_analyzer": "case_sensitive",
+      "fields": {
+        "keyword": {
+          "type": "keyword",
+          "ignore_above": 256
+        }
+      }
+    },
+    "dtuNo": {
+      "type": "text",
+      "analyzer": "case_sensitive",
+      "search_analyzer": "case_sensitive",
+      "fields": {
+        "keyword": {
+          "type": "keyword",
+          "ignore_above": 256
+        }
+      }
+    }
+  }
+}

elastic-publish-service/index-text/unit-agency.setting.json

@@ -0,0 +1,11 @@
+{
+  "analysis": {
+    "analyzer": {
+      "case_sensitive": {
+        "filter": "lowercase",
+        "type": "custom",
+        "tokenizer": "keyword"
+      }
+    }
+  }
+}

elastic-publish-service/index/unit-agency.mapping.json

@@ -0,0 +1,59 @@
+{
+  "properties": {
+    "agencyDetail": {
+      "type": "text",
+      "analyzer": "ik_max_word",
+      "search_analyzer": "ik_smart",
+      "fields": {
+        "keyword": {
+          "type": "keyword",
+          "ignore_above": 256
+        }
+      }
+    },
+    "agencyName": {
+      "type": "text",
+      "analyzer": "ik_max_word",
+      "search_analyzer": "ik_smart",
+      "fields": {
+        "keyword": {
+          "type": "keyword",
+          "ignore_above": 256
+        }
+      }
+    },
+    "nameOfBuilding": {
+      "type": "text",
+      "analyzer": "ik_max_word",
+      "search_analyzer": "ik_smart",
+      "fields": {
+        "keyword": {
+          "type": "keyword",
+          "ignore_above": 256
+        }
+      }
+    },
+    "unitType": {
+      "type": "text",
+      "analyzer": "ik_max_word",
+      "search_analyzer": "ik_smart",
+      "fields": {
+        "keyword": {
+          "type": "keyword",
+          "ignore_above": 256
+        }
+      }
+    },
+    "dtuNo": {
+      "type": "text",
+      "analyzer": "caseSensitive",
+      "search_analyzer": "caseSensitive",
+      "fields": {
+        "keyword": {
+          "type": "keyword",
+          "ignore_above": 256
+        }
+      }
+    }
+  }
+}

elastic-publish-service/index/unit-agency.setting.json

@@ -0,0 +1,11 @@
+{
+  "analysis": {
+    "analyzer": {
+      "caseSensitive": {
+        "filter": "lowercase",
+        "type": "custom",
+        "tokenizer": "keyword"
+      }
+    }
+  }
+}

elastic-publish-service/req/http-client.env.json

@@ -1,6 +1,6 @@
 {
   "dev": {
-    "host": "http://192.168.10.179:7080"
+    "host": "http://192.168.10.179:8092"
   },
   "beta": {
     "host": "http://192.168.10.100:8092"

elastic-publish-service/req/main-controller.http

@@ -3,7 +3,7 @@ Content-Type: application/json
 
 {
   "agencyPath": "/1",
-  "keyword": "SV190102725618",
+  "keyword": "重庆电子工程职业学院GLTL重庆高校",
   "page": 1,
   "size": 10,
   "regulatoryLevelId": "",
@@ -21,7 +21,7 @@ Content-Type: application/json
 
 {
   "agencyPath": "/1",
-  "keyword": "RSV1901027256184",
+  "keyword": "卡拉ok",
   "page": 1,
   "size": 10,
   "regulatoryLevelId": "",

elastic-publish-service/src/main/java/com/vcarecity/publish/controller/MainController.java

@@ -9,6 +9,7 @@ import com.vcarecity.publish.pojo.dto.TotalResultDTO;
 import com.vcarecity.publish.pojo.dto.UnitAgencyDTO;
 import com.vcarecity.publish.pojo.dto.UnitIdDTO;
 import com.vcarecity.publish.pojo.query.UnitAgencyQuery;
+import com.vcarecity.publish.service.LoadDataStatusService;
 import com.vcarecity.publish.sql.service.UnitService;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Value;
@@ -17,7 +18,6 @@ import org.springframework.web.bind.annotation.*;
 import javax.validation.Valid;
 import java.io.IOException;
 import java.util.List;
-import java.util.concurrent.atomic.AtomicInteger;
 
 /**
  * @author Kerry on 19/12/11
@@ -27,21 +27,20 @@ import java.util.concurrent.atomic.AtomicInteger;
 @RestController
 public class MainController implements PrepareController {
 
-    private AtomicInteger dataStatus;
 
     private final UnitService unitService;
     private final ElasticLoadDataService elasticLoadDataService;
+    private final LoadDataStatusService loadDataStatusService;
 
     public MainController(UnitService unitService,
                           ElasticLoadDataService elasticLoadDataService,
+                          LoadDataStatusService loadDataStatusService,
                           @Value("${app.scheduling.enable}") Boolean schedulingEnable) {
         this.unitService = unitService;
         this.elasticLoadDataService = elasticLoadDataService;
-
-        if (schedulingEnable) {
-            dataStatus = new AtomicInteger(ElasticConstant.DATA_STATUS_LOADED);
-        } else {
-            dataStatus = new AtomicInteger(ElasticConstant.DATA_STATUS_NOT_LOAD);
+        this.loadDataStatusService = loadDataStatusService;
+        if (!schedulingEnable) {
+            loadDataStatusService.setStatus(ElasticConstant.DATA_STATUS_LOADED);
         }
     }
 
@@ -50,20 +49,20 @@ public class MainController implements PrepareController {
     @RequestMapping(value = "/load", method = {RequestMethod.GET, RequestMethod.POST})
     public ApiResult loadData() {
         synchronized (this) {
-            if (dataStatus.get() == ElasticConstant.DATA_STATUS_NOT_LOAD) {
+            if (loadDataStatusService.getStatus() == ElasticConstant.DATA_STATUS_NOT_LOAD) {
                 logger.info("prepare to load data into elasticsearch");
-                dataStatus.set(ElasticConstant.DATA_STATUS_LOADING);
+                loadDataStatusService.setStatus(ElasticConstant.DATA_STATUS_LOADING);
                 new Thread(() -> {
                     try {
                         elasticLoadDataService.loadData();
-                        dataStatus.set(ElasticConstant.DATA_STATUS_LOADED);
+                        loadDataStatusService.setStatus(ElasticConstant.DATA_STATUS_LOADED);
                     } catch (IOException e) {
                         e.printStackTrace();
-                        dataStatus.set(ElasticConstant.DATA_STATUS_NOT_LOAD);
+                        loadDataStatusService.setStatus(ElasticConstant.DATA_STATUS_NOT_LOAD);
                     }
                 }).start();
                 return ApiResult.builder().status(40000).message("first load data into elasticsearch").build();
-            } else if (dataStatus.get() == ElasticConstant.DATA_STATUS_LOADING) {
+            } else if (loadDataStatusService.getStatus() == ElasticConstant.DATA_STATUS_LOADING) {
                 return ApiResult.builder().status(40000).message("loading data into elasticsearch").build();
             }
         }
@@ -97,6 +96,6 @@ public class MainController implements PrepareController {
     @SkipMethod
     @Override
     public int getDataStatus() {
-        return dataStatus.get();
+        return loadDataStatusService.getStatus();
     }
 }

elastic-publish-service/src/main/java/com/vcarecity/publish/elastic/service/impl/ElasticIndexServiceImpl.java

@@ -1,6 +1,7 @@
 package com.vcarecity.publish.elastic.service.impl;
 
 import com.vcarecity.publish.elastic.service.ElasticIndexService;
+import com.vcarecity.publish.util.Utils;
 import lombok.extern.slf4j.Slf4j;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
 import org.elasticsearch.client.RequestOptions;
@@ -9,8 +10,10 @@ import org.elasticsearch.client.indices.CreateIndexRequest;
 import org.elasticsearch.client.indices.CreateIndexResponse;
 import org.elasticsearch.client.indices.GetIndexRequest;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 
+import java.io.File;
 import java.io.IOException;
 import java.util.List;
 
@@ -22,6 +25,9 @@ import java.util.List;
 @Service
 public class ElasticIndexServiceImpl implements ElasticIndexService {
 
+    @Value("${app.index.config}")
+    private String indexConfig;
+
     private final RestHighLevelClient restHighLevelClient;
 
     public ElasticIndexServiceImpl(RestHighLevelClient restHighLevelClient) {
@@ -36,36 +42,24 @@ public class ElasticIndexServiceImpl implements ElasticIndexService {
 
     @Override
     public boolean createIndex(String index) throws IOException {
-
         CreateIndexRequest request = new CreateIndexRequest(index);
-
-        // mapping
-        //language=JSON
-        String mapping = "{\n" +
-                "  \"properties\": {\n" +
-                "    \"agencyName\": {\n" +
-                "      \"type\": \"text\",\n" +
-                "      \"analyzer\": \"ik_max_word\",\n" +
-                "      \"search_analyzer\": \"ik_smart\"\n" +
-                "    },\n" +
-                "    \"agencyDetail\": {\n" +
-                "      \"type\": \"text\",\n" +
-                "      \"analyzer\": \"ik_max_word\",\n" +
-                "      \"search_analyzer\": \"ik_smart\"\n" +
-                "    },\n" +
-                "    \"nameOfBuilding\": {\n" +
-                "      \"type\": \"text\",\n" +
-                "      \"analyzer\": \"ik_max_word\",\n" +
-                "      \"search_analyzer\": \"ik_smart\"\n" +
-                "    }\n" +
-                "  " +
-                "}\n" +
-                "}";
-
-        request.mapping(mapping, XContentType.JSON);
+        try {
+            File file = new File(indexConfig + "/" + index + ".setting.json");
+            String setting = Utils.readFile(file);
+            request.settings(setting, XContentType.JSON);
+        } catch (IOException e) {
+            logger.warn("setting file not found. usage default setting config");
+        }
+
+        try {
+            File file = new File(indexConfig + "/" + index + ".mapping.json");
+            String mapping = Utils.readFile(file);
+            request.mapping(mapping, XContentType.JSON);
+        } catch (IOException e) {
+            logger.warn("mapping file not found. usage default mapping config");
+        }
 
         CreateIndexResponse createIndexResponse = restHighLevelClient.indices().create(request, RequestOptions.DEFAULT);
-
         return createIndexResponse.isAcknowledged();
 
     }

elastic-publish-service/src/main/java/com/vcarecity/publish/elastic/service/impl/ElasticLoadDataServiceImpl.java

@@ -25,6 +25,7 @@ public class ElasticLoadDataServiceImpl implements ElasticLoadDataService {
     private final ElasticUpdateService elasticUpdateService;
     private final ElasticIndexService elasticIndexService;
     private final int pageCount;
+    private final String index;
 
     public ElasticLoadDataServiceImpl(UnitMapper unitMapper,
                                       ElasticUpdateService elasticUpdateService,
@@ -33,18 +34,22 @@ public class ElasticLoadDataServiceImpl implements ElasticLoadDataService {
         this.elasticUpdateService = elasticUpdateService;
         this.elasticIndexService = elasticIndexService;
         this.pageCount = ElasticConstant.QUERY_PAGE_COUNT;
+        this.index = ElasticConstant.UNIT_AGENCY_INDEX;
     }
 
 
     @Override
     public void loadData() throws IOException {
 
-        boolean exist = elasticIndexService.indexExist(ElasticConstant.UNIT_AGENCY_INDEX);
+        boolean exist = elasticIndexService.indexExist(index);
         if (exist) {
-            logger.warn("elastic index = [{}] already exits", ElasticConstant.UNIT_AGENCY_INDEX);
-            elasticIndexService.deleteIndex(ElasticConstant.UNIT_AGENCY_INDEX);
+            logger.warn("elastic index = [{}] already exits", index);
+            elasticIndexService.deleteIndex(index);
         }
 
+        boolean createResult = elasticIndexService.createIndex(this.index);
+        logger.info("create elastic index = [{}], create result = {}", index, createResult);
+
         long allTime = System.currentTimeMillis();
 
         // calc handler count

elastic-publish-service/src/main/java/com/vcarecity/publish/pojo/dto/UnitAgencyDTO.java

@@ -12,8 +12,8 @@ import lombok.ToString;
 @EqualsAndHashCode(callSuper = true)
 @Data
 public class UnitAgencyDTO extends UnitIdDTO {
+    private String dtuNo;
     private String agencyName;
-    private String agencyDetail;
     private String nameOfBuilding;
     private String unitType;
     private String unitUpdateStamp;

elastic-publish-service/src/main/java/com/vcarecity/publish/service/LoadDataStatusService.java

@@ -0,0 +1,53 @@
+package com.vcarecity.publish.service;
+
+/**
+ * @author VcKerry on 12/26/19
+ */
+
+public interface LoadDataStatusService {
+
+    /**
+     * 数据未加载
+     */
+    int DATA_STATUS_NOT_LOAD = -1;
+    /**
+     * 数据加载中
+     */
+    int DATA_STATUS_LOADING = 0;
+    /**
+     * 数据加载完成
+     */
+    int DATA_STATUS_LOADED = 1;
+
+
+    /**
+     * 获取当前状态
+     *
+     * @return
+     */
+    int getStatus();
+
+    /**
+     * 已经加载过了?
+     *
+     * @return
+     */
+    default boolean isLoaded() {
+        return getStatus() == DATA_STATUS_LOADED;
+    }
+
+    /**
+     * set status
+     *
+     * @param newStatus
+     */
+    void setStatus(int newStatus);
+
+    /**
+     * set loaded
+     */
+    default void setLoaded() {
+        setStatus(DATA_STATUS_LOADED);
+    }
+
+}

elastic-publish-service/src/main/java/com/vcarecity/publish/service/impl/LoadDataStatusServiceImpl.java

@@ -0,0 +1,30 @@
+package com.vcarecity.publish.service.impl;
+
+import com.vcarecity.publish.service.LoadDataStatusService;
+import org.springframework.stereotype.Service;
+
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * @author VcKerry on 12/26/19
+ */
+
+@Service
+public class LoadDataStatusServiceImpl implements LoadDataStatusService {
+
+    private AtomicInteger dataStatus;
+
+    public LoadDataStatusServiceImpl() {
+        this.dataStatus = new AtomicInteger(DATA_STATUS_NOT_LOAD);
+    }
+
+    @Override
+    public int getStatus() {
+        return dataStatus.get();
+    }
+
+    @Override
+    public void setStatus(int newStatus) {
+        dataStatus.set(newStatus);
+    }
+}

elastic-publish-service/src/main/java/com/vcarecity/publish/sql/service/impl/UnitServiceImpl.java

@@ -38,12 +38,19 @@ public class UnitServiceImpl implements UnitService {
     private final RestHighLevelClient restHighLevelClient;
     private final ObjectMapper objectMapper;
 
-    static final String[] SEARCH_KEY_NAMES = new String[]{
+    /*static final String[] SEARCH_KEY_NAMES = new String[]{
             "dtuNo.keyword",
             "agencyName.keyword",
-            //"agencyDetail.keyword",
             "unitType.keyword",
             "nameOfBuilding.keyword",
+    };*/
+
+
+    static final String[] SEARCH_KEY_NAMES = new String[]{
+            "dtuNo",
+            "agencyName",
+            "unitType",
+            "nameOfBuilding",
     };
 
     static final String[] MATCH_KEY_NAMES = new String[]{
@@ -330,7 +337,7 @@ public class UnitServiceImpl implements UnitService {
 
         for (String kn : SEARCH_KEY_NAMES) {
             QueryBuilder queryBuilder = QueryBuilders.wildcardQuery(kn, "*" + keyword + "*");
-            // QueryBuilder queryBuilder = QueryBuilders.matchQuery(kn, keyword);
+            // QueryBuilder queryBuilder = QueryBuilders.(kn, keyword);
             boolQueryBuilder.should(queryBuilder);
         }
         return boolQueryBuilder;

elastic-publish-service/src/main/java/com/vcarecity/publish/task/CheckUpdateTask.java

@@ -6,6 +6,7 @@ import com.vcarecity.publish.entity.UnitAgencyMergeEntity;
 import com.vcarecity.publish.mapper.AgencyMapper;
 import com.vcarecity.publish.mapper.UnitMapper;
 import com.vcarecity.publish.service.CheckTimeService;
+import com.vcarecity.publish.service.LoadDataStatusService;
 import com.vcarecity.publish.util.DateUtil;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.scheduling.annotation.Scheduled;
@@ -26,16 +27,18 @@ public class CheckUpdateTask {
 
     private final CheckTimeService checkTimeService;
     private final ElasticUpdateService elasticUpdateService;
+    private final LoadDataStatusService loadDataStatusService;
     private final AgencyMapper agencyMapper;
     private final UnitMapper unitMapper;
 
 
     public CheckUpdateTask(CheckTimeService checkTimeService,
                            ElasticUpdateService elasticUpdateService,
-                           AgencyMapper agencyMapper,
+                           LoadDataStatusService loadDataStatusService, AgencyMapper agencyMapper,
                            UnitMapper unitMapper) {
         this.checkTimeService = checkTimeService;
         this.elasticUpdateService = elasticUpdateService;
+        this.loadDataStatusService = loadDataStatusService;
         this.agencyMapper = agencyMapper;
         this.unitMapper = unitMapper;
     }
@@ -44,6 +47,11 @@ public class CheckUpdateTask {
     @Scheduled(initialDelay = 3 * 1000L, fixedDelay = ElasticConstant.CHECK_TIME_INTERVAL)
     public void checkUpdateTask() {
 
+        if (!loadDataStatusService.isLoaded()) {
+            return;
+        }
+
+
         // 当前更新时间
         long curCheckTime = System.currentTimeMillis();
         String toDateTime = LocalDateTime.ofInstant(new Date(curCheckTime).toInstant(), ZoneId.systemDefault()).format(DateUtil.DATE_TIME_FORMATTER);

elastic-publish-service/src/main/java/com/vcarecity/publish/util/Utils.java

@@ -1,5 +1,11 @@
 package com.vcarecity.publish.util;
 
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.FileSystemNotFoundException;
+import java.nio.file.Files;
+import java.util.List;
+
 /**
  * @author VcKerry on 12/19/19
  */
@@ -29,4 +35,17 @@ public class Utils {
     }
 
 
+    public static String readFile(String filePath) throws IOException {
+        return readFile(new File(filePath));
+    }
+
+    public static String readFile(File file) throws IOException {
+        if (!file.exists()) {
+            throw new FileSystemNotFoundException(file.getAbsolutePath());
+        }
+        List<String> strings = Files.readAllLines(file.toPath());
+        return String.join("\n", strings);
+    }
+
+
 }

elastic-publish-service/src/main/resources/application-beta.yml

@@ -24,5 +24,7 @@ mybatis:
     map-underscore-to-camel-case: true
 debug: true
 app:
+  index:
+    config: index-text
   scheduling:
     enable: true

elastic-publish-service/src/main/resources/application-dev.yml

@@ -1,5 +1,5 @@
 server:
-  port: 7080
+  port: 8092
 spring:
   datasource:
     url: jdbc:mysql:replication://192.168.10.210:6446,192.168.10.210:6447/fmmp?serverTimezone=Asia/Shanghai
@@ -11,7 +11,7 @@ spring:
         - http://192.168.10.241:9200
         - http://192.168.10.242:9200
         - http://192.168.10.243:9200
-      username: elastic
+      username: kerry
       password: abcd!234
   redis:
     database: 2
@@ -25,5 +25,7 @@ mybatis:
     map-underscore-to-camel-case: true
 debug: true
 app:
+  index:
+    config: index-text
   scheduling:
     enable: false

elastic-publish-service/src/test/java/com/vcarecity/publish/elastic/service/impl/ElasticIndexServiceImplTest.java

@@ -1,26 +1,35 @@
 package com.vcarecity.publish.elastic.service.impl;
 
+import com.vcarecity.publish.constants.ElasticConstant;
 import com.vcarecity.publish.elastic.service.ElasticIndexService;
 import org.junit.jupiter.api.Test;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.test.context.TestPropertySource;
 
 import java.io.IOException;
 
+@TestPropertySource(properties = "app.scheduling.enable=false")
 @SpringBootTest
 class ElasticIndexServiceImplTest {
 
     @Autowired
     private ElasticIndexService elasticIndexService;
 
+    public static final String INDEX = ElasticConstant.UNIT_AGENCY_INDEX;
 
     @Test
-    void indexExist() {
+    void indexExist() throws Exception {
+        boolean exist = elasticIndexService.indexExist(INDEX);
+        if (exist) {
+            elasticIndexService.deleteIndex(INDEX);
+        }
+        elasticIndexService.createIndex(INDEX);
     }
 
     @Test
     void createIndex() throws IOException {
-        elasticIndexService.createIndex("test");
+
     }
 
     @Test

elastic-publish-service/src/test/java/com/vcarecity/publish/util/UtilsTest.java

@@ -0,0 +1,18 @@
+package com.vcarecity.publish.util;
+
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+class UtilsTest {
+
+    @Test
+    void readFile() throws IOException {
+        String s = Utils.readFile("mapping/unit-agency.mapping.json");
+        System.out.println("-----");
+        System.out.println(s);
+        System.out.println("-----");
+    }
+}