init

.gitignore

@@ -0,0 +1,15 @@
+
+
+*.iml
+*.log
+*.class
+*.jar
+*.csv
+*.xls
+*.xlsx
+*.zip
+
+logs/
+target/
+build/
+.idea/

README.md

@@ -0,0 +1,14 @@
+
+
+
+
+````
+LOAD DATA INFILE '/var/lib/mysql-files/check_record_0006'
+INTO TABLE T_CHECK_RECORD_1
+FIELDS 
+    TERMINATED BY '\t'
+    ENCLOSED BY '\"'
+    ESCAPED BY '\\'
+LINES TERMINATED BY '\n'
+(CHECK_RECORD_ID,STAMP,USER_ID,CHECK_POINTS_ID,IS_NORMAL,IS_LATENT_DANGER,LATENT_DANGER_TYPE,LATENT_DANGER_DESC,UNDETECTED_NUM,LON,LAT,RULE_ID,IS_DEAL,STARTDATE,ENDDATE,VALID_CHECK,UPDATESTAMP);
+````

config/application-dev.yml

@@ -0,0 +1,4 @@
+database:
+  url: jdbc:mysql:replication://192.168.10.210:6446,192.168.10.210:6447/fmmp?serverTimezone=Asia/Shanghai
+  user: root
+  password: R00T@mysql

config/application.yml

@@ -0,0 +1,2 @@
+profile:
+  active: dev

pom.xml

@@ -0,0 +1,106 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>com.vcarecity.csv</groupId>
+    <artifactId>db-data-to-csv</artifactId>
+    <version>1.0.0</version>
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <configuration>
+                    <source>8</source>
+                    <target>8</target>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
+
+
+    <dependencies>
+
+        <dependency>
+            <groupId>mysql</groupId>
+            <artifactId>mysql-connector-java</artifactId>
+            <version>8.0.18</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-text</artifactId>
+            <version>1.8</version>
+        </dependency>
+
+
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-csv</artifactId>
+            <version>1.7</version>
+        </dependency>
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-api</artifactId>
+            <version>1.7.29</version>
+        </dependency>
+        <dependency>
+            <groupId>ch.qos.logback</groupId>
+            <artifactId>logback-classic</artifactId>
+            <version>1.2.3</version>
+        </dependency>
+        <dependency>
+            <groupId>org.projectlombok</groupId>
+            <artifactId>lombok</artifactId>
+            <version>1.18.10</version>
+            <scope>provided</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>com.zaxxer</groupId>
+            <artifactId>HikariCP</artifactId>
+            <version>3.4.1</version>
+        </dependency>
+        <dependency>
+            <groupId>com.google.inject</groupId>
+            <artifactId>guice</artifactId>
+            <version>4.2.2</version>
+        </dependency>
+        <dependency>
+            <groupId>com.google.inject.extensions</groupId>
+            <artifactId>guice-assistedinject</artifactId>
+            <version>4.2.2</version>
+        </dependency>
+
+        <dependency>
+            <groupId>com.fasterxml.jackson.dataformat</groupId>
+            <artifactId>jackson-dataformat-yaml</artifactId>
+            <version>2.10.1</version>
+        </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-databind</artifactId>
+            <version>2.10.1</version>
+        </dependency>
+
+        <dependency>
+            <groupId>me.yuxiaoyao.lang</groupId>
+            <artifactId>configuration-loader</artifactId>
+            <version>2.0.0</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.shardingsphere</groupId>
+            <artifactId>sharding-jdbc-core</artifactId>
+            <version>4.0.0-RC2</version>
+        </dependency>
+        <dependency>
+            <groupId>javax.persistence</groupId>
+            <artifactId>persistence-api</artifactId>
+            <version>1.0.2</version>
+        </dependency>
+
+    </dependencies>
+
+</project>

src/main/java/com/vcarecity/cvs/BuildPartitionSQL.java

@@ -0,0 +1,97 @@
+package com.vcarecity.cvs;
+
+import lombok.extern.slf4j.Slf4j;
+
+import java.time.LocalDate;
+import java.time.format.DateTimeFormatter;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * @author Kerry on 19/11/25
+ * <p>
+ * <p>
+ * 创建表分区 datetime
+ */
+
+@Slf4j
+public class BuildPartitionSQL {
+
+    private static final DateTimeFormatter DATE_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd");
+
+
+    public static void main(String[] args) {
+        BuildPartitionSQL buildPartitionSQL = new BuildPartitionSQL();
+        buildPartitionSQL.partitionFirstSql("T_CHECK_RECORD_1", "stamp", "2016-08-23", "2019-11-21", PartitionType.MONTH);
+    }
+
+
+    public String partitionFirstSql(String table, String field, String from, String to, PartitionType partitionType) {
+        return partitionFirstSql(table, field, LocalDate.parse(from, DATE_FORMATTER), LocalDate.parse(to, DATE_FORMATTER), partitionType);
+    }
+
+    public String partitionFirstSql(String table, String field, LocalDate from, LocalDate to, PartitionType partitionType) {
+
+        final String[] strings = partitionSql(from, to, partitionType);
+        final String join = String.join(",\n", strings);
+
+        String pre = "ALTER TABLE " + table + " PARTITION BY RANGE (TO_DAYS(" + field + ")) (";
+
+
+        logger.info("--------------------------first create table partition SQL.--------------------------");
+
+        String s = pre + "\n" + join + "\n" + ")";
+        System.out.println(s);
+
+        logger.info("--------------------------first create table partition SQL.--------------------------");
+        return s;
+    }
+
+    public String[] partitionSql(LocalDate from, LocalDate to, PartitionType partitionType) {
+        LocalDate index = from;
+        to = plusDateByType(to, partitionType);
+
+        List<String> partitionSqls = new ArrayList<>();
+        while (index.isBefore(to)) {
+            String pName = partitionType.getFormatter().format(index);
+            index = plusDateByType(index, partitionType);
+            final String format = DATE_FORMATTER.format(index);
+            String alter = "PARTITION p" + pName + " VALUES LESS THAN (TO_DAYS('" + format + "')) ENGINE = INNODB";
+            partitionSqls.add(alter);
+        }
+        return partitionSqls.toArray(new String[]{});
+    }
+
+    private enum PartitionType {
+        /**
+         * 天
+         */
+        DAY("yyyyMMdd"),
+        /**
+         * 月
+         */
+        MONTH("yyyyMM");
+        DateTimeFormatter formatter;
+
+        PartitionType(String format) {
+            this.formatter = DateTimeFormatter.ofPattern(format);
+        }
+
+        public DateTimeFormatter getFormatter() {
+            return formatter;
+        }
+    }
+
+    private static LocalDate plusDateByType(LocalDate localDate, PartitionType partitionType) {
+        switch (partitionType) {
+            case DAY:
+                return localDate.plusDays(1L);
+            case MONTH:
+                LocalDate date = localDate.plusMonths(1L);
+                return date.minusDays(date.getDayOfMonth() - 1);
+            default:
+                break;
+        }
+        throw new RuntimeException(partitionType.name());
+    }
+}

src/main/java/com/vcarecity/cvs/MySQLCSVWriter.java

@@ -0,0 +1,32 @@
+package com.vcarecity.cvs;
+
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import com.vcarecity.cvs.module.MySQLModule;
+import com.vcarecity.cvs.module.PropertiesModule;
+import com.vcarecity.cvs.properties.AppProperties;
+import me.yuxiaoyao.config.loader.ConfigLoader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * @author Kerry on 19/11/20
+ */
+
+public class MySQLCSVWriter {
+    private static Logger logger = LoggerFactory.getLogger(MySQLCSVWriter.class);
+    public static Injector injector;
+
+    public static void main(String[] args) throws Exception {
+        final AppProperties appProperties = ConfigLoader.parseConfig(args, AppProperties.class, null);
+        injector = Guice.createInjector(new PropertiesModule(appProperties), new MySQLModule());
+
+        //final TestInsertData testInsertData = injector.getInstance(TestInsertData.class);
+        //testInsertData.ins();
+
+        final MySQLStarter instance = injector.getInstance(MySQLStarter.class);
+        instance.run();
+
+
+    }
+}

src/main/java/com/vcarecity/cvs/MySQLStarter.java

@@ -0,0 +1,70 @@
+package com.vcarecity.cvs;
+
+import com.vcarecity.cvs.entity.CheckRecordEntity;
+import com.vcarecity.cvs.factory.ResultHandlerFactory;
+import com.vcarecity.cvs.properties.AppProperties;
+import com.vcarecity.cvs.service.ResultHandlerService;
+import com.vcarecity.cvs.service.SQLQueryService;
+import com.vcarecity.cvs.service.impl.CSVResultHandlerServiceImpl;
+import lombok.extern.slf4j.Slf4j;
+
+import javax.inject.Inject;
+import java.util.List;
+
+/**
+ * @author Kerry on 19/11/20
+ */
+
+@Slf4j
+public class MySQLStarter {
+    private final AppProperties properties;
+    private final SQLQueryService queryService;
+    private final ResultHandlerFactory resultHandlerFactory;
+
+    @Inject
+    public MySQLStarter(AppProperties properties,
+                        SQLQueryService queryService,
+                        ResultHandlerFactory resultHandlerFactory) {
+
+        this.properties = properties;
+        this.queryService = queryService;
+        this.resultHandlerFactory = resultHandlerFactory;
+
+    }
+
+    static final int PAGE_COUNT = 10000;
+
+    public void run() throws Exception {
+
+        logger.info("start query database...");
+
+        long startTime = System.currentTimeMillis();
+
+
+        String sql = "select * from T_CHECK_RECORD limit ?,?";
+        // sql = "SELECT * FROM websites  limit ?,?";
+
+        String properties = "check_record.csv";
+
+        int count = 0;
+        int currentSize;
+
+
+        ResultHandlerService resultHandler = resultHandlerFactory.createResultHandler(properties);
+
+        do {
+            List<CheckRecordEntity> result = queryService.queryByPage(sql, count, PAGE_COUNT, CheckRecordEntity.class);
+            resultHandler.resultHandler(result, CheckRecordEntity.class);
+            currentSize = result.size();
+            count += currentSize;
+
+        } while (currentSize == PAGE_COUNT);
+
+        if (resultHandler instanceof CSVResultHandlerServiceImpl) {
+            ((CSVResultHandlerServiceImpl) resultHandler).close();
+        }
+
+        logger.info("stop query database...{} /ms.", (System.currentTimeMillis() - startTime));
+    }
+
+}

src/main/java/com/vcarecity/cvs/TestInsertData.java

@@ -0,0 +1,55 @@
+package com.vcarecity.cvs;
+
+import com.google.inject.Inject;
+import com.zaxxer.hikari.HikariDataSource;
+
+import java.sql.Connection;
+import java.sql.Date;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+
+/**
+ * @author Kerry on 19/11/25
+ */
+
+public class TestInsertData {
+
+    private final HikariDataSource dataSource;
+
+    @Inject
+    public TestInsertData(HikariDataSource dataSource) {
+        this.dataSource = dataSource;
+    }
+
+    public void ins() throws SQLException {
+        //language=SQL
+        String sql = "insert into T_CHECK_RECORD_2 (STAMP, USER_ID, CHECK_POINTS_ID, IS_NORMAL, IS_LATENT_DANGER,\n" +
+                "                              LATENT_DANGER_TYPE, LATENT_DANGER_DESC, UNDETECTED_NUM, LON, LAT, RULE_ID, IS_DEAL,\n" +
+                "                              STARTDATE, ENDDATE, VALID_CHECK, UPDATESTAMP)\n" +
+                "values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)";
+
+        final Connection connection = dataSource.getConnection();
+        final PreparedStatement preparedStatement = connection.prepareStatement(sql);
+
+        preparedStatement.setDate(1, new Date(System.currentTimeMillis()));
+        preparedStatement.setInt(2, 2834);
+        preparedStatement.setInt(3, 21);
+        preparedStatement.setInt(4, 0);
+        preparedStatement.setInt(5, 8);
+        preparedStatement.setInt(6, 0);
+        preparedStatement.setString(7, "\t");
+        preparedStatement.setInt(8, 0);
+        preparedStatement.setDouble(9, 113.123456);
+        preparedStatement.setDouble(10, 22.123456);
+        preparedStatement.setInt(11, 0);
+        preparedStatement.setInt(12, 0);
+        preparedStatement.setDate(13, null);
+        preparedStatement.setDate(14, null);
+        preparedStatement.setInt(15, 0);
+        preparedStatement.setDate(16, new Date(System.currentTimeMillis() + 1000));
+
+        preparedStatement.execute();
+        preparedStatement.close();
+        connection.close();
+    }
+}

src/main/java/com/vcarecity/cvs/core/AutoMapper.java

@@ -0,0 +1,73 @@
+package com.vcarecity.cvs.core;
+
+import java.lang.reflect.Modifier;
+import java.sql.Date;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Timestamp;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.LocalTime;
+import java.util.List;
+
+/**
+ * @author Kerry on 19/11/21
+ */
+
+public class AutoMapper {
+
+
+    public static <T> T autoMapper(ResultSet resultSet, Class<T> cls) {
+        List<JavaMapColumn> fieldColumnList = ReflectionUtil.getFieldColumnList(cls);
+        try {
+            final T t = cls.newInstance();
+            for (JavaMapColumn fc : fieldColumnList) {
+                if (Modifier.isPrivate(fc.getField().getModifiers())) {
+                    fc.getField().setAccessible(true);
+                }
+                Class<?> type = fc.getField().getType();
+                Object object;
+                if (type == java.util.Date.class) {
+                    Timestamp timestamp = resultSet.getTimestamp(fc.getColumn());
+                    object = Date.from(timestamp.toInstant());
+                } else if (type == LocalDateTime.class) {
+                    Timestamp timestamp = resultSet.getTimestamp(fc.getColumn());
+                    object = timestamp.toLocalDateTime();
+                } else if (type == LocalDate.class) {
+                    Timestamp timestamp = resultSet.getTimestamp(fc.getColumn());
+                    object = timestamp.toLocalDateTime().toLocalDate();
+                } else if (type == LocalTime.class) {
+                    Timestamp timestamp = resultSet.getTimestamp(fc.getColumn());
+                    object = timestamp.toLocalDateTime().toLocalTime();
+                } else {
+                    // Timestamp 时出现 HOUR_OF_DAY 2->3 异常.请在mysql url 中加入 &serverTimezone=Asia/Shanghai
+                    object = resultSet.getObject(fc.getColumn(), type);
+                }
+                fc.getField().set(t, object);
+
+            }
+            return t;
+        } catch (InstantiationException | IllegalAccessException | SQLException e) {
+            e.printStackTrace();
+        }
+        return null;
+    }
+
+
+    public static <T> Object[] autoMapper(T t, Class<T> cls) {
+        List<JavaMapColumn> fieldColumnList = ReflectionUtil.getFieldColumnList(cls);
+        Object[] contents = new Object[fieldColumnList.size()];
+        try {
+            for (int i = 0; i < fieldColumnList.size(); i++) {
+                final JavaMapColumn fc = fieldColumnList.get(i);
+                if (Modifier.isPrivate(fc.getField().getModifiers())) {
+                    fc.getField().setAccessible(true);
+                }
+                contents[i] = fc.getField().get(t);
+            }
+        } catch (IllegalAccessException e) {
+            e.printStackTrace();
+        }
+        return contents;
+    }
+}

src/main/java/com/vcarecity/cvs/core/JavaMapColumn.java

@@ -0,0 +1,15 @@
+package com.vcarecity.cvs.core;
+
+import lombok.Data;
+
+import java.lang.reflect.Field;
+
+/**
+ * @author Kerry on 19/11/21
+ */
+
+@Data
+public class JavaMapColumn {
+    private final Field field;
+    private final String column;
+}

src/main/java/com/vcarecity/cvs/core/ReflectionUtil.java

@@ -0,0 +1,51 @@
+package com.vcarecity.cvs.core;
+
+import javax.persistence.Column;
+import javax.persistence.Transient;
+import java.lang.reflect.Field;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * @author Kerry on 19/11/21
+ */
+
+public class ReflectionUtil {
+
+
+    private static final Map<Class<?>, List<JavaMapColumn>> FIELD_CACHE = new ConcurrentHashMap<>();
+
+
+    public static List<JavaMapColumn> getFieldColumnList(Class<?> cls) {
+
+        List<JavaMapColumn> fields = FIELD_CACHE.get(cls);
+        if (fields != null) {
+            return fields;
+        }
+        synchronized (FIELD_CACHE) {
+            Field[] declaredFields = cls.getDeclaredFields();
+            fields = new ArrayList<>(declaredFields.length);
+
+            for (Field declaredField : declaredFields) {
+                Transient ta = declaredField.getAnnotation(Transient.class);
+                if (ta != null) {
+                    continue;
+                }
+                final Column column = declaredField.getAnnotation(Column.class);
+                String fieldName = declaredField.getName();
+                if (column != null) {
+                    final String name = column.name();
+                    if (name.length() > 0) {
+                        fieldName = name;
+                    }
+                }
+                fields.add(new JavaMapColumn(declaredField, fieldName));
+            }
+            FIELD_CACHE.put(cls, fields);
+            return FIELD_CACHE.get(cls);
+        }
+    }
+
+}

src/main/java/com/vcarecity/cvs/csv/DataBuilder.java

@@ -0,0 +1,137 @@
+package com.vcarecity.cvs.csv;
+
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.Random;
+
+/**
+ * @author Kerry on 19/11/21
+ */
+
+public class DataBuilder {
+
+    public static void main(String[] args) throws IOException {
+        DataBuilder builder = new DataBuilder();
+        builder.builder();
+    }
+
+    public void builder() throws IOException {
+        int count = 1000000;
+        dateTime = LocalDateTime.of(2018, 1, 1, 0, 0, 0);
+
+        String[] header = {
+                "CHECK_RECORD_ID",
+                "STAMP",
+                "USER_ID",
+                "CHECK_POINTS_ID",
+                "IS_NORMAL",
+                "IS_LATENT_DANGER",
+                "LATENT_DANGER_TYPE",
+                "LATENT_DANGER_DESC",
+                "UNDETECTED_NUM",
+                "LON",
+                "LAT",
+                "RULE_ID",
+                "IS_DEAL",
+                "STARTDATE",
+                "ENDDATE",
+                "VALID_CHECK",
+                "UPDATESTAMP"
+        };
+
+
+    }
+
+    public static final String NULL_VALUE = "\\N";
+
+    private final int min_user_id = 2000;
+    private Random random = new Random();
+    private long recordId = 1;
+    private LocalDateTime dateTime;
+    private int userId = min_user_id;
+
+
+    private String[] getContent(int index, int col) {
+        index++;
+        String[] line = new String[col];
+        line[0] = "" + (recordId++);
+        if (index % 1500 == 0) {
+            dateTime = dateTime.plusDays(1L);
+        }
+        line[1] = formatDateTime(dateTime);
+        line[2] = getUserId();
+        line[3] = getUserId();
+        line[4] = String.valueOf(random.nextInt(10));
+        line[5] = String.valueOf(random.nextInt(10));
+        line[6] = String.valueOf(random.nextInt(10));
+
+        line[7] = getMessage();
+
+        line[8] = String.valueOf(random.nextInt(55555));
+        line[9] = "" + Math.random() * 100;
+        line[10] = "" + Math.random() * 100;
+        line[11] = String.valueOf(random.nextInt(100000));
+        line[12] = String.valueOf(random.nextInt(12));
+        line[13] = null;
+        line[14] = null;
+        line[15] = String.valueOf(random.nextInt(12));
+        line[16] = formatDateTime(dateTime);
+
+        for (int i = 0; i < line.length; i++) {
+            if (line[i] == null) {
+                line[i] = NULL_VALUE;
+            }
+        }
+
+        return line;
+    }
+
+    private String getUserId() {
+        int max_user_id = 20000;
+        if (userId > max_user_id) {
+            userId = min_user_id;
+        }
+        return String.valueOf(userId++);
+    }
+
+    public static final DateTimeFormatter FORMATTER = DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss");
+
+    private static String formatDateTime(LocalDateTime localDateTime) {
+        return localDateTime.format(FORMATTER);
+    }
+
+    public String getMessage() {
+        final int i = random.nextInt(60);
+        StringBuffer sb = new StringBuffer();
+        for (int j = 0; j < i; j++) {
+            sb.append(getRandomChar());
+        }
+        return sb.toString();
+    }
+
+    public static String getRandomChar() {
+        String str = "";
+        int highCode;
+        int lowCode;
+
+        Random random = new Random();
+        //B0 + 0~39(16~55) 一级汉字所占区
+        highCode = (176 + Math.abs(random.nextInt(39)));
+        //A1 + 0~93 每区有94个汉字
+        lowCode = (161 + Math.abs(random.nextInt(93)));
+
+        byte[] b = new byte[2];
+        b[0] = (Integer.valueOf(highCode)).byteValue();
+        b[1] = (Integer.valueOf(lowCode)).byteValue();
+
+        try {
+            str = new String(b, "GBK");
+        } catch (UnsupportedEncodingException e) {
+            e.printStackTrace();
+        }
+        return str;
+    }
+
+}

src/main/java/com/vcarecity/cvs/csv/SQLCSVWriter.java

@@ -0,0 +1,13 @@
+package com.vcarecity.cvs.csv;
+
+
+
+/**
+ * @author Kerry on 19/11/21
+ */
+
+public class SQLCSVWriter {
+
+
+
+}

src/main/java/com/vcarecity/cvs/entity/CheckRecordEntity.java

@@ -0,0 +1,68 @@
+package com.vcarecity.cvs.entity;
+
+import lombok.Data;
+
+import javax.persistence.Column;
+import java.sql.Timestamp;
+
+
+/**
+ * @author Kerry on 19/11/21
+ */
+
+@Data
+public class CheckRecordEntity {
+
+    @Column(name = "CHECK_RECORD_ID")
+    private Long checkRecordId;
+
+    @Column(name = "STAMP")
+    private Timestamp stamp;
+
+    @Column(name = "USER_ID")
+    private Long userId;
+
+    @Column(name = "CHECK_POINTS_ID")
+    private Long checkPointsId;
+
+    @Column(name = "IS_NORMAL")
+    private Integer isNormal;
+
+    @Column(name = "IS_LATENT_DANGER")
+    private Integer isLatentDanger;
+
+    @Column(name = "LATENT_DANGER_TYPE")
+    private Integer latentDangerType;
+
+    @Column(name = "LATENT_DANGER_DESC")
+    private String latentDangerDesc;
+
+    @Column(name = "UNDETECTED_NUM")
+    private String undetectedNum;
+
+    @Column(name = "LON")
+    private Double lon;
+
+    @Column(name = "LAT")
+    private Double lat;
+
+    @Column(name = "RULE_ID")
+    private Long ruleId;
+
+    @Column(name = "IS_DEAL")
+    private Integer isDeal;
+
+    @Column(name = "STARTDATE")
+    private String startDate;
+
+    @Column(name = "ENDDATE")
+    private String endDate;
+
+    @Column(name = "VALID_CHECK")
+    private Integer validCheck;
+
+    @Column(name = "UPDATESTAMP")
+    private Timestamp updateStamp;
+
+
+}

src/main/java/com/vcarecity/cvs/entity/WebSiteEntity.java

@@ -0,0 +1,18 @@
+package com.vcarecity.cvs.entity;
+
+import lombok.Data;
+
+/**
+ * @author Kerry on 19/11/22
+ */
+
+@Data
+public class WebSiteEntity {
+    private Integer id;
+    private String name;
+    private String url;
+    private Integer alexa;
+    private String country;
+    private String tm;
+    private Integer ti;
+}

src/main/java/com/vcarecity/cvs/factory/ResultHandlerFactory.java

@@ -0,0 +1,19 @@
+package com.vcarecity.cvs.factory;
+
+import com.vcarecity.cvs.service.ResultHandlerService;
+
+/**
+ * @author Kerry on 19/11/21
+ */
+
+public interface ResultHandlerFactory {
+
+    /**
+     * create
+     *
+     * @param properties
+     * @return
+     */
+    ResultHandlerService createResultHandler(String properties);
+
+}

src/main/java/com/vcarecity/cvs/module/MySQLModule.java

@@ -0,0 +1,60 @@
+package com.vcarecity.cvs.module;
+
+import com.google.inject.AbstractModule;
+import com.google.inject.Inject;
+import com.google.inject.Provides;
+import com.google.inject.Singleton;
+import com.google.inject.assistedinject.FactoryModuleBuilder;
+import com.vcarecity.cvs.factory.ResultHandlerFactory;
+import com.vcarecity.cvs.properties.AppProperties;
+import com.vcarecity.cvs.properties.DbProperties;
+import com.vcarecity.cvs.service.ResultHandlerService;
+import com.vcarecity.cvs.service.SQLQueryService;
+import com.vcarecity.cvs.service.impl.CSVResultHandlerServiceImpl;
+import com.vcarecity.cvs.service.impl.MySQLQueryServiceImpl;
+import com.zaxxer.hikari.HikariConfig;
+import com.zaxxer.hikari.HikariDataSource;
+
+/**
+ * @author Kerry on 19/11/20
+ */
+
+public class MySQLModule extends AbstractModule {
+    @Override
+    protected void configure() {
+        bind(SQLQueryService.class).to(MySQLQueryServiceImpl.class);
+
+        install(new FactoryModuleBuilder()
+                .implement(ResultHandlerService.class, CSVResultHandlerServiceImpl.class)
+                .build(ResultHandlerFactory.class));
+
+    }
+
+    @Provides
+    @Singleton
+    @Inject
+    public HikariDataSource dataSource(AppProperties properties) {
+        // @see https://github.com/brettwooldridge/HikariCP/wiki/MySQL-Configuration
+
+        final DbProperties database = properties.getDatabase();
+        HikariConfig config = new HikariConfig();
+        config.setJdbcUrl(database.getUrl());
+        config.setUsername(database.getUser());
+        config.setPassword(database.getPassword());
+        config.addDataSourceProperty("cachePrepStmts", "true");
+        config.addDataSourceProperty("prepStmtCacheSize", "250");
+        config.addDataSourceProperty("prepStmtCacheSqlLimit", "2048");
+        config.addDataSourceProperty("useServerPrepStmts", "true");
+        config.addDataSourceProperty("useLocalSessionState", "true");
+        config.addDataSourceProperty("rewriteBatchedStatements", "true");
+        config.addDataSourceProperty("cacheResultSetMetadata", "true");
+        config.addDataSourceProperty("cacheServerConfiguration", "true");
+        config.addDataSourceProperty("elideSetAutoCommits", "true");
+        config.addDataSourceProperty("maintainTimeStats", "false");
+
+        final HikariDataSource hikariDataSource = new HikariDataSource(config);
+        Runtime.getRuntime().addShutdownHook(new Thread(hikariDataSource::close));
+        return hikariDataSource;
+    }
+
+}

src/main/java/com/vcarecity/cvs/module/PropertiesModule.java

@@ -0,0 +1,21 @@
+package com.vcarecity.cvs.module;
+
+import com.google.inject.AbstractModule;
+import com.vcarecity.cvs.properties.AppProperties;
+
+/**
+ * @author Kerry on 19/11/20
+ */
+
+public class PropertiesModule extends AbstractModule {
+    private final AppProperties properties;
+
+    public PropertiesModule(AppProperties properties) {
+        this.properties = properties;
+    }
+
+    @Override
+    protected void configure() {
+        bind(AppProperties.class).toInstance(this.properties);
+    }
+}

src/main/java/com/vcarecity/cvs/properties/AppProperties.java

@@ -0,0 +1,12 @@
+package com.vcarecity.cvs.properties;
+
+import lombok.Data;
+
+/**
+ * @author Kerry on 19/11/20
+ */
+
+@Data
+public class AppProperties {
+    private DbProperties database;
+}

src/main/java/com/vcarecity/cvs/properties/DbProperties.java

@@ -0,0 +1,14 @@
+package com.vcarecity.cvs.properties;
+
+import lombok.Data;
+
+/**
+ * @author Kerry on 19/11/20
+ */
+
+@Data
+public class DbProperties {
+    private String url;
+    private String user;
+    private String password;
+}

src/main/java/com/vcarecity/cvs/service/ResultHandlerService.java

@@ -0,0 +1,22 @@
+package com.vcarecity.cvs.service;
+
+import java.util.List;
+
+/**
+ * @author Kerry on 19/11/21
+ */
+
+public interface ResultHandlerService {
+
+
+    /**
+     * ins
+     *
+     * @param dataList
+     * @param cls
+     * @throws Exception
+     */
+    <T> void resultHandler(List<T> dataList, Class<T> cls) throws Exception;
+
+
+}

src/main/java/com/vcarecity/cvs/service/SQLQueryService.java

@@ -0,0 +1,27 @@
+package com.vcarecity.cvs.service;
+
+import java.sql.SQLException;
+import java.util.List;
+
+/**
+ * @author Kerry on 19/11/21
+ */
+
+public interface SQLQueryService {
+
+
+    /**
+     * query by page
+     *
+     * @param sql
+     * @param start
+     * @param pageCount
+     * @param clazz
+     * @param <T>
+     * @return
+     * @throws SQLException
+     */
+    <T> List<T> queryByPage(String sql, int start, int pageCount, Class<T> clazz) throws SQLException;
+
+
+}

src/main/java/com/vcarecity/cvs/service/impl/CSVResultHandlerServiceImpl.java

@@ -0,0 +1,188 @@
+package com.vcarecity.cvs.service.impl;
+
+
+import com.google.inject.Inject;
+import com.google.inject.assistedinject.Assisted;
+import com.vcarecity.cvs.core.AutoMapper;
+import com.vcarecity.cvs.core.JavaMapColumn;
+import com.vcarecity.cvs.core.ReflectionUtil;
+import com.vcarecity.cvs.service.ResultHandlerService;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.csv.CSVFormat;
+import org.apache.commons.csv.CSVPrinter;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.sql.Timestamp;
+import java.time.*;
+import java.time.format.DateTimeFormatter;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * @author Kerry on 19/11/21
+ */
+
+@Slf4j
+public class CSVResultHandlerServiceImpl implements ResultHandlerService {
+
+    private final CSVFormat csvFormat;
+
+    private BufferedWriter bufferedWriter = null;
+    private CSVPrinter writer = null;
+
+    private final String filePattern;
+    private int counter;
+    private int fileNameIndex = 0;
+
+
+    /**
+     * 分割文件行数
+     */
+    private static final int SPLIT_LINE_NUM = 200_0000;
+
+
+    @Inject
+    public CSVResultHandlerServiceImpl(@Assisted String filePattern) {
+        this.filePattern = filePattern;
+        this.csvFormat = CSVFormat.MYSQL;
+        this.counter = 0;
+    }
+
+    private File getFile() {
+        final int index = fileNameIndex;
+        fileNameIndex++;
+        return new File(filePattern + index + ".csv");
+    }
+
+
+    private void checkFile() {
+        synchronized (this) {
+            if (counter % SPLIT_LINE_NUM == 0) {
+                try {
+                    if (bufferedWriter != null) {
+                        bufferedWriter.close();
+                    }
+                    if (writer != null) {
+                        writer.close();
+                    }
+                    bufferedWriter = Files.newBufferedWriter(getFile().toPath());
+                    writer = new CSVPrinter(bufferedWriter, csvFormat);
+                } catch (IOException e) {
+                    e.printStackTrace();
+                }
+            }
+        }
+    }
+
+    @Override
+    public <T> void resultHandler(List<T> dataList, Class<T> cls) throws Exception {
+        checkFile();
+
+        logger.debug("start write to csv file size = {}", dataList.size());
+
+        long startTime = System.currentTimeMillis();
+
+        for (T t : dataList) {
+            Object[] row;
+            if (cls.isArray()) {
+                // array
+                row = (Object[]) t;
+            } else if (List.class.isAssignableFrom(cls)) {
+                // noinspection rawtypes
+                row = ((List) t).toArray();
+            } else {
+                row = resultMapper(t, cls);
+            }
+            Object[] handlerRow = toCsvContent(row);
+            writer.printRecord(handlerRow);
+        }
+        bufferedWriter.flush();
+
+        counter += dataList.size();
+
+
+        // 输出Header
+        final List<JavaMapColumn> fieldColumnList = ReflectionUtil.getFieldColumnList(cls);
+        final List<String> collect = fieldColumnList.stream().map(JavaMapColumn::getColumn).collect(Collectors.toList());
+        final String join = String.join(",", collect.toArray(new String[]{}));
+
+        logger.debug("write to csv file usage time {} /ms. header = ({})", (System.currentTimeMillis() - startTime), join);
+    }
+
+
+    protected <T> Object[] resultMapper(T t, Class<T> cls) {
+        return AutoMapper.autoMapper(t, cls);
+    }
+
+    public void close() throws IOException {
+        bufferedWriter.close();
+        writer.close();
+    }
+
+    private static Object[] toCsvContent(Object[] objects) {
+        Object[] contents = new Object[objects.length];
+
+        // 主要是对 时间格式的处理
+
+        for (int i = 0; i < objects.length; i++) {
+            final Object object = objects[i];
+            if (object == null) {
+                contents[i] = null;
+            } else if (object instanceof Timestamp) {
+                contents[i] = formatTimestamp((Timestamp) object);
+            } else if (object instanceof java.sql.Date) {
+                contents[i] = formatDate((java.sql.Date) object);
+            } else if (object instanceof java.util.Date) {
+                contents[i] = formatDate((java.util.Date) object);
+            } else if (object instanceof LocalDateTime) {
+                contents[i] = formatLocalDateTime((LocalDateTime) object);
+            } else if (object instanceof LocalDate) {
+                contents[i] = formatLocalDate((LocalDate) object);
+            } else if (object instanceof LocalTime) {
+                contents[i] = formatLocalTime((LocalTime) object);
+            } else {
+                contents[i] = object;
+            }
+        }
+        return contents;
+    }
+
+    public static final DateTimeFormatter DATE_FORMATTER = DateTimeFormatter.ofPattern("yyyy/MM/dd");
+
+    private static String formatLocalDate(LocalDate localDate) {
+        return localDate.format(DATE_FORMATTER);
+    }
+
+
+    private static String formatDate(java.sql.Date date) {
+        final LocalDate localDate = date.toLocalDate();
+        return localDate.format(DATE_FORMATTER);
+    }
+
+    private static String formatDate(java.util.Date date) {
+        final Instant instant = date.toInstant();
+        final LocalDateTime of = LocalDateTime.ofInstant(instant, ZoneId.systemDefault());
+        return of.format(DATE_TIME_FORMATTER);
+    }
+
+    public static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss");
+
+    private static String formatTimestamp(Timestamp timestamp) {
+        final LocalDateTime localDateTime = timestamp.toLocalDateTime();
+        return localDateTime.format(DATE_TIME_FORMATTER);
+    }
+
+    private static String formatLocalDateTime(LocalDateTime localDateTime) {
+        return localDateTime.format(DATE_TIME_FORMATTER);
+    }
+
+    public static final DateTimeFormatter TIME_FORMATTER = DateTimeFormatter.ofPattern("HH:mm:ss");
+
+    private static String formatLocalTime(LocalTime localTime) {
+        return localTime.format(TIME_FORMATTER);
+    }
+
+}

src/main/java/com/vcarecity/cvs/service/impl/MySQLQueryServiceImpl.java

@@ -0,0 +1,56 @@
+package com.vcarecity.cvs.service.impl;
+
+import com.google.inject.Inject;
+import com.vcarecity.cvs.core.AutoMapper;
+import com.vcarecity.cvs.service.SQLQueryService;
+import com.zaxxer.hikari.HikariDataSource;
+import lombok.extern.slf4j.Slf4j;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * @author Kerry on 19/11/21
+ */
+
+
+@Slf4j
+public class MySQLQueryServiceImpl implements SQLQueryService {
+    private final HikariDataSource dataSource;
+
+    @Inject
+    public MySQLQueryServiceImpl(HikariDataSource dataSource) {
+        this.dataSource = dataSource;
+    }
+
+    @Override
+    public <T> List<T> queryByPage(String sql, int start, int pageCount, Class<T> clazz) throws SQLException {
+        final Connection connection = dataSource.getConnection();
+        final PreparedStatement preparedStatement = connection.prepareStatement(sql);
+        preparedStatement.setInt(1, start);
+        preparedStatement.setInt(2, pageCount);
+
+        List<T> list = new ArrayList<>(pageCount);
+        long startTime = System.currentTimeMillis();
+
+        logger.debug("start query: {},{},{}.start on = {}", sql, start, pageCount, startTime);
+
+        final ResultSet resultSet = preparedStatement.executeQuery();
+        while (resultSet.next()) {
+            // mapper
+            final T t = AutoMapper.autoMapper(resultSet, clazz);
+            if (t != null) {
+                list.add(t);
+            }
+        }
+
+        logger.debug("start query: {},{},{}.usageTime= {}", sql, start, pageCount, (System.currentTimeMillis() - startTime));
+        connection.close();
+
+        return list;
+    }
+}

src/main/java/com/vcarecity/cvs/service/impl/MySQLResultHandlerServiceImpl.java

@@ -0,0 +1,42 @@
+package com.vcarecity.cvs.service.impl;
+
+import com.google.inject.Inject;
+import com.google.inject.assistedinject.Assisted;
+import com.vcarecity.cvs.service.ResultHandlerService;
+import com.zaxxer.hikari.HikariDataSource;
+import lombok.extern.slf4j.Slf4j;
+
+import java.sql.Connection;
+import java.time.LocalDateTime;
+import java.util.List;
+
+/**
+ * @author Kerry on 19/11/21
+ */
+
+
+@Slf4j
+public class MySQLResultHandlerServiceImpl implements ResultHandlerService {
+
+    private final HikariDataSource dataSource;
+    private final String sql;
+
+    @Inject
+    public MySQLResultHandlerServiceImpl(HikariDataSource dataSource, @Assisted String sql) {
+        this.dataSource = dataSource;
+        this.sql = sql;
+    }
+
+
+    @Override
+    public <T> void resultHandler(List<T> dataList, Class<T> cls) throws Exception {
+        logger.debug("start write to mysql size = {}, on = {}", dataList.size(), LocalDateTime.now());
+        long startTime = System.currentTimeMillis();
+        final Connection connection = dataSource.getConnection();
+
+
+        connection.close();
+
+        logger.debug("write to mysql usage time {} /ms", System.currentTimeMillis() - startTime);
+    }
+}

src/main/java/com/vcarecity/cvs/service/impl/OracleQueryServiceImpl.java

@@ -0,0 +1,55 @@
+package com.vcarecity.cvs.service.impl;
+
+import com.google.inject.Inject;
+import com.vcarecity.cvs.core.AutoMapper;
+import com.vcarecity.cvs.service.SQLQueryService;
+import com.zaxxer.hikari.HikariDataSource;
+import lombok.extern.slf4j.Slf4j;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * @author Kerry on 19/11/25
+ */
+
+@Slf4j
+public class OracleQueryServiceImpl implements SQLQueryService {
+    private final HikariDataSource dataSource;
+
+    @Inject
+    public OracleQueryServiceImpl(HikariDataSource dataSource) {
+        this.dataSource = dataSource;
+    }
+
+    @Override
+    public <T> List<T> queryByPage(String sql, int start, int pageCount, Class<T> clazz) throws SQLException {
+        final Connection connection = dataSource.getConnection();
+        final PreparedStatement preparedStatement = connection.prepareStatement(sql);
+        preparedStatement.setInt(1, start);
+        preparedStatement.setInt(2, pageCount);
+
+        List<T> list = new ArrayList<>(pageCount);
+        long startTime = System.currentTimeMillis();
+
+        logger.debug("start query: {},{},{}.start on = {}", sql, start, pageCount, startTime);
+
+        final ResultSet resultSet = preparedStatement.executeQuery();
+        while (resultSet.next()) {
+            // mapper
+            final T t = AutoMapper.autoMapper(resultSet, clazz);
+            if (t != null) {
+                list.add(t);
+            }
+        }
+
+        logger.debug("start query: {},{},{}.usageTime= {}", sql, start, pageCount, (System.currentTimeMillis() - startTime));
+        connection.close();
+
+        return list;
+    }
+}

src/main/lombok.config

@@ -0,0 +1 @@
+lombok.log.fieldName = logger

src/main/resources/logback-test.xml

@@ -0,0 +1,38 @@
+<configuration scan="true" scanPeriod="120 seconds">
+
+    <property name="LOG_HOME" value="./logs"/>
+    <statusListener class="ch.qos.logback.core.status.NopStatusListener"/>
+
+    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+        <encoder>
+            <pattern>%d{yyyy-MM-dd HH:mm:ss} %highlight(%-5level) [%15.15t] %cyan(%-40.40logger{39}): %msg %n</pattern>
+        </encoder>
+    </appender>
+
+    <appender name="FILE_APPENDER" class="ch.qos.logback.core.rolling.RollingFileAppender">
+
+        <file>${LOG_HOME}/application.log</file>
+        <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
+            <fileNamePattern>${LOG_HOME}/application.%d{yyyyMMdd}_%i.log</fileNamePattern>
+            <maxHistory>30</maxHistory>
+            <totalSizeCap>20GB</totalSizeCap>
+            <maxFileSize>50MB</maxFileSize>
+        </rollingPolicy>
+        <encoder>
+            <pattern>%d{yy-MM-dd HH:mm:ss} %-5level [%15.15t] %-40.40logger{39}: %msg %n</pattern>
+        </encoder>
+    </appender>
+
+    <appender name="ASYNC" class="ch.qos.logback.classic.AsyncAppender" additivity="false">
+        <appender-ref ref="FILE_APPENDER"/>
+    </appender>
+
+    <logger name="com.vcarecity" level="DEBUG" additivity="false">
+        <appender-ref ref="STDOUT"/>
+    </logger>
+
+    <root level="INFO">
+        <appender-ref ref="STDOUT"/>
+    </root>
+
+</configuration>

src/test/java/com/vcarecity/cvs/service/impl/CSVResultHandlerServiceImplTest.java

@@ -0,0 +1,7 @@
+package com.vcarecity.cvs.service.impl;
+
+
+
+class CSVResultHandlerServiceImplTest {
+
+}

src/test/lombok.config

@@ -0,0 +1 @@
+lombok.log.fieldName = logger