Browse Source

插入支持List

张泳健 5 years ago
parent
commit
8499e7907f

+ 1 - 1
config/application-oracle.yml View File

3
   user: C##FMMP
3
   user: C##FMMP
4
   password: vcare~1(^_^)
4
   password: vcare~1(^_^)
5
 to-db:
5
 to-db:
6
-  url: jdbc:mysql:replication://192.168.10.210:6446,192.168.10.210:6447/fmmp?serverTimezone=Asia/Shanghai
6
+  url: jdbc:mysql:replication://192.168.10.210:6446/fmmp?serverTimezone=Asia/Shanghai
7
   user: root
7
   user: root
8
   password: R00T@mysql
8
   password: R00T@mysql
9
 export:
9
 export:

+ 8 - 3
src/main/java/com/vcarecity/cvs/FileExporterApp.java View File

29
         ReflectionUtil.initTableClassMapper("com.vcarecity.cvs.entity");
29
         ReflectionUtil.initTableClassMapper("com.vcarecity.cvs.entity");
30
 
30
 
31
         AppProperties appProperties = ConfigLoader.parseConfig(args, AppProperties.class, null);
31
         AppProperties appProperties = ConfigLoader.parseConfig(args, AppProperties.class, null);
32
-        jdbcArgs(args, appProperties);
32
+        final boolean isTruncate = jdbcArgs(args, appProperties);
33
 
33
 
34
         String table = getTableByArgs(args);
34
         String table = getTableByArgs(args);
35
         if (table == null || table.length() == 0) {
35
         if (table == null || table.length() == 0) {
58
         injector = Guice.createInjector(new PropertiesModule(appProperties), new SQLModule(type));
58
         injector = Guice.createInjector(new PropertiesModule(appProperties), new SQLModule(type));
59
 
59
 
60
         final SQLStarter instance = injector.getInstance(SQLStarter.class);
60
         final SQLStarter instance = injector.getInstance(SQLStarter.class);
61
-        instance.exportDataAndConvert(table, properties, tableClass);
61
+        instance.exportDataAndConvert(table, properties, tableClass, isTruncate);
62
     }
62
     }
63
 
63
 
64
 
64
 
86
     }
86
     }
87
 
87
 
88
 
88
 
89
-    private static void jdbcArgs(String[] args, AppProperties properties) {
89
+    private static boolean jdbcArgs(String[] args, AppProperties properties) {
90
         final DbProperties database = properties.getFromDb();
90
         final DbProperties database = properties.getFromDb();
91
+        boolean isTruncateTable = false;
91
         for (String arg : args) {
92
         for (String arg : args) {
92
             if (arg.startsWith("jdbc.url=")) {
93
             if (arg.startsWith("jdbc.url=")) {
93
                 database.setUrl(arg.substring("jdbc.url=".length()).trim());
94
                 database.setUrl(arg.substring("jdbc.url=".length()).trim());
95
                 database.setUser(arg.substring("jdbc.user=".length()).trim());
96
                 database.setUser(arg.substring("jdbc.user=".length()).trim());
96
             } else if (arg.startsWith("jdbc.password=")) {
97
             } else if (arg.startsWith("jdbc.password=")) {
97
                 database.setPassword(arg.substring("jdbc.password=".length()).trim());
98
                 database.setPassword(arg.substring("jdbc.password=".length()).trim());
99
+            } else if (arg.startsWith("--truncate=true")) {
100
+                isTruncateTable = true;
98
             }
101
             }
99
         }
102
         }
100
         properties.setFromDb(database);
103
         properties.setFromDb(database);
104
+
105
+        return isTruncateTable;
101
     }
106
     }
102
 }
107
 }

+ 10 - 0
src/main/java/com/vcarecity/cvs/service/ResultHandlerService.java View File

10
 
10
 
11
 
11
 
12
     /**
12
     /**
13
+     * 清空表
14
+     *
15
+     * @param table
16
+     */
17
+    default void truncateTable(String table) {
18
+
19
+    }
20
+
21
+
22
+    /**
13
      * ins
23
      * ins
14
      *
24
      *
15
      * @param table
25
      * @param table

+ 18 - 0
src/main/java/com/vcarecity/cvs/service/impl/MySQLResultHandlerServiceImpl.java View File

2
 
2
 
3
 import com.google.inject.Inject;
3
 import com.google.inject.Inject;
4
 import com.google.inject.assistedinject.Assisted;
4
 import com.google.inject.assistedinject.Assisted;
5
+import com.vcarecity.cvs.core.AutoMapper;
5
 import com.vcarecity.cvs.core.ColumnUpdateMapper;
6
 import com.vcarecity.cvs.core.ColumnUpdateMapper;
6
 import com.vcarecity.cvs.properties.AppProperties;
7
 import com.vcarecity.cvs.properties.AppProperties;
7
 import com.vcarecity.cvs.service.ResultHandlerService;
8
 import com.vcarecity.cvs.service.ResultHandlerService;
12
 
13
 
13
 import java.sql.Connection;
14
 import java.sql.Connection;
14
 import java.sql.PreparedStatement;
15
 import java.sql.PreparedStatement;
16
+import java.sql.SQLException;
15
 import java.util.Arrays;
17
 import java.util.Arrays;
16
 import java.util.List;
18
 import java.util.List;
17
 import java.util.concurrent.atomic.AtomicLong;
19
 import java.util.concurrent.atomic.AtomicLong;
35
 
37
 
36
 
38
 
37
     @Override
39
     @Override
40
+    public void truncateTable(String table) {
41
+        try (final Connection connection = dataSource.getConnection();
42
+             final PreparedStatement statement = connection.prepareStatement("truncate table " + table)) {
43
+            statement.execute();
44
+        } catch (SQLException e) {
45
+            e.printStackTrace();
46
+        }
47
+    }
48
+
49
+    @Override
38
     public <T> void resultHandler(String table, String[] header, List<T> dataList) throws Exception {
50
     public <T> void resultHandler(String table, String[] header, List<T> dataList) throws Exception {
39
 
51
 
40
         final String sql = getSql(table, header);
52
         final String sql = getSql(table, header);
60
                     for (int i = 0; i < rows.size(); i++) {
72
                     for (int i = 0; i < rows.size(); i++) {
61
                         ps.setObject(i + 1, rows.get(i));
73
                         ps.setObject(i + 1, rows.get(i));
62
                     }
74
                     }
75
+                } else {
76
+                    //TODO final Object[] row = resultMapper(t, (Class<T>) t.getClass());
63
                 }
77
                 }
64
                 ps.addBatch();
78
                 ps.addBatch();
65
                 //if (index % 1000 == 0) {
79
                 //if (index % 1000 == 0) {
81
         logger.info("WRITE SUCCESS to mysql usage time {} /ms, size = {}", curUseTime, dataList.size());
95
         logger.info("WRITE SUCCESS to mysql usage time {} /ms, size = {}", curUseTime, dataList.size());
82
     }
96
     }
83
 
97
 
98
+    protected <T> Object[] resultMapper(T t, Class<T> cls) {
99
+        return AutoMapper.autoMapper(t, cls);
100
+    }
101
+
84
 
102
 
85
     private static String getSql(String table, String[] headers) {
103
     private static String getSql(String table, String[] headers) {
86
 
104
 

+ 49 - 26
src/main/java/com/vcarecity/cvs/service/impl/MySQLResultStatHandlerServiceImpl.java View File

10
 import com.zaxxer.hikari.HikariDataSource;
10
 import com.zaxxer.hikari.HikariDataSource;
11
 import lombok.extern.slf4j.Slf4j;
11
 import lombok.extern.slf4j.Slf4j;
12
 
12
 
13
-import java.sql.Connection;
14
-import java.sql.Date;
15
-import java.sql.Statement;
16
-import java.sql.Timestamp;
13
+import java.sql.*;
17
 import java.util.List;
14
 import java.util.List;
18
 import java.util.concurrent.atomic.AtomicLong;
15
 import java.util.concurrent.atomic.AtomicLong;
19
 
16
 
35
     }
32
     }
36
 
33
 
37
     @Override
34
     @Override
35
+    public void truncateTable(String table) {
36
+        try (final Connection connection = dataSource.getConnection();
37
+             final PreparedStatement statement = connection.prepareStatement("truncate table " + table)) {
38
+            statement.execute();
39
+        } catch (SQLException e) {
40
+            e.printStackTrace();
41
+        }
42
+    }
43
+
44
+    @Override
38
     public <T> void resultHandler(String table, String[] header, List<T> dataList) throws Exception {
45
     public <T> void resultHandler(String table, String[] header, List<T> dataList) throws Exception {
39
 
46
 
40
         if (dataList == null || dataList.isEmpty()) {
47
         if (dataList == null || dataList.isEmpty()) {
42
         }
49
         }
43
         long startTime = System.currentTimeMillis();
50
         long startTime = System.currentTimeMillis();
44
 
51
 
45
-        StringBuilder sb = new StringBuilder();
46
 
52
 
47
         final String[] newHeaders = ColumnUpdateMapper.updateColumnName(table, header);
53
         final String[] newHeaders = ColumnUpdateMapper.updateColumnName(table, header);
48
         final String column = String.join(",", newHeaders);
54
         final String column = String.join(",", newHeaders);
49
 
55
 
50
-
56
+        StringBuilder sb = new StringBuilder();
51
         sb.append("INSERT INTO ").append(table).append("(").append(String.join(",", column)).append(") VALUES ");
57
         sb.append("INSERT INTO ").append(table).append("(").append(String.join(",", column)).append(") VALUES ");
52
 
58
 
53
         for (T t : dataList) {
59
         for (T t : dataList) {
60
+            sb.append("(");
54
             if (t.getClass().isArray()) {
61
             if (t.getClass().isArray()) {
55
                 Object[] rows = (Object[]) t;
62
                 Object[] rows = (Object[]) t;
56
-                sb.append("(");
57
                 for (int i = 0; i < header.length; i++) {
63
                 for (int i = 0; i < header.length; i++) {
58
-
59
-                    Object row = rows[i];
60
-                    if (row instanceof String) {
61
-                        sb.append("'").append(row).append("'");
62
-                    } else if (row instanceof Date) {
63
-                        sb.append("'").append(row).append("'");
64
-                    } else if (row instanceof Timestamp) {
65
-                        sb.append("'").append(row).append("'");
66
-                    } else {
67
-                        sb.append(row);
64
+                    mapperRow(rows[i], sb);
65
+                    if (i != header.length - 1) {
66
+                        sb.append(",");
68
                     }
67
                     }
68
+                }
69
+            } else if (List.class.isAssignableFrom(t.getClass())) {
70
+                List list = (List) t;
71
+                for (int i = 0; i < list.size(); i++) {
72
+                    mapperRow(list.get(i), sb);
69
                     if (i != header.length - 1) {
73
                     if (i != header.length - 1) {
70
                         sb.append(",");
74
                         sb.append(",");
71
                     }
75
                     }
72
                 }
76
                 }
73
-                sb.append("),");
77
+
78
+            } else {
79
+                //TODO auto mapper
74
             }
80
             }
81
+            sb.append("),");
75
         }
82
         }
76
         String execSql = sb.deleteCharAt(sb.length() - 1).toString();
83
         String execSql = sb.deleteCharAt(sb.length() - 1).toString();
77
 
84
 
78
-        // logger.debug("{}", execSql);
79
-
80
-        final Connection connection = dataSource.getConnection();
81
-
82
-        final Statement statement = connection.createStatement();
83
-        statement.execute(execSql);
84
-        statement.close();
85
-        connection.close();
85
+        logger.info("{}", execSql);
86
+
87
+        try {
88
+            final Connection connection = dataSource.getConnection();
89
+            final Statement statement = connection.createStatement();
90
+            statement.execute(execSql);
91
+            statement.close();
92
+            connection.close();
93
+        } catch (SQLException e) {
94
+            logger.warn("error sql: {}", execSql);
95
+            throw e;
96
+        }
86
 
97
 
87
         final long curUseTime = System.currentTimeMillis() - startTime;
98
         final long curUseTime = System.currentTimeMillis() - startTime;
88
 
99
 
91
         logger.info("WRITE SUCCESS to mysql usage time {} /ms, size = {}", curUseTime, dataList.size());
102
         logger.info("WRITE SUCCESS to mysql usage time {} /ms, size = {}", curUseTime, dataList.size());
92
     }
103
     }
93
 
104
 
105
+    private void mapperRow(Object row, StringBuilder sb) {
106
+        if (row instanceof String) {
107
+            sb.append("'").append(row).append("'");
108
+        } else if (row instanceof Date) {
109
+            sb.append("'").append(row).append("'");
110
+        } else if (row instanceof Timestamp) {
111
+            sb.append("'").append(row).append("'");
112
+        } else {
113
+            sb.append(row);
114
+        }
115
+    }
116
+
94
 
117
 
95
     @Override
118
     @Override
96
     public long getUsageTime() {
119
     public long getUsageTime() {

+ 7 - 2
src/main/java/com/vcarecity/cvs/starter/SQLStarter.java View File

43
      *
43
      *
44
      * @param table
44
      * @param table
45
      * @param properties
45
      * @param properties
46
+     * @param isTruncateTable
46
      * @throws Exception
47
      * @throws Exception
47
      */
48
      */
48
-    public <T> void exportDataAndConvert(String table, Object properties, Class<T> cls) throws Exception {
49
+    public <T> void exportDataAndConvert(String table, Object properties, Class<T> cls, boolean isTruncateTable) throws Exception {
49
 
50
 
50
         if (cls == null) {
51
         if (cls == null) {
51
             //noinspection unchecked
52
             //noinspection unchecked
54
 
55
 
55
         ResultHandlerService resultHandler = resultHandlerFactory.createResultHandler(properties);
56
         ResultHandlerService resultHandler = resultHandlerFactory.createResultHandler(properties);
56
 
57
 
58
+        if (isTruncateTable) {
59
+            resultHandler.truncateTable(table);
60
+        }
61
+
57
 
62
 
58
         logger.info("start query database table = {}, file = {}, class = {}", table, properties, cls);
63
         logger.info("start query database table = {}, file = {}, class = {}", table, properties, cls);
59
 
64
 
73
             currentSize = result.size();
78
             currentSize = result.size();
74
             queryCount += currentSize;
79
             queryCount += currentSize;
75
 
80
 
76
-            logger.info("current query count is {}, already query: {} /ms, write: {} /msg", queryCount,
81
+            logger.info("current query count is {}, already query: {} /ms, write: {} /ms", queryCount,
77
                     ((TimerService) queryService).getUsageTime(),
82
                     ((TimerService) queryService).getUsageTime(),
78
                     ((TimerService) resultHandler).getUsageTime());
83
                     ((TimerService) resultHandler).getUsageTime());
79
 
84
 

+ 33 - 0
src/main/java/com/vcarecity/cvs/starter/ThreadPoolStarter.java View File

1
+package com.vcarecity.cvs.starter;
2
+
3
+import com.google.inject.Inject;
4
+import com.vcarecity.cvs.properties.AppProperties;
5
+import com.vcarecity.cvs.service.impl.OracleQueryServiceImpl;
6
+import com.vcarecity.cvs.util.CreateDatasource;
7
+import com.zaxxer.hikari.HikariDataSource;
8
+
9
+/**
10
+ * @author Kerry on 19/12/03
11
+ */
12
+
13
+public class ThreadPoolStarter {
14
+
15
+    private final AppProperties properties;
16
+
17
+    @Inject
18
+    public ThreadPoolStarter(AppProperties properties) {
19
+        this.properties = properties;
20
+    }
21
+
22
+    public void start() {
23
+        HikariDataSource fromDatasource = CreateDatasource.createDataSource(properties.getFromDb());
24
+        HikariDataSource toDatasource = CreateDatasource.createDataSource(properties.getToDb());
25
+
26
+
27
+        final OracleQueryServiceImpl oracleQueryService = new OracleQueryServiceImpl(fromDatasource);
28
+
29
+
30
+    }
31
+
32
+
33
+}