fengyanjie@sagacloud.cn 4 lat temu
rodzic
commit
cbb4755abf

+ 17 - 0
dmp-report/pom.xml

@@ -100,6 +100,23 @@
             <artifactId>hystrix-javanica</artifactId>
             <version>1.5.12</version>
         </dependency>
+
+        <!-- 分库分表 -->
+        <dependency>
+            <groupId>org.apache.shardingsphere</groupId>
+            <artifactId>sharding-jdbc-spring-boot-starter</artifactId>
+            <version>4.1.1</version>
+        </dependency>
+        <dependency>
+            <groupId>io.shardingsphere</groupId>
+            <artifactId>sharding-jdbc</artifactId>
+            <version>3.0.0.M3</version>
+        </dependency>
+        <dependency>
+            <groupId>com.xiaoju.uemc.tinyid</groupId>
+            <artifactId>tinyid-client</artifactId>
+            <version>0.1.0-SNAPSHOT</version>
+        </dependency>
     </dependencies>
 
     <build>

+ 0 - 6
dmp-report/src/main/java/com/persagy/dmp/report/repository/ReportAlarmRecordRepository.java

@@ -2,14 +2,8 @@ package com.persagy.dmp.report.repository;
 
 import com.persagy.dmp.report.entity.ReportAlarmRecord;
 import org.springframework.data.jpa.repository.JpaRepository;
-import org.springframework.data.jpa.repository.Modifying;
-import org.springframework.data.jpa.repository.Query;
 import org.springframework.data.querydsl.QuerydslPredicateExecutor;
 
 public interface ReportAlarmRecordRepository extends JpaRepository<ReportAlarmRecord, Integer>, QuerydslPredicateExecutor<ReportAlarmRecord> {
-
-    @Modifying
-    @Query(value = "delete from report_alarm_record where date_Type = ?1 and create_time >= ?2 and create_time < ?3", nativeQuery = true)
-    void deleteDateType(String dateType, String start, String end);
 }
 

+ 0 - 7
dmp-report/src/main/java/com/persagy/dmp/report/repository/ReportObjectRepository.java

@@ -2,15 +2,8 @@ package com.persagy.dmp.report.repository;
 
 import com.persagy.dmp.report.entity.ReportRwdObject;
 import org.springframework.data.jpa.repository.JpaRepository;
-import org.springframework.data.jpa.repository.Modifying;
-import org.springframework.data.jpa.repository.Query;
 import org.springframework.data.querydsl.QuerydslPredicateExecutor;
 
 public interface ReportObjectRepository extends JpaRepository<ReportRwdObject, Integer>, QuerydslPredicateExecutor<ReportRwdObject> {
-
-    @Modifying
-    @Query(value = "delete from report_rwd_object where date_Type = ?1 and create_time >= ?2 and create_time < ?3", nativeQuery = true)
-    void deleteDateType(String dateType, String start, String end);
-
 }
 

+ 12 - 3
dmp-report/src/main/java/com/persagy/dmp/report/service/DataExecutorService.java

@@ -7,6 +7,7 @@ import com.persagy.dmp.report.entity.*;
 import com.persagy.dmp.report.enumeration.DateType;
 import com.persagy.dmp.report.repository.ReportAlarmRecordRepository;
 import com.persagy.dmp.report.repository.ReportObjectRepository;
+import com.querydsl.core.types.dsl.BooleanExpression;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.MapUtils;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -45,7 +46,7 @@ public class DataExecutorService {
         StringBuilder sqlBuilder = new StringBuilder();
         sqlBuilder.append("        select project_id,group_code,obj_type,class_code,'" + dateType + "' as date_type,          \n");
         reportUtils.appendSql(sqlBuilder, dateType);
-        sqlBuilder.append("        count(0) rwd_object_num from dmp_rwd." + tableName + "_" + groupCode + "_" + projectId + "          \n");
+        sqlBuilder.append("        count(0) rwd_object_num from dmp_rwd." + tableName + "_" + groupCode.toLowerCase() + "_" + projectId.toLowerCase() + "          \n");
         sqlBuilder.append("        where create_time >= ? and create_time < ?          \n");
         sqlBuilder.append("          and project_id = ? and group_code = ?          \n");
         sqlBuilder.append("         group by project_id,group_code,obj_type,class_code,date_type,year,month,day,hour,season,date          \n");
@@ -83,7 +84,11 @@ public class DataExecutorService {
         Map<String, String> map = reportUtils.getDate(dateType, startTime, endTime);
         String start = map.get("start");
         String end = map.get("end");
-        reportObjectRepository.deleteDateType(dateType.name(), start, end);
+        QReportRwdObject qt = QReportRwdObject.reportRwdObject;
+        BooleanExpression ex = qt.dateType.eq(dateType).and(qt.createTime.goe(start)).and(qt.createTime.lt(end));
+        Iterable<ReportRwdObject> all = reportObjectRepository.findAll(ex);
+        reportObjectRepository.deleteAll(all);
+
         List<ReportRwdObject> save = new ArrayList<>();
         result.forEach(item -> save.add((ReportRwdObject) item));
         reportObjectRepository.saveAll(save);
@@ -139,7 +144,11 @@ public class DataExecutorService {
         Map<String, String> map = reportUtils.getDate(dateType, startTime, endTime);
         String start = map.get("start");
         String end = map.get("end");
-        reportAlarmRecordRepository.deleteDateType(dateType.name(), start, end);
+        QReportAlarmRecord qt = QReportAlarmRecord.reportAlarmRecord;
+        BooleanExpression ex = qt.dateType.eq(dateType).and(qt.createTime.goe(start)).and(qt.createTime.lt(end));
+        Iterable<ReportAlarmRecord> all = reportAlarmRecordRepository.findAll(ex);
+        reportAlarmRecordRepository.deleteAll(all);
+
         List<ReportAlarmRecord> save = new ArrayList<>();
         result.forEach(item -> save.add((ReportAlarmRecord) item));
         reportAlarmRecordRepository.saveAll(save);

+ 39 - 0
dmp-report/src/main/java/com/persagy/dmp/report/sharding/GroupKeyTinyidShardingAlgorithm.java

@@ -0,0 +1,39 @@
+package com.persagy.dmp.report.sharding;
+
+import com.xiaoju.uemc.tinyid.client.utils.TinyId;
+import org.apache.shardingsphere.spi.keygen.ShardingKeyGenerator;
+
+import java.util.Properties;
+
+/**
+ * @author: FengYanjie
+ * @date: 2020-12-30
+ * @verison: 1.0
+ */
+public class GroupKeyTinyidShardingAlgorithm implements ShardingKeyGenerator {
+    /**
+     * 自定义的生成方案类型
+     */
+    @Override
+    public String getType() {
+        return "tinyid";
+    }
+
+    /**
+     * 核心方法-生成主键ID
+     */
+    @Override
+    public Comparable generateKey() {
+        Long id = TinyId.nextId("test");
+        return id;
+    }
+
+    @Override
+    public Properties getProperties() {
+        return null;
+    }
+
+    @Override
+    public void setProperties(Properties properties) {
+    }
+}

+ 26 - 0
dmp-report/src/main/java/com/persagy/dmp/report/sharding/GroupPreciseShardingAlgorithm.java

@@ -0,0 +1,26 @@
+package com.persagy.dmp.report.sharding;
+
+import org.apache.shardingsphere.api.sharding.standard.PreciseShardingAlgorithm;
+import org.apache.shardingsphere.api.sharding.standard.PreciseShardingValue;
+
+import java.util.Collection;
+
+/**
+ * @author: FengYanjie
+ * @date: 2020-12-30
+ * @verison: 1.0
+ */
+public class GroupPreciseShardingAlgorithm implements PreciseShardingAlgorithm<String> {
+
+    @Override
+    public String doSharding(Collection<String> tableNames, PreciseShardingValue<String> preciseShardingValue) {
+        for (String tableName : tableNames) {
+            int val = Integer.parseInt(preciseShardingValue.getValue());
+            String value = val % tableNames.size() + "";
+            if (tableName.endsWith(value)) {
+                return tableName;
+            }
+        }
+        throw new IllegalArgumentException();
+    }
+}

+ 33 - 0
dmp-report/src/main/java/com/persagy/dmp/report/sharding/GroupRangeShardingAlgorithm.java

@@ -0,0 +1,33 @@
+package com.persagy.dmp.report.sharding;
+
+import org.apache.shardingsphere.api.sharding.standard.RangeShardingAlgorithm;
+import org.apache.shardingsphere.api.sharding.standard.RangeShardingValue;
+
+import java.util.Collection;
+import java.util.LinkedHashSet;
+import java.util.Set;
+
+/**
+ * @author: FengYanjie
+ * @date: 2020-12-30
+ * @verison: 1.0
+ */
+public class GroupRangeShardingAlgorithm implements RangeShardingAlgorithm<String> {
+
+    @Override
+    public Collection<String> doSharding(Collection<String> databaseNames, RangeShardingValue<String> rangeShardingValue) {
+        Set<String> result = new LinkedHashSet<>();
+        // between and 的起始值
+        int lower = Integer.parseInt(rangeShardingValue.getValueRange().lowerEndpoint());
+        int upper = Integer.parseInt(rangeShardingValue.getValueRange().upperEndpoint());
+        // 循环范围计算分库逻辑
+        for (int i = lower; i <= upper; i++) {
+            for (String databaseName : databaseNames) {
+                if (databaseName.endsWith(i % databaseNames.size() + "")) {
+                    result.add(databaseName);
+                }
+            }
+        }
+        return result;
+    }
+}

+ 1 - 0
dmp-report/src/main/resources/META-INF/services/org.apache.shardingsphere.spi.keygen.ShardingKeyGenerator

@@ -0,0 +1 @@
+com.persagy.dmp.report.sharding.GroupKeyTinyidShardingAlgorithm

+ 2 - 0
dmp-report/src/main/resources/tinyid_client.properties

@@ -0,0 +1,2 @@
+tinyid.server=127.0.0.1:9999
+tinyid.token=0f673adf80504e2eaa552f5d791b644c

+ 11 - 0
dmp-report/src/test/java/com/persagy/dmp.report/test.java

@@ -11,6 +11,7 @@ import com.persagy.dmp.report.enumeration.DateType;
 import com.persagy.dmp.report.service.DataExecutorService;
 import com.persagy.dmp.report.service.ReportUtils;
 import com.querydsl.jpa.impl.JPAQueryFactory;
+import com.xiaoju.uemc.tinyid.client.utils.TinyId;
 import lombok.extern.slf4j.Slf4j;
 import org.junit.jupiter.api.Test;
 import org.junit.runner.RunWith;
@@ -37,6 +38,16 @@ public class test {
     @Autowired
     ReportUtils reportUtils;
 
+    @Test
+    public void test() {
+        Long id = TinyId.nextId("test");
+        System.err.println(id);
+        List<Long> ids = TinyId.nextId("test", 10);
+        for (Long aLong : ids) {
+            System.err.println(aLong);
+        }
+    }
+
 //    @Test
 //    public void test1() {
 //        QAlarmItem qt = QAlarmItem.alarmItem;