Browse Source

initial version

jxing2 6 years ago
parent
commit
167c3426d6

+ 62 - 1
repairInfoTable/pom.xml

@@ -17,8 +17,47 @@
     <maven.compiler.source>1.7</maven.compiler.source>
     <maven.compiler.target>1.7</maven.compiler.target>
   </properties>
-
+    <repositories>
+        <repository>
+            <id>SagaCloud</id>
+            <name>SagaCloud</name>
+            <url>http://47.93.33.207:8082/repository/SagaCloud/</url>
+            <snapshots>
+                <enabled>true</enabled>
+            </snapshots>
+            <releases>
+                <enabled>true</enabled>
+            </releases>
+        </repository>
+        <repository>
+            <id>alimaven</id>
+            <name>aliyun maven</name>
+            <url>http://maven.aliyun.com/nexus/content/groups/public</url>
+        </repository>
+        <repository>
+            <id>mvnrepository</id>
+            <url>https://mvnrepository.com/</url>
+        </repository>
+        <!--
+        <repository>
+            <id>sonatype-nexus-snapshots</id>
+            <name>Sonatype Nexus Snapshots</name>
+            <url>http://oss.sonatype.org/content/repositories/snapshots</url>
+        </repository>
+        -->
+    </repositories>
   <dependencies>
+    <!-- https://mvnrepository.com/artifact/com.google.guava/guava -->
+    <dependency>
+      <groupId>com.google.guava</groupId>
+      <artifactId>guava</artifactId>
+      <version>27.1-jre</version>
+    </dependency>
+    <dependency>
+      <groupId>com.alibaba</groupId>
+      <artifactId>fastjson</artifactId>
+      <version>1.2.47</version>
+    </dependency>
     <dependency>
       <groupId>junit</groupId>
       <artifactId>junit</artifactId>
@@ -107,6 +146,18 @@
         </exclusion>
       </exclusions>
     </dependency>
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-api</artifactId>
+      <version>1.7.26</version>
+    </dependency>
+
+    <!-- https://mvnrepository.com/artifact/org.slf4j/slf4j-log4j12 -->
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-log4j12</artifactId>
+      <version>1.7.26</version>
+    </dependency>
   </dependencies>
 
   <build>
@@ -153,5 +204,15 @@
         </plugin>
       </plugins>
     </pluginManagement>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-compiler-plugin</artifactId>
+        <configuration>
+          <source>8</source>
+          <target>8</target>
+        </configuration>
+      </plugin>
+    </plugins>
   </build>
 </project>

+ 1 - 1
repairInfoTable/src/main/java/com/sagacloud/Config.java

@@ -29,7 +29,7 @@ public class Config {
                     }
                 } catch (IOException e) {
                     e.printStackTrace();
-                    log.error("Load properties failed.");
+                    //log.error("Load properties failed.");
                 }
             }
         }

+ 142 - 0
repairInfoTable/src/main/java/com/sagacloud/RepairTable.java

@@ -3,11 +3,153 @@ package com.sagacloud;
  * Author: Jxing
  * Create Time: 2019/3/12
  */
+import com.alibaba.fastjson.*;
+import com.google.common.base.Strings;
+import com.sagacloud.hbase.HbaseScanIndex;
+import com.sagacloud.hbase.ZillionOpUtil;
+import com.sagacloud.pojo.HBaseTable;
+import com.sagacloud.hbase.HBaseScanUtil;
+import com.sagacloud.pojo.InfosObj;
 import com.zillion.database.agent.ZillionAgent;
+import com.zillion.util.table.HTableUtil;
+import com.zillion.util.table.ZillionTableIndex;
+import com.zillion.util.table.ZillionTableSchema;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+
+import java.io.IOException;
+import java.util.*;
+
 public class RepairTable {
 
     public static void main(String[] args) {
+        //args = new String[]{"-b", "physical_world_v3_226"};
+        HBaseTable tableInfo = new HBaseTable();
+        boolean complete = HBaseTable.getInfo(args, tableInfo);
+        if(!complete){
+            System.out.println("参数错误, -b 数据库名");
+            return;
+        }
         ZillionAgent agent = ZillionAgentProvider.GetAgent();
+        ResultScanner scanner = null;
+        JSONObject criteria = new JSONObject();
+        ZillionTableSchema schema = agent.container.repositoryMap.get(tableInfo.getDatabase()).GetTable(tableInfo.getDatatable());
+        ZillionTableIndex index = HbaseScanIndex.zillionTableIndex(schema, criteria, null);
+        try {
+            scanner = getScanner(agent, tableInfo, criteria);
+        } catch (Exception e) {
+            System.out.println(e.getMessage());
+        }
+        fixData(scanner, index, schema, tableInfo);
+        agent.Stop();
+    }
+
+    private static void fixData(ResultScanner scanner, ZillionTableIndex index, ZillionTableSchema schema, HBaseTable tableInfo) {
+        if(scanner == null)
+            return;
+        // key --> objId + infoId , value -->  [ info_value, ArrayList<JSONObjectRow> ]
+        Map<String, ArrayList<InfosObj>> fixMap = new LinkedHashMap<>();
+        ArrayList<InfosObj> deleteArr = new ArrayList<>();
+        try {
+            String curProjectId = "aaa";
+            while(true) {
+                Result[] rs = scanner.next(2000);
+                if (rs == null || rs.length == 0) {
+                    break;
+                }
+                for (int i = 0; i < rs.length; ++i) {
+                    if (HTableUtil.valid(schema, rs[i].listCells())) {
+                        JSONObject resultObject = HBaseScanUtil.Convert(schema, index, rs[i]);
+                        if (!resultObject.containsKey("obj_type") || !resultObject.containsKey("obj_id")
+                                || !resultObject.containsKey("info_id") || !resultObject.containsKey("time")) {
+                            throw new Exception("该表不是数据平台3的obj_infos表");
+                        }
+                        InfosObj infosObj = resultObject.toJavaObject(InfosObj.class);
+                        String idKey = String.join("", infosObj.getProject_id(), infosObj.getObj_type(), infosObj.getObj_id(), infosObj.getInfo_id());
+                        putToFixMap(idKey, infosObj, fixMap);
+                        if(!(curProjectId == null && infosObj.getProject_id() == null) &&
+                                (curProjectId == null || infosObj.getProject_id() == null || !curProjectId.equals(infosObj.getProject_id()))){
+                            curProjectId = infosObj.getProject_id();
+                            System.out.println("正在处理项目: " + curProjectId);
+                            continue;
+                        }
+                    }
+                }
+                putToDeleteArr(fixMap, deleteArr);
+                if(deleteArr.size() > 2000){
+                    ZillionOpUtil.delete(deleteArr, tableInfo);
+                }
+            }
+            ZillionOpUtil.delete(deleteArr, tableInfo);
+        } catch (Exception e) {
+            System.out.println(e.getMessage());
+            return;
+        }finally {
+            scanner.close();
+        }
+        System.out.println(deleteArr);
+    }
+
+    private static void putToDeleteArr(Map<String, ArrayList<InfosObj>> fixMap, ArrayList<InfosObj> deleteArr) {
+        String lastKey = null;
+        for(String idKey : fixMap.keySet()){
+            lastKey = idKey;
+            ArrayList<InfosObj> arr = fixMap.get(idKey);
+            boolean isFirst = true;
+            Object lastValue = null;
+            for(int i = 0; i < arr.size(); ++i){
+                InfosObj single = arr.get(i);
+                if(isFirst){
+                    lastValue = single.getObjInfoValue();
+                    isFirst = false;
+                    continue;
+                }
+                if(single.getObjInfoValue() == null){
+                    if(lastValue == null) {
+                        // remove
+                        deleteArr.add(single);
+                        arr.remove(i);
+                        --i;
+                        continue;
+                    }else{
+                        lastValue = single.getObjInfoValue();
+                    }
+                }else if(single.getObjInfoValue().equals(lastValue)){
+                    // remove
+                    deleteArr.add(single);
+                    arr.remove(i);
+                    --i;
+                    continue;
+                }else{
+                    lastValue = single.getObjInfoValue();
+                }
+
+            }
+        }
+        if(lastKey == null ){
+            return;
+        }
+        Iterator<String> iter = fixMap.keySet().iterator();
+        while(iter.hasNext()) {
+            String key = iter.next();
+            if (!key.equals(lastKey)) {
+                iter.remove();
+            }
+        }
+    }
+
+    private static void putToFixMap(String idKey, InfosObj resultObject, Map<String, ArrayList<InfosObj>> fixMap) {
+        if(fixMap.containsKey(idKey)){
+            fixMap.get(idKey).add(resultObject);
+        }else{
+            ArrayList<InfosObj> arr = new ArrayList<>();
+            fixMap.put(idKey, arr);
+            arr.add(resultObject);
+        }
+    }
+
 
+    private static ResultScanner getScanner(ZillionAgent agent, HBaseTable tableInfo, JSONObject jsonObject) throws Exception {
+        return HBaseScanUtil.getScan(agent, tableInfo.getDatabase(), tableInfo.getDatatable(), tableInfo.getDatatable(), jsonObject, null);
     }
 }

+ 173 - 0
repairInfoTable/src/main/java/com/sagacloud/hbase/HBaseScanUtil.java

@@ -0,0 +1,173 @@
+package com.sagacloud.hbase;
+/*
+ * Author: Jxing
+ * Create Time: 2019/3/13
+ */
+
+import com.alibaba.fastjson.*;
+import com.zillion.database.agent.*;
+import com.zillion.database.util.TableWrapper_HBase;
+import com.zillion.util.common.ByteToObject;
+import com.zillion.util.table.HTableUtil;
+import com.zillion.util.table.ZillionTableIndex;
+import com.zillion.util.table.ZillionTableSchema;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.filter.Filter;
+import org.apache.hadoop.hbase.filter.FilterList;
+import org.apache.hadoop.hbase.filter.MultiRowRangeFilter;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.zillion.util.common.MyException;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+public class HBaseScanUtil {
+    /**
+     * @param
+     * @UpdateDate: 2018/9/14 10:51
+     * @Description: scan主方法
+     * @return: split_child : 无分表时和tablename相同
+     * Criteria,Criterias :  查询条件  ,只能传一个,另外一个传null
+     */
+    public static ResultScanner getScan(ZillionAgent agent, String database, String tablename, String split_child, JSONObject Criteria, JSONArray Criterias) throws Exception {
+        ZillionTableSchema schema = agent.container.repositoryMap.get(database).GetTable(tablename);
+        if (Criteria != null && Criterias != null) {
+            throw new MyException("Criteria and Criterias not all null");
+        }
+        if (Criteria != null) {
+            ZillionQueryCheckUtil.CheckCriteria(schema, Criteria);
+        } else {
+            Criteria = (JSONObject) Criterias.get(0);
+            for (int i = 0; i < Criterias.size(); i++) {
+                JSONObject CriteriaInner = (JSONObject) Criterias.get(i);
+                ZillionQueryCheckUtil.CheckCriteria(schema, CriteriaInner);
+            }
+        }
+        Scan scan = new Scan();
+        //设置获取列
+        scan.addColumn(ByteToObject.toBytes("f"), ByteToObject.toBytes("v"));
+        if (schema.table_type == null || schema.table_type.equals("index_data")
+                || (schema.table_type.equals("split_month") || schema.table_type.equals("split_year"))) {
+            for (int i = 0; i < schema.OtherColumn.length; i++) {
+                String column_name = schema.OtherColumn[i];
+                scan.addColumn(ByteToObject.toBytes("f"), ByteToObject.toBytes(column_name));
+            }
+        } else if (schema.table_type.equals("all_column")) {
+            String[] Columns = schema.ColumnMap.keySet().toArray(new String[0]);
+            for (int ii = 0; ii < Columns.length; ii++) {
+                String column_name = Columns[ii];
+                scan.addColumn(ByteToObject.toBytes("f"), ByteToObject.toBytes(column_name));
+            }
+        }
+        JSONArray Cris = new JSONArray();
+        Cris.add(Criteria);
+        String indexname = HbaseScanIndex.indexname(schema, "select", new JSONArray(), Cris);
+        TableWrapper_HBase wrapper = hTableWrapper(indexname, schema, agent, split_child);
+        ZillionTableIndex index = HbaseScanIndex.zillionTableIndex(schema, "select", new JSONArray(), Cris);
+
+        //设置过滤
+        List<ZillionCriteria> criteria_keyList = new ArrayList<>();
+        ZillionCriteria criteria_key = get_criteria_key(schema, index, Criteria);
+        if (criteria_key != null) {
+            criteria_keyList.add(criteria_key);
+            if (Criterias != null && Criterias.size() > 1) {
+                for (int i = 1; i < Criterias.size(); i++) {
+                    JSONObject CriteriaInner = (JSONObject) Criterias.get(i);
+                    ZillionCriteria criteria_tmp = get_criteria_key(schema, index, CriteriaInner);
+                    if (criteria_tmp == null) {
+                        throw new MyException("ZillionAgent: " + (i + 1) + " Criteria must follow key's rule!");
+                    }
+                    criteria_keyList.add(criteria_tmp);
+                }
+            }
+        } else {
+            if (Criterias != null && Criterias.size() > 1) {
+                throw new MyException("ZillionAgent: " + "table of type all_column Criterias must follow rule!");
+            }
+            ZillionCriteria_all_column zillionCriteria_all_column = ZillionMatchUtil.CriteriaMatch_all_column_best(schema, Criteria);
+            if (zillionCriteria_all_column.isKey) {
+                criteria_key = zillionCriteria_all_column.criteria;
+                criteria_keyList.add(criteria_key);
+            }
+        }
+        boolean exist_match_all = false;
+        List<MultiRowRangeFilter.RowRange> rangeList = new ArrayList<MultiRowRangeFilter.RowRange>();
+        for (ZillionCriteria criteria_tmp : criteria_keyList) {
+            if (criteria_tmp.matched == 0) {
+                exist_match_all = true;
+                break;
+            }
+            if (criteria_tmp.bytes_lt_include && Bytes.compareTo(criteria_tmp.bytes_gt, criteria_tmp.bytes_lt) == 0) {
+                throw new MyException("ZillionAgent: " + "criteria_key lt_include & gt is equal to lt!!!");
+            }
+            rangeList.add(new MultiRowRangeFilter.RowRange(criteria_tmp.bytes_gt, criteria_tmp.bytes_gt_include, criteria_tmp.bytes_lt, criteria_tmp.bytes_lt_include));
+        }
+        if (exist_match_all) {
+            if (criteria_key.filterList.size() > 0) {
+                if (criteria_key.filterList.size() == 1) {
+                    scan.setFilter(criteria_key.filterList.get(0));
+                } else {
+                    FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ALL);
+                    for (Filter filter : criteria_key.filterList) {
+                        filterList.addFilter(filter);
+                    }
+                    scan.setFilter(filterList);
+                }
+            }
+        } else {
+            Filter rowFilter = new MultiRowRangeFilter(rangeList);
+            if (criteria_key.filterList.size() > 0) {
+                FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ALL);
+                filterList.addFilter(rowFilter);
+                for (Filter filter : criteria_key.filterList) {
+                    filterList.addFilter(filter);
+                }
+                scan.setFilter(filterList);
+            } else {
+                scan.setFilter(rowFilter);
+            }
+        }
+
+        ResultScanner scanner = wrapper.htable.getScanner(scan);
+        return scanner;
+    }
+
+    private static TableWrapper_HBase hTableWrapper(String indexname, ZillionTableSchema schema, ZillionAgent agent, String split_child) throws Exception {
+        TableWrapper_HBase wrapper = null;
+        if (indexname != null) {
+            for (int i = 0; i < schema.Indexes.size(); i++) {
+                ZillionTableIndex index1 = schema.Indexes.get(i);
+                if (index1.index_name.equals(indexname)) {
+                    wrapper = (TableWrapper_HBase) index1.connect(agent.container, split_child, false);
+                    break;
+                }
+            }
+        } else {
+            wrapper = (TableWrapper_HBase) schema.connect(agent.container, split_child, false);
+        }
+        return wrapper;
+    }
+
+    private static ZillionCriteria get_criteria_key(ZillionTableSchema schema, ZillionTableIndex index, JSONObject Criteria) throws IOException {
+        ZillionCriteria criteria_key;
+        if (index == null) {
+            criteria_key = ZillionMatchUtil.CriteriaMatch(schema.ColumnMap, schema.Key, Criteria, true);
+        } else {
+            criteria_key = ZillionMatchUtil.CriteriaMatch(schema.ColumnMap, index.Key, Criteria, true);
+        }
+        return criteria_key;
+    }
+
+    public static JSONObject Convert(ZillionTableSchema schema, ZillionTableIndex index, Result rs) throws IOException {
+        JSONObject resultObject;
+        if (index != null) {
+            resultObject = HTableUtil.Convert_index_data(schema, index, rs.getRow(), rs.listCells());
+        } else {
+            resultObject = HTableUtil.Convert(schema, rs.getRow(), rs.listCells());
+        }
+        return resultObject;
+    }
+}

+ 235 - 0
repairInfoTable/src/main/java/com/sagacloud/hbase/HbaseScanIndex.java

@@ -0,0 +1,235 @@
+package com.sagacloud.hbase;
+/*
+ * Author: Jxing
+ * Create Time: 2019/3/13
+ */
+
+import com.alibaba.fastjson.*;
+import com.zillion.database.agent.ZillionCriteria;
+import com.zillion.database.agent.ZillionCriteria_all_column;
+import com.zillion.database.agent.ZillionMatchUtil;
+import com.zillion.database.agent.ZillionOrderByUtil;
+import com.zillion.util.table.ZillionTableIndex;
+import com.zillion.util.table.ZillionTableSchema;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class HbaseScanIndex {
+    /**
+     * @param
+     * @UpdateDate: 2018/9/14 10:55
+     * @Description: 根据条件获取属于哪个索引
+     * @return:
+     */
+    public static String indexname(ZillionTableSchema schema, String QueryType, JSONArray OrderBy, JSONArray Criterias) {
+        String indexname = null;
+        try {
+            JSONObject Criteria = null;
+            if (Criterias.size() > 0) {
+                Criteria = (JSONObject) Criterias.get(0);
+            }
+            boolean OrderBy_useless = true;
+            ZillionCriteria criteria_key = ZillionMatchUtil.CriteriaMatch(schema.ColumnMap, schema.Key, Criteria, true);
+            List<ZillionCriteria> criteria_keyList = new ArrayList();
+            ZillionTableIndex schema_index = null;
+            ZillionCriteria criteria_index = null;
+            List<ZillionCriteria> criteria_indexList = new ArrayList();
+            if (criteria_key != null) {
+                criteria_keyList.add(criteria_key);
+                if (QueryType.equals("select") && !ZillionOrderByUtil.Match(schema.Key, Criteria, OrderBy)) {
+                    OrderBy_useless = false;
+                }
+                if (Criterias != null && Criterias.size() > 1) {
+                    for (int i = 1; i < Criterias.size(); ++i) {
+                        JSONObject CriteriaInner = (JSONObject) Criterias.get(i);
+                        ZillionCriteria criteria_tmp = ZillionMatchUtil.CriteriaMatch(schema.ColumnMap, schema.Key, CriteriaInner, true);
+                        if (criteria_tmp == null) {
+                            throw new Exception(i + 1 + " Criteria must follow key's rule!");
+                        }
+
+                        criteria_keyList.add(criteria_tmp);
+                        if (QueryType.equals("select") && !ZillionOrderByUtil.Match(schema.Key, CriteriaInner, OrderBy)) {
+                            OrderBy_useless = false;
+                        }
+                    }
+                }
+            } else {
+                boolean isKey = false;
+                if (schema.table_type != null) {
+                    isKey = true;
+                }
+                ZillionCriteria criteria_tmp;
+                int i;
+                for (i = 0; i < schema.Indexes.size(); ++i) {
+                    ZillionTableIndex index_tmp = (ZillionTableIndex) schema.Indexes.get(i);
+                    criteria_tmp = ZillionMatchUtil.CriteriaMatch(schema.ColumnMap, index_tmp.Key, Criteria, isKey);
+                    if (criteria_tmp != null) {
+                        schema_index = index_tmp;
+                        indexname = schema_index.index_name;
+                        criteria_index = criteria_tmp;
+                        break;
+                    }
+                }
+                if (criteria_index != null) {
+                    criteria_indexList.add(criteria_index);
+                    if (QueryType.equals("select") && !ZillionOrderByUtil.Match(schema_index.Key, Criteria, OrderBy)) {
+                        OrderBy_useless = false;
+                    }
+                    if (Criterias != null && Criterias.size() > 1) {
+                        for (i = 1; i < Criterias.size(); ++i) {
+                            JSONObject CriteriaInner = (JSONObject) Criterias.get(i);
+                            criteria_tmp = ZillionMatchUtil.CriteriaMatch(schema.ColumnMap, schema_index.Key, CriteriaInner, isKey);
+                            if (criteria_tmp == null) {
+                                throw new Exception(i + 1 + " Criteria must follow index " + schema_index.index_name + "'s rule!");
+                            }
+                            criteria_indexList.add(criteria_tmp);
+                            if (QueryType.equals("select") && !ZillionOrderByUtil.Match(schema_index.Key, CriteriaInner, OrderBy)) {
+                                OrderBy_useless = false;
+                            }
+                        }
+                    }
+                } else if (schema.table_type != null && schema.table_type.equals("all_column")) {
+                    if (Criterias != null && Criterias.size() > 1) {
+                        throw new Exception("table of type all_column Criterias must follow rule!");
+                    }
+                    ZillionCriteria_all_column ZillionCriteria_all_column = ZillionMatchUtil.CriteriaMatch_all_column_best(schema, Criteria);
+                    if (ZillionCriteria_all_column.isKey) {
+                        criteria_key = ZillionCriteria_all_column.criteria;
+                        criteria_keyList.add(criteria_key);
+                        if (QueryType.equals("select") && !ZillionOrderByUtil.Match(schema.Key, Criteria, OrderBy)) {
+                            OrderBy_useless = false;
+                        }
+                    } else {
+                        schema_index = (ZillionTableIndex) schema.Indexes.get(ZillionCriteria_all_column.index_index);
+                        criteria_index = ZillionCriteria_all_column.criteria;
+                        criteria_indexList.add(criteria_index);
+                        if (QueryType.equals("select") && !ZillionOrderByUtil.Match(schema_index.Key, Criteria, OrderBy)) {
+                            OrderBy_useless = false;
+                        }
+                    }
+                }
+            }
+
+
+        } catch (Exception e) {
+
+        }
+
+
+        return indexname;
+    }
+
+
+    public static ZillionTableIndex zillionTableIndex(ZillionTableSchema schema, JSONObject Criteria, JSONArray Criterias) {
+        JSONArray Cris = new JSONArray();
+        if (Criteria == null) {
+            Cris.add((JSONObject) Criterias.get(0));
+        } else {
+            Cris.add(Criteria);
+        }
+        return zillionTableIndex(schema, "select", new JSONArray(), Cris);
+    }
+
+    /**
+     * @param
+     * @UpdateDate: 2018/9/14 10:55
+     * @Description: 根据条件获取是否是索引条件
+     * @return:
+     */
+    public static ZillionTableIndex zillionTableIndex(ZillionTableSchema schema, String QueryType, JSONArray OrderBy, JSONArray Criterias) {
+        ZillionTableIndex zillionTableIndex = null;
+        try {
+            JSONObject Criteria = null;
+            if (Criterias.size() > 0) {
+                Criteria = (JSONObject) Criterias.get(0);
+            }
+            boolean OrderBy_useless = true;
+            ZillionCriteria criteria_key = ZillionMatchUtil.CriteriaMatch(schema.ColumnMap, schema.Key, Criteria, true);
+            List<ZillionCriteria> criteria_keyList = new ArrayList();
+            ZillionTableIndex schema_index = null;
+            ZillionCriteria criteria_index = null;
+            List<ZillionCriteria> criteria_indexList = new ArrayList();
+            if (criteria_key != null) {
+                criteria_keyList.add(criteria_key);
+                if (QueryType.equals("select") && !ZillionOrderByUtil.Match(schema.Key, Criteria, OrderBy)) {
+                    OrderBy_useless = false;
+                }
+                if (Criterias != null && Criterias.size() > 1) {
+                    for (int i = 1; i < Criterias.size(); ++i) {
+                        JSONObject CriteriaInner = (JSONObject) Criterias.get(i);
+                        ZillionCriteria criteria_tmp = ZillionMatchUtil.CriteriaMatch(schema.ColumnMap, schema.Key, CriteriaInner, true);
+                        if (criteria_tmp == null) {
+                            throw new Exception(i + 1 + " Criteria must follow key's rule!");
+                        }
+
+                        criteria_keyList.add(criteria_tmp);
+                        if (QueryType.equals("select") && !ZillionOrderByUtil.Match(schema.Key, CriteriaInner, OrderBy)) {
+                            OrderBy_useless = false;
+                        }
+                    }
+                }
+            } else {
+                boolean isKey = false;
+                if (schema.table_type != null) {
+                    isKey = true;
+                }
+                ZillionCriteria criteria_tmp;
+                int i;
+                for (i = 0; i < schema.Indexes.size(); ++i) {
+                    ZillionTableIndex index_tmp = (ZillionTableIndex) schema.Indexes.get(i);
+                    criteria_tmp = ZillionMatchUtil.CriteriaMatch(schema.ColumnMap, index_tmp.Key, Criteria, isKey);
+                    if (criteria_tmp != null) {
+                        schema_index = index_tmp;
+                        zillionTableIndex = schema_index;
+                        criteria_index = criteria_tmp;
+                        break;
+                    }
+                }
+                if (criteria_index != null) {
+                    criteria_indexList.add(criteria_index);
+                    if (QueryType.equals("select") && !ZillionOrderByUtil.Match(schema_index.Key, Criteria, OrderBy)) {
+                        OrderBy_useless = false;
+                    }
+                    if (Criterias != null && Criterias.size() > 1) {
+                        for (i = 1; i < Criterias.size(); ++i) {
+                            JSONObject CriteriaInner = (JSONObject) Criterias.get(i);
+                            criteria_tmp = ZillionMatchUtil.CriteriaMatch(schema.ColumnMap, schema_index.Key, CriteriaInner, isKey);
+                            if (criteria_tmp == null) {
+                                throw new Exception(i + 1 + " Criteria must follow index " + schema_index.index_name + "'s rule!");
+                            }
+                            criteria_indexList.add(criteria_tmp);
+                            if (QueryType.equals("select") && !ZillionOrderByUtil.Match(schema_index.Key, CriteriaInner, OrderBy)) {
+                                OrderBy_useless = false;
+                            }
+                        }
+                    }
+                } else if (schema.table_type != null && schema.table_type.equals("all_column")) {
+                    if (Criterias != null && Criterias.size() > 1) {
+                        throw new Exception("table of type all_column Criterias must follow rule!");
+                    }
+                    ZillionCriteria_all_column ZillionCriteria_all_column = ZillionMatchUtil.CriteriaMatch_all_column_best(schema, Criteria);
+                    if (ZillionCriteria_all_column.isKey) {
+                        criteria_key = ZillionCriteria_all_column.criteria;
+                        criteria_keyList.add(criteria_key);
+                        if (QueryType.equals("select") && !ZillionOrderByUtil.Match(schema.Key, Criteria, OrderBy)) {
+                            OrderBy_useless = false;
+                        }
+                    } else {
+                        schema_index = (ZillionTableIndex) schema.Indexes.get(ZillionCriteria_all_column.index_index);
+                        criteria_index = ZillionCriteria_all_column.criteria;
+                        criteria_indexList.add(criteria_index);
+                        if (QueryType.equals("select") && !ZillionOrderByUtil.Match(schema_index.Key, Criteria, OrderBy)) {
+                            OrderBy_useless = false;
+                        }
+                    }
+                }
+            }
+        } catch (Exception e) {
+
+        }
+
+
+        return zillionTableIndex;
+    }
+}

+ 43 - 0
repairInfoTable/src/main/java/com/sagacloud/hbase/ZillionOpUtil.java

@@ -0,0 +1,43 @@
+package com.sagacloud.hbase;
+/*
+ * Author: Jxing
+ * Create Time: 2019/3/13
+ */
+
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
+import com.sagacloud.ZillionAgentProvider;
+import com.sagacloud.pojo.HBaseTable;
+import com.sagacloud.pojo.InfosObj;
+import com.zillion.database.agent.ZillionAgent;
+
+import java.util.ArrayList;
+
+public class ZillionOpUtil {
+    public static void delete(ArrayList<InfosObj> arr, HBaseTable table) throws Exception {
+        JSONArray criterias = new JSONArray();
+        for(int i = 0; i < arr.size(); ++i){
+            InfosObj single = arr.get(i);
+            JSONObject object = new JSONObject();
+            object.put("project_id", single.getProject_id());
+            object.put("obj_type", single.getObj_type());
+            object.put("obj_id", single.getObj_id());
+            object.put("info_id", single.getInfo_id());
+            object.put("time", single.getTime());
+            criterias.add(object);
+        }
+        arr.clear();
+        System.out.println(delete(true, table.getDatabase(), table.getDatatable(), criterias));
+    }
+    private static JSONObject delete(boolean isBatchQuery, String database, String datatable, Object criteria) throws Exception {
+        JSONObject json = JSONObject.parseObject("{\"QueryType\":\"delete\"}");
+        json.put("Database", database);
+        if (isBatchQuery) {
+            json.put("QueryType", "batch_delete");
+            json.put("Criterias", criteria);
+        } else
+            json.put("Criteria", criteria);
+        json.put("Datatable", datatable);
+        return ZillionAgentProvider.GetAgent().Query(json);
+    }
+}

+ 44 - 0
repairInfoTable/src/main/java/com/sagacloud/pojo/HBaseTable.java

@@ -0,0 +1,44 @@
+package com.sagacloud.pojo;
+/*
+ * Author: Jxing
+ * Create Time: 2019/3/13
+ */
+
+public class HBaseTable {
+    private String database;
+    private String datatable;
+
+    public String getDatabase() {
+        return database;
+    }
+
+    public void setDatabase(String database) {
+        this.database = database;
+    }
+
+    public String getDatatable() {
+        return datatable;
+    }
+
+    public void setDatatable(String datatable) {
+        this.datatable = datatable;
+    }
+
+    public static boolean getInfo(String[] args, HBaseTable tableInfo) {
+        boolean hasDatabaseName = false, hasDatatableName = true;
+        tableInfo.datatable = "obj_infos";
+        try {
+            for (int i = 0; i < args.length; ++i) {
+                if (args[i].equals("-t") || args[i].equals("-table") || args[i].equals("-datatable") ) {
+                    tableInfo.datatable = args[++i];
+                }else if(args[i].equals("-b") || args[i].equals("-database")){
+                    tableInfo.database = args[++i];
+                    hasDatabaseName = true;
+                }
+            }
+        }catch (Exception ex){
+            return false;
+        }
+        return hasDatabaseName;
+    }
+}

+ 112 - 0
repairInfoTable/src/main/java/com/sagacloud/pojo/InfosObj.java

@@ -0,0 +1,112 @@
+package com.sagacloud.pojo;
+/*
+ * Author: Jxing
+ * Create Time: 2019/3/13
+ */
+
+public class InfosObj {
+    private String project_id;
+    private String obj_type;
+    private String obj_id;
+    private String info_id;
+    private String time;
+    private Boolean b_value;
+    private Long l_value;
+    private String s_value;
+    private Double d_value;
+    private String json_value;
+
+    public String getProject_id() {
+        return project_id;
+    }
+
+    public void setProject_id(String project_id) {
+        this.project_id = project_id;
+    }
+
+    public String getObj_type() {
+        return obj_type;
+    }
+
+    public void setObj_type(String obj_type) {
+        this.obj_type = obj_type;
+    }
+
+    public String getObj_id() {
+        return obj_id;
+    }
+
+    public void setObj_id(String obj_id) {
+        this.obj_id = obj_id;
+    }
+
+    public String getInfo_id() {
+        return info_id;
+    }
+
+    public void setInfo_id(String info_id) {
+        this.info_id = info_id;
+    }
+
+    public String getTime() {
+        return time;
+    }
+
+    public void setTime(String time) {
+        this.time = time;
+    }
+
+    public Boolean getB_value() {
+        return b_value;
+    }
+
+    public void setB_value(Boolean b_value) {
+        this.b_value = b_value;
+    }
+
+    public Long getL_value() {
+        return l_value;
+    }
+
+    public void setL_value(Long l_value) {
+        this.l_value = l_value;
+    }
+
+    public String getS_value() {
+        return s_value;
+    }
+
+    public void setS_value(String s_value) {
+        this.s_value = s_value;
+    }
+
+    public Double getD_value() {
+        return d_value;
+    }
+
+    public void setD_value(Double d_value) {
+        this.d_value = d_value;
+    }
+
+    public String getJson_value() {
+        return json_value;
+    }
+
+    public void setJson_value(String json_value) {
+        this.json_value = json_value;
+    }
+
+    public Object getObjInfoValue() {
+        if(b_value != null)
+            return b_value;
+        if(l_value != null)
+            return l_value;
+        if(s_value != null)
+            return s_value;
+        if(d_value != null)
+            return d_value;
+        if(json_value != null)
+            return json_value;
+        return null;
+    }
+}

+ 1 - 1
repairInfoTable/src/main/resources/config.properties

@@ -1 +1 @@
-hbase.zookeeper.quorum=zookeeper1.saga
+hbase.zookeeper.quorum=zookeeper1,zookeeper2,zookeeper3

+ 30 - 0
repairInfoTable/src/main/resources/log4j.properties

@@ -0,0 +1,30 @@
+#log4j.rootLogger=INFO,loggerId
+log4j.rootLogger=INFO,ConsoleAppender,loggerId
+
+log4j.appender.RollingLogFileAppender=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.RollingLogFileAppender.Threshold=INFO
+log4j.appender.RollingLogFileAppender.ImmediateFlush=true
+log4j.appender.RollingLogFileAppender.File=log/data-platform-v3/data-platform-v3-repair.log
+log4j.appender.RollingLogFileAppender.DatePattern='.'yyyy-MM-dd
+log4j.appender.RollingLogFileAppender.Append=true
+log4j.appender.RollingLogFileAppender.layout=org.apache.log4j.PatternLayout
+log4j.appender.RollingLogFileAppender.layout.ConversionPattern=%d{yyyy-MM-dd HH\:mm\:ss.SSS} [%p][%c] %t %m%n
+log4j.appender.MAIN_LOG.MaxFileSize=1MB
+log4j.appender.MAIN_LOG.MaxBackupIndex=99
+
+
+log4j.appender.loggerId=org.apache.log4j.RollingFileAppender
+log4j.appender.loggerId.layout=org.apache.log4j.PatternLayout
+log4j.appender.loggerId.ImmediateFlush=true
+log4j.appender.loggerId.layout.ConversionPattern=%d{yyyy-MM-dd HH\:mm\:ss.SSS} [%p][%c] %t %m%n
+#log4j.appender.loggerId.File=E:/log/data-platform-v3/data-platform-v3.logss
+log4j.appender.loggerId.File=log/data-platform-v3/data-platform-v3-repair.log
+log4j.appender.loggerId.MaxFileSize=300MB
+log4j.appender.loggerId.MaxBackupIndex=10
+
+log4j.appender.ConsoleAppender=org.apache.log4j.ConsoleAppender
+log4j.appender.ConsoleAppender.Threshold=INFO
+log4j.appender.ConsoleAppender.ImmediateFlush=true
+log4j.appender.ConsoleAppender.Target=System.out
+log4j.appender.ConsoleAppender.layout=org.apache.log4j.PatternLayout
+log4j.appender.ConsoleAppender.layout.ConversionPattern=%d{yyyy-MM-dd HH\:mm\:ss.SSS} [%p][%c] %t %m%n