package com.sagacloud; /* * Author: Jxing * Create Time: 2019/3/12 */ import com.alibaba.fastjson.*; import com.google.common.base.Strings; import com.sagacloud.hbase.HbaseScanIndex; import com.sagacloud.hbase.ZillionOpUtil; import com.sagacloud.pojo.HBaseTable; import com.sagacloud.hbase.HBaseScanUtil; import com.sagacloud.pojo.InfosObj; import com.zillion.database.agent.ZillionAgent; import com.zillion.util.table.HTableUtil; import com.zillion.util.table.ZillionTableIndex; import com.zillion.util.table.ZillionTableSchema; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.ResultScanner; import java.io.IOException; import java.util.*; public class RepairTable { public static void main(String[] args) { //args = new String[]{"-b", "physical_world_v3_226", "-p", "1101080001"}; HBaseTable tableInfo = new HBaseTable(); boolean complete = HBaseTable.getInfo(args, tableInfo); if(!complete){ System.out.println("参数错误, -b 数据库名, -p 项目号(不带Pj)"); return; } System.out.println(tableInfo.getDatabase() + " + " + tableInfo.getDatatable() + " + " + tableInfo.getProjectId()); ZillionAgent agent = ZillionAgentProvider.GetAgent(); ResultScanner scanner = null; JSONObject criteria = new JSONObject(); generateCriteria(criteria, tableInfo); ZillionTableSchema schema = agent.container.repositoryMap.get(tableInfo.getDatabase()).GetTable(tableInfo.getDatatable()); ZillionTableIndex index = HbaseScanIndex.zillionTableIndex(schema, criteria, null); try { scanner = getScanner(agent, tableInfo, criteria); } catch (Exception e) { System.out.println(e.getMessage()); } fixData(scanner, index, schema, tableInfo); agent.Stop(); } private static void generateCriteria(JSONObject criteria, HBaseTable tableInfo) { if(tableInfo.isVotn()){ criteria.put("obj_type", "VOTn"); } if(tableInfo.getProjectId() != null && tableInfo.getProjectId().length() > 0){ criteria.put("project_id", tableInfo.getProjectId()); } } private static void fixData(ResultScanner scanner, ZillionTableIndex index, ZillionTableSchema schema, HBaseTable tableInfo) { if(scanner == null) return; // key --> objId + infoId , value --> [ info_value, ArrayList ] Map> fixMap = new LinkedHashMap<>(); ArrayList deleteArr = new ArrayList<>(); try { String curProjectId = "aaa"; while(true) { Result[] rs = scanner.next(2000); if (rs == null || rs.length == 0) { break; } for (int i = 0; i < rs.length; ++i) { if (HTableUtil.valid(schema, rs[i].listCells())) { JSONObject resultObject = HBaseScanUtil.Convert(schema, index, rs[i]); if (!resultObject.containsKey("obj_type") || !resultObject.containsKey("obj_id") || !resultObject.containsKey("info_id") || !resultObject.containsKey("time")) { throw new Exception("该表不是数据平台3的obj_infos表"); } InfosObj infosObj = resultObject.toJavaObject(InfosObj.class); String idKey = String.join("", infosObj.getProject_id(), infosObj.getObj_type(), infosObj.getObj_id(), infosObj.getInfo_id()); putToFixMap(idKey, infosObj, fixMap); if(!(curProjectId == null && infosObj.getProject_id() == null) && (curProjectId == null || infosObj.getProject_id() == null || !curProjectId.equals(infosObj.getProject_id()))){ curProjectId = infosObj.getProject_id(); System.out.println("正在处理项目: " + curProjectId); continue; } } } putToDeleteArr(fixMap, deleteArr, tableInfo); if(deleteArr.size() > 2000){ ZillionOpUtil.delete(deleteArr, tableInfo); System.out.println("正在修复 " + curProjectId); } } ZillionOpUtil.delete(deleteArr, tableInfo); } catch (Exception e) { System.out.println(e.getMessage()); return; }finally { scanner.close(); } System.out.println("已完成"); } private static void putToDeleteArr(Map> fixMap, ArrayList deleteArr, HBaseTable tableInfo) { String lastKey = null; for(String idKey : fixMap.keySet()){ lastKey = idKey; ArrayList arr = fixMap.get(idKey); boolean isFirst = true; Object lastValue = null; for(int i = 0; i < arr.size(); ++i){ InfosObj single = arr.get(i); if(isFirst){ lastValue = single.getObjInfoValue(); isFirst = false; continue; } if(single.getObjInfoValue() == null){ if(lastValue == null) { // remove deleteArr.add(single); arr.remove(i); --i; continue; }else{ lastValue = single.getObjInfoValue(); } }else if(single.getObjInfoValue().equals(lastValue)){ // remove deleteArr.add(single); arr.remove(i); --i; continue; }else{ lastValue = single.getObjInfoValue(); } } if(tableInfo.isVotn()){ for(; arr.size() > 1; ){ deleteArr.add(arr.get(0)); arr.remove(0); } } } if(lastKey == null ){ return; } Iterator iter = fixMap.keySet().iterator(); while(iter.hasNext()) { String key = iter.next(); if (!key.equals(lastKey)) { iter.remove(); } } } private static void putToFixMap(String idKey, InfosObj resultObject, Map> fixMap) { if(fixMap.containsKey(idKey)){ fixMap.get(idKey).add(resultObject); }else{ ArrayList arr = new ArrayList<>(); fixMap.put(idKey, arr); arr.add(resultObject); } } private static ResultScanner getScanner(ZillionAgent agent, HBaseTable tableInfo, JSONObject jsonObject) throws Exception { return HBaseScanUtil.getScan(agent, tableInfo.getDatabase(), tableInfo.getDatatable(), tableInfo.getDatatable(), jsonObject, null); } }