Browse Source

增加是否跳过脏数据的开关;增加数据处理进度的日志输出

cuixubin 4 years ago
parent
commit
6cc2fa54aa

+ 18 - 1
src/main/java/com/persagy/dptool/PrimaryController.java

@@ -4,6 +4,7 @@ import com.persagy.dptool.dialog.HelpDialog;
 import com.persagy.dptool.dialog.MessageDialog;
 import com.persagy.dptool.dto.PhysicalObject;
 import com.persagy.dptool.dto.VenderInfo;
+import javafx.beans.property.SimpleBooleanProperty;
 import javafx.concurrent.Task;
 import javafx.event.ActionEvent;
 import javafx.fxml.FXML;
@@ -57,6 +58,11 @@ public class PrimaryController implements Initializable {
     /** 底部状态栏 */
     @FXML
     public Label lblState;
+    /** 是否跳过脏数据按钮 */
+    public ToggleButton tgbSkipDirty;
+
+    private static final String skipDirtyTip = "跳过脏数据";
+    public static SimpleBooleanProperty skipDirtyData = new SimpleBooleanProperty(true);
 
     @Override
     public void initialize(URL location, ResourceBundle resources) {
@@ -69,7 +75,18 @@ public class PrimaryController implements Initializable {
             tableNameSBStr.append("  " + (i+1) + ". " + ConstantData.tablesNameList.get(i) + "\n");
         }
         txaContentJson.setText("目前只能校验HBase数据库中,物理世界的以下数据表数据:\n" + tableNameSBStr);
+        tgbSkipDirty.setSelected(true);
+        skipDirtyData.bind(tgbSkipDirty.selectedProperty());
+    }
 
+    public void skipDirty(ActionEvent e) {
+        if(tgbSkipDirty.isSelected()) {
+            tgbSkipDirty.setText(skipDirtyTip+"-是");
+            tgbSkipDirty.setStyle("-fx-background-color: #9eea6a");
+        }else {
+            tgbSkipDirty.setText(skipDirtyTip+"-否");
+            tgbSkipDirty.setStyle("-fx-background-color: #c6c6c6");
+        }
     }
 
     /**
@@ -252,7 +269,7 @@ public class PrimaryController implements Initializable {
     }
 
     public void showAbout(ActionEvent e) {
-        String aboutInfo = "Version:1.0\nJDK:1.8\nAuthor:cuixubin@persagy.com";
+        String aboutInfo = "Version:1.1\nJDK:1.8\nAuthor:cuixubin@persagy.com";
         new MessageDialog("About", aboutInfo, 300, 120, paneRoot.getScene().getWindow().getX() + 20, paneRoot.getScene().getWindow().getY() + 20).showAndWait();
     }
 }

+ 4 - 1
src/main/java/com/persagy/dptool/TaskFactory.java

@@ -34,13 +34,16 @@ public class TaskFactory {
                         controller.lblState.setText("正在处理数据...请不要编辑被处理的文件。");
                     });
 
+                    long doStart = System.currentTimeMillis();
                     Map<String, String> resultMap = VenderUtil.business(jsonFile, venderInfo);
+                    long doEnd = System.currentTimeMillis();
 
                     Platform.runLater(()->{
                         if(resultMap.containsKey(VenderUtil.keyError)) {
                             controller.lblVenderTip.setText(resultMap.get(VenderUtil.keyError));
                         }else {
-                            controller.lblVenderResult.setText(resultMap.get(VenderUtil.keyResult));
+                            String timeCost = "处理完成,耗时" + ((doEnd - doStart) / 1000) + "秒。";
+                            controller.lblVenderResult.setText(timeCost + resultMap.get(VenderUtil.keyResult));
                         }
                     });
                 }else {

+ 138 - 9
src/main/java/com/persagy/dptool/VenderUtil.java

@@ -6,11 +6,14 @@ import com.persagy.dptool.dto.vender.BrandDTO;
 import com.persagy.dptool.dto.vender.ObjectInfoRecord;
 import com.persagy.dptool.dto.vender.SpecificatDTO;
 import com.persagy.dptool.dto.vender.VenderContactDTO;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.*;
 import java.util.*;
 
 public class VenderUtil {
+    private static final Logger log = LoggerFactory.getLogger(VenderUtil.class);
     public static final String keyResult = "RESULT";
     public static final String keyError = "ERROR";
 
@@ -23,6 +26,33 @@ public class VenderUtil {
             return result;
         }
 
+        Boolean skipDirty = PrimaryController.skipDirtyData.getValue();
+        log.info("厂商库相关数据处理逻辑------start-----");
+        log.info("厂商库相关数据处理逻辑------分析数据-----跳过脏数据=" + skipDirty);
+        FileReader fileReader = null;
+        LineNumberReader lineNumberReader = null;
+        try {
+            fileReader = new FileReader(sourceFile);
+            lineNumberReader = new LineNumberReader(fileReader);
+            lineNumberReader.skip(Long.MAX_VALUE);
+            long lines = lineNumberReader.getLineNumber() + 1;
+
+            log.info(sourceFile.getName() + "文件总记录数约" + lines + "行");
+        }catch(Exception e) {
+
+        }finally {
+            try {
+                fileReader.close();
+            } catch (IOException e) {
+                e.printStackTrace();
+            }
+            try {
+                lineNumberReader.close();
+            } catch (IOException e) {
+                e.printStackTrace();
+            }
+        }
+
         String pjId = venderInfo.pjId;
         String pjIdField = "\"project_id\":\""+pjId+"\"";
 
@@ -46,27 +76,47 @@ public class VenderUtil {
                     continue;
                 }
 
-                if(lineStr.contains("\"obj_type\":\"Eq\"") && lineStr.contains(pjIdField) && lineStr.contains("obj_id") && lineStr.contains("info_id")) {
+                if(!lineStr.contains("project_id") || !lineStr.contains("obj_id") || !lineStr.contains("info_id")) {
+                    if(null == skipDirty || skipDirty) {
+                        continue;
+                    }
+                    throw new Exception("第" + lineNum + "行有脏数据");
+                }
+
+                if (lineStr.contains(pjIdField) && lineStr.contains("\"obj_type\":\"Eq\"") && rightString(lineStr)) {
                     // 判定为指定项目的设备记录
 
                     ObjectInfoRecord objectInfoRecord = CommonUtil.jsonStrToObj(lineStr, ObjectInfoRecord.class);
                     objId = objectInfoRecord.getObj_id();
                     infoIdAndTimeStr = objectInfoRecord.getInfo_id() + "@" + objectInfoRecord.getTime();
-                    if(obj2InfosMap.containsKey(objId)) {
+                    if (obj2InfosMap.containsKey(objId)) {
                         obj2InfosMap.get(objId).put(infoIdAndTimeStr, objectInfoRecord);
-                    }else {
+                    } else {
                         Map<String, ObjectInfoRecord> infoRecords = new HashMap<>();
-                        infoRecords.put(infoIdAndTimeStr, objectInfoRecord);;
+                        infoRecords.put(infoIdAndTimeStr, objectInfoRecord);
+                        ;
                         obj2InfosMap.put(objId, infoRecords);
                     }
 
-                }else {
+                } else {
                     // 写入目标文件
                     writer.write(lineStr + "\n");
                 }
                 lineNum ++;
+
+                if(lineNum % 1000 == 0) {
+                    System.out.println("--------已分析读取数据行数:" + lineNum);
+                }
             }
 
+            System.out.println("--------已分析读取数据行数:" + lineNum);
+
+            log.info("分析得到需要进行厂商库数据处理Eq数量为:" + obj2InfosMap.size());
+
+            log.info("厂商库相关数据处理逻辑------处理数据-----");
+
+            int handleNum = 0;
+
             for(Map<String, ObjectInfoRecord> item : obj2InfosMap.values()) {
                 //item对象结构 -> {信息点编码@时间:信息点记录}
 
@@ -100,9 +150,16 @@ public class VenderUtil {
                 for(ObjectInfoRecord record : item.values()) {
                     writer.write(record.toString() + "\n");
                 }
+
+                handleNum ++;
+
+                if(handleNum % 1000 == 0) {
+                    System.out.println("--------已处理Eq数:" + handleNum);
+                }
             }
+            System.out.println("--------已处理Eq数:" + handleNum);
 
-            result.put(keyResult, "处理完成。数据已存到:" + newFile.getAbsolutePath());
+            result.put(keyResult, "数据已存到:" + newFile.getAbsolutePath());
         }catch (Exception e) {
             e.printStackTrace();
             result.put(keyError, "处理第" + lineNum + "行数据出错! ErrorMsg=" + e.getMessage());
@@ -131,10 +188,39 @@ public class VenderUtil {
             }
         }
 
+        log.info("厂商库相关数据处理逻辑------end-----");
         return result;
     }
 
     /**
+     * 判断是否为需要处理的记录
+     * @param lineStr
+     * @return
+     */
+    private static boolean rightString(String lineStr) {
+        if(lineStr.contains("DPBrandID") || lineStr.contains("DPMaintainerID") ||
+                lineStr.contains("DPManufacturerID") || lineStr.contains("DPSupplierID") ||
+                lineStr.contains("DPSpecificationID")) {
+            return true;
+        }
+
+        if(lineStr.contains("Brand") || lineStr.contains("Specification") || lineStr.contains("Maintainer") ||
+                lineStr.contains("MaintainerContactor") || lineStr.contains("MaintainerPhone")) {
+            return true;
+        }
+
+        if(lineStr.contains("Manufacturer") || lineStr.contains("ManufacturerContactor") || lineStr.contains("ManufacturerPhone")) {
+            return true;
+        }
+
+        if(lineStr.contains("Supplier") || lineStr.contains("SupplierContactor") || lineStr.contains("SupplierPhone")) {
+            return true;
+        }
+
+        return false;
+    }
+
+    /**
      * 处理自定义信息点 DPSpecificationID,对应的 Specification 信息点值
      * @param item {信息点编码@时间:信息点记录}
      */
@@ -336,8 +422,51 @@ public class VenderUtil {
     }
 
     public static void main(String[] args) throws JsonProcessingException {
-        String time1 = "20211214212806";
-        String time2 = "null";
-        System.out.println(time2.compareTo(time1));
+        File file = new File("D:/test/obj_infos.json");
+        if (file.exists()) {
+            try {
+                FileReader fileReader = new FileReader(file);
+                LineNumberReader lineNumberReader = new LineNumberReader(fileReader);
+                lineNumberReader.skip(Long.MAX_VALUE);
+                long lines = lineNumberReader.getLineNumber() + 1;
+                fileReader.close();
+                lineNumberReader.close();
+                System.out.println(lines);
+
+
+                FileInputStream fis = new FileInputStream(file);
+                Scanner scanner = new Scanner(fis, "utf-8");
+
+                long dataNum = 0;
+                String lineStr = null;
+                // {设备id:{信息点编码@时间:信息点记录}}
+                Map<String, Map<String, ObjectInfoRecord>> obj2InfosMap = new HashMap<>();
+                long times = System.currentTimeMillis();
+                while(scanner.hasNext()) {
+                    lineStr = scanner.nextLine();
+//                    if(lineStr != null && (lineStr.contains("DPBrandID") || lineStr.contains("DPMaintainerID") || lineStr.contains("DPManufacturerID") || lineStr.contains("DPSupplierID") || lineStr.contains("DPSpecificationID"))) {
+//                        dataNum++;
+//                    }
+
+                    if(lineStr != null && (lineStr.indexOf("DPBrandID")>0 || lineStr.indexOf("DPMaintainerID")>0 ||
+                            lineStr.indexOf("DPManufacturerID")>0 || lineStr.indexOf("DPSupplierID")>0 || lineStr.indexOf("DPSpecificationID")>0)) {
+                        dataNum++;
+                    }
+                }
+                long timee = System.currentTimeMillis();
+                System.out.println(dataNum);
+                System.out.println("time=" + (timee-times));
+                if(null != fis) {
+                    fis.close();
+                }
+
+                if(null != scanner) {
+                    scanner.close();
+                }
+            } catch (IOException e) {
+                e.printStackTrace();
+            }
+
+        }
     }
 }

+ 23 - 11
src/main/resources/primary.fxml

@@ -1,16 +1,27 @@
 <?xml version="1.0" encoding="UTF-8"?>
 
-<?import javafx.scene.paint.*?>
-<?import javafx.scene.effect.*?>
-<?import javafx.scene.text.*?>
-<?import javafx.geometry.*?>
-<?import java.lang.*?>
-<?import java.util.*?>
-<?import javafx.scene.*?>
-<?import javafx.scene.control.*?>
-<?import javafx.scene.layout.*?>
+<?import javafx.geometry.Insets?>
+<?import javafx.scene.Cursor?>
+<?import javafx.scene.control.Button?>
+<?import javafx.scene.control.Label?>
+<?import javafx.scene.control.Menu?>
+<?import javafx.scene.control.MenuBar?>
+<?import javafx.scene.control.MenuItem?>
+<?import javafx.scene.control.ProgressIndicator?>
+<?import javafx.scene.control.SeparatorMenuItem?>
+<?import javafx.scene.control.Tab?>
+<?import javafx.scene.control.TabPane?>
+<?import javafx.scene.control.TextArea?>
+<?import javafx.scene.control.TextField?>
+<?import javafx.scene.control.ToggleButton?>
+<?import javafx.scene.layout.AnchorPane?>
+<?import javafx.scene.layout.BorderPane?>
+<?import javafx.scene.layout.FlowPane?>
+<?import javafx.scene.layout.HBox?>
+<?import javafx.scene.layout.StackPane?>
+<?import javafx.scene.text.Font?>
 
-<BorderPane fx:id="paneRoot" maxHeight="-Infinity" maxWidth="-Infinity" minHeight="-Infinity" minWidth="-Infinity" prefHeight="640.0" prefWidth="860.0" xmlns="http://javafx.com/javafx/8" xmlns:fx="http://javafx.com/fxml/1" fx:controller="com.persagy.dptool.PrimaryController">
+<BorderPane fx:id="paneRoot" maxHeight="-Infinity" maxWidth="-Infinity" minHeight="-Infinity" minWidth="-Infinity" prefHeight="640.0" prefWidth="860.0" xmlns="http://javafx.com/javafx/8.0.171" xmlns:fx="http://javafx.com/fxml/1" fx:controller="com.persagy.dptool.PrimaryController">
     <top>
         <MenuBar BorderPane.alignment="CENTER">
             <menus>
@@ -102,9 +113,10 @@
                         </Label>
                         <Label layoutX="55.0" layoutY="142.0" />
                         <Label layoutX="21.0" layoutY="171.0" />
-                        <Button layoutX="405.0" layoutY="137.0" mnemonicParsing="false" onAction="#venderValueHandle" text="处理" />
+                        <Button layoutX="450.0" layoutY="137.0" mnemonicParsing="false" onAction="#venderValueHandle" text="处理" />
                         <Label fx:id="lblVenderTip" layoutX="39.0" layoutY="191.0" prefHeight="32.0" prefWidth="782.0" textFill="RED" />
                         <Label fx:id="lblVenderResult" layoutX="39.0" layoutY="250.0" prefHeight="35.0" prefWidth="783.0" textFill="#3b8d4d" />
+                        <ToggleButton fx:id="tgbSkipDirty" layoutX="289.0" layoutY="137.0" mnemonicParsing="false" onAction="#skipDirty" style="-fx-background-color: #9eea6a;" text="跳过脏数据-是" />
                      </children></AnchorPane>
                     </content>
                 </Tab>