wudianlong 2 vuotta sitten
vanhempi
commit
e25917b3f3

+ 0 - 19
energy/clean_mysql.sh

@@ -1,19 +0,0 @@
-#! /bin/bash
-
-. ./config.sh
-
-url=$MYSQL_URL
-username=$MYSQL_USER
-password=$MYSQL_PASSWORD
-
-del_energy_15_min="truncate table energy_15_min;"
-del_energy_5_min="truncate table energy_5_min;"
-
-clean_energy_15(){
-mysql -h${url}  -P3306  -u${username} -p${password} ${result_database} -e "${del_energy_15_min}"
-}
-
-clean_energy_5(){
-mysql -h${url}  -P3306  -u${username} -p${password} ${result_database} -e "${del_energy_5_min}"
-}
-

+ 21 - 0
energy/dayTask/clean_mysql.sh

@@ -0,0 +1,21 @@
+#! /bin/bash
+
+. ./config.sh
+
+url=$MYSQL_URL
+username=$MYSQL_USER
+password=$MYSQL_PASSWORD
+database=saga_dw
+
+del_energy_15_min_history="truncate table energy_15_min_history;"
+del_energy_15_min_fjd_history="truncate table energy_15_min_fjd_history;"
+
+## 历史数据一次性导入后,需要删除 mysql 中的冗余数据
+clean_energy_15(){
+mysql -h${url}  -P3306  -u${username} -p${password} ${database} -e "${del_energy_15_min_history}"
+}
+
+clean_energy_5(){
+mysql -h${url}  -P3306  -u${username} -p${password} ${database} -e "${del_energy_15_min_fjd_history}"
+}
+

energy/config.sh → energy/dayTask/config.sh


+ 74 - 0
energy/dayTask/energy_hive.sql

@@ -0,0 +1,74 @@
+/** 能源原始数据 15min 历史数据 */
+create table ods_energy_15_min
+(
+    building   string,
+    func_id    string,
+    meter      string,
+    data_time  string,
+    data_value decimal(20, 15)
+) COMMENT '能源原始数据15min历史数据'
+PARTITIONED BY (`dt` string)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001'
+STORED AS TEXTFILE
+LOCATION
+    'hdfs://sagaCluster:8020/warehouse/saga_dw/ods/ods_energy_15_min'
+TBLPROPERTIES (
+    'orc.compress'='lzo');
+
+/** 能源原始数据 5min 历史数据 */
+create table ods_energy_5_min
+(
+    building   string,
+    func_id    string,
+    meter      string,
+    data_time  string,
+    data_value decimal(20, 15)
+) COMMENT '能源原始数据15min历史数据'
+PARTITIONED BY (`dt` string)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001'
+STORED AS TEXTFILE
+LOCATION
+    'hdfs://sagaCluster:8020/warehouse/saga_dw/ods/ods_energy_5_min'
+TBLPROPERTIES (
+    'orc.compress'='lzo');
+
+/** 能源数据 15min */
+create table ods_energy_15_min
+(
+    building   string,
+    func_id    string,
+    meter      string,
+    data_time  string,
+    data_value decimal(20, 15)
+) COMMENT '能源原始数据15min'
+PARTITIONED BY (`dt` string)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001'
+STORED AS TEXTFILE
+LOCATION
+    'hdfs://sagaCluster:8020/warehouse/saga_dw/ods/ods_energy_15_min'
+TBLPROPERTIES (
+    'orc.compress'='lzo');
+
+/** 能源数据 5min */
+create table ods_energy_5_min
+(
+    building   string,
+    func_id    string,
+    meter      string,
+    data_time  string,
+    data_value decimal(20, 15)
+) COMMENT '能源原始数据15min'
+PARTITIONED BY (`dt` string)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001'
+STORED AS TEXTFILE
+LOCATION
+    'hdfs://sagaCluster:8020/warehouse/saga_dw/ods/ods_energy_5_min'
+TBLPROPERTIES (
+    'orc.compress'='lzo');
+
+
+
+
+
+
+

+ 43 - 0
energy/dayTask/energy_mysql.sql

@@ -0,0 +1,43 @@
+/** 能源 15 分钟数据 */
+create table energy_15_min
+(
+    building   varchar(128)    comment '大楼id',
+    func_id    varchar(128)    comment '仪表功能号',
+    meter      varchar(128)    comment '仪表号',
+    data_time  datetime        comment '数据采集时间',
+    data_value decimal(30, 15) comment '采集值',
+    dt         varchar(64)     comment '采集日期'
+) comment '能源 15 分钟差值数据';
+
+/** 能源 5 分钟数据 */
+create table energy_5_min
+(
+    building   varchar(128)    comment '大楼id',
+    func_id    varchar(128)    comment '仪表功能号',
+    meter      varchar(128)    comment '仪表号',
+    data_time  datetime        comment '数据采集时间',
+    data_value decimal(30, 15) comment '采集值',
+    dt         varchar(64)     comment '采集日期'
+) comment '能源 5 分钟差值数据';
+
+/** 能源 15 分钟数据-fjd */
+create table energy_15_min_fjd
+(
+    building   varchar(128)    comment '大楼id',
+    func_id    varchar(128)    comment '仪表功能号',
+    meter      varchar(128)    comment '仪表号',
+    data_time  datetime        comment '数据采集时间',
+    data_value decimal(30, 15) comment '采集值',
+    dt         varchar(64)     comment '采集日期'
+) comment '能源 15 分钟 fjd 数据';
+
+/** 能源 5 分钟数据-fjd */
+create table energy_5_min_fjd
+(
+    building   varchar(128)    comment '大楼id',
+    func_id    varchar(128)    comment '仪表功能号',
+    meter      varchar(128)    comment '仪表号',
+    data_time  datetime        comment '数据采集时间',
+    data_value decimal(30, 15) comment '采集值',
+    dt         varchar(64)     comment '采集日期'
+) comment '能源 5 分钟 fjd 数据';

+ 52 - 0
energy/dayTask/mysql_to_hdfs.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+
+. ./config.sh
+
+url=$MYSQL_URL
+username=$MYSQL_USER
+password=$MYSQL_PASSWORD
+
+## 无分区
+collect_data_np() {
+sqoop import -D mapred.job.queue.name=hive  \
+--connect $1 \
+--username $2 \
+--password $3 \
+--target-dir /warehouse/saga_dw/ods/$2  \
+--delete-target-dir \
+--query "$5 and \$CONDITIONS" \
+--num-mappers 1 \
+--hive-drop-import-delims \
+--fields-terminated-by '\001' \
+--compress \
+--compression-codec lzo \
+--hive-import \
+--hive-database saga_dw \
+--hive-table $4 \
+--hive-overwrite \
+--null-string '\\N' \
+--null-non-string '\\N'
+}
+
+mysql_to_hdfs_lzo() {
+  sqoop import  \
+  -D mapred.job.queue.name=hive  \
+  --connect "jdbc:mysql://hadoop01:3306/saga_dev?useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&tinyInt1isBit=false" \
+  --username root \
+  --password j5ry0#jZ7vaUt5f4 \
+  --target-dir /warehouse/saga_dw/ods/ods_energy_15_min_history  \
+  --delete-target-dir \
+  --columns "building, func_id, meter, data_time, data_value, dt" \
+  --query "select building, func_id, meter, data_time, data_value, dt from energy_15_min where 1 = 1 and  \$CONDITIONS" \
+  --num-mappers 1 \
+  --hive-drop-import-delims \
+  --fields-terminated-by '\001' \
+  --compress \
+  --compression-codec lzo \
+  --hive-import \
+  --hive-database saga_dw \
+  --hive-table ods_energy_15_min_history \
+  --hive-overwrite \
+  --null-string '\\N' \
+  --null-non-string '\\N'
+}

+ 0 - 76
energy/energy_hive.sql

@@ -1,76 +0,0 @@
-/** 能源原始数据 15min 历史数据 */
-create table ods_energy_15_min_history
-(
-    building   string,
-    func_id    string,
-    meter      string,
-    data_time  string,
-    data_value decimal(20, 15),
-    `dt` string
-) COMMENT '能源原始数据15min历史数据'
-    partitioned by (`dt` string)
-    ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001'
-    STORED AS TEXTFILE
-    LOCATION
-        'hdfs://sagaCluster:8020/warehouse/saga_dw/ods/ods_energy_15_min_history'
-    TBLPROPERTIES (
-        'orc.compress'='lzo');
-
-/** 能源原始数据 5min 历史数据 */
-create table ods_energy_5_min_history
-(
-    building   string,
-    func_id    string,
-    meter      string,
-    data_time  string,
-    data_value decimal(20, 15),
-    `dt` string
-) COMMENT '能源原始数据15min历史数据'
-    partitioned by (`dt` string)
-    ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001'
-    STORED AS TEXTFILE
-    LOCATION
-        'hdfs://sagaCluster:8020/warehouse/saga_dw/ods/ods_energy_5_min_history'
-    TBLPROPERTIES (
-        'orc.compress'='lzo');
-
-/** 能源数据 15min */
-create table ods_energy_15_min
-(
-    building   string,
-    func_id    string,
-    meter      string,
-    data_time  string,
-    data_value decimal(20, 15)
-) COMMENT '能源原始数据15min'
-    partitioned by (`dt` string)
-    ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001'
-    STORED AS TEXTFILE
-    LOCATION
-        'hdfs://sagaCluster:8020/warehouse/saga_dw/ods/ods_energy_15_min'
-    TBLPROPERTIES (
-        'orc.compress'='lzo');
-
-/** 能源数据 5min */
-create table ods_energy_5_min
-(
-    building   string,
-    func_id    string,
-    meter      string,
-    data_time  string,
-    data_value decimal(20, 15)
-) COMMENT '能源原始数据15min'
-    partitioned by (`dt` string)
-    ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001'
-    STORED AS TEXTFILE
-    LOCATION
-        'hdfs://sagaCluster:8020/warehouse/saga_dw/ods/ods_energy_5_min'
-    TBLPROPERTIES (
-        'orc.compress'='lzo');
-
-
-
-
-
-
-

+ 0 - 43
energy/energy_mysql.sql

@@ -1,43 +0,0 @@
-/** 能源 15 分钟数据-历史 */
-create table energy_15_min_history
-(
-    building   varchar(128)    null,
-    func_id    varchar(128)    null,
-    meter      varchar(128)    null,
-    data_time  datetime        null,
-    data_value decimal(20, 15) null,
-    dt         varchar(64)     null
-);
-
-/** 能源 15 分钟数据 */
-create table energy_15_min
-(
-    building   varchar(128)    null,
-    func_id    varchar(128)    null,
-    meter      varchar(128)    null,
-    data_time  datetime        null,
-    data_value decimal(20, 15) null,
-    dt         varchar(64)     null
-);
-
-/** 能源 5 分钟数据-历史 */
-create table energy_5_min_history
-(
-    building   varchar(128)    null,
-    func_id    varchar(128)    null,
-    meter      varchar(128)    null,
-    data_time  datetime        null,
-    data_value decimal(20, 15) null,
-    dt         varchar(64)     null
-);
-
-/** 能源 5 分钟数据 */
-create table energy_5_min
-(
-    building   varchar(128)    null,
-    func_id    varchar(128)    null,
-    meter      varchar(128)    null,
-    data_time  datetime        null,
-    data_value decimal(20, 15) null,
-    dt         varchar(64)     null
-);

+ 21 - 0
energy/history/clean_mysql.sh

@@ -0,0 +1,21 @@
+#! /bin/bash
+
+. ./config.sh
+
+url=$MYSQL_URL
+username=$MYSQL_USER
+password=$MYSQL_PASSWORD
+database=saga_dw
+
+del_energy_15_min_history="truncate table energy_15_min_history;"
+del_energy_5_min_history="truncate table energy_5_min_history;"
+
+## 历史数据一次性导入后,需要删除 mysql 中的冗余数据
+clean_energy_15(){
+mysql -h${url}  -P3306  -u${username} -p${password} ${database} -e "${del_energy_15_min_history}"
+}
+
+clean_energy_5(){
+mysql -h${url}  -P3306  -u${username} -p${password} ${database} -e "${del_energy_5_min_history}"
+}
+

+ 5 - 0
energy/history/config.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+MYSQL_URL='jdbc:mysql://192.168.0.76:3306/sagacloud_review?useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&tinyInt1isBit=false'
+MYSQL_USER=root
+MYSQL_PASSWORD=123456

+ 71 - 0
energy/history/energy_hive.sql

@@ -0,0 +1,71 @@
+/** 能源差值数据 15min 历史数据 */
+create table ods_energy_15_min_history
+(
+    building   string   comment '大楼id',
+    func_id    string   comment '仪表功能号',
+    meter      string   comment '仪表号',
+    data_time  string   comment '数据采集时间',
+    data_value decimal(30, 15)  comment '采集值',
+    `dt` string  comment '采集日期'
+) COMMENT '能源原始数据 15min 历史数据'
+ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001'
+STORED AS TEXTFILE
+LOCATION
+    'hdfs://sagaCluster:8020/warehouse/saga_dw/ods/ods_energy_15_min_history'
+TBLPROPERTIES (
+    'orc.compress'='lzo');
+
+/** 能源差值数据 5min 历史数据 */
+create table ods_energy_5_min_history
+(
+    building   string  comment '大楼id',
+    func_id    string  comment '仪表功能号',
+    meter      string  comment '仪表号',
+    data_time  string  comment '数据采集时间',
+    data_value decimal(30, 15)  comment '采集值',
+    `dt` string  comment '采集日期'
+) COMMENT '能源原始数据 5min 历史数据'
+ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001'
+STORED AS TEXTFILE
+LOCATION
+    'hdfs://sagaCluster:8020/warehouse/saga_dw/ods/ods_energy_5_min_history'
+TBLPROPERTIES (
+    'orc.compress'='lzo');
+
+/** 能源 fjd 数据 15min 历史数据 */
+create table ods_energy_15_min_fjd_history
+(
+    building   string  comment '大楼id',
+    func_id    string  comment '仪表功能号',
+    meter      string  comment '仪表号',
+    data_time  string  comment '数据采集时间',
+    data_value decimal(30, 15)  comment '采集值',
+    `dt` string  comment '采集日期'
+) COMMENT '能源 fjd 数据 15min 历史数据'
+ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001'
+STORED AS TEXTFILE
+LOCATION
+    'hdfs://sagaCluster:8020/warehouse/saga_dw/ods/ods_energy_15_min_fjd_history'
+TBLPROPERTIES (
+    'orc.compress'='lzo');
+
+/** 能源 fjd 数据 5min 历史数据 */
+create table ods_energy_5_min_fjd_history
+(
+    building   string  comment '大楼id',
+    func_id    string  comment '仪表功能号',
+    meter      string  comment '仪表号',
+    data_time  string  comment '数据采集时间',
+    data_value decimal(30, 15)  comment '采集值',
+    `dt` string  comment '采集日期'
+) COMMENT '能源 fjd 数据 5min 历史数据'
+ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001'
+STORED AS TEXTFILE
+LOCATION
+    'hdfs://sagaCluster:8020/warehouse/saga_dw/ods/ods_energy_5_min_fjd_history'
+TBLPROPERTIES (
+    'orc.compress'='lzo');
+
+
+
+

+ 43 - 0
energy/history/energy_mysql.sql

@@ -0,0 +1,43 @@
+/** 能源 15 分钟差值数据-历史 */
+create table energy_15_min_history
+(
+    building   varchar(128)    comment '大楼id',
+    func_id    varchar(128)    comment '仪表功能号',
+    meter      varchar(128)    comment '仪表号',
+    data_time  datetime        comment '数据采集时间',
+    data_value decimal(30, 15) comment '采集值',
+    dt         varchar(64)     comment '采集日期'
+) comment '能源 15 分钟差值数据-历史';
+
+/** 能源 5 分钟差值数据-历史 */
+create table energy_5_min_history
+(
+    building   varchar(128)    comment '大楼id',
+    func_id    varchar(128)    comment '仪表功能号',
+    meter      varchar(128)    comment '仪表号',
+    data_time  datetime        comment '数据采集时间',
+    data_value decimal(30, 15) comment '采集值',
+    dt         varchar(64)     comment '采集日期'
+) comment '能源 5 分钟差值数据-历史';
+
+/** 能源 15 分钟 fjd 数据-历史 */
+create table energy_15_min_fjd_history
+(
+    building   varchar(128)    comment '大楼id',
+    func_id    varchar(128)    comment '仪表功能号',
+    meter      varchar(128)    comment '仪表号',
+    data_time  datetime        comment '数据采集时间',
+    data_value decimal(30, 15) comment '采集值',
+    dt         varchar(64)     comment '采集日期'
+) comment '能源 15 分钟 fjd 数据-历史';
+
+/** 能源 5 分钟 fjd 数据-历史 */
+create table energy_5_min_fjd_history
+(
+    building   varchar(128)    comment '大楼id',
+    func_id    varchar(128)    comment '仪表功能号',
+    meter      varchar(128)    comment '仪表号',
+    data_time  datetime        comment '数据采集时间',
+    data_value decimal(30, 15) comment '采集值',
+    dt         varchar(64)     comment '采集日期'
+) comment '能源 5 分钟 fjd 数据-历史';

+ 70 - 0
energy/history/mysql_to_hdfs.sh

@@ -0,0 +1,70 @@
+#!/bin/bash
+
+. ./config.sh
+
+url=$MYSQL_URL
+username=$MYSQL_USER
+password=$MYSQL_PASSWORD
+
+## 无分区
+import_no_partition() {
+  sqoop import  \
+  -D mapred.job.queue.name=hive  \
+  --connect $1 \
+  --username $2 \
+  --password $3 \
+  --target-dir $4   \
+  --delete-target-dir \
+  --columns $5  \
+  --query "$6  and  \$CONDITIONS" \
+  --num-mappers 1 \
+  --hive-drop-import-delims \
+  --fields-terminated-by '\001' \
+  --compress \
+  --compression-codec lzo \
+  --hive-import \
+  --hive-database saga_dw \
+  --hive-table $7 \
+  --hive-overwrite \
+  --null-string '\\N' \
+  --null-non-string '\\N'
+}
+
+ods_energy_15_min_history(){
+  import_no_partition $url $username $password /warehouse/saga_dw/ods/ods_energy_15_min_history "building, func_id, meter, data_time, data_value, dt" "select building, func_id, meter, data_time, data_value, dt from energy_15_min_history where 1 = 1" ods_energy_15_min_history
+}
+
+ods_energy_5_min_history(){
+  import_no_partition $url $username $password /warehouse/saga_dw/ods/ods_energy_5_min_history "building, func_id, meter, data_time, data_value, dt" "select building, func_id, meter, data_time, data_value, dt from energy_5_min_history where 1 = 1" ods_energy_5_min_history
+}
+
+ods_energy_15_min_fjd_history(){
+  import_no_partition $url $username $password /warehouse/saga_dw/ods/ods_energy_15_min_fjd_history "building, func_id, meter, data_time, data_value, dt" "select building, func_id, meter, data_time, data_value, dt from energy_15_min_fjd_history where 1 = 1" ods_energy_15_min_fjd_history
+}
+
+ods_energy_5_min_fjd_history(){
+  import_no_partition $url $username $password /warehouse/saga_dw/ods/ods_energy_5_min_fjd_history "building, func_id, meter, data_time, data_value, dt" "select building, func_id, meter, data_time, data_value, dt from energy_5_min_fjd_history where 1 = 1" ods_energy_5_min_fjd_history
+}
+
+case $1 in
+"energy_15_min")
+  ods_energy_15_min_history
+  ods_energy_15_min_fjd_history
+  ;;
+"energy_5_min")
+  ods_energy_5_min_history
+  ods_energy_5_min_fjd_history
+  ;;
+"ods_energy_15_min_history")
+  ods_energy_15_min_history
+  ;;
+"ods_energy_5_min_history")
+  ods_energy_5_min_history
+  ;;
+"ods_energy_15_min_fjd_history")
+  ods_energy_15_min_fjd_history
+  ;;
+"ods_energy_5_min_fjd_history")
+  ods_energy_5_min_fjd_history
+  ;;
+esac

+ 81 - 0
energy/history/ods_view_to_table.sh

@@ -0,0 +1,81 @@
+#!/bin/bash
+
+env_config="
+use saga_dw;
+SET mapreduce.job.queuename=hive;
+SET hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat;
+SET hive.exec.dynamic.partition.mode=nonstrict;"
+
+ods_energy_15_min="
+insert into saga_dw.ods_energy_15_min partition(dt)
+select
+  building,
+  func_id,
+  meter,
+  data_time,
+  data_value,
+  dt
+  from saga_dw.ods_energy_15_min_history;"
+
+ods_energy_5_min="
+insert into saga_dw.ods_energy_5_min partition(dt)
+select
+  building,
+  func_id,
+  meter,
+  data_time,
+  data_value,
+  dt
+  from saga_dw.ods_energy_5_min_history;"
+
+ods_energy_15_min_fjd="
+insert into saga_dw.ods_energy_15_min_fjd partition(dt)
+select
+ building,
+ func_id,
+ meter,
+ data_time,
+ data_value,
+ dt
+ from saga_dw.ods_energy_15_min_fjd_history;"
+
+ods_energy_5_min_fjd="
+insert into saga_dw.ods_energy_5_min_fjd partition(dt)
+select
+  building,
+  func_id,
+  meter,
+  data_time,
+  data_value,
+  dt
+  from saga_dw.ods_energy_5_min_fjd_history;"
+
+
+insert_into_table(){
+  case $1 in
+  "energy_15_min")
+      echo $1
+      hive -e "$env_config$ods_energy_15_min$ods_energy_15_min_fjd"
+    ;;
+  "energy_5_min")
+      echo $1
+      hive -e "$env_config$ods_energy_5_min$ods_energy_5_min_fjd"
+    ;;
+  "ods_energy_15_min")
+      echo $1
+      hive -e "$env_config$ods_energy_15_min"
+  ;;
+  "ods_energy_5_min")
+      echo $1
+      hive -e "$env_config$ods_energy_5_min"
+  ;;
+  "ods_energy_15_min_fjd")
+      echo $1
+      hive -e "$env_config$ods_energy_15_min_fjd"
+  ;;
+  "ods_energy_5_min_fjd")
+      echo $1
+      hive -e "$env_config$ods_energy_5_min_fjd"
+  ;;
+  esac
+}

+ 0 - 29
energy/mysql_to_hdfs.sh

@@ -1,29 +0,0 @@
-#!/bin/bash
-
-. ./config.sh
-
-url=$MYSQL_URL
-username=$MYSQL_USER
-password=$MYSQL_PASSWORD
-
-## 无分区
-collect_data_np() {
-sqoop import -D mapred.job.queue.name=hive  \
---connect $1 \
---username $2 \
---password $3 \
---target-dir /warehouse/saga_dw/ods/$2  \
---delete-target-dir \
---query "$5 and \$CONDITIONS" \
---num-mappers 1 \
---hive-drop-import-delims \
---fields-terminated-by '\001' \
---compress \
---compression-codec lzo \
---hive-import \
---hive-database saga_dw \
---hive-table $4 \
---hive-overwrite \
---null-string '\\N' \
---null-non-string '\\N'
-}