wdl 2 years ago
parent
commit
926f1ceffe

+ 7 - 0
task_day/config.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+# 数据智能组数据库
+MYSQL_HOST=hadoop01
+MYSQL_URL="jdbc:mysql://$MYSQL_HOST:3306/saga_dev?useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&tinyInt1isBit=false"
+MYSQL_USER=root
+MYSQL_PASSWORD=j5ry0#jZ7vaUt5f4

+ 209 - 0
task_day/energy_hive.sql

@@ -0,0 +1,209 @@
+/** 能源原始数据 15min 差值 */
+create table ods_energy_15_min
+(
+    building   string,
+    func_id    string,
+    meter      string,
+    data_time  string,
+    data_value decimal(30, 15)
+) COMMENT '能源原始数据15min差值数据'
+PARTITIONED BY (`dt` string)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001'
+STORED AS TEXTFILE
+LOCATION
+    'hdfs://sagaCluster:8020/warehouse/saga_dw/ods/ods_energy_15_min'
+TBLPROPERTIES (
+    'orc.compress'='lzo');
+
+/** 能源原始数据 15min 差值临时表 */
+create table ods_energy_15_min_tmp
+(
+    building   string,
+    func_id    string,
+    meter      string,
+    data_time  string,
+    data_value decimal(30, 15)
+) COMMENT '能源原始数据15min差值临时表'
+    PARTITIONED BY (`dt` string)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001'
+STORED AS TEXTFILE
+LOCATION
+    'hdfs://sagaCluster:8020/warehouse/saga_dw/ods/ods_energy_15_min_tmp'
+TBLPROPERTIES (
+    'orc.compress'='lzo');
+
+/** 能源原始数据 5min 差值 */
+create table ods_energy_5_min
+(
+    building   string,
+    func_id    string,
+    meter      string,
+    data_time  string,
+    data_value decimal(30, 15)
+) COMMENT '能源原始数据15min差值数据'
+PARTITIONED BY (`dt` string)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001'
+STORED AS TEXTFILE
+LOCATION
+    'hdfs://sagaCluster:8020/warehouse/saga_dw/ods/ods_energy_5_min'
+TBLPROPERTIES (
+    'orc.compress'='lzo');
+
+/** 能源数据 15min */
+create table ods_energy_15_min_fjd
+(
+    building   string,
+    func_id    string,
+    meter      string,
+    data_time  string,
+    data_value decimal(30, 15)
+) COMMENT '能源原始数据 15min'
+PARTITIONED BY (`dt` string)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001'
+STORED AS TEXTFILE
+LOCATION
+    'hdfs://sagaCluster:8020/warehouse/saga_dw/ods/ods_energy_15_min_fjd'
+TBLPROPERTIES (
+    'orc.compress'='lzo');
+
+/** 能源数据 5min */
+create table ods_energy_5_min_fjd
+(
+    building   string,
+    func_id    string,
+    meter      string,
+    data_time  string,
+    data_value decimal(30, 15)
+) COMMENT '能源原始数据 5min'
+PARTITIONED BY (`dt` string)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001'
+STORED AS TEXTFILE
+LOCATION
+    'hdfs://sagaCluster:8020/warehouse/saga_dw/ods/ods_energy_5_min_fjd'
+TBLPROPERTIES (
+    'orc.compress'='lzo');
+
+/** 能源数据按小时 */
+create table saga_dw.dwd_energy_15_min_hour
+(
+    building         string,
+    func_id          string,
+    branch_type      string,
+    branch_type_name string,
+    use_range_type string,
+    value_sum        decimal(30, 15),
+    current_hour     string comment '小时',
+    current_year     integer comment '年份',
+    year_mouth       string comment '年份-月份',
+    week_of_year     integer comment '这一年的第几周'
+) COMMENT '能源原始数据15min按小时维度数据'
+    PARTITIONED BY (`dt` string)
+    ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001'
+    STORED AS TEXTFILE
+    LOCATION
+        'hdfs://sagaCluster:8020/warehouse/saga_dw/dwd/dwd_energy_15_min_hour'
+    TBLPROPERTIES (
+        'orc.compress' = 'lzo');
+
+
+/** CO2 15min 分精度 */
+create table ods_co2_15_min
+(
+    building   string,
+    func_id    string,
+    meter      string,
+    data_time  string,
+    data_value decimal(30, 15)
+) COMMENT 'CO2 15min 分精度'
+    PARTITIONED BY (`dt` string)
+    ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001'
+    STORED AS TEXTFILE
+    LOCATION
+        'hdfs://sagaCluster:8020/warehouse/saga_dw/ods/ods_co2_15_min'
+    TBLPROPERTIES (
+        'orc.compress'='lzo');
+
+/** PM2.5 15min 分精度 */
+create table ods_pm25_15_min
+(
+    building   string,
+    func_id    string,
+    meter      string,
+    data_time  string,
+    data_value decimal(30, 15)
+) COMMENT 'PM2.5 15min 分精度'
+    PARTITIONED BY (`dt` string)
+    ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001'
+    STORED AS TEXTFILE
+    LOCATION
+        'hdfs://sagaCluster:8020/warehouse/saga_dw/ods/ods_pm25_15_min'
+    TBLPROPERTIES (
+        'orc.compress'='lzo');
+
+/** 甲醛 15 分钟分精度 */
+create table ods_hcho_15_min
+(
+    building   string,
+    func_id    string,
+    meter      string,
+    data_time  string,
+    data_value decimal(30, 15)
+) COMMENT '甲醛 15 分钟分精度'
+    PARTITIONED BY (`dt` string)
+    ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001'
+    STORED AS TEXTFILE
+    LOCATION
+        'hdfs://sagaCluster:8020/warehouse/saga_dw/ods/ods_hcho_15_min'
+    TBLPROPERTIES (
+        'orc.compress'='lzo');
+
+/** 温度 15 分钟分精度 */
+create table ods_temperature_15_min
+(
+    building   string,
+    func_id    string,
+    meter      string,
+    data_time  string,
+    data_value decimal(30, 15)
+) COMMENT '温度 15 分钟分精度'
+    PARTITIONED BY (`dt` string)
+    ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001'
+    STORED AS TEXTFILE
+    LOCATION
+        'hdfs://sagaCluster:8020/warehouse/saga_dw/ods/ods_temperature_15_min'
+    TBLPROPERTIES (
+        'orc.compress'='lzo');
+
+/** 湿度 15 分钟分精度 */
+create table ods_humidity_15_min
+(
+    building   string,
+    func_id    string,
+    meter      string,
+    data_time  string,
+    data_value decimal(30, 15)
+) COMMENT '湿度 15 分钟分精度'
+    PARTITIONED BY (`dt` string)
+    ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001'
+    STORED AS TEXTFILE
+    LOCATION
+        'hdfs://sagaCluster:8020/warehouse/saga_dw/ods/ods_humidity_15_min'
+    TBLPROPERTIES (
+        'orc.compress'='lzo');
+
+
+create table ods_history_tmp
+(
+    building   string,
+    func_id    string,
+    meter      string,
+    data_time  string,
+    data_value decimal(30, 15)
+) COMMENT '历史数据中间表'
+    PARTITIONED BY (`dt` string)
+    ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001'
+    STORED AS TEXTFILE
+    LOCATION
+        'hdfs://sagaCluster:8020/warehouse/saga_dw/ods/tmp/ods_history_tmp'
+    TBLPROPERTIES (
+        'orc.compress'='lzo');

+ 116 - 0
task_day/energy_mysql.sql

@@ -0,0 +1,116 @@
+/** 能源 15 分钟数据 */
+create table energy_15_min
+(
+    building   varchar(128)    comment '大楼id',
+    func_id    varchar(128)    comment '仪表功能号',
+    meter      varchar(128)    comment '仪表号',
+    data_time  datetime        comment '数据采集时间',
+    data_value decimal(30, 15) comment '采集值',
+    dt         varchar(64)     comment '采集日期'
+) comment '能源 15 分钟差值数据';
+
+/** 能源 5 分钟数据 */
+create table energy_5_min
+(
+    building   varchar(128)    comment '大楼id',
+    func_id    varchar(128)    comment '仪表功能号',
+    meter      varchar(128)    comment '仪表号',
+    data_time  datetime        comment '数据采集时间',
+    data_value decimal(30, 15) comment '采集值',
+    dt         varchar(64)     comment '采集日期'
+) comment '能源 5 分钟差值数据';
+
+/** 能源 15 分钟数据-fjd */
+create table energy_15_min_fjd
+(
+    building   varchar(128)    comment '大楼id',
+    func_id    varchar(128)    comment '仪表功能号',
+    meter      varchar(128)    comment '仪表号',
+    data_time  datetime        comment '数据采集时间',
+    data_value decimal(30, 15) comment '采集值',
+    dt         varchar(64)     comment '采集日期'
+) comment '能源 15 分钟 fjd 数据';
+
+/** 能源 5 分钟数据-fjd */
+create table energy_5_min_fjd
+(
+    building   varchar(128)    comment '大楼id',
+    func_id    varchar(128)    comment '仪表功能号',
+    meter      varchar(128)    comment '仪表号',
+    data_time  datetime        comment '数据采集时间',
+    data_value decimal(30, 15) comment '采集值',
+    dt         varchar(64)     comment '采集日期'
+) comment '能源 5 分钟 fjd 数据';
+
+
+create table app_energy_15_min
+(
+    building         varchar(128)   ,
+    func_id          varchar(128)   ,
+    parent_code      varchar(128)   ,
+    branch_type      varchar(128)   ,
+    branch_type_name varchar(128)   ,
+    use_range_type   varchar(128)   ,
+    value_sum        decimal(30, 15),
+    current_hour     varchar(128)    comment '小时',
+    current_year     int             comment '年份',
+    year_mouth       varchar(128)    comment '年份-月份',
+    week_of_year     int             comment '这一年的第几周',
+    dt               datetime
+);
+
+
+create table co2_15_min
+(
+    building   varchar(128)     comment '大楼id',
+    func_id    varchar(128)     comment '仪表功能号',
+    meter      varchar(128)     comment '仪表号',
+    data_time  datetime         comment '数据采集时间',
+    data_value decimal(30, 15)  comment '采集值',
+    dt         varchar(64)      comment '采集日期'
+)
+    comment 'CO2 15 分钟数据';
+
+create table pm25_15_min
+(
+    building   varchar(128)     comment '大楼id',
+    func_id    varchar(128)     comment '仪表功能号',
+    meter      varchar(128)     comment '仪表号',
+    data_time  datetime         comment '数据采集时间',
+    data_value decimal(30, 15)  comment '采集值',
+    dt         varchar(64)      comment '采集日期'
+)
+    comment 'pm2.5 15 分钟数据';
+
+create table temperature_15_min
+(
+    building   varchar(128)     comment '大楼id',
+    func_id    varchar(128)     comment '仪表功能号',
+    meter      varchar(128)     comment '仪表号',
+    data_time  datetime         comment '数据采集时间',
+    data_value decimal(30, 15)  comment '采集值',
+    dt         varchar(64)      comment '采集日期'
+)
+    comment '温度 15 分钟数据';
+
+create table hcho_15_min
+(
+    building   varchar(128)     comment '大楼id',
+    func_id    varchar(128)     comment '仪表功能号',
+    meter      varchar(128)     comment '仪表号',
+    data_time  datetime         comment '数据采集时间',
+    data_value decimal(30, 15)  comment '采集值',
+    dt         varchar(64)      comment '采集日期'
+)
+    comment '甲醛 15 分钟数据';
+
+create table humidity_15_min
+(
+    building   varchar(128)     comment '大楼id',
+    func_id    varchar(128)     comment '仪表功能号',
+    meter      varchar(128)     comment '仪表号',
+    data_time  datetime         comment '数据采集时间',
+    data_value decimal(30, 15)  comment '采集值',
+    dt         varchar(64)      comment '采集日期'
+)
+    comment '湿度 15 分钟数据';

+ 64 - 0
task_day/hdfs_to_mysql.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+
+. /mnt/datadisk0/saga/bi_saga/dwh_saga/task_day/config.sh
+
+if [ -n "$2" ] ;then
+    do_date=$2
+else
+    echo "====没有输入数据的日期,取当前时间的前一天===="
+    do_date=$(date -d yesterday +"%Y-%m-%d")
+fi
+
+echo "$do_date"
+
+url=$MYSQL_URL
+username=$MYSQL_USER
+password=$MYSQL_PASSWORD
+
+hdfs_to_mysql() {
+  sqoop export -D mapred.job.queue.name=hive \
+    --connect $1 \
+    --username $2 \
+    --password $3 \
+    --table $4 \
+    -m 1 \
+    --hcatalog-database saga_dw \
+    --hcatalog-table $5 \
+    --fields-terminated-by '\001' \
+    --input-null-string '\\N' \
+    --input-null-non-string '\\N'
+}
+
+hdfs_to_mysql_by_day() {
+  sqoop export -D mapred.job.queue.name=hive \
+    --connect $1 \
+    --username $2 \
+    --password $3 \
+    --table $4 \
+    -m 1 \
+    --hcatalog-database saga_dw \
+    --hcatalog-table $5 \
+    --hcatalog-partition-keys dt \
+    --hcatalog-partition-values $do_date \
+    --fields-terminated-by '\001' \
+    --input-null-string '\\N' \
+    --input-null-non-string '\\N'
+}
+
+
+app_energy(){
+  hdfs_to_mysql "$url" "$username" "$password" "app_energy_15_min" "dwd_energy_15_min_hour"
+}
+app_energy_day(){
+  hdfs_to_mysql_by_day "$url" "$username" "$password" "app_energy_15_min" "dwd_energy_15_min_hour"
+}
+
+case $1 in
+"app_energy")
+  app_energy
+  ;;
+"app_energy_day")
+  app_energy_day
+  ;;
+esac
+

+ 98 - 0
task_day/mysql_to_hdfs.sh

@@ -0,0 +1,98 @@
+#!/bin/bash
+
+. /mnt/datadisk0/saga/bi_saga/dwh_saga/task_day/config.sh
+
+if [ -n "$2" ] ;then
+    do_date=$2
+else
+    echo "====没有输入数据的日期,取当前时间的前一天===="
+    do_date=$(date -d yesterday +"%Y-%m-%d")
+fi
+
+url=$MYSQL_URL
+username=$MYSQL_USER
+password=$MYSQL_PASSWORD
+
+mysql_to_hdfs_lzo() {
+	sqoop import  \
+	-D mapred.job.queue.name=default  \
+	--connect $1 \
+	--username $2 \
+	--password $3 \
+	--target-dir $4$7/$do_date \
+	--delete-target-dir \
+	--columns "$5" \
+	--query "$6 and \$CONDITIONS" \
+	--num-mappers 1 \
+	--hive-drop-import-delims \
+	--fields-terminated-by '\001' \
+	--compress \
+	--compression-codec lzo \
+	--hive-import \
+	--hive-database saga_dw \
+	--hive-table $7 \
+	--hive-overwrite \
+	--hive-partition-key dt \
+	--hive-partition-value $do_date \
+	--null-string '\\N' \
+	--null-non-string '\\N'
+}
+
+## 能源 15 分钟差值
+ods_energy_15_min(){
+  mysql_to_hdfs_lzo "$url" "$username" "$password" /warehouse/saga_dw/ods/tmp/ "building, func_id, meter, data_time, data_value" "select building, func_id, meter, from_unixtime(unix_timestamp(data_time) + 28800) as data_time, data_value from energy_15_min where 1 = 1 " ods_energy_15_min
+}
+
+## CO2 15 分钟分精度
+ods_co2_15_min(){
+  mysql_to_hdfs_lzo "$url" "$username" "$password" /warehouse/saga_dw/ods/tmp/ "building, func_id, meter, data_time, data_value" "select building, func_id, meter, data_time, data_value from co2_15_min where dt = '$do_date'" ods_co2_15_min
+}
+
+## PM2.5 15 分钟分精度
+ods_pm25_15_min(){
+  mysql_to_hdfs_lzo "$url" "$username" "$password" /warehouse/saga_dw/ods/tmp/ "building, func_id, meter, data_time, data_value" "select building, func_id, meter, data_time, data_value from pm25_15_min where dt = '$do_date'" ods_pm25_15_min
+}
+
+## 甲醛 15 分钟分精度
+ods_hcho_15_min(){
+  mysql_to_hdfs_lzo "$url" "$username" "$password" /warehouse/saga_dw/ods/tmp/ "building, func_id, meter, data_time, data_value" "select building, func_id, meter, data_time, data_value from hcho_15_min where dt = '$do_date'" ods_hcho_15_min
+}
+
+## 温度 15 分钟分精度
+ods_temperature_15_min(){
+  mysql_to_hdfs_lzo "$url" "$username" "$password" /warehouse/saga_dw/ods/tmp/ "building, func_id, meter, data_time, data_value" "select building, func_id, meter, data_time, data_value from temperature_15_min where dt = '$do_date'" ods_temperature_15_min
+}
+
+## 湿度 15 分钟分精度
+ods_humidity_15_min(){
+  mysql_to_hdfs_lzo "$url" "$username" "$password" /warehouse/saga_dw/ods/tmp/ "building, func_id, meter, data_time, data_value" "select building, func_id, meter, data_time, data_value from humidity_15_min where dt = '$do_date'" ods_humidity_15_min
+}
+
+case $1 in
+"all")
+  ods_energy_15_min
+  ods_co2_15_min
+  ods_pm25_15_min
+  ods_hcho_15_min
+  ods_temperature_15_min
+  ods_humidity_15_min
+  ;;
+"ods_energy_15_min")
+  ods_energy_15_min
+  ;;
+"ods_co2_15_min")
+  ods_co2_15_min
+  ;;
+"ods_pm25_15_min")
+  ods_pm25_15_min
+  ;;
+"ods_hcho_15_min")
+  ods_hcho_15_min
+  ;;
+"ods_temperature_15_min")
+  ods_temperature_15_min
+  ;;
+"ods_humidity_15_min")
+  ods_humidity_15_min
+  ;;
+esac

+ 102 - 0
task_day/ods_to_dwd.sh

@@ -0,0 +1,102 @@
+#!/bin/bash
+
+. /mnt/datadisk0/saga/bi_saga/dwh_saga/task_day/config.sh
+
+if [ -n "$2" ] ;then
+    do_date=$2
+else
+    echo "====没有输入数据的日期,取当前时间的前一天===="
+    do_date=$(date -d yesterday +"%Y-%m-%d")
+fi
+echo "$do_date"
+
+env_config="
+use saga_dw;
+SET mapreduce.job.queuename=default;
+"
+
+dwd_energy_15_min_hour="
+insert overwrite table saga_dw.dwd_energy_15_min_hour partition (dt = '$do_date')
+select building,
+       func_id,
+       parent_code,
+       branch_type,
+       branch_type_name,
+       use_range_type,
+       cast(value_sum as decimal(30, 15)) as value_sum ,
+       hour_ as current_hour,
+       current_year,
+       year_mouth,
+       week_of_year
+from (
+    select building,
+           func_id,
+           parent_code,
+           branch_type,
+           branch_type_name,
+           use_range_type,
+           dt,
+           t1.hour_,
+           (ap_sum - al_sum) value_sum
+    from (
+      select building,
+             func_id,
+             'AP1' parent_code,
+             'ALU' as branch_type,
+             '其他' as branch_type_name,
+             'OTHER' use_range_type,
+             sum(if((branch_type == 'AP' and use_range_type == '1'), data_value, 0)) ap_sum,
+             sum(if((branch_type == 'AL' and use_range_type in (2, 3, 4, 6, 7)), data_value, 0)) al_sum,
+             dt,
+             o15mh.hour_
+      from (
+          select building,
+               func_id,
+               meter,
+               data_time,
+               substr(data_time, 0, 13) hour_,
+               data_value,
+               dt
+        from ods_energy_15_min oe15m
+        where dt = '$do_date'
+           ) o15mh
+          left join dim_office_meter dom on o15mh.meter = dom.meter
+      where dt = '$do_date' and  branch_type in ('AP', 'AL')
+      group by building, func_id, dt, o15mh.hour_ ) t1
+    union
+    select building,
+           func_id,
+           parent_code,
+           branch_type,
+           branch_type_name,
+           use_range_type,
+           dt,
+           o15mh.hour_,
+           sum(data_value) value_sum
+    from (
+        select building,
+               func_id,
+               meter,
+               data_time,
+               substr(data_time, 12, 2) hour_,
+               data_value,
+               dt
+        from ods_energy_15_min oe15m
+        where dt = '$do_date'
+         ) o15mh
+        left join dim_office_meter dom2 on o15mh.meter = dom2.meter
+    group by building, func_id, parent_code, branch_type, branch_type_name, use_range_type, dt, o15mh.hour_
+     ) teh
+    left join dim_date_day ddd on dt = date_id
+order by current_hour;"
+
+
+insert_into_table(){
+    case $1 in
+    "all")
+      hive -e "$env_config$dwd_energy_15_min_hour"
+      ;;
+    esac
+}
+
+insert_into_table "$1"

+ 96 - 0
task_day/py_mysql.sh

@@ -0,0 +1,96 @@
+#!/bin/bash
+
+#. /mnt/datadisk0/saga/bi_saga/dwh_saga/task_day/config.sh
+
+if [ -n "$2" ] ;then
+    do_date=$2
+    startSec=$(date -d "$do_date" "+%s")
+    echo "$startSec"
+    current_day=$(date -d "@$startSec" "+%Y-%m-%d")
+
+    start_time=$(date -d "-1 day $current_day" +%Y%m%d)000000
+    end_time=$(date -d "0 day $current_day" +%Y%m%d)000000
+
+    energy_start_time=$(date -d "-2 day $current_day" +%Y%m%d)160000
+    energy_end_time=$(date -d "-1 day $current_day" +%Y%m%d)160000
+else
+    echo "====没有输入数据的日期,取当前时间的前一天===="
+
+    start_time=$(date -d "-1 day" +"%Y%m%d")000000
+    end_time=$(date -d "0 day" +"%Y%m%d")000000
+
+    energy_start_time=$(date -d "-2 day" +"%Y%m%d")160000
+    energy_end_time=$(date -d "-1 day" +"%Y%m%d")160000
+fi
+
+echo "start_time:" "$start_time"
+echo "end_time:"  "$end_time"
+echo "energy_start_time:" "$energy_start_time"
+echo "energy_end_time:"  "$energy_end_time"
+
+#
+# 11301  CO2
+# 11401  PM2.5
+# 11305  甲醛
+# 11101  温度
+# 11201  湿度
+
+## 能源数据
+energy_hbase_to_mysql(){
+  /usr/local/bin/python3 /mnt/datadisk0/saga/bi_saga/python_transfer/start_mysql.py 10101 --start_time "$energy_start_time" --end_time "$energy_end_time"
+}
+
+## CO2
+co2_hbase_to_mysql(){
+  /usr/local/bin/python3 /mnt/datadisk0/saga/bi_saga/python_transfer/start_mysql.py 11301 --start_time "$start_time" --end_time "$end_time"
+}
+
+## PM2.5
+pm25_hbase_to_mysql(){
+  /usr/local/bin/python3 /mnt/datadisk0/saga/bi_saga/python_transfer/start_mysql.py 11401 --start_time "$start_time" --end_time "$end_time"
+}
+
+## 甲醛
+hcho_hbase_to_mysql(){
+  /usr/local/bin/python3 /mnt/datadisk0/saga/bi_saga/python_transfer/start_mysql.py 11305 --start_time "$start_time" --end_time "$end_time"
+}
+
+## 温度
+temperature_hbase_to_mysql(){
+  /usr/local/bin/python3 /mnt/datadisk0/saga/bi_saga/python_transfer/start_mysql.py 11101 --start_time "$start_time" --end_time "$end_time"
+}
+
+## 湿度
+humidity_hbase_to_mysql(){
+  /usr/local/bin/python3 /mnt/datadisk0/saga/bi_saga/python_transfer/start_mysql.py 11201 --start_time "$start_time" --end_time "$end_time"
+}
+
+
+case $1 in
+"all")
+    energy_hbase_to_mysql
+    co2_hbase_to_mysql
+    pm25_hbase_to_mysql
+    hcho_hbase_to_mysql
+    temperature_hbase_to_mysql
+    humidity_hbase_to_mysql
+  ;;
+"energy")
+    energy_hbase_to_mysql
+  ;;
+"co2")
+    co2_hbase_to_mysql
+  ;;
+"pm25")
+    pm25_hbase_to_mysql
+  ;;
+"hcho")
+    hcho_hbase_to_mysql
+  ;;
+"temperature")
+    temperature_hbase_to_mysql
+  ;;
+"humidity")
+    humidity_hbase_to_mysql
+  ;;
+esac

+ 42 - 0
task_history/history_day.sh

@@ -0,0 +1,42 @@
+#! /bin/bash
+
+. /mnt/datadisk0/saga/bi_saga/dwh_saga/task_day/config.sh
+
+startDate=$1
+endDate=$2
+startSec=$(date -d "$startDate" "+%s")
+endSec=$(date -d "$endDate" "+%s")
+source /etc/profile.d/saga_env.sh
+for((i=$startSec;i<=$endSec;i+=86400))
+do
+    current_day=$(date -d "@$i" "+%Y-%m-%d")
+
+    start_time=$(date -d "0 day $current_day" +%Y%m%d)000000
+    end_time=$(date -d "1 day $current_day" +%Y%m%d)000000
+
+    echo current_day:"$current_day"
+    echo start_time:"${start_time}"
+    echo end_time:"${end_time}"
+
+    echo "*******************能源数据需要减掉8小时*******************"
+    energy_start_time=$(date -d "-1 day $current_day" +%Y%m%d)160000
+    energy_end_time=$(date -d "-0 day $current_day" +%Y%m%d)160000
+    echo energy_start_time:"${energy_start_time}"
+    echo energy_end_time:"${energy_end_time}"
+
+    # 执行某天的数据导入 hbase —> mysql
+#    sh /mnt/datadisk0/saga/bi_saga/dwh_saga/task_day/py_mysql.sh energy --start_time $energy_start_time --end_time $energy_end_time
+
+    # mysql -> hive
+#    sh /mnt/datadisk0/saga/bi_saga/dwh_saga/task_day/mysql_to_hdfs.sh ods_energy_15_min $current_day
+
+    # ods -> dwd
+#    ssh hadoop01 sh /mnt/datadisk0/saga/bi_saga/dwh_saga/task_day/ods_to_dwd.sh all $current_day
+
+    # 清理当天mysql中的数据
+#    ssh hadoop01 sh /mnt/datadisk0/saga/bi_saga/dwh_saga/task_day/clean_mysql.sh clean_energy_15 $current_day
+
+    # hive -> mysql
+#    sh /mnt/datadisk0/saga/bi_saga/dwh_saga/task_day/hdfs_to_mysql.sh app_energy_day $current_day
+
+done