Browse Source

修改配置

wdl 2 years ago
parent
commit
3451dc4544
5 changed files with 41 additions and 99 deletions
  1. 10 2
      task_day/config.sh
  2. 21 87
      task_day/energy_hive.sql
  3. 3 3
      task_day/hdfs_to_mysql.sh
  4. 6 6
      task_day/mysql_to_hdfs.sh
  5. 1 1
      task_day/ods_to_dwd.sh

+ 10 - 2
task_day/config.sh

@@ -1,7 +1,15 @@
 #!/bin/bash
+# shellcheck disable=SC2034
 
 # 数据智能组数据库
-MYSQL_HOST=hadoop01
+MYSQL_HOST=10.10.10.58
 MYSQL_URL="jdbc:mysql://$MYSQL_HOST:3306/saga_dev?useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&tinyInt1isBit=false"
 MYSQL_USER=root
-MYSQL_PASSWORD=j5ry0#jZ7vaUt5f4
+MYSQL_PASSWORD=dwh@2022
+
+
+# 上格云数据库
+SAGA_HOST=39.106.8.246
+SAGA_URL="jdbc:mysql://$MYSQL_HOST:53309/saga_bi?useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&tinyInt1isBit=false"
+SAGA_USER=root
+SAGA_PASSWORD='H%k3!BHw1kQXIc70'

+ 21 - 87
task_day/energy_hive.sql

@@ -1,3 +1,5 @@
+create database saga_dw location "/saga/saga_dw";
+
 /** 能源原始数据 15min 差值 */
 create table ods_energy_15_min
 (
@@ -7,47 +9,13 @@ create table ods_energy_15_min
     data_time  string,
     data_value decimal(30, 15)
 ) COMMENT '能源原始数据15min差值数据'
-PARTITIONED BY (`dt` string)
-ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001'
-STORED AS TEXTFILE
-LOCATION
-    'hdfs://sagaCluster:8020/warehouse/saga_dw/ods/ods_energy_15_min'
-TBLPROPERTIES (
-    'orc.compress'='lzo');
-
-/** 能源原始数据 15min 差值临时表 */
-create table ods_energy_15_min_tmp
-(
-    building   string,
-    func_id    string,
-    meter      string,
-    data_time  string,
-    data_value decimal(30, 15)
-) COMMENT '能源原始数据15min差值临时表'
     PARTITIONED BY (`dt` string)
-ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001'
-STORED AS TEXTFILE
-LOCATION
-    'hdfs://sagaCluster:8020/warehouse/saga_dw/ods/ods_energy_15_min_tmp'
-TBLPROPERTIES (
-    'orc.compress'='lzo');
-
-/** 能源原始数据 5min 差值 */
-create table ods_energy_5_min
-(
-    building   string,
-    func_id    string,
-    meter      string,
-    data_time  string,
-    data_value decimal(30, 15)
-) COMMENT '能源原始数据15min差值数据'
-PARTITIONED BY (`dt` string)
-ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001'
-STORED AS TEXTFILE
-LOCATION
-    'hdfs://sagaCluster:8020/warehouse/saga_dw/ods/ods_energy_5_min'
-TBLPROPERTIES (
-    'orc.compress'='lzo');
+    ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001'
+    STORED AS TEXTFILE
+    LOCATION
+        'hdfs://HDFS85113:8020/saga/saga_dw/ods/ods_energy_15_min'
+    TBLPROPERTIES (
+        'orc.compress'='lzo');
 
 /** 能源数据 15min */
 create table ods_energy_15_min_fjd
@@ -58,30 +26,13 @@ create table ods_energy_15_min_fjd
     data_time  string,
     data_value decimal(30, 15)
 ) COMMENT '能源原始数据 15min'
-PARTITIONED BY (`dt` string)
-ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001'
-STORED AS TEXTFILE
-LOCATION
-    'hdfs://sagaCluster:8020/warehouse/saga_dw/ods/ods_energy_15_min_fjd'
-TBLPROPERTIES (
-    'orc.compress'='lzo');
-
-/** 能源数据 5min */
-create table ods_energy_5_min_fjd
-(
-    building   string,
-    func_id    string,
-    meter      string,
-    data_time  string,
-    data_value decimal(30, 15)
-) COMMENT '能源原始数据 5min'
-PARTITIONED BY (`dt` string)
-ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001'
-STORED AS TEXTFILE
-LOCATION
-    'hdfs://sagaCluster:8020/warehouse/saga_dw/ods/ods_energy_5_min_fjd'
-TBLPROPERTIES (
-    'orc.compress'='lzo');
+    PARTITIONED BY (`dt` string)
+    ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001'
+    STORED AS TEXTFILE
+    LOCATION
+        'hdfs://HDFS85113:8020/saga/saga_dw/ods/ods_energy_15_min_fjd'
+    TBLPROPERTIES (
+        'orc.compress'='lzo');
 
 /** 能源数据按小时 */
 create table saga_dw.dwd_energy_15_min_hour
@@ -101,7 +52,7 @@ create table saga_dw.dwd_energy_15_min_hour
     ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001'
     STORED AS TEXTFILE
     LOCATION
-        'hdfs://sagaCluster:8020/warehouse/saga_dw/dwd/dwd_energy_15_min_hour'
+        'hdfs://HDFS85113:8020/saga/saga_dw/dwd/dwd_energy_15_min_hour'
     TBLPROPERTIES (
         'orc.compress' = 'lzo');
 
@@ -119,7 +70,7 @@ create table ods_co2_15_min
     ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001'
     STORED AS TEXTFILE
     LOCATION
-        'hdfs://sagaCluster:8020/warehouse/saga_dw/ods/ods_co2_15_min'
+        'hdfs://HDFS85113:8020/saga/saga_dw/ods/ods_co2_15_min'
     TBLPROPERTIES (
         'orc.compress'='lzo');
 
@@ -136,7 +87,7 @@ create table ods_pm25_15_min
     ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001'
     STORED AS TEXTFILE
     LOCATION
-        'hdfs://sagaCluster:8020/warehouse/saga_dw/ods/ods_pm25_15_min'
+        'hdfs://HDFS85113:8020/saga/saga_dw/ods/ods_pm25_15_min'
     TBLPROPERTIES (
         'orc.compress'='lzo');
 
@@ -153,7 +104,7 @@ create table ods_hcho_15_min
     ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001'
     STORED AS TEXTFILE
     LOCATION
-        'hdfs://sagaCluster:8020/warehouse/saga_dw/ods/ods_hcho_15_min'
+        'hdfs://HDFS85113:8020/saga/saga_dw/ods/ods_hcho_15_min'
     TBLPROPERTIES (
         'orc.compress'='lzo');
 
@@ -170,7 +121,7 @@ create table ods_temperature_15_min
     ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001'
     STORED AS TEXTFILE
     LOCATION
-        'hdfs://sagaCluster:8020/warehouse/saga_dw/ods/ods_temperature_15_min'
+        'hdfs://HDFS85113:8020/saga/saga_dw/ods/ods_temperature_15_min'
     TBLPROPERTIES (
         'orc.compress'='lzo');
 
@@ -187,23 +138,6 @@ create table ods_humidity_15_min
     ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001'
     STORED AS TEXTFILE
     LOCATION
-        'hdfs://sagaCluster:8020/warehouse/saga_dw/ods/ods_humidity_15_min'
+        'hdfs://HDFS85113:8020/saga/saga_dw/ods/ods_humidity_15_min'
     TBLPROPERTIES (
         'orc.compress'='lzo');
-
-
-create table ods_history_tmp
-(
-    building   string,
-    func_id    string,
-    meter      string,
-    data_time  string,
-    data_value decimal(30, 15)
-) COMMENT '历史数据中间表'
-    PARTITIONED BY (`dt` string)
-    ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001'
-    STORED AS TEXTFILE
-    LOCATION
-        'hdfs://sagaCluster:8020/warehouse/saga_dw/ods/tmp/ods_history_tmp'
-    TBLPROPERTIES (
-        'orc.compress'='lzo');

+ 3 - 3
task_day/hdfs_to_mysql.sh

@@ -11,9 +11,9 @@ fi
 
 echo "$do_date"
 
-url=$MYSQL_URL
-username=$MYSQL_USER
-password=$MYSQL_PASSWORD
+url=$SAGA_URL
+username=$SAGA_USER
+password=$SAGA_PASSWORD
 
 hdfs_to_mysql() {
   sqoop export -D mapred.job.queue.name=hive \

+ 6 - 6
task_day/mysql_to_hdfs.sh

@@ -40,32 +40,32 @@ mysql_to_hdfs_lzo() {
 
 ## 能源 15 分钟差值
 ods_energy_15_min(){
-  mysql_to_hdfs_lzo "$url" "$username" "$password" /warehouse/saga_dw/ods/tmp/ "building, func_id, meter, data_time, data_value" "select building, func_id, meter, from_unixtime(unix_timestamp(data_time) + 28800) as data_time, data_value from energy_15_min where 1 = 1 " ods_energy_15_min
+  mysql_to_hdfs_lzo "$url" "$username" "$password" /saga/saga_dw/ods/tmp/ "building, func_id, meter, data_time, data_value" "select building, func_id, meter, from_unixtime(unix_timestamp(data_time) + 28800) as data_time, data_value from energy_15_min where 1 = 1 " ods_energy_15_min
 }
 
 ## CO2 15 分钟分精度
 ods_co2_15_min(){
-  mysql_to_hdfs_lzo "$url" "$username" "$password" /warehouse/saga_dw/ods/tmp/ "building, func_id, meter, data_time, data_value" "select building, func_id, meter, data_time, data_value from co2_15_min where dt = '$do_date'" ods_co2_15_min
+  mysql_to_hdfs_lzo "$url" "$username" "$password" /saga/saga_dw/ods/tmp/ "building, func_id, meter, data_time, data_value" "select building, func_id, meter, data_time, data_value from co2_15_min where dt = '$do_date'" ods_co2_15_min
 }
 
 ## PM2.5 15 分钟分精度
 ods_pm25_15_min(){
-  mysql_to_hdfs_lzo "$url" "$username" "$password" /warehouse/saga_dw/ods/tmp/ "building, func_id, meter, data_time, data_value" "select building, func_id, meter, data_time, data_value from pm25_15_min where dt = '$do_date'" ods_pm25_15_min
+  mysql_to_hdfs_lzo "$url" "$username" "$password" /saga/saga_dw/ods/tmp/ "building, func_id, meter, data_time, data_value" "select building, func_id, meter, data_time, data_value from pm25_15_min where dt = '$do_date'" ods_pm25_15_min
 }
 
 ## 甲醛 15 分钟分精度
 ods_hcho_15_min(){
-  mysql_to_hdfs_lzo "$url" "$username" "$password" /warehouse/saga_dw/ods/tmp/ "building, func_id, meter, data_time, data_value" "select building, func_id, meter, data_time, data_value from hcho_15_min where dt = '$do_date'" ods_hcho_15_min
+  mysql_to_hdfs_lzo "$url" "$username" "$password" /saga/saga_dw/ods/tmp/ "building, func_id, meter, data_time, data_value" "select building, func_id, meter, data_time, data_value from hcho_15_min where dt = '$do_date'" ods_hcho_15_min
 }
 
 ## 温度 15 分钟分精度
 ods_temperature_15_min(){
-  mysql_to_hdfs_lzo "$url" "$username" "$password" /warehouse/saga_dw/ods/tmp/ "building, func_id, meter, data_time, data_value" "select building, func_id, meter, data_time, data_value from temperature_15_min where dt = '$do_date'" ods_temperature_15_min
+  mysql_to_hdfs_lzo "$url" "$username" "$password" /saga/saga_dw/ods/tmp/ "building, func_id, meter, data_time, data_value" "select building, func_id, meter, data_time, data_value from temperature_15_min where dt = '$do_date'" ods_temperature_15_min
 }
 
 ## 湿度 15 分钟分精度
 ods_humidity_15_min(){
-  mysql_to_hdfs_lzo "$url" "$username" "$password" /warehouse/saga_dw/ods/tmp/ "building, func_id, meter, data_time, data_value" "select building, func_id, meter, data_time, data_value from humidity_15_min where dt = '$do_date'" ods_humidity_15_min
+  mysql_to_hdfs_lzo "$url" "$username" "$password" /saga/saga_dw/ods/tmp/ "building, func_id, meter, data_time, data_value" "select building, func_id, meter, data_time, data_value from humidity_15_min where dt = '$do_date'" ods_humidity_15_min
 }
 
 case $1 in

+ 1 - 1
task_day/ods_to_dwd.sh

@@ -54,7 +54,7 @@ from (
                func_id,
                meter,
                data_time,
-               substr(data_time, 0, 13) hour_,
+               substr(data_time, 12, 2) hour_,
                data_value,
                dt
         from ods_energy_15_min oe15m