wudianlong 2 gadi atpakaļ
vecāks
revīzija
7a2b3eb74c

+ 42 - 4
energy/dayTask/clean_mysql.sh

@@ -7,15 +7,53 @@ username=$MYSQL_USER
 password=$MYSQL_PASSWORD
 database=saga_dw
 
-del_energy_15_min_history="truncate table energy_15_min_history;"
-del_energy_15_min_fjd_history="truncate table energy_15_min_fjd_history;"
+del_energy_15_min="truncate table energy_15_min;"
+del_energy_5_min="truncate table energy_5_min;"
+del_energy_15_min_fjd="truncate table energy_15_min_fjd;"
+del_energy_5_min_fjd="truncate table energy_5_min_fjd;"
 
 ## 历史数据一次性导入后,需要删除 mysql 中的冗余数据
 clean_energy_15(){
-mysql -h${url}  -P3306  -u${username} -p${password} ${database} -e "${del_energy_15_min_history}"
+mysql -h${url}  -P3306  -u${username} -p${password} ${database} -e "${del_energy_15_min}"
 }
 
 clean_energy_5(){
-mysql -h${url}  -P3306  -u${username} -p${password} ${database} -e "${del_energy_15_min_fjd_history}"
+mysql -h${url}  -P3306  -u${username} -p${password} ${database} -e "${del_energy_15_min_fjd}"
 }
 
+clean_energy_15_fjd(){
+mysql -h${url}  -P3306  -u${username} -p${password} ${database} -e "${del_energy_15_min}"
+}
+
+clean_energy_5_fjd(){
+mysql -h${url}  -P3306  -u${username} -p${password} ${database} -e "${del_energy_15_min_fjd}"
+}
+
+case $1 in
+"all")
+    clean_energy_15
+    clean_energy_15_fjd
+    clean_energy_5
+    clean_energy_5_fjd
+  ;;
+"clean_15")
+    clean_energy_15
+    clean_energy_15_fjd
+  ;;
+"clean_5")
+    clean_energy_5
+    clean_energy_5_fjd
+  ;;
+"clean_energy_15")
+    clean_energy_15
+  ;;
+"clean_energy_15_fjd")
+    clean_energy_15_fjd
+  ;;
+"clean_energy_5")
+    clean_energy_5
+  ;;
+"clean_energy_5_fjd")
+    clean_energy_5_fjd
+  ;;
+esac

+ 10 - 10
energy/dayTask/energy_hive.sql

@@ -5,7 +5,7 @@ create table ods_energy_15_min
     func_id    string,
     meter      string,
     data_time  string,
-    data_value decimal(20, 15)
+    data_value decimal(30, 15)
 ) COMMENT '能源原始数据15min历史数据'
 PARTITIONED BY (`dt` string)
 ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001'
@@ -22,7 +22,7 @@ create table ods_energy_5_min
     func_id    string,
     meter      string,
     data_time  string,
-    data_value decimal(20, 15)
+    data_value decimal(30, 15)
 ) COMMENT '能源原始数据15min历史数据'
 PARTITIONED BY (`dt` string)
 ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001'
@@ -33,36 +33,36 @@ TBLPROPERTIES (
     'orc.compress'='lzo');
 
 /** 能源数据 15min */
-create table ods_energy_15_min
+create table ods_energy_15_min_fjd
 (
     building   string,
     func_id    string,
     meter      string,
     data_time  string,
-    data_value decimal(20, 15)
-) COMMENT '能源原始数据15min'
+    data_value decimal(30, 15)
+) COMMENT '能源原始数据 15min'
 PARTITIONED BY (`dt` string)
 ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001'
 STORED AS TEXTFILE
 LOCATION
-    'hdfs://sagaCluster:8020/warehouse/saga_dw/ods/ods_energy_15_min'
+    'hdfs://sagaCluster:8020/warehouse/saga_dw/ods/ods_energy_15_min_fjd'
 TBLPROPERTIES (
     'orc.compress'='lzo');
 
 /** 能源数据 5min */
-create table ods_energy_5_min
+create table ods_energy_5_min_fjd
 (
     building   string,
     func_id    string,
     meter      string,
     data_time  string,
-    data_value decimal(20, 15)
-) COMMENT '能源原始数据15min'
+    data_value decimal(30, 15)
+) COMMENT '能源原始数据 5min'
 PARTITIONED BY (`dt` string)
 ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001'
 STORED AS TEXTFILE
 LOCATION
-    'hdfs://sagaCluster:8020/warehouse/saga_dw/ods/ods_energy_5_min'
+    'hdfs://sagaCluster:8020/warehouse/saga_dw/ods/ods_energy_5_min_fjd'
 TBLPROPERTIES (
     'orc.compress'='lzo');
 

+ 57 - 30
energy/dayTask/mysql_to_hdfs.sh

@@ -2,42 +2,29 @@
 
 . ./config.sh
 
+
+if [ -n "$2" ] ;then
+    echo "如果是输入的日期按照取输入日期"
+    do_date=$2
+else
+    echo "====没有输入数据的日期,取当前时间的前一天===="
+    do_date=`date -d yesterday +"%Y-%m-%d"`
+fi
+
 url=$MYSQL_URL
 username=$MYSQL_USER
 password=$MYSQL_PASSWORD
 
-## 无分区
-collect_data_np() {
-sqoop import -D mapred.job.queue.name=hive  \
---connect $1 \
---username $2 \
---password $3 \
---target-dir /warehouse/saga_dw/ods/$2  \
---delete-target-dir \
---query "$5 and \$CONDITIONS" \
---num-mappers 1 \
---hive-drop-import-delims \
---fields-terminated-by '\001' \
---compress \
---compression-codec lzo \
---hive-import \
---hive-database saga_dw \
---hive-table $4 \
---hive-overwrite \
---null-string '\\N' \
---null-non-string '\\N'
-}
-
 mysql_to_hdfs_lzo() {
   sqoop import  \
   -D mapred.job.queue.name=hive  \
-  --connect "jdbc:mysql://hadoop01:3306/saga_dev?useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&tinyInt1isBit=false" \
-  --username root \
-  --password j5ry0#jZ7vaUt5f4 \
-  --target-dir /warehouse/saga_dw/ods/ods_energy_15_min_history  \
+  --connect $1 \
+  --username $2 \
+  --password $3 \
+  --target-dir $4$do_date  \
   --delete-target-dir \
-  --columns "building, func_id, meter, data_time, data_value, dt" \
-  --query "select building, func_id, meter, data_time, data_value, dt from energy_15_min where 1 = 1 and  \$CONDITIONS" \
+  --columns $5 \
+  --query "$6 and  \$CONDITIONS" \
   --num-mappers 1 \
   --hive-drop-import-delims \
   --fields-terminated-by '\001' \
@@ -45,8 +32,48 @@ mysql_to_hdfs_lzo() {
   --compression-codec lzo \
   --hive-import \
   --hive-database saga_dw \
-  --hive-table ods_energy_15_min_history \
+  --hive-table $7 \
   --hive-overwrite \
   --null-string '\\N' \
   --null-non-string '\\N'
-}
+}
+
+
+ods_energy_15_min(){
+  import_no_partition $url $username $password /warehouse/saga_dw/ods/ods_energy_15_min "building, func_id, meter, data_time, data_value" "select building, func_id, meter, data_time, data_value from energy_15_min where 1 = 1" ods_energy_15_min
+}
+
+ods_energy_5_min(){
+  import_no_partition $url $username $password /warehouse/saga_dw/ods/ods_energy_5_min "building, func_id, meter, data_time, data_value" "select building, func_id, meter, data_time, data_value from energy_5_min where 1 = 1" ods_energy_5_min
+}
+
+ods_energy_15_min_fjd(){
+  import_no_partition $url $username $password /warehouse/saga_dw/ods/ods_energy_15_min_fjd "building, func_id, meter, data_time, data_value" "select building, func_id, meter, data_time, data_value from energy_15_min_fjd where 1 = 1" ods_energy_15_min_fjd
+}
+
+ods_energy_5_min_fjd(){
+  import_no_partition $url $username $password /warehouse/saga_dw/ods/ods_energy_5_min_fjd "building, func_id, meter, data_time, data_value" "select building, func_id, meter, data_time, data_value from energy_5_min_fjd where 1 = 1" ods_energy_5_min_fjd
+}
+
+case $1 in
+"energy_15_min")
+  ods_energy_15_min
+  ods_energy_15_min_fjd
+  ;;
+"energy_5_min")
+  ods_energy_5_min
+  ods_energy_5_min_fjd
+  ;;
+"ods_energy_15_min")
+  ods_energy_15_min
+  ;;
+"ods_energy_5_min")
+  ods_energy_5_min
+  ;;
+"ods_energy_15_min_fjd")
+  ods_energy_15_min_fjd
+  ;;
+"ods_energy_5_min_fjd")
+  ods_energy_5_min_fjd
+  ;;
+esac