|
@@ -2,42 +2,29 @@
|
|
|
|
|
|
. ./config.sh
|
|
. ./config.sh
|
|
|
|
|
|
|
|
+
|
|
|
|
+if [ -n "$2" ] ;then
|
|
|
|
+ echo "如果是输入的日期按照取输入日期"
|
|
|
|
+ do_date=$2
|
|
|
|
+else
|
|
|
|
+ echo "====没有输入数据的日期,取当前时间的前一天===="
|
|
|
|
+ do_date=`date -d yesterday +"%Y-%m-%d"`
|
|
|
|
+fi
|
|
|
|
+
|
|
url=$MYSQL_URL
|
|
url=$MYSQL_URL
|
|
username=$MYSQL_USER
|
|
username=$MYSQL_USER
|
|
password=$MYSQL_PASSWORD
|
|
password=$MYSQL_PASSWORD
|
|
|
|
|
|
-## 无分区
|
|
|
|
-collect_data_np() {
|
|
|
|
-sqoop import -D mapred.job.queue.name=hive \
|
|
|
|
---connect $1 \
|
|
|
|
---username $2 \
|
|
|
|
---password $3 \
|
|
|
|
---target-dir /warehouse/saga_dw/ods/$2 \
|
|
|
|
---delete-target-dir \
|
|
|
|
---query "$5 and \$CONDITIONS" \
|
|
|
|
---num-mappers 1 \
|
|
|
|
---hive-drop-import-delims \
|
|
|
|
---fields-terminated-by '\001' \
|
|
|
|
---compress \
|
|
|
|
---compression-codec lzo \
|
|
|
|
---hive-import \
|
|
|
|
---hive-database saga_dw \
|
|
|
|
---hive-table $4 \
|
|
|
|
---hive-overwrite \
|
|
|
|
---null-string '\\N' \
|
|
|
|
---null-non-string '\\N'
|
|
|
|
-}
|
|
|
|
-
|
|
|
|
mysql_to_hdfs_lzo() {
|
|
mysql_to_hdfs_lzo() {
|
|
sqoop import \
|
|
sqoop import \
|
|
-D mapred.job.queue.name=hive \
|
|
-D mapred.job.queue.name=hive \
|
|
- --connect "jdbc:mysql://hadoop01:3306/saga_dev?useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&tinyInt1isBit=false" \
|
|
|
|
- --username root \
|
|
|
|
- --password j5ry0#jZ7vaUt5f4 \
|
|
|
|
- --target-dir /warehouse/saga_dw/ods/ods_energy_15_min_history \
|
|
|
|
|
|
+ --connect $1 \
|
|
|
|
+ --username $2 \
|
|
|
|
+ --password $3 \
|
|
|
|
+ --target-dir $4$do_date \
|
|
--delete-target-dir \
|
|
--delete-target-dir \
|
|
- --columns "building, func_id, meter, data_time, data_value, dt" \
|
|
|
|
- --query "select building, func_id, meter, data_time, data_value, dt from energy_15_min where 1 = 1 and \$CONDITIONS" \
|
|
|
|
|
|
+ --columns $5 \
|
|
|
|
+ --query "$6 and \$CONDITIONS" \
|
|
--num-mappers 1 \
|
|
--num-mappers 1 \
|
|
--hive-drop-import-delims \
|
|
--hive-drop-import-delims \
|
|
--fields-terminated-by '\001' \
|
|
--fields-terminated-by '\001' \
|
|
@@ -45,8 +32,48 @@ mysql_to_hdfs_lzo() {
|
|
--compression-codec lzo \
|
|
--compression-codec lzo \
|
|
--hive-import \
|
|
--hive-import \
|
|
--hive-database saga_dw \
|
|
--hive-database saga_dw \
|
|
- --hive-table ods_energy_15_min_history \
|
|
|
|
|
|
+ --hive-table $7 \
|
|
--hive-overwrite \
|
|
--hive-overwrite \
|
|
--null-string '\\N' \
|
|
--null-string '\\N' \
|
|
--null-non-string '\\N'
|
|
--null-non-string '\\N'
|
|
-}
|
|
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+
|
|
|
|
+ods_energy_15_min(){
|
|
|
|
+ import_no_partition $url $username $password /warehouse/saga_dw/ods/ods_energy_15_min "building, func_id, meter, data_time, data_value" "select building, func_id, meter, data_time, data_value from energy_15_min where 1 = 1" ods_energy_15_min
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+ods_energy_5_min(){
|
|
|
|
+ import_no_partition $url $username $password /warehouse/saga_dw/ods/ods_energy_5_min "building, func_id, meter, data_time, data_value" "select building, func_id, meter, data_time, data_value from energy_5_min where 1 = 1" ods_energy_5_min
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+ods_energy_15_min_fjd(){
|
|
|
|
+ import_no_partition $url $username $password /warehouse/saga_dw/ods/ods_energy_15_min_fjd "building, func_id, meter, data_time, data_value" "select building, func_id, meter, data_time, data_value from energy_15_min_fjd where 1 = 1" ods_energy_15_min_fjd
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+ods_energy_5_min_fjd(){
|
|
|
|
+ import_no_partition $url $username $password /warehouse/saga_dw/ods/ods_energy_5_min_fjd "building, func_id, meter, data_time, data_value" "select building, func_id, meter, data_time, data_value from energy_5_min_fjd where 1 = 1" ods_energy_5_min_fjd
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+case $1 in
|
|
|
|
+"energy_15_min")
|
|
|
|
+ ods_energy_15_min
|
|
|
|
+ ods_energy_15_min_fjd
|
|
|
|
+ ;;
|
|
|
|
+"energy_5_min")
|
|
|
|
+ ods_energy_5_min
|
|
|
|
+ ods_energy_5_min_fjd
|
|
|
|
+ ;;
|
|
|
|
+"ods_energy_15_min")
|
|
|
|
+ ods_energy_15_min
|
|
|
|
+ ;;
|
|
|
|
+"ods_energy_5_min")
|
|
|
|
+ ods_energy_5_min
|
|
|
|
+ ;;
|
|
|
|
+"ods_energy_15_min_fjd")
|
|
|
|
+ ods_energy_15_min_fjd
|
|
|
|
+ ;;
|
|
|
|
+"ods_energy_5_min_fjd")
|
|
|
|
+ ods_energy_5_min_fjd
|
|
|
|
+ ;;
|
|
|
|
+esac
|