瀏覽代碼

修改sqoop脚本,支持分区

wudianlong 2 年之前
父節點
當前提交
09c5772865
共有 1 個文件被更改,包括 26 次插入25 次删除
  1. 26 25
      energy/dayTask/mysql_to_hdfs.sh

+ 26 - 25
energy/dayTask/mysql_to_hdfs.sh

@@ -16,43 +16,44 @@ username=$MYSQL_USER
 password=$MYSQL_PASSWORD
 
 mysql_to_hdfs_lzo() {
-  sqoop import  \
-  -D mapred.job.queue.name=hive  \
-  --connect $1 \
-  --username $2 \
-  --password $3 \
-  --target-dir $4/dt=$do_date  \
-  --delete-target-dir \
-  --columns $5 \
-  --query "$6 and  \$CONDITIONS" \
-  --num-mappers 1 \
-  --hive-drop-import-delims \
-  --fields-terminated-by '\001' \
-  --compress \
-  --compression-codec lzo \
-  --hive-import \
-  --hive-database saga_dw \
-  --hive-table $7 \
-  --hive-overwrite \
-  --null-string '\\N' \
-  --null-non-string '\\N'
+	sqoop import  \
+	-D mapred.job.queue.name=hive  \
+	--connect $1 \
+	--username $2 \
+	--password $3 \
+	--target-dir /warehouse/saga_dw/ods/tmp/$7/$do_date \
+	--delete-target-dir \
+	--columns $5 \
+	--query "select building, func_id, meter, data_time, data_value from energy_15_min where dt = '$do_date' and \$CONDITIONS" \
+	--num-mappers 1 \
+	--hive-drop-import-delims \
+	--fields-terminated-by '\001' \
+	--compress \
+	--compression-codec lzo \
+	--hive-import \
+	--hive-database saga_dw \
+	--hive-table $7 \
+	--hive-overwrite \
+	--hive-partition-key dt \
+	--hive-partition-value $do_date \
+	--null-string '\\N' \
+	--null-non-string '\\N'
 }
 
-
 ods_energy_15_min(){
-  import_no_partition $url $username $password /warehouse/saga_dw/ods/ods_energy_15_min "building, func_id, meter, data_time, data_value" "select building, func_id, meter, data_time, data_value from energy_15_min where 1 = 1" ods_energy_15_min
+  import_no_partition $url $username $password /warehouse/saga_dw/ods/ods_energy_15_min "building, func_id, meter, data_time, data_value" "select building, func_id, meter, data_time, data_value from energy_15_min where where dt = '$do_date'" ods_energy_15_min
 }
 
 ods_energy_5_min(){
-  import_no_partition $url $username $password /warehouse/saga_dw/ods/ods_energy_5_min "building, func_id, meter, data_time, data_value" "select building, func_id, meter, data_time, data_value from energy_5_min where 1 = 1" ods_energy_5_min
+  import_no_partition $url $username $password /warehouse/saga_dw/ods/ods_energy_5_min "building, func_id, meter, data_time, data_value" "select building, func_id, meter, data_time, data_value from energy_5_min where where dt = '$do_date'" ods_energy_5_min
 }
 
 ods_energy_15_min_fjd(){
-  import_no_partition $url $username $password /warehouse/saga_dw/ods/ods_energy_15_min_fjd "building, func_id, meter, data_time, data_value" "select building, func_id, meter, data_time, data_value from energy_15_min_fjd where 1 = 1" ods_energy_15_min_fjd
+  import_no_partition $url $username $password /warehouse/saga_dw/ods/ods_energy_15_min_fjd "building, func_id, meter, data_time, data_value" "select building, func_id, meter, data_time, data_value from energy_15_min_fjd where where dt = '$do_date'" ods_energy_15_min_fjd
 }
 
 ods_energy_5_min_fjd(){
-  import_no_partition $url $username $password /warehouse/saga_dw/ods/ods_energy_5_min_fjd "building, func_id, meter, data_time, data_value" "select building, func_id, meter, data_time, data_value from energy_5_min_fjd where 1 = 1" ods_energy_5_min_fjd
+  import_no_partition $url $username $password /warehouse/saga_dw/ods/ods_energy_5_min_fjd "building, func_id, meter, data_time, data_value" "select building, func_id, meter, data_time, data_value from energy_5_min_fjd where where dt = '$do_date'" ods_energy_5_min_fjd
 }
 
 case $1 in