|
@@ -16,43 +16,44 @@ username=$MYSQL_USER
|
|
|
password=$MYSQL_PASSWORD
|
|
|
|
|
|
mysql_to_hdfs_lzo() {
|
|
|
- sqoop import \
|
|
|
- -D mapred.job.queue.name=hive \
|
|
|
- --connect $1 \
|
|
|
- --username $2 \
|
|
|
- --password $3 \
|
|
|
- --target-dir $4/dt=$do_date \
|
|
|
- --delete-target-dir \
|
|
|
- --columns $5 \
|
|
|
- --query "$6 and \$CONDITIONS" \
|
|
|
- --num-mappers 1 \
|
|
|
- --hive-drop-import-delims \
|
|
|
- --fields-terminated-by '\001' \
|
|
|
- --compress \
|
|
|
- --compression-codec lzo \
|
|
|
- --hive-import \
|
|
|
- --hive-database saga_dw \
|
|
|
- --hive-table $7 \
|
|
|
- --hive-overwrite \
|
|
|
- --null-string '\\N' \
|
|
|
- --null-non-string '\\N'
|
|
|
+ sqoop import \
|
|
|
+ -D mapred.job.queue.name=hive \
|
|
|
+ --connect $1 \
|
|
|
+ --username $2 \
|
|
|
+ --password $3 \
|
|
|
+ --target-dir /warehouse/saga_dw/ods/tmp/$7/$do_date \
|
|
|
+ --delete-target-dir \
|
|
|
+ --columns $5 \
|
|
|
+ --query "select building, func_id, meter, data_time, data_value from energy_15_min where dt = '$do_date' and \$CONDITIONS" \
|
|
|
+ --num-mappers 1 \
|
|
|
+ --hive-drop-import-delims \
|
|
|
+ --fields-terminated-by '\001' \
|
|
|
+ --compress \
|
|
|
+ --compression-codec lzo \
|
|
|
+ --hive-import \
|
|
|
+ --hive-database saga_dw \
|
|
|
+ --hive-table $7 \
|
|
|
+ --hive-overwrite \
|
|
|
+ --hive-partition-key dt \
|
|
|
+ --hive-partition-value $do_date \
|
|
|
+ --null-string '\\N' \
|
|
|
+ --null-non-string '\\N'
|
|
|
}
|
|
|
|
|
|
-
|
|
|
ods_energy_15_min(){
|
|
|
- import_no_partition $url $username $password /warehouse/saga_dw/ods/ods_energy_15_min "building, func_id, meter, data_time, data_value" "select building, func_id, meter, data_time, data_value from energy_15_min where 1 = 1" ods_energy_15_min
|
|
|
+ import_no_partition $url $username $password /warehouse/saga_dw/ods/ods_energy_15_min "building, func_id, meter, data_time, data_value" "select building, func_id, meter, data_time, data_value from energy_15_min where where dt = '$do_date'" ods_energy_15_min
|
|
|
}
|
|
|
|
|
|
ods_energy_5_min(){
|
|
|
- import_no_partition $url $username $password /warehouse/saga_dw/ods/ods_energy_5_min "building, func_id, meter, data_time, data_value" "select building, func_id, meter, data_time, data_value from energy_5_min where 1 = 1" ods_energy_5_min
|
|
|
+ import_no_partition $url $username $password /warehouse/saga_dw/ods/ods_energy_5_min "building, func_id, meter, data_time, data_value" "select building, func_id, meter, data_time, data_value from energy_5_min where where dt = '$do_date'" ods_energy_5_min
|
|
|
}
|
|
|
|
|
|
ods_energy_15_min_fjd(){
|
|
|
- import_no_partition $url $username $password /warehouse/saga_dw/ods/ods_energy_15_min_fjd "building, func_id, meter, data_time, data_value" "select building, func_id, meter, data_time, data_value from energy_15_min_fjd where 1 = 1" ods_energy_15_min_fjd
|
|
|
+ import_no_partition $url $username $password /warehouse/saga_dw/ods/ods_energy_15_min_fjd "building, func_id, meter, data_time, data_value" "select building, func_id, meter, data_time, data_value from energy_15_min_fjd where where dt = '$do_date'" ods_energy_15_min_fjd
|
|
|
}
|
|
|
|
|
|
ods_energy_5_min_fjd(){
|
|
|
- import_no_partition $url $username $password /warehouse/saga_dw/ods/ods_energy_5_min_fjd "building, func_id, meter, data_time, data_value" "select building, func_id, meter, data_time, data_value from energy_5_min_fjd where 1 = 1" ods_energy_5_min_fjd
|
|
|
+ import_no_partition $url $username $password /warehouse/saga_dw/ods/ods_energy_5_min_fjd "building, func_id, meter, data_time, data_value" "select building, func_id, meter, data_time, data_value from energy_5_min_fjd where where dt = '$do_date'" ods_energy_5_min_fjd
|
|
|
}
|
|
|
|
|
|
case $1 in
|