mysql_to_hdfs.sh 2.3 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889
  1. #!/bin/bash
  2. . ./config.sh
  3. if [ -n "$2" ] ;then
  4. echo "如果是输入的日期按照取输入日期"
  5. do_date=$2
  6. else
  7. echo "====没有输入数据的日期,取当前时间的前一天===="
  8. do_date=`date -d yesterday +"%Y-%m-%d"`
  9. fi
  10. url=$MYSQL_URL
  11. username=$MYSQL_USER
  12. password=$MYSQL_PASSWORD
  13. mysql_to_hdfs_lzo() {
  14. sqoop import \
  15. -D mapred.job.queue.name=hive \
  16. --connect $1 \
  17. --username $2 \
  18. --password $3 \
  19. --target-dir $4$7/$do_date \
  20. --delete-target-dir \
  21. --columns $5 \
  22. --query "$6 and \$CONDITIONS" \
  23. --num-mappers 1 \
  24. --hive-drop-import-delims \
  25. --fields-terminated-by '\001' \
  26. --compress \
  27. --compression-codec lzo \
  28. --hive-import \
  29. --hive-database saga_dw \
  30. --hive-table $7 \
  31. --hive-overwrite \
  32. --hive-partition-key dt \
  33. --hive-partition-value $do_date \
  34. --null-string '\\N' \
  35. --null-non-string '\\N'
  36. }
  37. mysql_to_hdfs(){
  38. echo $1
  39. echo $2
  40. echo $3
  41. echo $4$7/$do_date
  42. echo $5
  43. echo $6
  44. echo $7
  45. }
  46. ods_energy_15_min(){
  47. mysql_to_hdfs_lzo $url $username $password /warehouse/saga_dw/ods/tmp/ "building, func_id, meter, data_time, data_value" "select building, func_id, meter, data_time, data_value from energy_15_min where where dt = '$do_date'" ods_energy_15_min
  48. }
  49. ods_energy_5_min(){
  50. mysql_to_hdfs_lzo $url $username $password /warehouse/saga_dw/ods/tmp/ "building, func_id, meter, data_time, data_value" "select building, func_id, meter, data_time, data_value from energy_5_min where where dt = '$do_date'" ods_energy_5_min
  51. }
  52. ods_energy_15_min_fjd(){
  53. mysql_to_hdfs_lzo $url $username $password /warehouse/saga_dw/ods/tmp/ "building, func_id, meter, data_time, data_value" "select building, func_id, meter, data_time, data_value from energy_15_min_fjd where where dt = '$do_date'" ods_energy_15_min_fjd
  54. }
  55. ods_energy_5_min_fjd(){
  56. mysql_to_hdfs_lzo $url $username $password /warehouse/saga_dw/ods/tmp/ "building, func_id, meter, data_time, data_value" "select building, func_id, meter, data_time, data_value from energy_5_min_fjd where where dt = '$do_date'" ods_energy_5_min_fjd
  57. }
  58. case $1 in
  59. "energy_15_min")
  60. ods_energy_15_min
  61. ods_energy_15_min_fjd
  62. ;;
  63. "energy_5_min")
  64. ods_energy_5_min
  65. ods_energy_5_min_fjd
  66. ;;
  67. "ods_energy_15_min")
  68. ods_energy_15_min
  69. ;;
  70. "ods_energy_5_min")
  71. ods_energy_5_min
  72. ;;
  73. "ods_energy_15_min_fjd")
  74. ods_energy_15_min_fjd
  75. ;;
  76. "ods_energy_5_min_fjd")
  77. ods_energy_5_min_fjd
  78. ;;
  79. esac