mysql_to_hdfs.sh 2.2 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970
  1. #!/bin/bash
  2. . ./config.sh
  3. url=$MYSQL_URL
  4. username=$MYSQL_USER
  5. password=$MYSQL_PASSWORD
  6. ## 无分区
  7. import_no_partition() {
  8. sqoop import \
  9. -D mapred.job.queue.name=hive \
  10. --connect $1 \
  11. --username $2 \
  12. --password $3 \
  13. --target-dir $4$7 \
  14. --delete-target-dir \
  15. --columns "$5" \
  16. --query "$6 and \$CONDITIONS" \
  17. --num-mappers 1 \
  18. --hive-drop-import-delims \
  19. --fields-terminated-by '\001' \
  20. --compress \
  21. --compression-codec lzo \
  22. --hive-import \
  23. --hive-database saga_dw \
  24. --hive-table $7 \
  25. --hive-overwrite \
  26. --null-string '\\N' \
  27. --null-non-string '\\N'
  28. }
  29. ods_energy_15_min_history(){
  30. import_no_partition $url $username $password /warehouse/saga_dw/ods/ods_tmp_data/ "building, func_id, meter, data_time, data_value, dt" "select building, func_id, meter, data_time, data_value, dt from energy_15_min_history where 1 = 1" ods_energy_15_min_history
  31. }
  32. ods_energy_5_min_history(){
  33. import_no_partition $url $username $password /warehouse/saga_dw/ods/ods_tmp_data/ "building, func_id, meter, data_time, data_value, dt" "select building, func_id, meter, data_time, data_value, dt from energy_5_min_history where 1 = 1" ods_energy_5_min_history
  34. }
  35. ods_energy_15_min_fjd_history(){
  36. import_no_partition $url $username $password /warehouse/saga_dw/ods/ods_tmp_data/ "building, func_id, meter, data_time, data_value, dt" "select building, func_id, meter, data_time, data_value, dt from energy_15_min_fjd_history where 1 = 1" ods_energy_15_min_fjd_history
  37. }
  38. ods_energy_5_min_fjd_history(){
  39. import_no_partition $url $username $password /warehouse/saga_dw/ods/ods_tmp_data/ "building, func_id, meter, data_time, data_value, dt" "select building, func_id, meter, data_time, data_value, dt from energy_5_min_fjd_history where 1 = 1" ods_energy_5_min_fjd_history
  40. }
  41. case $1 in
  42. "energy_15_min")
  43. ods_energy_15_min_history
  44. ods_energy_15_min_fjd_history
  45. ;;
  46. "energy_5_min")
  47. ods_energy_5_min_history
  48. ods_energy_5_min_fjd_history
  49. ;;
  50. "ods_energy_15_min_history")
  51. ods_energy_15_min_history
  52. ;;
  53. "ods_energy_5_min_history")
  54. ods_energy_5_min_history
  55. ;;
  56. "ods_energy_15_min_fjd_history")
  57. ods_energy_15_min_fjd_history
  58. ;;
  59. "ods_energy_5_min_fjd_history")
  60. ods_energy_5_min_fjd_history
  61. ;;
  62. esac