mysql_to_hdfs.sh 3.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107
  1. #!/bin/bash
  2. . /opt/app/bi-app/energy/dayTask/config.sh
  3. if [ -n "$2" ] ;then
  4. echo "如果是输入的日期按照取输入日期"
  5. do_date=$2
  6. else
  7. echo "====没有输入数据的日期,取当前时间的前一天===="
  8. do_date=`date -d yesterday +"%Y-%m-%d"`
  9. fi
  10. url=$MYSQL_URL
  11. username=$MYSQL_USER
  12. password=$MYSQL_PASSWORD
  13. mysql_to_hdfs_lzo() {
  14. sqoop import \
  15. -D mapred.job.queue.name=default \
  16. --connect $1 \
  17. --username $2 \
  18. --password $3 \
  19. --target-dir $4$7/$do_date \
  20. --delete-target-dir \
  21. --columns "$5" \
  22. --query "$6 and \$CONDITIONS" \
  23. --num-mappers 1 \
  24. --hive-drop-import-delims \
  25. --fields-terminated-by '\001' \
  26. --compress \
  27. --compression-codec lzo \
  28. --hive-import \
  29. --hive-database saga_dw \
  30. --hive-table $7 \
  31. --hive-overwrite \
  32. --hive-partition-key dt \
  33. --hive-partition-value $do_date \
  34. --null-string '\\N' \
  35. --null-non-string '\\N'
  36. }
  37. ## 能源 15 分钟差值
  38. ods_energy_15_min(){
  39. mysql_to_hdfs_lzo $url $username $password /warehouse/saga_dw/ods/tmp/ "building, func_id, meter, data_time, data_value" "select building, func_id, meter, data_time, data_value from energy_15_min where 1 = 1 " ods_energy_15_min
  40. }
  41. ## 能源 15 分钟分精度
  42. ods_energy_15_min_fjd(){
  43. mysql_to_hdfs_lzo $url $username $password /warehouse/saga_dw/ods/tmp/ "building, func_id, meter, data_time, data_value" "select building, func_id, meter, data_time, data_value from energy_15_min_fjd where dt = '$do_date'" ods_energy_15_min_fjd
  44. }
  45. ## CO2 15 分钟分精度
  46. ods_co2_15_min(){
  47. mysql_to_hdfs_lzo $url $username $password /warehouse/saga_dw/ods/tmp/ "building, func_id, meter, data_time, data_value" "select building, func_id, meter, data_time, data_value from co2_15_min where dt = '$do_date'" ods_co2_15_min
  48. }
  49. ## PM2.5 15 分钟分精度
  50. ods_pm25_15_min(){
  51. mysql_to_hdfs_lzo $url $username $password /warehouse/saga_dw/ods/tmp/ "building, func_id, meter, data_time, data_value" "select building, func_id, meter, data_time, data_value from pm25_15_min where dt = '$do_date'" ods_pm25_15_min
  52. }
  53. ## 甲醛 15 分钟分精度
  54. ods_hcho_15_min(){
  55. mysql_to_hdfs_lzo $url $username $password /warehouse/saga_dw/ods/tmp/ "building, func_id, meter, data_time, data_value" "select building, func_id, meter, data_time, data_value from hcho_15_min where dt = '$do_date'" ods_hcho_15_min
  56. }
  57. ## 温度 15 分钟分精度
  58. ods_temperature_15_min(){
  59. mysql_to_hdfs_lzo $url $username $password /warehouse/saga_dw/ods/tmp/ "building, func_id, meter, data_time, data_value" "select building, func_id, meter, data_time, data_value from temperature_15_min where dt = '$do_date'" ods_temperature_15_min
  60. }
  61. ## 湿度 15 分钟分精度
  62. ods_humidity_15_min(){
  63. mysql_to_hdfs_lzo $url $username $password /warehouse/saga_dw/ods/tmp/ "building, func_id, meter, data_time, data_value" "select building, func_id, meter, data_time, data_value from humidity_15_min where dt = '$do_date'" ods_humidity_15_min
  64. }
  65. case $1 in
  66. "all")
  67. ods_energy_15_min
  68. ods_co2_15_min
  69. ods_pm25_15_min
  70. ods_hcho_15_min
  71. ods_temperature_15_min
  72. ods_humidity_15_min
  73. ;;
  74. "ods_energy_15_min")
  75. ods_energy_15_min
  76. ;;
  77. "ods_energy_15_min_fjd")
  78. ods_energy_15_min_fjd
  79. ;;
  80. "ods_co2_15_min")
  81. ods_co2_15_min
  82. ;;
  83. "ods_pm25_15_min")
  84. ods_pm25_15_min
  85. ;;
  86. "ods_hcho_15_min")
  87. ods_hcho_15_min
  88. ;;
  89. "ods_temperature_15_min")
  90. ods_temperature_15_min
  91. ;;
  92. "ods_humidity_15_min")
  93. ods_humidity_15_min
  94. ;;
  95. esac