hdfs_to_mysql.sh 1.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263
  1. #!/bin/bash
  2. . /opt/app/bi-app/energy/dayTask/config.sh
  3. if [ -n "$2" ] ;then
  4. echo "如果是输入的日期按照取输入日期"
  5. do_date=$2
  6. else
  7. echo "====没有输入数据的日期,取当前时间的前一天===="
  8. do_date=`date -d yesterday +"%Y-%m-%d"`
  9. fi
  10. url=$MYSQL_URL
  11. username=$MYSQL_USER
  12. password=$MYSQL_PASSWORD
  13. hdfs_to_mysql() {
  14. sqoop export -D mapred.job.queue.name=hive \
  15. --connect $1 \
  16. --username $2 \
  17. --password $3 \
  18. --table $4 \
  19. -m 1 \
  20. --hcatalog-database saga_dw \
  21. --hcatalog-table $5 \
  22. --fields-terminated-by '\001' \
  23. --input-null-string '\\N' \
  24. --input-null-non-string '\\N'
  25. }
  26. hdfs_to_mysql_by_day() {
  27. sqoop export -D mapred.job.queue.name=hive \
  28. --connect $1 \
  29. --username $2 \
  30. --password $3 \
  31. --table $4 \
  32. -m 1 \
  33. --hcatalog-database saga_dw \
  34. --hcatalog-table $5 \
  35. --hcatalog-partition-keys dt \
  36. --hcatalog-partition-values $do_date \
  37. --fields-terminated-by '\001' \
  38. --input-null-string '\\N' \
  39. --input-null-non-string '\\N'
  40. }
  41. app_energy(){
  42. hdfs_to_mysql $url $username $password "app_energy_15_min" "dwd_energy_15_min_hour"
  43. }
  44. app_energy_day(){
  45. hdfs_to_mysql_by_day $url $username $password "app_energy_15_min" "dwd_energy_15_min_hour"
  46. }
  47. case $1 in
  48. "app_energy")
  49. app_energy
  50. ;;
  51. "app_energy_day")
  52. app_energy_day
  53. ;;
  54. esac