bus_mysql_to_hdfs.sh 1.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960
  1. #!/bin/bash
  2. . /opt/app/bi-app/energy/dayTask/config.sh
  3. . ./config.sh
  4. if [ -n "$2" ] ;then
  5. echo "如果是输入的日期按照取输入日期"
  6. do_date=$2
  7. else
  8. echo "====没有输入数据的日期,取当前时间的前一天===="
  9. do_date=$(date -d yesterday +"%Y-%m-%d")
  10. fi
  11. SAGA_DATABASE=sagacloud_customization
  12. url=$SAGA_URL_PRE$SAGA_DATABASE$SAGA_URL_SUF
  13. username=$SAGA_USER
  14. password=$SAGA_PASSWORD
  15. echo "$do_date"
  16. mysql_to_hdfs_lzo() {
  17. sqoop import \
  18. -D mapred.job.queue.name=default \
  19. --connect "$1" \
  20. --username "$2" \
  21. --password "$3" \
  22. --target-dir "$4""$7"/"$do_date" \
  23. --delete-target-dir \
  24. --columns "$5" \
  25. --query "$6 and \$CONDITIONS" \
  26. --num-mappers 1 \
  27. --hive-drop-import-delims \
  28. --fields-terminated-by '\001' \
  29. --compress \
  30. --compression-codec lzo \
  31. --hive-import \
  32. --hive-database saga_dw \
  33. --hive-table "$7" \
  34. --hive-overwrite \
  35. --hive-partition-key dt \
  36. --hive-partition-value "$do_date" \
  37. --null-string '\\N' \
  38. --null-non-string '\\N'
  39. }
  40. ## 空调操作记录表
  41. ods_feedback(){
  42. mysql_to_hdfs_lzo "$url" "$username" "$password" /warehouse/saga_dw/ods/tmp/ \
  43. "id, project_id, object_id, source_type, user_id, user_phone, user_name, value_type, item_id, value, create_time, next_open_time, model, duration_type, custom_plan, curr_temp, nick_name, result, exe_result, fb_temp, remark" \
  44. "select id, project_id, object_id, source_type, user_id, user_phone, user_name, value_type, item_id, value, create_time, next_open_time, model, duration_type, custom_plan, curr_temp, nick_name, result, exe_result, fb_temp, remark from feedback where date_format(create_time, '%Y-%m-%d') = $do_date and 1 = 1 " \
  45. ods_bus_feedback
  46. }
  47. case $1 in
  48. "all")
  49. ods_feedback
  50. ;;
  51. "ods_feedback")
  52. ods_feedback
  53. ;;
  54. esac