sqoop_func.sh 1.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748
  1. #! /bin/bash
  2. . /opt/app/bi-app/example/config/config.sh
  3. if [ -n "$2" ] ;then
  4. echo "如果是输入的日期按照取输入日期"
  5. do_date=$2
  6. else
  7. echo "====没有输入数据的日期,取当前时间的前一天===="
  8. do_date=`date -d yesterday +"%Y-%m-%d"`
  9. fi
  10. url=$MYSQL_URL
  11. username=$MYSQL_USER
  12. password=$MYSQL_PASSWORD
  13. ## 无分区
  14. import_no_partition() {
  15. sqoop import \
  16. -D mapred.job.queue.name=hive \
  17. --connect $1 \
  18. --username $2 \
  19. --password $3 \
  20. --target-dir $4$7 \
  21. --delete-target-dir \
  22. --columns "$5" \
  23. --query "$6 and \$CONDITIONS" \
  24. --num-mappers 1 \
  25. --hive-drop-import-delims \
  26. --fields-terminated-by '\001' \
  27. --compress \
  28. --compression-codec lzo \
  29. --hive-import \
  30. --hive-database saga_dw \
  31. --hive-table $7 \
  32. --hive-overwrite \
  33. --null-string '\\N' \
  34. --null-non-string '\\N'
  35. }
  36. dim_date(){
  37. column_name="date_id, week_of_year, day_of_year, day_of_mouth, type, date_type, index_work_day_of_month, week_day, week_day_cn, week_day_en, current_year, current_quarter, year_quarter, mouth, year_mouth, is_first_day_mouth, is_last_day_mouth, type_des"
  38. query_sql="select date_id, week_of_year, day_of_year, day_of_mouth, type, date_type, index_work_day_of_month, week_day, week_day_cn, week_day_en, current_year, current_quarter, year_quarter, mouth, year_mouth, is_first_day_mouth, is_last_day_mouth, type_des from dim_date where 1 = 1 "
  39. import_no_partition $url $username $password /warehouse/saga_dw/dim/tmp/ "$column_name" "$query_sql" dim_date_day
  40. }
  41. dim_date