#! /bin/bash if [ -n "$2" ] ;then do_date=$2 else do_date=`date -d '-1 day' +%F` fi echo $do_date ## lzo 压缩 bin/sqoop import \ --connect "jdbc:mysql://192.168.0.76:3306/sagacloud_review?useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&tinyInt1isBit=false" \ --username root \ --password 123456 \ --target-dir /warehouse/saga_dw/ods/ods_energy_week_day \ --delete-target-dir \ --query "select * from energy_week_day where 1 = 1 and \$CONDITIONS" \ --num-mappers 1 \ --hive-drop-import-delims \ --fields-terminated-by '\001' \ --compress \ --compression-codec lzo \ --hive-import \ --hive-database saga_dw \ --hive-table ods_energy_week_day \ --hive-overwrite \ --null-string '\\N' \ --null-non-string '\\N' #按天写入分区 mysql_to_hdfs_by_day() { sqoop import -D mapred.job.queue.name=root.tianyan \ --connect "$1" \ --username $2 \ --password $3 \ --target-dir /warehouse/saga_dw/ods/$2/$do_date \ --delete-target-dir \ --query "$5 and \$CONDITIONS" \ --num-mappers 1 \ --hive-drop-import-delims \ --fields-terminated-by '\001' \ --compress \ --compression-codec snappy \ --hive-import \ --hive-database saga_dw \ --hive-table $4 \ --hive-overwrite \ --hive-partition-key dt \ --hive-partition-value $do_date \ --null-string '\\N' \ --null-non-string '\\N' } #写入没分区表 collect_data_np() { sqoop import -D mapred.job.queue.name=root.tianyan \ --connect "$1" \ --username $2 \ --password $3 \ --target-dir /warehouse/saga_dw/ods/$2 \ --delete-target-dir \ --query "$5 and \$CONDITIONS" \ --num-mappers 1 \ --hive-drop-import-delims \ --fields-terminated-by '\001' \ --compress \ --compression-codec snappy \ --hive-import \ --hive-database dw \ --hive-table $4 \ --hive-overwrite \ --null-string '\\N' \ --null-non-string '\\N' } ## lzo压缩,mysql -> hdfs mysql_to_hdfs_lzo () { bin/sqoop import \ -D mapred.job.queue.name=root.hive \ --connect "jdbc:mysql://192.168.0.76:3306/sagacloud_review?useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&tinyInt1isBit=false" \ --username root \ --password 123456 \ --target-dir /warehouse/saga_dw/ods/ods_energy_week_day \ --delete-target-dir \ --query "select * from energy_week_day where 1 = 1 and \$CONDITIONS" \ --num-mappers 1 \ --hive-drop-import-delims \ --fields-terminated-by '\001' \ --compress \ --compression-codec lzo \ --hive-import \ --hive-database saga_dw \ --hive-table ods_energy_week_day \ --hive-overwrite \ --null-string '\\N' \ --null-non-string '\\N' } hdfs_to_mysql(){ sqoop export \ -D \ -connect }