|
@@ -1,90 +1,31 @@
|
|
|
#! /bin/bash
|
|
|
|
|
|
-if [ -n "$2" ] ;then do_date=$2
|
|
|
+. /opt/app/bi-app/example/config/config.sh
|
|
|
+
|
|
|
+if [ -n "$2" ] ;then
|
|
|
+ echo "如果是输入的日期按照取输入日期"
|
|
|
+ do_date=$2
|
|
|
else
|
|
|
-do_date=`date -d '-1 day' +%F`
|
|
|
+ echo "====没有输入数据的日期,取当前时间的前一天===="
|
|
|
+ do_date=`date -d yesterday +"%Y-%m-%d"`
|
|
|
fi
|
|
|
|
|
|
-echo $do_date
|
|
|
-
|
|
|
-
|
|
|
-## lzo 压缩
|
|
|
-bin/sqoop import \
|
|
|
---connect "jdbc:mysql://192.168.0.76:3306/sagacloud_review?useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&tinyInt1isBit=false" \
|
|
|
---username root \
|
|
|
---password 123456 \
|
|
|
---target-dir /warehouse/saga_dw/ods/ods_energy_week_day \
|
|
|
---delete-target-dir \
|
|
|
---query "select * from energy_week_day where 1 = 1 and \$CONDITIONS" \
|
|
|
---num-mappers 1 \
|
|
|
---hive-drop-import-delims \
|
|
|
---fields-terminated-by '\001' \
|
|
|
---compress \
|
|
|
---compression-codec lzo \
|
|
|
---hive-import \
|
|
|
---hive-database saga_dw \
|
|
|
---hive-table ods_energy_week_day \
|
|
|
---hive-overwrite \
|
|
|
---null-string '\\N' \
|
|
|
---null-non-string '\\N'
|
|
|
-
|
|
|
-#按天写入分区
|
|
|
-mysql_to_hdfs_by_day() {
|
|
|
-sqoop import -D mapred.job.queue.name=root.tianyan \
|
|
|
---connect "$1" \
|
|
|
---username $2 \
|
|
|
---password $3 \
|
|
|
---target-dir /warehouse/saga_dw/ods/$2/$do_date \
|
|
|
---delete-target-dir \
|
|
|
---query "$5 and \$CONDITIONS" \
|
|
|
---num-mappers 1 \
|
|
|
---hive-drop-import-delims \
|
|
|
---fields-terminated-by '\001' \
|
|
|
---compress \
|
|
|
---compression-codec snappy \
|
|
|
---hive-import \
|
|
|
---hive-database saga_dw \
|
|
|
---hive-table $4 \
|
|
|
---hive-overwrite \
|
|
|
---hive-partition-key dt \
|
|
|
---hive-partition-value $do_date \
|
|
|
---null-string '\\N' \
|
|
|
---null-non-string '\\N'
|
|
|
-}
|
|
|
-
|
|
|
-#写入没分区表
|
|
|
-collect_data_np() {
|
|
|
-sqoop import -D mapred.job.queue.name=root.tianyan \
|
|
|
---connect "$1" \
|
|
|
---username $2 \
|
|
|
---password $3 \
|
|
|
---target-dir /warehouse/saga_dw/ods/$2 \
|
|
|
---delete-target-dir \
|
|
|
---query "$5 and \$CONDITIONS" \
|
|
|
---num-mappers 1 \
|
|
|
---hive-drop-import-delims \
|
|
|
---fields-terminated-by '\001' \
|
|
|
---compress \
|
|
|
---compression-codec snappy \
|
|
|
---hive-import \
|
|
|
---hive-database dw \
|
|
|
---hive-table $4 \
|
|
|
---hive-overwrite \
|
|
|
---null-string '\\N' \
|
|
|
---null-non-string '\\N'
|
|
|
-}
|
|
|
+url=$MYSQL_URL
|
|
|
+username=$MYSQL_USER
|
|
|
+password=$MYSQL_PASSWORD
|
|
|
|
|
|
|
|
|
-## lzo压缩,mysql -> hdfs
|
|
|
-mysql_to_hdfs_lzo () {
|
|
|
- bin/sqoop import \
|
|
|
- -D mapred.job.queue.name=root.hive \
|
|
|
- --connect "jdbc:mysql://192.168.0.76:3306/sagacloud_review?useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&tinyInt1isBit=false" \
|
|
|
- --username root \
|
|
|
- --password 123456 \
|
|
|
- --target-dir /warehouse/saga_dw/ods/ods_energy_week_day \
|
|
|
+## 无分区
|
|
|
+import_no_partition() {
|
|
|
+ sqoop import \
|
|
|
+ -D mapred.job.queue.name=hive \
|
|
|
+ --connect $1 \
|
|
|
+ --username $2 \
|
|
|
+ --password $3 \
|
|
|
+ --target-dir $4$7 \
|
|
|
--delete-target-dir \
|
|
|
- --query "select * from energy_week_day where 1 = 1 and \$CONDITIONS" \
|
|
|
+ --columns "$5" \
|
|
|
+ --query "$6 and \$CONDITIONS" \
|
|
|
--num-mappers 1 \
|
|
|
--hive-drop-import-delims \
|
|
|
--fields-terminated-by '\001' \
|
|
@@ -92,14 +33,16 @@ mysql_to_hdfs_lzo () {
|
|
|
--compression-codec lzo \
|
|
|
--hive-import \
|
|
|
--hive-database saga_dw \
|
|
|
- --hive-table ods_energy_week_day \
|
|
|
+ --hive-table $7 \
|
|
|
--hive-overwrite \
|
|
|
--null-string '\\N' \
|
|
|
--null-non-string '\\N'
|
|
|
}
|
|
|
|
|
|
-hdfs_to_mysql(){
|
|
|
- sqoop export \
|
|
|
- -D \
|
|
|
- -connect
|
|
|
+dim_date(){
|
|
|
+ column_name="date_id, week_of_year, day_of_year, day_of_mouth, type, date_type, index_work_day_of_month, week_day, week_day_cn, week_day_en, current_year, current_quarter, year_quarter, mouth, year_mouth, is_first_day_mouth, is_last_day_mouth, type_des"
|
|
|
+ query_sql="select date_id, week_of_year, day_of_year, day_of_mouth, type, date_type, index_work_day_of_month, week_day, week_day_cn, week_day_en, current_year, current_quarter, year_quarter, mouth, year_mouth, is_first_day_mouth, is_last_day_mouth, type_des from dim_date where 1 = 1 "
|
|
|
+ import_no_partition $url $username $password /warehouse/saga_dw/dim/tmp/ $column_name $query_sql dim_date_day
|
|
|
}
|
|
|
+
|
|
|
+dim_date
|