wudianlong 2 rokov pred
rodič
commit
856d1ecb18

+ 3 - 2
example/config/config.sh

@@ -1,5 +1,6 @@
 #!/bin/bash
 
-MYSQL_URL='jdbc:mysql://192.168.0.76:3306/sagacloud_review?useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&tinyInt1isBit=false'
+MYSQL_HOST=hadoop01
+MYSQL_URL="jdbc:mysql://$MYSQL_HOST:3306/saga_dev?useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&tinyInt1isBit=false"
 MYSQL_USER=root
-MYSQL_PASSWORD=123456
+MYSQL_PASSWORD=j5ry0#jZ7vaUt5f4

+ 27 - 84
example/function/sqoop_func.sh

@@ -1,90 +1,31 @@
 #! /bin/bash
 
-if [ -n "$2" ] ;then do_date=$2
+. /opt/app/bi-app/example/config/config.sh
+
+if [ -n "$2" ] ;then
+    echo "如果是输入的日期按照取输入日期"
+    do_date=$2
 else
-do_date=`date -d '-1 day' +%F`
+    echo "====没有输入数据的日期,取当前时间的前一天===="
+    do_date=`date -d yesterday +"%Y-%m-%d"`
 fi
 
-echo $do_date
-
-
-## lzo 压缩
-bin/sqoop import  \
---connect "jdbc:mysql://192.168.0.76:3306/sagacloud_review?useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&tinyInt1isBit=false" \
---username root \
---password 123456 \
---target-dir /warehouse/saga_dw/ods/ods_energy_week_day  \
---delete-target-dir \
---query "select * from energy_week_day where 1 = 1 and  \$CONDITIONS" \
---num-mappers 1 \
---hive-drop-import-delims \
---fields-terminated-by '\001' \
---compress \
---compression-codec lzo \
---hive-import \
---hive-database saga_dw \
---hive-table ods_energy_week_day \
---hive-overwrite \
---null-string '\\N' \
---null-non-string '\\N'
-
-#按天写入分区
-mysql_to_hdfs_by_day() {
-sqoop import -D mapred.job.queue.name=root.tianyan  \
---connect "$1" \
---username $2 \
---password $3 \
---target-dir /warehouse/saga_dw/ods/$2/$do_date  \
---delete-target-dir \
---query "$5 and  \$CONDITIONS" \
---num-mappers 1 \
---hive-drop-import-delims \
---fields-terminated-by '\001' \
---compress \
---compression-codec snappy \
---hive-import \
---hive-database saga_dw \
---hive-table $4 \
---hive-overwrite \
---hive-partition-key dt \
---hive-partition-value $do_date \
---null-string '\\N' \
---null-non-string '\\N'
-}
-
-#写入没分区表
-collect_data_np() {
-sqoop import -D mapred.job.queue.name=root.tianyan  \
---connect "$1" \
---username $2 \
---password $3 \
---target-dir /warehouse/saga_dw/ods/$2  \
---delete-target-dir \
---query "$5 and  \$CONDITIONS" \
---num-mappers 1 \
---hive-drop-import-delims \
---fields-terminated-by '\001' \
---compress \
---compression-codec snappy \
---hive-import \
---hive-database dw \
---hive-table $4 \
---hive-overwrite \
---null-string '\\N' \
---null-non-string '\\N'
-}
+url=$MYSQL_URL
+username=$MYSQL_USER
+password=$MYSQL_PASSWORD
 
 
-## lzo压缩,mysql -> hdfs
-mysql_to_hdfs_lzo () {
-  bin/sqoop import  \
-  -D mapred.job.queue.name=root.hive  \
-  --connect "jdbc:mysql://192.168.0.76:3306/sagacloud_review?useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&tinyInt1isBit=false" \
-  --username root \
-  --password 123456 \
-  --target-dir /warehouse/saga_dw/ods/ods_energy_week_day  \
+## 无分区
+import_no_partition() {
+  sqoop import  \
+  -D mapred.job.queue.name=hive  \
+  --connect $1 \
+  --username $2 \
+  --password $3 \
+  --target-dir $4$7   \
   --delete-target-dir \
-  --query "select * from energy_week_day where 1 = 1 and  \$CONDITIONS" \
+  --columns "$5"  \
+  --query "$6  and  \$CONDITIONS" \
   --num-mappers 1 \
   --hive-drop-import-delims \
   --fields-terminated-by '\001' \
@@ -92,14 +33,16 @@ mysql_to_hdfs_lzo () {
   --compression-codec lzo \
   --hive-import \
   --hive-database saga_dw \
-  --hive-table ods_energy_week_day \
+  --hive-table $7 \
   --hive-overwrite \
   --null-string '\\N' \
   --null-non-string '\\N'
 }
 
-hdfs_to_mysql(){
-  sqoop export \
-  -D \
-  -connect
+dim_date(){
+  column_name="date_id, week_of_year, day_of_year, day_of_mouth, type, date_type, index_work_day_of_month, week_day, week_day_cn, week_day_en, current_year, current_quarter, year_quarter, mouth, year_mouth, is_first_day_mouth, is_last_day_mouth, type_des"
+  query_sql="select date_id, week_of_year, day_of_year, day_of_mouth, type, date_type, index_work_day_of_month, week_day, week_day_cn, week_day_en, current_year, current_quarter, year_quarter, mouth, year_mouth, is_first_day_mouth, is_last_day_mouth, type_des from dim_date where 1 = 1 "
+  import_no_partition $url $username $password /warehouse/saga_dw/dim/tmp/  $column_name $query_sql dim_date_day
 }
+
+dim_date

+ 0 - 0
example/hive_ddl/saga_dw_dim.sql