wudianlong 2 éve
szülő
commit
9eaf15d23a

+ 5 - 5
energy/dayTask/clean_mysql.sh

@@ -2,7 +2,7 @@
 
 . ./config.sh
 
-url=$MYSQL_URL
+host=$MYSQL_HOST
 username=$MYSQL_USER
 password=$MYSQL_PASSWORD
 database=saga_dw
@@ -14,19 +14,19 @@ del_energy_5_min_fjd="truncate table energy_5_min_fjd;"
 
 ## 历史数据一次性导入后,需要删除 mysql 中的冗余数据
 clean_energy_15(){
-mysql -h${url}  -P3306  -u${username} -p${password} ${database} -e "${del_energy_15_min}"
+mysql -h${host}  -P3306  -u${username} -p${password} ${database} -e "${del_energy_15_min}"
 }
 
 clean_energy_5(){
-mysql -h${url}  -P3306  -u${username} -p${password} ${database} -e "${del_energy_15_min_fjd}"
+mysql -h${host}  -P3306  -u${username} -p${password} ${database} -e "${del_energy_15_min_fjd}"
 }
 
 clean_energy_15_fjd(){
-mysql -h${url}  -P3306  -u${username} -p${password} ${database} -e "${del_energy_15_min}"
+mysql -h${host}  -P3306  -u${username} -p${password} ${database} -e "${del_energy_5_min}"
 }
 
 clean_energy_5_fjd(){
-mysql -h${url}  -P3306  -u${username} -p${password} ${database} -e "${del_energy_15_min_fjd}"
+mysql -h${host}  -P3306  -u${username} -p${password} ${database} -e "${del_energy_5_min_fjd}"
 }
 
 case $1 in

+ 3 - 2
energy/dayTask/config.sh

@@ -1,5 +1,6 @@
 #!/bin/bash
 
-MYSQL_URL="jdbc:mysql://hadoop01:3306/saga_dev?useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&tinyInt1isBit=false"
+MYSQL_HOST=hadoop01
+MYSQL_URL="jdbc:mysql://$MYSQL_HOST:3306/saga_dev?useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&tinyInt1isBit=false"
 MYSQL_USER=root
-MYSQL_PASSWORD=j5ry0#jZ7vaUt5f4
+MYSQL_PASSWORD=j5ry0#jZ7vaUt5f4

+ 41 - 0
energy/dayTask/hdfs_to_mysql.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+
+if [ -n "$2" ] ;then
+    echo "如果是输入的日期按照取输入日期"
+    do_date=$2
+else
+    echo "====没有输入数据的日期,取当前时间的前一天===="
+    do_date=`date -d yesterday +"%Y-%m-%d"`
+fi
+
+url=$MYSQL_URL
+username=$MYSQL_USER
+password=$MYSQL_PASSWORD
+
+hdfs_to_mysql() {
+  sqoop export -D mapred.job.queue.name=hive \
+    --connect $1 \
+    --username $2 \
+    --password $3 \
+    --table $4 \
+    -m 1 \
+    --hcatalog-database saga_dw \
+    --hcatalog-table $5 \
+    --hcatalog-partition-keys dt \
+    --hcatalog-partition-values $do_date \
+    --fields-terminated-by '\001' \
+    --input-null-string '\\N' \
+    --input-null-non-string '\\N'
+}
+
+
+app_energy(){
+  mysql_to_hdfs_lzo $url $username $password mysql_tableName hive_tableName
+}
+
+case $1 in
+"table")
+  app_energy
+  ;;
+esac
+

+ 1 - 1
energy/history/mysql_to_hdfs.sh

@@ -15,7 +15,7 @@ import_no_partition() {
   --password $3 \
   --target-dir $4   \
   --delete-target-dir \
-  --columns $5  \
+  --columns "$5"  \
   --query "$6  and  \$CONDITIONS" \
   --num-mappers 1 \
   --hive-drop-import-delims \