Browse Source

每月1日执行

李莎 1 year ago
commit
814fa61a1f
5 changed files with 338 additions and 0 deletions
  1. 9 0
      Dockerfile
  2. 154 0
      MyUtils/DateUtils.py
  3. 115 0
      MyUtils/MysqlUtils.py
  4. 55 0
      main.py
  5. 5 0
      requirements.txt

+ 9 - 0
Dockerfile

@@ -0,0 +1,9 @@
+FROM python:3.7.15-slim
+
+
+WORKDIR ./saga_equipment_quarter_history
+ADD . .
+
+RUN  pip install -r requirements.txt
+
+CMD ["python", "-u","./main.py"]

+ 154 - 0
MyUtils/DateUtils.py

@@ -0,0 +1,154 @@
+import datetime
+from dateutil.relativedelta import relativedelta
+from dateutil import rrule
+
+YYmmddHHMMSS = "%Y%m%d%H%M%S"
+YYmmdd = "%Y%m%d"
+YYmm = "%Y%m"
+# 根据开始月份结束月份获取所有月份
+def get_each_month(start_month, end_month):
+    if str(start_month).count('-') != 1 or str(end_month).count('-') != 1:
+        print("Parameter Error: Pls input a string such as '2019-01'")
+        return []
+    if int(str(start_month).split('-')[1]) > 12 or int(str(end_month).split('-')[1]) > 12:
+        print('Parameter Error: Pls input correct month range such as between 1 to 12')
+        return []
+    if int(str(start_month).split('-')[1]) == 0 or int(str(end_month).split('-')[1]) == 13:
+        print('Parameter Error: Pls input correct month range such as between 1 to 12')
+        return []
+    start = datetime.datetime.strptime(start_month, "%Y-%m")
+    end = datetime.datetime.strptime(end_month, "%Y-%m")
+    month_count = rrule.rrule(rrule.MONTHLY, dtstart=start, until=end).count()  # 计算总月份数
+    if end < start:
+        print("Parameter Error: Pls input right date range,start_month can't latter than end_month")
+        return []
+    else:
+        list_month = []
+        year = int(str(start)[:7].split('-')[0])  # 截取起始年份
+        for m in range(month_count):  # 利用range函数填充结果列表
+            month = int(str(start)[:7].split('-')[1])  # 截取起始月份,写在for循环里,作为每次迭代的累加基数
+            month = month + m
+            if month > 12:
+                if month % 12 > 0:
+                    month = month % 12  # 计算结果大于12,取余数
+                    if month == 1:
+                        year += 1  # 只需在1月份的时候对年份加1,注意year的初始化在for循环外
+                else:
+                    month = 12
+            if len(str(month)) == 1:
+                list_month.append(str(year) + '-0' + str(month))
+            else:
+                list_month.append(str(year) + '-' + str(month))
+        return list_month
+
+
+# 转换格式,去掉"-",%Y%m
+def get_eachmonth(start_month, end_month):
+    startmonth = start_month[0:4] + "-" + start_month[4:6]
+    endmonth = end_month[0:4] + "-" + end_month[4:6]
+    months = get_each_month(startmonth, endmonth)
+    list_month = [i[0:4] + i[5:7] for i in months]
+    return list_month
+
+
+def get_month(starttime, endtime):
+    months = []
+    starttime = datetime.datetime.strptime(starttime, YYmmddHHMMSS)
+    endtime = datetime.datetime.strptime(endtime, YYmmddHHMMSS)
+    while starttime <= endtime:
+        start = starttime.strftime("%Y%m")
+        if start not in months:
+            months.append(start)
+        starttime = starttime + datetime.timedelta(days=1)
+    return months
+
+def get_month_range(starttime, endtime):
+    months = []
+    starttime = datetime.datetime.strptime(starttime[0:8], YYmmdd)
+    endtime = datetime.datetime.strptime(endtime[0:8], YYmmdd)
+    while starttime < endtime:
+        start = starttime.strftime(YYmmddHHMMSS)
+        endMonth = datetime.datetime.strptime(start[0:6], YYmm) + relativedelta(months=+1)
+
+        end = endMonth.strftime(YYmmddHHMMSS)
+        if endtime <= endMonth:
+            end = endtime.strftime(YYmmddHHMMSS)
+        starttime = endMonth
+        months.append([start, end])
+    return months
+
+
+
+def get_month1(starttime, endtime):
+    months = []
+    startyear = int(starttime[0:4])
+    startmonth = int(starttime[4:6])
+    endyear = int(endtime[0:4])
+    endmonth = int(endtime[4:6])
+    while startyear != endyear or startmonth != endmonth:
+        startyearstr = str(startyear)
+        startmonthstr = str(startmonth)
+        if startmonth < 10:
+            startmonthstr = "0" + str(startmonth)
+        months.append(startyearstr + startmonthstr)
+        startmonth += 1
+        if startmonth == 13:
+            startyear += 1
+            startmonth = 1
+    months.append(endtime[0:6])
+    return months
+
+#根据开始结束时间获取最大区间为1天的时间区间
+# def get_day(starttime,endtime):
+#     times = []
+#     starttime = datetime.datetime.strptime(starttime, YYmmddHHMMSS)
+#     endtime = datetime.datetime.strptime(endtime, YYmmddHHMMSS)
+#     while starttime < endtime:
+#         start = starttime.strftime(YYmmddHHMMSS)
+#         starttime_delta = starttime + datetime.timedelta(days=1)
+#         end = starttime_delta.strftime(YYmmddHHMMSS)
+#         if str(starttime)[0:7]<str(starttime_delta)[0:7] :
+#             end = starttime_delta.strftime("%Y%m") +"01000000"
+#             starttime = datetime.datetime.strptime(end, YYmmddHHMMSS)
+#             times.append([start, end])
+#             continue
+#         if starttime >= starttime_delta:
+#             end = starttime_delta.strftime(YYmmddHHMMSS)
+#         starttime = starttime + datetime.timedelta(days=1)
+#         times.append([start,end])
+#     return times
+#根据开始结束时间获取最大区间为1天的时间区间
+def get_day(starttime,endtime):
+    times = []
+    starttime = datetime.datetime.strptime(starttime, YYmmddHHMMSS)
+    endtime = datetime.datetime.strptime(endtime, YYmmddHHMMSS)
+    while starttime < endtime:
+        start = starttime.strftime(YYmmddHHMMSS)
+        stratDay = datetime.datetime.strptime(start[0:8],YYmmdd)
+        starttime_delta = stratDay + datetime.timedelta(days=1)
+        end = starttime_delta.strftime(YYmmddHHMMSS)
+        if endtime <= starttime_delta:
+            end = endtime.strftime(YYmmddHHMMSS)
+        starttime = stratDay + datetime.timedelta(days=1)
+        times.append([start,end])
+    return times
+
+
+def get_day_1(starttime,endtime):
+    times = []
+    starttime = datetime.datetime.strptime(starttime, YYmmdd)
+    endtime = datetime.datetime.strptime(endtime, YYmmdd)
+    while starttime < endtime:
+        start = starttime.strftime(YYmmdd)
+        stratDay = datetime.datetime.strptime(start[0:8],YYmmdd)
+        starttime_delta = stratDay + datetime.timedelta(days=1)
+        end = starttime_delta.strftime(YYmmdd)
+        if endtime <= starttime_delta:
+            end = endtime.strftime(YYmmdd)
+        starttime = stratDay + datetime.timedelta(days=1)
+        times.append(start)
+
+    return times
+
+if __name__ == '__main__':
+    print(get_day_1("20220128", "20220205"))

+ 115 - 0
MyUtils/MysqlUtils.py

@@ -0,0 +1,115 @@
+import pymysql
+from dbutils.pooled_db import PooledDB
+
+
+class MysqlUtils():
+    # 连接池对象
+    __pool = None
+    def __init__(self,**config):
+        self.host = config["host"]
+        self.port = int(config["port"])
+        self.user = config["user"]
+        self.passwd = config["passwd"]
+        self.database = None
+        if "database" in config:
+            self.database = config["database"]
+        self.pool = self.get_conn()
+        # self.pool = None
+
+    def get_conn(self):
+        if MysqlUtils.__pool is None:
+            __pool = PooledDB(pymysql, mincached=1, maxcached=5, maxconnections=5,
+                              host=self.host, port=self.port, user=self.user, passwd=self.passwd,
+                              database=self.database,
+                               use_unicode=True, blocking=False, charset="utf8")
+            self.pool = __pool
+            return __pool
+
+
+    def query(self,sql):
+        con = None
+        cur = None
+        try:
+            con = self.pool.connection()
+            cur = con.cursor()
+            count = cur.execute(sql)
+            if count >= 0:
+                result = cur.fetchall()
+                result = [[j.decode() if isinstance(j, bytes) else j for j in i] for i in result]
+                # print(result)
+            else:
+                result = False
+            return result
+        except Exception as e:
+            print(e)
+        finally:
+            if cur:
+                cur.close()
+            if con:
+                con.close()
+
+
+    def update(self,sql):
+        con = None
+        cur = None
+        try:
+            con = self.pool.connection()
+            cur = con.cursor()
+            cur.execute(sql)
+            con.commit()
+        except Exception as e:
+            con.rollback()  # 事务回滚
+            print(e)
+        finally:
+            if cur:
+                cur.close()
+            if con:
+                con.close()
+
+    def update_mult(self,sql1,sql2,sql3):
+        con = None
+        cur = None
+        try:
+            con = self.pool.connection()
+            cur = con.cursor()
+            cur.execute(sql1)
+            cur.execute(sql2)
+            cur.execute(sql3)
+            con.commit()
+            result = True
+        except Exception as e:
+            con.rollback()  # 事务回滚
+            print(e)
+            result = False
+
+        finally:
+            if cur:
+                cur.close()
+            if con:
+                con.close()
+        return  result
+
+    def update_two(self,sql1,sql2):
+        con = None
+        cur = None
+        try:
+            con = self.pool.connection()
+            cur = con.cursor()
+            cur.execute(sql1)
+            cur.execute(sql2)
+            con.commit()
+        except Exception as e:
+            con.rollback()  # 事务回滚
+            print(e)
+        finally:
+            if cur:
+                cur.close()
+            if con:
+                con.close()
+
+
+
+
+    def close(self):
+        if MysqlUtils.__pool:
+            MysqlUtils.__pool.close()

+ 55 - 0
main.py

@@ -0,0 +1,55 @@
+from MyUtils.MysqlUtils import MysqlUtils
+from MyUtils.DateUtils import get_day_1
+import datetime
+import dateutil.relativedelta
+import time
+import pytz
+import schedule
+
+REPLACE_EQUIP_SQL = "replace into sagacloud_review.custom_project_equipment_quarter_history SELECT * from sagacloud_review.custom_project_equipment_quarter where date = '%s'"
+REPLACE_SPACE_SQL = "replace into sagacloud_review.custom_space_quarter_history SELECT * from sagacloud_review.custom_space_quarter where date = '%s'"
+DELETE_EQUIP_SQL = "delete from sagacloud_review.custom_project_equipment_quarter where date = '%s'"
+DELETE_SPACE_SQL = "delete from sagacloud_review.custom_space_quarter where date = '%s'"
+REPLACE_TARGET_SQL = "replace into sagacloud_customization.custom_persist_target_history SELECT * from sagacloud_customization.custom_persist_target where date = '%s'"
+DELETE_TARGET_SQL = "delete from sagacloud_customization.custom_persist_target where date = '%s'"
+
+
+def datetime_now():
+    # datetime_now = datetime.datetime.now().strftime("%Y%m%d%H%M%S")
+    #容器时间
+    # tz = pytz.timezone('Asia/Shanghai')  # 东八区
+    datetime_now = datetime.datetime.fromtimestamp(int(time.time()),
+                                    pytz.timezone('Asia/Shanghai')).strftime('%Y-%m-%d %H:%M:%S')
+    return datetime_now
+
+
+mysql = {'database': 'sagacloud_review', 'host': '10.100.28.84', 'passwd': 'gWK5o9WmCBF5LiW', 'port': 9934, 'user': 'root'}
+
+start_date_month = (datetime.datetime.now() + dateutil.relativedelta.relativedelta(months=-2)).strftime("%Y%m")+"01"
+end_date_month = (datetime.datetime.now() + dateutil.relativedelta.relativedelta(months=-1)).strftime("%Y%m")+"01"
+# print(start_date_month,end_date_month)
+
+# start_date_month = "20220903"
+# end_date_month = "20230201"
+def job():
+    if datetime.date.today().day != 1:
+        print("%s 等待4号执行程序"%datetime_now())
+    else:
+        # 连接mysql
+        MysqlUtil = MysqlUtils(**mysql)
+
+        for date in days:
+            print("%s 开始导入%s的数据"%(datetime_now(),date))
+            MysqlUtil.update_two(REPLACE_EQUIP_SQL%(date),DELETE_EQUIP_SQL%(date))
+            MysqlUtil.update_two(REPLACE_SPACE_SQL%(date),DELETE_SPACE_SQL%(date))
+            MysqlUtil.update_two(REPLACE_TARGET_SQL%(date),DELETE_TARGET_SQL%(date))
+
+        # 关闭数据库
+        MysqlUtil.close()
+
+
+schedule.every().day.at("08:30").do(job)
+while True:
+    days = get_day_1(start_date_month, end_date_month)
+    schedule.run_pending()
+    time.sleep(30)

+ 5 - 0
requirements.txt

@@ -0,0 +1,5 @@
+pymysql==1.0.2
+pytz==2021.1
+python-dateutil==2.8.1
+dbutils==2.0.2
+schedule==1.1.0