|
|
@@ -0,0 +1,157 @@
|
|
|
+from datetime import datetime
|
|
|
+import requests
|
|
|
+import sqlite3
|
|
|
+import json
|
|
|
+import os
|
|
|
+import sys
|
|
|
+from weather import all_city
|
|
|
+import logging
|
|
|
+import django
|
|
|
+import time
|
|
|
+import random
|
|
|
+BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
|
+sys.path.append(BASE_DIR)
|
|
|
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'bigdataAPI.settings')
|
|
|
+django.setup()
|
|
|
+from apps.Weather.models import District, DayData, HistoryDayData
|
|
|
+# # 配置日志级别和格式
|
|
|
+# logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
|
|
|
+
|
|
|
+# # 创建一个文件处理器，指定日志文件名和写入模式
|
|
|
+# os.mkdir("/data/weather/") if not os.path.exists("/data/weather/") else None
|
|
|
+# file_handler = logging.FileHandler('/data/weather/app.log')
|
|
|
+# file_handler.setLevel(logging.INFO)
|
|
|
+
|
|
|
+# # 创建一个格式化程序，用于定义日志消息的显示方式
|
|
|
+# formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
|
|
|
+# file_handler.setFormatter(formatter)
|
|
|
+
|
|
|
+# # 将文件处理器添加到根日志记录器中
|
|
|
+# logging.getLogger('').addHandler(file_handler)
|
|
|
+
|
|
|
+app_id = "69334222"
|
|
|
+app_secret = "2u4bHXHD"
|
|
|
+
|
|
|
+province_dict = {
|
|
|
+"11": '北京市',
|
|
|
+"12": '天津市',
|
|
|
+"13": '河北省',
|
|
|
+"14": '山西省',
|
|
|
+"15": '内蒙古自治区',
|
|
|
+"21": '辽宁省',
|
|
|
+"22": '吉林省',
|
|
|
+"23": '黑龙江省',
|
|
|
+"31": '上海市',
|
|
|
+"32": '江苏省',
|
|
|
+"33": '浙江省',
|
|
|
+"34": '安徽省',
|
|
|
+"35": '福建省',
|
|
|
+"36": '江西省',
|
|
|
+"37": '山东省',
|
|
|
+"41": '河南省',
|
|
|
+"42": '湖北省',
|
|
|
+"43": '湖南省',
|
|
|
+"44": '广东省',
|
|
|
+"45": '广西壮族自治区',
|
|
|
+"46": '海南省',
|
|
|
+"50": '重庆市',
|
|
|
+"51": '四川省',
|
|
|
+"52": '贵州省',
|
|
|
+"53": '云南省',
|
|
|
+"54": '西藏自治区',
|
|
|
+"61": '陕西省',
|
|
|
+"62": '甘肃省',
|
|
|
+"63": '青海省',
|
|
|
+"64": '宁夏回族自治区',
|
|
|
+"65": '新疆维吾尔自治区'
|
|
|
+}
|
|
|
+
|
|
|
+def init_data():
|
|
|
+ # 导入全部的城市
|
|
|
+ for k in all_city:
|
|
|
+ print(k, "-------------")
|
|
|
+ district = District()
|
|
|
+ district.city_id = str(k[0])
|
|
|
+ district.city = k[1]
|
|
|
+ district.pid = str(k[2])
|
|
|
+ district.save()
|
|
|
+
|
|
|
+
|
|
|
+def main():
|
|
|
+ headers = {
|
|
|
+ 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 Edge/16.16299',
|
|
|
+ 'Accept-Language': 'en-US,en;q=0.8',
|
|
|
+ 'Accept-Encoding': 'gzip, deflate, br',
|
|
|
+ 'Connection': 'keep-alive',
|
|
|
+ }
|
|
|
+ print("up today ....")
|
|
|
+ day_data = DayData.objects.all()
|
|
|
+ today_error = 0
|
|
|
+ # 调用次数统计
|
|
|
+ day_ct = 0
|
|
|
+ now = datetime.now()
|
|
|
+ start_ct_time = now.strftime('%Y-%m-%d %H:%M:%S')
|
|
|
+ now_hour = now.hour
|
|
|
+ today_str = datetime.strptime(f'{now.year}-{now.month}-{now.day} 19:00:00', "%Y-%m-%d %H:%M:%S")
|
|
|
+ for day_obj in day_data:
|
|
|
+ time_out = False
|
|
|
+ print(day_obj.cityid)
|
|
|
+ today_url = f"http://v1.yiketianqi.com/api?unescape=1&version=v62&appid=69334222&appsecret=2ME6U58N&cityid={day_obj.cityid}"
|
|
|
+ today_response = requests.get(today_url, headers=headers, timeout=10)
|
|
|
+ day_ct += 1
|
|
|
+ try:
|
|
|
+ if today_response.status_code == 200:
|
|
|
+ today_data = json.loads(today_response.text)
|
|
|
+ if "errcode" not in today_data.keys():
|
|
|
+ day_obj.content = str(today_data)
|
|
|
+ day_obj.save()
|
|
|
+ try:
|
|
|
+ if now_hour == 19:
|
|
|
+ HistoryDayData.objects.create(
|
|
|
+ cityid = day_obj.cityid,
|
|
|
+ content = today_data,
|
|
|
+ addtime = int(today_str.timestamp())
|
|
|
+ )
|
|
|
+ except Exception as e:
|
|
|
+ print(e, "历史数据入库失败")
|
|
|
+ else:
|
|
|
+ print(f"cityid {day_obj.cityid} error_code {today_response.status_code}")
|
|
|
+ print(f"request fail again : {day_obj.cityid}")
|
|
|
+ time.sleep(2)
|
|
|
+ today_response = requests.get(today_url, headers=headers, timeout=5)
|
|
|
+ day_ct += 1
|
|
|
+ if today_response.status_code == 200:
|
|
|
+ today_data = json.loads(today_response.text)
|
|
|
+ if "errcode" not in today_data.keys():
|
|
|
+ day_obj.content = str(today_data)
|
|
|
+ day_obj.save()
|
|
|
+ print(f"request again success: {day_obj.cityid}")
|
|
|
+ try:
|
|
|
+ if now_hour == 19:
|
|
|
+ HistoryDayData.objects.create(
|
|
|
+ cityid = day_obj.cityid,
|
|
|
+ content = today_data,
|
|
|
+ addtime = int(today_str.timestamp())
|
|
|
+ )
|
|
|
+ except Exception as e:
|
|
|
+ print(e, "历史数据入库失败")
|
|
|
+ else:
|
|
|
+ print(f"cityid {day_obj.cityid} error_code {today_response.status_code} again request fail")
|
|
|
+ print(f"error {today_error}")
|
|
|
+ today_error += 1
|
|
|
+ time_out = True
|
|
|
+ except Exception as e:
|
|
|
+ print(f"request fail again : {day_obj.cityid} {e.args}")
|
|
|
+ if time_out:
|
|
|
+ time.sleep(1)
|
|
|
+ else:
|
|
|
+ time.sleep(0.5)
|
|
|
+
|
|
|
+ with open("/data/weather/weather_count.txt", 'a+', encoding='utf-8') as f:
|
|
|
+ f.write(f"当日天气接口： 开始时间：{start_ct_time} 结束时间：{datetime.now().strftime('%Y-%m-%d %H:%M:%S')} 调用次数：{day_ct} \n")
|
|
|
+
|
|
|
+ # 把19点的数据写入历史天气数据中
|
|
|
+
|
|
|
+
|
|
|
+if __name__ == "__main__":
|
|
|
+ main()
|
林轩