import requests import sqlite3 import json import os import sys from weather import all_city import logging import django import time import random BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.append(BASE_DIR) os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'bigdataAPI.settings') django.setup() from apps.Weather.models import District, DayData, ServerDayData # 配置日志级别和格式 logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') # 创建一个文件处理器，指定日志文件名和写入模式 os.mkdir("/data/weather/") if not os.path.exists("/data/weather/") else None file_handler = logging.FileHandler('/data/weather/app.log') file_handler.setLevel(logging.INFO) # 创建一个格式化程序，用于定义日志消息的显示方式 formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') file_handler.setFormatter(formatter) # 将文件处理器添加到根日志记录器中 logging.getLogger('').addHandler(file_handler) app_id = "69334222" app_secret = "2u4bHXHD" province_dict = { "11": '北京市', "12": '天津市', "13": '河北省', "14": '山西省', "15": '内蒙古自治区', "21": '辽宁省', "22": '吉林省', "23": '黑龙江省', "31": '上海市', "32": '江苏省', "33": '浙江省', "34": '安徽省', "35": '福建省', "36": '江西省', "37": '山东省', "41": '河南省', "42": '湖北省', "43": '湖南省', "44": '广东省', "45": '广西壮族自治区', "46": '海南省', "50": '重庆市', "51": '四川省', "52": '贵州省', "53": '云南省', "54": '西藏自治区', "61": '陕西省', "62": '甘肃省', "63": '青海省', "64": '宁夏回族自治区', "65": '新疆维吾尔自治区' } def init_data(): # 导入全部的城市 for k in all_city: print(k, "-------------") district = District() district.city_id = str(k[0]) district.city = k[1] district.pid = str(k[2]) district.save() def main(): headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 Edge/16.16299', 'Accept-Language': 'en-US,en;q=0.8', 'Accept-Encoding': 'gzip, deflate, br', 'Connection': 'keep-alive', } day_data = DayData.objects.filter(id__gte=2683) today_error = 0 for day_obj in day_data: time_out = False logging.warning(day_obj.cityid) today_url = f"http://v0.yiketianqi.com/api?unescape=1&version=v62&appid=69334222&appsecret=2ME6U58N&cityid={day_obj.cityid}" today_response = requests.get(today_url, headers=headers, timeout=5) try: if today_response.status_code == 200: today_data = json.loads(today_response.text) if "errcode" not in today_data.keys(): day_obj.content = str(today_data) day_obj.save() else: logging.warning(f"cityid {day_obj.cityid} error_code {today_response.status_code}") logging.warning(f"request fail again : {day_obj.cityid}") time.sleep(2) today_response = requests.get(today_url, headers=headers, timeout=5) if today_response.status_code == 200: today_data = json.loads(today_response.text) if "errcode" not in today_data.keys(): day_obj.content = str(today_data) day_obj.save() logging.warning(f"request again success: {day_obj.cityid}") else: logging.warning(f"cityid {day_obj.cityid} error_code {today_response.status_code} again request fail") logging.error(f"error {today_error}") today_error += 1 time_out = True except Exception as e: logging.warning(f"request fail again : {day_obj.cityid} {e.args}") if time_out: time.sleep(5) else: time.sleep(2) server_error = 0 logging.warning("up server day ...") server_days = ServerDayData.objects.all() for server_day_obj in server_days: time_out_k = False logging.warning(server_day_obj.cityid) server_day_url = f"http://v0.yiketianqi.com/api?unescape=1&version=v91&appid=69334222&appsecret=2ME6U58N&cityid={server_day_obj.cityid}&ext=hours,aqi,life" server_day_response = requests.get(server_day_url, headers=headers, timeout=5) try: if server_day_response.status_code == 200: server_day_data = json.loads(server_day_response.text) if "errcode" not in server_day_data.keys(): server_day_obj.content = str(server_day_data) server_day_obj.save() else: time.sleep(2) logging.warning(f"cityid {day_obj.cityid} error_code {server_day_response.status_code}") logging.warning(f"request fail again : {server_day_obj.cityid}") server_day_response = requests.get(server_day_url, headers=headers, timeout=5) if server_day_response.status_code == 200: server_day_data = json.loads(server_day_response.text) if "errcode" not in server_day_data.keys(): server_day_obj.content = str(server_day_data) server_day_obj.save() logging.warning(f"request again success: {server_day_obj.cityid}") else: logging.warning(f"cityid {day_obj.cityid} error_code {server_day_response.status_code} again request afil") logging.error(f"error {server_error}") server_error += 1 time_out_k = True except Exception as e: logging.warning(f"request fail again : {server_day_obj.cityid} {e.args}") if time_out_k: time.sleep(5) else: time.sleep(2) """ district = District.objects.all() # init_data() logging.warning("up...") results = District.objects.filter(id__gt = 31) # 更新数据 for row in results: temp = row.city # 东城区 city_id = row.city_id # 110101000000 pid = row.pid # 110100000000 province_id = city_id[:2] province = province_dict.get(province_id) if len(pid) == 2 and temp == "市辖区": # 处理 市辖区 只有两级，区为空 city = province district = "市辖区" elif len(pid) ==2 and temp != "市辖区": # 三级 省 市 区（县） city = temp district = "" else: dis = District.objects.get(city_id=pid) city = dis.city district = temp try: today_url = f"http://v0.yiketianqi.com/api?unescape=1&version=v62&appid=69334222&appsecret=2u4bHXHD&adcode={city_id}" today_response = requests.get(today_url, timeout=5) today_data = json.loads(today_response.text) if "errcode" not in today_data.keys(): cityid = today_data.get("cityid", "") DayData.objects.update_or_create( cityid=cityid, defaults={ "cityid": cityid, "province": province, "city": city, "district": district, "content": str(today_data) } ) server_day_url = f"http://v0.yiketianqi.com/api?unescape=1&version=v91&appid={app_id}&appsecret={app_secret}&adcode={city_id}&ext=hours,aqi,life" server_day_response = requests.get(server_day_url, timeout=5) server_day_data = json.loads(server_day_response.text) if "errcode" not in server_day_data.keys(): cityid = today_data.get("cityid", "") ServerDayData.objects.update_or_create( cityid=cityid, defaults={ "cityid": cityid, "province": province, "city": city, "district": district, "content": str(server_day_data) } ) except Exception as e: print(e) logging.warning("over...") """ if __name__ == "__main__": main()