bigdataAPI/crond_script/crond_weather.py

import requests
import sqlite3
import json
import os
import sys
from weather import all_city
import logging
import django
import time
import random
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.append(BASE_DIR)
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'bigdataAPI.settings')
django.setup()
from apps.Weather.models import District, DayData, ServerDayData
# 配置日志级别和格式
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')

# 创建一个文件处理器，指定日志文件名和写入模式
os.mkdir("/data/weather/") if not os.path.exists("/data/weather/") else None
file_handler = logging.FileHandler('/data/weather/app.log')
file_handler.setLevel(logging.INFO)

# 创建一个格式化程序，用于定义日志消息的显示方式
formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
file_handler.setFormatter(formatter)

# 将文件处理器添加到根日志记录器中
logging.getLogger('').addHandler(file_handler)

app_id = "69334222"
app_secret = "2u4bHXHD"

province_dict = {
"11": '北京市',
"12": '天津市',
"13": '河北省',
"14": '山西省',
"15": '内蒙古自治区',
"21": '辽宁省',
"22": '吉林省',
"23": '黑龙江省',
"31": '上海市',
"32": '江苏省',
"33": '浙江省',
"34": '安徽省',
"35": '福建省',
"36": '江西省',
"37": '山东省',
"41": '河南省',
"42": '湖北省',
"43": '湖南省',
"44": '广东省',
"45": '广西壮族自治区',
"46": '海南省',
"50": '重庆市',
"51": '四川省',
"52": '贵州省',
"53": '云南省',
"54": '西藏自治区',
"61": '陕西省',
"62": '甘肃省',
"63": '青海省',
"64": '宁夏回族自治区',
"65": '新疆维吾尔自治区'
}

def init_data():
    # 导入全部的城市
    for k in all_city:
        print(k, "-------------")
        district = District()
        district.city_id = str(k[0])
        district.city = k[1]
        district.pid = str(k[2])
        district.save()
    

def main():
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 Edge/16.16299',
        'Accept-Language': 'en-US,en;q=0.8',
        'Accept-Encoding': 'gzip, deflate, br',
        'Connection': 'keep-alive',
    }
    day_data = DayData.objects.all()
    today_error = 0
    for day_obj in day_data:
        time_out = False 
        logging.warning(day_obj.cityid)
        today_url = f"http://v0.yiketianqi.com/api?unescape=1&version=v62&appid=69334222&appsecret=2ME6U58N&cityid={day_obj.cityid}"
        today_response = requests.get(today_url, headers=headers, timeout=5)
        try:
            if today_response.status_code == 200:
                today_data = json.loads(today_response.text)
                if "errcode" not in today_data.keys():
                    day_obj.content = str(today_data)
                    day_obj.save()
            else:
                logging.warning(f"cityid {day_obj.cityid}  error_code {today_response.status_code}")
                logging.warning(f"request fail again : {day_obj.cityid}")
                time.sleep(2)
                today_response = requests.get(today_url, headers=headers, timeout=5)
                if today_response.status_code == 200:
                    today_data = json.loads(today_response.text)
                    if "errcode" not in today_data.keys():
                        day_obj.content = str(today_data)
                        day_obj.save()
                        logging.warning(f"request again success: {day_obj.cityid}")
                else:
                    logging.warning(f"cityid {day_obj.cityid}  error_code {today_response.status_code}  again request fail")
                    logging.error(f"error {today_error}")
                    today_error += 1
                    time_out = True
        except Exception as e:
            logging.warning(f"request fail again : {day_obj.cityid} {e.args}")
        if time_out:
            time.sleep(5)
        else:
            time.sleep(2)
        
        

    server_error = 0
    logging.warning("up server day ...")
    server_days = ServerDayData.objects.all()
    for server_day_obj in server_days:
        time_out_k = False 
        logging.warning(server_day_obj.cityid)
        server_day_url = f"http://v0.yiketianqi.com/api?unescape=1&version=v91&appid=69334222&appsecret=2ME6U58N&cityid={server_day_obj.cityid}&ext=hours,aqi,life"
        server_day_response = requests.get(server_day_url, headers=headers, timeout=5)
        try:
            if server_day_response.status_code == 200:
                server_day_data = json.loads(server_day_response.text)
                if "errcode" not in server_day_data.keys():
                    server_day_obj.content = str(server_day_data)
                    server_day_obj.save()
            else:
                time.sleep(2)
                logging.warning(f"cityid {day_obj.cityid}  error_code {server_day_response.status_code}")
                logging.warning(f"request fail again : {server_day_obj.cityid}")
                server_day_response = requests.get(server_day_url, headers=headers, timeout=5)
                if server_day_response.status_code == 200:
                    server_day_data = json.loads(server_day_response.text)
                    if "errcode" not in server_day_data.keys():
                        server_day_obj.content = str(server_day_data)
                        server_day_obj.save()
                        logging.warning(f"request again success: {server_day_obj.cityid}")
                else:
                    logging.warning(f"cityid {day_obj.cityid}  error_code {server_day_response.status_code}  again request afil")
                    logging.error(f"error {server_error}")
                    server_error += 1
                    time_out_k = True
        except Exception as e:
            logging.warning(f"request fail again : {server_day_obj.cityid} {e.args}")
        if time_out_k:
            time.sleep(5)
        else:
            time.sleep(2)
        


    """
    district = District.objects.all() 
    # init_data()
    logging.warning("up...")
    results = District.objects.filter(id__gt = 31)
    # 更新数据
    for row in results:
        temp = row.city   # 东城区
        city_id = row.city_id       # 110101000000
        pid = row.pid     # 110100000000
        province_id = city_id[:2]
        province = province_dict.get(province_id)
        if len(pid) == 2 and temp == "市辖区":
            # 处理 市辖区 只有两级，区为空
            city = province
            district = "市辖区"
        elif len(pid) ==2 and temp != "市辖区":
            # 三级 省 市  区（县）
            city = temp
            district = ""
        else:
            dis = District.objects.get(city_id=pid)
            city = dis.city
            district = temp

        try:
            today_url = f"http://v0.yiketianqi.com/api?unescape=1&version=v62&appid=69334222&appsecret=2u4bHXHD&adcode={city_id}"
            today_response = requests.get(today_url, timeout=5)
            today_data = json.loads(today_response.text)
            if "errcode" not in today_data.keys():
                cityid = today_data.get("cityid", "")
                DayData.objects.update_or_create(
                    cityid=cityid,
                    defaults={
                        "cityid": cityid,
                        "province": province,
                        "city": city,
                        "district": district,
                        "content": str(today_data)
                    }
                )
            server_day_url = f"http://v0.yiketianqi.com/api?unescape=1&version=v91&appid={app_id}&appsecret={app_secret}&adcode={city_id}&ext=hours,aqi,life"
            server_day_response = requests.get(server_day_url, timeout=5)
            server_day_data = json.loads(server_day_response.text)
            if "errcode" not in server_day_data.keys():
                cityid = today_data.get("cityid", "")
                ServerDayData.objects.update_or_create(
                    cityid=cityid,
                    defaults={
                        "cityid": cityid,
                        "province": province,
                        "city": city,
                        "district": district,
                        "content": str(server_day_data)
                    }
                )
        except Exception as e:
            print(e)
    logging.warning("over...")
    """

if __name__ == "__main__":
    main()