Merge branch 'master' of http://code.nyzhwlw.com:10202/yf_fyh/bigdataAPI

apps/PestAnalysis/models.py

@@ -1,3 +1,40 @@
 from django.db import models
 
-# Create your models here.
+class HeNanAddr(models.Model):
+    province = models.CharField(max_length=100, blank=True, null=True)
+    city = models.CharField(max_length=100, blank=True, null=True)
+    district = models.CharField(max_length=100, blank=True, null=True)
+    code = models.CharField(max_length=100, blank=True, null=True)
+
+    class Meta:
+        db_table = 'he_nan_addr'
+
+class CbdAddrPest(models.Model):
+    pest = models.CharField(max_length=100, blank=True, null=True)
+    pest_id = models.CharField(max_length=10, blank=True, null=True, default="0")
+    count = models.PositiveIntegerField(default=0)
+    date = models.PositiveIntegerField(default=0)
+    format_date = models.CharField(max_length=100, blank=True, null=True)
+    province = models.CharField(max_length=100, blank=True, null=True)
+    city = models.CharField(max_length=100, blank=True, null=True)
+    district = models.CharField(max_length=100, blank=True, null=True)
+    addr_code = models.CharField(max_length=100, blank=True, null=True)
+
+    class Meta:
+        db_table = 'cbd_addr_pest'
+
+
+class EnvTempHum(models.Model):
+    title = models.CharField(max_length=100, blank=True, null=True)
+    value = models.CharField(max_length=100, blank=True, null=True)
+    unit = models.CharField(max_length=100, blank=True, null=True)
+    type = models.CharField(max_length=100, blank=True, null=True)
+    date = models.PositiveIntegerField(default=0)
+    format_date = models.CharField(max_length=100, blank=True, null=True)
+    province = models.CharField(max_length=100, blank=True, null=True)
+    city = models.CharField(max_length=100, blank=True, null=True)
+    district = models.CharField(max_length=100, blank=True, null=True)
+    addr_code = models.CharField(max_length=100, blank=True, null=True)
+
+    class Meta:
+        db_table = 'env_temp_hum'

apps/PestAnalysis/serializers.py

@@ -1,5 +1,5 @@
 from rest_framework import serializers
-
+from apps.PestAnalysis.models import CbdAddrPest, EnvTempHum
 
 class ZhiBaoSelectViewSerializer(serializers.Serializer):
     identify_model = serializers.CharField(help_text="识别模型", required=True)
@@ -37,4 +37,16 @@ class ZhiBaoSelectViewSerializer(serializers.Serializer):
     def validate(self, attrs):
         if attrs["start_time"] > attrs["end_time"]:
             raise serializers.ValidationError("结束时间不能小于开始时间")
-        return attrs
+        return attrs
+    
+
+class CbdAddrPestSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = CbdAddrPest
+        fields = "__all__"
+
+
+class EnvTempHumSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = EnvTempHum
+        fields = "__all__"

apps/PestAnalysis/urls.py

@@ -5,4 +5,6 @@ from . import views
 urlpatterns = [
     url(r'^info/$', views.PestSelectView.as_view(), name='pest_info'),
     url(r'^pest/$', views.RecentMonthPestCount.as_view(), name='pest'),
+    url(r"^pest_count/$", views.PestCountAPIView.as_view(), name="pest_count"),
+    url(r"^env_temp_hum/$", views.EnvTempHumAPIView.as_view(), name="env_temp_hum"),
 ]

apps/PestAnalysis/views.py

@@ -2,17 +2,23 @@ from rest_framework.generics import GenericAPIView
 from rest_framework.response import Response
 import re
 import math
+import datetime
 from utils.JWTAuthentication_diy import APIAuthentication
 from utils.permissions import CbdDeviceDetailPermission
 from .serializers import ZhiBaoSelectViewSerializer
 from utils.JWTAuthentication_diy import MyJWTAuthentication
 from utils.permissions import ModulePermission
+from rest_framework.views import APIView
 
 from utils.db_utils import MongoDBTools
 from utils.all_dict import insect_dict
 from django.conf import settings
 
 
+from apps.PestAnalysis.models import CbdAddrPest, EnvTempHum
+from apps.PestAnalysis.serializers import CbdAddrPestSerializer, EnvTempHumSerializer
+
+
 class PestSelectView(GenericAPIView):
     authentication_classes = [MyJWTAuthentication]
     permission_classes = [ModulePermission]
@@ -209,3 +215,99 @@ class RecentMonthPestCount(GenericAPIView):
             return Response({"data": [], "imgs": [], "msg": e.args})
 
 
+class PestCountAPIView(APIView):
+
+    def get(self, request, *args, **kwargs):
+        try:
+            # regionCode=city_410100&page=1&pageSize=20&startDate=2025-12-04&endDate=2025-12-19
+            data = request.query_params
+            pest = data.get("pest")
+            code = data.get("regionCode") 
+            code = code.split("_")[-1]
+            start = data.get("startDate")
+            end = data.get("endDate")
+            page = data.get("page")
+            pageSize = data.get("pageSize")
+            if page:
+                try:
+                    page = int(page)
+                except:
+                    page = 1
+            else:
+                page = 1
+            if pageSize:
+                try:
+                    pageSize = int(pageSize)
+                except:
+                    pageSize = 20
+            else:
+                pageSize = 20
+            if code:
+                query = CbdAddrPest.objects.filter(addr_code__startswith=code).order_by("-date")
+                if start and end:
+                    start = int(datetime.datetime.strptime(start, "%Y-%m-%d").timestamp())
+                    end = int(datetime.datetime.strptime(end, "%Y-%m-%d").timestamp())
+                    query = query.filter(date__range=[start, end])
+                if pest:
+                    query = query.filter(pest__icontains=pest)
+                count = query.count()
+                query = query[(page-1)*pageSize:page*pageSize]
+                data = CbdAddrPestSerializer(query, many=True).data
+                response = { 
+                    "data": {
+                        "data": data,
+                        "total": count,
+                    }
+                }
+                return Response(response)
+        except Exception as e:
+            return Response({"data": [], "imgs": [], "msg": e.args})
+        
+
+class EnvTempHumAPIView(APIView):
+
+    def get(self, request, *args, **kwargs):
+        try:
+            # regionCode=city_410100&page=1&pageSize=20&startDate=2025-12-04&endDate=2025-12-19
+            data = request.query_params
+            pest = data.get("pest")
+            code = data.get("regionCode") 
+            code = code.split("_")[-1]
+            start = data.get("startDate")
+            end = data.get("endDate")
+            page = data.get("page")
+            pageSize = data.get("pageSize")
+            if page:
+                try:
+                    page = int(page)
+                except:
+                    page = 1
+            else:
+                page = 1
+            if pageSize:
+                try:
+                    pageSize = int(pageSize)
+                except:
+                    pageSize = 20
+            else:
+                pageSize = 20
+            if code:
+                query = EnvTempHum.objects.filter(addr_code__startswith=code).order_by("-date")
+                if start and end:
+                    start = int(datetime.datetime.strptime(start, "%Y-%m-%d").timestamp())
+                    end = int(datetime.datetime.strptime(end, "%Y-%m-%d").timestamp())
+                    query = query.filter(date__range=[start, end])
+                if pest:
+                    query = query.filter(pest__icontains=pest)
+                count = query.count()
+                query = query[(page-1)*pageSize:page*pageSize]
+                data = EnvTempHumSerializer(query, many=True).data
+                response = { 
+                    "data": {
+                        "data": data,
+                        "total": count,
+                    }
+                }
+                return Response(response)
+        except Exception as e:
+            return Response({"data": [], "imgs": [], "msg": e.args})

crond_script/cbd_analysis.py

@@ -0,0 +1,112 @@
+import requests
+import datetime
+import pandas as pd 
+import os 
+import sys 
+import django
+import json
+
+BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.append(BASE_DIR)
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'bigdataAPI.settings')
+django.setup()
+from apps.PestAnalysis.models import CbdAddrPest, HeNanAddr
+
+def init_henan_addr(): 
+    json_file = os.path.join(BASE_DIR, 'crond_script', 'henan-data.json')
+    with open(json_file, 'r') as f:
+        data = json.load(f)
+    province = data.get("provinces")
+    for p in province: 
+        cities = p.get("cities")
+        for c in cities:
+            city = c.get("name")
+            code = c.get("code")
+            HeNanAddr.objects.get_or_create(
+                code = code,
+                defaults={
+                    "province": "河南省",
+                    "city": city,
+                    "code": code
+                }
+            )
+            districts = c.get("districts")
+            for d in districts:
+                HeNanAddr.objects.create(
+                    province = "河南省",
+                    city = city,
+                    district = d.get("name"),
+                    code = d.get("code")
+                )
+
+
+def device_info(file_path):
+    df = pd.read_excel(file_path)
+    device_info = df.to_dict(orient='records')
+    return device_info
+
+
+def get_cbd_data(devices, start, end):
+    di = 0
+    for device in devices:
+        d_id = device.get("id")
+        di += 1
+        print(di, "+++++", d_id)
+        province = device.get("province")
+        city = device.get("city")
+        district = device.get("district")
+        url = "https://web.hnyfwlw.com/api/api_gateway?method=forecast.cbd_analysis.analysis_pest_result"
+        data = {
+            "d_id": str(d_id),
+            "start": start,
+            "end": end,
+            "offset": 0,
+            "model": "B",
+            "token": "pXQBKiumY1soyO854pFUfMARZV9pmJ0idqAePsqilVI="
+        }
+        response = requests.post(url, data=data)
+        result = response.json()
+        rk_data = result.get("data")
+        day = rk_data.get("day")
+        pest_title_list = rk_data.get("pest")
+        pest_name = rk_data.get("pest_name") 
+        for index, p in enumerate(pest_name):
+            pest_info = pest_title_list[index]
+            for dindex, d in enumerate(day):
+                if pest_info[dindex]:
+                    addr_obj = HeNanAddr.objects.filter(city = city, district = district)
+                    if addr_obj:
+                        addr_obj = addr_obj.first()
+                        addr_code = addr_obj.code
+                        if addr_code:
+                            cbd_pest_obj = CbdAddrPest.objects.filter(
+                                    format_date=d,
+                                    pest = p,
+                                    province = province,
+                                    city = city,
+                                    district = district,
+                                    addr_code = str(addr_code)
+                                )
+                            if cbd_pest_obj:
+                                cbd_pest_obj = cbd_pest_obj.first()
+                                org_count = cbd_pest_obj.count
+                                cbd_pest_obj.count = org_count + pest_info[dindex]
+                                cbd_pest_obj.save()
+                            else:
+                                CbdAddrPest.objects.create(
+                                    date = int(datetime.datetime.strptime(d, "%Y-%m-%d").timestamp()),
+                                    format_date=d,
+                                    pest = p,
+                                    province = province,
+                                    city = city,
+                                    district = district,
+                                    count = pest_info[dindex],
+                                    addr_code = str(addr_code)
+                                )
+
+
+if __name__ == '__main__':
+    devices = device_info(f"{BASE_DIR}/crond_script/find_query.xlsx")
+    get_cbd_data(devices, "2024-01-01", "2024-12-31")
+    # init_henan_addr()
+    

crond_script/nd_qxz_data.py

@@ -0,0 +1,178 @@
+from datetime import datetime
+import pymongo
+import pandas as pd
+from urllib import parse
+import requests
+import os 
+import sys
+import django
+
+
+BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.append(BASE_DIR)
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'bigdataAPI.settings')
+django.setup()
+from apps.PestAnalysis.models import EnvTempHum, HeNanAddr
+
+
+def bigata_mongo():
+    user = parse.quote_plus("root")
+    passwd = parse.quote_plus("yfkj@6020")
+    myclient = pymongo.MongoClient("mongodb://{0}:{1}@8.136.98.49:57017/".format(user,passwd))
+    return myclient
+
+def device_aggrate():
+    myclient = bigata_mongo() 
+    mydb = myclient.smartfarming
+    mycol = mydb.sa_device 
+    query = [
+        {
+            "$match": {
+                "province": "河南省",
+                "device_type_id": 15
+            }
+        },
+        {
+            "$group": {
+            "_id": {
+                "province": "$province",
+                "city": "$city",
+                "district": "$district"
+            },
+            "device_ids": { "$addToSet": "$device_id" },
+            "unique_count": { "$sum": 1 }
+            }
+        },
+        {
+            "$project": {
+            "location": "$_id",
+            "device_ids": 1,
+            "unique_count": 1,
+            "_id": 0
+            }
+        }
+    ]
+    result = mycol.aggregate(query)
+
+    # for i in list(result):
+    #     device_ids = i.get("device_ids")
+    #     count = i.get("unique_count")
+    #     location = i.get("location")
+    #     print("地址：", location, "数量", count)
+    myclient.close()
+    return list(result)
+
+
+def nd_qxz_data(device_ids, start, end):
+    myclient = bigata_mongo()
+    mydb = myclient.smartfarming
+    mycol = mydb.sa_nd_qxz_data
+    query = {
+        "upl_time": {
+            "$gte": start,
+            "$lte": end
+        },
+        "device_id": {
+            "$in": device_ids
+        }
+    }
+    project = {
+        "_id": 0,
+        "temp": 1,
+        "swc": 1
+    }
+    nd_data = mycol.find(query,project)
+    myclient.close()
+    nd_data = list(nd_data) 
+    tmp_list = {}
+    if nd_data:
+        for k in nd_data: 
+            temp = k.get("temp")
+            swc = k.get("swc")
+            if temp:
+                temp = temp.split(",")
+                for index, ti in enumerate(temp): 
+                    try:
+                        ti = float(ti)
+                    except:
+                        ti = 0
+                    if "temp_" + str(index) in tmp_list:
+                        tmp_list["temp_" + str(index)].append(ti)
+                    else:
+                        tmp_list["temp_" + str(index)] = [ti]
+            if swc:
+                swc = swc.split(",")
+                for index, si in enumerate(swc):
+                    try:
+                        si = float(si)
+                    except:
+                        si = 0
+                    if "swc_" + str(index) in tmp_list:
+                        tmp_list["swc_" + str(index)].append(si)
+                    else:
+                        tmp_list["swc_" + str(index)] = [si]
+    avg_data = {}
+    for k, v in tmp_list.items():
+        avg_data[k] = round(sum(v) / len(v), 2)
+    return avg_data
+
+
+
+
+def main(start_y, end_y):
+    start_year = datetime(start_y, 1, 1, 0, 0, 0).timestamp()
+    end_year = datetime(end_y, 1, 1, 0, 0, 0).timestamp()
+    device_addr = device_aggrate()
+    for i in device_addr:
+        device_ids = i.get("device_ids")
+        location = i.get("location")
+        province = location.get("province")
+        city = location.get("city")
+        district = location.get("district")
+        print(district)
+        henan = HeNanAddr.objects.filter(province=province, city=city, district=district)
+        if henan.exists():
+            henan = henan.first() 
+            addr_code = henan.code
+            if addr_code:
+                # 时间 
+                for r in range(int(start_year), int(end_year), 86400):
+                    data = nd_qxz_data(device_ids, r, r + 86400)
+                    format_date = datetime.fromtimestamp(r).strftime("%Y-%m-%d")
+                    for k, v in data.items():
+                        if k.startswith("temp"): 
+                            k_lst = k.split("_")
+                            if len(k_lst) == 2:
+                                title = f"{(int(k_lst[1]) + 1) * 10}cm土壤温度"
+                                EnvTempHum.objects.create(
+                                    title = title,
+                                    value = v,
+                                    unit = "℃",
+                                    type = "1",
+                                    date = r,
+                                    format_date = format_date,
+                                    province = province,
+                                    city = city,
+                                    district = district,
+                                    addr_code = addr_code
+                                )
+                        elif k.startswith("swc"):
+                            k_lst = k.split("_")
+                            if len(k_lst) == 2:
+                                title = f"{(int(k_lst[1]) + 1) * 10}cm土壤湿度"
+                                EnvTempHum.objects.create(
+                                    title = title,
+                                    value = v,
+                                    unit = "%",
+                                    type = "2",
+                                    date = r,
+                                    format_date = format_date,
+                                    province = province,
+                                    city = city,
+                                    district = district,
+                                    addr_code = addr_code
+                                )
+
+
+if __name__ == '__main__':
+    main(2024, 2025)

uwsgi.ini

@@ -8,8 +8,8 @@ module=bigdataAPI.wsgi:application
 # 指定sock的文件路径
 socket=/run/socker/bigdata_api.sock
 # 进程个数
-workers=8
-threads=8
+workers=10
+threads=12
 pidfile=/run/pid/bigdata_api.pid
 # 指定IP端口
 # ;http=127.0.0.1:50005
@@ -29,11 +29,11 @@ enable-threads=true
 # 设置缓冲
 post-buffering=4096
 # 设置日志目录
-daemonize=/var/log/bigdata_api/uwsgi.log
+daemonize=/home/logs/bigdataAPI/uwsgi.log
 # 启用日志按日期分割
 log-date = %Y-%m-%d
 # 设置日志备份文件的名称格式
-log-backupname = /var/log/bigdata_api/uwsgi.{date}.log
+log-backupname = /home/logs/bigdataAPI/uwsgi.{date}.log
 # 设置日志保留的时间（7天）
 log-keep = 7