From 6f2050eb005a0dadeed2bad6968f81b4cbeca3d1 Mon Sep 17 00:00:00 2001 From: liucouhua Date: Mon, 28 Oct 2024 10:39:01 +0800 Subject: [PATCH] =?UTF-8?q?=E6=94=B9=E8=BF=9B=E4=BA=86=E9=80=8F=E8=A7=86?= =?UTF-8?q?=E5=88=86=E6=9E=90=E6=A8=A1=E5=9D=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. 在mem.vector.score模块中拓展了中间量na的内容，使它不止包含综合准确样本数，还包含风向准确数、风速准确数。 2. 在mem.vector.score模块中增加了基于中间量na计算风向准确率、风速准确率的方法。 3. 在mps.middle_prepare模块中增加了tase_angle, tase_angle_uv,选项支持基于中间量统计风向的误差。 4. 在mps.score_df函数中对地形下数据结果设置为nan，在mps.middle_prepare.middle_df_grd函数中增加了dropna,支持地形一下数据的过滤。 5. 在score_df函数中改进了相关系数、bs评分中间量的合并方法，提升了效率。 --- meteva/method/vector/score.py | 64 ++++++++++-- meteva/perspact/middle_prepare.py | 167 +++++++++++++++++------------- meteva/perspact/score.py | 57 +++++++--- 3 files changed, 190 insertions(+), 98 deletions(-) diff --git a/meteva/method/vector/score.py b/meteva/method/vector/score.py index 191005ba..95e74a18 100644 --- a/meteva/method/vector/score.py +++ b/meteva/method/vector/score.py @@ -392,11 +392,10 @@ def nasws_s(s_ob,s_fo,min_s = 0,max_s = 300): shape = list(Fo_shape[:ind]) shape.append(5) nass_array = nass_array.reshape(shape) + return nass_array - return nass_array - def nasws_uv(u_ob,u_fo,v_ob,v_fo,min_s = 0,max_s = 300): ''' @@ -611,8 +610,8 @@ def na_ds(d_ob,d_fo,s_ob,s_fo): :param s_ob:观测的风速，numpy数组，shape和d_ob完全一致 :param d_fo:预报的风向，numpy数组，numpy可以比d_ob高一维（用于同时进行多家预报结果检验），fo.shape低维与ob.shape保持一致 :param s_fo:预报的风速，numpy数组，numpy可以比d_ob高一维（用于同时进行多家预报结果检验），fo.shape低维与ob.shape保持一致 - :return:根据预报和观测的风向风速数据，统计得到的样本数、风向和风速都正确样本数，d_fo和d_ob的shape一致，则说明只有一家预报，此时返回 - 一维的数据，size= 2。 如果d_fo和d_ob的shape不一致，则说明有多个预报，则返回的数据shape = （预报成员数，2） + :return:根据预报和观测的风向风速数据，统计得到的样本数、风向和风速都正确样本数，风向正确的样本数，风速正确的样本数，d_fo和d_ob的shape一致，则说明只有一家预报，此时返回 + 一维的数据，size= 4。 如果d_fo和d_ob的shape不一致，则说明有多个预报，则返回的数据shape = （预报成员数，4） ''' Ob_shape = d_ob.shape Fo_shape = d_fo.shape @@ -636,14 +635,28 @@ def na_ds(d_ob,d_fo,s_ob,s_fo): for line in range(new_Fo_shape[0]): fo_d = meteva.base.tool.math_tools.tran_direction_to_8angle(new_Fo_d[line, :]) fo_s = meteva.base.tool.math_tools.tran_speed_to_14grade(new_Fo_s[line,:]) - nasd_array = np.zeros(2) + nasd_array = np.zeros(4) nasd_array[0] = d_ob.size + + #综合准确 index = np.where((fo_d == ob_d)&(fo_s == ob_s)) - nasd_array[1] += len(index[0]) + nasd_array[1] = len(index[0]) + + + #风向准确 + index = np.where(fo_d == ob_d) + nasd_array[2] = len(index[0]) + + #风速准确 + index = np.where(fo_s == ob_s) + nasd_array[3] = len(index[0]) + nasd_list.append(nasd_array) + + nrag_array = np.array(nasd_list) shape = list(Fo_shape[:ind]) - shape.append(2) + shape.append(4) nrag_array = nrag_array.reshape(shape) return nrag_array @@ -657,8 +670,8 @@ def na_uv(u_ob,u_fo,v_ob,v_fo): :param u_fo: 预报的u分量，numpy数组，shape和u_ob完全一致或比u_ob高一维（用于同时进行多家预报结果检验），u_fo.shape低维与u_ob.shape保持一致 :param v_ob: 观测的u分量，numpy数组，shape 和u_ob完全一致 :param v_fo: 预报的v分量，numpy数组，shape和u_ob完全一致或比u_ob高一维（用于同时进行多家预报结果检验），v_fo.shape低维与v_ob.shape保持一致 - :return:根据预报和观测的风向风速数据，统计得到的样本数、风向和风速都正确样本数，d_fo和d_ob的shape一致，则说明只有一家预报，此时返回 - 一维的数据，size= 2。 如果d_fo和d_ob的shape不一致，则说明有多个预报，则返回的数据shape = （预报成员数，2） + :return:根据预报和观测的风向风速数据，统计得到的样本数、风向和风速都正确样本数，风向正确的样本数，风速正确的样本数，d_fo和d_ob的shape一致，则说明只有一家预报，此时返回 + 一维的数据，size= 4。 如果d_fo和d_ob的shape不一致，则说明有多个预报，则返回的数据shape = （预报成员数，4） ''' s_ob,d_ob = meteva.base.math_tools.u_v_to_s_d(u_ob,v_ob) s_fo,d_fo = meteva.base.math_tools.u_v_to_s_d(u_fo,v_fo) @@ -676,6 +689,39 @@ def acz_na(na_array): ac = sc/total return ac +def acd_na(na_array): + ''' + 基于中间结果计算风向准确率， + :param nrag_array: 输入nrag函数统计得到的样本数、风向风速都正确的样本数 + :return: 返回风预报综合准确率 + ''' + total = na_array[...,0] + sc = na_array[...,2] + ac = sc/total + return ac + +def acs_na(na_array): + ''' + 基于中间结果计算风速准确率， + :param nrag_array: 输入nrag函数统计得到的样本数、风向风速都正确的样本数 + :return: 返回风预报综合准确率 + ''' + total = na_array[...,0] + sc = na_array[...,3] + ac = sc/total + return ac + +def acs_nacs(na_array): + ''' + 基于中间结果计算风速准确率， + :param nrag_array: 输入nrag函数统计得到的样本数、风向风速都正确的样本数 + :return: 返回风预报综合准确率 + ''' + total = na_array[...,0] + sc = na_array[...,1] + ac = sc/total + return ac + def acz(d_ob,d_fo,s_ob,s_fo): ''' 根据输入的观测和预报风向（0-360度）和风速（m/s），计算风雨吧综合准确率， diff --git a/meteva/perspact/middle_prepare.py b/meteva/perspact/middle_prepare.py index 19d046dd..2f1fc97c 100644 --- a/meteva/perspact/middle_prepare.py +++ b/meteva/perspact/middle_prepare.py @@ -12,15 +12,19 @@ "hfmc" : ["H","F","M","C"], "hfmc_multi": ["H", "F", "M", "C"], "tase":["T", "E", "A", "S"], + "tase_angle": ["T", "E_angle", "A_angle", "S_angle"], + "tase_angle_uv": ["T", "E_angle", "A_angle", "S_angle"], + "tasem":["T", "E", "A", "S","M"], "tc_count":["T", "C"], "tlfo":["T","LFO"], "toar":["T","OAR"], "tmmsss": ["T", "MX","MY","SX","SY","SXY"], "tbask":["T","BASK"], - "nasws_uv": ["T", "AC", "SCORE", "SEVERER", "WEAK"], - "nas_uv": ["T", "AC", "SCORE"], - "na_uv":["T","AC"], - "nasws_s": ["T", "AC", "SCORE", "SEVERER", "WEAK"], + "nasws_uv": ["T", "ACS", "SCORE", "SEVERER", "WEAK"], + "nas_uv": ["T", "ACS", "SCORE"], + "na_uv":["T","ACZ","ACD","ACS"], + "nasws_s": ["T", "ACS", "SPEEDSCORE", "SEVERER", "WEAK"], + "na_s":["T","ACS"], "fss":["pob","pfo","fbs"], "tems":["T","E","MX","SX"], "cscs":["CX","SX","CY","SY"], @@ -42,6 +46,8 @@ def get_score_method_with_mid(method): hfmc_of_sum_rain_list = [meteva.method.pc_of_sun_rain] tase_list = [meteva.method.me, meteva.method.mae, meteva.method.mse, meteva.method.rmse] + tase_angle_list = [meteva.method.me_angle, meteva.method.mae_angle, meteva.method.rmse_angle] + tasem_list = [meteva.method.ob_fo_mean] tc_list = [meteva.method.correct_rate,meteva.method.wrong_rate] tmmsss_list = [meteva.method.residual_error, meteva.method.residual_error_rate,meteva.method.corr, meteva.method.nse,meteva.method.ob_fo_mean,meteva.method.ob_fo_sum,meteva.method.ob_fo_std] @@ -50,7 +56,7 @@ def get_score_method_with_mid(method): meteva.method.wind_severer_rate] nasws_uv_list = [meteva.method.acs_uv,meteva.method.scs_uv,meteva.method.wind_weaker_rate_uv,meteva.method.wind_severer_rate_uv] nas_uv_list = [meteva.method.acd_uv,meteva.method.scd_uv] - na_uv_list = [meteva.method.acz_uv] + na_uv_list = [meteva.method.acz_uv,meteva.method.acd_uv,meteva.method.acs_uv] toar_list = [meteva.method.mre] tlfo_list = [meteva.method.rmsf] fss_list = [meteva.method.fss] @@ -58,46 +64,51 @@ def get_score_method_with_mid(method): hnh_list = [meteva.method.roc_auc] cscs_list = [meteva.method.ob_fo_precipitation_strength] - method_mid = None + method_mid = [] method_name = method.__name__ if method in hfmc_of_sum_rain_list: - method_mid = getattr(meteva.method, method_name + "_hfmc") - elif method in hfmc_list: - method_mid = getattr(meteva.method, method_name +"_hfmc") - elif method in hfmc_multi_list: - method_mid = getattr(meteva.method, method_name[:-6] +"_hfmc") - elif method in tase_list: - method_mid = getattr(meteva.method, method_name + "_tase") - elif method in tc_list: - method_mid = getattr(meteva.method, method_name + "_tc") - elif method in tmmsss_list: - method_mid = getattr(meteva.method, method_name + "_tmmsss") - elif method in toar_list: - method_mid = getattr(meteva.method, method_name + "_toar") - elif method in tlfo_list: - method_mid = getattr(meteva.method, method_name + "_tlfo") - elif method in tbask_list: - method_mid = getattr(meteva.product, method_name + "_tbask") - elif method in nasws_s_list: - method_mid = getattr(meteva.method, method_name + "_nasws") - elif method in tems_list: - method_mid = getattr(meteva.method, method_name + "_tems") - elif method in nasws_uv_list: + method_mid.append(getattr(meteva.method, method_name + "_hfmc")) + if method in hfmc_list: + method_mid.append(getattr(meteva.method, method_name +"_hfmc")) + if method in hfmc_multi_list: + method_mid.append(getattr(meteva.method, method_name[:-6] +"_hfmc")) + if method in tase_list: + method_mid.append(getattr(meteva.method, method_name + "_tase")) + if method in tase_angle_list: + method_name1 = method_name.split("_")[0] + method_mid.append(getattr(meteva.method, method_name1 + "_tase")) + if method in tc_list: + method_mid.append(getattr(meteva.method, method_name + "_tc")) + if method in tmmsss_list: + method_mid.append(getattr(meteva.method, method_name + "_tmmsss")) + if method in toar_list: + method_mid.append(getattr(meteva.method, method_name + "_toar")) + if method in tlfo_list: + method_mid.append(getattr(meteva.method, method_name + "_tlfo")) + if method in tbask_list: + method_mid.append(getattr(meteva.product, method_name + "_tbask")) + if method in nasws_s_list: + method_mid.append(getattr(meteva.method, method_name + "_nasws")) + if method in tems_list: + method_mid.append(getattr(meteva.method, method_name + "_tems")) + if method in nasws_uv_list: method_name1 = method_name.replace("_uv","_nasws") - method_mid = getattr(meteva.method, method_name1) - elif method in nas_uv_list: + method_mid.append(getattr(meteva.method, method_name1)) + if method in nas_uv_list: method_name1 = method_name.replace("_uv","_nas") - method_mid = getattr(meteva.method, method_name1) - elif method in na_uv_list: + method_mid.append(getattr(meteva.method, method_name1)) + if method in na_uv_list: method_name1 = method_name.replace("_uv","_na") - method_mid = getattr(meteva.method, method_name1) - elif method in fss_list: - method_mid = getattr(meteva.method, method_name + "_fof") - elif method in hnh_list: - method_mid = getattr(meteva.method, method_name + "_hnh") - elif method in cscs_list: - method_mid = getattr(meteva.method, method_name + "_cscs") + method_mid.append(getattr(meteva.method, method_name1)) + if method in fss_list: + method_mid.append(getattr(meteva.method, method_name + "_fof")) + if method in hnh_list: + method_mid.append(getattr(meteva.method, method_name + "_hnh")) + if method in cscs_list: + method_mid.append(getattr(meteva.method, method_name + "_cscs")) + if method in tasem_list: + method_mid.append(getattr(meteva.method,method_name+"_tasem")) #print(method_mid) return method_mid @@ -115,6 +126,8 @@ def get_middle_method(method): hfmc_of_sum_rain_list = [meteva.method.pc_of_sun_rain] tase_list = [meteva.method.me, meteva.method.mae, meteva.method.mse, meteva.method.rmse] + tase_angle_list = [meteva.method.me_angle, meteva.method.mae_angle, meteva.method.rmse_angle] + tasem_list = [meteva.method.ob_fo_mean] tc_list = [meteva.method.correct_rate,meteva.method.wrong_rate] tmmsss_list = [meteva.method.residual_error, meteva.method.residual_error_rate,meteva.method.corr,meteva.method.ob_fo_sum,meteva.method.ob_fo_mean, meteva.method.nse,meteva.method.ob_fo_std] @@ -124,46 +137,51 @@ def get_middle_method(method): nasws_s_list = [meteva.method.acs,meteva.method.scs,meteva.method.wind_weaker_rate,meteva.method.wind_severer_rate] nasws_uv_list = [meteva.method.acs_uv,meteva.method.scs_uv,meteva.method.wind_weaker_rate_uv,meteva.method.wind_severer_rate_uv] nas_uv_list = [meteva.method.acd_uv,meteva.method.scd_uv] - na_uv_list = [meteva.method.acz_uv] + na_uv_list = [meteva.method.acz_uv,meteva.method.acs_uv,meteva.method.acd_uv] fss_list = [meteva.method.fss] tems_list = [meteva.method.bs, meteva.method.bss] hnh_list = [meteva.method.roc_auc] cscs_list = [meteva.method.ob_fo_precipitation_strength] - method_mid = None + method_mid = [] + if method in hfmc_of_sum_rain_list: - method_mid = meteva.method.hfmc_of_sun_rain - elif method in hfmc_list: - method_mid = meteva.method.hfmc - elif method in hfmc_multi_list: - method_mid = meteva.method.hfmc_multi - elif method in tase_list: - method_mid = meteva.method.tase - elif method in tc_list: - method_mid = meteva.method.tc_count - elif method in tmmsss_list: - method_mid = meteva.method.tmmsss - elif method in toar_list: - method_mid = meteva.method.toar - elif method in tlfo_list: - method_mid = meteva.method.tlfo - elif method in tbask_list: - method_mid = meteva.product.regulation.temperature.tbask - elif method in nasws_s_list: - method_mid = meteva.method.nasws_s - elif method in nasws_uv_list: - method_mid = meteva.method.nasws_uv - elif method in nas_uv_list: - method_mid = meteva.method.nas_uv - elif method in na_uv_list: - method_mid = meteva.method.na_uv - elif method in fss_list: - method_mid = meteva.method.fss - elif method in tems_list: - method_mid = meteva.method.tems - elif method in hnh_list: - method_mid = meteva.method.hnh - elif method in cscs_list: - method_mid = meteva.method.cscs + method_mid.append(meteva.method.hfmc_of_sun_rain) + if method in hfmc_list: + method_mid.append(meteva.method.hfmc) + if method in hfmc_multi_list: + method_mid.append(meteva.method.hfmc_multi) + if method in tase_list: + method_mid.append(meteva.method.tase) + if method in tase_angle_list: + method_mid.append(meteva.method.tase_angle) + if method in tc_list: + method_mid.append(meteva.method.tc_count) + if method in tmmsss_list: + method_mid.append(meteva.method.tmmsss) + if method in toar_list: + method_mid.append(meteva.method.toar) + if method in tlfo_list: + method_mid.append(meteva.method.tlfo) + if method in tbask_list: + method_mid.append(meteva.product.regulation.temperature.tbask) + if method in nasws_s_list: + method_mid.append(meteva.method.nasws_s) + if method in nasws_uv_list: + method_mid.append(meteva.method.nasws_uv) + if method in nas_uv_list: + method_mid.append(meteva.method.nas_uv) + if method in na_uv_list: + method_mid.append(meteva.method.na_uv) + if method in fss_list: + method_mid.append(meteva.method.fss) + if method in tems_list: + method_mid.append(meteva.method.tems) + if method in hnh_list: + method_mid.append(meteva.method.hnh) + if method in cscs_list: + method_mid.append(meteva.method.cscs) + if method in tasem_list: + method_mid.append(meteva.method.tasem) return method_mid def get_middle_columns(method): @@ -473,6 +491,7 @@ def middle_df_grd(grd_ob, grd_fo, method, grade_list=None, compare=None, marker= squere_error = (np.square(grd_ob.values - grd_fo.values) * grd_weight.values).flatten() mean_obs = (grd_ob.values* grd_weight.values).flatten() df = pd.DataFrame( {"T":total_count,"E":error,"A":abs_error,"S":squere_error,"M":mean_obs,"id":marker.values.flatten()}) + df.dropna(inplace=True) # 使用 agg 方法进行聚合计算 result_df = df.groupby('id').agg({ 'T': 'sum', # 权重和 diff --git a/meteva/perspact/score.py b/meteva/perspact/score.py index a1bec27d..23db350f 100644 --- a/meteva/perspact/score.py +++ b/meteva/perspact/score.py @@ -7,6 +7,7 @@ import copy + # 实现任意纬度分类的函数 def score_df(df, method, s = None,g=None,gll_dict = None,plot = None,excel_path = None,first = True,**kwargs): ''' @@ -23,16 +24,32 @@ def score_df(df, method, s = None,g=None,gll_dict = None,plot = None,excel_path :return: ''' method_name = method.__name__ + method_mid_list = meteva.perspact.get_middle_method(method) + method_mid = None + column_df = df.columns + column_list_all = [] + for method_mid1 in method_mid_list: + column_list = meteva.perspact.get_middle_columns(method_mid1) + if set(column_list) <= set(column_df): + method_mid = method_mid1 + else: + column_list_all.append(column_list) + if method_mid is None: + print("input pandas.DataFrame must contains columns in list of " + str( + column_list_all) + " for mem." + method_name + " caculation") + + return None, None + + score_method_with_mid_list = meteva.perspact.get_score_method_with_mid(method) + score_method_with_mid = None + for mid in score_method_with_mid_list: + mid_name = method_mid.__name__.split("_")[0] + if mid.__name__.find(mid_name)>0: + score_method_with_mid = mid - method_mid = meteva.perspact.get_middle_method(method) - column_list = meteva.perspact.get_middle_columns(method_mid) - score_method_with_mid = meteva.perspact.get_score_method_with_mid(method) df0 = meteva.base.sta_data(df) - column_df = df.columns - if not set(column_list) <= set(column_df): - print("input pandas.DataFrame must contains columns in list of "+ str(column_list) + " for mem." + method_name + " caculation") - return None,None + if s is not None: if "member" in s.keys(): @@ -62,6 +79,8 @@ def score_df(df, method, s = None,g=None,gll_dict = None,plot = None,excel_path df1 = meteva.base.sele_by_dict(df0, s) + + if len(df1.index) ==0: print("所选参数范围内没有检验数据") return None,None @@ -81,6 +100,12 @@ def score_df(df, method, s = None,g=None,gll_dict = None,plot = None,excel_path df_ob["CY"] = df1["CX"] df_ob["SY"] = df1["SX"] pass + if method_mid.__name__ =="tasem": + try: + df1["M"] = df1["M"] + df1["E"] + except: + print("中间数据中未包含M 列") + df_ob.drop_duplicates(subset=None, keep='first', inplace=True) df1 = pd.concat([df_ob,df1],axis=0) if g is None: @@ -163,15 +188,17 @@ def score_df(df, method, s = None,g=None,gll_dict = None,plot = None,excel_path for i in range(len(df1_list)): if method_mid == meteva.method.tmmsss: tmmsss_array = df1_list[i][column_list].values - mid_array = tmmsss_array[0, :] - for j in range(1, tmmsss_array.shape[0]): - mid_array = meteva.method.tmmsss_merge(mid_array, tmmsss_array[j, :]) + mid_array = meteva.method.tmmsss_merge_all(tmmsss_array) + # mid_array = tmmsss_array[0, :] + # for j in range(1, tmmsss_array.shape[0]): + # mid_array = meteva.method.tmmsss_merge(mid_array, tmmsss_array[j, :]) elif method_mid == meteva.method.tems: tems_array = df1_list[i][column_list].values - mid_array = tems_array[0, :] - for j in range(1, tems_array.shape[0]): - mid_array = meteva.method.tems_merge(mid_array, tems_array[j, :]) + mid_array = meteva.method.tems_merge_all(tems_array) + # mid_array = tems_array[0, :] + # for j in range(1, tems_array.shape[0]): + # mid_array = meteva.method.tems_merge(mid_array, tems_array[j, :]) else: mid_list = [] for column in column_list: @@ -970,7 +997,7 @@ def score_xy_df(df_mid,method,s = None,g = None,gll_dict = None,save_path = None grd_list_plot = [] for i in range(len(grd_list)): score_grd = grd_list[i].copy() - score_grd.values[score_grd.values == meteva.base.IV] = 0 + score_grd.values[score_grd.values == meteva.base.IV] = np.nan grd_list_plot.append(score_grd) if gnames0 is None: title_list = None @@ -1009,7 +1036,7 @@ def score_xy_df_delta(df_mid,method,reference_member,s = None,gll_dict = None,sa grd_list_plot = [] for i in range(len(grd_list)): score_grd = grd_list[i].copy() - score_grd.values[score_grd.values == meteva.base.IV] = 0 + score_grd.values[score_grd.values == meteva.base.IV] = np.nan grd_list_plot.append(score_grd) title_list = []