Introduced analysis for band, load agents of swarm as multi-level dat…

…aframe, and multiple condition files in a session
chiyu1203 · Oct 19, 2024 · 302d64d · 302d64d
1 parent ec7ee81
commit 302d64d
Show file tree

Hide file tree

Showing 5 changed files with 439 additions and 291 deletions.
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -9,7 +9,10 @@
             "type": "debugpy",
             "request": "launch",
             "program": "${file}",
-            "console": "integratedTerminal"
+            "console": "integratedTerminal",
+            "env": {
+                "PYDEVD_WARN_SLOW_RESOLVE_TIMEOUT": "3"
+            }
         }
     ]
 }
diff --git a/analysis_methods_dictionary.json b/analysis_methods_dictionary.json
@@ -1,29 +1,30 @@
 {
- "experiment_name": "swarm",
- "overwrite_curated_dataset": true,
- "save_output": false,
- "time_series_analysis": true,
- "filtering_method": "sg_filter",
- "plotting_trajectory": true,
- "load_individual_data": true,
- "select_animals_by_condition": true,
- "graph_colour_code": [
-  "r",
-  "b",
-  "g",
-  "k",
-  "c",
-  "y",
-  "m",
-  "r"
- ],
- "camera_fps": 100,
- "trackball_radius_cm": 0.5,
- "monitor_fps": 60,
- "body_length": 4,
- "growth_condition": "G",
- "analysis_window": [
-  -10,
-  10
- ]
+    "experiment_name": "band",
+    "overwrite_curated_dataset": true,
+    "save_output": true,
+    "agents_shared_across_vrs": false,
+    "time_series_analysis": false,
+    "filtering_method": "sg_filter",
+    "plotting_trajectory": true,
+    "load_individual_data": true,
+    "select_animals_by_condition": true,
+    "graph_colour_code": [
+        "r",
+        "b",
+        "g",
+        "k",
+        "c",
+        "y",
+        "m",
+        "r"
+    ],
+    "camera_fps": 100,
+    "trackball_radius_cm": 0.5,
+    "monitor_fps": 60,
+    "body_length": 4,
+    "growth_condition": "G",
+    "analysis_window": [
+        -10,
+        10
+    ]
 }
diff --git a/create_analysis_method.py b/create_analysis_method.py
@@ -2,10 +2,11 @@
 
 file_name = "analysis_methods_dictionary.json"
 analysis_methods = {
-    "experiment_name": "swarm",
+    "experiment_name": "band",
     "overwrite_curated_dataset": True,
-    "save_output": False,
-    "time_series_analysis": True,
+    "save_output": True,
+    "agents_shared_across_vrs": False,
+    "time_series_analysis": False,
     "filtering_method": "sg_filter",
     "plotting_trajectory": True,
     "load_individual_data": True,

diff --git a/data_exploration_tutorial.ipynb b/data_exploration_tutorial.ipynb
@@ -67,9 +67,10 @@
     "    \n",
     "\n",
     "#Put the folder of your Unity experiment below\n",
-    "thisDataset =\"D:/MatrexVR_Swarm_Data/RunData\"\n",
+    "#thisDataset =\"D:/MatrexVR_Swarm_Data/RunData\"\n",
     "#thisDataset =\"D:/MatrexVR_blackbackground_Data/RunData\"\n",
     "#thisDataset =\"D:/MatrexVR_grass1_Data/RunData\"\n",
+    "thisDataset =\"D:/MatrexVR_navigation_Data/RunData\"\n",
     "#parameter name means independent variable in the experiment\n",
     "#parameter_name='kappa' \n",
     "parameter_name='mu'\n",
@@ -105,13 +106,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 26,
+   "execution_count": 4,
    "metadata": {},
    "outputs": [],
    "source": [
     "##This cell is used to move data of the thermo-humidity logger to animals'folder\n",
     "import shutil\n",
-    "tmp_file_name='matrexVR240824-240901.txt'\n",
+    "#tmp_file_name='matrexVR240824-240901.txt'\n",
+    "tmp_file_name='DL220THP_Thermo2_241012_241014.csv'\n",
     "tmp_source=os.path.join('Z:/Users/chiyu',tmp_file_name)\n",
     "for this_dir in dir_list:\n",
     "    tmp_new_dir = os.path.join(this_dir,tmp_file_name)\n",
@@ -201,15 +203,6 @@
     "                dir_list.append(folder_path.replace(\"\\\\\", \"/\"))"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "dir_list"
-   ]
-  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -274,7 +267,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 44,
+   "execution_count": 11,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -295,6 +288,9 @@
     "        print(\"there is a bug\")\n",
     "        return df_all,dfxy_all\n",
     "    for this_dir,this_vr in dir_iterator:\n",
+    "        if Path(this_dir).is_dir()==False:\n",
+    "            print(f'no such a dir exist {this_dir}')\n",
+    "            continue\n",
     "        summary_pattern = f\"VR{this_vr}*score.h5\"\n",
     "        xy_pattern = f\"VR{this_vr}*XY.h5\"\n",
     "        found_result = find_file(Path(this_dir), summary_pattern)        \n",
@@ -315,6 +311,8 @@
     "                color_code={0: 0.1,45: 0.4,315: 0.7}\n",
     "            elif scene_name.lower()=='swarm':\n",
     "                color_code={0: 0.1, 45: 0.2, 90: 0.3,135:0.4,180: 0.5, 225: 0.6, 270: 0.7,315:0.8}\n",
+    "            elif scene_name.lower()=='band':\n",
+    "                color_code={0: 0.1, 45: 0.2, 90: 0.3, 270: 0.7,315:0.8}\n",
     "            else:\n",
     "                return Warning('scene name not found')\n",
     "        elif test_parameter == 'agent_speed':\n",
@@ -355,23 +353,23 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 19,
    "metadata": {},
    "outputs": [],
    "source": [
     "#introduce customised functions\n",
     "def plot_sercansincos(df,analysis_methods,parameters,parameter_name,vr_num='all'):\n",
-    "    dont_save_output= analysis_methods.get(\"dont_save_output\")\n",
+    "    save_output= analysis_methods.get(\"save_output\")\n",
     "    scene_name=analysis_methods.get(\"experiment_name\")\n",
     "    cos = df[\"cos\"]\n",
     "    sin = df[\"sin\"]\n",
     "    if 'density' in df.columns:\n",
     "        density=df[\"density\"].unique()[0]\n",
-    "        cos_fig_name=f\"{vr_num}_cos_{parameter_name}_{parameters}_density_{int(density)}.svg\"\n",
-    "        sin_fig_name=f\"{vr_num}_sin_{parameter_name}_{parameters}_density_{int(density)}.svg\" \n",
+    "        cos_fig_name=f\"{vr_num}_cos_{scene_name}_{parameter_name}_{parameters}_density_{int(density)}.svg\"\n",
+    "        sin_fig_name=f\"{vr_num}_sin_{scene_name}_{parameter_name}_{parameters}_density_{int(density)}.svg\" \n",
     "    elif scene_name=='choice':\n",
-    "        cos_fig_name=f\"{vr_num}_cos_{parameter_name}_{parameters}_single_target_{df['object_type'].values[0]}.svg\"\n",
-    "        sin_fig_name=f\"{vr_num}_sin_{parameter_name}_{parameters}_single_target_{df['object_type'].values[0]}.svg\"\n",
+    "        cos_fig_name=f\"{vr_num}_cos_{scene_name}_{parameter_name}_{parameters}_single_target_{df['object_type'].values[0]}.svg\"\n",
+    "        sin_fig_name=f\"{vr_num}_sin_{scene_name}_{parameter_name}_{parameters}_single_target_{df['object_type'].values[0]}.svg\"\n",
     "\n",
     "    fig, ax = plt.subplots(dpi=300, figsize=(1.1,0.25))\n",
     "    #plt.rcParams.update(plt.rcParamsDefault)\n",
@@ -392,7 +390,7 @@
     "    ax.set_yticks([])\n",
     "    plt.ylabel(\"\")\n",
     "    plt.xlabel(\"\")    \n",
-    "    if dont_save_output==False:\n",
+    "    if save_output==True:\n",
     "        plt.savefig(cos_fig_name)\n",
     "    fig, ax = plt.subplots(dpi=300, figsize=(1.1,0.25))\n",
     "    plt.subplots_adjust(bottom=0.4)\n",
@@ -406,21 +404,21 @@
     "    plt.ylabel(\"\")\n",
     "    plt.xlabel(\"\")\n",
     "    plt.title(\"r sin\\u03F4\")\n",
-    "    if dont_save_output==False:\n",
+    "    if save_output==True:\n",
     "        plt.savefig(sin_fig_name)\n",
     "    plt.show()\n",
     "def plot_sercantrajec(dfXY,analysis_methods,parameters,parameter_name,trajec_lim=1000,vr_num='all'):\n",
-    "    dont_save_output= analysis_methods.get(\"dont_save_output\")\n",
+    "    save_output= analysis_methods.get(\"save_output\")\n",
     "    scene_name=analysis_methods.get(\"experiment_name\")\n",
     "    # dfXY[parameter_name].unique()\n",
     "    \n",
     "    a = dfXY.groupby('VR')\n",
     "    if 'density' in dfXY.columns:\n",
     "        density=dfXY[\"density\"].unique()[0]\n",
     "        print(density)\n",
-    "        fig_name=f\"{vr_num}_summary_trajectory_{parameter_name}_{parameters}_density_{int(density)}.png\"\n",
+    "        fig_name=f\"{vr_num}_summary_trajectory_{scene_name}_{parameter_name}_{parameters}_density_{int(density)}.png\"\n",
     "    else:\n",
-    "        fig_name=f\"{vr_num}_summary_trajectory_{parameter_name}_{parameters}_single_target_{dfXY['object_type'].values[0]}.png\"\n",
+    "        fig_name=f\"{vr_num}_summary_trajectory_{scene_name}_{parameter_name}_{parameters}_single_target_{dfXY['object_type'].values[0]}.png\"\n",
     "\n",
     "    fig, ax = plt.subplots(figsize=(3,3), dpi=300) \n",
     "    plt.rcParams.update(plt.rcParamsDefault)\n",
@@ -440,15 +438,15 @@
     "        # Here plot agent's trajectory with hardcode parameters\n",
     "    if scene_name=='choice' and dfXY['object_type'].values[0] !='empty_trial':\n",
     "        agent_speed=2\n",
-    "        initial_distance=8\n",
+    "        radial_distance=8\n",
     "        duration=60\n",
     "        travel_direction=parameters*-np.pi/180#the radian circle is clockwise in Unity, so 45 degree should be used as -45 degree in the regular radian circle\n",
-    "        initial_distance_b=np.cos(travel_direction)*initial_distance\n",
+    "        radial_distance_b=np.cos(travel_direction)*radial_distance\n",
     "        delta_cos=np.cumsum(np.repeat(np.cos(travel_direction)*agent_speed, duration))\n",
-    "        agent_cos=initial_distance_b+delta_cos\n",
-    "        initial_distance_b=np.sin(travel_direction)*initial_distance\n",
+    "        agent_cos=radial_distance_b+delta_cos\n",
+    "        radial_distance_b=np.sin(travel_direction)*radial_distance\n",
     "        delta_sin=np.cumsum(np.repeat(np.sin(travel_direction)*agent_speed, duration))\n",
-    "        agent_sin=initial_distance_b+delta_sin\n",
+    "        agent_sin=radial_distance_b+delta_sin\n",
     "        plt.plot(agent_cos, agent_sin, color='k', linewidth=1)\n",
     "\n",
     "    plt.xlim(-1*trajec_lim, trajec_lim)\n",
@@ -457,7 +455,7 @@
     "    plt.xticks([-1*trajec_lim, 0, trajec_lim])\n",
     "        # Set the aspect ratio to be equal\n",
     "    plt.gca().set_aspect('equal')                                                  \n",
-    "    if dont_save_output==False:    \n",
+    "    if save_output==True:    \n",
     "        plt.savefig(fig_name)\n",
     "    plt.show()"
    ]
@@ -471,7 +469,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 64,
+   "execution_count": 14,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -550,7 +548,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 66,
+   "execution_count": 16,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -562,6 +560,15 @@
     "good_tracking=df_con['loss'] < 0.05"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_con"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -578,17 +585,17 @@
     "#plotting trajectory\n",
     "#differentiate between stim and ISI based on columns in dfxy_con\n",
     "if analysis_methods.get(\"experiment_name\")==\"choice\":\n",
-    "    stim_or_isi=dfxy_con['initial_distance']\n",
-    "elif analysis_methods.get(\"experiment_name\")==\"swarm\":\n",
+    "    stim_or_isi=dfxy_con['radial_distance']\n",
+    "elif analysis_methods.get(\"experiment_name\")==\"swarm\" or analysis_methods.get(\"experiment_name\")==\"band\":\n",
     "    stim_or_isi=dfxy_con['density']\n",
     "df_stim=dfxy_con.loc[(dfxy_con['VR'].isin(df_con[\"VR\"][good_tracking])) & (stim_or_isi>0)]\n",
     "for key, grp in df_stim.groupby(parameter_name):\n",
     "    print(f\"{parameter_name}:{key}\")\n",
-    "    plot_sercantrajec(grp,analysis_methods,key,parameter_name,2000)\n",
+    "    plot_sercantrajec(grp,analysis_methods,key,parameter_name,500)\n",
     "df_isi=dfxy_con.loc[(dfxy_con['VR'].isin(df_con[\"VR\"][good_tracking])) & (stim_or_isi==0)]\n",
     "for key, grp in df_isi.groupby(parameter_name):\n",
     "    print(f\"{parameter_name}:{key}\")\n",
-    "    plot_sercantrajec(grp,analysis_methods,key,parameter_name,2000)\n",
+    "    plot_sercantrajec(grp,analysis_methods,key,parameter_name,500)\n",
     "#xy_lim at around 2000 is good for trial lasts around 4 or 5 min\n",
     "#xy_lim at around 500 is good for trial lasts around 1 min"
    ]
@@ -609,8 +616,8 @@
     "#Visualise the distribution of  mean angle using seaborn kernel density estimation plot\n",
     "#differentiate between stim and ISI based on columns in df_con\n",
     "if analysis_methods.get(\"experiment_name\")==\"choice\":\n",
-    "    stim_or_isi=df_con['initial_distance']\n",
-    "elif analysis_methods.get(\"experiment_name\")==\"swarm\":\n",
+    "    stim_or_isi=df_con['radial_distance']\n",
+    "elif analysis_methods.get(\"experiment_name\")==\"swarm\" or analysis_methods.get(\"experiment_name\")==\"band\":\n",
     "    stim_or_isi=df_con['density']\n",
     "df_stim=df_con[stim_or_isi>0]\n",
     "for key, grp in df_stim.groupby(parameter_name):\n",
@@ -650,7 +657,7 @@
     "    found_result = find_file(Path(this_dir), locust_pattern)        \n",
     "    df = pd.read_hdf(found_result)\n",
     "    if analysis_methods.get(\"experiment_name\")==\"choice\":\n",
-    "        stim_or_isi=df['initial_distance']\n",
+    "        stim_or_isi=df['radial_distance']\n",
     "    elif analysis_methods.get(\"experiment_name\")==\"swarm\":\n",
     "        stim_or_isi=df['density']\n",
     "    df_stim=df[stim_or_isi>0]\n",