process_and_send_data_details.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "02598747",
   "metadata": {},
   "source": [
    "# Traitement des données de financements "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "a59f7608",
   "metadata": {},
   "outputs": [],
   "source": [
    "#importer les packages\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from project.server.main.utils import replace_all,get_scanR_structure,get_id\n",
    "from project.server.main.cached_data_handler import get_structure, get_person\n",
    "from project.server.main.features_into_dictionnary import persons, projects, address\n",
    "from tqdm import tqdm\n",
    "import pprint as pp\n",
    "tqdm.pandas()\n",
    "from project.server.main.pickle import load_cache,write_cache\n",
    "import os\n",
    "from dotenv import load_dotenv\n",
    "import requests\n",
    "\n",
    "load_dotenv()\n",
    "\n",
    "Authorization = os.getenv('Authorization_access_185')\n",
    "Authorization_ORCID = os.getenv('Authorization_cluster_BSO_ORCID')\n",
    "url_cluster = os.getenv('url_cluster')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "b60621a0",
   "metadata": {},
   "outputs": [],
   "source": [
    "sources=pd.read_json('sources.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "95e2a545",
   "metadata": {},
   "outputs": [],
   "source": [
    "source=list(sources.keys())[1]\n",
    "source"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7fad3e27",
   "metadata": {},
   "outputs": [],
   "source": [
    "#cache structures, personnes et orcid avec differentes sources de donnees\n",
    "cached_data = {}\n",
    "try:\n",
    "    cached_data = load_cache(cached_data,f\"./DATA/{source}/caches/cached_{source.lower()}_data.pkl\")\n",
    "except:\n",
    "    write_cache(cached_data,f\"./DATA/{source}/caches/cached_{source.lower()}_data.pkl\")\n",
    "    \n",
    "cached_data_persons = {}\n",
    "try:\n",
    "    cached_data_persons = load_cache(cached_data_persons,f\"./DATA/{source}/caches/cached_{source.lower()}_data_persons.pkl\")\n",
    "except:\n",
    "    write_cache(cached_data_persons,f\"./DATA/{source}/caches/cached_{source.lower()}_data_persons.pkl\")\n",
    "    \n",
    "cached_data_orcid = {}\n",
    "try:\n",
    "    cached_data_orcid = load_cache(cached_data_orcid,f\"./DATA/{source}/caches/cached_{source.lower()}_data_orcid.pkl\")\n",
    "except:\n",
    "    write_cache(cached_data_orcid,f\"./DATA/{source}/caches/cached_{source.lower()}_data_orcid.pkl\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6bcfb200",
   "metadata": {},
   "source": [
    "# Données partenaires"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "3c82fbd0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# amener les partenaires depuis le site \n",
    "if source=='ANR':\n",
    "    page_partenaires_10 = requests.get(sources[source]['url_partners']).json()\n",
    "    colonnes_partenaires_10 = page_partenaires_10['columns']\n",
    "    donnees_partenaires_10 = page_partenaires_10['data']\n",
    "    df_partenaires=pd.DataFrame(data=donnees_partenaires_10,columns=colonnes_partenaires_10)\n",
    "elif source=='ANSES':\n",
    "    df_from_anses=pd.read_excel(sources[source]['url_partners'])\n",
    "    df=df_from_anses.iloc[1:,:]\n",
    "    df.columns=list(df_from_anses.iloc[0,:])\n",
    "    dict_equipe={list(df_from_anses.columns)[k].replace('Équipe 10 ','Équipe 10').replace('Équipe13','Équipe 13'):k for k in range (len(list(df_from_anses.columns))) if list(df_from_anses.columns)[k].find('Équipe')>=0}\n",
    "    list_df=[]\n",
    "    number=3\n",
    "    for n in range(1,len(dict_equipe)+1):\n",
    "        equipe_n=pd.concat([df.iloc[:,0:3],df.iloc[:,number:number+6]], axis=1)\n",
    "        list_df.append(equipe_n)\n",
    "        number+=6\n",
    "    df_partenaires=pd.concat([list_df[k].dropna(subset=[sources[source]['nom'], sources[source]['prenom'],sources[source]['nom_structure'], sources[source]['nom'], 'Pays'], how='all') for k in range(len(list_df))])\n",
    "    \n",
    "elif source=='IRESP':\n",
    "    df_partenaires1=pd.read_csv(sources[source]['url_partners1'] ,sep=\";\", encoding='UTF-8')\n",
    "    df_partenaires2=pd.read_csv(sources[source]['url_partners2'] ,sep=\";\", encoding='UTF-8')\n",
    "    df_partenaires=pd.concat([df_partenaires1,df_partenaires2])\n",
    "elif source=='ADEME':\n",
    "    df_partenaires=pd.read_csv(sources[source]['url_partners'] ,sep=\",\", encoding='ISO-8859-1', on_bad_lines='skip')\n",
    "else:    \n",
    "    df_partenaires=pd.read_csv(sources[source]['url_partners'] ,sep=\";\", encoding='ISO-8859-1')\n",
    "\n",
    "df_partenaires=df_partenaires.reset_index()\n",
    "del df_partenaires['index']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "230e93cc",
   "metadata": {},
   "source": [
    "# Matcher établissement"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "eca75cf4",
   "metadata": {},
   "outputs": [],
   "source": [
    "id_struct=df_partenaires\n",
    "id_struct[f\"{sources[source]['nom_structure']}2\"]=id_struct.loc[:,sources[source]['nom_structure']].apply(lambda x: replace_all(str(x).lower().replace(\" d e\",\" d'e\").replace(\" d a\",\" d'a\").replace(\" d i\",\" d'i\").replace(\" d o\",\" d'o\").replace(\" d u\",\" d'u\").replace(\" d y\",\" d'y\").replace(\" d h\",\" d'h\").replace(\" l e\",\" l'e\").replace(\" l a\",\" l'a\").replace(\" l i\",\" l'i\").replace(\" l o\",\" l'o\").replace(\" l u\",\" l'u\").replace(\" l y\",\" l'y\").replace(\" l h\",\" l'h\")))\n",
    "id_struct=id_struct.drop_duplicates(subset=[f\"{sources[source]['nom_structure']}2\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cc0b7e40",
   "metadata": {},
   "outputs": [],
   "source": [
    "id_struct.progress_apply(lambda row: get_structure(row,source,cached_data,sources[source]['nom_structure'],sources[source]['ville'],sources[source]['pays'],sources[source]['code_projet'],False), axis=1) \n",
    "write_cache(cached_data,f\"./DATA/{source}/caches/cached_{source}_data.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "84ae48b2",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(cached_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "feeebfff",
   "metadata": {},
   "outputs": [],
   "source": [
    "id_struct['id_structure_matcher']=id_struct.loc[:,sources[source]['nom_structure']].apply(lambda x: cached_data[x])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0a6033b7",
   "metadata": {},
   "outputs": [],
   "source": [
    "id_struct"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "d968cf8c",
   "metadata": {},
   "outputs": [],
   "source": [
    "id_struct=id_struct.reset_index()\n",
    "del id_struct['index']\n",
    "id_struct.to_excel(f\"./DATA/{source}/df_partenaires.xlsx\")\n",
    "id_struct.to_json(f\"./DATA/{source}/df_partenaires.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5f3afeca",
   "metadata": {},
   "outputs": [],
   "source": [
    "id_struct=pd.read_json(f\"./DATA/{source}/df_partenaires.json\")\n",
    "id_struct=id_struct[[sources[source]['nom_structure'],'id_structure_matcher']]\n",
    "id_struct[f\"{sources[source]['nom_structure']}2\"]=id_struct.loc[:,sources[source]['nom_structure']].apply(lambda x: replace_all(str(x).lower().replace(\" d e\",\" d'e\").replace(\" d a\",\" d'a\").replace(\" d i\",\" d'i\").replace(\" d o\",\" d'o\").replace(\" d u\",\" d'u\").replace(\" d y\",\" d'y\").replace(\" d h\",\" d'h\").replace(\" l e\",\" l'e\").replace(\" l a\",\" l'a\").replace(\" l i\",\" l'i\").replace(\" l o\",\" l'o\").replace(\" l u\",\" l'u\").replace(\" l y\",\" l'y\").replace(\" l h\",\" l'h\")))\n",
    "\n",
    "df_partenaires[f\"{sources[source]['nom_structure']}2\"]=df_partenaires.loc[:,sources[source]['nom_structure']].apply(lambda x: replace_all(str(x).lower().replace(\" d e\",\" d'e\").replace(\" d a\",\" d'a\").replace(\" d i\",\" d'i\").replace(\" d o\",\" d'o\").replace(\" d u\",\" d'u\").replace(\" d y\",\" d'y\").replace(\" d h\",\" d'h\").replace(\" l e\",\" l'e\").replace(\" l a\",\" l'a\").replace(\" l i\",\" l'i\").replace(\" l o\",\" l'o\").replace(\" l u\",\" l'u\").replace(\" l y\",\" l'y\").replace(\" l h\",\" l'h\")))\n",
    "df_partenaires_struct=pd.merge(df_partenaires,id_struct[[f\"{sources[source]['nom_structure']}2\",'id_structure_matcher']], on=f\"{sources[source]['nom_structure']}2\", how='left')\n",
    "df_partenaires_struct"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d5f9b64f",
   "metadata": {},
   "outputs": [],
   "source": [
    "#compléter les données avec scanR\n",
    "url_scanr='https://storage.gra.cloud.ovh.net/v1/AUTH_32c5d10cb0fe4519b957064a111717e3/scanR/projects.json'\n",
    "requete_scanR = requests.get(url_scanr)\n",
    "page_scanR= requete_scanR.json()\n",
    "df_scanR=pd.DataFrame(page_scanR)\n",
    "scanR=df_scanR.explode('participants').loc[:,['id','participants']]\n",
    "scanR=scanR.rename(columns={'id':'id_anr'})\n",
    "scanR['index']=[x for x in range(len(scanR))]\n",
    "scanR=scanR.set_index('index')\n",
    "scanR['id_structure_scanr']=scanR['participants'].apply(lambda x: x.get(str('structure')) if isinstance(x, dict) else None )\n",
    "scanR['nom_struct']=scanR['participants'].apply(lambda x: get_scanR_structure(x))\n",
    "del scanR['participants']\n",
    "scanR_nettoye=scanR.drop_duplicates(subset='nom_struct')\n",
    "scanR_nettoye[f\"{sources[source]['nom_structure']}2\"]=scanR_nettoye.loc[:,'nom_struct'].apply(lambda x: replace_all(str(x).lower()))\n",
    "scanR_nettoye=scanR_nettoye[['id_structure_scanr',f\"{sources[source]['nom_structure']}2\"]]\n",
    "scanR_nettoye=scanR_nettoye.drop_duplicates(subset=f\"{sources[source]['nom_structure']}2\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ed71b425",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_partenaires_struct=pd.merge(df_partenaires_struct,scanR_nettoye, on=f\"{sources[source]['nom_structure']}2\", how='left')\n",
    "df_partenaires_struct"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "89dbba33",
   "metadata": {},
   "outputs": [],
   "source": [
    "#######fichier avec les identifiants structures rettrouvés à la main par Emmanuel ==> 'code'\n",
    "scanr_structures=pd.read_excel('scanr_partenaires_non_identifies.xlsx')\n",
    "scanr_structures[f\"{sources[source]['nom_structure']}2\"]=scanr_structures.loc[:,'Nom'].apply(lambda x: replace_all(str(x).lower().replace(\" d e\",\" d'e\").replace(\" d a\",\" d'a\").replace(\" d i\",\" d'i\").replace(\" d o\",\" d'o\").replace(\" d u\",\" d'u\").replace(\" d y\",\" d'y\").replace(\" d h\",\" d'h\").replace(\" l e\",\" l'e\").replace(\" l a\",\" l'a\").replace(\" l i\",\" l'i\").replace(\" l o\",\" l'o\").replace(\" l u\",\" l'u\").replace(\" l y\",\" l'y\").replace(\" l h\",\" l'h\")))\n",
    "scanr_structures=scanr_structures[[f\"{sources[source]['nom_structure']}2\",'code']]\n",
    "scanr_structures=scanr_structures.dropna().drop_duplicates(subset=f\"{sources[source]['nom_structure']}2\")\n",
    "df_partenaires_complet=pd.merge(df_partenaires_struct,scanr_structures, on=f\"{sources[source]['nom_structure']}2\", how='left')\n",
    "df_partenaires_complet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "03c18c20",
   "metadata": {},
   "outputs": [],
   "source": [
    "if 'finess' in list(df_partenaires.columns):\n",
    "    finess_siret=pd.read_json(\"finess_siret-siege.json\")\n",
    "    df_partenaires_complet=pd.merge(df_partenaires_complet,finess_siret,how='left', on='finess')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f8c8d110",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_partenaires_complet[sources[source]['identifiants_preferes_structure']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a67832c4",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_partenaires_complet.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "744c95bb",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_partenaires_complet['id_structure']=df_partenaires_complet.apply(lambda row: get_id(row,sources[source]['identifiants_preferes_structure']), axis=1)\n",
    "df_partenaires_complet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "18b49f3a",
   "metadata": {},
   "outputs": [],
   "source": [
    "#df_partenaires_complet[pd.isna(df_partenaires_complet.id_structure)]\n",
    "df_partenaires_complet.loc[(pd.isna(df_partenaires_complet['id_structure']))|(str(df_partenaires_complet['id_structure'])=='None')|(str(df_partenaires_complet['id_structure'])=='nan')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "583f5ec8",
   "metadata": {},
   "outputs": [],
   "source": [
    "#df_partenaires_complet.to_excel(f\"./DATA/{source}/df_partenaires_id_structures.xlsx\")\n",
    "df_partenaires_complet.to_json(f\"./DATA/{source}/df_partners_id_structures.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "1c690f9a",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_partenaires_complet=pd.read_json(f\"./DATA/{source}/df_partenaires_id_structures.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "ec322580",
   "metadata": {},
   "outputs": [],
   "source": [
    "########récupération des structures sans identifiants pour les donner à Emmanuel\n",
    "identifiants_a_remplir=df_partenaires_complet.loc[(pd.isna(df_partenaires_complet['id_structure']))|(str(df_partenaires_complet['id_structure'])=='None')|(str(df_partenaires_complet['id_structure'])=='nan')]\n",
    "identifiants_a_remplir\n",
    "identifiants_a_remplir=identifiants_a_remplir.drop_duplicates(subset=f\"{sources[source]['nom_structure']}2\")\n",
    "identifiants_a_remplir=identifiants_a_remplir.reset_index()\n",
    "del identifiants_a_remplir['index']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "51ae89d6",
   "metadata": {},
   "outputs": [],
   "source": [
    "if sources[source]['ville'] in list(identifiants_a_remplir.columns) and sources[source]['pays'] in list(identifiants_a_remplir.columns) and sources[source]['adresse'] not in list(identifiants_a_remplir.columns):\n",
    "    identifiants_a_remplir=identifiants_a_remplir[[sources[source]['nom_structure'],sources[source]['ville'],sources[source]['pays']]]\n",
    "elif sources[source]['ville'] in list(identifiants_a_remplir.columns) and sources[source]['pays'] in list(identifiants_a_remplir.columns) and sources[source]['adresse'] in list(identifiants_a_remplir.columns):\n",
    "    identifiants_a_remplir=identifiants_a_remplir[[sources[source]['nom_structure'],sources[source]['adresse'],sources[source]['ville'],sources[source]['pays']]]\n",
    "elif sources[source]['region'] in list(identifiants_a_remplir.columns):\n",
    "    identifiants_a_remplir=identifiants_a_remplir[[sources[source]['nom_structure'],sources[source]['region']]]\n",
    "elif sources[source]['ville'] in list(identifiants_a_remplir.columns) and sources[source]['pays'] not in list(identifiants_a_remplir.columns):\n",
    "    identifiants_a_remplir=identifiants_a_remplir[[sources[source]['nom_structure'],sources[source]['ville']]]\n",
    "identifiants_a_remplir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "90b660db",
   "metadata": {},
   "outputs": [],
   "source": [
    "identifiants_a_remplir.to_excel(f\"./structures_manquantes/partenaires_non_identifies_{source}.xlsx\", index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "55b3e1da",
   "metadata": {},
   "source": [
    "# Matcher des chercheurs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "89e9d1dd",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_partenaires=pd.read_json(f\"./DATA/{source}/df_partners_id_structures.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ba8ba43e",
   "metadata": {},
   "outputs": [],
   "source": [
    "if len([x for x in ['nom', 'prenom'] if x in list(sources[source].keys())])==2:\n",
    "    df_partenaires['id_personne']=df_partenaires.progress_apply(lambda row: get_person(row, cached_data_persons,sources[source]['nom'],sources[source]['prenom']), axis=1)\n",
    "    write_cache(cached_data_persons,f\"./DATA/{source}/caches/cached_data_persons.pkl\")\n",
    "    #df_partenaires.to_excel(f\"./DATA/{source}/df_partners_id_personne.xlsx\")\n",
    "    df_partenaires.to_json(f\"./DATA/{source}/df_partners_id_personne.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "14039fb3",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_partenaires['id_personne']=df_partenaires.progress_apply(lambda row: get_person(row, cached_data_persons,sources[source]['nom'],sources[source]['prenom']), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "adc7546d",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_partenaires.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c65d4fd7",
   "metadata": {},
   "outputs": [],
   "source": [
    "if sources[source]['id_ORCID'] in list(df_partenaires.columns):\n",
    "    df_partenaires=pd.read_json(f\"./DATA/{source}/df_partenaires_id_personne.json\")\n",
    "    df_partenaires['idref_ORCID']=df_partenaires.progress_apply(lambda row: orcid_to_idref(row,cached_data_orcid,sources[source]['id_ORCID'],Authorization_ORCID), axis=1)\n",
    "    write_cache(cached_data_orcid,f\"./DATA/{source}/caches/cached_data_orcid.pkl\")\n",
    "    df_partenaires.to_excel(f\"./DATA/{source}/df_partenaires_id_personne_ORCID.xlsx\")\n",
    "    df_partenaires.to_json(f\"./DATA/{source}/df_partenaires_id_personne_ORCID.json\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "12332b18",
   "metadata": {},
   "source": [
    "# ENVOI DES PROJETS SUR SCANR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "21cef2ea",
   "metadata": {},
   "outputs": [],
   "source": [
    "if len(sources[source]['identifiants_preferes_personne'])==2:\n",
    "    df_partenaires=pd.read_json(f\"./DATA/{source}/df_partners_id_person_ORCID.json\")\n",
    "elif len(sources[source]['identifiants_preferes_personne'])==1:\n",
    "    df_partenaires=pd.read_json(f\"./DATA/{source}/df_partners_id_person.json\")\n",
    "else:\n",
    "    df_partenaires=pd.read_json(f\"./DATA/{source}/df_partners_id_structures.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "57413951",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_partenaires.loc[df_partenaires.id_structure.apply(lambda x :isinstance(x,list)),'id_structure']=df_partenaires.loc[df_partenaires.id_structure.apply(lambda x :isinstance(x,list)),'id_structure'].apply(lambda y: y[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a9ed3582",
   "metadata": {},
   "outputs": [],
   "source": [
    "if len([x for x in ['nom', 'prenom'] if x in list(sources[source].keys())])==2:\n",
    "    df_partenaires['id_person']=df_partenaires.apply(lambda row: get_id(row,sources[source]['identifiants_preferes_personne']), axis=1)\n",
    "    df_partenaires['persons']=df_partenaires.progress_apply(lambda row: persons(row,sources[source]['prenom'],sources[source]['nom']) ,axis=1)\n",
    "else:\n",
    "    df_partenaires['persons']=np.nan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "9ee1687d",
   "metadata": {},
   "outputs": [],
   "source": [
    "if source != 'SIRANO':\n",
    "    df_partenaires=df_partenaires.groupby([sources[source]['code_projet']]).agg({'persons': lambda x: [ y for y in x.tolist() if pd.isna(y)==False]}, dropna=False).reset_index()\n",
    "else:\n",
    "    df_projets=df_partenaires.groupby([sources[source]['code_projet'], sources[source]['annee'], sources[source]['acronyme'],sources[source]['titre'],sources[source]['budget']], dropna=False).agg({'persons': lambda x: [ y for y in x.tolist() if pd.isna(y)==False]}, dropna=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "e6ff1d0f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# amener les projets depuis le site \n",
    "if source=='ANR':\n",
    "    page_projets_10 = requests.get(sources[source]['url_projets']).json()\n",
    "    colonnes_projets_10 = page_projets_10['columns']\n",
    "    donnees_projets_10 = page_projets_10['data']\n",
    "    df_projets=pd.DataFrame(data=donnees_projets_10,columns=colonnes_projets_10)\n",
    "elif source=='IRESP':\n",
    "    df_projets1=pd.read_csv(sources[source]['url_projets1'] ,sep=\";\", encoding='UTF-8')\n",
    "    df_projets2=pd.read_csv(sources[source]['url_projets2'] ,sep=\";\", encoding='UTF-8')\n",
    "    df_projets=pd.concat([df_projets1,df_projets2])\n",
    "    df_projets.loc[pd.isna(df_projets['Titre_du_projet_FR']),'Titre_du_projet_FR']=df_projets.loc[pd.isna(df_projets['Titre_du_projet_FR']),'Titre_du_projet']\n",
    "elif source!='SIRANO':\n",
    "    df_projets=pd.read_csv(sources[source]['url_projects'] ,sep=\";\", encoding='ISO-8859-1')\n",
    "\n",
    "df_projets=df_projets.reset_index()\n",
    "del df_projets['index']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7dd3868b",
   "metadata": {},
   "outputs": [],
   "source": [
    "if source!='SIRANO':\n",
    "    df_projets=pd.merge(df_projets,df_partenaires,on=sources[source]['code_projet'], how='left')\n",
    "else :\n",
    "    df_projets['id']=df_partenaires.apply(lambda row: f\"{row[sources[source]['code_projet']]}-{row[sources[source]['annee']]}-{row[sources[source]['acronyme']]}\" , axis=1)\n",
    "    del df_projets['code_projet']\n",
    "    sources[source]['code_projet']='id'\n",
    "\n",
    "df_projets['type']=source\n",
    "df_projets['name']=df_projets.progress_apply(lambda row: projects(row,sources[source]['titre_fr'],sources[source]['titre_en']) ,axis=1)\n",
    "df_projets['description']=df_projets.progress_apply(lambda row: projects(row,sources[source]['resume_fr'],sources[source]['resume_en']) ,axis=1)\n",
    "df_projets.loc[:,sources[source]['budget']]=df_projets.loc[:,sources[source]['budget']].apply(lambda x : float(str(x).replace('.0','').replace('.00','').replace(' ','').replace(',','.').replace('€','').replace('\\x80','')))\n",
    "df_projets=df_projets.rename(columns={sources[source]['annee']:'year',sources[source]['acronyme']:'acronym',\n",
    "                                      sources[source]['budget']:'budget_financed',sources[source]['code_projet']:'id'})\n",
    "df_projets=df_projets[['id','type','name','description','acronym','year','budget_financed','persons']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3c620c25",
   "metadata": {},
   "outputs": [],
   "source": [
    "dict_row=df_projets.iloc[0,:].to_dict()\n",
    "dict_row2={k:v for k,v in list(dict_row.items()) if ((str(v)!='nan')&(str(v)!='NaN')&(str(v)!='None')&(str(v)!='x')&(str(v)!='[]'))}\n",
    "dict_row2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3040b95d",
   "metadata": {},
   "outputs": [],
   "source": [
    "#envoi\n",
    "err=[]\n",
    "for i,row in df_projets.iterrows():\n",
    "    dict_row=row.to_dict()\n",
    "    dict_row2={k:v for k,v in list(dict_row.items()) if ((str(v)!='nan')&(str(v)!='NaN')&(str(v)!='None')&(str(v)!='x')&(str(v)!='[]'))}\n",
    "    try:\n",
    "       r=requests.post('http://185.161.45.213/projects/projects', json = dict_row2, headers={\"Authorization\":Authorization})\n",
    "       res= r.json()\n",
    "       if res.get('status')=='ERR':\n",
    "           err.append(res)\n",
    "           if res.get('error').get('code')!=422:\n",
    "               print(err)\n",
    "               pp.pprint(err)\n",
    "    except Exception as e:\n",
    "        pp.pprint(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "23a56567",
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.Series([x.get('issues').get('id') for x in err]).drop_duplicates().tolist() "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ff7e90ab",
   "metadata": {},
   "source": [
    "pour mettre à jour"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "51462f6b",
   "metadata": {},
   "outputs": [],
   "source": [
    "nbr_page=int(requests.get('http://185.161.45.213/projects/projects?where={\"type\":\"ANSES\"}&projection={\"id\":1}&max_results=500&page=1', headers={\"Authorization\":Authorization}).json()['hrefs']['last']['href'].split('page=')[1])\n",
    "\n",
    "list_ids=[]\n",
    "for i in range(1,nbr_page+1):\n",
    "    print(\"page\",i)\n",
    "    page=requests.get('http://185.161.45.213/projects/projects?where={\"type\":\"ANSES\"}&projection={\"id\":1}&max_results=500'+f\"&page={i}\", headers={\"Authorization\":Authorization}).json()\n",
    "    for k in range(len(page['data'])):\n",
    "        print(\"k\",k)\n",
    "        list_ids.append(page['data'][k]['id'])\n",
    "    \n",
    "projets_a_ajouter=[x for x in list(df_projets['id']) if x not in list_ids]\n",
    "\n",
    "projets_a_retirer=[x for x in list_ids if x not in list(df_projets['id'])]\n",
    "\n",
    "df_projets = df_projets[df_projets['id'].apply(lambda x: x in projets_a_ajouter)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "da0c308a",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(projets_a_ajouter)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d4747378",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(df_projets[df_projets['id'].apply(lambda x: x in projets_a_ajouter)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8cac5958",
   "metadata": {},
   "outputs": [],
   "source": [
    "dict_row=df_projets.iloc[0,:].to_dict()\n",
    "dict_row2={k:v for k,v in list(dict_row.items()) if ((str(v)!='nan')&(str(v)!='NaN')&(str(v)!='None')&(str(v)!='x')&(str(v)!='[]'))}\n",
    "dict_row2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "611054ce",
   "metadata": {},
   "outputs": [],
   "source": [
    "err=[]\n",
    "for i,row in df_projets.iterrows():\n",
    "    dict_row=row.to_dict()\n",
    "    dict_row2={k:v for k,v in list(dict_row.items()) if ((str(v)!='nan')&(str(v)!='NaN')&(str(v)!='None')&(str(v)!='x')&(str(v)!='[]'))}\n",
    "    try:\n",
    "       r=requests.post('http://185.161.45.213/projects/projects', json = dict_row2, headers={\"Authorization\":Authorization})\n",
    "       res= r.json()\n",
    "       if res.get('status')=='ERR':\n",
    "           err.append(res)\n",
    "           if res.get('error').get('code')!=422:\n",
    "               print(err)\n",
    "               pp.pprint(err)\n",
    "    except Exception as e:\n",
    "        pp.pprint(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d86e1a09",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(projets_a_ajouter)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "28038ffa",
   "metadata": {},
   "source": [
    "Modifications"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4858669e",
   "metadata": {},
   "outputs": [],
   "source": [
    "nbr_page=int(requests.get('http://185.161.45.213/projects/projects?where={\"type\":\"ANR\"}&projection={\"id\":1,\"year\":1}&max_results=500'+f\"&page={1}\", headers={\"Authorization\":Authorization}).json()['hrefs']['last']['href'].split('page=')[1])\n",
    "nbr_page"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "73786090",
   "metadata": {},
   "outputs": [],
   "source": [
    "list_ids=[]\n",
    "for i in range(1,nbr_page+1):\n",
    "    print(\"page\",i)\n",
    "    page=requests.get('http://185.161.45.213/projects/projects?where={%22type%22:%22ANR%22}&projection={%22id%22:1,%22year%22:1}&max_results=500'+f\"&page={i}\", headers={\"Authorization\":Authorization}).json()\n",
    "    for k in range(len(page['data'])):\n",
    "        list_ids.append(page['data'][k]['year'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aacd7b75",
   "metadata": {},
   "outputs": [],
   "source": [
    "max(list_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "227532a8",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_projets = df_projets[df_projets['id'].apply(lambda x: x in list_ids)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5a0578c8",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "en principe, il faut que je mette à jour tous les projets \n",
    "qui ne sont pas clos avec les titres résumés et persons, mais \n",
    "peut etre qu'on peut mettre à jour automatiquement les projets\n",
    "des 3 dernières années ? \n",
    "\n",
    "\"\"\"\n",
    "\n",
    "err=[]\n",
    "for id in df_projets.iterrows() :\n",
    "    url = f\"http://185.161.45.213/projects/projects/{id}\"\n",
    "    project=requests.get(url, headers={'Authorization': Authorization}).json()\n",
    "    head = {\"Authorization\": Authorization, \"If-Match\": project['etag'], \"Content-Type\": \"application/json\"}\n",
    "\n",
    "    r = requests.patch(url, json = {\"year\": 2023}, headers=head)\n",
    "    res= r.json()\n",
    "    if res.get('status')=='ERR':\n",
    "        err.append(res)\n",
    "        if res.get('error').get('code')!=422:\n",
    "            print(err)\n",
    "            pp.pprint(err)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a41e50a5",
   "metadata": {},
   "source": [
    "# ENVOI DES PARTENAIRES SUR SCANR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "11136681",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_partenaires=pd.read_json(f\"./DATA/{source}/df_partners_id_structures.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a42766a2",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_partenaires.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "c68b4f62",
   "metadata": {},
   "outputs": [],
   "source": [
    "### ATTENTION, vérifier que les projets sirano sont dans des structures françaises\n",
    "if source=='IRESP':\n",
    "    df_partenaires[sources[source]['pays']]=df_partenaires.loc[:,sources[source]['ville']].apply(lambda x: x.split('(')[1].replace(')','') if x.find('(')>=0 else 'France')\n",
    "    df_partenaires.loc[:,sources[source]['ville']]=df_partenaires.loc[:,sources[source]['ville']].apply(lambda x: x.split('(')[0] if x.find('(')>=0 else x)\n",
    "  \n",
    "df_partenaires['address']=df_partenaires.apply(lambda row: address(row,sources[source]['pays'],sources[source]['ville'],source), axis=1)\n",
    "df_partenaires.loc[:,'id_structure']=df_partenaires.loc[:,'id_structure'].apply(lambda x: x[0] if isinstance(x,list) else x )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f66425af",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_partenaires"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cf09f899",
   "metadata": {},
   "outputs": [],
   "source": [
    "if source in ['ANSES','SIRANO']:\n",
    "    df_partenaires['id']=df_partenaires.apply(lambda row: f\"{row[sources[source]['code_projet']]}-{row[str(sources[source]['nom_structure'])+'2']}-{row[sources[source]['nom']]}-{row[sources[source]['prenom']]}\" , axis=1)\n",
    "if source =='REG_IDF':\n",
    "    df_partenaires['id']=df_partenaires.apply(lambda row: f\"{row[sources[source]['code_projet']]}-{row[str(sources[source]['nom_structure'])+'2']}-{row['entite_role']}\" , axis=1)\n",
    "df_partenaires['address']=df_partenaires.apply(lambda row: address(row,sources[source]['pays'],sources[source]['ville'],source), axis=1)\n",
    "df_partenaires=df_partenaires.rename(columns={sources[source]['nom_structure']: 'name', sources[source]['code_projet']: 'project_id', 'id_structure':'participant_id','Projet.Partenaire.Code_Decision_ANR':'id'})\n",
    "df_partenaires=df_partenaires[['name','id','project_id','participant_id','address']]\n",
    "df_partenaires['project_type']=source\n",
    "df_partenaires['participant_id']=df_partenaires.loc[:,'participant_id'].apply(lambda x: str(x[0]).replace('.0','') if isinstance(x,list) else str(x).split(';')[0].replace('.0',''))\n",
    "df_partenaires=df_partenaires[['id','project_id', 'project_type', 'participant_id', 'name','address']]\n",
    "df_partenaires['name'] = df_partenaires['name'].astype(str)\n",
    "df_partenaires"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d3495663",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_partenaires[df_partenaires.duplicated(subset=['id'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dae2cc5b",
   "metadata": {},
   "outputs": [],
   "source": [
    "dict_row=df_partenaires.iloc[1,:].to_dict()\n",
    "dict_row2={k:v for k,v in list(dict_row.items()) if ((str(v)!='nan')&(str(v)!='NaN')&(str(v)!='None')&(str(v)!='x'))}\n",
    "dict_row2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e2f8d3b1",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(df_partenaires)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9b489768",
   "metadata": {},
   "outputs": [],
   "source": [
    "err=[]\n",
    "for i,row in df_partenaires.iterrows():\n",
    "    dict_row=row.to_dict()\n",
    "    dict_row2={k:v for k,v in list(dict_row.items()) if ((str(v)!='nan')&(str(v)!='NaN')&(str(v)!='None')&(str(v)!='x'))}\n",
    "    try:\n",
    "       r=requests.post('http://185.161.45.213/projects/participations', json = dict_row2, headers={\"Authorization\":Authorization})\n",
    "       res= r.json()\n",
    "       if res.get('status')=='ERR':\n",
    "           print(i)\n",
    "           err.append(res)\n",
    "           if res.get('error').get('code')!=422:\n",
    "               print(err)\n",
    "               pp.pprint(err)\n",
    "    except Exception as e:\n",
    "        pp.pprint(e)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e2ea3fe3",
   "metadata": {},
   "source": [
    "Mise à jour"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "137c389e",
   "metadata": {},
   "outputs": [],
   "source": [
    "nbr_page=int(requests.get('http://185.161.45.213/projects/participations?where={\"project_type\":\"ANSES\"}&projection={\"id\":1}&max_results=500&page=1', headers={\"Authorization\":Authorization}).json()['hrefs']['last']['href'].split('page=')[1])\n",
    "\n",
    "list_ids=[]\n",
    "for i in range(1,nbr_page+1):\n",
    "    print(\"page\",i)\n",
    "    page=requests.get('http://185.161.45.213/projects/participations?where={\"project_type\":\"ANSES\"}&projection={\"id\":1}&max_results=500'+f\"&page={i}\", headers={\"Authorization\":Authorization}).json()\n",
    "    for k in range(len(page['data'])):\n",
    "        print(\"k\",k)\n",
    "        list_ids.append(page['data'][k]['id'])\n",
    "    \n",
    "projets_a_ajouter=[x for x in list(df_partenaires['id'].drop_duplicates()) if x not in list(pd.Series(list_ids).drop_duplicates())]\n",
    "\n",
    "projets_a_retirer=[x for x in list_ids if x not in list(df_partenaires['id'])]\n",
    "\n",
    "df_partenaires = df_partenaires[df_partenaires['id'].apply(lambda x: x in projets_a_ajouter)]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c0a930be",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(projets_a_ajouter)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9b351c2e",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(df_partenaires)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "defef4ee",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(df_partenaires)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "id": "a8b50023",
   "metadata": {},
   "outputs": [],
   "source": [
    "err=[]\n",
    "for i,row in df_partenaires.iterrows():\n",
    "    dict_row=row.to_dict()\n",
    "    dict_row2={k:v for k,v in list(dict_row.items()) if ((str(v)!='nan')&(str(v)!='NaN')&(str(v)!='None')&(str(v)!='x'))}\n",
    "    try:\n",
    "       r=requests.post('http://185.161.45.213/projects/participations', json = dict_row2, headers={\"Authorization\":Authorization})\n",
    "       res= r.json()\n",
    "       if res.get('status')=='ERR':\n",
    "           print(i)\n",
    "           err.append(res)\n",
    "           if res.get('error').get('code')!=422:\n",
    "               print(err)\n",
    "               pp.pprint(err)\n",
    "    except Exception as e:\n",
    "        pp.pprint(e)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b66d0830",
   "metadata": {},
   "source": [
    "Modifications"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "041def3a",
   "metadata": {},
   "outputs": [],
   "source": [
    "nbr_page=int(requests.get('http://185.161.45.213/projects/participations?where={\"project_type\":\"ANR\",\"participant_id\":{\"$exists\":false}}&projection={\"id\":1}&max_results=500'+f\"&page={1}\", headers={\"Authorization\":Authorization}).json()['hrefs']['last']['href'].split('page=')[1])\n",
    "\n",
    "list_ids=[]\n",
    "for i in range(1,nbr_page+1):\n",
    "    print(\"page\",i)\n",
    "    page=requests.get('http://185.161.45.213/projects/participations?where={\"project_type\":\"ANR\",\"participant_id\":{\"$exists\":false}}&projection={\"id\":1}&max_results=500'+f\"&page={i}\", headers={\"Authorization\":Authorization}).json()\n",
    "    for k in range(len(page['data'])):\n",
    "        print(\"k\",k)\n",
    "        list_ids.append(page['data'][k]['id'])\n",
    "\n",
    "df_partenaires = df_partenaires[df_partenaires['id'].apply(lambda x: x in list_ids)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ae447f6e",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(list_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "id": "8c3dd659",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_partenaires=df_partenaires[(df_partenaires.participant_id!='x') & (df_partenaires.participant_id!='None')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e8a62be8",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_partenaires"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0eef494c",
   "metadata": {},
   "outputs": [],
   "source": [
    "err=[]\n",
    "for i,row in df_partenaires.iterrows() :\n",
    "    print(i)\n",
    "    id=row['id']\n",
    "    url = f\"http://185.161.45.213/projects/participations/{id}\"\n",
    "    project=requests.get(url, headers={'Authorization': Authorization}).json()\n",
    "    head = {\"Authorization\": Authorization, \"If-Match\": project['etag'], \"Content-Type\": \"application/json\"}\n",
    "\n",
    "    r = requests.patch(url, json = {\"participant_id\": row['participant_id']}, headers=head)\n",
    "    res= r.json()\n",
    "    if res.get('status')=='ERR':\n",
    "        err.append(res)\n",
    "        if res.get('error').get('code')!=422:\n",
    "            print(err)\n",
    "            pp.pprint(err)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3ab8add7",
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.Series([x.get('issues').get('id')[25:] for x in err]).drop_duplicates().tolist()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "14b25fb0",
   "metadata": {},
   "source": [
    "Faire un nouveau fichier avec toutes les structures manquantes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "41dd26a6",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "73cf7e77",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7a22e4ca",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0a973ce1",
   "metadata": {},
   "outputs": [],
   "source": [
    "struct_manq=pd.read_excel('scanr_partenaires_non_identifies.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "ae5cf822",
   "metadata": {},
   "outputs": [],
   "source": [
    "nom1=struct_manq[struct_manq.New=='IRESP'].drop_duplicates(subset='Nom').Nom"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "eab48e23",
   "metadata": {},
   "outputs": [],
   "source": [
    "iresp_actuel=pd.read_excel('./missing_ids_structures/partenaires_non_identifies_IRESP.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "72f4933c",
   "metadata": {},
   "outputs": [],
   "source": [
    "nom2=iresp_actuel['Nom_equipe']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a2a772bf",
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.Series(list(nom1)+list(nom2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c28cc4ac",
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.Series(list(nom1)+list(nom2)).drop_duplicates()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}