Step_1_Obtaining_data_Gemstons.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.keys import Keys\n",
    "\n",
    "from webdriver_manager.chrome import ChromeDriverManager\n",
    "from bs4 import BeautifulSoup\n",
    "import pandas as pd\n",
    "import time\n",
    "import urllib.request\n",
    "import cv2\n",
    "import requests\n",
    "from selenium.common.exceptions import NoSuchElementException\n",
    "from selenium.common.exceptions import ElementClickInterceptedException"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Get_Stones_Details(driverf):\n",
    "    time.sleep(1)\n",
    "    Formula = \"\"\n",
    "    Colours = \"\"\n",
    "    Elements = \"\"\n",
    "    soup = BeautifulSoup(driverf.page_source, 'html.parser')\n",
    "    try:\n",
    "        divs = soup.find_all(\"div\",{\"class\":\"collapsesection\"})\n",
    "        for div in divs:\n",
    "            text = div.text\n",
    "            EFinder = text.find(\"Elements \")\n",
    "            if(EFinder!=-1):\n",
    "                Elements = (text[EFinder+16:])\n",
    "                if(Elements.find(\"-\")!=-1):\n",
    "                    Elements = Elements[:Elements.find(\"-\")]\n",
    "                else:\n",
    "                    Elements = Elements[:14]\n",
    "            CFinder = text.find(\"Colour:\")\n",
    "            if(CFinder!=-1):\n",
    "                Colours = text[CFinder+7:]\n",
    "                if(Colours.find(\".\")!=-1):\n",
    "                    Colours = Colours[:Colours.find(\".\")]\n",
    "                else:\n",
    "                    Colours = Colours[:16]\n",
    "            FFinder = text.find(\"Formula:\")\n",
    "            if(FFinder!=-1):\n",
    "                Formula = text[FFinder+8:]\n",
    "                if(Formula.find(\"Colour\")!=-1):\n",
    "                    Formula = Formula[:Formula.find(\"Colour\")]\n",
    "                elif(Formula.find(\"Elements \")!=-1):\n",
    "                    Formula = Formula[:Formula.find(\"Elements \")]\n",
    "                elif(Formula.find(\"Formula \")!=-1):\n",
    "                    Formula = Formula[:Formula.find(\"Formula \")]\n",
    "                else:\n",
    "                    Formula = Formula[:10]\n",
    "\n",
    "    except NoSuchElementException:\n",
    "        pass\n",
    "    except TypeError:\n",
    "        pass\n",
    "    except AttributeError:\n",
    "        pass\n",
    "    return Formula, Colours, Elements\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "def Get_Locations(driverf, stone_name):\n",
    "    time.sleep(2)   \n",
    "    loc_list = []\n",
    "    loc_url = []\n",
    "    final_loc = []\n",
    "    soup = BeautifulSoup(driverf.page_source, 'html.parser')\n",
    "    try:\n",
    "        table = soup.find(\"table\",{\"class\":\"loclisttable\"})\n",
    "        if(len(table)!=0):\n",
    "            for t in table:\n",
    "                url_for_cords = t.find_all('a')\n",
    "                for url in url_for_cords:\n",
    "                    href = url.get('href')\n",
    "                    loc_url.append(href)\n",
    "                divs = t.find_all(\"div\")\n",
    "                for d in divs:\n",
    "                    loc_list.append(d.text)\n",
    "\n",
    "\n",
    "            for loc in loc_url:\n",
    "                if(loc.find('loc-')!=-1):\n",
    "                    http = \"https://www.mindat.org/\"+loc\n",
    "                    final_loc.append(http)\n",
    "    except NoSuchElementException:\n",
    "        pass\n",
    "    except TypeError:\n",
    "        pass\n",
    "    return loc_list,final_loc\n",
    "\n",
    "\n",
    "\n",
    "def save_photos(df):  \n",
    "    for ind in df.index:\n",
    "        i = 0\n",
    "        image_url_list = df['Image_url'][ind]\n",
    "        stone_name = df['Name'][ind]\n",
    "        for image_url in image_url_list:\n",
    "            FileName = \"images.csv\"\n",
    "            FilePath = 'Data\\Images\\images'\n",
    "            image_url = 'https://www.mindat.org/'+image_url\n",
    "            response = requests.get(image_url)\n",
    "            image_name = FilePath + 'mindat'+stone_name+'ind='+str(ind)+'num='+str(i) + '.jpg' \n",
    "            with open(image_name, 'wb') as file:\n",
    "                file.write(response.content)\n",
    "            i+=1\n",
    "            img = cv2.imread(image_name)\n",
    "    # cv2.imshow(img)\n",
    "\n",
    "\n",
    "\n",
    "def get_photos(driverf,stone_name):\n",
    "    time.sleep(1)\n",
    "    i = 0    \n",
    "    img_list = []\n",
    "    try:\n",
    "        driverf.find_element_by_css_selector(\"input[type='text']\").send_keys(stone_name)\n",
    "        time.sleep(1)\n",
    "        driverf.find_element_by_css_selector(\"input[type='text']\").send_keys(Keys.ENTER)\n",
    "    except ElementClickInterceptedException:\n",
    "        return img_list\n",
    "    except NoSuchElementException:\n",
    "        time.sleep(20)\n",
    "        driverf.close()\n",
    "        driverf = webdriver.Chrome(ChromeDriverManager().install())\n",
    "        driverf.get('https://geology.com/gemstones/')\n",
    "        pass\n",
    "    soup = BeautifulSoup(driverf.page_source, 'html.parser')\n",
    "    try:\n",
    "        table = soup.find_all(\"div\",{\"class\":\"userbigpicture noborder\"})\n",
    "        for img in table:\n",
    "            if(i==3):break\n",
    "            img_list.append(img.find('img')['src'])\n",
    "            i += 1\n",
    "    except NoSuchElementException:\n",
    "        pass\n",
    "    return img_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[WDM] - Current google-chrome version is 87.0.4280\n",
      "[WDM] - Get LATEST driver version for 87.0.4280\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[WDM] - Driver [C:\\Users\\Noam\\.wdm\\drivers\\chromedriver\\win32\\87.0.4280.88\\chromedriver.exe] found in cache\n",
      "[WDM] - Current google-chrome version is 87.0.4280\n",
      "[WDM] - Get LATEST driver version for 87.0.4280\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[WDM] - Driver [C:\\Users\\Noam\\.wdm\\drivers\\chromedriver\\win32\\87.0.4280.88\\chromedriver.exe] found in cache\n"
     ]
    }
   ],
   "source": [
    "driver = webdriver.Chrome(ChromeDriverManager().install())\n",
    "driverf = webdriver.Chrome(ChromeDriverManager().install())\n",
    "driver.get('https://geology.com/gemstones/') #getting most popular gemstone's names\n",
    "driverf.get('https://www.mindat.org/photoscroll.php') # getting thos gemstone's details\n",
    "\n",
    "soup = BeautifulSoup(driver.page_source, 'html.parser')\n",
    "table = soup.find(\"div\",{\"class\":\"right\"})\n",
    "Name_list = []\n",
    "\n",
    "href_list = table.find_all('a')\n",
    "for href in href_list:\n",
    "    time.sleep(3)\n",
    "    data_dict = {}\n",
    "    Name = href.text\n",
    "    if(Name.find(\" Diamond\")!=-1 or Name.find(\" Opal\")!=-1): continue\n",
    "    data_dict['Name'] = Name\n",
    "    image_list = get_photos(driverf, Name) ###### imagesssss\n",
    "    if(len(image_list)==0): continue\n",
    "    Formula, Colours, Elements = Get_Stones_Details(driverf)\n",
    "    data_dict['Formula'] = Formula\n",
    "    data_dict['Elements'] = Elements\n",
    "    data_dict['Colours'] = Colours\n",
    "    data_dict['Image_url'] = image_list\n",
    "    data_dict['Locations'], data_dict['Locations_url'] = Get_Locations(driverf, Name)\n",
    "    \n",
    "    Name_list.append(data_dict)\n",
    "\n",
    "driver.close()\n",
    "driverf.close()    \n",
    "df = pd.DataFrame(Name_list)\n",
    "save_photos(df) \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df.drop_duplicates(subset=['Name'],inplace=True)\n",
    "df.to_csv(\"project_data_gemstones_table.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Name</th>\n",
       "      <th>Formula</th>\n",
       "      <th>Elements</th>\n",
       "      <th>Colours</th>\n",
       "      <th>Image_url</th>\n",
       "      <th>Locations</th>\n",
       "      <th>Locations_url</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Amazonite</td>\n",
       "      <td>K(AlSi3O8)</td>\n",
       "      <td>Al, K, O, Si</td>\n",
       "      <td>NaN</td>\n",
       "      <td>['/imagecache/11/7a/04766080014946246016426.jp...</td>\n",
       "      <td>['Argentina', '\\xa0', 'Australia', '\\xa0', 'Au...</td>\n",
       "      <td>['https://www.mindat.org//loc-269722.html', 'h...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Amethyst</td>\n",
       "      <td>SiO2</td>\n",
       "      <td>O, Si</td>\n",
       "      <td>Violet - purpleC</td>\n",
       "      <td>['/imagecache/8a/ca/03039750014946246163278.jp...</td>\n",
       "      <td>['Afghanistan', '\\xa0', 'Angola', '\\xa0', 'Arg...</td>\n",
       "      <td>['https://www.mindat.org//loc-404929.html', 'h...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Ametrine</td>\n",
       "      <td>SiO2</td>\n",
       "      <td>O, Si</td>\n",
       "      <td>NaN</td>\n",
       "      <td>['/imagecache/d2/c6/04532760014946392196542.jp...</td>\n",
       "      <td>['Bolivia', '\\xa0', 'Brazil', '\\xa0', 'Canada'...</td>\n",
       "      <td>['https://www.mindat.org//loc-11804.html', 'ht...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Ammolite</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>['/imagecache/b4/a9/00207230014946603988222.jp...</td>\n",
       "      <td>['Austria', '\\xa0', 'Canada', '\\xa0', 'Madagas...</td>\n",
       "      <td>['https://www.mindat.org//loc-205904.html', 'h...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Andalusite</td>\n",
       "      <td>Al2(SiO4)OIMA Formula:Al2SiO5</td>\n",
       "      <td>Bonfim W</td>\n",
       "      <td>Pink to red brown, occasionally yellow, green,...</td>\n",
       "      <td>['/imagecache/ff/30/01398870014946330614953.jp...</td>\n",
       "      <td>['Andorra', '\\xa0', 'Antarctica', 'AmMin 36:36...</td>\n",
       "      <td>['https://www.mindat.org//loc-31239.html', 'ht...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73</th>\n",
       "      <td>Turquoise</td>\n",
       "      <td>CuAl6(PO4)4(OH)8 · 4H2O</td>\n",
       "      <td>Al, Cu, H, O, P</td>\n",
       "      <td>bright blue, sky-blue, pale green, blue-green,...</td>\n",
       "      <td>['/imagecache/6d/d1/05330790014956573955192.jp...</td>\n",
       "      <td>['Afghanistan', 'Gübelin, E., Wolgensinger, M....</td>\n",
       "      <td>['https://www.mindat.org/loc-14319.html', 'htt...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74</th>\n",
       "      <td>Unakite</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>['/imagecache/5a/7d/05648390015601981974323.jp...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75</th>\n",
       "      <td>Variscite</td>\n",
       "      <td>AlPO4 · 2H2OIMA Formula:Al(PO4) · 2H2O</td>\n",
       "      <td>Sn–W–Pb–Ag–Zn Huanuni Deposit, Bolivia. Minera...</td>\n",
       "      <td>Pale to emerald-green, bluish green, colourles...</td>\n",
       "      <td>['/imagecache/77/23/08123090014977192376273.jp...</td>\n",
       "      <td>['Antigua and Barbuda', '\\xa0', 'Argentina', '...</td>\n",
       "      <td>['https://www.mindat.org//loc-147083.html', 'h...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76</th>\n",
       "      <td>Zircon</td>\n",
       "      <td>Zr(SiO4)May contain minor U, Th, Pb, Hf, Y/REE...</td>\n",
       "      <td>, Pr, Nd, and Sm) from a Carbonatite Deposit: ...</td>\n",
       "      <td>Colourless, yellow, grey, reddish-brown, green...</td>\n",
       "      <td>['/imagecache/40/37/05576060014946320573225.jp...</td>\n",
       "      <td>['Afghanistan', '\\xa0', 'Algeria', '\\xa0', 'An...</td>\n",
       "      <td>['https://www.mindat.org//loc-3.html', 'https:...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77</th>\n",
       "      <td>Zoisite</td>\n",
       "      <td>Ca2Al3[Si2O7][SiO4]O(OH)</td>\n",
       "      <td>Bonfim W</td>\n",
       "      <td>Colourless, purple, greyish-white, grey, yello...</td>\n",
       "      <td>['/imagecache/ce/e7/04096150014946273104080.jp...</td>\n",
       "      <td>['Afghanistan', '\\xa0', 'Antarctica', 'AmMin 3...</td>\n",
       "      <td>['https://www.mindat.org//loc-8.html', 'https:...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>78 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          Name                                            Formula  \\\n",
       "0    Amazonite                                         K(AlSi3O8)   \n",
       "1     Amethyst                                               SiO2   \n",
       "2     Ametrine                                               SiO2   \n",
       "3     Ammolite                                                NaN   \n",
       "4   Andalusite                      Al2(SiO4)OIMA Formula:Al2SiO5   \n",
       "..         ...                                                ...   \n",
       "73   Turquoise                            CuAl6(PO4)4(OH)8 · 4H2O   \n",
       "74     Unakite                                                NaN   \n",
       "75   Variscite             AlPO4 · 2H2OIMA Formula:Al(PO4) · 2H2O   \n",
       "76      Zircon  Zr(SiO4)May contain minor U, Th, Pb, Hf, Y/REE...   \n",
       "77     Zoisite                           Ca2Al3[Si2O7][SiO4]O(OH)   \n",
       "\n",
       "                                             Elements  \\\n",
       "0                                       Al, K, O, Si    \n",
       "1                                              O, Si    \n",
       "2                                              O, Si    \n",
       "3                                                 NaN   \n",
       "4                                            Bonfim W   \n",
       "..                                                ...   \n",
       "73                                   Al, Cu, H, O, P    \n",
       "74                                                NaN   \n",
       "75  Sn–W–Pb–Ag–Zn Huanuni Deposit, Bolivia. Minera...   \n",
       "76  , Pr, Nd, and Sm) from a Carbonatite Deposit: ...   \n",
       "77                                           Bonfim W   \n",
       "\n",
       "                                              Colours  \\\n",
       "0                                                 NaN   \n",
       "1                                    Violet - purpleC   \n",
       "2                                                 NaN   \n",
       "3                                                 NaN   \n",
       "4   Pink to red brown, occasionally yellow, green,...   \n",
       "..                                                ...   \n",
       "73  bright blue, sky-blue, pale green, blue-green,...   \n",
       "74                                                NaN   \n",
       "75  Pale to emerald-green, bluish green, colourles...   \n",
       "76  Colourless, yellow, grey, reddish-brown, green...   \n",
       "77  Colourless, purple, greyish-white, grey, yello...   \n",
       "\n",
       "                                            Image_url  \\\n",
       "0   ['/imagecache/11/7a/04766080014946246016426.jp...   \n",
       "1   ['/imagecache/8a/ca/03039750014946246163278.jp...   \n",
       "2   ['/imagecache/d2/c6/04532760014946392196542.jp...   \n",
       "3   ['/imagecache/b4/a9/00207230014946603988222.jp...   \n",
       "4   ['/imagecache/ff/30/01398870014946330614953.jp...   \n",
       "..                                                ...   \n",
       "73  ['/imagecache/6d/d1/05330790014956573955192.jp...   \n",
       "74  ['/imagecache/5a/7d/05648390015601981974323.jp...   \n",
       "75  ['/imagecache/77/23/08123090014977192376273.jp...   \n",
       "76  ['/imagecache/40/37/05576060014946320573225.jp...   \n",
       "77  ['/imagecache/ce/e7/04096150014946273104080.jp...   \n",
       "\n",
       "                                            Locations  \\\n",
       "0   ['Argentina', '\\xa0', 'Australia', '\\xa0', 'Au...   \n",
       "1   ['Afghanistan', '\\xa0', 'Angola', '\\xa0', 'Arg...   \n",
       "2   ['Bolivia', '\\xa0', 'Brazil', '\\xa0', 'Canada'...   \n",
       "3   ['Austria', '\\xa0', 'Canada', '\\xa0', 'Madagas...   \n",
       "4   ['Andorra', '\\xa0', 'Antarctica', 'AmMin 36:36...   \n",
       "..                                                ...   \n",
       "73  ['Afghanistan', 'Gübelin, E., Wolgensinger, M....   \n",
       "74                                                 []   \n",
       "75  ['Antigua and Barbuda', '\\xa0', 'Argentina', '...   \n",
       "76  ['Afghanistan', '\\xa0', 'Algeria', '\\xa0', 'An...   \n",
       "77  ['Afghanistan', '\\xa0', 'Antarctica', 'AmMin 3...   \n",
       "\n",
       "                                        Locations_url  \n",
       "0   ['https://www.mindat.org//loc-269722.html', 'h...  \n",
       "1   ['https://www.mindat.org//loc-404929.html', 'h...  \n",
       "2   ['https://www.mindat.org//loc-11804.html', 'ht...  \n",
       "3   ['https://www.mindat.org//loc-205904.html', 'h...  \n",
       "4   ['https://www.mindat.org//loc-31239.html', 'ht...  \n",
       "..                                                ...  \n",
       "73  ['https://www.mindat.org/loc-14319.html', 'htt...  \n",
       "74                                                 []  \n",
       "75  ['https://www.mindat.org//loc-147083.html', 'h...  \n",
       "76  ['https://www.mindat.org//loc-3.html', 'https:...  \n",
       "77  ['https://www.mindat.org//loc-8.html', 'https:...  \n",
       "\n",
       "[78 rows x 7 columns]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_loc(loc):\n",
    "    if(loc.find(\"'\")!=-1):\n",
    "        loc = loc.replace(\"'\",'',1)\n",
    "    if(loc.find(\"]\")!=-1):\n",
    "        loc = loc.replace(\"]\",'',1)\n",
    "    if(loc.find(\"[\")!=-1):\n",
    "        loc = loc.replace(\"[\",'',1)\n",
    "    if(loc.find(\",\")!=-1):\n",
    "        loc = loc.replace(\",\",'',1)\n",
    "    if(loc.find(\"'\")!=-1):\n",
    "        loc = loc.replace(\"'\",'',1)\n",
    "    return loc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"project_data_gemstones_table.csv\")\n",
    "locs = df['Locations_url']\n",
    "locs = pd.DataFrame(locs)\n",
    "new_locs = []\n",
    "for i in locs.index:\n",
    "    for string in locs['Locations_url'][i].split():\n",
    "        string = clean_loc(string)\n",
    "           \n",
    "        new_locs.append(string)\n",
    "new_locs = pd.DataFrame(new_locs)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in new_locs.index:\n",
    "    if(i%10!=0):\n",
    "        new_locs = new_locs.drop(index = i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_locs = new_locs.drop_duplicates().reset_index()\n",
    "new_locs = pd.DataFrame(new_locs[0])\n",
    "new_locs.drop_duplicates()\n",
    "new_loc_df = pd.DataFrame(index=new_locs.index, columns=[\"Location_URL\",\"Stone_Name\"])\n",
    "\n",
    "dict_locs = {}\n",
    "\n",
    "for ind in new_locs.index:\n",
    "    if(ind%2==0):continue\n",
    "    for i in locs.index:\n",
    "\n",
    "        stones_for_loc = []\n",
    "        curr_loc = new_locs[0][ind]\n",
    "\n",
    "        for string in locs['Locations_url'][i].split():\n",
    "            if(string.find(curr_loc)!=-1):\n",
    "                stones_for_loc.append(df['Name'][i])\n",
    "                try:\n",
    "                    dict_locs[string] +=\" \" + df['Name'][i]\n",
    "                except KeyError:\n",
    "                    dict_locs[string] = df['Name'][i]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Stones_Name</th>\n",
       "      <th>URL</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Amazonite</td>\n",
       "      <td>'https://www.mindat.org//loc-63988.html',</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Amazonite Scapolite Sphene Titanite</td>\n",
       "      <td>'https://www.mindat.org//loc-21257.html',</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Amazonite</td>\n",
       "      <td>'https://www.mindat.org//loc-272378.html',</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Amazonite Beryl Diopside Serpentine Sphene Spi...</td>\n",
       "      <td>'https://www.mindat.org//loc-9525.html',</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Amazonite Amethyst Beryl Diopside</td>\n",
       "      <td>'https://www.mindat.org//loc-133112.html',</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3616</th>\n",
       "      <td>Zoisite</td>\n",
       "      <td>'https://www.mindat.org//loc-17176.html',</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3617</th>\n",
       "      <td>Amethyst Apatite Kyanite Sphene Titanite Zirco...</td>\n",
       "      <td>'https://www.mindat.org//loc-30133.html',</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3618</th>\n",
       "      <td>Diopside Sphene Titanite Zoisite</td>\n",
       "      <td>'https://www.mindat.org//loc-122349.html',</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3619</th>\n",
       "      <td>Apatite Diopside Zoisite</td>\n",
       "      <td>'https://www.mindat.org//loc-38903.html',</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3620</th>\n",
       "      <td>Beryl Zoisite</td>\n",
       "      <td>'https://www.mindat.org//loc-19571.html',</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3621 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            Stones_Name  \\\n",
       "0                                             Amazonite   \n",
       "1                   Amazonite Scapolite Sphene Titanite   \n",
       "2                                             Amazonite   \n",
       "3     Amazonite Beryl Diopside Serpentine Sphene Spi...   \n",
       "4                     Amazonite Amethyst Beryl Diopside   \n",
       "...                                                 ...   \n",
       "3616                                            Zoisite   \n",
       "3617  Amethyst Apatite Kyanite Sphene Titanite Zirco...   \n",
       "3618                   Diopside Sphene Titanite Zoisite   \n",
       "3619                           Apatite Diopside Zoisite   \n",
       "3620                                      Beryl Zoisite   \n",
       "\n",
       "                                             URL  \n",
       "0      'https://www.mindat.org//loc-63988.html',  \n",
       "1      'https://www.mindat.org//loc-21257.html',  \n",
       "2     'https://www.mindat.org//loc-272378.html',  \n",
       "3       'https://www.mindat.org//loc-9525.html',  \n",
       "4     'https://www.mindat.org//loc-133112.html',  \n",
       "...                                          ...  \n",
       "3616   'https://www.mindat.org//loc-17176.html',  \n",
       "3617   'https://www.mindat.org//loc-30133.html',  \n",
       "3618  'https://www.mindat.org//loc-122349.html',  \n",
       "3619   'https://www.mindat.org//loc-38903.html',  \n",
       "3620   'https://www.mindat.org//loc-19571.html',  \n",
       "\n",
       "[3621 rows x 2 columns]"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "index = []\n",
    "for i in dict_locs:\n",
    "    index.append(i)\n",
    "index\n",
    "new_loc_df = pd.DataFrame(dict_locs.values(), columns=[\"Stones_Name\"])\n",
    "new_loc_df[\"URL\"] = index\n",
    "new_loc_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_loc_df.to_csv(\"location_and_stones_table.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_mineral_commodities_recorded(loc_driver):\n",
    "    minerals = \"\"\n",
    "    try:\n",
    "        soup = BeautifulSoup(loc_driver.page_source, 'html.parser')\n",
    "        minerals_soup = soup.find(\"div\",{\"id\":\"ltmain\"})\n",
    "        minerals = soup.find(\"div\",{\"id\":\"newloccomlist\"})\n",
    "        minerals = minerals.text\n",
    "    except NoSuchElementException:\n",
    "        pass\n",
    "    except AttributeError:\n",
    "        pass\n",
    "    return minerals\n",
    "\n",
    "\n",
    "def find_loc_name(loc_soup):\n",
    "    time.sleep(1)\n",
    "    Name = \"\"\n",
    "    try:\n",
    "        h1 = loc_soup.find(\"div\",{\"class\":\"fpbox990nl\"})\n",
    "        names = h1.find_all(\"a\")\n",
    "        i = 1\n",
    "        for name in names:\n",
    "            Name +=name.text + \",\"\n",
    "            i += 1\n",
    "    except NoSuchElementException:\n",
    "        pass\n",
    "    except AttributeError:\n",
    "        pass\n",
    "    return Name\n",
    "\n",
    "\n",
    "def get_minerals(loc_driver, loc_soup):\n",
    "    time.sleep(1)\n",
    "    minerals = []\n",
    "    try:\n",
    "        div = loc_soup.find(\"div\",{\"id\":\"filters\"})\n",
    "        span = div.find_all(\"span\")\n",
    "        for sp in span:\n",
    "            minerals.append(sp.text)\n",
    "    except NoSuchElementException:\n",
    "        pass\n",
    "    except AttributeError:\n",
    "        pass\n",
    "\n",
    "        \n",
    "    return minerals\n",
    "\n",
    "def find_rock_types(loc_soup):\n",
    "    rock_type_list = []\n",
    "    try:\n",
    "        rock_types = loc_soup.find(\"div\",{\"id\":\"newlocrocklist\"})\n",
    "        rt = rock_types.find_all(\"a\")\n",
    "        for r in rt:\n",
    "            a=r.text\n",
    "            rock_type_list.append(a)\n",
    "    except NoSuchElementException:\n",
    "        pass\n",
    "    except AttributeError:\n",
    "        pass\n",
    "    return rock_type_list\n",
    "\n",
    "def find_locations_details(new_loc_df, loc_driver):\n",
    "    total = []\n",
    "    for i in new_loc_df.index:\n",
    "        time.sleep(1)\n",
    "        loc = clean_loc(new_loc_df['URL'][i])\n",
    "        if(loc.find(\"https://www.mindat.org/loc-293106.html\")!=-1): continue\n",
    "        try:\n",
    "            loc_soup = BeautifulSoup(loc_driver.page_source,'html.parser')\n",
    "            Dec_cords_list = \"\"\n",
    "            loc_climate = \"\"\n",
    "            Coordinates  = loc_soup.find_all(\"div\",{\"class\":\"LFtd\"})\n",
    "            for cor in Coordinates:\n",
    "                cor = cor.text\n",
    "                if(cor.find(\" : \")!=-1):###find climate\n",
    "                    loc_climate= cor\n",
    "                else:\n",
    "                    if(any(c.isalpha() for c in cor)==False): ##check for cords digits only\n",
    "                        Dec_cords_list=cor\n",
    "                    \n",
    "        except NoSuchElementException: \n",
    "            warn = webdriver.Chrome(ChromeDriverManager().install())\n",
    "            time.sleep(10)\n",
    "            loc_driver.close()\n",
    "            warn.close()\n",
    "            loc_driver = webdriver.Chrome(ChromeDriverManager().install())\n",
    "            loc_driver.get(loc)\n",
    "            pass\n",
    "        except WebDriverException:\n",
    "            loc_driver.close()\n",
    "            loc_driver = webdriver.Chrome(ChromeDriverManager().install())\n",
    "            time.sleep(10)\n",
    "            loc_driver.get(loc)\n",
    "            pass\n",
    "        except TypeError:\n",
    "            loc_driver.close()\n",
    "            loc_driver = webdriver.Chrome(ChromeDriverManager().install())\n",
    "            time.sleep(10)\n",
    "            loc_driver.get(loc)\n",
    "            pass\n",
    "            \n",
    "        loc_data_dict = {}\n",
    "        loc_data_dict[\"Location URL\"] = loc\n",
    "        loc_data_dict[\"Stone Refernced\"] = new_loc_df['Stones_Name'][i]\n",
    "        loc_data_dict[\"Minerals Recorded\"] = find_mineral_commodities_recorded(loc_driver)\n",
    "        loc_driver.get(loc)\n",
    "        time.sleep(1)\n",
    "        loc_name=find_loc_name(loc_soup)\n",
    "        loc_data_dict[\"Location Name\"] = loc_name\n",
    "        loc_data_dict[\"Rock Types\"] = find_rock_types(loc_soup)\n",
    "        \n",
    "        loc_data_dict[\"Climate\"] = loc_climate\n",
    "        loc_data_dict[\"Decimal Coordinates\"] = Dec_cords_list\n",
    "        loc_data_dict[\"Minerals\"] = get_minerals(loc_driver, loc_soup)\n",
    "        if(len(Dec_cords_list)==0):continue\n",
    "        total.append(loc_data_dict)\n",
    "    return total\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[WDM] - Current google-chrome version is 88.0.4324\n",
      "[WDM] - Get LATEST driver version for 88.0.4324\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[WDM] - Driver [C:\\Users\\Noam\\.wdm\\drivers\\chromedriver\\win32\\88.0.4324.96\\chromedriver.exe] found in cache\n"
     ]
    }
   ],
   "source": [
    "loc_driver = webdriver.Chrome(ChromeDriverManager().install())\n",
    "t = find_locations_details(new_loc_df, loc_driver)\n",
    "loc_driver.close()\n",
    "datf = pd.DataFrame(t)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Location URL</th>\n",
       "      <th>Stone Refernced</th>\n",
       "      <th>Minerals Recorded</th>\n",
       "      <th>Location Name</th>\n",
       "      <th>Rock Types</th>\n",
       "      <th>Climate</th>\n",
       "      <th>Decimal Coordinates</th>\n",
       "      <th>Minerals</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>https://www.mindat.org//loc-4047.html</td>\n",
       "      <td>Amazonite Apatite Aquamarine Beryl Opal Smoky ...</td>\n",
       "      <td>None</td>\n",
       "      <td>, Preitenegg, Wolfsberg District, Carinthia, A...</td>\n",
       "      <td>[]</td>\n",
       "      <td>Dfb : Warm-summer humid continental climate</td>\n",
       "      <td>46.96177,14.95181</td>\n",
       "      <td>[Valid Species, Al, C, Ca, Ce, F, Fe, H, K, La...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>https://www.mindat.org//loc-250161.html</td>\n",
       "      <td>Amazonite Aquamarine Beryl Kyanite Smoky Quartz</td>\n",
       "      <td>None</td>\n",
       "      <td>, Spruce Pine, Spruce Pine Mining District, Mi...</td>\n",
       "      <td>[]</td>\n",
       "      <td>Cfb : Temperate oceanic climate</td>\n",
       "      <td>35.91667,-82.06667</td>\n",
       "      <td>[Al, B, Be, C, Ca, Cd, Ce, Cl, Cu, F, Fe, H, K...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>https://www.mindat.org//loc-8991.html</td>\n",
       "      <td>Amethyst Apatite Spinel</td>\n",
       "      <td>None</td>\n",
       "      <td>, Thanh Son District, Phú Thọ Province, Vietna...</td>\n",
       "      <td>[]</td>\n",
       "      <td>Cwa : Monsoon-influenced humid subtropical cli...</td>\n",
       "      <td>21.18306,105.25000</td>\n",
       "      <td>[Al, B, Be, F, Fe, H, K, Mg, O, Si, Ti]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>https://www.mindat.org//loc-56146.html</td>\n",
       "      <td>Amethyst</td>\n",
       "      <td>None</td>\n",
       "      <td>, Eggenburg, Waldviertel, Lower Austria, Austr...</td>\n",
       "      <td>[]</td>\n",
       "      <td>Cfb : Temperate oceanic climate</td>\n",
       "      <td>48.64833,15.82083</td>\n",
       "      <td>[Valid Species, O, Si]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>https://www.mindat.org//loc-232658.html</td>\n",
       "      <td>Amethyst</td>\n",
       "      <td>None</td>\n",
       "      <td>, Krimml, Zell am See District, Salzburg, Aust...</td>\n",
       "      <td>[]</td>\n",
       "      <td>Dfb : Warm-summer humid continental climate</td>\n",
       "      <td>47.15161,12.13835</td>\n",
       "      <td>[Valid Species, O, Si]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1599</th>\n",
       "      <td>https://www.mindat.org//loc-193574.html</td>\n",
       "      <td>Diopside Zoisite</td>\n",
       "      <td>None</td>\n",
       "      <td>, Cochise Mining District, Little Dragoon Moun...</td>\n",
       "      <td>[]</td>\n",
       "      <td>BSk : Cold semi-arid (steppe) climate</td>\n",
       "      <td>32.06556,-110.04056</td>\n",
       "      <td>[Valid Species, Al, Ca, Cu, Fe, H, K, Mo, O, S...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1600</th>\n",
       "      <td>https://www.mindat.org//loc-17150.html</td>\n",
       "      <td>Diopside Zoisite</td>\n",
       "      <td>ⓘ Sand and Gravel</td>\n",
       "      <td>, Palm Springs, Riverside Co., California, USA...</td>\n",
       "      <td>[]</td>\n",
       "      <td>Csa : Hot-summer Mediterranean climate</td>\n",
       "      <td>33.89747,-116.53778</td>\n",
       "      <td>[Al, B, C, Ca, F, Fe, H, K, Mg, Mn, Na, O, Si,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1601</th>\n",
       "      <td>https://www.mindat.org//loc-6134.html</td>\n",
       "      <td>Apatite Cordierite Kyanite Zoisite</td>\n",
       "      <td>None</td>\n",
       "      <td>, Lyon Co., Nevada, USA,i,Silver City Mining D...</td>\n",
       "      <td>[]</td>\n",
       "      <td>Csb : Warm-summer Mediterranean climate</td>\n",
       "      <td>39.26361,-119.64056</td>\n",
       "      <td>[Ag, Al, As, Au, B, C, Ca, Cu, Fe, H, K, Mg, M...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1602</th>\n",
       "      <td>https://www.mindat.org//loc-17176.html</td>\n",
       "      <td>Zoisite</td>\n",
       "      <td>ⓘ Feldspar</td>\n",
       "      <td>, Mitchell Co., North Carolina, USA,i,Pine Mou...</td>\n",
       "      <td>[]</td>\n",
       "      <td>Cfb : Temperate oceanic climate</td>\n",
       "      <td>35.94305,-82.09000</td>\n",
       "      <td>[Al, Ca, Cu, F, Fe, H, K, Mg, Mn, O, P, Si, Ti...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1603</th>\n",
       "      <td>https://www.mindat.org//loc-19571.html</td>\n",
       "      <td>Beryl Zoisite</td>\n",
       "      <td>None</td>\n",
       "      <td>, Gold Hill Mining District (Clifton Mining Di...</td>\n",
       "      <td>[]</td>\n",
       "      <td>BWk : Cold desert climate</td>\n",
       "      <td>40.13920,-113.82505</td>\n",
       "      <td>[Al, Au, Bi, C, Ca, Cl, Cu, F, Fe, H, K, Mg, O...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1604 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                 Location URL  \\\n",
       "0       https://www.mindat.org//loc-4047.html   \n",
       "1     https://www.mindat.org//loc-250161.html   \n",
       "2       https://www.mindat.org//loc-8991.html   \n",
       "3      https://www.mindat.org//loc-56146.html   \n",
       "4     https://www.mindat.org//loc-232658.html   \n",
       "...                                       ...   \n",
       "1599  https://www.mindat.org//loc-193574.html   \n",
       "1600   https://www.mindat.org//loc-17150.html   \n",
       "1601    https://www.mindat.org//loc-6134.html   \n",
       "1602   https://www.mindat.org//loc-17176.html   \n",
       "1603   https://www.mindat.org//loc-19571.html   \n",
       "\n",
       "                                        Stone Refernced  Minerals Recorded  \\\n",
       "0     Amazonite Apatite Aquamarine Beryl Opal Smoky ...               None   \n",
       "1       Amazonite Aquamarine Beryl Kyanite Smoky Quartz               None   \n",
       "2                               Amethyst Apatite Spinel               None   \n",
       "3                                              Amethyst               None   \n",
       "4                                              Amethyst               None   \n",
       "...                                                 ...                ...   \n",
       "1599                                   Diopside Zoisite               None   \n",
       "1600                                   Diopside Zoisite  ⓘ Sand and Gravel   \n",
       "1601                 Apatite Cordierite Kyanite Zoisite               None   \n",
       "1602                                            Zoisite         ⓘ Feldspar   \n",
       "1603                                      Beryl Zoisite               None   \n",
       "\n",
       "                                          Location Name Rock Types  \\\n",
       "0     , Preitenegg, Wolfsberg District, Carinthia, A...         []   \n",
       "1     , Spruce Pine, Spruce Pine Mining District, Mi...         []   \n",
       "2     , Thanh Son District, Phú Thọ Province, Vietna...         []   \n",
       "3     , Eggenburg, Waldviertel, Lower Austria, Austr...         []   \n",
       "4     , Krimml, Zell am See District, Salzburg, Aust...         []   \n",
       "...                                                 ...        ...   \n",
       "1599  , Cochise Mining District, Little Dragoon Moun...         []   \n",
       "1600  , Palm Springs, Riverside Co., California, USA...         []   \n",
       "1601  , Lyon Co., Nevada, USA,i,Silver City Mining D...         []   \n",
       "1602  , Mitchell Co., North Carolina, USA,i,Pine Mou...         []   \n",
       "1603  , Gold Hill Mining District (Clifton Mining Di...         []   \n",
       "\n",
       "                                                Climate  Decimal Coordinates  \\\n",
       "0           Dfb : Warm-summer humid continental climate    46.96177,14.95181   \n",
       "1                       Cfb : Temperate oceanic climate   35.91667,-82.06667   \n",
       "2     Cwa : Monsoon-influenced humid subtropical cli...   21.18306,105.25000   \n",
       "3                       Cfb : Temperate oceanic climate    48.64833,15.82083   \n",
       "4           Dfb : Warm-summer humid continental climate    47.15161,12.13835   \n",
       "...                                                 ...                  ...   \n",
       "1599              BSk : Cold semi-arid (steppe) climate  32.06556,-110.04056   \n",
       "1600             Csa : Hot-summer Mediterranean climate  33.89747,-116.53778   \n",
       "1601            Csb : Warm-summer Mediterranean climate  39.26361,-119.64056   \n",
       "1602                    Cfb : Temperate oceanic climate   35.94305,-82.09000   \n",
       "1603                          BWk : Cold desert climate  40.13920,-113.82505   \n",
       "\n",
       "                                               Minerals  \n",
       "0     [Valid Species, Al, C, Ca, Ce, F, Fe, H, K, La...  \n",
       "1     [Al, B, Be, C, Ca, Cd, Ce, Cl, Cu, F, Fe, H, K...  \n",
       "2               [Al, B, Be, F, Fe, H, K, Mg, O, Si, Ti]  \n",
       "3                                [Valid Species, O, Si]  \n",
       "4                                [Valid Species, O, Si]  \n",
       "...                                                 ...  \n",
       "1599  [Valid Species, Al, Ca, Cu, Fe, H, K, Mo, O, S...  \n",
       "1600  [Al, B, C, Ca, F, Fe, H, K, Mg, Mn, Na, O, Si,...  \n",
       "1601  [Ag, Al, As, Au, B, C, Ca, Cu, Fe, H, K, Mg, M...  \n",
       "1602  [Al, Ca, Cu, F, Fe, H, K, Mg, Mn, O, P, Si, Ti...  \n",
       "1603  [Al, Au, Bi, C, Ca, Cl, Cu, F, Fe, H, K, Mg, O...  \n",
       "\n",
       "[1604 rows x 8 columns]"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "datf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "datf.to_csv(\"project_data_location_table1.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[WDM] - Current google-chrome version is 88.0.4324\n",
      "[WDM] - Get LATEST driver version for 88.0.4324\n",
      "[WDM] - Driver [C:\\Users\\Noam\\.wdm\\drivers\\chromedriver\\win32\\88.0.4324.96\\chromedriver.exe] found in cache\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " \n"
     ]
    }
   ],
   "source": [
    "israel_locations_list = [\"https://www.mindat.org/loc-14762.html\"]\n",
    "israel_driver = webdriver.Chrome(ChromeDriverManager().install())\n",
    "israel_driver.get(\"https://www.mindat.org/loc-14762.html\")\n",
    "israel_soup = BeautifulSoup(israel_driver.page_source,'html.parser')\n",
    "israel_locs = israel_soup.find(\"div\",{\"class\":\"loclistcontainer\"})\n",
    "mt = israel_locs.find_all(\"a\")\n",
    "for m in mt:\n",
    "    a=\"https://www.mindat.org/\"+m.get(\"href\")\n",
    "    if(a.find(\"/loc-\")!=-1):\n",
    "        israel_locations_list.append(a)\n",
    "    \n",
    "\n",
    "\n",
    "israel_loc_df = pd.DataFrame(columns=[\"Stones_Name\"])\n",
    "israel_loc_df[\"URL\"] = israel_locations_list\n",
    "israel_loc_df\n",
    "\n",
    "israel_t = find_locations_details(israel_loc_df, israel_driver)\n",
    "israel_driver.close()\n",
    "israel_df = pd.DataFrame(israel_t)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Location URL</th>\n",
       "      <th>Stone Refernced</th>\n",
       "      <th>Minerals Recorded</th>\n",
       "      <th>Location Name</th>\n",
       "      <th>Rock Types</th>\n",
       "      <th>Climate</th>\n",
       "      <th>Decimal Coordinates</th>\n",
       "      <th>Minerals</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>https://www.mindat.org/loc-14762.html</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ⓘ Copperⓘ Gemstones</td>\n",
       "      <td>,i,Israel,Country,Click here to sponsor this p...</td>\n",
       "      <td>[ⓘ, Alkali basalt, ⓘ, Analcimite, ⓘ, Basalt, ⓘ...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>[Ag, Al, B, Ba, C, Ca, Ce, Cl, Cr, Cu, F, Fe, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>https://www.mindat.org/loc-14762.html</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ⓘ Copperⓘ Gemstones</td>\n",
       "      <td>,i,Israel,Country,Click here to sponsor this p...</td>\n",
       "      <td>[ⓘ, Alkali basalt, ⓘ, Analcimite, ⓘ, Basalt, ⓘ...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>[Ag, Al, B, Ba, C, Ca, Ce, Cl, Cr, Cu, F, Fe, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>https://www.mindat.org//loc-145487.html</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ⓘ Copperⓘ Gemstones</td>\n",
       "      <td>,i,Israel,Country,Click here to sponsor this p...</td>\n",
       "      <td>[ⓘ, Alkali basalt, ⓘ, Analcimite, ⓘ, Basalt, ⓘ...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>[Ag, Al, B, Ba, C, Ca, Ce, Cl, Cr, Cu, F, Fe, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>https://www.mindat.org//loc-145486.html</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>, Israel,i,Central District (HaMerkaz District...</td>\n",
       "      <td>[]</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>https://www.mindat.org//loc-145489.html</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ⓘ Gemstones</td>\n",
       "      <td>, Israel,i,Haifa District,District, Israel,Cou...</td>\n",
       "      <td>[ⓘ, Alkali basalt, ⓘ, Kimberlite, ⓘ, Peridotit...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>[Al, B, C, Ca, Cl, Cr, F, Fe, H, K, Mg, N, Na,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>https://www.mindat.org//loc-145488.html</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>, Israel,i,Jerusalem District,District, Israel...</td>\n",
       "      <td>[]</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>[Al, C, Ca, H, O, Si]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>https://www.mindat.org//loc-205351.html</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>, Israel,i,Northern District (HaZafon District...</td>\n",
       "      <td>[ⓘ, Basalt, ⓘ, 'Dolerite', ⓘ, 'Melanocratic ne...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>[Al, C, Ca, Cl, Fe, H, K, Mg, Mn, Na, Ni, O, S...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>https://www.mindat.org/loc-14762.html</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ⓘ Copper</td>\n",
       "      <td>, Israel,i,Southern District (HaDarom District...</td>\n",
       "      <td>[ⓘ, Analcimite, ⓘ, Basalt, ⓘ, Basanite, ⓘ, Bre...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>[Ag, Al, B, Ba, C, Ca, Ce, Cl, Cr, Cu, F, Fe, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>https://www.mindat.org//loc-205351.html</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ⓘ Copperⓘ Gemstones</td>\n",
       "      <td>,i,Israel,Country,Click here to sponsor this p...</td>\n",
       "      <td>[ⓘ, Alkali basalt, ⓘ, Analcimite, ⓘ, Basalt, ⓘ...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>[Ag, Al, B, Ba, C, Ca, Ce, Cl, Cr, Cu, F, Fe, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>https://www.mindat.org//loc-16782.html</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ⓘ Copper</td>\n",
       "      <td>, Israel,i,Southern District (HaDarom District...</td>\n",
       "      <td>[ⓘ, Analcimite, ⓘ, Basalt, ⓘ, Basanite, ⓘ, Bre...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>[Ag, Al, B, Ba, C, Ca, Ce, Cl, Cr, Cu, F, Fe, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>https://www.mindat.org//loc-225964.html</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>, Southern District (HaDarom District), Israel...</td>\n",
       "      <td>[ⓘ, Marble]</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>[Al, Ba, C, Ca, Cl, Cr, F, Fe, H, Mg, O, P, S,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>https://www.mindat.org//loc-191489.html</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>, Southern District (HaDarom District), Israel...</td>\n",
       "      <td>[]</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>[Valid Species, Al, C, Ca, Cu, Fe, H, Mg, O, S...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>https://www.mindat.org//loc-14768.html</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>, Southern District (HaDarom District), Israel...</td>\n",
       "      <td>[]</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>[Valid Species, O, Si]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>https://www.mindat.org/loc-105764.html</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ⓘ Copper</td>\n",
       "      <td>, Southern District (HaDarom District), Israel...</td>\n",
       "      <td>[]</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>[Ag, Al, Ba, C, Ca, Cl, Cu, F, Fe, H, K, Mg, M...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                               Location URL  Stone Refernced  \\\n",
       "0     https://www.mindat.org/loc-14762.html              NaN   \n",
       "1     https://www.mindat.org/loc-14762.html              NaN   \n",
       "2   https://www.mindat.org//loc-145487.html              NaN   \n",
       "3   https://www.mindat.org//loc-145486.html              NaN   \n",
       "4   https://www.mindat.org//loc-145489.html              NaN   \n",
       "5   https://www.mindat.org//loc-145488.html              NaN   \n",
       "6   https://www.mindat.org//loc-205351.html              NaN   \n",
       "7     https://www.mindat.org/loc-14762.html              NaN   \n",
       "8   https://www.mindat.org//loc-205351.html              NaN   \n",
       "9    https://www.mindat.org//loc-16782.html              NaN   \n",
       "10  https://www.mindat.org//loc-225964.html              NaN   \n",
       "11  https://www.mindat.org//loc-191489.html              NaN   \n",
       "12   https://www.mindat.org//loc-14768.html              NaN   \n",
       "13   https://www.mindat.org/loc-105764.html              NaN   \n",
       "\n",
       "      Minerals Recorded                                      Location Name  \\\n",
       "0   ⓘ Copperⓘ Gemstones  ,i,Israel,Country,Click here to sponsor this p...   \n",
       "1   ⓘ Copperⓘ Gemstones  ,i,Israel,Country,Click here to sponsor this p...   \n",
       "2   ⓘ Copperⓘ Gemstones  ,i,Israel,Country,Click here to sponsor this p...   \n",
       "3                  None  , Israel,i,Central District (HaMerkaz District...   \n",
       "4           ⓘ Gemstones  , Israel,i,Haifa District,District, Israel,Cou...   \n",
       "5                  None  , Israel,i,Jerusalem District,District, Israel...   \n",
       "6                  None  , Israel,i,Northern District (HaZafon District...   \n",
       "7              ⓘ Copper  , Israel,i,Southern District (HaDarom District...   \n",
       "8   ⓘ Copperⓘ Gemstones  ,i,Israel,Country,Click here to sponsor this p...   \n",
       "9              ⓘ Copper  , Israel,i,Southern District (HaDarom District...   \n",
       "10                 None  , Southern District (HaDarom District), Israel...   \n",
       "11                 None  , Southern District (HaDarom District), Israel...   \n",
       "12                 None  , Southern District (HaDarom District), Israel...   \n",
       "13             ⓘ Copper  , Southern District (HaDarom District), Israel...   \n",
       "\n",
       "                                           Rock Types Climate  \\\n",
       "0   [ⓘ, Alkali basalt, ⓘ, Analcimite, ⓘ, Basalt, ⓘ...           \n",
       "1   [ⓘ, Alkali basalt, ⓘ, Analcimite, ⓘ, Basalt, ⓘ...           \n",
       "2   [ⓘ, Alkali basalt, ⓘ, Analcimite, ⓘ, Basalt, ⓘ...           \n",
       "3                                                  []           \n",
       "4   [ⓘ, Alkali basalt, ⓘ, Kimberlite, ⓘ, Peridotit...           \n",
       "5                                                  []           \n",
       "6   [ⓘ, Basalt, ⓘ, 'Dolerite', ⓘ, 'Melanocratic ne...           \n",
       "7   [ⓘ, Analcimite, ⓘ, Basalt, ⓘ, Basanite, ⓘ, Bre...           \n",
       "8   [ⓘ, Alkali basalt, ⓘ, Analcimite, ⓘ, Basalt, ⓘ...           \n",
       "9   [ⓘ, Analcimite, ⓘ, Basalt, ⓘ, Basanite, ⓘ, Bre...           \n",
       "10                                        [ⓘ, Marble]           \n",
       "11                                                 []           \n",
       "12                                                 []           \n",
       "13                                                 []           \n",
       "\n",
       "   Decimal Coordinates                                           Minerals  \n",
       "0                       [Ag, Al, B, Ba, C, Ca, Ce, Cl, Cr, Cu, F, Fe, ...  \n",
       "1                       [Ag, Al, B, Ba, C, Ca, Ce, Cl, Cr, Cu, F, Fe, ...  \n",
       "2                       [Ag, Al, B, Ba, C, Ca, Ce, Cl, Cr, Cu, F, Fe, ...  \n",
       "3                                                                      []  \n",
       "4                       [Al, B, C, Ca, Cl, Cr, F, Fe, H, K, Mg, N, Na,...  \n",
       "5                                                   [Al, C, Ca, H, O, Si]  \n",
       "6                       [Al, C, Ca, Cl, Fe, H, K, Mg, Mn, Na, Ni, O, S...  \n",
       "7                       [Ag, Al, B, Ba, C, Ca, Ce, Cl, Cr, Cu, F, Fe, ...  \n",
       "8                       [Ag, Al, B, Ba, C, Ca, Ce, Cl, Cr, Cu, F, Fe, ...  \n",
       "9                       [Ag, Al, B, Ba, C, Ca, Ce, Cl, Cr, Cu, F, Fe, ...  \n",
       "10                      [Al, Ba, C, Ca, Cl, Cr, F, Fe, H, Mg, O, P, S,...  \n",
       "11                      [Valid Species, Al, C, Ca, Cu, Fe, H, Mg, O, S...  \n",
       "12                                                 [Valid Species, O, Si]  \n",
       "13                      [Ag, Al, Ba, C, Ca, Cl, Cu, F, Fe, H, K, Mg, M...  "
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "israel_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Location URL</th>\n",
       "      <th>Stone Refernced</th>\n",
       "      <th>Minerals Recorded</th>\n",
       "      <th>Location Name</th>\n",
       "      <th>Rock Types</th>\n",
       "      <th>Climate</th>\n",
       "      <th>Decimal Coordinates</th>\n",
       "      <th>Minerals</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>https://www.mindat.org/loc-14762.html</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ⓘ Copperⓘ Gemstones</td>\n",
       "      <td>,i,Israel,Country,Click here to sponsor this p...</td>\n",
       "      <td>[ⓘ, Alkali basalt, ⓘ, Analcimite, ⓘ, Basalt, ⓘ...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>[Ag, Al, B, Ba, C, Ca, Ce, Cl, Cr, Cu, F, Fe, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>https://www.mindat.org//loc-145487.html</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ⓘ Copperⓘ Gemstones</td>\n",
       "      <td>,i,Israel,Country,Click here to sponsor this p...</td>\n",
       "      <td>[ⓘ, Alkali basalt, ⓘ, Analcimite, ⓘ, Basalt, ⓘ...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>[Ag, Al, B, Ba, C, Ca, Ce, Cl, Cr, Cu, F, Fe, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>https://www.mindat.org//loc-145486.html</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>, Israel,i,Central District (HaMerkaz District...</td>\n",
       "      <td>[]</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>https://www.mindat.org//loc-145489.html</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ⓘ Gemstones</td>\n",
       "      <td>, Israel,i,Haifa District,District, Israel,Cou...</td>\n",
       "      <td>[ⓘ, Alkali basalt, ⓘ, Kimberlite, ⓘ, Peridotit...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>[Al, B, C, Ca, Cl, Cr, F, Fe, H, K, Mg, N, Na,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>https://www.mindat.org//loc-145488.html</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>, Israel,i,Jerusalem District,District, Israel...</td>\n",
       "      <td>[]</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>[Al, C, Ca, H, O, Si]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1612</th>\n",
       "      <td>https://www.mindat.org//loc-193574.html</td>\n",
       "      <td>Diopside Zoisite</td>\n",
       "      <td>None</td>\n",
       "      <td>, Cochise Mining District, Little Dragoon Moun...</td>\n",
       "      <td>[]</td>\n",
       "      <td>BSk : Cold semi-arid (steppe) climate</td>\n",
       "      <td>32.06556,-110.04056</td>\n",
       "      <td>[Valid Species, Al, Ca, Cu, Fe, H, K, Mo, O, S...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1613</th>\n",
       "      <td>https://www.mindat.org//loc-17150.html</td>\n",
       "      <td>Diopside Zoisite</td>\n",
       "      <td>ⓘ Sand and Gravel</td>\n",
       "      <td>, Palm Springs, Riverside Co., California, USA...</td>\n",
       "      <td>[]</td>\n",
       "      <td>Csa : Hot-summer Mediterranean climate</td>\n",
       "      <td>33.89747,-116.53778</td>\n",
       "      <td>[Al, B, C, Ca, F, Fe, H, K, Mg, Mn, Na, O, Si,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1614</th>\n",
       "      <td>https://www.mindat.org//loc-6134.html</td>\n",
       "      <td>Apatite Cordierite Kyanite Zoisite</td>\n",
       "      <td>None</td>\n",
       "      <td>, Lyon Co., Nevada, USA,i,Silver City Mining D...</td>\n",
       "      <td>[]</td>\n",
       "      <td>Csb : Warm-summer Mediterranean climate</td>\n",
       "      <td>39.26361,-119.64056</td>\n",
       "      <td>[Ag, Al, As, Au, B, C, Ca, Cu, Fe, H, K, Mg, M...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1615</th>\n",
       "      <td>https://www.mindat.org//loc-17176.html</td>\n",
       "      <td>Zoisite</td>\n",
       "      <td>ⓘ Feldspar</td>\n",
       "      <td>, Mitchell Co., North Carolina, USA,i,Pine Mou...</td>\n",
       "      <td>[]</td>\n",
       "      <td>Cfb : Temperate oceanic climate</td>\n",
       "      <td>35.94305,-82.09000</td>\n",
       "      <td>[Al, Ca, Cu, F, Fe, H, K, Mg, Mn, O, P, Si, Ti...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1616</th>\n",
       "      <td>https://www.mindat.org//loc-19571.html</td>\n",
       "      <td>Beryl Zoisite</td>\n",
       "      <td>None</td>\n",
       "      <td>, Gold Hill Mining District (Clifton Mining Di...</td>\n",
       "      <td>[]</td>\n",
       "      <td>BWk : Cold desert climate</td>\n",
       "      <td>40.13920,-113.82505</td>\n",
       "      <td>[Al, Au, Bi, C, Ca, Cl, Cu, F, Fe, H, K, Mg, O...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1617 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                 Location URL  \\\n",
       "0       https://www.mindat.org/loc-14762.html   \n",
       "1     https://www.mindat.org//loc-145487.html   \n",
       "2     https://www.mindat.org//loc-145486.html   \n",
       "3     https://www.mindat.org//loc-145489.html   \n",
       "4     https://www.mindat.org//loc-145488.html   \n",
       "...                                       ...   \n",
       "1612  https://www.mindat.org//loc-193574.html   \n",
       "1613   https://www.mindat.org//loc-17150.html   \n",
       "1614    https://www.mindat.org//loc-6134.html   \n",
       "1615   https://www.mindat.org//loc-17176.html   \n",
       "1616   https://www.mindat.org//loc-19571.html   \n",
       "\n",
       "                         Stone Refernced    Minerals Recorded  \\\n",
       "0                                    NaN  ⓘ Copperⓘ Gemstones   \n",
       "1                                    NaN  ⓘ Copperⓘ Gemstones   \n",
       "2                                    NaN                 None   \n",
       "3                                    NaN          ⓘ Gemstones   \n",
       "4                                    NaN                 None   \n",
       "...                                  ...                  ...   \n",
       "1612                    Diopside Zoisite                 None   \n",
       "1613                    Diopside Zoisite    ⓘ Sand and Gravel   \n",
       "1614  Apatite Cordierite Kyanite Zoisite                 None   \n",
       "1615                             Zoisite           ⓘ Feldspar   \n",
       "1616                       Beryl Zoisite                 None   \n",
       "\n",
       "                                          Location Name  \\\n",
       "0     ,i,Israel,Country,Click here to sponsor this p...   \n",
       "1     ,i,Israel,Country,Click here to sponsor this p...   \n",
       "2     , Israel,i,Central District (HaMerkaz District...   \n",
       "3     , Israel,i,Haifa District,District, Israel,Cou...   \n",
       "4     , Israel,i,Jerusalem District,District, Israel...   \n",
       "...                                                 ...   \n",
       "1612  , Cochise Mining District, Little Dragoon Moun...   \n",
       "1613  , Palm Springs, Riverside Co., California, USA...   \n",
       "1614  , Lyon Co., Nevada, USA,i,Silver City Mining D...   \n",
       "1615  , Mitchell Co., North Carolina, USA,i,Pine Mou...   \n",
       "1616  , Gold Hill Mining District (Clifton Mining Di...   \n",
       "\n",
       "                                             Rock Types  \\\n",
       "0     [ⓘ, Alkali basalt, ⓘ, Analcimite, ⓘ, Basalt, ⓘ...   \n",
       "1     [ⓘ, Alkali basalt, ⓘ, Analcimite, ⓘ, Basalt, ⓘ...   \n",
       "2                                                    []   \n",
       "3     [ⓘ, Alkali basalt, ⓘ, Kimberlite, ⓘ, Peridotit...   \n",
       "4                                                    []   \n",
       "...                                                 ...   \n",
       "1612                                                 []   \n",
       "1613                                                 []   \n",
       "1614                                                 []   \n",
       "1615                                                 []   \n",
       "1616                                                 []   \n",
       "\n",
       "                                      Climate  Decimal Coordinates  \\\n",
       "0                                                                    \n",
       "1                                                                    \n",
       "2                                                                    \n",
       "3                                                                    \n",
       "4                                                                    \n",
       "...                                       ...                  ...   \n",
       "1612    BSk : Cold semi-arid (steppe) climate  32.06556,-110.04056   \n",
       "1613   Csa : Hot-summer Mediterranean climate  33.89747,-116.53778   \n",
       "1614  Csb : Warm-summer Mediterranean climate  39.26361,-119.64056   \n",
       "1615          Cfb : Temperate oceanic climate   35.94305,-82.09000   \n",
       "1616                BWk : Cold desert climate  40.13920,-113.82505   \n",
       "\n",
       "                                               Minerals  \n",
       "0     [Ag, Al, B, Ba, C, Ca, Ce, Cl, Cr, Cu, F, Fe, ...  \n",
       "1     [Ag, Al, B, Ba, C, Ca, Ce, Cl, Cr, Cu, F, Fe, ...  \n",
       "2                                                    []  \n",
       "3     [Al, B, C, Ca, Cl, Cr, F, Fe, H, K, Mg, N, Na,...  \n",
       "4                                 [Al, C, Ca, H, O, Si]  \n",
       "...                                                 ...  \n",
       "1612  [Valid Species, Al, Ca, Cu, Fe, H, K, Mo, O, S...  \n",
       "1613  [Al, B, C, Ca, F, Fe, H, K, Mg, Mn, Na, O, Si,...  \n",
       "1614  [Ag, Al, As, Au, B, C, Ca, Cu, Fe, H, K, Mg, M...  \n",
       "1615  [Al, Ca, Cu, F, Fe, H, K, Mg, Mn, O, P, Si, Ti...  \n",
       "1616  [Al, Au, Bi, C, Ca, Cl, Cu, F, Fe, H, K, Mg, O...  \n",
       "\n",
       "[1617 rows x 8 columns]"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "loc_df = pd.concat([israel_df,datf])\n",
    "loc_df = loc_df.drop_duplicates(subset=['Location URL','Location Name'])\n",
    "loc_df = loc_df.reset_index(drop = True)\n",
    "loc_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [],
   "source": [
    "loc_df.to_csv(\"raw_project_data_location_table.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "#world's rarest gemstones\n",
    "Tanzanite =\"https://www.mindat.org/min-3885.html\"\n",
    "Black_Opal = \"https://www.mindat.org/min-7998.html\"\n",
    "Ganterite = \"https://www.mindat.org/min-25676.html\"\n",
    "Alexandrite = \"https://www.mindat.org/min-109.html\"\n",
    "Benitoite = \"https://www.mindat.org/min-624.html\"\n",
    "Painite = \"https://www.mindat.org/min-3063.html\"\n",
    "Red_Beryl = \"https://www.mindat.org/min-690.html\"\n",
    "Jadeite =\"https://www.mindat.org/min-2062.html\"\n",
    "Ammolite = \"https://www.mindat.org/min-8393.html\"\n",
    "Muscovite = \"https://www.mindat.org/min-2815.html\"\n",
    "Poudretteite = \"https://www.mindat.org/min-3272.html\"\n",
    "Taaffeite = \"https://www.mindat.org/min-3863.html\"\n",
    "rare_stones_list = [Tanzanite,Black_Opal,Ganterite,Alexandrite,Benitoite,Painite,Red_Beryl,Jadeite,Ammolite,Muscovite,Poudretteite,Taaffeite]\n",
    "rare_stones_names=[\"Tanzanite\",\"Black_Opal\",\"Ganterite\",\"Alexandrite\",\"Benitoite\",\"Painite\",\"Red_Beryl\",\"Jadeite\",\"Ammolite\",\"Muscovite\",\"Poudretteite\",\"Taaffeite\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[WDM] - Current google-chrome version is 88.0.4324\n",
      "[WDM] - Get LATEST driver version for 88.0.4324\n",
      "[WDM] - Driver [C:\\Users\\Noam\\.wdm\\drivers\\chromedriver\\win32\\88.0.4324.96\\chromedriver.exe] found in cache\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Stones_Name</th>\n",
       "      <th>URL</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Tanzanite</td>\n",
       "      <td>https://www.mindat.org//loc-3246.html</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Tanzanite</td>\n",
       "      <td>https://www.mindat.org//loc-5699.html</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Tanzanite</td>\n",
       "      <td>https://www.mindat.org//loc-189430.html</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Tanzanite</td>\n",
       "      <td>https://www.mindat.org//loc-189431.html</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Tanzanite</td>\n",
       "      <td>https://www.mindat.org//loc-189432.html</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>376</th>\n",
       "      <td>Taaffeite</td>\n",
       "      <td>https://www.mindat.org//loc-3147.html</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>377</th>\n",
       "      <td>Taaffeite</td>\n",
       "      <td>https://www.mindat.org//loc-3148.html</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>378</th>\n",
       "      <td>Taaffeite</td>\n",
       "      <td>https://www.mindat.org//loc-3167.html</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>379</th>\n",
       "      <td>Taaffeite</td>\n",
       "      <td>https://www.mindat.org//loc-67493.html</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>380</th>\n",
       "      <td>Taaffeite</td>\n",
       "      <td>https://www.mindat.org//loc-108691.html</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>381 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    Stones_Name                                      URL\n",
       "0     Tanzanite    https://www.mindat.org//loc-3246.html\n",
       "1     Tanzanite    https://www.mindat.org//loc-5699.html\n",
       "2     Tanzanite  https://www.mindat.org//loc-189430.html\n",
       "3     Tanzanite  https://www.mindat.org//loc-189431.html\n",
       "4     Tanzanite  https://www.mindat.org//loc-189432.html\n",
       "..          ...                                      ...\n",
       "376   Taaffeite    https://www.mindat.org//loc-3147.html\n",
       "377   Taaffeite    https://www.mindat.org//loc-3148.html\n",
       "378   Taaffeite    https://www.mindat.org//loc-3167.html\n",
       "379   Taaffeite   https://www.mindat.org//loc-67493.html\n",
       "380   Taaffeite  https://www.mindat.org//loc-108691.html\n",
       "\n",
       "[381 rows x 2 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rare_stones_locs = []\n",
    "rare_stones_names_list = []\n",
    "rare_driver = webdriver.Chrome(ChromeDriverManager().install())\n",
    "j=0\n",
    "for i in rare_stones_list:\n",
    "    rare_driver.get(i)\n",
    "    rare_soup = BeautifulSoup(rare_driver.page_source,'html.parser')\n",
    "    rare_locs = rare_soup.find(\"table\",{\"class\":\"loclisttable\"})\n",
    "        \n",
    "    mt = rare_locs.find_all(\"a\")\n",
    "    for m in mt:\n",
    "        a=\"https://www.mindat.org/\"+m.get(\"href\")\n",
    "        if(a.find(\"/loc-\")!=-1):\n",
    "            rare_stones_locs.append(a)\n",
    "            rare_stones_names_list.append(rare_stones_names[j])\n",
    "    j+=1\n",
    "\n",
    "rare_stones_df = pd.DataFrame(columns=[\"Stones_Name\"])\n",
    "rare_stones_df[\"Stones_Name\"] = rare_stones_names_list\n",
    "rare_stones_df[\"URL\"] = rare_stones_locs\n",
    "rare_driver.close()\n",
    "rare_stones_df\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "\n",
    "rare_driver = webdriver.Chrome(ChromeDriverManager().install())\n",
    "rare_t = find_locations_details(rare_stones_df, rare_driver)\n",
    "rare_driver.close()\n",
    "rare_df = pd.DataFrame(rare_t)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Location URL</th>\n",
       "      <th>Stone Refernced</th>\n",
       "      <th>Minerals Recorded</th>\n",
       "      <th>Location Name</th>\n",
       "      <th>Rock Types</th>\n",
       "      <th>Climate</th>\n",
       "      <th>Decimal Coordinates</th>\n",
       "      <th>Minerals</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>https://www.mindat.org//loc-189430.html</td>\n",
       "      <td>Tanzanite</td>\n",
       "      <td>None</td>\n",
       "      <td>, Lelatema Mountains, Simanjiro District, Many...</td>\n",
       "      <td>[]</td>\n",
       "      <td>Aw : Tropical savanna, wet</td>\n",
       "      <td>-3.58333,37.00833</td>\n",
       "      <td>[Al, As, B, C, Ca, Ce, Cl, Cr, Cu, F, Fe, Ga, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>https://www.mindat.org/loc-151.html</td>\n",
       "      <td>Black_Opal</td>\n",
       "      <td>None</td>\n",
       "      <td>, Korogwe Rural District, Tanga Region, Tanzan...</td>\n",
       "      <td>[ⓘ, Gneiss, ⓘ, 'Pegmatite']</td>\n",
       "      <td>Am : Tropical monsoon climate</td>\n",
       "      <td>-4.52833,38.73139</td>\n",
       "      <td>[Al, B, C, Ca, Ce, Cl, Cr, Cu, F, Fe, Gd, H, L...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>https://www.mindat.org//loc-146632.html</td>\n",
       "      <td>Black_Opal</td>\n",
       "      <td>ⓘ Gemstones</td>\n",
       "      <td>, Finch Co., New South Wales, Australia,i,Ligh...</td>\n",
       "      <td>[ⓘ, Claystone, ⓘ, Gravel, ⓘ, Mudstone, ⓘ, Sand...</td>\n",
       "      <td>BSh : Hot semi-arid (steppe) climate</td>\n",
       "      <td>-29.43146,147.98010</td>\n",
       "      <td>[Al, Ca, H, Mg, Na, O, Si]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>https://www.mindat.org//loc-235813.html</td>\n",
       "      <td>Black_Opal</td>\n",
       "      <td>None</td>\n",
       "      <td>, Lightning Ridge, Finch Co., New South Wales,...</td>\n",
       "      <td>[]</td>\n",
       "      <td>BSh : Hot semi-arid (steppe) climate</td>\n",
       "      <td>-29.46333,147.76083</td>\n",
       "      <td>[H, O, Si]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>https://www.mindat.org//loc-146633.html</td>\n",
       "      <td>Black_Opal</td>\n",
       "      <td>None</td>\n",
       "      <td>, Coocoran Opal Fields, Lightning Ridge, Finch...</td>\n",
       "      <td>[]</td>\n",
       "      <td>BSh : Hot semi-arid (steppe) climate</td>\n",
       "      <td>-29.44833,147.78167</td>\n",
       "      <td>[Valid Species, H, O, Si]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>153</th>\n",
       "      <td>https://www.mindat.org/loc-3142.html</td>\n",
       "      <td>Taaffeite</td>\n",
       "      <td>ⓘ Molybdenumⓘ Vanadium</td>\n",
       "      <td>, Republic of Karelia, Russia,i,Pitkyaranta mi...</td>\n",
       "      <td>[ⓘ, Basalt, ⓘ, Gneiss, ⓘ, Granite, ⓘ, 'Graniti...</td>\n",
       "      <td>Dfc : Subarctic climate</td>\n",
       "      <td>61.55000,31.50000</td>\n",
       "      <td>[Ag, Al, As, Au, B, Ba, Be, Bi, C, Ca, Ce, Cl,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>154</th>\n",
       "      <td>https://www.mindat.org//loc-123069.html</td>\n",
       "      <td>Taaffeite</td>\n",
       "      <td>None</td>\n",
       "      <td>, Polonnaruwa District, North Central Province...</td>\n",
       "      <td>[]</td>\n",
       "      <td>Af : Tropical rainforest climate</td>\n",
       "      <td>7.78060,80.81840</td>\n",
       "      <td>[Al, Be, Ca, H, K, Mg, Na, O, Si, W]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>155</th>\n",
       "      <td>https://www.mindat.org//loc-3167.html</td>\n",
       "      <td>Taaffeite</td>\n",
       "      <td>None</td>\n",
       "      <td>, Ratnapura, Ratnapura District, Sabaragamuwa ...</td>\n",
       "      <td>[]</td>\n",
       "      <td>Af : Tropical rainforest climate</td>\n",
       "      <td>6.63702,80.36177</td>\n",
       "      <td>[Al, Be, Mg, O]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>156</th>\n",
       "      <td>https://www.mindat.org//loc-67493.html</td>\n",
       "      <td>Taaffeite</td>\n",
       "      <td>ⓘ Ironⓘ Manganese</td>\n",
       "      <td>, Långban Ore District, Filipstad, Värmland Co...</td>\n",
       "      <td>[ⓘ, Biotitite, ⓘ, Granite, ⓘ, Skarn]</td>\n",
       "      <td>Dfc : Subarctic climate</td>\n",
       "      <td>59.85538,14.26480</td>\n",
       "      <td>[Ag, Al, As, B, Ba, Be, Bi, C, Ca, Cd, Ce, Cl,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>157</th>\n",
       "      <td>https://www.mindat.org//loc-108691.html</td>\n",
       "      <td>Taaffeite</td>\n",
       "      <td>None</td>\n",
       "      <td>, Tunduru District, Ruvuma Region, Tanzania,i,...</td>\n",
       "      <td>[]</td>\n",
       "      <td>Aw : Tropical savanna, wet</td>\n",
       "      <td>-11.04056,37.32917</td>\n",
       "      <td>[Al, B, Be, C, Mg, O, Si, Zr]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>158 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                Location URL Stone Refernced  \\\n",
       "0    https://www.mindat.org//loc-189430.html       Tanzanite   \n",
       "1        https://www.mindat.org/loc-151.html      Black_Opal   \n",
       "2    https://www.mindat.org//loc-146632.html      Black_Opal   \n",
       "3    https://www.mindat.org//loc-235813.html      Black_Opal   \n",
       "4    https://www.mindat.org//loc-146633.html      Black_Opal   \n",
       "..                                       ...             ...   \n",
       "153     https://www.mindat.org/loc-3142.html       Taaffeite   \n",
       "154  https://www.mindat.org//loc-123069.html       Taaffeite   \n",
       "155    https://www.mindat.org//loc-3167.html       Taaffeite   \n",
       "156   https://www.mindat.org//loc-67493.html       Taaffeite   \n",
       "157  https://www.mindat.org//loc-108691.html       Taaffeite   \n",
       "\n",
       "          Minerals Recorded  \\\n",
       "0                      None   \n",
       "1                      None   \n",
       "2               ⓘ Gemstones   \n",
       "3                      None   \n",
       "4                      None   \n",
       "..                      ...   \n",
       "153  ⓘ Molybdenumⓘ Vanadium   \n",
       "154                    None   \n",
       "155                    None   \n",
       "156       ⓘ Ironⓘ Manganese   \n",
       "157                    None   \n",
       "\n",
       "                                         Location Name  \\\n",
       "0    , Lelatema Mountains, Simanjiro District, Many...   \n",
       "1    , Korogwe Rural District, Tanga Region, Tanzan...   \n",
       "2    , Finch Co., New South Wales, Australia,i,Ligh...   \n",
       "3    , Lightning Ridge, Finch Co., New South Wales,...   \n",
       "4    , Coocoran Opal Fields, Lightning Ridge, Finch...   \n",
       "..                                                 ...   \n",
       "153  , Republic of Karelia, Russia,i,Pitkyaranta mi...   \n",
       "154  , Polonnaruwa District, North Central Province...   \n",
       "155  , Ratnapura, Ratnapura District, Sabaragamuwa ...   \n",
       "156  , Långban Ore District, Filipstad, Värmland Co...   \n",
       "157  , Tunduru District, Ruvuma Region, Tanzania,i,...   \n",
       "\n",
       "                                            Rock Types  \\\n",
       "0                                                   []   \n",
       "1                          [ⓘ, Gneiss, ⓘ, 'Pegmatite']   \n",
       "2    [ⓘ, Claystone, ⓘ, Gravel, ⓘ, Mudstone, ⓘ, Sand...   \n",
       "3                                                   []   \n",
       "4                                                   []   \n",
       "..                                                 ...   \n",
       "153  [ⓘ, Basalt, ⓘ, Gneiss, ⓘ, Granite, ⓘ, 'Graniti...   \n",
       "154                                                 []   \n",
       "155                                                 []   \n",
       "156               [ⓘ, Biotitite, ⓘ, Granite, ⓘ, Skarn]   \n",
       "157                                                 []   \n",
       "\n",
       "                                  Climate  Decimal Coordinates  \\\n",
       "0              Aw : Tropical savanna, wet    -3.58333,37.00833   \n",
       "1           Am : Tropical monsoon climate    -4.52833,38.73139   \n",
       "2    BSh : Hot semi-arid (steppe) climate  -29.43146,147.98010   \n",
       "3    BSh : Hot semi-arid (steppe) climate  -29.46333,147.76083   \n",
       "4    BSh : Hot semi-arid (steppe) climate  -29.44833,147.78167   \n",
       "..                                    ...                  ...   \n",
       "153               Dfc : Subarctic climate    61.55000,31.50000   \n",
       "154      Af : Tropical rainforest climate     7.78060,80.81840   \n",
       "155      Af : Tropical rainforest climate     6.63702,80.36177   \n",
       "156               Dfc : Subarctic climate    59.85538,14.26480   \n",
       "157            Aw : Tropical savanna, wet   -11.04056,37.32917   \n",
       "\n",
       "                                              Minerals  \n",
       "0    [Al, As, B, C, Ca, Ce, Cl, Cr, Cu, F, Fe, Ga, ...  \n",
       "1    [Al, B, C, Ca, Ce, Cl, Cr, Cu, F, Fe, Gd, H, L...  \n",
       "2                           [Al, Ca, H, Mg, Na, O, Si]  \n",
       "3                                           [H, O, Si]  \n",
       "4                            [Valid Species, H, O, Si]  \n",
       "..                                                 ...  \n",
       "153  [Ag, Al, As, Au, B, Ba, Be, Bi, C, Ca, Ce, Cl,...  \n",
       "154               [Al, Be, Ca, H, K, Mg, Na, O, Si, W]  \n",
       "155                                    [Al, Be, Mg, O]  \n",
       "156  [Ag, Al, As, B, Ba, Be, Bi, C, Ca, Cd, Ce, Cl,...  \n",
       "157                      [Al, B, Be, C, Mg, O, Si, Zr]  \n",
       "\n",
       "[158 rows x 8 columns]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rare_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Stone_Name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Tanzanite</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Black_Opal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Ganterite</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Alexandrite</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Benitoite</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Painite</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Red_Beryl</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Jadeite</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Ammolite</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Muscovite</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Poudretteite</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Taaffeite</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Stone_Name\n",
       "0      Tanzanite\n",
       "1     Black_Opal\n",
       "2      Ganterite\n",
       "3    Alexandrite\n",
       "4      Benitoite\n",
       "5        Painite\n",
       "6      Red_Beryl\n",
       "7        Jadeite\n",
       "8       Ammolite\n",
       "9      Muscovite\n",
       "10  Poudretteite\n",
       "11     Taaffeite"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rare_stone_category = pd.DataFrame(index=range(12),columns=[\"Stone_Name\"])\n",
    "rare_stone_category[\"Stone_Name\"]=rare_stones_names\n",
    "rare_stone_category"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "rare_df.to_csv(\"rarest_stones_df.csv\")\n",
    "rare_stone_category.to_csv(\"rarest_stones_category_dictionary.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}