scripts/scrape-dry-run-getmeta.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from selenium import webdriver\n",
    "from selenium.webdriver.chrome.options import Options\n",
    "from selenium.webdriver.support.select import Select\n",
    "from IPython.core.display import Image, display"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "options = Options()\n",
    "\n",
    "options.add_experimental_option(\"prefs\", {\n",
    "      \"download.default_directory\": r\"/opt/data/bihar-all\",\n",
    "      \"download.prompt_for_download\": False,\n",
    "      \"download.directory_upgrade\": True,\n",
    "      \"safebrowsing.enabled\": True\n",
    "    })\n",
    " \n",
    "options.add_argument('headless')\n",
    "options.add_argument('--disable-infobars')\n",
    "options.add_argument('--disable-dev-shm-usage')\n",
    "options.add_argument('--no-sandbox')\n",
    "options.add_argument('--remote-debugging-port=9222')\n",
    "\n",
    "driver = webdriver.Chrome('/opt/tools/webdriver/chromedriver', options=options)\n",
    "\n",
    "driver.get('http://ele.bihar.gov.in/pdfsearch/')\n",
    "\n",
    "# get the image source"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 330 330\n",
      "1 309 639\n",
      "2 271 910\n",
      "3 282 1192\n",
      "4 245 1437\n",
      "5 251 1688\n",
      "6 276 1964\n",
      "7 275 2239\n",
      "8 264 2503\n",
      "9 279 2782\n",
      "10 282 3064\n",
      "11 284 3348\n",
      "12 253 3601\n",
      "13 254 3855\n",
      "14 257 4112\n",
      "15 234 4346\n",
      "16 315 4661\n",
      "17 243 4904\n",
      "18 298 5202\n",
      "19 280 5482\n",
      "20 290 5772\n",
      "21 288 6060\n",
      "22 306 6366\n",
      "23 300 6666\n",
      "24 310 6976\n",
      "25 298 7274\n",
      "26 305 7579\n",
      "27 288 7867\n",
      "28 275 8142\n",
      "29 263 8405\n",
      "30 272 8677\n",
      "31 295 8972\n",
      "32 299 9271\n",
      "33 314 9585\n",
      "34 323 9908\n",
      "35 350 10258\n",
      "36 331 10589\n",
      "37 320 10909\n",
      "38 323 11232\n",
      "39 342 11574\n",
      "40 307 11881\n",
      "41 287 12168\n",
      "42 295 12463\n",
      "43 273 12736\n",
      "44 297 13033\n",
      "45 296 13329\n",
      "46 297 13626\n",
      "47 292 13918\n",
      "48 280 14198\n",
      "49 271 14469\n",
      "50 287 14756\n",
      "51 274 15030\n",
      "52 273 15303\n",
      "53 271 15574\n",
      "54 247 15821\n",
      "55 300 16121\n",
      "56 261 16382\n",
      "57 274 16656\n",
      "58 301 16957\n",
      "59 306 17263\n",
      "60 317 17580\n",
      "61 298 17878\n",
      "62 253 18131\n",
      "63 265 18396\n",
      "64 327 18723\n",
      "65 290 19013\n",
      "66 282 19295\n",
      "67 250 19545\n",
      "68 262 19807\n",
      "69 332 20139\n",
      "70 301 20440\n",
      "71 311 20751\n",
      "72 341 21092\n",
      "73 312 21404\n",
      "74 347 21751\n",
      "75 338 22089\n",
      "76 307 22396\n",
      "77 248 22644\n",
      "78 248 22892\n",
      "79 283 23175\n",
      "80 269 23444\n",
      "81 277 23721\n",
      "82 294 24015\n",
      "83 246 24261\n",
      "84 293 24554\n",
      "85 294 24848\n",
      "86 303 25151\n",
      "87 314 25465\n",
      "88 311 25776\n",
      "89 278 26054\n",
      "90 272 26326\n",
      "91 261 26587\n",
      "92 286 26873\n",
      "93 304 27177\n",
      "94 302 27479\n",
      "95 283 27762\n",
      "96 311 28073\n",
      "97 303 28376\n",
      "98 305 28681\n",
      "99 285 28966\n",
      "100 313 29279\n",
      "101 335 29614\n",
      "102 366 29980\n",
      "103 302 30282\n",
      "104 297 30579\n",
      "105 273 30852\n",
      "106 319 31171\n",
      "107 287 31458\n",
      "108 313 31771\n",
      "109 298 32069\n",
      "110 321 32390\n",
      "111 314 32704\n",
      "112 297 33001\n",
      "113 298 33299\n",
      "114 321 33620\n",
      "115 297 33917\n",
      "116 279 34196\n",
      "117 317 34513\n",
      "118 300 34813\n",
      "119 263 35076\n",
      "120 270 35346\n",
      "121 282 35628\n",
      "122 318 35946\n",
      "123 334 36280\n",
      "124 326 36606\n",
      "125 283 36889\n",
      "126 275 37164\n",
      "127 333 37497\n",
      "128 284 37781\n",
      "129 270 38051\n",
      "130 318 38369\n",
      "131 301 38670\n",
      "132 269 38939\n",
      "133 295 39234\n",
      "134 259 39493\n",
      "135 269 39762\n",
      "136 255 40017\n",
      "137 252 40269\n",
      "138 318 40587\n",
      "139 274 40861\n",
      "140 241 41102\n",
      "141 299 41401\n",
      "142 278 41679\n",
      "143 325 42004\n",
      "144 245 42249\n",
      "145 312 42561\n",
      "146 244 42805\n",
      "147 252 43057\n",
      "148 251 43308\n",
      "149 299 43607\n",
      "150 300 43907\n",
      "151 240 44147\n",
      "152 260 44407\n",
      "153 323 44730\n",
      "154 330 45060\n",
      "155 317 45377\n",
      "156 325 45702\n",
      "157 307 46009\n",
      "158 308 46317\n",
      "159 326 46643\n",
      "160 267 46910\n",
      "161 261 47171\n",
      "162 329 47500\n",
      "163 324 47824\n",
      "164 335 48159\n",
      "165 332 48491\n",
      "166 332 48823\n",
      "167 364 49187\n",
      "168 263 49450\n",
      "169 234 49684\n",
      "170 294 49978\n",
      "171 382 50360\n",
      "172 293 50653\n",
      "173 305 50958\n",
      "174 322 51280\n",
      "175 322 51602\n",
      "176 330 51932\n",
      "177 274 52206\n",
      "178 289 52495\n",
      "179 275 52770\n",
      "180 408 53178\n",
      "181 346 53524\n",
      "182 372 53896\n",
      "183 325 54221\n",
      "184 281 54502\n",
      "185 325 54827\n",
      "186 335 55162\n",
      "187 357 55519\n",
      "188 382 55901\n",
      "189 310 56211\n",
      "190 341 56552\n",
      "191 289 56841\n",
      "192 314 57155\n",
      "193 339 57494\n",
      "194 276 57770\n",
      "195 320 58090\n",
      "196 324 58414\n",
      "197 300 58714\n",
      "198 323 59037\n",
      "199 286 59323\n",
      "200 325 59648\n",
      "201 331 59979\n",
      "202 283 60262\n",
      "203 285 60547\n",
      "204 290 60837\n",
      "205 348 61185\n",
      "206 306 61491\n",
      "207 347 61838\n",
      "208 351 62189\n",
      "209 308 62497\n",
      "210 289 62786\n",
      "211 281 63067\n",
      "212 332 63399\n",
      "213 251 63650\n",
      "214 257 63907\n",
      "215 316 64223\n",
      "216 289 64512\n",
      "217 252 64764\n",
      "218 326 65090\n",
      "219 336 65426\n",
      "220 305 65731\n",
      "221 286 66017\n",
      "222 322 66339\n",
      "223 348 66687\n",
      "224 329 67016\n",
      "225 286 67302\n",
      "226 329 67631\n",
      "227 317 67948\n",
      "228 345 68293\n",
      "229 227 68520\n",
      "230 351 68871\n",
      "231 288 69159\n",
      "232 327 69486\n",
      "233 309 69795\n",
      "234 307 70102\n",
      "235 366 70468\n",
      "236 338 70806\n",
      "237 311 71117\n",
      "238 343 71460\n",
      "239 308 71768\n",
      "240 298 72066\n",
      "241 336 72402\n",
      "242 321 72723\n"
     ]
    }
   ],
   "source": [
    "n = 0\n",
    "sel = Select(driver.find_element_by_xpath('//select[@id=\"DropDownList1\"]'))\n",
    "a = 0\n",
    "data = []\n",
    "while a < len(sel.options):\n",
    "    ac = sel.options[a].text\n",
    "    sel.select_by_index(a)\n",
    "    sel2 = Select(driver.find_element_by_xpath('//select[@id=\"DropDownList2\"]'))\n",
    "    c = len(sel2.options)\n",
    "    n += c\n",
    "    for d in sel2.options:\n",
    "        data.append({'assembly_segment': ac, 'part_number': d.text})\n",
    "    sel = Select(driver.find_element_by_xpath('//select[@id=\"DropDownList1\"]'))\n",
    "    print(a, c, n)\n",
    "    a += 1\n",
    "    #break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>assembly_segment</th>\n",
       "      <th>part_number</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1 - वाल्मीकिनगर</td>\n",
       "      <td>1 - उत्क्रमित मध्य विद्यालय, ठाढ़ी</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1 - वाल्मीकिनगर</td>\n",
       "      <td>2 - वन विकास भवन कोतरहा पूर्वी भाग</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1 - वाल्मीकिनगर</td>\n",
       "      <td>3 - वन विकास भवन कोतरहा पश्चिम भाग</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1 - वाल्मीकिनगर</td>\n",
       "      <td>4 - उत्क्रमित मध्य विद्यालय लक्ष्मीपुर पूर्वी भाग</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1 - वाल्मीकिनगर</td>\n",
       "      <td>5 - उत्क्रमित मध्य विद्यालय लक्ष्मीपुर पश्चिम भाग</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72718</th>\n",
       "      <td>243 - चकाई</td>\n",
       "      <td>317 - पंचायत भवन दुल्लमपुर दांया भाग</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72719</th>\n",
       "      <td>243 - चकाई</td>\n",
       "      <td>318 - पंचायत भवन दुलामपुर बांया भाग</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72720</th>\n",
       "      <td>243 - चकाई</td>\n",
       "      <td>319 - प्राथमिक विद्यालय चितरडीह</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72721</th>\n",
       "      <td>243 - चकाई</td>\n",
       "      <td>320 - वन विभाग  निरिक्षण भवन प्रतापपुर</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72722</th>\n",
       "      <td>243 - चकाई</td>\n",
       "      <td>321 - वन विभाग निरीक्षण भवन प्रतापपुर बांया भाग</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>72723 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      assembly_segment                                        part_number\n",
       "0      1 - वाल्मीकिनगर                 1 - उत्क्रमित मध्य विद्यालय, ठाढ़ी\n",
       "1      1 - वाल्मीकिनगर                 2 - वन विकास भवन कोतरहा पूर्वी भाग\n",
       "2      1 - वाल्मीकिनगर                 3 - वन विकास भवन कोतरहा पश्चिम भाग\n",
       "3      1 - वाल्मीकिनगर  4 - उत्क्रमित मध्य विद्यालय लक्ष्मीपुर पूर्वी भाग\n",
       "4      1 - वाल्मीकिनगर  5 - उत्क्रमित मध्य विद्यालय लक्ष्मीपुर पश्चिम भाग\n",
       "...                ...                                                ...\n",
       "72718       243 - चकाई               317 - पंचायत भवन दुल्लमपुर दांया भाग\n",
       "72719       243 - चकाई                318 - पंचायत भवन दुलामपुर बांया भाग\n",
       "72720       243 - चकाई                    319 - प्राथमिक विद्यालय चितरडीह\n",
       "72721       243 - चकाई             320 - वन विभाग  निरिक्षण भवन प्रतापपुर\n",
       "72722       243 - चकाई    321 - वन विभाग निरीक्षण भवन प्रतापपुर बांया भाग\n",
       "\n",
       "[72723 rows x 2 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame(data)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('../metadata_and_log_files/metadata.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}