drl_demo_v3.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from google.cloud import documentai_v1beta3 as documentai\n",
    "from google.cloud import storage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "storage_client = storage.Client()\n",
    "bucket = storage_client.get_bucket('jayadeep-etl-platform')\n",
    "blob = bucket.get_blob('docuai/demo-doc.pdf')\n",
    "content = blob.download_as_string()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "project_id = \"jayadeep-etl-platform\"\n",
    "location = \"us\"\n",
    "processor_id = \"900a4ac66b3ab487\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "doc_client = documentai.DocumentProcessorServiceClient()\n",
    "name = f\"projects/{project_id}/locations/{location}/processors/{processor_id}\"\n",
    "document = {\"content\": content, \"mime_type\": \"application/pdf\"}\n",
    "request = {\"name\": name, \"document\": document}\n",
    "result = doc_client.process_document(request=request)\n",
    "document = result.document"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "document_pages = document.pages"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Extract Information based on Normalized Vertices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_text(doc_element: dict, document: dict):\n",
    "    \"\"\"\n",
    "    Document AI identifies form fields by their offsets\n",
    "    in document text. This function converts offsets\n",
    "    to text snippets.\n",
    "    \"\"\"\n",
    "    response = \"\"\n",
    "    # If a text segment spans several lines, it will\n",
    "    # be stored in different text segments.\n",
    "    for segment in doc_element.text_anchor.text_segments:\n",
    "        start_index = (\n",
    "            int(segment.start_index)\n",
    "            if segment in doc_element.text_anchor.text_segments\n",
    "            else 0\n",
    "        )\n",
    "        end_index = int(segment.end_index)\n",
    "        response += document.text[start_index:end_index]\n",
    "    return response.replace('\\n','')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Extract All Information from Document"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Paragraph text: Sample document:\n",
      "\n",
      "Paragraph text: Description: This text needs to be extracted\n",
      "\n",
      "Paragraph text: Strength: This text needs to be extracted\n",
      "\n",
      "Paragraph text: Name: ABC\n",
      "\n",
      "Paragraph text: Quantity: 250\n",
      "\n",
      "Paragraph text: Issue No: PQ-43-97A\n",
      "\n",
      "Paragraph text: Date: 01-01-2020\n",
      "\n",
      "Paragraph text: Sequence\n",
      "\n",
      "Paragraph text: Frames\n",
      "\n",
      "Paragraph text: Tracking without Tracking with\n",
      "distance cue P/R/F distance cue P/R/F\n",
      "\n",
      "Paragraph text: UAV1\n",
      "\n",
      "Paragraph text: 616\n",
      "\n",
      "Paragraph text: 0.90/0.90/0.90\n",
      "\n",
      "Paragraph text: 0.94/0.94/0.94\n",
      "\n",
      "Paragraph text: UAV 2\n",
      "\n",
      "Paragraph text: 1833\n",
      "\n",
      "Paragraph text: 0.92/0.88/0.90\n",
      "\n",
      "Paragraph text: 0.96/0.99/0.97\n",
      "\n",
      "Paragraph text: Traffic 1\n",
      "\n",
      "Paragraph text: 156\n",
      "\n",
      "Paragraph text: 0.88/0.91/0.89\n",
      "\n",
      "Paragraph text: 0.96/0.98/0.97\n",
      "\n",
      "Paragraph text: Traffic 2\n",
      "\n",
      "Paragraph text: 227\n",
      "\n",
      "Paragraph text: 0.85/0.90/0.87\n",
      "\n",
      "Paragraph text: 0.97/0.97/0.97\n",
      "\n",
      "Paragraph text: Pedestrian 2\n",
      "\n",
      "Paragraph text: 338\n",
      "\n",
      "Paragraph text: 0.93/0.95/0.94\n",
      "\n",
      "Paragraph text: 1.00/1.00/1.00\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for page in document_pages:\n",
    "    paragraphs = page.paragraphs\n",
    "    for paragraph in paragraphs:\n",
    "        paragraph_text = get_text(paragraph.layout, document)\n",
    "        print(f\"Paragraph text: {paragraph_text}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Extract Table Information"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Page number: 1\n",
      "Table 0: \n",
      "Header Row 0: Strength: This text needs to be extracted\t\n",
      "Row 0: Name: ABC\tQuantity: 250\n",
      "Row 1: Issue No: PQ-43-97A\tDate: 01-01-2020\n",
      "Table 1: \n",
      "Header Row 0: Sequence\tFrames\tTracking without distance cue P/R/F \tTracking withdistance cue P/R/F\n",
      "Row 0: UAV1\t616\t0.90/0.90/0.90\t0.94/0.94/0.94\n",
      "Row 1: UAV 2\t1833\t0.92/0.88/0.90\t0.96/0.99/0.97\n",
      "Row 2: Traffic 1\t156\t0.88/0.91/0.89\t0.96/0.98/0.97\n",
      "Row 3: Traffic 2\t227\t0.85/0.90/0.87\t0.97/0.97/0.97\n",
      "Row 4: Pedestrian 2\t338\t0.93/0.95/0.94\t1.00/1.00/1.00\n"
     ]
    }
   ],
   "source": [
    "for page in document.pages:\n",
    "    print(\"Page number: {}\".format(page.page_number))\n",
    "    for table_num, table in enumerate(page.tables):\n",
    "        print(\"Table {}: \".format(table_num))\n",
    "        for row_num, row in enumerate(table.header_rows):\n",
    "            cells = \"\\t\".join([get_text(cell.layout,document) for cell in row.cells])\n",
    "            print(\"Header Row {}: {}\".format(row_num, cells))\n",
    "        for row_num, row in enumerate(table.body_rows):\n",
    "            #print (row)\n",
    "            cells = \"\\t\".join([get_text(cell.layout,document) for cell in row.cells])\n",
    "            #print (cells)\n",
    "            print(\"Row {}: {}\".format(row_num, cells))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Extract Key Value Pairs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Key Value Pairs: ('Date: ', '01-01-2020')\n",
      "Key Value Pairs: ('Quantity: ', '250')\n",
      "Key Value Pairs: ('Issue No: ', 'PQ-43-97A')\n",
      "Key Value Pairs: ('Name: ', 'ABC')\n",
      "Key Value Pairs: ('Strength: ', 'This text needs to be extracted')\n",
      "Key Value Pairs: ('Description: ', 'This text needs to be extracted')\n"
     ]
    }
   ],
   "source": [
    "for page in document.pages:\n",
    "    form_fields = page.form_fields\n",
    "    for form_field in form_fields:\n",
    "        field_name = get_text(form_field.field_name, document)\n",
    "        field_value = get_text(form_field.field_value, document)\n",
    "\n",
    "        print(f\"Key Value Pairs: {field_name, field_value}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "environment": {
   "name": "tf2-2-3-gpu.2-3.m59",
   "type": "gcloud",
   "uri": "gcr.io/deeplearning-platform-release/tf2-2-3-gpu.2-3:m59"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}