Data-analysis-DoH.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scapy.all import *\n",
    "load_layer(\"ssl\")\n",
    "load_layer(\"tls\")\n",
    "import pprint\n",
    "import json, time\n",
    "from os import listdir\n",
    "from os.path import isfile, join\n",
    "import pandas as pd\n",
    "mypath = \"/home/bx1/NUS/DoH Analysis/data2\"  # path where the data is stored"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "google.com\n",
      "youtube.com\n"
     ]
    }
   ],
   "source": [
    "# get urls from alexadomain.json \n",
    "rank_from = 1\n",
    "rank_to = 2\n",
    "\n",
    "with open(\"alexadomain.json\") as jsf:\n",
    "        data = json.load(jsf)\n",
    "\n",
    "for i in range(rank_from-1,rank_to):\n",
    "    print(data[i][\"DataUrl\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'DataUrl': '24smi.info',\n",
       " 'Country': {'Rank': '1687',\n",
       "  'Reach': {'PerMillion': '505'},\n",
       "  'PageViews': {'PerMillion': '24.7', 'PerUser': '1.77'}},\n",
       " 'Global': {'Rank': '1687'}}"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[1687]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4000\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'ocsp.pki.goog': ['74.125.24.94', '2404:6800:4003:c04::5e'],\n",
       " 'snippets.cdn.mozilla.net': ['13.224.253.29',\n",
       "  '13.224.253.122',\n",
       "  '13.224.253.72',\n",
       "  '13.224.253.12'],\n",
       " 'ocsp.digicert.com': ['117.18.237.29'],\n",
       " 'tracking-protection.cdn.mozilla.net': ['13.224.253.120',\n",
       "  '13.224.253.121',\n",
       "  '13.224.253.76',\n",
       "  '13.224.253.24'],\n",
       " 'ogs.google.com': ['74.125.24.102',\n",
       "  '74.125.24.101',\n",
       "  '74.125.24.138',\n",
       "  '74.125.24.113',\n",
       "  '74.125.24.100',\n",
       "  '74.125.24.139',\n",
       "  '2404:6800:4003:c03::66'],\n",
       " 'www.google.com.sg': ['172.217.194.94', '2404:6800:4003:c02::5e'],\n",
       " 'apis.google.com': ['2404:6800:4003:80a::200e',\n",
       "  '74.125.24.139',\n",
       "  '74.125.24.100',\n",
       "  '74.125.24.102',\n",
       "  '74.125.24.101',\n",
       "  '74.125.24.113',\n",
       "  '74.125.24.138'],\n",
       " 'www.gstatic.com': ['2404:6800:4003:c02::5e', '74.125.68.94'],\n",
       " 'adservice.google.com': ['2404:6800:4003:c02::9c',\n",
       "  '74.125.24.155',\n",
       "  '74.125.24.157',\n",
       "  '74.125.24.154',\n",
       "  '74.125.24.156'],\n",
       " 'www.google.com': ['172.217.194.147',\n",
       "  '172.217.194.106',\n",
       "  '172.217.194.103',\n",
       "  '172.217.194.99',\n",
       "  '172.217.194.104',\n",
       "  '172.217.194.105',\n",
       "  '2404:6800:4003:c03::67'],\n",
       " 'ssl.gstatic.com': ['2404:6800:4003:c04::5e', '74.125.200.94'],\n",
       " 'location.services.mozilla.com': ['52.215.71.87',\n",
       "  '54.72.168.141',\n",
       "  '34.253.23.107'],\n",
       " 'shavar.services.mozilla.com': ['54.149.19.17',\n",
       "  '52.88.59.72',\n",
       "  '35.164.3.68',\n",
       "  '54.148.248.23',\n",
       "  '52.33.61.229',\n",
       "  '35.165.44.141'],\n",
       " 'query-stime': 1571654861.8919137,\n",
       " 'query-status': 1,\n",
       " 'query-url': 'http://www.google.com',\n",
       " 'tdump-stat': 1,\n",
       " 'query-etime': 1571654863.4474425}"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# parse all urls.json files and store in <dns_res_stat>\n",
    "data_dir = mypath + \"/urls/\"\n",
    "\n",
    "onlyfiles = [f for f in listdir(data_dir) if isfile(join(data_dir, f))]\n",
    "\n",
    "dns_res_stat = []\n",
    "for i in range(1,len(onlyfiles)+1):\n",
    "    with open(data_dir + \"url\" + str(i) + \".json\"  ,\"r\") as f:\n",
    "        json_obj =  json.load(f)\n",
    "        dns_res_stat.append(json_obj)\n",
    "        \n",
    "        \n",
    "print(len(dns_res_stat))\n",
    "dns_res_stat[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# function to get the time taken by the data collecor script (uses <query-time>)\n",
    "def get_time_interval(dns_res_stat):\n",
    "    st = dns_res_stat[0][\"query-stime\"]\n",
    "    end = dns_res_stat[len(dns_res_stat)-1][\"query-stime\"]\n",
    "    \n",
    "    print(\"start-time: \" + time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(st)))\n",
    "    print(\"end-time: \" + time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(end)))\n",
    "    print(\"Took \" + str(end-st) + \" seconds for \" + str(len(dns_res_stat)) + \" websites\" )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# function to get the failed queries - due to URL format issue (https://www.) or network issue\n",
    "def get_failed_queries(dns_res_stat):\n",
    "    failed_query = []\n",
    "    \n",
    "    url_skip_stat = []\n",
    "    for i in range(0,len(dns_res_stat)):\n",
    "        if dns_res_stat[i][\"query-status\"] == 0:\n",
    "            #pint(str(dns_res_stat[i][\"query-url\"]) + \" : \" + str(dns_res_stat[i][\"query-stime\"]))\n",
    "            #print(\"----->\" + str(dns_res_stat[i]))\n",
    "            failed_query.append(i)\n",
    "    return failed_query "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "start-time: 2019-10-21 16:17:41\n",
      "end-time: 2019-10-24 15:06:21\n",
      "Took 254919.99915790558 seconds for 4000 websites\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "963"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#statistics from the DNS resolutions from firefox \n",
    "\n",
    "get_time_interval(dns_res_stat)   \n",
    "failed_query = get_failed_queries(dns_res_stat) #failed_query have the ranks of the URLS\n",
    "len(failed_query) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'http://www.yahoo.co.jp'"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#individual websites which is failed \n",
    "dns_res_stat[failed_query[3]][\"query-url\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1046, 472)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# parse pcaps and store in <pcap>\n",
    "data_dir = mypath + \"/traces/\" \n",
    "\n",
    "query_no = 0\n",
    "onlyfiles = [f for f in listdir(data_dir) if isfile(join(data_dir, f))]\n",
    "onlyfiles = sorted(onlyfiles)\n",
    "\n",
    "pcap = rdpcap(data_dir+onlyfiles[query_no])\n",
    "tls_pcap = pcap[TLS]\n",
    "len(pcap),len(tls_pcap)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Ether  dst=02:42:66:c2:79:b2 src=02:42:ac:11:00:02 type=IPv4 |<IP  version=4 ihl=5 tos=0x0 len=60 id=17227 flags=DF frag=0 ttl=64 proto=tcp chksum=0x3b4e src=172.17.0.2 dst=8.8.8.8 |<TCP  sport=55136 dport=https seq=1045311707 ack=0 dataofs=10 reserved=0 flags=S window=64240 chksum=0xbc51 urgptr=0 options=[('MSS', 1460), ('SAckOK', b''), ('Timestamp', (2515866414, 0)), ('NOP', None), ('WScale', 7)] |>>>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pcap[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1, b'dns.google')"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# rough work to identify how to parse the fields and get correspoding values\n",
    "temp = tls_pcap[0]\n",
    "layert = type(temp[TCP].payload)\n",
    "p = temp[layert].fields['msg'][0]\n",
    "p.msgtype,p.ext[0].servernames[0].servername"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['version', 'ihl', 'tos', 'len', 'id', 'flags', 'frag', 'ttl', 'proto', 'chksum', 'src', 'dst', 'options'] ['sport', 'dport', 'seq', 'ack', 'dataofs', 'reserved', 'flags', 'window', 'chksum', 'urgptr', 'options'] ['type', 'version', 'len', 'iv', 'msg', 'mac', 'pad', 'padlen']\n"
     ]
    }
   ],
   "source": [
    "# get all the fields in field-value pairs from IP,TCP and TLS layers\n",
    "temp_pcap = pcap\n",
    "ip_fields = [field.name for field in IP().fields_desc]\n",
    "tcp_fields = [field.name for field in TCP().fields_desc]\n",
    "tls_fields = [field.name for field in TLS().fields_desc]\n",
    "print(ip_fields,tcp_fields,tls_fields)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "########### Rough work BEGINs to get the fields and converting it into a DataFrame ###########\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>version</th>\n",
       "      <th>ihl</th>\n",
       "      <th>tos</th>\n",
       "      <th>len</th>\n",
       "      <th>id</th>\n",
       "      <th>flags</th>\n",
       "      <th>frag</th>\n",
       "      <th>ttl</th>\n",
       "      <th>proto</th>\n",
       "      <th>chksum</th>\n",
       "      <th>src</th>\n",
       "      <th>dst</th>\n",
       "      <th>options</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>69</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>91</td>\n",
       "      <td>31851</td>\n",
       "      <td>DF</td>\n",
       "      <td>0</td>\n",
       "      <td>64</td>\n",
       "      <td>6</td>\n",
       "      <td>527</td>\n",
       "      <td>172.17.0.2</td>\n",
       "      <td>8.8.8.8</td>\n",
       "      <td>0</td>\n",
       "      <td>1571654860.349295</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>52</td>\n",
       "      <td>51500</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "      <td>104</td>\n",
       "      <td>6</td>\n",
       "      <td>52596</td>\n",
       "      <td>8.8.8.8</td>\n",
       "      <td>172.17.0.2</td>\n",
       "      <td>0</td>\n",
       "      <td>1571654860.355563</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>71</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>60</td>\n",
       "      <td>30460</td>\n",
       "      <td>DF</td>\n",
       "      <td>0</td>\n",
       "      <td>64</td>\n",
       "      <td>6</td>\n",
       "      <td>39806</td>\n",
       "      <td>172.17.0.2</td>\n",
       "      <td>52.215.71.87</td>\n",
       "      <td>0</td>\n",
       "      <td>1571654860.409490</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>60</td>\n",
       "      <td>0</td>\n",
       "      <td>DF</td>\n",
       "      <td>0</td>\n",
       "      <td>227</td>\n",
       "      <td>6</td>\n",
       "      <td>28538</td>\n",
       "      <td>52.215.71.87</td>\n",
       "      <td>172.17.0.2</td>\n",
       "      <td>0</td>\n",
       "      <td>1571654860.575829</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>52</td>\n",
       "      <td>30461</td>\n",
       "      <td>DF</td>\n",
       "      <td>0</td>\n",
       "      <td>64</td>\n",
       "      <td>6</td>\n",
       "      <td>39813</td>\n",
       "      <td>172.17.0.2</td>\n",
       "      <td>52.215.71.87</td>\n",
       "      <td>0</td>\n",
       "      <td>1571654860.575851</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>569</td>\n",
       "      <td>30462</td>\n",
       "      <td>DF</td>\n",
       "      <td>0</td>\n",
       "      <td>64</td>\n",
       "      <td>6</td>\n",
       "      <td>39295</td>\n",
       "      <td>172.17.0.2</td>\n",
       "      <td>52.215.71.87</td>\n",
       "      <td>0</td>\n",
       "      <td>1571654860.576816</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>113</td>\n",
       "      <td>31852</td>\n",
       "      <td>DF</td>\n",
       "      <td>0</td>\n",
       "      <td>64</td>\n",
       "      <td>6</td>\n",
       "      <td>504</td>\n",
       "      <td>172.17.0.2</td>\n",
       "      <td>8.8.8.8</td>\n",
       "      <td>0</td>\n",
       "      <td>1571654860.601469</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>147</td>\n",
       "      <td>31853</td>\n",
       "      <td>DF</td>\n",
       "      <td>0</td>\n",
       "      <td>64</td>\n",
       "      <td>6</td>\n",
       "      <td>469</td>\n",
       "      <td>172.17.0.2</td>\n",
       "      <td>8.8.8.8</td>\n",
       "      <td>0</td>\n",
       "      <td>1571654860.601517</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>109</td>\n",
       "      <td>31854</td>\n",
       "      <td>DF</td>\n",
       "      <td>0</td>\n",
       "      <td>64</td>\n",
       "      <td>6</td>\n",
       "      <td>506</td>\n",
       "      <td>172.17.0.2</td>\n",
       "      <td>8.8.8.8</td>\n",
       "      <td>0</td>\n",
       "      <td>1571654860.601595</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>78</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>147</td>\n",
       "      <td>31855</td>\n",
       "      <td>DF</td>\n",
       "      <td>0</td>\n",
       "      <td>64</td>\n",
       "      <td>6</td>\n",
       "      <td>467</td>\n",
       "      <td>172.17.0.2</td>\n",
       "      <td>8.8.8.8</td>\n",
       "      <td>0</td>\n",
       "      <td>1571654860.601616</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>52</td>\n",
       "      <td>51634</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "      <td>104</td>\n",
       "      <td>6</td>\n",
       "      <td>52462</td>\n",
       "      <td>8.8.8.8</td>\n",
       "      <td>172.17.0.2</td>\n",
       "      <td>0</td>\n",
       "      <td>1571654860.603568</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>52</td>\n",
       "      <td>51635</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "      <td>104</td>\n",
       "      <td>6</td>\n",
       "      <td>52461</td>\n",
       "      <td>8.8.8.8</td>\n",
       "      <td>172.17.0.2</td>\n",
       "      <td>0</td>\n",
       "      <td>1571654860.603577</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>52</td>\n",
       "      <td>51636</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "      <td>104</td>\n",
       "      <td>6</td>\n",
       "      <td>52460</td>\n",
       "      <td>8.8.8.8</td>\n",
       "      <td>172.17.0.2</td>\n",
       "      <td>0</td>\n",
       "      <td>1571654860.603911</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>52</td>\n",
       "      <td>51637</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "      <td>104</td>\n",
       "      <td>6</td>\n",
       "      <td>52459</td>\n",
       "      <td>8.8.8.8</td>\n",
       "      <td>172.17.0.2</td>\n",
       "      <td>0</td>\n",
       "      <td>1571654860.603916</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>83</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>175</td>\n",
       "      <td>51646</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "      <td>104</td>\n",
       "      <td>6</td>\n",
       "      <td>52327</td>\n",
       "      <td>8.8.8.8</td>\n",
       "      <td>172.17.0.2</td>\n",
       "      <td>0</td>\n",
       "      <td>1571654860.611998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>265</td>\n",
       "      <td>51647</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "      <td>104</td>\n",
       "      <td>6</td>\n",
       "      <td>52236</td>\n",
       "      <td>8.8.8.8</td>\n",
       "      <td>172.17.0.2</td>\n",
       "      <td>0</td>\n",
       "      <td>1571654860.612008</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>83</td>\n",
       "      <td>51648</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "      <td>104</td>\n",
       "      <td>6</td>\n",
       "      <td>52417</td>\n",
       "      <td>8.8.8.8</td>\n",
       "      <td>172.17.0.2</td>\n",
       "      <td>0</td>\n",
       "      <td>1571654860.612160</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>91</td>\n",
       "      <td>51649</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "      <td>104</td>\n",
       "      <td>6</td>\n",
       "      <td>52408</td>\n",
       "      <td>8.8.8.8</td>\n",
       "      <td>172.17.0.2</td>\n",
       "      <td>0</td>\n",
       "      <td>1571654860.612162</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>87</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>310</td>\n",
       "      <td>51650</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "      <td>104</td>\n",
       "      <td>6</td>\n",
       "      <td>52188</td>\n",
       "      <td>8.8.8.8</td>\n",
       "      <td>172.17.0.2</td>\n",
       "      <td>0</td>\n",
       "      <td>1571654860.612162</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>52</td>\n",
       "      <td>31856</td>\n",
       "      <td>DF</td>\n",
       "      <td>0</td>\n",
       "      <td>64</td>\n",
       "      <td>6</td>\n",
       "      <td>561</td>\n",
       "      <td>172.17.0.2</td>\n",
       "      <td>8.8.8.8</td>\n",
       "      <td>0</td>\n",
       "      <td>1571654860.612195</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>91</td>\n",
       "      <td>31857</td>\n",
       "      <td>DF</td>\n",
       "      <td>0</td>\n",
       "      <td>64</td>\n",
       "      <td>6</td>\n",
       "      <td>521</td>\n",
       "      <td>172.17.0.2</td>\n",
       "      <td>8.8.8.8</td>\n",
       "      <td>0</td>\n",
       "      <td>1571654860.612316</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>113</td>\n",
       "      <td>31858</td>\n",
       "      <td>DF</td>\n",
       "      <td>0</td>\n",
       "      <td>64</td>\n",
       "      <td>6</td>\n",
       "      <td>498</td>\n",
       "      <td>172.17.0.2</td>\n",
       "      <td>8.8.8.8</td>\n",
       "      <td>0</td>\n",
       "      <td>1571654860.612987</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>91</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>142</td>\n",
       "      <td>31859</td>\n",
       "      <td>DF</td>\n",
       "      <td>0</td>\n",
       "      <td>64</td>\n",
       "      <td>6</td>\n",
       "      <td>468</td>\n",
       "      <td>172.17.0.2</td>\n",
       "      <td>8.8.8.8</td>\n",
       "      <td>0</td>\n",
       "      <td>1571654860.613031</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>52</td>\n",
       "      <td>51651</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "      <td>104</td>\n",
       "      <td>6</td>\n",
       "      <td>52445</td>\n",
       "      <td>8.8.8.8</td>\n",
       "      <td>172.17.0.2</td>\n",
       "      <td>0</td>\n",
       "      <td>1571654860.615688</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>125</td>\n",
       "      <td>51656</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "      <td>104</td>\n",
       "      <td>6</td>\n",
       "      <td>52367</td>\n",
       "      <td>8.8.8.8</td>\n",
       "      <td>172.17.0.2</td>\n",
       "      <td>0</td>\n",
       "      <td>1571654860.620835</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>94</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>227</td>\n",
       "      <td>51657</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "      <td>104</td>\n",
       "      <td>6</td>\n",
       "      <td>52264</td>\n",
       "      <td>8.8.8.8</td>\n",
       "      <td>172.17.0.2</td>\n",
       "      <td>0</td>\n",
       "      <td>1571654860.620847</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>83</td>\n",
       "      <td>51658</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "      <td>104</td>\n",
       "      <td>6</td>\n",
       "      <td>52407</td>\n",
       "      <td>8.8.8.8</td>\n",
       "      <td>172.17.0.2</td>\n",
       "      <td>0</td>\n",
       "      <td>1571654860.620962</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>91</td>\n",
       "      <td>51659</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "      <td>104</td>\n",
       "      <td>6</td>\n",
       "      <td>52398</td>\n",
       "      <td>8.8.8.8</td>\n",
       "      <td>172.17.0.2</td>\n",
       "      <td>0</td>\n",
       "      <td>1571654860.620965</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>52</td>\n",
       "      <td>31860</td>\n",
       "      <td>DF</td>\n",
       "      <td>0</td>\n",
       "      <td>64</td>\n",
       "      <td>6</td>\n",
       "      <td>557</td>\n",
       "      <td>172.17.0.2</td>\n",
       "      <td>8.8.8.8</td>\n",
       "      <td>0</td>\n",
       "      <td>1571654860.621065</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>91</td>\n",
       "      <td>31861</td>\n",
       "      <td>DF</td>\n",
       "      <td>0</td>\n",
       "      <td>64</td>\n",
       "      <td>6</td>\n",
       "      <td>517</td>\n",
       "      <td>172.17.0.2</td>\n",
       "      <td>8.8.8.8</td>\n",
       "      <td>0</td>\n",
       "      <td>1571654860.621107</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>60</td>\n",
       "      <td>18017</td>\n",
       "      <td>DF</td>\n",
       "      <td>0</td>\n",
       "      <td>64</td>\n",
       "      <td>6</td>\n",
       "      <td>65185</td>\n",
       "      <td>172.17.0.2</td>\n",
       "      <td>54.149.19.17</td>\n",
       "      <td>0</td>\n",
       "      <td>1571654860.621589</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>52</td>\n",
       "      <td>51662</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "      <td>104</td>\n",
       "      <td>6</td>\n",
       "      <td>52434</td>\n",
       "      <td>8.8.8.8</td>\n",
       "      <td>172.17.0.2</td>\n",
       "      <td>0</td>\n",
       "      <td>1571654860.627568</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>91</td>\n",
       "      <td>31862</td>\n",
       "      <td>DF</td>\n",
       "      <td>0</td>\n",
       "      <td>64</td>\n",
       "      <td>6</td>\n",
       "      <td>516</td>\n",
       "      <td>172.17.0.2</td>\n",
       "      <td>8.8.8.8</td>\n",
       "      <td>0</td>\n",
       "      <td>1571654860.706652</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>76</td>\n",
       "      <td>31863</td>\n",
       "      <td>DF</td>\n",
       "      <td>0</td>\n",
       "      <td>64</td>\n",
       "      <td>6</td>\n",
       "      <td>530</td>\n",
       "      <td>172.17.0.2</td>\n",
       "      <td>8.8.8.8</td>\n",
       "      <td>0</td>\n",
       "      <td>1571654860.706798</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>103</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>52</td>\n",
       "      <td>31864</td>\n",
       "      <td>DF</td>\n",
       "      <td>0</td>\n",
       "      <td>64</td>\n",
       "      <td>6</td>\n",
       "      <td>553</td>\n",
       "      <td>172.17.0.2</td>\n",
       "      <td>8.8.8.8</td>\n",
       "      <td>0</td>\n",
       "      <td>1571654860.706816</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>104</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>52</td>\n",
       "      <td>51706</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "      <td>104</td>\n",
       "      <td>6</td>\n",
       "      <td>52390</td>\n",
       "      <td>8.8.8.8</td>\n",
       "      <td>172.17.0.2</td>\n",
       "      <td>0</td>\n",
       "      <td>1571654860.708851</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>105</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>52</td>\n",
       "      <td>51707</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "      <td>104</td>\n",
       "      <td>6</td>\n",
       "      <td>52389</td>\n",
       "      <td>8.8.8.8</td>\n",
       "      <td>172.17.0.2</td>\n",
       "      <td>0</td>\n",
       "      <td>1571654860.709360</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>106</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>52</td>\n",
       "      <td>31865</td>\n",
       "      <td>DF</td>\n",
       "      <td>0</td>\n",
       "      <td>64</td>\n",
       "      <td>6</td>\n",
       "      <td>552</td>\n",
       "      <td>172.17.0.2</td>\n",
       "      <td>8.8.8.8</td>\n",
       "      <td>0</td>\n",
       "      <td>1571654860.709369</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>107</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>52</td>\n",
       "      <td>20353</td>\n",
       "      <td>DF</td>\n",
       "      <td>0</td>\n",
       "      <td>227</td>\n",
       "      <td>6</td>\n",
       "      <td>8193</td>\n",
       "      <td>52.215.71.87</td>\n",
       "      <td>172.17.0.2</td>\n",
       "      <td>0</td>\n",
       "      <td>1571654860.742312</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>108</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>1340</td>\n",
       "      <td>20354</td>\n",
       "      <td>DF</td>\n",
       "      <td>0</td>\n",
       "      <td>227</td>\n",
       "      <td>6</td>\n",
       "      <td>6904</td>\n",
       "      <td>52.215.71.87</td>\n",
       "      <td>172.17.0.2</td>\n",
       "      <td>0</td>\n",
       "      <td>1571654860.743695</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>109</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>52</td>\n",
       "      <td>30463</td>\n",
       "      <td>DF</td>\n",
       "      <td>0</td>\n",
       "      <td>64</td>\n",
       "      <td>6</td>\n",
       "      <td>39811</td>\n",
       "      <td>172.17.0.2</td>\n",
       "      <td>52.215.71.87</td>\n",
       "      <td>0</td>\n",
       "      <td>1571654860.743702</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>110</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>1340</td>\n",
       "      <td>20355</td>\n",
       "      <td>DF</td>\n",
       "      <td>0</td>\n",
       "      <td>227</td>\n",
       "      <td>6</td>\n",
       "      <td>6903</td>\n",
       "      <td>52.215.71.87</td>\n",
       "      <td>172.17.0.2</td>\n",
       "      <td>0</td>\n",
       "      <td>1571654860.743739</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>111</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>52</td>\n",
       "      <td>30464</td>\n",
       "      <td>DF</td>\n",
       "      <td>0</td>\n",
       "      <td>64</td>\n",
       "      <td>6</td>\n",
       "      <td>39810</td>\n",
       "      <td>172.17.0.2</td>\n",
       "      <td>52.215.71.87</td>\n",
       "      <td>0</td>\n",
       "      <td>1571654860.743745</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>112</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>868</td>\n",
       "      <td>20356</td>\n",
       "      <td>DF</td>\n",
       "      <td>0</td>\n",
       "      <td>227</td>\n",
       "      <td>6</td>\n",
       "      <td>7374</td>\n",
       "      <td>52.215.71.87</td>\n",
       "      <td>172.17.0.2</td>\n",
       "      <td>0</td>\n",
       "      <td>1571654860.743927</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>113</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>52</td>\n",
       "      <td>30465</td>\n",
       "      <td>DF</td>\n",
       "      <td>0</td>\n",
       "      <td>64</td>\n",
       "      <td>6</td>\n",
       "      <td>39809</td>\n",
       "      <td>172.17.0.2</td>\n",
       "      <td>52.215.71.87</td>\n",
       "      <td>0</td>\n",
       "      <td>1571654860.743931</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>114</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>178</td>\n",
       "      <td>30466</td>\n",
       "      <td>DF</td>\n",
       "      <td>0</td>\n",
       "      <td>64</td>\n",
       "      <td>6</td>\n",
       "      <td>39682</td>\n",
       "      <td>172.17.0.2</td>\n",
       "      <td>52.215.71.87</td>\n",
       "      <td>0</td>\n",
       "      <td>1571654860.746570</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>115</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>60</td>\n",
       "      <td>718</td>\n",
       "      <td>DF</td>\n",
       "      <td>0</td>\n",
       "      <td>64</td>\n",
       "      <td>6</td>\n",
       "      <td>31691</td>\n",
       "      <td>172.17.0.2</td>\n",
       "      <td>8.8.8.8</td>\n",
       "      <td>0</td>\n",
       "      <td>1571654860.750212</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>116</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>60</td>\n",
       "      <td>10453</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "      <td>104</td>\n",
       "      <td>6</td>\n",
       "      <td>28100</td>\n",
       "      <td>8.8.8.8</td>\n",
       "      <td>172.17.0.2</td>\n",
       "      <td>0</td>\n",
       "      <td>1571654860.752625</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>52</td>\n",
       "      <td>719</td>\n",
       "      <td>DF</td>\n",
       "      <td>0</td>\n",
       "      <td>64</td>\n",
       "      <td>6</td>\n",
       "      <td>31698</td>\n",
       "      <td>172.17.0.2</td>\n",
       "      <td>8.8.8.8</td>\n",
       "      <td>0</td>\n",
       "      <td>1571654860.752645</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>678</td>\n",
       "      <td>720</td>\n",
       "      <td>DF</td>\n",
       "      <td>0</td>\n",
       "      <td>64</td>\n",
       "      <td>6</td>\n",
       "      <td>31071</td>\n",
       "      <td>172.17.0.2</td>\n",
       "      <td>8.8.8.8</td>\n",
       "      <td>0</td>\n",
       "      <td>1571654860.753560</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>119</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>52</td>\n",
       "      <td>10455</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "      <td>104</td>\n",
       "      <td>6</td>\n",
       "      <td>28106</td>\n",
       "      <td>8.8.8.8</td>\n",
       "      <td>172.17.0.2</td>\n",
       "      <td>0</td>\n",
       "      <td>1571654860.756587</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>120</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>264</td>\n",
       "      <td>10456</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "      <td>104</td>\n",
       "      <td>6</td>\n",
       "      <td>27893</td>\n",
       "      <td>8.8.8.8</td>\n",
       "      <td>172.17.0.2</td>\n",
       "      <td>0</td>\n",
       "      <td>1571654860.756906</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>121</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>52</td>\n",
       "      <td>721</td>\n",
       "      <td>DF</td>\n",
       "      <td>0</td>\n",
       "      <td>64</td>\n",
       "      <td>6</td>\n",
       "      <td>31696</td>\n",
       "      <td>172.17.0.2</td>\n",
       "      <td>8.8.8.8</td>\n",
       "      <td>0</td>\n",
       "      <td>1571654860.756914</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    version ihl tos   len     id flags frag  ttl proto chksum           src  \\\n",
       "69        4   5   0    91  31851    DF    0   64     6    527    172.17.0.2   \n",
       "70        4   5   0    52  51500          0  104     6  52596       8.8.8.8   \n",
       "71        4   5   0    60  30460    DF    0   64     6  39806    172.17.0.2   \n",
       "72        4   5   0    60      0    DF    0  227     6  28538  52.215.71.87   \n",
       "73        4   5   0    52  30461    DF    0   64     6  39813    172.17.0.2   \n",
       "74        4   5   0   569  30462    DF    0   64     6  39295    172.17.0.2   \n",
       "75        4   5   0   113  31852    DF    0   64     6    504    172.17.0.2   \n",
       "76        4   5   0   147  31853    DF    0   64     6    469    172.17.0.2   \n",
       "77        4   5   0   109  31854    DF    0   64     6    506    172.17.0.2   \n",
       "78        4   5   0   147  31855    DF    0   64     6    467    172.17.0.2   \n",
       "79        4   5   0    52  51634          0  104     6  52462       8.8.8.8   \n",
       "80        4   5   0    52  51635          0  104     6  52461       8.8.8.8   \n",
       "81        4   5   0    52  51636          0  104     6  52460       8.8.8.8   \n",
       "82        4   5   0    52  51637          0  104     6  52459       8.8.8.8   \n",
       "83        4   5   0   175  51646          0  104     6  52327       8.8.8.8   \n",
       "84        4   5   0   265  51647          0  104     6  52236       8.8.8.8   \n",
       "85        4   5   0    83  51648          0  104     6  52417       8.8.8.8   \n",
       "86        4   5   0    91  51649          0  104     6  52408       8.8.8.8   \n",
       "87        4   5   0   310  51650          0  104     6  52188       8.8.8.8   \n",
       "88        4   5   0    52  31856    DF    0   64     6    561    172.17.0.2   \n",
       "89        4   5   0    91  31857    DF    0   64     6    521    172.17.0.2   \n",
       "90        4   5   0   113  31858    DF    0   64     6    498    172.17.0.2   \n",
       "91        4   5   0   142  31859    DF    0   64     6    468    172.17.0.2   \n",
       "92        4   5   0    52  51651          0  104     6  52445       8.8.8.8   \n",
       "93        4   5   0   125  51656          0  104     6  52367       8.8.8.8   \n",
       "94        4   5   0   227  51657          0  104     6  52264       8.8.8.8   \n",
       "95        4   5   0    83  51658          0  104     6  52407       8.8.8.8   \n",
       "96        4   5   0    91  51659          0  104     6  52398       8.8.8.8   \n",
       "97        4   5   0    52  31860    DF    0   64     6    557    172.17.0.2   \n",
       "98        4   5   0    91  31861    DF    0   64     6    517    172.17.0.2   \n",
       "99        4   5   0    60  18017    DF    0   64     6  65185    172.17.0.2   \n",
       "100       4   5   0    52  51662          0  104     6  52434       8.8.8.8   \n",
       "101       4   5   0    91  31862    DF    0   64     6    516    172.17.0.2   \n",
       "102       4   5   0    76  31863    DF    0   64     6    530    172.17.0.2   \n",
       "103       4   5   0    52  31864    DF    0   64     6    553    172.17.0.2   \n",
       "104       4   5   0    52  51706          0  104     6  52390       8.8.8.8   \n",
       "105       4   5   0    52  51707          0  104     6  52389       8.8.8.8   \n",
       "106       4   5   0    52  31865    DF    0   64     6    552    172.17.0.2   \n",
       "107       4   5   0    52  20353    DF    0  227     6   8193  52.215.71.87   \n",
       "108       4   5   0  1340  20354    DF    0  227     6   6904  52.215.71.87   \n",
       "109       4   5   0    52  30463    DF    0   64     6  39811    172.17.0.2   \n",
       "110       4   5   0  1340  20355    DF    0  227     6   6903  52.215.71.87   \n",
       "111       4   5   0    52  30464    DF    0   64     6  39810    172.17.0.2   \n",
       "112       4   5   0   868  20356    DF    0  227     6   7374  52.215.71.87   \n",
       "113       4   5   0    52  30465    DF    0   64     6  39809    172.17.0.2   \n",
       "114       4   5   0   178  30466    DF    0   64     6  39682    172.17.0.2   \n",
       "115       4   5   0    60    718    DF    0   64     6  31691    172.17.0.2   \n",
       "116       4   5   0    60  10453          0  104     6  28100       8.8.8.8   \n",
       "117       4   5   0    52    719    DF    0   64     6  31698    172.17.0.2   \n",
       "118       4   5   0   678    720    DF    0   64     6  31071    172.17.0.2   \n",
       "119       4   5   0    52  10455          0  104     6  28106       8.8.8.8   \n",
       "120       4   5   0   264  10456          0  104     6  27893       8.8.8.8   \n",
       "121       4   5   0    52    721    DF    0   64     6  31696    172.17.0.2   \n",
       "\n",
       "              dst options               time  \n",
       "69        8.8.8.8       0  1571654860.349295  \n",
       "70     172.17.0.2       0  1571654860.355563  \n",
       "71   52.215.71.87       0  1571654860.409490  \n",
       "72     172.17.0.2       0  1571654860.575829  \n",
       "73   52.215.71.87       0  1571654860.575851  \n",
       "74   52.215.71.87       0  1571654860.576816  \n",
       "75        8.8.8.8       0  1571654860.601469  \n",
       "76        8.8.8.8       0  1571654860.601517  \n",
       "77        8.8.8.8       0  1571654860.601595  \n",
       "78        8.8.8.8       0  1571654860.601616  \n",
       "79     172.17.0.2       0  1571654860.603568  \n",
       "80     172.17.0.2       0  1571654860.603577  \n",
       "81     172.17.0.2       0  1571654860.603911  \n",
       "82     172.17.0.2       0  1571654860.603916  \n",
       "83     172.17.0.2       0  1571654860.611998  \n",
       "84     172.17.0.2       0  1571654860.612008  \n",
       "85     172.17.0.2       0  1571654860.612160  \n",
       "86     172.17.0.2       0  1571654860.612162  \n",
       "87     172.17.0.2       0  1571654860.612162  \n",
       "88        8.8.8.8       0  1571654860.612195  \n",
       "89        8.8.8.8       0  1571654860.612316  \n",
       "90        8.8.8.8       0  1571654860.612987  \n",
       "91        8.8.8.8       0  1571654860.613031  \n",
       "92     172.17.0.2       0  1571654860.615688  \n",
       "93     172.17.0.2       0  1571654860.620835  \n",
       "94     172.17.0.2       0  1571654860.620847  \n",
       "95     172.17.0.2       0  1571654860.620962  \n",
       "96     172.17.0.2       0  1571654860.620965  \n",
       "97        8.8.8.8       0  1571654860.621065  \n",
       "98        8.8.8.8       0  1571654860.621107  \n",
       "99   54.149.19.17       0  1571654860.621589  \n",
       "100    172.17.0.2       0  1571654860.627568  \n",
       "101       8.8.8.8       0  1571654860.706652  \n",
       "102       8.8.8.8       0  1571654860.706798  \n",
       "103       8.8.8.8       0  1571654860.706816  \n",
       "104    172.17.0.2       0  1571654860.708851  \n",
       "105    172.17.0.2       0  1571654860.709360  \n",
       "106       8.8.8.8       0  1571654860.709369  \n",
       "107    172.17.0.2       0  1571654860.742312  \n",
       "108    172.17.0.2       0  1571654860.743695  \n",
       "109  52.215.71.87       0  1571654860.743702  \n",
       "110    172.17.0.2       0  1571654860.743739  \n",
       "111  52.215.71.87       0  1571654860.743745  \n",
       "112    172.17.0.2       0  1571654860.743927  \n",
       "113  52.215.71.87       0  1571654860.743931  \n",
       "114  52.215.71.87       0  1571654860.746570  \n",
       "115       8.8.8.8       0  1571654860.750212  \n",
       "116    172.17.0.2       0  1571654860.752625  \n",
       "117       8.8.8.8       0  1571654860.752645  \n",
       "118       8.8.8.8       0  1571654860.753560  \n",
       "119    172.17.0.2       0  1571654860.756587  \n",
       "120    172.17.0.2       0  1571654860.756906  \n",
       "121       8.8.8.8       0  1571654860.756914  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# rough work - IP fields into a Dataframe\n",
    "\n",
    "df_fields = ip_fields + ['time'] #+ tcp_fields  #+ ['payload']\n",
    "df = pd.DataFrame(columns=df_fields)\n",
    "\n",
    "for pkt in temp_pcap[IP]:\n",
    "\n",
    "    values = []    \n",
    "    \n",
    "    # IP layer fields\n",
    "    for fd in ip_fields:\n",
    "        \n",
    "        if fd == 'options':\n",
    "            values.append(len(pkt[IP].fields[fd]))\n",
    "        else:\n",
    "            values.append(pkt[IP].fields[fd])\n",
    "    \n",
    "    values.append(pkt.time)\n",
    "\n",
    "    \n",
    "    df_append = pd.DataFrame([values], columns=df_fields)\n",
    "    df = pd.concat([df, df_append], axis=0)\n",
    "\n",
    "df = df.reset_index()\n",
    "df = df.drop(columns=\"index\")\n",
    "df[69:122]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    1046\n",
       "Name: tos, dtype: int64"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.tos.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>version</th>\n",
       "      <th>ihl</th>\n",
       "      <th>tos</th>\n",
       "      <th>len</th>\n",
       "      <th>id</th>\n",
       "      <th>flags</th>\n",
       "      <th>frag</th>\n",
       "      <th>ttl</th>\n",
       "      <th>proto</th>\n",
       "      <th>chksum</th>\n",
       "      <th>src</th>\n",
       "      <th>dst</th>\n",
       "      <th>options</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [version, ihl, tos, len, id, flags, frag, ttl, proto, chksum, src, dst, options, time]\n",
       "Index: []"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df.id == 41572]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sport</th>\n",
       "      <th>dport</th>\n",
       "      <th>seq</th>\n",
       "      <th>ack</th>\n",
       "      <th>dataofs</th>\n",
       "      <th>reserved</th>\n",
       "      <th>flags</th>\n",
       "      <th>window</th>\n",
       "      <th>chksum</th>\n",
       "      <th>urgptr</th>\n",
       "      <th>options</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>55136</td>\n",
       "      <td>443</td>\n",
       "      <td>1045311707</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>S</td>\n",
       "      <td>64240</td>\n",
       "      <td>48209</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>443</td>\n",
       "      <td>55136</td>\n",
       "      <td>2995151223</td>\n",
       "      <td>1045311708</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>SA</td>\n",
       "      <td>62392</td>\n",
       "      <td>56464</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>55136</td>\n",
       "      <td>443</td>\n",
       "      <td>1045311708</td>\n",
       "      <td>2995151224</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>A</td>\n",
       "      <td>502</td>\n",
       "      <td>48201</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>55136</td>\n",
       "      <td>443</td>\n",
       "      <td>1045311708</td>\n",
       "      <td>2995151224</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>PA</td>\n",
       "      <td>502</td>\n",
       "      <td>48718</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>443</td>\n",
       "      <td>55136</td>\n",
       "      <td>2995151224</td>\n",
       "      <td>1045312225</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>A</td>\n",
       "      <td>248</td>\n",
       "      <td>64449</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   sport  dport         seq         ack dataofs reserved flags window chksum  \\\n",
       "0  55136    443  1045311707           0      10        0     S  64240  48209   \n",
       "1    443  55136  2995151223  1045311708      10        0    SA  62392  56464   \n",
       "2  55136    443  1045311708  2995151224       8        0     A    502  48201   \n",
       "3  55136    443  1045311708  2995151224       8        0    PA    502  48718   \n",
       "4    443  55136  2995151224  1045312225       8        0     A    248  64449   \n",
       "\n",
       "  urgptr options  \n",
       "0      0       2  \n",
       "1      0       2  \n",
       "2      0       2  \n",
       "3      0       2  \n",
       "4      0       2  "
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# rough work - TCP fields into a Dataframe\n",
    "tcp_fd = ['sport', 'dport', 'flags'] \n",
    "\n",
    "df_fields = tcp_fields  #+ ['payload']        \n",
    "                \n",
    "\n",
    "df = pd.DataFrame(columns=df_fields)\n",
    "\n",
    "for pkt in temp_pcap[IP]:\n",
    "\n",
    "    values = []    \n",
    "    \n",
    "    #tcp layer - getting type\n",
    "    layer_type = type(pkt[IP].payload)\n",
    "    for fd in tcp_fields:\n",
    "        try:\n",
    "            if fd == 'options':\n",
    "                values.append(len(pkt[layer_type].fields[fd][0]))\n",
    "            else:\n",
    "                values.append(pkt[layer_type].fields[fd])\n",
    "        except:\n",
    "            values.append(None)\n",
    "    \n",
    "    df_append = pd.DataFrame([values], columns=df_fields)\n",
    "    df = pd.concat([df, df_append], axis=0)\n",
    "\n",
    "df = df.reset_index()\n",
    "df = df.drop(columns=\"index\")\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df.flags.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>type</th>\n",
       "      <th>version</th>\n",
       "      <th>len</th>\n",
       "      <th>iv</th>\n",
       "      <th>mac</th>\n",
       "      <th>pad</th>\n",
       "      <th>padlen</th>\n",
       "      <th>sni</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>22</td>\n",
       "      <td>769</td>\n",
       "      <td>512</td>\n",
       "      <td>b''</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>b'dns.google'</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   type version   len    iv   mac   pad padlen            sni\n",
       "0  None    None  None  None  None  None   None           None\n",
       "1  None    None  None  None  None  None   None           None\n",
       "2  None    None  None  None  None  None   None           None\n",
       "3    22     769   512   b''  None  None   None  b'dns.google'\n",
       "4  None    None  None  None  None  None   None           None"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# rough work - TLS fields into a Dataframe\n",
    "\n",
    "tls_fields = ['type', 'version', 'len', 'iv', 'mac', 'pad', 'padlen', 'sni']\n",
    "df_fields = tls_fields  #+ ['payload']\n",
    "\n",
    "df = pd.DataFrame(columns=df_fields)\n",
    "\n",
    "for pkt in temp_pcap[IP]:\n",
    "\n",
    "    values = []    \n",
    "    \n",
    "    if type(pkt[TCP].payload) != scapy.packet.NoPayload: \n",
    "        #TLS layer - getting type is to just make sure\n",
    "        layer_type = type(pkt[TCP].payload)\n",
    "        for fd in tls_fields:\n",
    "            try:\n",
    "                if fd == 'sni':\n",
    "                    if pkt[layer_type].fields['msg'] and pkt[layer_type].fields['msg'][0].msgtype == 1:\n",
    "                        values.append(pkt[layer_type].fields['msg'][0].ext[0].servernames[0].servername)\n",
    "                    else:\n",
    "                        values.append(0)\n",
    "                else:\n",
    "                    values.append(pkt[layer_type].fields[fd])\n",
    "            except:\n",
    "                values.append(None)\n",
    "    else:\n",
    "        for fd in tls_fields:\n",
    "            values.append(None)\n",
    "    \n",
    "#     values.append(len(pkt[layer_type].payload))\n",
    "#     #field_values.append(packet[layer_type].payload.original)\n",
    "#     #field_values.append(binascii.hexlify(packet[layer_type].payload.original))\n",
    "\n",
    "    \n",
    "    df_append = pd.DataFrame([values], columns=df_fields)\n",
    "    df = pd.concat([df, df_append], axis=0)\n",
    "\n",
    "df = df.reset_index()\n",
    "df = df.drop(columns=\"index\")\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "\n",
    "####### Rough work ENDs ##########\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>src</th>\n",
       "      <th>dst</th>\n",
       "      <th>len</th>\n",
       "      <th>ttl</th>\n",
       "      <th>sport</th>\n",
       "      <th>dport</th>\n",
       "      <th>flags</th>\n",
       "      <th>type</th>\n",
       "      <th>version</th>\n",
       "      <th>sni</th>\n",
       "      <th>time</th>\n",
       "      <th>query-url</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>172.17.0.2</td>\n",
       "      <td>8.8.8.8</td>\n",
       "      <td>60</td>\n",
       "      <td>64</td>\n",
       "      <td>55136</td>\n",
       "      <td>443</td>\n",
       "      <td>S</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1571654858.891679</td>\n",
       "      <td>http://www.google.com</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>8.8.8.8</td>\n",
       "      <td>172.17.0.2</td>\n",
       "      <td>60</td>\n",
       "      <td>104</td>\n",
       "      <td>443</td>\n",
       "      <td>55136</td>\n",
       "      <td>SA</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1571654858.894632</td>\n",
       "      <td>http://www.google.com</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>172.17.0.2</td>\n",
       "      <td>8.8.8.8</td>\n",
       "      <td>52</td>\n",
       "      <td>64</td>\n",
       "      <td>55136</td>\n",
       "      <td>443</td>\n",
       "      <td>A</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1571654858.894650</td>\n",
       "      <td>http://www.google.com</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>172.17.0.2</td>\n",
       "      <td>8.8.8.8</td>\n",
       "      <td>569</td>\n",
       "      <td>64</td>\n",
       "      <td>55136</td>\n",
       "      <td>443</td>\n",
       "      <td>PA</td>\n",
       "      <td>22</td>\n",
       "      <td>769</td>\n",
       "      <td>b'dns.google'</td>\n",
       "      <td>1571654858.895607</td>\n",
       "      <td>http://www.google.com</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>8.8.8.8</td>\n",
       "      <td>172.17.0.2</td>\n",
       "      <td>52</td>\n",
       "      <td>104</td>\n",
       "      <td>443</td>\n",
       "      <td>55136</td>\n",
       "      <td>A</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1571654858.898672</td>\n",
       "      <td>http://www.google.com</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          src         dst  len  ttl  sport  dport flags  type version  \\\n",
       "0  172.17.0.2     8.8.8.8   60   64  55136    443     S  None    None   \n",
       "1     8.8.8.8  172.17.0.2   60  104    443  55136    SA  None    None   \n",
       "2  172.17.0.2     8.8.8.8   52   64  55136    443     A  None    None   \n",
       "3  172.17.0.2     8.8.8.8  569   64  55136    443    PA    22     769   \n",
       "4     8.8.8.8  172.17.0.2   52  104    443  55136     A  None    None   \n",
       "\n",
       "             sni               time              query-url  \n",
       "0           None  1571654858.891679  http://www.google.com  \n",
       "1           None  1571654858.894632  http://www.google.com  \n",
       "2           None  1571654858.894650  http://www.google.com  \n",
       "3  b'dns.google'  1571654858.895607  http://www.google.com  \n",
       "4           None  1571654858.898672  http://www.google.com  "
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Shortlisting needed fields and creatign a Dataframe\n",
    "ip_fd = ['src', 'dst', 'len', 'ttl'] \n",
    "tcp_fd = ['sport', 'dport', 'flags']  \n",
    "tls_fd = ['type', 'version', 'sni']\n",
    "\n",
    "df_fields = ip_fd + tcp_fd + tls_fd + ['time'] + ['query-url']\n",
    "\n",
    "df = pd.DataFrame(columns=df_fields)\n",
    "\n",
    "for pkt in temp_pcap[IP]:\n",
    "\n",
    "    values = []    \n",
    "    \n",
    "    # IP layer fields\n",
    "    for fd in ip_fd:\n",
    "        \n",
    "        if fd == 'options':\n",
    "            values.append(len(pkt[IP].fields[fd]))\n",
    "        else:\n",
    "            values.append(pkt[IP].fields[fd])\n",
    "\n",
    "\n",
    "    # TCP layer - getting type\n",
    "    layer_type = type(pkt[IP].payload)\n",
    "    for fd in tcp_fd:\n",
    "        try:\n",
    "            if fd == 'options':\n",
    "                values.append(len(pkt[layer_type].fields[fd][0]))\n",
    "            else:\n",
    "                values.append(pkt[layer_type].fields[fd])\n",
    "        except:\n",
    "            values.append(None)\n",
    "            \n",
    "    \n",
    "# TYPE:23 - application data\n",
    "#     22 - handshake proto\n",
    "#     21 - alert protocol\n",
    "#     20 - change cipher spec\n",
    "                   \n",
    "    # TLS fields\n",
    "    if type(pkt[TCP].payload) != scapy.packet.NoPayload: \n",
    "        layer_type = type(pkt[TCP].payload)\n",
    "        for fd in tls_fd:\n",
    "            try:\n",
    "                if fd == 'sni':\n",
    "                    if pkt[layer_type].fields['msg'] and pkt[layer_type].fields['msg'][0].msgtype == 1:\n",
    "                        values.append(pkt[layer_type].fields['msg'][0].ext[0].servernames[0].servername)\n",
    "                    else:\n",
    "                        values.append(0)\n",
    "                else:\n",
    "                    values.append(pkt[layer_type].fields[fd])\n",
    "            except:\n",
    "                values.append(0)\n",
    "    else:\n",
    "        for fd in tls_fd:\n",
    "            values.append(None)\n",
    "    \n",
    "    # Adding time and queried URL\n",
    "    values.append(pkt.time)\n",
    "    values.append(dns_res_stat[query_no]['query-url'])\n",
    "    \n",
    "#     values.append(len(pkt[layer_type].payload))\n",
    "#     #field_values.append(packet[layer_type].payload.original)\n",
    "#     #field_values.append(binascii.hexlify(packet[layer_type].payload.original))\n",
    "\n",
    "    \n",
    "    df_append = pd.DataFrame([values], columns=df_fields)\n",
    "    df = pd.concat([df, df_append], axis=0)\n",
    "    \n",
    "df = df.reset_index()\n",
    "df = df.drop(columns=\"index\")\n",
    "#df.columns = ['src', 'dst', 'iplen', 'ttl', 'sport', 'dport', 'flags', 'options', 'ack', 'dataofs', 'type', 'version', 'tlslen', 'sni', 'time', 'query-url']\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>src</th>\n",
       "      <th>dst</th>\n",
       "      <th>len</th>\n",
       "      <th>ttl</th>\n",
       "      <th>sport</th>\n",
       "      <th>dport</th>\n",
       "      <th>flags</th>\n",
       "      <th>type</th>\n",
       "      <th>version</th>\n",
       "      <th>sni</th>\n",
       "      <th>time</th>\n",
       "      <th>query-url</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>172.17.0.2</td>\n",
       "      <td>8.8.8.8</td>\n",
       "      <td>569</td>\n",
       "      <td>64</td>\n",
       "      <td>55136</td>\n",
       "      <td>443</td>\n",
       "      <td>PA</td>\n",
       "      <td>22</td>\n",
       "      <td>769</td>\n",
       "      <td>b'dns.google'</td>\n",
       "      <td>1571654858.895607</td>\n",
       "      <td>http://www.google.com</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>172.17.0.2</td>\n",
       "      <td>8.8.8.8</td>\n",
       "      <td>678</td>\n",
       "      <td>64</td>\n",
       "      <td>55166</td>\n",
       "      <td>443</td>\n",
       "      <td>PA</td>\n",
       "      <td>22</td>\n",
       "      <td>769</td>\n",
       "      <td>b'dns.google'</td>\n",
       "      <td>1571654860.321700</td>\n",
       "      <td>http://www.google.com</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74</th>\n",
       "      <td>172.17.0.2</td>\n",
       "      <td>52.215.71.87</td>\n",
       "      <td>569</td>\n",
       "      <td>64</td>\n",
       "      <td>38216</td>\n",
       "      <td>443</td>\n",
       "      <td>PA</td>\n",
       "      <td>22</td>\n",
       "      <td>769</td>\n",
       "      <td>b'location.services.mozilla.com'</td>\n",
       "      <td>1571654860.576816</td>\n",
       "      <td>http://www.google.com</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118</th>\n",
       "      <td>172.17.0.2</td>\n",
       "      <td>8.8.8.8</td>\n",
       "      <td>678</td>\n",
       "      <td>64</td>\n",
       "      <td>55182</td>\n",
       "      <td>443</td>\n",
       "      <td>PA</td>\n",
       "      <td>22</td>\n",
       "      <td>769</td>\n",
       "      <td>b'dns.google'</td>\n",
       "      <td>1571654860.753560</td>\n",
       "      <td>http://www.google.com</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>164</th>\n",
       "      <td>172.17.0.2</td>\n",
       "      <td>54.149.19.17</td>\n",
       "      <td>569</td>\n",
       "      <td>64</td>\n",
       "      <td>42432</td>\n",
       "      <td>443</td>\n",
       "      <td>PA</td>\n",
       "      <td>22</td>\n",
       "      <td>769</td>\n",
       "      <td>b'shavar.services.mozilla.com'</td>\n",
       "      <td>1571654860.847866</td>\n",
       "      <td>http://www.google.com</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>183</th>\n",
       "      <td>172.17.0.2</td>\n",
       "      <td>8.8.8.8</td>\n",
       "      <td>569</td>\n",
       "      <td>64</td>\n",
       "      <td>55188</td>\n",
       "      <td>443</td>\n",
       "      <td>PA</td>\n",
       "      <td>22</td>\n",
       "      <td>769</td>\n",
       "      <td>b'dns.google'</td>\n",
       "      <td>1571654860.972715</td>\n",
       "      <td>http://www.google.com</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>211</th>\n",
       "      <td>172.17.0.2</td>\n",
       "      <td>8.8.8.8</td>\n",
       "      <td>569</td>\n",
       "      <td>64</td>\n",
       "      <td>55190</td>\n",
       "      <td>443</td>\n",
       "      <td>PA</td>\n",
       "      <td>22</td>\n",
       "      <td>769</td>\n",
       "      <td>b'dns.google'</td>\n",
       "      <td>1571654862.471539</td>\n",
       "      <td>http://www.google.com</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>225</th>\n",
       "      <td>172.17.0.2</td>\n",
       "      <td>8.8.8.8</td>\n",
       "      <td>569</td>\n",
       "      <td>64</td>\n",
       "      <td>55192</td>\n",
       "      <td>443</td>\n",
       "      <td>PA</td>\n",
       "      <td>22</td>\n",
       "      <td>769</td>\n",
       "      <td>b'dns.google'</td>\n",
       "      <td>1571654862.487472</td>\n",
       "      <td>http://www.google.com</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>309</th>\n",
       "      <td>172.17.0.2</td>\n",
       "      <td>172.217.194.147</td>\n",
       "      <td>569</td>\n",
       "      <td>64</td>\n",
       "      <td>52414</td>\n",
       "      <td>443</td>\n",
       "      <td>PA</td>\n",
       "      <td>22</td>\n",
       "      <td>769</td>\n",
       "      <td>b'www.google.com'</td>\n",
       "      <td>1571654862.543603</td>\n",
       "      <td>http://www.google.com</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>342</th>\n",
       "      <td>172.17.0.2</td>\n",
       "      <td>13.224.253.29</td>\n",
       "      <td>569</td>\n",
       "      <td>64</td>\n",
       "      <td>38736</td>\n",
       "      <td>443</td>\n",
       "      <td>PA</td>\n",
       "      <td>22</td>\n",
       "      <td>769</td>\n",
       "      <td>b'snippets.cdn.mozilla.net'</td>\n",
       "      <td>1571654862.597467</td>\n",
       "      <td>http://www.google.com</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>434</th>\n",
       "      <td>172.17.0.2</td>\n",
       "      <td>74.125.200.94</td>\n",
       "      <td>569</td>\n",
       "      <td>64</td>\n",
       "      <td>56386</td>\n",
       "      <td>443</td>\n",
       "      <td>PA</td>\n",
       "      <td>22</td>\n",
       "      <td>769</td>\n",
       "      <td>b'ssl.gstatic.com'</td>\n",
       "      <td>1571654862.680942</td>\n",
       "      <td>http://www.google.com</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>638</th>\n",
       "      <td>172.17.0.2</td>\n",
       "      <td>172.217.194.94</td>\n",
       "      <td>569</td>\n",
       "      <td>64</td>\n",
       "      <td>60928</td>\n",
       "      <td>443</td>\n",
       "      <td>PA</td>\n",
       "      <td>22</td>\n",
       "      <td>769</td>\n",
       "      <td>b'www.google.com.sg'</td>\n",
       "      <td>1571654862.863514</td>\n",
       "      <td>http://www.google.com</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>671</th>\n",
       "      <td>172.17.0.2</td>\n",
       "      <td>13.224.253.120</td>\n",
       "      <td>569</td>\n",
       "      <td>64</td>\n",
       "      <td>34350</td>\n",
       "      <td>443</td>\n",
       "      <td>PA</td>\n",
       "      <td>22</td>\n",
       "      <td>769</td>\n",
       "      <td>b'tracking-protection.cdn.mozilla.net'</td>\n",
       "      <td>1571654862.900309</td>\n",
       "      <td>http://www.google.com</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>716</th>\n",
       "      <td>172.17.0.2</td>\n",
       "      <td>74.125.68.94</td>\n",
       "      <td>569</td>\n",
       "      <td>64</td>\n",
       "      <td>56438</td>\n",
       "      <td>443</td>\n",
       "      <td>PA</td>\n",
       "      <td>22</td>\n",
       "      <td>769</td>\n",
       "      <td>b'www.gstatic.com'</td>\n",
       "      <td>1571654862.919197</td>\n",
       "      <td>http://www.google.com</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            src              dst  len ttl  sport dport flags type version  \\\n",
       "3    172.17.0.2          8.8.8.8  569  64  55136   443    PA   22     769   \n",
       "35   172.17.0.2          8.8.8.8  678  64  55166   443    PA   22     769   \n",
       "74   172.17.0.2     52.215.71.87  569  64  38216   443    PA   22     769   \n",
       "118  172.17.0.2          8.8.8.8  678  64  55182   443    PA   22     769   \n",
       "164  172.17.0.2     54.149.19.17  569  64  42432   443    PA   22     769   \n",
       "183  172.17.0.2          8.8.8.8  569  64  55188   443    PA   22     769   \n",
       "211  172.17.0.2          8.8.8.8  569  64  55190   443    PA   22     769   \n",
       "225  172.17.0.2          8.8.8.8  569  64  55192   443    PA   22     769   \n",
       "309  172.17.0.2  172.217.194.147  569  64  52414   443    PA   22     769   \n",
       "342  172.17.0.2    13.224.253.29  569  64  38736   443    PA   22     769   \n",
       "434  172.17.0.2    74.125.200.94  569  64  56386   443    PA   22     769   \n",
       "638  172.17.0.2   172.217.194.94  569  64  60928   443    PA   22     769   \n",
       "671  172.17.0.2   13.224.253.120  569  64  34350   443    PA   22     769   \n",
       "716  172.17.0.2     74.125.68.94  569  64  56438   443    PA   22     769   \n",
       "\n",
       "                                        sni               time  \\\n",
       "3                             b'dns.google'  1571654858.895607   \n",
       "35                            b'dns.google'  1571654860.321700   \n",
       "74         b'location.services.mozilla.com'  1571654860.576816   \n",
       "118                           b'dns.google'  1571654860.753560   \n",
       "164          b'shavar.services.mozilla.com'  1571654860.847866   \n",
       "183                           b'dns.google'  1571654860.972715   \n",
       "211                           b'dns.google'  1571654862.471539   \n",
       "225                           b'dns.google'  1571654862.487472   \n",
       "309                       b'www.google.com'  1571654862.543603   \n",
       "342             b'snippets.cdn.mozilla.net'  1571654862.597467   \n",
       "434                      b'ssl.gstatic.com'  1571654862.680942   \n",
       "638                    b'www.google.com.sg'  1571654862.863514   \n",
       "671  b'tracking-protection.cdn.mozilla.net'  1571654862.900309   \n",
       "716                      b'www.gstatic.com'  1571654862.919197   \n",
       "\n",
       "                 query-url  \n",
       "3    http://www.google.com  \n",
       "35   http://www.google.com  \n",
       "74   http://www.google.com  \n",
       "118  http://www.google.com  \n",
       "164  http://www.google.com  \n",
       "183  http://www.google.com  \n",
       "211  http://www.google.com  \n",
       "225  http://www.google.com  \n",
       "309  http://www.google.com  \n",
       "342  http://www.google.com  \n",
       "434  http://www.google.com  \n",
       "638  http://www.google.com  \n",
       "671  http://www.google.com  \n",
       "716  http://www.google.com  "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get the packets with the SNIs\n",
    "df[(df.sni != 0) & df.sni.notnull()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###[ Ethernet ]### \n",
      "  dst       = 02:42:66:c2:79:b2\n",
      "  src       = 02:42:ac:11:00:02\n",
      "  type      = IPv4\n",
      "###[ IP ]### \n",
      "     version   = 4\n",
      "     ihl       = 5\n",
      "     tos       = 0x0\n",
      "     len       = 60\n",
      "     id        = 17227\n",
      "     flags     = DF\n",
      "     frag      = 0\n",
      "     ttl       = 64\n",
      "     proto     = tcp\n",
      "     chksum    = 0x3b4e\n",
      "     src       = 172.17.0.2\n",
      "     dst       = 8.8.8.8\n",
      "     \\options   \\\n",
      "###[ TCP ]### \n",
      "        sport     = 55136\n",
      "        dport     = https\n",
      "        seq       = 1045311707\n",
      "        ack       = 0\n",
      "        dataofs   = 10\n",
      "        reserved  = 0\n",
      "        flags     = S\n",
      "        window    = 64240\n",
      "        chksum    = 0xbc51\n",
      "        urgptr    = 0\n",
      "        options   = [('MSS', 1460), ('SAckOK', b''), ('Timestamp', (2515866414, 0)), ('NOP', None), ('WScale', 7)]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#get first packet after given time\n",
    "for pkt in pcap:\n",
    "    if pkt.time >= 1569299628.325551:\n",
    "        pkt.show()\n",
    "        break;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "\n",
    "####### Rough work BEGINs for getting more values from TLS field ##########\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###[ Ethernet ]### \n",
      "  dst       = 02:42:66:c2:79:b2\n",
      "  src       = 02:42:ac:11:00:02\n",
      "  type      = IPv4\n",
      "###[ IP ]### \n",
      "     version   = 4\n",
      "     ihl       = 5\n",
      "     tos       = 0x0\n",
      "     len       = 147\n",
      "     id        = 31855\n",
      "     flags     = DF\n",
      "     frag      = 0\n",
      "     ttl       = 64\n",
      "     proto     = tcp\n",
      "     chksum    = 0x1d3\n",
      "     src       = 172.17.0.2\n",
      "     dst       = 8.8.8.8\n",
      "     \\options   \\\n",
      "###[ TCP ]### \n",
      "        sport     = 55166\n",
      "        dport     = https\n",
      "        seq       = 691474357\n",
      "        ack       = 3576900200\n",
      "        dataofs   = 8\n",
      "        reserved  = 0\n",
      "        flags     = PA\n",
      "        window    = 501\n",
      "        chksum    = 0xbca8\n",
      "        urgptr    = 0\n",
      "        options   = [('NOP', None), ('NOP', None), ('Timestamp', (2515868124, 507616863))]\n",
      "###[ TLS ]### \n",
      "           type      = application_data\n",
      "           version   = TLS 1.2\n",
      "           len       = 90    [deciphered_len= 90]\n",
      "           iv        = b''\n",
      "           \\msg       \\\n",
      "            |###[ TLS Application Data ]### \n",
      "            |  data      = '\\n\\xd5\\x87\\xb9\\r\\xe1\\xf3`?\\x84\\x87\\xae9\\xe1\\xa0Ax\\xf9K\\x127\\x18O\\xfb1\\x94R\\xd6\\xaf\\x7f\\x04@*\\xf4\\x1b\\x83>\\x0f\\x8c\\x9c\\x9e\\xb1\\xedN\\x86\\xbb\\x08HF\\xd9\\xf0i8\\x82\\x98]8\\xbf\\xb8k\\x8f\\xcf\\xa1\\xe8\\xb4\\xf5\\\\k\\x8bs[\\x05\\x9fh\\xc1\\xf3\\xd3)\\xc9\\x0f~=`aU-o\\x0c\\xb72'\n",
      "           mac       = b''\n",
      "           pad       = b''\n",
      "           padlen    = None\n",
      "\n"
     ]
    }
   ],
   "source": [
    "tls_pcap[43].show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(tls_pcap)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[]\n"
     ]
    }
   ],
   "source": [
    "# trial and error method for checking and handling type of TLS packet(hackshake/app-data..) \n",
    "# and their respective numbers which are assigned to them in the packet (eg: client-hello:1 and server-key-exch:12)\n",
    "# caught[] will contain which are not handled and hence can be added to if statements. \n",
    "# This code will be further moved to portion where TLS fields are handles and made into Dataframe\n",
    "caught = []\n",
    "for k in range(0,len(tls_pcap)):\n",
    "    #record_stat = [0,0,0,0]\n",
    "    temp = tls_pcap[k]\n",
    "    layert = type(temp[TCP].payload)\n",
    "    temp = temp[layert]\n",
    "    p = temp.fields['msg'][0]\n",
    "    # temp.fields['type'],p.msgtype,p.ext[0].servernames[0].servername #client hello\n",
    "    #print(temp.fields['type'],p.msgtype)\n",
    "\n",
    "\n",
    "    #print(\"Length:\",len(temp.layers()))\n",
    "    for i in temp.iterpayloads():\n",
    "        fld_val = i.fields\n",
    "\n",
    "        try:\n",
    "            if fld_val['type']:\n",
    "                if fld_val['type'] == 22:                      #handshake\n",
    "\n",
    "                    #print(fld_val['type'])\n",
    "                    try:\n",
    "                        inter_fld_val = fld_val['msg'][0].fields\n",
    "                    except:\n",
    "                        #print(\"Handshake raw\")\n",
    "                        f=1\n",
    "\n",
    "                    \n",
    "                    if inter_fld_val['msgtype'] == 0:          \n",
    "                        #print(\"0:Hello request\")\n",
    "                        f=1\n",
    "\n",
    "                    elif inter_fld_val['msgtype'] == 1:            #client hello\n",
    "                        #print(\"1:Client hello\")\n",
    "                        f=1\n",
    "\n",
    "                    elif inter_fld_val['msgtype'] == 2:          #server hello\n",
    "                        #print(\"2:Server hello\")\n",
    "                        f=1\n",
    "                        \n",
    "                    elif inter_fld_val['msgtype'] == 4:          \n",
    "                        #print(\"4:Session Ticket\")\n",
    "                        f=1\n",
    "\n",
    "                    elif inter_fld_val['msgtype'] == 11:         \n",
    "                        #print(\"11:Certificate\")\n",
    "                        f=1\n",
    "\n",
    "                    elif inter_fld_val['msgtype'] == 12:          \n",
    "                        #print(\"Server Key exchange\")\n",
    "                        f=1\n",
    "\n",
    "                    elif inter_fld_val['msgtype'] == 14:          \n",
    "                        #print(\"Server Hello done\")\n",
    "                        f=1\n",
    "\n",
    "                    elif inter_fld_val['msgtype'] == 16:          \n",
    "                        #print(\"Client Key exchange\")\n",
    "                        f=1\n",
    "                    \n",
    "                    elif inter_fld_val['msgtype'] == 22:          \n",
    "                        #print(\"22:Certificate Status\")\n",
    "                        f=1\n",
    "                    \n",
    "            \n",
    "\n",
    "                    else:\n",
    "                        #print(\"Caught!!!!!!!! from handshake: \",inter_fld_val['msgtype'])\n",
    "                        caught.append(k)\n",
    "\n",
    "                elif fld_val['type'] == 20:                    #change cipher spec\n",
    "                    #print(fld_val['type'], \"Change cipher spec\")\n",
    "                    f=1\n",
    "                    \n",
    "                elif fld_val['type'] == 21:                    # alert - close_notify\n",
    "                    #print(fld_val['type'], \"Alert\")\n",
    "                    f=1\n",
    "                \n",
    "                elif fld_val['type'] == 23:                    #application data\n",
    "                    #print(fld_val['type'], \"Application data\")  \n",
    "                    f=1\n",
    "                else:\n",
    "                    print(\"Caught!!!!!!!!:\", fld_val['type'])\n",
    "                    caught.append(k)\n",
    "        except:\n",
    "            #print(\"Raw data\")\n",
    "            f=1\n",
    "           \n",
    "    #print(str(k)+\" packet finished\")\n",
    "    #print(\"-----------------\")\n",
    "print(caught)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# same as the code above but handles all files in the folder \n",
    "data_dir = mypath + \"/traces/\" \n",
    "\n",
    "files = [f for f in listdir(data_dir) if isfile(join(data_dir, f))]\n",
    "\n",
    "files = sorted(files)\n",
    "\n",
    "for query_no in range(0,len(files)):\n",
    "\n",
    "    pcap = rdpcap(data_dir+files[query_no])\n",
    "    tls_pcap = pcap[TLS]\n",
    "\n",
    "    caught = []\n",
    "    for k in range(0,len(tls_pcap)):\n",
    "        record_stat = [0,0,0,0]\n",
    "        temp = tls_pcap[k]\n",
    "        layert = type(temp[TCP].payload)\n",
    "        temp = temp[layert]\n",
    "        p = temp.fields['msg'][0]\n",
    "        # temp.fields['type'],p.msgtype,p.ext[0].servernames[0].servername #client hello\n",
    "        #print(temp.fields['type'],p.msgtype)\n",
    "\n",
    "\n",
    "        #print(\"Length:\",len(temp.layers()))\n",
    "        for i in temp.iterpayloads():\n",
    "            fld_val = i.fields\n",
    "\n",
    "            try:\n",
    "                if fld_val['type']:\n",
    "                    if fld_val['type'] == 22:                      #handshake\n",
    "\n",
    "                        #print(fld_val['type'])\n",
    "                        try:\n",
    "                            inter_fld_val = fld_val['msg'][0].fields\n",
    "                        except:\n",
    "                            #print(\"Handshake raw\")\n",
    "                            f=1\n",
    "\n",
    "\n",
    "                        if inter_fld_val['msgtype'] == 0:          \n",
    "                            #print(\"0:Hello request\")\n",
    "                            f=1\n",
    "\n",
    "                        elif inter_fld_val['msgtype'] == 1:            #client hello\n",
    "                            #print(\"1:Client hello\")\n",
    "                            f=1\n",
    "\n",
    "                        elif inter_fld_val['msgtype'] == 2:          #server hello\n",
    "                            #print(\"2:Server hello\")\n",
    "                            f=1\n",
    "\n",
    "                        elif inter_fld_val['msgtype'] == 4:          #server hello\n",
    "                            #print(\"4:Session Ticket\")\n",
    "                            f=1\n",
    "\n",
    "                        elif inter_fld_val['msgtype'] == 11:         \n",
    "                            #print(\"11:Certificate\")\n",
    "                            f=1\n",
    "\n",
    "                        elif inter_fld_val['msgtype'] == 12:          \n",
    "                            #print(\"Server Key exchange\")\n",
    "                            f=1\n",
    "\n",
    "                        elif inter_fld_val['msgtype'] == 14:          \n",
    "                            #print(\"Server Hello done\")\n",
    "                            f=1\n",
    "\n",
    "                        elif inter_fld_val['msgtype'] == 16:          \n",
    "                            #print(\"Client Key exchange\")\n",
    "                            f=1\n",
    "\n",
    "                        elif inter_fld_val['msgtype'] == 22:          \n",
    "                            #print(\"22:Certificate Status\")\n",
    "                            f=1\n",
    "\n",
    "\n",
    "\n",
    "                        else:\n",
    "                            print(\"Caught!!!!!!!! from handshake: \",inter_fld_val['msgtype'])\n",
    "                            caught.append(k)\n",
    "\n",
    "                    elif fld_val['type'] == 20:                    #change cipher spec\n",
    "                        #print(fld_val['type'], \"Change cipher spec\")\n",
    "                        f=1\n",
    "\n",
    "                    elif fld_val['type'] == 21:                    # alert - close_notify\n",
    "                        #print(fld_val['type'], \"Alert\")\n",
    "                        f=1\n",
    "\n",
    "                    elif fld_val['type'] == 23:                    #application data\n",
    "                        #print(fld_val['type'], \"Application data\")  \n",
    "                        f=1\n",
    "                    else:\n",
    "                        print(\"Caught!!!!!!!!:\", fld_val['type'])\n",
    "                        caught.append(k)\n",
    "            except:\n",
    "                #print(\"Raw data\")\n",
    "                f=1\n",
    "\n",
    "        #print(str(k)+\" packet finished\")\n",
    "        #print(\"-----------------\")\n",
    "    print(query_no,\":\",caught)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "temp.layers(),type(temp[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "temp[0].show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "type(temp[0].payload.payload), type(temp[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "temp[0].fields['msg'][0].fields"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "ethernet_frame = tls_pcap[1]\n",
    "ip_packet = ethernet_frame.payload\n",
    "segment = ip_packet.payload\n",
    "data = segment.payload\n",
    "\n",
    "print(ethernet_frame.summary())\n",
    "print(ip_packet.summary())\n",
    "print(segment.summary())\n",
    "print(data.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "tls = segment.payload\n",
    "tls.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tls.msg[0].gmt_unix_time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#msgtype = 2 is for server_hello\n",
    "tls.msg[0].msgtype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for tls_pkt in pcap[TLS]:\n",
    "    tls = tls_pkt.payload.payload.payload\n",
    "    # \"handshake\" (not application data) and \"client_hello\"\n",
    "    if tls.type == 22 and tls.msg[0].msgtype == 1:\n",
    "        print(tls.msg[0].ext[0].servernames[0].servername.decode('UTF-8'))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}