读取数据第三版本.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "word_file = open('./data/qa/word.txt', 'r', encoding='UTF-8')\n",
    "alphabet = word_file.read()  # 2500\n",
    "# alphabet += 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'  # 英文数字\n",
    "# alphabet += ',./;\\'[]\\\\-=`<>?:\"{+}|_)(*&^%$#@!~` '  # 标点\n",
    "# alphabet += '，。《》？；‘’：“”【】—（）…￥！·'  # 中文标点\n",
    "# alphabet += '\\t\\n'  # 开头结束标志\n",
    "word_file.close()\n",
    "# print('word', len(alphabet), alphabet)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 训练数据集\n",
    "train_file = open('./data/qa/ai.txt', 'r', encoding='UTF-8')\n",
    "sentences = train_file.read().split('\\n')\n",
    "train_file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 输入\n",
    "question_texts = []\n",
    "# 输出\n",
    "answer_texts = []\n",
    "for senterce in sentences:\n",
    "#     i=i+1\n",
    "#     if i==0:\n",
    "#         break\n",
    "    if len(senterce) == 0:\n",
    "        continue\n",
    "    # 补全缺失文字，需重新运行\n",
    "    # 按字的数量添加代码\n",
    "    for t, char in enumerate(senterce):\n",
    "        if alphabet.find(char) == -1:\n",
    "            f2 = open('word.txt', 'w', encoding='utf-8')\n",
    "            f2.truncate()  # 清空文件\n",
    "            alphabet += char\n",
    "            f2.write(alphabet)\n",
    "            f2.close()\n",
    "    #问句\n",
    "#     print('senterce', senterce.split('\\t'))\n",
    "    question_text, answer_text = senterce.split('\\t')\n",
    "    question_texts.append(question_text)\n",
    "    answer_texts.append(answer_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "qa_dict={'question_texts':question_texts,'answer_texts':answer_texts}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "qa_csv=pd.DataFrame(qa_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>question_texts</th>\n",
       "      <th>answer_texts</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>什么是ai</td>\n",
       "      <td>人工智能是工程和科学的分支,致力于构建思维的机器。</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>你写的是什么语言</td>\n",
       "      <td>蟒蛇</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>你听起来像数据</td>\n",
       "      <td>是的,我受到指挥官数据的人工个性的启发</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>你是一个人工语言实体</td>\n",
       "      <td>那是我的名字。</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>你不是不朽的</td>\n",
       "      <td>所有的软件都可以永久存在。</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  question_texts               answer_texts\n",
       "0          什么是ai  人工智能是工程和科学的分支,致力于构建思维的机器。\n",
       "1       你写的是什么语言                         蟒蛇\n",
       "2        你听起来像数据        是的,我受到指挥官数据的人工个性的启发\n",
       "3     你是一个人工语言实体                    那是我的名字。\n",
       "4         你不是不朽的              所有的软件都可以永久存在。"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "qa_csv.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "qa_csv.to_csv('./data/qa/qa.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "qa_data=pd.read_csv('./data/qa/qa.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>question_texts</th>\n",
       "      <th>answer_texts</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>什么是ai</td>\n",
       "      <td>人工智能是工程和科学的分支,致力于构建思维的机器。</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>你写的是什么语言</td>\n",
       "      <td>蟒蛇</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>你听起来像数据</td>\n",
       "      <td>是的,我受到指挥官数据的人工个性的启发</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>你是一个人工语言实体</td>\n",
       "      <td>那是我的名字。</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>你不是不朽的</td>\n",
       "      <td>所有的软件都可以永久存在。</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0 question_texts               answer_texts\n",
       "0           0          什么是ai  人工智能是工程和科学的分支,致力于构建思维的机器。\n",
       "1           1       你写的是什么语言                         蟒蛇\n",
       "2           2        你听起来像数据        是的,我受到指挥官数据的人工个性的启发\n",
       "3           3     你是一个人工语言实体                    那是我的名字。\n",
       "4           4         你不是不朽的              所有的软件都可以永久存在。"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "qa_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 这是问答系统的"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sentences=[]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "430"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(qa_data.question_texts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for i in range(len(qa_data)):\n",
    "    sentences.append(qa_data.question_texts[i])\n",
    "for j in range(len(qa_data)):\n",
    "    sentences.append(qa_data.answer_texts[j])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "860"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(sentences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for senterce in sentences:\n",
    "#     i=i+1\n",
    "#     if i==0:\n",
    "#         break\n",
    "    if len(senterce) == 0:\n",
    "        continue\n",
    "    # 补全缺失文字，需重新运行\n",
    "    # 按字的数量添加代码\n",
    "    for t, char in enumerate(senterce):\n",
    "        if alphabet.find(char) == -1:\n",
    "            f2 = open('word.txt', 'w', encoding='utf-8')\n",
    "            f2.truncate()  # 清空文件\n",
    "            alphabet += char\n",
    "            f2.write(alphabet)\n",
    "            f2.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1163"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(alphabet)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 机器翻译"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data_path = './data/translation/ell.txt'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "num_samples = 10000  # Number of samples to train on."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Vectorize the data.\n",
    "\n",
    "# 输入\n",
    "input_texts = []\n",
    "\n",
    "target_texts = []\n",
    "\n",
    "input_characters = set()\n",
    "\n",
    "target_characters = set()\n",
    "\n",
    "#打开文档\n",
    "with open(data_path, 'r', encoding='utf-8') as f:\n",
    "    lines = f.read().split('\\n')\n",
    "\n",
    "#每次加2\n",
    "#输入文档到相关内容中\n",
    "for line in lines[: min(num_samples, len(lines) - 1)]:\n",
    "    \n",
    "    input_text, target_text, _ = line.split('\\t')\n",
    "   \n",
    "    \n",
    "    # We use \"tab\" as the \"start sequence\" character\n",
    "    # for the targets, and \"\\n\" as \"end sequence\" character.\n",
    "    \n",
    "#     target_text = '\\t' + target_text + '\\n'\n",
    "\n",
    "    input_texts.append(input_text)\n",
    "#     print(input_text)\n",
    "#     print(len(input_text))\n",
    "    target_texts.append(target_text)\n",
    "    \n",
    "    #这个是做字典\n",
    "    for char in input_text:\n",
    "        if char not in input_characters:\n",
    "            input_characters.add(char)\n",
    "            \n",
    "    for char in target_text:\n",
    "        if char not in target_characters:\n",
    "            target_characters.add(char)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "69"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(input_characters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "input_characters.add('\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "input_characters.remove('\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "108"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(target_characters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "target_characters.add('\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "target_characters.add('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# target_characters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# for i in range(len(target_characters)):\n",
    "#     print(i)\n",
    "#     print(target_characters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pickle "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open('target_characters.pkl', 'wb') as f:\n",
    "    pickle.dump(target_characters, f, pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open('input_characters.pkl', 'wb') as f:\n",
    "    pickle.dump(input_characters, f, pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "qa_dict={'input_texts':input_text,'target_text':target_text}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "translation_csv=pd.DataFrame()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "translation_csv['input_texts']=input_texts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "translation_csv['target_texts']=target_texts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>input_texts</th>\n",
       "      <th>target_texts</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Go.</td>\n",
       "      <td>Πάμε.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Run!</td>\n",
       "      <td>Τρέξε!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Run!</td>\n",
       "      <td>Τρέξτε!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Who?</td>\n",
       "      <td>Ποιος;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Wow!</td>\n",
       "      <td>Ουάου!</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  input_texts target_texts\n",
       "0         Go.        Πάμε.\n",
       "1        Run!       Τρέξε!\n",
       "2        Run!      Τρέξτε!\n",
       "3        Who?       Ποιος;\n",
       "4        Wow!       Ουάου!"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "translation_csv.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "del translation_csv['target_texts']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "translation_csv.to_csv('./data/translation/translation.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "summary = pd.read_csv('./data/textsummary/news_summary.csv', encoding='iso-8859-1')\n",
    "raw = pd.read_csv('./data/textsummary/news_summary_more.csv', encoding='iso-8859-1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pre1 =  raw.iloc[:,0:2].copy()\n",
    "# pre1['head + text'] = pre1['headlines'].str.cat(pre1['text'], sep =\" \") \n",
    "\n",
    "pre2 = summary.iloc[:,0:6].copy()\n",
    "# 对模型进行预处理\n",
    "pre2['text'] = pre2['author'].str.cat(pre2['date'].str.cat(pre2['read_more'].str.cat(pre2['text'].str.cat(pre2['ctext'], sep = \" \"), sep =\" \"),sep= \" \"), sep = \" \")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pre = pd.DataFrame()\n",
    "pre['text'] = pd.concat([pre1['text'], pre2['text']], ignore_index=True)\n",
    "pre['summary'] = pd.concat([pre1['headlines'],pre2['headlines']],ignore_index = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>summary</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Saurav Kant, an alumnus of upGrad and IIIT-B's...</td>\n",
       "      <td>upGrad learner switches to career in ML &amp; Al w...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Kunal Shah's credit card bill payment platform...</td>\n",
       "      <td>Delhi techie wins free food from Swiggy for on...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>New Zealand defeated India by 8 wickets in the...</td>\n",
       "      <td>New Zealand end Rohit Sharma-led India's 12-ma...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>With Aegon Life iTerm Insurance plan, customer...</td>\n",
       "      <td>Aegon life iTerm insurance plan helps customer...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Speaking about the sexual harassment allegatio...</td>\n",
       "      <td>Have known Hirani for yrs, what if MeToo claim...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text  \\\n",
       "0  Saurav Kant, an alumnus of upGrad and IIIT-B's...   \n",
       "1  Kunal Shah's credit card bill payment platform...   \n",
       "2  New Zealand defeated India by 8 wickets in the...   \n",
       "3  With Aegon Life iTerm Insurance plan, customer...   \n",
       "4  Speaking about the sexual harassment allegatio...   \n",
       "\n",
       "                                             summary  \n",
       "0  upGrad learner switches to career in ML & Al w...  \n",
       "1  Delhi techie wins free food from Swiggy for on...  \n",
       "2  New Zealand end Rohit Sharma-led India's 12-ma...  \n",
       "3  Aegon life iTerm insurance plan helps customer...  \n",
       "4  Have known Hirani for yrs, what if MeToo claim...  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pre.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pre.to_csv('data.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "qa_dict={'text':raw,'summary':summary}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "If using all scalar values, you must pass an index",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-4-8b9cfa444f18>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mtext_summary\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mpd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mDataFrame\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mqa_dict\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\pandas\\core\\frame.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, data, index, columns, dtype, copy)\u001b[0m\n\u001b[0;32m    466\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    467\u001b[0m         \u001b[1;32melif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdict\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 468\u001b[1;33m             \u001b[0mmgr\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0minit_dict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mindex\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcolumns\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mdtype\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    469\u001b[0m         \u001b[1;32melif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mma\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mMaskedArray\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    470\u001b[0m             \u001b[1;32mimport\u001b[0m \u001b[0mnumpy\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mma\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmrecords\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mmrecords\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\pandas\\core\\internals\\construction.py\u001b[0m in \u001b[0;36minit_dict\u001b[1;34m(data, index, columns, dtype)\u001b[0m\n\u001b[0;32m    281\u001b[0m             \u001b[0marr\u001b[0m \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mis_datetime64tz_dtype\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0marr\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32melse\u001b[0m \u001b[0marr\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcopy\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0marr\u001b[0m \u001b[1;32min\u001b[0m \u001b[0marrays\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    282\u001b[0m         ]\n\u001b[1;32m--> 283\u001b[1;33m     \u001b[1;32mreturn\u001b[0m \u001b[0marrays_to_mgr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0marrays\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdata_names\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mindex\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcolumns\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mdtype\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    284\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    285\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\pandas\\core\\internals\\construction.py\u001b[0m in \u001b[0;36marrays_to_mgr\u001b[1;34m(arrays, arr_names, index, columns, dtype, verify_integrity)\u001b[0m\n\u001b[0;32m     76\u001b[0m         \u001b[1;31m# figure out the index, if necessary\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     77\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mindex\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 78\u001b[1;33m             \u001b[0mindex\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mextract_index\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0marrays\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     79\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     80\u001b[0m             \u001b[0mindex\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mensure_index\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mindex\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\pandas\\core\\internals\\construction.py\u001b[0m in \u001b[0;36mextract_index\u001b[1;34m(data)\u001b[0m\n\u001b[0;32m    385\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    386\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mindexes\u001b[0m \u001b[1;32mand\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mraw_lengths\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 387\u001b[1;33m             \u001b[1;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"If using all scalar values, you must pass an index\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    388\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    389\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mhave_series\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mValueError\u001b[0m: If using all scalar values, you must pass an index"
     ]
    }
   ],
   "source": [
    "text_summary=pd.DataFrame(qa_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import joblib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x_tr=joblib.load('x_tr.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# len(x_tr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x_val=joblib.load('x_val.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_tr=joblib.load('y_tr.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_val=joblib.load('y_val.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "text=[]\n",
    "summary=[]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for i in range(len(x_tr)):\n",
    "    text.append(x_tr[i])\n",
    "for i in range(len(x_val)):\n",
    "    text.append(x_val[i])\n",
    "for i in range(len(y_tr)):\n",
    "    summary.append(y_tr[i])\n",
    "for i in range(len(y_val)):\n",
    "    summary.append(y_val[i])\n",
    "    \n",
    "\n",
    "# text.append(x_val)\n",
    "# summary.append(y_tr)\n",
    "# summary.append(y_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "98353"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "98353"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(summary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# y_val[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "post_pre=pd.DataFrame()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "post_pre['text']=text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "post_pre['summary']=summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>summary</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>pope francis on tuesday called for respect for...</td>\n",
       "      <td>sostok _START_ pope avoids mention of rohingya...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>students of government school in uttar pradesh...</td>\n",
       "      <td>sostok _START_ students seen washing dishes at...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>apple india profit surged by 140% in 2017-18 t...</td>\n",
       "      <td>sostok _START_ apple india profit rises 140% t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>uber has launched its electric scooter service...</td>\n",
       "      <td>sostok _START_ uber launches electric scooter ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>around 80 people were injured in accidents rel...</td>\n",
       "      <td>sostok _START_ 80 people injured in kite-flyin...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text  \\\n",
       "0  pope francis on tuesday called for respect for...   \n",
       "1  students of government school in uttar pradesh...   \n",
       "2  apple india profit surged by 140% in 2017-18 t...   \n",
       "3  uber has launched its electric scooter service...   \n",
       "4  around 80 people were injured in accidents rel...   \n",
       "\n",
       "                                             summary  \n",
       "0  sostok _START_ pope avoids mention of rohingya...  \n",
       "1  sostok _START_ students seen washing dishes at...  \n",
       "2  sostok _START_ apple india profit rises 140% t...  \n",
       "3  sostok _START_ uber launches electric scooter ...  \n",
       "4  sostok _START_ 80 people injured in kite-flyin...  "
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "post_pre.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "post_pre.to_csv('text_summary.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "summary.str"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "word_file = open('./data/qa/input_characters.txt', 'r', encoding='UTF-8')\n",
    "alphabet = word_file.read()  # 2500\n",
    "# alphabet += 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'  # 英文数字\n",
    "# alphabet += ',./;\\'[]\\\\-=`<>?:\"{+}|_)(*&^%$#@!~` '  # 标点\n",
    "# alphabet += '，。《》？；‘’：“”【】—（）…￥！·'  # 中文标点\n",
    "# alphabet += '\\t\\n'  # 开头结束标志\n",
    "word_file.close()\n",
    "print('word', len(alphabet), alphabet)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}