Chapter2.1.1.3.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 从sklearn.datasets里导入新闻数据抓取器 fetch_20newsgroups"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.datasets import fetch_20newsgroups\n",
    "import  pandas as  pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "news = fetch_20newsgroups(subset='all')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "18846\n"
     ]
    }
   ],
   "source": [
    "# 输出数据的条数\n",
    "print len(news.data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "From: Mamatha Devineni Ratnam <mr47+@andrew.cmu.edu>\n",
      "Subject: Pens fans reactions\n",
      "Organization: Post Office, Carnegie Mellon, Pittsburgh, PA\n",
      "Lines: 12\n",
      "NNTP-Posting-Host: po4.andrew.cmu.edu\n",
      "\n",
      "\n",
      "\n",
      "I am sure some bashers of Pens fans are pretty confused about the lack\n",
      "of any kind of posts about the recent Pens massacre of the Devils. Actually,\n",
      "I am  bit puzzled too and a bit relieved. However, I am going to put an end\n",
      "to non-PIttsburghers' relief with a bit of praise for the Pens. Man, they\n",
      "are killing those Devils worse than I thought. Jagr just showed you why\n",
      "he is much better than his regular season stats. He is also a lot\n",
      "fo fun to watch in the playoffs. Bowman should let JAgr have a lot of\n",
      "fun in the next couple of games since the Pens are going to beat the pulp out of Jersey anyway. I was very disappointed not to see the Islanders lose the final\n",
      "regular season game.          PENS RULE!!!\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# 查看一条数据形式\n",
    "print news.data[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 20类新闻文本数据分割"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 从sklearn.cross_validation导入train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda/envs/root2/lib/python2.7/site-packages/sklearn/cross_validation.py:44: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.cross_validation import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 随机采样25%的数据样本作为测试集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train,X_test,y_train,y_test = train_test_split(news.data,news.target,test_size=0.25,random_state=33)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 从sklearn.feature_extraction.text里导入文本特征向量化模块"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "vec = CountVectorizer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train = vec.fit_transform(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_test = vec.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 从sklean.naive_bayes里导入朴素贝叶斯模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.naive_bayes import MultinomialNB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 使用默认配置初始化朴素贝叶斯"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "mnb = MultinomialNB()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 利用训练数据对模型参数进行估计"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MultinomialNB(alpha=1.0, class_prior=None, fit_prior=True)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mnb.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 对参数进行预测"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_predict = mnb.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 对性能进行评估（准确性；召回率；精确率和F1指标）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 从sklearn.metrics里导入classification_report用于详细的分类性能报告"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "# print 'The accuracy of Naive Bayes Classifier is', mnb.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The Accuracy of Naive Bayes Classifier is: 0.839770797963\n"
     ]
    }
   ],
   "source": [
    "#print 'Accuracy of LR Classifier:', lr.score(X_test, y_test)\n",
    "print 'The Accuracy of Naive Bayes Classifier is:', mnb.score(X_test,y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print classification_report(y_test,y_predict,target_names=news.target_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                          precision    recall  f1-score   support\n",
      "\n",
      "             alt.atheism       0.86      0.86      0.86       201\n",
      "           comp.graphics       0.59      0.86      0.70       250\n",
      " comp.os.ms-windows.misc       0.89      0.10      0.17       248\n",
      "comp.sys.ibm.pc.hardware       0.60      0.88      0.72       240\n",
      "   comp.sys.mac.hardware       0.93      0.78      0.85       242\n",
      "          comp.windows.x       0.82      0.84      0.83       263\n",
      "            misc.forsale       0.91      0.70      0.79       257\n",
      "               rec.autos       0.89      0.89      0.89       238\n",
      "         rec.motorcycles       0.98      0.92      0.95       276\n",
      "      rec.sport.baseball       0.98      0.91      0.95       251\n",
      "        rec.sport.hockey       0.93      0.99      0.96       233\n",
      "               sci.crypt       0.86      0.98      0.91       238\n",
      "         sci.electronics       0.85      0.88      0.86       249\n",
      "                 sci.med       0.92      0.94      0.93       245\n",
      "               sci.space       0.89      0.96      0.92       221\n",
      "  soc.religion.christian       0.78      0.96      0.86       232\n",
      "      talk.politics.guns       0.88      0.96      0.92       251\n",
      "   talk.politics.mideast       0.90      0.98      0.94       231\n",
      "      talk.politics.misc       0.79      0.89      0.84       188\n",
      "      talk.religion.misc       0.93      0.44      0.60       158\n",
      "\n",
      "             avg / total       0.86      0.84      0.82      4712\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print classification_report(y_test, y_predict, target_names = news.target_names)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}