example_get_stats_list.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import urllib.request\n",
    "import urllib.parse\n",
    "import json\n",
    "import pandas as pd\n",
    "import pprint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "    \"statsCode\": \"00200524\",\n",
      "    \"surveyYears\": \"2017\"\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "with open('setting/get_stats_list_sample.json') as f:\n",
    "    print(f.read())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open('setting/get_stats_list_sample.json') as f:\n",
    "    p = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'statsCode': '00200524', 'surveyYears': '2017'}\n"
     ]
    }
   ],
   "source": [
    "print(p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'dict'>\n"
     ]
    }
   ],
   "source": [
    "print(type(p))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "    \"appId\": \"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\"\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "with open('setting/app_id_sample.json') as f:\n",
    "    print(f.read())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open('setting/app_id.json') as f:\n",
    "    p_id = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "p.update(p_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "url = ('http://api.e-stat.go.jp/rest/2.1/app/json/getStatsList?'\n",
    "       + urllib.parse.urlencode(p))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with urllib.request.urlopen(url) as response:\n",
    "    data = response.read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'bytes'>\n"
     ]
    }
   ],
   "source": [
    "print(type(data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "d = json.loads(data.decode())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'GET_STATS_LIST': {'DATALIST_INF': {'NUMBER': 21,\n",
      "                                     'RESULT_INF': {...},\n",
      "                                     'TABLE_INF': [...]},\n",
      "                    'PARAMETER': {'DATA_FORMAT': 'J',\n",
      "                                  'LANG': 'J',\n",
      "                                  'STATS_CODE': '00200524',\n",
      "                                  'SURVEY_YEARS': 2017},\n",
      "                    'RESULT': {'DATE': '2018-05-24T19:38:42.172+09:00',\n",
      "                               'ERROR_MSG': '正常に終了しました。',\n",
      "                               'STATUS': 0}}}\n"
     ]
    }
   ],
   "source": [
    "pprint.pprint(d, depth=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "table_info_list = d['GET_STATS_LIST']['DATALIST_INF']['TABLE_INF']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'list'>\n"
     ]
    }
   ],
   "source": [
    "print(type(table_info_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "21\n"
     ]
    }
   ],
   "source": [
    "print(len(table_info_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'dict'>\n"
     ]
    }
   ],
   "source": [
    "print(type(table_info_list[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'@id': '0003215840',\n",
      " 'CYCLE': '-',\n",
      " 'GOV_ORG': {'$': '総務省', '@code': '00200'},\n",
      " 'MAIN_CATEGORY': {'$': '人口・世帯', '@code': '02'},\n",
      " 'OPEN_DATE': '2018-04-13',\n",
      " 'OVERALL_TOTAL_NUMBER': 816,\n",
      " 'SMALL_AREA': 0,\n",
      " 'STATISTICS_NAME': '人口推計 平成29年10月1日現在人口推計',\n",
      " 'STATISTICS_NAME_SPEC': {'TABULATION_CATEGORY': '人口推計',\n",
      "                          'TABULATION_SUB_CATEGORY1': '平成29年10月1日現在人口推計'},\n",
      " 'STAT_NAME': {'$': '人口推計', '@code': '00200524'},\n",
      " 'SUB_CATEGORY': {'$': '人口', '@code': '01'},\n",
      " 'SURVEY_DATE': 201710,\n",
      " 'TITLE': {'$': '年齢（各歳），男女別人口及び人口性比－総人口，日本人人口', '@no': '001'},\n",
      " 'TITLE_SPEC': {'TABLE_NAME': '年齢（各歳），男女別人口及び人口性比－総人口，日本人人口'},\n",
      " 'UPDATED_DATE': '2018-04-14'}\n"
     ]
    }
   ],
   "source": [
    "pprint.pprint(table_info_list[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = pd.io.json.json_normalize(table_info_list, sep='_')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['@id', 'CYCLE', 'GOV_ORG_$', 'GOV_ORG_@code', 'MAIN_CATEGORY_$',\n",
      "       'MAIN_CATEGORY_@code', 'OPEN_DATE', 'OVERALL_TOTAL_NUMBER',\n",
      "       'SMALL_AREA', 'STATISTICS_NAME',\n",
      "       'STATISTICS_NAME_SPEC_TABULATION_CATEGORY',\n",
      "       'STATISTICS_NAME_SPEC_TABULATION_SUB_CATEGORY1', 'STAT_NAME_$',\n",
      "       'STAT_NAME_@code', 'SUB_CATEGORY_$', 'SUB_CATEGORY_@code',\n",
      "       'SURVEY_DATE', 'TITLE_$', 'TITLE_@no', 'TITLE_SPEC_TABLE_NAME',\n",
      "       'UPDATED_DATE'],\n",
      "      dtype='object')\n"
     ]
    }
   ],
   "source": [
    "print(df.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['id', 'CYCLE', 'GOV_ORG_val', 'GOV_ORG_code', 'MAIN_CATEGORY_val',\n",
      "       'MAIN_CATEGORY_code', 'OPEN_DATE', 'OVERALL_TOTAL_NUMBER', 'SMALL_AREA',\n",
      "       'STATISTICS_NAME', 'STATISTICS_NAME_SPEC_TABULATION_CATEGORY',\n",
      "       'STATISTICS_NAME_SPEC_TABULATION_SUB_CATEGORY1', 'STAT_NAME_val',\n",
      "       'STAT_NAME_code', 'SUB_CATEGORY_val', 'SUB_CATEGORY_code',\n",
      "       'SURVEY_DATE', 'TITLE_val', 'TITLE_no', 'TITLE_SPEC_TABLE_NAME',\n",
      "       'UPDATED_DATE'],\n",
      "      dtype='object')\n"
     ]
    }
   ],
   "source": [
    "df.columns = [s.replace('@', '').replace('$', 'val') for s in df.columns]\n",
    "print(df.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df.to_csv('download/stats_list_{}_{}.csv'.format(p['statsCode'], p[\"surveyYears\"]), index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}