Unify_NutriName&ServingUnit.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Purpose of Preprocessing\n",
    "- Select only the nutrients to use\n",
    "- Unify nutrient names\n",
    "- Unify serving unit names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import json\n",
    "import re"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 0 ) Import data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "vit_path = '../output/vitamins/'\n",
    "min_path = '../output/minerals/'\n",
    "amino_path = '../output/amino-acids/'\n",
    "\n",
    "vit_list = [i for i in os.listdir(vit_path) if i != '.DS_Store']\n",
    "min_list = [i for i in os.listdir(min_path) if i != '.DS_Store']\n",
    "amino_list = [i for i in os.listdir(amino_path) if i != '.DS_Store']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(len(vit_list), len(min_list), len(amino_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(len(vit_list)+len(min_list)+len(amino_list))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1 ) Unify nutrients name"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.1) get_nutri_key\n",
    "- Get total nutrients name\n",
    "- Check if there is scrap error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_nutri_key(prod_path, prod_list):\n",
    "    total_nutri_list = []\n",
    "    error_cnt = 0\n",
    "    \n",
    "    for prod in prod_list:\n",
    "        with open(prod_path+prod, encoding = 'utf-8-sig') as f:\n",
    "            prod_data = json.load(f)\n",
    "            try:\n",
    "                prod_nutri_list = list(prod_data['nutrients'].keys())\n",
    "                total_nutri_list.extend(prod_nutri_list)\n",
    "            except:\n",
    "                error_cnt += 1\n",
    "                print(prod_data['prod_cd'])\n",
    "    print(error_cnt) # To check scrap error\n",
    "\n",
    "    total_nutri_list = sorted(list(set(total_nutri_list)))\n",
    "    return total_nutri_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "vit_nutri_list = get_nutri_key(vit_path, vit_list)\n",
    "min_nutri_list = get_nutri_key(min_path, min_list)\n",
    "amino_nutri_list = get_nutri_key(amino_path, amino_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Create \"total_nutri_list\" which has total nutrients names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "total_nutri_list = vit_nutri_list + min_nutri_list + amino_nutri_list # 모든 nutrients 명이 담겨있는 리스트\n",
    "total_nutri_list = list(set(total_nutri_list))\n",
    "print(len(total_nutri_list))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Query nutrient name within total_nutri_list\n",
    "- Create a list of analogous nutrient names for each nutrient in Excel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Nutrients name with English\n",
    "[i for i in total_nutri_list if 'vitamind' in i.lower()]\n",
    "# ['VitaminD', 'VitaminD-3', 'VitaminD2', 'VitaminD3', 'VitaminD35', 'VitaminDOrganic', 'VitaminDs']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Nutrients name with Korean\n",
    "[i for i in total_nutri_list if '단백질' in i]\n",
    "# ['단백질', '단백질최대', '분리대두단백질및제공성분', '쌀단백질농축물', '완두콩단백질', '통현미단백질', '현미단백질']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.2) change_nutri_nm\n",
    "- Delete 'Calories' key\n",
    "- Delete other keys except for what I will use\n",
    "- Unify nutrient names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def change_nutri_nm(json):\n",
    "    \n",
    "    for key in list(json['nutrients'].keys()):\n",
    "\n",
    "        # 1. Protein\n",
    "        if key in ['Protein','RiceProteinConcentrate', '단백질', '단백질최대', '분리대두단백질및제공성분', '쌀단백질농축물', '완두콩단백질', '통현미단백질', '현미단백질']:\n",
    "            json['nutrients']['Protein'] = json['nutrients'].pop(key)\n",
    "        \n",
    "        # 2. Fat\n",
    "        elif key in ['Fat','TotalFat', 'EssentialFattyAcid-RichWholeFoodEnergy','EssentialFattyAcidComplex','총지방']:\n",
    "            json['nutrients']['Fat'] = json['nutrients'].pop(key)\n",
    "        \n",
    "        # 3. Carbohydrate\n",
    "        elif key in ['Carbohydrate','TotalCarb', 'TotalCarbohydrate', 'TotalCarbohydrates','총탄수화물', '탄수화물']:\n",
    "            json['nutrients']['Carbohydrate'] = json['nutrients'].pop(key)\n",
    "        \n",
    "        # 4. Dietary_Fiber\n",
    "        elif key in ['DietaryFiber', '식이섬유', '프리바이오틱식이섬유']:\n",
    "            json['nutrients']['Dietary_Fiber'] = json['nutrients'].pop(key)\n",
    "        \n",
    "        # 5. Calcium\n",
    "        elif key in ['Calcium','CoralCalcium', '구연산칼슘','미결정하이드록시아파타이트칼슘','미세결정질칼슘하이드록시아파타이트','산호칼슘',\n",
    "                    '유기농식물칼슘혼합물유기농해조류','칼슘','칼슘D-글루카레이트','칼슘프룩토보레이트','칼슘프룩토보레이트보론6mg공급','화석화된산호칼슘']:\n",
    "            json['nutrients']['Calcium'] = json['nutrients'].pop(key)\n",
    "        \n",
    "        # 6. Iron\n",
    "        elif key in ['Iron', '철', '철분']:\n",
    "            json['nutrients']['Iron'] = json['nutrients'].pop(key)\n",
    "        \n",
    "        # 7. Magnesium\n",
    "        elif key in ['Magnesium','PureHighGradeIonicMagnesium', '마그네슘','마그네슘L-트레오네이트','마그네슘말레이트트라이하이드레이트',\n",
    "                        '마그네슘말산염트라이하이드레이트','마그네슘원소', '순수고급이온성마그네슘','원소마그네슘']:\n",
    "            json['nutrients']['Magnesium'] = json['nutrients'].pop(key)\n",
    "        \n",
    "        # 8. Phosphorus\n",
    "        elif key in ['Phosphorus','인']:\n",
    "            json['nutrients']['Phosphorus'] = json['nutrients'].pop(key)\n",
    "        \n",
    "        # 9. Potassium\n",
    "        elif key in ['Potassium','구연산칼륨', '칼륨']:\n",
    "            json['nutrients']['Potassium'] = json['nutrients'].pop(key)\n",
    "        \n",
    "        # 10. Sodium\n",
    "        elif key in ['Sodium','나트륨']:\n",
    "            json['nutrients']['Sodium'] = json['nutrients'].pop(key)\n",
    "        \n",
    "        # 11. Zinc\n",
    "        elif key in ['Zinc','산화아연', '아연', '아연-L-카르노신', '아연L-카르노신']:\n",
    "            json['nutrients']['Zinc'] = json['nutrients'].pop(key)\n",
    "        \n",
    "        # 12. Copper\n",
    "        elif key in ['Copper','구리']:\n",
    "            json['nutrients']['Copper'] = json['nutrients'].pop(key)\n",
    "        \n",
    "        # 13. Manganese\n",
    "        elif key in ['Manganese','구연산망간', '망간']:\n",
    "            json['nutrients']['Manganese'] = json['nutrients'].pop(key)\n",
    "        \n",
    "        # 14. Selenium\n",
    "        elif key in ['Selenium','셀레늄']:\n",
    "            json['nutrients']['Selenium'] = json['nutrients'].pop(key)\n",
    "\n",
    "        # 15. Vitamin_A\n",
    "        elif key in ['VitaminA', 'VitaminA100', '비타민A', '전체식품영양소비타민A', '카로티노이드함유비타민A']:\n",
    "            json['nutrients']['Vitamin_A'] = json['nutrients'].pop(key)\n",
    "\n",
    "        # 16. Vitamin_D\n",
    "        elif key in ['VitaminD','VitaminD-3','VitaminD2','VitaminD3','VitaminD35','VitaminDOrganic', 'VitaminDs',\n",
    "                    '비타민D', '비타민D-3', '비타민D2', '비타민D25mcg', '비타민D3']:\n",
    "            json['nutrients']['Vitamin_D'] = json['nutrients'].pop(key)\n",
    "\n",
    "        # 17. Niacin\n",
    "        elif key in ['Niacin','나이아신', '비타민B-3나이아신']:\n",
    "            json['nutrients']['Niacin'] = json['nutrients'].pop(key)\n",
    "\n",
    "        # 18. Folic_acid\n",
    "        elif key in ['Folic', 'FolicAcid', 'Folicacid', 'Folate','엽산', '엽산C', '엽산염', '엽산해당량']:\n",
    "            json['nutrients']['Folic_acid'] = json['nutrients'].pop(key)\n",
    "\n",
    "        # 19. Vitamin_B12\n",
    "        elif key in ['VitaminB-12','VitaminB-12asCyanocobalamin', 'VitaminB12','비타민B-12', '비타민B12']:\n",
    "            json['nutrients']['Vitamin_B12'] = json['nutrients'].pop(key)\n",
    "\n",
    "        # 20. Vitamin_B6\n",
    "        elif key in ['VitaminB-6', 'VitaminB6', '비타민B-6', '비타민B6']:\n",
    "            json['nutrients']['Vitamin_B6'] = json['nutrients'].pop(key)\n",
    "\n",
    "        # 21. Vitamin_C\n",
    "        elif key in ['VitaminC','VitaminCSupportBase','VitaminCasPureWay-C','VitaminCfromPureway-C',\n",
    "                    'Living비타민C복합체유기농아세로라체리','PureWay-C에서추출한비타민C','비타민C','비타민C보조물','퓨어웨이-C에서추출한비타민C']:\n",
    "            json['nutrients']['Vitamin_C'] = json['nutrients'].pop(key)\n",
    "\n",
    "        # 22. Vitamin_E\n",
    "        elif key in ['VitaminE','감마-비타민E복합체','비타민E','비타민E함유','비타민E혼합토코페롤','토코페롤및토코트리에놀함유비타민E']:\n",
    "            json['nutrients']['Vitamin_E'] = json['nutrients'].pop(key)\n",
    "\n",
    "        # 23. Vitamin_K\n",
    "        elif key in ['VitaminK', 'VitaminK-2', 'VitaminK1', 'VitaminK2', 'VitaminKactivityfrom', \n",
    "                 '비타민K','비타민K-1','비타민K-2','비타민K1','비타민K12','비타민K2','비타민K2-7','비타민K2MK-4',\n",
    "                '비타민K2MK-7','비타민KK-2','비타민KK1','비타민K비타민K2','비타민K활성량비타민K종류','비타민K활성성분']:\n",
    "            json['nutrients']['Vitamin_K'] = json['nutrients'].pop(key)\n",
    "\n",
    "        # 24. Leucine\n",
    "        elif key in ['Leucine','InstantlySolubleBCAAsL-Leucine','L-Leucine','ProprietaryBranchedChainAminoAcidComplexL-Leucine',\n",
    "        \t'AjiPureMP9EAAMatrixL-류신','L-류신','류신','류신펩티드','미분화된L-류신', '이소류신','즉시용해되는BCAAL-류신']:\n",
    "            json['nutrients']['Leucine'] = json['nutrients'].pop(key)\n",
    "\n",
    "        # 25. Iso_Leucine\n",
    "        elif key in ['Isoleucine', 'L-Isoleucine', 'L-아이소류신','L-이소류신', '미분화된L-이소류신','아이소류신','이소류신']:\n",
    "            json['nutrients']['Iso_Leucine'] = json['nutrients'].pop(key)\n",
    "\n",
    "        # 26. Histidine\n",
    "        elif key in ['Histidine','L-HistidineHCL','L-히스티딘', 'L-히스티딘HCI', 'L-히스티딘HCl', 'L-히스티딘에이치씨엘', '히스티딘']:\n",
    "            json['nutrients']['Histidine'] = json['nutrients'].pop(key)\n",
    "\n",
    "        # 27. Linoleic_Acid\n",
    "        elif key in ['ConjugatedLinoleicAcidPowder']:\n",
    "            json['nutrients']['Linoleic_Acid'] = json['nutrients'].pop(key)\n",
    "\n",
    "        # 28. Alpha_Linolenic_Acid\n",
    "        elif key in ['필수지방산복합체알파리놀렌산']:\n",
    "            json['nutrients']['Alpha_Linolenic_Acid'] = json['nutrients'].pop(key)\n",
    "\n",
    "        # 29. Lysine\n",
    "        elif key in ['Lysine','Hydroxylysine','L-Lysine','L-LysineFree-Form','L-LysineHCL','L-LysineHCl','L-lysine',\n",
    "                'L-라이신','L-라이신HCI','L-라이신HCl','L-라이신에이치씨엘','L-라이신염산염','라이신', '라이신염산염','비건콜라겐강화혼합물L-라이신HCI',\n",
    "                '알파뉴로복합체글라이신L-라이신HCIL-글루타민L-알지닌HCI알파-GPC','필수아미노산L-라이신', 'L-리신', 'L-리신HCl']:\n",
    "            json['nutrients']['Lysine'] = json['nutrients'].pop(key)\n",
    "\n",
    "        # 30. Methionine\n",
    "        elif key in ['Methionine','L-Methionine', 'L-메티오닌', 'dl-메티오닌', '메티오닌']:\n",
    "            json['nutrients']['Methionine'] = json['nutrients'].pop(key)\n",
    "\n",
    "        # 31. Phenylalanine\n",
    "        elif key in ['Phenylalanine','DL-Phenylalanine', 'L-Phenylalanine', 'D-페닐알라닌', 'DL-페닐알라닌', 'L-페닐알라닌', '페닐알라닌']:\n",
    "            json['nutrients']['Phenylalanine'] = json['nutrients'].pop(key)\n",
    "\n",
    "        # 32. Tyrosine\n",
    "        elif key in ['Tyrosine','L-Tyrosine', 'L-티로신', 'N-아세틸L-티로신', '티로신']:\n",
    "            json['nutrients']['Tyrosine'] = json['nutrients'].pop(key)\n",
    "\n",
    "        # 33. Threonine\n",
    "        elif key in ['Threonine','L-Threonine','L-트레오닌', '트레오닌']:\n",
    "            json['nutrients']['Threonine'] = json['nutrients'].pop(key)\n",
    "\n",
    "        # 34. Valine\n",
    "        elif key in ['Valine','L-Valine', 'L-발린', '미분화된L-발린', '발린']:\n",
    "            json['nutrients']['Valine'] = json['nutrients'].pop(key)\n",
    "\n",
    "        # 35. Cholesterol\n",
    "        elif key in ['Cholesterol', '콜레스테롤']:\n",
    "            json['nutrients']['Cholesterol'] = json['nutrients'].pop(key)\n",
    "        \n",
    "        # 칼로리, Calories 삭제\n",
    "        elif key in ['칼로리', 'Calories']:\n",
    "            del json['nutrients'][key]\n",
    "        \n",
    "        # Others\n",
    "        else:\n",
    "            del json['nutrients'][key]  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2 ) Unify unit and serving unit"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Check the number of serving errors when scraping data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "serving_error_list = []\n",
    "for vit in vit_list:\n",
    "    with open(vit_path+vit) as f:\n",
    "        vit_data = json.load(f)\n",
    "        if vit_data['serving'] == 'serving unit error':\n",
    "            serving_error_list.append(vit_data)\n",
    "\n",
    "for min in min_list:\n",
    "    with open(min_path+min) as f:\n",
    "        min_data = json.load(f)\n",
    "        if min_data['serving'] == 'serving unit error':\n",
    "            serving_error_list.append(min_data)\n",
    "\n",
    "for amino in amino_list:\n",
    "    with open(amino_path+amino) as f:\n",
    "        amino_data = json.load(f)\n",
    "        if amino_data['serving'] == 'serving unit error':\n",
    "            serving_error_list.append(amino_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(len(serving_error_list))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.1 ) get_serving\n",
    "- Get serving amount & unit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_serving(prod_path, prod_list):\n",
    "    serving_amount_list = []\n",
    "    serving_unit_list = []\n",
    "    error_cnt = 0\n",
    "    \n",
    "    for prod in prod_list:\n",
    "        with open(prod_path+prod) as f:\n",
    "            prod_data = json.load(f)\n",
    "            try:\n",
    "                serving_amount_list.append(prod_data['serving'][0])\n",
    "                serving_unit_list.append(prod_data['serving'][1])\n",
    "            except:\n",
    "                error_cnt += 1\n",
    "    print(error_cnt)\n",
    "    return serving_amount_list, serving_unit_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "0\n",
      "0\n"
     ]
    }
   ],
   "source": [
    "vit_serving_amount_list, vit_serving_unit_list = get_serving(vit_path, vit_list)\n",
    "min_serving_amount_list, min_serving_unit_list = get_serving(min_path, min_list)\n",
    "amino_serving_amount_list, amino_serving_unit_list = get_serving(amino_path, amino_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Create lists of total serving amount and unit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "total_serving_amount_list = vit_serving_amount_list + min_serving_amount_list + amino_serving_amount_list\n",
    "total_serving_unit_list = vit_serving_unit_list + min_serving_unit_list + amino_serving_unit_list"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Create dictionaries to check the frequency of each serving amount and unit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# frequency of each serving amount\n",
    "total_serving_amount_dict = {i:total_serving_amount_list.count(i) for i in total_serving_amount_list}\n",
    "total_serving_amount_dict = sorted(total_serving_amount_dict.items(), key=lambda x: -x[1])\n",
    "\n",
    "# frequency of each serving unit\n",
    "total_serving_unit_dict = {i:total_serving_unit_list.count(i) for i in total_serving_unit_list}\n",
    "total_serving_unit_dict = sorted(total_serving_unit_dict.items(), key=lambda x: -x[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('1', 1938),\n",
       " ('2', 510),\n",
       " ('3', 205),\n",
       " ('4', 140),\n",
       " ('6', 33),\n",
       " ('5', 32),\n",
       " ('8', 12),\n",
       " ('30', 8),\n",
       " ('60', 4),\n",
       " ('10', 4),\n",
       " ('25', 3),\n",
       " ('9', 2),\n",
       " ('15', 2),\n",
       " ('75', 2),\n",
       " ('13', 2),\n",
       " ('12', 2),\n",
       " ('0', 1),\n",
       " ('50', 1),\n",
       " ('120', 1),\n",
       " ('35', 1),\n",
       " ('54', 1)]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "total_serving_amount_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('정', 1684),\n",
       " ('캡슐', 104),\n",
       " ('Tablet', 97),\n",
       " ('티스푼', 86),\n",
       " ('Capsule', 83),\n",
       " ('스쿱', 72),\n",
       " ('개', 67),\n",
       " ('Softgel', 50),\n",
       " ('방울', 48),\n",
       " ('Capsules', 47),\n",
       " ('Tablets', 30),\n",
       " ('테이블스푼', 27),\n",
       " ('ml', 26),\n",
       " ('g', 25),\n",
       " ('capsule', 18),\n",
       " ('식물성', 17),\n",
       " ('tablet', 16),\n",
       " ('세', 15),\n",
       " ('회', 15),\n",
       " ('채식', 14),\n",
       " ('Lozenge', 13),\n",
       " ('Softgels', 12),\n",
       " ('VegCap', 12),\n",
       " ('Teaspoon', 12),\n",
       " ('팩', 11),\n",
       " ('봉지', 11),\n",
       " ('Drops', 10),\n",
       " ('Vegetarian', 10),\n",
       " ('scoop', 10),\n",
       " ('Scoop', 10),\n",
       " ('Caplet', 9),\n",
       " ('MicroLingual', 8),\n",
       " ('Chewable', 7),\n",
       " ('곰', 7),\n",
       " ('소프트젤', 7),\n",
       " ('Quick', 7),\n",
       " ('Level', 7),\n",
       " ('VegCaps', 7),\n",
       " ('teaspoons', 7),\n",
       " ('큰', 6),\n",
       " ('Veg', 6),\n",
       " ('Vegetable', 6),\n",
       " ('찻', 6),\n",
       " ('capsules', 5),\n",
       " ('Stick', 5),\n",
       " ('mL', 5),\n",
       " ('Tube', 5),\n",
       " ('tsp', 5),\n",
       " ('츄어블', 5),\n",
       " ('tbsp', 4),\n",
       " ('소프트', 4),\n",
       " ('마리', 4),\n",
       " ('Drop', 4),\n",
       " ('Veggie', 4),\n",
       " ('종류의', 4),\n",
       " ('tablets', 4),\n",
       " ('Sprays', 4),\n",
       " ('국자', 4),\n",
       " ('특종', 4),\n",
       " ('scoops', 4),\n",
       " ('stick', 3),\n",
       " ('정을', 3),\n",
       " ('드롭퍼', 3),\n",
       " ('vegetarian', 3),\n",
       " ('Packet', 3),\n",
       " ('drop', 3),\n",
       " ('정량', 3),\n",
       " ('수준', 3),\n",
       " ('fl', 3),\n",
       " ('스푼', 3),\n",
       " ('Spray', 2),\n",
       " ('softgel', 2),\n",
       " ('레벨', 2),\n",
       " ('Gummies', 2),\n",
       " ('Tablespoons', 2),\n",
       " ('마름모꼴', 2),\n",
       " ('Small', 2),\n",
       " ('packet', 2),\n",
       " ('Coated', 2),\n",
       " ('teaspoon', 2),\n",
       " ('Tablespoon', 2),\n",
       " ('젤리', 1),\n",
       " ('Micro', 1),\n",
       " ('Soft', 1),\n",
       " ('Jellies', 1),\n",
       " ('Fast', 1),\n",
       " ('Fluid', 1),\n",
       " ('VeganCaps', 1),\n",
       " ('Organitab', 1),\n",
       " ('Tbsp', 1),\n",
       " ('웨이퍼', 1),\n",
       " ('Pieces', 1),\n",
       " ('츄어블정', 1),\n",
       " ('드롭', 1),\n",
       " ('Pure', 1),\n",
       " ('chewable', 1),\n",
       " ('multi', 1),\n",
       " ('Grams', 1),\n",
       " ('Sustained', 1),\n",
       " ('lozenge', 1),\n",
       " ('VCAP', 1),\n",
       " ('vegetable', 1),\n",
       " ('개의', 1),\n",
       " ('gummy', 1),\n",
       " ('tables', 1),\n",
       " ('완전', 1),\n",
       " ('Dropper', 1),\n",
       " ('caplets', 1),\n",
       " ('액상', 1),\n",
       " ('그램', 1),\n",
       " ('숟가락', 1),\n",
       " ('찻숫가락', 1),\n",
       " ('Scoops', 1),\n",
       " ('둥근', 1),\n",
       " ('쌓아', 1),\n",
       " ('작은', 1),\n",
       " ('Milliliter', 1)]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "total_serving_unit_dict"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Create a list of analogous units using this dictionary in Excel to unify serving units<br>\n",
    "<br>\n",
    "ex. ['VegCap','Vegetarian','VegCaps','VeganCaps','Veg Capsule','VCAP'] -> Vege Capsule"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.2 ) change_serving_unit_nm\n",
    "- Unify serving units"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def change_serving_unit_nm(json):\n",
    "\n",
    "    if json['serving'][1] in ['Tablet','정','개', 'Tablets', 'tablet', '회', 'Lozenge',\n",
    "                                'MicroLingual','Chewable','츄어블','마리','tablets','정을','정량','마름모꼴',\n",
    "                                'Micro','Organitab','Pieces','츄어블정','chewable','lozenge','개의', 'tables','caplets']:\n",
    "        json['serving'][1] = 'Tablet'\n",
    "\n",
    "    elif json ['serving'][1] in ['Capsule','캡슐', 'Capsules', 'capsule','Caplet','capsules','Coated']:\n",
    "        json['serving'][1] = 'Capsule'\n",
    "\n",
    "    elif json ['serving'][1] in ['Softgel','Softgels','소프트젤','소프트','softgel','Soft']:\n",
    "        json['serving'][1] = 'Softgel'\n",
    "\n",
    "    elif json ['serving'][1] in ['Scoop','스쿱','scoop','scoops','국자','Scoops']:\n",
    "        json['serving'][1] = 'Scoop'\n",
    "\n",
    "    elif json ['serving'][1] in ['tbsp','테이블스푼','큰','Tablespoons','Tbsp']:\n",
    "        json['serving'][1] = 'tbsp'\n",
    "\n",
    "    elif json ['serving'][1] in ['tsp','티스푼','teaspoons','Teaspoon','teaspoon','찻','찻숫가락']:\n",
    "        json['serving'][1] = 'tsp'\n",
    "\n",
    "    elif json ['serving'][1] in ['Drop','방울','드롭퍼','Drops','Dropper','drop','드롭']:\n",
    "        json['serving'][1] = 'Drop'\n",
    "\n",
    "    elif json ['serving'][1] in ['Packet','봉지', '팩','packet']:\n",
    "        json['serving'][1] = 'Packet'\n",
    "\n",
    "    elif json ['serving'][1] in ['g','Grams','그램']:\n",
    "        json['serving'][1] = 'g'\n",
    "\n",
    "    elif json ['serving'][1] in ['ml','mL','액상','Milliliter']:\n",
    "        json['serving'][1] = 'ml'\n",
    "\n",
    "    elif json ['serving'][1] in ['Gummies','젤리','Jellies','gummy']:\n",
    "        json['serving'][1] = 'Gummies'\n",
    "        \n",
    "    elif json ['serving'][1] in ['Veg Capsule','식물성','채식','VegCap','Vegetarian','VegCaps','Veg','Vegetable','Veggie',\n",
    "                                    'vegeterian','VeganCaps','Veg Capsule','VCAP','vegetable']:\n",
    "        json['serving'][1] = 'Veg Capsule'\n",
    "\n",
    "    elif json ['serving'][1] in ['Stick','stick']:\n",
    "        json['serving'][1] = 'Stick'\n",
    "\n",
    "    elif json ['serving'][1] in ['Spray','Sprays']:\n",
    "        json['serving'][1] = 'Spray'\n",
    "\n",
    "    elif json['serving'][1] in ['Tube', 'mg']:\n",
    "        pass\n",
    "\n",
    "    else:\n",
    "        del json['serving']\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.3 ) filter_serving_amount\n",
    "- Filter data that has zero serving amount <- maybe scrap error.."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def filter_serving_amount(json):\n",
    "    if json['serving'][0] == '0':\n",
    "        del json['serving']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3 ) apply_change\n",
    "- apply change to data with functions above and save to new directory\n",
    "    - change_nutri_nm\n",
    "    - change_serving_unit_nm\n",
    "    - filter_serving_amount functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "def apply_change_nutri_nm(prod_path, prod_list, prod_new_path):\n",
    "    \n",
    "    for prod in prod_list:\n",
    "        with open(prod_path + prod, 'r') as file:\n",
    "            prod_data = json.load(file)\n",
    "        if (prod_data['nutrients'] != 'nutrients_dict error') & (prod_data['serving'] != \"serving unit error\"):\n",
    "            change_nutri_nm(prod_data)\n",
    "            change_serving_unit_nm(prod_data)\n",
    "            if 'serving' in list(prod_data.keys()):\n",
    "                filter_serving_amount(prod_data)\n",
    "\n",
    "            try:\n",
    "                if prod_data['nutrients']:\n",
    "                    if 'serving' in list(prod_data.keys()):\n",
    "                        with open(prod_new_path+'iherb_{}.json'.format(prod_data['prod_cd']), 'w',\n",
    "                                encoding = 'UTF-8-sig') as new_file:\n",
    "                            new_file.write(json.dumps(prod_data, ensure_ascii=False))\n",
    "            except:\n",
    "                print(prod_data['prod_cd'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "vit_new_path = '../output_processed/vitamins/'\n",
    "apply_change_nutri_nm(vit_path, vit_list, vit_new_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "min_new_path = '../output_processed/minerals/'\n",
    "apply_change_nutri_nm(min_path, min_list, min_new_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "amino_new_path = '../output_processed/amino-acids/'\n",
    "apply_change_nutri_nm(amino_path, amino_list, amino_new_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Difference in the number of data before and after preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1680 789 435\n",
      "2904\n",
      "1526 678 197\n",
      "2401\n",
      "503\n"
     ]
    }
   ],
   "source": [
    "vit_processed_list = os.listdir(vit_new_path)\n",
    "min_processed_list = os.listdir(min_new_path)\n",
    "amino_processed_list = os.listdir(amino_new_path)\n",
    "\n",
    "# Before\n",
    "print(len(vit_list), len(min_list), len(amino_list))\n",
    "print(len(vit_list)+len(min_list)+len(amino_list))\n",
    "\n",
    "# After\n",
    "print(len(vit_processed_list), len(min_processed_list), len(amino_processed_list))\n",
    "print(len(vit_processed_list)+len(min_processed_list)+len(amino_processed_list))\n",
    "\n",
    "# Difference\n",
    "print(len(vit_list)+len(min_list)+len(amino_list)-len(vit_processed_list)-len(min_processed_list)-len(amino_processed_list))"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "aee8b7b246df8f9039afb4144a1f6fd8d2ca17a180786b69acc140d282b71a49"
  },
  "kernelspec": {
   "display_name": "Python 3.9.5 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.5"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}