Generate Docs.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Generate Docs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports and definitions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     /Users/gabriel.dutradias/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n",
      "[nltk_data] Downloading package vader_lexicon to\n",
      "[nltk_data]     /Users/gabriel.dutradias/nltk_data...\n",
      "[nltk_data]   Package vader_lexicon is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "import statistics\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from utils import create_folder, cleanup_folder, human_to_kebab_case\n",
    "from comment_classifier.utils import preprocess_comment\n",
    "from comment_classifier.sentence_scorer import SentenceScorer\n",
    "from doc_page import TagDocPage, PostQuestionDocPage\n",
    "import joblib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "output_dir = 'output'\n",
    "n_questions_per_tag = 15\n",
    "redundant_tags = ['android']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create DataFrames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "comments_df = pd.read_csv('data/comments.csv')\n",
    "posts_questions_df = pd.read_csv('data/posts_questions.csv').drop_duplicates(subset=['id'])\n",
    "posts_answers_df = pd.read_csv('data/posts_answers.csv')\n",
    "posts_tag_wiki_df = pd.read_csv('data/posts_tag_wiki.csv')\n",
    "posts_tag_wiki_excerpt_df = pd.read_csv('data/posts_tag_wiki_excerpt.csv')\n",
    "selected_tags_df = pd.read_csv('data/selected_tags.csv')\n",
    "tags_df = pd.read_csv('data/tags.csv')\n",
    "users_df = pd.read_csv('data/users.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Development"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluation functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pre-compute important values for evaluation\n",
    "\n",
    "# add user reputation to post answers\n",
    "posts_answers_with_user_df = pd.merge(left=posts_answers_df, right=users_df[['reputation', 'id']].add_prefix('user_'), left_on='owner_user_id', right_on='user_id')\n",
    "\n",
    "# scaler\n",
    "scaler = MinMaxScaler()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def add_scaled_fields_to_questions(questions_df):\n",
    "    questions_df[['scaled_view_count', 'scaled_score']] = scaler.fit_transform(questions_df[['view_count', 'score']])\n",
    "\n",
    "def add_scaled_fields_to_answers(answers_df):\n",
    "    answers_df[['scaled_score', 'scaled_user_reputation']] = scaler.fit_transform(answers_df[['score', 'user_reputation']])\n",
    "\n",
    "def eval_answer(answer_row):\n",
    "    return statistics.mean([answer_row.scaled_score, answer_row.scaled_user_reputation])\n",
    "\n",
    "def get_eval_question(answers_df):\n",
    "    def eval_question(question_row):\n",
    "        answer_rows = answers_df.loc[answers_df.parent_id == question_row.id].sort_values(by='eval', ascending=False)\n",
    "        if len(answer_rows) == 0:\n",
    "            return 0\n",
    "        answer_row = answer_rows.iloc[0]\n",
    "        return statistics.mean([question_row.scaled_view_count, question_row.scaled_score, answer_row.scaled_score, answer_row.scaled_user_reputation])\n",
    "    return eval_question"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prepare files and variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "cleanup_folder(output_dir)\n",
    "questions_for_docs = []"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Generate docs for tags\n",
    "\n",
    "Docs for tags are based on the `posts_tag_wiki` table from Stackoverflow. Content should be a long description for what each tag represents as well as pointing out relevant complementary docs."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "for idx, row in selected_tags_df.iterrows():\n",
    "    tag_id = row.id\n",
    "    tag_name = row.tag_name\n",
    "    tag_description = posts_tag_wiki_excerpt_df.loc[posts_tag_wiki_excerpt_df.id == row.excerpt_post_id]['body'].values[0]\n",
    "    tag_wiki_body = posts_tag_wiki_df.loc[posts_tag_wiki_df.id == row.wiki_post_id]['body'].values[0]\n",
    "    # save tag doc page\n",
    "    TagDocPage(tag_name, tag_description, tag_wiki_body).save(f\"{output_dir}/{tag_name}.md\")\n",
    "\n",
    "    # prepare post questions for tag\n",
    "    create_folder(f\"{output_dir}/{tag_name}\")\n",
    "    tag_questions_df = posts_questions_df[posts_questions_df.tag_id == tag_id].copy()\n",
    "    tag_answers_df = posts_answers_with_user_df[posts_answers_with_user_df.parent_id.isin(tag_questions_df.id)].copy()\n",
    "    add_scaled_fields_to_answers(tag_answers_df)\n",
    "    add_scaled_fields_to_questions(tag_questions_df)\n",
    "    tag_answers_df['eval'] = tag_answers_df.apply(eval_answer, axis=1)\n",
    "    tag_questions_df['eval'] = tag_questions_df.apply(get_eval_question(tag_answers_df), axis=1)\n",
    "    selected_questions_df = tag_questions_df.sort_values(by='eval', ascending=False).head(n_questions_per_tag)\n",
    "    for _, question_row in selected_questions_df.iterrows():\n",
    "        question_answers_df = tag_answers_df[tag_answers_df.parent_id == question_row.id].copy()\n",
    "        selected_answer = question_answers_df.sort_values(by='eval', ascending=False).iloc[0]\n",
    "        questions_for_docs.append((question_row, selected_answer))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Generate docs for questions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving 'output/android-actionbar/how-do-i-hide-a-menu-item-in-the-actionbar.md'\n",
      "save done\n",
      "Saving 'output/android-actionbar/how-to-disable-action-bar-permanently.md'\n",
      "save done\n",
      "Saving 'output/android-actionbar/actionbar-text-color.md'\n",
      "save done\n",
      "Saving 'output/android-actionbar/full-screen-theme-for-appcompat.md'\n",
      "save done\n",
      "Saving 'output/android-actionbar/how-to-hide-action-bar-before-activity-is-created,-and-then-show-it-again.md'\n",
      "save done\n",
      "Saving 'output/android-actionbar/remove-application-icon-and-title-from-honeycomb-action-bar.md'\n",
      "save done\n",
      "Saving 'output/android-actionbar/how-to-use-searchview-in-toolbar-android.md'\n",
      "save done\n",
      "Saving 'output/android-actionbar/how-to-set-custom-actionbar-color--style.md'\n",
      "save done\n",
      "Saving 'output/android-actionbar/how-do-you-remove-the-title-text-from-the-android-actionbar.md'\n",
      "save done\n",
      "Saving 'output/android-actionbar/android:-how-to-hide-actionbar-on-certain-activities.md'\n",
      "save done\n",
      "Saving 'output/android-actionbar/how-to-customize-the-back-button-on-actionbar.md'\n",
      "save done\n",
      "Saving 'output/android-actionbar/how-can-i-force-the-action-bar-to-be-at-the-bottom-in-ics.md'\n",
      "save done\n",
      "Saving 'output/android-actionbar/how-to-get-the-actionbar-height.md'\n",
      "save done\n",
      "Saving 'output/android-actionbar/getting-drawerlayout-to-slide-over-the-actionbar.md'\n",
      "save done\n",
      "Saving 'output/android-actionbar/change-toolbar-color-in-appcompat-21.md'\n",
      "save done\n",
      "Saving 'output/android-emulator/how-can-you-get-the-buildversion-number-of-your-android-application.md'\n",
      "save done\n",
      "Saving 'output/android-emulator/how-to-start-a-service-when-.apk-is-installed-for-the-first-time.md'\n",
      "save done\n",
      "Saving 'output/android-emulator/how-to-emulate-gps-location-in-the-android-emulator.md'\n",
      "save done\n",
      "Saving 'output/android-emulator/callback-in-android.md'\n",
      "save done\n",
      "Saving 'output/android-emulator/how-do-i-open-the-android-emulator's-terminal-window.md'\n",
      "save done\n",
      "Saving 'output/android-emulator/android-app-google-maps-showing-grey-tiles-and-not-map!.md'\n",
      "save done\n",
      "Saving 'output/android-emulator/how-to-get-to-the-file-explorer-in-android-studio.md'\n",
      "save done\n",
      "Saving 'output/android-emulator/the-android-emulator-is-out-of-view,-how-can-i-move-it.md'\n",
      "save done\n",
      "Saving 'output/android-emulator/android---showing-phonebook-contacts-and-selecting-one.md'\n",
      "save done\n",
      "Saving 'output/android-emulator/how-to-download-google-play-services-in-an-android-emulator.md'\n",
      "save done\n",
      "Saving 'output/android-emulator/in-avd-emulator-how-to-see-sdcard-folder-and-install-apk-to-avd.md'\n",
      "save done\n",
      "Saving 'output/android-emulator/how-to-emulate-pressing-home-button-in-android-emulator.md'\n",
      "save done\n",
      "Saving 'output/android-emulator/data-directory-has-no-readwrite-permission-in-android.md'\n",
      "save done\n",
      "Saving 'output/android-emulator/seek-bar,-change-path-color-from-yellow-to-white.md'\n",
      "save done\n",
      "Saving 'output/android-emulator/no-internet-on-android-emulator---why-and-how-to-fix.md'\n",
      "save done\n",
      "Saving 'output/android-edittext/get-value-of-a-edit-text-field.md'\n",
      "save done\n",
      "Saving 'output/android-edittext/how-to-hide-underbar-in-edittext.md'\n",
      "save done\n",
      "Saving 'output/android-edittext/how-to-use-the-textwatcher-class-in-android.md'\n",
      "save done\n",
      "Saving 'output/android-edittext/edittext,-inputtype-values-(xml).md'\n",
      "save done\n",
      "Saving 'output/android-edittext/how-to-change-the-floating-label-color-of-textinputlayout.md'\n",
      "save done\n",
      "Saving 'output/android-edittext/android:-show-soft-keyboard-automatically-when-focus-is-on-an-edittext.md'\n",
      "save done\n",
      "Saving 'output/android-edittext/change-edittext-hint-color-when-using-textinputlayout.md'\n",
      "save done\n",
      "Saving 'output/android-edittext/android:-how-can-i-validate-edittext-input.md'\n",
      "save done\n",
      "Saving 'output/android-edittext/disable-soft-keyboard-on-numberpicker.md'\n",
      "save done\n",
      "Saving 'output/android-edittext/how-do-i-show-the-number-keyboard-on-an-edittext-in-android.md'\n",
      "save done\n",
      "Saving 'output/android-edittext/disable-edittext-blinking-cursor.md'\n",
      "save done\n",
      "Saving 'output/android-edittext/how-to-make-edittext-not-editable-through-xml-in-android.md'\n",
      "save done\n",
      "Saving 'output/android-edittext/how-to-make-a-edittext-box-in-a-dialog.md'\n",
      "save done\n",
      "Saving 'output/android-edittext/how-do-i-handle-imeoptions'-done-button-click.md'\n",
      "save done\n",
      "Saving 'output/android-edittext/move-to-another-edittext-when-soft-keyboard-next-is-clicked-on-android.md'\n",
      "save done\n",
      "Saving 'output/android-intent/how-can-i-open-a-url-in-android's-web-browser-from-my-application.md'\n",
      "save done\n",
      "Saving 'output/android-intent/how-to-use-putextra()-and-getextra()-for-string-data.md'\n",
      "save done\n",
      "Saving 'output/android-intent/sending-an-intent-to-browser-to-open-specific-url.md'\n",
      "save done\n",
      "Saving 'output/android-intent/how-to-open-the-google-play-store-directly-from-my-android-application.md'\n",
      "save done\n",
      "Saving 'output/android-intent/how-to-tell-which-app-was-selected-by-intent.createchooser.md'\n",
      "save done\n",
      "Saving 'output/android-intent/android:-how-to-get-the-intent-received-by-a-service.md'\n",
      "save done\n",
      "Saving 'output/android-intent/send-email-intent.md'\n",
      "save done\n",
      "Saving 'output/android-intent/how-to-set-default-app-launcher-programmatically.md'\n",
      "save done\n",
      "Saving 'output/android-intent/unable-to-start-uiautomatorviewer.md'\n",
      "save done\n",
      "Saving 'output/android-intent/open-another-application-from-your-own-(intent).md'\n",
      "save done\n",
      "Saving 'output/android-intent/launch-custom-android-application-from-android-browser.md'\n",
      "save done\n",
      "Saving 'output/android-intent/how-to-get-a-list-of-installed-android-applications-and-pick-one-to-run.md'\n",
      "save done\n",
      "Saving 'output/android-intent/launching-google-maps-directions-via-an-intent-on-android.md'\n",
      "save done\n",
      "Saving 'output/android-intent/difference-between-intent.flag_activity_clear_task-and-intent.flag_activity_task_on_home.md'\n",
      "save done\n",
      "Saving 'output/android-intent/android-notification-intent-to-clear-it-self.md'\n",
      "save done\n",
      "Saving 'output/android-studio/android-studio-doesn't-see-device.md'\n",
      "save done\n",
      "Saving 'output/android-studio/you-have-not-accepted-the-license-agreements-of-the-following-sdk-components.md'\n",
      "save done\n",
      "Saving 'output/android-studio/how-to-add-'libs'-folder-in-android-studio.md'\n",
      "save done\n",
      "Saving 'output/android-studio/how-to-add-a-jar-in-external-libraries-in-android-studio.md'\n",
      "save done\n",
      "Saving 'output/android-studio/how-to-print-to-the-console-in-android-studio.md'\n",
      "save done\n",
      "Saving 'output/android-studio/android-webview-gives-net::err_cache_miss-message.md'\n",
      "save done\n",
      "Saving 'output/android-studio/filter-logcat-to-get-only-the-messages-from-my-application-in-android.md'\n",
      "save done\n",
      "Saving 'output/android-studio/building-and-running-app-via-gradle-and-android-studio-is-slower-than-via-eclipse.md'\n",
      "save done\n",
      "Saving 'output/android-studio/create-aar-file-in-android-studio.md'\n",
      "save done\n",
      "Saving 'output/android-studio/connecting-to-gitlab-repositories-on-android-studio.md'\n",
      "save done\n",
      "Saving 'output/android-studio/file-explorer-in-android-studio.md'\n",
      "save done\n",
      "Saving 'output/android-studio/android-studio-no-module.md'\n",
      "save done\n",
      "Saving 'output/android-studio/how-to-include-*.so-library-in-android-studio.md'\n",
      "save done\n",
      "Saving 'output/android-studio/where-is-android-studio-layout-preview.md'\n",
      "save done\n",
      "Saving 'output/android-studio/android-studio:-how-to-attach-android-sdk-sources.md'\n",
      "save done\n",
      "Saving 'output/android-activity/how-do-i-pass-data-between-activities-in-android-application.md'\n",
      "save done\n",
      "Saving 'output/android-activity/how-can-i-save-an-activity-state-using-the-save-instance-state.md'\n",
      "save done\n",
      "Saving 'output/android-activity/how-to-start-new-activity-on-button-click.md'\n",
      "save done\n",
      "Saving 'output/android-activity/onactivityresult-is-not-being-called-in-fragment.md'\n",
      "save done\n",
      "Saving 'output/android-activity/startactivity()-from-broadcastreceiver.md'\n",
      "save done\n",
      "Saving 'output/android-activity/activity-restart-on-rotation-android.md'\n",
      "save done\n",
      "Saving 'output/android-activity/get-root-view-from-current-activity.md'\n",
      "save done\n",
      "Saving 'output/android-activity/change-application's-starting-activity.md'\n",
      "save done\n",
      "Saving 'output/android-activity/how-to-change-title-of-activity-in-android.md'\n",
      "save done\n",
      "Saving 'output/android-activity/dilemma:-when-to-use-fragments-vs-activities:.md'\n",
      "save done\n",
      "Saving 'output/android-activity/example:-communication-between-activity-and-service-using-messaging.md'\n",
      "save done\n",
      "Saving 'output/android-activity/how-to-handle-screen-orientation-change-when-progress-dialog-and-background-thread-active.md'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "save done\n",
      "Saving 'output/android-activity/how-to-show-a-dialog-to-confirm-that-the-user-wishes-to-exit-an-android-activity.md'\n",
      "save done\n",
      "Saving 'output/android-activity/how-to-get-current-foreground-activity-context-in-android.md'\n",
      "save done\n",
      "Saving 'output/android-activity/start-an-activity-with-a-parameter.md'\n",
      "save done\n",
      "Saving 'output/android-gradle-plugin/how-can-i-access-a-buildconfig-value-in-my-androidmanifest.xml-file.md'\n",
      "save done\n",
      "Saving 'output/android-gradle-plugin/multi-flavor-app-based-on-multi-flavor-library-in-android-gradle.md'\n",
      "save done\n",
      "Saving 'output/android-gradle-plugin/publish-an-android-library-to-maven-with-aar-and-source-jar.md'\n",
      "save done\n",
      "Saving 'output/android-gradle-plugin/how-to-use-local-aar-dependency.md'\n",
      "save done\n",
      "Saving 'output/android-gradle-plugin/is-it-possible-to-use-multiple-manifest-files-for-different-gradle-buildsflavors.md'\n",
      "save done\n",
      "Saving 'output/android-gradle-plugin/build-flavors-for-different-version-of-same-class.md'\n",
      "save done\n",
      "Saving 'output/android-gradle-plugin/android-studio:-switching-a-url-depending-on-buildtype-(used-for-testing-in-debug-release).md'\n",
      "save done\n",
      "Saving 'output/android-gradle-plugin/how-to-include-an-android-library-module-in-another-library.md'\n",
      "save done\n",
      "Saving 'output/android-gradle-plugin/where-is-a-\"sync-project-with-gradle-files\"-button-in-android-studio-3.md'\n",
      "save done\n",
      "Saving 'output/android-gradle-plugin/gradle-dependencies-update.md'\n",
      "save done\n",
      "Saving 'output/android-gradle-plugin/is-there-any-way-to-integrate-eclipse-with-gradle-in-android-project.md'\n",
      "save done\n",
      "Saving 'output/android-gradle-plugin/how-can-i-use-android-dexoptions.md'\n",
      "save done\n",
      "Saving 'output/android-gradle-plugin/android-studio-gradle--bin.zip-vs.-gradle--all.zip.md'\n",
      "save done\n",
      "Saving 'output/android-gradle-plugin/how-do-i-set-an-environment-variable-in-android-studio-before-it-runs-my-gradle-build.md'\n",
      "save done\n",
      "Saving 'output/android-gradle-plugin/how-to-manage-different-support-library-versions-for-3rd-party-deps-with-gradle.md'\n",
      "save done\n",
      "Saving 'output/android-recyclerview/how-to-add-dividers-and-spaces-between-items-in-recyclerview.md'\n",
      "save done\n",
      "Saving 'output/android-recyclerview/how-to-updaterefresh-specific-item-in-recyclerview.md'\n",
      "save done\n",
      "Saving 'output/android-recyclerview/android-recyclerview-addition-&-removal-of-items.md'\n",
      "save done\n",
      "Saving 'output/android-recyclerview/how-to-implement-itemanimator-of-recyclerview-to-disable-the-animation-of-notifyitemchanged.md'\n",
      "save done\n",
      "Saving 'output/android-recyclerview/recyclerview-change-data-set.md'\n",
      "save done\n",
      "Saving 'output/android-recyclerview/how-to-prevent-recyclerview-item-from-blinking-after-notifyitemchanged(pos).md'\n",
      "save done\n",
      "Saving 'output/android-recyclerview/hideshow-toolbar-programmatically-on-coordinatorlayout.md'\n",
      "save done\n",
      "Saving 'output/android-recyclerview/detect-when-recyclerview-reaches-the-bottom-most-position-while-scrolling.md'\n",
      "save done\n",
      "Saving 'output/android-recyclerview/nestedscrollview-won't-start-from-top.md'\n",
      "save done\n",
      "Saving 'output/android-recyclerview/how-to-show-an-empty-view-with-a-recyclerview.md'\n",
      "save done\n",
      "Saving 'output/android-recyclerview/how-to-filter-a-recyclerview-with-a-searchview.md'\n",
      "save done\n",
      "Saving 'output/android-recyclerview/get-clicked-item-and-its-position-in-recyclerview.md'\n",
      "save done\n",
      "Saving 'output/android-recyclerview/need-an-example-about-recyclerview.adapter.notifyitemchanged(int-position,-object-payload).md'\n",
      "save done\n",
      "Saving 'output/android-recyclerview/simple-android-grid-example-using-recyclerview-with-gridlayoutmanager-(like-the-old-gridview).md'\n",
      "save done\n",
      "Saving 'output/android-recyclerview/what's-better-notifydatasetchanged-or-notifyitemchanged-in-loop.md'\n",
      "save done\n",
      "Saving 'output/android-fragments/illegalstateexception:-can-not-perform-this-action-after-onsaveinstancestate-with-viewpager.md'\n",
      "save done\n",
      "Saving 'output/android-fragments/how-to-determine-when-fragment-becomes-visible-in-viewpager.md'\n",
      "save done\n",
      "Saving 'output/android-fragments/getting-the-current-fragment-instance-in-the-viewpager.md'\n",
      "save done\n",
      "Saving 'output/android-fragments/basic-communication-between-two-fragments.md'\n",
      "save done\n",
      "Saving 'output/android-fragments/how-to-get-toolbar-from-fragment.md'\n",
      "save done\n",
      "Saving 'output/android-fragments/why-do-i-want-to-avoid-non-default-constructors-in-fragments.md'\n",
      "save done\n",
      "Saving 'output/android-fragments/oncreateoptionsmenu()-not-called-in-fragment.md'\n",
      "save done\n",
      "Saving 'output/android-fragments/update-data-in-listfragment-as-part-of-viewpager.md'\n",
      "save done\n",
      "Saving 'output/android-fragments/how-to-use-setarguments()-and-getarguments()-methods-in-fragments.md'\n",
      "save done\n",
      "Saving 'output/android-fragments/how-to-put-google-maps-v2-on-a-fragment-using-viewpager.md'\n",
      "save done\n",
      "Saving 'output/android-fragments/android:-how-can-fragment-take-a-global-variable-of-activity.md'\n",
      "save done\n",
      "Saving 'output/android-fragments/android-fragments-and-animation.md'\n",
      "save done\n",
      "Saving 'output/android-fragments/how-to-pass-values-between-fragments.md'\n",
      "save done\n",
      "Saving 'output/android-fragments/remove-old-fragment-from-fragment-manager.md'\n",
      "save done\n",
      "Saving 'output/android-fragments/calling-a-fragment-method-from-a-parent-activity.md'\n",
      "save done\n",
      "Saving 'output/android-asynctask/download-a-file-with-android,-and-showing-the-progress-in-a-progressdialog.md'\n",
      "save done\n",
      "Saving 'output/android-asynctask/using-the-\"animated-circle\"-in-an-imageview-while-loading-stuff.md'\n",
      "save done\n",
      "Saving 'output/android-asynctask/handler-vs-asynctask-vs-thread.md'\n",
      "save done\n",
      "Saving 'output/android-asynctask/android-two-asynctasks-serially-or-parallel-execution---the-second-is-freezing-but-the-result-is-ok.md'\n",
      "save done\n",
      "Saving 'output/android-asynctask/android-check-null-or-empty-string-in-android.md'\n",
      "save done\n",
      "Saving 'output/android-asynctask/android:-how-can-i-pass-parameters-to-asynctask's-onpreexecute().md'\n",
      "save done\n",
      "Saving 'output/android-asynctask/asynctask-won't-stop-even-when-the-activity-has-destroyed.md'\n",
      "save done\n",
      "Saving 'output/android-asynctask/return-a-value-from-asynctask-in-android.md'\n",
      "save done\n",
      "Saving 'output/android-asynctask/android---cancel-asynctask-forcefully.md'\n",
      "save done\n",
      "Saving 'output/android-asynctask/ideal-way-to-cancel-an-executing-asynctask.md'\n",
      "save done\n",
      "Saving 'output/android-asynctask/asynctaskloader-vs-asynctask.md'\n",
      "save done\n",
      "Saving 'output/android-asynctask/getting-context-in-asynctask.md'\n",
      "save done\n",
      "Saving 'output/android-asynctask/how-do-i-return-a-boolean-from-asynctask.md'\n",
      "save done\n",
      "Saving 'output/android-asynctask/asynctask:-pass-two-or-more-values-from-doinbackground-to-onpostexecute.md'\n",
      "save done\n",
      "Saving 'output/android-asynctask/asynctaskloader-basic-example.-(android).md'\n",
      "save done\n",
      "Saving 'output/listview/how-to-implement-android-pull-to-refresh.md'\n",
      "save done\n",
      "Saving 'output/listview/custom-adapter-for-list-view.md'\n",
      "save done\n",
      "Saving 'output/listview/listview-item-background-via-custom-selector.md'\n",
      "save done\n",
      "Saving 'output/listview/android-listview-with-different-layouts-for-each-row.md'\n",
      "save done\n",
      "Saving 'output/listview/android-listview-setselection()-does-not-seem-to-work.md'\n",
      "save done\n",
      "Saving 'output/listview/how-can-i-get-android-wifi-scan-results-into-a-list.md'\n",
      "save done\n",
      "Saving 'output/listview/android-listview-scrolling-to-top.md'\n",
      "save done\n",
      "Saving 'output/listview/android-listview-headers.md'\n",
      "save done\n",
      "Saving 'output/listview/showing-empty-view-when-listview-is-empty.md'\n",
      "save done\n",
      "Saving 'output/listview/android:-listview-elements-with-multiple-clickable-buttons.md'\n",
      "save done\n",
      "Saving 'output/listview/update-listview-dynamically-with-adapter.md'\n",
      "save done\n",
      "Saving 'output/listview/changing-background-color-of-listview-items-on-android.md'\n",
      "save done\n",
      "Saving 'output/listview/notifydatasetchange-not-working-from-custom-adapter.md'\n",
      "save done\n",
      "Saving 'output/listview/xamarin.forms-listview:-set-the-highlight-color-of-a-tapped-item.md'\n",
      "save done\n",
      "Saving 'output/listview/listview-setonitemclicklistener-not-working-by-adding-button.md'\n",
      "save done\n",
      "Saving 'output/android-layout/how-to-change-fontfamily-of-textview-in-android.md'\n",
      "save done\n",
      "Saving 'output/android-layout/how-do-i-get-the-currently-displayed-fragment.md'\n",
      "save done\n",
      "Saving 'output/android-layout/what-does-android:layout_weight-mean.md'\n",
      "save done\n",
      "Saving 'output/android-layout/how-to-change-color-of-textview-hyperlink.md'\n",
      "save done\n",
      "Saving 'output/android-layout/how-to-build-a-horizontal-listview-with-recyclerview.md'\n",
      "save done\n",
      "Saving 'output/android-layout/view's-getwidth()-and-getheight()-returns-0.md'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "save done\n",
      "Saving 'output/android-layout/how-to-center-items-of-a-recyclerview.md'\n",
      "save done\n",
      "Saving 'output/android-layout/how-to-change-the-color-of-a-checkbox.md'\n",
      "save done\n",
      "Saving 'output/android-layout/changing-edittext-bottom-line-color-with-appcompat-v7.md'\n",
      "save done\n",
      "Saving 'output/android-layout/put-buttons-at-bottom-of-screen-with-linearlayout.md'\n",
      "save done\n",
      "Saving 'output/android-layout/border-in-shape-xml.md'\n",
      "save done\n",
      "Saving 'output/android-layout/how-to-animate-recyclerview-items-when-they-appear.md'\n",
      "save done\n",
      "Saving 'output/android-layout/placingoverlapping(z-index)-a-view-above-another-view-in-android.md'\n",
      "save done\n",
      "Saving 'output/android-layout/android:-how-to-draw-a-border-to-a-linearlayout.md'\n",
      "save done\n",
      "Saving 'output/android-layout/how-to-customize-a-spinner-in-android.md'\n",
      "save done\n"
     ]
    }
   ],
   "source": [
    "for selected_question, selected_answer in questions_for_docs:\n",
    "    parent_tag_name = selected_question.tag_name\n",
    "    question_title = selected_question.title\n",
    "    question_body = selected_question.body\n",
    "    question_tags = [tag for tag in selected_question.tags.split('|') if tag not in redundant_tags and tag != selected_question.tag_name]\n",
    "    \n",
    "    # select answer\n",
    "    comments = list(comments_df[comments_df.post_id == selected_answer.id].sort_values(by='score', ascending=False)['text'])\n",
    "    answer_body = selected_answer.body\n",
    "\n",
    "    # save post question doc page\n",
    "    output_file_name = human_to_kebab_case(question_title)\n",
    "    print(f\"Saving '{output_dir}/{parent_tag_name}/{output_file_name}.md'\")\n",
    "    PostQuestionDocPage(question_title, question_body, question_tags, answer_body, comments).save(f\"{output_dir}/{parent_tag_name}/{output_file_name}.md\")\n",
    "    print('save done')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "output/android-actionbar/how-do-i-hide-a-menu-item-in-the-actionbar.md\n",
      "output/android-actionbar/how-to-disable-action-bar-permanently.md\n",
      "output/android-actionbar/actionbar-text-color.md\n",
      "output/android-actionbar/full-screen-theme-for-appcompat.md\n",
      "output/android-actionbar/how-to-hide-action-bar-before-activity-is-created,-and-then-show-it-again.md\n",
      "output/android-actionbar/remove-application-icon-and-title-from-honeycomb-action-bar.md\n",
      "output/android-actionbar/how-to-use-searchview-in-toolbar-android.md\n",
      "output/android-actionbar/how-to-set-custom-actionbar-color--style.md\n",
      "output/android-actionbar/how-do-you-remove-the-title-text-from-the-android-actionbar.md\n",
      "output/android-actionbar/android:-how-to-hide-actionbar-on-certain-activities.md\n",
      "output/android-actionbar/how-to-customize-the-back-button-on-actionbar.md\n",
      "output/android-actionbar/how-can-i-force-the-action-bar-to-be-at-the-bottom-in-ics.md\n",
      "output/android-actionbar/how-to-get-the-actionbar-height.md\n",
      "output/android-actionbar/getting-drawerlayout-to-slide-over-the-actionbar.md\n",
      "output/android-actionbar/change-toolbar-color-in-appcompat-21.md\n",
      "output/android-emulator/how-can-you-get-the-buildversion-number-of-your-android-application.md\n",
      "output/android-emulator/how-to-start-a-service-when-.apk-is-installed-for-the-first-time.md\n",
      "output/android-emulator/how-to-emulate-gps-location-in-the-android-emulator.md\n",
      "output/android-emulator/callback-in-android.md\n",
      "output/android-emulator/how-do-i-open-the-android-emulator's-terminal-window.md\n",
      "output/android-emulator/android-app-google-maps-showing-grey-tiles-and-not-map!.md\n",
      "output/android-emulator/how-to-get-to-the-file-explorer-in-android-studio.md\n",
      "output/android-emulator/the-android-emulator-is-out-of-view,-how-can-i-move-it.md\n",
      "output/android-emulator/android---showing-phonebook-contacts-and-selecting-one.md\n",
      "output/android-emulator/how-to-download-google-play-services-in-an-android-emulator.md\n",
      "output/android-emulator/in-avd-emulator-how-to-see-sdcard-folder-and-install-apk-to-avd.md\n",
      "output/android-emulator/how-to-emulate-pressing-home-button-in-android-emulator.md\n",
      "output/android-emulator/data-directory-has-no-readwrite-permission-in-android.md\n",
      "output/android-emulator/seek-bar,-change-path-color-from-yellow-to-white.md\n",
      "output/android-emulator/no-internet-on-android-emulator---why-and-how-to-fix.md\n",
      "output/android-edittext/get-value-of-a-edit-text-field.md\n",
      "output/android-edittext/how-to-hide-underbar-in-edittext.md\n",
      "output/android-edittext/how-to-use-the-textwatcher-class-in-android.md\n",
      "output/android-edittext/edittext,-inputtype-values-(xml).md\n",
      "output/android-edittext/how-to-change-the-floating-label-color-of-textinputlayout.md\n",
      "output/android-edittext/android:-show-soft-keyboard-automatically-when-focus-is-on-an-edittext.md\n",
      "output/android-edittext/change-edittext-hint-color-when-using-textinputlayout.md\n",
      "output/android-edittext/android:-how-can-i-validate-edittext-input.md\n",
      "output/android-edittext/disable-soft-keyboard-on-numberpicker.md\n",
      "output/android-edittext/how-do-i-show-the-number-keyboard-on-an-edittext-in-android.md\n",
      "output/android-edittext/disable-edittext-blinking-cursor.md\n",
      "output/android-edittext/how-to-make-edittext-not-editable-through-xml-in-android.md\n",
      "output/android-edittext/how-to-make-a-edittext-box-in-a-dialog.md\n",
      "output/android-edittext/how-do-i-handle-imeoptions'-done-button-click.md\n",
      "output/android-edittext/move-to-another-edittext-when-soft-keyboard-next-is-clicked-on-android.md\n",
      "output/android-intent/how-can-i-open-a-url-in-android's-web-browser-from-my-application.md\n",
      "output/android-intent/how-to-use-putextra()-and-getextra()-for-string-data.md\n",
      "output/android-intent/sending-an-intent-to-browser-to-open-specific-url.md\n",
      "output/android-intent/how-to-open-the-google-play-store-directly-from-my-android-application.md\n",
      "output/android-intent/how-to-tell-which-app-was-selected-by-intent.createchooser.md\n",
      "output/android-intent/android:-how-to-get-the-intent-received-by-a-service.md\n",
      "output/android-intent/send-email-intent.md\n",
      "output/android-intent/how-to-set-default-app-launcher-programmatically.md\n",
      "output/android-intent/unable-to-start-uiautomatorviewer.md\n",
      "output/android-intent/open-another-application-from-your-own-(intent).md\n",
      "output/android-intent/launch-custom-android-application-from-android-browser.md\n",
      "output/android-intent/how-to-get-a-list-of-installed-android-applications-and-pick-one-to-run.md\n",
      "output/android-intent/launching-google-maps-directions-via-an-intent-on-android.md\n",
      "output/android-intent/difference-between-intent.flag_activity_clear_task-and-intent.flag_activity_task_on_home.md\n",
      "output/android-intent/android-notification-intent-to-clear-it-self.md\n",
      "output/android-studio/android-studio-doesn't-see-device.md\n",
      "output/android-studio/you-have-not-accepted-the-license-agreements-of-the-following-sdk-components.md\n",
      "output/android-studio/how-to-add-'libs'-folder-in-android-studio.md\n",
      "output/android-studio/how-to-add-a-jar-in-external-libraries-in-android-studio.md\n",
      "output/android-studio/how-to-print-to-the-console-in-android-studio.md\n",
      "output/android-studio/android-webview-gives-net::err_cache_miss-message.md\n",
      "output/android-studio/filter-logcat-to-get-only-the-messages-from-my-application-in-android.md\n",
      "output/android-studio/building-and-running-app-via-gradle-and-android-studio-is-slower-than-via-eclipse.md\n",
      "output/android-studio/create-aar-file-in-android-studio.md\n",
      "output/android-studio/connecting-to-gitlab-repositories-on-android-studio.md\n",
      "output/android-studio/file-explorer-in-android-studio.md\n",
      "output/android-studio/android-studio-no-module.md\n",
      "output/android-studio/how-to-include-*.so-library-in-android-studio.md\n",
      "output/android-studio/where-is-android-studio-layout-preview.md\n",
      "output/android-studio/android-studio:-how-to-attach-android-sdk-sources.md\n",
      "output/android-activity/how-do-i-pass-data-between-activities-in-android-application.md\n",
      "output/android-activity/how-can-i-save-an-activity-state-using-the-save-instance-state.md\n",
      "output/android-activity/how-to-start-new-activity-on-button-click.md\n",
      "output/android-activity/onactivityresult-is-not-being-called-in-fragment.md\n",
      "output/android-activity/startactivity()-from-broadcastreceiver.md\n",
      "output/android-activity/activity-restart-on-rotation-android.md\n",
      "output/android-activity/get-root-view-from-current-activity.md\n",
      "output/android-activity/change-application's-starting-activity.md\n",
      "output/android-activity/how-to-change-title-of-activity-in-android.md\n",
      "output/android-activity/dilemma:-when-to-use-fragments-vs-activities:.md\n",
      "output/android-activity/example:-communication-between-activity-and-service-using-messaging.md\n",
      "output/android-activity/how-to-handle-screen-orientation-change-when-progress-dialog-and-background-thread-active.md\n",
      "output/android-activity/how-to-show-a-dialog-to-confirm-that-the-user-wishes-to-exit-an-android-activity.md\n",
      "output/android-activity/how-to-get-current-foreground-activity-context-in-android.md\n",
      "output/android-activity/start-an-activity-with-a-parameter.md\n",
      "output/android-gradle-plugin/how-can-i-access-a-buildconfig-value-in-my-androidmanifest.xml-file.md\n",
      "output/android-gradle-plugin/multi-flavor-app-based-on-multi-flavor-library-in-android-gradle.md\n",
      "output/android-gradle-plugin/publish-an-android-library-to-maven-with-aar-and-source-jar.md\n",
      "output/android-gradle-plugin/how-to-use-local-aar-dependency.md\n",
      "output/android-gradle-plugin/is-it-possible-to-use-multiple-manifest-files-for-different-gradle-buildsflavors.md\n",
      "output/android-gradle-plugin/build-flavors-for-different-version-of-same-class.md\n",
      "output/android-gradle-plugin/android-studio:-switching-a-url-depending-on-buildtype-(used-for-testing-in-debug-release).md\n",
      "output/android-gradle-plugin/how-to-include-an-android-library-module-in-another-library.md\n",
      "output/android-gradle-plugin/where-is-a-\"sync-project-with-gradle-files\"-button-in-android-studio-3.md\n",
      "output/android-gradle-plugin/gradle-dependencies-update.md\n",
      "output/android-gradle-plugin/is-there-any-way-to-integrate-eclipse-with-gradle-in-android-project.md\n",
      "output/android-gradle-plugin/how-can-i-use-android-dexoptions.md\n",
      "output/android-gradle-plugin/android-studio-gradle--bin.zip-vs.-gradle--all.zip.md\n",
      "output/android-gradle-plugin/how-do-i-set-an-environment-variable-in-android-studio-before-it-runs-my-gradle-build.md\n",
      "output/android-gradle-plugin/how-to-manage-different-support-library-versions-for-3rd-party-deps-with-gradle.md\n",
      "output/android-recyclerview/how-to-add-dividers-and-spaces-between-items-in-recyclerview.md\n",
      "output/android-recyclerview/how-to-updaterefresh-specific-item-in-recyclerview.md\n",
      "output/android-recyclerview/android-recyclerview-addition-&-removal-of-items.md\n",
      "output/android-recyclerview/how-to-implement-itemanimator-of-recyclerview-to-disable-the-animation-of-notifyitemchanged.md\n",
      "output/android-recyclerview/recyclerview-change-data-set.md\n",
      "output/android-recyclerview/how-to-prevent-recyclerview-item-from-blinking-after-notifyitemchanged(pos).md\n",
      "output/android-recyclerview/hideshow-toolbar-programmatically-on-coordinatorlayout.md\n",
      "output/android-recyclerview/detect-when-recyclerview-reaches-the-bottom-most-position-while-scrolling.md\n",
      "output/android-recyclerview/nestedscrollview-won't-start-from-top.md\n",
      "output/android-recyclerview/how-to-show-an-empty-view-with-a-recyclerview.md\n",
      "output/android-recyclerview/how-to-filter-a-recyclerview-with-a-searchview.md\n",
      "output/android-recyclerview/get-clicked-item-and-its-position-in-recyclerview.md\n",
      "output/android-recyclerview/need-an-example-about-recyclerview.adapter.notifyitemchanged(int-position,-object-payload).md\n",
      "output/android-recyclerview/simple-android-grid-example-using-recyclerview-with-gridlayoutmanager-(like-the-old-gridview).md\n",
      "output/android-recyclerview/what's-better-notifydatasetchanged-or-notifyitemchanged-in-loop.md\n",
      "output/android-fragments/illegalstateexception:-can-not-perform-this-action-after-onsaveinstancestate-with-viewpager.md\n",
      "output/android-fragments/how-to-determine-when-fragment-becomes-visible-in-viewpager.md\n",
      "output/android-fragments/getting-the-current-fragment-instance-in-the-viewpager.md\n",
      "output/android-fragments/basic-communication-between-two-fragments.md\n",
      "output/android-fragments/how-to-get-toolbar-from-fragment.md\n",
      "output/android-fragments/why-do-i-want-to-avoid-non-default-constructors-in-fragments.md\n",
      "output/android-fragments/oncreateoptionsmenu()-not-called-in-fragment.md\n",
      "output/android-fragments/update-data-in-listfragment-as-part-of-viewpager.md\n",
      "output/android-fragments/how-to-use-setarguments()-and-getarguments()-methods-in-fragments.md\n",
      "output/android-fragments/how-to-put-google-maps-v2-on-a-fragment-using-viewpager.md\n",
      "output/android-fragments/android:-how-can-fragment-take-a-global-variable-of-activity.md\n",
      "output/android-fragments/android-fragments-and-animation.md\n",
      "output/android-fragments/how-to-pass-values-between-fragments.md\n",
      "output/android-fragments/remove-old-fragment-from-fragment-manager.md\n",
      "output/android-fragments/calling-a-fragment-method-from-a-parent-activity.md\n",
      "output/android-asynctask/download-a-file-with-android,-and-showing-the-progress-in-a-progressdialog.md\n",
      "output/android-asynctask/using-the-\"animated-circle\"-in-an-imageview-while-loading-stuff.md\n",
      "output/android-asynctask/handler-vs-asynctask-vs-thread.md\n",
      "output/android-asynctask/android-two-asynctasks-serially-or-parallel-execution---the-second-is-freezing-but-the-result-is-ok.md\n",
      "output/android-asynctask/android-check-null-or-empty-string-in-android.md\n",
      "output/android-asynctask/android:-how-can-i-pass-parameters-to-asynctask's-onpreexecute().md\n",
      "output/android-asynctask/asynctask-won't-stop-even-when-the-activity-has-destroyed.md\n",
      "output/android-asynctask/return-a-value-from-asynctask-in-android.md\n",
      "output/android-asynctask/android---cancel-asynctask-forcefully.md\n",
      "output/android-asynctask/ideal-way-to-cancel-an-executing-asynctask.md\n",
      "output/android-asynctask/asynctaskloader-vs-asynctask.md\n",
      "output/android-asynctask/getting-context-in-asynctask.md\n",
      "output/android-asynctask/how-do-i-return-a-boolean-from-asynctask.md\n",
      "output/android-asynctask/asynctask:-pass-two-or-more-values-from-doinbackground-to-onpostexecute.md\n",
      "output/android-asynctask/asynctaskloader-basic-example.-(android).md\n",
      "output/listview/how-to-implement-android-pull-to-refresh.md\n",
      "output/listview/custom-adapter-for-list-view.md\n",
      "output/listview/listview-item-background-via-custom-selector.md\n",
      "output/listview/android-listview-with-different-layouts-for-each-row.md\n",
      "output/listview/android-listview-setselection()-does-not-seem-to-work.md\n",
      "output/listview/how-can-i-get-android-wifi-scan-results-into-a-list.md\n",
      "output/listview/android-listview-scrolling-to-top.md\n",
      "output/listview/android-listview-headers.md\n",
      "output/listview/showing-empty-view-when-listview-is-empty.md\n",
      "output/listview/android:-listview-elements-with-multiple-clickable-buttons.md\n",
      "output/listview/update-listview-dynamically-with-adapter.md\n",
      "output/listview/changing-background-color-of-listview-items-on-android.md\n",
      "output/listview/notifydatasetchange-not-working-from-custom-adapter.md\n",
      "output/listview/xamarin.forms-listview:-set-the-highlight-color-of-a-tapped-item.md\n",
      "output/listview/listview-setonitemclicklistener-not-working-by-adding-button.md\n",
      "output/android-layout/how-to-change-fontfamily-of-textview-in-android.md\n",
      "output/android-layout/how-do-i-get-the-currently-displayed-fragment.md\n",
      "output/android-layout/what-does-android:layout_weight-mean.md\n",
      "output/android-layout/how-to-change-color-of-textview-hyperlink.md\n",
      "output/android-layout/how-to-build-a-horizontal-listview-with-recyclerview.md\n",
      "output/android-layout/view's-getwidth()-and-getheight()-returns-0.md\n",
      "output/android-layout/how-to-center-items-of-a-recyclerview.md\n",
      "output/android-layout/how-to-change-the-color-of-a-checkbox.md\n",
      "output/android-layout/changing-edittext-bottom-line-color-with-appcompat-v7.md\n",
      "output/android-layout/put-buttons-at-bottom-of-screen-with-linearlayout.md\n",
      "output/android-layout/border-in-shape-xml.md\n",
      "output/android-layout/how-to-animate-recyclerview-items-when-they-appear.md\n",
      "output/android-layout/placingoverlapping(z-index)-a-view-above-another-view-in-android.md\n",
      "output/android-layout/android:-how-to-draw-a-border-to-a-linearlayout.md\n",
      "output/android-layout/how-to-customize-a-spinner-in-android.md\n"
     ]
    }
   ],
   "source": [
    "for selected_question, selected_answer in questions_for_docs:\n",
    "    parent_tag_name = selected_question.tag_name\n",
    "    question_title = selected_question.title\n",
    "    question_body = selected_question.body\n",
    "    question_tags = [tag for tag in selected_question.tags.split('|') if tag not in redundant_tags and tag != selected_question.tag_name]\n",
    "    \n",
    "    # select answer\n",
    "    comments = list(comments_df[comments_df.post_id == selected_answer.id].sort_values(by='score', ascending=False)['text'])\n",
    "    answer_body = selected_answer.body\n",
    "\n",
    "    # save post question doc page\n",
    "    output_file_name = human_to_kebab_case(question_title)\n",
    "    print(f\"{output_dir}/{parent_tag_name}/{output_file_name}.md\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}