deal_with_subscript.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "import numpy as np\n",
    "import re\n",
    "from collections import Counter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 文本存储路径,改成自己的就好\n",
    "path = 'C:/Users/13126/Desktop/test'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['C:/Users/13126/Desktop/test\\\\1.常速+慢速发音_高效提高英语听力训练.txt', 'C:/Users/13126/Desktop/test\\\\2.超实用英语听力训练119句.txt', 'C:/Users/13126/Desktop/test\\\\3.高效练耳朵英语听力_-_快速提高英文听力水平.txt', 'C:/Users/13126/Desktop/test\\\\4.高效练耳朵英语听力_-_在训练中提升你的英文听力技能.txt', 'C:/Users/13126/Desktop/test\\\\5.高效练耳朵英语听力90句.txt', 'C:/Users/13126/Desktop/test\\\\6.高效练耳朵英语听力100句(饮食篇).txt', 'C:/Users/13126/Desktop/test\\\\7.高效练耳朵英语听力348句_-_喜怒哀乐篇.txt', 'C:/Users/13126/Desktop/test\\\\8.高效练习英语听力49句.txt', 'C:/Users/13126/Desktop/test\\\\9.高效练英语听力65句(交通出行篇).txt']\n"
     ]
    }
   ],
   "source": [
    "# read the file list\n",
    "text_lst = os.listdir(path)\n",
    "text_path_lst = []\n",
    "for text in text_lst:\n",
    "    text_path_lst.append(os.sep.join([path, text]))\n",
    "print(text_path_lst)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "pattern = re.compile(r'\\w[\\'\\d\\w\\s]*\\.')\n",
    "for i, text_path in enumerate(text_path_lst):\n",
    "    with open(text_path) as f:\n",
    "        for line in f:\n",
    "            sentence_lst = pattern.findall(line)\n",
    "            counter_lst = Counter(sentence_lst)\n",
    "            sentence_filter_lst = [k for k, v in counter_lst.items() if v > 1]\n",
    "            new_file_name = str(i) + '.txt'\n",
    "            new_file_path = os.sep.join([path, 'new', new_file_name])\n",
    "            with open(new_file_path, 'w') as new_f:\n",
    "                for sentence in sentence_filter_lst:\n",
    "                    new_f.write(sentence+'\\n')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}