duplicate_image_finder.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Finds duplicate images  \n",
    "Makes a copy of the original and the duplicate into individual new folders"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from PIL import Image\n",
    "import six\n",
    "import imagehash\n",
    "import os, os.path\n",
    "from shutil import copyfile"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/eagle/p4/fastai\r\n"
     ]
    }
   ],
   "source": [
    "!pwd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "gpath = '/home/eagle/Abc1/protein/small/train/'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###### I grab color of image by len of the filename. This notebook uses green. Change 3 cells down.\n",
    "red 44  \n",
    "blue 45  \n",
    "green 46  \n",
    "yellow 47  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "46"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len('12206ccc-bbc1-11e8-b2bb-ac1f6b6435d0_green.png')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Our green image file names are the only ones with a length of 46"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "green = [i for i in (os.listdir(gpath)) if len(i) == 46]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "31072"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(green)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/home/eagle/Abc1/protein/small/train/'"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gpath"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Path to an empty directory you want to store and compare possible duplicates\n",
    "duplicates = '/home/eagle/Abc1/protein/small/pdupes/'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Creates a unique hash for images. Stores in hash as key in dict with value 0 original, 1 & greater dupes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "images = {}\n",
    "for img in (green):\n",
    "    hash = imagehash.dhash(Image.open(f'{gpath}{img}'))\n",
    "    images[hash] = images.get(hash, []) + [img]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Put the filename of each copy and each original in lists of lists"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "copies = []\n",
    "for k, img_list in six.iteritems(images):\n",
    "    if len(img_list) > 1:\n",
    "        copies.append(img_list)      "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Make a new numbered folder for each instance of a duplicate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "counter = 1\n",
    "for i in copies:\n",
    "    os.makedirs(f'{duplicates}{counter}', exist_ok=True)\n",
    "    counter += 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### We now take every duplicate and it's original and COPY into individual folders"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "counter = 1\n",
    "for i in copies:\n",
    "    for xx in i:\n",
    "        copyfile(f'{gpath}{xx}', f'{duplicates}{counter}/{xx}')\n",
    "    counter += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/home/eagle/Abc1/protein/small/pdupes/'"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "duplicates"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Go open your directory for duplicates and compare"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###### Move copies.\n",
    "I'm not responsible for lost files on your computer if you'd like to comment out the cell below\n",
    "If you are working with a images in a training set for a competition JUST LEAVE THEM if it's\n",
    "just a few duplicates.\n",
    "If you move make sure to change the corralating csv or any other training set labels!\n",
    "\n",
    "If one image has more than one copy you will need to hash all your images again\n",
    "This only moves ONE copy."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for i in copies:\n",
    "#     print(i)\n",
    "#     (os.rename(f'{gpath}{i[:1]}', f{DIR/FOR/COPIES/TO/MOVE/TO/}{I[:1]})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}