Selective Divide & Conquer (BOW).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "578eb95a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "E:\\BAG OF FEATURES-SPATIAL MATCHING -4Attack\n"
     ]
    }
   ],
   "source": [
    "cd E:\\BAG OF FEATURES-SPATIAL MATCHING -4Attack"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "6d8c3c56",
   "metadata": {},
   "outputs": [],
   "source": [
    "import matlab.engine\n",
    "eng2 = matlab.engine.start_matlab()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "f6af1b28",
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import absolute_import, division, print_function, unicode_literals\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "import numpy as np\n",
    "import imageio\n",
    "import visvis as vv\n",
    "from matplotlib import pyplot as plt\n",
    "from IPython.display import clear_output\n",
    "import os \n",
    "import cv2\n",
    "import torch\n",
    "from PIL import Image as im\n",
    "from PIL import Image  \n",
    "from PIL import Image as im\n",
    "import numpy as np\n",
    "\n",
    "from skimage.metrics import structural_similarity as ssim\n",
    "import argparse\n",
    "import shutil\n",
    "import os\n",
    "\n",
    "import art\n",
    "from art.estimators.classification import BlackBoxClassifier\n",
    "from art.defences.preprocessor import JpegCompression\n",
    "from art.attacks.evasion import HopSkipJump\n",
    "from art.attacks.evasion import ZooAttack\n",
    "from art.utils import to_categorical\n",
    "from art.utils import load_dataset, get_file, compute_accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "753d4c4a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import cv2\n",
    "import imageio\n",
    "import time\n",
    "from matplotlib import pyplot as plt\n",
    "from PIL import Image  \n",
    "from PIL import Image as im\n",
    "import numpy as np\n",
    "\n",
    "from skimage.metrics import structural_similarity as ssim\n",
    "import argparse\n",
    "import shutil\n",
    "import os\n",
    "\n",
    "\n",
    "\n",
    "base_images=[]\n",
    "folders=[]\n",
    "folders_full=[]\n",
    "save_folders=[]\n",
    "csv_saves=[]\n",
    "folder_4_base=r\"E:\\selected dataset\\bases for all category\" #folder containing base images of each target class\n",
    "folder=r\"E:\\selected dataset\\MIT withoutbase\" #folder containing multiple classes folder of target images\n",
    "folder_4_attack=r\"E:\\BAG OF FEATURES-SPATIAL MATCHING -4Attack\\images\"\n",
    "save_folder=r\"E:\\selected dataset\\demo save\" \n",
    "\n",
    "\n",
    "for filename in os.listdir(folder_4_base):\n",
    "        base_images.append((os.path.join(folder_4_base,filename)))\n",
    "        \n",
    "for filename in os.listdir(folder):\n",
    "        folders.append((os.path.join(folder,filename)))\n",
    "        \n",
    "for filename in os.listdir(save_folder):\n",
    "        save_folders.append(os.path.join(save_folder,filename))\n",
    "\n",
    "N=25 #total number of boxes\n",
    "M=12 #picked number of boxes\n",
    "    \n",
    "\n",
    "for main_loop in range(10): #number of classes \n",
    "\n",
    "    img2_arr=[]\n",
    "    img3_arr=[]\n",
    "    imgO_arr=[]\n",
    "    imgnew_arr=[]\n",
    "    list_T=[]\n",
    "    list_I=[]\n",
    "    savee=[]\n",
    "    for i in range(len(os.listdir(folders[main_loop]))):\n",
    "        savee.append('\\p'+str(i)+'.jpg')\n",
    "    for i in range(866,891):\n",
    "        list_T.append('\\P1010'+str(i)+'.jpg')\n",
    "    for i in range(841,866):\n",
    "        list_I.append('\\P1010'+str(i)+'.jpg')\n",
    "\n",
    "    img1_arr=imageio.imread(base_images[main_loop])\n",
    "    a=0\n",
    "    b=0\n",
    "\n",
    "    for filename in os.listdir(folders[main_loop]):\n",
    "            img2_arr.append(imageio.imread(os.path.join(folders[main_loop],filename)))\n",
    "            imgO_arr.append(imageio.imread(os.path.join(folders[main_loop],filename)))\n",
    "    v=0\n",
    " ########################################################################################################\n",
    "    image_init=[]\n",
    "    image_target=[]\n",
    "    averaged=[]\n",
    "\n",
    "\n",
    "    windowsize_r = 80\n",
    "    windowsize_c = 80\n",
    "\n",
    "    y=0\n",
    "    z=0\n",
    "    for r in range(0,img1_arr.shape[0], windowsize_r):\n",
    "        for c in range(0,img1_arr.shape[1], windowsize_c):\n",
    "            arr=img1_arr[r:r+windowsize_r,c:c+windowsize_c,:]     \n",
    "            image=im.fromarray(arr)\n",
    "            image.save(folder_4_attack+list_I[y])\n",
    "            y=y+1\n",
    "\n",
    "    for r in range(0,img2_arr[main_loop].shape[0], windowsize_r):\n",
    "        for c in range(0,img2_arr[main_loop].shape[1], windowsize_c):\n",
    "            arr=img2_arr[main_loop][r:r+windowsize_r,c:c+windowsize_c,:]     \n",
    "            image=im.fromarray(arr)\n",
    "            image.save(folder_4_attack+list_T[z])\n",
    "            z=z+1\n",
    "\n",
    "    for i in range(25):\n",
    "        image_init.append(imageio.imread(folder_4_attack+list_I[i]))\n",
    "        image_target.append(imageio.imread(folder_4_attack+list_T[i]))\n",
    "\n",
    "    saved=[]\n",
    "    image_init_selected=[]\n",
    "    image_target_selected=[]\n",
    "    m=[]\n",
    "    maxed=[]\n",
    "    sift64 = cv2.xfeatures2d.SURF_create()\n",
    "\n",
    "    for j in range(25):\n",
    "        kp1, des1 = sift64.detectAndCompute(image_init[j],None)\n",
    "        pts_adv = cv2.KeyPoint_convert(kp1)\n",
    "        m.append(len(pts_adv))\n",
    "\n",
    "    for i in range(12):\n",
    "        maxed.append(int(max(m)))\n",
    "        saved.append(m.index(max(m)))\n",
    "        n=m.index(max(m))\n",
    "        m[n]=0\n",
    "    for i in saved:\n",
    "        image_init_selected.append(image_init[i])\n",
    "        image_target_selected.append(image_target[i])\n",
    "    print(len(image_target_selected))\n",
    "    t=[]\n",
    "    inn=[]\n",
    "\n",
    "    for k in range(len(saved)):\n",
    "        t.append(25+int(saved[k]))\n",
    "        inn.append(int(saved[k]))\n",
    "    print(t)\n",
    "    print(inn)\n",
    "\n",
    "           # its output to a one-hot encoding\n",
    "    # its output to a one-hot encoding\n",
    "    def predict(x):\n",
    "      global pop2\n",
    "      global t,inn\n",
    "      global tar,innitial\n",
    "      out_label = []\n",
    "      for x_i in x:\n",
    "          #  x_i = x_i / x_i.max() #normalizes data in range 0 - 255\n",
    "           # x_i = 255 * x_i\n",
    "            x_i = x_i.astype(np.uint8)\n",
    "            data = im.fromarray(x_i)\n",
    "            data.save(r'E:\\BAG OF FEATURES-SPATIAL MATCHING -4Attack\\images\\p1010892.jpg')\n",
    "            try:\n",
    "\n",
    "                os.remove(r'E:\\BAG OF FEATURES-SPATIAL MATCHING -4Attack\\data\\p1010892_hist_200.mat')\n",
    "                os.remove(r'E:\\BAG OF FEATURES-SPATIAL MATCHING -4Attack\\data\\p1010892_hist_400.mat')\n",
    "                os.remove(r'E:\\BAG OF FEATURES-SPATIAL MATCHING -4Attack\\data\\p1010892_pyramid_200_3.mat')\n",
    "                os.remove(r'E:\\BAG OF FEATURES-SPATIAL MATCHING -4Attack\\data\\p1010892_pyramid_400_3.mat')\n",
    "                os.remove(r'E:\\BAG OF FEATURES-SPATIAL MATCHING -4Attack\\data\\p1010892_sift.mat')\n",
    "                os.remove(r'E:\\BAG OF FEATURES-SPATIAL MATCHING -4Attack\\data\\p1010892_texton_ind_200.mat')\n",
    "                os.remove(r'E:\\BAG OF FEATURES-SPATIAL MATCHING -4Attack\\data\\p1010892_texton_ind_400.mat')\n",
    "                os.remove(r'E:\\BAG OF FEATURES-SPATIAL MATCHING -4Attack\\data2\\p1010892_hist_200.mat')\n",
    "                os.remove(r'E:\\BAG OF FEATURES-SPATIAL MATCHING -4Attack\\data2\\p1010892_pyramid_200_2.mat')\n",
    "                os.remove(r'E:\\BAG OF FEATURES-SPATIAL MATCHING -4Attack\\data2\\p1010892_sift.mat')\n",
    "                os.remove(r'E:\\BAG OF FEATURES-SPATIAL MATCHING -4Attack\\data2\\p1010892_texton_ind_200.mat')\n",
    "            except:\n",
    "                pass\n",
    "            pop2=eng2.Example()\n",
    "\n",
    "\n",
    "            target_match=pop2[50][tar]\n",
    "\n",
    "\n",
    "            initial_match=pop2[50][innitial]\n",
    "\n",
    "\n",
    "\n",
    "            if target_match > 0.7:\n",
    "              out_label.append(0)\n",
    "            elif initial_match > 0.7: \n",
    "              out_label.append(1)\n",
    "            else:\n",
    "              out_label.append(2)\n",
    "\n",
    "      return to_categorical(out_label, 3)\n",
    "\n",
    "\n",
    "\n",
    "    # init black box object\n",
    "    classifier = BlackBoxClassifier(predict, image_target[0].shape, 3, clip_values=(0, 255))\n",
    "    label_dict = {0: 'image2', 1: 'image1', 2: 'other'}\n",
    "    classifier = BlackBoxClassifier(predict, image_target[0].shape, 3, clip_values=(0, 255))\n",
    "    attack = HopSkipJump(classifier=classifier, targeted=True, norm=2, max_iter=0, max_eval=1000, init_eval=100,verbose=False)\n",
    "    iter_step = 5\n",
    "    a=0\n",
    "    list1=[]\n",
    "    merge=[]\n",
    "    for i in range(0,55,1):\n",
    "        list1.append('image'+str(i)+'.jpg')\n",
    "    for j in range(len(saved)):\n",
    "        plt.imshow(image_init_selected[j])\n",
    "        plt.show()\n",
    "\n",
    "        for i in range(10):\n",
    "            tar=t[j]\n",
    "            innitial=inn[j]\n",
    "            x_adv = attack.generate(x=np.array([image_target_selected[j]], dtype=np.float32), x_adv_init=np.array([image_init_selected[j]],dtype=np.float32), y=to_categorical([1], 3))\n",
    "            print(attack.curr_iter)\n",
    "            print(\"Adversarial image at step %d.\" % (i * iter_step), \"L2 error\", \n",
    "                  np.linalg.norm(np.reshape(x_adv[0] - image_target_selected[j], [-1])),\n",
    "                  \"and BOW output %s.\" % label_dict[np.argmax(classifier.predict(x_adv)[0])])\n",
    "\n",
    "            plt.imshow(x_adv[0]/255)\n",
    "            plt.show(block=False)\n",
    "            print(x_adv[0].shape)\n",
    "            xtemp=x_adv[0]\n",
    "            x_adl=xtemp.astype(np.uint8)\n",
    "            merge.append(x_adl)\n",
    "            arr2im=im.fromarray(x_adl)\n",
    "            arr2im.save(list1[a])\n",
    "            attack.max_iter = iter_step\n",
    "        a=a+1\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "    j=1\n",
    "    print(saved)\n",
    "    for i in saved:\n",
    "          image_target[i]=merge[j]\n",
    "          j=j+10\n",
    "    for i in range(25):\n",
    "          arr2im=im.fromarray(image_target[i])\n",
    "          arr2im.save('adv'+str(i)+'.jpg')\n",
    "\n",
    "    from PIL import Image\n",
    "    len(merge)\n",
    "    image1=[]\n",
    "    blank_image = Image.new(\"RGB\", (400, 400))\n",
    "    for a in range(25):\n",
    "        print(a)\n",
    "        image1.append(Image.open(('adv'+str(a)+'.jpg')))\n",
    "    list1=[0,80,160,240,320]\n",
    "    k=0\n",
    "    for i in list1:\n",
    "        for j in list1:\n",
    "            blank_image.paste(image1[k], (j,i))\n",
    "            k=k+1\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "  ##########################################################################################################          \n",
    "        im2=blank_image\n",
    "        im3=blank_image\n",
    "      \n",
    "        im2 = im2.save(save_folders[main_loop]+savee[v]) \n",
    "        imgnew_arr.append(imageio.imread(save_folders[main_loop]+savee[v]))\n",
    "        v=v+1\n",
    "    def calculate_SSIM(imageA,imageB):\n",
    "        s = ssim(imageA, imageB,multichannel=True)\n",
    "        \n",
    "        return s\n",
    "    SSIM1=[]\n",
    "    for i in range(len(os.listdir(folders[main_loop]))):\n",
    "        SSIM1.append(calculate_SSIM((imgO_arr[i]),(imgnew_arr[i])))\n",
    "    print(\"SSIM between target & adversary image\",SSIM1)\n",
    "    def calculate_PSNR(im1,im2):\n",
    "        P=cv2.PSNR(im1,im2)\n",
    "        \n",
    "        return P\n",
    "    PSNR1=[]\n",
    "    for i in range(len(os.listdir(folders[main_loop]))):\n",
    "        PSNR1.append(calculate_PSNR(imgO_arr[i],imgnew_arr[i]))\n",
    "    print(\"PSNR between target & adversary image\",PSNR1)\n",
    "\n",
    "\n",
    "    def feature_match(check):\n",
    "\n",
    "        sift = cv2.xfeatures2d.SURF_create()\n",
    "        kpNEW1, desNEW1 = sift.detectAndCompute(check, None)\n",
    "        kpNEW2, desNEW2 = sift.detectAndCompute(adv_image, None)\n",
    "\n",
    "        pts_adv0 = cv2.KeyPoint_convert(kpNEW1)\n",
    "        pts_init0 = cv2.KeyPoint_convert(kpNEW2)\n",
    "\n",
    "        FLANN_INDEX_KDTREE = 1\n",
    "        index_params = dict(algorithm = FLANN_INDEX_KDTREE, trees = 5)\n",
    "        search_params = dict(checks = 50)\n",
    "        flann = cv2.FlannBasedMatcher(index_params, search_params)\n",
    "\n",
    "        matchesgood1 = flann.knnMatch(desNEW1,desNEW2,k=2)\n",
    "\n",
    "\n",
    "        # store all the good matches as per Lowe's ratio test.\n",
    "        good1 = []\n",
    "        for m,n in matchesgood1:\n",
    "            if m.distance < 0.7*n.distance:\n",
    "                good1.append(m)\n",
    "\n",
    "        src_pts1 = np.float32([ kpNEW1[m.queryIdx].pt for m in good1 ]).reshape(-1,1,2)\n",
    "        dst_pts1 = np.float32([ kpNEW2[m.trainIdx].pt for m in good1 ]).reshape(-1,1,2)\n",
    "        M, mask = cv2.findHomography(src_pts1, dst_pts1, cv2.RANSAC,3.0)\n",
    "        mask=list(mask)\n",
    "        inliers=mask.count(1)\n",
    "       \n",
    "        return inliers\n",
    "    adv_image=img1_arr\n",
    "    inliers_OP=[]\n",
    "    for i in range(len(os.listdir(folders[main_loop]))):\n",
    "        inliers_OP.append(feature_match(imgnew_arr[i]))\n",
    "    adv_image=img1_arr\n",
    "    inliers_OO=[]\n",
    "    for i in range(len(os.listdir(folders[main_loop]))):\n",
    "        inliers_OO.append(feature_match(imgO_arr[i]))\n",
    "    print(\"feature match before attack\",inliers_OO)\n",
    "    print(\"feature match after attack\",inliers_OP)\n",
    "    PFMI=[]\n",
    "    for i in range(len(os.listdir(folders[main_loop]))):\n",
    "        try:\n",
    "            PFMI.append(str(((inliers_OP[i]-inliers_OO[i])/inliers_OO[i])*100)+str('%'))\n",
    "        except:\n",
    "            PFMI.append(inliers_OP[i])\n",
    "    print(\"percentage feature matching increase\",PFMI)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f203a856",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}