FCSNA-QFVS.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "xNi5I5MGoMQ3"
   },
   "outputs": [],
   "source": [
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "pTeh7_BpoOJW",
    "outputId": "ff0c2afb-a8bb-4425-b76f-c9bd3881257a"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.cuda.is_available()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "xYXuUluJ74GC",
    "outputId": "982fca94-fa69-41ce-8311-b6e6171d804c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting wandb\n",
      "  Downloading wandb-0.16.6-py3-none-any.whl (2.2 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.2/2.2 MB\u001b[0m \u001b[31m9.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: Click!=8.0.0,>=7.1 in /usr/local/lib/python3.10/dist-packages (from wandb) (8.1.7)\n",
      "Collecting GitPython!=3.1.29,>=1.0.0 (from wandb)\n",
      "  Downloading GitPython-3.1.43-py3-none-any.whl (207 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m207.3/207.3 kB\u001b[0m \u001b[31m24.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: requests<3,>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from wandb) (2.31.0)\n",
      "Requirement already satisfied: psutil>=5.0.0 in /usr/local/lib/python3.10/dist-packages (from wandb) (5.9.5)\n",
      "Collecting sentry-sdk>=1.0.0 (from wandb)\n",
      "  Downloading sentry_sdk-2.1.1-py2.py3-none-any.whl (277 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m277.3/277.3 kB\u001b[0m \u001b[31m29.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting docker-pycreds>=0.4.0 (from wandb)\n",
      "  Downloading docker_pycreds-0.4.0-py2.py3-none-any.whl (9.0 kB)\n",
      "Requirement already satisfied: PyYAML in /usr/local/lib/python3.10/dist-packages (from wandb) (6.0.1)\n",
      "Collecting setproctitle (from wandb)\n",
      "  Downloading setproctitle-1.3.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (30 kB)\n",
      "Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from wandb) (67.7.2)\n",
      "Requirement already satisfied: appdirs>=1.4.3 in /usr/local/lib/python3.10/dist-packages (from wandb) (1.4.4)\n",
      "Requirement already satisfied: protobuf!=4.21.0,<5,>=3.19.0 in /usr/local/lib/python3.10/dist-packages (from wandb) (3.20.3)\n",
      "Requirement already satisfied: six>=1.4.0 in /usr/local/lib/python3.10/dist-packages (from docker-pycreds>=0.4.0->wandb) (1.16.0)\n",
      "Collecting gitdb<5,>=4.0.1 (from GitPython!=3.1.29,>=1.0.0->wandb)\n",
      "  Downloading gitdb-4.0.11-py3-none-any.whl (62 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m62.7/62.7 kB\u001b[0m \u001b[31m8.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2.0.0->wandb) (3.3.2)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2.0.0->wandb) (3.7)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2.0.0->wandb) (2.0.7)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2.0.0->wandb) (2024.2.2)\n",
      "Collecting smmap<6,>=3.0.1 (from gitdb<5,>=4.0.1->GitPython!=3.1.29,>=1.0.0->wandb)\n",
      "  Downloading smmap-5.0.1-py3-none-any.whl (24 kB)\n",
      "Installing collected packages: smmap, setproctitle, sentry-sdk, docker-pycreds, gitdb, GitPython, wandb\n",
      "Successfully installed GitPython-3.1.43 docker-pycreds-0.4.0 gitdb-4.0.11 sentry-sdk-2.1.1 setproctitle-1.3.3 smmap-5.0.1 wandb-0.16.6\n"
     ]
    }
   ],
   "source": [
    "pip install wandb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "kD1kTkYp7k4A"
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "\n",
    "# import wandb # logger\n",
    "import math\n",
    "from collections import OrderedDict\n",
    "import h5py\n",
    "import os\n",
    "import pickle\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 121
    },
    "id": "3i3sbyca0jSA",
    "outputId": "b93541f5-b9e2-4aeb-9369-47097bc16f80"
   },
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "        window._wandbApiKey = new Promise((resolve, reject) => {\n",
       "            function loadScript(url) {\n",
       "            return new Promise(function(resolve, reject) {\n",
       "                let newScript = document.createElement(\"script\");\n",
       "                newScript.onerror = reject;\n",
       "                newScript.onload = resolve;\n",
       "                document.body.appendChild(newScript);\n",
       "                newScript.src = url;\n",
       "            });\n",
       "            }\n",
       "            loadScript(\"https://cdn.jsdelivr.net/npm/postmate/build/postmate.min.js\").then(() => {\n",
       "            const iframe = document.createElement('iframe')\n",
       "            iframe.style.cssText = \"width:0;height:0;border:none\"\n",
       "            document.body.appendChild(iframe)\n",
       "            const handshake = new Postmate({\n",
       "                container: iframe,\n",
       "                url: 'https://wandb.ai/authorize'\n",
       "            });\n",
       "            const timeout = setTimeout(() => reject(\"Couldn't auto authenticate\"), 5000)\n",
       "            handshake.then(function(child) {\n",
       "                child.on('authorize', data => {\n",
       "                    clearTimeout(timeout)\n",
       "                    resolve(data)\n",
       "                });\n",
       "            });\n",
       "            })\n",
       "        });\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m: Logging into wandb.ai. (Learn how to deploy a W&B server locally: https://wandb.me/wandb-server)\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: You can find your API key in your browser here: https://wandb.ai/authorize\n",
      "wandb: Paste an API key from your profile and hit enter, or press ctrl+c to quit:"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " ··········\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m: Appending key for api.wandb.ai to your netrc file: /root/.netrc\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 3 login to logger\n",
    "wandb.login()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 125
    },
    "id": "q-mSzTM20vwy",
    "outputId": "ea599196-d356-4921-e312-4e7e2b6d931a"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "Tracking run with wandb version 0.16.6"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Run data is saved locally in <code>/content/wandb/run-20240506_133939-fya0dup4</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Syncing run <strong><a href='https://wandb.ai/nirld/FCSN_ATTENTION_FU_1/runs/fya0dup4' target=\"_blank\">pleasant-sun-10</a></strong> to <a href='https://wandb.ai/nirld/FCSN_ATTENTION_FU_1' target=\"_blank\">Weights & Biases</a> (<a href='https://wandb.me/run' target=\"_blank\">docs</a>)<br/>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View project at <a href='https://wandb.ai/nirld/FCSN_ATTENTION_FU_1' target=\"_blank\">https://wandb.ai/nirld/FCSN_ATTENTION_FU_1</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View run at <a href='https://wandb.ai/nirld/FCSN_ATTENTION_FU_1/runs/fya0dup4' target=\"_blank\">https://wandb.ai/nirld/FCSN_ATTENTION_FU_1/runs/fya0dup4</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<button onClick=\"this.nextSibling.style.display='block';this.style.display='none';\">Display W&B run</button><iframe src='https://wandb.ai/nirld/FCSN_ATTENTION_FU_1/runs/fya0dup4?jupyter=true' style='border:none;width:100%;height:420px;display:none;'></iframe>"
      ],
      "text/plain": [
       "<wandb.sdk.wandb_run.Run at 0x7d6b941e0f10>"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wandb.init(\n",
    "    project=\"FCSN_ATTENTION_FU_1\",\n",
    "    config={\n",
    "        \"epochs\": 20,\n",
    "        \"batch_size\": 5,\n",
    "        \"lr\": 0.0001,\n",
    "        \"decay_rate\": 0.8,\n",
    "        \"fusion_dimention\": 1200,\n",
    "        \"dropout\": 0.3,\n",
    "        \"Description\": \"300 fusion dimention, train 234 test 1\"\n",
    "    }\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "_rtG8E9z7wq0"
   },
   "outputs": [],
   "source": [
    "config = {\n",
    "  \"max_segment_num\": 20,\n",
    "  \"max_frame_num\": 200,\n",
    "  \"similarity_dim\": 1000,\n",
    "  \"concept_dim\": 300,\n",
    "  \"in_channel\": 2048,\n",
    "  \"conv1_channel\": 512,\n",
    "  \"conv2_channel\": 256,\n",
    "  \"deconv1_channel\": 1024,\n",
    "  \"deconv2_channel\": 1024,\n",
    "\n",
    "  \"gpu\": \"0\",\n",
    "  \"num_workers\": 0,\n",
    "  \"epoch\": 20,\n",
    "  \"batch_size\": 5,\n",
    "  \"lr\": 0.0001,\n",
    "  \"train_videos\": [1,2,3],\n",
    "  \"test_video\": 4,\n",
    "  \"top_percent\": 0.02\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "X508Yeae8RRa"
   },
   "outputs": [],
   "source": [
    "\n",
    "# Function to load the pickle file\n",
    "def load_pickle(filename):\n",
    "    with open(filename,'rb') as f:\n",
    "        return pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "c0ja96fePSQm"
   },
   "outputs": [],
   "source": [
    "# Custom class to load the dataset\n",
    "class UTEDataset(Dataset):\n",
    "    def __init__(self, videos):\n",
    "        self.dataset = []\n",
    "        train_videos = videos\n",
    "        for video_id in train_videos:\n",
    "            for _, _, files in os.walk(f\"/content/drive/MyDrive/Research/data/Oracle_summaries/P0{video_id}\"):\n",
    "                for file in files:\n",
    "                    self.dataset.append(file[:file.find(\"_oracle.txt\")]+f\"_{video_id}\") # eg. Food_Hands_1 -> (q1_q2_video_id)\n",
    "        self.embedding=load_pickle(\"/content/drive/MyDrive/Research/data/query_dictionary.pkl\")\n",
    "\n",
    "    def list_dataset(self):\n",
    "        return self.dataset\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        \"\"\"\n",
    "        This method will be callded when access indexed.\n",
    "        \"\"\"\n",
    "        video_id = self.dataset[index].split(\"_\")[2] # get video id from Food_Hands_1 1 here is video id\n",
    "        f = h5py.File(f'/content/drive/MyDrive/Research/data/processed/V{video_id}_resnet_avg.h5', 'r') # loading features\n",
    "        features=torch.tensor(f[\"features\"][()], dtype=torch.float32)\n",
    "        seg_len=torch.tensor(f[\"seg_len\"][()], dtype=torch.int32)\n",
    "\n",
    "\n",
    "        query_1, query_2 = self.dataset[index].split(\"_\")[0:2]\n",
    "\n",
    "        query_1_GT = torch.zeros(20*200) # GT of shape 20*200 (max_seg_size*max_shot_size)\n",
    "        query_2_GT = torch.zeros(20*200)\n",
    "\n",
    "        transfer={\"Cupglass\":\"Glass\",\n",
    "                  \"Musicalinstrument\":\"Instrument\",\n",
    "                  \"Petsanimal\":\"Animal\"}\n",
    "\n",
    "        with open(f\"/content/drive/MyDrive/Research/data/Dense_per_shot_tags/P0{video_id}/P0{video_id}.txt\", \"r\") as f:\n",
    "            lines = f.readlines()\n",
    "            for index, line in enumerate(lines):\n",
    "                queries = line.strip().split(',')\n",
    "                if query_1 in queries:\n",
    "                    query_1_GT[index] = 1\n",
    "                if query_2 in queries:\n",
    "                    query_2_GT[index] = 1\n",
    "                # NO need of the condition given below 🟥\n",
    "                # if index == 19:\n",
    "                #     break\n",
    "\n",
    "        shot_num = seg_len.sum() # getting total count of shots by summing individual segment length\n",
    "        mask_GT = torch.zeros(20*200, dtype=torch.bool)\n",
    "        for i in range(shot_num):\n",
    "            mask_GT[i] = 1\n",
    "\n",
    "        if query_1 in transfer:\n",
    "            query_1=transfer[query_1]\n",
    "        if query_2 in transfer:\n",
    "            query_2=transfer[query_2]\n",
    "\n",
    "        q1_text = query_1\n",
    "        q2_text = query_2\n",
    "        # print(\"Q1: \", query_1)\n",
    "        # print(\"Q2: \", query_2)\n",
    "        query_1 = torch.tensor(self.embedding[query_1], dtype=torch.float32)\n",
    "        query_2 = torch.tensor(self.embedding[query_2], dtype=torch.float32)\n",
    "\n",
    "        return features, seg_len, query_1, query_2, query_1_GT, query_2_GT, mask_GT, q1_text, q2_text\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "DzqbuEJ6Re_Q"
   },
   "outputs": [],
   "source": [
    "dtst = UTEDataset([1,2,3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "cD2iJ4EkRuCM",
    "outputId": "a0ebdf99-fb64-4364-c58d-244936f3fa56"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(dtst[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "1mNEYVZ-ObjI"
   },
   "outputs": [],
   "source": [
    "class Attention(torch.nn.Module):\n",
    "    def __init__(self, Q_dim, K_dim, head_size):\n",
    "        \"\"\"\n",
    "        head_size: it will be 256 but can be changed and its being used to project into same feature dimentions of q, k , and v.\n",
    "        \"\"\"\n",
    "        super().__init__()\n",
    "        self.query = torch.nn.Linear(Q_dim, head_size, bias=False)\n",
    "        self.key = torch.nn.Linear(K_dim, head_size, bias=False)\n",
    "        self.value = torch.nn.Linear(K_dim, head_size, bias=False)\n",
    "\n",
    "    def forward(self, Q_in, K_in, attention_mask):\n",
    "        q = self.query(Q_in)\n",
    "        k = self.key(K_in) # (B,T,C) same for all q, k, v\n",
    "        v = self.value(K_in)\n",
    "\n",
    "        print(f\"k shape : {k.shape} | q shape: {q.shape}\")\n",
    "\n",
    "        wei = q @ k.transpose(1,2)  * (1 / math.sqrt(q.size(-1)))  # (B, T, 256) @ (B, 256, T)   ---> (B, T, T)\n",
    "        print(\"wei shape: \", wei.shape)\n",
    "        # wei = torch.zeros((T,T))\n",
    "        wei = wei.masked_fill(attention_mask == 0, float('-inf'))\n",
    "        # wei = wei.masked_fill(attention_mask[:,:50,:50] == 0, float('-inf'))\n",
    "        # wei=wei.masked_fill(~attention_mask,-1e10)\n",
    "        wei = F.softmax(wei, dim=-1)\n",
    "\n",
    "        out = wei @ v\n",
    "        return out, wei"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "tFkblX1V8VAZ"
   },
   "outputs": [],
   "source": [
    "import math\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "# from .attention import Attention\n",
    "\n",
    "\n",
    "class CHAN(nn.Module):\n",
    "    def __init__(self,config, n_class=256):\n",
    "        nn.Module.__init__(self)\n",
    "        self.config=config\n",
    "\n",
    "        self.conv1 = nn.Sequential(OrderedDict([\n",
    "            ('conv1_1', nn.Conv1d(2048, 2048, 3, padding=1)),\n",
    "            ('bn1_1', nn.BatchNorm1d(2048)),\n",
    "            ('relu1_1', nn.ReLU(inplace=True)),\n",
    "            ('conv1_2', nn.Conv1d(2048, 2048, 3, padding=1)),\n",
    "            ('bn1_2', nn.BatchNorm1d(2048)),\n",
    "            ('relu1_2', nn.ReLU(inplace=True)),\n",
    "            ('pool1', nn.MaxPool1d(2, stride=2, ceil_mode=True))\n",
    "            ])) # 1/2\n",
    "\n",
    "        self.conv2 = nn.Sequential(OrderedDict([\n",
    "            ('conv2_1', nn.Conv1d(2048, 2048, 3, padding=1)),\n",
    "            ('bn2_1', nn.BatchNorm1d(2048)),\n",
    "            ('relu2_1', nn.ReLU(inplace=True)),\n",
    "            ('conv2_2', nn.Conv1d(2048, 2048, 3, padding=1)),\n",
    "            ('bn2_2', nn.BatchNorm1d(2048)),\n",
    "            ('relu2_2', nn.ReLU(inplace=True)),\n",
    "            ('pool2', nn.MaxPool1d(2, stride=2, ceil_mode=True))\n",
    "            ])) # 1/4\n",
    "\n",
    "        self.conv3 = nn.Sequential(OrderedDict([\n",
    "            ('conv3_1', nn.Conv1d(2048, 2048, 3, padding=1)),\n",
    "            ('bn3_1', nn.BatchNorm1d(2048)),\n",
    "            ('relu3_1', nn.ReLU(inplace=True)),\n",
    "            ('conv3_2', nn.Conv1d(2048, 2048, 3, padding=1)),\n",
    "            ('bn3_2', nn.BatchNorm1d(2048)),\n",
    "            ('relu3_2', nn.ReLU(inplace=True)),\n",
    "            ('conv3_3', nn.Conv1d(2048, 2048, 3, padding=1)),\n",
    "            ('bn3_3', nn.BatchNorm1d(2048)),\n",
    "            ('relu3_3', nn.ReLU(inplace=True)),\n",
    "            ('pool3', nn.MaxPool1d(2, stride=2, ceil_mode=True))\n",
    "            ])) # 1/8\n",
    "\n",
    "        self.conv4 = nn.Sequential(OrderedDict([\n",
    "            ('conv4_1', nn.Conv1d(2048, 2048, 3)),\n",
    "            ('bn4_1', nn.BatchNorm1d(2048)),\n",
    "            ('relu4_1', nn.ReLU(inplace=True)),\n",
    "            ('conv4_2', nn.Conv1d(2048, 2048, 3)),\n",
    "            ('bn4_2', nn.BatchNorm1d(2048)),\n",
    "            ('relu4_2', nn.ReLU(inplace=True)),\n",
    "            ('conv4_3', nn.Conv1d(2048, 2048, 3)),\n",
    "            ('bn4_3', nn.BatchNorm1d(2048)),\n",
    "            ('relu4_3', nn.ReLU(inplace=True)),\n",
    "            # ('pool4', nn.MaxPool1d(2, stride=2, ceil_mode=True))\n",
    "            ])) # 1/16\n",
    "\n",
    "        self.conv5 = nn.Sequential(OrderedDict([\n",
    "            ('conv5_1', nn.Conv1d(2048, 2048, 3, padding=1)),\n",
    "            ('bn5_1', nn.BatchNorm1d(2048)),\n",
    "            ('relu5_1', nn.ReLU(inplace=True)),\n",
    "            ('conv5_2', nn.Conv1d(2048, 2048, 3, padding=1)),\n",
    "            ('bn5_2', nn.BatchNorm1d(2048)),\n",
    "            ('relu5_2', nn.ReLU(inplace=True)),\n",
    "            ('conv5_3', nn.Conv1d(2048, 2048, 3, padding=1)),\n",
    "            ('bn5_3', nn.BatchNorm1d(2048)),\n",
    "            ('relu5_3', nn.ReLU(inplace=True)),\n",
    "            ('pool5', nn.MaxPool1d(2, stride=2, ceil_mode=True))\n",
    "            ])) # 1/32\n",
    "\n",
    "        self.conv6 = nn.Sequential(OrderedDict([\n",
    "            ('fc6', nn.Conv1d(2048, 4096, 1)),\n",
    "            ('bn6', nn.BatchNorm1d(4096)),\n",
    "            ('relu6', nn.ReLU(inplace=True)),\n",
    "            ('drop6', nn.Dropout(0.3))\n",
    "            ]))\n",
    "\n",
    "        self.conv7 = nn.Sequential(OrderedDict([\n",
    "            ('fc7', nn.Conv1d(4096, 4096, 1)),\n",
    "            ('bn7', nn.BatchNorm1d(4096)),\n",
    "            ('relu7', nn.ReLU(inplace=True)),\n",
    "            ('drop7', nn.Dropout(0.3))\n",
    "            ]))\n",
    "\n",
    "        self.conv8 = nn.Sequential(OrderedDict([\n",
    "            ('fc8', nn.Conv1d(4096, n_class, 1)),\n",
    "            ('bn8', nn.BatchNorm1d(n_class)),\n",
    "            ('relu8', nn.ReLU(inplace=True)),\n",
    "            ]))\n",
    "\n",
    "        self.conv_pool4 = nn.Conv1d(2048, 1024, 1)\n",
    "        self.bn_pool4 = nn.BatchNorm1d(1024)\n",
    "\n",
    "        # self.conv1d_1=nn.Conv1d(self.config[\"in_channel\"],self.config[\"conv1_channel\"],kernel_size=5,stride=1,padding=2)\n",
    "        # self.max_pooling_1=nn.MaxPool1d(2,stride=2,padding=0)\n",
    "        # self.conv1d_2=nn.Conv1d(self.config[\"conv1_channel\"],self.config[\"conv2_channel\"],kernel_size=5,stride=1,padding=2)\n",
    "        # self.max_pooling_2=nn.MaxPool1d(2,stride=2,padding=0)\n",
    "        # self.self_attention=Attention(self.config[\"conv2_channel\"],self.config[\"conv2_channel\"],self.config[\"conv2_channel\"])\n",
    "        # self.concept_attention=Attention(self.config[\"concept_dim\"],self.config[\"conv2_channel\"],self.config[\"conv2_channel\"])\n",
    "\n",
    "        # self.self_attention=Attention(256,256,256)\n",
    "        # self.concept_attention=Attention(300,256,256)\n",
    "\n",
    "        self.self_attention=Attention(self.config[\"conv2_channel\"],self.config[\"conv2_channel\"],self.config[\"conv2_channel\"])\n",
    "        self.query_relevence_attention=Attention(self.config[\"concept_dim\"],self.config[\"conv2_channel\"],self.config[\"conv2_channel\"])\n",
    "        self.global_attention=Attention(self.config[\"conv2_channel\"],self.config[\"conv2_channel\"],self.config[\"conv2_channel\"])\n",
    "\n",
    "\n",
    "        self.deconv1 = nn.ConvTranspose1d(768, 1024, 3, padding=1, stride=2, bias=False)\n",
    "        self.deconv2 = nn.ConvTranspose1d(1024, 1024, 20, stride=10, bias=False)\n",
    "\n",
    "        self.similarity_linear1=torch.nn.Linear(1024,300,bias=False)\n",
    "        self.similarity_linear2=torch.nn.Linear(300,300,bias=False)\n",
    "\n",
    "        # self.MLP=torch.nn.Linear(self.config[\"similarity_dim\"],1)\n",
    "        self.MLP = torch.nn.Linear(1200, 1)\n",
    "\n",
    "    # batch tensor: batch_size * max_seg_num * max_seg_length * 2048/4096\n",
    "    # seg_len list(list(int)) : batch_size * seg_num (num of frame)\n",
    "    # concept : batch_size * 300\n",
    "    def forward(self,batch,seg_len,concept1,concept2):\n",
    "        batch_size=batch.size()[0]\n",
    "        max_seg_num=batch.size()[1]\n",
    "        max_seg_length=batch.size()[2]\n",
    "        B, seg, T, C = batch.shape\n",
    "\n",
    "        # print(\"Seg Len: \", seg_len)\n",
    "        # print(\"batch_size: \", batch_size)\n",
    "        # print(\"Concept Shape: \", concept1.shape)\n",
    "        # print(\"Conv1d in shape-> \", batch.view(batch_size*max_seg_num,max_seg_length,-1).transpose(1,2).shape)\n",
    "        # (batch_size * max_seg_num) * 128 * max_seg_length\n",
    "        # tmp1=self.conv1d_1(batch.view(batch_size*max_seg_num,max_seg_length,-1).transpose(1,2))\n",
    "\n",
    "        # # print(\"Conv1d : \", tmp1.shape)\n",
    "        # # (batch_size * max_seg_num) * 128 * max_seg_length/2\n",
    "        # tmp1=self.max_pooling_1(tmp1)\n",
    "\n",
    "        # # print(\"Max Pooling: \", tmp1.shape)\n",
    "\n",
    "        # # (batch_size * max_seg_num) * 256 * max_seg_length/2\n",
    "        # tmp2=self.conv1d_2(tmp1)\n",
    "\n",
    "        # # print(\"Conv2 : \", tmp2.shape)\n",
    "        # # (batch_size * max_seg_num) * max_seg_length/4 * 256\n",
    "        # tmp2=self.max_pooling_2(tmp2).transpose(1,2)\n",
    "        h = batch.view(batch_size*max_seg_num, max_seg_length,-1).transpose(1,2)\n",
    "        h = self.conv1(h)\n",
    "        print(f\"conv1 {h.shape}\")\n",
    "        h = self.conv2(h)\n",
    "        print(f\"conv2 {h.shape}\")\n",
    "        h = self.conv3(h)\n",
    "        print(f\"conv3 {h.shape}\")\n",
    "        # print(\"H's Shape: \", h.shape)\n",
    "        h = self.conv4(h)\n",
    "        print(f\"conv4 {h.shape}\")\n",
    "        pool4 = h\n",
    "\n",
    "        # print(\"H's Shape: \", h.shape)\n",
    "\n",
    "        h = self.conv5(h)\n",
    "        print(f\"conv5 {h.shape}\")\n",
    "        h = self.conv6(h)\n",
    "        print(f\"conv6 {h.shape}\")\n",
    "        h = self.conv7(h)\n",
    "        print(f\"conv7 {h.shape}\")\n",
    "        # print(\"H's Shape: \", h.shape)\n",
    "        h = self.conv8(h)\n",
    "        print(f\"conv8 {h.shape}\")\n",
    "        h = h.transpose(1,2)\n",
    "\n",
    "        # print(\"H's Shape: \", h.shape) # [20, 1024, 7] --> (B*seg, C, T)\n",
    "\n",
    "        # print(\"Max Pool : \", tmp2.shape)\n",
    "        # batch_size * max_seg_num * max_seg_length/4\n",
    "\n",
    "        # attention_mask=torch.zeros(batch_size,max_seg_num,int(max_seg_length/20),dtype=torch.bool)\n",
    "        # for i in range(batch_size):\n",
    "        #     print(\"i: \", i)\n",
    "        #     print(\"len: \", len(seg_len[i]))\n",
    "        #     for j in range(len(seg_len[i])):\n",
    "        #         for k in range(int(math.ceil(seg_len[i][j]/20.0))):\n",
    "        #             attention_mask[i][j][k]=1\n",
    "\n",
    "        # tmp2 = tmp2.transpose(1,2)\n",
    "\n",
    "        # print(\"Max Pool : \", tmp2.shape)\n",
    "        # batch_size * max_seg_num * max_seg_length/4\n",
    "        attention_mask=torch.zeros(batch_size,max_seg_num,int(max_seg_length/20),dtype=torch.bool).to('cuda')\n",
    "        for i in range(batch_size):\n",
    "            # print(\"i: \", i)\n",
    "            # print(\"len: \", len(seg_len[i]))\n",
    "            for j in range(len(seg_len[i])):\n",
    "                for k in range(int(math.ceil(seg_len[i][j]/20.0))):\n",
    "                    attention_mask[i][j][k]=1\n",
    "\n",
    "        # print(\"Attention mask: \", attention_mask.shape) # 📍\n",
    "\n",
    "        # (batch_size * max_seg_num) * max_seg_length/4 * max_seg_length/4\n",
    "        # attention_mask=attention_mask.view(batch_size*max_seg_num,-1).unsqueeze(1)\n",
    "        attention_mask=attention_mask.view(B*seg,-1).unsqueeze(1) # (B*seg, 1, T/4) --> (200, 1, 50)\n",
    "\n",
    "        q_mat = h.view(B*seg, int(T//20), -1)\n",
    "        k_mat = h.view(B*seg, int(T//20), -1)\n",
    "\n",
    "        #### self-attention\n",
    "        self_attention_result, self_attention_wei = self.self_attention(q_mat, k_mat, attention_mask)\n",
    "\n",
    "        #### query_relevence_attention\n",
    "\n",
    "        # average representation of both query\n",
    "        avg_query = (concept1 * concept2)/2\n",
    "        t_q = avg_query.unsqueeze(1).unsqueeze(1).expand(\n",
    "                    B,seg,int(\n",
    "                        T/20\n",
    "                    ),300).contiguous().view(B*seg,int(T/20),300)\n",
    "\n",
    "        query_relevance_attention_result, query_relevance_attention_wei = self.query_relevence_attention(t_q, k_mat, attention_mask)\n",
    "\n",
    "        query_relevance_attention_agg_result = query_relevance_attention_result.mean(dim=1) # (B*seg, C) -> (200, 256) which is segment wise query relevence score, 1 vector per segment\n",
    "        query_relevance_attention_agg_result = query_relevance_attention_agg_result.unsqueeze(0).expand(int(T//20),B*seg,256)\n",
    "        q_mat = q_mat.contiguous().view(int(T/20), B*seg, 256) # (B*seg, T/20, 256) --> (T/20, B*seg, 256) == (50, 200, 256)\n",
    "\n",
    "        ####  Global Attention\n",
    "        attention_mask = attention_mask.view(int(T//20), 1, B*seg) # (B*seg, 1, T/20) --> (T/20, 1, B*seg)\n",
    "        global_attention_result, global_attention_wei = self.global_attention(q_mat,query_relevance_attention_agg_result, attention_mask) # (T/20, B*seg, 256)\n",
    "        global_attention_result = global_attention_result.view(B*seg, int(T//20), -1) # (T/20, B*seg, 256) --> (B*seg, T/20, 256)\n",
    "\n",
    "        ## Concat the feature\n",
    "        final_concat_result=torch.cat((h,self_attention_result,global_attention_result),dim=-1) # (B*seg, T/4, 768)\n",
    "\n",
    "        # (batch_size * max_seg_num) * max_seg_length/4 * 1024\n",
    "        # attention_result=torch.cat((h,self_attention_result,concept1_attention_result,concept2_attention_result),dim=-1)\n",
    "\n",
    "\n",
    "        # print(\"Attn concat result: \", attention_result.shape)\n",
    "\n",
    "        attention_result = final_concat_result.transpose(1,2)\n",
    "        # print(\"Attn transpose result: \", attention_result.shape)\n",
    "        h = self.deconv1(attention_result)\n",
    "        print(f\"deconv1 {h.shape}\")\n",
    "        upscore2 = h\n",
    "        h = self.conv_pool4(pool4)\n",
    "        h = self.bn_pool4(h)\n",
    "        score_pool4 = h\n",
    "        # print(\"upscore2: \", upscore2.shape)\n",
    "        # print(\"score_pool4: \", score_pool4.shape)\n",
    "\n",
    "        h = upscore2 + score_pool4\n",
    "\n",
    "        h = self.deconv2(h)\n",
    "        print(f\"deconv2 {h.shape}\")\n",
    "\n",
    "        # print(\"final deconv op: \", h.shape)\n",
    "        h = h.transpose(1,2)\n",
    "        # print(\"final deconv op tp: \", h.shape)\n",
    "        # print(\"Cat attention_result : \", attention_result.shape) # 📍\n",
    "        # (batch_size * max_seg_num) * 200 * max_seg_length/2\n",
    "        # result=self.transpose_conv1d_1(attention_result.transpose(1,2))\n",
    "\n",
    "        # print(\"deconv 1 ip: \", attention_result.transpose(1,2).shape)\n",
    "        # print(\"deconv 1: \", result.shape)\n",
    "        # print(\"deconv 2: ip \", self.transpose_conv1d_2(result).transpose(1,2).shape)\n",
    "\n",
    "        # batch_size * max_seg_num * max_seg_length * 400\n",
    "        # result=self.transpose_conv1d_2(result).transpose(1,2).contiguous().view(batch_size,max_seg_num*max_seg_length,-1)\n",
    "        # print(\"deconv 2: \", result.shape)\n",
    "\n",
    "        # batch_size * (max_seg_num * max_seg_length) * 50\n",
    "        similar_1=self.similarity_linear1(h.transpose(1,2).contiguous().view(batch_size, max_seg_num*max_seg_length,-1))\n",
    "\n",
    "        print(\"similar1: \", similar_1.shape)\n",
    "\n",
    "        # batch_size * 50\n",
    "        concept1_similar=self.similarity_linear2(concept1)\n",
    "        concept2_similar=self.similarity_linear2(concept2)\n",
    "\n",
    "        print(\"concept1_similar.unsqueeze(1): \", concept1_similar.unsqueeze(1).shape)\n",
    "\n",
    "        # batch_size * (max_seg_num * max_seg_length) * 50\n",
    "        # concept1_similar=concept1_similar.unsqueeze(1)*similar_1\n",
    "        # concept2_similar=concept2_similar.unsqueeze(1)*similar_1\n",
    "\n",
    "        # fusion unit -----new added\n",
    "        mul1 = concept1_similar.unsqueeze(1)*similar_1\n",
    "        mul2 = concept2_similar.unsqueeze(1)*similar_1\n",
    "\n",
    "        add1 = concept1_similar.unsqueeze(1)+similar_1\n",
    "        add2 = concept2_similar.unsqueeze(1)+similar_1\n",
    "\n",
    "        op_fusion1 = torch.cat((concept1_similar.unsqueeze(1).expand(B,4000,300), similar_1, add1, mul1), dim=-1)\n",
    "        op_fusion2 = torch.cat((concept2_similar.unsqueeze(1).expand(B,4000,300), similar_1, add2, mul2), dim=-1)\n",
    "\n",
    "        # batch_size * (max_seg_num * max_seg_length) * 1\n",
    "        concept1_score=self.MLP(op_fusion1)\n",
    "        concept2_score=self.MLP(op_fusion2)\n",
    "\n",
    "        # batch_size * (max_seg_num * max_seg_length) * 1\n",
    "        concept1_score=torch.sigmoid(concept1_score)\n",
    "        concept2_score=torch.sigmoid(concept2_score)\n",
    "\n",
    "        # batch_size * max_seg_num * max_seg_length\n",
    "        concept1_score=concept1_score.squeeze(-1).view(batch_size,max_seg_num,max_seg_length)\n",
    "        concept2_score=concept2_score.squeeze(-1).view(batch_size,max_seg_num,max_seg_length)\n",
    "\n",
    "        # batch_size * max_seg_num * max_seg_length\n",
    "        return concept1_score,concept2_score,self_attention_wei, query_relevance_attention_wei, global_attention_wei\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "z0CtR68eOjbH"
   },
   "outputs": [],
   "source": [
    "# Copyright (c) Meta Platforms, Inc. and affiliates.\n",
    "# All rights reserved.\n",
    "\n",
    "# This source code is licensed under the license found in the\n",
    "# LICENSE file in the root directory of this source tree.\n",
    "\n",
    "\n",
    "import networkx as nx\n",
    "from sklearn.metrics import pairwise_distances\n",
    "import numpy as np\n",
    "import scipy.io\n",
    "\n",
    "def process_video_mat(video_mat):\n",
    "    result = []\n",
    "    for shot_vec in video_mat:\n",
    "        shot_vec= shot_vec[0][0]\n",
    "        result.append(shot_vec)\n",
    "    result = np.array(result)\n",
    "    return result\n",
    "\n",
    "\n",
    "def process_mat(mat):\n",
    "    videos = mat['Tags'][0]\n",
    "    result = []\n",
    "    for video_mat in videos:\n",
    "        video_mat = video_mat[0]\n",
    "        video_data = process_video_mat(video_mat)\n",
    "        result.append(video_data)\n",
    "    return result\n",
    "\n",
    "\n",
    "def load_videos_tag(mat_path=\"/content/drive/MyDrive/Research/data/evaluation/Tags.mat\"):\n",
    "    mat = scipy.io.loadmat(mat_path)\n",
    "    videos_tag = process_mat(mat)\n",
    "    return videos_tag\n",
    "\n",
    "def semantic_iou(a, b):\n",
    "    intersection = a * b\n",
    "    intersection_num = sum(intersection)\n",
    "    union = a + b\n",
    "    union[union>0] = 1\n",
    "    union_num = sum(union)\n",
    "    if union_num != 0:\n",
    "        return intersection_num / union_num\n",
    "    else:\n",
    "        return 0\n",
    "\n",
    "\n",
    "def build_graph_from_pariwise_weights(weight_matrix):\n",
    "    B = nx.Graph()\n",
    "    bottom_nodes = list(map(lambda x: \"b-{}\".format(x), list(range(weight_matrix.shape[0]))))\n",
    "    top_nodes = list(map(lambda x: \"t-{}\".format(x), list(range(weight_matrix.shape[1]))))\n",
    "    edges = []\n",
    "    for i in range(weight_matrix.shape[0]):\n",
    "        for j in range(weight_matrix.shape[1]):\n",
    "            weight = weight_matrix[i][j]\n",
    "            edges.append((\"b-{}\".format(i), \"t-{}\".format(j), weight))\n",
    "    B.add_weighted_edges_from(edges)\n",
    "    return B\n",
    "\n",
    "\n",
    "def calculate_semantic_matching(machine_summary, gt_summary, video_shots_tag, video_id):\n",
    "    video_shots_tag = video_shots_tag[video_id]\n",
    "    # print(\"v4 length: \",len(video_shots_tag))\n",
    "    # print(\"v4 length: \",machine_summary)\n",
    "    machine_summary_mat = video_shots_tag[machine_summary]\n",
    "    gt_summary_mat = video_shots_tag[gt_summary]\n",
    "    weights = pairwise_distances(machine_summary_mat, gt_summary_mat, metric=semantic_iou)\n",
    "    B = build_graph_from_pariwise_weights(weights)\n",
    "    matching_edges = nx.algorithms.matching.max_weight_matching(B)\n",
    "    sum_weights = 0\n",
    "    i = 0\n",
    "    for edge in matching_edges:\n",
    "        edge_data = B.get_edge_data(edge[0], edge[1])\n",
    "        sum_weights += edge_data['weight']\n",
    "        i += 1\n",
    "    precision = sum_weights / machine_summary_mat.shape[0]\n",
    "    recall = sum_weights / gt_summary_mat.shape[0]\n",
    "    f1 = 2 * precision * recall / (precision + recall)\n",
    "    return precision, recall, f1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "128yWnkAPeyl"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import h5py\n",
    "import torch\n",
    "from torch.utils.data import DataLoader\n",
    "# import matlab.engine\n",
    "\n",
    "# from model import CHAN\n",
    "# from dataset import UCTDataset\n",
    "# from utils import load_pickle\n",
    "\n",
    "\n",
    "class Runner():\n",
    "    def __init__(self,config, train_vid, test_vid):\n",
    "        self.config=config\n",
    "        self.device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
    "        self.train_vid = train_vid\n",
    "        self.test_vid = test_vid\n",
    "        self.loss_list = []\n",
    "        self.f1_list = []\n",
    "        self.train_f1_list = []\n",
    "        os.environ[\"CUDA_VISIBLE_DEVICES\"] = self.config[\"gpu\"]\n",
    "        self._build_dataloader()\n",
    "        self._build_model()\n",
    "        self._build_optimizer()\n",
    "        self.max_f1=0\n",
    "        self.max_p=0\n",
    "        self.max_r=0\n",
    "\n",
    "\n",
    "    def _build_model(self):\n",
    "        self.model = CHAN(self.config).to(self.device)\n",
    "\n",
    "    def _build_dataset(self):\n",
    "        return UTEDataset(self.train_vid)\n",
    "\n",
    "    def _build_dataloader(self):\n",
    "        dataset=self._build_dataset()\n",
    "        self.dataloader=DataLoader(dataset, batch_size=self.config[\"batch_size\"], shuffle=True, num_workers=self.config[\"num_workers\"])\n",
    "\n",
    "    def _build_optimizer(self):\n",
    "        self.optimizer = torch.optim.Adam(self.model.parameters(), lr=self.config[\"lr\"], weight_decay=0.8)\n",
    "\n",
    "    def output(self):\n",
    "        print(\" max_p = \",self.max_p,\" max_r = \",self.max_r,\" max_f1 = \",self.max_f1)\n",
    "\n",
    "\n",
    "    def train(self):\n",
    "        print(\"start to evaluate random result\")\n",
    "        # self.evaluate(self.test_vid,self.config[\"top_percent\"])\n",
    "        print(\"end to evaluate random result\")\n",
    "\n",
    "        criterion=torch.nn.BCELoss()\n",
    "        self.model.train()\n",
    "        for epoch in range(self.config[\"epoch\"]):\n",
    "            batch_count=0\n",
    "            # loss_list = []\n",
    "            for features,seg_len,concept1,concept2,concept1_GT,concept2_GT,mask_GT, q1t, q2t in self.dataloader:\n",
    "                train_num=seg_len.shape[0]\n",
    "                batch_count+=1\n",
    "                self.optimizer.zero_grad()\n",
    "\n",
    "                mask=torch.zeros(train_num,self.config[\"max_segment_num\"],self.config[\"max_frame_num\"],dtype=torch.bool).to(self.device)\n",
    "                for i in range(train_num):\n",
    "                    for j in range(len(seg_len[i])):\n",
    "                        for k in range(seg_len[i][j]):\n",
    "                            mask[i][j][k]=1\n",
    "\n",
    "                # batch_size * max_seg_num * max_seg_length\n",
    "                # print(f\"Model: -----------\")\n",
    "                # print(f\"Segment Length: {seg_len}\")\n",
    "                # print(f\"Feature shape: {features.shape}\")\n",
    "                # print(f\"Concept shape: {features.shape}\")\n",
    "                # print(f\"Concept 1: {concept1}, Concept 2: {concept2}\")\n",
    "                # print(f\"features: {features}\")\n",
    "                concept1_score,concept2_score, self_attention_wei, query_relevance_attention_wei, global_attention_wei=self.model(features.to(self.device),seg_len.to(self.device),concept1.to(self.device),concept2.to(self.device))\n",
    "                # concept1_score,concept2_score, self_attention_wei, query_relevance_attention_wei, global_attention_wei=self.model(features,seg_len,concept1,concept2)\n",
    "\n",
    "                loss=torch.zeros(1).to(self.device)\n",
    "                for i in range(train_num):\n",
    "                    concept1_score_tmp=concept1_score[i].masked_select(mask[i]).unsqueeze(0)\n",
    "                    concept2_score_tmp=concept2_score[i].masked_select(mask[i]).unsqueeze(0)\n",
    "                    concept1_GT_tmp=concept1_GT[i].masked_select(mask_GT[i]).unsqueeze(0)\n",
    "                    concept2_GT_tmp=concept2_GT[i].masked_select(mask_GT[i]).unsqueeze(0)\n",
    "\n",
    "                    # print(f\"concept1_score: {concept1_score_tmp[:20]}, concept2_score: {concept2_score_tmp}\")\n",
    "                    # print(f\"concept1_score_GT: {concept1_GT_tmp[:20]}, concept2_score_GT: {concept2_GT_tmp}\")\n",
    "\n",
    "                    loss1=criterion(concept1_score_tmp,concept1_GT_tmp.to(self.device))\n",
    "                    loss2=criterion(concept2_score_tmp,concept2_GT_tmp.to(self.device))\n",
    "                    # loss1=criterion(concept1_score_tmp,concept1_GT_tmp)\n",
    "                    # loss2=criterion(concept2_score_tmp,concept2_GT_tmp)\n",
    "                    # print(f\"CE l1 = {loss1}\")\n",
    "                    # print(f\"CE l2 = {loss2}\")\n",
    "                    loss+=loss1+loss2\n",
    "                print(\"Loss: \", loss)\n",
    "                # wandb.log({\n",
    "                #   \"train/loss\": loss.item()/train_num\n",
    "                # })\n",
    "\n",
    "                if (batch_count+1)%5==0:\n",
    "                    print(\"epoch \",epoch+1,\" batch \",batch_count+1,\" loss \",loss.item()/train_num)\n",
    "                self.loss_list.append(loss.item()/train_num)\n",
    "                loss.backward()\n",
    "                self.optimizer.step()\n",
    "            _, _, f1 = self.evaluate(self.test_vid,self.config[\"top_percent\"]) # evaluating test dataset\n",
    "            _, _, f1_train = self.evaluate(4,self.config[\"top_percent\"]) # evaluating train dataset currently its 4\n",
    "            # wandb.log({\n",
    "            #   \"epoch\": epoch+1,\n",
    "            #   \"test/f1\": f1,\n",
    "            #   \"train/f1\": f1_train\n",
    "            # })\n",
    "            print(\"Loss: \",loss.item()/train_num)\n",
    "            self.f1_list.append(f1)\n",
    "            self.train_f1_list.append(f1_train)\n",
    "\n",
    "\n",
    "    def evaluate(self,video_id,top_percent):\n",
    "        \"\"\"\n",
    "        method takes test_video id and top_percent which is criteria for selecting a shot like 0.28 etc.\n",
    "        \"\"\"\n",
    "        self.model.eval()\n",
    "        f1_sum = 0\n",
    "        p_sum = 0\n",
    "        r_sum = 0\n",
    "\n",
    "        # loading pickle file that is pretrained model to get features of query(text)\n",
    "        embedding = load_pickle(\"/content/drive/MyDrive/Research/data/query_dictionary.pkl\")\n",
    "\n",
    "        evaluation_num = 0\n",
    "        print(\"Evaluating on video:\", video_id)\n",
    "        # directory ../data/P01 contains Oracle summaries in txt file format\n",
    "        # each concept1_concept2_oracle.txt file contains GT summary shot numbers\n",
    "        for _, _, files in os.walk(f\"/content/drive/MyDrive/Research/data/Oracle_summaries/P0{video_id}\"): # getting all files in files\n",
    "            evaluation_num = len(files)\n",
    "            for file in files: # looping through each file or we can say training example that is concept1_concept2_oracle.txt\n",
    "                # GTSummary list generation converting numbers given in file to list like [2, 3, ... ,443]\n",
    "                summaries_GT = []\n",
    "                with open(f\"/content/drive/MyDrive/Research/data/Oracle_summaries/P0{video_id}/\"+file, \"r\") as f:\n",
    "                    for line in f.readlines():\n",
    "                        summaries_GT.append(int(line.strip())) # this will append each GT shot number converting it to int and making its list like [1, 3,..434]\n",
    "                    # print(summaries_GT)\n",
    "\n",
    "                f = h5py.File(f'/content/drive/MyDrive/Research/data/processed/V{video_id}_resnet_avg.h5', 'r') # loading video features\n",
    "                features=torch.tensor(f[\"features\"][()], dtype=torch.float32)\n",
    "                seg_len=torch.tensor(f[\"seg_len\"][()], dtype=torch.int32)\n",
    "                # features = torch.tensor(f[\"feature\"][()], dtype=torch.float32) # converting feature values to tensor and loading features\n",
    "\n",
    "                transfer={\"Cupglass\":\"Glass\",\"Musicalinstrument\":\"Instrument\",\"Petsanimal\":\"Animal\"}\n",
    "\n",
    "                query1, query2 = file.split('_')[:2]\n",
    "                if query1 in transfer:\n",
    "                    query1=transfer[query1]\n",
    "                if query2 in transfer:\n",
    "                    query2=transfer[query2]\n",
    "\n",
    "                query1 = torch.tensor(embedding[query1], dtype=torch.float32)\n",
    "                query2 = torch.tensor(embedding[query2], dtype=torch.float32)\n",
    "                # print(len(features))\n",
    "                mask = torch.zeros(1, 20, 200, dtype=torch.bool).to(self.device)\n",
    "                for i in range(1):\n",
    "                    for j in range(len(seg_len.unsqueeze(0)[i])):\n",
    "                        for k in range(seg_len.unsqueeze(0)[i][j]):\n",
    "                            mask[i][j][k]=1\n",
    "                q1_score, q2_score, self_attention_wei, query_relevance_attention_wei, global_attention_wei  = self.model(features.unsqueeze(0).to(self.device), seg_len.unsqueeze(0).to(self.device),query1.unsqueeze(0).to(self.device), query2.unsqueeze(0).to(self.device))\n",
    "                # q1_score, q2_score, self_attention_wei, query_relevance_attention_wei, global_attention_wei = self.model(features.unsqueeze(0), seg_len.unsqueeze(0),query1.unsqueeze(0), query2.unsqueeze(0))\n",
    "                # print(\"q1:\", q1_score.shape)\n",
    "                # q1_score dims [1, 2783, 1]\n",
    "\n",
    "                score = q1_score + q2_score\n",
    "                # print(score.shape)  # dims [1, 2783, 1]\n",
    "                # score = score.transpose(1,2)\n",
    "                score = score.masked_select(mask) # dims [7745089]\n",
    "                # print(score.shape)\n",
    "\n",
    "                _,top_index=score.topk(int(score.shape[0]*0.02))\n",
    "\n",
    "                if 3588 in top_index and video_id == 4:\n",
    "                    print(\"removing 3588\")\n",
    "                    mk = top_index != 3588\n",
    "                    top_index = top_index[mk]\n",
    "\n",
    "\n",
    "                # print(\"before summaries_GT.shape : \", summaries_GT)\n",
    "                if 3692 in summaries_GT and video_id == 2:\n",
    "                    # removing this index from video 2 using remove() method because summaries_GT is python list and not the tensor\n",
    "                    print(\"removing 3692\")\n",
    "                    summaries_GT.remove(3692)\n",
    "                    # mk = top_index != 3692\n",
    "                    # print(mk)\n",
    "                    # top_index = top_index[mk]\n",
    "\n",
    "                    # mk = summaries_GT != 3692\n",
    "                    # summaries_GT = summaries_GT[mk]\n",
    "\n",
    "\n",
    "                # print(\"before summaries_GT.shape : \", summaries_GT)\n",
    "                top_index_to_print = top_index + 1\n",
    "                print(file.split('_')[:])\n",
    "                print(\"evaluation video: \", video_id-1)\n",
    "                # print(top_index)\n",
    "                print(top_index_to_print)\n",
    "                video_shots_tag = load_videos_tag(mat_path=\"/content/drive/MyDrive/Research/data/evaluation/Tags.mat\")\n",
    "                p, r, f1 = calculate_semantic_matching(list(top_index.cpu().numpy()), summaries_GT, video_shots_tag, video_id=video_id-1) # video id = 0,1,2,3\n",
    "\n",
    "                f1_sum += f1\n",
    "                p_sum += p\n",
    "                r_sum += r\n",
    "                # print(p, r , f1)\n",
    "                print(\"p \", p, \"r \", r, \"f1 \", f1)\n",
    "                # self.dataset.append(file[:file.find(\"_oracle.txt\")]+\"_\"+\"1\")            print(\"p \", p_sum/evaluation_num, \"r \", r_sum/evaluation_num, \"f1 \", f1_sum/evaluation_num)\n",
    "            print(\"p \", p_sum/evaluation_num, \"r \", r_sum/evaluation_num, \"f1 \", f1_sum/evaluation_num)\n",
    "            self.model.train()\n",
    "            return (p_sum/evaluation_num), (r_sum/evaluation_num), (f1_sum/evaluation_num)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "6Kuxu7k4Pfdt"
   },
   "outputs": [],
   "source": [
    "exp1 = Runner(config, [2,3,4], 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "id": "ttLvg8jFkYFo",
    "outputId": "c6203268-789b-4fb5-bae6-ab14f5564946"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "start to evaluate random result\n",
      "end to evaluate random result\n",
      "conv1 torch.Size([100, 2048, 100])\n",
      "conv2 torch.Size([100, 2048, 50])\n",
      "conv3 torch.Size([100, 2048, 25])\n",
      "conv4 torch.Size([100, 2048, 19])\n",
      "conv5 torch.Size([100, 2048, 10])\n",
      "conv6 torch.Size([100, 4096, 10])\n",
      "conv7 torch.Size([100, 4096, 10])\n",
      "conv8 torch.Size([100, 256, 10])\n",
      "k shape : torch.Size([100, 10, 256]) | q shape: torch.Size([100, 10, 256])\n",
      "wei shape:  torch.Size([100, 10, 10])\n",
      "k shape : torch.Size([100, 10, 256]) | q shape: torch.Size([100, 10, 256])\n",
      "wei shape:  torch.Size([100, 10, 10])\n",
      "k shape : torch.Size([10, 100, 256]) | q shape: torch.Size([10, 100, 256])\n",
      "wei shape:  torch.Size([10, 100, 100])\n",
      "deconv1 torch.Size([100, 1024, 19])\n",
      "deconv2 torch.Size([100, 1024, 200])\n",
      "similar1:  torch.Size([5, 4000, 300])\n",
      "concept1_similar.unsqueeze(1):  torch.Size([5, 1, 300])\n",
      "Loss:  tensor([6.9563], device='cuda:0', grad_fn=<AddBackward0>)\n",
      "conv1 torch.Size([100, 2048, 100])\n",
      "conv2 torch.Size([100, 2048, 50])\n",
      "conv3 torch.Size([100, 2048, 25])\n",
      "conv4 torch.Size([100, 2048, 19])\n",
      "conv5 torch.Size([100, 2048, 10])\n",
      "conv6 torch.Size([100, 4096, 10])\n",
      "conv7 torch.Size([100, 4096, 10])\n",
      "conv8 torch.Size([100, 256, 10])\n",
      "k shape : torch.Size([100, 10, 256]) | q shape: torch.Size([100, 10, 256])\n",
      "wei shape:  torch.Size([100, 10, 10])\n",
      "k shape : torch.Size([100, 10, 256]) | q shape: torch.Size([100, 10, 256])\n",
      "wei shape:  torch.Size([100, 10, 10])\n",
      "k shape : torch.Size([10, 100, 256]) | q shape: torch.Size([10, 100, 256])\n",
      "wei shape:  torch.Size([10, 100, 100])\n",
      "deconv1 torch.Size([100, 1024, 19])\n",
      "deconv2 torch.Size([100, 1024, 200])\n",
      "similar1:  torch.Size([5, 4000, 300])\n",
      "concept1_similar.unsqueeze(1):  torch.Size([5, 1, 300])\n",
      "Loss:  tensor([7.0474], device='cuda:0', grad_fn=<AddBackward0>)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.10/dist-packages/torch/autograd/graph.py:744: UserWarning: Plan failed with an OutOfMemoryError: CUDA out of memory. Tried to allocate 6.25 GiB. GPU  (Triggered internally at ../aten/src/ATen/native/cudnn/Conv_v8.cpp:924.)\n",
      "  return Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "conv1 torch.Size([100, 2048, 100])\n",
      "conv2 torch.Size([100, 2048, 50])\n",
      "conv3 torch.Size([100, 2048, 25])\n",
      "conv4 torch.Size([100, 2048, 19])\n",
      "conv5 torch.Size([100, 2048, 10])\n",
      "conv6 torch.Size([100, 4096, 10])\n",
      "conv7 torch.Size([100, 4096, 10])\n",
      "conv8 torch.Size([100, 256, 10])\n",
      "k shape : torch.Size([100, 10, 256]) | q shape: torch.Size([100, 10, 256])\n",
      "wei shape:  torch.Size([100, 10, 10])\n",
      "k shape : torch.Size([100, 10, 256]) | q shape: torch.Size([100, 10, 256])\n",
      "wei shape:  torch.Size([100, 10, 10])\n",
      "k shape : torch.Size([10, 100, 256]) | q shape: torch.Size([10, 100, 256])\n",
      "wei shape:  torch.Size([10, 100, 100])\n",
      "deconv1 torch.Size([100, 1024, 19])\n",
      "deconv2 torch.Size([100, 1024, 200])\n",
      "similar1:  torch.Size([5, 4000, 300])\n",
      "concept1_similar.unsqueeze(1):  torch.Size([5, 1, 300])\n",
      "Loss:  tensor([6.8377], device='cuda:0', grad_fn=<AddBackward0>)\n",
      "conv1 torch.Size([100, 2048, 100])\n",
      "conv2 torch.Size([100, 2048, 50])\n",
      "conv3 torch.Size([100, 2048, 25])\n",
      "conv4 torch.Size([100, 2048, 19])\n",
      "conv5 torch.Size([100, 2048, 10])\n",
      "conv6 torch.Size([100, 4096, 10])\n",
      "conv7 torch.Size([100, 4096, 10])\n",
      "conv8 torch.Size([100, 256, 10])\n",
      "k shape : torch.Size([100, 10, 256]) | q shape: torch.Size([100, 10, 256])\n",
      "wei shape:  torch.Size([100, 10, 10])\n",
      "k shape : torch.Size([100, 10, 256]) | q shape: torch.Size([100, 10, 256])\n",
      "wei shape:  torch.Size([100, 10, 10])\n",
      "k shape : torch.Size([10, 100, 256]) | q shape: torch.Size([10, 100, 256])\n",
      "wei shape:  torch.Size([10, 100, 100])\n",
      "deconv1 torch.Size([100, 1024, 19])\n",
      "deconv2 torch.Size([100, 1024, 200])\n",
      "similar1:  torch.Size([5, 4000, 300])\n",
      "concept1_similar.unsqueeze(1):  torch.Size([5, 1, 300])\n",
      "Loss:  tensor([6.7270], device='cuda:0', grad_fn=<AddBackward0>)\n",
      "epoch  1  batch  5  loss  1.3453996658325196\n",
      "conv1 torch.Size([100, 2048, 100])\n",
      "conv2 torch.Size([100, 2048, 50])\n",
      "conv3 torch.Size([100, 2048, 25])\n",
      "conv4 torch.Size([100, 2048, 19])\n",
      "conv5 torch.Size([100, 2048, 10])\n",
      "conv6 torch.Size([100, 4096, 10])\n",
      "conv7 torch.Size([100, 4096, 10])\n",
      "conv8 torch.Size([100, 256, 10])\n",
      "k shape : torch.Size([100, 10, 256]) | q shape: torch.Size([100, 10, 256])\n",
      "wei shape:  torch.Size([100, 10, 10])\n",
      "k shape : torch.Size([100, 10, 256]) | q shape: torch.Size([100, 10, 256])\n",
      "wei shape:  torch.Size([100, 10, 10])\n",
      "k shape : torch.Size([10, 100, 256]) | q shape: torch.Size([10, 100, 256])\n",
      "wei shape:  torch.Size([10, 100, 100])\n",
      "deconv1 torch.Size([100, 1024, 19])\n",
      "deconv2 torch.Size([100, 1024, 200])\n",
      "similar1:  torch.Size([5, 4000, 300])\n",
      "concept1_similar.unsqueeze(1):  torch.Size([5, 1, 300])\n",
      "Loss:  tensor([6.8268], device='cuda:0', grad_fn=<AddBackward0>)\n",
      "conv1 torch.Size([100, 2048, 100])\n",
      "conv2 torch.Size([100, 2048, 50])\n",
      "conv3 torch.Size([100, 2048, 25])\n",
      "conv4 torch.Size([100, 2048, 19])\n",
      "conv5 torch.Size([100, 2048, 10])\n",
      "conv6 torch.Size([100, 4096, 10])\n",
      "conv7 torch.Size([100, 4096, 10])\n",
      "conv8 torch.Size([100, 256, 10])\n",
      "k shape : torch.Size([100, 10, 256]) | q shape: torch.Size([100, 10, 256])\n",
      "wei shape:  torch.Size([100, 10, 10])\n",
      "k shape : torch.Size([100, 10, 256]) | q shape: torch.Size([100, 10, 256])\n",
      "wei shape:  torch.Size([100, 10, 10])\n",
      "k shape : torch.Size([10, 100, 256]) | q shape: torch.Size([10, 100, 256])\n",
      "wei shape:  torch.Size([10, 100, 100])\n",
      "deconv1 torch.Size([100, 1024, 19])\n",
      "deconv2 torch.Size([100, 1024, 200])\n",
      "similar1:  torch.Size([5, 4000, 300])\n",
      "concept1_similar.unsqueeze(1):  torch.Size([5, 1, 300])\n",
      "Loss:  tensor([6.6874], device='cuda:0', grad_fn=<AddBackward0>)\n",
      "conv1 torch.Size([100, 2048, 100])\n",
      "conv2 torch.Size([100, 2048, 50])\n",
      "conv3 torch.Size([100, 2048, 25])\n",
      "conv4 torch.Size([100, 2048, 19])\n",
      "conv5 torch.Size([100, 2048, 10])\n",
      "conv6 torch.Size([100, 4096, 10])\n",
      "conv7 torch.Size([100, 4096, 10])\n",
      "conv8 torch.Size([100, 256, 10])\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-13-d48ceca3fd09>\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mexp1\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-11-1a0ae6fea047>\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     73\u001b[0m                 \u001b[0;31m# print(f\"Concept 1: {concept1}, Concept 2: {concept2}\")\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     74\u001b[0m                 \u001b[0;31m# print(f\"features: {features}\")\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 75\u001b[0;31m                 \u001b[0mconcept1_score\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mconcept2_score\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself_attention_wei\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mquery_relevance_attention_wei\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mglobal_attention_wei\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfeatures\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mseg_len\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mconcept1\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mconcept2\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     76\u001b[0m                 \u001b[0;31m# concept1_score,concept2_score, self_attention_wei, query_relevance_attention_wei, global_attention_wei=self.model(features,seg_len,concept1,concept2)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     77\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1530\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_compiled_call_impl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# type: ignore[misc]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1531\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1532\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_impl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1533\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1534\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_call_impl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1539\u001b[0m                 \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_pre_hooks\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_hooks\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1540\u001b[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1541\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1542\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1543\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-8-51403a6d84e5>\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, batch, seg_len, concept1, concept2)\u001b[0m\n\u001b[1;32m    186\u001b[0m         \u001b[0;31m# print(\"Max Pool : \", tmp2.shape)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    187\u001b[0m         \u001b[0;31m# batch_size * max_seg_num * max_seg_length/4\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 188\u001b[0;31m         \u001b[0mattention_mask\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzeros\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mmax_seg_num\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmax_seg_length\u001b[0m\u001b[0;34m/\u001b[0m\u001b[0;36m20\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbool\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'cuda'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    189\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    190\u001b[0m             \u001b[0;31m# print(\"i: \", i)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "exp1.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 269,
     "referenced_widgets": [
      "cbcf14b1071d41be99050330c7163190",
      "d445c4c70a4e45fc86016f678dbfbc89",
      "d9975f2c972f450c81d4a9f21b83c1da",
      "75e073f8fa7d4357b07ac432d693d93d",
      "15e26a6fdbf6482b8a20edb05da9b2ad",
      "3651269ccce94a67be74ebcd7d7526cb",
      "4db13340e3a1408a86477cdecd54af00",
      "19a019befca44497a855cc607bb9f96b"
     ]
    },
    "id": "2PWsiKAopqEz",
    "outputId": "990f11c5-68ad-462b-85d5-41a36b5968d7"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cbcf14b1071d41be99050330c7163190",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(Label(value='0.001 MB of 0.001 MB uploaded\\r'), FloatProgress(value=1.0, max=1.0)))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style>\n",
       "    table.wandb td:nth-child(1) { padding: 0 10px; text-align: left ; width: auto;} td:nth-child(2) {text-align: left ; width: 100%}\n",
       "    .wandb-row { display: flex; flex-direction: row; flex-wrap: wrap; justify-content: flex-start; width: 100% }\n",
       "    .wandb-col { display: flex; flex-direction: column; flex-basis: 100%; flex: 1; padding: 10px; }\n",
       "    </style>\n",
       "<div class=\"wandb-row\"><div class=\"wandb-col\"><h3>Run history:</h3><br/><table class=\"wandb\"><tr><td>epoch</td><td>▁▁▂▂▂▃▃▄▄▄▅▅▅▆▆▇▇▇██</td></tr><tr><td>test/f1</td><td>▆▇▁▄▄▃▁▆█▇▇▇█▅█▇███▆</td></tr><tr><td>train/f1</td><td>██▆▆▄▁▁▆██████▇▇█▅██</td></tr><tr><td>train/loss</td><td>█▇▆▄▅▃▇▄▄▅▃▅▃▄▃▅▅▂▃▄▄▄▂▄▄▂▄▃▄▄▄▃▃▂▂▄▅▁▃▁</td></tr></table><br/></div><div class=\"wandb-col\"><h3>Run summary:</h3><br/><table class=\"wandb\"><tr><td>epoch</td><td>20</td></tr><tr><td>test/f1</td><td>0.45153</td></tr><tr><td>train/f1</td><td>0.36427</td></tr><tr><td>train/loss</td><td>0.86613</td></tr></table><br/></div></div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View run <strong style=\"color:#cdcd00\">pleasant-sun-10</strong> at: <a href='https://wandb.ai/nirld/FCSN_ATTENTION_FU_1/runs/fya0dup4' target=\"_blank\">https://wandb.ai/nirld/FCSN_ATTENTION_FU_1/runs/fya0dup4</a><br/> View project at: <a href='https://wandb.ai/nirld/FCSN_ATTENTION_FU_1' target=\"_blank\">https://wandb.ai/nirld/FCSN_ATTENTION_FU_1</a><br/>Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Find logs at: <code>./wandb/run-20240506_133939-fya0dup4/logs</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "wandb.finish()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "gQgPDPFvhrVy"
   },
   "outputs": [],
   "source": [
    "model_scripted = torch.jit.script(exp1.model) # Export to TorchScript\n",
    "model_scripted.save('/content/drive/MyDrive/Research/FCSN_ATTN_FU/train4_test1.pt')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Wq4dLijXmcy9"
   },
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "2TwqPW8KmK76"
   },
   "outputs": [],
   "source": [
    "# del exp1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "zoB21f04mi7_"
   },
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "VaMiP-J-S5x3"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "gpuType": "T4",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.10"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "15e26a6fdbf6482b8a20edb05da9b2ad": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "19a019befca44497a855cc607bb9f96b": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "3651269ccce94a67be74ebcd7d7526cb": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "4db13340e3a1408a86477cdecd54af00": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "75e073f8fa7d4357b07ac432d693d93d": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "cbcf14b1071d41be99050330c7163190": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "VBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "VBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "VBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_d445c4c70a4e45fc86016f678dbfbc89",
       "IPY_MODEL_d9975f2c972f450c81d4a9f21b83c1da"
      ],
      "layout": "IPY_MODEL_75e073f8fa7d4357b07ac432d693d93d"
     }
    },
    "d445c4c70a4e45fc86016f678dbfbc89": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "LabelModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "LabelModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "LabelView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_15e26a6fdbf6482b8a20edb05da9b2ad",
      "placeholder": "​",
      "style": "IPY_MODEL_3651269ccce94a67be74ebcd7d7526cb",
      "value": "1.971 MB of 1.971 MB uploaded\r"
     }
    },
    "d9975f2c972f450c81d4a9f21b83c1da": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_4db13340e3a1408a86477cdecd54af00",
      "max": 1,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_19a019befca44497a855cc607bb9f96b",
      "value": 1
     }
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}