experiments.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Demo\n",
    "Demo experiment for quick start."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=32, out_features=256, bias=True)\n",
      "      (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wk): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=4, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=512, bias=True)\n",
      "          (linear2): Linear(in_features=512, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 145515\n",
      "2023-08-16 17:20:51.243172: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n",
      "2023-08-16 17:20:51.288234: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2023-08-16 17:20:52.125030: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
      "===================Running===================\n",
      "ground_truth:data/fMRI/sim15_gt_processed.csv\n",
      "===================Results===================\n",
      "0 causes 0 with a delay of 1 time steps.\n",
      "2 causes 0 with a delay of 1 time steps.\n",
      "4 causes 0 with a delay of 1 time steps.\n",
      "1 causes 1 with a delay of 1 time steps.\n",
      "2 causes 1 with a delay of 15 time steps.\n",
      "2 causes 2 with a delay of 1 time steps.\n",
      "3 causes 2 with a delay of 1 time steps.\n",
      "4 causes 2 with a delay of 1 time steps.\n",
      "3 causes 3 with a delay of 1 time steps.\n",
      "4 causes 3 with a delay of 1 time steps.\n",
      "4 causes 4 with a delay of 1 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 0\n",
      "Total True Positives': 11\n",
      "Total False Negatives: 1\n",
      "Total Direct False Positives: 2\n",
      "Total Direct True Positives: 9\n",
      "TPs': ['0->0', '2->0', '4->0', '1->1', '2->1', '2->2', '3->2', '4->2', '3->3', '4->3', '4->4']\n",
      "FPs': []\n",
      "TPs direct: ['0->0', '4->0', '1->1', '2->1', '2->2', '3->2', '3->3', '4->3', '4->4']\n",
      "FPs direct: ['2->0', '4->2']\n",
      "FNs: ['1->0']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 1.0\n",
      "Recall': 0.9166666666666666\n",
      "F1' score: 0.9565217391304348\n",
      "(includes only direct causal relationships)\n",
      "Precision: 0.8181818181818182\n",
      "Recall: 0.9\n",
      "F1 score: 0.8571428571428572\n",
      "Percentage of delays that are correctly discovered: 88.88888888888889%\n",
      "===================Summary===================\n",
      "\t   Precision'   Recall'       F1'  Precision  Recall        F1       PoD\n",
      "\t1         1.0  0.916667  0.956522   0.818182     0.9  0.857143  0.888889\n"
     ]
    }
   ],
   "source": [
    "! python runner.py -c config/config_fMRI.json -t demo"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Experiments\n",
    "The following commands are applied sequentially to the dataset:\n",
    "- Diamond\n",
    "- Mediator\n",
    "- V-structure\n",
    "- Fork\n",
    "- Lorenz\n",
    "- fMRI"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=16, out_features=256, bias=True)\n",
      "      (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wk): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=4, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=256, bias=True)\n",
      "          (linear2): Linear(in_features=256, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 138375\n",
      "2023-08-16 18:16:40.727919: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n",
      "2023-08-16 18:16:40.773867: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2023-08-16 18:16:41.479733: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
      "===================Running===================\n",
      "ground_truth:data/basic/diamond/groundtruth.csv\n",
      "===================Results===================\n",
      "V1 causes V1 with a delay of 1 time steps.\n",
      "V2 causes V2 with a delay of 1 time steps.\n",
      "V3 causes V3 with a delay of 7 time steps.\n",
      "V3 causes V4 with a delay of 6 time steps.\n",
      "V4 causes V4 with a delay of 1 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 0\n",
      "Total True Positives': 5\n",
      "Total False Negatives: 3\n",
      "Total Direct False Positives: 0\n",
      "Total Direct True Positives: 5\n",
      "TPs': ['0->0', '1->1', '2->2', '2->3', '3->3']\n",
      "FPs': []\n",
      "TPs direct: ['0->0', '1->1', '2->2', '2->3', '3->3']\n",
      "FPs direct: []\n",
      "FNs: ['0->1', '0->2', '1->3']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 1.0\n",
      "Recall': 0.625\n",
      "F1' score: 0.7692307692307693\n",
      "(includes only direct causal relationships)\n",
      "Precision: 1.0\n",
      "Recall: 0.625\n",
      "F1 score: 0.7692307692307693\n",
      "Percentage of delays that are correctly discovered: 60.0%\n",
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=16, out_features=256, bias=True)\n",
      "      (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wk): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=4, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=256, bias=True)\n",
      "          (linear2): Linear(in_features=256, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 138375\n",
      "===================Running===================\n",
      "ground_truth:data/basic/diamond/groundtruth.csv\n",
      "===================Results===================\n",
      "V1 causes V1 with a delay of 1 time steps.\n",
      "V2 causes V2 with a delay of 1 time steps.\n",
      "V3 causes V3 with a delay of 1 time steps.\n",
      "V4 causes V4 with a delay of 1 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 0\n",
      "Total True Positives': 4\n",
      "Total False Negatives: 4\n",
      "Total Direct False Positives: 0\n",
      "Total Direct True Positives: 4\n",
      "TPs': ['0->0', '1->1', '2->2', '3->3']\n",
      "FPs': []\n",
      "TPs direct: ['0->0', '1->1', '2->2', '3->3']\n",
      "FPs direct: []\n",
      "FNs: ['0->1', '0->2', '1->3', '2->3']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 1.0\n",
      "Recall': 0.5\n",
      "F1' score: 0.6666666666666666\n",
      "(includes only direct causal relationships)\n",
      "Precision: 1.0\n",
      "Recall: 0.5\n",
      "F1 score: 0.6666666666666666\n",
      "Percentage of delays that are correctly discovered: 100.0%\n",
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=16, out_features=256, bias=True)\n",
      "      (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wk): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=4, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=256, bias=True)\n",
      "          (linear2): Linear(in_features=256, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 138375\n",
      "===================Running===================\n",
      "ground_truth:data/basic/diamond/groundtruth.csv\n",
      "===================Results===================\n",
      "V1 causes V1 with a delay of 10 time steps.\n",
      "V2 causes V2 with a delay of 1 time steps.\n",
      "V3 causes V3 with a delay of 7 time steps.\n",
      "V4 causes V4 with a delay of 1 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 0\n",
      "Total True Positives': 4\n",
      "Total False Negatives: 4\n",
      "Total Direct False Positives: 0\n",
      "Total Direct True Positives: 4\n",
      "TPs': ['0->0', '1->1', '2->2', '3->3']\n",
      "FPs': []\n",
      "TPs direct: ['0->0', '1->1', '2->2', '3->3']\n",
      "FPs direct: []\n",
      "FNs: ['0->1', '0->2', '1->3', '2->3']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 1.0\n",
      "Recall': 0.5\n",
      "F1' score: 0.6666666666666666\n",
      "(includes only direct causal relationships)\n",
      "Precision: 1.0\n",
      "Recall: 0.5\n",
      "F1 score: 0.6666666666666666\n",
      "Percentage of delays that are correctly discovered: 50.0%\n",
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=16, out_features=256, bias=True)\n",
      "      (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wk): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=4, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=256, bias=True)\n",
      "          (linear2): Linear(in_features=256, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 138375\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "===================Running===================\n",
      "ground_truth:data/basic/diamond/groundtruth.csv\n",
      "===================Results===================\n",
      "V1 causes V1 with a delay of 1 time steps.\n",
      "V2 causes V2 with a delay of 1 time steps.\n",
      "V3 causes V3 with a delay of 1 time steps.\n",
      "V4 causes V4 with a delay of 8 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 0\n",
      "Total True Positives': 4\n",
      "Total False Negatives: 4\n",
      "Total Direct False Positives: 0\n",
      "Total Direct True Positives: 4\n",
      "TPs': ['0->0', '1->1', '2->2', '3->3']\n",
      "FPs': []\n",
      "TPs direct: ['0->0', '1->1', '2->2', '3->3']\n",
      "FPs direct: []\n",
      "FNs: ['0->1', '0->2', '1->3', '2->3']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 1.0\n",
      "Recall': 0.5\n",
      "F1' score: 0.6666666666666666\n",
      "(includes only direct causal relationships)\n",
      "Precision: 1.0\n",
      "Recall: 0.5\n",
      "F1 score: 0.6666666666666666\n",
      "Percentage of delays that are correctly discovered: 75.0%\n",
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=16, out_features=256, bias=True)\n",
      "      (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wk): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=4, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=256, bias=True)\n",
      "          (linear2): Linear(in_features=256, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 138375\n",
      "===================Running===================\n",
      "ground_truth:data/basic/diamond/groundtruth.csv\n",
      "===================Results===================\n",
      "V1 causes V1 with a delay of 1 time steps.\n",
      "V2 causes V2 with a delay of 1 time steps.\n",
      "V3 causes V3 with a delay of 1 time steps.\n",
      "V4 causes V4 with a delay of 1 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 0\n",
      "Total True Positives': 4\n",
      "Total False Negatives: 4\n",
      "Total Direct False Positives: 0\n",
      "Total Direct True Positives: 4\n",
      "TPs': ['0->0', '1->1', '2->2', '3->3']\n",
      "FPs': []\n",
      "TPs direct: ['0->0', '1->1', '2->2', '3->3']\n",
      "FPs direct: []\n",
      "FNs: ['0->1', '0->2', '1->3', '2->3']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 1.0\n",
      "Recall': 0.5\n",
      "F1' score: 0.6666666666666666\n",
      "(includes only direct causal relationships)\n",
      "Precision: 1.0\n",
      "Recall: 0.5\n",
      "F1 score: 0.6666666666666666\n",
      "Percentage of delays that are correctly discovered: 100.0%\n",
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=16, out_features=256, bias=True)\n",
      "      (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wk): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=4, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=256, bias=True)\n",
      "          (linear2): Linear(in_features=256, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 138375\n",
      "===================Running===================\n",
      "ground_truth:data/basic/diamond/groundtruth.csv\n",
      "===================Results===================\n",
      "V1 causes V1 with a delay of 1 time steps.\n",
      "V2 causes V2 with a delay of 10 time steps.\n",
      "V3 causes V3 with a delay of 1 time steps.\n",
      "V4 causes V4 with a delay of 5 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 0\n",
      "Total True Positives': 4\n",
      "Total False Negatives: 4\n",
      "Total Direct False Positives: 0\n",
      "Total Direct True Positives: 4\n",
      "TPs': ['0->0', '1->1', '2->2', '3->3']\n",
      "FPs': []\n",
      "TPs direct: ['0->0', '1->1', '2->2', '3->3']\n",
      "FPs direct: []\n",
      "FNs: ['0->1', '0->2', '1->3', '2->3']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 1.0\n",
      "Recall': 0.5\n",
      "F1' score: 0.6666666666666666\n",
      "(includes only direct causal relationships)\n",
      "Precision: 1.0\n",
      "Recall: 0.5\n",
      "F1 score: 0.6666666666666666\n",
      "Percentage of delays that are correctly discovered: 50.0%\n",
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=16, out_features=256, bias=True)\n",
      "      (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wk): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=4, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=256, bias=True)\n",
      "          (linear2): Linear(in_features=256, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 138375\n",
      "===================Running===================\n",
      "ground_truth:data/basic/diamond/groundtruth.csv\n",
      "===================Results===================\n",
      "V2 causes V1 with a delay of 0 time steps.\n",
      "V2 causes V2 with a delay of 1 time steps.\n",
      "V3 causes V3 with a delay of 1 time steps.\n",
      "V4 causes V4 with a delay of 2 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 1\n",
      "Total True Positives': 3\n",
      "Total False Negatives: 5\n",
      "Total Direct False Positives: 1\n",
      "Total Direct True Positives: 3\n",
      "TPs': ['1->1', '2->2', '3->3']\n",
      "FPs': ['1->0']\n",
      "TPs direct: ['1->1', '2->2', '3->3']\n",
      "FPs direct: ['1->0']\n",
      "FNs: ['0->0', '0->1', '0->2', '1->3', '2->3']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 0.75\n",
      "Recall': 0.375\n",
      "F1' score: 0.5\n",
      "(includes only direct causal relationships)\n",
      "Precision: 0.75\n",
      "Recall: 0.375\n",
      "F1 score: 0.5\n",
      "Percentage of delays that are correctly discovered: 66.66666666666666%\n",
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=16, out_features=256, bias=True)\n",
      "      (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wk): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=4, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=256, bias=True)\n",
      "          (linear2): Linear(in_features=256, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 138375\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "===================Running===================\n",
      "ground_truth:data/basic/diamond/groundtruth.csv\n",
      "===================Results===================\n",
      "V1 causes V1 with a delay of 3 time steps.\n",
      "V2 causes V2 with a delay of 1 time steps.\n",
      "V3 causes V3 with a delay of 1 time steps.\n",
      "V4 causes V4 with a delay of 1 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 0\n",
      "Total True Positives': 4\n",
      "Total False Negatives: 4\n",
      "Total Direct False Positives: 0\n",
      "Total Direct True Positives: 4\n",
      "TPs': ['0->0', '1->1', '2->2', '3->3']\n",
      "FPs': []\n",
      "TPs direct: ['0->0', '1->1', '2->2', '3->3']\n",
      "FPs direct: []\n",
      "FNs: ['0->1', '0->2', '1->3', '2->3']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 1.0\n",
      "Recall': 0.5\n",
      "F1' score: 0.6666666666666666\n",
      "(includes only direct causal relationships)\n",
      "Precision: 1.0\n",
      "Recall: 0.5\n",
      "F1 score: 0.6666666666666666\n",
      "Percentage of delays that are correctly discovered: 75.0%\n",
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=16, out_features=256, bias=True)\n",
      "      (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wk): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=4, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=256, bias=True)\n",
      "          (linear2): Linear(in_features=256, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 138375\n",
      "===================Running===================\n",
      "ground_truth:data/basic/diamond/groundtruth.csv\n",
      "===================Results===================\n",
      "V1 causes V1 with a delay of 1 time steps.\n",
      "V1 causes V2 with a delay of 0 time steps.\n",
      "V2 causes V2 with a delay of 4 time steps.\n",
      "V1 causes V3 with a delay of 0 time steps.\n",
      "V3 causes V3 with a delay of 1 time steps.\n",
      "V1 causes V4 with a delay of 0 time steps.\n",
      "V4 causes V4 with a delay of 1 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 0\n",
      "Total True Positives': 7\n",
      "Total False Negatives: 2\n",
      "Total Direct False Positives: 1\n",
      "Total Direct True Positives: 6\n",
      "TPs': ['0->0', '0->1', '1->1', '0->2', '2->2', '0->3', '3->3']\n",
      "FPs': []\n",
      "TPs direct: ['0->0', '0->1', '1->1', '0->2', '2->2', '3->3']\n",
      "FPs direct: ['0->3']\n",
      "FNs: ['1->3', '2->3']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 1.0\n",
      "Recall': 0.7777777777777778\n",
      "F1' score: 0.8750000000000001\n",
      "(includes only direct causal relationships)\n",
      "Precision: 0.8571428571428571\n",
      "Recall: 0.75\n",
      "F1 score: 0.7999999999999999\n",
      "Percentage of delays that are correctly discovered: 60.0%\n",
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=16, out_features=256, bias=True)\n",
      "      (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wk): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=4, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=256, bias=True)\n",
      "          (linear2): Linear(in_features=256, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 138375\n",
      "===================Running===================\n",
      "ground_truth:data/basic/diamond/groundtruth.csv\n",
      "===================Results===================\n",
      "V1 causes V1 with a delay of 1 time steps.\n",
      "V2 causes V2 with a delay of 1 time steps.\n",
      "V3 causes V3 with a delay of 1 time steps.\n",
      "V2 causes V4 with a delay of 1 time steps.\n",
      "V4 causes V4 with a delay of 1 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 0\n",
      "Total True Positives': 5\n",
      "Total False Negatives: 3\n",
      "Total Direct False Positives: 0\n",
      "Total Direct True Positives: 5\n",
      "TPs': ['0->0', '1->1', '2->2', '1->3', '3->3']\n",
      "FPs': []\n",
      "TPs direct: ['0->0', '1->1', '2->2', '1->3', '3->3']\n",
      "FPs direct: []\n",
      "FNs: ['0->1', '0->2', '2->3']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 1.0\n",
      "Recall': 0.625\n",
      "F1' score: 0.7692307692307693\n",
      "(includes only direct causal relationships)\n",
      "Precision: 1.0\n",
      "Recall: 0.625\n",
      "F1 score: 0.7692307692307693\n",
      "Percentage of delays that are correctly discovered: 100.0%\n",
      "===================Summary===================\n",
      "\t    Precision'   Recall'       F1'  Precision  Recall        F1       PoD\n",
      "\t1         1.00  0.625000  0.769231   1.000000   0.625  0.769231  0.600000\n",
      "\t2         1.00  0.500000  0.666667   1.000000   0.500  0.666667  1.000000\n",
      "\t3         1.00  0.500000  0.666667   1.000000   0.500  0.666667  0.500000\n",
      "\t4         1.00  0.500000  0.666667   1.000000   0.500  0.666667  0.750000\n",
      "\t5         1.00  0.500000  0.666667   1.000000   0.500  0.666667  1.000000\n",
      "\t6         1.00  0.500000  0.666667   1.000000   0.500  0.666667  0.500000\n",
      "\t7         0.75  0.375000  0.500000   0.750000   0.375  0.500000  0.666667\n",
      "\t8         1.00  0.500000  0.666667   1.000000   0.500  0.666667  0.750000\n",
      "\t9         1.00  0.777778  0.875000   0.857143   0.750  0.800000  0.600000\n",
      "\t10        1.00  0.625000  0.769231   1.000000   0.625  0.769231  1.000000\n"
     ]
    }
   ],
   "source": [
    "! python runner.py -c config/config_basic_diamond_mediator.json -t diamond"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=16, out_features=256, bias=True)\n",
      "      (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wk): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=4, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=256, bias=True)\n",
      "          (linear2): Linear(in_features=256, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 137899\n",
      "2023-08-16 19:44:12.225385: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n",
      "2023-08-16 19:44:12.271912: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2023-08-16 19:44:12.978333: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
      "===================Running===================\n",
      "ground_truth:data/basic/mediator/groundtruth.csv\n",
      "===================Results===================\n",
      "V1 causes V1 with a delay of 1 time steps.\n",
      "V2 causes V2 with a delay of 1 time steps.\n",
      "V3 causes V3 with a delay of 1 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 0\n",
      "Total True Positives': 3\n",
      "Total False Negatives: 3\n",
      "Total Direct False Positives: 0\n",
      "Total Direct True Positives: 3\n",
      "TPs': ['0->0', '1->1', '2->2']\n",
      "FPs': []\n",
      "TPs direct: ['0->0', '1->1', '2->2']\n",
      "FPs direct: []\n",
      "FNs: ['0->1', '0->2', '1->2']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 1.0\n",
      "Recall': 0.5\n",
      "F1' score: 0.6666666666666666\n",
      "(includes only direct causal relationships)\n",
      "Precision: 1.0\n",
      "Recall: 0.5\n",
      "F1 score: 0.6666666666666666\n",
      "Percentage of delays that are correctly discovered: 100.0%\n",
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=16, out_features=256, bias=True)\n",
      "      (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wk): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=4, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=256, bias=True)\n",
      "          (linear2): Linear(in_features=256, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 137899\n",
      "===================Running===================\n",
      "ground_truth:data/basic/mediator/groundtruth.csv\n",
      "===================Results===================\n",
      "V1 causes V1 with a delay of 1 time steps.\n",
      "V2 causes V2 with a delay of 8 time steps.\n",
      "V3 causes V3 with a delay of 1 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 0\n",
      "Total True Positives': 3\n",
      "Total False Negatives: 3\n",
      "Total Direct False Positives: 0\n",
      "Total Direct True Positives: 3\n",
      "TPs': ['0->0', '1->1', '2->2']\n",
      "FPs': []\n",
      "TPs direct: ['0->0', '1->1', '2->2']\n",
      "FPs direct: []\n",
      "FNs: ['0->1', '0->2', '1->2']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 1.0\n",
      "Recall': 0.5\n",
      "F1' score: 0.6666666666666666\n",
      "(includes only direct causal relationships)\n",
      "Precision: 1.0\n",
      "Recall: 0.5\n",
      "F1 score: 0.6666666666666666\n",
      "Percentage of delays that are correctly discovered: 66.66666666666666%\n",
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=16, out_features=256, bias=True)\n",
      "      (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wk): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=4, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=256, bias=True)\n",
      "          (linear2): Linear(in_features=256, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 137899\n",
      "===================Running===================\n",
      "ground_truth:data/basic/mediator/groundtruth.csv\n",
      "===================Results===================\n",
      "V1 causes V1 with a delay of 1 time steps.\n",
      "V1 causes V2 with a delay of 14 time steps.\n",
      "V2 causes V2 with a delay of 5 time steps.\n",
      "V2 causes V3 with a delay of 1 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 0\n",
      "Total True Positives': 4\n",
      "Total False Negatives: 2\n",
      "Total Direct False Positives: 0\n",
      "Total Direct True Positives: 4\n",
      "TPs': ['0->0', '0->1', '1->1', '1->2']\n",
      "FPs': []\n",
      "TPs direct: ['0->0', '0->1', '1->1', '1->2']\n",
      "FPs direct: []\n",
      "FNs: ['0->2', '2->2']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 1.0\n",
      "Recall': 0.6666666666666666\n",
      "F1' score: 0.8\n",
      "(includes only direct causal relationships)\n",
      "Precision: 1.0\n",
      "Recall: 0.6666666666666666\n",
      "F1 score: 0.8\n",
      "Percentage of delays that are correctly discovered: 50.0%\n",
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=16, out_features=256, bias=True)\n",
      "      (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wk): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=4, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=256, bias=True)\n",
      "          (linear2): Linear(in_features=256, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 137899\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "===================Running===================\n",
      "ground_truth:data/basic/mediator/groundtruth.csv\n",
      "===================Results===================\n",
      "V1 causes V1 with a delay of 4 time steps.\n",
      "V1 causes V2 with a delay of 7 time steps.\n",
      "V3 causes V3 with a delay of 1 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 0\n",
      "Total True Positives': 3\n",
      "Total False Negatives: 3\n",
      "Total Direct False Positives: 0\n",
      "Total Direct True Positives: 3\n",
      "TPs': ['0->0', '0->1', '2->2']\n",
      "FPs': []\n",
      "TPs direct: ['0->0', '0->1', '2->2']\n",
      "FPs direct: []\n",
      "FNs: ['1->1', '0->2', '1->2']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 1.0\n",
      "Recall': 0.5\n",
      "F1' score: 0.6666666666666666\n",
      "(includes only direct causal relationships)\n",
      "Precision: 1.0\n",
      "Recall: 0.5\n",
      "F1 score: 0.6666666666666666\n",
      "Percentage of delays that are correctly discovered: 33.33333333333333%\n",
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=16, out_features=256, bias=True)\n",
      "      (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wk): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=4, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=256, bias=True)\n",
      "          (linear2): Linear(in_features=256, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 137899\n",
      "===================Running===================\n",
      "ground_truth:data/basic/mediator/groundtruth.csv\n",
      "===================Results===================\n",
      "V1 causes V1 with a delay of 2 time steps.\n",
      "V1 causes V2 with a delay of 2 time steps.\n",
      "V2 causes V2 with a delay of 2 time steps.\n",
      "V3 causes V3 with a delay of 2 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 0\n",
      "Total True Positives': 4\n",
      "Total False Negatives: 2\n",
      "Total Direct False Positives: 0\n",
      "Total Direct True Positives: 4\n",
      "TPs': ['0->0', '0->1', '1->1', '2->2']\n",
      "FPs': []\n",
      "TPs direct: ['0->0', '0->1', '1->1', '2->2']\n",
      "FPs direct: []\n",
      "FNs: ['0->2', '1->2']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 1.0\n",
      "Recall': 0.6666666666666666\n",
      "F1' score: 0.8\n",
      "(includes only direct causal relationships)\n",
      "Precision: 1.0\n",
      "Recall: 0.6666666666666666\n",
      "F1 score: 0.8\n",
      "Percentage of delays that are correctly discovered: 0.0%\n",
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=16, out_features=256, bias=True)\n",
      "      (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wk): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=4, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=256, bias=True)\n",
      "          (linear2): Linear(in_features=256, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 137899\n",
      "===================Running===================\n",
      "ground_truth:data/basic/mediator/groundtruth.csv\n",
      "===================Results===================\n",
      "V1 causes V1 with a delay of 1 time steps.\n",
      "V2 causes V2 with a delay of 1 time steps.\n",
      "V2 causes V3 with a delay of 14 time steps.\n",
      "V3 causes V3 with a delay of 1 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 0\n",
      "Total True Positives': 4\n",
      "Total False Negatives: 2\n",
      "Total Direct False Positives: 0\n",
      "Total Direct True Positives: 4\n",
      "TPs': ['0->0', '1->1', '1->2', '2->2']\n",
      "FPs': []\n",
      "TPs direct: ['0->0', '1->1', '1->2', '2->2']\n",
      "FPs direct: []\n",
      "FNs: ['0->1', '0->2']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 1.0\n",
      "Recall': 0.6666666666666666\n",
      "F1' score: 0.8\n",
      "(includes only direct causal relationships)\n",
      "Precision: 1.0\n",
      "Recall: 0.6666666666666666\n",
      "F1 score: 0.8\n",
      "Percentage of delays that are correctly discovered: 75.0%\n",
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=16, out_features=256, bias=True)\n",
      "      (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wk): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=4, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=256, bias=True)\n",
      "          (linear2): Linear(in_features=256, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 137899\n",
      "===================Running===================\n",
      "ground_truth:data/basic/mediator/groundtruth.csv\n",
      "===================Results===================\n",
      "V1 causes V1 with a delay of 1 time steps.\n",
      "V2 causes V2 with a delay of 1 time steps.\n",
      "V3 causes V3 with a delay of 1 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 0\n",
      "Total True Positives': 3\n",
      "Total False Negatives: 3\n",
      "Total Direct False Positives: 0\n",
      "Total Direct True Positives: 3\n",
      "TPs': ['0->0', '1->1', '2->2']\n",
      "FPs': []\n",
      "TPs direct: ['0->0', '1->1', '2->2']\n",
      "FPs direct: []\n",
      "FNs: ['0->1', '0->2', '1->2']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 1.0\n",
      "Recall': 0.5\n",
      "F1' score: 0.6666666666666666\n",
      "(includes only direct causal relationships)\n",
      "Precision: 1.0\n",
      "Recall: 0.5\n",
      "F1 score: 0.6666666666666666\n",
      "Percentage of delays that are correctly discovered: 100.0%\n",
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=16, out_features=256, bias=True)\n",
      "      (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wk): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=4, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=256, bias=True)\n",
      "          (linear2): Linear(in_features=256, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 137899\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "===================Running===================\n",
      "ground_truth:data/basic/mediator/groundtruth.csv\n",
      "===================Results===================\n",
      "V1 causes V1 with a delay of 2 time steps.\n",
      "V2 causes V2 with a delay of 4 time steps.\n",
      "V3 causes V3 with a delay of 2 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 0\n",
      "Total True Positives': 3\n",
      "Total False Negatives: 3\n",
      "Total Direct False Positives: 0\n",
      "Total Direct True Positives: 3\n",
      "TPs': ['0->0', '1->1', '2->2']\n",
      "FPs': []\n",
      "TPs direct: ['0->0', '1->1', '2->2']\n",
      "FPs direct: []\n",
      "FNs: ['0->1', '0->2', '1->2']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 1.0\n",
      "Recall': 0.5\n",
      "F1' score: 0.6666666666666666\n",
      "(includes only direct causal relationships)\n",
      "Precision: 1.0\n",
      "Recall: 0.5\n",
      "F1 score: 0.6666666666666666\n",
      "Percentage of delays that are correctly discovered: 0.0%\n",
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=16, out_features=256, bias=True)\n",
      "      (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wk): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=4, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=256, bias=True)\n",
      "          (linear2): Linear(in_features=256, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 137899\n",
      "===================Running===================\n",
      "ground_truth:data/basic/mediator/groundtruth.csv\n",
      "===================Results===================\n",
      "V1 causes V1 with a delay of 1 time steps.\n",
      "V2 causes V2 with a delay of 1 time steps.\n",
      "V3 causes V3 with a delay of 1 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 0\n",
      "Total True Positives': 3\n",
      "Total False Negatives: 3\n",
      "Total Direct False Positives: 0\n",
      "Total Direct True Positives: 3\n",
      "TPs': ['0->0', '1->1', '2->2']\n",
      "FPs': []\n",
      "TPs direct: ['0->0', '1->1', '2->2']\n",
      "FPs direct: []\n",
      "FNs: ['0->1', '0->2', '1->2']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 1.0\n",
      "Recall': 0.5\n",
      "F1' score: 0.6666666666666666\n",
      "(includes only direct causal relationships)\n",
      "Precision: 1.0\n",
      "Recall: 0.5\n",
      "F1 score: 0.6666666666666666\n",
      "Percentage of delays that are correctly discovered: 100.0%\n",
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=16, out_features=256, bias=True)\n",
      "      (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wk): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=4, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=256, bias=True)\n",
      "          (linear2): Linear(in_features=256, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 137899\n",
      "===================Running===================\n",
      "ground_truth:data/basic/mediator/groundtruth.csv\n",
      "===================Results===================\n",
      "V1 causes V1 with a delay of 1 time steps.\n",
      "V2 causes V2 with a delay of 1 time steps.\n",
      "V3 causes V3 with a delay of 1 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 0\n",
      "Total True Positives': 3\n",
      "Total False Negatives: 3\n",
      "Total Direct False Positives: 0\n",
      "Total Direct True Positives: 3\n",
      "TPs': ['0->0', '1->1', '2->2']\n",
      "FPs': []\n",
      "TPs direct: ['0->0', '1->1', '2->2']\n",
      "FPs direct: []\n",
      "FNs: ['0->1', '0->2', '1->2']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 1.0\n",
      "Recall': 0.5\n",
      "F1' score: 0.6666666666666666\n",
      "(includes only direct causal relationships)\n",
      "Precision: 1.0\n",
      "Recall: 0.5\n",
      "F1 score: 0.6666666666666666\n",
      "Percentage of delays that are correctly discovered: 100.0%\n",
      "===================Summary===================\n",
      "\t    Precision'   Recall'       F1'  Precision    Recall        F1       PoD\n",
      "\t1          1.0  0.500000  0.666667        1.0  0.500000  0.666667  1.000000\n",
      "\t2          1.0  0.500000  0.666667        1.0  0.500000  0.666667  0.666667\n",
      "\t3          1.0  0.666667  0.800000        1.0  0.666667  0.800000  0.500000\n",
      "\t4          1.0  0.500000  0.666667        1.0  0.500000  0.666667  0.333333\n",
      "\t5          1.0  0.666667  0.800000        1.0  0.666667  0.800000  0.000000\n",
      "\t6          1.0  0.666667  0.800000        1.0  0.666667  0.800000  0.750000\n",
      "\t7          1.0  0.500000  0.666667        1.0  0.500000  0.666667  1.000000\n",
      "\t8          1.0  0.500000  0.666667        1.0  0.500000  0.666667  0.000000\n",
      "\t9          1.0  0.500000  0.666667        1.0  0.500000  0.666667  1.000000\n",
      "\t10         1.0  0.500000  0.666667        1.0  0.500000  0.666667  1.000000\n"
     ]
    }
   ],
   "source": [
    "! python runner.py -c config/config_basic_diamond_mediator.json -t mediator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=16, out_features=256, bias=True)\n",
      "      (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wk): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=4, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=256, bias=True)\n",
      "          (linear2): Linear(in_features=256, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 137899\n",
      "2023-08-16 20:26:05.077262: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n",
      "2023-08-16 20:26:05.124327: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2023-08-16 20:26:05.980181: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
      "===================Running===================\n",
      "ground_truth:data/basic/v/groundtruth.csv\n",
      "===================Results===================\n",
      "V1 causes V1 with a delay of 1 time steps.\n",
      "V2 causes V2 with a delay of 1 time steps.\n",
      "V3 causes V3 with a delay of 1 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 0\n",
      "Total True Positives': 3\n",
      "Total False Negatives: 2\n",
      "Total Direct False Positives: 0\n",
      "Total Direct True Positives: 3\n",
      "TPs': ['0->0', '1->1', '2->2']\n",
      "FPs': []\n",
      "TPs direct: ['0->0', '1->1', '2->2']\n",
      "FPs direct: []\n",
      "FNs: ['0->2', '1->2']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 1.0\n",
      "Recall': 0.6\n",
      "F1' score: 0.7499999999999999\n",
      "(includes only direct causal relationships)\n",
      "Precision: 1.0\n",
      "Recall: 0.6\n",
      "F1 score: 0.7499999999999999\n",
      "Percentage of delays that are correctly discovered: 100.0%\n",
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=16, out_features=256, bias=True)\n",
      "      (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wk): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=4, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=256, bias=True)\n",
      "          (linear2): Linear(in_features=256, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 137899\n",
      "===================Running===================\n",
      "ground_truth:data/basic/v/groundtruth.csv\n",
      "===================Results===================\n",
      "V1 causes V1 with a delay of 11 time steps.\n",
      "V2 causes V2 with a delay of 1 time steps.\n",
      "V3 causes V3 with a delay of 1 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 0\n",
      "Total True Positives': 3\n",
      "Total False Negatives: 2\n",
      "Total Direct False Positives: 0\n",
      "Total Direct True Positives: 3\n",
      "TPs': ['0->0', '1->1', '2->2']\n",
      "FPs': []\n",
      "TPs direct: ['0->0', '1->1', '2->2']\n",
      "FPs direct: []\n",
      "FNs: ['0->2', '1->2']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 1.0\n",
      "Recall': 0.6\n",
      "F1' score: 0.7499999999999999\n",
      "(includes only direct causal relationships)\n",
      "Precision: 1.0\n",
      "Recall: 0.6\n",
      "F1 score: 0.7499999999999999\n",
      "Percentage of delays that are correctly discovered: 66.66666666666666%\n",
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=16, out_features=256, bias=True)\n",
      "      (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wk): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=4, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=256, bias=True)\n",
      "          (linear2): Linear(in_features=256, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 137899\n",
      "===================Running===================\n",
      "ground_truth:data/basic/v/groundtruth.csv\n",
      "===================Results===================\n",
      "V1 causes V1 with a delay of 1 time steps.\n",
      "V2 causes V2 with a delay of 1 time steps.\n",
      "V3 causes V3 with a delay of 1 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 0\n",
      "Total True Positives': 3\n",
      "Total False Negatives: 2\n",
      "Total Direct False Positives: 0\n",
      "Total Direct True Positives: 3\n",
      "TPs': ['0->0', '1->1', '2->2']\n",
      "FPs': []\n",
      "TPs direct: ['0->0', '1->1', '2->2']\n",
      "FPs direct: []\n",
      "FNs: ['0->2', '1->2']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 1.0\n",
      "Recall': 0.6\n",
      "F1' score: 0.7499999999999999\n",
      "(includes only direct causal relationships)\n",
      "Precision: 1.0\n",
      "Recall: 0.6\n",
      "F1 score: 0.7499999999999999\n",
      "Percentage of delays that are correctly discovered: 100.0%\n",
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=16, out_features=256, bias=True)\n",
      "      (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wk): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=4, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=256, bias=True)\n",
      "          (linear2): Linear(in_features=256, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 137899\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "===================Running===================\n",
      "ground_truth:data/basic/v/groundtruth.csv\n",
      "===================Results===================\n",
      "V1 causes V1 with a delay of 8 time steps.\n",
      "V2 causes V1 with a delay of 2 time steps.\n",
      "V1 causes V2 with a delay of 0 time steps.\n",
      "V2 causes V2 with a delay of 8 time steps.\n",
      "V1 causes V3 with a delay of 4 time steps.\n",
      "V3 causes V3 with a delay of 4 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 2\n",
      "Total True Positives': 4\n",
      "Total False Negatives: 1\n",
      "Total Direct False Positives: 2\n",
      "Total Direct True Positives: 4\n",
      "TPs': ['0->0', '1->1', '0->2', '2->2']\n",
      "FPs': ['1->0', '0->1']\n",
      "TPs direct: ['0->0', '1->1', '0->2', '2->2']\n",
      "FPs direct: ['1->0', '0->1']\n",
      "FNs: ['1->2']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 0.6666666666666666\n",
      "Recall': 0.8\n",
      "F1' score: 0.7272727272727272\n",
      "(includes only direct causal relationships)\n",
      "Precision: 0.6666666666666666\n",
      "Recall: 0.8\n",
      "F1 score: 0.7272727272727272\n",
      "Percentage of delays that are correctly discovered: 0.0%\n",
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=16, out_features=256, bias=True)\n",
      "      (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wk): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=4, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=256, bias=True)\n",
      "          (linear2): Linear(in_features=256, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 137899\n",
      "===================Running===================\n",
      "ground_truth:data/basic/v/groundtruth.csv\n",
      "===================Results===================\n",
      "V1 causes V1 with a delay of 5 time steps.\n",
      "V2 causes V2 with a delay of 1 time steps.\n",
      "V3 causes V3 with a delay of 6 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 0\n",
      "Total True Positives': 3\n",
      "Total False Negatives: 2\n",
      "Total Direct False Positives: 0\n",
      "Total Direct True Positives: 3\n",
      "TPs': ['0->0', '1->1', '2->2']\n",
      "FPs': []\n",
      "TPs direct: ['0->0', '1->1', '2->2']\n",
      "FPs direct: []\n",
      "FNs: ['0->2', '1->2']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 1.0\n",
      "Recall': 0.6\n",
      "F1' score: 0.7499999999999999\n",
      "(includes only direct causal relationships)\n",
      "Precision: 1.0\n",
      "Recall: 0.6\n",
      "F1 score: 0.7499999999999999\n",
      "Percentage of delays that are correctly discovered: 33.33333333333333%\n",
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=16, out_features=256, bias=True)\n",
      "      (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wk): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=4, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=256, bias=True)\n",
      "          (linear2): Linear(in_features=256, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 137899\n",
      "===================Running===================\n",
      "ground_truth:data/basic/v/groundtruth.csv\n",
      "===================Results===================\n",
      "V1 causes V1 with a delay of 1 time steps.\n",
      "V2 causes V1 with a delay of 4 time steps.\n",
      "V1 causes V2 with a delay of 11 time steps.\n",
      "V2 causes V2 with a delay of 1 time steps.\n",
      "V1 causes V3 with a delay of 5 time steps.\n",
      "V3 causes V3 with a delay of 1 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 2\n",
      "Total True Positives': 4\n",
      "Total False Negatives: 1\n",
      "Total Direct False Positives: 2\n",
      "Total Direct True Positives: 4\n",
      "TPs': ['0->0', '1->1', '0->2', '2->2']\n",
      "FPs': ['1->0', '0->1']\n",
      "TPs direct: ['0->0', '1->1', '0->2', '2->2']\n",
      "FPs direct: ['1->0', '0->1']\n",
      "FNs: ['1->2']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 0.6666666666666666\n",
      "Recall': 0.8\n",
      "F1' score: 0.7272727272727272\n",
      "(includes only direct causal relationships)\n",
      "Precision: 0.6666666666666666\n",
      "Recall: 0.8\n",
      "F1 score: 0.7272727272727272\n",
      "Percentage of delays that are correctly discovered: 100.0%\n",
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=16, out_features=256, bias=True)\n",
      "      (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wk): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=4, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=256, bias=True)\n",
      "          (linear2): Linear(in_features=256, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 137899\n",
      "===================Running===================\n",
      "ground_truth:data/basic/v/groundtruth.csv\n",
      "===================Results===================\n",
      "V1 causes V1 with a delay of 1 time steps.\n",
      "V3 causes V1 with a delay of 4 time steps.\n",
      "V2 causes V2 with a delay of 9 time steps.\n",
      "V1 causes V3 with a delay of 2 time steps.\n",
      "V3 causes V3 with a delay of 9 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 1\n",
      "Total True Positives': 4\n",
      "Total False Negatives: 1\n",
      "Total Direct False Positives: 1\n",
      "Total Direct True Positives: 4\n",
      "TPs': ['0->0', '1->1', '0->2', '2->2']\n",
      "FPs': ['2->0']\n",
      "TPs direct: ['0->0', '1->1', '0->2', '2->2']\n",
      "FPs direct: ['2->0']\n",
      "FNs: ['1->2']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 0.8\n",
      "Recall': 0.8\n",
      "F1' score: 0.8000000000000002\n",
      "(includes only direct causal relationships)\n",
      "Precision: 0.8\n",
      "Recall: 0.8\n",
      "F1 score: 0.8000000000000002\n",
      "Percentage of delays that are correctly discovered: 33.33333333333333%\n",
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=16, out_features=256, bias=True)\n",
      "      (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wk): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=4, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=256, bias=True)\n",
      "          (linear2): Linear(in_features=256, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 137899\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "===================Running===================\n",
      "ground_truth:data/basic/v/groundtruth.csv\n",
      "===================Results===================\n",
      "V1 causes V1 with a delay of 1 time steps.\n",
      "V1 causes V2 with a delay of 1 time steps.\n",
      "V2 causes V2 with a delay of 5 time steps.\n",
      "V1 causes V3 with a delay of 4 time steps.\n",
      "V3 causes V3 with a delay of 4 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 1\n",
      "Total True Positives': 4\n",
      "Total False Negatives: 1\n",
      "Total Direct False Positives: 1\n",
      "Total Direct True Positives: 4\n",
      "TPs': ['0->0', '1->1', '0->2', '2->2']\n",
      "FPs': ['0->1']\n",
      "TPs direct: ['0->0', '1->1', '0->2', '2->2']\n",
      "FPs direct: ['0->1']\n",
      "FNs: ['1->2']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 0.8\n",
      "Recall': 0.8\n",
      "F1' score: 0.8000000000000002\n",
      "(includes only direct causal relationships)\n",
      "Precision: 0.8\n",
      "Recall: 0.8\n",
      "F1 score: 0.8000000000000002\n",
      "Percentage of delays that are correctly discovered: 33.33333333333333%\n",
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=16, out_features=256, bias=True)\n",
      "      (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wk): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=4, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=256, bias=True)\n",
      "          (linear2): Linear(in_features=256, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 137899\n",
      "===================Running===================\n",
      "ground_truth:data/basic/v/groundtruth.csv\n",
      "===================Results===================\n",
      "V1 causes V1 with a delay of 1 time steps.\n",
      "V2 causes V2 with a delay of 1 time steps.\n",
      "V3 causes V3 with a delay of 1 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 0\n",
      "Total True Positives': 3\n",
      "Total False Negatives: 2\n",
      "Total Direct False Positives: 0\n",
      "Total Direct True Positives: 3\n",
      "TPs': ['0->0', '1->1', '2->2']\n",
      "FPs': []\n",
      "TPs direct: ['0->0', '1->1', '2->2']\n",
      "FPs direct: []\n",
      "FNs: ['0->2', '1->2']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 1.0\n",
      "Recall': 0.6\n",
      "F1' score: 0.7499999999999999\n",
      "(includes only direct causal relationships)\n",
      "Precision: 1.0\n",
      "Recall: 0.6\n",
      "F1 score: 0.7499999999999999\n",
      "Percentage of delays that are correctly discovered: 100.0%\n",
      "===================Summary===================\n",
      "\t    Precision'  Recall'       F1'  Precision  Recall        F1       PoD\n",
      "\t1     1.000000      0.6  0.750000   1.000000     0.6  0.750000  1.000000\n",
      "\t2     1.000000      0.6  0.750000   1.000000     0.6  0.750000  0.666667\n",
      "\t3     1.000000      0.6  0.750000   1.000000     0.6  0.750000  1.000000\n",
      "\t4     0.666667      0.8  0.727273   0.666667     0.8  0.727273  0.000000\n",
      "\t5     1.000000      0.6  0.750000   1.000000     0.6  0.750000  0.333333\n",
      "\t6     0.666667      0.8  0.727273   0.666667     0.8  0.727273  1.000000\n",
      "\t7     0.800000      0.8  0.800000   0.800000     0.8  0.800000  0.333333\n",
      "\t8     0.800000      0.8  0.800000   0.800000     0.8  0.800000  0.333333\n",
      "\t9     1.000000      0.8  0.888889   1.000000     0.8  0.888889  0.250000\n",
      "\t10    1.000000      0.6  0.750000   1.000000     0.6  0.750000  1.000000\n"
     ]
    }
   ],
   "source": [
    "! python runner.py -c config/config_basic_v_fork.json -t v"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=16, out_features=256, bias=True)\n",
      "      (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wk): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=4, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=256, bias=True)\n",
      "          (linear2): Linear(in_features=256, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 137899\n",
      "2023-08-16 21:29:29.557906: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n",
      "2023-08-16 21:29:29.602300: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2023-08-16 21:29:30.294965: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
      "===================Running===================\n",
      "ground_truth:data/basic/fork/groundtruth.csv\n",
      "===================Results===================\n",
      "V1 causes V1 with a delay of 10 time steps.\n",
      "V2 causes V2 with a delay of 10 time steps.\n",
      "V3 causes V3 with a delay of 8 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 0\n",
      "Total True Positives': 3\n",
      "Total False Negatives: 2\n",
      "Total Direct False Positives: 0\n",
      "Total Direct True Positives: 3\n",
      "TPs': ['0->0', '1->1', '2->2']\n",
      "FPs': []\n",
      "TPs direct: ['0->0', '1->1', '2->2']\n",
      "FPs direct: []\n",
      "FNs: ['0->1', '0->2']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 1.0\n",
      "Recall': 0.6\n",
      "F1' score: 0.7499999999999999\n",
      "(includes only direct causal relationships)\n",
      "Precision: 1.0\n",
      "Recall: 0.6\n",
      "F1 score: 0.7499999999999999\n",
      "Percentage of delays that are correctly discovered: 0.0%\n",
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=16, out_features=256, bias=True)\n",
      "      (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wk): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=4, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=256, bias=True)\n",
      "          (linear2): Linear(in_features=256, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 137899\n",
      "===================Running===================\n",
      "ground_truth:data/basic/fork/groundtruth.csv\n",
      "===================Results===================\n",
      "V1 causes V1 with a delay of 1 time steps.\n",
      "V2 causes V2 with a delay of 1 time steps.\n",
      "V3 causes V3 with a delay of 11 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 0\n",
      "Total True Positives': 3\n",
      "Total False Negatives: 2\n",
      "Total Direct False Positives: 0\n",
      "Total Direct True Positives: 3\n",
      "TPs': ['0->0', '1->1', '2->2']\n",
      "FPs': []\n",
      "TPs direct: ['0->0', '1->1', '2->2']\n",
      "FPs direct: []\n",
      "FNs: ['0->1', '0->2']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 1.0\n",
      "Recall': 0.6\n",
      "F1' score: 0.7499999999999999\n",
      "(includes only direct causal relationships)\n",
      "Precision: 1.0\n",
      "Recall: 0.6\n",
      "F1 score: 0.7499999999999999\n",
      "Percentage of delays that are correctly discovered: 66.66666666666666%\n",
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=16, out_features=256, bias=True)\n",
      "      (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wk): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=4, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=256, bias=True)\n",
      "          (linear2): Linear(in_features=256, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 137899\n",
      "===================Running===================\n",
      "ground_truth:data/basic/fork/groundtruth.csv\n",
      "===================Results===================\n",
      "V1 causes V1 with a delay of 15 time steps.\n",
      "V3 causes V1 with a delay of 7 time steps.\n",
      "V1 causes V2 with a delay of 1 time steps.\n",
      "V1 causes V3 with a delay of 7 time steps.\n",
      "V3 causes V3 with a delay of 1 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 1\n",
      "Total True Positives': 4\n",
      "Total False Negatives: 1\n",
      "Total Direct False Positives: 1\n",
      "Total Direct True Positives: 4\n",
      "TPs': ['0->0', '0->1', '0->2', '2->2']\n",
      "FPs': ['2->0']\n",
      "TPs direct: ['0->0', '0->1', '0->2', '2->2']\n",
      "FPs direct: ['2->0']\n",
      "FNs: ['1->1']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 0.8\n",
      "Recall': 0.8\n",
      "F1' score: 0.8000000000000002\n",
      "(includes only direct causal relationships)\n",
      "Precision: 0.8\n",
      "Recall: 0.8\n",
      "F1 score: 0.8000000000000002\n",
      "Percentage of delays that are correctly discovered: 66.66666666666666%\n",
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=16, out_features=256, bias=True)\n",
      "      (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wk): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=4, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=256, bias=True)\n",
      "          (linear2): Linear(in_features=256, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 137899\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "===================Running===================\n",
      "ground_truth:data/basic/fork/groundtruth.csv\n",
      "===================Results===================\n",
      "V1 causes V1 with a delay of 5 time steps.\n",
      "V2 causes V2 with a delay of 1 time steps.\n",
      "V3 causes V3 with a delay of 1 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 0\n",
      "Total True Positives': 3\n",
      "Total False Negatives: 2\n",
      "Total Direct False Positives: 0\n",
      "Total Direct True Positives: 3\n",
      "TPs': ['0->0', '1->1', '2->2']\n",
      "FPs': []\n",
      "TPs direct: ['0->0', '1->1', '2->2']\n",
      "FPs direct: []\n",
      "FNs: ['0->1', '0->2']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 1.0\n",
      "Recall': 0.6\n",
      "F1' score: 0.7499999999999999\n",
      "(includes only direct causal relationships)\n",
      "Precision: 1.0\n",
      "Recall: 0.6\n",
      "F1 score: 0.7499999999999999\n",
      "Percentage of delays that are correctly discovered: 66.66666666666666%\n",
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=16, out_features=256, bias=True)\n",
      "      (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wk): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=4, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=256, bias=True)\n",
      "          (linear2): Linear(in_features=256, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 137899\n",
      "===================Running===================\n",
      "ground_truth:data/basic/fork/groundtruth.csv\n",
      "===================Results===================\n",
      "V1 causes V1 with a delay of 5 time steps.\n",
      "V1 causes V2 with a delay of 1 time steps.\n",
      "V2 causes V2 with a delay of 6 time steps.\n",
      "V3 causes V3 with a delay of 5 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 0\n",
      "Total True Positives': 4\n",
      "Total False Negatives: 1\n",
      "Total Direct False Positives: 0\n",
      "Total Direct True Positives: 4\n",
      "TPs': ['0->0', '0->1', '1->1', '2->2']\n",
      "FPs': []\n",
      "TPs direct: ['0->0', '0->1', '1->1', '2->2']\n",
      "FPs direct: []\n",
      "FNs: ['0->2']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 1.0\n",
      "Recall': 0.8\n",
      "F1' score: 0.888888888888889\n",
      "(includes only direct causal relationships)\n",
      "Precision: 1.0\n",
      "Recall: 0.8\n",
      "F1 score: 0.888888888888889\n",
      "Percentage of delays that are correctly discovered: 25.0%\n",
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=16, out_features=256, bias=True)\n",
      "      (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wk): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=4, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=256, bias=True)\n",
      "          (linear2): Linear(in_features=256, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 137899\n",
      "===================Running===================\n",
      "ground_truth:data/basic/fork/groundtruth.csv\n",
      "===================Results===================\n",
      "V1 causes V1 with a delay of 1 time steps.\n",
      "V1 causes V2 with a delay of 3 time steps.\n",
      "V2 causes V2 with a delay of 1 time steps.\n",
      "V1 causes V3 with a delay of 6 time steps.\n",
      "V3 causes V3 with a delay of 1 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 0\n",
      "Total True Positives': 5\n",
      "Total False Negatives: 0\n",
      "Total Direct False Positives: 0\n",
      "Total Direct True Positives: 5\n",
      "TPs': ['0->0', '0->1', '1->1', '0->2', '2->2']\n",
      "FPs': []\n",
      "TPs direct: ['0->0', '0->1', '1->1', '0->2', '2->2']\n",
      "FPs direct: []\n",
      "FNs: []\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 1.0\n",
      "Recall': 1.0\n",
      "F1' score: 1.0\n",
      "(includes only direct causal relationships)\n",
      "Precision: 1.0\n",
      "Recall: 1.0\n",
      "F1 score: 1.0\n",
      "Percentage of delays that are correctly discovered: 75.0%\n",
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=16, out_features=256, bias=True)\n",
      "      (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wk): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=4, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=256, bias=True)\n",
      "          (linear2): Linear(in_features=256, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 137899\n",
      "===================Running===================\n",
      "ground_truth:data/basic/fork/groundtruth.csv\n",
      "===================Results===================\n",
      "V1 causes V1 with a delay of 1 time steps.\n",
      "V2 causes V2 with a delay of 1 time steps.\n",
      "V3 causes V3 with a delay of 1 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 0\n",
      "Total True Positives': 3\n",
      "Total False Negatives: 2\n",
      "Total Direct False Positives: 0\n",
      "Total Direct True Positives: 3\n",
      "TPs': ['0->0', '1->1', '2->2']\n",
      "FPs': []\n",
      "TPs direct: ['0->0', '1->1', '2->2']\n",
      "FPs direct: []\n",
      "FNs: ['0->1', '0->2']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 1.0\n",
      "Recall': 0.6\n",
      "F1' score: 0.7499999999999999\n",
      "(includes only direct causal relationships)\n",
      "Precision: 1.0\n",
      "Recall: 0.6\n",
      "F1 score: 0.7499999999999999\n",
      "Percentage of delays that are correctly discovered: 100.0%\n",
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=16, out_features=256, bias=True)\n",
      "      (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wk): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=4, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=256, bias=True)\n",
      "          (linear2): Linear(in_features=256, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 137899\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "===================Running===================\n",
      "ground_truth:data/basic/fork/groundtruth.csv\n",
      "===================Results===================\n",
      "V1 causes V1 with a delay of 4 time steps.\n",
      "V2 causes V2 with a delay of 7 time steps.\n",
      "V2 causes V3 with a delay of 3 time steps.\n",
      "V3 causes V3 with a delay of 7 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 1\n",
      "Total True Positives': 3\n",
      "Total False Negatives: 2\n",
      "Total Direct False Positives: 1\n",
      "Total Direct True Positives: 3\n",
      "TPs': ['0->0', '1->1', '2->2']\n",
      "FPs': ['1->2']\n",
      "TPs direct: ['0->0', '1->1', '2->2']\n",
      "FPs direct: ['1->2']\n",
      "FNs: ['0->1', '0->2']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 0.75\n",
      "Recall': 0.6\n",
      "F1' score: 0.6666666666666665\n",
      "(includes only direct causal relationships)\n",
      "Precision: 0.75\n",
      "Recall: 0.6\n",
      "F1 score: 0.6666666666666665\n",
      "Percentage of delays that are correctly discovered: 0.0%\n",
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=16, out_features=256, bias=True)\n",
      "      (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wk): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=4, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=256, bias=True)\n",
      "          (linear2): Linear(in_features=256, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 137899\n",
      "===================Running===================\n",
      "ground_truth:data/basic/fork/groundtruth.csv\n",
      "===================Results===================\n",
      "V1 causes V1 with a delay of 9 time steps.\n",
      "V2 causes V1 with a delay of 1 time steps.\n",
      "V1 causes V2 with a delay of 1 time steps.\n",
      "V2 causes V2 with a delay of 9 time steps.\n",
      "V1 causes V3 with a delay of 1 time steps.\n",
      "V3 causes V3 with a delay of 9 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 1\n",
      "Total True Positives': 5\n",
      "Total False Negatives: 0\n",
      "Total Direct False Positives: 1\n",
      "Total Direct True Positives: 5\n",
      "TPs': ['0->0', '0->1', '1->1', '0->2', '2->2']\n",
      "FPs': ['1->0']\n",
      "TPs direct: ['0->0', '0->1', '1->1', '0->2', '2->2']\n",
      "FPs direct: ['1->0']\n",
      "FNs: []\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 0.8333333333333334\n",
      "Recall': 1.0\n",
      "F1' score: 0.9090909090909091\n",
      "(includes only direct causal relationships)\n",
      "Precision: 0.8333333333333334\n",
      "Recall: 1.0\n",
      "F1 score: 0.9090909090909091\n",
      "Percentage of delays that are correctly discovered: 25.0%\n",
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=16, out_features=256, bias=True)\n",
      "      (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wk): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=4, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=256, bias=True)\n",
      "          (linear2): Linear(in_features=256, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((16, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 137899\n",
      "===================Running===================\n",
      "ground_truth:data/basic/fork/groundtruth.csv\n",
      "===================Results===================\n",
      "V1 causes V1 with a delay of 13 time steps.\n",
      "V2 causes V2 with a delay of 12 time steps.\n",
      "V3 causes V2 with a delay of 2 time steps.\n",
      "V3 causes V3 with a delay of 1 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 1\n",
      "Total True Positives': 3\n",
      "Total False Negatives: 2\n",
      "Total Direct False Positives: 1\n",
      "Total Direct True Positives: 3\n",
      "TPs': ['0->0', '1->1', '2->2']\n",
      "FPs': ['2->1']\n",
      "TPs direct: ['0->0', '1->1', '2->2']\n",
      "FPs direct: ['2->1']\n",
      "FNs: ['0->1', '0->2']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 0.75\n",
      "Recall': 0.6\n",
      "F1' score: 0.6666666666666665\n",
      "(includes only direct causal relationships)\n",
      "Precision: 0.75\n",
      "Recall: 0.6\n",
      "F1 score: 0.6666666666666665\n",
      "Percentage of delays that are correctly discovered: 33.33333333333333%\n",
      "===================Summary===================\n",
      "\t    Precision'  Recall'       F1'  Precision  Recall        F1       PoD\n",
      "\t1     1.000000      0.6  0.750000   1.000000     0.6  0.750000  0.000000\n",
      "\t2     1.000000      0.6  0.750000   1.000000     0.6  0.750000  0.666667\n",
      "\t3     0.800000      0.8  0.800000   0.800000     0.8  0.800000  0.666667\n",
      "\t4     1.000000      0.6  0.750000   1.000000     0.6  0.750000  0.666667\n",
      "\t5     1.000000      0.8  0.888889   1.000000     0.8  0.888889  0.250000\n",
      "\t6     1.000000      1.0  1.000000   1.000000     1.0  1.000000  0.750000\n",
      "\t7     1.000000      0.6  0.750000   1.000000     0.6  0.750000  1.000000\n",
      "\t8     0.750000      0.6  0.666667   0.750000     0.6  0.666667  0.000000\n",
      "\t9     0.833333      1.0  0.909091   0.833333     1.0  0.909091  0.250000\n",
      "\t10    0.750000      0.6  0.666667   0.750000     0.6  0.666667  0.333333\n"
     ]
    }
   ],
   "source": [
    "! python runner.py -c config/config_basic_v_fork.json -t fork"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=32, out_features=512, bias=True)\n",
      "      (norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=512, out_features=512, bias=True)\n",
      "          (Wk): Linear(in_features=512, out_features=512, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=8, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=512, bias=True)\n",
      "          (linear2): Linear(in_features=512, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 571307\n",
      "2023-08-16 22:28:03.646027: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n",
      "2023-08-16 22:28:03.691771: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2023-08-16 22:28:04.386376: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
      "===================Running===================\n",
      "ground_truth:data/lorenz96/groundtruth.csv\n",
      "===================Results===================\n",
      "T0 causes T0 with a delay of 2 time steps.\n",
      "T8 causes T0 with a delay of 0 time steps.\n",
      "T1 causes T1 with a delay of 2 time steps.\n",
      "T9 causes T1 with a delay of 1 time steps.\n",
      "T0 causes T2 with a delay of 0 time steps.\n",
      "T2 causes T2 with a delay of 2 time steps.\n",
      "T3 causes T2 with a delay of 2 time steps.\n",
      "T1 causes T3 with a delay of 0 time steps.\n",
      "T3 causes T3 with a delay of 2 time steps.\n",
      "T8 causes T3 with a delay of 6 time steps.\n",
      "T2 causes T4 with a delay of 1 time steps.\n",
      "T4 causes T4 with a delay of 2 time steps.\n",
      "T2 causes T5 with a delay of 0 time steps.\n",
      "T3 causes T5 with a delay of 0 time steps.\n",
      "T5 causes T5 with a delay of 2 time steps.\n",
      "T6 causes T5 with a delay of 0 time steps.\n",
      "T7 causes T5 with a delay of 0 time steps.\n",
      "T9 causes T5 with a delay of 0 time steps.\n",
      "T4 causes T6 with a delay of 1 time steps.\n",
      "T6 causes T6 with a delay of 2 time steps.\n",
      "T5 causes T7 with a delay of 1 time steps.\n",
      "T7 causes T7 with a delay of 2 time steps.\n",
      "T6 causes T8 with a delay of 1 time steps.\n",
      "T8 causes T8 with a delay of 2 time steps.\n",
      "T1 causes T9 with a delay of 0 time steps.\n",
      "T3 causes T9 with a delay of 0 time steps.\n",
      "T7 causes T9 with a delay of 1 time steps.\n",
      "T9 causes T9 with a delay of 2 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 3\n",
      "Total True Positives': 25\n",
      "Total False Negatives: 18\n",
      "Total Direct False Positives: 6\n",
      "Total Direct True Positives: 22\n",
      "TPs': ['0->0', '8->0', '1->1', '9->1', '0->2', '2->2', '3->2', '1->3', '3->3', '2->4', '4->4', '2->5', '3->5', '5->5', '6->5', '7->5', '4->6', '6->6', '5->7', '7->7', '6->8', '8->8', '1->9', '7->9', '9->9']\n",
      "FPs': ['8->3', '9->5', '3->9']\n",
      "TPs direct: ['0->0', '8->0', '1->1', '9->1', '0->2', '2->2', '3->2', '1->3', '3->3', '2->4', '4->4', '3->5', '5->5', '6->5', '4->6', '6->6', '5->7', '7->7', '6->8', '8->8', '7->9', '9->9']\n",
      "FPs direct: ['8->3', '2->5', '7->5', '9->5', '1->9', '3->9']\n",
      "FNs: ['1->0', '9->0', '0->1', '2->1', '1->2', '2->3', '4->3', '3->4', '5->4', '4->5', '5->6', '7->6', '6->7', '8->7', '7->8', '9->8', '0->9', '8->9']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 0.8928571428571429\n",
      "Recall': 0.5813953488372093\n",
      "F1' score: 0.7042253521126761\n",
      "(includes only direct causal relationships)\n",
      "Precision: 0.7857142857142857\n",
      "Recall: 0.55\n",
      "F1 score: 0.6470588235294117\n",
      "Percentage of delays that are correctly discovered: 27.27272727272727%\n",
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=32, out_features=512, bias=True)\n",
      "      (norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=512, out_features=512, bias=True)\n",
      "          (Wk): Linear(in_features=512, out_features=512, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=8, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=512, bias=True)\n",
      "          (linear2): Linear(in_features=512, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 571307\n",
      "===================Running===================\n",
      "ground_truth:data/lorenz96/groundtruth.csv\n",
      "===================Results===================\n",
      "T0 causes T0 with a delay of 3 time steps.\n",
      "T1 causes T0 with a delay of 27 time steps.\n",
      "T2 causes T0 with a delay of 0 time steps.\n",
      "T5 causes T0 with a delay of 1 time steps.\n",
      "T8 causes T0 with a delay of 1 time steps.\n",
      "T1 causes T1 with a delay of 13 time steps.\n",
      "T7 causes T1 with a delay of 5 time steps.\n",
      "T8 causes T1 with a delay of 0 time steps.\n",
      "T9 causes T1 with a delay of 0 time steps.\n",
      "T0 causes T2 with a delay of 1 time steps.\n",
      "T2 causes T2 with a delay of 7 time steps.\n",
      "T0 causes T3 with a delay of 0 time steps.\n",
      "T1 causes T3 with a delay of 0 time steps.\n",
      "T3 causes T3 with a delay of 3 time steps.\n",
      "T4 causes T3 with a delay of 0 time steps.\n",
      "T2 causes T4 with a delay of 0 time steps.\n",
      "T4 causes T4 with a delay of 4 time steps.\n",
      "T1 causes T5 with a delay of 0 time steps.\n",
      "T2 causes T5 with a delay of 6 time steps.\n",
      "T3 causes T5 with a delay of 0 time steps.\n",
      "T5 causes T5 with a delay of 2 time steps.\n",
      "T6 causes T5 with a delay of 14 time steps.\n",
      "T7 causes T5 with a delay of 0 time steps.\n",
      "T4 causes T6 with a delay of 1 time steps.\n",
      "T6 causes T6 with a delay of 8 time steps.\n",
      "T7 causes T6 with a delay of 0 time steps.\n",
      "T8 causes T6 with a delay of 0 time steps.\n",
      "T5 causes T7 with a delay of 1 time steps.\n",
      "T7 causes T7 with a delay of 2 time steps.\n",
      "T8 causes T7 with a delay of 0 time steps.\n",
      "T0 causes T8 with a delay of 0 time steps.\n",
      "T3 causes T8 with a delay of 0 time steps.\n",
      "T6 causes T8 with a delay of 0 time steps.\n",
      "T7 causes T8 with a delay of 0 time steps.\n",
      "T1 causes T9 with a delay of 0 time steps.\n",
      "T7 causes T9 with a delay of 0 time steps.\n",
      "T9 causes T9 with a delay of 2 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 2\n",
      "Total True Positives': 35\n",
      "Total False Negatives: 15\n",
      "Total Direct False Positives: 12\n",
      "Total Direct True Positives: 25\n",
      "TPs': ['0->0', '1->0', '2->0', '8->0', '1->1', '7->1', '8->1', '9->1', '0->2', '2->2', '0->3', '1->3', '3->3', '4->3', '2->4', '4->4', '1->5', '2->5', '3->5', '5->5', '6->5', '7->5', '4->6', '6->6', '7->6', '8->6', '5->7', '7->7', '8->7', '0->8', '6->8', '7->8', '1->9', '7->9', '9->9']\n",
      "FPs': ['5->0', '3->8']\n",
      "TPs direct: ['0->0', '1->0', '8->0', '1->1', '9->1', '0->2', '2->2', '1->3', '3->3', '4->3', '2->4', '4->4', '3->5', '5->5', '6->5', '4->6', '6->6', '7->6', '5->7', '7->7', '8->7', '6->8', '7->8', '7->9', '9->9']\n",
      "FPs direct: ['2->0', '5->0', '7->1', '8->1', '0->3', '1->5', '2->5', '7->5', '8->6', '0->8', '3->8', '1->9']\n",
      "FNs: ['9->0', '0->1', '2->1', '1->2', '3->2', '2->3', '3->4', '5->4', '4->5', '5->6', '6->7', '8->8', '9->8', '0->9', '8->9']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 0.9459459459459459\n",
      "Recall': 0.7\n",
      "F1' score: 0.8045977011494254\n",
      "(includes only direct causal relationships)\n",
      "Precision: 0.6756756756756757\n",
      "Recall: 0.625\n",
      "F1 score: 0.6493506493506493\n",
      "Percentage of delays that are correctly discovered: 16.0%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=32, out_features=512, bias=True)\n",
      "      (norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=512, out_features=512, bias=True)\n",
      "          (Wk): Linear(in_features=512, out_features=512, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=8, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=512, bias=True)\n",
      "          (linear2): Linear(in_features=512, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 571307\n",
      "===================Running===================\n",
      "ground_truth:data/lorenz96/groundtruth.csv\n",
      "===================Results===================\n",
      "T0 causes T0 with a delay of 1 time steps.\n",
      "T8 causes T0 with a delay of 1 time steps.\n",
      "T1 causes T1 with a delay of 4 time steps.\n",
      "T9 causes T1 with a delay of 1 time steps.\n",
      "T0 causes T2 with a delay of 1 time steps.\n",
      "T2 causes T2 with a delay of 1 time steps.\n",
      "T1 causes T3 with a delay of 0 time steps.\n",
      "T3 causes T3 with a delay of 1 time steps.\n",
      "T5 causes T3 with a delay of 0 time steps.\n",
      "T2 causes T4 with a delay of 1 time steps.\n",
      "T4 causes T4 with a delay of 2 time steps.\n",
      "T3 causes T5 with a delay of 1 time steps.\n",
      "T5 causes T5 with a delay of 1 time steps.\n",
      "T4 causes T6 with a delay of 0 time steps.\n",
      "T6 causes T6 with a delay of 1 time steps.\n",
      "T5 causes T7 with a delay of 1 time steps.\n",
      "T7 causes T7 with a delay of 1 time steps.\n",
      "T6 causes T8 with a delay of 1 time steps.\n",
      "T8 causes T8 with a delay of 1 time steps.\n",
      "T7 causes T9 with a delay of 0 time steps.\n",
      "T9 causes T9 with a delay of 1 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 0\n",
      "Total True Positives': 21\n",
      "Total False Negatives: 20\n",
      "Total Direct False Positives: 1\n",
      "Total Direct True Positives: 20\n",
      "TPs': ['0->0', '8->0', '1->1', '9->1', '0->2', '2->2', '1->3', '3->3', '5->3', '2->4', '4->4', '3->5', '5->5', '4->6', '6->6', '5->7', '7->7', '6->8', '8->8', '7->9', '9->9']\n",
      "FPs': []\n",
      "TPs direct: ['0->0', '8->0', '1->1', '9->1', '0->2', '2->2', '1->3', '3->3', '2->4', '4->4', '3->5', '5->5', '4->6', '6->6', '5->7', '7->7', '6->8', '8->8', '7->9', '9->9']\n",
      "FPs direct: ['5->3']\n",
      "FNs: ['1->0', '9->0', '0->1', '2->1', '1->2', '3->2', '2->3', '4->3', '3->4', '5->4', '4->5', '6->5', '5->6', '7->6', '6->7', '8->7', '7->8', '9->8', '0->9', '8->9']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 1.0\n",
      "Recall': 0.5121951219512195\n",
      "F1' score: 0.6774193548387097\n",
      "(includes only direct causal relationships)\n",
      "Precision: 0.9523809523809523\n",
      "Recall: 0.5\n",
      "F1 score: 0.6557377049180327\n",
      "Percentage of delays that are correctly discovered: 75.0%\n",
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=32, out_features=512, bias=True)\n",
      "      (norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=512, out_features=512, bias=True)\n",
      "          (Wk): Linear(in_features=512, out_features=512, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=8, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=512, bias=True)\n",
      "          (linear2): Linear(in_features=512, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 571307\n",
      "===================Running===================\n",
      "ground_truth:data/lorenz96/groundtruth.csv\n",
      "===================Results===================\n",
      "T1 causes T0 with a delay of 1 time steps.\n",
      "T8 causes T0 with a delay of 2 time steps.\n",
      "T1 causes T1 with a delay of 1 time steps.\n",
      "T9 causes T1 with a delay of 10 time steps.\n",
      "T2 causes T2 with a delay of 1 time steps.\n",
      "T3 causes T2 with a delay of 1 time steps.\n",
      "T1 causes T3 with a delay of 2 time steps.\n",
      "T3 causes T3 with a delay of 1 time steps.\n",
      "T4 causes T3 with a delay of 0 time steps.\n",
      "T9 causes T3 with a delay of 0 time steps.\n",
      "T2 causes T4 with a delay of 11 time steps.\n",
      "T4 causes T4 with a delay of 1 time steps.\n",
      "T5 causes T4 with a delay of 1 time steps.\n",
      "T3 causes T5 with a delay of 11 time steps.\n",
      "T5 causes T5 with a delay of 1 time steps.\n",
      "T6 causes T5 with a delay of 1 time steps.\n",
      "T8 causes T5 with a delay of 0 time steps.\n",
      "T4 causes T6 with a delay of 2 time steps.\n",
      "T6 causes T6 with a delay of 1 time steps.\n",
      "T7 causes T6 with a delay of 1 time steps.\n",
      "T8 causes T6 with a delay of 16 time steps.\n",
      "T5 causes T7 with a delay of 2 time steps.\n",
      "T7 causes T7 with a delay of 1 time steps.\n",
      "T7 causes T8 with a delay of 0 time steps.\n",
      "T8 causes T8 with a delay of 1 time steps.\n",
      "T9 causes T8 with a delay of 1 time steps.\n",
      "T0 causes T9 with a delay of 1 time steps.\n",
      "T8 causes T9 with a delay of 0 time steps.\n",
      "T9 causes T9 with a delay of 1 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 1\n",
      "Total True Positives': 28\n",
      "Total False Negatives: 14\n",
      "Total Direct False Positives: 3\n",
      "Total Direct True Positives: 26\n",
      "TPs': ['1->0', '8->0', '1->1', '9->1', '2->2', '3->2', '1->3', '3->3', '4->3', '9->3', '2->4', '4->4', '5->4', '3->5', '5->5', '6->5', '4->6', '6->6', '7->6', '8->6', '5->7', '7->7', '7->8', '8->8', '9->8', '0->9', '8->9', '9->9']\n",
      "FPs': ['8->5']\n",
      "TPs direct: ['1->0', '8->0', '1->1', '9->1', '2->2', '3->2', '1->3', '3->3', '4->3', '2->4', '4->4', '5->4', '3->5', '5->5', '6->5', '4->6', '6->6', '7->6', '5->7', '7->7', '7->8', '8->8', '9->8', '0->9', '8->9', '9->9']\n",
      "FPs direct: ['9->3', '8->5', '8->6']\n",
      "FNs: ['0->0', '9->0', '0->1', '2->1', '0->2', '1->2', '2->3', '3->4', '4->5', '5->6', '6->7', '8->7', '6->8', '7->9']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 0.9655172413793104\n",
      "Recall': 0.6666666666666666\n",
      "F1' score: 0.7887323943661971\n",
      "(includes only direct causal relationships)\n",
      "Precision: 0.896551724137931\n",
      "Recall: 0.65\n",
      "F1 score: 0.7536231884057972\n",
      "Percentage of delays that are correctly discovered: 61.53846153846154%\n",
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=32, out_features=512, bias=True)\n",
      "      (norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=512, out_features=512, bias=True)\n",
      "          (Wk): Linear(in_features=512, out_features=512, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=8, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=512, bias=True)\n",
      "          (linear2): Linear(in_features=512, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 571307\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "===================Running===================\n",
      "ground_truth:data/lorenz96/groundtruth.csv\n",
      "===================Results===================\n",
      "T0 causes T0 with a delay of 4 time steps.\n",
      "T1 causes T0 with a delay of 0 time steps.\n",
      "T2 causes T0 with a delay of 0 time steps.\n",
      "T6 causes T0 with a delay of 0 time steps.\n",
      "T8 causes T0 with a delay of 0 time steps.\n",
      "T2 causes T1 with a delay of 1 time steps.\n",
      "T9 causes T1 with a delay of 1 time steps.\n",
      "T0 causes T2 with a delay of 0 time steps.\n",
      "T6 causes T2 with a delay of 0 time steps.\n",
      "T9 causes T2 with a delay of 0 time steps.\n",
      "T1 causes T3 with a delay of 23 time steps.\n",
      "T3 causes T3 with a delay of 1 time steps.\n",
      "T4 causes T3 with a delay of 0 time steps.\n",
      "T2 causes T4 with a delay of 0 time steps.\n",
      "T4 causes T4 with a delay of 1 time steps.\n",
      "T5 causes T4 with a delay of 1 time steps.\n",
      "T3 causes T5 with a delay of 0 time steps.\n",
      "T5 causes T5 with a delay of 2 time steps.\n",
      "T6 causes T5 with a delay of 11 time steps.\n",
      "T7 causes T5 with a delay of 0 time steps.\n",
      "T9 causes T5 with a delay of 0 time steps.\n",
      "T0 causes T6 with a delay of 0 time steps.\n",
      "T4 causes T6 with a delay of 0 time steps.\n",
      "T6 causes T6 with a delay of 1 time steps.\n",
      "T3 causes T7 with a delay of 0 time steps.\n",
      "T5 causes T7 with a delay of 0 time steps.\n",
      "T7 causes T7 with a delay of 9 time steps.\n",
      "T7 causes T8 with a delay of 0 time steps.\n",
      "T8 causes T8 with a delay of 1 time steps.\n",
      "T9 causes T8 with a delay of 0 time steps.\n",
      "T0 causes T9 with a delay of 1 time steps.\n",
      "T1 causes T9 with a delay of 1 time steps.\n",
      "T6 causes T9 with a delay of 0 time steps.\n",
      "T8 causes T9 with a delay of 0 time steps.\n",
      "T9 causes T9 with a delay of 1 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 3\n",
      "Total True Positives': 32\n",
      "Total False Negatives: 15\n",
      "Total Direct False Positives: 10\n",
      "Total Direct True Positives: 25\n",
      "TPs': ['0->0', '1->0', '2->0', '6->0', '8->0', '2->1', '9->1', '0->2', '9->2', '1->3', '3->3', '4->3', '2->4', '4->4', '5->4', '3->5', '5->5', '6->5', '7->5', '4->6', '6->6', '3->7', '5->7', '7->7', '7->8', '8->8', '9->8', '0->9', '1->9', '6->9', '8->9', '9->9']\n",
      "FPs': ['6->2', '9->5', '0->6']\n",
      "TPs direct: ['0->0', '1->0', '8->0', '2->1', '9->1', '0->2', '1->3', '3->3', '4->3', '2->4', '4->4', '5->4', '3->5', '5->5', '6->5', '4->6', '6->6', '5->7', '7->7', '7->8', '8->8', '9->8', '0->9', '8->9', '9->9']\n",
      "FPs direct: ['2->0', '6->0', '6->2', '9->2', '7->5', '9->5', '0->6', '3->7', '1->9', '6->9']\n",
      "FNs: ['9->0', '0->1', '1->1', '1->2', '2->2', '3->2', '2->3', '3->4', '4->5', '5->6', '7->6', '6->7', '8->7', '6->8', '7->9']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 0.9142857142857143\n",
      "Recall': 0.6808510638297872\n",
      "F1' score: 0.7804878048780487\n",
      "(includes only direct causal relationships)\n",
      "Precision: 0.7142857142857143\n",
      "Recall: 0.625\n",
      "F1 score: 0.6666666666666666\n",
      "Percentage of delays that are correctly discovered: 36.0%\n",
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=32, out_features=512, bias=True)\n",
      "      (norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=512, out_features=512, bias=True)\n",
      "          (Wk): Linear(in_features=512, out_features=512, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=8, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=512, bias=True)\n",
      "          (linear2): Linear(in_features=512, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 571307\n",
      "===================Running===================\n",
      "ground_truth:data/lorenz96/groundtruth.csv\n",
      "===================Results===================\n",
      "T0 causes T0 with a delay of 1 time steps.\n",
      "T8 causes T0 with a delay of 1 time steps.\n",
      "T1 causes T1 with a delay of 11 time steps.\n",
      "T9 causes T1 with a delay of 1 time steps.\n",
      "T0 causes T2 with a delay of 0 time steps.\n",
      "T2 causes T2 with a delay of 11 time steps.\n",
      "T1 causes T3 with a delay of 1 time steps.\n",
      "T3 causes T3 with a delay of 1 time steps.\n",
      "T2 causes T4 with a delay of 0 time steps.\n",
      "T4 causes T4 with a delay of 3 time steps.\n",
      "T3 causes T5 with a delay of 0 time steps.\n",
      "T5 causes T5 with a delay of 16 time steps.\n",
      "T4 causes T6 with a delay of 0 time steps.\n",
      "T6 causes T6 with a delay of 4 time steps.\n",
      "T5 causes T7 with a delay of 0 time steps.\n",
      "T7 causes T7 with a delay of 16 time steps.\n",
      "T6 causes T8 with a delay of 1 time steps.\n",
      "T8 causes T8 with a delay of 24 time steps.\n",
      "T7 causes T9 with a delay of 0 time steps.\n",
      "T9 causes T9 with a delay of 2 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 0\n",
      "Total True Positives': 20\n",
      "Total False Negatives: 20\n",
      "Total Direct False Positives: 0\n",
      "Total Direct True Positives: 20\n",
      "TPs': ['0->0', '8->0', '1->1', '9->1', '0->2', '2->2', '1->3', '3->3', '2->4', '4->4', '3->5', '5->5', '4->6', '6->6', '5->7', '7->7', '6->8', '8->8', '7->9', '9->9']\n",
      "FPs': []\n",
      "TPs direct: ['0->0', '8->0', '1->1', '9->1', '0->2', '2->2', '1->3', '3->3', '2->4', '4->4', '3->5', '5->5', '4->6', '6->6', '5->7', '7->7', '6->8', '8->8', '7->9', '9->9']\n",
      "FPs direct: []\n",
      "FNs: ['1->0', '9->0', '0->1', '2->1', '1->2', '3->2', '2->3', '4->3', '3->4', '5->4', '4->5', '6->5', '5->6', '7->6', '6->7', '8->7', '7->8', '9->8', '0->9', '8->9']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 1.0\n",
      "Recall': 0.5\n",
      "F1' score: 0.6666666666666666\n",
      "(includes only direct causal relationships)\n",
      "Precision: 1.0\n",
      "Recall: 0.5\n",
      "F1 score: 0.6666666666666666\n",
      "Percentage of delays that are correctly discovered: 30.0%\n",
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=32, out_features=512, bias=True)\n",
      "      (norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=512, out_features=512, bias=True)\n",
      "          (Wk): Linear(in_features=512, out_features=512, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=8, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=512, bias=True)\n",
      "          (linear2): Linear(in_features=512, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 571307\n",
      "===================Running===================\n",
      "ground_truth:data/lorenz96/groundtruth.csv\n",
      "===================Results===================\n",
      "T0 causes T0 with a delay of 1 time steps.\n",
      "T1 causes T0 with a delay of 1 time steps.\n",
      "T8 causes T0 with a delay of 0 time steps.\n",
      "T1 causes T1 with a delay of 6 time steps.\n",
      "T2 causes T1 with a delay of 3 time steps.\n",
      "T3 causes T1 with a delay of 0 time steps.\n",
      "T9 causes T1 with a delay of 1 time steps.\n",
      "T0 causes T2 with a delay of 1 time steps.\n",
      "T2 causes T2 with a delay of 4 time steps.\n",
      "T3 causes T2 with a delay of 7 time steps.\n",
      "T1 causes T3 with a delay of 0 time steps.\n",
      "T3 causes T3 with a delay of 1 time steps.\n",
      "T4 causes T3 with a delay of 1 time steps.\n",
      "T2 causes T4 with a delay of 9 time steps.\n",
      "T4 causes T4 with a delay of 1 time steps.\n",
      "T5 causes T4 with a delay of 1 time steps.\n",
      "T3 causes T5 with a delay of 0 time steps.\n",
      "T5 causes T5 with a delay of 4 time steps.\n",
      "T4 causes T6 with a delay of 0 time steps.\n",
      "T6 causes T6 with a delay of 1 time steps.\n",
      "T5 causes T7 with a delay of 0 time steps.\n",
      "T7 causes T7 with a delay of 4 time steps.\n",
      "T6 causes T8 with a delay of 1 time steps.\n",
      "T8 causes T8 with a delay of 1 time steps.\n",
      "T9 causes T8 with a delay of 1 time steps.\n",
      "T0 causes T9 with a delay of 8 time steps.\n",
      "T7 causes T9 with a delay of 0 time steps.\n",
      "T9 causes T9 with a delay of 9 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 0\n",
      "Total True Positives': 28\n",
      "Total False Negatives: 13\n",
      "Total Direct False Positives: 1\n",
      "Total Direct True Positives: 27\n",
      "TPs': ['0->0', '1->0', '8->0', '1->1', '2->1', '3->1', '9->1', '0->2', '2->2', '3->2', '1->3', '3->3', '4->3', '2->4', '4->4', '5->4', '3->5', '5->5', '4->6', '6->6', '5->7', '7->7', '6->8', '8->8', '9->8', '0->9', '7->9', '9->9']\n",
      "FPs': []\n",
      "TPs direct: ['0->0', '1->0', '8->0', '1->1', '2->1', '9->1', '0->2', '2->2', '3->2', '1->3', '3->3', '4->3', '2->4', '4->4', '5->4', '3->5', '5->5', '4->6', '6->6', '5->7', '7->7', '6->8', '8->8', '9->8', '0->9', '7->9', '9->9']\n",
      "FPs direct: ['3->1']\n",
      "FNs: ['9->0', '0->1', '1->2', '2->3', '3->4', '4->5', '6->5', '5->6', '7->6', '6->7', '8->7', '7->8', '8->9']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 1.0\n",
      "Recall': 0.6829268292682927\n",
      "F1' score: 0.8115942028985507\n",
      "(includes only direct causal relationships)\n",
      "Precision: 0.9642857142857143\n",
      "Recall: 0.675\n",
      "F1 score: 0.7941176470588236\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Percentage of delays that are correctly discovered: 44.44444444444444%\n",
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=32, out_features=512, bias=True)\n",
      "      (norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=512, out_features=512, bias=True)\n",
      "          (Wk): Linear(in_features=512, out_features=512, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=8, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=512, bias=True)\n",
      "          (linear2): Linear(in_features=512, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 571307\n",
      "===================Running===================\n",
      "ground_truth:data/lorenz96/groundtruth.csv\n",
      "===================Results===================\n",
      "T0 causes T0 with a delay of 1 time steps.\n",
      "T8 causes T0 with a delay of 1 time steps.\n",
      "T1 causes T1 with a delay of 12 time steps.\n",
      "T9 causes T1 with a delay of 1 time steps.\n",
      "T0 causes T2 with a delay of 1 time steps.\n",
      "T2 causes T2 with a delay of 15 time steps.\n",
      "T3 causes T2 with a delay of 26 time steps.\n",
      "T4 causes T2 with a delay of 0 time steps.\n",
      "T1 causes T3 with a delay of 1 time steps.\n",
      "T3 causes T3 with a delay of 14 time steps.\n",
      "T5 causes T3 with a delay of 0 time steps.\n",
      "T2 causes T4 with a delay of 0 time steps.\n",
      "T4 causes T4 with a delay of 1 time steps.\n",
      "T3 causes T5 with a delay of 0 time steps.\n",
      "T5 causes T5 with a delay of 5 time steps.\n",
      "T4 causes T6 with a delay of 0 time steps.\n",
      "T6 causes T6 with a delay of 12 time steps.\n",
      "T5 causes T7 with a delay of 1 time steps.\n",
      "T7 causes T7 with a delay of 3 time steps.\n",
      "T6 causes T8 with a delay of 0 time steps.\n",
      "T8 causes T8 with a delay of 12 time steps.\n",
      "T3 causes T9 with a delay of 0 time steps.\n",
      "T7 causes T9 with a delay of 0 time steps.\n",
      "T9 causes T9 with a delay of 4 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 1\n",
      "Total True Positives': 23\n",
      "Total False Negatives: 19\n",
      "Total Direct False Positives: 3\n",
      "Total Direct True Positives: 21\n",
      "TPs': ['0->0', '8->0', '1->1', '9->1', '0->2', '2->2', '3->2', '4->2', '1->3', '3->3', '5->3', '2->4', '4->4', '3->5', '5->5', '4->6', '6->6', '5->7', '7->7', '6->8', '8->8', '7->9', '9->9']\n",
      "FPs': ['3->9']\n",
      "TPs direct: ['0->0', '8->0', '1->1', '9->1', '0->2', '2->2', '3->2', '1->3', '3->3', '2->4', '4->4', '3->5', '5->5', '4->6', '6->6', '5->7', '7->7', '6->8', '8->8', '7->9', '9->9']\n",
      "FPs direct: ['4->2', '5->3', '3->9']\n",
      "FNs: ['1->0', '9->0', '0->1', '2->1', '1->2', '2->3', '4->3', '3->4', '5->4', '4->5', '6->5', '5->6', '7->6', '6->7', '8->7', '7->8', '9->8', '0->9', '8->9']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 0.9583333333333334\n",
      "Recall': 0.5476190476190477\n",
      "F1' score: 0.696969696969697\n",
      "(includes only direct causal relationships)\n",
      "Precision: 0.875\n",
      "Recall: 0.525\n",
      "F1 score: 0.6562500000000001\n",
      "Percentage of delays that are correctly discovered: 33.33333333333333%\n",
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=32, out_features=512, bias=True)\n",
      "      (norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=512, out_features=512, bias=True)\n",
      "          (Wk): Linear(in_features=512, out_features=512, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=8, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=512, bias=True)\n",
      "          (linear2): Linear(in_features=512, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 571307\n",
      "===================Running===================\n",
      "ground_truth:data/lorenz96/groundtruth.csv\n",
      "===================Results===================\n",
      "T0 causes T0 with a delay of 1 time steps.\n",
      "T1 causes T0 with a delay of 1 time steps.\n",
      "T2 causes T0 with a delay of 3 time steps.\n",
      "T8 causes T0 with a delay of 4 time steps.\n",
      "T9 causes T0 with a delay of 0 time steps.\n",
      "T1 causes T1 with a delay of 1 time steps.\n",
      "T2 causes T1 with a delay of 1 time steps.\n",
      "T8 causes T1 with a delay of 0 time steps.\n",
      "T9 causes T1 with a delay of 9 time steps.\n",
      "T0 causes T2 with a delay of 2 time steps.\n",
      "T2 causes T2 with a delay of 1 time steps.\n",
      "T3 causes T2 with a delay of 1 time steps.\n",
      "T1 causes T3 with a delay of 1 time steps.\n",
      "T3 causes T3 with a delay of 15 time steps.\n",
      "T4 causes T3 with a delay of 11 time steps.\n",
      "T0 causes T4 with a delay of 0 time steps.\n",
      "T2 causes T4 with a delay of 1 time steps.\n",
      "T4 causes T4 with a delay of 3 time steps.\n",
      "T3 causes T5 with a delay of 7 time steps.\n",
      "T5 causes T5 with a delay of 1 time steps.\n",
      "T6 causes T5 with a delay of 1 time steps.\n",
      "T3 causes T6 with a delay of 1 time steps.\n",
      "T4 causes T6 with a delay of 7 time steps.\n",
      "T6 causes T6 with a delay of 1 time steps.\n",
      "T7 causes T6 with a delay of 1 time steps.\n",
      "T4 causes T7 with a delay of 3 time steps.\n",
      "T5 causes T7 with a delay of 6 time steps.\n",
      "T7 causes T7 with a delay of 1 time steps.\n",
      "T6 causes T8 with a delay of 1 time steps.\n",
      "T7 causes T8 with a delay of 0 time steps.\n",
      "T8 causes T8 with a delay of 1 time steps.\n",
      "T9 causes T8 with a delay of 1 time steps.\n",
      "T7 causes T9 with a delay of 0 time steps.\n",
      "T9 causes T9 with a delay of 1 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 0\n",
      "Total True Positives': 34\n",
      "Total False Negatives: 11\n",
      "Total Direct False Positives: 5\n",
      "Total Direct True Positives: 29\n",
      "TPs': ['0->0', '1->0', '2->0', '8->0', '9->0', '1->1', '2->1', '8->1', '9->1', '0->2', '2->2', '3->2', '1->3', '3->3', '4->3', '0->4', '2->4', '4->4', '3->5', '5->5', '6->5', '3->6', '4->6', '6->6', '7->6', '4->7', '5->7', '7->7', '6->8', '7->8', '8->8', '9->8', '7->9', '9->9']\n",
      "FPs': []\n",
      "TPs direct: ['0->0', '1->0', '8->0', '9->0', '1->1', '2->1', '9->1', '0->2', '2->2', '3->2', '1->3', '3->3', '4->3', '2->4', '4->4', '3->5', '5->5', '6->5', '4->6', '6->6', '7->6', '5->7', '7->7', '6->8', '7->8', '8->8', '9->8', '7->9', '9->9']\n",
      "FPs direct: ['2->0', '8->1', '0->4', '3->6', '4->7']\n",
      "FNs: ['0->1', '1->2', '2->3', '3->4', '5->4', '4->5', '5->6', '6->7', '8->7', '0->9', '8->9']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 1.0\n",
      "Recall': 0.7555555555555555\n",
      "F1' score: 0.8607594936708861\n",
      "(includes only direct causal relationships)\n",
      "Precision: 0.8529411764705882\n",
      "Recall: 0.725\n",
      "F1 score: 0.7837837837837837\n",
      "Percentage of delays that are correctly discovered: 58.620689655172406%\n",
      "===================Summary===================\n",
      "\t    Precision'   Recall'       F1'  Precision  Recall        F1       PoD\n",
      "\t1     0.892857  0.581395  0.704225   0.785714   0.550  0.647059  0.272727\n",
      "\t2     0.945946  0.700000  0.804598   0.675676   0.625  0.649351  0.160000\n",
      "\t3     1.000000  0.512195  0.677419   0.952381   0.500  0.655738  0.750000\n",
      "\t4     0.965517  0.666667  0.788732   0.896552   0.650  0.753623  0.615385\n",
      "\t5     1.000000  0.500000  0.666667   1.000000   0.500  0.666667  0.350000\n",
      "\t6     0.914286  0.680851  0.780488   0.714286   0.625  0.666667  0.360000\n",
      "\t7     1.000000  0.500000  0.666667   1.000000   0.500  0.666667  0.300000\n",
      "\t8     1.000000  0.682927  0.811594   0.964286   0.675  0.794118  0.444444\n",
      "\t9     0.958333  0.547619  0.696970   0.875000   0.525  0.656250  0.333333\n",
      "\t10    1.000000  0.755556  0.860759   0.852941   0.725  0.783784  0.586207\n"
     ]
    }
   ],
   "source": [
    "! python runner.py -c config/config_lorenz.json -t lorenz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=32, out_features=256, bias=True)\n",
      "      (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wk): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=4, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=512, bias=True)\n",
      "          (linear2): Linear(in_features=512, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 145515\n",
      "2023-08-16 23:56:36.926800: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n",
      "2023-08-16 23:56:36.974106: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2023-08-16 23:56:37.714252: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
      "===================Running===================\n",
      "ground_truth:data/fMRI/sim1_gt_processed.csv\n",
      "===================Results===================\n",
      "0 causes 0 with a delay of 18 time steps.\n",
      "2 causes 0 with a delay of 1 time steps.\n",
      "0 causes 1 with a delay of 1 time steps.\n",
      "1 causes 1 with a delay of 1 time steps.\n",
      "2 causes 1 with a delay of 1 time steps.\n",
      "2 causes 2 with a delay of 1 time steps.\n",
      "3 causes 2 with a delay of 1 time steps.\n",
      "3 causes 3 with a delay of 28 time steps.\n",
      "4 causes 3 with a delay of 6 time steps.\n",
      "3 causes 4 with a delay of 1 time steps.\n",
      "4 causes 4 with a delay of 9 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 2\n",
      "Total True Positives': 9\n",
      "Total False Negatives: 2\n",
      "Total Direct False Positives: 3\n",
      "Total Direct True Positives: 8\n",
      "TPs': ['0->0', '2->0', '1->1', '2->1', '2->2', '3->2', '3->3', '4->3', '4->4']\n",
      "FPs': ['0->1', '3->4']\n",
      "TPs direct: ['0->0', '1->1', '2->1', '2->2', '3->2', '3->3', '4->3', '4->4']\n",
      "FPs direct: ['2->0', '0->1', '3->4']\n",
      "FNs: ['1->0', '4->0']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 0.8181818181818182\n",
      "Recall': 0.8181818181818182\n",
      "F1' score: 0.8181818181818182\n",
      "(includes only direct causal relationships)\n",
      "Precision: 0.7272727272727273\n",
      "Recall: 0.8\n",
      "F1 score: 0.761904761904762\n",
      "Percentage of delays that are correctly discovered: 50.0%\n",
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=32, out_features=256, bias=True)\n",
      "      (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wk): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=4, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=512, bias=True)\n",
      "          (linear2): Linear(in_features=512, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 155415\n",
      "===================Running===================\n",
      "ground_truth:data/fMRI/sim2_gt_processed.csv\n",
      "===================Results===================\n",
      "0 causes 0 with a delay of 12 time steps.\n",
      "2 causes 0 with a delay of 12 time steps.\n",
      "0 causes 1 with a delay of 1 time steps.\n",
      "1 causes 1 with a delay of 1 time steps.\n",
      "2 causes 2 with a delay of 1 time steps.\n",
      "3 causes 2 with a delay of 1 time steps.\n",
      "3 causes 3 with a delay of 1 time steps.\n",
      "4 causes 3 with a delay of 1 time steps.\n",
      "1 causes 4 with a delay of 1 time steps.\n",
      "2 causes 4 with a delay of 1 time steps.\n",
      "4 causes 4 with a delay of 5 time steps.\n",
      "0 causes 5 with a delay of 7 time steps.\n",
      "5 causes 5 with a delay of 1 time steps.\n",
      "5 causes 6 with a delay of 1 time steps.\n",
      "6 causes 6 with a delay of 1 time steps.\n",
      "2 causes 7 with a delay of 1 time steps.\n",
      "3 causes 7 with a delay of 1 time steps.\n",
      "7 causes 7 with a delay of 1 time steps.\n",
      "8 causes 8 with a delay of 1 time steps.\n",
      "9 causes 8 with a delay of 1 time steps.\n",
      "5 causes 9 with a delay of 1 time steps.\n",
      "9 causes 9 with a delay of 1 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 8\n",
      "Total True Positives': 14\n",
      "Total False Negatives: 8\n",
      "Total Direct False Positives: 9\n",
      "Total Direct True Positives: 13\n",
      "TPs': ['0->0', '2->0', '1->1', '2->2', '3->2', '3->3', '4->3', '4->4', '5->5', '6->6', '7->7', '8->8', '9->8', '9->9']\n",
      "FPs': ['0->1', '1->4', '2->4', '0->5', '5->6', '2->7', '3->7', '5->9']\n",
      "TPs direct: ['0->0', '1->1', '2->2', '3->2', '3->3', '4->3', '4->4', '5->5', '6->6', '7->7', '8->8', '9->8', '9->9']\n",
      "FPs direct: ['2->0', '0->1', '1->4', '2->4', '0->5', '5->6', '2->7', '3->7', '5->9']\n",
      "FNs: ['1->0', '4->0', '2->1', '7->2', '6->5', '9->5', '7->6', '8->7']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 0.6363636363636364\n",
      "Recall': 0.6363636363636364\n",
      "F1' score: 0.6363636363636364\n",
      "(includes only direct causal relationships)\n",
      "Precision: 0.5909090909090909\n",
      "Recall: 0.6190476190476191\n",
      "F1 score: 0.6046511627906977\n",
      "Percentage of delays that are correctly discovered: 84.61538461538461%\n",
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=32, out_features=256, bias=True)\n",
      "      (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wk): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=4, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=512, bias=True)\n",
      "          (linear2): Linear(in_features=512, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 171915\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "===================Running===================\n",
      "ground_truth:data/fMRI/sim3_gt_processed.csv\n",
      "===================Results===================\n",
      "0 causes 0 with a delay of 19 time steps.\n",
      "1 causes 1 with a delay of 6 time steps.\n",
      "2 causes 2 with a delay of 31 time steps.\n",
      "3 causes 2 with a delay of 31 time steps.\n",
      "3 causes 3 with a delay of 28 time steps.\n",
      "4 causes 4 with a delay of 5 time steps.\n",
      "5 causes 5 with a delay of 11 time steps.\n",
      "6 causes 6 with a delay of 16 time steps.\n",
      "7 causes 7 with a delay of 1 time steps.\n",
      "8 causes 8 with a delay of 10 time steps.\n",
      "9 causes 9 with a delay of 6 time steps.\n",
      "10 causes 10 with a delay of 2 time steps.\n",
      "11 causes 10 with a delay of 2 time steps.\n",
      "11 causes 11 with a delay of 2 time steps.\n",
      "3 causes 12 with a delay of 1 time steps.\n",
      "12 causes 12 with a delay of 11 time steps.\n",
      "13 causes 13 with a delay of 3 time steps.\n",
      "3 causes 14 with a delay of 7 time steps.\n",
      "14 causes 14 with a delay of 2 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 2\n",
      "Total True Positives': 17\n",
      "Total False Negatives: 16\n",
      "Total Direct False Positives: 2\n",
      "Total Direct True Positives: 17\n",
      "TPs': ['0->0', '1->1', '2->2', '3->2', '3->3', '4->4', '5->5', '6->6', '7->7', '8->8', '9->9', '10->10', '11->10', '11->11', '12->12', '13->13', '14->14']\n",
      "FPs': ['3->12', '3->14']\n",
      "TPs direct: ['0->0', '1->1', '2->2', '3->2', '3->3', '4->4', '5->5', '6->6', '7->7', '8->8', '9->9', '10->10', '11->10', '11->11', '12->12', '13->13', '14->14']\n",
      "FPs direct: ['3->12', '3->14']\n",
      "FNs: ['1->0', '4->0', '2->1', '7->2', '12->2', '4->3', '6->5', '9->5', '7->6', '8->7', '12->7', '9->8', '14->10', '12->11', '13->12', '14->13']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 0.8947368421052632\n",
      "Recall': 0.5151515151515151\n",
      "F1' score: 0.6538461538461537\n",
      "(includes only direct causal relationships)\n",
      "Precision: 0.8947368421052632\n",
      "Recall: 0.5151515151515151\n",
      "F1 score: 0.6538461538461537\n",
      "Percentage of delays that are correctly discovered: 5.88235294117647%\n",
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=32, out_features=256, bias=True)\n",
      "      (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wk): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=4, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=512, bias=True)\n",
      "          (linear2): Linear(in_features=512, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 472215\n",
      "===================Running===================\n",
      "ground_truth:data/fMRI/sim4_gt_processed.csv\n",
      "===================Results===================\n",
      "0 causes 0 with a delay of 10 time steps.\n",
      "1 causes 1 with a delay of 28 time steps.\n",
      "2 causes 2 with a delay of 27 time steps.\n",
      "3 causes 3 with a delay of 5 time steps.\n",
      "4 causes 4 with a delay of 31 time steps.\n",
      "5 causes 5 with a delay of 1 time steps.\n",
      "6 causes 6 with a delay of 17 time steps.\n",
      "7 causes 7 with a delay of 3 time steps.\n",
      "8 causes 8 with a delay of 1 time steps.\n",
      "9 causes 9 with a delay of 1 time steps.\n",
      "10 causes 10 with a delay of 5 time steps.\n",
      "11 causes 11 with a delay of 12 time steps.\n",
      "12 causes 12 with a delay of 22 time steps.\n",
      "13 causes 13 with a delay of 1 time steps.\n",
      "14 causes 14 with a delay of 13 time steps.\n",
      "15 causes 15 with a delay of 30 time steps.\n",
      "16 causes 16 with a delay of 28 time steps.\n",
      "17 causes 17 with a delay of 3 time steps.\n",
      "18 causes 18 with a delay of 7 time steps.\n",
      "19 causes 19 with a delay of 21 time steps.\n",
      "20 causes 20 with a delay of 1 time steps.\n",
      "21 causes 21 with a delay of 1 time steps.\n",
      "22 causes 22 with a delay of 17 time steps.\n",
      "23 causes 23 with a delay of 1 time steps.\n",
      "24 causes 24 with a delay of 27 time steps.\n",
      "25 causes 25 with a delay of 4 time steps.\n",
      "26 causes 26 with a delay of 11 time steps.\n",
      "27 causes 27 with a delay of 6 time steps.\n",
      "28 causes 28 with a delay of 6 time steps.\n",
      "29 causes 29 with a delay of 11 time steps.\n",
      "30 causes 30 with a delay of 16 time steps.\n",
      "31 causes 31 with a delay of 1 time steps.\n",
      "32 causes 32 with a delay of 30 time steps.\n",
      "33 causes 33 with a delay of 6 time steps.\n",
      "34 causes 34 with a delay of 1 time steps.\n",
      "35 causes 35 with a delay of 1 time steps.\n",
      "36 causes 36 with a delay of 1 time steps.\n",
      "37 causes 37 with a delay of 4 time steps.\n",
      "38 causes 38 with a delay of 1 time steps.\n",
      "39 causes 39 with a delay of 3 time steps.\n",
      "40 causes 40 with a delay of 1 time steps.\n",
      "41 causes 41 with a delay of 4 time steps.\n",
      "42 causes 42 with a delay of 1 time steps.\n",
      "43 causes 43 with a delay of 8 time steps.\n",
      "44 causes 44 with a delay of 13 time steps.\n",
      "45 causes 45 with a delay of 1 time steps.\n",
      "46 causes 46 with a delay of 1 time steps.\n",
      "47 causes 47 with a delay of 28 time steps.\n",
      "48 causes 48 with a delay of 1 time steps.\n",
      "49 causes 49 with a delay of 4 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 0\n",
      "Total True Positives': 50\n",
      "Total False Negatives: 61\n",
      "Total Direct False Positives: 0\n",
      "Total Direct True Positives: 50\n",
      "TPs': ['0->0', '1->1', '2->2', '3->3', '4->4', '5->5', '6->6', '7->7', '8->8', '9->9', '10->10', '11->11', '12->12', '13->13', '14->14', '15->15', '16->16', '17->17', '18->18', '19->19', '20->20', '21->21', '22->22', '23->23', '24->24', '25->25', '26->26', '27->27', '28->28', '29->29', '30->30', '31->31', '32->32', '33->33', '34->34', '35->35', '36->36', '37->37', '38->38', '39->39', '40->40', '41->41', '42->42', '43->43', '44->44', '45->45', '46->46', '47->47', '48->48', '49->49']\n",
      "FPs': []\n",
      "TPs direct: ['0->0', '1->1', '2->2', '3->3', '4->4', '5->5', '6->6', '7->7', '8->8', '9->9', '10->10', '11->11', '12->12', '13->13', '14->14', '15->15', '16->16', '17->17', '18->18', '19->19', '20->20', '21->21', '22->22', '23->23', '24->24', '25->25', '26->26', '27->27', '28->28', '29->29', '30->30', '31->31', '32->32', '33->33', '34->34', '35->35', '36->36', '37->37', '38->38', '39->39', '40->40', '41->41', '42->42', '43->43', '44->44', '45->45', '46->46', '47->47', '48->48', '49->49']\n",
      "FPs direct: []\n",
      "FNs: ['1->0', '4->0', '2->1', '3->2', '7->2', '22->2', '27->2', '4->3', '6->5', '9->5', '7->6', '8->7', '12->7', '9->8', '11->10', '14->10', '12->11', '13->12', '17->12', '14->13', '16->15', '19->15', '17->16', '18->17', '22->17', '19->18', '21->20', '24->20', '22->21', '23->22', '24->23', '26->25', '29->25', '27->26', '28->27', '32->27', '47->27', '29->28', '31->30', '34->30', '32->31', '33->32', '37->32', '34->33', '36->35', '39->35', '37->36', '38->37', '42->37', '39->38', '41->40', '44->40', '42->41', '43->42', '47->42', '44->43', '46->45', '49->45', '47->46', '48->47', '49->48']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 1.0\n",
      "Recall': 0.45045045045045046\n",
      "F1' score: 0.6211180124223602\n",
      "(includes only direct causal relationships)\n",
      "Precision: 1.0\n",
      "Recall: 0.45045045045045046\n",
      "F1 score: 0.6211180124223602\n",
      "Percentage of delays that are correctly discovered: 34.0%\n",
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=32, out_features=256, bias=True)\n",
      "      (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wk): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=4, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=512, bias=True)\n",
      "          (linear2): Linear(in_features=512, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 145515\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "===================Running===================\n",
      "ground_truth:data/fMRI/sim5_gt_processed.csv\n",
      "===================Results===================\n",
      "0 causes 0 with a delay of 1 time steps.\n",
      "4 causes 0 with a delay of 9 time steps.\n",
      "1 causes 1 with a delay of 1 time steps.\n",
      "2 causes 2 with a delay of 2 time steps.\n",
      "4 causes 2 with a delay of 10 time steps.\n",
      "3 causes 3 with a delay of 1 time steps.\n",
      "4 causes 3 with a delay of 9 time steps.\n",
      "4 causes 4 with a delay of 1 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 0\n",
      "Total True Positives': 8\n",
      "Total False Negatives: 3\n",
      "Total Direct False Positives: 1\n",
      "Total Direct True Positives: 7\n",
      "TPs': ['0->0', '4->0', '1->1', '2->2', '4->2', '3->3', '4->3', '4->4']\n",
      "FPs': []\n",
      "TPs direct: ['0->0', '4->0', '1->1', '2->2', '3->3', '4->3', '4->4']\n",
      "FPs direct: ['4->2']\n",
      "FNs: ['1->0', '2->1', '3->2']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 1.0\n",
      "Recall': 0.7272727272727273\n",
      "F1' score: 0.8421052631578948\n",
      "(includes only direct causal relationships)\n",
      "Precision: 0.875\n",
      "Recall: 0.7\n",
      "F1 score: 0.7777777777777777\n",
      "Percentage of delays that are correctly discovered: 57.14285714285714%\n",
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=32, out_features=256, bias=True)\n",
      "      (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wk): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=4, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=512, bias=True)\n",
      "          (linear2): Linear(in_features=512, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 155415\n",
      "===================Running===================\n",
      "ground_truth:data/fMRI/sim6_gt_processed.csv\n",
      "===================Results===================\n",
      "0 causes 0 with a delay of 1 time steps.\n",
      "1 causes 0 with a delay of 13 time steps.\n",
      "0 causes 1 with a delay of 13 time steps.\n",
      "1 causes 1 with a delay of 1 time steps.\n",
      "2 causes 2 with a delay of 1 time steps.\n",
      "3 causes 3 with a delay of 1 time steps.\n",
      "4 causes 4 with a delay of 1 time steps.\n",
      "5 causes 5 with a delay of 1 time steps.\n",
      "6 causes 6 with a delay of 1 time steps.\n",
      "5 causes 7 with a delay of 12 time steps.\n",
      "7 causes 7 with a delay of 1 time steps.\n",
      "8 causes 7 with a delay of 14 time steps.\n",
      "8 causes 8 with a delay of 1 time steps.\n",
      "9 causes 9 with a delay of 26 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 2\n",
      "Total True Positives': 12\n",
      "Total False Negatives: 9\n",
      "Total Direct False Positives: 2\n",
      "Total Direct True Positives: 12\n",
      "TPs': ['0->0', '1->0', '1->1', '2->2', '3->3', '4->4', '5->5', '6->6', '7->7', '8->7', '8->8', '9->9']\n",
      "FPs': ['0->1', '5->7']\n",
      "TPs direct: ['0->0', '1->0', '1->1', '2->2', '3->3', '4->4', '5->5', '6->6', '7->7', '8->7', '8->8', '9->9']\n",
      "FPs direct: ['0->1', '5->7']\n",
      "FNs: ['4->0', '2->1', '3->2', '7->2', '4->3', '6->5', '9->5', '7->6', '9->8']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 0.8571428571428571\n",
      "Recall': 0.5714285714285714\n",
      "F1' score: 0.6857142857142857\n",
      "(includes only direct causal relationships)\n",
      "Precision: 0.8571428571428571\n",
      "Recall: 0.5714285714285714\n",
      "F1 score: 0.6857142857142857\n",
      "Percentage of delays that are correctly discovered: 75.0%\n",
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=32, out_features=256, bias=True)\n",
      "      (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wk): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=4, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=512, bias=True)\n",
      "          (linear2): Linear(in_features=512, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 145515\n",
      "===================Running===================\n",
      "ground_truth:data/fMRI/sim7_gt_processed.csv\n",
      "===================Results===================\n",
      "0 causes 0 with a delay of 2 time steps.\n",
      "3 causes 0 with a delay of 18 time steps.\n",
      "1 causes 1 with a delay of 2 time steps.\n",
      "3 causes 1 with a delay of 28 time steps.\n",
      "0 causes 2 with a delay of 27 time steps.\n",
      "2 causes 2 with a delay of 2 time steps.\n",
      "3 causes 3 with a delay of 2 time steps.\n",
      "1 causes 4 with a delay of 16 time steps.\n",
      "4 causes 4 with a delay of 2 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 3\n",
      "Total True Positives': 6\n",
      "Total False Negatives: 5\n",
      "Total Direct False Positives: 4\n",
      "Total Direct True Positives: 5\n",
      "TPs': ['0->0', '1->1', '3->1', '2->2', '3->3', '4->4']\n",
      "FPs': ['3->0', '0->2', '1->4']\n",
      "TPs direct: ['0->0', '1->1', '2->2', '3->3', '4->4']\n",
      "FPs direct: ['3->0', '3->1', '0->2', '1->4']\n",
      "FNs: ['1->0', '4->0', '2->1', '3->2', '4->3']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 0.6666666666666666\n",
      "Recall': 0.5454545454545454\n",
      "F1' score: 0.6\n",
      "(includes only direct causal relationships)\n",
      "Precision: 0.5555555555555556\n",
      "Recall: 0.5\n",
      "F1 score: 0.5263157894736842\n",
      "Percentage of delays that are correctly discovered: 0.0%\n",
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=32, out_features=256, bias=True)\n",
      "      (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wk): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=4, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=512, bias=True)\n",
      "          (linear2): Linear(in_features=512, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 145515\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "===================Running===================\n",
      "ground_truth:data/fMRI/sim8_gt_processed.csv\n",
      "===================Results===================\n",
      "0 causes 0 with a delay of 1 time steps.\n",
      "4 causes 0 with a delay of 18 time steps.\n",
      "1 causes 1 with a delay of 1 time steps.\n",
      "2 causes 1 with a delay of 1 time steps.\n",
      "4 causes 1 with a delay of 1 time steps.\n",
      "2 causes 2 with a delay of 1 time steps.\n",
      "4 causes 2 with a delay of 1 time steps.\n",
      "3 causes 3 with a delay of 1 time steps.\n",
      "4 causes 3 with a delay of 28 time steps.\n",
      "3 causes 4 with a delay of 1 time steps.\n",
      "4 causes 4 with a delay of 1 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 2\n",
      "Total True Positives': 9\n",
      "Total False Negatives: 2\n",
      "Total Direct False Positives: 3\n",
      "Total Direct True Positives: 8\n",
      "TPs': ['0->0', '4->0', '1->1', '2->1', '2->2', '4->2', '3->3', '4->3', '4->4']\n",
      "FPs': ['4->1', '3->4']\n",
      "TPs direct: ['0->0', '4->0', '1->1', '2->1', '2->2', '3->3', '4->3', '4->4']\n",
      "FPs direct: ['4->1', '4->2', '3->4']\n",
      "FNs: ['1->0', '3->2']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 0.8181818181818182\n",
      "Recall': 0.8181818181818182\n",
      "F1' score: 0.8181818181818182\n",
      "(includes only direct causal relationships)\n",
      "Precision: 0.7272727272727273\n",
      "Recall: 0.8\n",
      "F1 score: 0.761904761904762\n",
      "Percentage of delays that are correctly discovered: 75.0%\n",
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=32, out_features=256, bias=True)\n",
      "      (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wk): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=4, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=512, bias=True)\n",
      "          (linear2): Linear(in_features=512, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 145515\n",
      "===================Running===================\n",
      "ground_truth:data/fMRI/sim9_gt_processed.csv\n",
      "===================Results===================\n",
      "0 causes 0 with a delay of 21 time steps.\n",
      "4 causes 0 with a delay of 25 time steps.\n",
      "1 causes 1 with a delay of 21 time steps.\n",
      "2 causes 2 with a delay of 1 time steps.\n",
      "4 causes 2 with a delay of 25 time steps.\n",
      "3 causes 3 with a delay of 1 time steps.\n",
      "4 causes 4 with a delay of 1 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 0\n",
      "Total True Positives': 7\n",
      "Total False Negatives: 4\n",
      "Total Direct False Positives: 1\n",
      "Total Direct True Positives: 6\n",
      "TPs': ['0->0', '4->0', '1->1', '2->2', '4->2', '3->3', '4->4']\n",
      "FPs': []\n",
      "TPs direct: ['0->0', '4->0', '1->1', '2->2', '3->3', '4->4']\n",
      "FPs direct: ['4->2']\n",
      "FNs: ['1->0', '2->1', '3->2', '4->3']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 1.0\n",
      "Recall': 0.6363636363636364\n",
      "F1' score: 0.7777777777777778\n",
      "(includes only direct causal relationships)\n",
      "Precision: 0.8571428571428571\n",
      "Recall: 0.6\n",
      "F1 score: 0.7058823529411764\n",
      "Percentage of delays that are correctly discovered: 50.0%\n",
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=32, out_features=256, bias=True)\n",
      "      (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wk): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=4, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=512, bias=True)\n",
      "          (linear2): Linear(in_features=512, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 145515\n",
      "===================Running===================\n",
      "ground_truth:data/fMRI/sim10_gt_processed.csv\n",
      "===================Results===================\n",
      "0 causes 0 with a delay of 5 time steps.\n",
      "1 causes 1 with a delay of 1 time steps.\n",
      "2 causes 2 with a delay of 5 time steps.\n",
      "3 causes 3 with a delay of 13 time steps.\n",
      "4 causes 4 with a delay of 16 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 0\n",
      "Total True Positives': 5\n",
      "Total False Negatives: 5\n",
      "Total Direct False Positives: 0\n",
      "Total Direct True Positives: 5\n",
      "TPs': ['0->0', '1->1', '2->2', '3->3', '4->4']\n",
      "FPs': []\n",
      "TPs direct: ['0->0', '1->1', '2->2', '3->3', '4->4']\n",
      "FPs direct: []\n",
      "FNs: ['1->0', '4->0', '2->1', '3->2', '4->3']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 1.0\n",
      "Recall': 0.5\n",
      "F1' score: 0.6666666666666666\n",
      "(includes only direct causal relationships)\n",
      "Precision: 1.0\n",
      "Recall: 0.5\n",
      "F1 score: 0.6666666666666666\n",
      "Percentage of delays that are correctly discovered: 20.0%\n",
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=32, out_features=256, bias=True)\n",
      "      (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wk): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=4, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=512, bias=True)\n",
      "          (linear2): Linear(in_features=512, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 155415\n",
      "===================Running===================\n",
      "ground_truth:data/fMRI/sim11_gt_processed.csv\n",
      "===================Results===================\n",
      "0 causes 0 with a delay of 1 time steps.\n",
      "2 causes 0 with a delay of 1 time steps.\n",
      "0 causes 1 with a delay of 1 time steps.\n",
      "1 causes 1 with a delay of 1 time steps.\n",
      "2 causes 2 with a delay of 1 time steps.\n",
      "3 causes 2 with a delay of 1 time steps.\n",
      "3 causes 3 with a delay of 1 time steps.\n",
      "2 causes 4 with a delay of 1 time steps.\n",
      "4 causes 4 with a delay of 1 time steps.\n",
      "5 causes 5 with a delay of 1 time steps.\n",
      "6 causes 6 with a delay of 1 time steps.\n",
      "0 causes 7 with a delay of 1 time steps.\n",
      "7 causes 7 with a delay of 1 time steps.\n",
      "8 causes 8 with a delay of 1 time steps.\n",
      "9 causes 9 with a delay of 1 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 3\n",
      "Total True Positives': 12\n",
      "Total False Negatives: 10\n",
      "Total Direct False Positives: 4\n",
      "Total Direct True Positives: 11\n",
      "TPs': ['0->0', '2->0', '1->1', '2->2', '3->2', '3->3', '4->4', '5->5', '6->6', '7->7', '8->8', '9->9']\n",
      "FPs': ['0->1', '2->4', '0->7']\n",
      "TPs direct: ['0->0', '1->1', '2->2', '3->2', '3->3', '4->4', '5->5', '6->6', '7->7', '8->8', '9->9']\n",
      "FPs direct: ['2->0', '0->1', '2->4', '0->7']\n",
      "FNs: ['1->0', '4->0', '2->1', '7->2', '4->3', '6->5', '9->5', '7->6', '8->7', '9->8']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 0.8\n",
      "Recall': 0.5454545454545454\n",
      "F1' score: 0.6486486486486486\n",
      "(includes only direct causal relationships)\n",
      "Precision: 0.7333333333333333\n",
      "Recall: 0.5238095238095238\n",
      "F1 score: 0.611111111111111\n",
      "Percentage of delays that are correctly discovered: 100.0%\n",
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=32, out_features=256, bias=True)\n",
      "      (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wk): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=4, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=512, bias=True)\n",
      "          (linear2): Linear(in_features=512, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 155415\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "===================Running===================\n",
      "ground_truth:data/fMRI/sim12_gt_processed.csv\n",
      "===================Results===================\n",
      "0 causes 0 with a delay of 1 time steps.\n",
      "2 causes 0 with a delay of 26 time steps.\n",
      "0 causes 1 with a delay of 2 time steps.\n",
      "1 causes 1 with a delay of 1 time steps.\n",
      "2 causes 1 with a delay of 1 time steps.\n",
      "1 causes 2 with a delay of 10 time steps.\n",
      "2 causes 2 with a delay of 1 time steps.\n",
      "3 causes 3 with a delay of 1 time steps.\n",
      "2 causes 4 with a delay of 1 time steps.\n",
      "4 causes 4 with a delay of 1 time steps.\n",
      "0 causes 5 with a delay of 19 time steps.\n",
      "5 causes 5 with a delay of 1 time steps.\n",
      "5 causes 6 with a delay of 1 time steps.\n",
      "6 causes 6 with a delay of 1 time steps.\n",
      "2 causes 7 with a delay of 1 time steps.\n",
      "7 causes 7 with a delay of 1 time steps.\n",
      "8 causes 8 with a delay of 1 time steps.\n",
      "9 causes 8 with a delay of 1 time steps.\n",
      "5 causes 9 with a delay of 1 time steps.\n",
      "9 causes 9 with a delay of 1 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 7\n",
      "Total True Positives': 13\n",
      "Total False Negatives: 9\n",
      "Total Direct False Positives: 8\n",
      "Total Direct True Positives: 12\n",
      "TPs': ['0->0', '2->0', '1->1', '2->1', '2->2', '3->3', '4->4', '5->5', '6->6', '7->7', '8->8', '9->8', '9->9']\n",
      "FPs': ['0->1', '1->2', '2->4', '0->5', '5->6', '2->7', '5->9']\n",
      "TPs direct: ['0->0', '1->1', '2->1', '2->2', '3->3', '4->4', '5->5', '6->6', '7->7', '8->8', '9->8', '9->9']\n",
      "FPs direct: ['2->0', '0->1', '1->2', '2->4', '0->5', '5->6', '2->7', '5->9']\n",
      "FNs: ['1->0', '4->0', '3->2', '7->2', '4->3', '6->5', '9->5', '7->6', '8->7']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 0.65\n",
      "Recall': 0.5909090909090909\n",
      "F1' score: 0.6190476190476191\n",
      "(includes only direct causal relationships)\n",
      "Precision: 0.6\n",
      "Recall: 0.5714285714285714\n",
      "F1 score: 0.5853658536585366\n",
      "Percentage of delays that are correctly discovered: 100.0%\n",
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=32, out_features=256, bias=True)\n",
      "      (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wk): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=4, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=512, bias=True)\n",
      "          (linear2): Linear(in_features=512, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 145515\n",
      "===================Running===================\n",
      "ground_truth:data/fMRI/sim13_gt_processed.csv\n",
      "===================Results===================\n",
      "0 causes 0 with a delay of 30 time steps.\n",
      "1 causes 1 with a delay of 30 time steps.\n",
      "2 causes 2 with a delay of 5 time steps.\n",
      "3 causes 3 with a delay of 7 time steps.\n",
      "4 causes 4 with a delay of 7 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 0\n",
      "Total True Positives': 5\n",
      "Total False Negatives: 8\n",
      "Total Direct False Positives: 0\n",
      "Total Direct True Positives: 5\n",
      "TPs': ['0->0', '1->1', '2->2', '3->3', '4->4']\n",
      "FPs': []\n",
      "TPs direct: ['0->0', '1->1', '2->2', '3->3', '4->4']\n",
      "FPs direct: []\n",
      "FNs: ['1->0', '4->0', '0->1', '2->1', '1->2', '3->2', '4->3', '3->4']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 1.0\n",
      "Recall': 0.38461538461538464\n",
      "F1' score: 0.5555555555555556\n",
      "(includes only direct causal relationships)\n",
      "Precision: 1.0\n",
      "Recall: 0.38461538461538464\n",
      "F1 score: 0.5555555555555556\n",
      "Percentage of delays that are correctly discovered: 0.0%\n",
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=32, out_features=256, bias=True)\n",
      "      (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wk): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=4, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=512, bias=True)\n",
      "          (linear2): Linear(in_features=512, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 145515\n",
      "===================Running===================\n",
      "ground_truth:data/fMRI/sim14_gt_processed.csv\n",
      "===================Results===================\n",
      "0 causes 0 with a delay of 31 time steps.\n",
      "4 causes 0 with a delay of 27 time steps.\n",
      "0 causes 1 with a delay of 25 time steps.\n",
      "1 causes 1 with a delay of 14 time steps.\n",
      "1 causes 2 with a delay of 1 time steps.\n",
      "2 causes 2 with a delay of 31 time steps.\n",
      "3 causes 2 with a delay of 1 time steps.\n",
      "3 causes 3 with a delay of 1 time steps.\n",
      "1 causes 4 with a delay of 1 time steps.\n",
      "3 causes 4 with a delay of 29 time steps.\n",
      "4 causes 4 with a delay of 22 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 4\n",
      "Total True Positives': 7\n",
      "Total False Negatives: 4\n",
      "Total Direct False Positives: 5\n",
      "Total Direct True Positives: 6\n",
      "TPs': ['0->0', '1->1', '2->2', '3->2', '3->3', '1->4', '4->4']\n",
      "FPs': ['4->0', '0->1', '1->2', '3->4']\n",
      "TPs direct: ['0->0', '1->1', '2->2', '3->2', '3->3', '4->4']\n",
      "FPs direct: ['4->0', '0->1', '1->2', '1->4', '3->4']\n",
      "FNs: ['1->0', '2->1', '4->3', '0->4']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 0.6363636363636364\n",
      "Recall': 0.6363636363636364\n",
      "F1' score: 0.6363636363636364\n",
      "(includes only direct causal relationships)\n",
      "Precision: 0.5454545454545454\n",
      "Recall: 0.6\n",
      "F1 score: 0.5714285714285713\n",
      "Percentage of delays that are correctly discovered: 33.33333333333333%\n",
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=32, out_features=256, bias=True)\n",
      "      (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wk): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=4, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=512, bias=True)\n",
      "          (linear2): Linear(in_features=512, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 145515\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "===================Running===================\n",
      "ground_truth:data/fMRI/sim15_gt_processed.csv\n",
      "===================Results===================\n",
      "0 causes 0 with a delay of 1 time steps.\n",
      "2 causes 0 with a delay of 1 time steps.\n",
      "4 causes 0 with a delay of 1 time steps.\n",
      "1 causes 1 with a delay of 1 time steps.\n",
      "2 causes 1 with a delay of 15 time steps.\n",
      "2 causes 2 with a delay of 1 time steps.\n",
      "3 causes 2 with a delay of 1 time steps.\n",
      "4 causes 2 with a delay of 1 time steps.\n",
      "3 causes 3 with a delay of 1 time steps.\n",
      "4 causes 3 with a delay of 1 time steps.\n",
      "4 causes 4 with a delay of 1 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 0\n",
      "Total True Positives': 11\n",
      "Total False Negatives: 1\n",
      "Total Direct False Positives: 2\n",
      "Total Direct True Positives: 9\n",
      "TPs': ['0->0', '2->0', '4->0', '1->1', '2->1', '2->2', '3->2', '4->2', '3->3', '4->3', '4->4']\n",
      "FPs': []\n",
      "TPs direct: ['0->0', '4->0', '1->1', '2->1', '2->2', '3->2', '3->3', '4->3', '4->4']\n",
      "FPs direct: ['2->0', '4->2']\n",
      "FNs: ['1->0']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 1.0\n",
      "Recall': 0.9166666666666666\n",
      "F1' score: 0.9565217391304348\n",
      "(includes only direct causal relationships)\n",
      "Precision: 0.8181818181818182\n",
      "Recall: 0.9\n",
      "F1 score: 0.8571428571428572\n",
      "Percentage of delays that are correctly discovered: 88.88888888888889%\n",
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=32, out_features=256, bias=True)\n",
      "      (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wk): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=4, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=512, bias=True)\n",
      "          (linear2): Linear(in_features=512, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 145515\n",
      "===================Running===================\n",
      "ground_truth:data/fMRI/sim16_gt_processed.csv\n",
      "===================Results===================\n",
      "0 causes 0 with a delay of 17 time steps.\n",
      "0 causes 1 with a delay of 1 time steps.\n",
      "1 causes 1 with a delay of 13 time steps.\n",
      "2 causes 2 with a delay of 19 time steps.\n",
      "3 causes 2 with a delay of 11 time steps.\n",
      "3 causes 3 with a delay of 9 time steps.\n",
      "4 causes 4 with a delay of 12 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 1\n",
      "Total True Positives': 6\n",
      "Total False Negatives: 6\n",
      "Total Direct False Positives: 1\n",
      "Total Direct True Positives: 6\n",
      "TPs': ['0->0', '1->1', '2->2', '3->2', '3->3', '4->4']\n",
      "FPs': ['0->1']\n",
      "TPs direct: ['0->0', '1->1', '2->2', '3->2', '3->3', '4->4']\n",
      "FPs direct: ['0->1']\n",
      "FNs: ['1->0', '4->0', '2->1', '3->1', '4->2', '4->3']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 0.8571428571428571\n",
      "Recall': 0.5\n",
      "F1' score: 0.631578947368421\n",
      "(includes only direct causal relationships)\n",
      "Precision: 0.8571428571428571\n",
      "Recall: 0.5\n",
      "F1 score: 0.631578947368421\n",
      "Percentage of delays that are correctly discovered: 0.0%\n",
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=32, out_features=256, bias=True)\n",
      "      (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wk): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=4, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=512, bias=True)\n",
      "          (linear2): Linear(in_features=512, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 155415\n",
      "===================Running===================\n",
      "ground_truth:data/fMRI/sim17_gt_processed.csv\n",
      "===================Results===================\n",
      "0 causes 0 with a delay of 1 time steps.\n",
      "2 causes 0 with a delay of 1 time steps.\n",
      "0 causes 1 with a delay of 6 time steps.\n",
      "1 causes 1 with a delay of 1 time steps.\n",
      "2 causes 1 with a delay of 5 time steps.\n",
      "2 causes 2 with a delay of 1 time steps.\n",
      "3 causes 2 with a delay of 1 time steps.\n",
      "3 causes 3 with a delay of 6 time steps.\n",
      "4 causes 3 with a delay of 6 time steps.\n",
      "0 causes 4 with a delay of 1 time steps.\n",
      "2 causes 4 with a delay of 1 time steps.\n",
      "3 causes 4 with a delay of 1 time steps.\n",
      "4 causes 4 with a delay of 5 time steps.\n",
      "5 causes 5 with a delay of 6 time steps.\n",
      "5 causes 6 with a delay of 1 time steps.\n",
      "6 causes 6 with a delay of 1 time steps.\n",
      "7 causes 6 with a delay of 1 time steps.\n",
      "3 causes 7 with a delay of 1 time steps.\n",
      "5 causes 7 with a delay of 1 time steps.\n",
      "7 causes 7 with a delay of 1 time steps.\n",
      "8 causes 8 with a delay of 1 time steps.\n",
      "9 causes 8 with a delay of 1 time steps.\n",
      "3 causes 9 with a delay of 7 time steps.\n",
      "5 causes 9 with a delay of 1 time steps.\n",
      "9 causes 9 with a delay of 1 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 9\n",
      "Total True Positives': 16\n",
      "Total False Negatives: 6\n",
      "Total Direct False Positives: 10\n",
      "Total Direct True Positives: 15\n",
      "TPs': ['0->0', '2->0', '1->1', '2->1', '2->2', '3->2', '3->3', '4->3', '4->4', '5->5', '6->6', '7->6', '7->7', '8->8', '9->8', '9->9']\n",
      "FPs': ['0->1', '0->4', '2->4', '3->4', '5->6', '3->7', '5->7', '3->9', '5->9']\n",
      "TPs direct: ['0->0', '1->1', '2->1', '2->2', '3->2', '3->3', '4->3', '4->4', '5->5', '6->6', '7->6', '7->7', '8->8', '9->8', '9->9']\n",
      "FPs direct: ['2->0', '0->1', '0->4', '2->4', '3->4', '5->6', '3->7', '5->7', '3->9', '5->9']\n",
      "FNs: ['1->0', '4->0', '7->2', '6->5', '9->5', '8->7']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 0.64\n",
      "Recall': 0.7272727272727273\n",
      "F1' score: 0.6808510638297872\n",
      "(includes only direct causal relationships)\n",
      "Precision: 0.6\n",
      "Recall: 0.7142857142857143\n",
      "F1 score: 0.6521739130434783\n",
      "Percentage of delays that are correctly discovered: 66.66666666666666%\n",
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=32, out_features=256, bias=True)\n",
      "      (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wk): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=4, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=512, bias=True)\n",
      "          (linear2): Linear(in_features=512, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 145515\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "===================Running===================\n",
      "ground_truth:data/fMRI/sim18_gt_processed.csv\n",
      "===================Results===================\n",
      "0 causes 0 with a delay of 1 time steps.\n",
      "2 causes 0 with a delay of 1 time steps.\n",
      "0 causes 1 with a delay of 11 time steps.\n",
      "1 causes 1 with a delay of 1 time steps.\n",
      "2 causes 2 with a delay of 1 time steps.\n",
      "3 causes 2 with a delay of 1 time steps.\n",
      "3 causes 3 with a delay of 1 time steps.\n",
      "4 causes 3 with a delay of 1 time steps.\n",
      "3 causes 4 with a delay of 1 time steps.\n",
      "4 causes 4 with a delay of 1 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 2\n",
      "Total True Positives': 8\n",
      "Total False Negatives: 3\n",
      "Total Direct False Positives: 3\n",
      "Total Direct True Positives: 7\n",
      "TPs': ['0->0', '2->0', '1->1', '2->2', '3->2', '3->3', '4->3', '4->4']\n",
      "FPs': ['0->1', '3->4']\n",
      "TPs direct: ['0->0', '1->1', '2->2', '3->2', '3->3', '4->3', '4->4']\n",
      "FPs direct: ['2->0', '0->1', '3->4']\n",
      "FNs: ['1->0', '4->0', '2->1']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 0.8\n",
      "Recall': 0.7272727272727273\n",
      "F1' score: 0.761904761904762\n",
      "(includes only direct causal relationships)\n",
      "Precision: 0.7\n",
      "Recall: 0.7\n",
      "F1 score: 0.7\n",
      "Percentage of delays that are correctly discovered: 100.0%\n",
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=32, out_features=256, bias=True)\n",
      "      (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wk): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=4, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=512, bias=True)\n",
      "          (linear2): Linear(in_features=512, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 145515\n",
      "===================Running===================\n",
      "ground_truth:data/fMRI/sim19_gt_processed.csv\n",
      "===================Results===================\n",
      "0 causes 0 with a delay of 5 time steps.\n",
      "1 causes 1 with a delay of 4 time steps.\n",
      "2 causes 2 with a delay of 4 time steps.\n",
      "3 causes 3 with a delay of 4 time steps.\n",
      "4 causes 4 with a delay of 4 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 0\n",
      "Total True Positives': 5\n",
      "Total False Negatives: 5\n",
      "Total Direct False Positives: 0\n",
      "Total Direct True Positives: 5\n",
      "TPs': ['0->0', '1->1', '2->2', '3->3', '4->4']\n",
      "FPs': []\n",
      "TPs direct: ['0->0', '1->1', '2->2', '3->3', '4->4']\n",
      "FPs direct: []\n",
      "FNs: ['1->0', '4->0', '2->1', '3->2', '4->3']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 1.0\n",
      "Recall': 0.5\n",
      "F1' score: 0.6666666666666666\n",
      "(includes only direct causal relationships)\n",
      "Precision: 1.0\n",
      "Recall: 0.5\n",
      "F1 score: 0.6666666666666666\n",
      "Percentage of delays that are correctly discovered: 0.0%\n",
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=32, out_features=256, bias=True)\n",
      "      (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wk): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=4, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=512, bias=True)\n",
      "          (linear2): Linear(in_features=512, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 145515\n",
      "===================Running===================\n",
      "ground_truth:data/fMRI/sim20_gt_processed.csv\n",
      "===================Results===================\n",
      "0 causes 0 with a delay of 4 time steps.\n",
      "1 causes 1 with a delay of 4 time steps.\n",
      "2 causes 2 with a delay of 4 time steps.\n",
      "3 causes 3 with a delay of 4 time steps.\n",
      "4 causes 4 with a delay of 4 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 0\n",
      "Total True Positives': 5\n",
      "Total False Negatives: 5\n",
      "Total Direct False Positives: 0\n",
      "Total Direct True Positives: 5\n",
      "TPs': ['0->0', '1->1', '2->2', '3->3', '4->4']\n",
      "FPs': []\n",
      "TPs direct: ['0->0', '1->1', '2->2', '3->3', '4->4']\n",
      "FPs direct: []\n",
      "FNs: ['1->0', '4->0', '2->1', '3->2', '4->3']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 1.0\n",
      "Recall': 0.5\n",
      "F1' score: 0.6666666666666666\n",
      "(includes only direct causal relationships)\n",
      "Precision: 1.0\n",
      "Recall: 0.5\n",
      "F1 score: 0.6666666666666666\n",
      "Percentage of delays that are correctly discovered: 0.0%\n",
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=32, out_features=256, bias=True)\n",
      "      (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wk): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=4, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=512, bias=True)\n",
      "          (linear2): Linear(in_features=512, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 145515\n",
      "===================Running===================\n",
      "ground_truth:data/fMRI/sim21_gt_processed.csv\n",
      "===================Results===================\n",
      "0 causes 0 with a delay of 18 time steps.\n",
      "2 causes 0 with a delay of 1 time steps.\n",
      "0 causes 1 with a delay of 1 time steps.\n",
      "1 causes 1 with a delay of 1 time steps.\n",
      "2 causes 1 with a delay of 1 time steps.\n",
      "2 causes 2 with a delay of 1 time steps.\n",
      "3 causes 2 with a delay of 1 time steps.\n",
      "3 causes 3 with a delay of 28 time steps.\n",
      "4 causes 3 with a delay of 6 time steps.\n",
      "3 causes 4 with a delay of 1 time steps.\n",
      "4 causes 4 with a delay of 9 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 2\n",
      "Total True Positives': 9\n",
      "Total False Negatives: 2\n",
      "Total Direct False Positives: 3\n",
      "Total Direct True Positives: 8\n",
      "TPs': ['0->0', '2->0', '1->1', '2->1', '2->2', '3->2', '3->3', '4->3', '4->4']\n",
      "FPs': ['0->1', '3->4']\n",
      "TPs direct: ['0->0', '1->1', '2->1', '2->2', '3->2', '3->3', '4->3', '4->4']\n",
      "FPs direct: ['2->0', '0->1', '3->4']\n",
      "FNs: ['1->0', '4->0']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 0.8181818181818182\n",
      "Recall': 0.8181818181818182\n",
      "F1' score: 0.8181818181818182\n",
      "(includes only direct causal relationships)\n",
      "Precision: 0.7272727272727273\n",
      "Recall: 0.8\n",
      "F1 score: 0.761904761904762\n",
      "Percentage of delays that are correctly discovered: 50.0%\n",
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=32, out_features=256, bias=True)\n",
      "      (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wk): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=4, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=512, bias=True)\n",
      "          (linear2): Linear(in_features=512, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 145515\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "===================Running===================\n",
      "ground_truth:data/fMRI/sim22_gt_processed.csv\n",
      "===================Results===================\n",
      "0 causes 0 with a delay of 29 time steps.\n",
      "1 causes 1 with a delay of 5 time steps.\n",
      "2 causes 2 with a delay of 28 time steps.\n",
      "3 causes 3 with a delay of 30 time steps.\n",
      "4 causes 4 with a delay of 19 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 0\n",
      "Total True Positives': 5\n",
      "Total False Negatives: 5\n",
      "Total Direct False Positives: 0\n",
      "Total Direct True Positives: 5\n",
      "TPs': ['0->0', '1->1', '2->2', '3->3', '4->4']\n",
      "FPs': []\n",
      "TPs direct: ['0->0', '1->1', '2->2', '3->3', '4->4']\n",
      "FPs direct: []\n",
      "FNs: ['1->0', '4->0', '2->1', '3->2', '4->3']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 1.0\n",
      "Recall': 0.5\n",
      "F1' score: 0.6666666666666666\n",
      "(includes only direct causal relationships)\n",
      "Precision: 1.0\n",
      "Recall: 0.5\n",
      "F1 score: 0.6666666666666666\n",
      "Percentage of delays that are correctly discovered: 0.0%\n",
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=32, out_features=256, bias=True)\n",
      "      (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wk): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=4, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=512, bias=True)\n",
      "          (linear2): Linear(in_features=512, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 145515\n",
      "===================Running===================\n",
      "ground_truth:data/fMRI/sim23_gt_processed.csv\n",
      "===================Results===================\n",
      "0 causes 0 with a delay of 1 time steps.\n",
      "1 causes 1 with a delay of 1 time steps.\n",
      "2 causes 2 with a delay of 1 time steps.\n",
      "3 causes 3 with a delay of 1 time steps.\n",
      "4 causes 4 with a delay of 1 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 0\n",
      "Total True Positives': 5\n",
      "Total False Negatives: 5\n",
      "Total Direct False Positives: 0\n",
      "Total Direct True Positives: 5\n",
      "TPs': ['0->0', '1->1', '2->2', '3->3', '4->4']\n",
      "FPs': []\n",
      "TPs direct: ['0->0', '1->1', '2->2', '3->3', '4->4']\n",
      "FPs direct: []\n",
      "FNs: ['1->0', '4->0', '2->1', '3->2', '4->3']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 1.0\n",
      "Recall': 0.5\n",
      "F1' score: 0.6666666666666666\n",
      "(includes only direct causal relationships)\n",
      "Precision: 1.0\n",
      "Recall: 0.5\n",
      "F1 score: 0.6666666666666666\n",
      "Percentage of delays that are correctly discovered: 100.0%\n",
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=32, out_features=256, bias=True)\n",
      "      (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wk): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=4, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=512, bias=True)\n",
      "          (linear2): Linear(in_features=512, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 145515\n",
      "===================Running===================\n",
      "ground_truth:data/fMRI/sim24_gt_processed.csv\n",
      "===================Results===================\n",
      "0 causes 0 with a delay of 19 time steps.\n",
      "1 causes 1 with a delay of 1 time steps.\n",
      "2 causes 2 with a delay of 6 time steps.\n",
      "3 causes 3 with a delay of 7 time steps.\n",
      "4 causes 4 with a delay of 7 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 0\n",
      "Total True Positives': 5\n",
      "Total False Negatives: 5\n",
      "Total Direct False Positives: 0\n",
      "Total Direct True Positives: 5\n",
      "TPs': ['0->0', '1->1', '2->2', '3->3', '4->4']\n",
      "FPs': []\n",
      "TPs direct: ['0->0', '1->1', '2->2', '3->3', '4->4']\n",
      "FPs direct: []\n",
      "FNs: ['1->0', '4->0', '2->1', '3->2', '4->3']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 1.0\n",
      "Recall': 0.5\n",
      "F1' score: 0.6666666666666666\n",
      "(includes only direct causal relationships)\n",
      "Precision: 1.0\n",
      "Recall: 0.5\n",
      "F1 score: 0.6666666666666666\n",
      "Percentage of delays that are correctly discovered: 20.0%\n",
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=32, out_features=256, bias=True)\n",
      "      (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wk): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=4, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=512, bias=True)\n",
      "          (linear2): Linear(in_features=512, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 145515\n",
      "===================Running===================\n",
      "ground_truth:data/fMRI/sim25_gt_processed.csv\n",
      "===================Results===================\n",
      "0 causes 0 with a delay of 16 time steps.\n",
      "0 causes 1 with a delay of 4 time steps.\n",
      "1 causes 1 with a delay of 30 time steps.\n",
      "2 causes 2 with a delay of 23 time steps.\n",
      "3 causes 2 with a delay of 5 time steps.\n",
      "3 causes 3 with a delay of 13 time steps.\n",
      "4 causes 4 with a delay of 5 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 1\n",
      "Total True Positives': 6\n",
      "Total False Negatives: 4\n",
      "Total Direct False Positives: 1\n",
      "Total Direct True Positives: 6\n",
      "TPs': ['0->0', '1->1', '2->2', '3->2', '3->3', '4->4']\n",
      "FPs': ['0->1']\n",
      "TPs direct: ['0->0', '1->1', '2->2', '3->2', '3->3', '4->4']\n",
      "FPs direct: ['0->1']\n",
      "FNs: ['1->0', '4->0', '2->1', '4->3']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 0.8571428571428571\n",
      "Recall': 0.6\n",
      "F1' score: 0.7058823529411764\n",
      "(includes only direct causal relationships)\n",
      "Precision: 0.8571428571428571\n",
      "Recall: 0.6\n",
      "F1 score: 0.7058823529411764\n",
      "Percentage of delays that are correctly discovered: 0.0%\n",
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=32, out_features=256, bias=True)\n",
      "      (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wk): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=4, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=512, bias=True)\n",
      "          (linear2): Linear(in_features=512, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 145515\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "===================Running===================\n",
      "ground_truth:data/fMRI/sim26_gt_processed.csv\n",
      "===================Results===================\n",
      "0 causes 0 with a delay of 19 time steps.\n",
      "4 causes 0 with a delay of 18 time steps.\n",
      "1 causes 1 with a delay of 6 time steps.\n",
      "1 causes 2 with a delay of 30 time steps.\n",
      "4 causes 2 with a delay of 5 time steps.\n",
      "2 causes 3 with a delay of 14 time steps.\n",
      "4 causes 3 with a delay of 21 time steps.\n",
      "1 causes 4 with a delay of 18 time steps.\n",
      "4 causes 4 with a delay of 21 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 3\n",
      "Total True Positives': 6\n",
      "Total False Negatives: 5\n",
      "Total Direct False Positives: 4\n",
      "Total Direct True Positives: 5\n",
      "TPs': ['0->0', '4->0', '1->1', '4->2', '4->3', '4->4']\n",
      "FPs': ['1->2', '2->3', '1->4']\n",
      "TPs direct: ['0->0', '4->0', '1->1', '4->3', '4->4']\n",
      "FPs direct: ['1->2', '4->2', '2->3', '1->4']\n",
      "FNs: ['1->0', '2->1', '2->2', '3->2', '3->3']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 0.6666666666666666\n",
      "Recall': 0.5454545454545454\n",
      "F1' score: 0.6\n",
      "(includes only direct causal relationships)\n",
      "Precision: 0.5555555555555556\n",
      "Recall: 0.5\n",
      "F1 score: 0.5263157894736842\n",
      "Percentage of delays that are correctly discovered: 0.0%\n",
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=32, out_features=256, bias=True)\n",
      "      (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wk): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=4, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=512, bias=True)\n",
      "          (linear2): Linear(in_features=512, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 145515\n",
      "===================Running===================\n",
      "ground_truth:data/fMRI/sim27_gt_processed.csv\n",
      "===================Results===================\n",
      "0 causes 0 with a delay of 1 time steps.\n",
      "1 causes 0 with a delay of 22 time steps.\n",
      "0 causes 1 with a delay of 23 time steps.\n",
      "1 causes 1 with a delay of 6 time steps.\n",
      "1 causes 2 with a delay of 18 time steps.\n",
      "4 causes 2 with a delay of 21 time steps.\n",
      "1 causes 3 with a delay of 15 time steps.\n",
      "4 causes 3 with a delay of 24 time steps.\n",
      "1 causes 4 with a delay of 18 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 4\n",
      "Total True Positives': 5\n",
      "Total False Negatives: 6\n",
      "Total Direct False Positives: 5\n",
      "Total Direct True Positives: 4\n",
      "TPs': ['0->0', '1->0', '1->1', '4->2', '4->3']\n",
      "FPs': ['0->1', '1->2', '1->3', '1->4']\n",
      "TPs direct: ['0->0', '1->0', '1->1', '4->3']\n",
      "FPs direct: ['0->1', '1->2', '4->2', '1->3', '1->4']\n",
      "FNs: ['4->0', '2->1', '2->2', '3->2', '3->3', '4->4']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 0.5555555555555556\n",
      "Recall': 0.45454545454545453\n",
      "F1' score: 0.5\n",
      "(includes only direct causal relationships)\n",
      "Precision: 0.4444444444444444\n",
      "Recall: 0.4\n",
      "F1 score: 0.4210526315789474\n",
      "Percentage of delays that are correctly discovered: 25.0%\n",
      "PredictModel(\n",
      "  (encoder): Encoder(\n",
      "    (emb): Embedding(\n",
      "      (feature_emb): Linear(in_features=32, out_features=256, bias=True)\n",
      "      (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "      (drop_out): Dropout(p=0, inplace=False)\n",
      "    )\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (qk): Clone()\n",
      "        (attention): MultiHeadAttention(\n",
      "          (attention): ScaleDotProductAttention(\n",
      "            (qk_mul): einsum()\n",
      "            (softmax): Softmax(dim=-1)\n",
      "            (hardmard_product): einsum()\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (Wq): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wk): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (Wv): CausalConv(\n",
      "            (mul): einsum()\n",
      "          )\n",
      "          (w_concat): Linear(in_features=4, out_features=1, bias=False)\n",
      "        )\n",
      "        (norm1): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout1): Dropout(p=0, inplace=False)\n",
      "        (ffn): PositionwiseFeedForward(\n",
      "          (linear1): Linear(in_features=1, out_features=512, bias=True)\n",
      "          (linear2): Linear(in_features=512, out_features=1, bias=True)\n",
      "          (activation): LeakyReLU(negative_slope=0.01)\n",
      "          (dropout): Dropout(p=0, inplace=False)\n",
      "        )\n",
      "        (norm2): LayerNorm((32, 1), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout2): Dropout(p=0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable parameters: 145515\n",
      "===================Running===================\n",
      "ground_truth:data/fMRI/sim28_gt_processed.csv\n",
      "===================Results===================\n",
      "0 causes 0 with a delay of 4 time steps.\n",
      "4 causes 0 with a delay of 1 time steps.\n",
      "0 causes 1 with a delay of 31 time steps.\n",
      "1 causes 1 with a delay of 31 time steps.\n",
      "2 causes 2 with a delay of 23 time steps.\n",
      "3 causes 2 with a delay of 5 time steps.\n",
      "3 causes 3 with a delay of 24 time steps.\n",
      "2 causes 4 with a delay of 30 time steps.\n",
      "4 causes 4 with a delay of 30 time steps.\n",
      "===================Evaluation===================\n",
      "Total False Positives': 2\n",
      "Total True Positives': 7\n",
      "Total False Negatives: 3\n",
      "Total Direct False Positives: 2\n",
      "Total Direct True Positives: 7\n",
      "TPs': ['0->0', '4->0', '1->1', '2->2', '3->2', '3->3', '4->4']\n",
      "FPs': ['0->1', '2->4']\n",
      "TPs direct: ['0->0', '4->0', '1->1', '2->2', '3->2', '3->3', '4->4']\n",
      "FPs direct: ['0->1', '2->4']\n",
      "FNs: ['1->0', '2->1', '4->3']\n",
      "(includes direct and indirect causal relationships)\n",
      "Precision': 0.7777777777777778\n",
      "Recall': 0.7\n",
      "F1' score: 0.7368421052631577\n",
      "(includes only direct causal relationships)\n",
      "Precision: 0.7777777777777778\n",
      "Recall: 0.7\n",
      "F1 score: 0.7368421052631577\n",
      "Percentage of delays that are correctly discovered: 14.285714285714285%\n",
      "===================Summary===================\n",
      "\t    Precision'   Recall'       F1'  Precision    Recall        F1       PoD\n",
      "\t1     0.818182  0.818182  0.818182   0.727273  0.800000  0.761905  0.500000\n",
      "\t2     0.636364  0.636364  0.636364   0.590909  0.619048  0.604651  0.846154\n",
      "\t3     0.894737  0.515152  0.653846   0.894737  0.515152  0.653846  0.058824\n",
      "\t4     1.000000  0.450450  0.621118   1.000000  0.450450  0.621118  0.340000\n",
      "\t5     1.000000  0.727273  0.842105   0.875000  0.700000  0.777778  0.571429\n",
      "\t6     0.857143  0.571429  0.685714   0.857143  0.571429  0.685714  0.750000\n",
      "\t7     0.666667  0.545455  0.600000   0.555556  0.500000  0.526316  0.000000\n",
      "\t8     0.818182  0.818182  0.818182   0.727273  0.800000  0.761905  0.750000\n",
      "\t9     1.000000  0.636364  0.777778   0.857143  0.600000  0.705882  0.500000\n",
      "\t10    1.000000  0.500000  0.666667   1.000000  0.500000  0.666667  0.200000\n",
      "\t11    0.800000  0.545455  0.648649   0.733333  0.523810  0.611111  1.000000\n",
      "\t12    0.650000  0.590909  0.619048   0.600000  0.571429  0.585366  1.000000\n",
      "\t13    1.000000  0.384615  0.555556   1.000000  0.384615  0.555556  0.000000\n",
      "\t14    0.636364  0.636364  0.636364   0.545455  0.600000  0.571429  0.333333\n",
      "\t15    1.000000  0.916667  0.956522   0.818182  0.900000  0.857143  0.888889\n",
      "\t16    0.857143  0.500000  0.631579   0.857143  0.500000  0.631579  0.000000\n",
      "\t17    0.640000  0.727273  0.680851   0.600000  0.714286  0.652174  0.666667\n",
      "\t18    0.800000  0.727273  0.761905   0.700000  0.700000  0.700000  1.000000\n",
      "\t19    1.000000  0.500000  0.666667   1.000000  0.500000  0.666667  0.000000\n",
      "\t20    1.000000  0.500000  0.666667   1.000000  0.500000  0.666667  0.000000\n",
      "\t21    0.818182  0.818182  0.818182   0.727273  0.800000  0.761905  0.500000\n",
      "\t22    1.000000  0.500000  0.666667   1.000000  0.500000  0.666667  0.000000\n",
      "\t23    1.000000  0.500000  0.666667   1.000000  0.500000  0.666667  1.000000\n",
      "\t24    1.000000  0.500000  0.666667   1.000000  0.500000  0.666667  0.200000\n",
      "\t25    0.857143  0.600000  0.705882   0.857143  0.600000  0.705882  0.000000\n",
      "\t26    0.666667  0.545455  0.600000   0.555556  0.500000  0.526316  0.000000\n",
      "\t27    0.555556  0.454545  0.500000   0.444444  0.400000  0.421053  0.250000\n",
      "\t28    0.777778  0.700000  0.736842   0.777778  0.700000  0.736842  0.142857\n"
     ]
    }
   ],
   "source": [
    "! python runner.py -c config/config_fMRI.json -t fMRI"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}