diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d04d206 --- /dev/null +++ b/.gitignore @@ -0,0 +1,132 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ +ckpt/ +results/ +DIFRINT_TEMP/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..17de8be --- /dev/null +++ b/README.md @@ -0,0 +1,97 @@ +

DUT: Learning Video Stabilization by Simply Watching Unstable Videos + Open In Colab +

+ +

+ Introduction | + Results Demo | + Installation | + Inference Code | + News | + Statement | + Reference +

+ +## Introduction + +

This repository contains the code, models, test results for the paper DUT: Learning Video Stabilization by Simply Watching Unstable Videos. It contains a keypoint detetcion module for robust keypoint detection, a motion propagation module for grid-based trajectories estimation, and a trajectory smoothing module for dynamic trajectory smoothing. The DUT model is totally unsupervised and just need unstable videos for training. + +

We have released the inference code, a pretrained model, which can be found in section inference code for more details. The inference code for DIFRINT and StabNet are also included in this repo.

+ + + +## Results Demo + +We test DUT on the NUS dataset and provide a result overview below (odd rows for unstable inputs and even rows for stabilized videos). + + + +For the motion propagation module, we propose a multi-homography estimation strategy instead of directly using RANSAC for single homography estimation to deal with multiple planes cases. The outliers recognized by RANSAC and the multi-planes identified with the multi-homography strategy are as follows: + + + +The trajectory smoothing module predicts dynamic kernels for iterative smoothing according to the trajectories themselves, the smoothing process can be viewed below: + + + +## Installation +Requirements: + +- Python 3.6.5+ +- Pytorch (version 1.4.0) +- Torchvision (version 0.5.0) + +1. Clone this repository + + `git clone https://github.com/Annbless/DUTCode.git` + +2. Go into the repository + + `cd DUTCode` + +3. Create conda environment and activate + + `conda create -n DUTCode python=3.6.5` + + `conda activate DUTCode` + +4. Install dependencies + + `conda install pytorch==1.4.0 torchvision==0.5.0 cudatoolkit=10.1 -c pytorch` + + `pip install -r requirements.txt` + +Our code has been tested with Python 3.6.5, Pytorch 1.4.0, Torchvision 0.5.0, CUDA 10.1 on Ubuntu 18.04. + + +## Inference Code + +Here we provide the procedure of testing on sample videos by our pretrained models: + +1. Download pretrained model and unzip them to ckpt folder + +2. Put clipped frames from unstable videos in the images folder (we have provided one example in the folder) + +3. Run the stabilization script + + `chmod +x scripts/*` + + `./scripts/deploy_samples.sh` + +4. The results of our DUT, DIFRINT, and StabNet are provided under the results folder + +## News +- [x] release inference code for the DUT model; +- [x] include inference code for the DIFRINT model, thanks to jinsc37; +- [x] include pytorch version StabNet model, thanks to cxjyxxme; +- [ ] include traditional stabilizers in python; +- [ ] release metrics calculation code; +- [ ] release training code for the DUT model; + +## Statement +This project is for research purpose only, please contact us for the licence of commercial use. For any other questions please contact [yuxu7116@uni.sydney.edu.au](mailto:yuxu7116@uni.sydney.edu.au). + +## Reference + +- DIFRINT +- StabNet (tensorflow version) diff --git a/configs/__init__.py b/configs/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/configs/config.py b/configs/config.py new file mode 100644 index 0000000..0fb2bf5 --- /dev/null +++ b/configs/config.py @@ -0,0 +1,89 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from easydict import EasyDict as edict + +__C = edict() +cfg = __C + +""" +Model options +""" +__C.MODEL = edict() + +# gaussian kernel size +__C.MODEL.GAUSSIAN_KSIZE = 15 + +# gaussian kernel sigma +__C.MODEL.GAUSSIAN_SIGMA = 0.5 + +# Descriptor Threshold +__C.MODEL.DES_THRSH = 1.0 + +# Coordinate Threshold +__C.MODEL.COO_THRSH = 5.0 + +# Ksize +__C.MODEL.KSIZE = 3 + +# padding +__C.MODEL.padding = 1 + +# dilation +__C.MODEL.dilation = 1 + +# scale_list +__C.MODEL.scale_list = [3.0, 5.0, 7.0, 9.0, 11.0, 13.0, 15.0, 17.0, 19.0, 21.0] + +# grid size +__C.MODEL.PIXELS = 16 + +# neighbor radius +__C.MODEL.RADIUS = 200 + +# input height +__C.MODEL.HEIGHT = 480 + +# input width +__C.MODEL.WIDTH = 640 + +# normalization amplitude in optical flow +__C.MODEL.FLOWC = 20 + +# cluster threshold +__C.MODEL.THRESHOLDPOINT = 102 + +""" +Training options +""" +__C.TRAIN = edict() + +# score strength weight +__C.TRAIN.score_com_strength = 100.0 + +# scale strength weight +__C.TRAIN.scale_com_strength = 100.0 + +# non maximum supression threshold +__C.TRAIN.NMS_THRESH = 0.0 + +# nms kernel size +__C.TRAIN.NMS_KSIZE = 5 + +# top k patch +__C.TRAIN.TOPK = 512 + +""" +Threshold options +""" +__C.Threshold = edict() + +__C.Threshold.MANG = 2 +__C.Threshold.ROT = 5 + +""" +Infer options +""" +__C.INFER = edict() +__C.INFER.ALLINFER = False diff --git a/demo/Cluster.gif b/demo/Cluster.gif new file mode 100644 index 0000000..8c6b5bb Binary files /dev/null and b/demo/Cluster.gif differ diff --git a/demo/NetworkStructure.png b/demo/NetworkStructure.png new file mode 100644 index 0000000..930d92d Binary files /dev/null and b/demo/NetworkStructure.png differ diff --git a/demo/RANSAC.gif b/demo/RANSAC.gif new file mode 100644 index 0000000..a08135e Binary files /dev/null and b/demo/RANSAC.gif differ diff --git a/demo/Trajectory.gif b/demo/Trajectory.gif new file mode 100644 index 0000000..b822abc Binary files /dev/null and b/demo/Trajectory.gif differ diff --git a/demo/overview1.gif b/demo/overview1.gif new file mode 100644 index 0000000..e579d25 Binary files /dev/null and b/demo/overview1.gif differ diff --git a/demo/overview2.gif b/demo/overview2.gif new file mode 100644 index 0000000..ec066c0 Binary files /dev/null and b/demo/overview2.gif differ diff --git a/demo/overview3.gif b/demo/overview3.gif new file mode 100644 index 0000000..465b4d5 Binary files /dev/null and b/demo/overview3.gif differ diff --git a/demo/overview4.gif b/demo/overview4.gif new file mode 100644 index 0000000..4ff5880 Binary files /dev/null and b/demo/overview4.gif differ diff --git a/images/0.jpg b/images/0.jpg new file mode 100644 index 0000000..4338630 Binary files /dev/null and b/images/0.jpg differ diff --git a/images/1.jpg b/images/1.jpg new file mode 100644 index 0000000..596e7cc Binary files /dev/null and b/images/1.jpg differ diff --git a/images/10.jpg b/images/10.jpg new file mode 100644 index 0000000..bf6ac1b Binary files /dev/null and b/images/10.jpg differ diff --git a/images/100.jpg b/images/100.jpg new file mode 100644 index 0000000..267a811 Binary files /dev/null and b/images/100.jpg differ diff --git a/images/101.jpg b/images/101.jpg new file mode 100644 index 0000000..69551cb Binary files /dev/null and b/images/101.jpg differ diff --git a/images/102.jpg b/images/102.jpg new file mode 100644 index 0000000..96a8b60 Binary files /dev/null and b/images/102.jpg differ diff --git a/images/103.jpg b/images/103.jpg new file mode 100644 index 0000000..b6ab38a Binary files /dev/null and b/images/103.jpg differ diff --git a/images/104.jpg b/images/104.jpg new file mode 100644 index 0000000..ea90029 Binary files /dev/null and b/images/104.jpg differ diff --git a/images/105.jpg b/images/105.jpg new file mode 100644 index 0000000..6485835 Binary files /dev/null and b/images/105.jpg differ diff --git a/images/106.jpg b/images/106.jpg new file mode 100644 index 0000000..dcf511b Binary files /dev/null and b/images/106.jpg differ diff --git a/images/107.jpg b/images/107.jpg new file mode 100644 index 0000000..6ff6d97 Binary files /dev/null and b/images/107.jpg differ diff --git a/images/108.jpg b/images/108.jpg new file mode 100644 index 0000000..c27ca45 Binary files /dev/null and b/images/108.jpg differ diff --git a/images/109.jpg b/images/109.jpg new file mode 100644 index 0000000..d430720 Binary files /dev/null and b/images/109.jpg differ diff --git a/images/11.jpg b/images/11.jpg new file mode 100644 index 0000000..57bf5ac Binary files /dev/null and b/images/11.jpg differ diff --git a/images/110.jpg b/images/110.jpg new file mode 100644 index 0000000..d44a63b Binary files /dev/null and b/images/110.jpg differ diff --git a/images/111.jpg b/images/111.jpg new file mode 100644 index 0000000..cbc811c Binary files /dev/null and b/images/111.jpg differ diff --git a/images/112.jpg b/images/112.jpg new file mode 100644 index 0000000..dd83bbd Binary files /dev/null and b/images/112.jpg differ diff --git a/images/113.jpg b/images/113.jpg new file mode 100644 index 0000000..b554719 Binary files /dev/null and b/images/113.jpg differ diff --git a/images/114.jpg b/images/114.jpg new file mode 100644 index 0000000..82bb9c5 Binary files /dev/null and b/images/114.jpg differ diff --git a/images/115.jpg b/images/115.jpg new file mode 100644 index 0000000..9a866bf Binary files /dev/null and b/images/115.jpg differ diff --git a/images/116.jpg b/images/116.jpg new file mode 100644 index 0000000..29e0896 Binary files /dev/null and b/images/116.jpg differ diff --git a/images/117.jpg b/images/117.jpg new file mode 100644 index 0000000..23c4fad Binary files /dev/null and b/images/117.jpg differ diff --git a/images/118.jpg b/images/118.jpg new file mode 100644 index 0000000..2369cbb Binary files /dev/null and b/images/118.jpg differ diff --git a/images/119.jpg b/images/119.jpg new file mode 100644 index 0000000..e6174c7 Binary files /dev/null and b/images/119.jpg differ diff --git a/images/12.jpg b/images/12.jpg new file mode 100644 index 0000000..b706aa6 Binary files /dev/null and b/images/12.jpg differ diff --git a/images/120.jpg b/images/120.jpg new file mode 100644 index 0000000..b6395f8 Binary files /dev/null and b/images/120.jpg differ diff --git a/images/121.jpg b/images/121.jpg new file mode 100644 index 0000000..7cc952d Binary files /dev/null and b/images/121.jpg differ diff --git a/images/122.jpg b/images/122.jpg new file mode 100644 index 0000000..3027a24 Binary files /dev/null and b/images/122.jpg differ diff --git a/images/123.jpg b/images/123.jpg new file mode 100644 index 0000000..b00bbe6 Binary files /dev/null and b/images/123.jpg differ diff --git a/images/124.jpg b/images/124.jpg new file mode 100644 index 0000000..5504319 Binary files /dev/null and b/images/124.jpg differ diff --git a/images/125.jpg b/images/125.jpg new file mode 100644 index 0000000..50d8231 Binary files /dev/null and b/images/125.jpg differ diff --git a/images/126.jpg b/images/126.jpg new file mode 100644 index 0000000..a6a5add Binary files /dev/null and b/images/126.jpg differ diff --git a/images/127.jpg b/images/127.jpg new file mode 100644 index 0000000..05c512f Binary files /dev/null and b/images/127.jpg differ diff --git a/images/128.jpg b/images/128.jpg new file mode 100644 index 0000000..d162052 Binary files /dev/null and b/images/128.jpg differ diff --git a/images/129.jpg b/images/129.jpg new file mode 100644 index 0000000..32fd722 Binary files /dev/null and b/images/129.jpg differ diff --git a/images/13.jpg b/images/13.jpg new file mode 100644 index 0000000..2e7721a Binary files /dev/null and b/images/13.jpg differ diff --git a/images/130.jpg b/images/130.jpg new file mode 100644 index 0000000..fa734de Binary files /dev/null and b/images/130.jpg differ diff --git a/images/131.jpg b/images/131.jpg new file mode 100644 index 0000000..fd4a07e Binary files /dev/null and b/images/131.jpg differ diff --git a/images/132.jpg b/images/132.jpg new file mode 100644 index 0000000..ad111a5 Binary files /dev/null and b/images/132.jpg differ diff --git a/images/133.jpg b/images/133.jpg new file mode 100644 index 0000000..44fbaca Binary files /dev/null and b/images/133.jpg differ diff --git a/images/134.jpg b/images/134.jpg new file mode 100644 index 0000000..389eddf Binary files /dev/null and b/images/134.jpg differ diff --git a/images/135.jpg b/images/135.jpg new file mode 100644 index 0000000..327b888 Binary files /dev/null and b/images/135.jpg differ diff --git a/images/136.jpg b/images/136.jpg new file mode 100644 index 0000000..7544117 Binary files /dev/null and b/images/136.jpg differ diff --git a/images/137.jpg b/images/137.jpg new file mode 100644 index 0000000..8922dd2 Binary files /dev/null and b/images/137.jpg differ diff --git a/images/138.jpg b/images/138.jpg new file mode 100644 index 0000000..bbe7413 Binary files /dev/null and b/images/138.jpg differ diff --git a/images/139.jpg b/images/139.jpg new file mode 100644 index 0000000..73621dc Binary files /dev/null and b/images/139.jpg differ diff --git a/images/14.jpg b/images/14.jpg new file mode 100644 index 0000000..3a1fcb2 Binary files /dev/null and b/images/14.jpg differ diff --git a/images/140.jpg b/images/140.jpg new file mode 100644 index 0000000..3204351 Binary files /dev/null and b/images/140.jpg differ diff --git a/images/141.jpg b/images/141.jpg new file mode 100644 index 0000000..b68a822 Binary files /dev/null and b/images/141.jpg differ diff --git a/images/142.jpg b/images/142.jpg new file mode 100644 index 0000000..e17ab70 Binary files /dev/null and b/images/142.jpg differ diff --git a/images/143.jpg b/images/143.jpg new file mode 100644 index 0000000..fa4c3b9 Binary files /dev/null and b/images/143.jpg differ diff --git a/images/144.jpg b/images/144.jpg new file mode 100644 index 0000000..8ece8d0 Binary files /dev/null and b/images/144.jpg differ diff --git a/images/145.jpg b/images/145.jpg new file mode 100644 index 0000000..02ccb7f Binary files /dev/null and b/images/145.jpg differ diff --git a/images/146.jpg b/images/146.jpg new file mode 100644 index 0000000..efecd8b Binary files /dev/null and b/images/146.jpg differ diff --git a/images/147.jpg b/images/147.jpg new file mode 100644 index 0000000..5b3ce7b Binary files /dev/null and b/images/147.jpg differ diff --git a/images/148.jpg b/images/148.jpg new file mode 100644 index 0000000..db5bc1f Binary files /dev/null and b/images/148.jpg differ diff --git a/images/149.jpg b/images/149.jpg new file mode 100644 index 0000000..3c33bd3 Binary files /dev/null and b/images/149.jpg differ diff --git a/images/15.jpg b/images/15.jpg new file mode 100644 index 0000000..b0d503b Binary files /dev/null and b/images/15.jpg differ diff --git a/images/150.jpg b/images/150.jpg new file mode 100644 index 0000000..bb37c38 Binary files /dev/null and b/images/150.jpg differ diff --git a/images/151.jpg b/images/151.jpg new file mode 100644 index 0000000..afde029 Binary files /dev/null and b/images/151.jpg differ diff --git a/images/152.jpg b/images/152.jpg new file mode 100644 index 0000000..7f50beb Binary files /dev/null and b/images/152.jpg differ diff --git a/images/153.jpg b/images/153.jpg new file mode 100644 index 0000000..6287d60 Binary files /dev/null and b/images/153.jpg differ diff --git a/images/154.jpg b/images/154.jpg new file mode 100644 index 0000000..2d31b5b Binary files /dev/null and b/images/154.jpg differ diff --git a/images/155.jpg b/images/155.jpg new file mode 100644 index 0000000..563dfdd Binary files /dev/null and b/images/155.jpg differ diff --git a/images/156.jpg b/images/156.jpg new file mode 100644 index 0000000..0019ce8 Binary files /dev/null and b/images/156.jpg differ diff --git a/images/157.jpg b/images/157.jpg new file mode 100644 index 0000000..a3a16bb Binary files /dev/null and b/images/157.jpg differ diff --git a/images/158.jpg b/images/158.jpg new file mode 100644 index 0000000..d8dc3ae Binary files /dev/null and b/images/158.jpg differ diff --git a/images/159.jpg b/images/159.jpg new file mode 100644 index 0000000..b793fe3 Binary files /dev/null and b/images/159.jpg differ diff --git a/images/16.jpg b/images/16.jpg new file mode 100644 index 0000000..6e293ce Binary files /dev/null and b/images/16.jpg differ diff --git a/images/160.jpg b/images/160.jpg new file mode 100644 index 0000000..098c2a5 Binary files /dev/null and b/images/160.jpg differ diff --git a/images/161.jpg b/images/161.jpg new file mode 100644 index 0000000..01d49b2 Binary files /dev/null and b/images/161.jpg differ diff --git a/images/162.jpg b/images/162.jpg new file mode 100644 index 0000000..6d41dec Binary files /dev/null and b/images/162.jpg differ diff --git a/images/163.jpg b/images/163.jpg new file mode 100644 index 0000000..1221b25 Binary files /dev/null and b/images/163.jpg differ diff --git a/images/164.jpg b/images/164.jpg new file mode 100644 index 0000000..f384b6b Binary files /dev/null and b/images/164.jpg differ diff --git a/images/165.jpg b/images/165.jpg new file mode 100644 index 0000000..59f033e Binary files /dev/null and b/images/165.jpg differ diff --git a/images/166.jpg b/images/166.jpg new file mode 100644 index 0000000..2c3ff71 Binary files /dev/null and b/images/166.jpg differ diff --git a/images/167.jpg b/images/167.jpg new file mode 100644 index 0000000..b06f403 Binary files /dev/null and b/images/167.jpg differ diff --git a/images/168.jpg b/images/168.jpg new file mode 100644 index 0000000..a1b950a Binary files /dev/null and b/images/168.jpg differ diff --git a/images/169.jpg b/images/169.jpg new file mode 100644 index 0000000..125aa8c Binary files /dev/null and b/images/169.jpg differ diff --git a/images/17.jpg b/images/17.jpg new file mode 100644 index 0000000..8cb0eec Binary files /dev/null and b/images/17.jpg differ diff --git a/images/170.jpg b/images/170.jpg new file mode 100644 index 0000000..1e7b718 Binary files /dev/null and b/images/170.jpg differ diff --git a/images/171.jpg b/images/171.jpg new file mode 100644 index 0000000..25043c2 Binary files /dev/null and b/images/171.jpg differ diff --git a/images/172.jpg b/images/172.jpg new file mode 100644 index 0000000..152e83c Binary files /dev/null and b/images/172.jpg differ diff --git a/images/173.jpg b/images/173.jpg new file mode 100644 index 0000000..8fa599c Binary files /dev/null and b/images/173.jpg differ diff --git a/images/174.jpg b/images/174.jpg new file mode 100644 index 0000000..cbd93bf Binary files /dev/null and b/images/174.jpg differ diff --git a/images/175.jpg b/images/175.jpg new file mode 100644 index 0000000..e9c1ae8 Binary files /dev/null and b/images/175.jpg differ diff --git a/images/176.jpg b/images/176.jpg new file mode 100644 index 0000000..01850cd Binary files /dev/null and b/images/176.jpg differ diff --git a/images/177.jpg b/images/177.jpg new file mode 100644 index 0000000..ed90279 Binary files /dev/null and b/images/177.jpg differ diff --git a/images/178.jpg b/images/178.jpg new file mode 100644 index 0000000..a7da4bc Binary files /dev/null and b/images/178.jpg differ diff --git a/images/179.jpg b/images/179.jpg new file mode 100644 index 0000000..02d522f Binary files /dev/null and b/images/179.jpg differ diff --git a/images/18.jpg b/images/18.jpg new file mode 100644 index 0000000..2642210 Binary files /dev/null and b/images/18.jpg differ diff --git a/images/180.jpg b/images/180.jpg new file mode 100644 index 0000000..5568f38 Binary files /dev/null and b/images/180.jpg differ diff --git a/images/181.jpg b/images/181.jpg new file mode 100644 index 0000000..d7c6351 Binary files /dev/null and b/images/181.jpg differ diff --git a/images/182.jpg b/images/182.jpg new file mode 100644 index 0000000..a9fa37b Binary files /dev/null and b/images/182.jpg differ diff --git a/images/183.jpg b/images/183.jpg new file mode 100644 index 0000000..fe25550 Binary files /dev/null and b/images/183.jpg differ diff --git a/images/184.jpg b/images/184.jpg new file mode 100644 index 0000000..f739dec Binary files /dev/null and b/images/184.jpg differ diff --git a/images/185.jpg b/images/185.jpg new file mode 100644 index 0000000..7f0a37b Binary files /dev/null and b/images/185.jpg differ diff --git a/images/186.jpg b/images/186.jpg new file mode 100644 index 0000000..75a0e24 Binary files /dev/null and b/images/186.jpg differ diff --git a/images/187.jpg b/images/187.jpg new file mode 100644 index 0000000..7b95f00 Binary files /dev/null and b/images/187.jpg differ diff --git a/images/188.jpg b/images/188.jpg new file mode 100644 index 0000000..4474722 Binary files /dev/null and b/images/188.jpg differ diff --git a/images/189.jpg b/images/189.jpg new file mode 100644 index 0000000..ee20326 Binary files /dev/null and b/images/189.jpg differ diff --git a/images/19.jpg b/images/19.jpg new file mode 100644 index 0000000..5dc7520 Binary files /dev/null and b/images/19.jpg differ diff --git a/images/190.jpg b/images/190.jpg new file mode 100644 index 0000000..67d09d3 Binary files /dev/null and b/images/190.jpg differ diff --git a/images/191.jpg b/images/191.jpg new file mode 100644 index 0000000..5733051 Binary files /dev/null and b/images/191.jpg differ diff --git a/images/192.jpg b/images/192.jpg new file mode 100644 index 0000000..b0f5ddb Binary files /dev/null and b/images/192.jpg differ diff --git a/images/193.jpg b/images/193.jpg new file mode 100644 index 0000000..aa06fca Binary files /dev/null and b/images/193.jpg differ diff --git a/images/194.jpg b/images/194.jpg new file mode 100644 index 0000000..23fdb9d Binary files /dev/null and b/images/194.jpg differ diff --git a/images/195.jpg b/images/195.jpg new file mode 100644 index 0000000..fcfe71d Binary files /dev/null and b/images/195.jpg differ diff --git a/images/196.jpg b/images/196.jpg new file mode 100644 index 0000000..25b687e Binary files /dev/null and b/images/196.jpg differ diff --git a/images/197.jpg b/images/197.jpg new file mode 100644 index 0000000..7930ea7 Binary files /dev/null and b/images/197.jpg differ diff --git a/images/198.jpg b/images/198.jpg new file mode 100644 index 0000000..cb8cb5e Binary files /dev/null and b/images/198.jpg differ diff --git a/images/199.jpg b/images/199.jpg new file mode 100644 index 0000000..e243abc Binary files /dev/null and b/images/199.jpg differ diff --git a/images/2.jpg b/images/2.jpg new file mode 100644 index 0000000..1249e21 Binary files /dev/null and b/images/2.jpg differ diff --git a/images/20.jpg b/images/20.jpg new file mode 100644 index 0000000..29d6c29 Binary files /dev/null and b/images/20.jpg differ diff --git a/images/200.jpg b/images/200.jpg new file mode 100644 index 0000000..0f04bc6 Binary files /dev/null and b/images/200.jpg differ diff --git a/images/201.jpg b/images/201.jpg new file mode 100644 index 0000000..30c1f44 Binary files /dev/null and b/images/201.jpg differ diff --git a/images/202.jpg b/images/202.jpg new file mode 100644 index 0000000..65e27ba Binary files /dev/null and b/images/202.jpg differ diff --git a/images/203.jpg b/images/203.jpg new file mode 100644 index 0000000..66b25fb Binary files /dev/null and b/images/203.jpg differ diff --git a/images/204.jpg b/images/204.jpg new file mode 100644 index 0000000..c1f010c Binary files /dev/null and b/images/204.jpg differ diff --git a/images/205.jpg b/images/205.jpg new file mode 100644 index 0000000..ff552b4 Binary files /dev/null and b/images/205.jpg differ diff --git a/images/206.jpg b/images/206.jpg new file mode 100644 index 0000000..03dffbe Binary files /dev/null and b/images/206.jpg differ diff --git a/images/207.jpg b/images/207.jpg new file mode 100644 index 0000000..25ccc5a Binary files /dev/null and b/images/207.jpg differ diff --git a/images/208.jpg b/images/208.jpg new file mode 100644 index 0000000..86c8a3e Binary files /dev/null and b/images/208.jpg differ diff --git a/images/209.jpg b/images/209.jpg new file mode 100644 index 0000000..d4e392e Binary files /dev/null and b/images/209.jpg differ diff --git a/images/21.jpg b/images/21.jpg new file mode 100644 index 0000000..c69337b Binary files /dev/null and b/images/21.jpg differ diff --git a/images/210.jpg b/images/210.jpg new file mode 100644 index 0000000..685f670 Binary files /dev/null and b/images/210.jpg differ diff --git a/images/211.jpg b/images/211.jpg new file mode 100644 index 0000000..8e76996 Binary files /dev/null and b/images/211.jpg differ diff --git a/images/212.jpg b/images/212.jpg new file mode 100644 index 0000000..7d04022 Binary files /dev/null and b/images/212.jpg differ diff --git a/images/213.jpg b/images/213.jpg new file mode 100644 index 0000000..241bb5c Binary files /dev/null and b/images/213.jpg differ diff --git a/images/214.jpg b/images/214.jpg new file mode 100644 index 0000000..a376396 Binary files /dev/null and b/images/214.jpg differ diff --git a/images/215.jpg b/images/215.jpg new file mode 100644 index 0000000..2131361 Binary files /dev/null and b/images/215.jpg differ diff --git a/images/216.jpg b/images/216.jpg new file mode 100644 index 0000000..ad248f5 Binary files /dev/null and b/images/216.jpg differ diff --git a/images/217.jpg b/images/217.jpg new file mode 100644 index 0000000..8e0cb6f Binary files /dev/null and b/images/217.jpg differ diff --git a/images/218.jpg b/images/218.jpg new file mode 100644 index 0000000..7676ec9 Binary files /dev/null and b/images/218.jpg differ diff --git a/images/219.jpg b/images/219.jpg new file mode 100644 index 0000000..9ce0912 Binary files /dev/null and b/images/219.jpg differ diff --git a/images/22.jpg b/images/22.jpg new file mode 100644 index 0000000..eeff95b Binary files /dev/null and b/images/22.jpg differ diff --git a/images/220.jpg b/images/220.jpg new file mode 100644 index 0000000..d86d314 Binary files /dev/null and b/images/220.jpg differ diff --git a/images/221.jpg b/images/221.jpg new file mode 100644 index 0000000..735596c Binary files /dev/null and b/images/221.jpg differ diff --git a/images/222.jpg b/images/222.jpg new file mode 100644 index 0000000..2406ff2 Binary files /dev/null and b/images/222.jpg differ diff --git a/images/223.jpg b/images/223.jpg new file mode 100644 index 0000000..93fa4de Binary files /dev/null and b/images/223.jpg differ diff --git a/images/224.jpg b/images/224.jpg new file mode 100644 index 0000000..6a474e1 Binary files /dev/null and b/images/224.jpg differ diff --git a/images/225.jpg b/images/225.jpg new file mode 100644 index 0000000..44beb50 Binary files /dev/null and b/images/225.jpg differ diff --git a/images/226.jpg b/images/226.jpg new file mode 100644 index 0000000..59de532 Binary files /dev/null and b/images/226.jpg differ diff --git a/images/227.jpg b/images/227.jpg new file mode 100644 index 0000000..a34026f Binary files /dev/null and b/images/227.jpg differ diff --git a/images/228.jpg b/images/228.jpg new file mode 100644 index 0000000..7ec8870 Binary files /dev/null and b/images/228.jpg differ diff --git a/images/229.jpg b/images/229.jpg new file mode 100644 index 0000000..1575949 Binary files /dev/null and b/images/229.jpg differ diff --git a/images/23.jpg b/images/23.jpg new file mode 100644 index 0000000..27a8700 Binary files /dev/null and b/images/23.jpg differ diff --git a/images/230.jpg b/images/230.jpg new file mode 100644 index 0000000..ab86266 Binary files /dev/null and b/images/230.jpg differ diff --git a/images/231.jpg b/images/231.jpg new file mode 100644 index 0000000..ff95476 Binary files /dev/null and b/images/231.jpg differ diff --git a/images/232.jpg b/images/232.jpg new file mode 100644 index 0000000..992d562 Binary files /dev/null and b/images/232.jpg differ diff --git a/images/233.jpg b/images/233.jpg new file mode 100644 index 0000000..6c7ae2d Binary files /dev/null and b/images/233.jpg differ diff --git a/images/234.jpg b/images/234.jpg new file mode 100644 index 0000000..403f288 Binary files /dev/null and b/images/234.jpg differ diff --git a/images/235.jpg b/images/235.jpg new file mode 100644 index 0000000..b99b820 Binary files /dev/null and b/images/235.jpg differ diff --git a/images/236.jpg b/images/236.jpg new file mode 100644 index 0000000..2190f36 Binary files /dev/null and b/images/236.jpg differ diff --git a/images/237.jpg b/images/237.jpg new file mode 100644 index 0000000..ea617c9 Binary files /dev/null and b/images/237.jpg differ diff --git a/images/238.jpg b/images/238.jpg new file mode 100644 index 0000000..9994e5e Binary files /dev/null and b/images/238.jpg differ diff --git a/images/239.jpg b/images/239.jpg new file mode 100644 index 0000000..c8f9c44 Binary files /dev/null and b/images/239.jpg differ diff --git a/images/24.jpg b/images/24.jpg new file mode 100644 index 0000000..578f15a Binary files /dev/null and b/images/24.jpg differ diff --git a/images/240.jpg b/images/240.jpg new file mode 100644 index 0000000..88cdd11 Binary files /dev/null and b/images/240.jpg differ diff --git a/images/241.jpg b/images/241.jpg new file mode 100644 index 0000000..14775f7 Binary files /dev/null and b/images/241.jpg differ diff --git a/images/242.jpg b/images/242.jpg new file mode 100644 index 0000000..9ad91a9 Binary files /dev/null and b/images/242.jpg differ diff --git a/images/243.jpg b/images/243.jpg new file mode 100644 index 0000000..b4958bb Binary files /dev/null and b/images/243.jpg differ diff --git a/images/244.jpg b/images/244.jpg new file mode 100644 index 0000000..1f0f82c Binary files /dev/null and b/images/244.jpg differ diff --git a/images/245.jpg b/images/245.jpg new file mode 100644 index 0000000..8f3f610 Binary files /dev/null and b/images/245.jpg differ diff --git a/images/246.jpg b/images/246.jpg new file mode 100644 index 0000000..d49deee Binary files /dev/null and b/images/246.jpg differ diff --git a/images/247.jpg b/images/247.jpg new file mode 100644 index 0000000..dde01dd Binary files /dev/null and b/images/247.jpg differ diff --git a/images/248.jpg b/images/248.jpg new file mode 100644 index 0000000..c34b5fa Binary files /dev/null and b/images/248.jpg differ diff --git a/images/249.jpg b/images/249.jpg new file mode 100644 index 0000000..944fd42 Binary files /dev/null and b/images/249.jpg differ diff --git a/images/25.jpg b/images/25.jpg new file mode 100644 index 0000000..a203e6a Binary files /dev/null and b/images/25.jpg differ diff --git a/images/250.jpg b/images/250.jpg new file mode 100644 index 0000000..284dc83 Binary files /dev/null and b/images/250.jpg differ diff --git a/images/251.jpg b/images/251.jpg new file mode 100644 index 0000000..e0cfd45 Binary files /dev/null and b/images/251.jpg differ diff --git a/images/252.jpg b/images/252.jpg new file mode 100644 index 0000000..1b7e1c7 Binary files /dev/null and b/images/252.jpg differ diff --git a/images/253.jpg b/images/253.jpg new file mode 100644 index 0000000..324fe14 Binary files /dev/null and b/images/253.jpg differ diff --git a/images/254.jpg b/images/254.jpg new file mode 100644 index 0000000..d9cda9a Binary files /dev/null and b/images/254.jpg differ diff --git a/images/255.jpg b/images/255.jpg new file mode 100644 index 0000000..38d2ee3 Binary files /dev/null and b/images/255.jpg differ diff --git a/images/256.jpg b/images/256.jpg new file mode 100644 index 0000000..29937df Binary files /dev/null and b/images/256.jpg differ diff --git a/images/257.jpg b/images/257.jpg new file mode 100644 index 0000000..9e131dc Binary files /dev/null and b/images/257.jpg differ diff --git a/images/258.jpg b/images/258.jpg new file mode 100644 index 0000000..dbeab6d Binary files /dev/null and b/images/258.jpg differ diff --git a/images/259.jpg b/images/259.jpg new file mode 100644 index 0000000..5e78d31 Binary files /dev/null and b/images/259.jpg differ diff --git a/images/26.jpg b/images/26.jpg new file mode 100644 index 0000000..cfaa8cb Binary files /dev/null and b/images/26.jpg differ diff --git a/images/260.jpg b/images/260.jpg new file mode 100644 index 0000000..b09c663 Binary files /dev/null and b/images/260.jpg differ diff --git a/images/261.jpg b/images/261.jpg new file mode 100644 index 0000000..269ef5e Binary files /dev/null and b/images/261.jpg differ diff --git a/images/262.jpg b/images/262.jpg new file mode 100644 index 0000000..c97a0e5 Binary files /dev/null and b/images/262.jpg differ diff --git a/images/263.jpg b/images/263.jpg new file mode 100644 index 0000000..8ec57e2 Binary files /dev/null and b/images/263.jpg differ diff --git a/images/264.jpg b/images/264.jpg new file mode 100644 index 0000000..82d1fd4 Binary files /dev/null and b/images/264.jpg differ diff --git a/images/265.jpg b/images/265.jpg new file mode 100644 index 0000000..740d89b Binary files /dev/null and b/images/265.jpg differ diff --git a/images/266.jpg b/images/266.jpg new file mode 100644 index 0000000..675ab7c Binary files /dev/null and b/images/266.jpg differ diff --git a/images/267.jpg b/images/267.jpg new file mode 100644 index 0000000..c4a0fc3 Binary files /dev/null and b/images/267.jpg differ diff --git a/images/268.jpg b/images/268.jpg new file mode 100644 index 0000000..b231bc8 Binary files /dev/null and b/images/268.jpg differ diff --git a/images/269.jpg b/images/269.jpg new file mode 100644 index 0000000..a9d3cb4 Binary files /dev/null and b/images/269.jpg differ diff --git a/images/27.jpg b/images/27.jpg new file mode 100644 index 0000000..61d9304 Binary files /dev/null and b/images/27.jpg differ diff --git a/images/270.jpg b/images/270.jpg new file mode 100644 index 0000000..18e40f9 Binary files /dev/null and b/images/270.jpg differ diff --git a/images/271.jpg b/images/271.jpg new file mode 100644 index 0000000..68d2140 Binary files /dev/null and b/images/271.jpg differ diff --git a/images/272.jpg b/images/272.jpg new file mode 100644 index 0000000..611ce82 Binary files /dev/null and b/images/272.jpg differ diff --git a/images/273.jpg b/images/273.jpg new file mode 100644 index 0000000..9afa61b Binary files /dev/null and b/images/273.jpg differ diff --git a/images/274.jpg b/images/274.jpg new file mode 100644 index 0000000..e883bb1 Binary files /dev/null and b/images/274.jpg differ diff --git a/images/275.jpg b/images/275.jpg new file mode 100644 index 0000000..db845e1 Binary files /dev/null and b/images/275.jpg differ diff --git a/images/276.jpg b/images/276.jpg new file mode 100644 index 0000000..2f46249 Binary files /dev/null and b/images/276.jpg differ diff --git a/images/277.jpg b/images/277.jpg new file mode 100644 index 0000000..e64cc9d Binary files /dev/null and b/images/277.jpg differ diff --git a/images/278.jpg b/images/278.jpg new file mode 100644 index 0000000..960fd50 Binary files /dev/null and b/images/278.jpg differ diff --git a/images/279.jpg b/images/279.jpg new file mode 100644 index 0000000..831d733 Binary files /dev/null and b/images/279.jpg differ diff --git a/images/28.jpg b/images/28.jpg new file mode 100644 index 0000000..33b79cb Binary files /dev/null and b/images/28.jpg differ diff --git a/images/280.jpg b/images/280.jpg new file mode 100644 index 0000000..e449628 Binary files /dev/null and b/images/280.jpg differ diff --git a/images/281.jpg b/images/281.jpg new file mode 100644 index 0000000..330a354 Binary files /dev/null and b/images/281.jpg differ diff --git a/images/282.jpg b/images/282.jpg new file mode 100644 index 0000000..13019b5 Binary files /dev/null and b/images/282.jpg differ diff --git a/images/283.jpg b/images/283.jpg new file mode 100644 index 0000000..dbced6e Binary files /dev/null and b/images/283.jpg differ diff --git a/images/284.jpg b/images/284.jpg new file mode 100644 index 0000000..bf1a27b Binary files /dev/null and b/images/284.jpg differ diff --git a/images/285.jpg b/images/285.jpg new file mode 100644 index 0000000..1f783fb Binary files /dev/null and b/images/285.jpg differ diff --git a/images/286.jpg b/images/286.jpg new file mode 100644 index 0000000..d3f5a4d Binary files /dev/null and b/images/286.jpg differ diff --git a/images/287.jpg b/images/287.jpg new file mode 100644 index 0000000..c38b30b Binary files /dev/null and b/images/287.jpg differ diff --git a/images/288.jpg b/images/288.jpg new file mode 100644 index 0000000..0c48671 Binary files /dev/null and b/images/288.jpg differ diff --git a/images/289.jpg b/images/289.jpg new file mode 100644 index 0000000..15f8d14 Binary files /dev/null and b/images/289.jpg differ diff --git a/images/29.jpg b/images/29.jpg new file mode 100644 index 0000000..8a3cba8 Binary files /dev/null and b/images/29.jpg differ diff --git a/images/290.jpg b/images/290.jpg new file mode 100644 index 0000000..12c7b54 Binary files /dev/null and b/images/290.jpg differ diff --git a/images/291.jpg b/images/291.jpg new file mode 100644 index 0000000..88d35f7 Binary files /dev/null and b/images/291.jpg differ diff --git a/images/292.jpg b/images/292.jpg new file mode 100644 index 0000000..43f0374 Binary files /dev/null and b/images/292.jpg differ diff --git a/images/293.jpg b/images/293.jpg new file mode 100644 index 0000000..86e83f2 Binary files /dev/null and b/images/293.jpg differ diff --git a/images/294.jpg b/images/294.jpg new file mode 100644 index 0000000..0694ecc Binary files /dev/null and b/images/294.jpg differ diff --git a/images/295.jpg b/images/295.jpg new file mode 100644 index 0000000..5aa08a1 Binary files /dev/null and b/images/295.jpg differ diff --git a/images/296.jpg b/images/296.jpg new file mode 100644 index 0000000..35d4974 Binary files /dev/null and b/images/296.jpg differ diff --git a/images/297.jpg b/images/297.jpg new file mode 100644 index 0000000..2b0178b Binary files /dev/null and b/images/297.jpg differ diff --git a/images/298.jpg b/images/298.jpg new file mode 100644 index 0000000..e571e46 Binary files /dev/null and b/images/298.jpg differ diff --git a/images/299.jpg b/images/299.jpg new file mode 100644 index 0000000..92a178c Binary files /dev/null and b/images/299.jpg differ diff --git a/images/3.jpg b/images/3.jpg new file mode 100644 index 0000000..78e6016 Binary files /dev/null and b/images/3.jpg differ diff --git a/images/30.jpg b/images/30.jpg new file mode 100644 index 0000000..a65284b Binary files /dev/null and b/images/30.jpg differ diff --git a/images/300.jpg b/images/300.jpg new file mode 100644 index 0000000..79f7d93 Binary files /dev/null and b/images/300.jpg differ diff --git a/images/301.jpg b/images/301.jpg new file mode 100644 index 0000000..7066994 Binary files /dev/null and b/images/301.jpg differ diff --git a/images/302.jpg b/images/302.jpg new file mode 100644 index 0000000..ee98854 Binary files /dev/null and b/images/302.jpg differ diff --git a/images/303.jpg b/images/303.jpg new file mode 100644 index 0000000..2856382 Binary files /dev/null and b/images/303.jpg differ diff --git a/images/304.jpg b/images/304.jpg new file mode 100644 index 0000000..081d64c Binary files /dev/null and b/images/304.jpg differ diff --git a/images/305.jpg b/images/305.jpg new file mode 100644 index 0000000..cb8f79c Binary files /dev/null and b/images/305.jpg differ diff --git a/images/306.jpg b/images/306.jpg new file mode 100644 index 0000000..e829db3 Binary files /dev/null and b/images/306.jpg differ diff --git a/images/307.jpg b/images/307.jpg new file mode 100644 index 0000000..25199f8 Binary files /dev/null and b/images/307.jpg differ diff --git a/images/308.jpg b/images/308.jpg new file mode 100644 index 0000000..62cc195 Binary files /dev/null and b/images/308.jpg differ diff --git a/images/309.jpg b/images/309.jpg new file mode 100644 index 0000000..b0be9e2 Binary files /dev/null and b/images/309.jpg differ diff --git a/images/31.jpg b/images/31.jpg new file mode 100644 index 0000000..4c35ca1 Binary files /dev/null and b/images/31.jpg differ diff --git a/images/310.jpg b/images/310.jpg new file mode 100644 index 0000000..2a61997 Binary files /dev/null and b/images/310.jpg differ diff --git a/images/311.jpg b/images/311.jpg new file mode 100644 index 0000000..1a2b901 Binary files /dev/null and b/images/311.jpg differ diff --git a/images/312.jpg b/images/312.jpg new file mode 100644 index 0000000..678d38a Binary files /dev/null and b/images/312.jpg differ diff --git a/images/313.jpg b/images/313.jpg new file mode 100644 index 0000000..42c622e Binary files /dev/null and b/images/313.jpg differ diff --git a/images/314.jpg b/images/314.jpg new file mode 100644 index 0000000..333ed3a Binary files /dev/null and b/images/314.jpg differ diff --git a/images/315.jpg b/images/315.jpg new file mode 100644 index 0000000..e7f4742 Binary files /dev/null and b/images/315.jpg differ diff --git a/images/316.jpg b/images/316.jpg new file mode 100644 index 0000000..42adf61 Binary files /dev/null and b/images/316.jpg differ diff --git a/images/317.jpg b/images/317.jpg new file mode 100644 index 0000000..b6b8266 Binary files /dev/null and b/images/317.jpg differ diff --git a/images/318.jpg b/images/318.jpg new file mode 100644 index 0000000..3ded53d Binary files /dev/null and b/images/318.jpg differ diff --git a/images/319.jpg b/images/319.jpg new file mode 100644 index 0000000..75a5f02 Binary files /dev/null and b/images/319.jpg differ diff --git a/images/32.jpg b/images/32.jpg new file mode 100644 index 0000000..2c15340 Binary files /dev/null and b/images/32.jpg differ diff --git a/images/320.jpg b/images/320.jpg new file mode 100644 index 0000000..9575b22 Binary files /dev/null and b/images/320.jpg differ diff --git a/images/321.jpg b/images/321.jpg new file mode 100644 index 0000000..e9958e4 Binary files /dev/null and b/images/321.jpg differ diff --git a/images/322.jpg b/images/322.jpg new file mode 100644 index 0000000..b22b476 Binary files /dev/null and b/images/322.jpg differ diff --git a/images/323.jpg b/images/323.jpg new file mode 100644 index 0000000..83410a4 Binary files /dev/null and b/images/323.jpg differ diff --git a/images/324.jpg b/images/324.jpg new file mode 100644 index 0000000..27d1756 Binary files /dev/null and b/images/324.jpg differ diff --git a/images/325.jpg b/images/325.jpg new file mode 100644 index 0000000..2349058 Binary files /dev/null and b/images/325.jpg differ diff --git a/images/326.jpg b/images/326.jpg new file mode 100644 index 0000000..0241e7c Binary files /dev/null and b/images/326.jpg differ diff --git a/images/327.jpg b/images/327.jpg new file mode 100644 index 0000000..fc4d641 Binary files /dev/null and b/images/327.jpg differ diff --git a/images/328.jpg b/images/328.jpg new file mode 100644 index 0000000..e38abf0 Binary files /dev/null and b/images/328.jpg differ diff --git a/images/329.jpg b/images/329.jpg new file mode 100644 index 0000000..c9c7b9f Binary files /dev/null and b/images/329.jpg differ diff --git a/images/33.jpg b/images/33.jpg new file mode 100644 index 0000000..37b9810 Binary files /dev/null and b/images/33.jpg differ diff --git a/images/330.jpg b/images/330.jpg new file mode 100644 index 0000000..dc0a79f Binary files /dev/null and b/images/330.jpg differ diff --git a/images/331.jpg b/images/331.jpg new file mode 100644 index 0000000..eacd498 Binary files /dev/null and b/images/331.jpg differ diff --git a/images/332.jpg b/images/332.jpg new file mode 100644 index 0000000..8cdc9fd Binary files /dev/null and b/images/332.jpg differ diff --git a/images/333.jpg b/images/333.jpg new file mode 100644 index 0000000..af0655e Binary files /dev/null and b/images/333.jpg differ diff --git a/images/334.jpg b/images/334.jpg new file mode 100644 index 0000000..af960cc Binary files /dev/null and b/images/334.jpg differ diff --git a/images/335.jpg b/images/335.jpg new file mode 100644 index 0000000..d00671f Binary files /dev/null and b/images/335.jpg differ diff --git a/images/336.jpg b/images/336.jpg new file mode 100644 index 0000000..3f71875 Binary files /dev/null and b/images/336.jpg differ diff --git a/images/337.jpg b/images/337.jpg new file mode 100644 index 0000000..7a03228 Binary files /dev/null and b/images/337.jpg differ diff --git a/images/338.jpg b/images/338.jpg new file mode 100644 index 0000000..45f93d0 Binary files /dev/null and b/images/338.jpg differ diff --git a/images/339.jpg b/images/339.jpg new file mode 100644 index 0000000..ef87fc0 Binary files /dev/null and b/images/339.jpg differ diff --git a/images/34.jpg b/images/34.jpg new file mode 100644 index 0000000..54df658 Binary files /dev/null and b/images/34.jpg differ diff --git a/images/340.jpg b/images/340.jpg new file mode 100644 index 0000000..3667a78 Binary files /dev/null and b/images/340.jpg differ diff --git a/images/341.jpg b/images/341.jpg new file mode 100644 index 0000000..7a7a91e Binary files /dev/null and b/images/341.jpg differ diff --git a/images/342.jpg b/images/342.jpg new file mode 100644 index 0000000..4c7b806 Binary files /dev/null and b/images/342.jpg differ diff --git a/images/343.jpg b/images/343.jpg new file mode 100644 index 0000000..3b808a0 Binary files /dev/null and b/images/343.jpg differ diff --git a/images/344.jpg b/images/344.jpg new file mode 100644 index 0000000..d6f7f91 Binary files /dev/null and b/images/344.jpg differ diff --git a/images/345.jpg b/images/345.jpg new file mode 100644 index 0000000..7d77a70 Binary files /dev/null and b/images/345.jpg differ diff --git a/images/346.jpg b/images/346.jpg new file mode 100644 index 0000000..0c6ca12 Binary files /dev/null and b/images/346.jpg differ diff --git a/images/347.jpg b/images/347.jpg new file mode 100644 index 0000000..30e777b Binary files /dev/null and b/images/347.jpg differ diff --git a/images/348.jpg b/images/348.jpg new file mode 100644 index 0000000..de87d79 Binary files /dev/null and b/images/348.jpg differ diff --git a/images/349.jpg b/images/349.jpg new file mode 100644 index 0000000..b4baa81 Binary files /dev/null and b/images/349.jpg differ diff --git a/images/35.jpg b/images/35.jpg new file mode 100644 index 0000000..3564393 Binary files /dev/null and b/images/35.jpg differ diff --git a/images/350.jpg b/images/350.jpg new file mode 100644 index 0000000..9666496 Binary files /dev/null and b/images/350.jpg differ diff --git a/images/351.jpg b/images/351.jpg new file mode 100644 index 0000000..5c8bcae Binary files /dev/null and b/images/351.jpg differ diff --git a/images/352.jpg b/images/352.jpg new file mode 100644 index 0000000..720d639 Binary files /dev/null and b/images/352.jpg differ diff --git a/images/353.jpg b/images/353.jpg new file mode 100644 index 0000000..a8067b9 Binary files /dev/null and b/images/353.jpg differ diff --git a/images/354.jpg b/images/354.jpg new file mode 100644 index 0000000..14dfc5a Binary files /dev/null and b/images/354.jpg differ diff --git a/images/355.jpg b/images/355.jpg new file mode 100644 index 0000000..5bcb1a9 Binary files /dev/null and b/images/355.jpg differ diff --git a/images/356.jpg b/images/356.jpg new file mode 100644 index 0000000..f5a81d3 Binary files /dev/null and b/images/356.jpg differ diff --git a/images/357.jpg b/images/357.jpg new file mode 100644 index 0000000..1d46444 Binary files /dev/null and b/images/357.jpg differ diff --git a/images/358.jpg b/images/358.jpg new file mode 100644 index 0000000..dc363cb Binary files /dev/null and b/images/358.jpg differ diff --git a/images/359.jpg b/images/359.jpg new file mode 100644 index 0000000..0e8ab27 Binary files /dev/null and b/images/359.jpg differ diff --git a/images/36.jpg b/images/36.jpg new file mode 100644 index 0000000..ff5dc57 Binary files /dev/null and b/images/36.jpg differ diff --git a/images/360.jpg b/images/360.jpg new file mode 100644 index 0000000..56c2e27 Binary files /dev/null and b/images/360.jpg differ diff --git a/images/361.jpg b/images/361.jpg new file mode 100644 index 0000000..9b0e6e5 Binary files /dev/null and b/images/361.jpg differ diff --git a/images/362.jpg b/images/362.jpg new file mode 100644 index 0000000..6e0a9a8 Binary files /dev/null and b/images/362.jpg differ diff --git a/images/363.jpg b/images/363.jpg new file mode 100644 index 0000000..48ab085 Binary files /dev/null and b/images/363.jpg differ diff --git a/images/364.jpg b/images/364.jpg new file mode 100644 index 0000000..925977e Binary files /dev/null and b/images/364.jpg differ diff --git a/images/365.jpg b/images/365.jpg new file mode 100644 index 0000000..a2f9b47 Binary files /dev/null and b/images/365.jpg differ diff --git a/images/366.jpg b/images/366.jpg new file mode 100644 index 0000000..3ee2809 Binary files /dev/null and b/images/366.jpg differ diff --git a/images/367.jpg b/images/367.jpg new file mode 100644 index 0000000..6d11f90 Binary files /dev/null and b/images/367.jpg differ diff --git a/images/368.jpg b/images/368.jpg new file mode 100644 index 0000000..7dad469 Binary files /dev/null and b/images/368.jpg differ diff --git a/images/369.jpg b/images/369.jpg new file mode 100644 index 0000000..d0097bc Binary files /dev/null and b/images/369.jpg differ diff --git a/images/37.jpg b/images/37.jpg new file mode 100644 index 0000000..d21f356 Binary files /dev/null and b/images/37.jpg differ diff --git a/images/370.jpg b/images/370.jpg new file mode 100644 index 0000000..9d77d4f Binary files /dev/null and b/images/370.jpg differ diff --git a/images/371.jpg b/images/371.jpg new file mode 100644 index 0000000..ef1e433 Binary files /dev/null and b/images/371.jpg differ diff --git a/images/372.jpg b/images/372.jpg new file mode 100644 index 0000000..57873cb Binary files /dev/null and b/images/372.jpg differ diff --git a/images/373.jpg b/images/373.jpg new file mode 100644 index 0000000..f5f0fff Binary files /dev/null and b/images/373.jpg differ diff --git a/images/374.jpg b/images/374.jpg new file mode 100644 index 0000000..945fa88 Binary files /dev/null and b/images/374.jpg differ diff --git a/images/375.jpg b/images/375.jpg new file mode 100644 index 0000000..eda7d16 Binary files /dev/null and b/images/375.jpg differ diff --git a/images/376.jpg b/images/376.jpg new file mode 100644 index 0000000..ad86828 Binary files /dev/null and b/images/376.jpg differ diff --git a/images/377.jpg b/images/377.jpg new file mode 100644 index 0000000..8d3ce17 Binary files /dev/null and b/images/377.jpg differ diff --git a/images/378.jpg b/images/378.jpg new file mode 100644 index 0000000..5d8a624 Binary files /dev/null and b/images/378.jpg differ diff --git a/images/379.jpg b/images/379.jpg new file mode 100644 index 0000000..c3bbf1c Binary files /dev/null and b/images/379.jpg differ diff --git a/images/38.jpg b/images/38.jpg new file mode 100644 index 0000000..2a715c4 Binary files /dev/null and b/images/38.jpg differ diff --git a/images/380.jpg b/images/380.jpg new file mode 100644 index 0000000..e096bed Binary files /dev/null and b/images/380.jpg differ diff --git a/images/381.jpg b/images/381.jpg new file mode 100644 index 0000000..74bec1b Binary files /dev/null and b/images/381.jpg differ diff --git a/images/382.jpg b/images/382.jpg new file mode 100644 index 0000000..1f1d1f6 Binary files /dev/null and b/images/382.jpg differ diff --git a/images/383.jpg b/images/383.jpg new file mode 100644 index 0000000..f89ef25 Binary files /dev/null and b/images/383.jpg differ diff --git a/images/384.jpg b/images/384.jpg new file mode 100644 index 0000000..719b4fa Binary files /dev/null and b/images/384.jpg differ diff --git a/images/385.jpg b/images/385.jpg new file mode 100644 index 0000000..3edadb6 Binary files /dev/null and b/images/385.jpg differ diff --git a/images/386.jpg b/images/386.jpg new file mode 100644 index 0000000..f817e13 Binary files /dev/null and b/images/386.jpg differ diff --git a/images/387.jpg b/images/387.jpg new file mode 100644 index 0000000..1eb9af2 Binary files /dev/null and b/images/387.jpg differ diff --git a/images/388.jpg b/images/388.jpg new file mode 100644 index 0000000..010ffa3 Binary files /dev/null and b/images/388.jpg differ diff --git a/images/389.jpg b/images/389.jpg new file mode 100644 index 0000000..fc70c11 Binary files /dev/null and b/images/389.jpg differ diff --git a/images/39.jpg b/images/39.jpg new file mode 100644 index 0000000..616d92a Binary files /dev/null and b/images/39.jpg differ diff --git a/images/390.jpg b/images/390.jpg new file mode 100644 index 0000000..a23d09e Binary files /dev/null and b/images/390.jpg differ diff --git a/images/391.jpg b/images/391.jpg new file mode 100644 index 0000000..6d1cf91 Binary files /dev/null and b/images/391.jpg differ diff --git a/images/392.jpg b/images/392.jpg new file mode 100644 index 0000000..ca9469f Binary files /dev/null and b/images/392.jpg differ diff --git a/images/393.jpg b/images/393.jpg new file mode 100644 index 0000000..9270cef Binary files /dev/null and b/images/393.jpg differ diff --git a/images/394.jpg b/images/394.jpg new file mode 100644 index 0000000..87a3d30 Binary files /dev/null and b/images/394.jpg differ diff --git a/images/395.jpg b/images/395.jpg new file mode 100644 index 0000000..e484792 Binary files /dev/null and b/images/395.jpg differ diff --git a/images/396.jpg b/images/396.jpg new file mode 100644 index 0000000..d53fc81 Binary files /dev/null and b/images/396.jpg differ diff --git a/images/397.jpg b/images/397.jpg new file mode 100644 index 0000000..784a7e9 Binary files /dev/null and b/images/397.jpg differ diff --git a/images/398.jpg b/images/398.jpg new file mode 100644 index 0000000..3a0a9ae Binary files /dev/null and b/images/398.jpg differ diff --git a/images/399.jpg b/images/399.jpg new file mode 100644 index 0000000..7cf0522 Binary files /dev/null and b/images/399.jpg differ diff --git a/images/4.jpg b/images/4.jpg new file mode 100644 index 0000000..d359854 Binary files /dev/null and b/images/4.jpg differ diff --git a/images/40.jpg b/images/40.jpg new file mode 100644 index 0000000..f844472 Binary files /dev/null and b/images/40.jpg differ diff --git a/images/400.jpg b/images/400.jpg new file mode 100644 index 0000000..59c8620 Binary files /dev/null and b/images/400.jpg differ diff --git a/images/401.jpg b/images/401.jpg new file mode 100644 index 0000000..ade59cb Binary files /dev/null and b/images/401.jpg differ diff --git a/images/402.jpg b/images/402.jpg new file mode 100644 index 0000000..94a5785 Binary files /dev/null and b/images/402.jpg differ diff --git a/images/403.jpg b/images/403.jpg new file mode 100644 index 0000000..85db327 Binary files /dev/null and b/images/403.jpg differ diff --git a/images/404.jpg b/images/404.jpg new file mode 100644 index 0000000..23de0af Binary files /dev/null and b/images/404.jpg differ diff --git a/images/405.jpg b/images/405.jpg new file mode 100644 index 0000000..03729e5 Binary files /dev/null and b/images/405.jpg differ diff --git a/images/406.jpg b/images/406.jpg new file mode 100644 index 0000000..5292b97 Binary files /dev/null and b/images/406.jpg differ diff --git a/images/407.jpg b/images/407.jpg new file mode 100644 index 0000000..f15815d Binary files /dev/null and b/images/407.jpg differ diff --git a/images/408.jpg b/images/408.jpg new file mode 100644 index 0000000..6a6b30f Binary files /dev/null and b/images/408.jpg differ diff --git a/images/409.jpg b/images/409.jpg new file mode 100644 index 0000000..78d916e Binary files /dev/null and b/images/409.jpg differ diff --git a/images/41.jpg b/images/41.jpg new file mode 100644 index 0000000..9ecf393 Binary files /dev/null and b/images/41.jpg differ diff --git a/images/410.jpg b/images/410.jpg new file mode 100644 index 0000000..804ab16 Binary files /dev/null and b/images/410.jpg differ diff --git a/images/411.jpg b/images/411.jpg new file mode 100644 index 0000000..2342a38 Binary files /dev/null and b/images/411.jpg differ diff --git a/images/412.jpg b/images/412.jpg new file mode 100644 index 0000000..6316046 Binary files /dev/null and b/images/412.jpg differ diff --git a/images/413.jpg b/images/413.jpg new file mode 100644 index 0000000..55d83db Binary files /dev/null and b/images/413.jpg differ diff --git a/images/414.jpg b/images/414.jpg new file mode 100644 index 0000000..ea59ba2 Binary files /dev/null and b/images/414.jpg differ diff --git a/images/415.jpg b/images/415.jpg new file mode 100644 index 0000000..ce886ef Binary files /dev/null and b/images/415.jpg differ diff --git a/images/416.jpg b/images/416.jpg new file mode 100644 index 0000000..c9f7b21 Binary files /dev/null and b/images/416.jpg differ diff --git a/images/417.jpg b/images/417.jpg new file mode 100644 index 0000000..48c934b Binary files /dev/null and b/images/417.jpg differ diff --git a/images/418.jpg b/images/418.jpg new file mode 100644 index 0000000..b20513a Binary files /dev/null and b/images/418.jpg differ diff --git a/images/419.jpg b/images/419.jpg new file mode 100644 index 0000000..26addad Binary files /dev/null and b/images/419.jpg differ diff --git a/images/42.jpg b/images/42.jpg new file mode 100644 index 0000000..b2ebca4 Binary files /dev/null and b/images/42.jpg differ diff --git a/images/420.jpg b/images/420.jpg new file mode 100644 index 0000000..07193a3 Binary files /dev/null and b/images/420.jpg differ diff --git a/images/421.jpg b/images/421.jpg new file mode 100644 index 0000000..b3b15e0 Binary files /dev/null and b/images/421.jpg differ diff --git a/images/422.jpg b/images/422.jpg new file mode 100644 index 0000000..9bf4205 Binary files /dev/null and b/images/422.jpg differ diff --git a/images/423.jpg b/images/423.jpg new file mode 100644 index 0000000..f9dc686 Binary files /dev/null and b/images/423.jpg differ diff --git a/images/424.jpg b/images/424.jpg new file mode 100644 index 0000000..3f916c6 Binary files /dev/null and b/images/424.jpg differ diff --git a/images/425.jpg b/images/425.jpg new file mode 100644 index 0000000..49b1e8a Binary files /dev/null and b/images/425.jpg differ diff --git a/images/426.jpg b/images/426.jpg new file mode 100644 index 0000000..7fb5f7a Binary files /dev/null and b/images/426.jpg differ diff --git a/images/427.jpg b/images/427.jpg new file mode 100644 index 0000000..5409598 Binary files /dev/null and b/images/427.jpg differ diff --git a/images/428.jpg b/images/428.jpg new file mode 100644 index 0000000..49e4fdb Binary files /dev/null and b/images/428.jpg differ diff --git a/images/429.jpg b/images/429.jpg new file mode 100644 index 0000000..ee0616b Binary files /dev/null and b/images/429.jpg differ diff --git a/images/43.jpg b/images/43.jpg new file mode 100644 index 0000000..16fc7b0 Binary files /dev/null and b/images/43.jpg differ diff --git a/images/430.jpg b/images/430.jpg new file mode 100644 index 0000000..dcff97b Binary files /dev/null and b/images/430.jpg differ diff --git a/images/431.jpg b/images/431.jpg new file mode 100644 index 0000000..85e2988 Binary files /dev/null and b/images/431.jpg differ diff --git a/images/432.jpg b/images/432.jpg new file mode 100644 index 0000000..735f239 Binary files /dev/null and b/images/432.jpg differ diff --git a/images/433.jpg b/images/433.jpg new file mode 100644 index 0000000..a97f64f Binary files /dev/null and b/images/433.jpg differ diff --git a/images/434.jpg b/images/434.jpg new file mode 100644 index 0000000..66f7d91 Binary files /dev/null and b/images/434.jpg differ diff --git a/images/435.jpg b/images/435.jpg new file mode 100644 index 0000000..4960042 Binary files /dev/null and b/images/435.jpg differ diff --git a/images/436.jpg b/images/436.jpg new file mode 100644 index 0000000..be8a934 Binary files /dev/null and b/images/436.jpg differ diff --git a/images/437.jpg b/images/437.jpg new file mode 100644 index 0000000..48d3443 Binary files /dev/null and b/images/437.jpg differ diff --git a/images/438.jpg b/images/438.jpg new file mode 100644 index 0000000..bae060e Binary files /dev/null and b/images/438.jpg differ diff --git a/images/439.jpg b/images/439.jpg new file mode 100644 index 0000000..07532e8 Binary files /dev/null and b/images/439.jpg differ diff --git a/images/44.jpg b/images/44.jpg new file mode 100644 index 0000000..3f467c5 Binary files /dev/null and b/images/44.jpg differ diff --git a/images/440.jpg b/images/440.jpg new file mode 100644 index 0000000..84d552c Binary files /dev/null and b/images/440.jpg differ diff --git a/images/441.jpg b/images/441.jpg new file mode 100644 index 0000000..0228d45 Binary files /dev/null and b/images/441.jpg differ diff --git a/images/442.jpg b/images/442.jpg new file mode 100644 index 0000000..f4a6775 Binary files /dev/null and b/images/442.jpg differ diff --git a/images/443.jpg b/images/443.jpg new file mode 100644 index 0000000..f1cd8cc Binary files /dev/null and b/images/443.jpg differ diff --git a/images/444.jpg b/images/444.jpg new file mode 100644 index 0000000..812348c Binary files /dev/null and b/images/444.jpg differ diff --git a/images/445.jpg b/images/445.jpg new file mode 100644 index 0000000..6695468 Binary files /dev/null and b/images/445.jpg differ diff --git a/images/446.jpg b/images/446.jpg new file mode 100644 index 0000000..c87821d Binary files /dev/null and b/images/446.jpg differ diff --git a/images/447.jpg b/images/447.jpg new file mode 100644 index 0000000..e9cb391 Binary files /dev/null and b/images/447.jpg differ diff --git a/images/448.jpg b/images/448.jpg new file mode 100644 index 0000000..f2346a5 Binary files /dev/null and b/images/448.jpg differ diff --git a/images/449.jpg b/images/449.jpg new file mode 100644 index 0000000..7094387 Binary files /dev/null and b/images/449.jpg differ diff --git a/images/45.jpg b/images/45.jpg new file mode 100644 index 0000000..388eebf Binary files /dev/null and b/images/45.jpg differ diff --git a/images/450.jpg b/images/450.jpg new file mode 100644 index 0000000..d931618 Binary files /dev/null and b/images/450.jpg differ diff --git a/images/451.jpg b/images/451.jpg new file mode 100644 index 0000000..33fdb41 Binary files /dev/null and b/images/451.jpg differ diff --git a/images/452.jpg b/images/452.jpg new file mode 100644 index 0000000..8d056ce Binary files /dev/null and b/images/452.jpg differ diff --git a/images/453.jpg b/images/453.jpg new file mode 100644 index 0000000..3e792f4 Binary files /dev/null and b/images/453.jpg differ diff --git a/images/454.jpg b/images/454.jpg new file mode 100644 index 0000000..0476948 Binary files /dev/null and b/images/454.jpg differ diff --git a/images/455.jpg b/images/455.jpg new file mode 100644 index 0000000..cbceac7 Binary files /dev/null and b/images/455.jpg differ diff --git a/images/456.jpg b/images/456.jpg new file mode 100644 index 0000000..51498ce Binary files /dev/null and b/images/456.jpg differ diff --git a/images/457.jpg b/images/457.jpg new file mode 100644 index 0000000..e738163 Binary files /dev/null and b/images/457.jpg differ diff --git a/images/458.jpg b/images/458.jpg new file mode 100644 index 0000000..1e7ab19 Binary files /dev/null and b/images/458.jpg differ diff --git a/images/459.jpg b/images/459.jpg new file mode 100644 index 0000000..8ac9fd4 Binary files /dev/null and b/images/459.jpg differ diff --git a/images/46.jpg b/images/46.jpg new file mode 100644 index 0000000..a3dc911 Binary files /dev/null and b/images/46.jpg differ diff --git a/images/460.jpg b/images/460.jpg new file mode 100644 index 0000000..1c67372 Binary files /dev/null and b/images/460.jpg differ diff --git a/images/461.jpg b/images/461.jpg new file mode 100644 index 0000000..00edd70 Binary files /dev/null and b/images/461.jpg differ diff --git a/images/462.jpg b/images/462.jpg new file mode 100644 index 0000000..ff1f4d8 Binary files /dev/null and b/images/462.jpg differ diff --git a/images/463.jpg b/images/463.jpg new file mode 100644 index 0000000..a43c8e4 Binary files /dev/null and b/images/463.jpg differ diff --git a/images/464.jpg b/images/464.jpg new file mode 100644 index 0000000..baf9cd4 Binary files /dev/null and b/images/464.jpg differ diff --git a/images/465.jpg b/images/465.jpg new file mode 100644 index 0000000..8ec2303 Binary files /dev/null and b/images/465.jpg differ diff --git a/images/466.jpg b/images/466.jpg new file mode 100644 index 0000000..4390d6b Binary files /dev/null and b/images/466.jpg differ diff --git a/images/467.jpg b/images/467.jpg new file mode 100644 index 0000000..b636364 Binary files /dev/null and b/images/467.jpg differ diff --git a/images/468.jpg b/images/468.jpg new file mode 100644 index 0000000..07e63e3 Binary files /dev/null and b/images/468.jpg differ diff --git a/images/469.jpg b/images/469.jpg new file mode 100644 index 0000000..4bf5f18 Binary files /dev/null and b/images/469.jpg differ diff --git a/images/47.jpg b/images/47.jpg new file mode 100644 index 0000000..7293171 Binary files /dev/null and b/images/47.jpg differ diff --git a/images/470.jpg b/images/470.jpg new file mode 100644 index 0000000..45fd63f Binary files /dev/null and b/images/470.jpg differ diff --git a/images/471.jpg b/images/471.jpg new file mode 100644 index 0000000..194dff4 Binary files /dev/null and b/images/471.jpg differ diff --git a/images/472.jpg b/images/472.jpg new file mode 100644 index 0000000..1a5bc27 Binary files /dev/null and b/images/472.jpg differ diff --git a/images/473.jpg b/images/473.jpg new file mode 100644 index 0000000..d4c054b Binary files /dev/null and b/images/473.jpg differ diff --git a/images/474.jpg b/images/474.jpg new file mode 100644 index 0000000..79f3a0b Binary files /dev/null and b/images/474.jpg differ diff --git a/images/475.jpg b/images/475.jpg new file mode 100644 index 0000000..9d361f3 Binary files /dev/null and b/images/475.jpg differ diff --git a/images/476.jpg b/images/476.jpg new file mode 100644 index 0000000..7aa3fc3 Binary files /dev/null and b/images/476.jpg differ diff --git a/images/477.jpg b/images/477.jpg new file mode 100644 index 0000000..a9b30d6 Binary files /dev/null and b/images/477.jpg differ diff --git a/images/478.jpg b/images/478.jpg new file mode 100644 index 0000000..766f7b9 Binary files /dev/null and b/images/478.jpg differ diff --git a/images/479.jpg b/images/479.jpg new file mode 100644 index 0000000..b49c470 Binary files /dev/null and b/images/479.jpg differ diff --git a/images/48.jpg b/images/48.jpg new file mode 100644 index 0000000..7a8ae78 Binary files /dev/null and b/images/48.jpg differ diff --git a/images/49.jpg b/images/49.jpg new file mode 100644 index 0000000..8c55ff8 Binary files /dev/null and b/images/49.jpg differ diff --git a/images/5.jpg b/images/5.jpg new file mode 100644 index 0000000..67c2585 Binary files /dev/null and b/images/5.jpg differ diff --git a/images/50.jpg b/images/50.jpg new file mode 100644 index 0000000..485eb83 Binary files /dev/null and b/images/50.jpg differ diff --git a/images/51.jpg b/images/51.jpg new file mode 100644 index 0000000..878d490 Binary files /dev/null and b/images/51.jpg differ diff --git a/images/52.jpg b/images/52.jpg new file mode 100644 index 0000000..97be07d Binary files /dev/null and b/images/52.jpg differ diff --git a/images/53.jpg b/images/53.jpg new file mode 100644 index 0000000..554fdce Binary files /dev/null and b/images/53.jpg differ diff --git a/images/54.jpg b/images/54.jpg new file mode 100644 index 0000000..28919e6 Binary files /dev/null and b/images/54.jpg differ diff --git a/images/55.jpg b/images/55.jpg new file mode 100644 index 0000000..fdd7850 Binary files /dev/null and b/images/55.jpg differ diff --git a/images/56.jpg b/images/56.jpg new file mode 100644 index 0000000..0632a06 Binary files /dev/null and b/images/56.jpg differ diff --git a/images/57.jpg b/images/57.jpg new file mode 100644 index 0000000..1ee8a9d Binary files /dev/null and b/images/57.jpg differ diff --git a/images/58.jpg b/images/58.jpg new file mode 100644 index 0000000..5b5bc51 Binary files /dev/null and b/images/58.jpg differ diff --git a/images/59.jpg b/images/59.jpg new file mode 100644 index 0000000..71411b0 Binary files /dev/null and b/images/59.jpg differ diff --git a/images/6.jpg b/images/6.jpg new file mode 100644 index 0000000..e13e8fa Binary files /dev/null and b/images/6.jpg differ diff --git a/images/60.jpg b/images/60.jpg new file mode 100644 index 0000000..faf0e62 Binary files /dev/null and b/images/60.jpg differ diff --git a/images/61.jpg b/images/61.jpg new file mode 100644 index 0000000..1bd08bf Binary files /dev/null and b/images/61.jpg differ diff --git a/images/62.jpg b/images/62.jpg new file mode 100644 index 0000000..2071cea Binary files /dev/null and b/images/62.jpg differ diff --git a/images/63.jpg b/images/63.jpg new file mode 100644 index 0000000..4e55cfb Binary files /dev/null and b/images/63.jpg differ diff --git a/images/64.jpg b/images/64.jpg new file mode 100644 index 0000000..a59aabf Binary files /dev/null and b/images/64.jpg differ diff --git a/images/65.jpg b/images/65.jpg new file mode 100644 index 0000000..92bb4a9 Binary files /dev/null and b/images/65.jpg differ diff --git a/images/66.jpg b/images/66.jpg new file mode 100644 index 0000000..8a837f6 Binary files /dev/null and b/images/66.jpg differ diff --git a/images/67.jpg b/images/67.jpg new file mode 100644 index 0000000..1c5ce06 Binary files /dev/null and b/images/67.jpg differ diff --git a/images/68.jpg b/images/68.jpg new file mode 100644 index 0000000..feda40d Binary files /dev/null and b/images/68.jpg differ diff --git a/images/69.jpg b/images/69.jpg new file mode 100644 index 0000000..8d63097 Binary files /dev/null and b/images/69.jpg differ diff --git a/images/7.jpg b/images/7.jpg new file mode 100644 index 0000000..a3c65b3 Binary files /dev/null and b/images/7.jpg differ diff --git a/images/70.jpg b/images/70.jpg new file mode 100644 index 0000000..a3eb707 Binary files /dev/null and b/images/70.jpg differ diff --git a/images/71.jpg b/images/71.jpg new file mode 100644 index 0000000..1155329 Binary files /dev/null and b/images/71.jpg differ diff --git a/images/72.jpg b/images/72.jpg new file mode 100644 index 0000000..dd9afa9 Binary files /dev/null and b/images/72.jpg differ diff --git a/images/73.jpg b/images/73.jpg new file mode 100644 index 0000000..008c373 Binary files /dev/null and b/images/73.jpg differ diff --git a/images/74.jpg b/images/74.jpg new file mode 100644 index 0000000..7432f8a Binary files /dev/null and b/images/74.jpg differ diff --git a/images/75.jpg b/images/75.jpg new file mode 100644 index 0000000..9dd36d0 Binary files /dev/null and b/images/75.jpg differ diff --git a/images/76.jpg b/images/76.jpg new file mode 100644 index 0000000..6d99b79 Binary files /dev/null and b/images/76.jpg differ diff --git a/images/77.jpg b/images/77.jpg new file mode 100644 index 0000000..ad35e8d Binary files /dev/null and b/images/77.jpg differ diff --git a/images/78.jpg b/images/78.jpg new file mode 100644 index 0000000..3d2be05 Binary files /dev/null and b/images/78.jpg differ diff --git a/images/79.jpg b/images/79.jpg new file mode 100644 index 0000000..c85a458 Binary files /dev/null and b/images/79.jpg differ diff --git a/images/8.jpg b/images/8.jpg new file mode 100644 index 0000000..982f515 Binary files /dev/null and b/images/8.jpg differ diff --git a/images/80.jpg b/images/80.jpg new file mode 100644 index 0000000..314c85f Binary files /dev/null and b/images/80.jpg differ diff --git a/images/81.jpg b/images/81.jpg new file mode 100644 index 0000000..d6a9472 Binary files /dev/null and b/images/81.jpg differ diff --git a/images/82.jpg b/images/82.jpg new file mode 100644 index 0000000..cbeab89 Binary files /dev/null and b/images/82.jpg differ diff --git a/images/83.jpg b/images/83.jpg new file mode 100644 index 0000000..1dc006a Binary files /dev/null and b/images/83.jpg differ diff --git a/images/84.jpg b/images/84.jpg new file mode 100644 index 0000000..b6c2614 Binary files /dev/null and b/images/84.jpg differ diff --git a/images/85.jpg b/images/85.jpg new file mode 100644 index 0000000..41f39fa Binary files /dev/null and b/images/85.jpg differ diff --git a/images/86.jpg b/images/86.jpg new file mode 100644 index 0000000..e0216de Binary files /dev/null and b/images/86.jpg differ diff --git a/images/87.jpg b/images/87.jpg new file mode 100644 index 0000000..248616e Binary files /dev/null and b/images/87.jpg differ diff --git a/images/88.jpg b/images/88.jpg new file mode 100644 index 0000000..afde643 Binary files /dev/null and b/images/88.jpg differ diff --git a/images/89.jpg b/images/89.jpg new file mode 100644 index 0000000..9812deb Binary files /dev/null and b/images/89.jpg differ diff --git a/images/9.jpg b/images/9.jpg new file mode 100644 index 0000000..3ed3cd7 Binary files /dev/null and b/images/9.jpg differ diff --git a/images/90.jpg b/images/90.jpg new file mode 100644 index 0000000..7c1848b Binary files /dev/null and b/images/90.jpg differ diff --git a/images/91.jpg b/images/91.jpg new file mode 100644 index 0000000..309b4ff Binary files /dev/null and b/images/91.jpg differ diff --git a/images/92.jpg b/images/92.jpg new file mode 100644 index 0000000..0662a5d Binary files /dev/null and b/images/92.jpg differ diff --git a/images/93.jpg b/images/93.jpg new file mode 100644 index 0000000..aecdbd4 Binary files /dev/null and b/images/93.jpg differ diff --git a/images/94.jpg b/images/94.jpg new file mode 100644 index 0000000..b36b481 Binary files /dev/null and b/images/94.jpg differ diff --git a/images/95.jpg b/images/95.jpg new file mode 100644 index 0000000..49fe4e9 Binary files /dev/null and b/images/95.jpg differ diff --git a/images/96.jpg b/images/96.jpg new file mode 100644 index 0000000..a382ebe Binary files /dev/null and b/images/96.jpg differ diff --git a/images/97.jpg b/images/97.jpg new file mode 100644 index 0000000..cfdb406 Binary files /dev/null and b/images/97.jpg differ diff --git a/images/98.jpg b/images/98.jpg new file mode 100644 index 0000000..3038bd5 Binary files /dev/null and b/images/98.jpg differ diff --git a/images/99.jpg b/images/99.jpg new file mode 100644 index 0000000..4354f34 Binary files /dev/null and b/images/99.jpg differ diff --git a/models/DIFRINT/__init__.py b/models/DIFRINT/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/models/DIFRINT/flowlib.py b/models/DIFRINT/flowlib.py new file mode 100644 index 0000000..805f308 --- /dev/null +++ b/models/DIFRINT/flowlib.py @@ -0,0 +1,517 @@ +#!/usr/bin/python +""" +# ============================== +# flowlib.py +# library for optical flow processing +# Author: Ruoteng Li +# Date: 6th Aug 2016 +# ============================== +""" +import png +import numpy as np +import matplotlib.colors as cl +import matplotlib.pyplot as plt +from PIL import Image + + +UNKNOWN_FLOW_THRESH = 1e7 +SMALLFLOW = 0.0 +LARGEFLOW = 1e8 + +""" +============= +Flow Section +============= +""" + + +def show_flow(filename): + """ + visualize optical flow map using matplotlib + :param filename: optical flow file + :return: None + """ + flow = read_flow(filename) + img = flow_to_image(flow) + plt.imshow(img) + plt.show() + + +def visualize_flow(flow, mode='Y'): + """ + this function visualize the input flow + :param flow: input flow in array + :param mode: choose which color mode to visualize the flow (Y: Ccbcr, RGB: RGB color) + :return: None + """ + if mode == 'Y': + # Ccbcr color wheel + img = flow_to_image(flow) + plt.imshow(img) + plt.show() + elif mode == 'RGB': + (h, w) = flow.shape[0:2] + du = flow[:, :, 0] + dv = flow[:, :, 1] + valid = flow[:, :, 2] + max_flow = max(np.max(du), np.max(dv)) + img = np.zeros((h, w, 3), dtype=np.float64) + # angle layer + img[:, :, 0] = np.arctan2(dv, du) / (2 * np.pi) + # magnitude layer, normalized to 1 + img[:, :, 1] = np.sqrt(du * du + dv * dv) * 8 / max_flow + # phase layer + img[:, :, 2] = 8 - img[:, :, 1] + # clip to [0,1] + small_idx = img[:, :, 0:3] < 0 + large_idx = img[:, :, 0:3] > 1 + img[small_idx] = 0 + img[large_idx] = 1 + # convert to rgb + img = cl.hsv_to_rgb(img) + # remove invalid point + img[:, :, 0] = img[:, :, 0] * valid + img[:, :, 1] = img[:, :, 1] * valid + img[:, :, 2] = img[:, :, 2] * valid + # show + plt.imshow(img) + plt.show() + + return None + + +def read_flow(filename): + """ + read optical flow from Middlebury .flo file + :param filename: name of the flow file + :return: optical flow data in matrix + """ + f = open(filename, 'rb') + magic = np.fromfile(f, np.float32, count=1) + data2d = None + + if 202021.25 != magic: + print('Magic number incorrect. Invalid .flo file') + else: + w = np.fromfile(f, np.int32, count=1) + h = np.fromfile(f, np.int32, count=1) + print("Reading %d x %d flo file" % (h, w)) + data2d = np.fromfile(f, np.float32, count=2*w[0]*h[0]) + # reshape data into 3D array (columns, rows, channels) + data2d = np.resize(data2d, (h[0], w[0], 2)) + f.close() + return data2d + + +def read_flow_png(flow_file): + """ + Read optical flow from KITTI .png file + :param flow_file: name of the flow file + :return: optical flow data in matrix + """ + flow_object = png.Reader(filename=flow_file) + flow_direct = flow_object.asDirect() + flow_data = list(flow_direct[2]) + (w, h) = flow_direct[3]['size'] + flow = np.zeros((h, w, 3), dtype=np.float64) + for i in range(len(flow_data)): + flow[i, :, 0] = flow_data[i][0::3] + flow[i, :, 1] = flow_data[i][1::3] + flow[i, :, 2] = flow_data[i][2::3] + + invalid_idx = (flow[:, :, 2] == 0) + flow[:, :, 0:2] = (flow[:, :, 0:2] - 2 ** 15) / 64.0 + flow[invalid_idx, 0] = 0 + flow[invalid_idx, 1] = 0 + return flow + + +def write_flow(flow, filename): + """ + write optical flow in Middlebury .flo format + :param flow: optical flow map + :param filename: optical flow file path to be saved + :return: None + """ + f = open(filename, 'wb') + magic = np.array([202021.25], dtype=np.float32) + (height, width) = flow.shape[0:2] + w = np.array([width], dtype=np.int32) + h = np.array([height], dtype=np.int32) + magic.tofile(f) + w.tofile(f) + h.tofile(f) + flow.tofile(f) + f.close() + + +def segment_flow(flow): + h = flow.shape[0] + w = flow.shape[1] + u = flow[:, :, 0] + v = flow[:, :, 1] + + idx = ((abs(u) > LARGEFLOW) | (abs(v) > LARGEFLOW)) + idx2 = (abs(u) == SMALLFLOW) + class0 = (v == 0) & (u == 0) + u[idx2] = 0.00001 + tan_value = v / u + + class1 = (tan_value < 1) & (tan_value >= 0) & (u > 0) & (v >= 0) + class2 = (tan_value >= 1) & (u >= 0) & (v >= 0) + class3 = (tan_value < -1) & (u <= 0) & (v >= 0) + class4 = (tan_value < 0) & (tan_value >= -1) & (u < 0) & (v >= 0) + class8 = (tan_value >= -1) & (tan_value < 0) & (u > 0) & (v <= 0) + class7 = (tan_value < -1) & (u >= 0) & (v <= 0) + class6 = (tan_value >= 1) & (u <= 0) & (v <= 0) + class5 = (tan_value >= 0) & (tan_value < 1) & (u < 0) & (v <= 0) + + seg = np.zeros((h, w)) + + seg[class1] = 1 + seg[class2] = 2 + seg[class3] = 3 + seg[class4] = 4 + seg[class5] = 5 + seg[class6] = 6 + seg[class7] = 7 + seg[class8] = 8 + seg[class0] = 0 + seg[idx] = 0 + + return seg + + +def flow_error(tu, tv, u, v): + """ + Calculate average end point error + :param tu: ground-truth horizontal flow map + :param tv: ground-truth vertical flow map + :param u: estimated horizontal flow map + :param v: estimated vertical flow map + :return: End point error of the estimated flow + """ + smallflow = 0.0 + ''' + stu = tu[bord+1:end-bord,bord+1:end-bord] + stv = tv[bord+1:end-bord,bord+1:end-bord] + su = u[bord+1:end-bord,bord+1:end-bord] + sv = v[bord+1:end-bord,bord+1:end-bord] + ''' + stu = tu[:] + stv = tv[:] + su = u[:] + sv = v[:] + + idxUnknow = (abs(stu) > UNKNOWN_FLOW_THRESH) | (abs(stv) > UNKNOWN_FLOW_THRESH) + stu[idxUnknow] = 0 + stv[idxUnknow] = 0 + su[idxUnknow] = 0 + sv[idxUnknow] = 0 + + ind2 = [(np.absolute(stu) > smallflow) | (np.absolute(stv) > smallflow)] + index_su = su[ind2] + index_sv = sv[ind2] + an = 1.0 / np.sqrt(index_su ** 2 + index_sv ** 2 + 1) + un = index_su * an + vn = index_sv * an + + index_stu = stu[ind2] + index_stv = stv[ind2] + tn = 1.0 / np.sqrt(index_stu ** 2 + index_stv ** 2 + 1) + tun = index_stu * tn + tvn = index_stv * tn + + ''' + angle = un * tun + vn * tvn + (an * tn) + index = [angle == 1.0] + angle[index] = 0.999 + ang = np.arccos(angle) + mang = np.mean(ang) + mang = mang * 180 / np.pi + ''' + + epe = np.sqrt((stu - su) ** 2 + (stv - sv) ** 2) + epe = epe[ind2] + mepe = np.mean(epe) + return mepe + + +def flow_to_image(flow): + """ + Convert flow into middlebury color code image + :param flow: optical flow map + :return: optical flow image in middlebury color + """ + u = flow[:, :, 0] + v = flow[:, :, 1] + + maxu = -999. + maxv = -999. + minu = 999. + minv = 999. + + idxUnknow = (abs(u) > UNKNOWN_FLOW_THRESH) | (abs(v) > UNKNOWN_FLOW_THRESH) + u[idxUnknow] = 0 + v[idxUnknow] = 0 + + maxu = max(maxu, np.max(u)) + minu = min(minu, np.min(u)) + + maxv = max(maxv, np.max(v)) + minv = min(minv, np.min(v)) + + rad = np.sqrt(u ** 2 + v ** 2) + maxrad = max(-1, np.max(rad)) + + print("max flow: %.4f\nflow range:\nu = %.3f .. %.3f\nv = %.3f .. %.3f" % (maxrad, minu,maxu, minv, maxv)) + + u = u/(maxrad + np.finfo(float).eps) + v = v/(maxrad + np.finfo(float).eps) + + img = compute_color(u, v) + + idx = np.repeat(idxUnknow[:, :, np.newaxis], 3, axis=2) + img[idx] = 0 + + return np.uint8(img) + + +def evaluate_flow_file(gt, pred): + """ + evaluate the estimated optical flow end point error according to ground truth provided + :param gt: ground truth file path + :param pred: estimated optical flow file path + :return: end point error, float32 + """ + # Read flow files and calculate the errors + gt_flow = read_flow(gt) # ground truth flow + eva_flow = read_flow(pred) # predicted flow + # Calculate errors + average_pe = flow_error(gt_flow[:, :, 0], gt_flow[:, :, 1], eva_flow[:, :, 0], eva_flow[:, :, 1]) + return average_pe + + +def evaluate_flow(gt_flow, pred_flow): + """ + gt: ground-truth flow + pred: estimated flow + """ + average_pe = flow_error(gt_flow[:, :, 0], gt_flow[:, :, 1], pred_flow[:, :, 0], pred_flow[:, :, 1]) + return average_pe + + +""" +============== +Disparity Section +============== +""" + + +def read_disp_png(file_name): + """ + Read optical flow from KITTI .png file + :param file_name: name of the flow file + :return: optical flow data in matrix + """ + image_object = png.Reader(filename=file_name) + image_direct = image_object.asDirect() + image_data = list(image_direct[2]) + (w, h) = image_direct[3]['size'] + channel = len(image_data[0]) / w + flow = np.zeros((h, w, channel), dtype=np.uint16) + for i in range(len(image_data)): + for j in range(channel): + flow[i, :, j] = image_data[i][j::channel] + return flow[:, :, 0] / 256 + + +def disp_to_flowfile(disp, filename): + """ + Read KITTI disparity file in png format + :param disp: disparity matrix + :param filename: the flow file name to save + :return: None + """ + f = open(filename, 'wb') + magic = np.array([202021.25], dtype=np.float32) + (height, width) = disp.shape[0:2] + w = np.array([width], dtype=np.int32) + h = np.array([height], dtype=np.int32) + empty_map = np.zeros((height, width), dtype=np.float32) + data = np.dstack((disp, empty_map)) + magic.tofile(f) + w.tofile(f) + h.tofile(f) + data.tofile(f) + f.close() + + +""" +============== +Image Section +============== +""" + + +def read_image(filename): + """ + Read normal image of any format + :param filename: name of the image file + :return: image data in matrix uint8 type + """ + img = Image.open(filename) + im = np.array(img) + return im + + +def warp_image(im, flow): + """ + Use optical flow to warp image to the next + :param im: image to warp + :param flow: optical flow + :return: warped image + """ + from scipy import interpolate + image_height = im.shape[0] + image_width = im.shape[1] + flow_height = flow.shape[0] + flow_width = flow.shape[1] + n = image_height * image_width + (iy, ix) = np.mgrid[0:image_height, 0:image_width] + (fy, fx) = np.mgrid[0:flow_height, 0:flow_width] + fx += flow[:,:,0] + fy += flow[:,:,1] + mask = np.logical_or(fx <0 , fx > flow_width) + mask = np.logical_or(mask, fy < 0) + mask = np.logical_or(mask, fy > flow_height) + fx = np.minimum(np.maximum(fx, 0), flow_width) + fy = np.minimum(np.maximum(fy, 0), flow_height) + points = np.concatenate((ix.reshape(n,1), iy.reshape(n,1)), axis=1) + xi = np.concatenate((fx.reshape(n, 1), fy.reshape(n,1)), axis=1) + warp = np.zeros((image_height, image_width, im.shape[2])) + for i in range(im.shape[2]): + channel = im[:, :, i] + plt.imshow(channel, cmap='gray') + values = channel.reshape(n, 1) + new_channel = interpolate.griddata(points, values, xi, method='cubic') + new_channel = np.reshape(new_channel, [flow_height, flow_width]) + new_channel[mask] = 1 + warp[:, :, i] = new_channel.astype(np.uint8) + + return warp.astype(np.uint8) + + +""" +============== +Others +============== +""" + +def scale_image(image, new_range): + """ + Linearly scale the image into desired range + :param image: input image + :param new_range: the new range to be aligned + :return: image normalized in new range + """ + min_val = np.min(image).astype(np.float32) + max_val = np.max(image).astype(np.float32) + min_val_new = np.array(min(new_range), dtype=np.float32) + max_val_new = np.array(max(new_range), dtype=np.float32) + scaled_image = (image - min_val) / (max_val - min_val) * (max_val_new - min_val_new) + min_val_new + return scaled_image.astype(np.uint8) + + +def compute_color(u, v): + """ + compute optical flow color map + :param u: optical flow horizontal map + :param v: optical flow vertical map + :return: optical flow in color code + """ + [h, w] = u.shape + img = np.zeros([h, w, 3]) + nanIdx = np.isnan(u) | np.isnan(v) + u[nanIdx] = 0 + v[nanIdx] = 0 + + colorwheel = make_color_wheel() + ncols = np.size(colorwheel, 0) + + rad = np.sqrt(u**2+v**2) + + a = np.arctan2(-v, -u) / np.pi + + fk = (a+1) / 2 * (ncols - 1) + 1 + + k0 = np.floor(fk).astype(int) + + k1 = k0 + 1 + k1[k1 == ncols+1] = 1 + f = fk - k0 + + for i in range(0, np.size(colorwheel,1)): + tmp = colorwheel[:, i] + col0 = tmp[k0-1] / 255 + col1 = tmp[k1-1] / 255 + col = (1-f) * col0 + f * col1 + + idx = rad <= 1 + col[idx] = 1-rad[idx]*(1-col[idx]) + notidx = np.logical_not(idx) + + col[notidx] *= 0.75 + img[:, :, i] = np.uint8(np.floor(255 * col*(1-nanIdx))) + + return img + + +def make_color_wheel(): + """ + Generate color wheel according Middlebury color code + :return: Color wheel + """ + RY = 15 + YG = 6 + GC = 4 + CB = 11 + BM = 13 + MR = 6 + + ncols = RY + YG + GC + CB + BM + MR + + colorwheel = np.zeros([ncols, 3]) + + col = 0 + + # RY + colorwheel[0:RY, 0] = 255 + colorwheel[0:RY, 1] = np.transpose(np.floor(255*np.arange(0, RY) / RY)) + col += RY + + # YG + colorwheel[col:col+YG, 0] = 255 - np.transpose(np.floor(255*np.arange(0, YG) / YG)) + colorwheel[col:col+YG, 1] = 255 + col += YG + + # GC + colorwheel[col:col+GC, 1] = 255 + colorwheel[col:col+GC, 2] = np.transpose(np.floor(255*np.arange(0, GC) / GC)) + col += GC + + # CB + colorwheel[col:col+CB, 1] = 255 - np.transpose(np.floor(255*np.arange(0, CB) / CB)) + colorwheel[col:col+CB, 2] = 255 + col += CB + + # BM + colorwheel[col:col+BM, 2] = 255 + colorwheel[col:col+BM, 0] = np.transpose(np.floor(255*np.arange(0, BM) / BM)) + col += + BM + + # MR + colorwheel[col:col+MR, 2] = 255 - np.transpose(np.floor(255 * np.arange(0, MR) / MR)) + colorwheel[col:col+MR, 0] = 255 + + return colorwheel \ No newline at end of file diff --git a/models/DIFRINT/models.py b/models/DIFRINT/models.py new file mode 100644 index 0000000..4b90358 --- /dev/null +++ b/models/DIFRINT/models.py @@ -0,0 +1,744 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.nn.init as init + +from .pwcNet import PwcNet + +import math +import pdb + + +class UNet1(nn.Module): + def __init__(self): + super(UNet1, self).__init__() + + class Encoder(nn.Module): + def __init__(self, in_nc, out_nc, stride, k_size=3, pad=1): + super(Encoder, self).__init__() + + self.seq = nn.Sequential( + nn.ReflectionPad2d(pad), + nn.Conv2d(in_nc, out_nc, kernel_size=k_size, + stride=stride, padding=0), + nn.LeakyReLU(0.2) + ) + self.GateConv = nn.Sequential( + nn.ReflectionPad2d(pad), + nn.Conv2d(in_nc, out_nc, kernel_size=k_size, + stride=stride, padding=0), + nn.Sigmoid() + ) + + def forward(self, x): + return self.seq(x) * self.GateConv(x) + + class Decoder(nn.Module): + def __init__(self, in_nc, out_nc, stride, k_size=3, pad=1, tanh=False): + super(Decoder, self).__init__() + + self.seq = nn.Sequential( + nn.ReflectionPad2d(pad), + nn.Conv2d(in_nc, in_nc, kernel_size=k_size, + stride=stride, padding=0), + nn.ReflectionPad2d(pad), + nn.Conv2d(in_nc, out_nc, kernel_size=k_size, + stride=stride, padding=0) + ) + + if tanh: + self.activ = nn.Tanh() + else: + self.activ = nn.LeakyReLU(0.2) + + self.GateConv = nn.Sequential( + nn.ReflectionPad2d(pad), + nn.Conv2d(in_nc, in_nc, kernel_size=k_size, + stride=stride, padding=0), + nn.ReflectionPad2d(pad), + nn.Conv2d(in_nc, out_nc, kernel_size=k_size, + stride=stride, padding=0), + nn.Sigmoid() + ) + + def forward(self, x): + s = self.seq(x) + s = self.activ(s) + return s * self.GateConv(x) + + self.enc0 = Encoder(6, 8, stride=1) + self.enc1 = Encoder(8, 16, stride=2) + self.enc2 = Encoder(16, 16, stride=2) + self.enc3 = Encoder(16, 16, stride=2) + + self.dec0 = Decoder(16, 16, stride=1) + # up-scaling + concat + self.dec1 = Decoder(16+16, 16, stride=1) + self.dec2 = Decoder(16+16, 16, stride=1) + self.dec3 = Decoder(16+8, 16, stride=1) + + self.dec4 = Decoder(16, 3, stride=1, tanh=True) + + def forward(self, x1, x2): + s0 = self.enc0(torch.cat([x1, x2], 1).cuda()) + s1 = self.enc1(s0) + s2 = self.enc2(s1) + s3 = self.enc3(s2) + + s4 = self.dec0(s3) + # up-scaling + concat + s4 = F.interpolate(s4, scale_factor=2, mode='nearest') + s5 = self.dec1(torch.cat([s4, s2], 1).cuda()) + s5 = F.interpolate(s5, scale_factor=2, mode='nearest') + s6 = self.dec2(torch.cat([s5, s1], 1).cuda()) + s6 = F.interpolate(s6, scale_factor=2, mode='nearest') + s7 = self.dec3(torch.cat([s6, s0], 1).cuda()) + + out = self.dec4(s7) + return out + +class ResNet(nn.Module): + def __init__(self): + super(ResNet, self).__init__() + + class ConvBlock(nn.Module): + def __init__(self, in_ch, out_ch): + super(ConvBlock, self).__init__() + + self.seq = nn.Sequential( + nn.Conv2d(in_ch, out_ch, kernel_size=1, + stride=1, padding=0), + nn.LeakyReLU(0.2) + ) + + self.GateConv = nn.Sequential( + nn.ReflectionPad2d(1), + nn.Conv2d(in_ch, in_ch, kernel_size=3, + stride=1, padding=0), + nn.ReflectionPad2d(1), + nn.Conv2d(in_ch, out_ch, kernel_size=3, + stride=1, padding=0), + nn.Sigmoid() + ) + + def forward(self, x): + return self.seq(x) * self.GateConv(x) + + class ResBlock(nn.Module): + def __init__(self, num_ch): + super(ResBlock, self).__init__() + + self.seq = nn.Sequential( + nn.Conv2d(num_ch, num_ch, kernel_size=1, + stride=1, padding=0), + nn.LeakyReLU(0.2) + ) + + self.GateConv = nn.Sequential( + nn.ReflectionPad2d(1), + nn.Conv2d(num_ch, num_ch, kernel_size=3, + stride=1, padding=0), + nn.ReflectionPad2d(1), + nn.Conv2d(num_ch, num_ch, kernel_size=3, + stride=1, padding=0), + nn.Sigmoid() + ) + + def forward(self, x): + return self.seq(x) * self.GateConv(x) + x + + self.seq = nn.Sequential( + ConvBlock(6, 32), + ResBlock(32), + ResBlock(32), + ResBlock(32), + ResBlock(32), + ResBlock(32), + ConvBlock(32, 3), + nn.Tanh() + ) + + def forward(self, x1, x2): + return self.seq(torch.cat([x1, x2], 1).cuda()) + + +############################################################################################################# + +class UNet2(nn.Module): + def __init__(self): + super(UNet2, self).__init__() + + class Encoder(nn.Module): + def __init__(self, in_nc, out_nc, stride, k_size=3, pad=1): + super(Encoder, self).__init__() + + self.seq = nn.Sequential( + nn.ReflectionPad2d(pad), + nn.Conv2d(in_nc, out_nc, kernel_size=k_size, + stride=stride, padding=0), + nn.ReLU() + ) + self.GateConv = nn.Sequential( + nn.ReflectionPad2d(pad), + nn.Conv2d(in_nc, out_nc, kernel_size=k_size, + stride=stride, padding=0), + nn.Sigmoid() + ) + + def forward(self, x): + return self.seq(x) * self.GateConv(x) + + class Decoder(nn.Module): + def __init__(self, in_nc, out_nc, stride, k_size=3, pad=1, tanh=False): + super(Decoder, self).__init__() + + self.seq = nn.Sequential( + nn.ReflectionPad2d(pad), + nn.Conv2d(in_nc, in_nc, kernel_size=k_size, + stride=stride, padding=0), + nn.ReflectionPad2d(pad), + nn.Conv2d(in_nc, out_nc, kernel_size=k_size, + stride=stride, padding=0) + ) + + if tanh: + self.activ = nn.Tanh() + else: + self.activ = nn.ReLU() + + self.GateConv = nn.Sequential( + nn.ReflectionPad2d(pad), + nn.Conv2d(in_nc, in_nc, kernel_size=k_size, + stride=stride, padding=0), + nn.ReflectionPad2d(pad), + nn.Conv2d(in_nc, out_nc, kernel_size=k_size, + stride=stride, padding=0), + nn.Sigmoid() + ) + + def forward(self, x): + s = self.seq(x) + s = self.activ(s) + return s * self.GateConv(x) + + self.enc0 = Encoder(16, 32, stride=1) + self.enc1 = Encoder(32, 32, stride=2) + self.enc2 = Encoder(32, 32, stride=2) + self.enc3 = Encoder(32, 32, stride=2) + + self.dec0 = Decoder(32, 32, stride=1) + # up-scaling + concat + self.dec1 = Decoder(32+32, 32, stride=1) + self.dec2 = Decoder(32+32, 32, stride=1) + self.dec3 = Decoder(32+32, 32, stride=1) + + self.dec4 = Decoder(32, 3, stride=1, tanh=True) + + def forward(self, w1, w2, flo1, flo2, fr1, fr2): + s0 = self.enc0(torch.cat([w1, w2, flo1, flo1, fr1, fr2], 1).cuda()) + s1 = self.enc1(s0) + s2 = self.enc2(s1) + s3 = self.enc3(s2) + + s4 = self.dec0(s3) + # up-scaling + concat + s4 = F.interpolate(s4, scale_factor=2, mode='nearest') + s5 = self.dec1(torch.cat([s4, s2], 1).cuda()) + s5 = F.interpolate(s5, scale_factor=2, mode='nearest') + s6 = self.dec2(torch.cat([s5, s1], 1).cuda()) + s6 = F.interpolate(s6, scale_factor=2, mode='nearest') + s7 = self.dec3(torch.cat([s6, s0], 1).cuda()) + + out = self.dec4(s7) + return out + + +class DIFNet2(nn.Module): + def __init__(self): + super(DIFNet2, self).__init__() + + class Backward(torch.nn.Module): + def __init__(self): + super(Backward, self).__init__() + # end + + def forward(self, tensorInput, tensorFlow, scale=1.0): + if hasattr(self, 'tensorPartial') == False or self.tensorPartial.size(0) != tensorFlow.size(0) or self.tensorPartial.size(2) != tensorFlow.size(2) or self.tensorPartial.size(3) != tensorFlow.size(3): + self.tensorPartial = torch.FloatTensor().resize_(tensorFlow.size( + 0), 1, tensorFlow.size(2), tensorFlow.size(3)).fill_(1.0).cuda() + # end + + if hasattr(self, 'tensorGrid') == False or self.tensorGrid.size(0) != tensorFlow.size(0) or self.tensorGrid.size(2) != tensorFlow.size(2) or self.tensorGrid.size(3) != tensorFlow.size(3): + tensorHorizontal = torch.linspace(-1.0, 1.0, tensorFlow.size(3)).view( + 1, 1, 1, tensorFlow.size(3)).expand(tensorFlow.size(0), -1, tensorFlow.size(2), -1) + tensorVertical = torch.linspace(-1.0, 1.0, tensorFlow.size(2)).view( + 1, 1, tensorFlow.size(2), 1).expand(tensorFlow.size(0), -1, -1, tensorFlow.size(3)) + + self.tensorGrid = torch.cat( + [tensorHorizontal, tensorVertical], 1).cuda() + # end + # pdb.set_trace() + tensorInput = torch.cat([tensorInput, self.tensorPartial], 1) + tensorFlow = torch.cat([tensorFlow[:, 0:1, :, :] / ((tensorInput.size( + 3) - 1.0) / 2.0), tensorFlow[:, 1:2, :, :] / ((tensorInput.size(2) - 1.0) / 2.0)], 1) + + tensorOutput = torch.nn.functional.grid_sample(input=tensorInput, grid=( + self.tensorGrid + tensorFlow*scale).permute(0, 2, 3, 1), mode='bilinear', padding_mode='zeros') + + tensorMask = tensorOutput[:, -1:, :, :] + tensorMask[tensorMask > 0.999] = 1.0 + tensorMask[tensorMask < 1.0] = 0.0 + + return tensorOutput[:, :-1, :, :] * tensorMask + + # PWC + self.pwc = PwcNet() + self.pwc.load_state_dict(torch.load('./ckpt/sintel.pytorch')) + self.pwc.eval() + + # Warping layer + self.warpLayer = Backward() + self.warpLayer.eval() + + # UNets + self.UNet2 = UNet2() + self.ResNet2 = ResNet2() + + def warpFrame(self, fr_1, fr_2, scale=1.0): + with torch.no_grad(): + # Due to Pyramid method? + temp_w = int(math.floor(math.ceil(fr_1.size(3) / 64.0) * 64.0)) + temp_h = int(math.floor(math.ceil(fr_1.size(2) / 64.0) * 64.0)) + + temp_fr_1 = torch.nn.functional.interpolate( + input=fr_1, size=(temp_h, temp_w), mode='nearest') + temp_fr_2 = torch.nn.functional.interpolate( + input=fr_2, size=(temp_h, temp_w), mode='nearest') + + flo = 20.0 * torch.nn.functional.interpolate(input=self.pwc(temp_fr_1, temp_fr_2), size=( + fr_1.size(2), fr_1.size(3)), mode='bilinear', align_corners=False) + return self.warpLayer(fr_2, flo, scale), flo + + def forward(self, fr1, fr2, f3, fs2, fs1, scale): + w1, flo1 = self.warpFrame(fs2, fr1, scale=scale) + w2, flo2 = self.warpFrame(fs1, fr2, scale=scale) + + I_int = self.UNet2(w1, w2, flo1, flo2, fr1, fr2) + f_int, flo_int = self.warpFrame(I_int, f3) + + fhat = self.ResNet2(I_int, f_int, flo_int, f3) + return fhat, I_int + + +class ResNet2(nn.Module): + def __init__(self): + super(ResNet2, self).__init__() + + class ConvBlock(nn.Module): + def __init__(self, in_ch, out_ch): + super(ConvBlock, self).__init__() + + self.seq = nn.Sequential( + nn.Conv2d(in_ch, out_ch, kernel_size=1, + stride=1, padding=0), + nn.ReLU() + ) + + self.GateConv = nn.Sequential( + nn.ReflectionPad2d(1), + nn.Conv2d(in_ch, in_ch, kernel_size=3, + stride=1, padding=0), + nn.ReflectionPad2d(1), + nn.Conv2d(in_ch, out_ch, kernel_size=3, + stride=1, padding=0), + nn.Sigmoid() + ) + + def forward(self, x): + return self.seq(x) * self.GateConv(x) + + class ResBlock(nn.Module): + def __init__(self, num_ch): + super(ResBlock, self).__init__() + + self.seq = nn.Sequential( + nn.Conv2d(num_ch, num_ch, kernel_size=1, + stride=1, padding=0), + nn.ReLU() + ) + + self.GateConv = nn.Sequential( + nn.ReflectionPad2d(1), + nn.Conv2d(num_ch, num_ch, kernel_size=3, + stride=1, padding=0), + nn.ReflectionPad2d(1), + nn.Conv2d(num_ch, num_ch, kernel_size=3, + stride=1, padding=0), + nn.Sigmoid() + ) + + def forward(self, x): + return self.seq(x) * self.GateConv(x) + x + + self.seq = nn.Sequential( + ConvBlock(11, 32), + ResBlock(32), + ResBlock(32), + ResBlock(32), + ResBlock(32), + ResBlock(32), + ConvBlock(32, 3), + nn.Tanh() + ) + + def forward(self, I_int, f_int, flo_int, f3): + return self.seq(torch.cat([I_int, f_int, flo_int, f3], 1).cuda()) + + +############################################################################################################# +class UNetFlow(nn.Module): + def __init__(self): + super(UNetFlow, self).__init__() + + class Encoder(nn.Module): + def __init__(self, in_nc, out_nc, stride, k_size=3, pad=1): + super(Encoder, self).__init__() + + self.seq = nn.Sequential( + nn.ReflectionPad2d(pad), + nn.Conv2d(in_nc, out_nc, kernel_size=k_size, + stride=stride, padding=0), + nn.LeakyReLU(0.2) + ) + self.GateConv = nn.Sequential( + nn.ReflectionPad2d(pad), + nn.Conv2d(in_nc, out_nc, kernel_size=k_size, + stride=stride, padding=0), + nn.Sigmoid() + ) + + def forward(self, x): + return self.seq(x) * self.GateConv(x) + + class Decoder(nn.Module): + def __init__(self, in_nc, out_nc, stride, k_size=3, pad=1, tanh=False): + super(Decoder, self).__init__() + + self.seq = nn.Sequential( + nn.ReflectionPad2d(pad), + nn.Conv2d(in_nc, out_nc, kernel_size=k_size, + stride=stride, padding=0) + ) + + if tanh: + self.activ = nn.Tanh() + else: + self.activ = nn.LeakyReLU(0.2) + + self.GateConv = nn.Sequential( + nn.ReflectionPad2d(pad), + nn.Conv2d(in_nc, out_nc, kernel_size=k_size, + stride=stride, padding=0), + nn.Sigmoid() + ) + + def forward(self, x): + s = self.seq(x) + s = self.activ(s) + return s * self.GateConv(x) + + self.enc0 = Encoder(4, 32, stride=1) + self.enc1 = Encoder(32, 32, stride=2) + self.enc2 = Encoder(32, 32, stride=2) + self.enc3 = Encoder(32, 32, stride=2) + + self.dec0 = Decoder(32, 32, stride=1) + # up-scaling + concat + self.dec1 = Decoder(32+32, 32, stride=1) + self.dec2 = Decoder(32+32, 32, stride=1) + self.dec3 = Decoder(32+32, 32, stride=1) + + self.dec4 = Decoder(32, 2, stride=1) + + def forward(self, x1, x2): + s0 = self.enc0(torch.cat([x1, x2], 1).cuda()) + s1 = self.enc1(s0) + s2 = self.enc2(s1) + s3 = self.enc3(s2) + + s4 = self.dec0(s3) + # up-scaling + concat + s4 = F.interpolate(s4, scale_factor=2, mode='nearest') + s5 = self.dec1(torch.cat([s4, s2], 1).cuda()) + s5 = F.interpolate(s5, scale_factor=2, mode='nearest') + s6 = self.dec2(torch.cat([s5, s1], 1).cuda()) + s6 = F.interpolate(s6, scale_factor=2, mode='nearest') + s7 = self.dec3(torch.cat([s6, s0], 1).cuda()) + + out = self.dec4(s7) + return out + + +class UNet3(nn.Module): + def __init__(self): + super(UNet3, self).__init__() + + class Encoder(nn.Module): + def __init__(self, in_nc, out_nc, stride, k_size=3, pad=1): + super(Encoder, self).__init__() + + self.seq = nn.Sequential( + nn.ReflectionPad2d(pad), + nn.Conv2d(in_nc, out_nc, kernel_size=k_size, + stride=stride, padding=0), + nn.ReLU() + ) + self.GateConv = nn.Sequential( + nn.ReflectionPad2d(pad), + nn.Conv2d(in_nc, out_nc, kernel_size=k_size, + stride=stride, padding=0), + nn.Sigmoid() + ) + + def forward(self, x): + return self.seq(x) * self.GateConv(x) + + class Decoder(nn.Module): + def __init__(self, in_nc, out_nc, stride, k_size=3, pad=1, tanh=False): + super(Decoder, self).__init__() + + self.seq = nn.Sequential( + nn.ReflectionPad2d(pad), + nn.Conv2d(in_nc, out_nc, kernel_size=k_size, + stride=stride, padding=0) + ) + + if tanh: + self.activ = nn.Tanh() + else: + self.activ = nn.ReLU() + + self.GateConv = nn.Sequential( + nn.ReflectionPad2d(pad), + nn.Conv2d(in_nc, out_nc, kernel_size=k_size, + stride=stride, padding=0), + nn.Sigmoid() + ) + + def forward(self, x): + s = self.seq(x) + s = self.activ(s) + return s * self.GateConv(x) + + self.enc0 = Encoder(6, 32, stride=1) + self.enc1 = Encoder(32, 32, stride=2) + self.enc2 = Encoder(32, 32, stride=2) + self.enc3 = Encoder(32, 32, stride=2) + + self.dec0 = Decoder(32, 32, stride=1) + # up-scaling + concat + self.dec1 = Decoder(32+32, 32, stride=1) + self.dec2 = Decoder(32+32, 32, stride=1) + self.dec3 = Decoder(32+32, 32, stride=1) + + self.dec4 = Decoder(32, 3, stride=1, tanh=True) + + def forward(self, w1, w2): + s0 = self.enc0(torch.cat([w1, w2], 1).cuda()) + s1 = self.enc1(s0) + s2 = self.enc2(s1) + s3 = self.enc3(s2) + + s4 = self.dec0(s3) + # up-scaling + concat + s4 = F.interpolate(s4, scale_factor=2, mode='nearest') + s5 = self.dec1(torch.cat([s4, s2], 1).cuda()) + s5 = F.interpolate(s5, scale_factor=2, mode='nearest') + s6 = self.dec2(torch.cat([s5, s1], 1).cuda()) + s6 = F.interpolate(s6, scale_factor=2, mode='nearest') + s7 = self.dec3(torch.cat([s6, s0], 1).cuda()) + + out = self.dec4(s7) + return out + + +class DIFNet3(nn.Module): + def __init__(self): + super(DIFNet3, self).__init__() + + class Backward(torch.nn.Module): + def __init__(self): + super(Backward, self).__init__() + # end + + def forward(self, tensorInput, tensorFlow, scale=1.0): + if hasattr(self, 'tensorPartial') == False or self.tensorPartial.size(0) != tensorFlow.size(0) or self.tensorPartial.size(2) != tensorFlow.size(2) or self.tensorPartial.size(3) != tensorFlow.size(3): + self.tensorPartial = torch.FloatTensor().resize_(tensorFlow.size( + 0), 1, tensorFlow.size(2), tensorFlow.size(3)).fill_(1.0).cuda() + # end + + if hasattr(self, 'tensorGrid') == False or self.tensorGrid.size(0) != tensorFlow.size(0) or self.tensorGrid.size(2) != tensorFlow.size(2) or self.tensorGrid.size(3) != tensorFlow.size(3): + tensorHorizontal = torch.linspace(-1.0, 1.0, tensorFlow.size(3)).view( + 1, 1, 1, tensorFlow.size(3)).expand(tensorFlow.size(0), -1, tensorFlow.size(2), -1) + tensorVertical = torch.linspace(-1.0, 1.0, tensorFlow.size(2)).view( + 1, 1, tensorFlow.size(2), 1).expand(tensorFlow.size(0), -1, -1, tensorFlow.size(3)) + + self.tensorGrid = torch.cat( + [tensorHorizontal, tensorVertical], 1).cuda() + # end + # pdb.set_trace() + tensorInput = torch.cat([tensorInput, self.tensorPartial], 1) + tensorFlow = torch.cat([tensorFlow[:, 0:1, :, :] / ((tensorInput.size( + 3) - 1.0) / 2.0), tensorFlow[:, 1:2, :, :] / ((tensorInput.size(2) - 1.0) / 2.0)], 1) + + tensorOutput = torch.nn.functional.grid_sample(input=tensorInput, grid=( + self.tensorGrid + tensorFlow*scale).permute(0, 2, 3, 1), mode='bilinear', padding_mode='zeros') + + tensorMask = tensorOutput[:, -1:, :, :] + tensorMask[tensorMask > 0.999] = 1.0 + tensorMask[tensorMask < 1.0] = 0.0 + + return tensorOutput[:, :-1, :, :] * tensorMask + + # PWC + self.pwc = PwcNet() + self.pwc.load_state_dict(torch.load('./trained_models/sintel.pytorch')) + self.pwc.eval() + + # Warping layer + self.warpLayer = Backward() + self.warpLayer.eval() + + # UNets + self.UNetFlow = UNetFlow() + self.UNet = UNet3() + self.ResNet = ResNet3() + + def warpFrame(self, fr_1, fr_2, scale=1.0): + with torch.no_grad(): + # Due to Pyramid method? + temp_w = int(math.floor(math.ceil(fr_1.size(3) / 64.0) * 64.0)) + temp_h = int(math.floor(math.ceil(fr_1.size(2) / 64.0) * 64.0)) + + temp_fr_1 = torch.nn.functional.interpolate( + input=fr_1, size=(temp_h, temp_w), mode='nearest') + temp_fr_2 = torch.nn.functional.interpolate( + input=fr_2, size=(temp_h, temp_w), mode='nearest') + + flo = 20.0 * torch.nn.functional.interpolate(input=self.pwc(temp_fr_1, temp_fr_2), size=( + fr_1.size(2), fr_1.size(3)), mode='bilinear', align_corners=False) + return self.warpLayer(fr_2, flo, scale), flo + + def forward(self, fr1, fr2, f3, fs2, fs1, scale): + _, flo1 = self.warpFrame(fs2, fr1, scale=scale) + _, flo2 = self.warpFrame(fs1, fr2, scale=scale) + + # refine flow + flo1_ = self.UNetFlow(flo1, flo2) + flo2_ = self.UNetFlow(flo2, flo1) + + w1 = self.warpLayer(fr1, flo1_, scale) + w2 = self.warpLayer(fr2, flo2_, scale) + + I_int = self.UNet(w1, w2) + f_int, _ = self.warpFrame(I_int, f3) + + fhat = self.ResNet(I_int, f_int) + return fhat, I_int + + +class ResNet3(nn.Module): + def __init__(self): + super(ResNet3, self).__init__() + + class ConvBlock(nn.Module): + def __init__(self, in_ch, out_ch): + super(ConvBlock, self).__init__() + + self.seq = nn.Sequential( + nn.Conv2d(in_ch, out_ch, kernel_size=1, + stride=1, padding=0), + nn.ReLU() + ) + + self.GateConv = nn.Sequential( + nn.ReflectionPad2d(1), + nn.Conv2d(in_ch, in_ch, kernel_size=3, + stride=1, padding=0), + nn.ReflectionPad2d(1), + nn.Conv2d(in_ch, out_ch, kernel_size=3, + stride=1, padding=0), + nn.Sigmoid() + ) + + def forward(self, x): + return self.seq(x) * self.GateConv(x) + + class ResBlock(nn.Module): + def __init__(self, num_ch): + super(ResBlock, self).__init__() + + self.seq = nn.Sequential( + nn.Conv2d(num_ch, num_ch, kernel_size=1, + stride=1, padding=0), + nn.ReLU() + ) + + self.GateConv = nn.Sequential( + nn.ReflectionPad2d(1), + nn.Conv2d(num_ch, num_ch, kernel_size=3, + stride=1, padding=0), + nn.ReflectionPad2d(1), + nn.Conv2d(num_ch, num_ch, kernel_size=3, + stride=1, padding=0), + nn.Sigmoid() + ) + + def forward(self, x): + return self.seq(x) * self.GateConv(x) + x + + self.seq = nn.Sequential( + ConvBlock(6, 32), + ResBlock(32), + ResBlock(32), + ResBlock(32), + ResBlock(32), + ResBlock(32), + ConvBlock(32, 3), + nn.Tanh() + ) + + def forward(self, I_int, f_int): + return self.seq(torch.cat([I_int, f_int], 1).cuda()) + + +############################ GAN Discriminator############################### + +class Discriminator(nn.Module): + def __init__(self, in_channels): + super(Discriminator, self).__init__() + self.seq = nn.Sequential( + nn.Conv2d(in_channels, 64, kernel_size=4, stride=2, padding=1), + nn.LeakyReLU(0.2), + + nn.Conv2d(64, 128, kernel_size=4, stride=2, padding=1), + nn.InstanceNorm2d(128), + nn.LeakyReLU(0.2), + + nn.Conv2d(128, 256, kernel_size=4, stride=2, padding=1), + nn.InstanceNorm2d(256), + nn.LeakyReLU(0.2), + + nn.Conv2d(256, 512, kernel_size=4, padding=1), + nn.InstanceNorm2d(512), + nn.LeakyReLU(0.2), + + nn.Conv2d(512, 1, kernel_size=4, padding=1) + ) + + def forward(self, fr_2): + #x = torch.cat((fr_1, fr_2), 1) + x = self.seq(fr_2) + x = F.avg_pool2d(x, x.size()[2:]).view(-1, x.size()[0]).squeeze() + out = torch.sigmoid(x) + return out diff --git a/models/DIFRINT/pwcNet.py b/models/DIFRINT/pwcNet.py new file mode 100644 index 0000000..952e2c7 --- /dev/null +++ b/models/DIFRINT/pwcNet.py @@ -0,0 +1,366 @@ +import getopt +import math +import numpy +import os +import PIL +import PIL.Image +import sys +import torch +import torch.nn.functional as F +#import torch.utils.serialization +import torchvision.transforms as transforms +import matplotlib.pyplot as plt +from .flowlib import * +import pdb +# from correlation import correlation +try: + from models.correlation import correlation # the custom cost volume layer +except: + parentddir = os.path.abspath(os.path.join( + os.path.dirname(__file__), os.path.pardir, 'correlation')) + sys.path.append(parentddir) + import correlation # you should consider upgrading python +# end + +########################################################## + +# assert(int(torch.__version__.replace('.', '')) >= 40) # requires at least pytorch version 0.4.0 + +#torch.set_grad_enabled(True) # make sure to not compute gradients for computational performance + +#torch.cuda.device(0) # change this if you have a multiple graphics cards and you want to utilize them + +#torch.backends.cudnn.enabled = True # make sure to use cudnn for computational performance + +########################################################## + +########################################################## + + +class PwcNet(torch.nn.Module): + def __init__(self, strModel='sintel'): + super(PwcNet, self).__init__() + + class Extractor(torch.nn.Module): + def __init__(self): + super(Extractor, self).__init__() + + self.moduleOne = torch.nn.Sequential( + torch.nn.Conv2d(in_channels=3, out_channels=16, kernel_size=3, stride=2, padding=1), + torch.nn.LeakyReLU(inplace=False, negative_slope=0.1), + torch.nn.Conv2d(in_channels=16, out_channels=16, kernel_size=3, stride=1, padding=1), + torch.nn.LeakyReLU(inplace=False, negative_slope=0.1), + torch.nn.Conv2d(in_channels=16, out_channels=16, kernel_size=3, stride=1, padding=1), + torch.nn.LeakyReLU(inplace=False, negative_slope=0.1) + ) + + self.moduleTwo = torch.nn.Sequential( + torch.nn.Conv2d(in_channels=16, out_channels=32, kernel_size=3, stride=2, padding=1), + torch.nn.LeakyReLU(inplace=False, negative_slope=0.1), + torch.nn.Conv2d(in_channels=32, out_channels=32, kernel_size=3, stride=1, padding=1), + torch.nn.LeakyReLU(inplace=False, negative_slope=0.1), + torch.nn.Conv2d(in_channels=32, out_channels=32, kernel_size=3, stride=1, padding=1), + torch.nn.LeakyReLU(inplace=False, negative_slope=0.1) + ) + + self.moduleThr = torch.nn.Sequential( + torch.nn.Conv2d(in_channels=32, out_channels=64, kernel_size=3, stride=2, padding=1), + torch.nn.LeakyReLU(inplace=False, negative_slope=0.1), + torch.nn.Conv2d(in_channels=64, out_channels=64, kernel_size=3, stride=1, padding=1), + torch.nn.LeakyReLU(inplace=False, negative_slope=0.1), + torch.nn.Conv2d(in_channels=64, out_channels=64, kernel_size=3, stride=1, padding=1), + torch.nn.LeakyReLU(inplace=False, negative_slope=0.1) + ) + + self.moduleFou = torch.nn.Sequential( + torch.nn.Conv2d(in_channels=64, out_channels=96, kernel_size=3, stride=2, padding=1), + torch.nn.LeakyReLU(inplace=False, negative_slope=0.1), + torch.nn.Conv2d(in_channels=96, out_channels=96, kernel_size=3, stride=1, padding=1), + torch.nn.LeakyReLU(inplace=False, negative_slope=0.1), + torch.nn.Conv2d(in_channels=96, out_channels=96, kernel_size=3, stride=1, padding=1), + torch.nn.LeakyReLU(inplace=False, negative_slope=0.1) + ) + + self.moduleFiv = torch.nn.Sequential( + torch.nn.Conv2d(in_channels=96, out_channels=128, kernel_size=3, stride=2, padding=1), + torch.nn.LeakyReLU(inplace=False, negative_slope=0.1), + torch.nn.Conv2d(in_channels=128, out_channels=128, kernel_size=3, stride=1, padding=1), + torch.nn.LeakyReLU(inplace=False, negative_slope=0.1), + torch.nn.Conv2d(in_channels=128, out_channels=128, kernel_size=3, stride=1, padding=1), + torch.nn.LeakyReLU(inplace=False, negative_slope=0.1) + ) + + self.moduleSix = torch.nn.Sequential( + torch.nn.Conv2d(in_channels=128, out_channels=196, kernel_size=3, stride=2, padding=1), + torch.nn.LeakyReLU(inplace=False, negative_slope=0.1), + torch.nn.Conv2d(in_channels=196, out_channels=196, kernel_size=3, stride=1, padding=1), + torch.nn.LeakyReLU(inplace=False, negative_slope=0.1), + torch.nn.Conv2d(in_channels=196, out_channels=196, kernel_size=3, stride=1, padding=1), + torch.nn.LeakyReLU(inplace=False, negative_slope=0.1) + ) + # end + + def forward(self, tensorInput): + tensorOne = self.moduleOne(tensorInput) + tensorTwo = self.moduleTwo(tensorOne) + tensorThr = self.moduleThr(tensorTwo) + tensorFou = self.moduleFou(tensorThr) + tensorFiv = self.moduleFiv(tensorFou) + tensorSix = self.moduleSix(tensorFiv) + + return [ tensorOne, tensorTwo, tensorThr, tensorFou, tensorFiv, tensorSix ] + # end + # end + + class Backward(torch.nn.Module): + def __init__(self): + super(Backward, self).__init__() + # end + + def forward(self, tensorInput, tensorFlow): + if hasattr(self, 'tensorPartial') == False or self.tensorPartial.size(0) != tensorFlow.size(0) or self.tensorPartial.size(2) != tensorFlow.size(2) or self.tensorPartial.size(3) != tensorFlow.size(3): + self.tensorPartial = torch.FloatTensor().resize_(tensorFlow.size(0), 1, tensorFlow.size(2), tensorFlow.size(3)).fill_(1.0).cuda() + # end + + if hasattr(self, 'tensorGrid') == False or self.tensorGrid.size(0) != tensorFlow.size(0) or self.tensorGrid.size(2) != tensorFlow.size(2) or self.tensorGrid.size(3) != tensorFlow.size(3): + tensorHorizontal = torch.linspace(-1.0, 1.0, tensorFlow.size(3)).view(1, 1, 1, tensorFlow.size(3)).expand(tensorFlow.size(0), -1, tensorFlow.size(2), -1) + tensorVertical = torch.linspace(-1.0, 1.0, tensorFlow.size(2)).view(1, 1, tensorFlow.size(2), 1).expand(tensorFlow.size(0), -1, -1, tensorFlow.size(3)) + + self.tensorGrid = torch.cat([ tensorHorizontal, tensorVertical ], 1).cuda() + # end + + tensorInput = torch.cat([ tensorInput, self.tensorPartial ], 1) + tensorFlow = torch.cat([ tensorFlow[:, 0:1, :, :] / ((tensorInput.size(3) - 1.0) / 2.0), tensorFlow[:, 1:2, :, :] / ((tensorInput.size(2) - 1.0) / 2.0) ], 1) + + tensorOutput = torch.nn.functional.grid_sample(input=tensorInput, grid=(self.tensorGrid + tensorFlow).permute(0, 2, 3, 1), mode='bilinear', padding_mode='zeros') + + tensorMask = tensorOutput[:, -1:, :, :]; tensorMask[tensorMask > 0.999] = 1.0; tensorMask[tensorMask < 1.0] = 0.0 + + return tensorOutput[:, :-1, :, :] * tensorMask + # end + # end + + class Decoder(torch.nn.Module): + def __init__(self, intLevel): + super(Decoder, self).__init__() + + intPrevious = [ None, None, 81 + 32 + 2 + 2, 81 + 64 + 2 + 2, 81 + 96 + 2 + 2, 81 + 128 + 2 + 2, 81, None ][intLevel + 1] + intCurrent = [ None, None, 81 + 32 + 2 + 2, 81 + 64 + 2 + 2, 81 + 96 + 2 + 2, 81 + 128 + 2 + 2, 81, None ][intLevel + 0] + + if intLevel < 6: self.moduleUpflow = torch.nn.ConvTranspose2d(in_channels=2, out_channels=2, kernel_size=4, stride=2, padding=1) + if intLevel < 6: self.moduleUpfeat = torch.nn.ConvTranspose2d(in_channels=intPrevious + 128 + 128 + 96 + 64 + 32, out_channels=2, kernel_size=4, stride=2, padding=1) + + if intLevel < 6: self.dblBackward = [ None, None, None, 5.0, 2.5, 1.25, 0.625, None ][intLevel + 1] + if intLevel < 6: self.moduleBackward = Backward() + + self.moduleCorrelation = correlation.ModuleCorrelation() + self.moduleCorreleaky = torch.nn.LeakyReLU(inplace=False, negative_slope=0.1) + + self.moduleOne = torch.nn.Sequential( + torch.nn.Conv2d(in_channels=intCurrent, out_channels=128, kernel_size=3, stride=1, padding=1), + torch.nn.LeakyReLU(inplace=False, negative_slope=0.1) + ) + + self.moduleTwo = torch.nn.Sequential( + torch.nn.Conv2d(in_channels=intCurrent + 128, out_channels=128, kernel_size=3, stride=1, padding=1), + torch.nn.LeakyReLU(inplace=False, negative_slope=0.1) + ) + + self.moduleThr = torch.nn.Sequential( + torch.nn.Conv2d(in_channels=intCurrent + 128 + 128, out_channels=96, kernel_size=3, stride=1, padding=1), + torch.nn.LeakyReLU(inplace=False, negative_slope=0.1) + ) + + self.moduleFou = torch.nn.Sequential( + torch.nn.Conv2d(in_channels=intCurrent + 128 + 128 + 96, out_channels=64, kernel_size=3, stride=1, padding=1), + torch.nn.LeakyReLU(inplace=False, negative_slope=0.1) + ) + + self.moduleFiv = torch.nn.Sequential( + torch.nn.Conv2d(in_channels=intCurrent + 128 + 128 + 96 + 64, out_channels=32, kernel_size=3, stride=1, padding=1), + torch.nn.LeakyReLU(inplace=False, negative_slope=0.1) + ) + + self.moduleSix = torch.nn.Sequential( + torch.nn.Conv2d(in_channels=intCurrent + 128 + 128 + 96 + 64 + 32, out_channels=2, kernel_size=3, stride=1, padding=1) + ) + # end + + def forward(self, tensorFirst, tensorSecond, objectPrevious): + tensorFlow = None + tensorFeat = None + + if objectPrevious is None: + tensorFlow = None + tensorFeat = None + + tensorVolume = self.moduleCorreleaky(self.moduleCorrelation(tensorFirst, tensorSecond)) + + tensorFeat = torch.cat([ tensorVolume ], 1) + + elif objectPrevious is not None: + tensorFlow = self.moduleUpflow(objectPrevious['tensorFlow']) + tensorFeat = self.moduleUpfeat(objectPrevious['tensorFeat']) + + tensorVolume = self.moduleCorreleaky(self.moduleCorrelation(tensorFirst, self.moduleBackward(tensorSecond, tensorFlow * self.dblBackward))) + + tensorFeat = torch.cat([ tensorVolume, tensorFirst, tensorFlow, tensorFeat ], 1) + + # end + + tensorFeat = torch.cat([ self.moduleOne(tensorFeat), tensorFeat ], 1) + tensorFeat = torch.cat([ self.moduleTwo(tensorFeat), tensorFeat ], 1) + tensorFeat = torch.cat([ self.moduleThr(tensorFeat), tensorFeat ], 1) + tensorFeat = torch.cat([ self.moduleFou(tensorFeat), tensorFeat ], 1) + tensorFeat = torch.cat([ self.moduleFiv(tensorFeat), tensorFeat ], 1) + + tensorFlow = self.moduleSix(tensorFeat) + + return { + 'tensorFlow': tensorFlow, + 'tensorFeat': tensorFeat + } + # end + # end + + class Refiner(torch.nn.Module): + def __init__(self): + super(Refiner, self).__init__() + + self.moduleMain = torch.nn.Sequential( + torch.nn.Conv2d(in_channels=81 + 32 + 2 + 2 + 128 + 128 + 96 + 64 + 32, out_channels=128, kernel_size=3, stride=1, padding=1, dilation=1), + torch.nn.LeakyReLU(inplace=False, negative_slope=0.1), + torch.nn.Conv2d(in_channels=128, out_channels=128, kernel_size=3, stride=1, padding=2, dilation=2), + torch.nn.LeakyReLU(inplace=False, negative_slope=0.1), + torch.nn.Conv2d(in_channels=128, out_channels=128, kernel_size=3, stride=1, padding=4, dilation=4), + torch.nn.LeakyReLU(inplace=False, negative_slope=0.1), + torch.nn.Conv2d(in_channels=128, out_channels=96, kernel_size=3, stride=1, padding=8, dilation=8), + torch.nn.LeakyReLU(inplace=False, negative_slope=0.1), + torch.nn.Conv2d(in_channels=96, out_channels=64, kernel_size=3, stride=1, padding=16, dilation=16), + torch.nn.LeakyReLU(inplace=False, negative_slope=0.1), + torch.nn.Conv2d(in_channels=64, out_channels=32, kernel_size=3, stride=1, padding=1, dilation=1), + torch.nn.LeakyReLU(inplace=False, negative_slope=0.1), + torch.nn.Conv2d(in_channels=32, out_channels=2, kernel_size=3, stride=1, padding=1, dilation=1) + ) + # end + + def forward(self, tensorInput): + return self.moduleMain(tensorInput) + # end + # end + + self.moduleExtractor = Extractor() + + self.moduleTwo = Decoder(2) + self.moduleThr = Decoder(3) + self.moduleFou = Decoder(4) + self.moduleFiv = Decoder(5) + self.moduleSix = Decoder(6) + + self.moduleRefiner = Refiner() + + #self.load_state_dict(torch.load('./trained_models/' + strModel + '.pytorch')) + # end + + def forward(self, tensorFirst, tensorSecond): + tensorFirst = self.moduleExtractor(tensorFirst) + tensorSecond = self.moduleExtractor(tensorSecond) + + objectEstimate = self.moduleSix(tensorFirst[-1], tensorSecond[-1], None) + objectEstimate = self.moduleFiv(tensorFirst[-2], tensorSecond[-2], objectEstimate) + objectEstimate = self.moduleFou(tensorFirst[-3], tensorSecond[-3], objectEstimate) + objectEstimate = self.moduleThr(tensorFirst[-4], tensorSecond[-4], objectEstimate) + objectEstimate = self.moduleTwo(tensorFirst[-5], tensorSecond[-5], objectEstimate) + + return objectEstimate['tensorFlow'] + self.moduleRefiner(objectEstimate['tensorFeat']) + # end +# end + +moduleNetwork = PwcNet().cuda() + +########################################################## + +def estimate(tensorInputFirst, tensorInputSecond): + tensorOutput = torch.FloatTensor() + + assert(tensorInputFirst.size(1) == tensorInputSecond.size(1)) + assert(tensorInputFirst.size(2) == tensorInputSecond.size(2)) + + intWidth = tensorInputFirst.size(2) + intHeight = tensorInputFirst.size(1) + + assert(intWidth == 1024) # remember that there is no guarantee for correctness, comment this line out if you acknowledge this and want to continue + assert(intHeight == 436) # remember that there is no guarantee for correctness, comment this line out if you acknowledge this and want to continue + + if True: + tensorInputFirst = tensorInputFirst.cuda() + tensorInputSecond = tensorInputSecond.cuda() + tensorOutput = tensorOutput.cuda() + # end + + if True: + tensorPreprocessedFirst = tensorInputFirst.view(1, 3, intHeight, intWidth) + tensorPreprocessedSecond = tensorInputSecond.view(1, 3, intHeight, intWidth) + + intPreprocessedWidth = int(math.floor(math.ceil(intWidth / 64.0) * 64.0)) # Due to Pyramid method? + intPreprocessedHeight = int(math.floor(math.ceil(intHeight / 64.0) * 64.0)) + + tensorPreprocessedFirst = torch.nn.functional.interpolate(input=tensorPreprocessedFirst, size=(intPreprocessedHeight, intPreprocessedWidth), mode='bilinear', align_corners=False) + tensorPreprocessedSecond = torch.nn.functional.interpolate(input=tensorPreprocessedSecond, size=(intPreprocessedHeight, intPreprocessedWidth), mode='bilinear', align_corners=False) + #pdb.set_trace() + tensorFlow = 20.0 * torch.nn.functional.interpolate(input=moduleNetwork(tensorPreprocessedFirst, tensorPreprocessedSecond), size=(intHeight, intWidth), mode='bilinear', align_corners=False) + + tensorFlow[:, 0, :, :] *= float(intWidth) / float(intPreprocessedWidth) + tensorFlow[:, 1, :, :] *= float(intHeight) / float(intPreprocessedHeight) + + tensorOutput.resize_(2, intHeight, intWidth).copy_(tensorFlow[0, :, :, :]) + # end + + if True: + tensorInputFirst = tensorInputFirst.cpu() + tensorInputSecond = tensorInputSecond.cpu() + tensorOutput = tensorOutput.cpu() + # end + + return tensorOutput +# end + +########################################################## + +if __name__ == '__main__': + tensorInputFirst = torch.FloatTensor(numpy.array(PIL.Image.open(arguments_strFirst))[:, :, ::-1].transpose(2, 0, 1).astype(numpy.float32) / 255.0) + tensorInputSecond = torch.FloatTensor(numpy.array(PIL.Image.open(arguments_strSecond))[:, :, ::-1].transpose(2, 0, 1).astype(numpy.float32) / 255.0) + + tensorOutput = estimate(tensorInputFirst, tensorInputSecond) + #pdb.set_trace() + # Output *.flo file + objectOutput = open(arguments_strOut, 'wb') + + numpy.array([ 80, 73, 69, 72 ], numpy.uint8).tofile(objectOutput) + numpy.array([ tensorOutput.size(2), tensorOutput.size(1) ], numpy.int32).tofile(objectOutput) + numpy.array(tensorOutput.permute(1, 2, 0), numpy.float32).tofile(objectOutput) + + objectOutput.close() + + # Visualize *.flo file + #flowlib.show_flow(arguments_strOut) + #pdb.set_trace() + + # Output warped *.png file + tensorInputFirst = torch.FloatTensor(numpy.array(PIL.Image.open(arguments_strFirst)).transpose(2, 0, 1).astype(numpy.float32) / 255.0) + tensorInputSecond = torch.FloatTensor(numpy.array(PIL.Image.open(arguments_strSecond)).transpose(2, 0, 1).astype(numpy.float32) / 255.0) + tensorOutput = tensorOutput.permute(1, 2, 0)[None,:,:,:] + #pdb.set_trace() + tensorOutput[:,:,:,0] = tensorOutput[:,:,:,0]/(1024/2) + tensorOutput[:,:,:,1] = tensorOutput[:,:,:,1]/(436/2) + + + scale = 0.5 + mesh = numpy.array(numpy.meshgrid(numpy.linspace(-1,1,1024), numpy.linspace(-1,1,436))) + mesh = torch.FloatTensor(mesh) + outwarp = F.grid_sample(tensorInputSecond[None,:,:,:], mesh.permute(1, 2, 0)[None,:,:,:] + tensorOutput*scale) + outwarp = outwarp.squeeze().permute(1, 2, 0) + + plt.imshow(outwarp.numpy()) + plt.show() +# end diff --git a/models/DUT/DUT.py b/models/DUT/DUT.py new file mode 100644 index 0000000..c8fb5cf --- /dev/null +++ b/models/DUT/DUT.py @@ -0,0 +1,346 @@ +import torch +import torch.nn as nn +import numpy as np +from .Smoother import Smoother +from .rf_det_so import RFDetSO +from configs.config import cfg +from utils.MedianFilter import SingleMotionPropagate, MultiMotionPropagate +from utils.image_utils import topk_map +from utils.IterativeSmooth import generateSmooth +from .MotionPro import MotionPro +import cv2 +from .PWCNet import Network as PWCNet +from .PWCNet import estimate as opticalFlowEstimate + +class KeypointDetction(nn.Module): + def __init__(self, RFDetPath='', topK=cfg.TRAIN.TOPK, detectorType=0): + super(KeypointDetction, self).__init__() + self.feature_params = dict(maxCorners = topK, + qualityLevel = 0.3, + minDistance = 7, + blockSize = 7) + + self.TOPK = topK + self.type = detectorType + + def forward(self, im_data): + ''' + @param im_data [B, 1, H, W] gray images + @return im_topk [B, 1, H, W] + @return kpts [[N, 4] for B] (B, 0, H, W) + ''' + + device = im_data.device + im1 = im_data + im1 = (im1.cpu().numpy() * 255).astype(np.uint8) + batch = im1.shape[0] + assert im1.shape[1] == 1 + im_topK = torch.zeros((batch, 1, im1.shape[2], im1.shape[3]), device=device) + for idx in range(batch): + im = im1[idx, 0] + + if self.type == 0: + p = cv2.goodFeaturesToTrack(im, mask=None, **self.feature_params) + p = p[:, 0, :] # N, 2 + im_topK[idx, 0, p[:, 1], p[:, 0]] = 1. + kpts = im_topK.nonzero() + kpts = [kpts[kpts[:, 0] == idx, :] for idx in range(batch)] # B, N, 4 + return im_topK, kpts + +class RFDetection(nn.Module): + def __init__(self, RFDetPath, topK=cfg.TRAIN.TOPK): + super(RFDetection, self).__init__() + + self.det = RFDetSO( + cfg.TRAIN.score_com_strength, + cfg.TRAIN.scale_com_strength, + cfg.TRAIN.NMS_THRESH, + cfg.TRAIN.NMS_KSIZE, + cfg.TRAIN.TOPK, + cfg.MODEL.GAUSSIAN_KSIZE, + cfg.MODEL.GAUSSIAN_SIGMA, + cfg.MODEL.KSIZE, + cfg.MODEL.padding, + cfg.MODEL.dilation, + cfg.MODEL.scale_list, + ) + + self.TOPK = topK + + def forward(self, im_data, batch=2, allInfer=False): + ''' + @param im_data [B, 1, H, W] + @return im_topk [B, 1, H, W] + @return kpts [[N, 4] for B] (B, 0, H, W) + ''' + if allInfer: + im_data = im_data + im_rawsc, _, _ = self.det(im_data) + im_score = self.det.process(im_rawsc)[0] + im_topk = topk_map(im_score, self.TOPK).permute(0, 3, 1, 2) # B, 1, H, W + kpts = im_topk.nonzero() # (B*topk, 4) + kpts = [kpts[kpts[:, 0] == idx, :] for idx in range(im_data.shape[0])] # [[N, 4] for B] + im_topk = im_topk.float() + else: + im_topK_ = [] + kpts_ = [] + for j in range(0, im_data.shape[0], batch): + im_data_clip = im_data[j:j+batch] + im_rawsc, _, _ = self.det(im_data_clip) + im_score = self.det.process(im_rawsc)[0] + im_topk = topk_map(im_score, self.TOPK).permute(0, 3, 1, 2) # B, 1, H, W + kpts = im_topk.nonzero() # (B*topk, 4) + kpts = [kpts[kpts[:, 0] == idx, :] for idx in range(im_data_clip.shape[0])] # [[N, 4] for B] + im_topk = im_topk.float() + im_topK_.append(im_topk) + kpts_ = kpts_ + kpts + kpts = kpts_ + im_topk = torch.cat(im_topK_, 0) + + return im_topk, kpts # B, 1, H, W; N, 4; + + def reload(self, RFDetPath): + + # self.pwcNet.load_state_dict({strKey.replace('module', 'net'): tenWeight for strKey, tenWeight in torch.load(PWCNetPath).items()}) + print('reload RFDet Model') + pretrained_dict = torch.load(RFDetPath)['state_dict'] + model_dict = self.det.state_dict() + pretrained_dict = {k[4:]:v for k, v in pretrained_dict.items() if k[:3]=='det' and k[4:] in model_dict} + assert len(pretrained_dict.keys()) > 0 + model_dict.update(pretrained_dict) + assert len(model_dict.keys()) == len(pretrained_dict.keys()), 'mismatch for RFDet' + self.det.load_state_dict(model_dict) + print("successfully load {} params for RFDet".format(len(model_dict))) + +class MotionEstimation(nn.Module): + def __init__(self, PWCNetPath=''): + super(MotionEstimation, self).__init__() + if PWCNetPath == '': + self.PWCNet = None + else: + self.PWCNet = PWCNet() + # self.PWCNet.eval() + # self.PWCNet.cuda() + + def forward(self, x, x_RGB, im_topk, kpts): + ''' + @param im_data [B, 1, H, W] + @param im_topk [B, 1, H, W] + @param kpts [[N, 4] for B] (B, 0, H, W) + @param OpticalFlow [B, 2, H, W] precomputed optical flow; optional, default None + @param RGBImages [B, 3, H, W] RGB images for optical flow computation, optional, default None + ''' + if self.PWCNet is None: + raise NotImplementedError() + + x_RGB = x_RGB * 1. / 255. + + optical_flow = [opticalFlowEstimate(x_RGB[:, i, :, :, :].cuda(), x_RGB[:, (i+1), :, :, :].cuda(), self.PWCNet) + for i in range(0, x_RGB.shape[1] - 1)] + x_RGB = x_RGB.cpu() + torch.cuda.empty_cache() + optical_flow = torch.cat(optical_flow, 0) + + flow_masked = optical_flow * im_topk[:-1] # B - 1, 2, H, W + + return flow_masked + + def reload(self, PWCNetPath): + if PWCNetPath == '': + pass + else: + self.PWCNet.load_state_dict({strKey.replace('module', 'net'): tenWeight for strKey, tenWeight in torch.load(PWCNetPath).items()}) + +class KLT(nn.Module): + def __init__(self, PWCNetPath=''): + super(KLT, self).__init__() + self.lk_params = dict(winSize = (15, 15), + maxLevel = 2, + criteria = (cv2.TERM_CRITERIA_EPS | cv2.TERM_CRITERIA_COUNT, 20, 0.03)) + + def forward(self, x, x_RGB, im_topk, kpts): + ''' + @param im_data [B, 1, H, W] + @param im_topk [B, 1, H, W] + @param kpts [[N, 4] for B] (B, 0, H, W) + @param OpticalFlow [B, 2, H, W] precomputed optical flow; optional, default None + @param RGBImages [B, 3, H, W] RGB images for optical flow computation, optional, default None + ''' + batch, _, height, width = x.shape + im_cpu = (x.cpu().numpy() * 255.).astype(np.uint8)[:, 0, :, :] + OpticalFlow = np.zeros((batch - 1, 2, height, width)) + for j in range(batch - 1): + p0 = kpts[j].detach().cpu().numpy()[:, ::-1] + p0 = np.expand_dims(p0[:, :2], 1).astype(np.float32) + + p1, _, _ = cv2.calcOpticalFlowPyrLK(im_cpu[j], im_cpu[j + 1], p0, None, **self.lk_params) + op = p1 - p0 + p0 = p0.astype(np.uint8) + OpticalFlow[j, :, p0[:, 0, 1], p0[:, 0, 0]] = op[:, 0, :] + + return torch.from_numpy(OpticalFlow.astype(np.float32)).to(x.device) + +class motionPropagate(object): + def __init__(self, inferenceMethod): + self.inference = inferenceMethod + +class JacobiSolver(nn.Module): + def __init__(self): + super(JacobiSolver, self).__init__() + self.generateSmooth = generateSmooth + self.KernelSmooth = Smoother().KernelSmooth + + def forward(self, x): + return None + +class DUT(nn.Module): + def __init__(self, SmootherPath='', RFDetPath='', PWCNetPath='', MotionProPath='', homo=True): + super(DUT, self).__init__() + print("-------------model configuration------------------------") + if RFDetPath != '': + print('using RFNet ...') + self.keypointModule = RFDetection(RFDetPath) + else: + print('using corner keypoint detector...') + self.keypointModule = KeypointDetction() + + if PWCNetPath != '': + print('using PWCNet for motion estimation...') + self.motionEstimation = MotionEstimation(PWCNetPath) + else: + print('using KLT tracker for motion estimation...') + self.motionEstimation = KLT() + + if MotionProPath != '': + if homo: + print('using Motion Propagation model with multi homo...') + self.motionPro = MotionPro(globalchoice='multi') + else: + print('using Motion Propagation model with single homo...') + self.motionPro = MotionPro(globalchoice='single') + else: + if homo: + print('using median filter with multi homo...') + self.motionPro = motionPropagate(MultiMotionPropagate) + else: + print('using median filter with single homo...') + self.motionPro = motionPropagate(SingleMotionPropagate) + + if SmootherPath != '': + print('using Deep Smoother Model...') + self.smoother = Smoother() + else: + print('using Jacobi Solver ...') + self.smoother = JacobiSolver() + + self.reload(SmootherPath, RFDetPath, PWCNetPath, MotionProPath) + + def forward(self, x, x_RGB, repeat=50): + return self.inference(x, x_RGB, repeat) + + def inference(self, x, x_RGB, repeat=50): + """ + @param: x [B, C, T, H, W] Assume B is 1 here, a set of Gray images + @param: x_RGB [B, C, T, H, W] Assume B is 1 here, a set of RGB images + @param: repeat int repeat time for the smoother module + + @return: smoothPath + """ + + x = x.permute(0, 2, 1, 3, 4).squeeze(0) # T, C, H, W + + # keypoint extraction + print("detect keypoints ....") + im_topk, kpts = self.keypointModule.forward(x) # T, 1, H, W; list([N, 4]) + + # This will slow down the code but save GPU memory + x = x.cpu() + torch.cuda.empty_cache() + + print("estimate motion ....") + masked_flow = self.motionEstimation.forward(x, x_RGB, im_topk, kpts) + + x_RGB = x_RGB.cpu() + im_topk = im_topk.cpu() + torch.cuda.empty_cache() + + del x + del x_RGB + del im_topk + + print("motion propagation ....") + origin_motion = [self.motionPro.inference(masked_flow[i:i+1, 0:1, :, :].cuda(), masked_flow[i:i+1, 1:2, :, :].cuda(), kpts[i]).cpu() + for i in range(len(kpts) - 1)] + + origin_motion = torch.stack(origin_motion, 2).cuda() # B, 2, T, H, W + origin_motion = torch.cat([torch.zeros_like(origin_motion[:, :, 0:1, :, :]).to(origin_motion.device), origin_motion], 2) + + origin_motion = torch.cumsum(origin_motion, 2) + min_value = torch.min(origin_motion) + origin_motion = origin_motion - min_value + max_value = torch.max(origin_motion) + 1e-5 + origin_motion = origin_motion / max_value + + smoothKernel = self.smoother(origin_motion.cuda()) + + smoothPath = torch.cat(self.smoother.KernelSmooth(smoothKernel, origin_motion.cuda(), repeat), 1) # B, 2, T, H, W + smoothPath = smoothPath * max_value + min_value + origin_motion = origin_motion * max_value + min_value + + return origin_motion, smoothPath + + def reload(self, SmootherPath, RFDetPath, PWCNetPath, MotionProPath): + print('------------------reload parameters-------------------------') + + if SmootherPath == '': + print("No parameters for JacobiSolver") + else: + print("reload Smoother params") + pretrained_dict = torch.load(SmootherPath) + model_dict = self.smoother.state_dict() + pretrained_dict = {k:v for k, v in pretrained_dict.items() if k in model_dict} + assert len(pretrained_dict.keys()) > 0 + assert len(model_dict.keys()) == len(pretrained_dict.keys()) + model_dict.update(pretrained_dict) + assert len(model_dict.keys()) == len(pretrained_dict.keys()) + self.smoother.load_state_dict(model_dict) + print("successfully load {} params for smoother".format(len(model_dict))) + + if RFDetPath != '': + self.keypointModule.reload(RFDetPath) + else: + print("No parameters for Keypoint detector") + + if PWCNetPath == '': + print("No parameters for Optical flow") + else: + print('reload PWCNet Model') + self.motionEstimation.reload(PWCNetPath) + + if MotionProPath == '': + print("No parameters for motion propagation") + else: + print('reload MotionPropagation Model') + model_dict_motion = torch.load(MotionProPath) + model_dict = self.motionPro.state_dict() + model_dict_motion = {k:v for k, v in model_dict_motion.items() if k in model_dict} + assert len(model_dict_motion.keys()) > 0 + model_dict.update(model_dict_motion) + assert len(model_dict_motion.keys()) == len(model_dict.keys()) + self.motionPro.load_state_dict(model_dict) + print("successfully load {} params for MotionPropagation".format(len(model_dict))) + + +if __name__ == "__main__": + + im_raw = np.random.randn(1, 3, 20, 240, 320).astype(np.float32) # B, 3, T, H, W (RGB) + im_data = im_raw[:, 0:1, :, :, :] + im_raw = torch.from_numpy(im_raw).cuda() + im_data = torch.from_numpy(im_data).cuda() + + model = DUT('1', '2', '3', '4') + model.cuda() + model.eval() + # optimizer = torch.optim.Adam(model.parameters()) + # optimizer.zero_grad() + smoothPath = model.inference(im_data, im_raw) diff --git a/models/DUT/MotionPro.py b/models/DUT/MotionPro.py new file mode 100644 index 0000000..a9232c1 --- /dev/null +++ b/models/DUT/MotionPro.py @@ -0,0 +1,121 @@ +import torch +import torch.nn as nn +import numpy as np +import os +import math +from configs.config import cfg +from utils.MedianFilter import MedianPool2d +import cv2 +from utils.ProjectionUtils import multiHomoEstimate, singleHomoEstimate + + +class MotionPro(nn.Module): + + def __init__(self, + inplanes=2, + embeddingSize=64, + hiddenSize=128, + number_points=512, + kernel=5, + globalchoice='multi'): + + super(MotionPro, self).__init__() + self.embedding = nn.Sequential( + nn.Conv1d(inplanes, embeddingSize, 1), + nn.ReLU(), + ) + self.embedding_motion = nn.Sequential( + nn.Conv1d(inplanes, embeddingSize, 1), + nn.ReLU(), + ) + + self.pad = kernel // 2 + self.conv1 = nn.Conv1d(embeddingSize, embeddingSize, 1) + self.conv2 = nn.Conv1d(embeddingSize, embeddingSize // 2, 1) + self.conv3 = nn.Conv1d(embeddingSize // 2, 1, 1) + + self.weighted = nn.Softmax(dim=2) + + self.relu = nn.ReLU() + self.leakyRelu = nn.LeakyReLU(0.1) + + self.m_conv1 = nn.Conv1d(embeddingSize, 2 * embeddingSize, 1) + self.m_conv2 = nn.Conv1d(2 * embeddingSize, 2 * embeddingSize, 1) + self.m_conv3 = nn.Conv1d(2 * embeddingSize, embeddingSize, 1) + + self.fuse_conv1 = nn.Conv1d( + embeddingSize + embeddingSize // 2, embeddingSize, 1) + self.fuse_conv2 = nn.Conv1d(embeddingSize, embeddingSize, 1) + + self.decoder = nn.Linear(embeddingSize, 2, bias=False) + + if globalchoice == 'multi': + self.homoEstimate = multiHomoEstimate + elif globalchoice == 'single': + self.homoEstimate = singleHomoEstimate + + self.meidanPool = MedianPool2d(5, same=True) + + def forward(self, motion): + ''' + @param: motion contains distance info and motion info of keypoints + + @return: return predicted motion for each grid vertex + ''' + distance_info = motion[:, 0:2, :] + motion_info = motion[0:1, 2:4, :] + + embedding_distance = self.embedding(distance_info) + embedding_distance = self.leakyRelu(self.conv1(embedding_distance)) + embedding_distance = self.leakyRelu(self.conv2(embedding_distance)) + distance_weighted = self.weighted(self.conv3(embedding_distance)) + + embedding_motion = self.embedding_motion(motion_info) + embedding_motion = self.leakyRelu(self.m_conv1(embedding_motion)) + embedding_motion = self.leakyRelu(self.m_conv2(embedding_motion)) + embedding_motion = self.leakyRelu(self.m_conv3(embedding_motion)) + embedding_motion = embedding_motion.repeat( + distance_info.shape[0], 1, 1) + + embedding_motion = torch.cat([embedding_motion, embedding_distance], 1) + embedding_motion = self.leakyRelu(self.fuse_conv1(embedding_motion)) + embedding_motion = self.leakyRelu(self.fuse_conv2(embedding_motion)) + + embedding_motion = torch.sum(embedding_motion * distance_weighted, 2) + + out_motion = self.decoder(embedding_motion) + + return out_motion + + def inference(self, x_flow, y_flow, kp): + """ + @param x_flow [B, 1, H, W] + @param y_flow [B, 1, H, W] + @param kp [B*topk, 4 / 2]->[N, 4/2] + """ + if kp.shape[1] == 4: + kp = kp[:, 2:] + index = kp.long() + origin_motion = torch.cat([x_flow, y_flow], 1) + extracted_motion = origin_motion[0, :, index[:, 0], index[:, 1]] + kp = kp.permute(1, 0).float() + concat_motion = torch.cat( + [kp[1:2, :], kp[0:1, :], extracted_motion], 0) + + motion, gridsMotion, _ = self.homoEstimate(concat_motion, kp) + GridMotion = (self.forward(motion) + + gridsMotion.squeeze(-1)) * cfg.MODEL.FLOWC + GridMotion = GridMotion.view( + cfg.MODEL.HEIGHT // cfg.MODEL.PIXELS, cfg.MODEL.WIDTH // cfg.MODEL.PIXELS, 2) + GridMotion = GridMotion.permute(2, 0, 1).unsqueeze(0) + GridMotion = self.meidanPool(GridMotion) + return GridMotion + + +if __name__ == "__main__": + model = MotionPro() + model.train() + model.cuda() + x = torch.from_numpy(np.random.randn(4, 512).astype(np.float32)).cuda() + kp = torch.from_numpy(np.random.randn(512, 2).astype(np.float32)).cuda() + model.train_step(x, kp) diff --git a/models/DUT/PWCNet.py b/models/DUT/PWCNet.py new file mode 100644 index 0000000..aa69dba --- /dev/null +++ b/models/DUT/PWCNet.py @@ -0,0 +1,392 @@ +#!/usr/bin/env python + +import torch + +import getopt +import math +import numpy +import os +import PIL +import PIL.Image +import sys + +try: + from .correlation import correlation # the custom cost volume layer +except: + parentddir = os.path.abspath(os.path.join( + os.path.dirname(__file__), os.path.pardir, 'correlation')) + sys.path.append(parentddir) + import correlation # you should consider upgrading python +# end + +########################################################## + +# requires at least pytorch version 1.3.0 +assert(int(str('').join(torch.__version__.split('.')[0:2])) >= 13) + +# torch.set_grad_enabled(False) # make sure to not compute gradients for computational performance + +# torch.backends.cudnn.enabled = True # make sure to use cudnn for computational performance + +########################################################## + +# arguments_strModel = 'default' +# arguments_strFirst = './images/first.png' +# arguments_strSecond = './images/second.png' +# arguments_strOut = './out.flo' + +# for strOption, strArgument in getopt.getopt(sys.argv[1:], '', [ strParameter[2:] + '=' for strParameter in sys.argv[1::2] ])[0]: +# if strOption == '--model' and strArgument != '': arguments_strModel = strArgument # which model to use +# if strOption == '--first' and strArgument != '': arguments_strFirst = strArgument # path to the first frame +# if strOption == '--second' and strArgument != '': arguments_strSecond = strArgument # path to the second frame +# if strOption == '--out' and strArgument != '': arguments_strOut = strArgument # path to where the output should be stored +# # end + +########################################################## + +backwarp_tenGrid = {} +backwarp_tenPartial = {} + + +def backwarp(tenInput, tenFlow): + if str(tenFlow.size()) not in backwarp_tenGrid: + tenHorizontal = torch.linspace(-1.0, 1.0, tenFlow.shape[3]).view( + 1, 1, 1, tenFlow.shape[3]).expand(tenFlow.shape[0], -1, tenFlow.shape[2], -1) + tenVertical = torch.linspace(-1.0, 1.0, tenFlow.shape[2]).view( + 1, 1, tenFlow.shape[2], 1).expand(tenFlow.shape[0], -1, -1, tenFlow.shape[3]) + + backwarp_tenGrid[str(tenFlow.size())] = torch.cat( + [tenHorizontal, tenVertical], 1).cuda() + # end + + if str(tenFlow.size()) not in backwarp_tenPartial: + backwarp_tenPartial[str(tenFlow.size())] = tenFlow.new_ones( + [tenFlow.shape[0], 1, tenFlow.shape[2], tenFlow.shape[3]]) + # end + + tenFlow = torch.cat([tenFlow[:, 0:1, :, :] / ((tenInput.shape[3] - 1.0) / 2.0), + tenFlow[:, 1:2, :, :] / ((tenInput.shape[2] - 1.0) / 2.0)], 1) + tenInput = torch.cat( + [tenInput, backwarp_tenPartial[str(tenFlow.size())]], 1) + + tenOutput = torch.nn.functional.grid_sample(input=tenInput, grid=(backwarp_tenGrid[str(tenFlow.size( + ))] + tenFlow).permute(0, 2, 3, 1), mode='bilinear', padding_mode='zeros', align_corners=True) + + tenMask = tenOutput[:, -1:, :, :] + tenMask[tenMask > 0.999] = 1.0 + tenMask[tenMask < 1.0] = 0.0 + + return tenOutput[:, :-1, :, :] * tenMask +# end + +########################################################## + + +class Network(torch.nn.Module): + def __init__(self): + super(Network, self).__init__() + + class Extractor(torch.nn.Module): + def __init__(self): + super(Extractor, self).__init__() + + self.netOne = torch.nn.Sequential( + torch.nn.Conv2d(in_channels=3, out_channels=16, + kernel_size=3, stride=2, padding=1), + torch.nn.LeakyReLU(inplace=False, negative_slope=0.1), + torch.nn.Conv2d(in_channels=16, out_channels=16, + kernel_size=3, stride=1, padding=1), + torch.nn.LeakyReLU(inplace=False, negative_slope=0.1), + torch.nn.Conv2d(in_channels=16, out_channels=16, + kernel_size=3, stride=1, padding=1), + torch.nn.LeakyReLU(inplace=False, negative_slope=0.1) + ) + + self.netTwo = torch.nn.Sequential( + torch.nn.Conv2d(in_channels=16, out_channels=32, + kernel_size=3, stride=2, padding=1), + torch.nn.LeakyReLU(inplace=False, negative_slope=0.1), + torch.nn.Conv2d(in_channels=32, out_channels=32, + kernel_size=3, stride=1, padding=1), + torch.nn.LeakyReLU(inplace=False, negative_slope=0.1), + torch.nn.Conv2d(in_channels=32, out_channels=32, + kernel_size=3, stride=1, padding=1), + torch.nn.LeakyReLU(inplace=False, negative_slope=0.1) + ) + + self.netThr = torch.nn.Sequential( + torch.nn.Conv2d(in_channels=32, out_channels=64, + kernel_size=3, stride=2, padding=1), + torch.nn.LeakyReLU(inplace=False, negative_slope=0.1), + torch.nn.Conv2d(in_channels=64, out_channels=64, + kernel_size=3, stride=1, padding=1), + torch.nn.LeakyReLU(inplace=False, negative_slope=0.1), + torch.nn.Conv2d(in_channels=64, out_channels=64, + kernel_size=3, stride=1, padding=1), + torch.nn.LeakyReLU(inplace=False, negative_slope=0.1) + ) + + self.netFou = torch.nn.Sequential( + torch.nn.Conv2d(in_channels=64, out_channels=96, + kernel_size=3, stride=2, padding=1), + torch.nn.LeakyReLU(inplace=False, negative_slope=0.1), + torch.nn.Conv2d(in_channels=96, out_channels=96, + kernel_size=3, stride=1, padding=1), + torch.nn.LeakyReLU(inplace=False, negative_slope=0.1), + torch.nn.Conv2d(in_channels=96, out_channels=96, + kernel_size=3, stride=1, padding=1), + torch.nn.LeakyReLU(inplace=False, negative_slope=0.1) + ) + + self.netFiv = torch.nn.Sequential( + torch.nn.Conv2d(in_channels=96, out_channels=128, + kernel_size=3, stride=2, padding=1), + torch.nn.LeakyReLU(inplace=False, negative_slope=0.1), + torch.nn.Conv2d(in_channels=128, out_channels=128, + kernel_size=3, stride=1, padding=1), + torch.nn.LeakyReLU(inplace=False, negative_slope=0.1), + torch.nn.Conv2d(in_channels=128, out_channels=128, + kernel_size=3, stride=1, padding=1), + torch.nn.LeakyReLU(inplace=False, negative_slope=0.1) + ) + + self.netSix = torch.nn.Sequential( + torch.nn.Conv2d(in_channels=128, out_channels=196, + kernel_size=3, stride=2, padding=1), + torch.nn.LeakyReLU(inplace=False, negative_slope=0.1), + torch.nn.Conv2d(in_channels=196, out_channels=196, + kernel_size=3, stride=1, padding=1), + torch.nn.LeakyReLU(inplace=False, negative_slope=0.1), + torch.nn.Conv2d(in_channels=196, out_channels=196, + kernel_size=3, stride=1, padding=1), + torch.nn.LeakyReLU(inplace=False, negative_slope=0.1) + ) + # end + + def forward(self, tenInput): + tenOne = self.netOne(tenInput) + tenTwo = self.netTwo(tenOne) + tenThr = self.netThr(tenTwo) + tenFou = self.netFou(tenThr) + tenFiv = self.netFiv(tenFou) + tenSix = self.netSix(tenFiv) + + return [tenOne, tenTwo, tenThr, tenFou, tenFiv, tenSix] + # end + # end + + class Decoder(torch.nn.Module): + def __init__(self, intLevel): + super(Decoder, self).__init__() + + intPrevious = [None, None, 81 + 32 + 2 + 2, 81 + 64 + 2 + 2, + 81 + 96 + 2 + 2, 81 + 128 + 2 + 2, 81, None][intLevel + 1] + intCurrent = [None, None, 81 + 32 + 2 + 2, 81 + 64 + 2 + 2, + 81 + 96 + 2 + 2, 81 + 128 + 2 + 2, 81, None][intLevel + 0] + + if intLevel < 6: + self.netUpflow = torch.nn.ConvTranspose2d( + in_channels=2, out_channels=2, kernel_size=4, stride=2, padding=1) + if intLevel < 6: + self.netUpfeat = torch.nn.ConvTranspose2d( + in_channels=intPrevious + 128 + 128 + 96 + 64 + 32, out_channels=2, kernel_size=4, stride=2, padding=1) + if intLevel < 6: + self.fltBackwarp = [None, None, None, 5.0, + 2.5, 1.25, 0.625, None][intLevel + 1] + + self.netOne = torch.nn.Sequential( + torch.nn.Conv2d( + in_channels=intCurrent, out_channels=128, kernel_size=3, stride=1, padding=1), + torch.nn.LeakyReLU(inplace=False, negative_slope=0.1) + ) + + self.netTwo = torch.nn.Sequential( + torch.nn.Conv2d(in_channels=intCurrent + 128, + out_channels=128, kernel_size=3, stride=1, padding=1), + torch.nn.LeakyReLU(inplace=False, negative_slope=0.1) + ) + + self.netThr = torch.nn.Sequential( + torch.nn.Conv2d(in_channels=intCurrent + 128 + 128, + out_channels=96, kernel_size=3, stride=1, padding=1), + torch.nn.LeakyReLU(inplace=False, negative_slope=0.1) + ) + + self.netFou = torch.nn.Sequential( + torch.nn.Conv2d(in_channels=intCurrent + 128 + 128 + 96, + out_channels=64, kernel_size=3, stride=1, padding=1), + torch.nn.LeakyReLU(inplace=False, negative_slope=0.1) + ) + + self.netFiv = torch.nn.Sequential( + torch.nn.Conv2d(in_channels=intCurrent + 128 + 128 + 96 + + 64, out_channels=32, kernel_size=3, stride=1, padding=1), + torch.nn.LeakyReLU(inplace=False, negative_slope=0.1) + ) + + self.netSix = torch.nn.Sequential( + torch.nn.Conv2d(in_channels=intCurrent + 128 + 128 + 96 + + 64 + 32, out_channels=2, kernel_size=3, stride=1, padding=1) + ) + # end + + def forward(self, tenFirst, tenSecond, objPrevious): + tenFlow = None + tenFeat = None + + if objPrevious is None: + tenFlow = None + tenFeat = None + + tenVolume = torch.nn.functional.leaky_relu(input=correlation.FunctionCorrelation( + tenFirst=tenFirst, tenSecond=tenSecond), negative_slope=0.1, inplace=False) + + tenFeat = torch.cat([tenVolume], 1) + + elif objPrevious is not None: + tenFlow = self.netUpflow(objPrevious['tenFlow']) + tenFeat = self.netUpfeat(objPrevious['tenFeat']) + + tenVolume = torch.nn.functional.leaky_relu(input=correlation.FunctionCorrelation(tenFirst=tenFirst, tenSecond=backwarp( + tenInput=tenSecond, tenFlow=tenFlow * self.fltBackwarp)), negative_slope=0.1, inplace=False) + + tenFeat = torch.cat( + [tenVolume, tenFirst, tenFlow, tenFeat], 1) + + # end + + tenFeat = torch.cat([self.netOne(tenFeat), tenFeat], 1) + tenFeat = torch.cat([self.netTwo(tenFeat), tenFeat], 1) + tenFeat = torch.cat([self.netThr(tenFeat), tenFeat], 1) + tenFeat = torch.cat([self.netFou(tenFeat), tenFeat], 1) + tenFeat = torch.cat([self.netFiv(tenFeat), tenFeat], 1) + + tenFlow = self.netSix(tenFeat) + + return { + 'tenFlow': tenFlow, + 'tenFeat': tenFeat + } + # end + # end + + class Refiner(torch.nn.Module): + def __init__(self): + super(Refiner, self).__init__() + + self.netMain = torch.nn.Sequential( + torch.nn.Conv2d(in_channels=81 + 32 + 2 + 2 + 128 + 128 + 96 + 64 + 32, + out_channels=128, kernel_size=3, stride=1, padding=1, dilation=1), + torch.nn.LeakyReLU(inplace=False, negative_slope=0.1), + torch.nn.Conv2d(in_channels=128, out_channels=128, + kernel_size=3, stride=1, padding=2, dilation=2), + torch.nn.LeakyReLU(inplace=False, negative_slope=0.1), + torch.nn.Conv2d(in_channels=128, out_channels=128, + kernel_size=3, stride=1, padding=4, dilation=4), + torch.nn.LeakyReLU(inplace=False, negative_slope=0.1), + torch.nn.Conv2d(in_channels=128, out_channels=96, + kernel_size=3, stride=1, padding=8, dilation=8), + torch.nn.LeakyReLU(inplace=False, negative_slope=0.1), + torch.nn.Conv2d(in_channels=96, out_channels=64, + kernel_size=3, stride=1, padding=16, dilation=16), + torch.nn.LeakyReLU(inplace=False, negative_slope=0.1), + torch.nn.Conv2d(in_channels=64, out_channels=32, + kernel_size=3, stride=1, padding=1, dilation=1), + torch.nn.LeakyReLU(inplace=False, negative_slope=0.1), + torch.nn.Conv2d(in_channels=32, out_channels=2, + kernel_size=3, stride=1, padding=1, dilation=1) + ) + # end + + def forward(self, tenInput): + return self.netMain(tenInput) + # end + # end + + self.netExtractor = Extractor() + + self.netTwo = Decoder(2) + self.netThr = Decoder(3) + self.netFou = Decoder(4) + self.netFiv = Decoder(5) + self.netSix = Decoder(6) + + self.netRefiner = Refiner() + + # self.load_state_dict({ strKey.replace('module', 'net'): tenWeight for strKey, tenWeight in torch.load(__file__.replace('run.py', 'network-' + arguments_strModel + '.pytorch')).items() }) + # end + + def forward(self, tenFirst, tenSecond): + tenFirst = self.netExtractor(tenFirst) + tenSecond = self.netExtractor(tenSecond) + + objEstimate = self.netSix(tenFirst[-1], tenSecond[-1], None) + objEstimate = self.netFiv(tenFirst[-2], tenSecond[-2], objEstimate) + objEstimate = self.netFou(tenFirst[-3], tenSecond[-3], objEstimate) + objEstimate = self.netThr(tenFirst[-4], tenSecond[-4], objEstimate) + objEstimate = self.netTwo(tenFirst[-5], tenSecond[-5], objEstimate) + + return objEstimate['tenFlow'] + self.netRefiner(objEstimate['tenFeat']) + # end +# end + + +netNetwork = None + +########################################################## + + +def estimate(tenFirst, tenSecond, netNetwork): + + assert(tenFirst.shape[1] == tenSecond.shape[1]) + assert(tenFirst.shape[2] == tenSecond.shape[2]) + + intWidth = tenFirst.shape[3] + intHeight = tenFirst.shape[2] + # import ipdb; ipdb.set_trace() + tenPreprocessedFirst = tenFirst.view(-1, 3, intHeight, intWidth) + tenPreprocessedSecond = tenSecond.view(-1, 3, intHeight, intWidth) + + intPreprocessedWidth = int(math.floor(math.ceil(intWidth / 64.0) * 64.0)) + intPreprocessedHeight = int(math.floor(math.ceil(intHeight / 64.0) * 64.0)) + + tenPreprocessedFirst = torch.nn.functional.interpolate(input=tenPreprocessedFirst, size=( + intPreprocessedHeight, intPreprocessedWidth), mode='bilinear', align_corners=False) + tenPreprocessedSecond = torch.nn.functional.interpolate(input=tenPreprocessedSecond, size=( + intPreprocessedHeight, intPreprocessedWidth), mode='bilinear', align_corners=False) + + tenFlow = 20.0 * torch.nn.functional.interpolate(input=netNetwork( + tenPreprocessedFirst, tenPreprocessedSecond), size=(intHeight, intWidth), mode='bilinear', align_corners=False) + + tenFlow[:, 0, :, :] *= float(intWidth) / float(intPreprocessedWidth) + tenFlow[:, 1, :, :] *= float(intHeight) / float(intPreprocessedHeight) + + global backwarp_tenGrid + global backwarp_tenPartial + backwarp_tenGrid = {} + backwarp_tenPartial = {} + + return tenFlow +# end + +########################################################## + + +if __name__ == '__main__': + tenFirst = torch.FloatTensor(numpy.ascontiguousarray(numpy.array(PIL.Image.open( + arguments_strFirst))[:, :, ::-1].transpose(2, 0, 1).astype(numpy.float32) * (1.0 / 255.0))) + tenSecond = torch.FloatTensor(numpy.ascontiguousarray(numpy.array(PIL.Image.open( + arguments_strSecond))[:, :, ::-1].transpose(2, 0, 1).astype(numpy.float32) * (1.0 / 255.0))) + + tenOutput = estimate(tenFirst, tenSecond) + + objOutput = open(arguments_strOut, 'wb') + + numpy.array([80, 73, 69, 72], numpy.uint8).tofile(objOutput) + numpy.array([tenOutput.shape[2], tenOutput.shape[1]], + numpy.int32).tofile(objOutput) + numpy.array(tenOutput.numpy().transpose(1, 2, 0), + numpy.float32).tofile(objOutput) + + objOutput.close() +# end diff --git a/models/DUT/Smoother.py b/models/DUT/Smoother.py new file mode 100644 index 0000000..149860f --- /dev/null +++ b/models/DUT/Smoother.py @@ -0,0 +1,88 @@ +import torch +import torch.nn as nn +import numpy as np +import math +from utils.IterativeSmooth import generateSmooth + +class Smoother(nn.Module): + + def __init__(self, inplanes=2, embeddingSize=64, hiddenSize=64, kernel=5): + super(Smoother, self).__init__() + self.embedding = nn.Sequential( + nn.Linear(inplanes, embeddingSize), + nn.ReLU() + ) + self.pad = kernel // 2 + self.conv1 = nn.Conv3d(embeddingSize, embeddingSize, + (kernel, 3, 3), padding=(self.pad, 1, 1)) + self.conv3 = nn.Conv3d(embeddingSize, embeddingSize, + (kernel, 3, 3), padding=(self.pad, 1, 1)) + self.conv2 = nn.Conv3d(embeddingSize, embeddingSize, + (kernel, 3, 3), padding=(self.pad, 1, 1)) + self.decoder = nn.Linear(embeddingSize, 12, bias=True) + self.scale = nn.Linear(embeddingSize, 1, bias=True) + self.activation = nn.Sigmoid() + self.relu = nn.ReLU() + self.generateSmooth = generateSmooth + + def forward(self, trajectory): + ''' + @param trajectory: Unstable trajectory with shape [B, 2, T, H, W] + + @return kernel: dynamic smooth kernel with shape [B, 12, T, H, W] + ''' + + trajectory = trajectory.permute(0, 2, 3, 4, 1) + embedding_trajectory = self.embedding(trajectory).permute(0, 4, 1, 2, 3) + hidden = embedding_trajectory + hidden = self.relu(self.conv1(hidden)) + hidden = self.relu(self.conv3(hidden)) + hidden = self.relu(self.conv2(hidden)) + kernel = self.activation(self.decoder(hidden.permute( + 0, 2, 3, 4, 1)).permute(0, 4, 1, 2, 3)) + kernel = self.scale(hidden.permute(0, 2, 3, 4, 1)).permute( + 0, 4, 1, 2, 3) * kernel + return kernel + + def inference(self, x_paths, y_paths, repeat=50): + ''' + @param x_paths: Unstable trajectory in x direction, [B, T, H, W] + @param y_paths: Unstable trajectory in y direction, [B, T, H, W] + @param repeat: iterations for smoother, int + + @return smooth_x: Smoothed trajectory in x direction, [B, T, H, W] + @return smooth_y: Smoothed trajectory in y direction, [B, T, H, W] + ''' + path = np.concatenate([np.expand_dims(x_paths, -1), + np.expand_dims(y_paths, -1)], -1) + + # regularization + min_v = np.min(path, keepdims=True) + path = path - min_v + max_v = np.max(path, keepdims=True) + 1e-5 + path = path / max_v + path = np.transpose(np.expand_dims(path, 0), (0, 4, 3, 1, 2)) + path_t = torch.from_numpy(path.astype(np.float32)).cuda() + + # get smooth kernel + kernel_t = self.forward(path_t) + + # iterative smooth + smooth_x, smooth_y = self.KernelSmooth(kernel_t, path_t, repeat) + + smooth_x = smooth_x.cpu().squeeze().permute(1, 2, 0).numpy() * max_v + min_v + smooth_y = smooth_y.cpu().squeeze().permute(1, 2, 0).numpy() * max_v + min_v + return smooth_x, smooth_y + + def KernelSmooth(self, kernel, path, repeat=20): + if kernel is None: + smooth_x = self.generateSmooth( + path[:, 0:1, :, :, :], None, repeat) + smooth_y = self.generateSmooth( + path[:, 1:2, :, :, :], None, repeat) + else: + smooth_x = self.generateSmooth( + path[:, 0:1, :, :, :], kernel[:, 0:6, :, :, :], repeat) + smooth_y = self.generateSmooth( + path[:, 1:2, :, :, :], kernel[:, 6:12, :, :, :], repeat) + return smooth_x, smooth_y diff --git a/models/DUT/__init__.py b/models/DUT/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/models/DUT/rf_det_module.py b/models/DUT/rf_det_module.py new file mode 100644 index 0000000..28966b8 --- /dev/null +++ b/models/DUT/rf_det_module.py @@ -0,0 +1,244 @@ +# -*- coding: utf-8 -*- +# @Time : 2018-9-27 15:39 +# @Author : xylon + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from utils.image_utils import filter_border, nms, topk_map, get_gauss_filter_weight + + +class RFDetModule(nn.Module): + def __init__( + self, + score_com_strength, + scale_com_strength, + nms_thresh, + nms_ksize, + topk, + gauss_ksize, + gauss_sigma, + ksize, + padding, + dilation, + scale_list, + ): + super(RFDetModule, self).__init__() + + self.score_com_strength = score_com_strength + self.scale_com_strength = scale_com_strength + self.NMS_THRESH = nms_thresh + self.NMS_KSIZE = nms_ksize + self.TOPK = topk + self.GAUSSIAN_KSIZE = gauss_ksize + self.GAUSSIAN_SIGMA = gauss_sigma + + self.conv1 = nn.Conv2d( + in_channels=1, + out_channels=16, + kernel_size=ksize, + stride=1, + padding=padding, + dilation=dilation, + ) # 3 RF + self.insnorm1 = nn.InstanceNorm2d(16, affine=True) + self.conv_s3 = nn.Conv2d( + in_channels=16, out_channels=1, kernel_size=1, stride=1, padding=0 + ) + self.insnorm_s3 = nn.InstanceNorm2d(1, affine=True) + + self.conv2 = nn.Conv2d( + in_channels=16, + out_channels=16, + kernel_size=ksize, + stride=1, + padding=padding, + dilation=dilation, + ) # 5 RF + self.insnorm2 = nn.InstanceNorm2d(16, affine=True) + self.conv_s5 = nn.Conv2d( + in_channels=16, out_channels=1, kernel_size=1, stride=1, padding=0 + ) + self.insnorm_s5 = nn.InstanceNorm2d(1, affine=True) + + self.conv3 = nn.Conv2d( + in_channels=16, + out_channels=16, + kernel_size=ksize, + stride=1, + padding=padding, + dilation=dilation, + ) # 7 RF + self.insnorm3 = nn.InstanceNorm2d(16, affine=True) + self.conv_s7 = nn.Conv2d( + in_channels=16, out_channels=1, kernel_size=1, stride=1, padding=0 + ) + self.insnorm_s7 = nn.InstanceNorm2d(1, affine=True) + + self.conv4 = nn.Conv2d( + in_channels=16, + out_channels=16, + kernel_size=ksize, + stride=1, + padding=padding, + dilation=dilation, + ) # 9 RF + self.insnorm4 = nn.InstanceNorm2d(16, affine=True) + self.conv_s9 = nn.Conv2d( + in_channels=16, out_channels=1, kernel_size=1, stride=1, padding=0 + ) + self.insnorm_s9 = nn.InstanceNorm2d(1, affine=True) + + self.conv5 = nn.Conv2d( + in_channels=16, + out_channels=16, + kernel_size=ksize, + stride=1, + padding=padding, + dilation=dilation, + ) # 11 RF + self.insnorm5 = nn.InstanceNorm2d(16, affine=True) + self.conv_s11 = nn.Conv2d( + in_channels=16, out_channels=1, kernel_size=1, stride=1, padding=0 + ) + self.insnorm_s11 = nn.InstanceNorm2d(1, affine=True) + + self.conv6 = nn.Conv2d( + in_channels=16, + out_channels=16, + kernel_size=ksize, + stride=1, + padding=padding, + dilation=dilation, + ) # 13 RF + self.insnorm6 = nn.InstanceNorm2d(16, affine=True) + self.conv_s13 = nn.Conv2d( + in_channels=16, out_channels=1, kernel_size=1, stride=1, padding=0 + ) + self.insnorm_s13 = nn.InstanceNorm2d(1, affine=True) + + self.conv7 = nn.Conv2d( + in_channels=16, + out_channels=16, + kernel_size=ksize, + stride=1, + padding=padding, + dilation=dilation, + ) # 15 RF + self.insnorm7 = nn.InstanceNorm2d(16, affine=True) + self.conv_s15 = nn.Conv2d( + in_channels=16, out_channels=1, kernel_size=1, stride=1, padding=0 + ) + self.insnorm_s15 = nn.InstanceNorm2d(1, affine=True) + + self.conv8 = nn.Conv2d( + in_channels=16, + out_channels=16, + kernel_size=ksize, + stride=1, + padding=padding, + dilation=dilation, + ) # 17 RF + self.insnorm8 = nn.InstanceNorm2d(16, affine=True) + self.conv_s17 = nn.Conv2d( + in_channels=16, out_channels=1, kernel_size=1, stride=1, padding=0 + ) + self.insnorm_s17 = nn.InstanceNorm2d(1, affine=True) + + self.conv9 = nn.Conv2d( + in_channels=16, + out_channels=16, + kernel_size=ksize, + stride=1, + padding=padding, + dilation=dilation, + ) # 19 RF + self.insnorm9 = nn.InstanceNorm2d(16, affine=True) + self.conv_s19 = nn.Conv2d( + in_channels=16, out_channels=1, kernel_size=1, stride=1, padding=0 + ) + self.insnorm_s19 = nn.InstanceNorm2d(1, affine=True) + + self.conv10 = nn.Conv2d( + in_channels=16, + out_channels=16, + kernel_size=ksize, + stride=1, + padding=padding, + dilation=dilation, + ) # 21 RF + self.insnorm10 = nn.InstanceNorm2d(16, affine=True) + self.conv_s21 = nn.Conv2d( + in_channels=16, out_channels=1, kernel_size=1, stride=1, padding=0 + ) + self.insnorm_s21 = nn.InstanceNorm2d(1, affine=True) + + self.scale_list = torch.tensor(scale_list) + + def forward(self, **kwargs): + pass + + def process(self, im1w_score): + """ + nms(n), topk(t), gaussian kernel(g) operation + :param im1w_score: warped score map + :return: processed score map, topk mask, topk value + """ + im1w_score = filter_border(im1w_score) + + # apply nms to im1w_score + nms_mask = nms(im1w_score, thresh=self.NMS_THRESH, ksize=self.NMS_KSIZE) + im1w_score = im1w_score * nms_mask + topk_value = im1w_score + + # apply topk to im1w_score + topk_mask = topk_map(im1w_score, self.TOPK) + im1w_score = topk_mask.to(torch.float) * im1w_score + + # apply gaussian kernel to im1w_score + psf = im1w_score.new_tensor( + get_gauss_filter_weight(self.GAUSSIAN_KSIZE, self.GAUSSIAN_SIGMA)[ + None, None, :, : + ] + ).to(im1w_score.device) + im1w_score = F.conv2d( + input=im1w_score.permute(0, 3, 1, 2), + weight=psf, + stride=1, + padding=self.GAUSSIAN_KSIZE // 2, + ).permute( + 0, 2, 3, 1 + ) # (B, H, W, 1) + + """ + apply tf.clamp to make sure all value in im1w_score isn't greater than 1 + but this won't happend in correct way + """ + im1w_score = im1w_score.clamp(min=0.0, max=1.0) + + return im1w_score, topk_mask, topk_value + + # @staticmethod + # def loss(left_score, im1gt_score, im1visible_mask): + # im1_score = left_score + + # l2_element_diff = (im1_score - im1gt_score) ** 2 + # # visualization numbers + # Nvi = torch.clamp(im1visible_mask.sum(dim=(3, 2, 1)), min=2.0) + # loss = ( + # torch.sum(l2_element_diff * im1visible_mask, dim=(3, 2, 1)) / (Nvi + 1e-8) + # ).mean() + + # return loss + + @staticmethod + def weights_init(m): + if isinstance(m, nn.Conv2d): + nn.init.xavier_uniform_( + m.weight.data, gain=nn.init.calculate_gain("leaky_relu") + ) + try: + nn.init.xavier_uniform_(m.bias.data) + except: + pass diff --git a/models/DUT/rf_det_so.py b/models/DUT/rf_det_so.py new file mode 100644 index 0000000..0a5da8b --- /dev/null +++ b/models/DUT/rf_det_so.py @@ -0,0 +1,242 @@ +# -*- coding: utf-8 -*- +# @Time : 2018-9-13 16:03 +# @Author : xylon + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from utils.image_utils import soft_nms_3d, soft_max_and_argmax_1d +from utils.math_utils import L2Norm +from .rf_det_module import RFDetModule + + +class RFDetSO(RFDetModule): + def __init__( + self, + score_com_strength, + scale_com_strength, + nms_thresh, + nms_ksize, + topk, + gauss_ksize, + gauss_sigma, + ksize, + padding, + dilation, + scale_list, + ): + super(RFDetSO, self).__init__( + score_com_strength, + scale_com_strength, + nms_thresh, + nms_ksize, + topk, + gauss_ksize, + gauss_sigma, + ksize, + padding, + dilation, + scale_list, + ) + + self.conv_o3 = nn.Conv2d( + in_channels=16, out_channels=2, kernel_size=1, stride=1, padding=0 + ) + self.conv_o5 = nn.Conv2d( + in_channels=16, out_channels=2, kernel_size=1, stride=1, padding=0 + ) + self.conv_o7 = nn.Conv2d( + in_channels=16, out_channels=2, kernel_size=1, stride=1, padding=0 + ) + self.conv_o9 = nn.Conv2d( + in_channels=16, out_channels=2, kernel_size=1, stride=1, padding=0 + ) + self.conv_o11 = nn.Conv2d( + in_channels=16, out_channels=2, kernel_size=1, stride=1, padding=0 + ) + self.conv_o13 = nn.Conv2d( + in_channels=16, out_channels=2, kernel_size=1, stride=1, padding=0 + ) + self.conv_o15 = nn.Conv2d( + in_channels=16, out_channels=2, kernel_size=1, stride=1, padding=0 + ) + self.conv_o17 = nn.Conv2d( + in_channels=16, out_channels=2, kernel_size=1, stride=1, padding=0 + ) + self.conv_o19 = nn.Conv2d( + in_channels=16, out_channels=2, kernel_size=1, stride=1, padding=0 + ) + self.conv_o21 = nn.Conv2d( + in_channels=16, out_channels=2, kernel_size=1, stride=1, padding=0 + ) + + def forward(self, photos): + + # Extract score map in scale space from 3 to 21 + score_featmaps_s3 = F.leaky_relu(self.insnorm1(self.conv1(photos))) + score_map_s3 = self.insnorm_s3(self.conv_s3(score_featmaps_s3)).permute( + 0, 2, 3, 1 + ) + orint_map_s3 = ( + L2Norm(self.conv_o3(score_featmaps_s3), dim=1) + .permute(0, 2, 3, 1) + .unsqueeze(-2) + ) + + score_featmaps_s5 = F.leaky_relu(self.insnorm2(self.conv2(score_featmaps_s3))) + score_map_s5 = self.insnorm_s5(self.conv_s5(score_featmaps_s5)).permute( + 0, 2, 3, 1 + ) + orint_map_s5 = ( + L2Norm(self.conv_o5(score_featmaps_s5), dim=1) + .permute(0, 2, 3, 1) + .unsqueeze(-2) + ) + score_featmaps_s5 = score_featmaps_s5 + score_featmaps_s3 + + score_featmaps_s7 = F.leaky_relu(self.insnorm3(self.conv3(score_featmaps_s5))) + score_map_s7 = self.insnorm_s7(self.conv_s7(score_featmaps_s7)).permute( + 0, 2, 3, 1 + ) + orint_map_s7 = ( + L2Norm(self.conv_o7(score_featmaps_s7), dim=1) + .permute(0, 2, 3, 1) + .unsqueeze(-2) + ) + score_featmaps_s7 = score_featmaps_s7 + score_featmaps_s5 + + score_featmaps_s9 = F.leaky_relu(self.insnorm4(self.conv4(score_featmaps_s7))) + score_map_s9 = self.insnorm_s9(self.conv_s9(score_featmaps_s9)).permute( + 0, 2, 3, 1 + ) + orint_map_s9 = ( + L2Norm(self.conv_o9(score_featmaps_s9), dim=1) + .permute(0, 2, 3, 1) + .unsqueeze(-2) + ) + score_featmaps_s9 = score_featmaps_s9 + score_featmaps_s7 + + score_featmaps_s11 = F.leaky_relu(self.insnorm5(self.conv5(score_featmaps_s9))) + score_map_s11 = self.insnorm_s11(self.conv_s11(score_featmaps_s11)).permute( + 0, 2, 3, 1 + ) + orint_map_s11 = ( + L2Norm(self.conv_o11(score_featmaps_s11), dim=1) + .permute(0, 2, 3, 1) + .unsqueeze(-2) + ) + score_featmaps_s11 = score_featmaps_s11 + score_featmaps_s9 + + score_featmaps_s13 = F.leaky_relu(self.insnorm6(self.conv6(score_featmaps_s11))) + score_map_s13 = self.insnorm_s13(self.conv_s13(score_featmaps_s13)).permute( + 0, 2, 3, 1 + ) + orint_map_s13 = ( + L2Norm(self.conv_o13(score_featmaps_s13), dim=1) + .permute(0, 2, 3, 1) + .unsqueeze(-2) + ) + score_featmaps_s13 = score_featmaps_s13 + score_featmaps_s11 + + score_featmaps_s15 = F.leaky_relu(self.insnorm7(self.conv7(score_featmaps_s13))) + score_map_s15 = self.insnorm_s15(self.conv_s15(score_featmaps_s15)).permute( + 0, 2, 3, 1 + ) + orint_map_s15 = ( + L2Norm(self.conv_o15(score_featmaps_s15), dim=1) + .permute(0, 2, 3, 1) + .unsqueeze(-2) + ) + score_featmaps_s15 = score_featmaps_s15 + score_featmaps_s13 + + score_featmaps_s17 = F.leaky_relu(self.insnorm8(self.conv8(score_featmaps_s15))) + score_map_s17 = self.insnorm_s17(self.conv_s17(score_featmaps_s17)).permute( + 0, 2, 3, 1 + ) + orint_map_s17 = ( + L2Norm(self.conv_o17(score_featmaps_s17), dim=1) + .permute(0, 2, 3, 1) + .unsqueeze(-2) + ) + score_featmaps_s17 = score_featmaps_s17 + score_featmaps_s15 + + score_featmaps_s19 = F.leaky_relu(self.insnorm9(self.conv9(score_featmaps_s17))) + score_map_s19 = self.insnorm_s19(self.conv_s19(score_featmaps_s19)).permute( + 0, 2, 3, 1 + ) + orint_map_s19 = ( + L2Norm(self.conv_o19(score_featmaps_s19), dim=1) + .permute(0, 2, 3, 1) + .unsqueeze(-2) + ) + score_featmaps_s19 = score_featmaps_s19 + score_featmaps_s17 + + score_featmaps_s21 = F.leaky_relu( + self.insnorm10(self.conv10(score_featmaps_s19)) + ) + score_map_s21 = self.insnorm_s21(self.conv_s21(score_featmaps_s21)).permute( + 0, 2, 3, 1 + ) + orint_map_s21 = ( + L2Norm(self.conv_o21(score_featmaps_s21), dim=1) + .permute(0, 2, 3, 1) + .unsqueeze(-2) + ) + + score_maps = torch.cat( + ( + score_map_s3, + score_map_s5, + score_map_s7, + score_map_s9, + score_map_s11, + score_map_s13, + score_map_s15, + score_map_s17, + score_map_s19, + score_map_s21, + ), + -1, + ) # (B, H, W, C) + + orint_maps = torch.cat( + ( + orint_map_s3, + orint_map_s5, + orint_map_s7, + orint_map_s9, + orint_map_s11, + orint_map_s13, + orint_map_s15, + orint_map_s17, + orint_map_s19, + orint_map_s21, + ), + -2, + ) # (B, H, W, 10, 2) + + # get each pixel probability in all scale + scale_probs = soft_nms_3d(score_maps, ksize=15, com_strength=3.0) + + # get each pixel probability summary from all scale space and correspond scale value + score_map, scale_map, orint_map = soft_max_and_argmax_1d( + input=scale_probs, + orint_maps=orint_maps, + dim=-1, + scale_list=self.scale_list, + keepdim=True, + com_strength1=self.score_com_strength, + com_strength2=self.scale_com_strength, + ) + + return score_map, scale_map, orint_map + + @staticmethod + def convO_init(m): + if isinstance(m, nn.Conv2d): + nn.init.zeros_(m.weight.data) + try: + nn.init.ones_(m.bias.data) + except: + pass \ No newline at end of file diff --git a/models/StabNet/__init__.py b/models/StabNet/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/models/StabNet/model.py b/models/StabNet/model.py new file mode 100644 index 0000000..d77e5f9 --- /dev/null +++ b/models/StabNet/model.py @@ -0,0 +1,360 @@ +from __future__ import print_function +import torch +import torch.nn as nn +import torch.nn.functional as F +import torchvision +import matplotlib.pyplot as plt +import numpy as np +import cv2 +from .pytorch_resnet_v2_50 import KitModel +from .v2_93 import * +import numpy as np +from PIL import Image +import cv2 +import time +import os +import math + +def transformer(U, theta, name='SpatialTransformer', **kwargs): + """Spatial Transformer Layer + + Implements a spatial transformer layer as described in [1]_. + Based on [2]_ and edited by David Dao for Tensorflow. + + Parameters + ---------- + U : float + The output of a convolutional net should have the + shape [num_batch, height, width, num_channels]. + theta: float + The output of the + localisation network should be [num_batch, 6]. + out_size: tuple of two ints + The size of the output of the network (height, width) + + References + ---------- + .. [1] Spatial Transformer Networks + Max Jaderberg, Karen Simonyan, Andrew Zisserman, Koray Kavukcuoglu + Submitted on 5 Jun 2015 + .. [2] https://github.com/skaae/transformer_network/blob/master/transformerlayer.py + + Notes + ----- + To initialize the network to the identity transform init + ``theta`` to : + identity = np.array([[1., 0., 0.], + [0., 1., 0.]]) + identity = identity.flatten() + theta = tf.Variable(initial_value=identity) + + """ + def _repeat(x, n_repeats): + rep = torch._cast_Long(torch.transpose(torch.ones([n_repeats, ]).unsqueeze(1), 1, 0)) + x = torch.matmul(x.view(-1, 1), rep) + return x.view(-1) + + def _interpolate(im, x, y, out_size): + num_batch, height, width, channels = im.size() # to be sure the input dims is NHWC + x = torch._cast_Float(x).cuda() + y = torch._cast_Float(y).cuda() + height_f = torch._cast_Float(torch.Tensor([height]))[0].cuda() + width_f = torch._cast_Float(torch.Tensor([width]))[0].cuda() + out_height = out_size[0] + out_width = out_size[1] + zero = torch.zeros([], dtype=torch.int32).cuda() + max_y = torch._cast_Long(torch.Tensor([height - 1]))[0].cuda() + max_x = torch._cast_Long(torch.Tensor([width - 1]))[0].cuda() + + # scale indices from [-1, 1] to [0, width/height] + x = (x + 1.0) * width_f / 2.0 + y = (y + 1.0) * height_f / 2.0 + + # do sampling + x0 = torch._cast_Long(torch.floor(x)).cuda() + x1 = x0 + 1 + y0 = torch._cast_Long(torch.floor(y)).cuda() + y1 = y0 + 1 + + x0 = torch.clamp(x0, zero, max_x) + x1 = torch.clamp(x1, zero, max_x) + y0 = torch.clamp(y0, zero, max_y) + y1 = torch.clamp(y1, zero, max_y) + dim2 = width + dim1 = width * height + base = _repeat(torch.arange(num_batch) * dim1, out_height * out_width).cuda() + base_y0 = base + y0 * dim2 + base_y1 = base + y1 * dim2 + idx_a = base_y0 + x0 + idx_b = base_y1 + x0 + idx_c = base_y0 + x1 + idx_d = base_y1 + x1 + + # use indices to look up pixels in the flate images + # and restore channels dim + im_flat = im.contiguous().view(-1, channels) + im_flat = torch._cast_Float(im_flat) + Ia = im_flat[idx_a] # as in tf, the default dim is row first + Ib = im_flat[idx_b] + Ic = im_flat[idx_c] + Id = im_flat[idx_d] + + # calculate interpolated values + x0_f = torch._cast_Float(x0).cuda() + x1_f = torch._cast_Float(x1).cuda() + y0_f = torch._cast_Float(y0).cuda() + y1_f = torch._cast_Float(y1).cuda() + wa = ((x1_f - x) * (y1_f - y)).unsqueeze(1) + wb = ((x1_f - x) * (y - y0_f)).unsqueeze(1) + wc = ((x - x0_f) * (y1_f - y)).unsqueeze(1) + wd = ((x - x0_f) * (y - y0_f)).unsqueeze(1) + + return wa * Ia + wb * Ib + wc * Ic + wd * Id + + def _meshgrid(height, width): + x_t = torch.matmul(torch.ones([height, 1]), + torch.transpose(torch.linspace(-1.0, 1.0, width).unsqueeze(1), 1, 0)) + y_t = torch.matmul(torch.linspace(-1.0, 1.0, height).unsqueeze(1), + torch.ones([1, width])) + + x_t_flat = x_t.view(1, -1) + y_t_flat = y_t.view(1, -1) + + ones = torch.ones_like(x_t_flat) + grid = torch.cat([x_t_flat, y_t_flat, ones], 0) + return grid + + def pinv(A): + A = A.cpu() + torch.eye(8) * 1e-4 + return torch.inverse(A).cuda() + + def get_H(ori, tar): + num_batch = ori.size()[0] + one = torch.ones([num_batch, 1]).cuda() + zero = torch.zeros([num_batch, 1]).cuda() + x = [ori[:, 0:1], ori[:, 2:3], ori[:, 4:5], ori[:, 6:7]] + y = [ori[:, 1:2], ori[:, 3:4], ori[:, 5:6], ori[:, 7:8]] + u = [tar[:, 0:1], tar[:, 2:3], tar[:, 4:5], tar[:, 6:7]] + v = [tar[:, 1:2], tar[:, 3:4], tar[:, 5:6], tar[:, 7:8]] + + A_ = [] + A_.extend([x[0], y[0], one, zero, zero, zero, -x[0] * u[0], -y[0] * u[0]]) + A_.extend([x[1], y[1], one, zero, zero, zero, -x[1] * u[1], -y[1] * u[1]]) + A_.extend([x[2], y[2], one, zero, zero, zero, -x[2] * u[2], -y[2] * u[2]]) + A_.extend([x[3], y[3], one, zero, zero, zero, -x[3] * u[3], -y[3] * u[3]]) + A_.extend([zero, zero, zero, x[0], y[0], one, -x[0] * v[0], -y[0] * v[0]]) + A_.extend([zero, zero, zero, x[1], y[1], one, -x[1] * v[1], -y[1] * v[1]]) + A_.extend([zero, zero, zero, x[2], y[2], one, -x[2] * v[2], -y[2] * v[2]]) + A_.extend([zero, zero, zero, x[3], y[3], one, -x[3] * v[3], -y[3] * v[3]]) + A = torch.cat(A_, dim=1).view(num_batch, 8, 8) + b_ = [u[0], u[1], u[2], u[3], v[0],v[1], v[2], v[3]] + b = torch.cat(b_, dim=1).view([num_batch, 8, 1]) + + ans = torch.cat([torch.matmul(pinv(A), b).view([num_batch, 8]), torch.ones([num_batch, 1]).cuda()], + dim=1) + return ans + + # check pass + def get_Hs(theta): + num_batch = theta.size()[0] + h = 2.0 / grid_h + w = 2.0 / grid_w + Hs = [] + for i in range(grid_h): + for j in range(grid_w): + hh = i * h - 1 + ww = j * w - 1 + ori = torch._cast_Float(torch.Tensor([ww, hh, ww + w, hh, ww, hh + h, ww + w, hh + h])).\ + view([1, 8]).repeat([num_batch, 1]).cuda() + id = i * (grid_w + 1) + grid_w + tar = torch.cat([theta[:, i:i+1, j:j+1, :], + theta[:, i:i+1, j+1:j+2, :], + theta[:, i+1:i+2, j:j+1, :], + theta[:, i+1:i+2, j+1:j+2, :]], dim=1) + tar = tar.view([num_batch, 8]) + Hs.append(get_H(ori, tar).view([num_batch, 1, 9])) + + Hs = torch.cat(Hs, dim=1).view([num_batch, grid_h, grid_w, 9]) + return Hs + + def _meshgrid2(height, width, sh, eh, sw, ew): + hn = eh - sh + 1 + wn = ew - sw + 1 + + x_t = torch.matmul(torch.ones([hn, 1]).cuda(), + torch.transpose(torch.linspace(-1.0, 1.0, width)[sw:sw+wn].unsqueeze(1), 1, 0).cuda()) + y_t = torch.matmul(torch.linspace(-1.0, 1.0, height)[sh:sh+hn].unsqueeze(1).cuda(), + torch.ones([1, wn]).cuda()) + + x_t_flat = x_t.view(1, -1) + y_t_flat = y_t.view(1, -1) + + ones = torch.ones_like(x_t_flat) + grid = torch.cat([x_t_flat, y_t_flat, ones], 0) + return grid + + def _transform3(theta, input_dim): + input_dim = input_dim.permute([0, 2, 3, 1]) + num_batch = input_dim.size()[0] + num_channels = input_dim.size()[3] + theta = torch._cast_Float(theta) + Hs = get_Hs(theta) + gh = int(math.floor(height / grid_h)) + gw = int(math.floor(width / grid_w)) + x_ = [] + y_ = [] + + for i in range(grid_h): + row_x_ = [] + row_y_ = [] + for j in range(grid_w): + H = Hs[:, i:i+1, j:j+1, :].view(num_batch, 3, 3) + sh = i * gh + eh = (i + 1) * gh - 1 + sw = j * gw + ew = (j + 1) * gw - 1 + if (i == grid_h - 1): + eh = height - 1 + if (j == grid_w - 1): + ew = width - 1 + grid = _meshgrid2(height, width, sh, eh, sw, ew) + grid = grid.unsqueeze(0) + grid = grid.repeat([num_batch, 1, 1]) + + T_g = torch.matmul(H, grid) + x_s = T_g[:, 0:1, :] + y_s = T_g[:, 1:2, :] + z_s = T_g[:, 2:3, :] + + z_s_flat = z_s.contiguous().view(-1) + t_1 = torch.ones(z_s_flat.size()).cuda() + t_0 = torch.zeros(z_s_flat.size()).cuda() + + sign_z_flat = torch.where(z_s_flat >= 0, t_1, t_0) * 2 - 1 + z_s_flat = z_s.contiguous().view(-1) + sign_z_flat * 1e-8 + x_s_flat = x_s.contiguous().view(-1) / z_s_flat + y_s_flat = y_s.contiguous().view(-1) / z_s_flat + + x_s = x_s_flat.view([num_batch, eh - sh + 1, ew - sw + 1]) + y_s = y_s_flat.view([num_batch, eh - sh + 1, ew - sw + 1]) + row_x_.append(x_s) + row_y_.append(y_s) + row_x = torch.cat(row_x_, dim=2) + row_y = torch.cat(row_y_, dim=2) + x_.append(row_x) + y_.append(row_y) + + x = torch.cat(x_, dim=1).view([num_batch, height, width, 1]) + y = torch.cat(y_, dim=1).view([num_batch, height, width, 1]) + x_map_ = x.clone() + y_map_ = y.clone() + img = torch.cat([x, y], dim=3) + x_s_flat = x.view(-1) + y_s_flat = y.view(-1) + + t_1 = torch.ones(x_s_flat.size()).cuda() + t_0 = torch.zeros(x_s_flat.size()).cuda() + + cond = (torch.gt(t_1 * -1, x_s_flat) | torch.gt(x_s_flat, t_1)) | \ + (torch.gt(t_1 * -1, y_s_flat) | torch.gt(y_s_flat, t_1)) + + black_pix = torch.where(cond, t_1, t_0).view([num_batch, height, width]) + + out_size = (height, width) + input_transformed = _interpolate( + input_dim, x_s_flat, y_s_flat, + out_size) + + output = input_transformed.view([num_batch, height, width, num_channels]) + output = output.permute([0, 3, 1, 2]) + + return output, black_pix, img, x_map_, y_map_ + + output = _transform3(theta, U) + return output + +class stabNet(nn.Module): + def __init__(self): + super(stabNet, self).__init__() + self.resnet50 = KitModel() + self.resnet50.resnet_v2_50_conv1_Conv2D = nn.Conv2d(in_channels=13, out_channels=64, kernel_size=(7, 7), stride=(2, 2), groups=1, bias=True) + self.regressor = nn.Sequential(nn.Linear(2048, 2048), + nn.ReLU(), + nn.Linear(2048, 1024), + nn.ReLU(), + nn.Linear(1024, 512), + nn.ReLU(), + nn.Linear(512, ((grid_h + 1) * (grid_w + 1) * 2)), + ) + self.use_black_loss = 0 + self.one_ = torch.ones([batch_size, grid_h, grid_w, 8]).cuda() + self.zero_ = torch.zeros([batch_size, grid_h, grid_w, 8]).cuda() + + def get_black_pos(self, pts): + one_ = self.one_ / do_crop_rate + zero_ = self.zero_ + black_err = torch.where(torch.gt(pts, one_), pts - one_, zero_) + \ + torch.where(torch.gt(-1 * one_, pts), one_ * -1 - pts, zero_) + return black_err.view([batch_size, -1]) + + def get_4_pts(self, theta, batch_size): + pts1_ = [] + pts2_ = [] + pts = [] + h = 2.0 / grid_h + w = 2.0 / grid_w + tot = 0 + for i in range(grid_h + 1): + pts.append([]) + for j in range(grid_w + 1): + hh = i * h - 1 + ww = j * w - 1 + p = torch._cast_Float(torch.Tensor([ww, hh]).view(2)).cuda() + temp = theta[:, tot * 2: tot * 2 + 2] + tot += 1 + p = (p + temp).view([batch_size, 1, 2]) + p = torch.clamp(p, -1. / do_crop_rate, 1. / do_crop_rate) + pts[i].append(p.view([batch_size, 2, 1])) + pts2_.append(p) + + for i in range(grid_h): + for j in range(grid_w): + g = torch.cat([pts[i][j], pts[i][j + 1], pts[i + 1][j], pts[i + 1][j + 1]], dim = 2) + pts1_.append(g.view([batch_size, 1, 8])) + + pts1 = torch.cat(pts1_, 1).view([batch_size, grid_h, grid_w, 8]) + pts2 = torch.cat(pts2_, 1).view([batch_size, grid_h + 1, grid_w + 1, 2]) + + return pts1, pts2 + + def warp_pts(self, pts, flow): + x = pts[:, :, 0] + x = torch.clamp((x + 1) / 2 * width, 0, width - 1) + x = torch._cast_Int(torch.round(x)) + y = pts[:, :, 1] + y = torch.clamp((y + 1) / 2 * height, 0, height - 1) + y = torch._cast_Int(torch.round(y)) + + out = [] + for i in range(batch_size): + flow_ = flow[i, :, :, :].view([-1, 2]) + xy = x[i, :] + y[i, :] * width + xy = torch._cast_Long(xy) + temp = flow_[xy] + out.append(temp.view([1, max_matches, 2])) + return torch.cat(out, 0) + + def forward(self, x_tensor): + x_batch_size = x_tensor.size()[0] + x = x_tensor[:, 12:13, :, :] + + # summary 1, dismiss now + x_tensor = self.resnet50(x_tensor) + x_tensor = torch.mean(x_tensor, dim=[2, 3]) + theta = self.regressor(x_tensor) + + pts1, pts2 = self.get_4_pts(theta, x_batch_size) + + h_trans, black_pix, flow, x_map, y_map = transformer(x, pts2) # NCHW NHWC NHWC + + return h_trans.permute([0, 2, 3, 1]), black_pix, x_map, y_map diff --git a/models/StabNet/pytorch_resnet_v2_50.py b/models/StabNet/pytorch_resnet_v2_50.py new file mode 100644 index 0000000..6ef93c6 --- /dev/null +++ b/models/StabNet/pytorch_resnet_v2_50.py @@ -0,0 +1,341 @@ +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +import math + +__weights_dict = dict() + +def load_weights(weight_file): + if weight_file == None: + return + + try: + weights_dict = np.load(weight_file, allow_pickle=True).item() + except: + weights_dict = np.load(weight_file, encoding='bytes').item() + + return weights_dict + +class KitModel(nn.Module): + + # def __init__(self, weight_file): + def __init__(self): + super(KitModel, self).__init__() + # global __weights_dict + # __weights_dict = load_weights(weight_file) + + self.resnet_v2_50_conv1_Conv2D = self.__conv(2, name='resnet_v2_50/conv1/Conv2D', in_channels=3, out_channels=64, kernel_size=(7, 7), stride=(2, 2), groups=1, bias=True) + self.resnet_v2_50_block1_unit_1_bottleneck_v2_preact_FusedBatchNorm = self.__batch_normalization(2, 'resnet_v2_50/block1/unit_1/bottleneck_v2/preact/FusedBatchNorm', num_features=64, eps=1e-5, momentum=0.003) + self.resnet_v2_50_block1_unit_1_bottleneck_v2_shortcut_Conv2D = self.__conv(2, name='resnet_v2_50/block1/unit_1/bottleneck_v2/shortcut/Conv2D', in_channels=64, out_channels=256, kernel_size=(1, 1), stride=(1, 1), groups=1, bias=True) + self.resnet_v2_50_block1_unit_1_bottleneck_v2_conv1_Conv2D = self.__conv(2, name='resnet_v2_50/block1/unit_1/bottleneck_v2/conv1/Conv2D', in_channels=64, out_channels=64, kernel_size=(1, 1), stride=(1, 1), groups=1, bias=None) + self.resnet_v2_50_block1_unit_1_bottleneck_v2_conv1_BatchNorm_FusedBatchNorm = self.__batch_normalization(2, 'resnet_v2_50/block1/unit_1/bottleneck_v2/conv1/BatchNorm/FusedBatchNorm', num_features=64, eps=1e-5, momentum=0.003) + self.resnet_v2_50_block1_unit_1_bottleneck_v2_conv2_Conv2D = self.__conv(2, name='resnet_v2_50/block1/unit_1/bottleneck_v2/conv2/Conv2D', in_channels=64, out_channels=64, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=None) + self.resnet_v2_50_block1_unit_1_bottleneck_v2_conv2_BatchNorm_FusedBatchNorm = self.__batch_normalization(2, 'resnet_v2_50/block1/unit_1/bottleneck_v2/conv2/BatchNorm/FusedBatchNorm', num_features=64, eps=1e-5, momentum=0.003) + self.resnet_v2_50_block1_unit_1_bottleneck_v2_conv3_Conv2D = self.__conv(2, name='resnet_v2_50/block1/unit_1/bottleneck_v2/conv3/Conv2D', in_channels=64, out_channels=256, kernel_size=(1, 1), stride=(1, 1), groups=1, bias=True) + self.resnet_v2_50_block1_unit_2_bottleneck_v2_preact_FusedBatchNorm = self.__batch_normalization(2, 'resnet_v2_50/block1/unit_2/bottleneck_v2/preact/FusedBatchNorm', num_features=256, eps=1e-5, momentum=0.003) + self.resnet_v2_50_block1_unit_2_bottleneck_v2_conv1_Conv2D = self.__conv(2, name='resnet_v2_50/block1/unit_2/bottleneck_v2/conv1/Conv2D', in_channels=256, out_channels=64, kernel_size=(1, 1), stride=(1, 1), groups=1, bias=None) + self.resnet_v2_50_block1_unit_2_bottleneck_v2_conv1_BatchNorm_FusedBatchNorm = self.__batch_normalization(2, 'resnet_v2_50/block1/unit_2/bottleneck_v2/conv1/BatchNorm/FusedBatchNorm', num_features=64, eps=1e-5, momentum=0.003) + self.resnet_v2_50_block1_unit_2_bottleneck_v2_conv2_Conv2D = self.__conv(2, name='resnet_v2_50/block1/unit_2/bottleneck_v2/conv2/Conv2D', in_channels=64, out_channels=64, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=None) + self.resnet_v2_50_block1_unit_2_bottleneck_v2_conv2_BatchNorm_FusedBatchNorm = self.__batch_normalization(2, 'resnet_v2_50/block1/unit_2/bottleneck_v2/conv2/BatchNorm/FusedBatchNorm', num_features=64, eps=1e-5, momentum=0.003) + self.resnet_v2_50_block1_unit_2_bottleneck_v2_conv3_Conv2D = self.__conv(2, name='resnet_v2_50/block1/unit_2/bottleneck_v2/conv3/Conv2D', in_channels=64, out_channels=256, kernel_size=(1, 1), stride=(1, 1), groups=1, bias=True) + self.resnet_v2_50_block1_unit_3_bottleneck_v2_preact_FusedBatchNorm = self.__batch_normalization(2, 'resnet_v2_50/block1/unit_3/bottleneck_v2/preact/FusedBatchNorm', num_features=256, eps=1e-5, momentum=0.003) + self.resnet_v2_50_block1_unit_3_bottleneck_v2_conv1_Conv2D = self.__conv(2, name='resnet_v2_50/block1/unit_3/bottleneck_v2/conv1/Conv2D', in_channels=256, out_channels=64, kernel_size=(1, 1), stride=(1, 1), groups=1, bias=None) + self.resnet_v2_50_block1_unit_3_bottleneck_v2_conv1_BatchNorm_FusedBatchNorm = self.__batch_normalization(2, 'resnet_v2_50/block1/unit_3/bottleneck_v2/conv1/BatchNorm/FusedBatchNorm', num_features=64, eps=1e-5, momentum=0.003) + self.resnet_v2_50_block1_unit_3_bottleneck_v2_conv2_Conv2D = self.__conv(2, name='resnet_v2_50/block1/unit_3/bottleneck_v2/conv2/Conv2D', in_channels=64, out_channels=64, kernel_size=(3, 3), stride=(2, 2), groups=1, bias=None) + self.resnet_v2_50_block1_unit_3_bottleneck_v2_conv2_BatchNorm_FusedBatchNorm = self.__batch_normalization(2, 'resnet_v2_50/block1/unit_3/bottleneck_v2/conv2/BatchNorm/FusedBatchNorm', num_features=64, eps=1e-5, momentum=0.003) + self.resnet_v2_50_block1_unit_3_bottleneck_v2_conv3_Conv2D = self.__conv(2, name='resnet_v2_50/block1/unit_3/bottleneck_v2/conv3/Conv2D', in_channels=64, out_channels=256, kernel_size=(1, 1), stride=(1, 1), groups=1, bias=True) + self.resnet_v2_50_block2_unit_1_bottleneck_v2_preact_FusedBatchNorm = self.__batch_normalization(2, 'resnet_v2_50/block2/unit_1/bottleneck_v2/preact/FusedBatchNorm', num_features=256, eps=1e-5, momentum=0.003) + self.resnet_v2_50_block2_unit_1_bottleneck_v2_shortcut_Conv2D = self.__conv(2, name='resnet_v2_50/block2/unit_1/bottleneck_v2/shortcut/Conv2D', in_channels=256, out_channels=512, kernel_size=(1, 1), stride=(1, 1), groups=1, bias=True) + self.resnet_v2_50_block2_unit_1_bottleneck_v2_conv1_Conv2D = self.__conv(2, name='resnet_v2_50/block2/unit_1/bottleneck_v2/conv1/Conv2D', in_channels=256, out_channels=128, kernel_size=(1, 1), stride=(1, 1), groups=1, bias=None) + self.resnet_v2_50_block2_unit_1_bottleneck_v2_conv1_BatchNorm_FusedBatchNorm = self.__batch_normalization(2, 'resnet_v2_50/block2/unit_1/bottleneck_v2/conv1/BatchNorm/FusedBatchNorm', num_features=128, eps=1e-5, momentum=0.003) + self.resnet_v2_50_block2_unit_1_bottleneck_v2_conv2_Conv2D = self.__conv(2, name='resnet_v2_50/block2/unit_1/bottleneck_v2/conv2/Conv2D', in_channels=128, out_channels=128, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=None) + self.resnet_v2_50_block2_unit_1_bottleneck_v2_conv2_BatchNorm_FusedBatchNorm = self.__batch_normalization(2, 'resnet_v2_50/block2/unit_1/bottleneck_v2/conv2/BatchNorm/FusedBatchNorm', num_features=128, eps=1e-5, momentum=0.003) + self.resnet_v2_50_block2_unit_1_bottleneck_v2_conv3_Conv2D = self.__conv(2, name='resnet_v2_50/block2/unit_1/bottleneck_v2/conv3/Conv2D', in_channels=128, out_channels=512, kernel_size=(1, 1), stride=(1, 1), groups=1, bias=True) + self.resnet_v2_50_block2_unit_2_bottleneck_v2_preact_FusedBatchNorm = self.__batch_normalization(2, 'resnet_v2_50/block2/unit_2/bottleneck_v2/preact/FusedBatchNorm', num_features=512, eps=1e-5, momentum=0.003) + self.resnet_v2_50_block2_unit_2_bottleneck_v2_conv1_Conv2D = self.__conv(2, name='resnet_v2_50/block2/unit_2/bottleneck_v2/conv1/Conv2D', in_channels=512, out_channels=128, kernel_size=(1, 1), stride=(1, 1), groups=1, bias=None) + self.resnet_v2_50_block2_unit_2_bottleneck_v2_conv1_BatchNorm_FusedBatchNorm = self.__batch_normalization(2, 'resnet_v2_50/block2/unit_2/bottleneck_v2/conv1/BatchNorm/FusedBatchNorm', num_features=128, eps=1e-5, momentum=0.003) + self.resnet_v2_50_block2_unit_2_bottleneck_v2_conv2_Conv2D = self.__conv(2, name='resnet_v2_50/block2/unit_2/bottleneck_v2/conv2/Conv2D', in_channels=128, out_channels=128, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=None) + self.resnet_v2_50_block2_unit_2_bottleneck_v2_conv2_BatchNorm_FusedBatchNorm = self.__batch_normalization(2, 'resnet_v2_50/block2/unit_2/bottleneck_v2/conv2/BatchNorm/FusedBatchNorm', num_features=128, eps=1e-5, momentum=0.003) + self.resnet_v2_50_block2_unit_2_bottleneck_v2_conv3_Conv2D = self.__conv(2, name='resnet_v2_50/block2/unit_2/bottleneck_v2/conv3/Conv2D', in_channels=128, out_channels=512, kernel_size=(1, 1), stride=(1, 1), groups=1, bias=True) + self.resnet_v2_50_block2_unit_3_bottleneck_v2_preact_FusedBatchNorm = self.__batch_normalization(2, 'resnet_v2_50/block2/unit_3/bottleneck_v2/preact/FusedBatchNorm', num_features=512, eps=1e-5, momentum=0.003) + self.resnet_v2_50_block2_unit_3_bottleneck_v2_conv1_Conv2D = self.__conv(2, name='resnet_v2_50/block2/unit_3/bottleneck_v2/conv1/Conv2D', in_channels=512, out_channels=128, kernel_size=(1, 1), stride=(1, 1), groups=1, bias=None) + self.resnet_v2_50_block2_unit_3_bottleneck_v2_conv1_BatchNorm_FusedBatchNorm = self.__batch_normalization(2, 'resnet_v2_50/block2/unit_3/bottleneck_v2/conv1/BatchNorm/FusedBatchNorm', num_features=128, eps=1e-5, momentum=0.003) + self.resnet_v2_50_block2_unit_3_bottleneck_v2_conv2_Conv2D = self.__conv(2, name='resnet_v2_50/block2/unit_3/bottleneck_v2/conv2/Conv2D', in_channels=128, out_channels=128, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=None) + self.resnet_v2_50_block2_unit_3_bottleneck_v2_conv2_BatchNorm_FusedBatchNorm = self.__batch_normalization(2, 'resnet_v2_50/block2/unit_3/bottleneck_v2/conv2/BatchNorm/FusedBatchNorm', num_features=128, eps=1e-5, momentum=0.003) + self.resnet_v2_50_block2_unit_3_bottleneck_v2_conv3_Conv2D = self.__conv(2, name='resnet_v2_50/block2/unit_3/bottleneck_v2/conv3/Conv2D', in_channels=128, out_channels=512, kernel_size=(1, 1), stride=(1, 1), groups=1, bias=True) + self.resnet_v2_50_block2_unit_4_bottleneck_v2_preact_FusedBatchNorm = self.__batch_normalization(2, 'resnet_v2_50/block2/unit_4/bottleneck_v2/preact/FusedBatchNorm', num_features=512, eps=1e-5, momentum=0.003) + self.resnet_v2_50_block2_unit_4_bottleneck_v2_conv1_Conv2D = self.__conv(2, name='resnet_v2_50/block2/unit_4/bottleneck_v2/conv1/Conv2D', in_channels=512, out_channels=128, kernel_size=(1, 1), stride=(1, 1), groups=1, bias=None) + self.resnet_v2_50_block2_unit_4_bottleneck_v2_conv1_BatchNorm_FusedBatchNorm = self.__batch_normalization(2, 'resnet_v2_50/block2/unit_4/bottleneck_v2/conv1/BatchNorm/FusedBatchNorm', num_features=128, eps=1e-5, momentum=0.003) + self.resnet_v2_50_block2_unit_4_bottleneck_v2_conv2_Conv2D = self.__conv(2, name='resnet_v2_50/block2/unit_4/bottleneck_v2/conv2/Conv2D', in_channels=128, out_channels=128, kernel_size=(3, 3), stride=(2, 2), groups=1, bias=None) + self.resnet_v2_50_block2_unit_4_bottleneck_v2_conv2_BatchNorm_FusedBatchNorm = self.__batch_normalization(2, 'resnet_v2_50/block2/unit_4/bottleneck_v2/conv2/BatchNorm/FusedBatchNorm', num_features=128, eps=1e-5, momentum=0.003) + self.resnet_v2_50_block2_unit_4_bottleneck_v2_conv3_Conv2D = self.__conv(2, name='resnet_v2_50/block2/unit_4/bottleneck_v2/conv3/Conv2D', in_channels=128, out_channels=512, kernel_size=(1, 1), stride=(1, 1), groups=1, bias=True) + self.resnet_v2_50_block3_unit_1_bottleneck_v2_preact_FusedBatchNorm = self.__batch_normalization(2, 'resnet_v2_50/block3/unit_1/bottleneck_v2/preact/FusedBatchNorm', num_features=512, eps=1e-5, momentum=0.003) + self.resnet_v2_50_block3_unit_1_bottleneck_v2_shortcut_Conv2D = self.__conv(2, name='resnet_v2_50/block3/unit_1/bottleneck_v2/shortcut/Conv2D', in_channels=512, out_channels=1024, kernel_size=(1, 1), stride=(1, 1), groups=1, bias=True) + self.resnet_v2_50_block3_unit_1_bottleneck_v2_conv1_Conv2D = self.__conv(2, name='resnet_v2_50/block3/unit_1/bottleneck_v2/conv1/Conv2D', in_channels=512, out_channels=256, kernel_size=(1, 1), stride=(1, 1), groups=1, bias=None) + self.resnet_v2_50_block3_unit_1_bottleneck_v2_conv1_BatchNorm_FusedBatchNorm = self.__batch_normalization(2, 'resnet_v2_50/block3/unit_1/bottleneck_v2/conv1/BatchNorm/FusedBatchNorm', num_features=256, eps=1e-5, momentum=0.003) + self.resnet_v2_50_block3_unit_1_bottleneck_v2_conv2_Conv2D = self.__conv(2, name='resnet_v2_50/block3/unit_1/bottleneck_v2/conv2/Conv2D', in_channels=256, out_channels=256, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=None) + self.resnet_v2_50_block3_unit_1_bottleneck_v2_conv2_BatchNorm_FusedBatchNorm = self.__batch_normalization(2, 'resnet_v2_50/block3/unit_1/bottleneck_v2/conv2/BatchNorm/FusedBatchNorm', num_features=256, eps=1e-5, momentum=0.003) + self.resnet_v2_50_block3_unit_1_bottleneck_v2_conv3_Conv2D = self.__conv(2, name='resnet_v2_50/block3/unit_1/bottleneck_v2/conv3/Conv2D', in_channels=256, out_channels=1024, kernel_size=(1, 1), stride=(1, 1), groups=1, bias=True) + self.resnet_v2_50_block3_unit_2_bottleneck_v2_preact_FusedBatchNorm = self.__batch_normalization(2, 'resnet_v2_50/block3/unit_2/bottleneck_v2/preact/FusedBatchNorm', num_features=1024, eps=1e-5, momentum=0.003) + self.resnet_v2_50_block3_unit_2_bottleneck_v2_conv1_Conv2D = self.__conv(2, name='resnet_v2_50/block3/unit_2/bottleneck_v2/conv1/Conv2D', in_channels=1024, out_channels=256, kernel_size=(1, 1), stride=(1, 1), groups=1, bias=None) + self.resnet_v2_50_block3_unit_2_bottleneck_v2_conv1_BatchNorm_FusedBatchNorm = self.__batch_normalization(2, 'resnet_v2_50/block3/unit_2/bottleneck_v2/conv1/BatchNorm/FusedBatchNorm', num_features=256, eps=1e-5, momentum=0.003) + self.resnet_v2_50_block3_unit_2_bottleneck_v2_conv2_Conv2D = self.__conv(2, name='resnet_v2_50/block3/unit_2/bottleneck_v2/conv2/Conv2D', in_channels=256, out_channels=256, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=None) + self.resnet_v2_50_block3_unit_2_bottleneck_v2_conv2_BatchNorm_FusedBatchNorm = self.__batch_normalization(2, 'resnet_v2_50/block3/unit_2/bottleneck_v2/conv2/BatchNorm/FusedBatchNorm', num_features=256, eps=1e-5, momentum=0.003) + self.resnet_v2_50_block3_unit_2_bottleneck_v2_conv3_Conv2D = self.__conv(2, name='resnet_v2_50/block3/unit_2/bottleneck_v2/conv3/Conv2D', in_channels=256, out_channels=1024, kernel_size=(1, 1), stride=(1, 1), groups=1, bias=True) + self.resnet_v2_50_block3_unit_3_bottleneck_v2_preact_FusedBatchNorm = self.__batch_normalization(2, 'resnet_v2_50/block3/unit_3/bottleneck_v2/preact/FusedBatchNorm', num_features=1024, eps=1e-5, momentum=0.003) + self.resnet_v2_50_block3_unit_3_bottleneck_v2_conv1_Conv2D = self.__conv(2, name='resnet_v2_50/block3/unit_3/bottleneck_v2/conv1/Conv2D', in_channels=1024, out_channels=256, kernel_size=(1, 1), stride=(1, 1), groups=1, bias=None) + self.resnet_v2_50_block3_unit_3_bottleneck_v2_conv1_BatchNorm_FusedBatchNorm = self.__batch_normalization(2, 'resnet_v2_50/block3/unit_3/bottleneck_v2/conv1/BatchNorm/FusedBatchNorm', num_features=256, eps=1e-5, momentum=0.003) + self.resnet_v2_50_block3_unit_3_bottleneck_v2_conv2_Conv2D = self.__conv(2, name='resnet_v2_50/block3/unit_3/bottleneck_v2/conv2/Conv2D', in_channels=256, out_channels=256, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=None) + self.resnet_v2_50_block3_unit_3_bottleneck_v2_conv2_BatchNorm_FusedBatchNorm = self.__batch_normalization(2, 'resnet_v2_50/block3/unit_3/bottleneck_v2/conv2/BatchNorm/FusedBatchNorm', num_features=256, eps=1e-5, momentum=0.003) + self.resnet_v2_50_block3_unit_3_bottleneck_v2_conv3_Conv2D = self.__conv(2, name='resnet_v2_50/block3/unit_3/bottleneck_v2/conv3/Conv2D', in_channels=256, out_channels=1024, kernel_size=(1, 1), stride=(1, 1), groups=1, bias=True) + self.resnet_v2_50_block3_unit_4_bottleneck_v2_preact_FusedBatchNorm = self.__batch_normalization(2, 'resnet_v2_50/block3/unit_4/bottleneck_v2/preact/FusedBatchNorm', num_features=1024, eps=1e-5, momentum=0.003) + self.resnet_v2_50_block3_unit_4_bottleneck_v2_conv1_Conv2D = self.__conv(2, name='resnet_v2_50/block3/unit_4/bottleneck_v2/conv1/Conv2D', in_channels=1024, out_channels=256, kernel_size=(1, 1), stride=(1, 1), groups=1, bias=None) + self.resnet_v2_50_block3_unit_4_bottleneck_v2_conv1_BatchNorm_FusedBatchNorm = self.__batch_normalization(2, 'resnet_v2_50/block3/unit_4/bottleneck_v2/conv1/BatchNorm/FusedBatchNorm', num_features=256, eps=1e-5, momentum=0.003) + self.resnet_v2_50_block3_unit_4_bottleneck_v2_conv2_Conv2D = self.__conv(2, name='resnet_v2_50/block3/unit_4/bottleneck_v2/conv2/Conv2D', in_channels=256, out_channels=256, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=None) + self.resnet_v2_50_block3_unit_4_bottleneck_v2_conv2_BatchNorm_FusedBatchNorm = self.__batch_normalization(2, 'resnet_v2_50/block3/unit_4/bottleneck_v2/conv2/BatchNorm/FusedBatchNorm', num_features=256, eps=1e-5, momentum=0.003) + self.resnet_v2_50_block3_unit_4_bottleneck_v2_conv3_Conv2D = self.__conv(2, name='resnet_v2_50/block3/unit_4/bottleneck_v2/conv3/Conv2D', in_channels=256, out_channels=1024, kernel_size=(1, 1), stride=(1, 1), groups=1, bias=True) + self.resnet_v2_50_block3_unit_5_bottleneck_v2_preact_FusedBatchNorm = self.__batch_normalization(2, 'resnet_v2_50/block3/unit_5/bottleneck_v2/preact/FusedBatchNorm', num_features=1024, eps=1e-5, momentum=0.003) + self.resnet_v2_50_block3_unit_5_bottleneck_v2_conv1_Conv2D = self.__conv(2, name='resnet_v2_50/block3/unit_5/bottleneck_v2/conv1/Conv2D', in_channels=1024, out_channels=256, kernel_size=(1, 1), stride=(1, 1), groups=1, bias=None) + self.resnet_v2_50_block3_unit_5_bottleneck_v2_conv1_BatchNorm_FusedBatchNorm = self.__batch_normalization(2, 'resnet_v2_50/block3/unit_5/bottleneck_v2/conv1/BatchNorm/FusedBatchNorm', num_features=256, eps=1e-5, momentum=0.003) + self.resnet_v2_50_block3_unit_5_bottleneck_v2_conv2_Conv2D = self.__conv(2, name='resnet_v2_50/block3/unit_5/bottleneck_v2/conv2/Conv2D', in_channels=256, out_channels=256, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=None) + self.resnet_v2_50_block3_unit_5_bottleneck_v2_conv2_BatchNorm_FusedBatchNorm = self.__batch_normalization(2, 'resnet_v2_50/block3/unit_5/bottleneck_v2/conv2/BatchNorm/FusedBatchNorm', num_features=256, eps=1e-5, momentum=0.003) + self.resnet_v2_50_block3_unit_5_bottleneck_v2_conv3_Conv2D = self.__conv(2, name='resnet_v2_50/block3/unit_5/bottleneck_v2/conv3/Conv2D', in_channels=256, out_channels=1024, kernel_size=(1, 1), stride=(1, 1), groups=1, bias=True) + self.resnet_v2_50_block3_unit_6_bottleneck_v2_preact_FusedBatchNorm = self.__batch_normalization(2, 'resnet_v2_50/block3/unit_6/bottleneck_v2/preact/FusedBatchNorm', num_features=1024, eps=1e-5, momentum=0.003) + self.resnet_v2_50_block3_unit_6_bottleneck_v2_conv1_Conv2D = self.__conv(2, name='resnet_v2_50/block3/unit_6/bottleneck_v2/conv1/Conv2D', in_channels=1024, out_channels=256, kernel_size=(1, 1), stride=(1, 1), groups=1, bias=None) + self.resnet_v2_50_block3_unit_6_bottleneck_v2_conv1_BatchNorm_FusedBatchNorm = self.__batch_normalization(2, 'resnet_v2_50/block3/unit_6/bottleneck_v2/conv1/BatchNorm/FusedBatchNorm', num_features=256, eps=1e-5, momentum=0.003) + self.resnet_v2_50_block3_unit_6_bottleneck_v2_conv2_Conv2D = self.__conv(2, name='resnet_v2_50/block3/unit_6/bottleneck_v2/conv2/Conv2D', in_channels=256, out_channels=256, kernel_size=(3, 3), stride=(2, 2), groups=1, bias=None) + self.resnet_v2_50_block3_unit_6_bottleneck_v2_conv2_BatchNorm_FusedBatchNorm = self.__batch_normalization(2, 'resnet_v2_50/block3/unit_6/bottleneck_v2/conv2/BatchNorm/FusedBatchNorm', num_features=256, eps=1e-5, momentum=0.003) + self.resnet_v2_50_block3_unit_6_bottleneck_v2_conv3_Conv2D = self.__conv(2, name='resnet_v2_50/block3/unit_6/bottleneck_v2/conv3/Conv2D', in_channels=256, out_channels=1024, kernel_size=(1, 1), stride=(1, 1), groups=1, bias=True) + self.resnet_v2_50_block4_unit_1_bottleneck_v2_preact_FusedBatchNorm = self.__batch_normalization(2, 'resnet_v2_50/block4/unit_1/bottleneck_v2/preact/FusedBatchNorm', num_features=1024, eps=1e-5, momentum=0.003) + self.resnet_v2_50_block4_unit_1_bottleneck_v2_shortcut_Conv2D = self.__conv(2, name='resnet_v2_50/block4/unit_1/bottleneck_v2/shortcut/Conv2D', in_channels=1024, out_channels=2048, kernel_size=(1, 1), stride=(1, 1), groups=1, bias=True) + self.resnet_v2_50_block4_unit_1_bottleneck_v2_conv1_Conv2D = self.__conv(2, name='resnet_v2_50/block4/unit_1/bottleneck_v2/conv1/Conv2D', in_channels=1024, out_channels=512, kernel_size=(1, 1), stride=(1, 1), groups=1, bias=None) + self.resnet_v2_50_block4_unit_1_bottleneck_v2_conv1_BatchNorm_FusedBatchNorm = self.__batch_normalization(2, 'resnet_v2_50/block4/unit_1/bottleneck_v2/conv1/BatchNorm/FusedBatchNorm', num_features=512, eps=1e-5, momentum=0.003) + self.resnet_v2_50_block4_unit_1_bottleneck_v2_conv2_Conv2D = self.__conv(2, name='resnet_v2_50/block4/unit_1/bottleneck_v2/conv2/Conv2D', in_channels=512, out_channels=512, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=None) + self.resnet_v2_50_block4_unit_1_bottleneck_v2_conv2_BatchNorm_FusedBatchNorm = self.__batch_normalization(2, 'resnet_v2_50/block4/unit_1/bottleneck_v2/conv2/BatchNorm/FusedBatchNorm', num_features=512, eps=1e-5, momentum=0.003) + self.resnet_v2_50_block4_unit_1_bottleneck_v2_conv3_Conv2D = self.__conv(2, name='resnet_v2_50/block4/unit_1/bottleneck_v2/conv3/Conv2D', in_channels=512, out_channels=2048, kernel_size=(1, 1), stride=(1, 1), groups=1, bias=True) + self.resnet_v2_50_block4_unit_2_bottleneck_v2_preact_FusedBatchNorm = self.__batch_normalization(2, 'resnet_v2_50/block4/unit_2/bottleneck_v2/preact/FusedBatchNorm', num_features=2048, eps=1e-5, momentum=0.003) + self.resnet_v2_50_block4_unit_2_bottleneck_v2_conv1_Conv2D = self.__conv(2, name='resnet_v2_50/block4/unit_2/bottleneck_v2/conv1/Conv2D', in_channels=2048, out_channels=512, kernel_size=(1, 1), stride=(1, 1), groups=1, bias=None) + self.resnet_v2_50_block4_unit_2_bottleneck_v2_conv1_BatchNorm_FusedBatchNorm = self.__batch_normalization(2, 'resnet_v2_50/block4/unit_2/bottleneck_v2/conv1/BatchNorm/FusedBatchNorm', num_features=512, eps=1e-5, momentum=0.003) + self.resnet_v2_50_block4_unit_2_bottleneck_v2_conv2_Conv2D = self.__conv(2, name='resnet_v2_50/block4/unit_2/bottleneck_v2/conv2/Conv2D', in_channels=512, out_channels=512, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=None) + self.resnet_v2_50_block4_unit_2_bottleneck_v2_conv2_BatchNorm_FusedBatchNorm = self.__batch_normalization(2, 'resnet_v2_50/block4/unit_2/bottleneck_v2/conv2/BatchNorm/FusedBatchNorm', num_features=512, eps=1e-5, momentum=0.003) + self.resnet_v2_50_block4_unit_2_bottleneck_v2_conv3_Conv2D = self.__conv(2, name='resnet_v2_50/block4/unit_2/bottleneck_v2/conv3/Conv2D', in_channels=512, out_channels=2048, kernel_size=(1, 1), stride=(1, 1), groups=1, bias=True) + self.resnet_v2_50_block4_unit_3_bottleneck_v2_preact_FusedBatchNorm = self.__batch_normalization(2, 'resnet_v2_50/block4/unit_3/bottleneck_v2/preact/FusedBatchNorm', num_features=2048, eps=1e-5, momentum=0.003) + self.resnet_v2_50_block4_unit_3_bottleneck_v2_conv1_Conv2D = self.__conv(2, name='resnet_v2_50/block4/unit_3/bottleneck_v2/conv1/Conv2D', in_channels=2048, out_channels=512, kernel_size=(1, 1), stride=(1, 1), groups=1, bias=None) + self.resnet_v2_50_block4_unit_3_bottleneck_v2_conv1_BatchNorm_FusedBatchNorm = self.__batch_normalization(2, 'resnet_v2_50/block4/unit_3/bottleneck_v2/conv1/BatchNorm/FusedBatchNorm', num_features=512, eps=1e-5, momentum=0.003) + self.resnet_v2_50_block4_unit_3_bottleneck_v2_conv2_Conv2D = self.__conv(2, name='resnet_v2_50/block4/unit_3/bottleneck_v2/conv2/Conv2D', in_channels=512, out_channels=512, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=None) + self.resnet_v2_50_block4_unit_3_bottleneck_v2_conv2_BatchNorm_FusedBatchNorm = self.__batch_normalization(2, 'resnet_v2_50/block4/unit_3/bottleneck_v2/conv2/BatchNorm/FusedBatchNorm', num_features=512, eps=1e-5, momentum=0.003) + self.resnet_v2_50_block4_unit_3_bottleneck_v2_conv3_Conv2D = self.__conv(2, name='resnet_v2_50/block4/unit_3/bottleneck_v2/conv3/Conv2D', in_channels=512, out_channels=2048, kernel_size=(1, 1), stride=(1, 1), groups=1, bias=True) + self.resnet_v2_50_postnorm_FusedBatchNorm = self.__batch_normalization(2, 'resnet_v2_50/postnorm/FusedBatchNorm', num_features=2048, eps=1e-5, momentum=0.003) + self.resnet_v2_50_logits_Conv2D = self.__conv(2, name='resnet_v2_50/logits/Conv2D', in_channels=2048, out_channels=1001, kernel_size=(1, 1), stride=(1, 1), groups=1, bias=True) + + def forward(self, x): + resnet_v2_50_Pad = F.pad(x, (3, 3, 3, 3), mode = 'constant', value = 0) + resnet_v2_50_conv1_Conv2D = self.resnet_v2_50_conv1_Conv2D(resnet_v2_50_Pad) + resnet_v2_50_pool1_MaxPool_pad = F.pad(resnet_v2_50_conv1_Conv2D, (0, 1, 0, 1), value=float('-inf')) + resnet_v2_50_pool1_MaxPool = F.max_pool2d(resnet_v2_50_pool1_MaxPool_pad, kernel_size=(3, 3), stride=(2, 2), padding=0, ceil_mode=False) + resnet_v2_50_block1_unit_1_bottleneck_v2_preact_FusedBatchNorm = self.resnet_v2_50_block1_unit_1_bottleneck_v2_preact_FusedBatchNorm(resnet_v2_50_pool1_MaxPool) + resnet_v2_50_block1_unit_1_bottleneck_v2_preact_Relu = F.relu(resnet_v2_50_block1_unit_1_bottleneck_v2_preact_FusedBatchNorm) + resnet_v2_50_block1_unit_1_bottleneck_v2_shortcut_Conv2D = self.resnet_v2_50_block1_unit_1_bottleneck_v2_shortcut_Conv2D(resnet_v2_50_block1_unit_1_bottleneck_v2_preact_Relu) + resnet_v2_50_block1_unit_1_bottleneck_v2_conv1_Conv2D = self.resnet_v2_50_block1_unit_1_bottleneck_v2_conv1_Conv2D(resnet_v2_50_block1_unit_1_bottleneck_v2_preact_Relu) + resnet_v2_50_block1_unit_1_bottleneck_v2_conv1_BatchNorm_FusedBatchNorm = self.resnet_v2_50_block1_unit_1_bottleneck_v2_conv1_BatchNorm_FusedBatchNorm(resnet_v2_50_block1_unit_1_bottleneck_v2_conv1_Conv2D) + resnet_v2_50_block1_unit_1_bottleneck_v2_conv1_Relu = F.relu(resnet_v2_50_block1_unit_1_bottleneck_v2_conv1_BatchNorm_FusedBatchNorm) + resnet_v2_50_block1_unit_1_bottleneck_v2_conv2_Conv2D_pad = F.pad(resnet_v2_50_block1_unit_1_bottleneck_v2_conv1_Relu, (1, 1, 1, 1)) + resnet_v2_50_block1_unit_1_bottleneck_v2_conv2_Conv2D = self.resnet_v2_50_block1_unit_1_bottleneck_v2_conv2_Conv2D(resnet_v2_50_block1_unit_1_bottleneck_v2_conv2_Conv2D_pad) + resnet_v2_50_block1_unit_1_bottleneck_v2_conv2_BatchNorm_FusedBatchNorm = self.resnet_v2_50_block1_unit_1_bottleneck_v2_conv2_BatchNorm_FusedBatchNorm(resnet_v2_50_block1_unit_1_bottleneck_v2_conv2_Conv2D) + resnet_v2_50_block1_unit_1_bottleneck_v2_conv2_Relu = F.relu(resnet_v2_50_block1_unit_1_bottleneck_v2_conv2_BatchNorm_FusedBatchNorm) + resnet_v2_50_block1_unit_1_bottleneck_v2_conv3_Conv2D = self.resnet_v2_50_block1_unit_1_bottleneck_v2_conv3_Conv2D(resnet_v2_50_block1_unit_1_bottleneck_v2_conv2_Relu) + resnet_v2_50_block1_unit_1_bottleneck_v2_add = resnet_v2_50_block1_unit_1_bottleneck_v2_shortcut_Conv2D + resnet_v2_50_block1_unit_1_bottleneck_v2_conv3_Conv2D + resnet_v2_50_block1_unit_2_bottleneck_v2_preact_FusedBatchNorm = self.resnet_v2_50_block1_unit_2_bottleneck_v2_preact_FusedBatchNorm(resnet_v2_50_block1_unit_1_bottleneck_v2_add) + resnet_v2_50_block1_unit_2_bottleneck_v2_preact_Relu = F.relu(resnet_v2_50_block1_unit_2_bottleneck_v2_preact_FusedBatchNorm) + resnet_v2_50_block1_unit_2_bottleneck_v2_conv1_Conv2D = self.resnet_v2_50_block1_unit_2_bottleneck_v2_conv1_Conv2D(resnet_v2_50_block1_unit_2_bottleneck_v2_preact_Relu) + resnet_v2_50_block1_unit_2_bottleneck_v2_conv1_BatchNorm_FusedBatchNorm = self.resnet_v2_50_block1_unit_2_bottleneck_v2_conv1_BatchNorm_FusedBatchNorm(resnet_v2_50_block1_unit_2_bottleneck_v2_conv1_Conv2D) + resnet_v2_50_block1_unit_2_bottleneck_v2_conv1_Relu = F.relu(resnet_v2_50_block1_unit_2_bottleneck_v2_conv1_BatchNorm_FusedBatchNorm) + resnet_v2_50_block1_unit_2_bottleneck_v2_conv2_Conv2D_pad = F.pad(resnet_v2_50_block1_unit_2_bottleneck_v2_conv1_Relu, (1, 1, 1, 1)) + resnet_v2_50_block1_unit_2_bottleneck_v2_conv2_Conv2D = self.resnet_v2_50_block1_unit_2_bottleneck_v2_conv2_Conv2D(resnet_v2_50_block1_unit_2_bottleneck_v2_conv2_Conv2D_pad) + resnet_v2_50_block1_unit_2_bottleneck_v2_conv2_BatchNorm_FusedBatchNorm = self.resnet_v2_50_block1_unit_2_bottleneck_v2_conv2_BatchNorm_FusedBatchNorm(resnet_v2_50_block1_unit_2_bottleneck_v2_conv2_Conv2D) + resnet_v2_50_block1_unit_2_bottleneck_v2_conv2_Relu = F.relu(resnet_v2_50_block1_unit_2_bottleneck_v2_conv2_BatchNorm_FusedBatchNorm) + resnet_v2_50_block1_unit_2_bottleneck_v2_conv3_Conv2D = self.resnet_v2_50_block1_unit_2_bottleneck_v2_conv3_Conv2D(resnet_v2_50_block1_unit_2_bottleneck_v2_conv2_Relu) + resnet_v2_50_block1_unit_2_bottleneck_v2_add = resnet_v2_50_block1_unit_1_bottleneck_v2_add + resnet_v2_50_block1_unit_2_bottleneck_v2_conv3_Conv2D + resnet_v2_50_block1_unit_3_bottleneck_v2_preact_FusedBatchNorm = self.resnet_v2_50_block1_unit_3_bottleneck_v2_preact_FusedBatchNorm(resnet_v2_50_block1_unit_2_bottleneck_v2_add) + resnet_v2_50_block1_unit_3_bottleneck_v2_shortcut_MaxPool = F.max_pool2d(resnet_v2_50_block1_unit_2_bottleneck_v2_add, kernel_size=(1, 1), stride=(2, 2), padding=0, ceil_mode=False) + resnet_v2_50_block1_unit_3_bottleneck_v2_preact_Relu = F.relu(resnet_v2_50_block1_unit_3_bottleneck_v2_preact_FusedBatchNorm) + resnet_v2_50_block1_unit_3_bottleneck_v2_conv1_Conv2D = self.resnet_v2_50_block1_unit_3_bottleneck_v2_conv1_Conv2D(resnet_v2_50_block1_unit_3_bottleneck_v2_preact_Relu) + resnet_v2_50_block1_unit_3_bottleneck_v2_conv1_BatchNorm_FusedBatchNorm = self.resnet_v2_50_block1_unit_3_bottleneck_v2_conv1_BatchNorm_FusedBatchNorm(resnet_v2_50_block1_unit_3_bottleneck_v2_conv1_Conv2D) + resnet_v2_50_block1_unit_3_bottleneck_v2_conv1_Relu = F.relu(resnet_v2_50_block1_unit_3_bottleneck_v2_conv1_BatchNorm_FusedBatchNorm) + resnet_v2_50_block1_unit_3_bottleneck_v2_Pad = F.pad(resnet_v2_50_block1_unit_3_bottleneck_v2_conv1_Relu, (1, 1, 1, 1), mode = 'constant', value = 0) + resnet_v2_50_block1_unit_3_bottleneck_v2_conv2_Conv2D = self.resnet_v2_50_block1_unit_3_bottleneck_v2_conv2_Conv2D(resnet_v2_50_block1_unit_3_bottleneck_v2_Pad) + resnet_v2_50_block1_unit_3_bottleneck_v2_conv2_BatchNorm_FusedBatchNorm = self.resnet_v2_50_block1_unit_3_bottleneck_v2_conv2_BatchNorm_FusedBatchNorm(resnet_v2_50_block1_unit_3_bottleneck_v2_conv2_Conv2D) + resnet_v2_50_block1_unit_3_bottleneck_v2_conv2_Relu = F.relu(resnet_v2_50_block1_unit_3_bottleneck_v2_conv2_BatchNorm_FusedBatchNorm) + resnet_v2_50_block1_unit_3_bottleneck_v2_conv3_Conv2D = self.resnet_v2_50_block1_unit_3_bottleneck_v2_conv3_Conv2D(resnet_v2_50_block1_unit_3_bottleneck_v2_conv2_Relu) + resnet_v2_50_block1_unit_3_bottleneck_v2_add = resnet_v2_50_block1_unit_3_bottleneck_v2_shortcut_MaxPool + resnet_v2_50_block1_unit_3_bottleneck_v2_conv3_Conv2D + resnet_v2_50_block2_unit_1_bottleneck_v2_preact_FusedBatchNorm = self.resnet_v2_50_block2_unit_1_bottleneck_v2_preact_FusedBatchNorm(resnet_v2_50_block1_unit_3_bottleneck_v2_add) + resnet_v2_50_block2_unit_1_bottleneck_v2_preact_Relu = F.relu(resnet_v2_50_block2_unit_1_bottleneck_v2_preact_FusedBatchNorm) + resnet_v2_50_block2_unit_1_bottleneck_v2_shortcut_Conv2D = self.resnet_v2_50_block2_unit_1_bottleneck_v2_shortcut_Conv2D(resnet_v2_50_block2_unit_1_bottleneck_v2_preact_Relu) + resnet_v2_50_block2_unit_1_bottleneck_v2_conv1_Conv2D = self.resnet_v2_50_block2_unit_1_bottleneck_v2_conv1_Conv2D(resnet_v2_50_block2_unit_1_bottleneck_v2_preact_Relu) + resnet_v2_50_block2_unit_1_bottleneck_v2_conv1_BatchNorm_FusedBatchNorm = self.resnet_v2_50_block2_unit_1_bottleneck_v2_conv1_BatchNorm_FusedBatchNorm(resnet_v2_50_block2_unit_1_bottleneck_v2_conv1_Conv2D) + resnet_v2_50_block2_unit_1_bottleneck_v2_conv1_Relu = F.relu(resnet_v2_50_block2_unit_1_bottleneck_v2_conv1_BatchNorm_FusedBatchNorm) + resnet_v2_50_block2_unit_1_bottleneck_v2_conv2_Conv2D_pad = F.pad(resnet_v2_50_block2_unit_1_bottleneck_v2_conv1_Relu, (1, 1, 1, 1)) + resnet_v2_50_block2_unit_1_bottleneck_v2_conv2_Conv2D = self.resnet_v2_50_block2_unit_1_bottleneck_v2_conv2_Conv2D(resnet_v2_50_block2_unit_1_bottleneck_v2_conv2_Conv2D_pad) + resnet_v2_50_block2_unit_1_bottleneck_v2_conv2_BatchNorm_FusedBatchNorm = self.resnet_v2_50_block2_unit_1_bottleneck_v2_conv2_BatchNorm_FusedBatchNorm(resnet_v2_50_block2_unit_1_bottleneck_v2_conv2_Conv2D) + resnet_v2_50_block2_unit_1_bottleneck_v2_conv2_Relu = F.relu(resnet_v2_50_block2_unit_1_bottleneck_v2_conv2_BatchNorm_FusedBatchNorm) + resnet_v2_50_block2_unit_1_bottleneck_v2_conv3_Conv2D = self.resnet_v2_50_block2_unit_1_bottleneck_v2_conv3_Conv2D(resnet_v2_50_block2_unit_1_bottleneck_v2_conv2_Relu) + resnet_v2_50_block2_unit_1_bottleneck_v2_add = resnet_v2_50_block2_unit_1_bottleneck_v2_shortcut_Conv2D + resnet_v2_50_block2_unit_1_bottleneck_v2_conv3_Conv2D + resnet_v2_50_block2_unit_2_bottleneck_v2_preact_FusedBatchNorm = self.resnet_v2_50_block2_unit_2_bottleneck_v2_preact_FusedBatchNorm(resnet_v2_50_block2_unit_1_bottleneck_v2_add) + resnet_v2_50_block2_unit_2_bottleneck_v2_preact_Relu = F.relu(resnet_v2_50_block2_unit_2_bottleneck_v2_preact_FusedBatchNorm) + resnet_v2_50_block2_unit_2_bottleneck_v2_conv1_Conv2D = self.resnet_v2_50_block2_unit_2_bottleneck_v2_conv1_Conv2D(resnet_v2_50_block2_unit_2_bottleneck_v2_preact_Relu) + resnet_v2_50_block2_unit_2_bottleneck_v2_conv1_BatchNorm_FusedBatchNorm = self.resnet_v2_50_block2_unit_2_bottleneck_v2_conv1_BatchNorm_FusedBatchNorm(resnet_v2_50_block2_unit_2_bottleneck_v2_conv1_Conv2D) + resnet_v2_50_block2_unit_2_bottleneck_v2_conv1_Relu = F.relu(resnet_v2_50_block2_unit_2_bottleneck_v2_conv1_BatchNorm_FusedBatchNorm) + resnet_v2_50_block2_unit_2_bottleneck_v2_conv2_Conv2D_pad = F.pad(resnet_v2_50_block2_unit_2_bottleneck_v2_conv1_Relu, (1, 1, 1, 1)) + resnet_v2_50_block2_unit_2_bottleneck_v2_conv2_Conv2D = self.resnet_v2_50_block2_unit_2_bottleneck_v2_conv2_Conv2D(resnet_v2_50_block2_unit_2_bottleneck_v2_conv2_Conv2D_pad) + resnet_v2_50_block2_unit_2_bottleneck_v2_conv2_BatchNorm_FusedBatchNorm = self.resnet_v2_50_block2_unit_2_bottleneck_v2_conv2_BatchNorm_FusedBatchNorm(resnet_v2_50_block2_unit_2_bottleneck_v2_conv2_Conv2D) + resnet_v2_50_block2_unit_2_bottleneck_v2_conv2_Relu = F.relu(resnet_v2_50_block2_unit_2_bottleneck_v2_conv2_BatchNorm_FusedBatchNorm) + resnet_v2_50_block2_unit_2_bottleneck_v2_conv3_Conv2D = self.resnet_v2_50_block2_unit_2_bottleneck_v2_conv3_Conv2D(resnet_v2_50_block2_unit_2_bottleneck_v2_conv2_Relu) + resnet_v2_50_block2_unit_2_bottleneck_v2_add = resnet_v2_50_block2_unit_1_bottleneck_v2_add + resnet_v2_50_block2_unit_2_bottleneck_v2_conv3_Conv2D + resnet_v2_50_block2_unit_3_bottleneck_v2_preact_FusedBatchNorm = self.resnet_v2_50_block2_unit_3_bottleneck_v2_preact_FusedBatchNorm(resnet_v2_50_block2_unit_2_bottleneck_v2_add) + resnet_v2_50_block2_unit_3_bottleneck_v2_preact_Relu = F.relu(resnet_v2_50_block2_unit_3_bottleneck_v2_preact_FusedBatchNorm) + resnet_v2_50_block2_unit_3_bottleneck_v2_conv1_Conv2D = self.resnet_v2_50_block2_unit_3_bottleneck_v2_conv1_Conv2D(resnet_v2_50_block2_unit_3_bottleneck_v2_preact_Relu) + resnet_v2_50_block2_unit_3_bottleneck_v2_conv1_BatchNorm_FusedBatchNorm = self.resnet_v2_50_block2_unit_3_bottleneck_v2_conv1_BatchNorm_FusedBatchNorm(resnet_v2_50_block2_unit_3_bottleneck_v2_conv1_Conv2D) + resnet_v2_50_block2_unit_3_bottleneck_v2_conv1_Relu = F.relu(resnet_v2_50_block2_unit_3_bottleneck_v2_conv1_BatchNorm_FusedBatchNorm) + resnet_v2_50_block2_unit_3_bottleneck_v2_conv2_Conv2D_pad = F.pad(resnet_v2_50_block2_unit_3_bottleneck_v2_conv1_Relu, (1, 1, 1, 1)) + resnet_v2_50_block2_unit_3_bottleneck_v2_conv2_Conv2D = self.resnet_v2_50_block2_unit_3_bottleneck_v2_conv2_Conv2D(resnet_v2_50_block2_unit_3_bottleneck_v2_conv2_Conv2D_pad) + resnet_v2_50_block2_unit_3_bottleneck_v2_conv2_BatchNorm_FusedBatchNorm = self.resnet_v2_50_block2_unit_3_bottleneck_v2_conv2_BatchNorm_FusedBatchNorm(resnet_v2_50_block2_unit_3_bottleneck_v2_conv2_Conv2D) + resnet_v2_50_block2_unit_3_bottleneck_v2_conv2_Relu = F.relu(resnet_v2_50_block2_unit_3_bottleneck_v2_conv2_BatchNorm_FusedBatchNorm) + resnet_v2_50_block2_unit_3_bottleneck_v2_conv3_Conv2D = self.resnet_v2_50_block2_unit_3_bottleneck_v2_conv3_Conv2D(resnet_v2_50_block2_unit_3_bottleneck_v2_conv2_Relu) + resnet_v2_50_block2_unit_3_bottleneck_v2_add = resnet_v2_50_block2_unit_2_bottleneck_v2_add + resnet_v2_50_block2_unit_3_bottleneck_v2_conv3_Conv2D + resnet_v2_50_block2_unit_4_bottleneck_v2_preact_FusedBatchNorm = self.resnet_v2_50_block2_unit_4_bottleneck_v2_preact_FusedBatchNorm(resnet_v2_50_block2_unit_3_bottleneck_v2_add) + resnet_v2_50_block2_unit_4_bottleneck_v2_shortcut_MaxPool = F.max_pool2d(resnet_v2_50_block2_unit_3_bottleneck_v2_add, kernel_size=(1, 1), stride=(2, 2), padding=0, ceil_mode=False) + resnet_v2_50_block2_unit_4_bottleneck_v2_preact_Relu = F.relu(resnet_v2_50_block2_unit_4_bottleneck_v2_preact_FusedBatchNorm) + resnet_v2_50_block2_unit_4_bottleneck_v2_conv1_Conv2D = self.resnet_v2_50_block2_unit_4_bottleneck_v2_conv1_Conv2D(resnet_v2_50_block2_unit_4_bottleneck_v2_preact_Relu) + resnet_v2_50_block2_unit_4_bottleneck_v2_conv1_BatchNorm_FusedBatchNorm = self.resnet_v2_50_block2_unit_4_bottleneck_v2_conv1_BatchNorm_FusedBatchNorm(resnet_v2_50_block2_unit_4_bottleneck_v2_conv1_Conv2D) + resnet_v2_50_block2_unit_4_bottleneck_v2_conv1_Relu = F.relu(resnet_v2_50_block2_unit_4_bottleneck_v2_conv1_BatchNorm_FusedBatchNorm) + resnet_v2_50_block2_unit_4_bottleneck_v2_Pad = F.pad(resnet_v2_50_block2_unit_4_bottleneck_v2_conv1_Relu, (1, 1, 1, 1), mode = 'constant', value = 0) + resnet_v2_50_block2_unit_4_bottleneck_v2_conv2_Conv2D = self.resnet_v2_50_block2_unit_4_bottleneck_v2_conv2_Conv2D(resnet_v2_50_block2_unit_4_bottleneck_v2_Pad) + resnet_v2_50_block2_unit_4_bottleneck_v2_conv2_BatchNorm_FusedBatchNorm = self.resnet_v2_50_block2_unit_4_bottleneck_v2_conv2_BatchNorm_FusedBatchNorm(resnet_v2_50_block2_unit_4_bottleneck_v2_conv2_Conv2D) + resnet_v2_50_block2_unit_4_bottleneck_v2_conv2_Relu = F.relu(resnet_v2_50_block2_unit_4_bottleneck_v2_conv2_BatchNorm_FusedBatchNorm) + resnet_v2_50_block2_unit_4_bottleneck_v2_conv3_Conv2D = self.resnet_v2_50_block2_unit_4_bottleneck_v2_conv3_Conv2D(resnet_v2_50_block2_unit_4_bottleneck_v2_conv2_Relu) + resnet_v2_50_block2_unit_4_bottleneck_v2_add = resnet_v2_50_block2_unit_4_bottleneck_v2_shortcut_MaxPool + resnet_v2_50_block2_unit_4_bottleneck_v2_conv3_Conv2D + resnet_v2_50_block3_unit_1_bottleneck_v2_preact_FusedBatchNorm = self.resnet_v2_50_block3_unit_1_bottleneck_v2_preact_FusedBatchNorm(resnet_v2_50_block2_unit_4_bottleneck_v2_add) + resnet_v2_50_block3_unit_1_bottleneck_v2_preact_Relu = F.relu(resnet_v2_50_block3_unit_1_bottleneck_v2_preact_FusedBatchNorm) + resnet_v2_50_block3_unit_1_bottleneck_v2_shortcut_Conv2D = self.resnet_v2_50_block3_unit_1_bottleneck_v2_shortcut_Conv2D(resnet_v2_50_block3_unit_1_bottleneck_v2_preact_Relu) + resnet_v2_50_block3_unit_1_bottleneck_v2_conv1_Conv2D = self.resnet_v2_50_block3_unit_1_bottleneck_v2_conv1_Conv2D(resnet_v2_50_block3_unit_1_bottleneck_v2_preact_Relu) + resnet_v2_50_block3_unit_1_bottleneck_v2_conv1_BatchNorm_FusedBatchNorm = self.resnet_v2_50_block3_unit_1_bottleneck_v2_conv1_BatchNorm_FusedBatchNorm(resnet_v2_50_block3_unit_1_bottleneck_v2_conv1_Conv2D) + resnet_v2_50_block3_unit_1_bottleneck_v2_conv1_Relu = F.relu(resnet_v2_50_block3_unit_1_bottleneck_v2_conv1_BatchNorm_FusedBatchNorm) + resnet_v2_50_block3_unit_1_bottleneck_v2_conv2_Conv2D_pad = F.pad(resnet_v2_50_block3_unit_1_bottleneck_v2_conv1_Relu, (1, 1, 1, 1)) + resnet_v2_50_block3_unit_1_bottleneck_v2_conv2_Conv2D = self.resnet_v2_50_block3_unit_1_bottleneck_v2_conv2_Conv2D(resnet_v2_50_block3_unit_1_bottleneck_v2_conv2_Conv2D_pad) + resnet_v2_50_block3_unit_1_bottleneck_v2_conv2_BatchNorm_FusedBatchNorm = self.resnet_v2_50_block3_unit_1_bottleneck_v2_conv2_BatchNorm_FusedBatchNorm(resnet_v2_50_block3_unit_1_bottleneck_v2_conv2_Conv2D) + resnet_v2_50_block3_unit_1_bottleneck_v2_conv2_Relu = F.relu(resnet_v2_50_block3_unit_1_bottleneck_v2_conv2_BatchNorm_FusedBatchNorm) + resnet_v2_50_block3_unit_1_bottleneck_v2_conv3_Conv2D = self.resnet_v2_50_block3_unit_1_bottleneck_v2_conv3_Conv2D(resnet_v2_50_block3_unit_1_bottleneck_v2_conv2_Relu) + resnet_v2_50_block3_unit_1_bottleneck_v2_add = resnet_v2_50_block3_unit_1_bottleneck_v2_shortcut_Conv2D + resnet_v2_50_block3_unit_1_bottleneck_v2_conv3_Conv2D + resnet_v2_50_block3_unit_2_bottleneck_v2_preact_FusedBatchNorm = self.resnet_v2_50_block3_unit_2_bottleneck_v2_preact_FusedBatchNorm(resnet_v2_50_block3_unit_1_bottleneck_v2_add) + resnet_v2_50_block3_unit_2_bottleneck_v2_preact_Relu = F.relu(resnet_v2_50_block3_unit_2_bottleneck_v2_preact_FusedBatchNorm) + resnet_v2_50_block3_unit_2_bottleneck_v2_conv1_Conv2D = self.resnet_v2_50_block3_unit_2_bottleneck_v2_conv1_Conv2D(resnet_v2_50_block3_unit_2_bottleneck_v2_preact_Relu) + resnet_v2_50_block3_unit_2_bottleneck_v2_conv1_BatchNorm_FusedBatchNorm = self.resnet_v2_50_block3_unit_2_bottleneck_v2_conv1_BatchNorm_FusedBatchNorm(resnet_v2_50_block3_unit_2_bottleneck_v2_conv1_Conv2D) + resnet_v2_50_block3_unit_2_bottleneck_v2_conv1_Relu = F.relu(resnet_v2_50_block3_unit_2_bottleneck_v2_conv1_BatchNorm_FusedBatchNorm) + resnet_v2_50_block3_unit_2_bottleneck_v2_conv2_Conv2D_pad = F.pad(resnet_v2_50_block3_unit_2_bottleneck_v2_conv1_Relu, (1, 1, 1, 1)) + resnet_v2_50_block3_unit_2_bottleneck_v2_conv2_Conv2D = self.resnet_v2_50_block3_unit_2_bottleneck_v2_conv2_Conv2D(resnet_v2_50_block3_unit_2_bottleneck_v2_conv2_Conv2D_pad) + resnet_v2_50_block3_unit_2_bottleneck_v2_conv2_BatchNorm_FusedBatchNorm = self.resnet_v2_50_block3_unit_2_bottleneck_v2_conv2_BatchNorm_FusedBatchNorm(resnet_v2_50_block3_unit_2_bottleneck_v2_conv2_Conv2D) + resnet_v2_50_block3_unit_2_bottleneck_v2_conv2_Relu = F.relu(resnet_v2_50_block3_unit_2_bottleneck_v2_conv2_BatchNorm_FusedBatchNorm) + resnet_v2_50_block3_unit_2_bottleneck_v2_conv3_Conv2D = self.resnet_v2_50_block3_unit_2_bottleneck_v2_conv3_Conv2D(resnet_v2_50_block3_unit_2_bottleneck_v2_conv2_Relu) + resnet_v2_50_block3_unit_2_bottleneck_v2_add = resnet_v2_50_block3_unit_1_bottleneck_v2_add + resnet_v2_50_block3_unit_2_bottleneck_v2_conv3_Conv2D + resnet_v2_50_block3_unit_3_bottleneck_v2_preact_FusedBatchNorm = self.resnet_v2_50_block3_unit_3_bottleneck_v2_preact_FusedBatchNorm(resnet_v2_50_block3_unit_2_bottleneck_v2_add) + resnet_v2_50_block3_unit_3_bottleneck_v2_preact_Relu = F.relu(resnet_v2_50_block3_unit_3_bottleneck_v2_preact_FusedBatchNorm) + resnet_v2_50_block3_unit_3_bottleneck_v2_conv1_Conv2D = self.resnet_v2_50_block3_unit_3_bottleneck_v2_conv1_Conv2D(resnet_v2_50_block3_unit_3_bottleneck_v2_preact_Relu) + resnet_v2_50_block3_unit_3_bottleneck_v2_conv1_BatchNorm_FusedBatchNorm = self.resnet_v2_50_block3_unit_3_bottleneck_v2_conv1_BatchNorm_FusedBatchNorm(resnet_v2_50_block3_unit_3_bottleneck_v2_conv1_Conv2D) + resnet_v2_50_block3_unit_3_bottleneck_v2_conv1_Relu = F.relu(resnet_v2_50_block3_unit_3_bottleneck_v2_conv1_BatchNorm_FusedBatchNorm) + resnet_v2_50_block3_unit_3_bottleneck_v2_conv2_Conv2D_pad = F.pad(resnet_v2_50_block3_unit_3_bottleneck_v2_conv1_Relu, (1, 1, 1, 1)) + resnet_v2_50_block3_unit_3_bottleneck_v2_conv2_Conv2D = self.resnet_v2_50_block3_unit_3_bottleneck_v2_conv2_Conv2D(resnet_v2_50_block3_unit_3_bottleneck_v2_conv2_Conv2D_pad) + resnet_v2_50_block3_unit_3_bottleneck_v2_conv2_BatchNorm_FusedBatchNorm = self.resnet_v2_50_block3_unit_3_bottleneck_v2_conv2_BatchNorm_FusedBatchNorm(resnet_v2_50_block3_unit_3_bottleneck_v2_conv2_Conv2D) + resnet_v2_50_block3_unit_3_bottleneck_v2_conv2_Relu = F.relu(resnet_v2_50_block3_unit_3_bottleneck_v2_conv2_BatchNorm_FusedBatchNorm) + resnet_v2_50_block3_unit_3_bottleneck_v2_conv3_Conv2D = self.resnet_v2_50_block3_unit_3_bottleneck_v2_conv3_Conv2D(resnet_v2_50_block3_unit_3_bottleneck_v2_conv2_Relu) + resnet_v2_50_block3_unit_3_bottleneck_v2_add = resnet_v2_50_block3_unit_2_bottleneck_v2_add + resnet_v2_50_block3_unit_3_bottleneck_v2_conv3_Conv2D + resnet_v2_50_block3_unit_4_bottleneck_v2_preact_FusedBatchNorm = self.resnet_v2_50_block3_unit_4_bottleneck_v2_preact_FusedBatchNorm(resnet_v2_50_block3_unit_3_bottleneck_v2_add) + resnet_v2_50_block3_unit_4_bottleneck_v2_preact_Relu = F.relu(resnet_v2_50_block3_unit_4_bottleneck_v2_preact_FusedBatchNorm) + resnet_v2_50_block3_unit_4_bottleneck_v2_conv1_Conv2D = self.resnet_v2_50_block3_unit_4_bottleneck_v2_conv1_Conv2D(resnet_v2_50_block3_unit_4_bottleneck_v2_preact_Relu) + resnet_v2_50_block3_unit_4_bottleneck_v2_conv1_BatchNorm_FusedBatchNorm = self.resnet_v2_50_block3_unit_4_bottleneck_v2_conv1_BatchNorm_FusedBatchNorm(resnet_v2_50_block3_unit_4_bottleneck_v2_conv1_Conv2D) + resnet_v2_50_block3_unit_4_bottleneck_v2_conv1_Relu = F.relu(resnet_v2_50_block3_unit_4_bottleneck_v2_conv1_BatchNorm_FusedBatchNorm) + resnet_v2_50_block3_unit_4_bottleneck_v2_conv2_Conv2D_pad = F.pad(resnet_v2_50_block3_unit_4_bottleneck_v2_conv1_Relu, (1, 1, 1, 1)) + resnet_v2_50_block3_unit_4_bottleneck_v2_conv2_Conv2D = self.resnet_v2_50_block3_unit_4_bottleneck_v2_conv2_Conv2D(resnet_v2_50_block3_unit_4_bottleneck_v2_conv2_Conv2D_pad) + resnet_v2_50_block3_unit_4_bottleneck_v2_conv2_BatchNorm_FusedBatchNorm = self.resnet_v2_50_block3_unit_4_bottleneck_v2_conv2_BatchNorm_FusedBatchNorm(resnet_v2_50_block3_unit_4_bottleneck_v2_conv2_Conv2D) + resnet_v2_50_block3_unit_4_bottleneck_v2_conv2_Relu = F.relu(resnet_v2_50_block3_unit_4_bottleneck_v2_conv2_BatchNorm_FusedBatchNorm) + resnet_v2_50_block3_unit_4_bottleneck_v2_conv3_Conv2D = self.resnet_v2_50_block3_unit_4_bottleneck_v2_conv3_Conv2D(resnet_v2_50_block3_unit_4_bottleneck_v2_conv2_Relu) + resnet_v2_50_block3_unit_4_bottleneck_v2_add = resnet_v2_50_block3_unit_3_bottleneck_v2_add + resnet_v2_50_block3_unit_4_bottleneck_v2_conv3_Conv2D + resnet_v2_50_block3_unit_5_bottleneck_v2_preact_FusedBatchNorm = self.resnet_v2_50_block3_unit_5_bottleneck_v2_preact_FusedBatchNorm(resnet_v2_50_block3_unit_4_bottleneck_v2_add) + resnet_v2_50_block3_unit_5_bottleneck_v2_preact_Relu = F.relu(resnet_v2_50_block3_unit_5_bottleneck_v2_preact_FusedBatchNorm) + resnet_v2_50_block3_unit_5_bottleneck_v2_conv1_Conv2D = self.resnet_v2_50_block3_unit_5_bottleneck_v2_conv1_Conv2D(resnet_v2_50_block3_unit_5_bottleneck_v2_preact_Relu) + resnet_v2_50_block3_unit_5_bottleneck_v2_conv1_BatchNorm_FusedBatchNorm = self.resnet_v2_50_block3_unit_5_bottleneck_v2_conv1_BatchNorm_FusedBatchNorm(resnet_v2_50_block3_unit_5_bottleneck_v2_conv1_Conv2D) + resnet_v2_50_block3_unit_5_bottleneck_v2_conv1_Relu = F.relu(resnet_v2_50_block3_unit_5_bottleneck_v2_conv1_BatchNorm_FusedBatchNorm) + resnet_v2_50_block3_unit_5_bottleneck_v2_conv2_Conv2D_pad = F.pad(resnet_v2_50_block3_unit_5_bottleneck_v2_conv1_Relu, (1, 1, 1, 1)) + resnet_v2_50_block3_unit_5_bottleneck_v2_conv2_Conv2D = self.resnet_v2_50_block3_unit_5_bottleneck_v2_conv2_Conv2D(resnet_v2_50_block3_unit_5_bottleneck_v2_conv2_Conv2D_pad) + resnet_v2_50_block3_unit_5_bottleneck_v2_conv2_BatchNorm_FusedBatchNorm = self.resnet_v2_50_block3_unit_5_bottleneck_v2_conv2_BatchNorm_FusedBatchNorm(resnet_v2_50_block3_unit_5_bottleneck_v2_conv2_Conv2D) + resnet_v2_50_block3_unit_5_bottleneck_v2_conv2_Relu = F.relu(resnet_v2_50_block3_unit_5_bottleneck_v2_conv2_BatchNorm_FusedBatchNorm) + resnet_v2_50_block3_unit_5_bottleneck_v2_conv3_Conv2D = self.resnet_v2_50_block3_unit_5_bottleneck_v2_conv3_Conv2D(resnet_v2_50_block3_unit_5_bottleneck_v2_conv2_Relu) + resnet_v2_50_block3_unit_5_bottleneck_v2_add = resnet_v2_50_block3_unit_4_bottleneck_v2_add + resnet_v2_50_block3_unit_5_bottleneck_v2_conv3_Conv2D + resnet_v2_50_block3_unit_6_bottleneck_v2_preact_FusedBatchNorm = self.resnet_v2_50_block3_unit_6_bottleneck_v2_preact_FusedBatchNorm(resnet_v2_50_block3_unit_5_bottleneck_v2_add) + resnet_v2_50_block3_unit_6_bottleneck_v2_shortcut_MaxPool = F.max_pool2d(resnet_v2_50_block3_unit_5_bottleneck_v2_add, kernel_size=(1, 1), stride=(2, 2), padding=0, ceil_mode=False) + resnet_v2_50_block3_unit_6_bottleneck_v2_preact_Relu = F.relu(resnet_v2_50_block3_unit_6_bottleneck_v2_preact_FusedBatchNorm) + resnet_v2_50_block3_unit_6_bottleneck_v2_conv1_Conv2D = self.resnet_v2_50_block3_unit_6_bottleneck_v2_conv1_Conv2D(resnet_v2_50_block3_unit_6_bottleneck_v2_preact_Relu) + resnet_v2_50_block3_unit_6_bottleneck_v2_conv1_BatchNorm_FusedBatchNorm = self.resnet_v2_50_block3_unit_6_bottleneck_v2_conv1_BatchNorm_FusedBatchNorm(resnet_v2_50_block3_unit_6_bottleneck_v2_conv1_Conv2D) + resnet_v2_50_block3_unit_6_bottleneck_v2_conv1_Relu = F.relu(resnet_v2_50_block3_unit_6_bottleneck_v2_conv1_BatchNorm_FusedBatchNorm) + resnet_v2_50_block3_unit_6_bottleneck_v2_Pad = F.pad(resnet_v2_50_block3_unit_6_bottleneck_v2_conv1_Relu, (1, 1, 1, 1), mode = 'constant', value = 0) + resnet_v2_50_block3_unit_6_bottleneck_v2_conv2_Conv2D = self.resnet_v2_50_block3_unit_6_bottleneck_v2_conv2_Conv2D(resnet_v2_50_block3_unit_6_bottleneck_v2_Pad) + resnet_v2_50_block3_unit_6_bottleneck_v2_conv2_BatchNorm_FusedBatchNorm = self.resnet_v2_50_block3_unit_6_bottleneck_v2_conv2_BatchNorm_FusedBatchNorm(resnet_v2_50_block3_unit_6_bottleneck_v2_conv2_Conv2D) + resnet_v2_50_block3_unit_6_bottleneck_v2_conv2_Relu = F.relu(resnet_v2_50_block3_unit_6_bottleneck_v2_conv2_BatchNorm_FusedBatchNorm) + resnet_v2_50_block3_unit_6_bottleneck_v2_conv3_Conv2D = self.resnet_v2_50_block3_unit_6_bottleneck_v2_conv3_Conv2D(resnet_v2_50_block3_unit_6_bottleneck_v2_conv2_Relu) + resnet_v2_50_block3_unit_6_bottleneck_v2_add = resnet_v2_50_block3_unit_6_bottleneck_v2_shortcut_MaxPool + resnet_v2_50_block3_unit_6_bottleneck_v2_conv3_Conv2D + resnet_v2_50_block4_unit_1_bottleneck_v2_preact_FusedBatchNorm = self.resnet_v2_50_block4_unit_1_bottleneck_v2_preact_FusedBatchNorm(resnet_v2_50_block3_unit_6_bottleneck_v2_add) + resnet_v2_50_block4_unit_1_bottleneck_v2_preact_Relu = F.relu(resnet_v2_50_block4_unit_1_bottleneck_v2_preact_FusedBatchNorm) + resnet_v2_50_block4_unit_1_bottleneck_v2_shortcut_Conv2D = self.resnet_v2_50_block4_unit_1_bottleneck_v2_shortcut_Conv2D(resnet_v2_50_block4_unit_1_bottleneck_v2_preact_Relu) + resnet_v2_50_block4_unit_1_bottleneck_v2_conv1_Conv2D = self.resnet_v2_50_block4_unit_1_bottleneck_v2_conv1_Conv2D(resnet_v2_50_block4_unit_1_bottleneck_v2_preact_Relu) + resnet_v2_50_block4_unit_1_bottleneck_v2_conv1_BatchNorm_FusedBatchNorm = self.resnet_v2_50_block4_unit_1_bottleneck_v2_conv1_BatchNorm_FusedBatchNorm(resnet_v2_50_block4_unit_1_bottleneck_v2_conv1_Conv2D) + resnet_v2_50_block4_unit_1_bottleneck_v2_conv1_Relu = F.relu(resnet_v2_50_block4_unit_1_bottleneck_v2_conv1_BatchNorm_FusedBatchNorm) + resnet_v2_50_block4_unit_1_bottleneck_v2_conv2_Conv2D_pad = F.pad(resnet_v2_50_block4_unit_1_bottleneck_v2_conv1_Relu, (1, 1, 1, 1)) + resnet_v2_50_block4_unit_1_bottleneck_v2_conv2_Conv2D = self.resnet_v2_50_block4_unit_1_bottleneck_v2_conv2_Conv2D(resnet_v2_50_block4_unit_1_bottleneck_v2_conv2_Conv2D_pad) + resnet_v2_50_block4_unit_1_bottleneck_v2_conv2_BatchNorm_FusedBatchNorm = self.resnet_v2_50_block4_unit_1_bottleneck_v2_conv2_BatchNorm_FusedBatchNorm(resnet_v2_50_block4_unit_1_bottleneck_v2_conv2_Conv2D) + resnet_v2_50_block4_unit_1_bottleneck_v2_conv2_Relu = F.relu(resnet_v2_50_block4_unit_1_bottleneck_v2_conv2_BatchNorm_FusedBatchNorm) + resnet_v2_50_block4_unit_1_bottleneck_v2_conv3_Conv2D = self.resnet_v2_50_block4_unit_1_bottleneck_v2_conv3_Conv2D(resnet_v2_50_block4_unit_1_bottleneck_v2_conv2_Relu) + resnet_v2_50_block4_unit_1_bottleneck_v2_add = resnet_v2_50_block4_unit_1_bottleneck_v2_shortcut_Conv2D + resnet_v2_50_block4_unit_1_bottleneck_v2_conv3_Conv2D + resnet_v2_50_block4_unit_2_bottleneck_v2_preact_FusedBatchNorm = self.resnet_v2_50_block4_unit_2_bottleneck_v2_preact_FusedBatchNorm(resnet_v2_50_block4_unit_1_bottleneck_v2_add) + resnet_v2_50_block4_unit_2_bottleneck_v2_preact_Relu = F.relu(resnet_v2_50_block4_unit_2_bottleneck_v2_preact_FusedBatchNorm) + resnet_v2_50_block4_unit_2_bottleneck_v2_conv1_Conv2D = self.resnet_v2_50_block4_unit_2_bottleneck_v2_conv1_Conv2D(resnet_v2_50_block4_unit_2_bottleneck_v2_preact_Relu) + resnet_v2_50_block4_unit_2_bottleneck_v2_conv1_BatchNorm_FusedBatchNorm = self.resnet_v2_50_block4_unit_2_bottleneck_v2_conv1_BatchNorm_FusedBatchNorm(resnet_v2_50_block4_unit_2_bottleneck_v2_conv1_Conv2D) + resnet_v2_50_block4_unit_2_bottleneck_v2_conv1_Relu = F.relu(resnet_v2_50_block4_unit_2_bottleneck_v2_conv1_BatchNorm_FusedBatchNorm) + resnet_v2_50_block4_unit_2_bottleneck_v2_conv2_Conv2D_pad = F.pad(resnet_v2_50_block4_unit_2_bottleneck_v2_conv1_Relu, (1, 1, 1, 1)) + resnet_v2_50_block4_unit_2_bottleneck_v2_conv2_Conv2D = self.resnet_v2_50_block4_unit_2_bottleneck_v2_conv2_Conv2D(resnet_v2_50_block4_unit_2_bottleneck_v2_conv2_Conv2D_pad) + resnet_v2_50_block4_unit_2_bottleneck_v2_conv2_BatchNorm_FusedBatchNorm = self.resnet_v2_50_block4_unit_2_bottleneck_v2_conv2_BatchNorm_FusedBatchNorm(resnet_v2_50_block4_unit_2_bottleneck_v2_conv2_Conv2D) + resnet_v2_50_block4_unit_2_bottleneck_v2_conv2_Relu = F.relu(resnet_v2_50_block4_unit_2_bottleneck_v2_conv2_BatchNorm_FusedBatchNorm) + resnet_v2_50_block4_unit_2_bottleneck_v2_conv3_Conv2D = self.resnet_v2_50_block4_unit_2_bottleneck_v2_conv3_Conv2D(resnet_v2_50_block4_unit_2_bottleneck_v2_conv2_Relu) + resnet_v2_50_block4_unit_2_bottleneck_v2_add = resnet_v2_50_block4_unit_1_bottleneck_v2_add + resnet_v2_50_block4_unit_2_bottleneck_v2_conv3_Conv2D + resnet_v2_50_block4_unit_3_bottleneck_v2_preact_FusedBatchNorm = self.resnet_v2_50_block4_unit_3_bottleneck_v2_preact_FusedBatchNorm(resnet_v2_50_block4_unit_2_bottleneck_v2_add) + resnet_v2_50_block4_unit_3_bottleneck_v2_preact_Relu = F.relu(resnet_v2_50_block4_unit_3_bottleneck_v2_preact_FusedBatchNorm) + resnet_v2_50_block4_unit_3_bottleneck_v2_conv1_Conv2D = self.resnet_v2_50_block4_unit_3_bottleneck_v2_conv1_Conv2D(resnet_v2_50_block4_unit_3_bottleneck_v2_preact_Relu) + resnet_v2_50_block4_unit_3_bottleneck_v2_conv1_BatchNorm_FusedBatchNorm = self.resnet_v2_50_block4_unit_3_bottleneck_v2_conv1_BatchNorm_FusedBatchNorm(resnet_v2_50_block4_unit_3_bottleneck_v2_conv1_Conv2D) + resnet_v2_50_block4_unit_3_bottleneck_v2_conv1_Relu = F.relu(resnet_v2_50_block4_unit_3_bottleneck_v2_conv1_BatchNorm_FusedBatchNorm) + resnet_v2_50_block4_unit_3_bottleneck_v2_conv2_Conv2D_pad = F.pad(resnet_v2_50_block4_unit_3_bottleneck_v2_conv1_Relu, (1, 1, 1, 1)) + resnet_v2_50_block4_unit_3_bottleneck_v2_conv2_Conv2D = self.resnet_v2_50_block4_unit_3_bottleneck_v2_conv2_Conv2D(resnet_v2_50_block4_unit_3_bottleneck_v2_conv2_Conv2D_pad) + resnet_v2_50_block4_unit_3_bottleneck_v2_conv2_BatchNorm_FusedBatchNorm = self.resnet_v2_50_block4_unit_3_bottleneck_v2_conv2_BatchNorm_FusedBatchNorm(resnet_v2_50_block4_unit_3_bottleneck_v2_conv2_Conv2D) + resnet_v2_50_block4_unit_3_bottleneck_v2_conv2_Relu = F.relu(resnet_v2_50_block4_unit_3_bottleneck_v2_conv2_BatchNorm_FusedBatchNorm) + resnet_v2_50_block4_unit_3_bottleneck_v2_conv3_Conv2D = self.resnet_v2_50_block4_unit_3_bottleneck_v2_conv3_Conv2D(resnet_v2_50_block4_unit_3_bottleneck_v2_conv2_Relu) + resnet_v2_50_block4_unit_3_bottleneck_v2_add = resnet_v2_50_block4_unit_2_bottleneck_v2_add + resnet_v2_50_block4_unit_3_bottleneck_v2_conv3_Conv2D + resnet_v2_50_postnorm_FusedBatchNorm = self.resnet_v2_50_postnorm_FusedBatchNorm(resnet_v2_50_block4_unit_3_bottleneck_v2_add) + resnet_v2_50_postnorm_Relu = F.relu(resnet_v2_50_postnorm_FusedBatchNorm) + return resnet_v2_50_postnorm_Relu + + + @staticmethod + def __conv(dim, name, **kwargs): + if dim == 1: layer = nn.Conv1d(**kwargs) + elif dim == 2: layer = nn.Conv2d(**kwargs) + elif dim == 3: layer = nn.Conv3d(**kwargs) + else: raise NotImplementedError() + + return layer + + @staticmethod + def __batch_normalization(dim, name, **kwargs): + if dim == 0 or dim == 1: layer = nn.BatchNorm1d(**kwargs) + elif dim == 2: layer = nn.BatchNorm2d(**kwargs) + elif dim == 3: layer = nn.BatchNorm3d(**kwargs) + else: raise NotImplementedError() + return layer + diff --git a/models/StabNet/v2_93.py b/models/StabNet/v2_93.py new file mode 100644 index 0000000..f6121e5 --- /dev/null +++ b/models/StabNet/v2_93.py @@ -0,0 +1,51 @@ +# imgloss + tmploss + featureloss +import numpy as np +height = 288 +width = 512 +batch_size = 10 +initial_learning_rate = 2e-5 +feature_mul = 1#10 * width +theta_mul = 400/2500#400 +regu_mul = 30/2500#30 +img_mul = 50#height * width * 1 +temp_mul = 500#10#height * width * 10 +black_mul = 300000/2500#300000 +id_mul = 10/2500#10 +training_iter = 100000 +step_size = 40000 +train_data_size = 27000 +test_data_size = 2500 +crop_rate = 1 +before_ch = 6 +after_ch = 0 +tot_ch = before_ch + after_ch + 1 +test_batches = 10 +random_crop_rate = 0.9 +max_crop_rate = 0.6 +disp_freq = 100 +test_freq = 500 +save_freq = 1000 +no_theta_iter = 1000000 +do_temp_loss_iter = 10000 +do_theta_10_iter = -1 +do_black_loss_iter = 1500 +do_theta_only_iter = 600 +tfrecord_item_num = 10 +log_dir = 'log/v2_93/' +model_dir = 'models/v2_93/' +data_dir = 'data/' +rand_H_max = np.array([[1.1, 0.1, 0.5], [0.1, 1.1, 0.5], [0.1, 0.1, 1]]) +rand_H_min = np.array([[0.9, -0.1, -0.5], [-0.1, 0.9, -0.5], [-0.1, -0.1, 1]]) +max_matches = 3000 +input_mask = True +do_crop_rate = 0.8 +indices = [0, 1, 2, 4, 8, 16, 32] +rand_H_change_rate = 1 +distortion_mul = 1#2500 +consistency_mul = 20#50000 +grid_large_h = 2 +grid_large_w = 2 +grid_h = 4 +grid_w = 4 +grid_theta_mul = 0 +grid_max_move = 1 diff --git a/models/__init__.py b/models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/models/correlation/correlation.py b/models/correlation/correlation.py new file mode 100644 index 0000000..4039eb4 --- /dev/null +++ b/models/correlation/correlation.py @@ -0,0 +1,397 @@ +#!/usr/bin/env python + +import torch + +import cupy +import re + +kernel_Correlation_rearrange = ''' + extern "C" __global__ void kernel_Correlation_rearrange( + const int n, + const float* input, + float* output + ) { + int intIndex = (blockIdx.x * blockDim.x) + threadIdx.x; + + if (intIndex >= n) { + return; + } + + int intSample = blockIdx.z; + int intChannel = blockIdx.y; + + float fltValue = input[(((intSample * SIZE_1(input)) + intChannel) * SIZE_2(input) * SIZE_3(input)) + intIndex]; + + __syncthreads(); + + int intPaddedY = (intIndex / SIZE_3(input)) + 4; + int intPaddedX = (intIndex % SIZE_3(input)) + 4; + int intRearrange = ((SIZE_3(input) + 8) * intPaddedY) + intPaddedX; + + output[(((intSample * SIZE_1(output) * SIZE_2(output)) + intRearrange) * SIZE_1(input)) + intChannel] = fltValue; + } +''' + +kernel_Correlation_updateOutput = ''' + extern "C" __global__ void kernel_Correlation_updateOutput( + const int n, + const float* rbot0, + const float* rbot1, + float* top + ) { + extern __shared__ char patch_data_char[]; + + float *patch_data = (float *)patch_data_char; + + // First (upper left) position of kernel upper-left corner in current center position of neighborhood in image 1 + int x1 = blockIdx.x + 4; + int y1 = blockIdx.y + 4; + int item = blockIdx.z; + int ch_off = threadIdx.x; + + // Load 3D patch into shared shared memory + for (int j = 0; j < 1; j++) { // HEIGHT + for (int i = 0; i < 1; i++) { // WIDTH + int ji_off = (j + i) * SIZE_3(rbot0); + for (int ch = ch_off; ch < SIZE_3(rbot0); ch += 32) { // CHANNELS + int idx1 = ((item * SIZE_1(rbot0) + y1+j) * SIZE_2(rbot0) + x1+i) * SIZE_3(rbot0) + ch; + int idxPatchData = ji_off + ch; + patch_data[idxPatchData] = rbot0[idx1]; + } + } + } + + __syncthreads(); + + __shared__ float sum[32]; + + // Compute correlation + for (int top_channel = 0; top_channel < SIZE_1(top); top_channel++) { + sum[ch_off] = 0; + + int s2o = top_channel % 9 - 4; + int s2p = top_channel / 9 - 4; + + for (int j = 0; j < 1; j++) { // HEIGHT + for (int i = 0; i < 1; i++) { // WIDTH + int ji_off = (j + i) * SIZE_3(rbot0); + for (int ch = ch_off; ch < SIZE_3(rbot0); ch += 32) { // CHANNELS + int x2 = x1 + s2o; + int y2 = y1 + s2p; + + int idxPatchData = ji_off + ch; + int idx2 = ((item * SIZE_1(rbot0) + y2+j) * SIZE_2(rbot0) + x2+i) * SIZE_3(rbot0) + ch; + + sum[ch_off] += patch_data[idxPatchData] * rbot1[idx2]; + } + } + } + + __syncthreads(); + + if (ch_off == 0) { + float total_sum = 0; + for (int idx = 0; idx < 32; idx++) { + total_sum += sum[idx]; + } + const int sumelems = SIZE_3(rbot0); + const int index = ((top_channel*SIZE_2(top) + blockIdx.y)*SIZE_3(top))+blockIdx.x; + top[index + item*SIZE_1(top)*SIZE_2(top)*SIZE_3(top)] = total_sum / (float)sumelems; + } + } + } +''' + +kernel_Correlation_updateGradFirst = ''' + #define ROUND_OFF 50000 + + extern "C" __global__ void kernel_Correlation_updateGradFirst( + const int n, + const int intSample, + const float* rbot0, + const float* rbot1, + const float* gradOutput, + float* gradFirst, + float* gradSecond + ) { for (int intIndex = (blockIdx.x * blockDim.x) + threadIdx.x; intIndex < n; intIndex += blockDim.x * gridDim.x) { + int n = intIndex % SIZE_1(gradFirst); // channels + int l = (intIndex / SIZE_1(gradFirst)) % SIZE_3(gradFirst) + 4; // w-pos + int m = (intIndex / SIZE_1(gradFirst) / SIZE_3(gradFirst)) % SIZE_2(gradFirst) + 4; // h-pos + + // round_off is a trick to enable integer division with ceil, even for negative numbers + // We use a large offset, for the inner part not to become negative. + const int round_off = ROUND_OFF; + const int round_off_s1 = round_off; + + // We add round_off before_s1 the int division and subtract round_off after it, to ensure the formula matches ceil behavior: + int xmin = (l - 4 + round_off_s1 - 1) + 1 - round_off; // ceil (l - 4) + int ymin = (m - 4 + round_off_s1 - 1) + 1 - round_off; // ceil (l - 4) + + // Same here: + int xmax = (l - 4 + round_off_s1) - round_off; // floor (l - 4) + int ymax = (m - 4 + round_off_s1) - round_off; // floor (m - 4) + + float sum = 0; + if (xmax>=0 && ymax>=0 && (xmin<=SIZE_3(gradOutput)-1) && (ymin<=SIZE_2(gradOutput)-1)) { + xmin = max(0,xmin); + xmax = min(SIZE_3(gradOutput)-1,xmax); + + ymin = max(0,ymin); + ymax = min(SIZE_2(gradOutput)-1,ymax); + + for (int p = -4; p <= 4; p++) { + for (int o = -4; o <= 4; o++) { + // Get rbot1 data: + int s2o = o; + int s2p = p; + int idxbot1 = ((intSample * SIZE_1(rbot0) + (m+s2p)) * SIZE_2(rbot0) + (l+s2o)) * SIZE_3(rbot0) + n; + float bot1tmp = rbot1[idxbot1]; // rbot1[l+s2o,m+s2p,n] + + // Index offset for gradOutput in following loops: + int op = (p+4) * 9 + (o+4); // index[o,p] + int idxopoffset = (intSample * SIZE_1(gradOutput) + op); + + for (int y = ymin; y <= ymax; y++) { + for (int x = xmin; x <= xmax; x++) { + int idxgradOutput = (idxopoffset * SIZE_2(gradOutput) + y) * SIZE_3(gradOutput) + x; // gradOutput[x,y,o,p] + sum += gradOutput[idxgradOutput] * bot1tmp; + } + } + } + } + } + const int sumelems = SIZE_1(gradFirst); + const int bot0index = ((n * SIZE_2(gradFirst)) + (m-4)) * SIZE_3(gradFirst) + (l-4); + gradFirst[bot0index + intSample*SIZE_1(gradFirst)*SIZE_2(gradFirst)*SIZE_3(gradFirst)] = sum / (float)sumelems; + } } +''' + +kernel_Correlation_updateGradSecond = ''' + #define ROUND_OFF 50000 + + extern "C" __global__ void kernel_Correlation_updateGradSecond( + const int n, + const int intSample, + const float* rbot0, + const float* rbot1, + const float* gradOutput, + float* gradFirst, + float* gradSecond + ) { for (int intIndex = (blockIdx.x * blockDim.x) + threadIdx.x; intIndex < n; intIndex += blockDim.x * gridDim.x) { + int n = intIndex % SIZE_1(gradSecond); // channels + int l = (intIndex / SIZE_1(gradSecond)) % SIZE_3(gradSecond) + 4; // w-pos + int m = (intIndex / SIZE_1(gradSecond) / SIZE_3(gradSecond)) % SIZE_2(gradSecond) + 4; // h-pos + + // round_off is a trick to enable integer division with ceil, even for negative numbers + // We use a large offset, for the inner part not to become negative. + const int round_off = ROUND_OFF; + const int round_off_s1 = round_off; + + float sum = 0; + for (int p = -4; p <= 4; p++) { + for (int o = -4; o <= 4; o++) { + int s2o = o; + int s2p = p; + + //Get X,Y ranges and clamp + // We add round_off before_s1 the int division and subtract round_off after it, to ensure the formula matches ceil behavior: + int xmin = (l - 4 - s2o + round_off_s1 - 1) + 1 - round_off; // ceil (l - 4 - s2o) + int ymin = (m - 4 - s2p + round_off_s1 - 1) + 1 - round_off; // ceil (l - 4 - s2o) + + // Same here: + int xmax = (l - 4 - s2o + round_off_s1) - round_off; // floor (l - 4 - s2o) + int ymax = (m - 4 - s2p + round_off_s1) - round_off; // floor (m - 4 - s2p) + + if (xmax>=0 && ymax>=0 && (xmin<=SIZE_3(gradOutput)-1) && (ymin<=SIZE_2(gradOutput)-1)) { + xmin = max(0,xmin); + xmax = min(SIZE_3(gradOutput)-1,xmax); + + ymin = max(0,ymin); + ymax = min(SIZE_2(gradOutput)-1,ymax); + + // Get rbot0 data: + int idxbot0 = ((intSample * SIZE_1(rbot0) + (m-s2p)) * SIZE_2(rbot0) + (l-s2o)) * SIZE_3(rbot0) + n; + float bot0tmp = rbot0[idxbot0]; // rbot1[l+s2o,m+s2p,n] + + // Index offset for gradOutput in following loops: + int op = (p+4) * 9 + (o+4); // index[o,p] + int idxopoffset = (intSample * SIZE_1(gradOutput) + op); + + for (int y = ymin; y <= ymax; y++) { + for (int x = xmin; x <= xmax; x++) { + int idxgradOutput = (idxopoffset * SIZE_2(gradOutput) + y) * SIZE_3(gradOutput) + x; // gradOutput[x,y,o,p] + sum += gradOutput[idxgradOutput] * bot0tmp; + } + } + } + } + } + const int sumelems = SIZE_1(gradSecond); + const int bot1index = ((n * SIZE_2(gradSecond)) + (m-4)) * SIZE_3(gradSecond) + (l-4); + gradSecond[bot1index + intSample*SIZE_1(gradSecond)*SIZE_2(gradSecond)*SIZE_3(gradSecond)] = sum / (float)sumelems; + } } +''' + +def cupy_kernel(strFunction, objVariables): + strKernel = globals()[strFunction] + + while True: + objMatch = re.search('(SIZE_)([0-4])(\()([^\)]*)(\))', strKernel) + + if objMatch is None: + break + # end + + intArg = int(objMatch.group(2)) + + strTensor = objMatch.group(4) + intSizes = objVariables[strTensor].size() + + strKernel = strKernel.replace(objMatch.group(), str(intSizes[intArg])) + # end + + while True: + objMatch = re.search('(VALUE_)([0-4])(\()([^\)]+)(\))', strKernel) + + if objMatch is None: + break + # end + + intArgs = int(objMatch.group(2)) + strArgs = objMatch.group(4).split(',') + + strTensor = strArgs[0] + intStrides = objVariables[strTensor].stride() + strIndex = [ '((' + strArgs[intArg + 1].replace('{', '(').replace('}', ')').strip() + ')*' + str(intStrides[intArg]) + ')' for intArg in range(intArgs) ] + + strKernel = strKernel.replace(objMatch.group(0), strTensor + '[' + str.join('+', strIndex) + ']') + # end + + return strKernel +# end + +@cupy.util.memoize(for_each_device=True) +def cupy_launch(strFunction, strKernel): + return cupy.cuda.compile_with_cache(strKernel).get_function(strFunction) +# end + +class _FunctionCorrelation(torch.autograd.Function): + @staticmethod + def forward(self, first, second): + rbot0 = first.new_zeros([ first.shape[0], first.shape[2] + 8, first.shape[3] + 8, first.shape[1] ]) + rbot1 = first.new_zeros([ first.shape[0], first.shape[2] + 8, first.shape[3] + 8, first.shape[1] ]) + + self.save_for_backward(first, second, rbot0, rbot1) + + assert(first.is_contiguous() == True) + assert(second.is_contiguous() == True) + + output = first.new_zeros([ first.shape[0], 81, first.shape[2], first.shape[3] ]) + + if first.is_cuda == True: + n = first.shape[2] * first.shape[3] + cupy_launch('kernel_Correlation_rearrange', cupy_kernel('kernel_Correlation_rearrange', { + 'input': first, + 'output': rbot0 + }))( + grid=tuple([ int((n + 16 - 1) / 16), first.shape[1], first.shape[0] ]), + block=tuple([ 16, 1, 1 ]), + args=[ n, first.data_ptr(), rbot0.data_ptr() ] + ) + + n = second.shape[2] * second.shape[3] + cupy_launch('kernel_Correlation_rearrange', cupy_kernel('kernel_Correlation_rearrange', { + 'input': second, + 'output': rbot1 + }))( + grid=tuple([ int((n + 16 - 1) / 16), second.shape[1], second.shape[0] ]), + block=tuple([ 16, 1, 1 ]), + args=[ n, second.data_ptr(), rbot1.data_ptr() ] + ) + + n = output.shape[1] * output.shape[2] * output.shape[3] + cupy_launch('kernel_Correlation_updateOutput', cupy_kernel('kernel_Correlation_updateOutput', { + 'rbot0': rbot0, + 'rbot1': rbot1, + 'top': output + }))( + grid=tuple([ output.shape[3], output.shape[2], output.shape[0] ]), + block=tuple([ 32, 1, 1 ]), + shared_mem=first.shape[1] * 4, + args=[ n, rbot0.data_ptr(), rbot1.data_ptr(), output.data_ptr() ] + ) + + elif first.is_cuda == False: + raise NotImplementedError() + + # end + + return output + # end + + @staticmethod + def backward(self, gradOutput): + first, second, rbot0, rbot1 = self.saved_tensors + + assert(gradOutput.is_contiguous() == True) + + gradFirst = first.new_zeros([ first.shape[0], first.shape[1], first.shape[2], first.shape[3] ]) if self.needs_input_grad[0] == True else None + gradSecond = first.new_zeros([ first.shape[0], first.shape[1], first.shape[2], first.shape[3] ]) if self.needs_input_grad[1] == True else None + + if first.is_cuda == True: + if gradFirst is not None: + for intSample in range(first.shape[0]): + n = first.shape[1] * first.shape[2] * first.shape[3] + cupy_launch('kernel_Correlation_updateGradFirst', cupy_kernel('kernel_Correlation_updateGradFirst', { + 'rbot0': rbot0, + 'rbot1': rbot1, + 'gradOutput': gradOutput, + 'gradFirst': gradFirst, + 'gradSecond': None + }))( + grid=tuple([ int((n + 512 - 1) / 512), 1, 1 ]), + block=tuple([ 512, 1, 1 ]), + args=[ n, intSample, rbot0.data_ptr(), rbot1.data_ptr(), gradOutput.data_ptr(), gradFirst.data_ptr(), None ] + ) + # end + # end + + if gradSecond is not None: + for intSample in range(first.shape[0]): + n = first.shape[1] * first.shape[2] * first.shape[3] + cupy_launch('kernel_Correlation_updateGradSecond', cupy_kernel('kernel_Correlation_updateGradSecond', { + 'rbot0': rbot0, + 'rbot1': rbot1, + 'gradOutput': gradOutput, + 'gradFirst': None, + 'gradSecond': gradSecond + }))( + grid=tuple([ int((n + 512 - 1) / 512), 1, 1 ]), + block=tuple([ 512, 1, 1 ]), + args=[ n, intSample, rbot0.data_ptr(), rbot1.data_ptr(), gradOutput.data_ptr(), None, gradSecond.data_ptr() ] + ) + # end + # end + + elif first.is_cuda == False: + raise NotImplementedError() + + # end + + return gradFirst, gradSecond + # end +# end + +def FunctionCorrelation(tenFirst, tenSecond): + return _FunctionCorrelation.apply(tenFirst, tenSecond) +# end + +class ModuleCorrelation(torch.nn.Module): + def __init__(self): + super(ModuleCorrelation, self).__init__() + # end + + def forward(self, tenFirst, tenSecond): + return _FunctionCorrelation.apply(tenFirst, tenSecond) + # end +# end \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..cf2681a --- /dev/null +++ b/requirements.txt @@ -0,0 +1,18 @@ +cupy==7.5.0 +easydict==1.9 +imageio==2.8.0 +ipdb==0.13.3 +matplotlib==3.2.2 +networkx==2.4 +numpy==1.18.2 +opencv-python==4.2.0.34 +Pillow==7.1.2 +pypng==0.0.20 +scikit-image==0.17.2 +scikit-learn==0.23.2 +scipy==1.1.0 +scs==2.1.2 +six==1.14.0 +sklearn==0.0 +tensorboardX==2.0 +tqdm==4.54.0 diff --git a/scripts/DIFRINTStabilizer.py b/scripts/DIFRINTStabilizer.py new file mode 100644 index 0000000..73eec63 --- /dev/null +++ b/scripts/DIFRINTStabilizer.py @@ -0,0 +1,133 @@ +import argparse +import os +import sys +from shutil import copyfile + +import torch +import torch.nn as nn +from torch.autograd import Variable +parentddir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir)) +sys.path.append(parentddir) +from models.DIFRINT.models import DIFNet2 +from models.DIFRINT.pwcNet import PwcNet + +from PIL import Image +import numpy as np +import math +import pdb +import time +import cv2 + +parser = argparse.ArgumentParser() +parser.add_argument('--modelPath', default='./trained_models/DIFNet2.pth') # 2 +parser.add_argument('--InputBasePath', default='') +parser.add_argument('--OutputBasePath', default='./') +parser.add_argument('--temp_file', default='./DIFRINT_TEMP/') +parser.add_argument('--n_iter', type=int, default=3, + help='number of stabilization interations') +parser.add_argument('--skip', type=int, default=2, + help='number of frame skips for interpolation') +parser.add_argument('--desiredWidth', type=int, default=640, + help='width of the input video') +parser.add_argument('--desiredHeight', type=int, default=480, + help='height of the input video') +parser.add_argument('--cuda', action='store_true', help='use GPU computation') +opt = parser.parse_args() +print(opt) + +if torch.cuda.is_available() and not opt.cuda: + print("WARNING: You have a CUDA device, so you should probably run with --cuda") + +########################################################## + +# Networks +DIFNet = DIFNet2() + +# Place Network in cuda memory +if opt.cuda: + DIFNet.cuda() + +# DataParallel +DIFNet = nn.DataParallel(DIFNet) +DIFNet.load_state_dict(torch.load(opt.modelPath)) +DIFNet.eval() + +if not os.path.exists(opt.OutputBasePath): + os.mkdir(opt.OutputBasePath) + +if not os.path.exists(opt.temp_file): + os.mkdir(opt.temp_file) + +########################################################## + +frameList = [ele for ele in os.listdir(opt.InputBasePath) if ele[-4:] == '.jpg'] +frameList = sorted(frameList, key=lambda x: int(x[:-4])) + +if os.path.exists(opt.temp_file): + copyfile(opt.InputBasePath + frameList[0], opt.temp_file + frameList[0]) + copyfile(opt.InputBasePath + frameList[-1], opt.temp_file + frameList[-1]) +else: + os.makedirs(opt.temp_file) + copyfile(opt.InputBasePath + frameList[0], opt.temp_file + frameList[0]) + copyfile(opt.InputBasePath + frameList[-1], opt.temp_file + frameList[-1]) +# end + +# Generate output sequence +for num_iter in range(opt.n_iter): + idx = 1 + print('\nIter: ' + str(num_iter+1)) + for f in frameList[1:-1]: + if f.endswith('.jpg'): + if num_iter == 0: + src = opt.InputBasePath + else: + src = opt.temp_file + # end + + if idx < opt.skip or idx > (len(frameList)-1-opt.skip): + skip = 1 + else: + skip = opt.skip + + + fr_g1 = torch.cuda.FloatTensor(np.array(Image.open(opt.temp_file + '%d.jpg' % ( + int(f[:-4])-skip)).resize((opt.desiredWidth, opt.desiredHeight))).transpose(2, 0, 1).astype(np.float32)[None, :, :, :] / 255.0) + + fr_g3 = torch.cuda.FloatTensor(np.array(Image.open( + src + '%d.jpg' % (int(f[:-4])+skip)).resize((opt.desiredWidth, opt.desiredHeight))).transpose(2, 0, 1).astype(np.float32)[None, :, :, :] / 255.0) + + + fr_o2 = torch.cuda.FloatTensor(np.array(Image.open( + opt.InputBasePath + f).resize((opt.desiredWidth, opt.desiredHeight))).transpose(2, 0, 1).astype(np.float32)[None, :, :, :] / 255.0) + + with torch.no_grad(): + fhat, I_int = DIFNet(fr_g1, fr_g3, fr_o2, + fr_g3, fr_g1, 0.5) # Notice 0.5 + + # Save image + img = Image.fromarray( + np.uint8(fhat.cpu().squeeze().permute(1, 2, 0)*255)) + img.save(opt.temp_file + f) + + sys.stdout.write('\rFrame: ' + str(idx) + + '/' + str(len(frameList)-2)) + sys.stdout.flush() + + idx += 1 + # end + # end + +frame_rate = 25 +frame_width = opt.desiredWidth +frame_height = opt.desiredHeight + +print("generate stabilized video...") +fourcc = cv2.VideoWriter_fourcc(*'MP4V') +out = cv2.VideoWriter(opt.OutputBasePath + '/DIFRINT_stable.mp4', fourcc, frame_rate, (frame_width, frame_height)) + +for f in frameList: + if f.endswith('.jpg'): + img = cv2.imread(os.path.join(opt.temp_file, f)) + out.write(img) + +out.release() diff --git a/scripts/DUTStabilizer.py b/scripts/DUTStabilizer.py new file mode 100644 index 0000000..738885a --- /dev/null +++ b/scripts/DUTStabilizer.py @@ -0,0 +1,111 @@ +import torch +import torch.nn as nn +import numpy as np +import os +import math +import cv2 +import sys +parentddir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir)) +sys.path.append(parentddir) + +from models.DUT.DUT import DUT +from tqdm import tqdm +from utils.WarpUtils import warpListImage +from configs.config import cfg +import argparse + +torch.set_grad_enabled(False) + +def parse_args(): + parser = argparse.ArgumentParser(description='Control for stabilization model') + parser.add_argument('--SmootherPath', help='the path to pretrained smoother model, blank for jacobi solver', default='') + parser.add_argument('--RFDetPath', help='pretrained RFNet path, blank for corner detection', default='') + parser.add_argument('--PWCNetPath', help='pretrained pwcnet path, blank for KTL tracker', default='') + parser.add_argument('--MotionProPath', help='pretrained motion propagation model path, blank for median', default='') + parser.add_argument('--SingleHomo', help='whether use multi homograph to do motion estimation', action='store_true') + parser.add_argument('--InputBasePath', help='path to input videos (cliped as frames)', default='') + parser.add_argument('--OutputBasePath', help='path to save output stable videos', default='./') + parser.add_argument('--OutNamePrefix', help='prefix name before the output video name', default='') + parser.add_argument('--MaxLength', help='max number of frames can be dealt with one time', type=int, default=1200) + parser.add_argument('--Repeat', help='max number of frames can be dealt with one time', type=int, default=50) + return parser.parse_args() + +def generateStable(model, base_path, outPath, outPrefix, max_length, args): + + image_base_path = base_path + image_len = min(len([ele for ele in os.listdir(image_base_path) if ele[-4:] == '.jpg']), max_length) + # read input video + images = [] + rgbimages = [] + for i in range(image_len): + image = cv2.imread(os.path.join(image_base_path, '{}.jpg'.format(i)), 0) + image = image * (1. / 255.) + image = cv2.resize(image, (cfg.MODEL.WIDTH, cfg.MODEL.HEIGHT)) + images.append(image.reshape(1, 1, cfg.MODEL.HEIGHT, cfg.MODEL.WIDTH)) + + image = cv2.imread(os.path.join(image_base_path, '{}.jpg'.format(i))) + image = cv2.resize(image, (cfg.MODEL.WIDTH, cfg.MODEL.HEIGHT)) + rgbimages.append(np.expand_dims(np.transpose(image, (2, 0, 1)), 0)) + + x = np.concatenate(images, 1).astype(np.float32) + x = torch.from_numpy(x).unsqueeze(0) + + x_RGB = np.concatenate(rgbimages, 0).astype(np.float32) + x_RGB = torch.from_numpy(x_RGB).unsqueeze(0) + + with torch.no_grad(): + origin_motion, smoothPath = model.inference(x.cuda(), x_RGB.cuda(), repeat=args.Repeat) + + origin_motion = origin_motion.cpu().numpy() + smoothPath = smoothPath.cpu().numpy() + origin_motion = np.transpose(origin_motion[0], (2, 3, 1, 0)) + smoothPath = np.transpose(smoothPath[0], (2, 3, 1, 0)) + + x_paths = origin_motion[:, :, :, 0] + y_paths = origin_motion[:, :, :, 1] + sx_paths = smoothPath[:, :, :, 0] + sy_paths = smoothPath[:, :, :, 1] + + frame_rate = 25 + frame_width = cfg.MODEL.WIDTH + frame_height = cfg.MODEL.HEIGHT + + print("generate stabilized video...") + fourcc = cv2.VideoWriter_fourcc(*'MP4V') + out = cv2.VideoWriter(os.path.join(outPath, outPrefix + 'DUT_stable.mp4'), fourcc, frame_rate, (frame_width, frame_height)) + + new_x_motion_meshes = sx_paths - x_paths + new_y_motion_meshes = sy_paths - y_paths + + outImages = warpListImage(rgbimages, new_x_motion_meshes, new_y_motion_meshes) + outImages = outImages.numpy().astype(np.uint8) + outImages = [np.transpose(outImages[idx], (1, 2, 0)) for idx in range(outImages.shape[0])] + for frame in tqdm(outImages): + VERTICAL_BORDER = 60 + HORIZONTAL_BORDER = 80 + + new_frame = frame[VERTICAL_BORDER:-VERTICAL_BORDER, HORIZONTAL_BORDER:-HORIZONTAL_BORDER] + new_frame = cv2.resize(new_frame, (frame.shape[1], frame.shape[0]), interpolation=cv2.INTER_CUBIC) + out.write(new_frame) + out.release() + +if __name__ == "__main__": + + args = parse_args() + print(args) + + smootherPath = args.SmootherPath + RFDetPath = args.RFDetPath + PWCNetPath = args.PWCNetPath + MotionProPath = args.MotionProPath + homo = not args.SingleHomo + inPath = args.InputBasePath + outPath = args.OutputBasePath + outPrefix = args.OutNamePrefix + maxlength = args.MaxLength + + model = DUT(SmootherPath=smootherPath, RFDetPath=RFDetPath, PWCNetPath=PWCNetPath, MotionProPath=MotionProPath, homo=homo) + model.cuda() + model.eval() + + generateStable(model, inPath, outPath, outPrefix, maxlength, args) diff --git a/scripts/StabNetStabilizer.py b/scripts/StabNetStabilizer.py new file mode 100644 index 0000000..dbdf668 --- /dev/null +++ b/scripts/StabNetStabilizer.py @@ -0,0 +1,238 @@ +import torch +import torch.nn as nn +import argparse +from PIL import Image +import cv2 +import os +import traceback +import math +import time +import sys +parentddir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir)) +sys.path.append(parentddir) + +from models.StabNet.v2_93 import * +from models.StabNet.model import stabNet + +parser = argparse.ArgumentParser() +parser.add_argument('--modelPath', default='./models') +parser.add_argument('--before-ch', type=int) +parser.add_argument('--OutputBasePath', default='data_video_local') +parser.add_argument('--InputBasePath', default='') +parser.add_argument('--max-span', type=int, default=1) +parser.add_argument('--refine', type=int, default=1) +parser.add_argument('--no_bm', type=int, default=1) +args = parser.parse_args() + +MaxSpan = args.max_span +args.indices = indices[1:] +batch_size = 1 + +before_ch = max(args.indices)#args.before_ch +after_ch = max(1, -min(args.indices) + 1) + +model = stabNet() +r_model = torch.load(args.modelPath) +model.load_state_dict(r_model) +model.cuda() +model.eval() + +def cvt_img2train(img, crop_rate = 1): + img = Image.fromarray(cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)) + if (crop_rate != 1): + h = int(height / crop_rate) + dh = int((h - height) / 2) + w = int(width / crop_rate) + dw = int((w - width) / 2) + + img = img.resize((w, h), Image.BILINEAR) + img = img.crop((dw, dh, dw + width, dh + height)) + else: + img = img.resize((width, height), Image.BILINEAR) + img = np.array(img) + img = img * (1. / 255) - 0.5 + img = img.reshape((1, height, width, 1)) + return img + +def make_dirs(path): + if not os.path.exists(path): os.makedirs(path) + +cvt_train2img = lambda x: ((np.reshape(x, (height, width)) + 0.5) * 255).astype(np.uint8) + +def warpRevBundle2(img, x_map, y_map): + assert(img.ndim == 3) + assert(img.shape[-1] == 3) + rate = 4 + x_map = cv2.resize(cv2.resize(x_map, (int(width / rate), int(height / rate))), (width, height)) + y_map = cv2.resize(cv2.resize(y_map, (int(width / rate), int(height / rate))), (width, height)) + x_map = (x_map + 1) / 2 * width + y_map = (y_map + 1) / 2 * height + dst = cv2.remap(img, x_map, y_map, cv2.INTER_LINEAR) + assert(dst.shape == (height, width, 3)) + return dst + +production_dir = args.OutputBasePath +make_dirs(production_dir) + +image_len = len([ele for ele in os.listdir(args.InputBasePath) if ele[-4:] == '.jpg']) +images = [] + +for i in range(image_len): + + image = cv2.imread(os.path.join(args.InputBasePath, '{}.jpg'.format(i))) + image = cv2.resize(image, (width, height)) + images.append(image) + +print('inference with {}'.format(args.indices)) + +tot_time = 0 + +print('totally {} frames for stabilization'.format(len(images))) + +before_frames = [] +before_masks = [] +after_frames = [] +after_temp = [] + +cnt = 0 + +frame = images[cnt] + +cnt += 1 + +for i in range(before_ch): + before_frames.append(cvt_img2train(frame, crop_rate)) + before_masks.append(np.zeros([1, height, width, 1], dtype=np.float)) + temp = before_frames[i] + temp = ((np.reshape(temp, (height, width)) + 0.5) * 255).astype(np.uint8) + + temp = np.concatenate([temp, np.zeros_like(temp)], axis=1) + temp = np.concatenate([temp, np.zeros_like(temp)], axis=0) + + +for i in range(after_ch): + frame = images[cnt] + cnt = cnt + 1 + frame_unstable = frame + after_temp.append(frame) + after_frames.append(cvt_img2train(frame, 1)) + +length = 0 +in_xs = [] +delta = 0 + +dh = int(height * 0.8 / 2) +dw = int(width * 0.8 / 2) +all_black = np.zeros([height, width], dtype=np.int64) +frames = [] + +black_mask = np.zeros([dh, width], dtype=np.float) +temp_mask = np.concatenate([np.zeros([height - 2 * dh, dw], dtype=np.float), np.ones([height - 2 * dh, width - 2 * dw], dtype=np.float), np.zeros([height - 2 * dh, dw], dtype=np.float)], axis=1) +black_mask = np.reshape(np.concatenate([black_mask, temp_mask, black_mask], axis=0),[1, height, width, 1]) + +try: + while(True): + + in_x = [] + if input_mask: + for i in args.indices: + if (i > 0): + in_x.append(before_masks[-i]) + for i in args.indices: + if (i > 0): + in_x.append(before_frames[-i]) + in_x.append(after_frames[0]) + for i in args.indices: + if (i < 0): + in_x.append(after_frames[-i]) + if (args.no_bm == 0): + in_x.append(black_mask) + # for i in range(after_ch + 1): + in_x = np.concatenate(in_x, axis = 3) + # for max span + if MaxSpan != 1: + in_xs.append(in_x) + if len(in_xs) > MaxSpan: + in_xs = in_xs[-1:] + print('cut') + in_x = in_xs[0].copy() + in_x[0, ..., before_ch] = after_frames[0][..., 0] + tmp_in_x = np.array(in_x.copy()) + for j in range(args.refine): + start = time.time() + img, black, x_map_, y_map_ = model.forward(torch.Tensor(tmp_in_x.transpose((0, 3, 1, 2))).cuda()) + img = img.cpu().clone().detach().numpy() + black = black.cpu().clone().detach().numpy() + x_map_ = x_map_.cpu().clone().detach().numpy() + y_map_ = y_map_.cpu().clone().detach().numpy() + tot_time += time.time() - start + black = black[0, :, :] + xmap = x_map_[0, :, :, 0] + ymap = y_map_[0, :, :, 0] + all_black = all_black + np.round(black).astype(np.int64) + img = img[0, :, :, :].reshape(height, width) + frame = img + black * (-1) + frame = frame.reshape(1, height, width, 1) + tmp_in_x[..., -1] = frame[..., 0] + img = ((np.reshape(img + 0.5, (height, width))) * 255).astype(np.uint8) + + net_output = img + + img_warped = warpRevBundle2(cv2.resize(after_temp[0], (width, height)), xmap, ymap) + frames.append(img_warped) + + if cnt + 1 <= len(images): + frame_unstable = images[cnt] + cnt = cnt + 1 + ret = True + else: + ret = False + + if (not ret): + break + length = length + 1 + if (length % 10 == 0): + print("length: " + str(length)) + print('fps={}'.format(length / tot_time)) + + before_frames.append(frame) + before_masks.append(black.reshape((1, height, width, 1))) + before_frames.pop(0) + before_masks.pop(0) + after_frames.append(cvt_img2train(frame_unstable, 1)) + after_frames.pop(0) + after_temp.append(frame_unstable) + after_temp.pop(0) +except Exception as e: + traceback.print_exc() +finally: + print('total length={}'.format(length + 2)) + + black_sum = np.zeros([height + 1, width + 1], dtype=np.int64) + for i in range(height): + for j in range(width): + black_sum[i + 1][j + 1] = black_sum[i][j + 1] + black_sum[i + 1][j] - black_sum[i][j] + all_black[i][j] + max_s = 0 + ans = [] + for i in range(0, int(math.floor(height * 0.5)), 10): + print(i) + print(max_s) + for j in range(0, int(math.floor(width * 0.5)), 10): + if (all_black[i][j] > 0): + continue + for hh in range(i, height): + dw = int(math.floor(float(max_s) / (hh - i + 1))) + for ww in range(j, width): + if (black_sum[hh + 1][ww + 1] - black_sum[hh + 1][j] - black_sum[i][ww + 1] + black_sum[i][j] > 0): + break + else: + s = (hh - i + 1) * (ww - j + 1) + if (s > max_s): + max_s = s + ans = [i, j, hh, ww] + videoWriter = cv2.VideoWriter(os.path.join(production_dir, 'StabNet_stable.mp4'), + cv2.VideoWriter_fourcc(*'MP4V'), 25, (ans[3] - ans[1] + 1, ans[2] - ans[0] + 1)) + for frame in frames: + frame_ = frame[ans[0]:ans[2] + 1, ans[1]:ans[3] + 1, :] + videoWriter.write(frame_) + videoWriter.release() \ No newline at end of file diff --git a/scripts/deploy_samples.sh b/scripts/deploy_samples.sh new file mode 100644 index 0000000..f2941f3 --- /dev/null +++ b/scripts/deploy_samples.sh @@ -0,0 +1,49 @@ +#/bin/bash + +OutputBasePath='results/' +SmootherPath='ckpt/smoother.pth' +RFDetPath='ckpt/RFDet_640.pth.tar' +PWCNetPath='ckpt/network-default.pytorch' +MotionProPath='ckpt/MotionPro.pth' +DIFPath='ckpt/DIFNet2.pth' +StabNetPath='ckpt/stabNet.pth' +InputPath='images/' + + +if [ -d "$OutputBasePath" ]; then + echo "Directory exists" ; +else + `mkdir -p $OutputBasePath`; +fi + +# Run the DUT model +echo " Stabiling using the DUT model " +echo "-----------------------------------" + +python ./scripts/DUTStabilizer.py \ + --SmootherPath=$SmootherPath \ + --RFDetPath=$RFDetPath \ + --PWCNetPath=$PWCNetPath \ + --MotionPro=$MotionProPath \ + --InputBasePath=$InputPath \ + --OutputBasePath=$OutputBasePath + +# Run the DIFRINT model +echo " Stabiling using the DIFRINT model " +echo "-----------------------------------" + +python ./scripts/DIFRINTStabilizer.py \ + --modelPath=$DIFPath \ + --InputBasePath=$InputPath \ + --OutputBasePath=$OutputBasePath \ + --cuda + +# Run the StabNet model +echo " Stabiling using the DIFRINT model " +echo "-----------------------------------" + +python ./scripts/StabNetStabilizer.py \ + --modelPath=$StabNetPath \ + --OutputBasePath=$OutputBasePath \ + --InputBasePath=$InputPath + diff --git a/utils/IterativeSmooth.py b/utils/IterativeSmooth.py new file mode 100644 index 0000000..eaf02c7 --- /dev/null +++ b/utils/IterativeSmooth.py @@ -0,0 +1,89 @@ +import torch +import torch.nn as nn +import numpy as np +import os +import math + + +def gauss(t, r=0, window_size=3): + """ + @param window_size is the size of window over which gaussian to be applied + @param t is the index of current point + @param r is the index of point in window + + @return guassian weights over a window size + """ + if np.abs(r-t) > window_size: + return 0 + else: + return np.exp((-9*(r-t)**2)/window_size**2) + + +def generateSmooth(originPath, kernel=None, repeat=20): + # B, 1, T, H, W; B, 6, T, H, W + smooth = originPath + + temp_smooth_3 = originPath[:, :, 3:-3, :, :] + + kernel = kernel + + if kernel is None: + kernel = torch.Tensor([gauss(i) + for i in range(-3, 4)]).to(originPath.device) + kernel = torch.cat([kernel[:3], kernel[4:]]) + kernel = kernel.unsqueeze(0).unsqueeze(2).unsqueeze(3).unsqueeze(4) + kernel = kernel.repeat(*originPath.shape) + + abskernel = torch.abs(kernel) + lambda_t = 100 + + for _ in range(repeat): + # import ipdb; ipdb.set_trace() + temp_smooth = torch.zeros_like(smooth, device=smooth.device) + temp_smooth_0 = smooth[:, :, 0:-6, :, :] * \ + kernel[:, 0:1, 3:-3, :, :] * lambda_t + temp_smooth_1 = smooth[:, :, 1:-5, :, :] * \ + kernel[:, 1:2, 3:-3, :, :] * lambda_t + temp_smooth_2 = smooth[:, :, 2:-4, :, :] * \ + kernel[:, 2:3, 3:-3, :, :] * lambda_t + + temp_smooth_4 = smooth[:, :, 4:-2, :, :] * \ + kernel[:, 3:4, 3:-3, :, :] * lambda_t + temp_smooth_5 = smooth[:, :, 5:-1, :, :] * \ + kernel[:, 4:5, 3:-3, :, :] * lambda_t + temp_smooth_6 = smooth[:, :, 6:, :, :] * \ + kernel[:, 5:6, 3:-3, :, :] * lambda_t + + temp_smooth[:, :, 3:-3, :, :] = ((temp_smooth_0 + temp_smooth_1 + temp_smooth_2 + temp_smooth_3 + temp_smooth_4 + temp_smooth_5 + temp_smooth_6) + / (1 + lambda_t * torch.sum(abskernel[:, :, 3:-3, :, :], dim=1, keepdim=True))) + + # 0 + temp = smooth[:, :, 1:4, :, :] + temp_smooth[:, :, 0, :, :] = (torch.sum(kernel[:, 3:, 0, :, :].unsqueeze( + 1) * temp, 2) * lambda_t + originPath[:, :, 0, :, :]) / (1 + lambda_t * torch.sum(abskernel[:, 3:, 0, :, :].unsqueeze(1), 2)) + # 1 + temp = torch.cat([smooth[:, :, :1, :, :], smooth[:, :, 2:5, :, :]], 2) + temp_smooth[:, :, 1, :, :] = (torch.sum(kernel[:, 2:, 1, :, :].unsqueeze( + 1) * temp, 2) * lambda_t + originPath[:, :, 1, :, :]) / (1 + lambda_t * torch.sum(abskernel[:, 2:, 1, :, :].unsqueeze(1), 2)) + # 2 + temp = torch.cat([smooth[:, :, :2, :, :], smooth[:, :, 3:6, :, :]], 2) + temp_smooth[:, :, 2, :, :] = (torch.sum(kernel[:, 1:, 2, :, :].unsqueeze( + 1) * temp, 2) * lambda_t + originPath[:, :, 2, :, :]) / (1 + lambda_t * torch.sum(abskernel[:, 1:, 2, :, :].unsqueeze(1), 2)) + # -1 + temp = smooth[:, :, -4:-1] + temp_smooth[:, :, -1, :, :] = (torch.sum(kernel[:, :3, -1, :, :].unsqueeze(1) * temp, 2) * lambda_t + + originPath[:, :, -1, :, :]) / (1 + lambda_t * torch.sum(abskernel[:, :3, -1, :, :].unsqueeze(1), 2)) + # -2 + temp = torch.cat([smooth[:, :, -5:-2, :, :], + smooth[:, :, -1:, :, :]], 2) + temp_smooth[:, :, -2, :, :] = (torch.sum(kernel[:, :4, -2, :, :].unsqueeze(1) * temp, 2) * lambda_t + + originPath[:, :, -2, :, :]) / (1 + lambda_t * torch.sum(abskernel[:, :4, -2, :, :].unsqueeze(1), 2)) + # -3 + temp = torch.cat([smooth[:, :, -6:-3, :, :], + smooth[:, :, -2:, :, :]], 2) + temp_smooth[:, :, -3, :, :] = (torch.sum(kernel[:, :5, -3, :, :].unsqueeze(1) * temp, 2) * lambda_t + + originPath[:, :, -3, :, :]) / (1 + lambda_t * torch.sum(abskernel[:, :5, -3, :, :].unsqueeze(1), 2)) + + smooth = temp_smooth + + return smooth diff --git a/utils/MedianFilter.py b/utils/MedianFilter.py new file mode 100644 index 0000000..dce81e0 --- /dev/null +++ b/utils/MedianFilter.py @@ -0,0 +1,326 @@ +import math +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.nn.modules.utils import _pair, _quadruple +import cv2 +import numpy as np + +from configs.config import cfg +from utils.ProjectionUtils import HomoCalc +from utils.ProjectionUtils import HomoProj +from utils.ProjectionUtils import MotionDistanceMeasure + +class MedianPool2d(nn.Module): + """ Median pool (usable as median filter when stride=1) module. + + Args: + kernel_size: size of pooling kernel, int or 2-tuple + stride: pool stride, int or 2-tuple + padding: pool padding, int or 4-tuple (l, r, t, b) as in pytorch F.pad + same: override padding and enforce same padding, boolean + """ + + def __init__(self, kernel_size=3, stride=1, padding=0, same=False): + super(MedianPool2d, self).__init__() + self.k = _pair(kernel_size) + self.stride = _pair(stride) + self.padding = _quadruple(padding) # convert to l, r, t, b + self.same = same + + def _padding(self, x): + if self.same: + ih, iw = x.size()[2:] + if ih % self.stride[0] == 0: + ph = max(self.k[0] - self.stride[0], 0) + else: + ph = max(self.k[0] - (ih % self.stride[0]), 0) + if iw % self.stride[1] == 0: + pw = max(self.k[1] - self.stride[1], 0) + else: + pw = max(self.k[1] - (iw % self.stride[1]), 0) + pl = pw // 2 + pr = pw - pl + pt = ph // 2 + pb = ph - pt + padding = (pl, pr, pt, pb) + else: + padding = self.padding + return padding + + def forward(self, x): + # using existing pytorch functions and tensor ops so that we get autograd, + # would likely be more efficient to implement from scratch at C/Cuda level + x = F.pad(x, self._padding(x), mode='reflect') + x = x.unfold(2, self.k[0], self.stride[0]).unfold( + 3, self.k[1], self.stride[1]) + x = x.contiguous().view(x.size()[:4] + (-1,)).median(dim=-1)[0] + return x + +def SingleMotionPropagate(x_flow, y_flow, pts): + """ + Traditional median filter for motion propagation + @param: x_flow [B, 1, H, W] + @param: y_flow [B, 1, H, W] + @param: pts [B*topk, 4] + """ + + pts = pts.float() + + medfilt = MedianPool2d(same=True) + + _, _, H, W = x_flow.shape + grids = torch.stack(torch.meshgrid( + torch.arange(W), + torch.arange(H)), 0).to(x_flow.device).permute(0, 2, 1) # 2, W, H --> 2, H, W + grids = grids.unsqueeze(0) # 1, 2, H, W + grids = grids.float() + new_points = grids + torch.cat([x_flow, y_flow], 1) # B, 2, H, W + new_points_S = new_points.clone() + new_points = new_points[0, :, pts[:, 2].long(), pts[:, 3].long()].permute(1, 0) # B*topK, 2 + old_points = grids[0, :, pts[:, 2].long(), pts[:, 3].long()].permute(1, 0) # B*topK, 2 + + old_points_numpy = old_points.detach().cpu().numpy() + new_points_numpy = new_points.detach().cpu().numpy() + + # pre-warping with global homography + Homo, state = cv2.findHomography( + old_points_numpy, new_points_numpy, cv2.RANSAC) + + if Homo is None: + Homo = np.array([[1., 0., 0.], + [0., 1., 0.], + [0., 0., 1.]]) + + Homo = torch.from_numpy(Homo.astype(np.float32)).to(old_points.device) + + meshes_x, meshes_y = torch.meshgrid(torch.arange(0, W, cfg.MODEL.PIXELS), + torch.arange(0, H, cfg.MODEL.PIXELS)) + meshes_x = meshes_x.float().permute(1, 0) + meshes_y = meshes_y.float().permute(1, 0) + meshes_x = meshes_x.to(old_points.device) + meshes_y = meshes_y.to(old_points.device) + meshes_z = torch.ones_like(meshes_x).to(old_points.device) + meshes = torch.stack([meshes_x, meshes_y, meshes_z], 0) + + meshes_projected = torch.mm(Homo, meshes.view(3, -1)).view(*meshes.shape) + x_motions = meshes[0, :, :] - meshes_projected[0, :, :] \ + / (meshes_projected[2, :, :] + 1e-5) + y_motions = meshes[1, :, :] - meshes_projected[1, :, :] \ + / (meshes_projected[2, :, :] + 1e-5) + + temp_x_motion = torch.zeros_like(x_motions) + temp_y_motion = torch.zeros_like(x_motions) + + for i in range(x_motions.shape[0]): + for j in range(x_motions.shape[1]): + distance = torch.sqrt( + (pts[:, 2] - i * cfg.MODEL.PIXELS) ** 2 + (pts[:, 3] - j * cfg.MODEL.PIXELS) ** 2) + distance = distance < cfg.MODEL.RADIUS + index = distance.nonzero() # the indexes whose distances are smaller than RADIUS + if index.shape[0] == 0: + continue + old_points_median = pts[index[:, 0].long(), :] # N', 4(B, C, H, W) + dominator = old_points_median[:, 3:4] * Homo[2, 0] + \ + old_points_median[:, 2:3] * Homo[2, 1] + \ + Homo[2, 2] + 1e-5 # N', 1 + x_nominator = old_points_median[:, 3:4] * Homo[0, 0] + \ + old_points_median[:, 2:3] * Homo[0, 1] + Homo[0, 2] + y_nominator = old_points_median[:, 3:4] * Homo[1, 0] + \ + old_points_median[:, 2:3] * Homo[1, 1] + Homo[1, 2] + new_points_homo = torch.cat( + [x_nominator / dominator, y_nominator / dominator], -1) # N', 2 + new_points_flow = new_points_S[0, :, + old_points_median[:, 2].long(), + old_points_median[:, 3].long()].permute(1, 0) # N', 2 + temp_motion = new_points_flow - new_points_homo + temp_x_motion[i, j] = temp_motion[:, 0].median() + temp_y_motion[i, j] = temp_motion[:, 1].median() + + x_motions = x_motions + temp_x_motion + y_motions = y_motions + temp_y_motion + + # apply second median filter (f-2) over the motion mesh for outliers + x_motion_mesh = medfilt(x_motions.unsqueeze(0).unsqueeze(0)) + y_motion_mesh = medfilt(y_motions.unsqueeze(0).unsqueeze(0)) + + return torch.cat([x_motion_mesh, y_motion_mesh], 1) + + +def MultiMotionPropagate(x_flow, y_flow, pts): + """ + Median filter for propagation with multi homography + @param: x_flow B, 1, H, W + @param: y_flow B, 1, H, W + @param: pts B*topk, 4 + """ + + medfilt = MedianPool2d(same=True) + + # spreads motion over the mesh for the old_frame + from sklearn.cluster import KMeans + pts = pts.float() + + B, C, H, W = x_flow.shape + grids = torch.stack(torch.meshgrid( + torch.arange(W), + torch.arange(H)), 0).to(x_flow.device).permute(0, 2, 1) # 2, W, H --> 2, H, W + grids = grids.unsqueeze(0) # 1, 2, H, W + grids = grids.float() + new_points = grids + torch.cat([x_flow, y_flow], 1) # B, 2, H, W + new_points = new_points[0, :, pts[:, 2].long(), pts[:, 3].long()].permute(1, 0) # B*topK, 2 + old_points = grids[0, :, pts[:, 2].long(), pts[:, 3].long()].permute(1, 0) # B*topK, 2 + + old_points_numpy = old_points.detach().cpu().numpy() + new_points_numpy = new_points.detach().cpu().numpy() + motion_numpy = new_points_numpy - old_points_numpy + pred_Y = KMeans(n_clusters=2, random_state=2).fit_predict(motion_numpy) + if np.sum(pred_Y) > cfg.TRAIN.TOPK / 2: + pred_Y = 1 - pred_Y + cluster1_old_points = old_points_numpy[(pred_Y == 0).nonzero()[0], :] + cluster1_new_points = new_points_numpy[(pred_Y == 0).nonzero()[0], :] + + # pre-warping with global homography + Homo, _ = cv2.findHomography( + cluster1_old_points, cluster1_new_points, cv2.RANSAC) + + if Homo is None: + Homo = np.array([[1., 0., 0.], + [0., 1., 0.], + [0., 0., 1.]]) + + dominator = (Homo[2, 0] * old_points_numpy[:, 0] + + Homo[2, 1] * old_points_numpy[:, 1] + Homo[2, 2]) + new_points_projected = np.stack([ + (Homo[0, 0] * old_points_numpy[:, 0] + Homo[0, 1] + * old_points_numpy[:, 1] + Homo[0, 2]) / dominator, + (Homo[1, 0] * old_points_numpy[:, 0] + Homo[1, 1] + * old_points_numpy[:, 1] + Homo[1, 2]) / dominator + ], 1) + + index = (pred_Y == 1).nonzero()[0] + attribute = np.zeros_like(new_points_numpy[:, 0:1]) # N', 1 + old_points_numpy_chosen = old_points_numpy[index, :] + new_points_numpy_chosen = new_points_numpy[index, :] + + cluster1_motion = cluster1_new_points - cluster1_old_points + clsuter2_motion = new_points_numpy_chosen - old_points_numpy_chosen + cluster1_meanMotion = np.mean(cluster1_motion, 0) + cluster2_meanMotion = np.mean(clsuter2_motion, 0) + distanceMeasure = MotionDistanceMeasure( + cluster1_meanMotion, cluster2_meanMotion) + + if np.sum(pred_Y) > cfg.MODEL.THRESHOLDPOINT and distanceMeasure: + + attribute[index, :] = np.expand_dims(np.ones_like(index), 1) + + Homo_2, _ = cv2.findHomography( + old_points_numpy_chosen, new_points_numpy_chosen, cv2.RANSAC) + if Homo_2 is None: + Homo_2 = Homo + + meshes_x, meshes_y = np.meshgrid(np.arange(0, W, cfg.MODEL.PIXELS), + np.arange(0, H, cfg.MODEL.PIXELS)) + + x_dominator = Homo[0, 0] * meshes_x + \ + Homo[0, 1] * meshes_y + Homo[0, 2] + y_dominator = Homo[1, 0] * meshes_x + \ + Homo[1, 1] * meshes_y + Homo[1, 2] + noiminator = Homo[2, 0] * meshes_x + Homo[2, 1] * meshes_y + Homo[2, 2] + + projected_1 = np.reshape( + np.stack([x_dominator / noiminator, y_dominator / noiminator], 2), (-1, 2)) + + x_dominator = Homo_2[0, 0] * meshes_x + \ + Homo_2[0, 1] * meshes_y + Homo_2[0, 2] + y_dominator = Homo_2[1, 0] * meshes_x + \ + Homo_2[1, 1] * meshes_y + Homo_2[1, 2] + noiminator = Homo_2[2, 0] * meshes_x + \ + Homo_2[2, 1] * meshes_y + Homo_2[2, 2] + + projected_2 = np.reshape( + np.stack([x_dominator / noiminator, y_dominator / noiminator], 2), (-1, 2)) + + distance_x = np.expand_dims( + new_points_numpy[:, 0], 0) - np.reshape(meshes_x, (-1, 1)) + distance_y = np.expand_dims( + new_points_numpy[:, 1], 0) - np.reshape(meshes_y, (-1, 1)) + distance = distance_x ** 2 + distance_y ** 2 # N, N' + distance_mask = (distance < (cfg.MODEL.RADIUS ** 2)) # N, N' + distance_mask_value = (distance_mask.astype( + np.float32) * attribute.transpose(1, 0)) # N, N' + distance = np.sum(distance_mask_value, 1) / \ + (np.sum(distance_mask, 1) + 1e-9) # N + + project_pos = np.reshape(np.expand_dims(distance, 1) * projected_2 + np.expand_dims((1 - distance), 1) + * projected_1, (cfg.MODEL.HEIGHT // cfg.MODEL.PIXELS, cfg.MODEL.WIDTH // cfg.MODEL.PIXELS, 2)) + + meshes_projected = torch.from_numpy(project_pos.astype( + np.float32)).to(new_points.device).permute(2, 0, 1) + + meshes_x, meshes_y = torch.meshgrid(torch.arange(0, W, cfg.MODEL.PIXELS), + torch.arange(0, H, cfg.MODEL.PIXELS)) + meshes_x = meshes_x.float().permute(1, 0) + meshes_y = meshes_y.float().permute(1, 0) + meshes_x = meshes_x.to(old_points.device) + meshes_y = meshes_y.to(old_points.device) + meshes = torch.stack([meshes_x, meshes_y], 0) + + x_motions = meshes[0, :, :] - meshes_projected[0, :, :] + y_motions = meshes[1, :, :] - meshes_projected[1, :, :] + + homo_cal = HomoCalc(meshes, meshes_projected) + project_pts = HomoProj(homo_cal, old_points) + new_points_projected = project_pts + + Homo = torch.from_numpy(Homo.astype(np.float32)).to(old_points.device) + + else: + + Homo = torch.from_numpy(Homo.astype(np.float32)).to(old_points.device) + meshes_x, meshes_y = torch.meshgrid(torch.arange(0, W, cfg.MODEL.PIXELS), + torch.arange(0, H, cfg.MODEL.PIXELS)) + meshes_x = meshes_x.float().permute(1, 0) + meshes_y = meshes_y.float().permute(1, 0) + meshes_x = meshes_x.to(old_points.device) + meshes_y = meshes_y.to(old_points.device) + meshes_z = torch.ones_like(meshes_x).to(old_points.device) + meshes = torch.stack([meshes_x, meshes_y, meshes_z], 0) + + meshes_projected = torch.mm( + Homo, meshes.view(3, -1)).view(*meshes.shape) + + x_motions = meshes[0, :, :] - meshes_projected[0, + :, :] / (meshes_projected[2, :, :]) + y_motions = meshes[1, :, :] - meshes_projected[1, + :, :] / (meshes_projected[2, :, :]) + new_points_projected = torch.from_numpy( + new_points_projected).to(old_points.device) + + temp_x_motion = torch.zeros_like(x_motions) + temp_y_motion = torch.zeros_like(x_motions) + + for i in range(x_motions.shape[0]): + for j in range(x_motions.shape[1]): + distance = torch.sqrt((old_points[:, 0] - i * cfg.MODEL.PIXELS) ** 2 + ( + old_points[:, 1] - j * cfg.MODEL.PIXELS) ** 2) + distance = distance < cfg.MODEL.RADIUS # B * topK + index = distance.nonzero() + if index.shape[0] == 0: + continue + + new_points_homo = new_points_projected[index[:, 0].long(), :] + + new_points_flow = new_points[index[:, 0].long(), :] + temp_motion = -(new_points_homo - new_points_flow) + temp_x_motion[i, j] = temp_motion[:, 0].median() + temp_y_motion[i, j] = temp_motion[:, 1].median() + + x_motions = x_motions + temp_x_motion + y_motions = y_motions + temp_y_motion + + # apply second median filter (f-2) over the motion mesh for outliers + x_motion_mesh = medfilt(x_motions.unsqueeze(0).unsqueeze(0)) + y_motion_mesh = medfilt(y_motions.unsqueeze(0).unsqueeze(0)) + + return torch.cat([x_motion_mesh, y_motion_mesh], 1) diff --git a/utils/ProjectionUtils.py b/utils/ProjectionUtils.py new file mode 100644 index 0000000..8f4af4c --- /dev/null +++ b/utils/ProjectionUtils.py @@ -0,0 +1,402 @@ +import torch +import cv2 +import numpy as np +from configs.config import cfg +import math + + +def HomoCalc(grids, new_grids_loc): + """ + @param: grids the location of origin grid vertices [2, H, W] + @param: new_grids_loc the location of desired grid vertices [2, H, W] + + @return: homo_t homograph projection matrix for each grid [3, 3, H-1, W-1] + """ + + _, H, W = grids.shape + + new_grids = new_grids_loc.unsqueeze(0) + + Homo = torch.zeros(1, 3, 3, H-1, W-1).to(grids.device) + + grids = grids.unsqueeze(0) + + try: + # for common cases if all the homograph can be calculated + one = torch.ones_like(grids[:, 0:1, :-1, :-1], device=grids.device) + zero = torch.zeros_like(grids[:, 1:2, :-1, :-1], device=grids.device) + + A = torch.cat([ + torch.stack([grids[:, 0:1, :-1, :-1], grids[:, 1:2, :-1, :-1], one, zero, zero, zero, + -1 * grids[:, 0:1, :-1, :-1] * new_grids[:, 0:1, :-1, :-1], -1 * grids[:, 1:2, :-1, :-1] * new_grids[:, 0:1, :-1, :-1]], 2), # 1, 1, 8, h-1, w-1 + torch.stack([grids[:, 0:1, 1:, :-1], grids[:, 1:2, 1:, :-1], one, zero, zero, zero, + -1 * grids[:, 0:1, 1:, :-1] * new_grids[:, 0:1, 1:, :-1], -1 * grids[:, 1:2, 1:, :-1] * new_grids[:, 0:1, 1:, :-1]], 2), + torch.stack([grids[:, 0:1, :-1, 1:], grids[:, 1:2, :-1, 1:], one, zero, zero, zero, + -1 * grids[:, 0:1, :-1, 1:] * new_grids[:, 0:1, :-1, 1:], -1 * grids[:, 1:2, :-1, 1:] * new_grids[:, 0:1, :-1, 1:]], 2), + torch.stack([grids[:, 0:1, 1:, 1:], grids[:, 1:2, 1:, 1:], one, zero, zero, zero, + -1 * grids[:, 0:1, 1:, 1:] * new_grids[:, 0:1, 1:, 1:], -1 * grids[:, 1:2, 1:, 1:] * new_grids[:, 0:1, 1:, 1:]], 2), + torch.stack([zero, zero, zero, grids[:, 0:1, :-1, :-1], grids[:, 1:2, :-1, :-1], one, + -1 * grids[:, 0:1, :-1, :-1] * new_grids[:, 1:2, :-1, :-1], -1 * grids[:, 1:2, :-1, :-1] * new_grids[:, 1:2, :-1, :-1]], 2), + torch.stack([zero, zero, zero, grids[:, 0:1, 1:, :-1], grids[:, 1:2, 1:, :-1], one, + -1 * grids[:, 0:1, 1:, :-1] * new_grids[:, 1:2, 1:, :-1], -1 * grids[:, 1:2, 1:, :-1] * new_grids[:, 1:2, 1:, :-1]], 2), + torch.stack([zero, zero, zero, grids[:, 0:1, :-1, 1:], grids[:, 1:2, :-1, 1:], one, + -1 * grids[:, 0:1, :-1, 1:] * new_grids[:, 1:2, :-1, 1:], -1 * grids[:, 1:2, :-1, 1:] * new_grids[:, 1:2, :-1, 1:]], 2), + torch.stack([zero, zero, zero, grids[:, 0:1, 1:, 1:], grids[:, 1:2, 1:, 1:], one, + -1 * grids[:, 0:1, 1:, 1:] * new_grids[:, 1:2, 1:, 1:], -1 * grids[:, 1:2, 1:, 1:] * new_grids[:, 1:2, 1:, 1:]], 2), + ], 1).view(8, 8, -1).permute(2, 0, 1) # 1, 8, 8, h-1, w-1 + B_ = torch.stack([ + new_grids[:, 0, :-1, :-1], + new_grids[:, 0, 1:, :-1], + new_grids[:, 0, :-1, 1:], + new_grids[:, 0, 1:, 1:], + new_grids[:, 1, :-1, :-1], + new_grids[:, 1, 1:, :-1], + new_grids[:, 1, :-1, 1:], + new_grids[:, 1, 1:, 1:], + ], 1).view(8, -1).permute(1, 0) # B, 8, h-1, w-1 ==> A @ H = B ==> H = A^-1 @ B + A_inverse = torch.inverse(A) + # B, 8, 8 @ B, 8, 1 --> B, 8, 1 + H_recovered = torch.bmm(A_inverse, B_.unsqueeze(2)) + + H_ = torch.cat([H_recovered, torch.ones_like( + H_recovered[:, 0:1, :], device=H_recovered.device)], 1).view(H_recovered.shape[0], 3, 3) + + H_ = H_.permute(1, 2, 0) + H_ = H_.view(Homo.shape) + Homo = H_ + except: + # if some of the homography can not be calculated + one = torch.ones_like(grids[:, 0:1, 0, 0], device=grids.device) + zero = torch.zeros_like(grids[:, 1:2, 0, 0], device=grids.device) + H_ = torch.eye(3, device=grids.device) + for i in range(H - 1): + for j in range(W - 1): + A = torch.cat([ + torch.stack([grids[:, 0:1, i, j], grids[:, 1:2, i, j], one, zero, zero, zero, + -1 * grids[:, 0:1, i, j] * new_grids[:, 0:1, i, j], -1 * grids[:, 1:2, i, j] * new_grids[:, 0:1, i, j]], 2), + torch.stack([grids[:, 0:1, i+1, j], grids[:, 1:2, i+1, j], one, zero, zero, zero, + -1 * grids[:, 0:1, i+1, j] * new_grids[:, 0:1, i+1, j], -1 * grids[:, 1:2, i+1, j] * new_grids[:, 0:1, i+1, j]], 2), + torch.stack([grids[:, 0:1, i, j+1], grids[:, 1:2, i, j+1], one, zero, zero, zero, + -1 * grids[:, 0:1, i, j+1] * new_grids[:, 0:1, i, j+1], -1 * grids[:, 1:2, i, j+1] * new_grids[:, 0:1, i, j+1]], 2), + torch.stack([grids[:, 0:1, i+1, j+1], grids[:, 1:2, i+1, j+1], one, zero, zero, zero, + -1 * grids[:, 0:1, i+1, j+1] * new_grids[:, 0:1, i+1, j+1], -1 * grids[:, 1:2, i+1, j+1] * new_grids[:, 0:1, i+1, j+1]], 2), + torch.stack([zero, zero, zero, grids[:, 0:1, i, j], grids[:, 1:2, i, j], one, + -1 * grids[:, 0:1, i, j] * new_grids[:, 1:2, i, j], -1 * grids[:, 1:2, i, j] * new_grids[:, 1:2, i, j]], 2), + torch.stack([zero, zero, zero, grids[:, 0:1, i+1, j], grids[:, 1:2, i+1, j], one, + -1 * grids[:, 0:1, i+1, j] * new_grids[:, 1:2, i+1, j], -1 * grids[:, 1:2, i+1, j] * new_grids[:, 1:2, i+1, j]], 2), + torch.stack([zero, zero, zero, grids[:, 0:1, i, j+1], grids[:, 1:2, i, j+1], one, + -1 * grids[:, 0:1, i, j+1] * new_grids[:, 1:2, i, j+1], -1 * grids[:, 1:2, i, j+1] * new_grids[:, 1:2, i, j+1]], 2), + torch.stack([zero, zero, zero, grids[:, 0:1, i+1, j+1], grids[:, 1:2, i+1, j+1], one, + -1 * grids[:, 0:1, i+1, j+1] * new_grids[:, 1:2, i+1, j+1], -1 * grids[:, 1:2, i+1, j+1] * new_grids[:, 1:2, i+1, j+1]], 2), + ], 1) # B, 8, 8 + B_ = torch.stack([ + new_grids[:, 0, i, j], + new_grids[:, 0, i+1, j], + new_grids[:, 0, i, j+1], + new_grids[:, 0, i+1, j+1], + new_grids[:, 1, i, j], + new_grids[:, 1, i+1, j], + new_grids[:, 1, i, j+1], + new_grids[:, 1, i+1, j+1], + ], 1) # B, 8 ==> A @ H = B ==> H = A^-1 @ B + try: + A_inverse = torch.inverse(A) + + # B, 8, 8 @ B, 8, 1 --> B, 8, 1 + H_recovered = torch.bmm(A_inverse, B_.unsqueeze(2)) + + H_ = torch.cat([H_recovered, torch.ones_like(H_recovered[:, 0:1, :]).to( + H_recovered.device)], 1).view(H_recovered.shape[0], 3, 3) + except: + pass + Homo[:, :, :, i, j] = H_ + + homo_t = Homo.view(3, 3, H-1, W-1) + + return homo_t + + +def HomoProj(homo, pts): + """ + @param: homo [3, 3, G_H-1, G_W-1] + @param: pts [N, 2(W, H)] - [:, 0] for width and [:, 1] for height + + @return: projected pts [N, 2(W, H)] - [:, 0] for width and [:, 1] for height + """ + + pts_location_x = (pts[:, 0:1] // cfg.MODEL.PIXELS).long() + pts_location_y = (pts[:, 1:2] // cfg.MODEL.PIXELS).long() + + # if the grid is outside of the image + index = (pts_location_x[:, 0] >= 39).nonzero().long() + pts_location_x[index, :] = 38 + index = (pts_location_y[:, 0] >= 29).nonzero().long() + pts_location_y[index, :] = 28 + + homo = homo.to(pts.device) + + # calculate the projection + x_dominator = pts[:, 0] * homo[0, 0, pts_location_y[:, 0], pts_location_x[:, 0]] + pts[:, 1] * \ + homo[0, 1, pts_location_y[:, 0], pts_location_x[:, 0]] + homo[0, 2, pts_location_y[:, 0], pts_location_x[:, 0]] + y_dominator = pts[:, 0] * homo[1, 0, pts_location_y[:, 0], pts_location_x[:, 0]] + pts[:, 1] * \ + homo[1, 1, pts_location_y[:, 0], pts_location_x[:, 0]] + homo[1, 2, pts_location_y[:, 0], pts_location_x[:, 0]] + noiminator = pts[:, 0] * homo[2, 0, pts_location_y[:, 0], pts_location_x[:, 0]] + pts[:, 1] * \ + homo[2, 1, pts_location_y[:, 0], pts_location_x[:, 0]] + homo[2, 2, pts_location_y[:, 0], pts_location_x[:, 0]] + noiminator = noiminator + + new_kp_x = x_dominator / noiminator + new_kp_y = y_dominator / noiminator + + return torch.stack([new_kp_x, new_kp_y], 1) + + +def multiHomoEstimate(motion, kp): + """ + @param: motion [4, N] + @param: kp [2, N] + """ + + from sklearn.cluster import KMeans + + new_kp = torch.cat([kp[1:2, :], kp[0:1, :]], 0) + motion[2:, :] + new_points_numpy = new_kp.cpu().detach().numpy().transpose(1, 0) + old_points = torch.stack([kp[1, :], kp[0, :]], 1).to(motion.device) + old_points_numpy = torch.cat( + [kp[1:2, :], kp[0:1, :]], 0).cpu().detach().numpy().transpose(1, 0) + motion_numpy = new_points_numpy - old_points_numpy + + pred_Y = KMeans(n_clusters=2, random_state=2).fit_predict(motion_numpy) + if np.sum(pred_Y) > cfg.TRAIN.TOPK / 2: + pred_Y = 1 - pred_Y + cluster1_old_points = old_points_numpy[(pred_Y == 0).nonzero()[0], :] + cluster1_new_points = new_points_numpy[(pred_Y == 0).nonzero()[0], :] + + # pre-warping with global homography + Homo, _ = cv2.findHomography( + cluster1_old_points, cluster1_new_points, cv2.RANSAC) + + if Homo is None: + Homo = np.array([[1., 0., 0.], + [0., 1., 0.], + [0., 0., 1.]]) + + dominator = (Homo[2, 0] * old_points_numpy[:, 0] + + Homo[2, 1] * old_points_numpy[:, 1] + Homo[2, 2]) + new_points_projected = torch.from_numpy(np.stack([ + (Homo[0, 0] * old_points_numpy[:, 0] + Homo[0, 1] + * old_points_numpy[:, 1] + Homo[0, 2]) / dominator, + (Homo[1, 0] * old_points_numpy[:, 0] + Homo[1, 1] + * old_points_numpy[:, 1] + Homo[1, 2]) / dominator + ], 1).astype(np.float32)).to(old_points.device).permute(1, 0) + + index = (pred_Y == 1).nonzero()[0] + attribute = np.zeros_like(new_points_numpy[:, 0:1]) # N', 1 + cluster2_old_points = old_points_numpy[index, :] + cluster2_new_points = new_points_numpy[index, :] + attribute[index, :] = np.expand_dims(np.ones_like(index), 1) + + cluster1_motion = cluster1_new_points - cluster1_old_points + clsuter2_motion = cluster2_new_points - cluster2_old_points + cluster1_meanMotion = np.mean(cluster1_motion, 0) + cluster2_meanMotion = np.mean(clsuter2_motion, 0) + distanceMeasure = MotionDistanceMeasure( + cluster1_meanMotion, cluster2_meanMotion) + + threhold = (np.sum(pred_Y) > cfg.MODEL.THRESHOLDPOINT) and distanceMeasure + + if threhold: + + Homo_2, _ = cv2.findHomography( + cluster2_old_points, cluster2_new_points, cv2.RANSAC) + if Homo_2 is None: + Homo_2 = Homo + + meshes_x, meshes_y = np.meshgrid(np.arange(0, cfg.MODEL.WIDTH, cfg.MODEL.PIXELS), + np.arange(0, cfg.MODEL.HEIGHT, cfg.MODEL.PIXELS)) + + # Use first cluster to do projection + x_dominator = Homo[0, 0] * meshes_x + \ + Homo[0, 1] * meshes_y + Homo[0, 2] + y_dominator = Homo[1, 0] * meshes_x + \ + Homo[1, 1] * meshes_y + Homo[1, 2] + noiminator = Homo[2, 0] * meshes_x + Homo[2, 1] * meshes_y + Homo[2, 2] + + projected_1 = np.reshape( + np.stack([x_dominator / noiminator, y_dominator / noiminator], 2), (-1, 2)) + + # Use second cluster to do projection + x_dominator = Homo_2[0, 0] * meshes_x + \ + Homo_2[0, 1] * meshes_y + Homo_2[0, 2] + y_dominator = Homo_2[1, 0] * meshes_x + \ + Homo_2[1, 1] * meshes_y + Homo_2[1, 2] + noiminator = Homo_2[2, 0] * meshes_x + \ + Homo_2[2, 1] * meshes_y + Homo_2[2, 2] + + projected_2 = np.reshape( + np.stack([x_dominator / noiminator, y_dominator / noiminator], 2), (-1, 2)) + + # Determine use which projected position + distance_x = np.expand_dims( + new_points_numpy[:, 0], 0) - np.reshape(meshes_x, (-1, 1)) + distance_y = np.expand_dims( + new_points_numpy[:, 1], 0) - np.reshape(meshes_y, (-1, 1)) + distance = distance_x ** 2 + distance_y ** 2 # N, N' + distance_mask = (distance < (cfg.MODEL.RADIUS ** 2)) # N, N' + distance_mask_value = (distance_mask.astype( + np.float32) * attribute.transpose(1, 0)) # N, N' + distance = np.sum(distance_mask_value, 1) / \ + (np.sum(distance_mask, 1) + 1e-9) # N + + project_pos = np.reshape(np.expand_dims(distance, 1) * projected_2 + np.expand_dims((1 - distance), 1) + * projected_1, (cfg.MODEL.HEIGHT // cfg.MODEL.PIXELS, cfg.MODEL.WIDTH // cfg.MODEL.PIXELS, 2)) + + meshes_projected = torch.from_numpy(project_pos.astype( + np.float32)).to(old_points.device).permute(2, 0, 1) + + # calculate reference location for each keypoint + meshes_x, meshes_y = torch.meshgrid(torch.arange(0, cfg.MODEL.WIDTH, cfg.MODEL.PIXELS), + torch.arange(0, cfg.MODEL.HEIGHT, cfg.MODEL.PIXELS)) + meshes_x = meshes_x.float().permute(1, 0) + meshes_y = meshes_y.float().permute(1, 0) + meshes_x = meshes_x.to(old_points.device) + meshes_y = meshes_y.to(old_points.device) + meshes = torch.stack([meshes_x, meshes_y], 0) + + x_motions = meshes[0, :, :] - \ + meshes_projected[0, :, :] + y_motions = meshes[1, :, :] - meshes_projected[1, :, :] + + homo_cal = HomoCalc(meshes, meshes_projected) + project_pts = HomoProj(homo_cal, old_points) + new_points_projected = project_pts.to(old_points.device).permute(1, 0) + + else: + Homo = torch.from_numpy(Homo.astype(np.float32)).to( + old_points.device) + meshes_x, meshes_y = torch.meshgrid(torch.arange(0, cfg.MODEL.WIDTH, cfg.MODEL.PIXELS), + torch.arange(0, cfg.MODEL.HEIGHT, cfg.MODEL.PIXELS)) + meshes_x = meshes_x.float().permute(1, 0) + meshes_y = meshes_y.float().permute(1, 0) + meshes_x = meshes_x.to(old_points.device) + meshes_y = meshes_y.to(old_points.device) + meshes_z = torch.ones_like(meshes_x).to(old_points.device) + meshes = torch.stack([meshes_x, meshes_y, meshes_z], 0) + meshes_projected = torch.mm( + Homo, meshes.view(3, -1)).view(*meshes.shape) + + x_motions = meshes[0, :, :] - meshes_projected[0, :, :] / \ + (meshes_projected[2, :, :]) + y_motions = meshes[1, :, :] - meshes_projected[1, :, :] / \ + (meshes_projected[2, :, :]) + + grids = torch.stack(torch.meshgrid( + torch.arange(0, cfg.MODEL.WIDTH, cfg.MODEL.PIXELS), + torch.arange(0, cfg.MODEL.HEIGHT, cfg.MODEL.PIXELS)), 0).to(motion.device).permute(0, 2, 1).reshape(2, -1).permute(1, 0) + + grids = grids.unsqueeze(2).float() # N', 2, 1 + projected_motion = torch.stack( + [x_motions, y_motions], 2).view(-1, 2, 1).to(motion.device) # G_H, G_W, 2 + + redisual_kp_motion = new_points_projected - torch.cat([kp[1:2, :], kp[0:1, :]], 0) + + motion[:2, :] = motion[:2, :] + motion[2:, :] + motion = motion.unsqueeze(0).repeat(grids.shape[0], 1, 1) # N', 4, N + motion[:, :2, :] = (motion[:, :2, :] - grids) / cfg.MODEL.WIDTH + origin_motion = motion[:, 2:, :] / cfg.MODEL.FLOWC + motion[:, 2:, :] = (redisual_kp_motion.unsqueeze( + 0) - motion[:, 2:, :]) / cfg.MODEL.FLOWC + + return motion, projected_motion / cfg.MODEL.FLOWC, origin_motion + + +def singleHomoEstimate(motion, kp): + """ + @param: motion [4, N] + @param: kp [2, N] + """ + new_kp = torch.cat([kp[1:2, :], kp[0:1, :]], 0) + motion[2:, :] + new_points_numpy = new_kp.cpu().detach().numpy().transpose(1, 0) + old_points = torch.stack([kp[1, :], kp[0, :]], 1).to(motion.device) + old_points_numpy = torch.cat( + [kp[1:2, :], kp[0:1, :]], 0).cpu().detach().numpy().transpose(1, 0) + + cluster1_old_points = old_points_numpy + cluster1_new_points = new_points_numpy + + # pre-warping with global homography + Homo, _ = cv2.findHomography( + cluster1_old_points, cluster1_new_points, cv2.RANSAC) + + if Homo is None: + Homo = np.array([[1., 0., 0.], + [0., 1., 0.], + [0., 0., 1.]]) + + dominator = (Homo[2, 0] * old_points_numpy[:, 0] + + Homo[2, 1] * old_points_numpy[:, 1] + Homo[2, 2]) + new_points_projected = torch.from_numpy(np.stack([ + (Homo[0, 0] * old_points_numpy[:, 0] + Homo[0, 1] + * old_points_numpy[:, 1] + Homo[0, 2]) / dominator, + (Homo[1, 0] * old_points_numpy[:, 0] + Homo[1, 1] + * old_points_numpy[:, 1] + Homo[1, 2]) / dominator + ], 1).astype(np.float32)).to(old_points.device).permute(1, 0) + + Homo = torch.from_numpy(Homo.astype(np.float32)).to( + old_points.device) # 3, 3 + meshes_x, meshes_y = torch.meshgrid(torch.arange(0, cfg.MODEL.WIDTH, cfg.MODEL.PIXELS), + torch.arange(0, cfg.MODEL.HEIGHT, cfg.MODEL.PIXELS)) + meshes_x = meshes_x.float().permute(1, 0) + meshes_y = meshes_y.float().permute(1, 0) + meshes_x = meshes_x.to(old_points.device) + meshes_y = meshes_y.to(old_points.device) + meshes_z = torch.ones_like(meshes_x).to(old_points.device) + meshes = torch.stack([meshes_x, meshes_y, meshes_z], + 0) # 3, H // PIXELS, W // PIXELS + meshes_projected = torch.mm(Homo, meshes.view(3, -1)).view(*meshes.shape) + x_motions = meshes[0, :, :] - meshes_projected[0, :, :] / \ + (meshes_projected[2, :, :]) # H//PIXELS, W//PIXELS + y_motions = meshes[1, :, :] - \ + meshes_projected[1, :, :] / (meshes_projected[2, :, :]) + + grids = torch.stack(torch.meshgrid( + torch.arange(0, cfg.MODEL.WIDTH, cfg.MODEL.PIXELS), + torch.arange(0, cfg.MODEL.HEIGHT, cfg.MODEL.PIXELS)), 0).to(motion.device).permute(0, 2, 1).reshape(2, -1).permute(1, 0) # 2, W, H --> 2, H, W --> 2, N' + + grids = grids.unsqueeze(2).float() # N', 2, 1 + projected_motion = torch.stack( + [x_motions, y_motions], 2).view(-1, 2, 1).to(motion.device) # G_H, G_W, 2 + + redisual_kp_motion = new_points_projected - torch.cat([kp[1:2, :], kp[0:1, :]], 0) + + # to kp_flow (kp(t)) location + motion[:2, :] = motion[:2, :] + motion[2:, :] + motion = motion.unsqueeze(0).repeat(grids.shape[0], 1, 1) # N', 4, N + motion[:, :2, :] = (motion[:, :2, :] - grids) / cfg.MODEL.WIDTH + origin_motion = motion[:, 2:, :] / cfg.MODEL.FLOWC + motion[:, 2:, :] = (redisual_kp_motion.unsqueeze( + 0) - motion[:, 2:, :]) / cfg.MODEL.FLOWC + + return motion, projected_motion / cfg.MODEL.FLOWC, origin_motion + + +def MotionDistanceMeasure(motion1, motion2): + """ + MotionDistanceMeasure + @params motion1 np.array(2) (w, h) + @params motion2 np.array(2) (w, h) + + @return bool describe whether the two motion are close or not, True for far and False for close + """ + + mangnitue_motion1 = np.sqrt(np.sum(motion1 ** 2)) + mangnitue_motion2 = np.sqrt(np.sum(motion2 ** 2)) + diff_mangnitude = np.abs(mangnitue_motion1 - mangnitue_motion2) + + rot = lambda x: math.atan2(x[1], x[0]) / math.pi * 180 + rot_motion1 = rot(motion1) + rot_motion2 = rot(motion2) + diff_rot = np.abs(rot_motion1 - rot_motion2) + if diff_rot > 180: + diff_rot = 360 - diff_rot + + return (diff_mangnitude >= cfg.Threshold.MANG) or (diff_rot >= cfg.Threshold.ROT) \ No newline at end of file diff --git a/utils/WarpUtils.py b/utils/WarpUtils.py new file mode 100644 index 0000000..ebf4da8 --- /dev/null +++ b/utils/WarpUtils.py @@ -0,0 +1,63 @@ +import torch +import torch.nn.functional as F +import numpy as np +from tqdm import tqdm + +from configs.config import cfg +from .ProjectionUtils import HomoCalc +from .ProjectionUtils import HomoProj + + +def mesh_warp_frame(frame, x_motion, y_motion): + """ + @param frame current frame [N, 1, H, W] + @param x_motion [N, 1, G_H, G_W] + @param y_motion [N, 1, G_H, G_W] + + @return mesh warping according to given motion + """ + + target_device = frame.device + + src_grids = torch.stack(torch.meshgrid(torch.arange(0, cfg.MODEL.WIDTH, cfg.MODEL.PIXELS, device=target_device), + torch.arange(0, cfg.MODEL.HEIGHT, cfg.MODEL.PIXELS, device=target_device)), 0).permute(0, 2, 1).unsqueeze(0).float() # 2, G_H, G_W + + des_grids = src_grids + torch.cat([x_motion, y_motion], 1) + + projection = [] + + for i in range(des_grids.shape[0]): + homo = HomoCalc(src_grids[0], des_grids[i]) + + origin_kp = torch.stack(torch.meshgrid(torch.arange(0, cfg.MODEL.WIDTH, device=target_device), + torch.arange(0, cfg.MODEL.HEIGHT, device=target_device)), 0).permute(0, 2, 1).float() # 2, H, W + + projected_kp = HomoProj(homo, origin_kp.contiguous().view( + 2, -1).permute(1, 0)).permute(1, 0) + + projection.append(projected_kp.contiguous().view( + *origin_kp.shape).permute(1, 2, 0)) # 2, H, W --> H, W, 2 + projection = torch.stack(projection, 0) + + projection[:, :, :, 0] = projection[:, :, :, 0] / cfg.MODEL.WIDTH * 2. - 1. + projection[:, :, :, 1] = projection[:, :, :, 1] / \ + cfg.MODEL.HEIGHT * 2. - 1. + generated_frame = F.grid_sample(frame, projection, align_corners=True) + + return generated_frame + + +def warpListImage(images, x_motion, y_motion): + """ + @param images List(image [1, 1, H, W]) + @param x_motion [G_H, G_W, N] + @param y_motion [G_H, G_W, N] + """ + + frames = np.concatenate(images, 0) + x_motion = np.expand_dims(np.transpose(x_motion, (2, 0, 1)), 1) + y_motion = np.expand_dims(np.transpose(y_motion, (2, 0, 1)), 1) + frames = torch.from_numpy(frames.astype(np.float32)) + x_motion = torch.from_numpy(x_motion.astype(np.float32)) + y_motion = torch.from_numpy(y_motion.astype(np.float32)) + return mesh_warp_frame(frames, x_motion, y_motion) diff --git a/utils/__init__.py b/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/utils/image_utils.py b/utils/image_utils.py new file mode 100644 index 0000000..c999a3d --- /dev/null +++ b/utils/image_utils.py @@ -0,0 +1,390 @@ +# -*- coding: utf-8 -*- +# @Time : 2018-9-21 14:36 +# @Author : xylon +import torch +from torch.nn import functional as F +from skimage import transform + +from utils.math_utils import L2Norm + + +def clip_patch(kpts_byxc, kpts_scale, kpts_ori, im_info, images, PSIZE): + """ + clip patch from im_C, im_S, im_info, im_raw. + :param kpts_byxc: tensor #(B*topk, 4): the 4 correspond to (b, y, x, 0) each element in it has length B*topk + :param kpts_scale: tensor(B*topk): image scale value corresponding to topk keypoints in all batch + :param kpts_ori: tensor(B*topk, 2): image orintation value corresponding to topk keypoints in all batch + :param im_info: tensor (B, 2): a list contain rescale ratio sh and sw + :param images: tensor(B, 1, H, W): like 960*720 gray image before image rescaled to 320*240 + :param PSIZE: should be cfg.PATCH.size + :return: torch(B*topk, psize, psize): B*topk patch resized + """ + assert kpts_byxc.size(0) == kpts_scale.size(0) + out_width = out_height = PSIZE + device = kpts_byxc.device + B, C, im_height, im_width = images.size() + num_kp = kpts_byxc.size(0) # B*K + max_y = int(im_height - 1) + max_x = int(im_width - 1) + y_t, x_t = torch.meshgrid( + [ + torch.linspace(-1, 1, out_height, dtype=torch.float, device=device), + torch.linspace(-1, 1, out_width, dtype=torch.float, device=device), + ] + ) + one_t = x_t.new_full(x_t.size(), fill_value=1) + x_t = x_t.contiguous().view(-1) + y_t = y_t.contiguous().view(-1) + one_t = one_t.view(-1) + grid = torch.stack((x_t, y_t, one_t)) # (3, out_width*out_height) + grid = grid.view(-1) # (3*out_width*out_height) + grid = grid.repeat(num_kp) # (numkp*3*out_width*out_height) + # [num_kp, 3, 81] # this grid is designed to mask on keypoint from its left-up[-1, -1] to right-bottom[1, 1] + grid = grid.view(num_kp, 3, -1) + + # + # create 6D affine from scale and orientation + # [s, 0, 0] [cos, -sin, 0] + # [0, s, 0] * [sin, cos, 0] + # [0, 0, 1] [0, 0, 1] + # + thetas = torch.eye( + 2, 3, dtype=torch.float, device=device + ) # [[ 1., 0., 0.],[ 0., 1., 0.]] (2, 3) + thetas = thetas.unsqueeze(0).repeat(num_kp, 1, 1) # (num_kp, 2, 3) + im_info = im_info[:, 0].unsqueeze(-1) # (B, 1) + kpts_scale = kpts_scale.view(im_info.size(0), -1) / im_info # (B, topk) + kpts_scale = kpts_scale.view(-1) / 2.0 # (numkp) + thetas = thetas * kpts_scale[:, None, None] + ones = torch.tensor([[[0, 0, 1]]], dtype=torch.float, device=device).repeat( + num_kp, 1, 1 + ) # (numkp, 1, 1) + thetas = torch.cat((thetas, ones), 1) # (num_kp, 3, 3) + # thetas like this + # [sw, 0, 0] + # [0, sh, 0] + # [0, 0, 1] + + if kpts_ori is not None: + cos = kpts_ori[:, 0].unsqueeze(-1) # [num_kp, 1] + sin = kpts_ori[:, 1].unsqueeze(-1) # [num_kp, 1] + zeros = cos.new_full(cos.size(), fill_value=0) + ones = cos.new_full(cos.size(), fill_value=1) + R = torch.cat((cos, -sin, zeros, sin, cos, zeros, zeros, zeros, ones), dim=-1) + R = R.view(-1, 3, 3) + thetas = torch.matmul(thetas, R) + + # Apply transformation to regular grid + # [num_kp,3,3] * [num_kp,3,H*W] = [num_kp, 3, 81] # magnify grid to each keypoint scale + T_g = torch.matmul(thetas, grid) + x = T_g[:, 0, :] # (numkp, 81) + y = T_g[:, 1, :] # (numkp, 81) + + # get each keypoint x + kp_x_ofst = kpts_byxc[:, 2].view(B, -1).float() / im_info # (B, topk) + kp_x_ofst = kp_x_ofst.view(-1, 1) # (numkp, 1) get each keypoint x + # get each keypoint y + kp_y_ofst = kpts_byxc[:, 1].view(B, -1).float() / im_info # (B, topk) + kp_y_ofst = kp_y_ofst.view(-1, 1) # (numkp, 1) get each keypoint y + + # centerize on keypoints + # [num_kp,81] + # [num_kp,1] # move grid center on each keypoint + x = x + kp_x_ofst + # [num_kp,81] + # [num_kp,1] # move grid center on each keypoint + y = y + kp_y_ofst + x = x.view(-1) # [num_kp*81] + y = y.view(-1) # [num_kp*81] + + # interpolation + x0 = x.floor().long() # [num_kp*81] + x1 = x0 + 1 # [num_kp*81] + y0 = y.floor().long() # [num_kp*81] + y1 = y0 + 1 # [num_kp*81] + + x0 = x0.clamp(min=0, max=max_x) # [num_kp*81] + x1 = x1.clamp(min=0, max=max_x) # [num_kp*81] + y0 = y0.clamp(min=0, max=max_y) # [num_kp*81] + y1 = y1.clamp(min=0, max=max_y) # [num_kp*81] + + dim2 = im_width + dim1 = im_width * im_height + batch_inds = kpts_byxc[:, 0].unsqueeze( + -1 + ) # (num_kp, 1) get each keypoint batch number + base = batch_inds.repeat( + 1, out_height * out_width + ) # [num_kp, 81] # means batch indexes correspond to each grid pixel + # [num_kp*81] # correspond to each grid pixel start index if all pixel flatten to a vector + base = base.view(-1) * dim1 + base_y0 = ( + base + y0 * dim2 + ) # correspond each grid pixel y0 pixel if all pixel flatten to a vector + base_y1 = ( + base + y1 * dim2 + ) # correspond each grid pixel y1 pixel if all pixel flatten to a vector + idx_a = ( + base_y0 + x0 + ) # correspond left_up point pixel index if all pixel flatten to a vector + idx_b = base_y1 + x0 # left-bottom pixel + idx_c = base_y0 + x1 # right-up pixel + idx_d = base_y1 + x1 # right-bottom pixel + + im_flat = images.view(-1) # [B*height*width] # flatten all pixel + + # [num_kp*81] # get pixel value in index idx_a + Ia = im_flat.gather(0, idx_a) + # [num_kp*81] # get pixel value in index idx_b + Ib = im_flat.gather(0, idx_b) + # [num_kp*81] # get pixel value in index idx_c + Ic = im_flat.gather(0, idx_c) + # [num_kp*81] # get pixel value in index idx_d + Id = im_flat.gather(0, idx_d) + + x0_f = x0.float() # [num_kp*81] + x1_f = x1.float() # [num_kp*81] + y0_f = y0.float() # [num_kp*81] + y1_f = y1.float() # [num_kp*81] + + # [num_kp*81] # interpolation weight which is the distance from x to x1 times y to y1 + wa = (x1_f - x) * (y1_f - y) + wb = (x1_f - x) * (y - y0_f) # [num_kp*81] # interpolation weight + wc = (x - x0_f) * (y1_f - y) # [num_kp*81] # interpolation weight + wd = (x - x0_f) * (y - y0_f) # [num_kp*81] # interpolation weight + + output = ( + wa * Ia + wb * Ib + wc * Ic + wd * Id + ) # interpolation value in each keypoints grid + output = output.view(num_kp, out_height, out_width) + return output.unsqueeze(1) + + +def warp(im1_data, homo21): + """ + warp im1 to im2 + cause we get pixel valu ein im2 from im1 + so we warp grid in im2 to im1 that we need homo21 + :param im1_data: (B, H, W, C) + :param homo21: (B, 3, 3) + :return: out_image (B, H, W, C) + """ + B, imH, imW, C = im1_data.size() + outH, outW = imH, imW + gy, gx = torch.meshgrid([torch.arange(outH), torch.arange(outW)]) + gx, gy = gx.float().unsqueeze(-1), gy.float().unsqueeze(-1) + ones = gy.new_full(gy.size(), fill_value=1) + grid = torch.cat((gx, gy, ones), -1) # (H, W, 3) + grid = grid.unsqueeze(0) # (1, H, W, 3) + grid = grid.repeat(B, 1, 1, 1) # (B, H, W, 3) + grid = grid.view(grid.size(0), -1, grid.size(-1)) # (B, H*W, 3) + grid = grid.permute(0, 2, 1) # (B, 3, H*W) + grid = grid.type_as(homo21).to(homo21.device) + + # (B, 3, 3) matmul (B, 3, H*W) => (B, 3, H*W) + grid_w = torch.matmul(homo21, grid) + grid_w = grid_w.permute(0, 2, 1) # (B, H*W, 3) + grid_w = grid_w.div(grid_w[:, :, 2].unsqueeze(-1) + 1e-8) # (B, H*W, 3) + grid_w = grid_w.view(B, outH, outW, -1)[:, :, :, :2] # (B, H, W, 2) + grid_w[:, :, :, 0] = grid_w[:, :, :, 0].div(imW - 1) * 2 - 1 + grid_w[:, :, :, 1] = grid_w[:, :, :, 1].div(imH - 1) * 2 - 1 + + out_image = torch.nn.functional.grid_sample( + im1_data.permute(0, 3, 1, 2), grid_w + ) # (B, C, H, W) + + return out_image.permute(0, 2, 3, 1) + + +def filtbordmask(imscore, radius): + bs, height, width, c = imscore.size() + mask = imscore.new_full( + (1, height - 2 * radius, width - 2 * radius, 1), fill_value=1 + ) + mask = F.pad( + input=mask, + pad=(0, 0, radius, radius, radius, radius, 0, 0), + mode="constant", + value=0, + ) + return mask + + +def filter_border(imscore, radius=8): + imscore = imscore * filtbordmask(imscore, radius=radius) + return imscore + + +def nms(input, thresh=0.0, ksize=5): + """ + non maximum depression in each pixel if it is not maximum probability in its ksize*ksize range + :param input: (B, H, W, 1) + :param thresh: float + :param ksize: int + :return: mask (B, H, W, 1) + """ + dtype, device = input.dtype, input.device + batch, height, width, channel = input.size() + pad = ksize // 2 + zeros = torch.zeros_like(input) + input = torch.where(input < thresh, zeros, input) + input_pad = F.pad( + input=input, + pad=(0, 0, 2 * pad, 2 * pad, 2 * pad, 2 * pad, 0, 0), + mode="constant", + value=0, + ) + slice_map = torch.tensor([], dtype=input_pad.dtype, device=device) + for i in range(ksize): + for j in range(ksize): + slice = input_pad[:, i : height + 2 * pad + i, j : width + 2 * pad + j, :] + slice_map = torch.cat((slice_map, slice), -1) + + max_slice = slice_map.max(dim=-1, keepdim=True)[0] + center_map = slice_map[:, :, :, slice_map.size(-1) // 2].unsqueeze(-1) + mask = torch.ge(center_map, max_slice) + + mask = mask[:, pad : height + pad, pad : width + pad, :] + + return mask.type_as(input) + + +def topk_map(maps, k=512): + """ + find the top k maximum pixel probability in a maps + :param maps: (B, H, W, 1) + :param k: int + :return: mask (B, H, W, 1) + """ + batch, height, width, _ = maps.size() + maps_flat = maps.view(batch, -1) + + indices = maps_flat.sort(dim=-1, descending=True)[1][:, :k] + batch_idx = ( + torch.arange(0, batch, dtype=indices.dtype, device=indices.device) + .unsqueeze(-1) + .repeat(1, k) + ) + batch_idx = batch_idx.view(-1).cpu().detach().numpy() + row_idx = indices.contiguous().view(-1).cpu().detach().numpy() + batch_indexes = (batch_idx, row_idx) + + topk_mask_flat = torch.zeros(maps_flat.size(), dtype=torch.uint8).to(maps.device) + topk_mask_flat[batch_indexes] = 1 + + mask = topk_mask_flat.view(batch, height, width, -1) + return mask + + +def get_gauss_filter_weight(ksize, sig): + """ + generate a gaussian kernel + :param ksize: int + :param sig: float + :return: numpy(ksize*ksize) + """ + mu_x = mu_y = ksize // 2 + if sig == 0: + psf = torch.zeros((ksize, ksize)).float() + psf[mu_y, mu_x] = 1.0 + else: + sig = torch.tensor(sig).float() + x = torch.arange(ksize)[None, :].repeat(ksize, 1).float() + y = torch.arange(ksize)[:, None].repeat(1, ksize).float() + psf = torch.exp( + -((x - mu_x) ** 2 / (2 * sig ** 2) + (y - mu_y) ** 2 / (2 * sig ** 2)) + ) + return psf + + +def soft_nms_3d(scale_logits, ksize, com_strength): + """ + calculate probability for each pixel in each scale space + :param scale_logits: (B, H, W, C) + :param ksize: int + :param com_strength: magnify parameter + :return: probability for each pixel in each scale, size is (B, H, W, C) + """ + num_scales = scale_logits.size(-1) + + max_each_scale = F.max_pool2d( + input=scale_logits.permute(0, 3, 1, 2), + kernel_size=ksize, + padding=ksize // 2, + stride=1, + ).permute( + 0, 2, 3, 1 + ) # (B, H, W, C) + max_all_scale, max_all_scale_idx = max_each_scale.max( + dim=-1, keepdim=True + ) # (B, H, W, 1) + exp_maps = torch.exp(com_strength * (scale_logits - max_all_scale)) # (B, H, W, C) + sum_exp = F.conv2d( + input=exp_maps.permute(0, 3, 1, 2).contiguous(), + weight=exp_maps.new_full([1, num_scales, ksize, ksize], fill_value=1).contiguous(), + stride=1, + padding=ksize // 2, + ).permute( + 0, 2, 3, 1 + ) # (B, H, W, 1) + probs = exp_maps / (sum_exp + 1e-8) + return probs + + +def soft_max_and_argmax_1d( + input, orint_maps, scale_list, com_strength1, com_strength2, dim=-1, keepdim=True +): + """ + input should be pixel probability in each scale + this function calculate the final pixel probability summary from all scale and each pixel correspond scale + :param input: scale_probs(B, H, W, 10) + :param orint_maps: (B, H, W, 10, 2) + :param dim: final channel + :param scale_list: scale space list + :param keepdim: kepp dimension + :param com_strength1: magnify argument of score + :param com_strength2: magnify argument of scale + :return: score_map(B, H, W, 1), scale_map(B, H, W, 1), (orint_map(B, H, W, 1, 2)) + """ + inputs_exp1 = torch.exp( + com_strength1 * (input - torch.max(input, dim=dim, keepdim=True)[0]) + ) + input_softmax1 = inputs_exp1 / ( + inputs_exp1.sum(dim=dim, keepdim=True) + 1e-8 + ) # (B, H, W, 10) + + inputs_exp2 = torch.exp( + com_strength2 * (input - torch.max(input, dim=dim, keepdim=True)[0]) + ) + input_softmax2 = inputs_exp2 / ( + inputs_exp2.sum(dim=dim, keepdim=True) + 1e-8 + ) # (B, H, W, 10) + + score_map = torch.sum(input * input_softmax1, dim=dim, keepdim=keepdim) + + scale_list_shape = [1] * len(input.size()) + scale_list_shape[dim] = -1 + scale_list = scale_list.view(scale_list_shape).to(input_softmax2.device) + scale_map = torch.sum(scale_list * input_softmax2, dim=dim, keepdim=keepdim) + + if orint_maps is not None: + orint_map = torch.sum( + orint_maps * input_softmax1.unsqueeze(-1), dim=dim - 1, keepdim=keepdim + ) # (B, H, W, 1, 2) + orint_map = L2Norm(orint_map, dim=-1) + return score_map, scale_map, orint_map + else: + return score_map, scale_map + + +def im_rescale(im, output_size): + h, w = im.shape[:2] + if isinstance(output_size, int): + if h > w: + new_h, new_w = output_size * h / w, output_size + else: + new_h, new_w = output_size, output_size * w / h + else: + new_h, new_w = output_size + new_h, new_w = int(new_h), int(new_w) + img = transform.resize(im, (new_h, new_w), mode="constant") + + return img, h, w, new_w / w, new_h / h diff --git a/utils/math_utils.py b/utils/math_utils.py new file mode 100644 index 0000000..e0123f9 --- /dev/null +++ b/utils/math_utils.py @@ -0,0 +1,125 @@ +# -*- coding: utf-8 -*- +# @Time : 2018-9-21 14:36 +# @Author : xylon +import numpy as np +import torch + + +def distance_matrix_vector(anchor, positive): + """ + Given batch of anchor descriptors and positive descriptors calculate distance matrix + :param anchor: (B, 128) + :param positive: (B, 128) + :return: + """ + eps = 1e-8 + FeatSimi_Mat = 2 - 2 * torch.mm(anchor, positive.t()) # [0, 4] + FeatSimi_Mat = FeatSimi_Mat.clamp(min=eps, max=4.0) + FeatSimi_Mat = torch.sqrt(FeatSimi_Mat) # euc [0, 2] + return FeatSimi_Mat + + +def pairwise_distances(x, y=None): + """ + Input: x is a Nxd matrix + y is an optional Mxd matirx + Output: dist is a NxM matrix where dist[i,j] is the square norm between x[i,:] and y[j,:] + if y is not given then use 'y=x'. + i.e. dist[i,j] = ||x[i,:]-y[j,:]||^2 + """ + x_norm = (x ** 2).sum(1).view(-1, 1) + if y is not None: + y_t = y.transpose(0, 1) + y_norm = (y ** 2).sum(1).view(1, -1) + else: + y_t = x.transpose(0, 1) + y_norm = x_norm.view(1, -1) + + dist = x_norm + y_norm - 2.0 * torch.mm(x, y_t) + eps = 1e-8 + return torch.sqrt(dist.clamp(min=eps, max=np.inf)) + + +def ptCltoCr(leftC, homolr, right_imscale, right_imorint=None, clamp=True): + """ + ptCltoCr is the abbreviation of projective transform keypoints Coordinates in left back to Coordinates in right + :param leftC: tensor #(B*topk, 4): the 4 correspond to (b, y, x, 0) each element in it has length B*topk + :param homolr: torch(B, 3, 3): homogeneous matrix + :param right_imscale: (B, H, W, 1) + :param right_imorint: (B, H, W, 1, 2) + :param clamp: whether clamp rightC_homo + :return: tuple (b, y, x, 0) each element in that has length B*topk + """ + # projective transform im1_C back to im2 called im2_Cw + B, maxh, maxw, C = right_imscale.size() # tuple (b, h, w) max size of image + leftC_homo = leftC.clone() + leftC_homo[:, 3] = leftC_homo[:, 3] + 1 # (B*topk, 4) (b, y, x, 1) + leftC_homo = leftC_homo[:, 1:] # (B*topk, 3) (y, x, 1) + leftC_homo = leftC_homo.index_select( + 1, leftC_homo.new_tensor([1, 0, 2]) + ) # (B*topk, 3) [[x], [y], [1]] + leftC_homo = leftC_homo.view(B, -1, 3) # (B, topk, 3) + leftC_homo = leftC_homo.permute(0, 2, 1) # (B, 3, topk) + + rightC_homo = torch.matmul(homolr, leftC_homo.float()) # (B, 3, topk) (x, y, h) + rightC_homo = rightC_homo.permute(0, 2, 1) # (B, topk, 3) (x, y, h) + # (B, topk, 3) (x, y, h) to 1 + rightC_homo = rightC_homo / (torch.unsqueeze(rightC_homo[:, :, 2], -1) + 1e-8) + rightC_homo = rightC_homo.round().long() + if clamp: + rightC_homo[:, :, 0] = rightC_homo[:, :, 0].clamp(min=0, max=maxw - 1) + rightC_homo[:, :, 1] = rightC_homo[:, :, 1].clamp(min=0, max=maxh - 1) + + topk = rightC_homo.size(1) + batch_v = ( + torch.arange(B, device=rightC_homo.device).view(B, 1, 1).repeat(1, topk, 1) + ) # (B, topk, 1) + # (B, topk, 4) (B, x, y, h) + rightC_homo = torch.cat((batch_v, rightC_homo), -1) + rightC_homo = rightC_homo.contiguous().view(-1, 4) # (B*topk, 4) (B, x, y, h) + rightC_homo = rightC_homo.index_select( + 1, rightC_homo.new_tensor([0, 2, 1, 3]) + ) # (B*topk, 4) (B, y, x, h) + rightC_homo[:, 3] = rightC_homo[:, 3] - 1 # (B*topk, 4) (B, y, x, 0) + + right_imS = right_imscale.view(-1) # (B*H*W) + dim1 = maxw + dim2 = maxh * maxw + scale_idx = rightC_homo[:, 0] * dim2 + rightC_homo[:, 1] * dim1 + rightC_homo[:, 2] + scale_idx = scale_idx.clamp(min=0, max=dim2 * B - 1) + right_imS = right_imS.gather(0, scale_idx) # (B*topk) + + if right_imorint is None: + right_imO = None + else: + right_cos, right_sin = right_imorint.squeeze().chunk( + chunks=2, dim=-1 + ) # each is (B, H, W, 1) + right_cos = right_cos.view(-1) # (B*H*W) + right_sin = right_sin.view(-1) # (B*H*W) + right_cos = right_cos.gather(0, scale_idx) # (B*topk) + right_sin = right_sin.gather(0, scale_idx) # (B*topk) + right_imO = torch.cat( + (right_cos.unsqueeze(-1), right_sin.unsqueeze(-1)), dim=-1 + ) # (B*topk, 2) + + return rightC_homo, right_imS, right_imO + + +def L2Norm(input, dim=-1): + input = input / torch.norm(input, p=2, dim=dim, keepdim=True) + return input + + +def MSD(x, y): + """ + mean square distance + :param x: (B, H, W, 2) 2 corresponds to XY + :param y: (B, H, W, 2) 2 corresponds to XY + :return: distance: (B, H, W, 1) + """ + sub = x - y + square = sub ** 2 + sm = square.sum(keepdim=True, dim=-1) + sqr = torch.sqrt((sm + 1e-8).float()) + return sqr * 2