initial commit

.gitignore

@@ -0,0 +1,132 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+ckpt/
+results/
+DIFRINT_TEMP/

README.md

@@ -0,0 +1,97 @@
+<h1 align="left">DUT: Learning Video Stabilization by Simply Watching Unstable Videos <a href=""><img  src="https://img.shields.io/badge/arXiv-Paper-<COLOR>.svg" ></a> <a href="https://colab.research.google.com/drive/1i-cK-6uFKbWRjxF26uxUqHHSEvq1Ln8h?usp=sharing">
+  <img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/>
+</a> </h1> 
+
+<p align="center">
+  <a href="#introduction">Introduction</a> |
+  <a href="#results-demo">Results Demo</a> |
+  <a href="#installation">Installation</a> |
+  <a href="#inference-code">Inference Code</a> |
+  <a href="#news">News</a> |
+  <a href="#statement">Statement</a> |
+  <a href="#reference">Reference</a>
+</p>
+
+## Introduction
+
+<p align="justify">This repository contains the code, models, test results for the paper <a href="">DUT: Learning Video Stabilization by Simply Watching Unstable Videos</a>. It contains a keypoint detetcion module for robust keypoint detection, a motion propagation module for grid-based trajectories estimation, and a trajectory smoothing module for dynamic trajectory smoothing. The DUT model is totally unsupervised and just need unstable videos for training.
+
+<p align="justify">We have released the inference code, a pretrained model, which can be found in section <a href="#inference-code"><i>inference code</i></a> for more details. The inference code for DIFRINT and StabNet are also included in this repo.</p>
+
+<img src="demo/NetworkStructure.png">
+
+## Results Demo
+
+We test DUT on the NUS dataset and provide a result overview below (odd rows for unstable inputs and even rows for stabilized videos).
+
+<img src="demo/overview1.gif"><img src="demo/overview2.gif"><img src="demo/overview3.gif">
+
+For the motion propagation module, we propose a multi-homography estimation strategy instead of directly using RANSAC for single homography estimation to deal with multiple planes cases. The outliers recognized by RANSAC and the multi-planes identified with the multi-homography strategy are as follows:
+
+<img src="demo/RANSAC.gif" class="left" width='40%'><img src="demo/Cluster.gif" class="right" width='40%'>
+
+The trajectory smoothing module predicts dynamic kernels for iterative smoothing according to the trajectories themselves, the smoothing process can be viewed below:
+
+<img src="demo/Trajectory.gif" width='80%'>
+
+## Installation
+Requirements:
+
+- Python 3.6.5+
+- Pytorch (version 1.4.0)
+- Torchvision (version 0.5.0)
+
+1. Clone this repository
+
+    `git clone https://github.com/Annbless/DUTCode.git`
+
+2. Go into the repository
+
+    `cd DUTCode`
+
+3. Create conda environment and activate
+
+    `conda create -n DUTCode python=3.6.5`
+
+    `conda activate DUTCode`
+
+4. Install dependencies
+
+    `conda install pytorch==1.4.0 torchvision==0.5.0 cudatoolkit=10.1 -c pytorch`
+
+    `pip install -r requirements.txt`
+
+Our code has been tested with Python 3.6.5, Pytorch 1.4.0, Torchvision 0.5.0, CUDA 10.1 on Ubuntu 18.04.
+
+
+## Inference Code
+
+Here we provide the procedure of testing on sample videos by our pretrained models:
+
+1. Download <a href="https://drive.google.com/drive/folders/15T8Wwf1OL99AKDGTgECzwubwTqbkmGn6?usp=sharing">pretrained model</a> and unzip them to ckpt folder
+
+2. Put clipped frames from unstable videos in the images folder (we have provided one example in the folder)
+
+3. Run the stabilization script
+
+    `chmod +x scripts/*`
+
+    `./scripts/deploy_samples.sh`
+
+4. The results of our DUT, DIFRINT, and StabNet are provided under the results folder
+
+## News
+- [x] release inference code for the DUT model;
+- [x] include inference code for the DIFRINT model, thanks to <a href="https://github.com/jinsc37">jinsc37</a>;
+- [x] include pytorch version StabNet model, thanks to <a href="https://github.com/cxjyxxme">cxjyxxme</a>;
+- [ ] include traditional stabilizers in python;
+- [ ] release metrics calculation code;
+- [ ] release training code for the DUT model;
+
+## Statement
+This project is for research purpose only, please contact us for the licence of commercial use. For any other questions please contact [[email protected]](mailto:[email protected]).
+
+## Reference
+
+- <a href="https://github.com/jinsc37/DIFRINT">DIFRINT</a>
+- <a href="https://github.com/cxjyxxme/deep-online-video-stabilization-deploy">StabNet (tensorflow version)</a>

configs/config.py

@@ -0,0 +1,89 @@
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from easydict import EasyDict as edict
+
+__C = edict()
+cfg = __C
+
+"""
+Model options
+"""
+__C.MODEL = edict()
+
+# gaussian kernel size
+__C.MODEL.GAUSSIAN_KSIZE = 15
+
+# gaussian kernel sigma
+__C.MODEL.GAUSSIAN_SIGMA = 0.5
+
+# Descriptor Threshold
+__C.MODEL.DES_THRSH = 1.0
+
+# Coordinate Threshold
+__C.MODEL.COO_THRSH = 5.0
+
+# Ksize
+__C.MODEL.KSIZE = 3
+
+# padding
+__C.MODEL.padding = 1
+
+# dilation
+__C.MODEL.dilation = 1
+
+# scale_list
+__C.MODEL.scale_list = [3.0, 5.0, 7.0, 9.0, 11.0, 13.0, 15.0, 17.0, 19.0, 21.0]
+
+# grid size
+__C.MODEL.PIXELS = 16
+
+# neighbor radius
+__C.MODEL.RADIUS = 200
+
+# input height
+__C.MODEL.HEIGHT = 480
+
+# input width
+__C.MODEL.WIDTH = 640
+
+# normalization amplitude in optical flow
+__C.MODEL.FLOWC = 20
+
+# cluster threshold
+__C.MODEL.THRESHOLDPOINT = 102
+
+"""
+Training options
+"""
+__C.TRAIN = edict()
+
+# score strength weight
+__C.TRAIN.score_com_strength = 100.0
+
+# scale strength weight
+__C.TRAIN.scale_com_strength = 100.0
+
+# non maximum supression threshold
+__C.TRAIN.NMS_THRESH = 0.0
+
+# nms kernel size
+__C.TRAIN.NMS_KSIZE = 5
+
+# top k patch
+__C.TRAIN.TOPK = 512
+
+"""
+Threshold options
+"""
+__C.Threshold = edict()
+
+__C.Threshold.MANG = 2
+__C.Threshold.ROT = 5
+
+"""
+Infer options
+"""
+__C.INFER = edict()
+__C.INFER.ALLINFER = False