diff --git a/.gitignore b/.gitignore index 72669209..2fbe70aa 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ -/weights +*.pth *.pyc *._ __pycache__ diff --git a/evaluate/readme.md b/evaluate/readme.md index 689e0088..8ed2e1b3 100644 --- a/evaluate/readme.md +++ b/evaluate/readme.md @@ -66,7 +66,8 @@ Only `--bop` is needed to be passed to load a bop scene. You can pass which scen We assume that you have the intrinsics stored in the camera data. If you do not have them, the script uses 512 x 512 with a fov of 0.78. If the camera data is complete, like with NViSII data, it will use the camera intrinsics. + +- make a script to visualize the json files from DOPE --> diff --git a/inference/README.md b/inference/README.md index aabbed1b..9608e374 100644 --- a/inference/README.md +++ b/inference/README.md @@ -22,7 +22,7 @@ The `inference.py` script will take a trained model to run inference. In order t Below is an example of running inference: ``` -python inference.py --weights ../output/weights --data ../sample_data --object cracker +python inference.py --weights ../weights --data ../sample_data --object cracker ``` ### Configuration Files diff --git a/readme.md b/readme.md index 25e7b1ad..ba44962f 100644 --- a/readme.md +++ b/readme.md @@ -1,79 +1,48 @@ [![License CC BY-NC-SA 4.0](https://img.shields.io/badge/License-CC%20BY--NC--SA%204.0-blue.svg)](https://creativecommons.org/licenses/by-nc-sa/4.0/legalcode) ![Python 3.8](https://img.shields.io/badge/python-3.8-blue.svg) -# Deep Object Pose Estimation - ROS Inference +# Deep Object Pose Estimation -This is the official DOPE ROS package for detection and 6-DoF pose estimation of **known objects** from an RGB camera. The network has been trained on the following YCB objects: cracker box, sugar box, tomato soup can, mustard bottle, potted meat can, and gelatin box. For more details, see our [CoRL 2018 paper](https://arxiv.org/abs/1809.10790) and [video](https://youtu.be/yVGViBqWtBI). +This is the official DOPE ROS package for detection and 6-DoF pose estimation of **known objects** from an RGB camera. For full details, see our [CoRL 2018 paper](https://arxiv.org/abs/1809.10790) and [video](https://youtu.be/yVGViBqWtBI). ![DOPE Objects](dope_objects.png) -## Updates -2024/03/07 - New training code. New synthetic data generation code, using Blenderproc. Repo reorganization +## Contents +This repository contains complete code for [training](train), [inference](inference), numerical [evaluation](evaluate) of results, and synthetic [data generation](data_generation) using either [NVISII](https://github.com/owl-project/NVISII) or [Blenderproc](https://github.com/DLR-RM/BlenderProc). We also provide a [ROS1 Noetic package](ros1) that performs inference on images from a USB camera. -2022/07/13 - Added a script with a simple example for computing the ADD and ADD-S metric on data. Please refer to [script/metrics/](https://github.com/NVlabs/Deep_Object_Pose/tree/master/scripts/metrics). +Hardware-accelerated ROS2 inference can be done with the +[Isaac ROS DOPE](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_pose_estimation/tree/main/isaac_ros_dope) project. -2022/03/30 - Update on the NViSII script to handle [symmetrical objects](https://github.com/NVlabs/Deep_Object_Pose/tree/master/scripts/nvisii_data_gen#handling-objects-with-symmetries). Also the NViSII script is compatible with the original training script. Thanks to Martin Günther. - -2021/12/13 - Added a NViSII script to generate synthetic data for training DOPE. See this [readme](https://github.com/NVlabs/Deep_Object_Pose/tree/master/scripts/nvisii_data_gen) for more details. We also added the update training and inference (without ROS) scripts for the NViSII paper [here](https://github.com/NVlabs/Deep_Object_Pose/tree/master/scripts/train2). - -2021/10/20 - Added ROS2 Foxy inference support through [Isaac ROS DOPE package](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_pose_estimation) for Jetson and x86+CUDA-capable GPU. - -2021/08/07 - Added publishing belief maps. Thank you to Martin Günther. - -2020/03/09 - Added HOPE [weights to google drive](https://drive.google.com/open?id=1DfoA3m_Bm0fW8tOWXGVxi4ETlLEAgmcg), [the 3d models](https://drive.google.com/drive/folders/1jiJS9KgcYAkfb8KJPp5MRlB0P11BStft), and the objects dimensions to config. [Tremblay et al., IROS 2020](https://arxiv.org/abs/2008.11822). The HOPE dataset can be found [here](https://github.com/swtyree/hope-dataset/) and is also part of the [BOP challenge](https://bop.felk.cvut.cz/datasets/#HOPE) - - -
-
## Tested Configurations -We have tested on Ubuntu 20.04 with ROS Noetic with an NVIDIA Titan X and RTX 2080ti with Python 3.8. The code may work on other systems. +We have tested our standalone training, inference and evaluation scripts on Ubuntu 20.04 and 22.04 with Python 3.8+, using an NVIDIA Titan X, 2080Ti, and Titan RTX. ---- -***NOTE*** - -For hardware-accelerated ROS2 inference support, please visit [Isaac ROS DOPE](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_pose_estimation/tree/main/isaac_ros_dope) which has been tested with ROS2 Foxy on Jetson AGX Xavier/JetPack 4.6 and on x86/Ubuntu 20.04 with RTX3060i. +The ROS1 node has been tested with ROS Noetic using Python 3.10. The Isaac ROS2 DOPE node has been tested with ROS2 Foxy on Jetson AGX Xavier with JetPack 4.6; and on x86/Ubuntu 20.04 with a NVIDIA Titan X, 2080Ti, and Titan RTX. ---- -
-
-## Synthetic Data Generation -Code and instructions for generating synthetic training data are found in the `data_generation` directory. There are two options for the render engine: you can use [NVISII](https://github.com/owl-project/NVISII) or [Blenderproc](https://github.com/DLR-RM/BlenderProc) +## Datasets -## Training -Code and instructions for training DOPE are found in the `train` directory. +We have trained and tested DOPE with two publicaly available datasets: YCB, and HOPE. The trained weights can be [downloaded from Google Drive](https://drive.google.com/drive/folders/1DfoA3m_Bm0fW8tOWXGVxi4ETlLEAgmcg). -## Inference -Code and instructions for command-line inference using PyTorch are found in the `inference` directory -## Evaluation -Code and instructions for evaluating the quality of your results are found in the `evaluate` directory ---- +### YCB 3D Models +YCB models can be downloaded from the [YCB website](http://www.ycbbenchmarks.com/), or by using [NVDU](https://github.com/NVIDIA/Dataset_Utilities) (see the `nvdu_ycb` command). -## YCB 3D Models -DOPE returns the poses of the objects in the camera coordinate frame. DOPE uses the aligned YCB models, which can be obtained using [NVDU](https://github.com/NVIDIA/Dataset_Utilities) (see the `nvdu_ycb` command). +### HOPE 3D Models +The [HOPE dataset](https://github.com/swtyree/hope-dataset/) is a collection of RGBD images and video sequences with labeled 6-DoF poses for 28 toy grocery objects. The 3D models [can be downloaded here](https://drive.google.com/drive/folders/1jiJS9KgcYAkfb8KJPp5MRlB0P11BStft). +The folders are organized in the style of the YCB 3d models. ---- +The physical objects can be purchased online (details and links to Amazon can be found in the [HOPE repository README](https://github.com/swtyree/hope-dataset/). -## HOPE 3D Models +

-![HOPE 3D models rendered in UE4](https://i.imgur.com/V6wX64p.png) +--- -We introduce new toy 3d models that you download [here](https://drive.google.com/drive/folders/1jiJS9KgcYAkfb8KJPp5MRlB0P11BStft). -The folders are arranged like the YCB 3d models organization. -You can buy the real objects using the following links -[set 1](https://www.amazon.com/gp/product/B071ZMT9S2), -[set 2](https://www.amazon.com/gp/product/B007EA6PKS), -[set 3](https://www.amazon.com/gp/product/B00H4SKSPS), -and -[set 4](https://www.amazon.com/gp/product/B072M2PGX9). -The HOPE dataset can be found [here](https://github.com/swtyree/hope-dataset/) and is also part of the [BOP challenge](https://bop.felk.cvut.cz/datasets/#HOPE). ## How to cite DOPE @@ -90,7 +59,7 @@ If you use this tool in a research project, please cite as follows: ## License -Copyright (C) 2018 NVIDIA Corporation. All rights reserved. Licensed under the [CC BY-NC-SA 4.0 license](https://creativecommons.org/licenses/by-nc-sa/4.0/legalcode). +Copyright (C) 2018-2024 NVIDIA Corporation. All rights reserved. This code is licensed under the [NVIDIA Source Code License](https://github.com/NVlabs/HANDAL/blob/main/LICENSE.txt). ## Acknowledgment diff --git a/train/README.md b/train/README.md index 7190f3ca..187658bc 100644 --- a/train/README.md +++ b/train/README.md @@ -1,6 +1,6 @@ # Deep Object Pose Estimation (DOPE) - Training -This repo contains a simplified version of the **training** script for DOPE. +This repo contains a simplified version of the training pipeline for DOPE. Scripts for inference, evaluation, and data visualization can be found in this repo's top-level directories `inference` and `evaluate`. A user report of training DOPE on a single GPU using NVISII-created synthetic data can [be found here](https://github.com/NVlabs/Deep_Object_Pose/issues/155#issuecomment-791148200). @@ -18,7 +18,7 @@ source ./output/dope_training/bin/activate --- To install the required dependencies, run: ``` -pip install -r requirements.txt +pip install -r ../requirements.txt ``` ## Training @@ -65,6 +65,3 @@ python debug.py --data PATH_TO_IMAGES 2. If you are running into dependency issues when installing, you can try to install the version specific dependencies that are commented out in `requirements.txt`. Be sure to do this in a virtual environment. -## License - -Copyright (C) 2018 NVIDIA Corporation. All rights reserved. Licensed under the [CC BY-NC-SA 4.0 license](https://creativecommons.org/licenses/by-nc-sa/4.0/legalcode). diff --git a/train/run_pipeline_on_ngc.py b/train/run_pipeline_on_ngc.py deleted file mode 100644 index 878e7bfb..00000000 --- a/train/run_pipeline_on_ngc.py +++ /dev/null @@ -1,85 +0,0 @@ -# This script is intended to make it easier to run the training, inference, and evaluation pipeline on NGC - -if __name__ == "__main__": - - import argparse - import subprocess - - parser = argparse.ArgumentParser() - - parser.add_argument("--endpoint", "--endpoint_url", type=str, required=True) - parser.add_argument( - "--train_buckets", - nargs="+", - help="s3 buckets containing training data. Can list multiple buckets separated by a space.", - required=True - ) - parser.add_argument( - "--output_bucket", - default="output_bucket", - help="Name of the bucket to write output to.", - required=True - ) - parser.add_argument( - "--object", - required=True, - help='Object to train network for. Must match "class" field in groundtruth .json file. For best performance, only put one object of interest.', - ) - parser.add_argument( - "--inference_bucket", - required=True, - help="Bucket that stores inference data.", - ) - parser.add_argument( - "--num_gpus", type=int, help="number of GPUs to be used in training.", default=1 - ) - parser.add_argument( - "--epochs", - type=int, - default=60, - help="Number of epochs to train for", - ) - parser.add_argument( - "--batchsize", "--batch_size", type=int, default=32, help="input batch size" - ) - - opt = parser.parse_args() - opt.train_buckets = " ".join(opt.train_buckets) - - # Run Training - train_command = ["python", "-m", "torch.distributed.launch", f"--nproc_per_node={opt.num_gpus}", "train.py"] - train_command += ["--use_s3"] - train_command += ["--endpoint", f"{opt.endpoint}"] - train_command += ["--train_buckets", f"{opt.train_buckets}"] - train_command += ["--object", f"{opt.object}"] - train_command += ["--batchsize", f"{opt.batchsize}"] - # 1 epoch on n GPUs is equivalent to n epochs on 1 GPU - train_command += ["--epochs", f"{opt.epochs // opt.num_gpus }"] - - subprocess.run(train_command) - - # copy inference data locally - subprocess.run(["mkdir", "sample_data/inference_data"]) - subprocess.run(["s3cmd", "sync", f"s3://{opt.inference_bucket}", "sample_data/inference_data"]) - - # Run Inference - inference_command = ["python", "inference.py"] - inference_command += ["--weights", "../output/weights"] - inference_command += ["--data", "../sample_data/inference_data"] - inference_command += ["--object", f"{opt.object}"] - - subprocess.run(inference_command, cwd="./inference") - - # Run Evaluate - evaluate_command = ["python", "evaluate.py"] - evaluate_command += ["--data_prediction", "../inference/output"] - evaluate_command += ["--data", "../sample_data/inference_data"] - evaluate_command += ["--outf", "../output"] - evaluate_command += ["--cuboid"] - - subprocess.run(evaluate_command, cwd="./evaluate") - - # Upload Results to s3 - subprocess.run(f"s3cmd mb s3://{opt.output_bucket}".split(" ")) - subprocess.run(f"s3cmd sync output/ s3://{opt.output_bucket}".split(" ")) - \ No newline at end of file diff --git a/weights/readme.md b/weights/readme.md index 7d1e6d35..76fcb387 100644 --- a/weights/readme.md +++ b/weights/readme.md @@ -1 +1,4 @@ -This is where you need to store the weights. +We have trained and tested DOPE with two publicaly available datasets: YCB, and HOPE. These trained weights can be +[downloaded from Google Drive](https://drive.google.com/drive/folders/1DfoA3m_Bm0fW8tOWXGVxi4ETlLEAgmcg). + +