diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..4ca178e --- /dev/null +++ b/.gitmodules @@ -0,0 +1,4 @@ +[submodule "tensorflow_yolov3"] + path = tensorflow_yolov3 + url = https://github.com/umtclskn/tensorflow-yolov3.git + branch = carla_simulator_yolov3 diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..608c451 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2020 Umut Çalışkan + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/carla_yolov3.png b/carla_yolov3.png new file mode 100644 index 0000000..2823449 Binary files /dev/null and b/carla_yolov3.png differ diff --git a/yolov3_object_detection.py b/yolov3_object_detection.py new file mode 100644 index 0000000..3ebbe9f --- /dev/null +++ b/yolov3_object_detection.py @@ -0,0 +1,298 @@ +#!/usr/bin/env python + +# Copyright (c) 2019 Aptiv +# +# This work is licensed under the terms of the MIT license. +# For a copy, see . + +""" +An example of client-side bounding boxes with basic car controls. +Controls: + W : throttle + S : brake + AD : steer + Space : hand-brake + ESC : quit +""" + +# ============================================================================== +# -- find carla module --------------------------------------------------------- +# ============================================================================== + + +import cv2 +import time +import numpy as np +import tensorflow_yolov3.carla.utils as utils + +import tensorflow as tf +from PIL import Image + + +import glob +import os +import sys + +try: + sys.path.append(glob.glob('../carla/dist/carla-*%d.%d-%s.egg' % ( + sys.version_info.major, + sys.version_info.minor, + 'win-amd64' if os.name == 'nt' else 'linux-x86_64'))[0]) +except IndexError: + pass + + +# ============================================================================== +# -- imports ------------------------------------------------------------------- +# ============================================================================== + +import carla + +import weakref +import random + +try: + import pygame + from pygame.locals import K_ESCAPE + from pygame.locals import K_SPACE + from pygame.locals import K_a + from pygame.locals import K_d + from pygame.locals import K_s + from pygame.locals import K_w +except ImportError: + raise RuntimeError('cannot import pygame, make sure pygame package is installed') + +try: + import numpy as np +except ImportError: + raise RuntimeError('cannot import numpy, make sure numpy package is installed') + +VIEW_WIDTH = 1920//2 +VIEW_HEIGHT = 1080//2 +VIEW_FOV = 90 + +BB_COLOR = (248, 64, 24) + +# ============================================================================== +# -- BasicSynchronousClient ---------------------------------------------------- +# ============================================================================== + + +class BasicSynchronousClient(object): + """ + Basic implementation of a synchronous client. + """ + + def __init__(self): + self.client = None + self.world = None + self.camera = None + self.car = None + + self.display = None + self.image = None + self.raw_image = None + self.capture = True + + def camera_blueprint(self): + """ + Returns camera blueprint. + """ + + camera_bp = self.world.get_blueprint_library().find('sensor.camera.rgb') + camera_bp.set_attribute('image_size_x', str(VIEW_WIDTH)) + camera_bp.set_attribute('image_size_y', str(VIEW_HEIGHT)) + camera_bp.set_attribute('fov', str(VIEW_FOV)) + + return camera_bp + + def set_synchronous_mode(self, synchronous_mode): + """ + Sets synchronous mode. + """ + settings = self.world.get_settings() + settings.synchronous_mode = synchronous_mode + self.world.apply_settings(settings) + + def setup_car(self): + """ + Spawns actor-vehicle to be controled. + """ + + car_bp = self.world.get_blueprint_library().filter('vehicle.*')[0] + location = random.choice(self.world.get_map().get_spawn_points()) + self.car = self.world.spawn_actor(car_bp, location) + + def setup_camera(self): + """ + Spawns actor-camera to be used to render view. + Sets calibration for client-side boxes rendering. + """ + + #camera_transform = carla.Transform(carla.Location(x=-5.5, z=2.8), carla.Rotation(pitch=-15)) + #First person view transform settings + camera_transform = carla.Transform(carla.Location(x=1.6, z=1.7), carla.Rotation(pitch=0)) + self.camera = self.world.spawn_actor(self.camera_blueprint(), camera_transform, attach_to=self.car) + weak_self = weakref.ref(self) + self.camera.listen(lambda image: weak_self().set_image(weak_self, image)) + + calibration = np.identity(3) + calibration[0, 2] = VIEW_WIDTH / 2.0 + calibration[1, 2] = VIEW_HEIGHT / 2.0 + calibration[0, 0] = calibration[1, 1] = VIEW_WIDTH / (2.0 * np.tan(VIEW_FOV * np.pi / 360.0)) + self.camera.calibration = calibration + + + def control(self, car): + """ + Applies control to main car based on pygame pressed keys. + Will return True If ESCAPE is hit, otherwise False to end main loop. + """ + + keys = pygame.key.get_pressed() + if keys[K_ESCAPE]: + return True + + control = car.get_control() + control.throttle = 0 + if keys[K_w]: + control.throttle = 1 + control.reverse = False + elif keys[K_s]: + control.throttle = 1 + control.reverse = True + if keys[K_a]: + control.steer = max(-1., min(control.steer - 0.05, 0)) + elif keys[K_d]: + control.steer = min(1., max(control.steer + 0.05, 0)) + else: + control.steer = 0 + control.hand_brake = keys[K_SPACE] + + car.apply_control(control) + return False + + @staticmethod + def set_image(weak_self, img): + """ + Sets image coming from camera sensor. + The self.capture flag is a mean of synchronization - once the flag is + set, next coming image will be stored. + """ + + self = weak_self() + if self.capture: + self.image = img + self.capture = False + + def render(self, display): + """ + Transforms image from camera sensor and blits it to main pygame display. + """ + + if self.image is not None: + array = np.frombuffer(self.image.raw_data, dtype=np.dtype("uint8")) + array = np.reshape(array, (self.image.height, self.image.width, 4)) + array = array[:, :, :3] + array = array[:, :, ::-1] + self.raw_image = cv2.cvtColor(array,cv2.COLOR_BGR2RGB) + surface = pygame.surfarray.make_surface(array.swapaxes(0, 1)) + display.blit(surface, (0, 0)) + + def game_loop(self, num_classes, input_size, graph, return_tensors): + """ + Main program loop. + """ + + + try: + pygame.init() + + self.client = carla.Client('127.0.0.1', 2000) + self.client.set_timeout(2.0) + self.world = self.client.get_world() + + self.setup_car() + self.setup_camera() + + self.display = pygame.display.set_mode((VIEW_WIDTH, VIEW_HEIGHT), pygame.HWSURFACE | pygame.DOUBLEBUF) + pygame_clock = pygame.time.Clock() + + self.set_synchronous_mode(True) + + with tf.Session(graph=graph) as sess: + while True: + self.world.tick() + + self.capture = True + pygame_clock.tick_busy_loop(20) + + self.render(self.display) + + self.raw_image = cv2.cvtColor(self.raw_image, cv2.COLOR_BGR2RGB) + frame_size = self.raw_image.shape[:2] + image_data = utils.image_preporcess(np.copy(self.raw_image), [input_size, input_size]) + image_data = image_data[np.newaxis, ...] + + pred_sbbox, pred_mbbox, pred_lbbox = sess.run( + [return_tensors[1], return_tensors[2], return_tensors[3]], + feed_dict={ return_tensors[0]: image_data}) + + pred_bbox = np.concatenate([np.reshape(pred_sbbox, (-1, 5 + num_classes)), + np.reshape(pred_mbbox, (-1, 5 + num_classes)), + np.reshape(pred_lbbox, (-1, 5 + num_classes))], axis=0) + + bboxes = utils.postprocess_boxes(pred_bbox, frame_size, input_size, 0.3) + bboxes = utils.nms(bboxes, 0.45, method='nms') + utils.draw_bounding_boxes(pygame, self.display, self.raw_image, bboxes) + + pygame.display.flip() + + pygame.event.pump() + if self.control(self.car): + return + + finally: + self.set_synchronous_mode(False) + self.camera.destroy() + self.car.destroy() + pygame.quit() + + +# ============================================================================== +# -- main() -------------------------------------------------------------------- +# ============================================================================== + + +def main(): + """ + Initializes the client-side bounding box demo. + """ + + try: + return_elements = ["input/input_data:0", "pred_sbbox/concat_2:0", "pred_mbbox/concat_2:0", "pred_lbbox/concat_2:0"] + pb_file = "tensorflow_yolov3\yolov3_coco.pb" + + # video_path = 0 + num_classes = 80 + input_size = 416 + graph = tf.Graph() + + THIS_FOLDER = os.path.dirname(os.path.abspath(__file__)) + my_file = os.path.join(THIS_FOLDER, pb_file) + print("my_file:", my_file) + + return_tensors = utils.read_pb_return_tensors(graph, my_file, return_elements) + + client = BasicSynchronousClient() + client.game_loop(num_classes, input_size, graph, return_tensors) + finally: + print('EXIT') + + +if __name__ == '__main__': + main() + + + +