Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add camera tracking module #1469

Draft
wants to merge 28 commits into
base: master
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
28 commits
Select commit Hold shift + click to select a range
8c007d1
mp_image: support macOS
srmainwaring Sep 29, 2024
e40835e
mp_image: add check that tracker is not None
srmainwaring Sep 29, 2024
0fc69b1
mavproxy_SIYI: support macOS
srmainwaring Sep 29, 2024
60ef255
camtrack: camera tracking module
srmainwaring Sep 27, 2024
99acc66
mp_image: handle each event in a separate function to allow override …
srmainwaring Oct 8, 2024
8ebb9a4
camtrack: handle camera tracking image status updates
srmainwaring Oct 8, 2024
3432d62
camtrack: camera and gimbal component ids should match autopilot
srmainwaring Oct 10, 2024
01f5846
camtrack: request gimbal device attitude status
srmainwaring Oct 10, 2024
f134a8a
camtrack: tracker_image: correct setting of tracked image position
srmainwaring Oct 10, 2024
feab200
camtrack: add settings and pid tuning tools
srmainwaring Oct 10, 2024
6f06e35
camtrack: onboard_controller: add option for tracker algo
srmainwaring Oct 15, 2024
d9587c1
camtrack: onboard_controller: add profiling to main loop
srmainwaring Oct 16, 2024
fd05617
camtrack: onboard_controller: add globals for update rates
srmainwaring Oct 16, 2024
ae658ad
camtrack: onboard_controller: add constant for attitude status update…
srmainwaring Oct 17, 2024
e442637
camtrack: onboard_controller: add micro-second props to profiler
srmainwaring Oct 17, 2024
5f5922e
camtrack: onboard_controller: add profiling to gimbal control loop
srmainwaring Oct 17, 2024
5c79cb5
camtrack: onboard_controller: update legacy cv2 trackers to latest
srmainwaring Oct 17, 2024
de99ae2
camtrack: onboard_controller: add rate limiter to gimbal controller
srmainwaring Oct 17, 2024
0452b6a
camtrack: onboard_controller: add rate limiter to gimbal controller
srmainwaring Oct 17, 2024
aec522a
camtrack: onboard_controller: check if cv2 has gstreamer
srmainwaring Oct 17, 2024
90deeea
camtrack: onboard_controller: use cv2 gstreamer if available
srmainwaring Oct 17, 2024
b2a9a24
camtrack: onboard_controller: add option to use hardware pipeline for…
srmainwaring Oct 21, 2024
c1dbace
camtrack: onboard_controller: more robust handling of frame shape
srmainwaring Oct 21, 2024
6ffb593
camtrack: onboard_controller: rescale and rate limit default pipeline
srmainwaring Oct 21, 2024
42f0358
camtrack: onboard_controller: add MOSSE tracker
srmainwaring Oct 21, 2024
ba44fee
camtrack: onboard_controller: update tracker pids
srmainwaring Oct 21, 2024
87e3bf6
camtrack: onboard_controller: display average fps
srmainwaring Oct 21, 2024
c55c1bb
camtrack: onboard_controller: update video frame rate and tracker pids
srmainwaring Oct 22, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
101 changes: 58 additions & 43 deletions MAVProxy/modules/lib/mp_image.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
'''

import time
from MAVProxy.modules.lib import wx_processguard
from MAVProxy.modules.lib.wx_loader import wx
import cv2
import numpy as np
Expand Down Expand Up @@ -575,6 +576,49 @@ def handle_osd(self, obj):
return
self.osd_elements[obj.label] = obj

def process_event(self, obj):
"""Process a single event"""
if isinstance(obj, MPImageOSD_Element):
self.handle_osd(obj)
if isinstance(obj, MPImageData):
self.set_image_data(obj.data, obj.width, obj.height)
if isinstance(obj, MPImageTitle):
state.frame.SetTitle(obj.title)
if isinstance(obj, MPImageRecenter):
self.on_recenter(obj.location)
if isinstance(obj, MPImageMenu):
self.set_menu(obj.menu)
if isinstance(obj, MPImagePopupMenu):
self.set_popup_menu(obj.menu)
if isinstance(obj, MPImageBrightness):
state.brightness = obj.brightness
self.need_redraw = True
if isinstance(obj, MPImageFullSize):
self.full_size()
if isinstance(obj, MPImageFitToWindow):
self.fit_to_window()
if isinstance(obj, win_layout.WinLayout):
win_layout.set_wx_window_layout(state.frame, obj)
if isinstance(obj, MPImageGStreamer):
self.start_gstreamer(obj.pipeline)
if isinstance(obj, MPImageVideo):
self.start_video(obj.filename)
if isinstance(obj, MPImageFPSMax):
self.fps_max = obj.fps_max
print("FPS_MAX: ", self.fps_max)
if isinstance(obj, MPImageSeekPercent):
self.seek_video(obj.percent)
if isinstance(obj, MPImageSeekFrame):
self.seek_video_frame(obj.frame)
if isinstance(obj, MPImageColormap):
self.colormap = obj.colormap
if isinstance(obj, MPImageColormapIndex):
self.colormap_index = obj.colormap_index
if isinstance(obj, MPImageStartTracker):
self.start_tracker(obj)
if isinstance(obj, MPImageEndTracker):
self.tracker = None

def on_redraw_timer(self, event):
'''the redraw timer ensures we show new map tiles as they
are downloaded'''
Expand All @@ -585,46 +629,8 @@ def on_redraw_timer(self, event):
except Exception:
time.sleep(0.05)
return
if isinstance(obj, MPImageOSD_Element):
self.handle_osd(obj)
if isinstance(obj, MPImageData):
self.set_image_data(obj.data, obj.width, obj.height)
if isinstance(obj, MPImageTitle):
state.frame.SetTitle(obj.title)
if isinstance(obj, MPImageRecenter):
self.on_recenter(obj.location)
if isinstance(obj, MPImageMenu):
self.set_menu(obj.menu)
if isinstance(obj, MPImagePopupMenu):
self.set_popup_menu(obj.menu)
if isinstance(obj, MPImageBrightness):
state.brightness = obj.brightness
self.need_redraw = True
if isinstance(obj, MPImageFullSize):
self.full_size()
if isinstance(obj, MPImageFitToWindow):
self.fit_to_window()
if isinstance(obj, win_layout.WinLayout):
win_layout.set_wx_window_layout(state.frame, obj)
if isinstance(obj, MPImageGStreamer):
self.start_gstreamer(obj.pipeline)
if isinstance(obj, MPImageVideo):
self.start_video(obj.filename)
if isinstance(obj, MPImageFPSMax):
self.fps_max = obj.fps_max
print("FPS_MAX: ", self.fps_max)
if isinstance(obj, MPImageSeekPercent):
self.seek_video(obj.percent)
if isinstance(obj, MPImageSeekFrame):
self.seek_video_frame(obj.frame)
if isinstance(obj, MPImageColormap):
self.colormap = obj.colormap
if isinstance(obj, MPImageColormapIndex):
self.colormap_index = obj.colormap_index
if isinstance(obj, MPImageStartTracker):
self.start_tracker(obj)
if isinstance(obj, MPImageEndTracker):
self.tracker = None

self.process_event(obj)

if self.need_redraw:
self.redraw()
Expand Down Expand Up @@ -696,8 +702,11 @@ def video_thread(self, url, cap_options):

frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
(width, height) = (frame.shape[1], frame.shape[0])
if self.tracker:
if self.tracker is not None:
self.tracker.update(frame)
# TODO: may need a lock? tracker may be set to None after update()
# but before get_position().
if self.tracker is not None:
pos = self.tracker.get_position()
if pos is not None:
startX = int(pos.left())
Expand Down Expand Up @@ -953,11 +962,17 @@ def full_size(self):
continue
if isinstance(event, MPImageTrackPos):
continue
if event.ClassName == 'wxMouseEvent':
if (
hasattr(event, "ClassName")
and event.ClassName == 'wxMouseEvent'
):
if event.leftIsDown and event.shiftDown:
im.start_tracker(event.X, event.Y, 50, 50)
if event.leftIsDown and event.controlDown:
im.end_tracking()
if event.ClassName == 'wxKeyEvent':
if (
hasattr(event, "ClassName")
and event.ClassName == 'wxKeyEvent'
):
print('key %u' % event.KeyCode)
time.sleep(0.1)
Loading