Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Le video Take 2 #22

Open
wants to merge 7 commits into
base: wip
Choose a base branch
from
Open
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -50,3 +50,5 @@ apps/examples/hello_world/resources/images/earth_clouds.jpg
apps/examples/hello_world/resources/images/images.tar.gz
apps/examples/hello_world/resources/images/white.tga
apps/examples/hello_world/resources/images/world_winter.jpg
.idea/
cmake-build-*
1 change: 1 addition & 0 deletions apps/examples/hello_video/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
resources/test.mp4
47 changes: 47 additions & 0 deletions apps/examples/hello_video/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
cmake_minimum_required(VERSION 3.7.2)
set(CMAKE_CXX_STANDARD 17)

add_compile_definitions(DEBUG_GENERATE_DOT_GRAPH=false)

set(PROJECT_NAME "Island-HelloVideo")

# Set global property (all targets are impacted)
# set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE "${CMAKE_COMMAND} -E time")
# set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK "${CMAKE_COMMAND} -E time")

project(${PROJECT_NAME})

# set to number of worker threads if you wish to use multi-threaded rendering
# add_compile_definitions( LE_MT=4 )

# Vulkan Validation layers are enabled by default for Debug builds.
# Uncomment the next line to disable loading Vulkan Validation Layers.
# add_compile_definitions( SHOULD_USE_VALIDATION_LAYERS=false )

# Point this to the base directory of your Island installation
set(ISLAND_BASE_DIR "${PROJECT_SOURCE_DIR}/../../../")

# Select which standard Island modules to use
set(REQUIRES_ISLAND_LOADER ON)
set(REQUIRES_ISLAND_CORE ON)

# Loads Island framework, based on selected Island modules from above
include("${ISLAND_BASE_DIR}CMakeLists.txt.island_prolog.in")

# Add application module, and (optional) any other private
# island modules which should not be part of the shared framework.
add_subdirectory(hello_video_app)

# Specify any optional modules from the standard framework here
add_island_module(le_video)

# Main application c++ file. Not much to see there,
set(SOURCES main.cpp)

# Sets up Island framework linkage and housekeeping, based on user selections
include("${ISLAND_BASE_DIR}CMakeLists.txt.island_epilog.in")

# (optional) create a link to local resources
link_resources(${PROJECT_SOURCE_DIR}/resources ${CMAKE_BINARY_DIR}/local_resources)

source_group(${PROJECT_NAME} FILES ${SOURCES})
5 changes: 5 additions & 0 deletions apps/examples/hello_video/download_assets.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
#!/bin/bash

pushd resources
curl -L https://test-videos.co.uk/vids/jellyfish/mp4/h264/1080/Jellyfish_1080_10s_20MB.mp4 -o test.mp4
popd
27 changes: 27 additions & 0 deletions apps/examples/hello_video/hello_video_app/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
set (TARGET hello_video_app)

set (SOURCES "hello_video_app.cpp")
set (SOURCES ${SOURCES} "hello_video_app.h")

if (${PLUGINS_DYNAMIC})

add_library(${TARGET} SHARED ${SOURCES})


add_dynamic_linker_flags()

target_compile_definitions(${TARGET} PUBLIC "PLUGINS_DYNAMIC")

else()

# Adding a static library means to also add a linker dependency for our target
# to the library.
set (STATIC_LIBS ${STATIC_LIBS} ${TARGET} PARENT_SCOPE)

add_library(${TARGET} STATIC ${SOURCES})

endif()

target_link_libraries(${TARGET} PUBLIC ${LINKER_FLAGS})

source_group(${TARGET} FILES ${SOURCES})
192 changes: 192 additions & 0 deletions apps/examples/hello_video/hello_video_app/hello_video_app.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,192 @@
#include "hello_video_app.h"

#include "le_window/le_window.h"
#include "le_renderer/le_renderer.h"
#include "le_pipeline_builder/le_pipeline_builder.h"
#include "le_ui_event/le_ui_event.h"
#include "le_video/le_video.h"
#include "le_resource_manager/le_resource_manager.h"
#include "le_log/le_log.h"

#define GLM_FORCE_DEPTH_ZERO_TO_ONE // vulkan clip space is from 0 to 1
#define GLM_FORCE_RIGHT_HANDED // glTF uses right handed coordinate system, and we're following its lead.

#include <vector>

struct hello_video_app_o {
le::Window window;
le::Renderer renderer;
uint64_t frame_counter = 0;

LeResourceManager resource_manager;

le::Video video;
};

constexpr le_resource_handle_t VIDEO_HANDLE = LE_IMG_RESOURCE( "video" );

typedef hello_video_app_o app_o;

// ----------------------------------------------------------------------

static void app_initialize() {
le::Window::init();
le::Video::init();
};

// ----------------------------------------------------------------------

static void app_terminate() {
le::Video::terminate();
le::Window::terminate();
};

// ----------------------------------------------------------------------

static app_o *app_create() {
auto app = new ( app_o );

le::Window::Settings settings;
settings
.setWidth( 1280 )
.setHeight( 720 )
.setTitle( "Island // HelloVideoApp" );

// create a new window
app->window.setup( settings );

app->renderer.setup( le::RendererInfoBuilder( app->window ).build() );

app->video.setup( app->resource_manager, VIDEO_HANDLE );
app->video.load( "./local_resources/test.mp4" );
app->video.set_loop( true );
app->video.play();

return app;
}

// ----------------------------------------------------------------------

static void pass_main_exec( le_command_buffer_encoder_o *encoder_, void *user_data ) {
auto app = static_cast<app_o *>( user_data );

le::Encoder encoder{ encoder_ };

static auto shaderVert = app->renderer.createShaderModule( "./local_resources/shaders/fullscreen.vert", le::ShaderStage::eVertex );
static auto shaderFrag = app->renderer.createShaderModule( "./local_resources/shaders/fullscreen.frag", le::ShaderStage::eFragment );

static auto video_texture = le::Renderer::produceTextureHandle( "video" );

static auto pipelineHelloVideoExample =
LeGraphicsPipelineBuilder( encoder.getPipelineManager() )
.addShaderStage( shaderVert )
.addShaderStage( shaderFrag )
.build();

encoder
.bindGraphicsPipeline( pipelineHelloVideoExample )
.setArgumentTexture( LE_ARGUMENT_NAME( "src_video" ), video_texture )
.draw( 4 );
}

// ----------------------------------------------------------------------
static void app_process_ui_events( app_o *self ) {
using namespace le_window;
uint32_t numEvents;
LeUiEvent const *pEvents;
window_i.get_ui_event_queue( self->window, &pEvents, numEvents );

std::vector<LeUiEvent> events{ pEvents, pEvents + numEvents };

bool wantsToggle = false;

for ( auto &event : events ) {
switch ( event.event ) {
case ( LeUiEvent::Type::eKey ): {
auto &e = event.key;
if ( e.action == LeUiEvent::ButtonAction::eRelease ) {

if ( e.key == LeUiEvent::NamedKey::eF11 ) {
wantsToggle ^= true;
} else if ( e.key == LeUiEvent::NamedKey::eSpace ) {
self->video.pause();
}

} // if ButtonAction == eRelease

} break;
default:
// do nothing
break;
}
}

if ( wantsToggle ) {
self->window.toggleFullscreen();
}
}

// ----------------------------------------------------------------------

static bool app_update( app_o *self ) {

// Polls events for all windows
// Use `self->window.getUIEventQueue()` to fetch events.
le::Window::pollEvents();

if ( self->window.shouldClose() ) {
return false;
}

app_process_ui_events( self );

static auto video_texture = le::Renderer::produceTextureHandle( "video" );

le::RenderModule mainModule{};
{
self->resource_manager.update( mainModule );

auto video_tex_info =
le::ImageSamplerInfoBuilder()
.withImageViewInfo()
.setImage( VIDEO_HANDLE )
.end()
.build();

auto renderPassFinal =
le::RenderPass( "root", LE_RENDER_PASS_TYPE_DRAW )
.addColorAttachment( LE_SWAPCHAIN_IMAGE_HANDLE )
.sampleTexture( video_texture, video_tex_info ) // Declare texture video
.setExecuteCallback( self, pass_main_exec ) //
;

mainModule.addRenderPass( renderPassFinal );
}

self->renderer.update( mainModule );

self->frame_counter++;

return true; // keep app alive
}

// ----------------------------------------------------------------------

static void app_destroy( app_o *self ) {

delete ( self ); // deletes camera
}

// ----------------------------------------------------------------------

LE_MODULE_REGISTER_IMPL( hello_video_app, api ) {
auto hello_video_app_api_i = static_cast<hello_video_app_api *>( api );
auto &hello_video_app_i = hello_video_app_api_i->hello_video_app_i;

hello_video_app_i.initialize = app_initialize;
hello_video_app_i.terminate = app_terminate;

hello_video_app_i.create = app_create;
hello_video_app_i.destroy = app_destroy;
hello_video_app_i.update = app_update;
}
60 changes: 60 additions & 0 deletions apps/examples/hello_video/hello_video_app/hello_video_app.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
#ifndef GUARD_hello_video_app_H
#define GUARD_hello_video_app_H
#endif

#include "le_core/le_core.h"

struct hello_video_app_o;

// clang-format off
struct hello_video_app_api {

struct hello_video_app_interface_t {
hello_video_app_o * ( *create )();
void ( *destroy )( hello_video_app_o *self );
bool ( *update )( hello_video_app_o *self );
void ( *initialize )(); // static methods
void ( *terminate )(); // static methods
};

hello_video_app_interface_t hello_video_app_i;
};
// clang-format on

LE_MODULE( hello_video_app );
LE_MODULE_LOAD_DEFAULT( hello_video_app );

#ifdef __cplusplus

namespace hello_video_app {
static const auto &api = hello_video_app_api_i;
static const auto &hello_video_app_i = api -> hello_video_app_i;
} // namespace hello_video_app

class HelloVideoApp : NoCopy, NoMove {

hello_video_app_o *self;

public:
HelloVideoApp()
: self( hello_video_app::hello_video_app_i.create() ) {
}

bool update() {
return hello_video_app::hello_video_app_i.update( self );
}

~HelloVideoApp() {
hello_video_app::hello_video_app_i.destroy( self );
}

static void initialize() {
hello_video_app::hello_video_app_i.initialize();
}

static void terminate() {
hello_video_app::hello_video_app_i.terminate();
}
};

#endif
33 changes: 33 additions & 0 deletions apps/examples/hello_video/main.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
#include "hello_video_app/hello_video_app.h"

// ----------------------------------------------------------------------

int main( int argc, char const *argv[] ) {

HelloVideoApp::initialize();

{
// We instantiate HelloVideoApp in its own scope - so that
// it will be destroyed before HelloVideoApp::terminate
// is called.

HelloVideoApp HelloVideoApp{};

for ( ;; ) {

#ifdef PLUGINS_DYNAMIC
le_core_poll_for_module_reloads();
#endif
auto result = HelloVideoApp.update();

if ( !result ) {
break;
}
}
}

// Must only be called once last HelloVideoApp is destroyed
HelloVideoApp::terminate();

return 0;
}
18 changes: 18 additions & 0 deletions apps/examples/hello_video/resources/shaders/fullscreen.frag
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
#version 450 core

#extension GL_ARB_separate_shader_objects : enable
#extension GL_ARB_shading_language_420pack : enable

// vertex shader inputs
layout (location = 0) in vec2 inTexCoord;

// uniforms
layout (set = 0, binding = 0) uniform sampler2D src_video;

// outputs
layout (location = 0) out vec4 outFragColor;

void main(){
vec4 col = texture(src_video, inTexCoord.xy);
outFragColor = vec4(col.b, col.g, col.r, col.a);
}
25 changes: 25 additions & 0 deletions apps/examples/hello_video/resources/shaders/fullscreen.vert
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
#version 450 core

// This shader built after a technique introduced in:
// http://www.saschawillems.de/?page_id=2122

#extension GL_ARB_separate_shader_objects : enable
#extension GL_ARB_shading_language_420pack : enable

// inputs // Note: no inputs!

// outputs
layout (location = 0) out vec2 outTexCoord;

// Override the built-in fixed function outputs
// to have more control over the SPIR-V code created.
out gl_PerVertex
{
vec4 gl_Position;
};

void main()
{
outTexCoord = vec2((gl_VertexIndex << 1) & 2, gl_VertexIndex & 2);
gl_Position = vec4(outTexCoord * 2.0f + -1.0f, 0.0f, 1.0f);
}
94 changes: 76 additions & 18 deletions modules/le_pixels/le_pixels.cpp
Original file line number Diff line number Diff line change
@@ -1,14 +1,19 @@
#include "le_pixels.h"
#include "le_core/le_core.h"
#include "3rdparty/stb_image.h"
#include "assert.h"

#include <cassert>
#include <iostream>
#include <iomanip>

#include <mutex>

struct le_pixels_o {
// members
void * image_data = nullptr;
void *image_data = nullptr;
// std::string file_path;
le_pixels_info info{};
std::mutex mtx;
};

// ----------------------------------------------------------------------
@@ -19,6 +24,7 @@ struct image_source_info_t {
eUndefined = 0,
eBuffer = 1,
eFile = 2,
eAllocate = 3
};

union {
@@ -29,6 +35,10 @@ struct image_source_info_t {
struct {
char const *file_path;
} as_file;
struct {
int width;
int height;
} as_allocate;
} data;

Type type;
@@ -64,6 +74,22 @@ static le_pixels_o *le_pixels_create( image_source_info_t const &info ) {
int num_channels;
int num_channels_in_file;

// depending on type info has to be filled before or after the pixels allocation, could also be a standalone utility function
const auto fill_info = [ & ] {
if ( info.requested_num_channels == 0 ) {
num_channels = num_channels_in_file;
} else {
num_channels = info.requested_num_channels;
}

self->info.bpp = 8 * get_num_bytes_for_type( info.requested_pixel_type ) * uint32_t( num_channels ); // note * 8, since we're returning *bits* per pixel!
self->info.width = uint32_t( width );
self->info.height = uint32_t( height );
self->info.depth = 1;
self->info.num_channels = uint32_t( num_channels );
self->info.byte_count = ( self->info.bpp / 8 ) * ( self->info.width * self->info.height * self->info.depth );
};

if ( info.type == image_source_info_t::Type::eBuffer ) {

switch ( info.requested_pixel_type ) {
@@ -78,6 +104,8 @@ static le_pixels_o *le_pixels_create( image_source_info_t const &info ) {
break;
}

fill_info();

} else if ( info.type == image_source_info_t::Type::eFile ) {

switch ( info.requested_pixel_type ) {
@@ -92,24 +120,32 @@ static le_pixels_o *le_pixels_create( image_source_info_t const &info ) {
break;
}

} else {
assert( false ); // unreachable
}
fill_info();

} else if ( info.type == image_source_info_t::Type::eAllocate ) {

width = info.data.as_allocate.width;
height = info.data.as_allocate.height;

fill_info();

self->image_data = calloc( self->info.byte_count, 1 );

if ( info.requested_num_channels == 0 ) {
num_channels = num_channels_in_file;
} else {
num_channels = info.requested_num_channels;
assert( false ); // unreachable
}

if ( !self->image_data ) {

if ( info.type == image_source_info_t::Type::eFile ) {
std::cerr << "ERROR: Could not load image from file: " << info.data.as_file.file_path << std::endl
<< std::flush;
} else {
} else if ( info.type == image_source_info_t::Type::eBuffer ) {
std::cerr << "ERROR: Could not load image from buffer at address: " << std::hex << info.data.as_buffer.buffer << std::endl
<< std::flush;
} else {
std::cerr << "ERROR: Could not allocate pixels " << std::endl
<< std::flush;
}

// If we didn't manage to load an image, this object is invalid,
@@ -123,18 +159,25 @@ static le_pixels_o *le_pixels_create( image_source_info_t const &info ) {

// ----------| invariant: load was successful

self->info.bpp = 8 * get_num_bytes_for_type( info.requested_pixel_type ) * uint32_t( num_channels ); // note * 8, since we're returning *bits* per pixel!
self->info.width = uint32_t( width );
self->info.height = uint32_t( height );
self->info.depth = 1;
self->info.num_channels = uint32_t( num_channels );
self->info.byte_count = ( self->info.bpp / 8 ) * ( self->info.width * self->info.height * self->info.depth );

return self;
}

// ----------------------------------------------------------------------

static le_pixels_o *le_pixels_create( int width, int height, int num_channels_requested = 4, le_pixels_info::Type type = le_pixels_info::Type::eUInt8 ) {

image_source_info_t info{};

info.type = image_source_info_t::Type::eAllocate;
info.data.as_allocate = { width, height };
info.requested_pixel_type = type;
info.requested_num_channels = num_channels_requested;

return le_pixels_create( info );
}

// ----------------------------------------------------------------------

static le_pixels_o *le_pixels_create_from_file( char const *file_path, int num_channels_requested = 0, le_pixels_info::Type type = le_pixels_info::Type::eUInt8 ) {

image_source_info_t info{};
@@ -176,6 +219,18 @@ static void *le_pixels_get_data( le_pixels_o *self ) {

// ----------------------------------------------------------------------

static void le_pixels_lock( le_pixels_o *self ) {
self->mtx.lock();
}

// ----------------------------------------------------------------------

static void le_pixels_unlock( le_pixels_o *self ) {
self->mtx.unlock();
}

// ----------------------------------------------------------------------

static bool le_pixels_get_info_from_source( image_source_info_t const &source, le_pixels_info *info ) {

if ( info == nullptr ) {
@@ -271,7 +326,8 @@ static bool le_pixels_get_info_from_memory( unsigned char const *buffer, size_t
LE_MODULE_REGISTER_IMPL( le_pixels, api ) {
auto &le_pixels_i = static_cast<le_pixels_api *>( api )->le_pixels_i;

le_pixels_i.create = le_pixels_create_from_file;
le_pixels_i.create = le_pixels_create;
le_pixels_i.create_from_file = le_pixels_create_from_file;
le_pixels_i.create_from_memory = le_pixels_create_from_memory;

le_pixels_i.get_info_from_memory = le_pixels_get_info_from_memory;
@@ -280,4 +336,6 @@ LE_MODULE_REGISTER_IMPL( le_pixels, api ) {
le_pixels_i.destroy = le_pixels_destroy;
le_pixels_i.get_data = le_pixels_get_data;
le_pixels_i.get_info = le_pixels_get_info;
}
le_pixels_i.lock = le_pixels_lock;
le_pixels_i.unlock = le_pixels_unlock;
}
9 changes: 6 additions & 3 deletions modules/le_pixels/le_pixels.h
Original file line number Diff line number Diff line change
@@ -29,13 +29,16 @@ struct le_pixels_api {

bool (* get_info_from_memory ) ( unsigned char const * buffer, size_t buffer_byte_count, le_pixels_info * info);
bool (* get_info_from_file ) ( char const * file_name, le_pixels_info * info);

le_pixels_o * ( * create )(int width, int height, int num_channels_requested, le_pixels_info::Type type );
le_pixels_o * ( * create_from_memory )( unsigned char const * buffer, size_t buffer_byte_count, int num_channels_requested, le_pixels_info::Type type);
le_pixels_o * ( * create ) ( char const * file_path, int num_channels_requested, le_pixels_info::Type type);
le_pixels_o * ( * create_from_file ) ( char const * file_path, int num_channels_requested, le_pixels_info::Type type);
void ( * destroy ) ( le_pixels_o* self );

le_pixels_info ( * get_info ) ( le_pixels_o* self );
void * ( * get_data ) ( le_pixels_o* self );

void ( * lock ) ( le_pixels_o* self );
void ( * unlock ) ( le_pixels_o* self );
};

le_pixels_interface_t le_pixels_i;
@@ -60,7 +63,7 @@ class Pixels : NoCopy, NoMove {

public:
Pixels( char const *path, int const &numChannelsRequested = 0, le_pixels_info::Type const &type = le_pixels_info::eUInt8 )
: self( le_pixels::le_pixels_i.create( path, numChannelsRequested, type ) ) {
: self( le_pixels::le_pixels_i.create_from_file( path, numChannelsRequested, type ) ) {
}

~Pixels() {
136 changes: 99 additions & 37 deletions modules/le_resource_manager/le_resource_manager.cpp
Original file line number Diff line number Diff line change
@@ -6,23 +6,34 @@
#include <string>
#include <vector>
#include <assert.h>
// ----------------------------------------------------------------------
#include <atomic>
#include <map>

struct le_resource_manager_o {
// ----------------------------------------------------------------------

struct le_resource_item_t {
struct image_data_layer_t {
le_pixels_o *pixels;
std::string path;
bool was_uploaded = false;
};
// std::string path;
le_pixels_o * pixels = nullptr;
bool pixels_owner = false;
std::atomic_bool was_uploaded = false;

image_data_layer_t() = default;

struct resource_item_t {
le_resource_handle_t image_handle;
le_resource_info_t image_info;
std::vector<image_data_layer_t> image_layers; // must have at least one element
image_data_layer_t( const le_resource_item_t::image_data_layer_t &src ) {
pixels = src.pixels;
pixels_owner = src.pixels_owner;
was_uploaded = bool( src.was_uploaded );
}
};

std::vector<resource_item_t> resources;
le_resource_handle_t image_handle;
le_resource_info_t image_info;
std::vector<image_data_layer_t> image_layers; // must have at least one element
};

struct le_resource_manager_o {
std::vector<le_resource_item_t> resources;
};

// TODO:
@@ -108,12 +119,20 @@ static void execTransferPass( le_command_buffer_encoder_o *pEncoder, void *user_
.setImageD( depth )
.build();

auto pixels = r.image_layers[ layer ].pixels;
auto pixels = r.image_layers[ layer ].pixels;
auto pixels_owner = r.image_layers[ layer ].pixels_owner;

if ( !pixels_owner )
le_pixels_i.lock( pixels );

auto info = le_pixels_i.get_info( pixels );
uint32_t num_bytes = info.byte_count; // TODO: make sure to get correct byte count for mip level, or compressed image.
void * bytes = le_pixels_i.get_data( pixels );

encoder.writeToImage( r.image_handle, write_info, bytes, num_bytes );

if ( !pixels_owner )
le_pixels_i.unlock( pixels );
}
r.image_layers[ layer ].was_uploaded = true;
}
@@ -163,19 +182,28 @@ static void infer_from_le_format( le::Format const &format, uint32_t *num_channe
}

// ----------------------------------------------------------------------
// NOTE: You must provide an array of paths in image_paths, and the
// array's size must match `image_info.image.arrayLayers`
// Most meta-data about the image file is loaded via image_info
static void le_resource_manager_add_item( le_resource_manager_o * self,
le_resource_handle_t const *image_handle,
le_resource_info_t const * image_info,
char const *const * image_paths ) {

le_resource_manager_o::resource_item_t item{};
static le_resource_item_t *le_resource_manager_get_item( le_resource_manager_o *self, const le_resource_handle_t &resource_handle ) {
// TODO: consider a lookup table
for ( auto &item : self->resources ) {
if ( item.image_handle == resource_handle ) {
return &item;
}
}
return nullptr;
}

// ----------------------------------------------------------------------

static void le_resource_manager_add_item_pixels( le_resource_manager_o *self, le_resource_handle_t const *image_handle, le_resource_info_t const *image_info, le_pixels_o **pixels, bool is_owner = false ) {
// TODO, merge this with filepaths code
assert( le_resource_manager_get_item( self, *image_handle ) == nullptr );

le_resource_item_t item{};

item.image_handle = *image_handle;
item.image_info = *image_info;
item.image_layers.reserve( image_info->image.arrayLayers );
//item.image_layers.reserve( image_info->image.arrayLayers );

bool extents_inferred = false;
if ( item.image_info.image.extent.width == 0 ||
@@ -185,17 +213,12 @@ static void le_resource_manager_add_item( le_resource_manager_o * self,
}

for ( size_t i = 0; i != item.image_info.image.arrayLayers; ++i ) {
le_resource_manager_o::image_data_layer_t layer_data{};
layer_data.path = std::string{ image_paths[ i ] };
le_resource_item_t::image_data_layer_t layer_data{};
// layer_data.path = std::string{ image_paths[ i ] };
layer_data.was_uploaded = false;

// we must find out the pixels type from image info format
uint32_t num_channels = 0;
le_pixels_info::Type pixels_type{};

infer_from_le_format( item.image_info.image.format, &num_channels, &pixels_type );

layer_data.pixels = le_pixels::le_pixels_i.create( layer_data.path.c_str(), num_channels, pixels_type );
layer_data.pixels = pixels[ i ];
layer_data.pixels_owner = is_owner;

if ( extents_inferred ) {
auto info = le_pixels::le_pixels_i.get_info( layer_data.pixels );
@@ -204,7 +227,7 @@ static void le_resource_manager_add_item( le_resource_manager_o * self,
item.image_info.image.extent.height = std::max( item.image_info.image.extent.height, info.height );
}

item.image_layers.emplace_back( layer_data );
item.image_layers.push_back( layer_data );
}

assert( item.image_info.image.extent.width != 0 &&
@@ -215,6 +238,42 @@ static void le_resource_manager_add_item( le_resource_manager_o * self,
self->resources.emplace_back( item );
}

// ----------------------------------------------------------------------
// NOTE: You must provide an array of paths in image_paths, and the
// array's size must match `image_info.image.arrayLayers`
// Most meta-data about the image file is loaded via image_info
static void le_resource_manager_add_item_filepaths( le_resource_manager_o * self,
le_resource_handle_t const *image_handle,
le_resource_info_t const * image_info,
char const *const * image_paths ) {

std::vector<le_pixels_o *> pixels( image_info->image.arrayLayers );

// we must find out the pixels type from image info format
uint32_t num_channels = 0;
le_pixels_info::Type pixels_type{};

infer_from_le_format( image_info->image.format, &num_channels, &pixels_type );

for ( size_t i = 0; i < pixels.size(); ++i ) {
pixels[ i ] = le_pixels::le_pixels_i.create_from_file( image_paths[ i ], num_channels, pixels_type );
}
le_resource_manager_add_item_pixels( self, image_handle, image_info, pixels.data(), true );
}

// ----------------------------------------------------------------------

static void le_resource_manager_update_pixels( le_resource_manager_o *self, le_resource_handle_t const *image_handle, le_pixels_o ** = nullptr ) {
auto item = le_resource_manager_get_item( self, *image_handle );
if ( !item )
return;

// TODO: use the new pixels if provided
for ( auto &layer : item->image_layers ) {
layer.was_uploaded = false;
}
}

// ----------------------------------------------------------------------

static le_resource_manager_o *le_resource_manager_create() {
@@ -230,12 +289,13 @@ static void le_resource_manager_destroy( le_resource_manager_o *self ) {

for ( auto &r : self->resources ) {
for ( auto &l : r.image_layers ) {
if ( l.pixels ) {
if ( l.pixels_owner && l.pixels ) {
le_pixels_i.destroy( l.pixels );
l.pixels = nullptr;
}
}
}
self->resources.clear();
delete ( self );
}

@@ -244,8 +304,10 @@ static void le_resource_manager_destroy( le_resource_manager_o *self ) {
LE_MODULE_REGISTER_IMPL( le_resource_manager, api ) {
auto &le_resource_manager_i = static_cast<le_resource_manager_api *>( api )->le_resource_manager_i;

le_resource_manager_i.create = le_resource_manager_create;
le_resource_manager_i.destroy = le_resource_manager_destroy;
le_resource_manager_i.update = le_resource_manager_update;
le_resource_manager_i.add_item = le_resource_manager_add_item;
}
le_resource_manager_i.create = le_resource_manager_create;
le_resource_manager_i.destroy = le_resource_manager_destroy;
le_resource_manager_i.update = le_resource_manager_update;
le_resource_manager_i.add_item_filepaths = le_resource_manager_add_item_filepaths;
le_resource_manager_i.add_item_pixels = le_resource_manager_add_item_pixels;
le_resource_manager_i.update_pixels = le_resource_manager_update_pixels;
}
16 changes: 13 additions & 3 deletions modules/le_resource_manager/le_resource_manager.h
Original file line number Diff line number Diff line change
@@ -77,6 +77,7 @@ struct le_resource_manager_o;
struct le_render_module_o; // ffdecl. (from le_renderer)
struct le_resource_handle_t; // ffdecl. (from le_renderer)
struct le_resource_info_t; // ffdecl. (from le_renderer)
struct le_pixels_o; // ffdelc. (from le_pixels)

// clang-format off
struct le_resource_manager_api {
@@ -86,8 +87,9 @@ struct le_resource_manager_api {
le_resource_manager_o * ( * create ) ( );
void ( * destroy ) ( le_resource_manager_o* self );
void ( * update ) ( le_resource_manager_o* self, le_render_module_o* module );
void ( * add_item ) ( le_resource_manager_o* self, le_resource_handle_t const * image_handle, le_resource_info_t const * image_info, char const * const * arr_image_paths);

void ( * add_item_filepaths ) (le_resource_manager_o* self, le_resource_handle_t const * image_handle, le_resource_info_t const * image_info, char const * const * arr_image_paths);
void ( * add_item_pixels ) ( le_resource_manager_o* self, le_resource_handle_t const * image_handle, le_resource_info_t const * image_info, le_pixels_o**, bool take_ownership);
void ( * update_pixels ) ( le_resource_manager_o* self, le_resource_handle_t const * image_handle, le_pixels_o** pixels);
};

le_resource_manager_interface_t le_resource_manager_i;
@@ -122,7 +124,15 @@ class LeResourceManager : NoCopy, NoMove {
}

void add_item( le_resource_handle_t const &image_handle, le_resource_info_t const &image_info, char const *const *arr_image_paths ) {
le_resource_manager::le_resource_manager_i.add_item( self, &image_handle, &image_info, arr_image_paths );
le_resource_manager::le_resource_manager_i.add_item_filepaths( self, &image_handle, &image_info, arr_image_paths );
}

void add_item( le_resource_handle_t const &image_handle, le_resource_info_t const &image_info, le_pixels_o **pixels, bool take_ownership = false ) {
return le_resource_manager::le_resource_manager_i.add_item_pixels( self, &image_handle, &image_info, pixels, take_ownership );
}

void update_pixels( le_resource_handle_t const &image_handle, le_pixels_o **pixels = nullptr ) {
le_resource_manager::le_resource_manager_i.update_pixels( self, &image_handle, pixels );
}

operator auto() {
101 changes: 101 additions & 0 deletions modules/le_video/3rdparty/FindLIBVLC.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
#############################################################################
# VLC - CMake module
# Copyright (C) 2014 Tadej Novak <tadej@tano.si>
# Original author: Rohit Yadav <rohityadav89@gmail.com>
#
# This library is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this library. If not, see <http://www.gnu.org/licenses/>.
#############################################################################
# If it's found it sets LIBVLC_FOUND to TRUE
# and following variables are set:
# LIBVLC_INCLUDE_DIR
# LIBVLC_LIBRARY


# FIND_PATH and FIND_LIBRARY normally search standard locations
# before the specified paths. To search non-standard paths first,
# FIND_* is invoked first with specified paths and NO_DEFAULT_PATH
# and then again with no specified paths to search the default
# locations. When an earlier FIND_* succeeds, subsequent FIND_*s
# searching for the same item do nothing.

#Put here path to custom location
#example: /home/user/vlc/include etc..
FIND_PATH(LIBVLC_INCLUDE_DIR vlc/vlc.h
HINTS "$ENV{LIBVLC_INCLUDE_PATH}"
PATHS
#Mac OS and Contribs
"${CMAKE_CURRENT_SOURCE_DIR}/contribs/include"
"${CMAKE_CURRENT_SOURCE_DIR}/contribs/include/vlc"
"/Applications/VLC.app/Contents/MacOS/include"
"/Applications/VLC.app/Contents/MacOS/include/vlc"
# Env
"$ENV{LIB_DIR}/include"
"$ENV{LIB_DIR}/include/vlc"
#
"/usr/include"
"/usr/include/vlc"
"/usr/local/include"
"/usr/local/include/vlc"
#mingw
c:/msys/local/include
"c:/Program Files (x86)/VideoLAN/VLC/sdk/include"
)
FIND_PATH(LIBVLC_INCLUDE_DIR PATHS "${CMAKE_INCLUDE_PATH}/vlc" NAMES vlc.h)

#Put here path to custom location
#example: /home/user/vlc/lib etc..
FIND_LIBRARY(LIBVLC_LIBRARY NAMES vlc libvlc
HINTS "$ENV{LIBVLC_LIBRARY_PATH}"
PATHS
"$ENV{LIB_DIR}/lib"
#Mac OS
"${CMAKE_CURRENT_SOURCE_DIR}/contribs/lib"
"${CMAKE_CURRENT_SOURCE_DIR}/contribs/plugins"
"/Applications/VLC.app/Contents/MacOS/lib"
"/Applications/VLC.app/Contents/MacOS/plugins"
#mingw
c:/msys/local/lib
"c:/Program Files (x86)/VideoLAN/VLC/sdk/lib"
)
FIND_LIBRARY(LIBVLC_LIBRARY NAMES vlc libvlc)
FIND_LIBRARY(LIBVLCCORE_LIBRARY NAMES vlccore libvlccore
HINTS "$ENV{LIBVLC_LIBRARY_PATH}"
PATHS
"$ENV{LIB_DIR}/lib"
#Mac OS
"${CMAKE_CURRENT_SOURCE_DIR}/contribs/lib"
"${CMAKE_CURRENT_SOURCE_DIR}/contribs/plugins"
"/Applications/VLC.app/Contents/MacOS/lib"
"/Applications/VLC.app/Contents/MacOS/plugins"
#mingw
c:/msys/local/lib
"c:/Program Files (x86)/VideoLAN/VLC/sdk/lib"
)
FIND_LIBRARY(LIBVLCCORE_LIBRARY NAMES vlccore libvlccore)

IF (LIBVLC_INCLUDE_DIR AND LIBVLC_LIBRARY AND LIBVLCCORE_LIBRARY)
SET(LIBVLC_FOUND TRUE)
ENDIF (LIBVLC_INCLUDE_DIR AND LIBVLC_LIBRARY AND LIBVLCCORE_LIBRARY)

IF (LIBVLC_FOUND)
IF (NOT LIBVLC_FIND_QUIETLY)
MESSAGE(STATUS "Found LibVLC include-dir path: ${LIBVLC_INCLUDE_DIR}")
MESSAGE(STATUS "Found LibVLC library path:${LIBVLC_LIBRARY}")
MESSAGE(STATUS "Found LibVLCcore library path:${LIBVLCCORE_LIBRARY}")
ENDIF (NOT LIBVLC_FIND_QUIETLY)
ELSE (LIBVLC_FOUND)
IF (LIBVLC_FIND_REQUIRED)
MESSAGE(FATAL_ERROR "Could not find LibVLC")
ENDIF (LIBVLC_FIND_REQUIRED)
ENDIF (LIBVLC_FOUND)
44 changes: 44 additions & 0 deletions modules/le_video/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
set(TARGET le_video)

# list modules this module depends on
depends_on_island_module(le_resource_manager)
depends_on_island_module(le_pixels)
depends_on_island_module(le_log)

list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty)

FIND_PACKAGE(LIBVLC REQUIRED)

set(SOURCES "le_video.cpp")
set(SOURCES ${SOURCES} "le_video.h")

if (${PLUGINS_DYNAMIC})

add_library(${TARGET} SHARED ${SOURCES})

# IMPORTANT: --no-gnu-unique
# This compiler flag is neccesary as otherwise the library may get compiled
# with some symbols exported as UNIQUE, which implicitly makes this library
# un-closeable. This means, any calls to `dlclose` on this library, once
# loaded, will have no effect, and autoreload for this library will not work
# as the first version of the library will remain resident.
target_compile_options(${TARGET} PUBLIC --no-gnu-unique)

target_compile_definitions(${TARGET} PUBLIC "PLUGINS_DYNAMIC")

else ()

add_library(${TARGET} STATIC ${SOURCES})

set(STATIC_LIBS ${STATIC_LIBS} ${TARGET} PARENT_SCOPE)

endif ()

# set (LINKER_FLAGS ${LINKER_FLAGS} stdc++fs)

target_include_directories(${TARGET} PRIVATE ${LIBVLC_INCLUDE_DIRS})

# TODO: use the actual filepath from findVLC for maximum platform compatibility, ${LIBVLCCORE_LIBRARY}
target_link_libraries(${TARGET} PUBLIC ${LINKER_FLAGS} vlc)

source_group(${TARGET} FILES ${SOURCES})
249 changes: 249 additions & 0 deletions modules/le_video/le_video.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,249 @@
#include <vlc/vlc.h>
#include <iostream>
#include <filesystem>
#include <atomic>
#include <thread>

#include <le_renderer/le_renderer.h>
#include <le_pixels/le_pixels.h>
#include <le_resource_manager/le_resource_manager.h>

#include "le_video.h"
#include "le_core/le_core.h"
#include "le_log/le_log.h"

struct le_video_o {
libvlc_instance_t * libvlc = nullptr;
le_resource_manager_o *resource_manager = nullptr;
libvlc_media_player_t *player = nullptr;
le_pixels_o * pixels = nullptr;
uint64_t duration = 0;
std::atomic_bool loop = false;
le_video_load_params load_params;
le_resource_handle_t image_handle;
};

//struct le_video_item_t {
// le_resource_handle_t handle;
// libvlc_media_player_t *player;
//};

static libvlc_instance_t *libvlc;
static le::Log log( "le_video" );

static int le_video_init() {
libvlc = libvlc_new( 0, nullptr );

if ( !libvlc ) {
log.error( "could not initialize" );
} else {
log.info( "initialized" );
}

const_cast<le_video_api *>( le_video_api_i )->le_video_context = libvlc;

return libvlc != nullptr;
}

static void le_video_terminate() {
libvlc_release( libvlc );
log.info( "terminated" );
}

// ----------------------------------------------------------------------

static le_video_o *le_video_create() {
auto self = new le_video_o();
return self;
}

static bool le_video_setup( le_video_o *self, le_resource_manager_o *resource_manager, le_resource_handle_t const *image_handle ) {

self->libvlc = libvlc;
self->resource_manager = resource_manager;
self->image_handle = *image_handle;

if ( !self->libvlc ) {
std::cerr << "Error no VLC context set" << std::endl;
return false;
}

return true;
}

// ----------------------------------------------------------------------

static void le_video_destroy( le_video_o *self ) {
if ( self->player ) {
libvlc_media_player_release( self->player );
}
le_pixels::le_pixels_i.destroy( self->pixels );
delete self;
}

// ------------------------------------------------------- CALLBACKS (all these happen on their own thread)

inline le_video_o *to_video( void *ptr ) {
return static_cast<le_video_o *>( ptr );
}

static void *cb_lock( void *opaque, void **planes ) {
auto video = to_video( opaque );
le_pixels::le_pixels_i.lock( video->pixels );
*planes = le_pixels::le_pixels_i.get_data( video->pixels );
return video->pixels;
}

static void cb_unlock( void *opaque, void *picture, void *const *planes ) {
auto video = to_video( opaque );
le_pixels::le_pixels_i.unlock( video->pixels );
}

static void cb_display( void *opaque, void *picture ) {
auto video = to_video( opaque );
le_resource_manager::le_resource_manager_i.update_pixels( video->resource_manager, &video->image_handle, nullptr );
}

// forward declare this one because we need some core video player functions
static void cb_evt( const libvlc_event_t *event, void *opaque );

// ----------------------------------------------------------------------

static void le_video_update( le_video_o *self ) {
// do something with self
}

static bool le_video_load( le_video_o *self, const le_video_load_params &params ) {
if ( !std::filesystem::exists( params.file_path ) ) {
log.error( "Videofile does not exist '%s'", params.file_path );
return false;
}

const char *chroma;
unsigned num_channels = 0;

switch ( params.output_format ) {
// case le::Format::eR8G8Uint:
// chroma = "RV32";
// num_channels = 2;
// break;
case le::Format::eR8G8B8Unorm:
chroma = "RV32";
num_channels = 3;
break;
case le::Format::eR8G8B8A8Unorm:
chroma = "RGBA";
num_channels = 4;
break;
default:
// TODO more formats
// log.error( "Only eR8G8B8A8Uint or eR8G8B8A8Uint video output format is supported" );
return false;
}

auto media = libvlc_media_new_path( self->libvlc, params.file_path );

// libvlc_media_add_option(media, ":avcodec-hw=none");
libvlc_media_parse( media );
// libvlc_media_parse_with_options(media, )
self->duration = libvlc_media_get_duration( media );

self->player = libvlc_media_player_new_from_media( media );

unsigned width, height;
libvlc_video_get_size( self->player, 0, &width, &height );

self->pixels = le_pixels::le_pixels_i.create( int( width ), int( height ), 4, le_pixels_info::eUInt8 );

log.info( "loaded '%s' %dx%d - %dms", params.file_path, width, height, self->duration );

// set callbacks
libvlc_video_set_callbacks( self->player, cb_lock, cb_unlock, cb_display, self );

auto event_manager = libvlc_media_player_event_manager( self->player );
static libvlc_event_e event_types[] = { libvlc_MediaPlayerEncounteredError, libvlc_MediaPlayerPositionChanged,
libvlc_MediaPlayerEndReached, libvlc_MediaPlayerLengthChanged,
libvlc_MediaPlayerSeekableChanged, libvlc_MediaPlayerStopped };
for ( auto event : event_types ) {
libvlc_event_attach( event_manager, event, cb_evt, self );
}

libvlc_video_set_format( self->player, chroma, width, height, width * 4 );

auto image_info =
le::ImageInfoBuilder()
.setImageType( le::ImageType::e2D )
.setExtent( width, height )
.build();

le_resource_manager::le_resource_manager_i.add_item_pixels( self->resource_manager, &self->image_handle, &image_info, &self->pixels, false );

libvlc_media_release( media );

return true;
}

// ----------------------------------------------------------------------

static void le_video_play( le_video_o *self ) {
libvlc_media_player_play( self->player );
}

// ----------------------------------------------------------------------

static void le_video_pause( le_video_o *self ) {
libvlc_media_player_pause( self->player );
}

// ----------------------------------------------------------------------

static void le_video_set_loop( le_video_o *self, bool state ) {
self->loop = state;
}

// ----------------------------------------------------------------------

static void le_video_set_position( le_video_o *self, int64_t position ) {
auto percent = position / float( self->duration );
libvlc_media_player_set_position( self->player, percent );
}

// ----------------------------------------------------------------------

static void cb_evt( const libvlc_event_t *event, void *opaque ) {
auto video = to_video( opaque );
switch ( event->type ) {
case libvlc_MediaPlayerEndReached:
// if ( video->loop ) {
// le_video_play( video );
// le_video_set_position( video, 0 );
// }
break;
case libvlc_MediaPlayerPositionChanged:
break;
//log.info( "%d", event->u.media_player_time_changed.new_time );
}
}

// ----------------------------------------------------------------------

LE_MODULE_REGISTER_IMPL( le_video, api ) {
auto videoApi = static_cast<le_video_api *>( api );
videoApi->init = le_video_init;
videoApi->terminate = le_video_terminate;

auto &le_video_i = videoApi->le_video_i;
le_video_i.create = le_video_create;
le_video_i.destroy = le_video_destroy;
le_video_i.setup = le_video_setup;
le_video_i.update = le_video_update;
le_video_i.load = le_video_load;
le_video_i.play = le_video_play;
le_video_i.pause = le_video_pause;
le_video_i.set_position = le_video_set_position;
le_video_i.set_loop = le_video_set_loop;

if ( videoApi->le_video_context ) {
libvlc = static_cast<libvlc_instance_t *>( videoApi->le_video_context );
}
}
111 changes: 111 additions & 0 deletions modules/le_video/le_video.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
#ifndef GUARD_le_video_H
#define GUARD_le_video_H

#include <string>

#include "le_core/le_core.h"

struct le_video_o;

struct le_video_load_params {
char const *file_path{};
le::Format output_format{ le::Format::eR8G8B8Unorm };
};

// forward declaration
struct le_resource_manager_o;

// clang-format off
struct le_video_api {

struct le_video_interface_t {
le_video_o * ( * create ) ( );
bool ( * setup ) ( le_video_o* self, le_resource_manager_o* resource_manager, le_resource_handle_t const * image_handle );
void ( * destroy ) ( le_video_o* self );
void ( * update ) ( le_video_o* self );
bool ( * load ) ( le_video_o* self, const le_video_load_params &params );
void ( * play ) ( le_video_o* self );
void ( * pause ) ( le_video_o* self );
void ( * set_position ) ( le_video_o* self, int64_t from );
void ( * set_loop ) ( le_video_o* self, bool state );
};

int ( *init ) ();
void ( *terminate ) ();

void* le_video_context{nullptr};
le_video_interface_t le_video_i;
};
// clang-format on

LE_MODULE( le_video );
LE_MODULE_LOAD_DEFAULT( le_video );

#ifdef __cplusplus

namespace le_video {
static const auto &api = le_video_api_i;
static const auto &le_video_i = api -> le_video_i;
} // namespace le_video

namespace le {

class Video : NoCopy, NoMove {

le_video_o *self;

public:
static int init() {
return le_video::api->init();
}

static void terminate() {
le_video::api->terminate();
}

Video()
: self( le_video::le_video_i.create() ) {
}

~Video() {
le_video::le_video_i.destroy( self );
}

bool setup( le_resource_manager_o *resource_manager, le_resource_handle_t const &image_handle ) {
return le_video::le_video_i.setup( self, resource_manager, &image_handle );
}

void update() {
le_video::le_video_i.update( self );
}

bool load( const std::string &path ) {
return le_video::le_video_i.load( self, { path.c_str() } );
}

void play() {
le_video::le_video_i.play( self );
}

void pause() {
le_video::le_video_i.pause( self );
}

void set_position( int64_t millis ) {
le_video::le_video_i.set_position( self, millis );
}

void set_loop( bool state ) {
le_video::le_video_i.set_loop( self, state );
}

operator auto() {
return self;
}
};

} // namespace le

#endif // __cplusplus

#endif