Refactoring the multiple tracing mechanisms.

Old system implemented 2 methods for tracing events: - many macros allow to record fxt events that depict various events in starpu (task submission, data transfers, scheduling points, memory management, synchronization, ...) - a few function calls allow to invoke external tracing tools (eg. APEX, EZTrace) callbacks. This is mainly for task submission/execution and data transfers. New system: - calls to fxt macros (_STARPU_TRACE_*) are replaced with calls to functions (_starpu_trace_) implemented in src/profiling/starpu_tracing.c - the tracing functions may then call a tracing tool (eg. fxt, APEX, EZTrace, ...) to record an event Tracing tools can now record a wide variety of events describing StarPU internals (the ones that were currently recorded with fxt). The drawback is that StarPU will perform a function call for each tracing point, even if no tracing tool is running. Experiments show that this does not impact performance (with running tests/microbench/tasks_overhead.c)
starpu-runtime · Feb 14, 2025 · 26e20ea · 26e20ea
1 parent e717199
commit 26e20ea
Show file tree

Hide file tree

Showing 76 changed files with 4,036 additions and 2,401 deletions.
diff --git a/ChangeLog b/ChangeLog
@@ -22,6 +22,7 @@ Changes:
    * Allow large sizes for vector, matrix, block, tensor and ndim data
      interfaces, and use proper MPI datatypes to exchange them.
    * Add soon_callback in tasks.
+   * Refactor the multiple tracing mechanisms
 
 Small changes:
   * Fix build system for StarPU Python interface

diff --git a/README.dev b/README.dev
@@ -194,16 +194,20 @@ Writing a new driver is essentially:
 
 
 
-Adding a new FXT state
+Adding a new trace event
 ----------------------
 
 This consists in:
 
-- Adding a code number in src/common/fxt.h
+- Adding a new function in src/profiling/starpu_tracing.h and in src/profiling/starpu_tracing.c
 
-- Adding the callable runtime macro in src/common/fxt.h
+- Calling this function in the wanted place in the runtime
 
-- Calling these macros in the wanted place in the runtime
+Implementing this event with FxT consists in
+
+- Adding a code number in src/profiling/fxt/fxt.h
+
+- Adding the callable runtime macro in src/profiling/fxt/fxt.h
 
 - Adding a paje state in states_list src/debug/traces/starpu_fxt.c and in
   src/debug/traces/starpu_paje.c

diff --git a/configure.ac b/configure.ac
@@ -6,6 +6,7 @@
 # Copyright (C) 2017-2017   Guillaume Beauchamp
 # Copyright (C) 2013-2013   Thibaut Lambert
 # Copyright (C) 2011-2011   Télécom Sud Paris
+# Copyright (C) 2023-2025   École de Technologie Supérieure (ETS, Montréal)
 #
 # StarPU is free software; you can redistribute it and/or modify
 # it under the terms of the GNU Lesser General Public License as published by
@@ -157,6 +158,91 @@ fi
 AC_MSG_CHECKING([for profiling tool support])
 AC_MSG_RESULT($enable_prof_tool)
 
+# Taskstubs
+AC_ARG_ENABLE(taskstubs, [AS_HELP_STRING([--enable-taskstubs],
+			[enable support for TaskStubs])],
+			enable_taskstubs=$enableval, enable_taskstubs=no)
+if test x$enable_taskstubs = xyes; then
+	AC_DEFINE(STARPU_PROF_TASKSTUBS, [1], [Define this to enable TaskStubs support])
+fi
+AC_MSG_CHECKING([for TaskStubs support])
+AC_MSG_RESULT($enable_taskstubs)
+
+AC_ARG_WITH(taskstubs-dir,
+	[AS_HELP_STRING([--with-taskstubs-dir=<path>],
+	[specify TaskStubs installation directory])],
+	[
+		taskstubs_dir="$withval"
+		# in case this was not explicit yet
+		enable_taskstubs=yes
+	], taskstubs_dir=no)
+
+AC_ARG_WITH(taskstubs-include-dir,
+	[AS_HELP_STRING([--with-taskstubs-include-dir=<path>],
+	[specify TaskStubs includes directory])],
+	[
+		taskstubs_include_dir="$withval"
+		# in case this was not explicit yet
+		enable_taskstubs=yes
+	], taskstubs_include_dir=no)
+
+AC_ARG_WITH(taskstubs-lib-dir,
+	[AS_HELP_STRING([--with-taskstubs-lib-dir=<path>],
+	[specify TaskStubs libs directory])],
+	[
+		taskstubs_lib_dir="$withval"
+		# in case this was not explicit yet
+		enable_taskstubs=yes
+	], taskstubs_lib_dir=no)
+
+if test x$enable_taskstubs = xyes ; then
+	AC_DEFINE(STARPU_PROF_TASKSTUBS, [1], [Define this to enable TaskStubs support])
+	PKG_CHECK_MODULES([TASKSTUBS], [TASKSTUBS], [have_pkg_taskstubs=yes], [have_pkg_taskstubs=no])
+
+	if test x$have_pkg_taskstubs = xyes; then
+		have_taskstubs=yes
+	else
+		if test "$taskstubs_include_dir" = "no" ; then
+		   if test "$taskstubs_dir" != "no" ; then
+		      taskstubs_include_dir="$taskstubs_dir/include/"
+		   fi		
+		fi
+
+		if test "$taskstubs_lib_dir" = "no" ; then
+		   if test "$taskstubs_dir" != "no" ; then
+		      taskstubs_lib_dir="$taskstubs_dir/lib"
+		   fi		
+		fi
+
+		if test "$taskstubs_include_dir" != "no" ; then
+		   AC_MSG_NOTICE(Using TaskStubs include dir $taskstubs_include_dir)
+		   TASKSTUBS_CFLAGS="-I$taskstubs_include_dir -I$taskstubs_include_dir/timer_plugin $TASKSTUBS_CFLAGS"
+		else
+		   AC_MSG_WARN([TaskStubs dir not found])
+		fi
+
+		if test "$taskstubs_lib_dir" != "no" ; then
+		   AC_MSG_NOTICE(Using TaskStubs lib dir $taskstubs_lib_dir)
+		   TASKSTUBS_LIBS="-L$taskstubs_lib_dir/ -ltimer_plugin"
+		else
+		   AC_MSG_WARN([TaskStubs lib not found])
+		fi
+
+		if test "$taskstubs_lib_dir" != "no" ; then
+		   if test "$taskstubs_lib_dir" != "no" ; then
+		      have_taskstubs=yes
+		   fi
+		fi
+	fi
+
+	if test -n "$TASKSTUBS_CFLAGS" ; then
+	   	CFLAGS="$TASKSTUBS_CFLAGS $CFLAGS"
+	fi
+	if test -n "$TASKSTUBS_LIBS" ; then
+	   	LDFLAGS="$TASKSTUBS_LIBS $LDFLAGS"
+	fi
+fi
+
 ###############################################################################
 #                                                                             #
 #                            Recursive tasks support                          #
@@ -4788,6 +4874,8 @@ AC_MSG_NOTICE([
 	BLAS library:      $blas_lib
 	hwloc:             $have_valid_hwloc
 	FxT trace enabled: $enable_fxt
+	Profiling tool:    $enable_prof_tool
+	TaskStubs enabled: $enable_taskstubs
 
         Documentation HTML:  $enable_build_doc
         Documentation PDF:   $enable_build_doc_pdf

diff --git a/doc/doxygen_dev/Makefile.am b/doc/doxygen_dev/Makefile.am
@@ -73,11 +73,11 @@ dox_inputs = $(DOX_CONFIG) 				\
 	$(top_srcdir)/src/parallel_worker/starpu_parallel_worker_create.h	\
 	$(top_srcdir)/src/profiling/profiling.h	\
 	$(top_srcdir)/src/profiling/bound.h	\
+	$(top_srcdir)/src/profiling/fxt/fxt.h	\
 	$(top_srcdir)/src/util/starpu_data_cpy.h	\
 	$(top_srcdir)/src/util/openmp_runtime_support.h	\
 	$(top_srcdir)/src/util/starpu_task_insert_utils.h	\
 	$(top_srcdir)/src/common/graph.h	\
-	$(top_srcdir)/src/common/fxt.h	\
 	$(top_srcdir)/src/common/starpu_spinlock.h	\
 	$(top_srcdir)/src/common/rbtree_i.h	\
 	$(top_srcdir)/src/common/rbtree.h	\

diff --git a/doc/doxygen_dev/doxygen-config.cfg.in b/doc/doxygen_dev/doxygen-config.cfg.in
@@ -51,11 +51,11 @@ INPUT                  = @top_srcdir@/doc/doxygen_dev/chapters/000_introduction.
 			 @top_srcdir@/src/parallel_worker/starpu_parallel_worker_create.h \
 			 @top_srcdir@/src/profiling/profiling.h \
 			 @top_srcdir@/src/profiling/bound.h \
+			 @top_srcdir@/src/profiling/fxt/fxt.h \
 			 @top_srcdir@/src/util/starpu_data_cpy.h \
 			 @top_srcdir@/src/util/openmp_runtime_support.h \
 			 @top_srcdir@/src/util/starpu_task_insert_utils.h \
 			 @top_srcdir@/src/common/graph.h \
-			 @top_srcdir@/src/common/fxt.h \
 			 @top_srcdir@/src/common/starpu_spinlock.h \
 			 @top_srcdir@/src/common/rbtree_i.h \
 			 @top_srcdir@/src/common/rbtree.h \

diff --git a/mpi/src/starpu_mpi_fxt.h b/mpi/src/starpu_mpi_fxt.h
@@ -20,7 +20,7 @@
 
 #include <starpu.h>
 #include <common/config.h>
-#include <common/fxt.h>
+#include <profiling/fxt/fxt.h>
 
 /** @file */
 

diff --git a/mpi/src/starpu_mpi_task_insert.c b/mpi/src/starpu_mpi_task_insert.c
@@ -331,7 +331,7 @@ int _starpu_mpi_task_decode_v(struct starpu_codelet *codelet, int me, int nb_nod
 	int prio = 0;
 	int select_node_policy = STARPU_MPI_NODE_SELECTION_CURRENT_POLICY;
 
-	_STARPU_TRACE_TASK_MPI_DECODE_START();
+	_starpu_trace_task_mpi_decode_start();
 
 	_STARPU_MPI_MALLOC(descrs, nb_allocated_data * sizeof(struct starpu_data_descr));
 	nb_data = 0;
@@ -354,7 +354,7 @@ int _starpu_mpi_task_decode_v(struct starpu_codelet *codelet, int me, int nb_nod
 				{
 					free(descrs);
 					va_end(varg_list_copy);
-					_STARPU_TRACE_TASK_MPI_DECODE_END();
+					_starpu_trace_task_mpi_decode_end();
 					return ret;
 				}
 			}
@@ -412,7 +412,7 @@ int _starpu_mpi_task_decode_v(struct starpu_codelet *codelet, int me, int nb_nod
 					{
 						free(descrs);
 						va_end(varg_list_copy);
-						_STARPU_TRACE_TASK_MPI_DECODE_END();
+						_starpu_trace_task_mpi_decode_end();
 						return ret;
 					}
 				}
@@ -443,7 +443,7 @@ int _starpu_mpi_task_decode_v(struct starpu_codelet *codelet, int me, int nb_nod
 					{
 						free(descrs);
 						va_end(varg_list_copy);
-						_STARPU_TRACE_TASK_MPI_DECODE_END();
+						_starpu_trace_task_mpi_decode_end();
 						return ret;
 					}
 				}
@@ -718,7 +718,7 @@ int _starpu_mpi_task_decode_v(struct starpu_codelet *codelet, int me, int nb_nod
 	*nb_data_p = nb_data;
 	*prio_p = prio;
 
-	_STARPU_TRACE_TASK_MPI_DECODE_END();
+	_starpu_trace_task_mpi_decode_end();
 	return 0;
 }
 
@@ -741,9 +741,10 @@ int _starpu_mpi_task_build_v(MPI_Comm comm, int me, struct starpu_codelet *codel
 	if (ret < 0)
 		return ret;
 
-	_STARPU_TRACE_TASK_MPI_PRE_START();
+	_starpu_trace_task_mpi_pre_start();
 	if (exchange_needed)
 		*exchange_needed = 0;
+
 	/* Send and receive data as requested */
 	for(i=0 ; i<nb_data ; i++)
 	{
@@ -799,7 +800,7 @@ int _starpu_mpi_task_build_v(MPI_Comm comm, int me, struct starpu_codelet *codel
 		}
 	}
 
-	_STARPU_TRACE_TASK_MPI_PRE_END();
+	_starpu_trace_task_mpi_pre_end();
 
 	return do_execute;
 }
@@ -808,7 +809,8 @@ int _starpu_mpi_task_postbuild_v(MPI_Comm comm, int me, int xrank, int do_execut
 {
 	int i;
 
-	_STARPU_TRACE_TASK_MPI_POST_START();
+	_starpu_trace_task_mpi_post_start();
+	starpu_mpi_comm_rank(comm, &me);
 
 	if (exchange_needed)
 	{
@@ -826,7 +828,7 @@ int _starpu_mpi_task_postbuild_v(MPI_Comm comm, int me, int xrank, int do_execut
 		}
 	}
 
-	_STARPU_TRACE_TASK_MPI_POST_END();
+	_starpu_trace_task_mpi_post_end();
 	_STARPU_MPI_LOG_OUT();
 	return 0;
 }

diff --git a/mpi/src/starpu_mpi_task_insert_fortran.c b/mpi/src/starpu_mpi_task_insert_fortran.c
@@ -38,7 +38,7 @@ int _fstarpu_mpi_task_decode_v(struct starpu_codelet *codelet, int me, int nb_no
 	int prio = 0;
 	int select_node_policy = STARPU_MPI_NODE_SELECTION_CURRENT_POLICY;
 
-	_STARPU_TRACE_TASK_MPI_DECODE_START();
+	_starpu_trace_task_mpi_decode_start();
 
 	_STARPU_MPI_MALLOC(descrs, nb_allocated_data * sizeof(struct starpu_data_descr));
 	nb_data = 0;
@@ -62,7 +62,7 @@ int _fstarpu_mpi_task_decode_v(struct starpu_codelet *codelet, int me, int nb_no
 				if (ret == -EINVAL)
 				{
 					free(descrs);
-					_STARPU_TRACE_TASK_MPI_DECODE_END();
+					_starpu_trace_task_mpi_decode_end();
 					return ret;
 				}
 			}
@@ -112,7 +112,7 @@ int _fstarpu_mpi_task_decode_v(struct starpu_codelet *codelet, int me, int nb_no
 					if (ret == -EINVAL)
 					{
 						free(descrs);
-						_STARPU_TRACE_TASK_MPI_DECODE_END();
+						_starpu_trace_task_mpi_decode_end();
 						return ret;
 					}
 				}
@@ -144,7 +144,7 @@ int _fstarpu_mpi_task_decode_v(struct starpu_codelet *codelet, int me, int nb_no
 					if (ret == -EINVAL)
 					{
 						free(descrs);
-						_STARPU_TRACE_TASK_MPI_DECODE_END();
+						_starpu_trace_task_mpi_decode_end();
 						return ret;
 					}
 				}
@@ -326,7 +326,7 @@ int _fstarpu_mpi_task_decode_v(struct starpu_codelet *codelet, int me, int nb_no
 	*nb_data_p = nb_data;
 	*prio_p = prio;
 
-	_STARPU_TRACE_TASK_MPI_DECODE_END();
+	_starpu_trace_task_mpi_decode_end();
 	return 0;
 }
 
@@ -350,7 +350,7 @@ int _fstarpu_mpi_task_build_v(MPI_Comm comm, struct starpu_codelet *codelet, str
 	if (ret < 0)
 		return ret;
 
-	_STARPU_TRACE_TASK_MPI_PRE_START();
+	_starpu_trace_task_mpi_pre_start();
 	/* Send and receive data as requested */
 	for(i=0 ; i<nb_data ; i++)
 	{
@@ -403,7 +403,7 @@ int _fstarpu_mpi_task_build_v(MPI_Comm comm, struct starpu_codelet *codelet, str
 		}
 	}
 
-	_STARPU_TRACE_TASK_MPI_PRE_END();
+	_starpu_trace_task_mpi_pre_end();
 
 	return do_execute;
 }

diff --git a/src/Makefile.am b/src/Makefile.am
@@ -125,7 +125,6 @@ noinst_HEADERS = 						\
 	common/list.h						\
 	common/rwlock.h						\
 	common/starpu_spinlock.h				\
-	common/fxt.h						\
 	common/utils.h						\
 	common/thread.h						\
 	common/barrier.h					\
@@ -157,9 +156,11 @@ noinst_HEADERS = 						\
 	drivers/disk/driver_disk.h				\
 	debug/traces/starpu_fxt.h				\
 	parallel_worker/starpu_parallel_worker_create.h		\
+	profiling/starpu_tracing.h		 		\
+	profiling/fxt/fxt.h			 		\
 	profiling/bound.h					\
 	profiling/profiling.h					\
-	profiling/callbacks.h					\
+	profiling/callbacks/callbacks.h				\
 	util/openmp_runtime_support.h				\
 	util/starpu_task_insert_utils.h				\
 	util/starpu_data_cpy.h					\
@@ -176,7 +177,6 @@ libstarpu_@STARPU_EFFECTIVE_VERSION@_la_SOURCES = 		\
 	common/rwlock.c						\
 	common/starpu_spinlock.c				\
 	common/timing.c						\
-	common/fxt.c						\
 	common/utils.c						\
 	common/thread.c						\
 	common/rbtree.c						\
@@ -288,10 +288,12 @@ libstarpu_@STARPU_EFFECTIVE_VERSION@_la_SOURCES = 		\
 	debug/traces/anim.c					\
 	debug/latency.c						\
 	debug/structures_size.c					\
+	profiling/starpu_tracing.c		 		\
+	profiling/fxt/fxt.c					\
 	profiling/profiling.c					\
 	profiling/bound.c					\
 	profiling/profiling_helpers.c				\
-	profiling/callbacks.c					\
+	profiling/callbacks/callbacks.c				\
 	worker_collection/worker_list.c				\
 	worker_collection/worker_tree.c				\
 	sched_policies/component_worker.c				\