Merge branch 'main' into nkpatel/maxpool_perf

.clang-tidy

@@ -3,7 +3,7 @@ Checks: >
   performance-*,
   modernize-*,
   readability-*,
-  cppcoreguidelines-*
+  cppcoreguidelines-*,
   -modernize-use-trailing-return-type
 
 CheckOptions:

.github/workflows/stress-fast-dispatch-build-and-unit-tests.yaml

@@ -30,6 +30,7 @@ jobs:
       TT_METAL_ENV: ${{ vars.TT_METAL_ENV }}
       ARCH_NAME: ${{ matrix.runner-info.arch }}
       TT_METAL_WATCHER: 60
+      TT_METAL_WATCHER_NOINLINE: 1
       LD_LIBRARY_PATH: ${{ github.workspace }}/build/lib
     environment: dev
     runs-on: ${{ matrix.runner-info.runs-on }}

.github/workflows/ttnn-run-sweeps.yaml

@@ -220,6 +220,12 @@ on:
           - eltwise.binary_backward.subalpha_bw.subalpha_bw
           - eltwise.binary_backward.xlogy_bw.xlogy_bw
           - eltwise.binary_backward.hypot_bw.hypot_bw
+          - eltwise.binary_backward.add_bw.add_bw
+          - eltwise.binary_backward.sub_bw.sub_bw
+          - eltwise.binary_backward.mul_bw.mul_bw
+          - eltwise.binary_backward.div_bw.div_bw
+          - eltwise.binary_backward.fmod_bw.fmod_bw
+          - eltwise.binary_backward.remainder_bw.remainder_bw
           - eltwise.binary_backward.rsub_bw.rsub_bw
           - eltwise.binary_backward.squared_difference_bw.squared_difference_bw
           - eltwise.composite.binary.addalpha.addalpha

CMakeLists.txt

@@ -24,13 +24,6 @@ else()
     FIND_AND_SET_CLANG17()
 endif()
 
-if(${PROJECT_SOURCE_DIR} STREQUAL ${PROJECT_BINARY_DIR})
-    message(
-        FATAL_ERROR
-        "CMake generation is not allowed within source directory!! Please set a build folder with '-B'!!"
-    )
-endif()
-
 project(
     Metalium
     VERSION 0.50.0
@@ -39,8 +32,40 @@ project(
     LANGUAGES
         CXX
 )
-include(CTest)
+
+if(${PROJECT_SOURCE_DIR} STREQUAL ${PROJECT_BINARY_DIR})
+    message(
+        FATAL_ERROR
+        "CMake generation is not allowed within source directory!! Please set a build folder with '-B'!!"
+    )
+endif()
+
 list(PREPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake)
+
+include(project_options)
+
+set(CMAKE_CXX_STANDARD 20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS OFF)
+
+if(ENABLE_LIBCXX)
+    add_compile_options($<$<COMPILE_LANG_AND_ID:CXX,Clang>:-stdlib=libc++>)
+    #add_link_options(
+    #    $<$<LINK_LANG_AND_ID:CXX,Clang>:-lc++>
+    #    $<$<LINK_LANG_AND_ID:CXX,Clang>:-lc++abi>
+    #)
+endif()
+
+# Using below until we can move to CMake >= 3.18 for LINK_LANG_AND_ID
+if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang" AND ENABLE_LIBCXX)
+    set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -lc++ -lc++abi")
+    set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -lc++ -lc++abi")
+endif()
+
+add_compile_options($<$<COMPILE_LANG_AND_ID:CXX,GNU>:-fsized-deallocation>)
+
+include(CTest)
+
 get_property(isMultiConfig GLOBAL PROPERTY GENERATOR_IS_MULTI_CONFIG)
 
 # Global settings if we're the top-level project
@@ -63,7 +88,6 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
                 ON
     )
 
-    option(ENABLE_CCACHE FALSE)
     if(ENABLE_CCACHE)
         include(cmake/ccache.cmake)
     endif()
@@ -81,31 +105,13 @@ set(CMAKE_CXX_FLAGS_DEBUG "-O0 -g -DDEBUG=DEBUG")
 set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O3 -g -DDEBUG=DEBUG")
 set(CMAKE_CXX_FLAGS_CI "-O3 -DDEBUG=DEBUG")
 
-set(CMAKE_CXX_STANDARD 20)
-set(CMAKE_CXX_STANDARD_REQUIRED ON)
-set(CMAKE_CXX_EXTENSIONS OFF)
-
 # Set default values for variables/options
 set(UMD_HOME "${PROJECT_SOURCE_DIR}/tt_metal/third_party/umd")
 
 ############################################################################################################################
 # Project Options
 #   The following options and their defaults impact what artifacts get built
 ############################################################################################################################
-option(WITH_PYTHON_BINDINGS "Enables build of python bindings" ON)
-option(ENABLE_CODE_TIMERS "Enable code timers" OFF)
-option(ENABLE_TRACY "Enable Tracy Profiling" OFF)
-option(ENABLE_LIBCXX "Enable using libc++" ON)
-option(ENABLE_BUILD_TIME_TRACE "Enable build time trace (Clang only -ftime-trace)" OFF)
-option(BUILD_SHARED_LIBS "Create shared libraries" ON)
-option(ENABLE_ASAN "Enable build with AddressSanitizer" OFF)
-option(ENABLE_MSAN "Enable build with MemorySanitizer" OFF)
-option(ENABLE_TSAN "Enable build with ThreadSanitizer" OFF)
-option(ENABLE_UBSAN "Enable build with UndefinedBehaviorSanitizer" OFF)
-option(BUILD_PROGRAMMING_EXAMPLES "Enables build of tt_metal programming examples" OFF)
-option(TT_METAL_BUILD_TESTS "Enables build of tt_metal tests" OFF)
-option(TTNN_BUILD_TESTS "Enables build of ttnn tests" OFF)
-
 message(STATUS "Build shared libs: ${BUILD_SHARED_LIBS}")
 message(STATUS "Build with ASAN: ${ENABLE_ASAN}")
 message(STATUS "Build with MSAN: ${ENABLE_MSAN}")
@@ -177,29 +183,6 @@ add_subdirectory(tt_metal/third_party/umd)
 #   These interface libs are linked with PUBLIC scope at lowest common target (tt_metal/common) and at tt_metal_libs level
 #   in order to propogate to the rest of tt_metal, tt_eager, etc.
 ############################################################################################################################
-add_library(stdlib INTERFACE)
-if(CMAKE_CXX_COMPILER_ID MATCHES "Clang" AND ENABLE_LIBCXX)
-    find_library(LIBC++ c++)
-    find_library(LIBC++ABI c++abi)
-    if(NOT LIBC++ OR NOT LIBC++ABI)
-        message(
-            FATAL_ERROR
-            "libc++ or libc++abi not found. Make sure you have libc++ and libc++abi installed and in your PATH"
-        )
-    endif()
-
-    target_link_libraries(
-        stdlib
-        INTERFACE
-            ${LIBC++}
-            ${LIBC++ABI}
-    )
-    target_compile_options(stdlib INTERFACE -stdlib=libc++)
-else()
-    target_link_libraries(stdlib INTERFACE stdc++)
-    target_compile_options(stdlib INTERFACE -fsized-deallocation)
-endif()
-
 add_library(metal_common_libs INTERFACE)
 target_link_libraries(
     metal_common_libs
@@ -210,7 +193,6 @@ target_link_libraries(
         atomic
         hwloc
         numa
-        stdlib # system libraries, hwloc has no cmake support, find_package won't find it
 )
 
 # Note on flags:
@@ -239,7 +221,6 @@ target_link_libraries(
     INTERFACE
         linker_flags
         compiler_warnings
-        stdlib
 )
 target_compile_options(
     compiler_flags

README.md

@@ -53,6 +53,7 @@
 | [ViT](./models/demos/grayskull/vit)                                         | 9     | [e150](https://tenstorrent.com/hardware/grayskull)       | 1,360   | 2,000      |             |
 | [ViT](./models/demos/wormhole/vit)                                          | 8     | [n150](https://tenstorrent.com/hardware/wormhole)        | 912     | 1,600      |             |
 | [Stable Diffusion 1.4 (512x512)](./models/demos/wormhole/stable_diffusion)  | 1     | [n150](https://tenstorrent.com/hardware/wormhole)        | 0.167   | 0.3        |             |
+| [Yolo V4 (320x320)](./models/experimental/yolov4)                           | 1     | [n150](https://tenstorrent.com/hardware/wormhole)        | 95      | 300        |             |
 
 
 ## NLPs

best_practices.md

@@ -18,13 +18,13 @@ void write_buffer(queue_id cq_id, Tensor& dst, std::vector<std::shared_ptr<void>
 void write_buffer(queue_id cq_id, Tensor& dst, const std::vector<std::shared_ptr<void>>& src, const std::optional<std::size_t>& transfer_size = std::nullopt); // Right!
 ```
 
-## 2. Use `std::span` for Input Parameters
+## 2. Use `tt::stl::Span` for Input Parameters
 
 ### Practice
-Consider using `std::span` as input instead of `std::vector`. This allows `std::array` to be used as an argument as well.
+Consider using `tt::stl::Span` as input instead of `std::vector`. This allows `std::array` to be used as an argument as well.
 
 ### Explanation
-`std::span` is a lightweight view over a contiguous sequence of objects, such as arrays and vectors. It provides a safe and flexible way to handle array-like data structures without copying them.
+`tt::stl::Spann` is a lightweight view over a contiguous sequence of objects, such as arrays and vectors. It provides a safe and flexible way to handle array-like data structures without copying them.
 
 ### Motivation
 - **Flexibility**: Enables functions to accept both `std::vector` and `std::array`.
@@ -33,7 +33,7 @@ Consider using `std::span` as input instead of `std::vector`. This allows `std::
 ### Example
 ```
 template <typename T>
-void print_elements(std::span<T> data) {
+void print_elements(tt::stl::Span<const T> data) {
     for (const auto& element : data) {
         std::cout << element << " ";
     }
@@ -217,7 +217,7 @@ Use the Copy-and-Swap idiom to avoid duplicating code between different construc
 ### Explanation
 The Copy-and-Swap idiom is a robust and elegant method to implement copy assignment operators. It leverages the copy constructor and the swap method to provide strong exception safety and reduce code duplication.
 
-### Example 
+### Example
 https://stackoverflow.com/questions/3279543/what-is-the-copy-and-swap-idiom
 
 
@@ -279,7 +279,7 @@ Prefer:
 enum class ThreadingOption { SingleCore, MultiCore };
 tensor = tt::tt_metal::tilize_with_val_padding(tensor, output_shape, 0, output_memory_config, dtype, ThreadingOption::MultiCore);
 ```
-Also consider giving enums power-of-2 values to pass them all as a single argument, e.g. 
+Also consider giving enums power-of-2 values to pass them all as a single argument, e.g.
 ```cpp
 Options::FOO | Options::BAR
 ```
@@ -343,7 +343,7 @@ void doSomething(...) {
 Prefer:
 ```cpp
 void doSomething(...) {
-    if (!contractCheck) 
+    if (!contractCheck)
         return;
 
     // Do a lot of things

cmake/compilers.cmake

@@ -14,6 +14,16 @@ function(CHECK_COMPILERS)
     message(STATUS "Checking compilers")
 
     if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
+        if(ENABLE_LIBCXX)
+            find_library(LIBC++ c++)
+            find_library(LIBC++ABI c++abi)
+            if(NOT LIBC++ OR NOT LIBC++ABI)
+                message(
+                    FATAL_ERROR
+                    "libc++ or libc++abi not found. Make sure you have libc++ and libc++abi installed and in your PATH"
+                )
+            endif()
+        endif()
         if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS "17.0.0" OR CMAKE_CXX_COMPILER_VERSION GREATER_EQUAL "18.0.0")
             message(WARNING "Only Clang-17 is tested right now")
         endif()

cmake/dependencies.cmake

@@ -8,6 +8,7 @@ include(${PROJECT_SOURCE_DIR}/cmake/fetch_boost.cmake)
 
 fetch_boost_library(core)
 fetch_boost_library(smart_ptr)
+fetch_boost_library(container)
 
 add_library(span INTERFACE)
 target_link_libraries(span INTERFACE Boost::core)
@@ -27,7 +28,6 @@ CPMAddPackage(
 )
 
 if(yaml-cpp_ADDED)
-    target_link_libraries(yaml-cpp PRIVATE stdlib)
     set_target_properties(
         yaml-cpp
         PROPERTIES
@@ -51,8 +51,6 @@ CPMAddPackage(
 
 if(googletest_ADDED)
     target_compile_options(gtest PRIVATE -Wno-implicit-int-float-conversion)
-    target_link_libraries(gtest PRIVATE stdlib)
-    target_link_libraries(gtest_main PRIVATE stdlib)
 endif()
 
 ############################################################################################################################

cmake/project_options.cmake

@@ -0,0 +1,19 @@
+###########################################################################################
+# Project Options
+#   The following options and their defaults impact what artifacts get built
+###########################################################################################
+option(WITH_PYTHON_BINDINGS "Enables build of python bindings" ON)
+option(ENABLE_CODE_TIMERS "Enable code timers" OFF)
+option(ENABLE_TRACY "Enable Tracy Profiling" OFF)
+option(ENABLE_LIBCXX "Enable using libc++" ON)
+option(ENABLE_BUILD_TIME_TRACE "Enable build time trace (Clang only -ftime-trace)" OFF)
+option(BUILD_SHARED_LIBS "Create shared libraries" ON)
+option(ENABLE_ASAN "Enable build with AddressSanitizer" OFF)
+option(ENABLE_MSAN "Enable build with MemorySanitizer" OFF)
+option(ENABLE_TSAN "Enable build with ThreadSanitizer" OFF)
+option(ENABLE_UBSAN "Enable build with UndefinedBehaviorSanitizer" OFF)
+option(BUILD_PROGRAMMING_EXAMPLES "Enables build of tt_metal programming examples" OFF)
+option(TT_METAL_BUILD_TESTS "Enables build of tt_metal tests" OFF)
+option(TTNN_BUILD_TESTS "Enables build of ttnn tests" OFF)
+option(ENABLE_CCACHE "Build with compiler cache" FALSE)
+###########################################################################################

models/demos/t3000/llama2_70b/demo/demo.py

@@ -262,7 +262,7 @@ def run_decode(
     if trace_mode:
         logger.info("Capturing trace")
         trace_id, tt_inp_emb, rot_mat, cache_idxs_tt, tt_logits, _ = model.capture_trace(
-            tokens[:, prev_pos:min_prompt_len], prev_pos
+            tokens[:, prev_pos : prev_pos + 1], prev_pos
         )
 
     for cur_pos in range(min_prompt_len, total_len):

models/demos/t3000/llama2_70b/tt/llama_generation.py

@@ -188,6 +188,7 @@ def decode_forward_trace(
         tt_logits,
         page_table=None,
         tt_page_table=None,
+        read_from_device=True,
     ):
         batch = tokens.shape[0]
 
@@ -207,12 +208,20 @@ def decode_forward_trace(
 
         # Run TT model
         ttnn.execute_trace(self.mesh_device, trace_id, cq_id=0, blocking=False)
+        if read_from_device:
+            logits = self.read_forward_trace(tt_logits, unpadded_batch=batch)
+            return logits
+        else:
+            return tt_logits
+
+    def read_forward_trace(self, tt_logits, unpadded_batch=None):
         updated_tt_logits = ttnn.from_device(tt_logits)
 
         logits = self._process_logits(updated_tt_logits)
 
         logits = logits.permute(2, 1, 0, 3).squeeze().unsqueeze(1)  # [batch, 1, vocab_size]
-        logits = logits[:batch]  # Remove padded users
+        if unpadded_batch is not None:
+            logits = logits[:unpadded_batch]  # Remove padded users
 
         return logits