Merge branch 'ad-freiburg:master' into magic-service-query-text-search

.clang-format

@@ -6,4 +6,8 @@ LineEnding: LF
 InsertNewlineAtEOF: true
 RequiresClausePosition: SingleLine
 IndentRequiresClause: false
+AttributeMacros: [CPP_member, CPP_auto_member, CPP_fun, CPP_auto_fun]
+# TODO<joka921> Update to Clang-format-17 and then we can have much nicer CPP_... macros.
+#Macros:
+#  - CPP_template_2(x)=template<x> requires
 ...

benchmark/JoinAlgorithmBenchmark.cpp

@@ -770,27 +770,29 @@ class GeneralInterfaceImplementation : public BenchmarkInterface {
       };
     };
     auto generateBiggerEqualLambdaDesc =
-        [](const ad_utility::isInstantiation<
-               ad_utility::ConstConfigOptionProxy> auto& option,
-           const auto& minimumValue, bool canBeEqual) {
-          return absl::StrCat("'", option.getConfigOption().getIdentifier(),
-                              "' must be bigger than",
-                              canBeEqual ? ", or equal to," : "", " ",
-                              minimumValue, ".");
-        };
+        []<typename OptionType,
+           typename = std::enable_if_t<ad_utility::isInstantiation<
+               OptionType, ad_utility::ConstConfigOptionProxy>>>(
+            const OptionType& option, const auto& minimumValue,
+            bool canBeEqual) {
+      return absl::StrCat("'", option.getConfigOption().getIdentifier(),
+                          "' must be bigger than",
+                          canBeEqual ? ", or equal to," : "", " ", minimumValue,
+                          ".");
+    };
 
     // Object with a `operator()` for the `<=` operator.
     auto lessEqualLambda = std::less_equal<size_t>{};
     auto generateLessEqualLambdaDesc =
-        [](const ad_utility::isInstantiation<
-               ad_utility::ConstConfigOptionProxy> auto& lhs,
-           const ad_utility::isInstantiation<
-               ad_utility::ConstConfigOptionProxy> auto& rhs) {
-          return absl::StrCat("'", lhs.getConfigOption().getIdentifier(),
-                              "' must be smaller than, or equal to, "
-                              "'",
-                              rhs.getConfigOption().getIdentifier(), "'.");
-        };
+        []<typename OptionType,
+           typename = std::enable_if_t<ad_utility::isInstantiation<
+               OptionType, ad_utility::ConstConfigOptionProxy>>>(
+            const OptionType& lhs, const OptionType& rhs) {
+      return absl::StrCat("'", lhs.getConfigOption().getIdentifier(),
+                          "' must be smaller than, or equal to, "
+                          "'",
+                          rhs.getConfigOption().getIdentifier(), "'.");
+    };
 
     // Adding the validators.
 

benchmark/infrastructure/BenchmarkMeasurementContainer.h

@@ -227,8 +227,8 @@ class ResultTable : public BenchmarkMetadataGetter {
 
   @param row, column Which table entry to read. Starts with `(0,0)`.
   */
-  template <ad_utility::SameAsAnyTypeIn<EntryType> T>
-  T getEntry(const size_t row, const size_t column) const {
+  CPP_template(typename T)(requires ad_utility::SameAsAnyTypeIn<T, EntryType>) T
+      getEntry(const size_t row, const size_t column) const {
     AD_CONTRACT_CHECK(row < numRows() && column < numColumns());
     static_assert(!ad_utility::isSimilar<T, std::monostate>);
 

benchmark/infrastructure/BenchmarkToJson.cpp

@@ -31,12 +31,12 @@ provided translation function for the vector entries.
 @tparam TranslationFunction Has to be a function, that takes `VectorType`
 and returns a `nlohmann:json` object.
 */
-template <typename VectorType, ad_utility::InvocableWithExactReturnType<
-                                   nlohmann::ordered_json, VectorType>
-                                   TranslationFunction>
-static nlohmann::json transformIntoJsonArray(
-    const std::vector<VectorType>& vec,
-    TranslationFunction translationFunction) {
+CPP_template(typename VectorType, typename TranslationFunction)(
+    requires ad_utility::InvocableWithExactReturnType<
+        TranslationFunction, nlohmann::ordered_json,
+        VectorType>) static nlohmann::json
+    transformIntoJsonArray(const std::vector<VectorType>& vec,
+                           TranslationFunction translationFunction) {
   /*
   Without explicit instantiation, `nlohmann::nlohmann::ordered_json` is not
   guaranteed, to always interpret a `push_back` correctly. For instance,

benchmark/util/ResultTableColumnOperations.h

@@ -21,8 +21,9 @@ Column number together with the type of value, that can be found inside the
 column. Note, that **all** entries in the column must have the same type,
 because of `ResultTable::getEntry`.
 */
-template <ad_utility::SameAsAnyTypeIn<ResultTable::EntryType> Type>
-struct ColumnNumWithType {
+CPP_template(typename Type)(
+    requires ad_utility::SameAsAnyTypeIn<
+        Type, ResultTable::EntryType>) struct ColumnNumWithType {
   using ColumnType = Type;
   const size_t columnNum_;
 };

src/backports/concepts.h

@@ -5,6 +5,8 @@
 #pragma once
 
 #include <concepts/concepts.hpp>
+
+#include "backports/cppTemplate2.h"
 #ifndef QLEVER_CPP_17
 #include <concepts>
 #endif
@@ -32,24 +34,39 @@
 // NOTE: The macros are variadic to allow for commas in the argument, like in
 // the second example above.
 
+// Additionally define the macros `CPP_template_2` and `CPP_and_2` that can
+// be used to constrain member functions of classes where the outer class
+// has already been constrained with `CPP_template`. For a detailed example, see
+// the `test/backports/ConceptsTest.cpp` file.
+
 #ifdef QLEVER_CPP_17
 #define QL_CONCEPT_OR_NOTHING(...)
 #define QL_CONCEPT_OR_TYPENAME(...) typename
+#define CPP_template_2 CPP_template_2_SFINAE
+#define CPP_and_2 CPP_and_2_sfinae
+#define CPP_and_def CPP_and_sfinae_def
+#define CPP_and_2_def CPP_and_2_def_sfinae
+#define CPP_variadic_template CPP_template_NO_DEFAULT_SFINAE
+#define CPP_member_def CPP_member_def_sfinae
 #else
 #define QL_CONCEPT_OR_NOTHING(...) __VA_ARGS__
 #define QL_CONCEPT_OR_TYPENAME(...) __VA_ARGS__
+#define CPP_template_2 CPP_template
+#define CPP_and_2 CPP_and
+#define CPP_and_def CPP_and
+#define CPP_and_2_def CPP_and
+#define CPP_variadic_template CPP_template
+#define CPP_member_def CPP_member
 #endif
 
 // The namespace `ql::concepts` includes concepts that are contained in the
 // C++20 standard as well as in `range-v3`.
-namespace ql {
-namespace concepts {
+namespace ql::concepts {
 
 #ifdef QLEVER_CPP_17
 using namespace ::concepts;
 #else
 using namespace std;
 #endif
 
-}  // namespace concepts
-}  // namespace ql
+}  // namespace ql::concepts

src/backports/cppTemplate2.h

@@ -0,0 +1,60 @@
+// Copyright 2024, University of Freiburg
+// Chair of Algorithms and Data Structures
+// Author: Johannes Kalmbach <[email protected]>
+
+#pragma once
+#include <concepts/concepts.hpp>
+
+// The internal reimplementation of a `CPP_variadic_template` macro
+// that can be used for variadic template functions.
+#define CPP_TEMPLATE_NO_DEFAULT_SFINAE_AUX_0(...)                         \
+  , std::enable_if_t<CPP_PP_CAT(CPP_TEMPLATE_SFINAE_AUX_3_, __VA_ARGS__), \
+                     int> = 0 > CPP_PP_IGNORE_CXX2A_COMPAT_END
+
+#define CPP_TEMPLATE_NO_DEFAULT_SFINAE_AUX_3_requires
+
+#define CPP_TEMPLATE_NO_DEFAULT_SFINAE_AUX_WHICH_(FIRST, ...) \
+  CPP_PP_EVAL(CPP_PP_CHECK,                                   \
+              CPP_PP_CAT(CPP_TEMPLATE_SFINAE_PROBE_CONCEPT_, FIRST))
+
+#define CPP_TEMPLATE_NO_DEFAULT_SFINAE_AUX_(...)                       \
+  CPP_PP_CAT(CPP_TEMPLATE_NO_DEFAULT_SFINAE_AUX_,                      \
+             CPP_TEMPLATE_NO_DEFAULT_SFINAE_AUX_WHICH_(__VA_ARGS__, )) \
+  (__VA_ARGS__)
+
+#define CPP_template_NO_DEFAULT_SFINAE(...) \
+  CPP_PP_IGNORE_CXX2A_COMPAT_BEGIN          \
+  template <__VA_ARGS__ CPP_TEMPLATE_NO_DEFAULT_SFINAE_AUX_
+
+// The internal reimplementation of a `CPP_template_2` and `CPP_and_2` macro
+// that can be used for class members when the outer class has already been
+// constrained using `CPP_template`.
+#define CPP_TEMPLATE_2_SFINAE_AUX_0(...)                                \
+  , bool CPP_true_2 = true,                                             \
+         std::enable_if_t <                                             \
+                 CPP_PP_CAT(CPP_TEMPLATE_SFINAE_AUX_3_, __VA_ARGS__) && \
+             CPP_BOOL(CPP_true_2),                                      \
+         int > = 0 > CPP_PP_IGNORE_CXX2A_COMPAT_END
+
+#define CPP_TEMPLATE_2_SFINAE_AUX_3_requires
+
+#define CPP_TEMPLATE_2_SFINAE_AUX_WHICH_(FIRST, ...) \
+  CPP_PP_EVAL(CPP_PP_CHECK,                          \
+              CPP_PP_CAT(CPP_TEMPLATE_SFINAE_PROBE_CONCEPT_, FIRST))
+
+#define CPP_TEMPLATE_2_SFINAE_AUX_(...)                       \
+  CPP_PP_CAT(CPP_TEMPLATE_2_SFINAE_AUX_,                      \
+             CPP_TEMPLATE_2_SFINAE_AUX_WHICH_(__VA_ARGS__, )) \
+  (__VA_ARGS__)
+
+#define CPP_template_2_SFINAE(...) \
+  CPP_PP_IGNORE_CXX2A_COMPAT_BEGIN \
+  template <__VA_ARGS__ CPP_TEMPLATE_2_SFINAE_AUX_
+
+/// INTERNAL ONLY
+#define CPP_and_def_sfinae &&CPP_BOOL(CPP_true), int >, std::enable_if_t <
+#define CPP_and_2_sfinae &&CPP_BOOL(CPP_true_2), int > = 0, std::enable_if_t <
+#define CPP_and_2_def_sfinae &&CPP_BOOL(CPP_true_2), int >, std::enable_if_t <
+
+#define CPP_member_def_sfinae \
+  template <bool (&CPP_true_fn)(::concepts::detail::xNil)>

src/engine/Bind.cpp

@@ -27,6 +27,9 @@ size_t Bind::getCostEstimate() {
   return _subtree->getCostEstimate() + _subtree->getSizeEstimate();
 }
 
+// We delegate the limit to the child operation, so we always support it.
+bool Bind::supportsLimit() const { return true; }
+
 float Bind::getMultiplicity(size_t col) {
   // this is the newly added column
   if (col == getResultWidth() - 1) {
@@ -93,6 +96,7 @@ IdTable Bind::cloneSubView(const IdTable& idTable,
 
 // _____________________________________________________________________________
 ProtoResult Bind::computeResult(bool requestLaziness) {
+  _subtree->setLimit(getLimit());
   LOG(DEBUG) << "Get input to BIND operation..." << std::endl;
   std::shared_ptr<const Result> subRes = _subtree->getResult(requestLaziness);
   LOG(DEBUG) << "Got input to Bind operation." << std::endl;

src/engine/Bind.h

@@ -30,6 +30,7 @@ class Bind : public Operation {
   [[nodiscard]] size_t getResultWidth() const override;
   std::vector<QueryExecutionTree*> getChildren() override;
   size_t getCostEstimate() override;
+  bool supportsLimit() const override;
 
  private:
   uint64_t getSizeEstimateBeforeLimit() override;

src/engine/CMakeLists.txt

@@ -14,5 +14,6 @@ add_library(engine
         CartesianProductJoin.cpp TextIndexScanForWord.cpp TextIndexScanForEntity.cpp
         TextLimit.cpp LazyGroupBy.cpp GroupByHashMapOptimization.cpp SpatialJoin.cpp
         CountConnectedSubgraphs.cpp SpatialJoinAlgorithms.cpp PathSearch.cpp ExecuteUpdate.cpp
-        Describe.cpp GraphStoreProtocol.cpp)
+        Describe.cpp GraphStoreProtocol.cpp
+        QueryExecutionContext.cpp)
 qlever_target_link_libraries(engine util index parser sparqlExpressions http SortPerformanceEstimator Boost::iostreams s2)

src/engine/ExecuteUpdate.cpp

@@ -176,9 +176,31 @@ ExecuteUpdate::computeGraphUpdateQuads(
       cancellationHandle->throwIfCancelled();
     }
   }
+  sortAndRemoveDuplicates(toInsert);
+  sortAndRemoveDuplicates(toDelete);
+  metadata.inUpdate_ = DeltaTriplesCount{static_cast<int64_t>(toInsert.size()),
+                                         static_cast<int64_t>(toDelete.size())};
+  toDelete = setMinus(toDelete, toInsert);
   metadata.triplePreparationTime_ = timer.msecs();
 
   return {
       IdTriplesAndLocalVocab{std::move(toInsert), std::move(localVocabInsert)},
       IdTriplesAndLocalVocab{std::move(toDelete), std::move(localVocabDelete)}};
 }
+
+// _____________________________________________________________________________
+void ExecuteUpdate::sortAndRemoveDuplicates(
+    std::vector<IdTriple<>>& container) {
+  ql::ranges::sort(container);
+  container.erase(std::unique(container.begin(), container.end()),
+                  container.end());
+}
+
+// _____________________________________________________________________________
+std::vector<IdTriple<>> ExecuteUpdate::setMinus(
+    const std::vector<IdTriple<>>& a, const std::vector<IdTriple<>>& b) {
+  std::vector<IdTriple<>> reducedToDelete;
+  reducedToDelete.reserve(a.size());
+  ql::ranges::set_difference(a, b, std::back_inserter(reducedToDelete));
+  return reducedToDelete;
+}

src/engine/ExecuteUpdate.h

@@ -17,6 +17,7 @@ struct UpdateMetadata {
   Milliseconds triplePreparationTime_ = Zero;
   Milliseconds insertionTime_ = Zero;
   Milliseconds deletionTime_ = Zero;
+  std::optional<DeltaTriplesCount> inUpdate_;
 };
 
 class ExecuteUpdate {
@@ -71,4 +72,16 @@ class ExecuteUpdate {
                           const CancellationHandle& cancellationHandle,
                           UpdateMetadata& metadata);
   FRIEND_TEST(ExecuteUpdate, computeGraphUpdateQuads);
+
+  // After the operation the vector is sorted and contains no duplicate
+  // elements.
+  static void sortAndRemoveDuplicates(std::vector<IdTriple<>>& container);
+  FRIEND_TEST(ExecuteUpdate, sortAndRemoveDuplicates);
+
+  // For two sorted vectors `A` and `B` return a new vector
+  // that contains the element of `A\B`.
+  // Precondition: the inputs must be sorted.
+  static std::vector<IdTriple<>> setMinus(const std::vector<IdTriple<>>& a,
+                                          const std::vector<IdTriple<>>& b);
+  FRIEND_TEST(ExecuteUpdate, setMinus);
 };

src/engine/ExportQueryExecutionTrees.cpp

@@ -901,7 +901,7 @@ cppcoro::generator<std::string> ExportQueryExecutionTrees::computeResult(
 
   auto inner =
       ad_utility::ConstexprSwitch<csv, tsv, octetStream, turtle, sparqlXml,
-                                  sparqlJson, qleverJson>(compute, mediaType);
+                                  sparqlJson, qleverJson>{}(compute, mediaType);
   return convertStreamGeneratorForChunkedTransfer(std::move(inner));
 }
 

src/engine/Filter.cpp

@@ -104,10 +104,10 @@ ProtoResult Filter::computeResult(bool requestLaziness) {
 }
 
 // _____________________________________________________________________________
-template <ad_utility::SimilarTo<IdTable> Table>
-IdTable Filter::filterIdTable(std::vector<ColumnIndex> sortedBy,
-                              Table&& idTable,
-                              const LocalVocab& localVocab) const {
+CPP_template_def(typename Table)(requires ad_utility::SimilarTo<Table, IdTable>)
+    IdTable Filter::filterIdTable(std::vector<ColumnIndex> sortedBy,
+                                  Table&& idTable,
+                                  const LocalVocab& localVocab) const {
   size_t width = idTable.numColumns();
   IdTable result{width, getExecutionContext()->getAllocator()};
 
@@ -120,10 +120,11 @@ IdTable Filter::filterIdTable(std::vector<ColumnIndex> sortedBy,
 }
 
 // _____________________________________________________________________________
-template <int WIDTH, ad_utility::SimilarTo<IdTable> Table>
-void Filter::computeFilterImpl(IdTable& dynamicResultTable, Table&& inputTable,
-                               const LocalVocab& localVocab,
-                               std::vector<ColumnIndex> sortedBy) const {
+CPP_template_def(int WIDTH, typename Table)(
+    requires ad_utility::SimilarTo<Table, IdTable>) void Filter::
+    computeFilterImpl(IdTable& dynamicResultTable, Table&& inputTable,
+                      const LocalVocab& localVocab,
+                      std::vector<ColumnIndex> sortedBy) const {
   AD_CONTRACT_CHECK(inputTable.numColumns() == WIDTH || WIDTH == 0);
   IdTableStatic<WIDTH> resultTable =
       std::move(dynamicResultTable).toStatic<static_cast<size_t>(WIDTH)>();

src/engine/Filter.h

@@ -70,13 +70,16 @@ class Filter : public Operation {
   ProtoResult computeResult(bool requestLaziness) override;
 
   // Perform the actual filter operation of the data provided.
-  template <int WIDTH, ad_utility::SimilarTo<IdTable> Table>
-  void computeFilterImpl(IdTable& dynamicResultTable, Table&& input,
-                         const LocalVocab& localVocab,
-                         std::vector<ColumnIndex> sortedBy) const;
+  CPP_template(int WIDTH, typename Table)(
+      requires ad_utility::SimilarTo<
+          Table, IdTable>) void computeFilterImpl(IdTable& dynamicResultTable,
+                                                  Table&& input,
+                                                  const LocalVocab& localVocab,
+                                                  std::vector<ColumnIndex>
+                                                      sortedBy) const;
 
   // Run `computeFilterImpl` on the provided IdTable
-  template <ad_utility::SimilarTo<IdTable> Table>
-  IdTable filterIdTable(std::vector<ColumnIndex> sortedBy, Table&& idTable,
-                        const LocalVocab& localVocab) const;
+  CPP_template(typename Table)(requires ad_utility::SimilarTo<Table, IdTable>)
+      IdTable filterIdTable(std::vector<ColumnIndex> sortedBy, Table&& idTable,
+                            const LocalVocab& localVocab) const;
 };

src/engine/OptionalJoin.h

@@ -58,14 +58,8 @@ class OptionalJoin : public Operation {
     return {_left.get(), _right.get()};
   }
 
-  /**
-   * @brief Joins two result tables on any number of columns, inserting the
-   *        special value ID_NO_VALUE for any entries marked as optional.
-   * @param a
-   * @param b
-   * @param joinColumns
-   * @param result
-   */
+  // Joins two result tables on any number of columns, inserting the special
+  // value `Id::makeUndefined()` for any entries marked as optional.
   void optionalJoin(
       const IdTable& left, const IdTable& right,
       const std::vector<std::array<ColumnIndex, 2>>& joinColumns,

src/engine/QueryExecutionContext.cpp

@@ -0,0 +1,11 @@
+//   Copyright 2025, University of Freiburg,
+//   Chair of Algorithms and Data Structures.
+//   Author: Robin Textor-Falconi <[email protected]>
+
+#include "engine/QueryExecutionContext.h"
+
+#include "global/RuntimeParameters.h"
+
+bool QueryExecutionContext::areWebSocketUpdatesEnabled() {
+  return RuntimeParameters().get<"websocket-updates-enabled">();
+}