From 5291c30b947425bee321cc5008b20febc5e61aa2 Mon Sep 17 00:00:00 2001
From: Jonathan Lifflander <jliffla@sandia.gov>
Date: Wed, 8 Feb 2023 16:53:34 -0800
Subject: [PATCH] #276: reduce: add synthesized reduce

---
 examples/hello_world/CMakeLists.txt           |  1 +
 examples/hello_world/hello_reduce.cc          | 62 ++++++++++++++++
 .../collective/reduce/operators/default_msg.h |  1 +
 .../reduce/operators/default_op.impl.h        |  8 ++-
 .../reduce/operators/functors/and_op.h        |  8 +++
 .../reduce/operators/functors/bit_and_op.h    |  8 +++
 .../reduce/operators/functors/bit_or_op.h     |  8 +++
 .../reduce/operators/functors/bit_xor_op.h    |  8 +++
 .../reduce/operators/functors/max_op.h        |  8 +++
 .../reduce/operators/functors/min_op.h        |  8 +++
 .../reduce/operators/functors/or_op.h         |  8 +++
 .../reduce/operators/functors/plus_op.h       |  8 +++
 .../operators/functors/tuple_op_helper.h      | 70 +++++++++++++++++++
 src/vt/collective/reduce/reduce.h             | 39 ++++++++++-
 src/vt/collective/reduce/reduce_msg.h         |  3 +-
 15 files changed, 244 insertions(+), 4 deletions(-)
 create mode 100644 examples/hello_world/hello_reduce.cc
 create mode 100644 src/vt/collective/reduce/operators/functors/tuple_op_helper.h

diff --git a/examples/hello_world/CMakeLists.txt b/examples/hello_world/CMakeLists.txt
index 1b7d464fc1..7cc336ffb5 100644
--- a/examples/hello_world/CMakeLists.txt
+++ b/examples/hello_world/CMakeLists.txt
@@ -11,6 +11,7 @@ set(
   hello_world_virtual_context_remote
   ring
   objgroup
+  hello_reduce
 )
 
 foreach(EXAMPLE_NAME ${HELLO_WORLD_EXAMPLES})
diff --git a/examples/hello_world/hello_reduce.cc b/examples/hello_world/hello_reduce.cc
new file mode 100644
index 0000000000..46e4dd2170
--- /dev/null
+++ b/examples/hello_world/hello_reduce.cc
@@ -0,0 +1,62 @@
+/*
+//@HEADER
+// *****************************************************************************
+//
+//                               hello_reduce.cc
+//                       DARMA/vt => Virtual Transport
+//
+// Copyright 2019-2021 National Technology & Engineering Solutions of Sandia, LLC
+// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S.
+// Government retains certain rights in this software.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// * Redistributions of source code must retain the above copyright notice,
+//   this list of conditions and the following disclaimer.
+//
+// * Redistributions in binary form must reproduce the above copyright notice,
+//   this list of conditions and the following disclaimer in the documentation
+//   and/or other materials provided with the distribution.
+//
+// * Neither the name of the copyright holder nor the names of its
+//   contributors may be used to endorse or promote products derived from this
+//   software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+// Questions? Contact darma@sandia.gov
+//
+// *****************************************************************************
+//@HEADER
+*/
+
+#include <vt/transport.h>
+
+void reduceResult(int result, double result2) {
+  auto num_nodes = vt::theContext()->getNumNodes();
+  fmt::print("reduction value={}, {}\n", result, result2);
+  vtAssert(num_nodes * 50 == result, "Must be equal");
+}
+
+int main(int argc, char** argv) {
+  vt::initialize(argc, argv);
+
+  vt::NodeType const root = 0;
+
+  auto r = vt::theCollective()->global();
+  r->reduce<vt::collective::PlusOp, reduceResult>(vt::Node{root}, 50, 52.334);
+
+  vt::finalize();
+  return 0;
+}
diff --git a/src/vt/collective/reduce/operators/default_msg.h b/src/vt/collective/reduce/operators/default_msg.h
index af6391b352..1ed218e0ab 100644
--- a/src/vt/collective/reduce/operators/default_msg.h
+++ b/src/vt/collective/reduce/operators/default_msg.h
@@ -80,6 +80,7 @@ struct ReduceDataMsg : SerializeIfNeeded<
     : MessageParentType(), ReduceCombine<void>(), val_(in_val)
   { }
 
+  DataType& getTuple() { return val_; }
   DataType const& getConstVal() const { return val_; }
   DataType& getVal() { return val_; }
   DataType&& getMoveVal() { return std::move(val_); }
diff --git a/src/vt/collective/reduce/operators/default_op.impl.h b/src/vt/collective/reduce/operators/default_op.impl.h
index 5f46db1794..a352ab94e9 100644
--- a/src/vt/collective/reduce/operators/default_op.impl.h
+++ b/src/vt/collective/reduce/operators/default_op.impl.h
@@ -49,6 +49,8 @@
 
 namespace vt { namespace collective { namespace reduce { namespace operators {
 
+struct NoCombine {};
+
 template <typename T>
 template <typename MsgT, typename Op, typename ActOp>
 /*static*/ void ReduceCombine<T>::msgHandler(MsgT* msg) {
@@ -61,8 +63,12 @@ template <typename MsgT, typename Op, typename ActOp>
     if (cb.valid()) {
       envelopeUnlockForForwarding(msg->env);
       cb.template send<MsgT>(msg);
+    } else if (msg->root_handler_ != uninitialized_handler) {
+      auto_registry::getAutoHandler(msg->root_handler_)->dispatch(msg, nullptr);
     } else {
-      ActOp()(msg);
+      if constexpr (not std::is_same_v<ActOp, NoCombine>) {
+        ActOp()(msg);
+      }
     }
   } else {
     MsgT* fst_msg = msg;
diff --git a/src/vt/collective/reduce/operators/functors/and_op.h b/src/vt/collective/reduce/operators/functors/and_op.h
index f97cf7fb56..de4156eff4 100644
--- a/src/vt/collective/reduce/operators/functors/and_op.h
+++ b/src/vt/collective/reduce/operators/functors/and_op.h
@@ -45,6 +45,7 @@
 #define INCLUDED_VT_COLLECTIVE_REDUCE_OPERATORS_FUNCTORS_AND_OP_H
 
 #include "vt/config.h"
+#include "vt/collective/reduce/operators/functors/tuple_op_helper.h"
 
 namespace vt { namespace collective { namespace reduce { namespace operators {
 
@@ -55,6 +56,13 @@ struct AndOp {
   }
 };
 
+template <typename... Params>
+struct AndOp<std::tuple<Params...>> {
+  void operator()(std::tuple<Params...>& v1, std::tuple<Params...> const& v2) {
+    opTuple<AndOp>(v1, v2);
+  }
+};
+
 template <typename T>
 struct AndOp< std::vector<T> > {
   void operator()(std::vector<T>& v1, std::vector<T> const& v2) {
diff --git a/src/vt/collective/reduce/operators/functors/bit_and_op.h b/src/vt/collective/reduce/operators/functors/bit_and_op.h
index 41055eb4e3..b9c687518a 100644
--- a/src/vt/collective/reduce/operators/functors/bit_and_op.h
+++ b/src/vt/collective/reduce/operators/functors/bit_and_op.h
@@ -45,6 +45,7 @@
 #define INCLUDED_VT_COLLECTIVE_REDUCE_OPERATORS_FUNCTORS_BIT_AND_OP_H
 
 #include "vt/config.h"
+#include "vt/collective/reduce/operators/functors/tuple_op_helper.h"
 
 namespace vt { namespace collective { namespace reduce { namespace operators {
 
@@ -55,6 +56,13 @@ struct BitAndOp {
   }
 };
 
+template <typename... Params>
+struct BitAndOp<std::tuple<Params...>> {
+  void operator()(std::tuple<Params...>& v1, std::tuple<Params...> const& v2) {
+    opTuple<BitAndOp>(v1, v2);
+  }
+};
+
 }}}} /* end namespace vt::collective::reduce::operators */
 
 namespace vt { namespace collective {
diff --git a/src/vt/collective/reduce/operators/functors/bit_or_op.h b/src/vt/collective/reduce/operators/functors/bit_or_op.h
index 0f94150563..6e4c4cb303 100644
--- a/src/vt/collective/reduce/operators/functors/bit_or_op.h
+++ b/src/vt/collective/reduce/operators/functors/bit_or_op.h
@@ -45,6 +45,7 @@
 #define INCLUDED_VT_COLLECTIVE_REDUCE_OPERATORS_FUNCTORS_BIT_OR_OP_H
 
 #include "vt/config.h"
+#include "vt/collective/reduce/operators/functors/tuple_op_helper.h"
 
 namespace vt { namespace collective { namespace reduce { namespace operators {
 
@@ -55,6 +56,13 @@ struct BitOrOp {
   }
 };
 
+template <typename... Params>
+struct BitOrOp<std::tuple<Params...>> {
+  void operator()(std::tuple<Params...>& v1, std::tuple<Params...> const& v2) {
+    opTuple<BitOrOp>(v1, v2);
+  }
+};
+
 }}}} /* end namespace vt::collective::reduce::operators */
 
 namespace vt { namespace collective {
diff --git a/src/vt/collective/reduce/operators/functors/bit_xor_op.h b/src/vt/collective/reduce/operators/functors/bit_xor_op.h
index 4da4e97b43..e38f85b141 100644
--- a/src/vt/collective/reduce/operators/functors/bit_xor_op.h
+++ b/src/vt/collective/reduce/operators/functors/bit_xor_op.h
@@ -45,6 +45,7 @@
 #define INCLUDED_VT_COLLECTIVE_REDUCE_OPERATORS_FUNCTORS_BIT_XOR_OP_H
 
 #include "vt/config.h"
+#include "vt/collective/reduce/operators/functors/tuple_op_helper.h"
 
 namespace vt { namespace collective { namespace reduce { namespace operators {
 
@@ -55,6 +56,13 @@ struct BitXorOp {
   }
 };
 
+template <typename... Params>
+struct BitXorOp<std::tuple<Params...>> {
+  void operator()(std::tuple<Params...>& v1, std::tuple<Params...> const& v2) {
+    opTuple<BitXorOp>(v1, v2);
+  }
+};
+
 }}}} /* end namespace vt::collective::reduce::operators */
 
 namespace vt { namespace collective {
diff --git a/src/vt/collective/reduce/operators/functors/max_op.h b/src/vt/collective/reduce/operators/functors/max_op.h
index a59fd6dcf9..94aa6ad349 100644
--- a/src/vt/collective/reduce/operators/functors/max_op.h
+++ b/src/vt/collective/reduce/operators/functors/max_op.h
@@ -45,6 +45,7 @@
 #define INCLUDED_VT_COLLECTIVE_REDUCE_OPERATORS_FUNCTORS_MAX_OP_H
 
 #include "vt/config.h"
+#include "vt/collective/reduce/operators/functors/tuple_op_helper.h"
 
 #include <algorithm>
 
@@ -57,6 +58,13 @@ struct MaxOp {
   }
 };
 
+template <typename... Params>
+struct MaxOp<std::tuple<Params...>> {
+  void operator()(std::tuple<Params...>& v1, std::tuple<Params...> const& v2) {
+    opTuple<MaxOp>(v1, v2);
+  }
+};
+
 template <typename T>
 struct MaxOp< std::vector<T> > {
   void operator()(std::vector<T>& v1, std::vector<T> const& v2) {
diff --git a/src/vt/collective/reduce/operators/functors/min_op.h b/src/vt/collective/reduce/operators/functors/min_op.h
index fab490eaa6..d95199ec99 100644
--- a/src/vt/collective/reduce/operators/functors/min_op.h
+++ b/src/vt/collective/reduce/operators/functors/min_op.h
@@ -45,6 +45,7 @@
 #define INCLUDED_VT_COLLECTIVE_REDUCE_OPERATORS_FUNCTORS_MIN_OP_H
 
 #include "vt/config.h"
+#include "vt/collective/reduce/operators/functors/tuple_op_helper.h"
 
 #include <algorithm>
 
@@ -57,6 +58,13 @@ struct MinOp {
   }
 };
 
+template <typename... Params>
+struct MinOp<std::tuple<Params...>> {
+  void operator()(std::tuple<Params...>& v1, std::tuple<Params...> const& v2) {
+    opTuple<MinOp>(v1, v2);
+  }
+};
+
 template <typename T>
 struct MinOp< std::vector<T> > {
   void operator()(std::vector<T>& v1, std::vector<T> const& v2) {
diff --git a/src/vt/collective/reduce/operators/functors/or_op.h b/src/vt/collective/reduce/operators/functors/or_op.h
index 9fe474203d..26cafde8bb 100644
--- a/src/vt/collective/reduce/operators/functors/or_op.h
+++ b/src/vt/collective/reduce/operators/functors/or_op.h
@@ -45,6 +45,7 @@
 #define INCLUDED_VT_COLLECTIVE_REDUCE_OPERATORS_FUNCTORS_OR_OP_H
 
 #include "vt/config.h"
+#include "vt/collective/reduce/operators/functors/tuple_op_helper.h"
 
 namespace vt { namespace collective { namespace reduce { namespace operators {
 
@@ -55,6 +56,13 @@ struct OrOp {
   }
 };
 
+template <typename... Params>
+struct OrOp<std::tuple<Params...>> {
+  void operator()(std::tuple<Params...>& v1, std::tuple<Params...> const& v2) {
+    opTuple<OrOp>(v1, v2);
+  }
+};
+
 template <typename T>
 struct OrOp<std::vector<T>> {
   void operator()(std::vector<T>& v1, std::vector<T> const& v2) {
diff --git a/src/vt/collective/reduce/operators/functors/plus_op.h b/src/vt/collective/reduce/operators/functors/plus_op.h
index 10120ffb47..7c748890d6 100644
--- a/src/vt/collective/reduce/operators/functors/plus_op.h
+++ b/src/vt/collective/reduce/operators/functors/plus_op.h
@@ -45,6 +45,7 @@
 #define INCLUDED_VT_COLLECTIVE_REDUCE_OPERATORS_FUNCTORS_PLUS_OP_H
 
 #include "vt/config.h"
+#include "vt/collective/reduce/operators/functors/tuple_op_helper.h"
 
 namespace vt { namespace collective { namespace reduce { namespace operators {
 
@@ -55,6 +56,13 @@ struct PlusOp {
   }
 };
 
+template <typename... Params>
+struct PlusOp<std::tuple<Params...>> {
+  void operator()(std::tuple<Params...>& v1, std::tuple<Params...> const& v2) {
+    opTuple<PlusOp>(v1, v2);
+  }
+};
+
 template <typename T>
 struct PlusOp< std::vector<T> > {
   void operator()(std::vector<T>& v1, std::vector<T> const& v2) {
diff --git a/src/vt/collective/reduce/operators/functors/tuple_op_helper.h b/src/vt/collective/reduce/operators/functors/tuple_op_helper.h
new file mode 100644
index 0000000000..e0e750293d
--- /dev/null
+++ b/src/vt/collective/reduce/operators/functors/tuple_op_helper.h
@@ -0,0 +1,70 @@
+/*
+//@HEADER
+// *****************************************************************************
+//
+//                              tuple_op_helper.h
+//                       DARMA/vt => Virtual Transport
+//
+// Copyright 2019-2021 National Technology & Engineering Solutions of Sandia, LLC
+// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S.
+// Government retains certain rights in this software.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// * Redistributions of source code must retain the above copyright notice,
+//   this list of conditions and the following disclaimer.
+//
+// * Redistributions in binary form must reproduce the above copyright notice,
+//   this list of conditions and the following disclaimer in the documentation
+//   and/or other materials provided with the distribution.
+//
+// * Neither the name of the copyright holder nor the names of its
+//   contributors may be used to endorse or promote products derived from this
+//   software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+// Questions? Contact darma@sandia.gov
+//
+// *****************************************************************************
+//@HEADER
+*/
+
+#if !defined INCLUDED_VT_COLLECTIVE_REDUCE_OPERATORS_FUNCTORS_TUPLE_OP_HELPER_H
+#define INCLUDED_VT_COLLECTIVE_REDUCE_OPERATORS_FUNCTORS_TUPLE_OP_HELPER_H
+
+#include <tuple>
+
+namespace vt { namespace collective { namespace reduce { namespace operators {
+
+template <
+  template <typename X> class Op,
+  typename... Ts, typename... Us, std::size_t... I
+>
+void opTuple(
+  std::tuple<Ts...>& t1, std::tuple<Us...> const& t2, std::index_sequence<I...>
+) {
+  std::forward_as_tuple(
+    (Op<std::decay_t<decltype(std::get<I>(t1))>>()(std::get<I>(t1),std::get<I>(t2)),0)...
+  );
+}
+
+template <template <typename X> class Op, typename... Ts, typename... Us>
+void opTuple(std::tuple<Ts...>& t1, std::tuple<Us...> const& t2) {
+  opTuple<Op>(t1, t2, std::make_index_sequence<sizeof...(Ts)>{});
+}
+
+}}}} /* end namespace vt::collective::reduce::operators */
+
+#endif /*INCLUDED_VT_COLLECTIVE_REDUCE_OPERATORS_FUNCTORS_TUPLE_OP_HELPER_H*/
diff --git a/src/vt/collective/reduce/reduce.h b/src/vt/collective/reduce/reduce.h
index 71a3809827..3b0934521c 100644
--- a/src/vt/collective/reduce/reduce.h
+++ b/src/vt/collective/reduce/reduce.h
@@ -168,6 +168,41 @@ struct Reduce : virtual collective::tree::Tree {
     return reduce<MsgT, f>(root, msg, id, num_contrib);
   }
 
+  //////////////////////////////////////////////////////////////
+  //////////////////////////////////////////////////////////////
+  //////////////////////////////////////////////////////////////
+  template <typename ReturnT, typename... Args>
+  struct FunctionTraitsArgs;
+
+  template <typename ReturnT, typename... Args>
+  struct FunctionTraitsArgs<ReturnT(*)(Args...)> {
+    using TupleType = std::tuple<std::decay_t<Args>...>;
+    using ReturnType = ReturnT;
+  };
+
+  template <template<typename Arg> class Op, auto f, typename... Params>
+  PendingSendType reduce(Node root, Params&&... params) {
+    using Tuple = typename FunctionTraitsArgs<decltype(f)>::TupleType;
+    using OpT = Op<Tuple>;
+    return reduce<OpT, f>(root, std::forward<Params>(params)...);
+  }
+
+  template <typename Op, auto f, typename... Params>
+  PendingSendType reduce(Node root, Params&&... params) {
+    using Tuple = typename FunctionTraitsArgs<decltype(f)>::TupleType;
+    using MsgT = ReduceTMsg<Tuple>;
+
+    auto msg = vt::makeMessage<MsgT>(std::tuple{std::forward<Params>(params)...});
+    auto id = detail::ReduceStamp{};
+    auto han = auto_registry::makeAutoHandlerParam<decltype(f), f, MsgT>();
+    msg->root_handler_ = han;
+
+    return reduce<Op, operators::NoCombine, MsgT>(root.get(), msg.get(), id, 1);
+  }
+  //////////////////////////////////////////////////////////////
+  //////////////////////////////////////////////////////////////
+  //////////////////////////////////////////////////////////////
+
   /**
    * \brief Reduce a message up the tree
    *
@@ -243,7 +278,7 @@ struct Reduce : virtual collective::tree::Tree {
         &MsgT::template msgHandler<
           MsgT,
           OpT,
-          collective::reduce::operators::ReduceCallback<MsgT>
+          operators::NoCombine
           >
         >(root, msg, cb, id, num_contrib);
     }
@@ -308,7 +343,7 @@ struct Reduce : virtual collective::tree::Tree {
       &MsgT::template msgHandler<
         MsgT,
         OpT,
-        collective::reduce::operators::ReduceCallback<MsgT>
+        operators::NoCombine
         >
       >(root, msg, cb, id, num_contrib);
   }
diff --git a/src/vt/collective/reduce/reduce_msg.h b/src/vt/collective/reduce/reduce_msg.h
index e2b5da9609..205192ddd4 100644
--- a/src/vt/collective/reduce/reduce_msg.h
+++ b/src/vt/collective/reduce/reduce_msg.h
@@ -88,12 +88,13 @@ struct ReduceMsg : SerializeSupported<
   NodeType reduce_root_ = uninitialized_destination;
   detail::ReduceIDImpl reduce_id_;
   HandlerType combine_handler_ = uninitialized_handler;
+  HandlerType root_handler_ = uninitialized_handler;
 
   template <typename SerializerT>
   void serialize(SerializerT& s) {
     MessageParentType::serialize(s);
     s | reduce_root_ | reduce_id_;
-    s | combine_handler_;
+    s | combine_handler_ | root_handler_;
   }
 };