Skip to content

Commit

Permalink
New container graph completed, including unit tests.
Browse files Browse the repository at this point in the history
  • Loading branch information
@massimim
massimim committed Sep 12, 2022
1 parent 3ca3d30 commit 08a3a40
Show file tree
Hide file tree
Showing 7 changed files with 143 additions and 113 deletions.
3 changes: 2 additions & 1 deletion libNeonDomain/include/Neon/domain/tools/TestData.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -342,7 +342,8 @@ auto TestData<G, T, C>::laplace(IODomain& A, NEON_IO IODomain& B)
}

template <typename G, typename T, int C>
auto TestData<G, T, C>::compare(FieldNames name, T tollerance) -> bool
auto TestData<G, T, C>::compare(FieldNames name,
[[maybe_unused]] T tollerance) -> bool
{
bool isTheSame = false;
if constexpr (std::is_integral_v<T>) {
Expand Down
59 changes: 46 additions & 13 deletions libNeonSet/include/Neon/set/container/Graph.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,14 @@

namespace Neon::set::container {

/**
* Abstraction for a graph of containers.
* Each graph node represents a Neon container,
* directed edges of the graph are dependencies between the containers.
* Dependencies my be data driven or user provided.
*
*
*/
struct Graph
{
using Uid = GraphData::Uid;
Expand Down Expand Up @@ -98,15 +106,24 @@ struct Graph
auto getSubsequentGraphNodes(const GraphNode& graphNode,
const std::vector<GraphDependencyType>& dependencyTypes = {GraphDependencyType::user,
GraphDependencyType::data}) -> std::vector<GraphNode*>;
/**
* Set the stream to run the graph.
* We provide a preset function so that some initialization
* could be only once, before executing the run method.
*/
auto runtimePreSet(int anchorStream)
-> void;

/**
* Execute the scheduling operation associated to the node
* Execute the graph on all devices
*/
auto run(int streamIdx = 0,
Neon::DataView dataView = Neon::DataView::STANDARD)
-> void;

/**
* Run the graph on a target device.
*/
auto run(Neon::SetIdx setIdx,
int streamIdx = 0,
Neon::DataView dataView = Neon::DataView::STANDARD)
Expand All @@ -116,21 +133,34 @@ struct Graph
const std::string& graphName,
bool debug) -> void;

/**
* Returns a reference for the used backend.
*/
auto getBackend() const -> const Neon::Backend&;

/**
* Recursively iterate through the graph nodes
* to expand any graph type of node.
*/
auto expandSubGraphs()
-> void;

auto getNumberOfNodes()
->int;

protected:
/**
* Invalidate all scheduling information that were computed
*/
auto helpInvalidateScheduling() -> void;

/**
* Helper - it checks the initialization of the backend.
*/
auto helpCheckBackendStatus() -> void;

/**
* Remove redundant dependencies
* Helper - it removes redundant dependencies
*/
auto helpRemoveRedundantDependencies() -> void;

Expand Down Expand Up @@ -205,55 +235,58 @@ struct Graph
-> void;

/**
* Execute
* Helper - it executes the graph on all devices
*/
auto helpExecute(int anchorStream)
-> void;

/**
* Helper - it executes the graph on a target device
*/
auto helpExecute(Neon::SetIdx setIdx, int anchorStream)
-> void;
/**
* Resetting node's data related to scheduling
* Helper - It resets node scheduling data
*/
auto helpComputeScheduling_00_resetData()
-> void;

/**
* Resetting node's data related to scheduling
* Helper - it generates a BFS visit structure for the graph
*/
auto helpComputeScheduling_01_generatingBFS(bool filterOutAnchors)
-> Bfs;

/**
* Maps node to streams.
* Helper - it maps node to streams.
* Returns the max stream Id used by the scheduling
*/
auto helpComputeScheduling_02_mappingStreams(Bfs& bfs, bool filterOutAnchors, int anchorStream)
-> int;

/**
* Define events to be waited and fired from each node
* Helper - it defines events to be waited and fired from each node
* Returns the max event Id used by the scheduling.
*/
auto helpComputeScheduling_03_events(Bfs& bfs)
-> int;

/**
* Booking the required resources from the backend.
* Helper - it Books the required resources from the backend.
*/
auto helpComputeScheduling_04_ensureResources(int maxStreamId, int maxEventId)
-> void;


using RawGraph = DiGraph<GraphNode, GraphDependency>;

Uid mUidCounter;
RawGraph mRawGraph;
Uid mUidCounter /**< internal counter to create node uids */;
RawGraph mRawGraph /**< raw graph structure */;
bool mSchedulingStatusIsValid;
int mMaxNumberStreams;
Bfs mBfs;
int mMaxNumberStreams /**< Max number of streams used in the scheduling phase */;
Bfs mBfs /**< Structure for the BFS visit. */;

Backend mBackend;
Backend mBackend /**< Backend used to create streams and events */;
bool mBackendIsSet = false;

bool mFilterOutAnchorsPreSet = false;
Expand Down
4 changes: 3 additions & 1 deletion libNeonSet/include/Neon/set/container/GraphContainer.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,9 @@ struct GraphContainer : ContainerAPI
* @param streamIdx
* @param dataView
*/
auto run(int streamIdx = 0, Neon::DataView dataView = Neon::DataView::STANDARD) -> void override;
auto run(int streamIdx = 0,
Neon::DataView dataView = Neon::DataView::STANDARD)
-> void override;

auto run(Neon::SetIdx setIdx,
int streamIdx = 0,
Expand Down
79 changes: 52 additions & 27 deletions libNeonSet/src/set/container/Graph.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -192,7 +192,8 @@ auto Graph::getProceedingGraphNodes(const GraphNode& grap
}

auto Graph::getSubsequentGraphNodes(const GraphNode& graphNode,
const std::vector<GraphDependencyType>& dependencyTypes) -> std::vector<GraphNode*>
const std::vector<GraphDependencyType>& dependencyTypes)
-> std::vector<GraphNode*>
{
std::vector<GraphNode*> nodes;

Expand Down Expand Up @@ -255,12 +256,14 @@ auto Graph::getDependencyType(const GraphNode& nodeA,
return dependencyType;
}

auto Graph::helpInvalidateScheduling() -> void
auto Graph::helpInvalidateScheduling()
-> void
{
mSchedulingStatusIsValid = false;
}

auto Graph::helpRemoveRedundantDependencies() -> void
auto Graph::helpRemoveRedundantDependencies()
-> void
{
// Vectors of edges to be removed
std::vector<std::pair<size_t, size_t>> edgesToBeRemoved;
Expand Down Expand Up @@ -330,7 +333,8 @@ auto Graph::helpRemoveRedundantDependencies() -> void

auto Graph::helpGetOutNeighbors(GraphData::Uid nodeUid,
bool filteredOut,
const std::vector<GraphDependencyType>& dependencyTypes) -> std::set<GraphData::Uid>
const std::vector<GraphDependencyType>& dependencyTypes)
-> std::set<GraphData::Uid>
{
std::set<GraphData::Uid> outNgh;
mRawGraph.forEachOutEdge(
Expand All @@ -350,7 +354,10 @@ auto Graph::helpGetOutNeighbors(GraphData::Uid nodeUid,
return outNgh;
}

auto Graph::helpGetInNeighbors(GraphData::Uid nodeUid, bool filterOutBegin, const std::vector<GraphDependencyType>& dependencyTypes) -> std::set<GraphData::Uid>
auto Graph::helpGetInNeighbors(GraphData::Uid nodeUid,
bool filterOutBegin,
const std::vector<GraphDependencyType>& dependencyTypes)
-> std::set<GraphData::Uid>
{
std::set<GraphData::Uid> inNgh;
mRawGraph.forEachInEdge(
Expand All @@ -371,7 +378,8 @@ auto Graph::helpGetInNeighbors(GraphData::Uid nodeUid, bool filterOutBegin, cons

auto Graph::helpGetOutEdges(GraphData::Uid nodeUid,
bool filterOutEnd,
const std::vector<GraphDependencyType>& dependencyTypes) -> std::set<std::pair<GraphData::Uid, GraphData::Uid>>
const std::vector<GraphDependencyType>& dependencyTypes)
-> std::set<std::pair<GraphData::Uid, GraphData::Uid>>
{
std::set<std::pair<GraphData::Uid, GraphData::Uid>> outEdges;
mRawGraph.forEachOutEdge(
Expand All @@ -392,7 +400,8 @@ auto Graph::helpGetOutEdges(GraphData::Uid nodeUid,

auto Graph::helpGetInEdges(GraphData::Uid nodeUid,
bool filterOutBegin,
const std::vector<GraphDependencyType>& dependencyTypes) -> std::set<std::pair<GraphData::Uid, GraphData::Uid>>
const std::vector<GraphDependencyType>& dependencyTypes)
-> std::set<std::pair<GraphData::Uid, GraphData::Uid>>
{
std::set<std::pair<GraphData::Uid, GraphData::Uid>> inEdges;
mRawGraph.forEachInEdge(
Expand Down Expand Up @@ -471,7 +480,8 @@ auto Graph::helpGetBFS(bool filterOutBeginEnd
return bfs;
}

auto Graph::helpComputeScheduling(bool filterOutAnchors, int anchorStream) -> void
auto Graph::helpComputeScheduling(bool filterOutAnchors, int anchorStream)
-> void
{
helpComputeScheduling_00_resetData();
mBfs = helpComputeScheduling_01_generatingBFS(filterOutAnchors);
Expand All @@ -480,7 +490,8 @@ auto Graph::helpComputeScheduling(bool filterOutAnchors, int anchorStream) -> vo
helpComputeScheduling_04_ensureResources(maxStreamId, maxEventId);
}

auto Graph::helpComputeScheduling_01_generatingBFS(bool filterOutAnchors) -> Bfs
auto Graph::helpComputeScheduling_01_generatingBFS(bool filterOutAnchors)
-> Bfs
{
return helpGetBFS(filterOutAnchors, {GraphDependencyType::data,
GraphDependencyType::user});
Expand Down Expand Up @@ -610,7 +621,8 @@ auto Graph::helpComputeScheduling_02_mappingStreams(Bfs& bfs,
return mMaxNumberStreams - 1;
}

auto Graph::helpComputeScheduling_03_events(Bfs& bfs) -> int
auto Graph::helpComputeScheduling_03_events(Bfs& bfs)
-> int
{
int eventCount = 0;

Expand Down Expand Up @@ -648,7 +660,8 @@ auto Graph::helpComputeScheduling_03_events(Bfs& bfs) -> int
return eventCount - 1;
}

auto Graph::helpComputeScheduling_04_ensureResources(int maxStreamId, int maxEventId)
auto Graph::helpComputeScheduling_04_ensureResources(int maxStreamId,
int maxEventId)
-> void
{
auto bk = getBackend();
Expand Down Expand Up @@ -707,7 +720,8 @@ Graph::Graph(const Backend& bk)
helpInvalidateScheduling();
}

auto Graph::getBackend() const -> const Neon::Backend&
auto Graph::getBackend() const
-> const Neon::Backend&
{
if (mBackendIsSet) {
return mBackend;
Expand All @@ -717,12 +731,17 @@ auto Graph::getBackend() const -> const Neon::Backend&
NEON_THROW(ex);
}

auto Graph::run(Neon::SetIdx /*setIdx*/,
int /*streamIdx*/,
Neon::DataView /*dataView*/)
auto Graph::run(Neon::SetIdx setIdx,
int streamIdx,
Neon::DataView dataView)
-> void
{
NEON_THROW_UNSUPPORTED_OPERATION("");
if (dataView != Neon::DataView::STANDARD) {
NEON_THROW_UNSUPPORTED_OPERATION("");
}

this->runtimePreSet(streamIdx);
this->helpExecute(setIdx, streamIdx);
}

auto Graph::run(int streamIdx,
Expand All @@ -732,12 +751,14 @@ auto Graph::run(int streamIdx,
if (dataView != Neon::DataView::STANDARD) {
NEON_THROW_UNSUPPORTED_OPERATION("");
}

this->runtimePreSet(streamIdx);
this->helpExecute(streamIdx);
}

auto Graph::helpExecute(Neon::SetIdx setIdx,
int anchorStream) -> void
int anchorStream)
-> void
{
if (anchorStream > -1 && (anchorStream != mAnchorStreamPreSet || mFilterOutAnchorsPreSet == true)) {
Neon::NeonException ex("");
Expand Down Expand Up @@ -815,11 +836,11 @@ auto Graph::expandSubGraphs() -> void
if (atLeastOneWasFound) {
helpInvalidateScheduling();
this->helpRemoveRedundantDependencies();
// this->ioToDot(std::to_string(i) + "_t_05", "kllkj", true);
// this->ioToDot(std::to_string(i) + "_t_05", "kllkj", true);
}
return;
}
// this->ioToDot(std::to_string(i) + "_t_00", "kllkj", true);
// this->ioToDot(std::to_string(i) + "_t_00", "kllkj", true);

auto& newTarget = mRawGraph.getVertexProperty(newTargetId);
const auto& subGraph = newTarget.getContainer().getContainerInterface().getGraph();
Expand All @@ -833,7 +854,7 @@ auto Graph::expandSubGraphs() -> void
fromOldToNew.insert(std::pair<size_t, size_t>(oldNode.getGraphData().getUid(),
newNode.getGraphData().getUid()));
});
///this->ioToDot(std::to_string(i) + "_t_01", "kllkj", true);
/// this->ioToDot(std::to_string(i) + "_t_01", "kllkj", true);

// Cloning the subGraph edges into the graph
subGraph.mRawGraph.forEachEdge([&](std::pair<size_t, size_t> edge) {
Expand All @@ -846,7 +867,7 @@ auto Graph::expandSubGraphs() -> void

this->addDependency(nodeA, nodeB, oldDep.getType());
});
//this->ioToDot(std::to_string(i) + "_t_02", "kllkj", true);
// this->ioToDot(std::to_string(i) + "_t_02", "kllkj", true);

{ // Removing the cloned begin and end from the subGraph
auto oldBeginId = subGraph.getBeginNode().getGraphData().getUid();
Expand All @@ -861,25 +882,29 @@ auto Graph::expandSubGraphs() -> void
this->removeNode(toBeRemovedBeginNode);
this->removeNode(toBeRemovedEndNode);
}
// this->ioToDot(std::to_string(i) + "_t_03", "kllkj", true);
// this->ioToDot(std::to_string(i) + "_t_03", "kllkj", true);

{ // Removing subGraph and depedencies

this->removeNodeAndItsDependencies(newTarget);
std::cout << "removing " << newTarget.getContainer().getName() << std::endl;
}
//this->ioToDot(std::to_string(i) + "_t_04", "kllkj", true);
// this->ioToDot(std::to_string(i) + "_t_04", "kllkj", true);
}

}
auto Graph::helpCheckBackendStatus() -> void
auto Graph::helpCheckBackendStatus()
-> void
{
if(!mBackendIsSet){
if (!mBackendIsSet) {
NeonException exception("Container Graph");
exception << "A backend was not registered with the Graph";
NEON_THROW(exception);
}
}
auto Graph::getNumberOfNodes() -> int
{
// We remove 2 because of the head and tail holders.
return mRawGraph.numVertices() - 2;
}


} // namespace Neon::set::container
Loading

0 comments on commit 08a3a40

Please sign in to comment.