Skip to content

Commit

Permalink
Merge pull request #189 from NexGenAnalytics/140-add-tpetra-tests-part-1
Browse files Browse the repository at this point in the history 
#140: Panzer:  add Tpetra tests for directories different than adapter-stk
  • Loading branch information
@stmcgovern
stmcgovern authored Sep 21, 2023
2 parents 7bd8d99 + 34b56b5 commit b7c73c3
Show file tree
Hide file tree
Showing 18 changed files with 973 additions and 335 deletions.
6 changes: 0 additions & 6 deletions packages/panzer/adapters-ioss/test/ioss_connmngr/tIOSSConnManager.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,12 +77,6 @@
#include "Ioss_IOFactory.h"
#include "Ioss_DatabaseIO.h"

//#ifdef HAVE_MPI
// #include "Epetra_MpiComm.h"
//#else
// #include "Epetra_SerialComm.h"
//#endif

using Teuchos::RCP;
using Teuchos::rcp;

Expand Down
2 changes: 0 additions & 2 deletions packages/panzer/adapters-stk/test/evaluator_tests/gather_orientation.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,8 +68,6 @@ using Teuchos::rcp;
#include "Teuchos_DefaultMpiComm.hpp"
#include "Teuchos_OpaqueWrapper.hpp"

#include "Epetra_MpiComm.h"

#include "user_app_EquationSetFactory.hpp"

#include <cstdio> // for get char
Expand Down
6 changes: 0 additions & 6 deletions packages/panzer/adapters-stk/test/stk_interface_test/tPointLocationSearch.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,12 +53,6 @@

#include "Shards_BasicTopologies.hpp"

#ifdef HAVE_MPI
#include "Epetra_MpiComm.h"
#else
#include "Epetra_SerialComm.h"
#endif

// STK_search objects
#include <stk_search/CoarseSearch.hpp>
#include <stk_search/BoundingBox.hpp>
Expand Down
6 changes: 0 additions & 6 deletions packages/panzer/adapters-stk/test/stk_interface_test/tUniformRef.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,12 +55,6 @@

#include "Shards_BasicTopologies.hpp"

#ifdef HAVE_MPI
#include "Epetra_MpiComm.h"
#else
#include "Epetra_SerialComm.h"
#endif

/*
#include "stk_mesh/base/GetEntities.hpp"
#include "stk_mesh/base/Selector.hpp"
Expand Down
188 changes: 171 additions & 17 deletions packages/panzer/disc-fe/src/lof/Panzer_BlockedTpetraLinearObjFactory.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,7 @@ template <typename Traits,typename ScalarT,typename LocalOrdinalT,typename Globa
class BlockedTpetraLinearObjFactory : public LinearObjFactory<Traits>
, public ThyraObjFactory<double> {
public:
typedef TpetraLinearObjContainer<ScalarT,LocalOrdinalT,GlobalOrdinalT,NodeT> TLOC;
typedef BlockedTpetraLinearObjContainer<ScalarT,LocalOrdinalT,GlobalOrdinalT,NodeT> BTLOC;
typedef Tpetra::Vector<ScalarT,LocalOrdinalT,GlobalOrdinalT,NodeT> VectorType;
typedef Tpetra::CrsMatrix<ScalarT,LocalOrdinalT,GlobalOrdinalT,NodeT> CrsMatrixType;
Expand Down Expand Up @@ -166,32 +167,62 @@ class BlockedTpetraLinearObjFactory : public LinearObjFactory<Traits>
//! Use preconstructed scatter evaluators
template <typename EvalT>
Teuchos::RCP<panzer::CloneableEvaluator> buildScatter() const
{ return Teuchos::rcp(new ScatterResidual_BlockedTpetra<EvalT,Traits,LocalOrdinalT,GlobalOrdinalT,NodeT>(blockedDOFManager_)); }
{
if(gidProviders_.front()->containsBlockedDOFManager()) {
return Teuchos::rcp(new ScatterResidual_BlockedTpetra<EvalT,Traits,LocalOrdinalT,GlobalOrdinalT,NodeT>(getGlobalIndexer()));
}
return Teuchos::rcp(new ScatterResidual_Tpetra<EvalT,Traits,LocalOrdinalT,GlobalOrdinalT,NodeT>(getRangeGlobalIndexer()));
}

//! Use preconstructed gather evaluators
template <typename EvalT>
Teuchos::RCP<panzer::CloneableEvaluator > buildGather() const
{ return Teuchos::rcp(new GatherSolution_BlockedTpetra<EvalT,Traits,ScalarT,LocalOrdinalT,GlobalOrdinalT,NodeT>(blockedDOFManager_)); }
{
if(gidProviders_.front()->containsBlockedDOFManager()) {
return Teuchos::rcp(new GatherSolution_BlockedTpetra<EvalT,Traits,ScalarT,LocalOrdinalT,GlobalOrdinalT,NodeT>(getGlobalIndexer()));
}
return Teuchos::rcp(new GatherSolution_Tpetra<EvalT,Traits,LocalOrdinalT,GlobalOrdinalT,NodeT>(getRangeGlobalIndexer()));
}

//! Use preconstructed gather evaluators
template <typename EvalT>
Teuchos::RCP<panzer::CloneableEvaluator > buildGatherTangent() const
{ return Teuchos::rcp(new GatherTangent_BlockedTpetra<EvalT,Traits,ScalarT,LocalOrdinalT,GlobalOrdinalT,NodeT>(blockedDOFManager_)); }
{
if(gidProviders_.front()->containsBlockedDOFManager()) {
return Teuchos::rcp(new GatherTangent_BlockedTpetra<EvalT,Traits,ScalarT,LocalOrdinalT,GlobalOrdinalT,NodeT>(getGlobalIndexer()));
}
return Teuchos::rcp(new GatherTangent_Tpetra<EvalT,Traits,LocalOrdinalT,GlobalOrdinalT,NodeT>(getRangeGlobalIndexer()));
}

//! Use preconstructed gather evaluators
template <typename EvalT>
Teuchos::RCP<panzer::CloneableEvaluator > buildGatherDomain() const
{ return Teuchos::rcp(new GatherSolution_BlockedTpetra<EvalT,Traits,ScalarT,LocalOrdinalT,GlobalOrdinalT,NodeT>(blockedDOFManager_)); }
{
if(gidProviders_.back()->containsBlockedDOFManager()) {
return Teuchos::rcp(new GatherSolution_BlockedTpetra<EvalT,Traits,ScalarT,LocalOrdinalT,GlobalOrdinalT,NodeT>(getDomainBlockedIndexer()));
}
return Teuchos::rcp(new GatherSolution_Tpetra<EvalT,Traits,LocalOrdinalT,GlobalOrdinalT,NodeT>(getDomainGlobalIndexer()));
}

//! Use preconstructed gather evaluators
template <typename EvalT>
Teuchos::RCP<panzer::CloneableEvaluator > buildGatherOrientation() const
{ return Teuchos::rcp(new GatherOrientation<EvalT,Traits,LocalOrdinalT,GlobalOrdinalT>(nc2c_vector(blockedDOFManager_->getFieldDOFManagers()))); }
{
if(gidProviders_.front()->containsBlockedDOFManager()) {
return Teuchos::rcp(new GatherOrientation<EvalT,Traits,LocalOrdinalT,GlobalOrdinalT>(nc2c_vector(getGlobalIndexer()->getFieldDOFManagers())));
}
return Teuchos::rcp(new GatherOrientation<EvalT,Traits,LocalOrdinalT,GlobalOrdinalT>(getRangeGlobalIndexer()));
}

//! Use preconstructed dirichlet scatter evaluators
template <typename EvalT>
Teuchos::RCP<panzer::CloneableEvaluator> buildScatterDirichlet() const
{ return Teuchos::rcp(new ScatterDirichletResidual_BlockedTpetra<EvalT,Traits,LocalOrdinalT,GlobalOrdinalT,NodeT>(blockedDOFManager_)); }
{
if(gidProviders_.front()->containsBlockedDOFManager()) {
return Teuchos::rcp(new ScatterDirichletResidual_BlockedTpetra<EvalT,Traits,LocalOrdinalT,GlobalOrdinalT,NodeT>(getGlobalIndexer()));
}
return Teuchos::rcp(new ScatterDirichletResidual_Tpetra<EvalT,Traits,LocalOrdinalT,GlobalOrdinalT,NodeT>(getRangeGlobalIndexer()));
}

/*************** Generic helper functions for container setup *******************/

Expand Down Expand Up @@ -267,9 +298,11 @@ class BlockedTpetraLinearObjFactory : public LinearObjFactory<Traits>

//! get the map from the matrix
virtual Teuchos::RCP<const MapType> getMap(int i) const;
virtual Teuchos::RCP<const MapType> getColMap(int i) const;

//! get the ghosted map from the matrix
virtual Teuchos::RCP<const MapType> getGhostedMap(int i) const;
virtual Teuchos::RCP<const MapType> getColGhostedMap(int i) const;

//! get the graph of the crs matrix
virtual Teuchos::RCP<const CrsGraphType> getGraph(int i,int j) const;
Expand All @@ -279,9 +312,11 @@ class BlockedTpetraLinearObjFactory : public LinearObjFactory<Traits>

//! get importer for converting an overalapped object to a "normal" object
virtual Teuchos::RCP<const ImportType> getGhostedImport(int i) const;
virtual Teuchos::RCP<const ImportType> getColGhostedImport(int i) const;

//! get exporter for converting an overalapped object to a "normal" object
virtual Teuchos::RCP<const ExportType> getGhostedExport(int j) const;
virtual Teuchos::RCP<const ExportType> getColGhostedExport(int j) const;

Teuchos::RCP<CrsMatrixType> getTpetraMatrix(int i,int j) const;
Teuchos::RCP<CrsMatrixType> getGhostedTpetraMatrix(int i,int j) const;
Expand All @@ -308,39 +343,145 @@ class BlockedTpetraLinearObjFactory : public LinearObjFactory<Traits>
virtual void endFill(LinearObjContainer & loc) const;

Teuchos::RCP<const panzer::BlockedDOFManager> getGlobalIndexer() const
{ return blockedDOFManager_; }
{
TEUCHOS_ASSERT(!gidProviders_.empty());
return gidProviders_.front()->getBlockedIndexer();
}

//! Get the domain unique global indexer this factory was created with.
Teuchos::RCP<const panzer::GlobalIndexer> getDomainGlobalIndexer() const
{ return blockProvider_; }
{
TEUCHOS_ASSERT(!gidProviders_.empty());
return gidProviders_.back()->getGlobalIndexer();
}

//! Get the domain unique global indexer this factory was created with.
Teuchos::RCP<const panzer::BlockedDOFManager> getDomainBlockedIndexer() const
{
TEUCHOS_ASSERT(!gidProviders_.empty());
return gidProviders_.back()->getBlockedIndexer();
}

//! Get the range unique global indexer this factory was created with.
Teuchos::RCP<const panzer::GlobalIndexer> getRangeGlobalIndexer() const
{ return blockProvider_; }
{
TEUCHOS_ASSERT(!gidProviders_.empty());
return gidProviders_.front()->getGlobalIndexer();
}

protected:
/*************** Utility class for handling blocked and nonblocked DOF managers *******************/

/** This classes is mean to abstract away the different global indexer types and hide
* if this is a blocked data structure or a unblocked one.
*/
class DOFManagerContainer
{
public:
DOFManagerContainer() {}
DOFManagerContainer(const Teuchos::RCP<const GlobalIndexer> &ugi)
{
setGlobalIndexer(ugi);
}

void setGlobalIndexer(const Teuchos::RCP<const GlobalIndexer> &ugi)
{
using Teuchos::RCP;
using Teuchos::rcp_dynamic_cast;

auto blockedDOFManager = rcp_dynamic_cast<const BlockedDOFManager>(ugi);
auto flatDOFManager = rcp_dynamic_cast<const GlobalIndexer>(ugi);

if (blockedDOFManager != Teuchos::null)
{
// set BlockedDOFManager
blockedDOFManager_ = blockedDOFManager;

// get all GID providers
auto dofManagers = blockedDOFManager_->getFieldDOFManagers();
for (auto itr = dofManagers.begin(); itr != dofManagers.end(); ++itr)
gidProviders_.push_back(*itr);
}
else if (flatDOFManager != Teuchos::null)
{
// for absolute clarity, nullify the blockedDOFManager_
blockedDOFManager_ = Teuchos::null;

// you have only a single GID provider
gidProviders_.push_back(flatDOFManager);
}
else
{
TEUCHOS_ASSERT(false);
}
}

//! Get the number of global indexers (not including the blocked one) contained.
int getFieldBlocks() const
{
return Teuchos::as<int>(gidProviders_.size());
}

/** Return true if this contains a blocked DOFManager as opposed to only a single DOFManager.
* If this returns true then <code>getGlobalIndexer</code> will return a <code>BlockedDOFManager<int,GO></code>,
* other wise it will return a <code>GlobalIndexer<int,GO></code>.
*/
bool containsBlockedDOFManager() const
{
return blockedDOFManager_ != Teuchos::null;
}

//! Get the "parent" global indexer (if <code>containsBlockedDOFManager()==false</code> this will throw)
Teuchos::RCP<const BlockedDOFManager> getBlockedIndexer() const
{
TEUCHOS_ASSERT(containsBlockedDOFManager());
return blockedDOFManager_;
}

//! Get the "parent" global indexer (if <code>getFieldBlocks()>1</code> this will be blocked, otherwise it may be either)
Teuchos::RCP<const GlobalIndexer> getGlobalIndexer() const
{
if (blockedDOFManager_ != Teuchos::null)
return blockedDOFManager_;

TEUCHOS_ASSERT(gidProviders_.size() == 1);
return gidProviders_[0];
}

//! Get DOFManagers associated with the blocks
const std::vector<Teuchos::RCP<const GlobalIndexer>> &getFieldDOFManagers() const
{
return gidProviders_;
}

private:
Teuchos::RCP<const BlockedDOFManager> blockedDOFManager_;
std::vector<Teuchos::RCP<const GlobalIndexer>> gidProviders_;
};

/*************** Generic methods/members *******************/

// Get the global indexer associated with a particular block
Teuchos::RCP<const GlobalIndexer> getGlobalIndexer(int i) const;
Teuchos::RCP<const GlobalIndexer> getColGlobalIndexer(int i) const;

//! Allocate the space in the std::vector objects so we can fill with appropriate Tpetra data
void makeRoomForBlocks(std::size_t blockCnt);
void makeRoomForBlocks(std::size_t blockCnt, std::size_t colBlockCnt = 0);

Teuchos::RCP<const GlobalIndexer> blockProvider_;
Teuchos::RCP<const BlockedDOFManager> blockedDOFManager_;
std::vector<Teuchos::RCP<const GlobalIndexer> > gidProviders_;
std::vector<Teuchos::RCP<const DOFManagerContainer> > gidProviders_;

bool useColGidProviders_;

// which block entries are ignored
std::unordered_set<std::pair<int,int>,panzer::pair_hash> excludedPairs_;

/*************** Thyra based methods/members *******************/

void ghostToGlobalThyraVector(const Teuchos::RCP<const Thyra::VectorBase<ScalarT> > & in,
const Teuchos::RCP<Thyra::VectorBase<ScalarT> > & out) const;
const Teuchos::RCP<Thyra::VectorBase<ScalarT> > & out, bool col) const;
void ghostToGlobalThyraMatrix(const Thyra::LinearOpBase<ScalarT> & in,Thyra::LinearOpBase<ScalarT> & out) const;
void globalToGhostThyraVector(const Teuchos::RCP<const Thyra::VectorBase<ScalarT> > & in,
const Teuchos::RCP<Thyra::VectorBase<ScalarT> > & out) const;
const Teuchos::RCP<Thyra::VectorBase<ScalarT> > & out, bool col) const;

mutable Teuchos::RCP<Thyra::ProductVectorSpaceBase<ScalarT> > rangeSpace_;
mutable Teuchos::RCP<Thyra::ProductVectorSpaceBase<ScalarT> > domainSpace_;
Expand All @@ -356,13 +497,16 @@ class BlockedTpetraLinearObjFactory : public LinearObjFactory<Traits>
const Teuchos::Ptr<CrsMatrixType> & A,
bool zeroVectorRows) const;

void ghostToGlobalTpetraVector(int i,const VectorType & in,VectorType & out) const;
void ghostToGlobalTpetraVector(int i,const VectorType & in,VectorType & out, bool col) const;
void ghostToGlobalTpetraMatrix(int blockRow,const CrsMatrixType & in,CrsMatrixType & out) const;
void globalToGhostTpetraVector(int i,const VectorType & in,VectorType & out) const;
void globalToGhostTpetraVector(int i,const VectorType & in,VectorType & out, bool col) const;

// get the map from the matrix
virtual Teuchos::RCP<const MapType> buildTpetraMap(int i) const;
virtual Teuchos::RCP<const MapType> buildColTpetraMap(int i) const;

virtual Teuchos::RCP<const MapType> buildTpetraGhostedMap(int i) const;
virtual Teuchos::RCP<const MapType> buildColTpetraGhostedMap(int i) const;

// get the graph of the crs matrix
virtual Teuchos::RCP<const CrsGraphType> buildTpetraGraph(int i,int j) const;
Expand All @@ -371,12 +515,22 @@ class BlockedTpetraLinearObjFactory : public LinearObjFactory<Traits>
// storage for Tpetra graphs and maps
Teuchos::RCP<const Teuchos::MpiComm<int> > comm_;
mutable std::vector<Teuchos::RCP<const MapType> > maps_;
mutable std::vector<Teuchos::RCP<const MapType> > colMaps_;

mutable std::vector<Teuchos::RCP<const MapType> > ghostedMaps_;
mutable std::vector<Teuchos::RCP<const MapType> > colGhostedMaps_;

mutable std::unordered_map<std::pair<int,int>,Teuchos::RCP<const CrsGraphType>,panzer::pair_hash> graphs_ ;
mutable std::unordered_map<std::pair<int,int>,Teuchos::RCP<const CrsGraphType>,panzer::pair_hash> colGraphs_ ;

mutable std::unordered_map<std::pair<int,int>,Teuchos::RCP<const CrsGraphType>,panzer::pair_hash> ghostedGraphs_;
mutable std::unordered_map<std::pair<int,int>,Teuchos::RCP<const CrsGraphType>,panzer::pair_hash> colGhostedGraphs_;

mutable std::vector<Teuchos::RCP<const ImportType> > importers_;
mutable std::vector<Teuchos::RCP<const ImportType> > colImporters_;

mutable std::vector<Teuchos::RCP<const ExportType> > exporters_;
mutable std::vector<Teuchos::RCP<const ExportType> > colExporters_;
};

}
Expand Down
Loading

0 comments on commit b7c73c3

Please sign in to comment.