From e5973c8aed385c20997249e7bb71e9381ad48f42 Mon Sep 17 00:00:00 2001 From: pelesh Date: Sat, 12 Oct 2024 01:58:16 -0400 Subject: [PATCH] Rename `copyData` -> `syncData` and ensure correct flags set when syncing (#199) * Wholesale name change copyData -> syncData * Set checks in Vector::syncData function * Only one argument is sufficient in Vector::syncData * Document matrix::*::syncData methods * Use explicit host-device syncing (#200) * Matrix getters don't sync. * Use explicit host-device syncing in tests. * Update examples --- examples/r_KLU_GLU.cpp | 13 +- examples/r_KLU_GLU_matrix_values_update.cpp | 99 ++++++++-------- .../r_KLU_cusolverrf_redo_factorization.cpp | 15 ++- examples/r_KLU_rf.cpp | 15 ++- examples/r_KLU_rf_FGMRES.cpp | 15 ++- .../r_KLU_rf_FGMRES_reuse_factorization.cpp | 15 ++- examples/r_KLU_rocSolverRf_FGMRES.cpp | 26 ++-- examples/r_KLU_rocsolverrf.cpp | 41 ++++--- .../r_KLU_rocsolverrf_redo_factorization.cpp | 44 ++++--- examples/r_SysSolverCuda.cpp | 2 +- examples/r_SysSolverHip.cpp | 8 +- examples/r_SysSolverHipRefine.cpp | 9 +- examples/r_randGMRES.cpp | 11 +- examples/r_randGMRES_CUDA.cpp | 2 +- resolve/GramSchmidt.cpp | 4 +- resolve/LinSolverDirectRocSolverRf.cpp | 2 +- resolve/SystemSolver.cpp | 4 +- resolve/matrix/Coo.cpp | 111 +++++++++++------- resolve/matrix/Coo.hpp | 2 +- resolve/matrix/Csc.cpp | 105 ++++++++++------- resolve/matrix/Csc.hpp | 2 +- resolve/matrix/Csr.cpp | 108 ++++++++++------- resolve/matrix/Csr.hpp | 2 +- resolve/matrix/Sparse.hpp | 2 +- resolve/vector/Vector.cpp | 62 ++++++---- resolve/vector/Vector.hpp | 2 +- resolve/vector/VectorHandler.cpp | 5 + tests/functionality/testKLU_GLU.cpp | 2 + tests/functionality/testKLU_Rf.cpp | 4 + tests/functionality/testKLU_Rf_FGMRES.cpp | 4 + tests/functionality/testKLU_RocSolver.cpp | 2 + .../testKLU_RocSolver_FGMRES.cpp | 2 + tests/functionality/testRandGMRES_Cuda.cpp | 5 +- tests/functionality/testRandGMRES_Rocm.cpp | 5 +- tests/functionality/testSysGLU.cpp | 2 + tests/functionality/testSysRandGMRES.cpp | 5 +- tests/functionality/testSysRefactor.cpp | 2 + .../unit/matrix/MatrixFactorizationTests.hpp | 6 +- tests/unit/matrix/MatrixHandlerTests.hpp | 4 +- tests/unit/vector/GramSchmidtTests.hpp | 2 +- tests/unit/vector/VectorHandlerTests.hpp | 2 +- 41 files changed, 466 insertions(+), 307 deletions(-) diff --git a/examples/r_KLU_GLU.cpp b/examples/r_KLU_GLU.cpp index 824bf7e3..92d7933c 100644 --- a/examples/r_KLU_GLU.cpp +++ b/examples/r_KLU_GLU.cpp @@ -96,19 +96,24 @@ int main(int argc, char *argv[]) ReSolve::io::updateMatrixFromFile(mat_file, A); ReSolve::io::updateArrayFromFile(rhs_file, &rhs); } - std::cout<<"Finished reading the matrix and rhs, size: "<getNumRows()<<" x "<getNumColumns()<< ", nnz: "<< A->getNnz()<< ", symmetric? "<symmetric()<< ", Expanded? "<expanded()<syncData(ReSolve::memory::DEVICE); + + std::cout << "Finished reading the matrix and rhs, size: " << A->getNumRows() << " x "<< A->getNumColumns() + << ", nnz: " << A->getNnz() + << ", symmetric? " << A->symmetric() + << ", Expanded? " << A->expanded() << std::endl; mat_file.close(); rhs_file.close(); // Update host and device data. if (i < 1) { vec_rhs->update(rhs, ReSolve::memory::HOST, ReSolve::memory::HOST); - vec_rhs->setDataUpdated(ReSolve::memory::HOST); } else { - A->copyData(ReSolve::memory::DEVICE); vec_rhs->update(rhs, ReSolve::memory::HOST, ReSolve::memory::DEVICE); } - std::cout<<"COO to CSR completed. Expanded NNZ: "<< A->getNnz()<getNnz() << std::endl; + //Now call direct solver int status; if (i < 1) { diff --git a/examples/r_KLU_GLU_matrix_values_update.cpp b/examples/r_KLU_GLU_matrix_values_update.cpp index 2717f4ff..95efb91c 100644 --- a/examples/r_KLU_GLU_matrix_values_update.cpp +++ b/examples/r_KLU_GLU_matrix_values_update.cpp @@ -105,57 +105,60 @@ int main(int argc, char *argv[]) //ReSolve::io::updateMatrixFromFile(mat_file, A); ReSolve::io::updateArrayFromFile(rhs_file, &rhs); } - std::cout<<"Finished reading the matrix and rhs, size: "<getNumRows()<<" x "<getNumColumns()<< ", nnz: "<< A->getNnz()<< ", symmetric? "<symmetric()<< ", Expanded? "<expanded()<update(rhs, ReSolve::memory::HOST, ReSolve::memory::HOST); - vec_rhs->setDataUpdated(ReSolve::memory::HOST); - } else { - A->copyData(ReSolve::memory::DEVICE); - vec_rhs->update(rhs, ReSolve::memory::HOST, ReSolve::memory::DEVICE); - } - std::cout<<"COO to CSR completed. Expanded NNZ: "<< A->getNnz()<setup(A); - status = KLU->analyze(); - std::cout<<"KLU analysis status: "<factorize(); - std::cout<<"KLU factorization status: "<getLFactor(); - matrix_type* U = KLU->getUFactor(); - if (L == nullptr) {printf("ERROR");} - index_type* P = KLU->getPOrdering(); - index_type* Q = KLU->getQOrdering(); - GLU->setup(A, L, U, P, Q); - status = GLU->solve(vec_rhs, vec_x); - std::cout<<"GLU solve status: "<solve(vec_rhs, vec_x); - // std::cout<<"KLU solve status: "<refactorize(); - std::cout<<"Using CUSOLVER GLU"<refactorize(); - std::cout<<"CUSOLVER GLU refactorization status: "<solve(vec_rhs, vec_x); - std::cout<<"CUSOLVER GLU solve status: "<update(rhs, ReSolve::memory::HOST, ReSolve::memory::DEVICE); - - - matrix_handler->setValuesChanged(true, ReSolve::memory::DEVICE); - matrix_handler->matvec(A, vec_x, vec_r, &ONE, &MINUSONE, ReSolve::memory::DEVICE); + // Copy matrix data to device + A->syncData(ReSolve::memory::DEVICE); + + std::cout << "Finished reading the matrix and rhs, size: " << A->getNumRows() << " x "<< A->getNumColumns() + << ", nnz: " << A->getNnz() + << ", symmetric? " << A->symmetric() + << ", Expanded? " << A->expanded() << std::endl; + mat_file.close(); + rhs_file.close(); + + // Update host and device data. + if (i < 1) { + vec_rhs->update(rhs, ReSolve::memory::HOST, ReSolve::memory::HOST); + } else { + vec_rhs->update(rhs, ReSolve::memory::HOST, ReSolve::memory::DEVICE); + } + std::cout << "CSR matrix loaded. Expanded NNZ: " << A->getNnz() << std::endl; + + //Now call direct solver + int status; + if (i < 1){ + KLU->setup(A); + status = KLU->analyze(); + std::cout<<"KLU analysis status: "<factorize(); + std::cout<<"KLU factorization status: "<getLFactor(); + matrix_type* U = KLU->getUFactor(); + if (L == nullptr) {printf("ERROR");} + index_type* P = KLU->getPOrdering(); + index_type* Q = KLU->getQOrdering(); + GLU->setup(A, L, U, P, Q); + status = GLU->solve(vec_rhs, vec_x); + std::cout<<"GLU solve status: "<solve(vec_rhs, vec_x); + // std::cout<<"KLU solve status: "<refactorize(); + std::cout<<"Using CUSOLVER GLU"<refactorize(); + std::cout<<"CUSOLVER GLU refactorization status: "<solve(vec_rhs, vec_x); + std::cout<<"CUSOLVER GLU solve status: "<update(rhs, ReSolve::memory::HOST, ReSolve::memory::DEVICE); - std::cout << "\t 2-Norm of the residual: " - << std::scientific << std::setprecision(16) - << sqrt(vector_handler->dot(vec_r, vec_r, ReSolve::memory::DEVICE)) << "\n"; + matrix_handler->setValuesChanged(true, ReSolve::memory::DEVICE); + matrix_handler->matvec(A, vec_x, vec_r, &ONE, &MINUSONE, ReSolve::memory::DEVICE); - } + std::cout << "\t 2-Norm of the residual: " + << std::scientific << std::setprecision(16) + << sqrt(vector_handler->dot(vec_r, vec_r, ReSolve::memory::DEVICE)) << "\n"; + } //now DELETE delete A; diff --git a/examples/r_KLU_cusolverrf_redo_factorization.cpp b/examples/r_KLU_cusolverrf_redo_factorization.cpp index db4d611f..1cb7f681 100644 --- a/examples/r_KLU_cusolverrf_redo_factorization.cpp +++ b/examples/r_KLU_cusolverrf_redo_factorization.cpp @@ -105,19 +105,24 @@ int main(int argc, char *argv[] ) ReSolve::io::updateMatrixFromFile(mat_file, A); ReSolve::io::updateArrayFromFile(rhs_file, &rhs); } - std::cout<<"Finished reading the matrix and rhs, size: "<getNumRows()<<" x "<getNumColumns()<< ", nnz: "<< A->getNnz()<< ", symmetric? "<symmetric()<< ", Expanded? "<expanded()<syncData(ReSolve::memory::DEVICE); + + std::cout << "Finished reading the matrix and rhs, size: " << A->getNumRows() << " x "<< A->getNumColumns() + << ", nnz: " << A->getNnz() + << ", symmetric? " << A->symmetric() + << ", Expanded? " << A->expanded() << std::endl; mat_file.close(); rhs_file.close(); // Update host and device data. if (i < 2) { vec_rhs->update(rhs, ReSolve::memory::HOST, ReSolve::memory::HOST); - vec_rhs->setDataUpdated(ReSolve::memory::HOST); } else { - A->copyData(ReSolve::memory::DEVICE); vec_rhs->update(rhs, ReSolve::memory::HOST, ReSolve::memory::DEVICE); } - std::cout<<"COO to CSR completed. Expanded NNZ: "<< A->getNnz()<getNnz() << std::endl; + //Now call direct solver if (i < 2) { KLU->setup(A); @@ -132,6 +137,8 @@ int main(int argc, char *argv[] ) U_csc = (ReSolve::matrix::Csc*) KLU->getUFactor(); ReSolve::matrix::Csr* L = new ReSolve::matrix::Csr(L_csc->getNumRows(), L_csc->getNumColumns(), L_csc->getNnz()); ReSolve::matrix::Csr* U = new ReSolve::matrix::Csr(U_csc->getNumRows(), U_csc->getNumColumns(), U_csc->getNnz()); + L_csc->syncData(ReSolve::memory::DEVICE); + U_csc->syncData(ReSolve::memory::DEVICE); matrix_handler->csc2csr(L_csc,L, ReSolve::memory::DEVICE); matrix_handler->csc2csr(U_csc,U, ReSolve::memory::DEVICE); if (L == nullptr) { diff --git a/examples/r_KLU_rf.cpp b/examples/r_KLU_rf.cpp index 40c39446..fa4c9b2e 100644 --- a/examples/r_KLU_rf.cpp +++ b/examples/r_KLU_rf.cpp @@ -95,19 +95,24 @@ int main(int argc, char *argv[] ) ReSolve::io::updateMatrixFromFile(mat_file, A); ReSolve::io::updateArrayFromFile(rhs_file, &rhs); } - std::cout<<"Finished reading the matrix and rhs, size: "<getNumRows()<<" x "<getNumColumns()<< ", nnz: "<< A->getNnz()<< ", symmetric? "<symmetric()<< ", Expanded? "<expanded()<syncData(ReSolve::memory::DEVICE); + + std::cout << "Finished reading the matrix and rhs, size: " << A->getNumRows() << " x "<< A->getNumColumns() + << ", nnz: " << A->getNnz() + << ", symmetric? " << A->symmetric() + << ", Expanded? " << A->expanded() << std::endl; mat_file.close(); rhs_file.close(); // Update host and device data. if (i < 2) { vec_rhs->update(rhs, ReSolve::memory::HOST, ReSolve::memory::HOST); - vec_rhs->setDataUpdated(ReSolve::memory::HOST); } else { - A->copyData(ReSolve::memory::DEVICE); vec_rhs->update(rhs, ReSolve::memory::HOST, ReSolve::memory::DEVICE); } - std::cout<<"COO to CSR completed. Expanded NNZ: "<< A->getNnz()<getNnz() << std::endl; + //Now call direct solver int status; if (i < 2){ @@ -123,6 +128,8 @@ int main(int argc, char *argv[] ) ReSolve::matrix::Csc* U_csc = (ReSolve::matrix::Csc*) KLU->getUFactor(); ReSolve::matrix::Csr* L = new ReSolve::matrix::Csr(L_csc->getNumRows(), L_csc->getNumColumns(), L_csc->getNnz()); ReSolve::matrix::Csr* U = new ReSolve::matrix::Csr(U_csc->getNumRows(), U_csc->getNumColumns(), U_csc->getNnz()); + L_csc->syncData(ReSolve::memory::DEVICE); + U_csc->syncData(ReSolve::memory::DEVICE); matrix_handler->csc2csr(L_csc,L, ReSolve::memory::DEVICE); matrix_handler->csc2csr(U_csc,U, ReSolve::memory::DEVICE); if (L == nullptr) {printf("ERROR");} diff --git a/examples/r_KLU_rf_FGMRES.cpp b/examples/r_KLU_rf_FGMRES.cpp index b2cb9737..1a8f3b88 100644 --- a/examples/r_KLU_rf_FGMRES.cpp +++ b/examples/r_KLU_rf_FGMRES.cpp @@ -99,19 +99,24 @@ int main(int argc, char *argv[]) ReSolve::io::updateMatrixFromFile(mat_file, A); ReSolve::io::updateArrayFromFile(rhs_file, &rhs); } - std::cout<<"Finished reading the matrix and rhs, size: "<getNumRows()<<" x "<getNumColumns()<< ", nnz: "<< A->getNnz()<< ", symmetric? "<symmetric()<< ", Expanded? "<expanded()<syncData(ReSolve::memory::DEVICE); + + std::cout << "Finished reading the matrix and rhs, size: " << A->getNumRows() << " x "<< A->getNumColumns() + << ", nnz: " << A->getNnz() + << ", symmetric? " << A->symmetric() + << ", Expanded? " << A->expanded() << std::endl; mat_file.close(); rhs_file.close(); // Update host and device data. if (i < 2) { vec_rhs->update(rhs, ReSolve::memory::HOST, ReSolve::memory::HOST); - vec_rhs->setDataUpdated(ReSolve::memory::HOST); } else { - A->copyData(ReSolve::memory::DEVICE); vec_rhs->update(rhs, ReSolve::memory::HOST, ReSolve::memory::DEVICE); } - std::cout<<"COO to CSR completed. Expanded NNZ: "<< A->getNnz()<getNnz() << std::endl; + //Now call direct solver int status; real_type norm_b; @@ -146,6 +151,8 @@ int main(int argc, char *argv[]) ReSolve::matrix::Csc* U_csc = (ReSolve::matrix::Csc*) KLU->getUFactor(); ReSolve::matrix::Csr* L = new ReSolve::matrix::Csr(L_csc->getNumRows(), L_csc->getNumColumns(), L_csc->getNnz()); ReSolve::matrix::Csr* U = new ReSolve::matrix::Csr(U_csc->getNumRows(), U_csc->getNumColumns(), U_csc->getNnz()); + L_csc->syncData(ReSolve::memory::DEVICE); + U_csc->syncData(ReSolve::memory::DEVICE); matrix_handler->csc2csr(L_csc,L, ReSolve::memory::DEVICE); matrix_handler->csc2csr(U_csc,U, ReSolve::memory::DEVICE); if (L == nullptr) { diff --git a/examples/r_KLU_rf_FGMRES_reuse_factorization.cpp b/examples/r_KLU_rf_FGMRES_reuse_factorization.cpp index dee8331b..8d070380 100644 --- a/examples/r_KLU_rf_FGMRES_reuse_factorization.cpp +++ b/examples/r_KLU_rf_FGMRES_reuse_factorization.cpp @@ -101,7 +101,13 @@ int main(int argc, char *argv[]) ReSolve::io::updateMatrixFromFile(mat_file, A); ReSolve::io::updateArrayFromFile(rhs_file, &rhs); } - std::cout<<"Finished reading the matrix and rhs, size: "<getNumRows()<<" x "<getNumColumns()<< ", nnz: "<< A->getNnz()<< ", symmetric? "<symmetric()<< ", Expanded? "<expanded()<syncData(ReSolve::memory::DEVICE); + + std::cout << "Finished reading the matrix and rhs, size: " << A->getNumRows() << " x "<< A->getNumColumns() + << ", nnz: " << A->getNnz() + << ", symmetric? " << A->symmetric() + << ", Expanded? " << A->expanded() << std::endl; mat_file.close(); rhs_file.close(); @@ -110,10 +116,11 @@ int main(int argc, char *argv[]) vec_rhs->update(rhs, ReSolve::memory::HOST, ReSolve::memory::HOST); vec_rhs->setDataUpdated(ReSolve::memory::HOST); } else { - A->copyData(ReSolve::memory::DEVICE); + A->syncData(ReSolve::memory::DEVICE); vec_rhs->update(rhs, ReSolve::memory::HOST, ReSolve::memory::DEVICE); } - std::cout<<"COO to CSR completed. Expanded NNZ: "<< A->getNnz()<getNnz() << std::endl; + //Now call direct solver int status; real_type norm_b; @@ -139,6 +146,8 @@ int main(int argc, char *argv[]) ReSolve::matrix::Csc* U_csc = (ReSolve::matrix::Csc*) KLU->getUFactor(); ReSolve::matrix::Csr* L = new ReSolve::matrix::Csr(L_csc->getNumRows(), L_csc->getNumColumns(), L_csc->getNnz()); ReSolve::matrix::Csr* U = new ReSolve::matrix::Csr(U_csc->getNumRows(), U_csc->getNumColumns(), U_csc->getNnz()); + L_csc->syncData(ReSolve::memory::DEVICE); + U_csc->syncData(ReSolve::memory::DEVICE); matrix_handler->csc2csr(L_csc,L, ReSolve::memory::DEVICE); matrix_handler->csc2csr(U_csc,U, ReSolve::memory::DEVICE); diff --git a/examples/r_KLU_rocSolverRf_FGMRES.cpp b/examples/r_KLU_rocSolverRf_FGMRES.cpp index 646d04aa..786a18c5 100644 --- a/examples/r_KLU_rocSolverRf_FGMRES.cpp +++ b/examples/r_KLU_rocSolverRf_FGMRES.cpp @@ -104,25 +104,25 @@ int main(int argc, char *argv[]) ReSolve::io::updateMatrixFromFile(mat_file, A); ReSolve::io::updateArrayFromFile(rhs_file, &rhs); } + // Copy matrix data to device + A->syncData(ReSolve::memory::DEVICE); + std::cout << "Finished reading the matrix and rhs, size: " << A->getNumRows() << " x "<< A->getNumColumns() << ", nnz: " << A->getNnz() << ", symmetric? " << A->symmetric() << ", Expanded? " << A->expanded() << std::endl; mat_file.close(); rhs_file.close(); - RESOLVE_RANGE_POP("Matrix Read"); // Update host and device data. - RESOLVE_RANGE_PUSH("Convert to CSR"); if (i < 2) { vec_rhs->update(rhs, ReSolve::memory::HOST, ReSolve::memory::HOST); - vec_rhs->setDataUpdated(ReSolve::memory::HOST); } else { - A->copyData(ReSolve::memory::DEVICE); vec_rhs->update(rhs, ReSolve::memory::HOST, ReSolve::memory::DEVICE); } - RESOLVE_RANGE_POP("Convert to CSR"); - std::cout<<"COO to CSR completed. Expanded NNZ: "<< A->getNnz()<getNnz() << std::endl; + int status; real_type norm_b; if (i < 2) { @@ -130,12 +130,12 @@ int main(int argc, char *argv[]) KLU->setup(A); matrix_handler->setValuesChanged(true, ReSolve::memory::DEVICE); status = KLU->analyze(); - std::cout<<"KLU analysis status: "<factorize(); - std::cout<<"KLU factorization status: "<solve(vec_rhs, vec_x); - std::cout<<"KLU solve status: "<update(rhs, ReSolve::memory::HOST, ReSolve::memory::DEVICE); norm_b = vector_handler->dot(vec_r, vec_r, ReSolve::memory::DEVICE); norm_b = sqrt(norm_b); @@ -155,18 +155,18 @@ int main(int argc, char *argv[]) Rf->setSolveMode(1); Rf->setup(A, L, U, P, Q, vec_rhs); Rf->refactorize(); - std::cout<<"about to set FGMRES" <setup(A); } RESOLVE_RANGE_POP("KLU"); } else { RESOLVE_RANGE_PUSH("RocSolver"); //status = KLU->refactorize(); - std::cout<<"Using ROCSOLVER RF"<refactorize(); - std::cout<<"ROCSOLVER RF refactorization status: "<solve(vec_rhs, vec_x); - std::cout<<"ROCSOLVER RF solve status: "<update(rhs, ReSolve::memory::HOST, ReSolve::memory::DEVICE); norm_b = vector_handler->dot(vec_r, vec_r, ReSolve::memory::DEVICE); norm_b = sqrt(norm_b); diff --git a/examples/r_KLU_rocsolverrf.cpp b/examples/r_KLU_rocsolverrf.cpp index a4d9f44d..d5ce923d 100644 --- a/examples/r_KLU_rocsolverrf.cpp +++ b/examples/r_KLU_rocsolverrf.cpp @@ -15,7 +15,7 @@ using namespace ReSolve::constants; -int main(int argc, char *argv[] ) +int main(int argc, char *argv[]) { // Use the same data types as those you specified in ReSolve build. using index_type = ReSolve::index_type; @@ -27,8 +27,8 @@ int main(int argc, char *argv[] ) std::string rhsFileName = argv[2]; index_type numSystems = atoi(argv[3]); - std::cout<<"Family mtx file name: "<< matrixFileName << ", total number of matrices: "<getNumRows()<<" x "<getNumColumns()<< ", nnz: "<< A->getNnz()<< ", symmetric? "<symmetric()<< ", Expanded? "<expanded()<syncData(ReSolve::memory::DEVICE); + + std::cout << "Finished reading the matrix and rhs, size: " << A->getNumRows() << " x "<< A->getNumColumns() + << ", nnz: " << A->getNnz() + << ", symmetric? " << A->symmetric() + << ", Expanded? " << A->expanded() << std::endl; mat_file.close(); rhs_file.close(); - // Update host and device data. + // Update right-hand-side vector. if (i < 2) { vec_rhs->update(rhs, ReSolve::memory::HOST, ReSolve::memory::HOST); - vec_rhs->setDataUpdated(ReSolve::memory::HOST); } else { - A->copyData(ReSolve::memory::DEVICE); vec_rhs->update(rhs, ReSolve::memory::HOST, ReSolve::memory::DEVICE); } - std::cout<<"COO to CSR completed. Expanded NNZ: "<< A->getNnz()<getNnz() << std::endl; + + // Now call direct solver int status; if (i < 2) { KLU->setup(A); status = KLU->analyze(); - std::cout<<"KLU analysis status: "<factorize(); - std::cout<<"KLU factorization status: "<solve(vec_rhs, vec_x); - std::cout<<"KLU solve status: "<getLFactor(); ReSolve::matrix::Csc* U = (ReSolve::matrix::Csc*) KLU->getUFactor(); @@ -128,13 +134,14 @@ int main(int argc, char *argv[] ) Rf->setup(A, L, U, P, Q, vec_rhs); } } else { - std::cout<<"Using rocsolver rf"<refactorize(); - std::cout<<"rocsolver rf refactorization status: "<solve(vec_rhs, vec_x); - std::cout<<"rocsolver rf solve status: "<update(rhs, ReSolve::memory::HOST, ReSolve::memory::DEVICE); real_type bnorm = sqrt(vector_handler->dot(vec_r, vec_r, ReSolve::memory::DEVICE)); matrix_handler->setValuesChanged(true, ReSolve::memory::DEVICE); @@ -154,7 +161,7 @@ int main(int argc, char *argv[] ) } // for (int i = 0; i < numSystems; ++i) - //now DELETE + // now DELETE delete A; delete KLU; delete Rf; diff --git a/examples/r_KLU_rocsolverrf_redo_factorization.cpp b/examples/r_KLU_rocsolverrf_redo_factorization.cpp index c82502dd..29daba80 100644 --- a/examples/r_KLU_rocsolverrf_redo_factorization.cpp +++ b/examples/r_KLU_rocsolverrf_redo_factorization.cpp @@ -27,8 +27,8 @@ int main(int argc, char *argv[] ) std::string rhsFileName = argv[2]; index_type numSystems = atoi(argv[3]); - std::cout<<"Family mtx file name: "<< matrixFileName << ", total number of matrices: "<getNumRows()<<" x "<getNumColumns()<< ", nnz: "<< A->getNnz()<< ", symmetric? "<symmetric()<< ", Expanded? "<expanded()<syncData(ReSolve::memory::DEVICE); + + std::cout << "Finished reading the matrix and rhs, size: " << A->getNumRows() << " x "<< A->getNumColumns() + << ", nnz: " << A->getNnz() + << ", symmetric? " << A->symmetric() + << ", Expanded? " << A->expanded() << std::endl; mat_file.close(); rhs_file.close(); // Update host and device data. if (i < 2) { vec_rhs->update(rhs, ReSolve::memory::HOST, ReSolve::memory::HOST); - vec_rhs->setDataUpdated(ReSolve::memory::HOST); } else { - A->copyData(ReSolve::memory::DEVICE); vec_rhs->update(rhs, ReSolve::memory::HOST, ReSolve::memory::DEVICE); } - std::cout<<"COO to CSR completed. Expanded NNZ: "<< A->getNnz()<getNnz() << std::endl; + //Now call direct solver int status; if (i < 2){ KLU->setup(A); status = KLU->analyze(); - std::cout<<"KLU analysis status: "<factorize(); - std::cout<<"KLU factorization status: "<solve(vec_rhs, vec_x); - std::cout<<"KLU solve status: "<getLFactor(); ReSolve::matrix::Csc* U = (ReSolve::matrix::Csc*) KLU->getUFactor(); @@ -130,16 +135,16 @@ int main(int argc, char *argv[] ) Rf->refactorize(); } } else { - std::cout<<"Using rocsolver rf"<refactorize(); - std::cout<<"rocsolver rf refactorization status: "<solve(vec_rhs, vec_x); - std::cout<<"rocsolver rf solve status: "<update(rhs, ReSolve::memory::HOST, ReSolve::memory::DEVICE); + // Check accuracy of the solution + vec_r->update(rhs, ReSolve::memory::HOST, ReSolve::memory::DEVICE); matrix_handler->setValuesChanged(true, ReSolve::memory::DEVICE); - matrix_handler->matvec(A, vec_x, vec_r, &ONE, &MINUSONE, ReSolve::memory::DEVICE); res_nrm = sqrt(vector_handler->dot(vec_r, vec_r, ReSolve::memory::DEVICE)); b_nrm = sqrt(vector_handler->dot(vec_rhs, vec_rhs, ReSolve::memory::DEVICE)); @@ -148,15 +153,16 @@ int main(int argc, char *argv[] ) << res_nrm/b_nrm << "\n"; if (!isnan(res_nrm)) { if (res_nrm/b_nrm > 1e-7 ) { - std::cout << "\n \t !!! ALERT !!! Residual norm is too large; redoing KLU symbolic and numeric factorization. !!! ALERT !!! \n \n"; + std::cout << "\n \t !!! ALERT !!! Residual norm is too large; " + << "redoing KLU symbolic and numeric factorization. !!! ALERT !!! \n\n"; KLU->setup(A); status = KLU->analyze(); - std::cout<<"KLU analysis status: "<factorize(); - std::cout<<"KLU factorization status: "<solve(vec_rhs, vec_x); - std::cout<<"KLU solve status: "<update(rhs, ReSolve::memory::HOST, ReSolve::memory::DEVICE); vec_r->update(rhs, ReSolve::memory::HOST, ReSolve::memory::DEVICE); @@ -166,7 +172,7 @@ int main(int argc, char *argv[] ) matrix_handler->matvec(A, vec_x, vec_r, &ONE, &MINUSONE, ReSolve::memory::DEVICE); res_nrm = sqrt(vector_handler->dot(vec_r, vec_r, ReSolve::memory::DEVICE)); - std::cout<<"\t New residual norm: " + std::cout << "\t New residual norm: " << std::scientific << std::setprecision(16) << res_nrm/b_nrm << "\n"; diff --git a/examples/r_SysSolverCuda.cpp b/examples/r_SysSolverCuda.cpp index ab7ea303..1f98ae42 100644 --- a/examples/r_SysSolverCuda.cpp +++ b/examples/r_SysSolverCuda.cpp @@ -104,7 +104,7 @@ int main(int argc, char *argv[]) vec_rhs->update(rhs, ReSolve::memory::HOST, ReSolve::memory::HOST); vec_rhs->setDataUpdated(ReSolve::memory::HOST); } else { - A->copyData(ReSolve::memory::DEVICE); + A->syncData(ReSolve::memory::DEVICE); vec_rhs->update(rhs, ReSolve::memory::HOST, ReSolve::memory::DEVICE); } std::cout<<"COO to CSR completed. Expanded NNZ: "<< A->getNnz()<syncData(ReSolve::memory::DEVICE); + std::cout << "Finished reading the matrix and rhs, size: " << A->getNumRows() << " x " << A->getNumColumns() << ", nnz: " << A->getNnz() @@ -102,12 +105,11 @@ int main(int argc, char *argv[] ) // Update host and device data. if (i < 2) { vec_rhs->update(rhs, ReSolve::memory::HOST, ReSolve::memory::HOST); - vec_rhs->setDataUpdated(ReSolve::memory::HOST); } else { - A->copyData(ReSolve::memory::DEVICE); vec_rhs->update(rhs, ReSolve::memory::HOST, ReSolve::memory::DEVICE); } - std::cout<<"COO to CSR completed. Expanded NNZ: "<< A->getNnz()<getNnz() << std::endl; + //Now call direct solver solver->setMatrix(A); int status = -1; diff --git a/examples/r_SysSolverHipRefine.cpp b/examples/r_SysSolverHipRefine.cpp index d340b51c..47e000e4 100644 --- a/examples/r_SysSolverHipRefine.cpp +++ b/examples/r_SysSolverHipRefine.cpp @@ -95,6 +95,9 @@ int main(int argc, char *argv[]) ReSolve::io::updateMatrixFromFile(mat_file, A); ReSolve::io::updateArrayFromFile(rhs_file, &rhs); } + // Copy matrix data to device + A->syncData(ReSolve::memory::DEVICE); + std::cout << "Finished reading the matrix and rhs, size: " << A->getNumRows() << " x " << A->getNumColumns() << ", nnz: " << A->getNnz() @@ -106,12 +109,12 @@ int main(int argc, char *argv[]) // Update host and device data. if (i < 2) { vec_rhs->update(rhs, ReSolve::memory::HOST, ReSolve::memory::HOST); - vec_rhs->setDataUpdated(ReSolve::memory::HOST); } else { - A->copyData(ReSolve::memory::DEVICE); vec_rhs->update(rhs, ReSolve::memory::HOST, ReSolve::memory::DEVICE); } - std::cout<<"COO to CSR completed. Expanded NNZ: "<< A->getNnz()<getNnz() << std::endl; + + // Now call solver solver->setMatrix(A); int status; if (i < 2) { diff --git a/examples/r_randGMRES.cpp b/examples/r_randGMRES.cpp index 2a7fb3f8..2c292740 100644 --- a/examples/r_randGMRES.cpp +++ b/examples/r_randGMRES.cpp @@ -72,13 +72,17 @@ int main(int argc, char *argv[]) vec_x->allocate(ReSolve::memory::DEVICE); vec_x->setToZero(ReSolve::memory::DEVICE); vec_r = new vector_type(A->getNumRows()); - std::cout<<"Finished reading the matrix and rhs, size: "<getNumRows()<<" x "<getNumColumns()<< ", nnz: "<< A->getNnz()<< ", symmetric? "<symmetric()<< ", Expanded? "<expanded()<getNumRows() << " x "<< A->getNumColumns() + << ", nnz: " << A->getNnz() + << ", symmetric? " << A->symmetric() + << ", Expanded? " << A->expanded() << std::endl; mat_file.close(); rhs_file.close(); - A->copyData(ReSolve::memory::DEVICE); + A->syncData(ReSolve::memory::DEVICE); vec_rhs->update(rhs, ReSolve::memory::HOST, ReSolve::memory::DEVICE); - //Now call direct solver + + //Now call the solver real_type norm_b; matrix_handler->setValuesChanged(true, ReSolve::memory::DEVICE); @@ -106,7 +110,6 @@ int main(int argc, char *argv[]) << FGMRES->getFinalResidualNorm()/norm_b << " iter: " << FGMRES->getNumIter() << "\n"; - delete A; delete Rf; delete [] x; diff --git a/examples/r_randGMRES_CUDA.cpp b/examples/r_randGMRES_CUDA.cpp index 9c60915a..c44aec3b 100644 --- a/examples/r_randGMRES_CUDA.cpp +++ b/examples/r_randGMRES_CUDA.cpp @@ -75,7 +75,7 @@ int main(int argc, char *argv[]) mat_file.close(); rhs_file.close(); - A->copyData(ReSolve::memory::DEVICE); + A->syncData(ReSolve::memory::DEVICE); vec_rhs->update(rhs, ReSolve::memory::HOST, ReSolve::memory::DEVICE); //Now call direct solver real_type norm_b; diff --git a/resolve/GramSchmidt.cpp b/resolve/GramSchmidt.cpp index b0911c4e..22a03a00 100644 --- a/resolve/GramSchmidt.cpp +++ b/resolve/GramSchmidt.cpp @@ -223,7 +223,7 @@ namespace ReSolve // vector_handler_->massDot2Vec(n, V, i + 1, vec_v_, vec_rv_, memspace_); vector_handler_->massDot2Vec(n, V, i + 1, vec_v_, vec_rv_, memspace_); vec_rv_->setDataUpdated(memspace_); - vec_rv_->copyData(memspace_, memory::HOST); + vec_rv_->syncData(memory::HOST); vec_rv_->deepCopyVectorData(&h_L_[idxmap(i, 0, num_vecs_ + 1)], 0, memory::HOST); h_rv = vec_rv_->getVectorData(1, memory::HOST); @@ -271,7 +271,7 @@ namespace ReSolve vector_handler_->massDot2Vec(n, V, i + 1, vec_v_, vec_rv_, memspace_); vec_rv_->setDataUpdated(memspace_); - vec_rv_->copyData(memspace_, memory::HOST); + vec_rv_->syncData(memory::HOST); vec_rv_->deepCopyVectorData(&h_L_[idxmap(i, 0, num_vecs_ + 1)], 0, memory::HOST); h_rv = vec_rv_->getVectorData(1, memory::HOST); diff --git a/resolve/LinSolverDirectRocSolverRf.cpp b/resolve/LinSolverDirectRocSolverRf.cpp index b7af7428..b7932883 100644 --- a/resolve/LinSolverDirectRocSolverRf.cpp +++ b/resolve/LinSolverDirectRocSolverRf.cpp @@ -44,7 +44,7 @@ namespace ReSolve addFactors(L, U); M_->setUpdated(ReSolve::memory::HOST); - M_->copyData(ReSolve::memory::DEVICE); + M_->syncData(ReSolve::memory::DEVICE); if (d_P_ == nullptr) { mem_.allocateArrayOnDevice(&d_P_, n); diff --git a/resolve/SystemSolver.cpp b/resolve/SystemSolver.cpp index 1eed3c71..d42757a0 100644 --- a/resolve/SystemSolver.cpp +++ b/resolve/SystemSolver.cpp @@ -389,7 +389,9 @@ namespace ReSolve } if (refactorizationMethod_ == "cusolverrf") { matrix::Csc* L_csc = dynamic_cast(L_); - matrix::Csc* U_csc = dynamic_cast(U_); + matrix::Csc* U_csc = dynamic_cast(U_); + L_csc->syncData(memory::DEVICE); + U_csc->syncData(memory::DEVICE); matrix::Csr* L_csr = new matrix::Csr(L_csc->getNumRows(), L_csc->getNumColumns(), L_csc->getNnz()); matrix::Csr* U_csr = new matrix::Csr(U_csc->getNumRows(), U_csc->getNumColumns(), U_csc->getNnz()); matrixHandler_->csc2csr(L_csc, L_csr, memory::DEVICE); diff --git a/resolve/matrix/Coo.cpp b/resolve/matrix/Coo.cpp index edba4c0f..cb3316a9 100644 --- a/resolve/matrix/Coo.cpp +++ b/resolve/matrix/Coo.cpp @@ -1,6 +1,7 @@ #include // <-- includes memcpy #include -#include +#include +#include #include #include "Coo.hpp" @@ -80,7 +81,7 @@ namespace ReSolve d_data_updated_ = true; owns_gpu_vals_ = true; owns_gpu_data_ = true; - copyData(memspaceDst); + syncData(memspaceDst); // Hijack data from the source *rows = nullptr; *cols = nullptr; @@ -94,7 +95,7 @@ namespace ReSolve h_data_updated_ = true; owns_cpu_vals_ = true; owns_cpu_data_ = true; - copyData(memspaceDst); + syncData(memspaceDst); // Hijack data from the source *rows = nullptr; *cols = nullptr; @@ -132,7 +133,7 @@ namespace ReSolve index_type* matrix::Coo::getRowData(memory::MemorySpace memspace) { using namespace ReSolve::memory; - copyData(memspace); + switch (memspace) { case HOST: return this->h_row_data_; @@ -146,7 +147,7 @@ namespace ReSolve index_type* matrix::Coo::getColData(memory::MemorySpace memspace) { using namespace ReSolve::memory; - copyData(memspace); + switch (memspace) { case HOST: return this->h_col_data_; @@ -160,7 +161,7 @@ namespace ReSolve real_type* matrix::Coo::getValues(memory::MemorySpace memspace) { using namespace ReSolve::memory; - copyData(memspace); + switch (memspace) { case HOST: return this->h_val_data_; @@ -286,55 +287,75 @@ namespace ReSolve return -1; } - int matrix::Coo::copyData(memory::MemorySpace memspaceOut) + /** + * @brief Sync data in memspace with the updated memory space. + * + * @param memspace - memory space to be synced up (HOST or DEVICE) + * @return int - 0 if successful, error code otherwise + * + * @todo Handle case when neither memory space is updated. Currently, + * this function does nothing in that situation, quitely ignoring + * the sync call. + */ + int matrix::Coo::syncData(memory::MemorySpace memspace) { using namespace ReSolve::memory; - index_type nnz_current = nnz_; - - switch (memspaceOut) { + switch (memspace) { case HOST: - if ((d_data_updated_ == true) && (h_data_updated_ == false)) { - if ((h_row_data_ == nullptr) != (h_col_data_ == nullptr)) { - out::error() << "In Coo::copyData one of host row or column data is null!\n"; - } - if ((h_row_data_ == nullptr) && (h_col_data_ == nullptr)) { - h_row_data_ = new index_type[nnz_current]; - h_col_data_ = new index_type[nnz_current]; - owns_cpu_data_ = true; - } - if (h_val_data_ == nullptr) { - h_val_data_ = new real_type[nnz_current]; - owns_cpu_vals_ = true; - } - mem_.copyArrayDeviceToHost(h_row_data_, d_row_data_, nnz_current); - mem_.copyArrayDeviceToHost(h_col_data_, d_col_data_, nnz_current); - mem_.copyArrayDeviceToHost(h_val_data_, d_val_data_, nnz_current); - h_data_updated_ = true; + if (h_data_updated_) { + out::misc() << "In Coo::syncData trying to sync host, but host already up to date!\n"; + return 0; + } + if (!d_data_updated_) { + out::error() << "In Coo::syncData trying to sync host with device, but device is out of date!\n"; + assert(d_data_updated_); } + if ((h_row_data_ == nullptr) != (h_col_data_ == nullptr)) { + out::error() << "In Coo::syncData one of host row or column data is null!\n"; + } + if ((h_row_data_ == nullptr) && (h_col_data_ == nullptr)) { + h_row_data_ = new index_type[nnz_]; + h_col_data_ = new index_type[nnz_]; + owns_cpu_data_ = true; + } + if (h_val_data_ == nullptr) { + h_val_data_ = new real_type[nnz_]; + owns_cpu_vals_ = true; + } + mem_.copyArrayDeviceToHost(h_row_data_, d_row_data_, nnz_); + mem_.copyArrayDeviceToHost(h_col_data_, d_col_data_, nnz_); + mem_.copyArrayDeviceToHost(h_val_data_, d_val_data_, nnz_); + h_data_updated_ = true; return 0; case DEVICE: - if ((d_data_updated_ == false) && (h_data_updated_ == true)) { - if ((d_row_data_ == nullptr) != (d_col_data_ == nullptr)) { - out::error() << "In Coo::copyData one of device row or column data is null!\n"; - } - if ((d_row_data_ == nullptr) && (d_col_data_ == nullptr)) { - mem_.allocateArrayOnDevice(&d_row_data_, nnz_current); - mem_.allocateArrayOnDevice(&d_col_data_, nnz_current); - owns_gpu_data_ = true; - } - if (d_val_data_ == nullptr) { - mem_.allocateArrayOnDevice(&d_val_data_, nnz_current); - owns_gpu_vals_ = true; - } - mem_.copyArrayHostToDevice(d_row_data_, h_row_data_, nnz_current); - mem_.copyArrayHostToDevice(d_col_data_, h_col_data_, nnz_current); - mem_.copyArrayHostToDevice(d_val_data_, h_val_data_, nnz_current); - d_data_updated_ = true; + if (d_data_updated_) { + out::misc() << "In Coo::syncData trying to sync device, but device already up to date!\n"; + return 0; + } + if (!h_data_updated_) { + out::error() << "In Coo::syncData trying to sync device with host, but host is out of date!\n"; + assert(h_data_updated_); } + if ((d_row_data_ == nullptr) != (d_col_data_ == nullptr)) { + out::error() << "In Coo::syncData one of device row or column data is null!\n"; + } + if ((d_row_data_ == nullptr) && (d_col_data_ == nullptr)) { + mem_.allocateArrayOnDevice(&d_row_data_, nnz_); + mem_.allocateArrayOnDevice(&d_col_data_, nnz_); + owns_gpu_data_ = true; + } + if (d_val_data_ == nullptr) { + mem_.allocateArrayOnDevice(&d_val_data_, nnz_); + owns_gpu_vals_ = true; + } + mem_.copyArrayHostToDevice(d_row_data_, h_row_data_, nnz_); + mem_.copyArrayHostToDevice(d_col_data_, h_col_data_, nnz_); + mem_.copyArrayHostToDevice(d_val_data_, h_val_data_, nnz_); + d_data_updated_ = true; return 0; default: - return -1; + return 1; } // switch } diff --git a/resolve/matrix/Coo.hpp b/resolve/matrix/Coo.hpp index b12b4227..0b9829af 100644 --- a/resolve/matrix/Coo.hpp +++ b/resolve/matrix/Coo.hpp @@ -36,7 +36,7 @@ namespace ReSolve { namespace matrix { virtual void print(std::ostream& file_out = std::cout, index_type indexing_base = 0); - virtual int copyData(memory::MemorySpace memspaceOut); + virtual int syncData(memory::MemorySpace memspaceOut); }; }} // namespace ReSolve::matrix diff --git a/resolve/matrix/Csc.cpp b/resolve/matrix/Csc.cpp index 589f97a4..899f37a3 100644 --- a/resolve/matrix/Csc.cpp +++ b/resolve/matrix/Csc.cpp @@ -1,5 +1,6 @@ #include // <-- includes memcpy #include +#include #include #include "Csc.hpp" @@ -33,7 +34,7 @@ namespace ReSolve index_type* matrix::Csc::getRowData(memory::MemorySpace memspace) { using namespace ReSolve::memory; - copyData(memspace); + switch (memspace) { case HOST: return this->h_row_data_; @@ -47,7 +48,7 @@ namespace ReSolve index_type* matrix::Csc::getColData(memory::MemorySpace memspace) { using namespace ReSolve::memory; - copyData(memspace); + switch (memspace) { case HOST: return this->h_col_data_; @@ -61,7 +62,7 @@ namespace ReSolve real_type* matrix::Csc::getValues(memory::MemorySpace memspace) { using namespace ReSolve::memory; - copyData(memspace); + switch (memspace) { case HOST: return this->h_val_data_; @@ -193,55 +194,75 @@ namespace ReSolve return -1; } - int matrix::Csc::copyData(memory::MemorySpace memspaceOut) + /** + * @brief Sync data in memspace with the updated memory space. + * + * @param memspace - memory space to be synced up (HOST or DEVICE) + * @return int - 0 if successful, error code otherwise + * + * @todo Handle case when neither memory space is updated. Currently, + * this function does nothing in that situation, quitely ignoring + * the sync call. + */ + int matrix::Csc::syncData(memory::MemorySpace memspace) { using namespace ReSolve::memory; - index_type nnz_current = nnz_; - - switch(memspaceOut) { + switch(memspace) { case HOST: - if ((d_data_updated_ == true) && (h_data_updated_ == false)) { - if ((h_row_data_ == nullptr) != (h_col_data_ == nullptr)) { - out::error() << "In Csc::copyData one of host row or column data is null!\n"; - } - if ((h_col_data_ == nullptr) && (h_row_data_ == nullptr)) { - h_col_data_ = new index_type[m_ + 1]; - h_row_data_ = new index_type[nnz_current]; - owns_cpu_data_ = true; - } - if (h_val_data_ == nullptr) { - h_val_data_ = new real_type[nnz_current]; - owns_cpu_vals_ = true; - } - mem_.copyArrayDeviceToHost(h_col_data_, d_col_data_, m_ + 1); - mem_.copyArrayDeviceToHost(h_row_data_, d_row_data_, nnz_current); - mem_.copyArrayDeviceToHost(h_val_data_, d_val_data_, nnz_current); - h_data_updated_ = true; + if (h_data_updated_) { + out::misc() << "In Csc::syncData trying to sync host, but host already up to date!\n"; + return 0; + } + if (!d_data_updated_) { + out::error() << "In Csc::syncData trying to sync host with device, but device is out of date!\n"; + assert(d_data_updated_); } + if ((h_row_data_ == nullptr) != (h_col_data_ == nullptr)) { + out::error() << "In Csc::syncData one of host row or column data is null!\n"; + } + if ((h_col_data_ == nullptr) && (h_row_data_ == nullptr)) { + h_col_data_ = new index_type[m_ + 1]; + h_row_data_ = new index_type[nnz_]; + owns_cpu_data_ = true; + } + if (h_val_data_ == nullptr) { + h_val_data_ = new real_type[nnz_]; + owns_cpu_vals_ = true; + } + mem_.copyArrayDeviceToHost(h_col_data_, d_col_data_, m_ + 1); + mem_.copyArrayDeviceToHost(h_row_data_, d_row_data_, nnz_); + mem_.copyArrayDeviceToHost(h_val_data_, d_val_data_, nnz_); + h_data_updated_ = true; return 0; case DEVICE: - if ((d_data_updated_ == false) && (h_data_updated_ == true)) { - if ((d_row_data_ == nullptr) != (d_col_data_ == nullptr)) { - out::error() << "In Csc::copyData one of device row or column data is null!\n"; - } - if ((d_col_data_ == nullptr) && (d_row_data_ == nullptr)) { - mem_.allocateArrayOnDevice(&d_col_data_, m_ + 1); - mem_.allocateArrayOnDevice(&d_row_data_, nnz_current); - owns_gpu_data_ = true; - } - if (d_val_data_ == nullptr) { - mem_.allocateArrayOnDevice(&d_val_data_, nnz_current); - owns_gpu_vals_ = true; - } - mem_.copyArrayHostToDevice(d_col_data_, h_col_data_, m_ + 1); - mem_.copyArrayHostToDevice(d_row_data_, h_row_data_, nnz_current); - mem_.copyArrayHostToDevice(d_val_data_, h_val_data_, nnz_current); - d_data_updated_ = true; + if (d_data_updated_) { + out::misc() << "In Csc::syncData trying to sync device, but device already up to date!\n"; + return 0; + } + if (!h_data_updated_) { + out::error() << "In Csc::syncData trying to sync device with host, but host is out of date!\n"; + assert(h_data_updated_); } + if ((d_row_data_ == nullptr) != (d_col_data_ == nullptr)) { + out::error() << "In Csc::syncData one of device row or column data is null!\n"; + } + if ((d_col_data_ == nullptr) && (d_row_data_ == nullptr)) { + mem_.allocateArrayOnDevice(&d_col_data_, m_ + 1); + mem_.allocateArrayOnDevice(&d_row_data_, nnz_); + owns_gpu_data_ = true; + } + if (d_val_data_ == nullptr) { + mem_.allocateArrayOnDevice(&d_val_data_, nnz_); + owns_gpu_vals_ = true; + } + mem_.copyArrayHostToDevice(d_col_data_, h_col_data_, m_ + 1); + mem_.copyArrayHostToDevice(d_row_data_, h_row_data_, nnz_); + mem_.copyArrayHostToDevice(d_val_data_, h_val_data_, nnz_); + d_data_updated_ = true; return 0; default: - return -1; + return 1; } // switch } diff --git a/resolve/matrix/Csc.hpp b/resolve/matrix/Csc.hpp index 3395ad6c..e679613c 100644 --- a/resolve/matrix/Csc.hpp +++ b/resolve/matrix/Csc.hpp @@ -35,7 +35,7 @@ namespace ReSolve { namespace matrix { virtual void print(std::ostream& file_out = std::cout, index_type indexing_base = 0); - virtual int copyData(memory::MemorySpace memspaceOut); + virtual int syncData(memory::MemorySpace memspaceOut); }; }} // namespace ReSolve::matrix diff --git a/resolve/matrix/Csr.cpp b/resolve/matrix/Csr.cpp index 36a22bc2..1e4dacb1 100644 --- a/resolve/matrix/Csr.cpp +++ b/resolve/matrix/Csr.cpp @@ -95,7 +95,7 @@ namespace ReSolve d_val_data_ = *vals; d_data_updated_ = true; owns_gpu_data_ = true; - copyData(memspaceDst); + syncData(memspaceDst); // Hijack data from the source *rows = nullptr; *cols = nullptr; @@ -108,7 +108,7 @@ namespace ReSolve h_val_data_ = *vals; h_data_updated_ = true; owns_cpu_data_ = true; - copyData(memspaceDst); + syncData(memspaceDst); // Hijack data from the source *rows = nullptr; @@ -151,7 +151,7 @@ namespace ReSolve index_type* matrix::Csr::getRowData(memory::MemorySpace memspace) { using namespace ReSolve::memory; - copyData(memspace); + switch (memspace) { case HOST: return this->h_row_data_; @@ -165,7 +165,7 @@ namespace ReSolve index_type* matrix::Csr::getColData(memory::MemorySpace memspace) { using namespace ReSolve::memory; - copyData(memspace); + switch (memspace) { case HOST: return this->h_col_data_; @@ -179,7 +179,7 @@ namespace ReSolve real_type* matrix::Csr::getValues(memory::MemorySpace memspace) { using namespace ReSolve::memory; - copyData(memspace); + switch (memspace) { case HOST: return this->h_val_data_; @@ -302,56 +302,76 @@ namespace ReSolve return -1; } - int matrix::Csr::copyData(memory::MemorySpace memspaceOut) + /** + * @brief Sync data in memspace with the updated memory space. + * + * @param memspace - memory space to be synced up (HOST or DEVICE) + * @return int - 0 if successful, error code otherwise + * + * @todo Handle case when neither memory space is updated. Currently, + * this function does nothing in that situation, quitely ignoring + * the sync call. + */ + int matrix::Csr::syncData(memory::MemorySpace memspace) { using namespace ReSolve::memory; - index_type nnz_current = nnz_; - - switch (memspaceOut) { + switch (memspace) { case HOST: //check if we need to copy or not - if ((d_data_updated_ == true) && (h_data_updated_ == false)) { - if ((h_row_data_ == nullptr) != (h_col_data_ == nullptr)) { - out::error() << "In Csr::copyData one of host row or column data is null!\n"; - } - if ((h_row_data_ == nullptr) && (h_col_data_ == nullptr)) { - h_row_data_ = new index_type[n_ + 1]; - h_col_data_ = new index_type[nnz_current]; - owns_cpu_data_ = true; - } - if (h_val_data_ == nullptr) { - h_val_data_ = new real_type[nnz_current]; - owns_cpu_vals_ = true; - } - mem_.copyArrayDeviceToHost(h_row_data_, d_row_data_, n_ + 1); - mem_.copyArrayDeviceToHost(h_col_data_, d_col_data_, nnz_current); - mem_.copyArrayDeviceToHost(h_val_data_, d_val_data_, nnz_current); - h_data_updated_ = true; + if (h_data_updated_) { + out::misc() << "In Csr::syncData trying to sync host, but host already up to date!\n"; + return 0; + } + if (!d_data_updated_) { + out::error() << "In Csr::syncData trying to sync host with device, but device is out of date!\n"; + assert(d_data_updated_); } + if ((h_row_data_ == nullptr) != (h_col_data_ == nullptr)) { + out::error() << "In Csr::syncData one of host row or column data is null!\n"; + } + if ((h_row_data_ == nullptr) && (h_col_data_ == nullptr)) { + h_row_data_ = new index_type[n_ + 1]; + h_col_data_ = new index_type[nnz_]; + owns_cpu_data_ = true; + } + if (h_val_data_ == nullptr) { + h_val_data_ = new real_type[nnz_]; + owns_cpu_vals_ = true; + } + mem_.copyArrayDeviceToHost(h_row_data_, d_row_data_, n_ + 1); + mem_.copyArrayDeviceToHost(h_col_data_, d_col_data_, nnz_); + mem_.copyArrayDeviceToHost(h_val_data_, d_val_data_, nnz_); + h_data_updated_ = true; return 0; case DEVICE: - if ((d_data_updated_ == false) && (h_data_updated_ == true)) { - if ((d_row_data_ == nullptr) != (d_col_data_ == nullptr)) { - out::error() << "In Csr::copyData one of device row or column data is null!\n"; - } - if ((d_row_data_ == nullptr) && (d_col_data_ == nullptr)) { - mem_.allocateArrayOnDevice(&d_row_data_, n_ + 1); - mem_.allocateArrayOnDevice(&d_col_data_, nnz_current); - owns_gpu_data_ = true; - } - if (d_val_data_ == nullptr) { - mem_.allocateArrayOnDevice(&d_val_data_, nnz_current); - owns_gpu_vals_ = true; - } - mem_.copyArrayHostToDevice(d_row_data_, h_row_data_, n_ + 1); - mem_.copyArrayHostToDevice(d_col_data_, h_col_data_, nnz_current); - mem_.copyArrayHostToDevice(d_val_data_, h_val_data_, nnz_current); - d_data_updated_ = true; + if (d_data_updated_) { + out::misc() << "In Csr::syncData trying to sync device, but device already up to date!\n"; + return 0; + } + if (!h_data_updated_) { + out::error() << "In Csr::syncData trying to sync device with host, but host is out of date!\n"; + assert(h_data_updated_); } + if ((d_row_data_ == nullptr) != (d_col_data_ == nullptr)) { + out::error() << "In Csr::syncData one of device row or column data is null!\n"; + } + if ((d_row_data_ == nullptr) && (d_col_data_ == nullptr)) { + mem_.allocateArrayOnDevice(&d_row_data_, n_ + 1); + mem_.allocateArrayOnDevice(&d_col_data_, nnz_); + owns_gpu_data_ = true; + } + if (d_val_data_ == nullptr) { + mem_.allocateArrayOnDevice(&d_val_data_, nnz_); + owns_gpu_vals_ = true; + } + mem_.copyArrayHostToDevice(d_row_data_, h_row_data_, n_ + 1); + mem_.copyArrayHostToDevice(d_col_data_, h_col_data_, nnz_); + mem_.copyArrayHostToDevice(d_val_data_, h_val_data_, nnz_); + d_data_updated_ = true; return 0; default: - return -1; + return 1; } // switch } diff --git a/resolve/matrix/Csr.hpp b/resolve/matrix/Csr.hpp index 2f597cd5..ae56d90d 100644 --- a/resolve/matrix/Csr.hpp +++ b/resolve/matrix/Csr.hpp @@ -43,7 +43,7 @@ namespace ReSolve { namespace matrix { virtual void print(std::ostream& file_out = std::cout, index_type indexing_base = 0); - virtual int copyData(memory::MemorySpace memspaceOut); + virtual int syncData(memory::MemorySpace memspaceOut); }; diff --git a/resolve/matrix/Sparse.hpp b/resolve/matrix/Sparse.hpp index 783fdb72..00094f32 100644 --- a/resolve/matrix/Sparse.hpp +++ b/resolve/matrix/Sparse.hpp @@ -65,7 +65,7 @@ namespace ReSolve { namespace matrix { virtual void print(std::ostream& file_out, index_type indexing_base) = 0; - virtual int copyData(memory::MemorySpace memspaceOut) = 0; + virtual int syncData(memory::MemorySpace memspaceOut) = 0; //update Values just updates values; it allocates if necessary. diff --git a/resolve/vector/Vector.cpp b/resolve/vector/Vector.cpp index d560091a..e29baa49 100644 --- a/resolve/vector/Vector.cpp +++ b/resolve/vector/Vector.cpp @@ -248,13 +248,12 @@ namespace ReSolve { namespace vector { real_type* Vector::getData(index_type i, memory::MemorySpace memspace) { if ((memspace == memory::HOST) && (cpu_updated_ == false) && (gpu_updated_ == true )) { - // remember IN FIRST OUT SECOND!!! - copyData(memory::DEVICE, memspace); + syncData(memspace); owns_cpu_data_ = true; } if ((memspace == memory::DEVICE) && (gpu_updated_ == false) && (cpu_updated_ == true )) { - copyData(memory::HOST, memspace); + syncData(memspace); owns_gpu_data_ = true; } if (memspace == memory::HOST) { @@ -272,40 +271,51 @@ namespace ReSolve { namespace vector { /** * @brief copy internal vector data from HOST to DEVICE or from DEVICE to HOST * - * @param[in] memspaceIn - Memory space of the data to copy FROM - * @param[in] memspaceOut - Memory space of the data to copy TO + * @param[in] memspaceOut - Memory space to sync * - * @return 0 if successful, -1 otherwise. + * @return 0 if successful, 1 otherwise. * */ - int Vector::copyData(memory::MemorySpace memspaceIn, memory::MemorySpace memspaceOut) + int Vector::syncData(memory::MemorySpace memspaceOut) { - int control=-1; - if ((memspaceIn == memory::HOST) && (memspaceOut == memory::DEVICE)){ control = 0;} - if ((memspaceIn == memory::DEVICE) && (memspaceOut == memory::HOST)) { control = 1;} + using namespace ReSolve::memory; - if ((memspaceOut == memory::HOST) && (h_data_ == nullptr)) { - //allocate first - h_data_ = new real_type[n_ * k_]; - owns_cpu_data_ = true; - } - if ((memspaceOut == memory::DEVICE) && (d_data_ == nullptr)) { - //allocate first - mem_.allocateArrayOnDevice(&d_data_, n_ * k_); - owns_gpu_data_ = true; - } - switch(control) { - case 0: //cpu->cuda + switch(memspaceOut) { + case DEVICE: // cpu->gpu + if (gpu_updated_) { + out::misc() << "Trying to sync device, but device already up to date!\n"; + return 0; + } + if (!cpu_updated_) { + out::error() << "Trying to sync device with host, but host is out of date!\n"; + } + if (d_data_ == nullptr) { + //allocate first + mem_.allocateArrayOnDevice(&d_data_, n_ * k_); + owns_gpu_data_ = true; + } mem_.copyArrayHostToDevice(d_data_, h_data_, n_current_ * k_); + gpu_updated_ = true; break; - case 1: //cuda->cpu + case HOST: //cuda->cpu + if (cpu_updated_) { + out::misc() << "Trying to sync host, but host already up to date!\n"; + return 0; + } + if (!gpu_updated_) { + out::error() << "Trying to sync host with device, but device is out of date!\n"; + } + if (h_data_ == nullptr) { + //allocate first + h_data_ = new real_type[n_ * k_]; + owns_cpu_data_ = true; + } mem_.copyArrayDeviceToHost(h_data_, d_data_, n_current_ * k_); + cpu_updated_ = true; break; default: - return -1; + return 1; } - cpu_updated_ = true; - gpu_updated_ = true; return 0; } diff --git a/resolve/vector/Vector.hpp b/resolve/vector/Vector.hpp index d566906e..968f6386 100644 --- a/resolve/vector/Vector.hpp +++ b/resolve/vector/Vector.hpp @@ -46,7 +46,7 @@ namespace ReSolve { namespace vector { void setToZero(index_type i, memory::MemorySpace memspace); // set i-th ivector to 0 void setToConst(real_type C, memory::MemorySpace memspace); void setToConst(index_type i, real_type C, memory::MemorySpace memspace); // set i-th vector to C - needed for unit tests, Gram Schmidt tests - int copyData(memory::MemorySpace memspaceIn, memory::MemorySpace memspaceOut); + int syncData(memory::MemorySpace memspaceOut); int setCurrentSize(index_type new_n_current); real_type* getVectorData(index_type i, memory::MemorySpace memspace); // get ith vector data out of multivector int deepCopyVectorData(real_type* dest, index_type i, memory::MemorySpace memspace); diff --git a/resolve/vector/VectorHandler.cpp b/resolve/vector/VectorHandler.cpp index 9701fed1..81ca3c77 100644 --- a/resolve/vector/VectorHandler.cpp +++ b/resolve/vector/VectorHandler.cpp @@ -123,6 +123,7 @@ namespace ReSolve { devImpl_->scal(alpha, x); break; } + x->setDataUpdated(memspace); } /** @@ -169,6 +170,7 @@ namespace ReSolve { devImpl_->axpy(alpha, x, y); break; } + y->setDataUpdated(memspace); } /** @@ -207,6 +209,7 @@ namespace ReSolve { devImpl_->gemv(transpose, n, k, alpha, beta, V, y, x); break; } + x->setDataUpdated(memspace); } /** @@ -232,6 +235,7 @@ namespace ReSolve { devImpl_->massAxpy(size, alpha, k, x, y); break; } + y->setDataUpdated(memspace); } /** @@ -259,6 +263,7 @@ namespace ReSolve { devImpl_->massDot2Vec(size, V, k, x, res); break; } + res->setDataUpdated(memspace); } /** diff --git a/tests/functionality/testKLU_GLU.cpp b/tests/functionality/testKLU_GLU.cpp index 53f0870b..9cf993d7 100644 --- a/tests/functionality/testKLU_GLU.cpp +++ b/tests/functionality/testKLU_GLU.cpp @@ -57,6 +57,7 @@ int main(int argc, char *argv[]) return -1; } ReSolve::matrix::Csr* A = ReSolve::io::createCsrFromFile(mat1); + A->syncData(ReSolve::memory::DEVICE); mat1.close(); // Read first rhs vector @@ -165,6 +166,7 @@ int main(int argc, char *argv[]) return -1; } ReSolve::io::updateMatrixFromFile(mat2, A); + A->syncData(ReSolve::memory::DEVICE); mat2.close(); // Load the second rhs vector diff --git a/tests/functionality/testKLU_Rf.cpp b/tests/functionality/testKLU_Rf.cpp index 60cd4afa..f9975460 100644 --- a/tests/functionality/testKLU_Rf.cpp +++ b/tests/functionality/testKLU_Rf.cpp @@ -57,6 +57,7 @@ int main(int argc, char *argv[]) return -1; } ReSolve::matrix::Csr* A = ReSolve::io::createCsrFromFile(mat1); + A->syncData(ReSolve::memory::DEVICE); mat1.close(); // Read first rhs vector @@ -145,6 +146,8 @@ int main(int argc, char *argv[]) ReSolve::matrix::Csc* L_csc = (ReSolve::matrix::Csc*) KLU->getLFactor(); ReSolve::matrix::Csc* U_csc = (ReSolve::matrix::Csc*) KLU->getUFactor(); + L_csc->syncData(ReSolve::memory::DEVICE); + U_csc->syncData(ReSolve::memory::DEVICE); ReSolve::matrix::Csr* L = new ReSolve::matrix::Csr(L_csc->getNumRows(), L_csc->getNumColumns(), L_csc->getNnz()); ReSolve::matrix::Csr* U = new ReSolve::matrix::Csr(U_csc->getNumRows(), U_csc->getNumColumns(), U_csc->getNnz()); error_sum += matrix_handler->csc2csr(L_csc,L, ReSolve::memory::DEVICE); @@ -164,6 +167,7 @@ int main(int argc, char *argv[]) return -1; } ReSolve::io::updateMatrixFromFile(mat2, A); + A->syncData(ReSolve::memory::DEVICE); mat2.close(); // Load the second rhs vector diff --git a/tests/functionality/testKLU_Rf_FGMRES.cpp b/tests/functionality/testKLU_Rf_FGMRES.cpp index 42f0d734..df9a5120 100644 --- a/tests/functionality/testKLU_Rf_FGMRES.cpp +++ b/tests/functionality/testKLU_Rf_FGMRES.cpp @@ -77,6 +77,7 @@ int main(int argc, char *argv[]) return -1; } ReSolve::matrix::Csr* A = ReSolve::io::createCsrFromFile(mat1); + A->syncData(ReSolve::memory::DEVICE); mat1.close(); // Read first rhs vector @@ -171,6 +172,8 @@ int main(int argc, char *argv[]) ReSolve::matrix::Csc* U_csc = (ReSolve::matrix::Csc*) KLU->getUFactor(); ReSolve::matrix::Csr* L = new ReSolve::matrix::Csr(L_csc->getNumRows(), L_csc->getNumColumns(), L_csc->getNnz()); ReSolve::matrix::Csr* U = new ReSolve::matrix::Csr(U_csc->getNumRows(), U_csc->getNumColumns(), U_csc->getNnz()); + L_csc->syncData(ReSolve::memory::DEVICE); + U_csc->syncData(ReSolve::memory::DEVICE); error_sum += matrix_handler->csc2csr(L_csc,L, ReSolve::memory::DEVICE); error_sum += matrix_handler->csc2csr(U_csc,U, ReSolve::memory::DEVICE); @@ -197,6 +200,7 @@ int main(int argc, char *argv[]) return -1; } ReSolve::io::updateMatrixFromFile(mat2, A); + A->syncData(ReSolve::memory::DEVICE); mat2.close(); // Load the second rhs vector diff --git a/tests/functionality/testKLU_RocSolver.cpp b/tests/functionality/testKLU_RocSolver.cpp index 97759667..0ceba109 100644 --- a/tests/functionality/testKLU_RocSolver.cpp +++ b/tests/functionality/testKLU_RocSolver.cpp @@ -61,6 +61,7 @@ int main(int argc, char *argv[]) return -1; } ReSolve::matrix::Csr* A = ReSolve::io::createCsrFromFile(mat1); + A->syncData(ReSolve::memory::DEVICE); mat1.close(); // Read first rhs vector @@ -173,6 +174,7 @@ int main(int argc, char *argv[]) return -1; } ReSolve::io::updateMatrixFromFile(mat2, A); + A->syncData(ReSolve::memory::DEVICE); mat2.close(); // Load the second rhs vector diff --git a/tests/functionality/testKLU_RocSolver_FGMRES.cpp b/tests/functionality/testKLU_RocSolver_FGMRES.cpp index c608868a..cca02b5c 100644 --- a/tests/functionality/testKLU_RocSolver_FGMRES.cpp +++ b/tests/functionality/testKLU_RocSolver_FGMRES.cpp @@ -77,6 +77,7 @@ int main(int argc, char *argv[]) return -1; } ReSolve::matrix::Csr* A = ReSolve::io::createCsrFromFile(mat1); + A->syncData(ReSolve::memory::DEVICE); mat1.close(); // Read first rhs vector @@ -192,6 +193,7 @@ int main(int argc, char *argv[]) return -1; } ReSolve::io::updateMatrixFromFile(mat2, A); + A->syncData(ReSolve::memory::DEVICE); mat2.close(); // Load the second rhs vector diff --git a/tests/functionality/testRandGMRES_Cuda.cpp b/tests/functionality/testRandGMRES_Cuda.cpp index e21a6664..05fc3c59 100644 --- a/tests/functionality/testRandGMRES_Cuda.cpp +++ b/tests/functionality/testRandGMRES_Cuda.cpp @@ -166,7 +166,8 @@ ReSolve::vector::Vector* generateRhs(const index_type N) data[i] = -111.0; } } - vec_rhs->copyData(ReSolve::memory::HOST, ReSolve::memory::DEVICE); + vec_rhs->setDataUpdated(ReSolve::memory::HOST); + vec_rhs->syncData(ReSolve::memory::DEVICE); return vec_rhs; } @@ -229,6 +230,6 @@ ReSolve::matrix::Csr* generateMatrix(const index_type N) A->setUpdated(ReSolve::memory::HOST); - A->copyData(ReSolve::memory::DEVICE); + A->syncData(ReSolve::memory::DEVICE); return A; } diff --git a/tests/functionality/testRandGMRES_Rocm.cpp b/tests/functionality/testRandGMRES_Rocm.cpp index 70801c66..3fdda823 100644 --- a/tests/functionality/testRandGMRES_Rocm.cpp +++ b/tests/functionality/testRandGMRES_Rocm.cpp @@ -166,7 +166,8 @@ ReSolve::vector::Vector* generateRhs(const index_type N) data[i] = -111.0; } } - vec_rhs->copyData(ReSolve::memory::HOST, ReSolve::memory::DEVICE); + vec_rhs->setDataUpdated(ReSolve::memory::HOST); + vec_rhs->syncData(ReSolve::memory::DEVICE); return vec_rhs; } @@ -229,6 +230,6 @@ ReSolve::matrix::Csr* generateMatrix(const index_type N) A->setUpdated(ReSolve::memory::HOST); - A->copyData(ReSolve::memory::DEVICE); + A->syncData(ReSolve::memory::DEVICE); return A; } diff --git a/tests/functionality/testSysGLU.cpp b/tests/functionality/testSysGLU.cpp index cf2aba15..c903229f 100644 --- a/tests/functionality/testSysGLU.cpp +++ b/tests/functionality/testSysGLU.cpp @@ -66,6 +66,7 @@ int main(int argc, char *argv[]) return -1; } ReSolve::matrix::Csr* A = ReSolve::io::createCsrFromFile(mat1); + A->syncData(ReSolve::memory::DEVICE); mat1.close(); // Read first rhs vector @@ -202,6 +203,7 @@ int main(int argc, char *argv[]) return -1; } ReSolve::io::updateMatrixFromFile(mat2, A); + A->syncData(ReSolve::memory::DEVICE); mat2.close(); // Load the second rhs vector diff --git a/tests/functionality/testSysRandGMRES.cpp b/tests/functionality/testSysRandGMRES.cpp index b090a1ff..cbc000c4 100644 --- a/tests/functionality/testSysRandGMRES.cpp +++ b/tests/functionality/testSysRandGMRES.cpp @@ -271,7 +271,8 @@ ReSolve::vector::Vector* generateRhs(const index_type N, ReSolve::memory::Memory data[i] = -111.0; } } - vec_rhs->copyData(ReSolve::memory::HOST, memspace); + vec_rhs->setDataUpdated(ReSolve::memory::HOST); + vec_rhs->syncData(memspace); return vec_rhs; } @@ -334,6 +335,6 @@ ReSolve::matrix::Csr* generateMatrix(const index_type N, ReSolve::memory::Memory A->setUpdated(ReSolve::memory::HOST); - A->copyData(memspace); + A->syncData(memspace); return A; } diff --git a/tests/functionality/testSysRefactor.cpp b/tests/functionality/testSysRefactor.cpp index 74b039ce..1ae563d7 100644 --- a/tests/functionality/testSysRefactor.cpp +++ b/tests/functionality/testSysRefactor.cpp @@ -92,6 +92,7 @@ int main(int argc, char *argv[]) return -1; } ReSolve::matrix::Csr* A = ReSolve::io::createCsrFromFile(mat1, true); + A->syncData(ReSolve::memory::DEVICE); mat1.close(); // Read first rhs vector @@ -224,6 +225,7 @@ int main(int argc, char *argv[]) return -1; } ReSolve::io::updateMatrixFromFile(mat2, A); + A->syncData(ReSolve::memory::DEVICE); mat2.close(); // Load the second rhs vector diff --git a/tests/unit/matrix/MatrixFactorizationTests.hpp b/tests/unit/matrix/MatrixFactorizationTests.hpp index 01977537..f3b880c2 100644 --- a/tests/unit/matrix/MatrixFactorizationTests.hpp +++ b/tests/unit/matrix/MatrixFactorizationTests.hpp @@ -167,7 +167,7 @@ class MatrixFactorizationTests : TestBase // A->print(); if ((memspace == "cuda") || (memspace == "hip")) { - A->copyData(memory::DEVICE); + A->syncData(memory::DEVICE); } return A; @@ -307,7 +307,7 @@ class MatrixFactorizationTests : TestBase { bool status = true; if (memspace != "cpu") { - A.copyData(memory::DEVICE); + A.syncData(memory::DEVICE); } size_t N = static_cast(A.getNumRows()); @@ -362,7 +362,7 @@ class MatrixFactorizationTests : TestBase { bool status = true; if (memspace != "cpu") { - x.copyData(memory::DEVICE, memory::HOST); + x.syncData(memory::HOST); } for (index_type i = 0; i < x.getSize(); ++i) { diff --git a/tests/unit/matrix/MatrixHandlerTests.hpp b/tests/unit/matrix/MatrixHandlerTests.hpp index 83656da0..6cd27451 100644 --- a/tests/unit/matrix/MatrixHandlerTests.hpp +++ b/tests/unit/matrix/MatrixHandlerTests.hpp @@ -87,7 +87,7 @@ class MatrixHandlerTests : TestBase { bool status = true; if (memspace_ == memory::DEVICE) { - x.copyData(memory::DEVICE, memory::HOST); + x.syncData(memory::HOST); } for (index_type i = 0; i < x.getSize(); ++i) { @@ -149,7 +149,7 @@ class MatrixHandlerTests : TestBase A->setUpdated(memory::HOST); if (memspace_ == memory::DEVICE) { - A->copyData(memspace_); + A->syncData(memspace_); } return A; diff --git a/tests/unit/vector/GramSchmidtTests.hpp b/tests/unit/vector/GramSchmidtTests.hpp index 5f7f020c..05398f69 100644 --- a/tests/unit/vector/GramSchmidtTests.hpp +++ b/tests/unit/vector/GramSchmidtTests.hpp @@ -96,7 +96,7 @@ namespace ReSolve } } V.setDataUpdated(memory::HOST); - V.copyData(memory::HOST, memspace_); + V.syncData(memspace_); //set the first vector to all 1s, normalize V.setToConst(0, 1.0, memspace_); diff --git a/tests/unit/vector/VectorHandlerTests.hpp b/tests/unit/vector/VectorHandlerTests.hpp index bbaa7c9d..f1ed4474 100644 --- a/tests/unit/vector/VectorHandlerTests.hpp +++ b/tests/unit/vector/VectorHandlerTests.hpp @@ -230,7 +230,7 @@ namespace ReSolve { bool status = true; if (memspace_ == memory::DEVICE) { - x.copyData(memory::DEVICE, memory::HOST); + x.syncData(memory::HOST); } for (index_type i = 0; i < x.getSize(); ++i) {