Skip to content

Commit

Permalink
Rename copyData -> syncData and ensure correct flags set when syn…
Browse files Browse the repository at this point in the history 
…cing (#199)

* Wholesale name change copyData -> syncData

* Set checks in Vector::syncData function

* Only one argument is sufficient in Vector::syncData

* Document matrix::*::syncData methods

* Use explicit host-device syncing (#200)

* Matrix getters don't sync.

* Use explicit host-device syncing in tests.

* Update examples
  • Loading branch information
@pelesh
pelesh authored Oct 12, 2024
1 parent 3c7e5d2 commit e5973c8
Show file tree
Hide file tree
Showing 41 changed files with 466 additions and 307 deletions.
13 changes: 9 additions & 4 deletions examples/r_KLU_GLU.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,19 +96,24 @@ int main(int argc, char *argv[])
ReSolve::io::updateMatrixFromFile(mat_file, A);
ReSolve::io::updateArrayFromFile(rhs_file, &rhs);
}
std::cout<<"Finished reading the matrix and rhs, size: "<<A->getNumRows()<<" x "<<A->getNumColumns()<< ", nnz: "<< A->getNnz()<< ", symmetric? "<<A->symmetric()<< ", Expanded? "<<A->expanded()<<std::endl;
// Copy matrix data to device
A->syncData(ReSolve::memory::DEVICE);

std::cout << "Finished reading the matrix and rhs, size: " << A->getNumRows() << " x "<< A->getNumColumns()
<< ", nnz: " << A->getNnz()
<< ", symmetric? " << A->symmetric()
<< ", Expanded? " << A->expanded() << std::endl;
mat_file.close();
rhs_file.close();

// Update host and device data.
if (i < 1) {
vec_rhs->update(rhs, ReSolve::memory::HOST, ReSolve::memory::HOST);
vec_rhs->setDataUpdated(ReSolve::memory::HOST);
} else {
A->copyData(ReSolve::memory::DEVICE);
vec_rhs->update(rhs, ReSolve::memory::HOST, ReSolve::memory::DEVICE);
}
std::cout<<"COO to CSR completed. Expanded NNZ: "<< A->getNnz()<<std::endl;
std::cout << "CSR matrix loaded. Expanded NNZ: " << A->getNnz() << std::endl;

//Now call direct solver
int status;
if (i < 1) {
Expand Down
99 changes: 51 additions & 48 deletions examples/r_KLU_GLU_matrix_values_update.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,57 +105,60 @@ int main(int argc, char *argv[])
//ReSolve::io::updateMatrixFromFile(mat_file, A);
ReSolve::io::updateArrayFromFile(rhs_file, &rhs);
}
std::cout<<"Finished reading the matrix and rhs, size: "<<A->getNumRows()<<" x "<<A->getNumColumns()<< ", nnz: "<< A->getNnz()<< ", symmetric? "<<A->symmetric()<< ", Expanded? "<<A->expanded()<<std::endl;
mat_file.close();
rhs_file.close();

// Update host and device data.
if (i < 1) {
vec_rhs->update(rhs, ReSolve::memory::HOST, ReSolve::memory::HOST);
vec_rhs->setDataUpdated(ReSolve::memory::HOST);
} else {
A->copyData(ReSolve::memory::DEVICE);
vec_rhs->update(rhs, ReSolve::memory::HOST, ReSolve::memory::DEVICE);
}
std::cout<<"COO to CSR completed. Expanded NNZ: "<< A->getNnz()<<std::endl;
//Now call direct solver
int status;
if (i < 1){
KLU->setup(A);
status = KLU->analyze();
std::cout<<"KLU analysis status: "<<status<<std::endl;
status = KLU->factorize();
std::cout<<"KLU factorization status: "<<status<<std::endl;
matrix_type* L = KLU->getLFactor();
matrix_type* U = KLU->getUFactor();
if (L == nullptr) {printf("ERROR");}
index_type* P = KLU->getPOrdering();
index_type* Q = KLU->getQOrdering();
GLU->setup(A, L, U, P, Q);
status = GLU->solve(vec_rhs, vec_x);
std::cout<<"GLU solve status: "<<status<<std::endl;
// status = KLU->solve(vec_rhs, vec_x);
// std::cout<<"KLU solve status: "<<status<<std::endl;
} else {
//status = KLU->refactorize();
std::cout<<"Using CUSOLVER GLU"<<std::endl;
status = GLU->refactorize();
std::cout<<"CUSOLVER GLU refactorization status: "<<status<<std::endl;
status = GLU->solve(vec_rhs, vec_x);
std::cout<<"CUSOLVER GLU solve status: "<<status<<std::endl;
}
vec_r->update(rhs, ReSolve::memory::HOST, ReSolve::memory::DEVICE);


matrix_handler->setValuesChanged(true, ReSolve::memory::DEVICE);
matrix_handler->matvec(A, vec_x, vec_r, &ONE, &MINUSONE, ReSolve::memory::DEVICE);
// Copy matrix data to device
A->syncData(ReSolve::memory::DEVICE);

std::cout << "Finished reading the matrix and rhs, size: " << A->getNumRows() << " x "<< A->getNumColumns()
<< ", nnz: " << A->getNnz()
<< ", symmetric? " << A->symmetric()
<< ", Expanded? " << A->expanded() << std::endl;
mat_file.close();
rhs_file.close();

// Update host and device data.
if (i < 1) {
vec_rhs->update(rhs, ReSolve::memory::HOST, ReSolve::memory::HOST);
} else {
vec_rhs->update(rhs, ReSolve::memory::HOST, ReSolve::memory::DEVICE);
}
std::cout << "CSR matrix loaded. Expanded NNZ: " << A->getNnz() << std::endl;

//Now call direct solver
int status;
if (i < 1){
KLU->setup(A);
status = KLU->analyze();
std::cout<<"KLU analysis status: "<<status<<std::endl;
status = KLU->factorize();
std::cout<<"KLU factorization status: "<<status<<std::endl;
matrix_type* L = KLU->getLFactor();
matrix_type* U = KLU->getUFactor();
if (L == nullptr) {printf("ERROR");}
index_type* P = KLU->getPOrdering();
index_type* Q = KLU->getQOrdering();
GLU->setup(A, L, U, P, Q);
status = GLU->solve(vec_rhs, vec_x);
std::cout<<"GLU solve status: "<<status<<std::endl;
// status = KLU->solve(vec_rhs, vec_x);
// std::cout<<"KLU solve status: "<<status<<std::endl;
} else {
//status = KLU->refactorize();
std::cout<<"Using CUSOLVER GLU"<<std::endl;
status = GLU->refactorize();
std::cout<<"CUSOLVER GLU refactorization status: "<<status<<std::endl;
status = GLU->solve(vec_rhs, vec_x);
std::cout<<"CUSOLVER GLU solve status: "<<status<<std::endl;
}
vec_r->update(rhs, ReSolve::memory::HOST, ReSolve::memory::DEVICE);

std::cout << "\t 2-Norm of the residual: "
<< std::scientific << std::setprecision(16)
<< sqrt(vector_handler->dot(vec_r, vec_r, ReSolve::memory::DEVICE)) << "\n";

matrix_handler->setValuesChanged(true, ReSolve::memory::DEVICE);
matrix_handler->matvec(A, vec_x, vec_r, &ONE, &MINUSONE, ReSolve::memory::DEVICE);

}
std::cout << "\t 2-Norm of the residual: "
<< std::scientific << std::setprecision(16)
<< sqrt(vector_handler->dot(vec_r, vec_r, ReSolve::memory::DEVICE)) << "\n";
}

//now DELETE
delete A;
Expand Down
15 changes: 11 additions & 4 deletions examples/r_KLU_cusolverrf_redo_factorization.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,19 +105,24 @@ int main(int argc, char *argv[] )
ReSolve::io::updateMatrixFromFile(mat_file, A);
ReSolve::io::updateArrayFromFile(rhs_file, &rhs);
}
std::cout<<"Finished reading the matrix and rhs, size: "<<A->getNumRows()<<" x "<<A->getNumColumns()<< ", nnz: "<< A->getNnz()<< ", symmetric? "<<A->symmetric()<< ", Expanded? "<<A->expanded()<<std::endl;
// Copy matrix data to device
A->syncData(ReSolve::memory::DEVICE);

std::cout << "Finished reading the matrix and rhs, size: " << A->getNumRows() << " x "<< A->getNumColumns()
<< ", nnz: " << A->getNnz()
<< ", symmetric? " << A->symmetric()
<< ", Expanded? " << A->expanded() << std::endl;
mat_file.close();
rhs_file.close();

// Update host and device data.
if (i < 2) {
vec_rhs->update(rhs, ReSolve::memory::HOST, ReSolve::memory::HOST);
vec_rhs->setDataUpdated(ReSolve::memory::HOST);
} else {
A->copyData(ReSolve::memory::DEVICE);
vec_rhs->update(rhs, ReSolve::memory::HOST, ReSolve::memory::DEVICE);
}
std::cout<<"COO to CSR completed. Expanded NNZ: "<< A->getNnz()<<std::endl;
std::cout << "CSR matrix loaded. Expanded NNZ: " << A->getNnz() << std::endl;

//Now call direct solver
if (i < 2) {
KLU->setup(A);
Expand All @@ -132,6 +137,8 @@ int main(int argc, char *argv[] )
U_csc = (ReSolve::matrix::Csc*) KLU->getUFactor();
ReSolve::matrix::Csr* L = new ReSolve::matrix::Csr(L_csc->getNumRows(), L_csc->getNumColumns(), L_csc->getNnz());
ReSolve::matrix::Csr* U = new ReSolve::matrix::Csr(U_csc->getNumRows(), U_csc->getNumColumns(), U_csc->getNnz());
L_csc->syncData(ReSolve::memory::DEVICE);
U_csc->syncData(ReSolve::memory::DEVICE);
matrix_handler->csc2csr(L_csc,L, ReSolve::memory::DEVICE);
matrix_handler->csc2csr(U_csc,U, ReSolve::memory::DEVICE);
if (L == nullptr) {
Expand Down
15 changes: 11 additions & 4 deletions examples/r_KLU_rf.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,19 +95,24 @@ int main(int argc, char *argv[] )
ReSolve::io::updateMatrixFromFile(mat_file, A);
ReSolve::io::updateArrayFromFile(rhs_file, &rhs);
}
std::cout<<"Finished reading the matrix and rhs, size: "<<A->getNumRows()<<" x "<<A->getNumColumns()<< ", nnz: "<< A->getNnz()<< ", symmetric? "<<A->symmetric()<< ", Expanded? "<<A->expanded()<<std::endl;
// Copy matrix data to device
A->syncData(ReSolve::memory::DEVICE);

std::cout << "Finished reading the matrix and rhs, size: " << A->getNumRows() << " x "<< A->getNumColumns()
<< ", nnz: " << A->getNnz()
<< ", symmetric? " << A->symmetric()
<< ", Expanded? " << A->expanded() << std::endl;
mat_file.close();
rhs_file.close();

// Update host and device data.
if (i < 2) {
vec_rhs->update(rhs, ReSolve::memory::HOST, ReSolve::memory::HOST);
vec_rhs->setDataUpdated(ReSolve::memory::HOST);
} else {
A->copyData(ReSolve::memory::DEVICE);
vec_rhs->update(rhs, ReSolve::memory::HOST, ReSolve::memory::DEVICE);
}
std::cout<<"COO to CSR completed. Expanded NNZ: "<< A->getNnz()<<std::endl;
std::cout << "CSR matrix loaded. Expanded NNZ: " << A->getNnz() << std::endl;

//Now call direct solver
int status;
if (i < 2){
Expand All @@ -123,6 +128,8 @@ int main(int argc, char *argv[] )
ReSolve::matrix::Csc* U_csc = (ReSolve::matrix::Csc*) KLU->getUFactor();
ReSolve::matrix::Csr* L = new ReSolve::matrix::Csr(L_csc->getNumRows(), L_csc->getNumColumns(), L_csc->getNnz());
ReSolve::matrix::Csr* U = new ReSolve::matrix::Csr(U_csc->getNumRows(), U_csc->getNumColumns(), U_csc->getNnz());
L_csc->syncData(ReSolve::memory::DEVICE);
U_csc->syncData(ReSolve::memory::DEVICE);
matrix_handler->csc2csr(L_csc,L, ReSolve::memory::DEVICE);
matrix_handler->csc2csr(U_csc,U, ReSolve::memory::DEVICE);
if (L == nullptr) {printf("ERROR");}
Expand Down
15 changes: 11 additions & 4 deletions examples/r_KLU_rf_FGMRES.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,19 +99,24 @@ int main(int argc, char *argv[])
ReSolve::io::updateMatrixFromFile(mat_file, A);
ReSolve::io::updateArrayFromFile(rhs_file, &rhs);
}
std::cout<<"Finished reading the matrix and rhs, size: "<<A->getNumRows()<<" x "<<A->getNumColumns()<< ", nnz: "<< A->getNnz()<< ", symmetric? "<<A->symmetric()<< ", Expanded? "<<A->expanded()<<std::endl;
// Copy matrix data to device
A->syncData(ReSolve::memory::DEVICE);

std::cout << "Finished reading the matrix and rhs, size: " << A->getNumRows() << " x "<< A->getNumColumns()
<< ", nnz: " << A->getNnz()
<< ", symmetric? " << A->symmetric()
<< ", Expanded? " << A->expanded() << std::endl;
mat_file.close();
rhs_file.close();

// Update host and device data.
if (i < 2) {
vec_rhs->update(rhs, ReSolve::memory::HOST, ReSolve::memory::HOST);
vec_rhs->setDataUpdated(ReSolve::memory::HOST);
} else {
A->copyData(ReSolve::memory::DEVICE);
vec_rhs->update(rhs, ReSolve::memory::HOST, ReSolve::memory::DEVICE);
}
std::cout<<"COO to CSR completed. Expanded NNZ: "<< A->getNnz()<<std::endl;
std::cout << "CSR matrix loaded. Expanded NNZ: " << A->getNnz() << std::endl;

//Now call direct solver
int status;
real_type norm_b;
Expand Down Expand Up @@ -146,6 +151,8 @@ int main(int argc, char *argv[])
ReSolve::matrix::Csc* U_csc = (ReSolve::matrix::Csc*) KLU->getUFactor();
ReSolve::matrix::Csr* L = new ReSolve::matrix::Csr(L_csc->getNumRows(), L_csc->getNumColumns(), L_csc->getNnz());
ReSolve::matrix::Csr* U = new ReSolve::matrix::Csr(U_csc->getNumRows(), U_csc->getNumColumns(), U_csc->getNnz());
L_csc->syncData(ReSolve::memory::DEVICE);
U_csc->syncData(ReSolve::memory::DEVICE);
matrix_handler->csc2csr(L_csc,L, ReSolve::memory::DEVICE);
matrix_handler->csc2csr(U_csc,U, ReSolve::memory::DEVICE);
if (L == nullptr) {
Expand Down
15 changes: 12 additions & 3 deletions examples/r_KLU_rf_FGMRES_reuse_factorization.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,13 @@ int main(int argc, char *argv[])
ReSolve::io::updateMatrixFromFile(mat_file, A);
ReSolve::io::updateArrayFromFile(rhs_file, &rhs);
}
std::cout<<"Finished reading the matrix and rhs, size: "<<A->getNumRows()<<" x "<<A->getNumColumns()<< ", nnz: "<< A->getNnz()<< ", symmetric? "<<A->symmetric()<< ", Expanded? "<<A->expanded()<<std::endl;
// Copy matrix data to device
A->syncData(ReSolve::memory::DEVICE);

std::cout << "Finished reading the matrix and rhs, size: " << A->getNumRows() << " x "<< A->getNumColumns()
<< ", nnz: " << A->getNnz()
<< ", symmetric? " << A->symmetric()
<< ", Expanded? " << A->expanded() << std::endl;
mat_file.close();
rhs_file.close();

Expand All @@ -110,10 +116,11 @@ int main(int argc, char *argv[])
vec_rhs->update(rhs, ReSolve::memory::HOST, ReSolve::memory::HOST);
vec_rhs->setDataUpdated(ReSolve::memory::HOST);
} else {
A->copyData(ReSolve::memory::DEVICE);
A->syncData(ReSolve::memory::DEVICE);
vec_rhs->update(rhs, ReSolve::memory::HOST, ReSolve::memory::DEVICE);
}
std::cout<<"COO to CSR completed. Expanded NNZ: "<< A->getNnz()<<std::endl;
std::cout << "CSR matrix loaded. Expanded NNZ: " << A->getNnz() << std::endl;

//Now call direct solver
int status;
real_type norm_b;
Expand All @@ -139,6 +146,8 @@ int main(int argc, char *argv[])
ReSolve::matrix::Csc* U_csc = (ReSolve::matrix::Csc*) KLU->getUFactor();
ReSolve::matrix::Csr* L = new ReSolve::matrix::Csr(L_csc->getNumRows(), L_csc->getNumColumns(), L_csc->getNnz());
ReSolve::matrix::Csr* U = new ReSolve::matrix::Csr(U_csc->getNumRows(), U_csc->getNumColumns(), U_csc->getNnz());
L_csc->syncData(ReSolve::memory::DEVICE);
U_csc->syncData(ReSolve::memory::DEVICE);

matrix_handler->csc2csr(L_csc,L, ReSolve::memory::DEVICE);
matrix_handler->csc2csr(U_csc,U, ReSolve::memory::DEVICE);
Expand Down
26 changes: 13 additions & 13 deletions examples/r_KLU_rocSolverRf_FGMRES.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,38 +104,38 @@ int main(int argc, char *argv[])
ReSolve::io::updateMatrixFromFile(mat_file, A);
ReSolve::io::updateArrayFromFile(rhs_file, &rhs);
}
// Copy matrix data to device
A->syncData(ReSolve::memory::DEVICE);

std::cout << "Finished reading the matrix and rhs, size: " << A->getNumRows() << " x "<< A->getNumColumns()
<< ", nnz: " << A->getNnz()
<< ", symmetric? " << A->symmetric()
<< ", Expanded? " << A->expanded() << std::endl;
mat_file.close();
rhs_file.close();
RESOLVE_RANGE_POP("Matrix Read");

// Update host and device data.
RESOLVE_RANGE_PUSH("Convert to CSR");
if (i < 2) {
vec_rhs->update(rhs, ReSolve::memory::HOST, ReSolve::memory::HOST);
vec_rhs->setDataUpdated(ReSolve::memory::HOST);
} else {
A->copyData(ReSolve::memory::DEVICE);
vec_rhs->update(rhs, ReSolve::memory::HOST, ReSolve::memory::DEVICE);
}
RESOLVE_RANGE_POP("Convert to CSR");
std::cout<<"COO to CSR completed. Expanded NNZ: "<< A->getNnz()<<std::endl;
RESOLVE_RANGE_POP("Matrix Read");
std::cout << "CSR matrix loaded. Expanded NNZ: " << A->getNnz() << std::endl;

int status;
real_type norm_b;
if (i < 2) {
RESOLVE_RANGE_PUSH("KLU");
KLU->setup(A);
matrix_handler->setValuesChanged(true, ReSolve::memory::DEVICE);
status = KLU->analyze();
std::cout<<"KLU analysis status: "<<status<<std::endl;
std::cout << "KLU analysis status: " << status << std::endl;
status = KLU->factorize();
std::cout<<"KLU factorization status: "<<status<<std::endl;
std::cout << "KLU factorization status: " << status << std::endl;

status = KLU->solve(vec_rhs, vec_x);
std::cout<<"KLU solve status: "<<status<<std::endl;
std::cout << "KLU solve status: " << status << std::endl;
vec_r->update(rhs, ReSolve::memory::HOST, ReSolve::memory::DEVICE);
norm_b = vector_handler->dot(vec_r, vec_r, ReSolve::memory::DEVICE);
norm_b = sqrt(norm_b);
Expand All @@ -155,18 +155,18 @@ int main(int argc, char *argv[])
Rf->setSolveMode(1);
Rf->setup(A, L, U, P, Q, vec_rhs);
Rf->refactorize();
std::cout<<"about to set FGMRES" <<std::endl;
std::cout << "About to set FGMRES ..." << std::endl;
FGMRES->setup(A);
}
RESOLVE_RANGE_POP("KLU");
} else {
RESOLVE_RANGE_PUSH("RocSolver");
//status = KLU->refactorize();
std::cout<<"Using ROCSOLVER RF"<<std::endl;
std::cout << "Using ROCSOLVER RF" << std::endl;
status = Rf->refactorize();
std::cout<<"ROCSOLVER RF refactorization status: "<<status<<std::endl;
std::cout << "ROCSOLVER RF refactorization status: " << status << std::endl;
status = Rf->solve(vec_rhs, vec_x);
std::cout<<"ROCSOLVER RF solve status: "<<status<<std::endl;
std::cout << "ROCSOLVER RF solve status: " << status << std::endl;
vec_r->update(rhs, ReSolve::memory::HOST, ReSolve::memory::DEVICE);
norm_b = vector_handler->dot(vec_r, vec_r, ReSolve::memory::DEVICE);
norm_b = sqrt(norm_b);
Expand Down
Loading

0 comments on commit e5973c8

Please sign in to comment.