Clean up CommMpi and CommSocket (useless Barrier) - The driver can now

automatically allocate and create a DSM buffer (server side) if the
environment variable H5FD_DSM_SERVER_CONFIG_PATH is set to the path of the
server config file - This mode is not fully tested yet

Author: soumagne

H5FDdsm.cxx

@@ -462,9 +462,13 @@ H5Pset_fapl_dsm(hid_t fapl_id, MPI_Comm dsmComm, void *dsmBuffer)
       dsmManagerSingleton->CreateDSM();
       if (dsmManagerSingleton->GetDsmIsServer()) {
         dsmManagerSingleton->PublishDSM();
+        while (!dsmManagerSingleton->GetDSMHandle()->GetIsConnected()) {
+          // Spin
+        }
       } else {
         dsmManagerSingleton->ConnectDSM();
       }
+      dsmManagerSingleton->GetDSMHandle()->SetIsAutoAllocated(true);
     }
     fa.buffer = dsmManagerSingleton->GetDSMHandle();
   }
@@ -626,6 +630,10 @@ H5FD_dsm_open(const char *name, unsigned UNUSED flags, hid_t fapl_id, haddr_t ma
         file->DsmBuffer->ClearStorage();
         file->DsmBuffer->GetComm()->Barrier();
       } else {
+        // Check that the DSM ready for update flag is set
+        while (!file->DsmBuffer->GetIsUpdateReady() && file->DsmBuffer->GetIsAutoAllocated()) {
+          // Spin
+        }
         // For now a file in a DSM Buffer which is not created is considered as open in read-only
         PRINT_INFO("SetIsReadOnly(true)");
         file->DsmBuffer->SetIsReadOnly(true);
@@ -787,6 +795,11 @@ H5FD_dsm_close(H5FD_t *_file)
       file->DsmBuffer->RequestPipelineUpdate();
     }
   } else {
+    if (file->DsmBuffer->GetIsUpdateReady() && file->DsmBuffer->GetIsAutoAllocated()) {
+      // file->DsmBuffer->GetComm()->Barrier();
+      file->DsmBuffer->SetIsUpdateReady(false);
+      file->DsmBuffer->RequestRemoteChannel();
+    }
     PRINT_INFO("SetIsReadOnly(false)");
     file->DsmBuffer->SetIsReadOnly(false);
   }

H5FDdsmBuffer.cxx

@@ -79,6 +79,7 @@ H5FDdsmBuffer::H5FDdsmBuffer() {
     H5FDdsmInt64 i;
     this->ThreadDsmReady = 0;
     this->DataPointer = 0;
+    this->IsAutoAllocated = false;
     this->IsServer = true;
     this->IsConnected = false;
     this->IsUpdateReady = false;
@@ -610,7 +611,6 @@ H5FDdsmBuffer::RequestPipelineUpdate() {
     H5FDdsmDebug("Send request pipeline update to " << i);
     this->SendCommandHeader(H5FD_DSM_PIPELINE_UPDATE, i, 0, 0);
   }
-
   this->Comm->Barrier();
   return(H5FD_DSM_SUCCESS);
 }

H5FDdsmBuffer.h

@@ -92,6 +92,10 @@ class H5FDdsm_EXPORT H5FDdsmBuffer : public H5FDdsmDriver {
     H5FDdsmGetValueMacro(IsUpdateReady, H5FDdsmBoolean);
     H5FDdsmSetValueMacro(IsUpdateReady, H5FDdsmBoolean);
 
+    // Is the DSMBuffer auto allocated within the driver or not
+    H5FDdsmGetValueMacro(IsAutoAllocated, bool);
+    H5FDdsmSetValueMacro(IsAutoAllocated, bool);
+
     // Is the DSMBuffer in server or client mode
     H5FDdsmGetValueMacro(IsServer, bool);
     H5FDdsmSetValueMacro(IsServer, bool);
@@ -132,6 +136,7 @@ class H5FDdsm_EXPORT H5FDdsmBuffer : public H5FDdsmDriver {
     volatile H5FDdsmInt32   ThreadDsmReady;
     volatile H5FDdsmBoolean IsConnected;
     volatile H5FDdsmBoolean IsUpdateReady;
+    bool                    IsAutoAllocated;
     bool                    IsServer;
     bool                    IsReadOnly;
     H5FDdsmInt32            ServiceThreadUseCopy;

H5FDdsmCommMpi.cxx

@@ -192,7 +192,6 @@ H5FDdsmCommMpi::OpenPort() {
   if (this->Id == 0) {
     MPI_Open_port(MPI_INFO_NULL, this->DsmMasterHostName);
   }
-  this->Barrier();
   return(H5FD_DSM_SUCCESS);
 }
 
@@ -203,7 +202,6 @@ H5FDdsmCommMpi::ClosePort() {
   if (this->Id == 0) {
     MPI_Close_port(this->DsmMasterHostName);
   }
-  this->Barrier();
   return(H5FD_DSM_SUCCESS);
 }
 
@@ -264,7 +262,7 @@ H5FDdsmInt32
 H5FDdsmCommMpi::RemoteCommDisconnect() {
 
   if(H5FDdsmComm::RemoteCommDisconnect() != H5FD_DSM_SUCCESS) return(H5FD_DSM_FAIL);
-  this->Barrier();
+
   if (this->InterComm) {
     MPI_Comm_disconnect(&this->InterComm);
     this->InterComm = MPI_COMM_NULL;

H5FDdsmCommSocket.cxx

@@ -233,7 +233,6 @@ H5FDdsmCommSocket::OpenPort()
     strcpy(this->DsmMasterHostName, this->DsmMasterSocket->GetHostName());
     this->DsmMasterSocket->Listen();
   }
-  this->Barrier();
 
   return(H5FD_DSM_SUCCESS);
 }
@@ -250,7 +249,6 @@ H5FDdsmCommSocket::ClosePort()
       return(H5FD_DSM_FAIL);
     }
   }
-  this->Barrier();
 
   return(H5FD_DSM_SUCCESS);
 }
@@ -326,7 +324,7 @@ H5FDdsmInt32
 H5FDdsmCommSocket::RemoteCommDisconnect()
 {
   if (H5FDdsmComm::RemoteCommDisconnect() != H5FD_DSM_SUCCESS) return(H5FD_DSM_FAIL);
-  this->Barrier();
+
   for (int i=0; i<H5FD_DSM_MAX_SOCKET; i++) {
     if (this->InterComm[i]) delete this->InterComm[i];
     this->InterComm[i] = NULL;

H5FDdsmManager.cxx

@@ -85,7 +85,7 @@ int H5FDdsmManager::GetAcceptedConnection()
 {
   int ret = 0;
   if (this->DSMBuffer) {
-    if (this->DSMBuffer->GetIsConnected()) return 1;
+    if (this->DSMBuffer->GetIsConnected()) ret = 1;
   }
   return ret;
 }
@@ -94,7 +94,7 @@ int H5FDdsmManager::GetDsmUpdateReady()
 {
   int ret = 0;
   if (this->DSMBuffer) {
-    if (this->DSMBuffer->GetIsUpdateReady()) return 1;
+    if (this->DSMBuffer->GetIsUpdateReady()) ret = 1;
   }
   return ret;
 }
@@ -356,12 +356,12 @@ void H5FDdsmManager::PublishDSM()
     std::string fullDsmConfigFilePath;
     const char *dsmEnvPath = getenv("H5FD_DSM_CONFIG_PATH");
     if (!dsmEnvPath) dsmEnvPath = getenv("HOME");
-    if (dsmEnvPath /*&& !this->GetDsmConfigFilePath()*/) {
+    if (dsmEnvPath) {
       this->SetDsmConfigFilePath(dsmEnvPath);
     }
     if (this->GetDsmConfigFilePath()) {
       fullDsmConfigFilePath = std::string(this->GetDsmConfigFilePath()) +
-          std::string("/.dsm_config");
+          std::string("/.dsm_client_config");
       dsmConfigFile.Create(fullDsmConfigFilePath);
       dsmConfigFile.AddSection("Comm", fullDsmConfigFilePath);
 
@@ -485,16 +485,21 @@ bool H5FDdsmManager::ReadDSMConfigFile()
 {
   H5FDdsmIniFile config;
   std::string configPath;
+  std::string mode = "client";
   const char *dsmEnvPath = getenv("H5FD_DSM_CONFIG_PATH");
-  if (!dsmEnvPath) dsmEnvPath = getenv("HOME");
-  if (dsmEnvPath) {
-    configPath = std::string(dsmEnvPath) + std::string("/.dsm_config");
+  const char *dsmServerEnvPath = getenv("H5FD_DSM_SERVER_CONFIG_PATH");
+
+  if (dsmServerEnvPath) {
+    mode = "server";
+    configPath = std::string(dsmServerEnvPath) + std::string("/.dsm_server_config");
+  } else {
+    if (!dsmEnvPath) dsmEnvPath = getenv("HOME");
+    configPath = std::string(dsmEnvPath) + std::string("/.dsm_client_config");
   }
   if (FileExists(configPath.c_str())) {
     if (this->UpdatePiece == 0) {
       std::cout << "Reading from " << configPath.c_str() << std::endl;
     }
-    std::string mode = config.GetValue("DSM_INIT_MODE",   "General", configPath);
     std::string size = config.GetValue("DSM_INIT_SIZE",   "General", configPath);
 
     std::string comm = config.GetValue("DSM_COMM_SYSTEM", "Comm", configPath);
@@ -512,11 +517,11 @@ bool H5FDdsmManager::ReadDSMConfigFile()
     // Comm settings
     if (comm == "socket") {
       this->SetDsmCommType(H5FD_DSM_COMM_SOCKET);
-      if (mode != "server") this->SetServerPort(atoi(port.c_str()));
+      this->SetServerPort(atoi(port.c_str()));
     } else if (comm == "mpi") {
       this->SetDsmCommType(H5FD_DSM_COMM_MPI);
     }
-    if (mode != "server") this->SetServerHostName(host.c_str());
+    this->SetServerHostName(host.c_str());
     return true;
   }
   return false;