Merge pull request #546 from lukemartinlogan/master

Add additional configuration to mdm

adapter/real_api.h

@@ -34,8 +34,6 @@ struct RealApiIter {
 };
 
 struct RealApi {
-  void *real_lib_next_;     /**< TODO(llogan): remove */
-  void *real_lib_default_;  /**< TODO(llogan): remove */
   void *real_lib_;
   bool is_intercepted_;
 
@@ -45,25 +43,27 @@ struct RealApi {
     auto exists = dlsym(lib, iter->symbol_name_);
     void *is_intercepted =
         (void*)dlsym(lib, iter->is_intercepted_);
-    if (!is_intercepted && exists) {
+    if (!is_intercepted && exists && !iter->lib_path_) {
       iter->lib_path_ = info->dlpi_name;
     }
     return 0;
   }
 
   RealApi(const char *symbol_name,
-          const char *is_intercepted)
-      : real_lib_next_(nullptr), real_lib_default_(nullptr) {
+          const char *is_intercepted) {
     RealApiIter iter(symbol_name, is_intercepted);
     dl_iterate_phdr(callback, (void*)&iter);
     if (iter.lib_path_) {
       real_lib_ = dlopen(iter.lib_path_, RTLD_GLOBAL | RTLD_NOW);
-      real_lib_next_ = real_lib_;
-      real_lib_default_ = real_lib_;
     }
     void *is_intercepted_ptr = (void*)dlsym(RTLD_DEFAULT,
                                              is_intercepted);
     is_intercepted_ = is_intercepted_ptr != nullptr;
+    /* if (is_intercepted_) {
+      real_lib_ = RTLD_NEXT;
+    } else {
+      real_lib_ = RTLD_DEFAULT;
+    }*/
   }
 };
 

ci/hermes/packages/hermes/package.py

@@ -9,6 +9,7 @@ class Hermes(CMakePackage):
     version('dev-priv', git='https://github.com/lukemartinlogan/hermes.git', branch='dev')
     variant('vfd', default=False, description='Enable HDF5 VFD')
     variant('ares', default=False, description='Enable full libfabric install')
+    variant('debug', default=False, description='Enable debug mode')
     depends_on('[email protected]')
     depends_on('cereal')
     depends_on('[email protected]')
@@ -26,19 +27,21 @@ def cmake_args(self):
                 '-DHERMES_RPC_THALLIUM=ON',
                 '-DHERMES_INSTALL_TESTS=ON',
                 '-DBUILD_TESTING=ON']
+        if '+debug' in self.spec:
+            args.append('-DCMAKE_BUILD_TYPE=Debug')
         if '+vfd' in self.spec:
             args.append(self.define('HERMES_ENABLE_VFD', 'ON'))
         return args
 
     def set_include(self, env, path):
         env.append_flags('CFLAGS', '-I{}'.format(path))
         env.append_flags('CXXFLAGS', '-I{}'.format(path))
-        env.append_flags('CPATH', '{}'.format(path))
-        env.append_flags('CMAKE_PREFIX_PATH', '-I{}'.format(path))
+        env.prepend_path('CPATH', '{}'.format(path))
+        env.prepend_path('CMAKE_PREFIX_PATH', '{}'.format(path))
 
     def set_lib(self, env, path):
         env.prepend_path('LD_LIBRARY_PATH', path)
-        env.append_flags('LIBRARY_PATH', '{}'.format(path))
+        env.prepend_path('LIBRARY_PATH', '{}'.format(path))
         env.append_flags('LDFLAGS', '-L{}'.format(path))
 
     def set_flags(self, env):

config/hermes_server_default.yaml

@@ -153,7 +153,15 @@ prefetch:
   epoch_ms: 50
   is_mpi: false
 
-# The shared memory prefix for the hermes shared memory segment. A user name
+### Define mdm properties
+mdm:
+  # This represents the number of blobs and buckets before collisions start
+  # to happen in the unordered_map tables.
+  est_blob_count: 100000
+  est_bucket_count: 100000
+  est_num_traits: 256
+
+# The shared memory prefix for the hermes shared memory segment. A username
 # will be automatically appended.
 shmem_name: "/hermes_shm_"
 

src/api/hermes.cc

@@ -104,7 +104,7 @@ void Hermes::LoadClientConfig(std::string config_path) {
   if (config_path.empty()) {
     config_path = GetEnvSafe(kHermesClientConf);
   }
-  // HILOG(kInfo, "Loading client configuration: {}", config_path)
+  HILOG(kDebug, "Loading client configuration: {}", config_path)
   client_config_.LoadFromFile(config_path);
 }
 

src/config_server.cc

@@ -167,6 +167,13 @@ void ServerConfig::ParseTraitInfo(YAML::Node yaml_conf) {
   }
 }
 
+/** parse prefetch information from YAML config */
+void ServerConfig::ParseMdmInfo(YAML::Node yaml_conf) {
+  mdm_.num_blobs_ = yaml_conf["est_blob_count"].as<size_t>();
+  mdm_.num_bkts_ = yaml_conf["est_blob_count"].as<size_t>();
+  mdm_.num_traits_ = yaml_conf["est_num_traits"].as<size_t>();
+}
+
 /** parse the YAML node */
 void ServerConfig::ParseYAML(YAML::Node &yaml_conf) {
   if (yaml_conf["devices"]) {
@@ -182,11 +189,14 @@ void ServerConfig::ParseYAML(YAML::Node &yaml_conf) {
     ParseBorgInfo(yaml_conf["buffer_organizer"]);
   }
   if (yaml_conf["tracing"]) {
-    ParsePrefetchInfo(yaml_conf["tracing"]);
+    ParseTracingInfo(yaml_conf["tracing"]);
   }
   if (yaml_conf["prefetch"]) {
     ParsePrefetchInfo(yaml_conf["prefetch"]);
   }
+  if (yaml_conf["mdm"]) {
+    ParseMdmInfo(yaml_conf["mdm"]);
+  }
   if (yaml_conf["system_view_state_update_interval_ms"]) {
     system_view_state_update_interval_ms =
         yaml_conf["system_view_state_update_interval_ms"].as<int>();

src/config_server.h

@@ -247,6 +247,18 @@ struct TracingInfo {
   std::string output_;
 };
 
+/** MDM information */
+struct MdmInfo {
+  /** Number of buckets in mdm bucket map before collisions */
+  size_t num_bkts_;
+
+  /** Number of blobs in mdm blob map before collisions */
+  size_t num_blobs_;
+
+  /** Number of traits in mdm trait map before collisions */
+  size_t num_traits_;
+};
+
 /**
  * System configuration for Hermes
  */
@@ -270,6 +282,9 @@ class ServerConfig : public BaseConfig {
   /** Prefetcher information */
   PrefetchInfo prefetcher_;
 
+  /** Metadata Manager information */
+  MdmInfo mdm_;
+
   /** Trait repo information */
   std::vector<std::string> trait_paths_;
 
@@ -298,6 +313,7 @@ class ServerConfig : public BaseConfig {
   void ParsePrefetchInfo(YAML::Node yaml_conf);
   void ParseTracingInfo(YAML::Node yaml_conf);
   void ParseTraitInfo(YAML::Node yaml_conf);
+  void ParseMdmInfo(YAML::Node yaml_conf);
 };
 
 }  // namespace hermes::config

src/config_server_default.h

@@ -156,7 +156,15 @@ const char* kServerDefaultConfigStr =
 "  epoch_ms: 50\n"
 "  is_mpi: false\n"
 "\n"
-"# The shared memory prefix for the hermes shared memory segment. A user name\n"
+"### Define mdm properties\n"
+"mdm:\n"
+"  # This represents the number of blobs and buckets before collisions start\n"
+"  # to happen in the unordered_map tables.\n"
+"  est_blob_count: 100000\n"
+"  est_bucket_count: 100000\n"
+"  est_num_traits: 256\n"
+"\n"
+"# The shared memory prefix for the hermes shared memory segment. A username\n"
 "# will be automatically appended.\n"
 "shmem_name: \"/hermes_shm_\"\n"
 "\n"

src/metadata_manager.cc

@@ -35,12 +35,12 @@ void MetadataManager::shm_init(hipc::ShmArchive<MetadataManagerShm> &header,
   header_->node_id_ = rpc_->node_id_;
 
   // Create the metadata maps
-  HSHM_MAKE_AR(header_->blob_id_map_, alloc, 32000000)
-  HSHM_MAKE_AR(header_->blob_map_, alloc, 32000000)
-  HSHM_MAKE_AR(header_->tag_id_map_, alloc, 32000000)
-  HSHM_MAKE_AR(header_->tag_map_, alloc, 32000000)
-  HSHM_MAKE_AR(header_->trait_id_map_, alloc, 256)
-  HSHM_MAKE_AR(header_->trait_map_, alloc, 256)
+  HSHM_MAKE_AR(header_->blob_id_map_, alloc, config->mdm_.num_blobs_)
+  HSHM_MAKE_AR(header_->blob_map_, alloc, config->mdm_.num_blobs_)
+  HSHM_MAKE_AR(header_->tag_id_map_, alloc, config->mdm_.num_bkts_)
+  HSHM_MAKE_AR(header_->tag_map_, alloc, config->mdm_.num_bkts_)
+  HSHM_MAKE_AR(header_->trait_id_map_, alloc, config->mdm_.num_traits_)
+  HSHM_MAKE_AR(header_->trait_map_, alloc, config->mdm_.num_traits_)
 
   // Create the DeviceInfo vector
   HSHM_MAKE_AR(header_->devices_, alloc, *config->devices_)