[sos-collector] Ability to use 'host group' files

This change adds the ability to use and write JSON-formatted files to
use as 'host groups' that define cluster-type, master, and a list of
nodes to use for collection.

Using the new --group option, sos-collector will look to
/var/lib/sos-collector for a host group with a matching name to load and
use for execution. Note that users may still use the --nodes option when
using --group - the two lists of nodes will be merged.

Using the new --save-group option will write out a host group file to
/var/lib/sos-collector/ with the same name as the value given to this
option. The values written to the host group file will be what was
determined for this run of sos-collector; meaning the cluster-type,
master node, and node list enumerated from the cluster. This means that
regex strings will not be saved to group files, but the results of those
regexes will be saved.

Fixes: #20

Signed-off-by: Jake Hunsaker <[email protected]>

Author: TurboTurtle

man/en/sos-collector.1

@@ -12,6 +12,8 @@ sos-collector \- Collect sosreports from multiple (cluster) nodes
     [\-\-case\-id CASE_ID]
     [\-\-cluster\-type CLUSTER_TYPE]
     [\-e ENABLE_PLUGINS]
+    [\-\-group GROUP]
+    [\-\-save\-group GROUP]
     [\-\-insecure-sudo]
     [\-k PLUGIN_OPTION]
     [\-\-label LABEL]
@@ -104,6 +106,32 @@ Sosreport option. Use this to enable a plugin that would otherwise not be run.
 
 This option supports providing a comma-delimited list of plugins.
 .TP
+\fB\-\-group\fR GROUP
+Specify an existing host group definition to use.
+
+Host groups are pre-defined settings for the cluster-type, master, and nodes options
+saved in JSON-formatted files under /var/lib/sos-collector/<GROUP>.
+
+If cluster_type and/or master are set in the group, sos-collector behaves as if
+these values were specified on the command-line.
+
+If nodes is defined, sos-collector \fBextends\fR the \fB\-\-nodes\fR option, if set,
+with the nodes or regexes listed in the group.
+
+Note that sos-collector will only write group definitions to /var/lib/sos-collector/
+however the GROUP value may be a filename for any group definitions that exist outside
+of the default location. If you are manually writing these files, use the value \fBnull\fR
+when a python NoneType is expected. Caveat: use \fBstring\fR 'none' if setting cluster_type
+to none.
+.TP
+\fB\-\-save\-group\fR GROUP
+Save the results of this run of sos-collector to a host group definition.
+
+sos-colllector will write a JSON-formatted file with name GROUP to /var/lib/sos-collector/
+with the settings for cluster-type, master, and the node list as discovered by cluster enumeration.
+Note that this means regexes are not directly saved to host groups, but the results of matching against
+those regexes are.
+.TP
 \fB\-\-insecure-sudo\fR
 Use this option when connecting as a non-root user that has passwordless sudo
 configured.

sos-collector

@@ -52,6 +52,10 @@ if __name__ == '__main__':
                         help="chroot executed commands to SYSROOT")
     parser.add_argument('-e', '--enable-plugins', action="append",
                         help='Enable specific plugins for sosreport')
+    parser.add_argument('--group', default=None,
+                        help='Use a predefined group JSON file')
+    parser.add_argument('--save-group', default='',
+                        help='Save the resulting node list to a group')
     parser.add_argument('--image', help=('Specify the container image to use'
                                          ' for atomic hosts. Defaults to '
                                          'the rhel7/support-tools image'

soscollector/configuration.py

@@ -47,7 +47,7 @@ def set_defaults(self):
         self['no_local'] = False
         self['tmp_dir'] = None
         self['out_dir'] = '/var/tmp/'
-        self['nodes'] = None
+        self['nodes'] = []
         self['debug'] = False
         self['tmp_dir_created'] = False
         self['cluster_type'] = None
@@ -87,6 +87,8 @@ def set_defaults(self):
         self['log_size'] = 0
         self['host_types'] = []
         self['password_per_node'] = False
+        self['group'] = None
+        self['save_group'] = ''
 
     def parse_node_strings(self):
         '''

soscollector/sos_collector.py

@@ -15,6 +15,7 @@
 
 import fnmatch
 import inspect
+import json
 import logging
 import os
 import random
@@ -38,6 +39,8 @@
 from soscollector import __version__
 from soscollector.exceptions import ControlPersistUnsupportedException
 
+COLLECTOR_LIB_DIR = '/var/lib/sos-collector'
+
 
 class SosCollector():
     '''Main sos-collector class'''
@@ -287,6 +290,60 @@ def _fmt_msg(self, msg):
             _fmt = _fmt + fill(line, width, replace_whitespace=False) + '\n'
         return _fmt
 
+    def _load_group_config(self):
+        '''
+        Attempts to load the host group specified on the command line.
+        Host groups are defined via JSON files, typically saved under
+        /var/lib/sos-collector/, although users can specify a full filepath
+        on the commandline to point to one existing anywhere on the system
+
+        Host groups define a list of nodes and/or regexes and optionally the
+        master and cluster-type options.
+        '''
+        if os.path.exists(self.config['group']):
+            fname = self.config['group']
+        elif os.path.exists(
+                os.path.join(COLLECTOR_LIB_DIR, self.config['group'])
+             ):
+                fname = os.path.join(COLLECTOR_LIB_DIR, self.config['group'])
+        else:
+            raise OSError('Group not found')
+
+        self.log_debug("Loading host group %s" % fname)
+
+        with open(fname, 'r') as hf:
+            _group = json.load(hf)
+            for key in ['master', 'cluster_type']:
+                if _group[key]:
+                    self.log_debug("Setting option '%s' to '%s' per host group"
+                                   % (key, _group[key]))
+                    self.config[key] = _group[key]
+            if _group['nodes']:
+                self.log_debug("Adding %s to node list" % _group['nodes'])
+                self.config['nodes'].extend(_group['nodes'])
+
+    def write_host_group(self):
+        '''
+        Saves the results of this run of sos-collector to a host group file
+        on the system so it can be used later on.
+
+        The host group will save the options master, cluster_type, and nodes
+        as determined by sos-collector prior to execution of sosreports.
+        '''
+        cfg = {
+            'name': self.config['save_group'],
+            'master': self.config['master'],
+            'cluster_type': self.config['cluster_type'],
+            'nodes': [n for n in self.node_list]
+        }
+        if not os.path.isdir(COLLECTOR_LIB_DIR):
+            raise OSError("%s no such directory" % COLLECTOR_LIB_DIR)
+        fname = COLLECTOR_LIB_DIR + '/' + cfg['name']
+        with open(fname, 'w') as hf:
+            json.dump(cfg, hf)
+        os.chmod(fname, 0o644)
+        return fname
+
     def prep(self):
         '''Based on configuration, performs setup for collection'''
         disclaimer = ("""\
@@ -349,6 +406,13 @@ def prep(self):
                               ' Ignoring request to change user on node')
                 self.config['become_root'] = False
 
+        if self.config['group']:
+            try:
+                self._load_group_config()
+            except Exception as err:
+                self.log_error("Could not load specified group %s: %s"
+                               % (self.config['group'], err))
+
         if self.config['master']:
             self.connect_to_master()
             self.config['no_local'] = True
@@ -380,6 +444,13 @@ def prep(self):
             self.config['cluster'].setup()
             self.config['cluster'].modify_sos_cmd()
         self.get_nodes()
+        if self.config['save_group']:
+            gname = self.config['save_group']
+            try:
+                fname = self.write_host_group()
+                self.log_info("Wrote group '%s' to %s" % (gname, fname))
+            except Exception as err:
+                self.log_error("Could not save group %s: %s" % (gname, err))
         self.intro()
         self.configure_sos_cmd()