[sos] Add 'upload' component to upload existing reports and files

This commit marks the beginning of the addition of a new 'upload'
component for sos, which can be used to upload already created
sos reports, collects, or other files like logs or vmcores to
a policy defined location.

The user needs to specify a file location, and can make use of any
of the options that exist nowadays for the --upload option.

This first commit includes:
- The initial framework for the 'upload' component.
- The new man page for 'sos upload'.
- The code in the component 'help' to show information about
the component.
- The code in sos/__init__.py to deal with the component.
- And modifications to setup.py to build the man pages.

Related: RHEL-23032, SUPDEV-138, CLIOT-481

Signed-off-by: Jose Castillo <[email protected]>

man/en/sos-upload.1

@@ -0,0 +1,111 @@
+.TH UPLOAD 1 "July 2024"
+
+.SH NAME
+sos_upload \- Upload files like previously generated sos reports or logs to a policy specific location
+.SH SYNOPSIS
+.B sos upload FILE [options]
+ [--case-id id]\fR
+ [--upload-url url]\fR
+ [--upload-user user]\fR
+ [--upload-pass pass]\fR
+ [--upload-directory dir]\fR
+ [--upload-method]\fR
+ [--upload-no-ssl-verify]\fR
+ [--upload-protocol protocol]\fR
+
+.PP
+.SH DESCRIPTION
+upload is an sos subcommand to upload sos reports, logs, vmcores, or other files to a policy defined remote location, or an user defined one.
+.SH REQUIRED ARGUMENTS
+.B FILE
+.TP
+The path to the archive that is to be uploaded.
+.SH OPTIONS
+.TP
+.B \--case-id NUMBER
+Specify a case identifier to associate with the archive.
+Identifiers may include alphanumeric characters, commas and periods ('.').
+.TP
+.B \--upload-url URL
+If a vendor does not provide a default upload location, or if you would like to upload
+the archive to a different location, specify the address here.
+
+A support protocol MUST be specified in this URL. Currently uploading is supported
+for HTTPS, SFTP, and FTP protocols.
+
+If your destination server listens on a non-standard port, specify the listening
+port in the URL.
+.TP 
+.B \-\-upload-user USER
+If a vendor does not provide a default user for uploading, specify the username here.
+
+If --batch is used and this option is omitted, no username will
+be collected and thus uploads will fail if no vendor default is set.
+
+You also have the option of providing this value via the SOSUPLOADUSER environment
+variable. If this variable is set, then no username prompt will occur and --batch
+may be used provided all other required values (case number, upload password)
+are provided. 
+.TP 
+.B \-\-upload-pass PASS 
+Specify the password to use for authentication with the destination server.
+
+If this option is omitted and upload is requested, you will be prompted for one.
+
+If --batch is used, this prompt will not occur, so any uploads are likely to fail unless
+this option is used.
+
+Note that this will result in the plaintext string appearing in `ps` output that may
+be collected by sos and be in the archive. If a password must be provided by you
+for uploading, it is strongly recommended to not use --batch and enter the password
+when prompted rather than using this option.
+
+You also have the option of providing this value via the SOSUPLOADPASSWORD environment
+variable. If this variable is set, then no password prompt will occur and --batch may
+be used provided all other required values (case number, upload user) are provided.
+.TP 
+.B \--upload-directory DIR
+Specify a directory to upload to, if one is not specified by a vendor default location
+or if your destination server does not allow writes to '/'.
+.TP 
+.B \--upload-method METHOD
+Specify the HTTP method to use for uploading to the provided --upload-url. Valid
+values are 'auto' (default), 'put', or 'post'. The use of 'auto' will default to
+the method required by the policy-default upload location, if one exists.
+
+This option has no effect on upload protocols other than HTTPS.
+.TP
+.B \--upload-no-ssl-verify
+Disable SSL verification for HTTPS uploads. This may be used to allow uploading
+to locations that have self-signed certificates, or certificates that are otherwise
+untrusted by the local system.
+
+Default behavior is to perform SSL verification against all upload locations.
+.TP 
+.B \--upload-protocol PROTO
+Manually specify the protocol to use for uploading to the target \fBupload-url\fR.
+
+Normally this is determined via the upload address, assuming that the protocol is part
+of the address provided, e.g. 'https://example.com'. By using this option, sos will skip
+the protocol check and use the method defined for the specified PROTO.
+
+For RHEL systems, setting this option to \fBsftp\fR will skip the initial attempt to
+upload to the Red Hat Customer Portal, and only attempt an upload to Red Hat's SFTP server,
+which is typically used as a fallback target.
+
+Valid values for PROTO are: 'auto' (default), 'https', 'ftp', 'sftp'.
+
+
+.SH SEE ALSO
+.BR sos (1)
+.BR sos-report (1)
+.BR sos-clean (1)
+.BR sos.conf (5)
+.BR sos-collect (1)
+
+.SH MAINTAINER
+.nf
+Maintained on GitHub at https://github.com/sosreport/sos
+.fi
+.SH AUTHORS & CONTRIBUTORS
+See \fBAUTHORS\fR file in the package documentation.

setup.py

@@ -24,7 +24,8 @@
  ('share/man/man1', ['man/en/sosreport.1', 'man/en/sos-report.1',
  'man/en/sos.1', 'man/en/sos-collect.1',
  'man/en/sos-collector.1', 'man/en/sos-clean.1',
- 'man/en/sos-mask.1', 'man/en/sos-help.1']),
+ 'man/en/sos-mask.1', 'man/en/sos-help.1',
+ 'man/en/sos-upload.1']),
  ('share/man/man5', ['man/en/sos.conf.5']),
  ('share/licenses/sos', ['LICENSE']),
  ('share/doc/sos', ['AUTHORS', 'README.md']),

sos/__init__.py

@@ -54,10 +54,12 @@ def __init__(self, args):
  import sos.report
  import sos.cleaner
  import sos.help
+ import sos.upload
  self._components = {
  'report': (sos.report.SoSReport, ['rep']),
  'clean': (sos.cleaner.SoSCleaner, ['cleaner', 'mask']),
- 'help': (sos.help.SoSHelper, [])
+ 'help': (sos.help.SoSHelper, []),
+ 'upload': (sos.upload.SoSUpload, [])
  }
  # some distros do not want pexpect as a default dep, so try to load
  # collector here, and if it fails add an entry that implies it is at

sos/help/__init__.py

@@ -100,7 +100,8 @@ def get_obj_for_topic(self):
  'collector': 'SoSCollector',
  'collector.transports': 'RemoteTransport',
  'collector.clusters': 'Cluster',
- 'policies': 'Policy'
+ 'policies': 'Policy',
+ 'upload': 'SoSUpload'
  }
 
  cls = None
@@ -206,7 +207,8 @@ def display_self_help(self):
  'report.plugins.$plugin': 'Information on a specific $plugin',
  'clean': 'Detailed help on the clean command',
  'collect': 'Detailed help on the collect command',
- 'policies': 'How sos operates on different distributions'
+ 'policies': 'How sos operates on different distributions',
+ 'upload': 'Detailed help on the upload command'
  }
 
  for sect, value in sections.items():

sos/policies/distros/redhat.py

@@ -298,7 +298,7 @@ def get_upload_url(self):
  self.ui_log.info("No case id provided, uploading to SFTP")
  return RH_SFTP_HOST
  rh_case_api = "/support/v1/cases/%s/attachments"
- return RH_API_HOST + rh_case_api % self.case_id
+ return RH_API_HOST + rh_case_api % self.commons['cmdlineopts'].case_id
 
  def _get_upload_https_auth(self):
  str_auth = f"Bearer {self._device_token}"
@@ -441,16 +441,15 @@ def check_file_too_big(self, archive):
  f"{convert_bytes(self._max_size_request)} "
  " via sos http upload. \n")
  )
- return RH_SFTP_HOST
- return RH_API_HOST
+ self.upload_url = RH_SFTP_HOST
 
  def upload_archive(self, archive):
  """Override the base upload_archive to provide for automatic failover
  from RHCP failures to the public RH dropbox
  """
  try:
  if self.get_upload_url().startswith(RH_API_HOST):
- self.upload_url = self.check_file_too_big(archive)
+ self.check_file_too_big(archive)
  uploaded = super().upload_archive(archive)
  except Exception as e:
  uploaded = False

sos/upload/__init__.py

@@ -0,0 +1,161 @@
+# Copyright 2024 Red Hat, Inc. Jose Castillo <[email protected]>
+
+# This file is part of the sos project: https://github.com/sosreport/sos
+#
+# This copyrighted material is made available to anyone wishing to use,
+# modify, copy, or redistribute it subject to the terms and conditions of
+# version 2 of the GNU General Public License.
+#
+# See the LICENSE file in the source distribution for further information.
+
+import os
+import sys
+from textwrap import fill
+from sos.component import SoSComponent
+from sos import _sos as _
+from sos import __version__
+
+
+class SoSUpload(SoSComponent):
+ """
+ This class is designed to upload files to a distribution
+ defined location. These files can be either sos reports,
+ sos collections, or other kind of files like: vmcores,
+ application cores, logs, etc.
+
+ """
+
+ desc = """
+ Upload a file (can be a sos report, a must-gather, or others) to
+ a distribution defined remote location
+ """
+
+ arg_defaults = {
+ 'upload_url': None,
+ 'upload_method': 'auto',
+ 'upload_no_ssl_verify': False,
+ 'upload_protocol': 'auto',
+ 'upload_file': '',
+ 'case_id': '',
+ 'upload_directory': None,
+ }
+
+ def __init__(self, parser, args, cmdline):
+ # we are running `sos upload` directly
+ # To Do: Work pending on hooking SoSReport or SoSCollector
+ # to this subsystem
+ super(SoSUpload, self).__init__(parser, args, cmdline)
+
+ # add manifest section for upload
+ self.manifest.components.add_section('upload')
+
+ @classmethod
+ def add_parser_options(cls, parser):
+ parser.usage = 'sos upload FILE [options]'
+ upload_grp = parser.add_argument_group(
+ 'Upload Options',
+ 'These options control how upload manages files'
+ )
+ upload_grp.add_argument("upload_file", metavar="FILE",
+ help="The file or archive to upload")
+ upload_grp.add_argument("--case-id", action="store", dest="case_id",
+ help="specify case identifier")
+ upload_grp.add_argument("--upload-url", default=None,
+ help="Upload the archive to specified server")
+ upload_grp.add_argument("--upload-user", default=None,
+ help="Username to authenticate with")
+ upload_grp.add_argument("--upload-pass", default=None,
+ help="Password to authenticate with")
+ upload_grp.add_argument("--upload-directory", action="store",
+ dest="upload_directory",
+ help="Specify upload directory for archive")
+ upload_grp.add_argument("--upload-method", default='auto',
+ choices=['auto', 'put', 'post'],
+ help="HTTP method to use for uploading")
+ upload_grp.add_argument("--upload-protocol", default='auto',
+ choices=['auto', 'https', 'ftp', 'sftp'],
+ help="Manually specify the upload protocol")
+ upload_grp.add_argument("--upload-no-ssl-verify", default=False,
+ action='store_true',
+ help="Disable SSL verification for upload url")
+
+ @classmethod
+ def display_help(cls, section):
+ section.set_title('SoS Upload Detailed Help')
+
+ section.add_text(
+ 'The upload command is designed to upload already existing '
+ 'sos reports, as well as other files like logs and vmcores '
+ 'to a distribution specific location.'
+ )
+
+ def _fmt_msg(self, msg):
+ width = 80
+ _fmt = ''
+ for line in msg.splitlines():
+ _fmt = _fmt + fill(line, width, replace_whitespace=False) + '\n'
+ return _fmt
+
+ def intro(self):
+ """Print the intro message and prompts for a case ID if one is not
+ provided on the command line
+ """
+ disclaimer = """\
+This utility is used to upload files to a policy-default location.
+
+The archive to be uploaded may contain data considered sensitive \
+and its content should be reviewed by the originating \
+organization before being passed to any third party.
+
+No configuration changes will be made to the system running \
+this utility.
+"""
+ self.ui_log.info(f"\nsos upload (version {__version__})")
+ intro_msg = self._fmt_msg(disclaimer)
+ self.ui_log.info(intro_msg)
+
+ prompt = "\nPress ENTER to continue, or CTRL-C to quit\n"
+ if not self.opts.batch:
+ try:
+ input(prompt)
+ self.ui_log.info("")
+ except KeyboardInterrupt:
+ self.exit("Exiting on user cancel", 130)
+ except Exception as e:
+ self.exit(e, 1)
+
+ def get_commons(self):
+ return {
+ 'cmdlineopts': self.opts,
+ 'policy': self.policy,
+ 'case_id': self.opts.case_id,
+ 'upload_directory': self.opts.upload_directory
+ }
+
+ def execute(self):
+
+ self.intro()
+ archive = self.opts.upload_file
+ self.policy.set_commons(self.get_commons())
+ try:
+ if os.stat(archive).st_size > 0:
+ if os.path.isfile(archive):
+ try:
+ self.ui_log.info(
+ _(f"Attempting to upload file {archive} "
+ f"to case {self.opts.case_id}")
+ )
+
+ self.policy.upload_archive(archive)
+ self.ui_log.info(
+ _(f"File {archive} uploaded successfully")
+ )
+ except Exception as err:
+ self.ui_log.error(_(f"Upload attempt failed: {err}"))
+ sys.exit(1)
+ else:
+ self.ui_log.error(_(f"{archive} is not a file."))
+ else:
+ self.ui_log.error(_(f"File {archive} is empty."))
+ except Exception as e:
+ self.ui_log.error(_(f"Cannot upload {archive}: {e} "))