From 98594120ecee23ee249018bc1b17232ed1d6de93 Mon Sep 17 00:00:00 2001 From: Jose Castillo Date: Thu, 8 Aug 2024 21:10:02 +0100 Subject: [PATCH 1/2] [sos] Add 'upload' component to upload existing reports and files This commit marks the beginning of the addition of a new 'upload' component for sos, which can be used to upload already created sos reports, collects, or other files like logs or vmcores to a policy defined location. The user needs to specify a file location, and can make use of any of the options that exist nowadays for the --upload option. This first commit includes: - The initial framework for the 'upload' component. - The new man page for 'sos upload'. - The code in the component 'help' to show information about the component. - The code in sos/__init__.py to deal with the component. - And modifications to setup.py to build the man pages. Related: RHEL-23032, SUPDEV-138, CLIOT-481 Signed-off-by: Jose Castillo --- man/en/sos-upload.1 | 111 +++++++++++++++++++++++ setup.py | 3 +- sos/__init__.py | 4 +- sos/help/__init__.py | 6 +- sos/policies/distros/redhat.py | 7 +- sos/upload/__init__.py | 161 +++++++++++++++++++++++++++++++++ 6 files changed, 284 insertions(+), 8 deletions(-) create mode 100644 man/en/sos-upload.1 create mode 100644 sos/upload/__init__.py diff --git a/man/en/sos-upload.1 b/man/en/sos-upload.1 new file mode 100644 index 0000000000..356dd61592 --- /dev/null +++ b/man/en/sos-upload.1 @@ -0,0 +1,111 @@ +.TH UPLOAD 1 "July 2024" + +.SH NAME +sos_upload \- Upload files like previously generated sos reports or logs to a policy specific location +.SH SYNOPSIS +.B sos upload FILE [options] + [--case-id id]\fR + [--upload-url url]\fR + [--upload-user user]\fR + [--upload-pass pass]\fR + [--upload-directory dir]\fR + [--upload-method]\fR + [--upload-no-ssl-verify]\fR + [--upload-protocol protocol]\fR + +.PP +.SH DESCRIPTION +upload is an sos subcommand to upload sos reports, logs, vmcores, or other files to a policy defined remote location, or a user defined one. +.SH REQUIRED ARGUMENTS +.B FILE +.TP +The path to the archive that is to be uploaded. +.SH OPTIONS +.TP +.B \--case-id NUMBER +Specify a case identifier to associate with the archive. +Identifiers may include alphanumeric characters, commas and periods ('.'). +.TP +.B \--upload-url URL +If a vendor does not provide a default upload location, or if you would like to upload +the archive to a different location, specify the address here. + +An upload protocol MUST be specified in this URL. Currently uploading is supported +for HTTPS, SFTP, and FTP protocols. + +If your destination server listens on a non-standard port, specify the listening +port in the URL. +.TP +.B \-\-upload-user USER +If a vendor does not provide a default user for uploading, specify the username here. + +If --batch is used and this option is omitted, no username will +be collected and thus uploads will fail if no vendor default is set. + +You also have the option of providing this value via the SOSUPLOADUSER environment +variable. If this variable is set, then no username prompt will occur and --batch +may be used provided all other required values (case number, upload password) +are provided. +.TP +.B \-\-upload-pass PASS +Specify the password to use for authentication with the destination server. + +If this option is omitted and upload is requested, you will be prompted for one. + +If --batch is used, this prompt will not occur, so any uploads are likely to fail unless +this option is used. + +Note that this will result in the plaintext string appearing in `ps` output that may +be collected by sos and be in the archive. If a password must be provided by you +for uploading, it is strongly recommended to not use --batch and enter the password +when prompted rather than using this option. + +You also have the option of providing this value via the SOSUPLOADPASSWORD environment +variable. If this variable is set, then no password prompt will occur and --batch may +be used provided all other required values (case number, upload user) are provided. +.TP +.B \--upload-directory DIR +Specify a directory to upload to, if one is not specified by a vendor default location +or if your destination server does not allow writes to '/'. +.TP +.B \--upload-method METHOD +Specify the HTTP method to use for uploading to the provided --upload-url. Valid +values are 'auto' (default), 'put', or 'post'. The use of 'auto' will default to +the method required by the policy-default upload location, if one exists. + +This option has no effect on upload protocols other than HTTPS. +.TP +.B \--upload-no-ssl-verify +Disable SSL verification for HTTPS uploads. This may be used to allow uploading +to locations that have self-signed certificates, or certificates that are otherwise +untrusted by the local system. + +Default behavior is to perform SSL verification against all upload locations. +.TP +.B \--upload-protocol PROTO +Manually specify the protocol to use for uploading to the target \fBupload-url\fR. + +Normally this is determined via the upload address, assuming that the protocol is part +of the address provided, e.g. 'https://example.com'. By using this option, sos will skip +the protocol check and use the method defined for the specified PROTO. + +For RHEL systems, setting this option to \fBsftp\fR will skip the initial attempt to +upload to the Red Hat Customer Portal, and only attempt an upload to Red Hat's SFTP server, +which is typically used as a fallback target. + +Valid values for PROTO are: 'auto' (default), 'https', 'ftp', 'sftp'. + + +.SH SEE ALSO +.BR sos (1) +.BR sos-report (1) +.BR sos-clean (1) +.BR sos.conf (5) +.BR sos-collect (1) + +.SH MAINTAINER +.nf +Maintained on GitHub at https://github.com/sosreport/sos +.fi +.SH AUTHORS & CONTRIBUTORS +See \fBAUTHORS\fR file in the package documentation. diff --git a/setup.py b/setup.py index 9ad4bb75f4..3942a26d98 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,8 @@ ('share/man/man1', ['man/en/sosreport.1', 'man/en/sos-report.1', 'man/en/sos.1', 'man/en/sos-collect.1', 'man/en/sos-collector.1', 'man/en/sos-clean.1', - 'man/en/sos-mask.1', 'man/en/sos-help.1']), + 'man/en/sos-mask.1', 'man/en/sos-help.1', + 'man/en/sos-upload.1']), ('share/man/man5', ['man/en/sos.conf.5']), ('share/licenses/sos', ['LICENSE']), ('share/doc/sos', ['AUTHORS', 'README.md']), diff --git a/sos/__init__.py b/sos/__init__.py index 4ccedddf32..5979640db5 100644 --- a/sos/__init__.py +++ b/sos/__init__.py @@ -54,10 +54,12 @@ def __init__(self, args): import sos.report import sos.cleaner import sos.help + import sos.upload self._components = { 'report': (sos.report.SoSReport, ['rep']), 'clean': (sos.cleaner.SoSCleaner, ['cleaner', 'mask']), - 'help': (sos.help.SoSHelper, []) + 'help': (sos.help.SoSHelper, []), + 'upload': (sos.upload.SoSUpload, []) } # some distros do not want pexpect as a default dep, so try to load # collector here, and if it fails add an entry that implies it is at diff --git a/sos/help/__init__.py b/sos/help/__init__.py index e5162d98e7..b057d17430 100644 --- a/sos/help/__init__.py +++ b/sos/help/__init__.py @@ -100,7 +100,8 @@ def get_obj_for_topic(self): 'collector': 'SoSCollector', 'collector.transports': 'RemoteTransport', 'collector.clusters': 'Cluster', - 'policies': 'Policy' + 'policies': 'Policy', + 'upload': 'SoSUpload' } cls = None @@ -206,7 +207,8 @@ def display_self_help(self): 'report.plugins.$plugin': 'Information on a specific $plugin', 'clean': 'Detailed help on the clean command', 'collect': 'Detailed help on the collect command', - 'policies': 'How sos operates on different distributions' + 'upload': 'Detailed help on the upload command', + 'policies': 'How sos operates on different distributions', } for sect, value in sections.items(): diff --git a/sos/policies/distros/redhat.py b/sos/policies/distros/redhat.py index 3dd47e04f3..68f76e4318 100644 --- a/sos/policies/distros/redhat.py +++ b/sos/policies/distros/redhat.py @@ -298,7 +298,7 @@ def get_upload_url(self): self.ui_log.info("No case id provided, uploading to SFTP") return RH_SFTP_HOST rh_case_api = "/support/v1/cases/%s/attachments" - return RH_API_HOST + rh_case_api % self.case_id + return RH_API_HOST + rh_case_api % self.commons['cmdlineopts'].case_id def _get_upload_https_auth(self): str_auth = f"Bearer {self._device_token}" @@ -441,8 +441,7 @@ def check_file_too_big(self, archive): f"{convert_bytes(self._max_size_request)} " " via sos http upload. \n") ) - return RH_SFTP_HOST - return RH_API_HOST + self.upload_url = RH_SFTP_HOST def upload_archive(self, archive): """Override the base upload_archive to provide for automatic failover @@ -450,7 +449,7 @@ def upload_archive(self, archive): """ try: if self.get_upload_url().startswith(RH_API_HOST): - self.upload_url = self.check_file_too_big(archive) + self.check_file_too_big(archive) uploaded = super().upload_archive(archive) except Exception as e: uploaded = False diff --git a/sos/upload/__init__.py b/sos/upload/__init__.py new file mode 100644 index 0000000000..5b9554e22a --- /dev/null +++ b/sos/upload/__init__.py @@ -0,0 +1,161 @@ +# Copyright 2024 Red Hat, Inc. Jose Castillo + +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +import os +import sys +from textwrap import fill +from sos.component import SoSComponent +from sos import _sos as _ +from sos import __version__ + + +class SoSUpload(SoSComponent): + """ + This class is designed to upload files to a distribution + defined location. These files can be either sos reports, + sos collections, or other kind of files like: vmcores, + application cores, logs, etc. + + """ + + desc = """ + Upload a file (can be a sos report, a must-gather, or others) to + a distribution defined remote location + """ + + arg_defaults = { + 'upload_url': None, + 'upload_method': 'auto', + 'upload_no_ssl_verify': False, + 'upload_protocol': 'auto', + 'upload_file': '', + 'case_id': '', + 'upload_directory': None, + } + + def __init__(self, parser, args, cmdline): + # we are running `sos upload` directly + # To Do: Work pending on hooking SoSReport or SoSCollector + # to this subsystem + super().__init__(parser, args, cmdline) + + # add manifest section for upload + self.manifest.components.add_section('upload') + + @classmethod + def add_parser_options(cls, parser): + parser.usage = 'sos upload FILE [options]' + upload_grp = parser.add_argument_group( + 'Upload Options', + 'These options control how upload manages files' + ) + upload_grp.add_argument("upload_file", metavar="FILE", + help="The file or archive to upload") + upload_grp.add_argument("--case-id", action="store", dest="case_id", + help="specify case identifier") + upload_grp.add_argument("--upload-url", default=None, + help="Upload the archive to specified server") + upload_grp.add_argument("--upload-user", default=None, + help="Username to authenticate with") + upload_grp.add_argument("--upload-pass", default=None, + help="Password to authenticate with") + upload_grp.add_argument("--upload-directory", action="store", + dest="upload_directory", + help="Specify upload directory for archive") + upload_grp.add_argument("--upload-method", default='auto', + choices=['auto', 'put', 'post'], + help="HTTP method to use for uploading") + upload_grp.add_argument("--upload-protocol", default='auto', + choices=['auto', 'https', 'ftp', 'sftp'], + help="Manually specify the upload protocol") + upload_grp.add_argument("--upload-no-ssl-verify", default=False, + action='store_true', + help="Disable SSL verification for upload url") + + @classmethod + def display_help(cls, section): + section.set_title('SoS Upload Detailed Help') + + section.add_text( + 'The upload command is designed to upload already existing ' + 'sos reports, as well as other files like logs and vmcores ' + 'to a distribution specific location.' + ) + + def _fmt_msg(self, msg): + width = 80 + _fmt = '' + for line in msg.splitlines(): + _fmt = _fmt + fill(line, width, replace_whitespace=False) + '\n' + return _fmt + + def intro(self): + """Print the intro message and prompts for a case ID if one is not + provided on the command line + """ + disclaimer = """\ +This utility is used to upload files to a policy-default location. + +The archive to be uploaded may contain data considered sensitive \ +and its content should be reviewed by the originating \ +organization before being passed to any third party. + +No configuration changes will be made to the system running \ +this utility. +""" + self.ui_log.info(f"\nsos upload (version {__version__})") + intro_msg = self._fmt_msg(disclaimer) + self.ui_log.info(intro_msg) + + prompt = "\nPress ENTER to continue, or CTRL-C to quit\n" + if not self.opts.batch: + try: + input(prompt) + self.ui_log.info("") + except KeyboardInterrupt: + self._exit("Exiting on user cancel", 130) + except Exception as e: + self._exit(e, 1) + + def get_commons(self): + return { + 'cmdlineopts': self.opts, + 'policy': self.policy, + 'case_id': self.opts.case_id, + 'upload_directory': self.opts.upload_directory + } + + def execute(self): + + self.intro() + archive = self.opts.upload_file + self.policy.set_commons(self.get_commons()) + try: + if os.stat(archive).st_size > 0: + if os.path.isfile(archive): + try: + self.ui_log.info( + _(f"Attempting to upload file {archive} " + f"to case {self.opts.case_id}") + ) + + self.policy.upload_archive(archive) + self.ui_log.info( + _(f"File {archive} uploaded successfully") + ) + except Exception as err: + self.ui_log.error(_(f"Upload attempt failed: {err}")) + sys.exit(1) + else: + self.ui_log.error(_(f"{archive} is not a file.")) + else: + self.ui_log.error(_(f"File {archive} is empty.")) + except Exception as e: + self.ui_log.error(_(f"Cannot upload {archive}: {e} ")) From 72dd27ce7d3887e6974952f01309d7472158eca3 Mon Sep 17 00:00:00 2001 From: Jose Castillo Date: Tue, 13 Aug 2024 16:04:49 +0100 Subject: [PATCH 2/2] [component] Move _fmt_msg() from collector,cleaner, upload to SosComponent The function _fmt_msg() was in three different subsystems. This commit removes the code from collector, cleaner, and upload into SosComponent. Signed-off-by: Jose Castillo --- sos/cleaner/__init__.py | 8 -------- sos/collector/__init__.py | 8 -------- sos/component.py | 8 ++++++++ sos/upload/__init__.py | 8 -------- 4 files changed, 8 insertions(+), 24 deletions(-) diff --git a/sos/cleaner/__init__.py b/sos/cleaner/__init__.py index b5ea40dad6..fbb7c55e55 100644 --- a/sos/cleaner/__init__.py +++ b/sos/cleaner/__init__.py @@ -19,7 +19,6 @@ from concurrent.futures import ThreadPoolExecutor from datetime import datetime from pwd import getpwuid -from textwrap import fill import sos.cleaner.preppers @@ -177,13 +176,6 @@ def log_info(self, msg, caller=None): def log_error(self, msg, caller=None): self.soslog.error(self._fmt_log_msg(msg, caller)) - def _fmt_msg(self, msg): - width = 80 - _fmt = '' - for line in msg.splitlines(): - _fmt = _fmt + fill(line, width, replace_whitespace=False) + '\n' - return _fmt - @classmethod def display_help(cls, section): section.set_title("SoS Cleaner Detailed Help") diff --git a/sos/collector/__init__.py b/sos/collector/__init__.py index e26f3efd3f..9485f86a06 100644 --- a/sos/collector/__init__.py +++ b/sos/collector/__init__.py @@ -26,7 +26,6 @@ from getpass import getpass from pathlib import Path from shlex import quote -from textwrap import fill from sos.cleaner import SoSCleaner from sos.collector.sosnode import SosNode from sos.options import ClusterOption, str_to_bool @@ -712,13 +711,6 @@ def _get_archive_path(self): compr = 'gz' return self.tmpdir + '/' + self.arc_name + '.tar.' + compr - def _fmt_msg(self, msg): - width = 80 - _fmt = '' - for line in msg.splitlines(): - _fmt = _fmt + fill(line, width, replace_whitespace=False) + '\n' - return _fmt - def _load_group_config(self): """ Attempts to load the host group specified on the command line. diff --git a/sos/component.py b/sos/component.py index 6f42046022..f440a8b21b 100644 --- a/sos/component.py +++ b/sos/component.py @@ -16,6 +16,7 @@ import sys import time +from textwrap import fill from argparse import SUPPRESS from datetime import datetime from getpass import getpass @@ -458,6 +459,13 @@ def _setup_logging(self): def get_temp_file(self): return self.tempfile_util.new() + def _fmt_msg(self, msg): + width = 80 + _fmt = '' + for line in msg.splitlines(): + _fmt = _fmt + fill(line, width, replace_whitespace=False) + '\n' + return _fmt + class SoSMetadata(): """This class is used to record metadata from a sos execution that will diff --git a/sos/upload/__init__.py b/sos/upload/__init__.py index 5b9554e22a..a924463b08 100644 --- a/sos/upload/__init__.py +++ b/sos/upload/__init__.py @@ -10,7 +10,6 @@ import os import sys -from textwrap import fill from sos.component import SoSComponent from sos import _sos as _ from sos import __version__ @@ -89,13 +88,6 @@ def display_help(cls, section): 'to a distribution specific location.' ) - def _fmt_msg(self, msg): - width = 80 - _fmt = '' - for line in msg.splitlines(): - _fmt = _fmt + fill(line, width, replace_whitespace=False) + '\n' - return _fmt - def intro(self): """Print the intro message and prompts for a case ID if one is not provided on the command line