Merge branch 'yugabyte:master' into master

Author: polarweasel

.arclint

@@ -12,13 +12,15 @@
      "(^managed/src/main/resources/v1[.]routes$)",
      "(^managed/src/main/public/swagger-ui/)",
      "(^managed/src/main/resources/swagger[.]json$)",
+     "(^managed/src/main/resources/swagger-strict[.]json$)",
      "(^managed/build[.]sbt$)",
      "(^managed/src/main/resources/aws_pricing/.*$)",
      "(^python/yb/test_data/.*[.](log|out)$)",
      "(^managed/src/main/resources/metric/Dashboard[.]json$)",
      "(^managed/src/main/resources/metric/recording_rules[.]yml$)",
      "(^managed/devops/replicated[.]yml$)",
-     "(^python/yb/py[.]typed$)"
+     "(^python/yb/py[.]typed$)",
+     "(^managed/RUNTIME-FLAGS[.]md$)"
   ],
   "linters": {
     "go-files": {

.github/ISSUE_TEMPLATE/cdc.yml

@@ -0,0 +1,39 @@
+name: YSQL-CDC
+description: Create a CDC issue.
+title: "[CDCSDK] Title"
+labels: ["area/cdcsdk", "status/awaiting-triage"]
+body:
+- type: textarea
+  id: cdcsdk
+  attributes:
+    label: Description
+    placeholder: |
+      **Provide a detailed description of the issue you want to report related to YSQL.**
+     
+      **Include steps to reproduce the issue.**
+  validations:
+    required: true
+- type: textarea
+  id: connectorversion
+  attributes:
+    label: Source connector version
+    placeholder: |
+      **v1.9.5.y.15**
+  validations:
+    required: true
+- type: textarea
+  id: configuration
+  attributes:
+    label: Connector configuration
+    placeholder: |
+      **Configuration properties used to deploy the YugabyteDB Source Connector**
+  validations:
+    required: true
+- type: textarea
+  id: ybversion
+  attributes:
+    label: YugabyteDB version
+    placeholder: |
+      **The YugabyteDB version with build number (if available)**
+  validations:
+    required: false

.github/ISSUE_TEMPLATE/docs.yml

@@ -1,7 +1,7 @@
 name: Docs
 description: Create a documentation issue.
 title: "[Docs] Title"
-labels: ["area/documentation"]
+labels: ["area/documentation", "status/awaiting-triage"]
 body:
 - type: textarea
   id: documentation
@@ -10,4 +10,4 @@ body:
     placeholder: |
       **Provide a detailed description of the information you want to update/include in the YugabyteDB docs.**
   validations:
-    required: true 
+    required: true

.github/ISSUE_TEMPLATE/ui.yml

@@ -9,7 +9,7 @@ body:
     label: Description
     placeholder: |
       **Provide a clear description of the issue you want to report related to the UI in Yugabyte.**
-      
-      **NOTE: If you can, please specify the product in question. For example, if this is a platform issue, include it in your issue like so: [UI] [Platform] Title**
+
+      **NOTE: If you can, please specify the product in question. For example, if this is a YugabyteDB Anywhere issue, include it in your issue like so: [UI] [YBA] Title**
   validations:
-    required: true 
+    required: true

.github/ISSUE_TEMPLATE/yugabyted.yml

@@ -0,0 +1,15 @@
+name: yugabyted
+description: Create a yugabyted issue.
+title: "[yugabyted] Title"
+labels: ["area/ecosystem", "area/ybd", "status/awaiting-triage"]
+body:
+- type: textarea
+  id: yugabyted
+  attributes:
+    label: Description
+    placeholder: |
+      **Provide a detailed description of the issue you want to report with the yugabyted based deployments.**
+
+      **Include the steps to reproduce the issue.**
+  validations:
+    required: true

bin/openssl_proxy.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/usr/bin/env bash
 
 root_certs_generation=false
 node_certs_generation=false
@@ -12,7 +12,7 @@ hostname=""
 
 show_help() {
   cat >&1 <<-EOT
-gen_certs.sh is to be used to create SSL certificates for secure deployment of YugabyteDB.
+openssl_proxy.sh is to be used to create SSL certificates for secure deployment of YugabyteDB.
 ---------------------------------------------------------------------------------------------------
 Usage: ${0##*/} [<action>] [<flags>]
 

bin/yugabyted

@@ -394,14 +394,30 @@ class ControlScript(object):
 
         self.start_processes()
 
+    # Kills all processes related to yugabyted
+    def kill_all_procs(self):
+        pid = self.script.get_pid()
+        if pid:
+            pgid = os.getpgid(pid)
+            if not pgid:
+                Output.log("PGID could not be found for {} process".format(SCRIPT_NAME) +
+                    "with PID {}. Is {} running?".format(pid, SCRIPT_NAME))
+                return ("No PGID", pid)
+            else:
+                try:
+                    os.killpg(pgid, SIGTERM)
+                except OSError as err:
+                    return (err, pid)
+        self.script.delete_pidfile()
+        return (None, pid)
+
     # Kills currently running yugabyted process if it exists.
     def stop(self, *args):
-        (err, pid) = self.script.kill()
+        (err, pid) = self.kill_all_procs()
         if err:
-            Output.print_out(
+            Output.log_error_and_exit(
                 "Failed to shut down {}: {}. Please check PID in {}".format(
                     SCRIPT_NAME, err, self.script.pidfile))
-            sys.exit(1)
         elif pid:
             self.script.wait_until_stop(pid)
             Output.print_out("Stopped {} using config {}.".format(SCRIPT_NAME, self.conf_file))
@@ -416,7 +432,7 @@ class ControlScript(object):
 
     # Destroy the YugabyteDB cluster.
     def destroy(self):
-        (err, pid) = self.script.kill()
+        (err, pid) = self.kill_all_procs()
         if err:
             Output.log_error_and_exit(
                 "Failed to shut down {}: {}. Please check PID in {}".format(
@@ -1360,6 +1376,9 @@ class ControlScript(object):
         if self.configs.saved_data.get("use_cassandra_authentication"):
             yb_tserver_cmd.extend(["--use_cassandra_authentication=true"])
 
+        if self.configs.saved_data.get("dns_enabled"):
+            yb_tserver_cmd.extend(["--use_node_hostname_for_local_tserver=true"])
+
         self.processes = {
             "master": YBProcessManager(
                 "master", yb_master_cmd, self.configs.saved_data.get("log_dir"),
@@ -1369,15 +1388,6 @@ class ControlScript(object):
                 self.configs.saved_data.get("data_dir")),
         }
 
-        # if self.configs.temp_data.get("ui"):
-        #     yugabyted_ui_cmd = [ find_binary_location("yugabyted-ui")] + \
-        #         [
-        #             "-database_host={}".format(bind_ip)
-        #         ]
-        #     self.processes["yugabyted-ui"] = ProcessManager(
-        #         "yugabyted-ui", yugabyted_ui_cmd, self.configs.saved_data.get("log_dir"),
-        #         self.configs.saved_data.get("data_dir"))
-
         for p in self.processes.values():
             pid = p.get_pid()
             if pid:
@@ -2025,7 +2035,7 @@ class ControlScript(object):
 
             placementInfoOfEveryNode = {}
             for node in dictOfAllNodes:
-                hostname = re.split('[-:]', node)[0]
+                hostname = node.split(":")[0]
                 cloudLocationOfHost = "{}.{}.{}".format(dictOfAllNodes[node]["cloud"],
                     dictOfAllNodes[node]["region"], dictOfAllNodes[node]["zone"])
                 placementInfoOfEveryNode[hostname] = cloudLocationOfHost
@@ -2636,13 +2646,22 @@ class ControlScript(object):
         # Regex expression for validating IPv6
         ipv6_regex = "((([0-9a-fA-F]){1,4})\\:){7}([0-9a-fA-F]){1,4}"
 
+        # Regex expression for validating DNS names
+        dns_regex = "^((?!-)[A-Za-z0-9-]" + "{1,63}(?<!-)\\.)" +"+[A-Za-z]{2,6}"
+
         ipv4_pattern = re.compile(ipv4_regex)
         ipv6_pattern = re.compile(ipv6_regex)
+        dns_pattern = re.compile(dns_regex)
 
         # Checking if it is not a valid IPv4 or IPv6 addresses
-        if not (re.search(ipv4_pattern, ip) or re.search(ipv6_pattern, ip)):
+        if not (re.search(ipv4_pattern, ip) or
+                re.search(ipv6_pattern, ip) or
+                re.search(dns_pattern, ip)):
             return False
 
+        if re.search(dns_pattern, ip):
+            self.configs.saved_data["dns_enabled"] = True
+
         return True
 
     # Returns a list of invalid IPs from the list of IPs provided.
@@ -3135,8 +3154,12 @@ class ControlScript(object):
                     self.configs.saved_data["ca_cert_file_path"] = os.path.join(args.certs_dir,
                                                                                     "ca.crt")
 
-            args.ui = self.parse_bool(args.ui)
             args.background = self.parse_bool(args.background)
+            if args.ui is not None:
+                args.ui = self.parse_bool(args.ui)
+            else:
+                args.ui = self.configs.saved_data.get("ui")
+
             if args.callhome is not None:
                 args.callhome = self.parse_bool(args.callhome)
             elif os.environ.get("YB_DISABLE_CALLHOME") is not None:
@@ -3162,15 +3185,6 @@ class ControlScript(object):
                         self.setup_env_init.is_exists('YCQL_PASSWORD'):
                     args.use_cassandra_authentication = True
 
-            # Output.log("Downloading the Yugabyted UI Binaries.")
-            # if args.ui and not self.download_extract_yugabyted_ui_binary():
-            #     Output.print_out('WARNING: UI disabled - Unable to download ui binaries.')
-            #     args.ui = False
-
-            # if args.ui and not self.java_installed():
-            #     Output.print_out('WARNING: UI disabled - Java 1.8 or higher is required.')
-            #     args.ui = False
-
             self.configs.temp_data["background"] = args.background
             self.configs.saved_data["ui"] = args.ui
             self.configs.temp_data["initial_scripts_dir"] = args.initial_scripts_dir
@@ -3392,7 +3406,7 @@ class ControlScript(object):
                 "--callhome", choices=BOOL_CHOICES,
                 help=argparse.SUPPRESS)
             cur_parser.add_argument(
-                "--ui", choices=BOOL_CHOICES, default="true", metavar="",
+                "--ui", choices=BOOL_CHOICES, metavar="",
                 help="Toggle enabling or disabling webserver UI. Default true.")
             cur_parser.add_argument(
                 "--ysql_enable_auth", choices=BOOL_CHOICES,
@@ -3560,6 +3574,7 @@ class Configs(object):
             "database_password": None,
             "current_masters": "",
             "ui": True,
+            "dns_enabled": False,
         }
         # Used to store data specific to certain functions that we don't want to save.
         self.temp_data = {
@@ -3743,7 +3758,6 @@ class ProcessManager(object):
         def temp_func():
             return self.is_proc_running(pid)
         retry_op(temp_func, timeout=60)
-        time.sleep(2)
         return
 
     # Delete corresponding pidfile for this process.
@@ -4142,7 +4156,12 @@ class YsqlProxy(object):
         env_var = os.environ.copy()
         env_var["PGUSER"] = self.username
         shell = subprocess.Popen(cmd, env=env_var)
-        shell.communicate()
+        while True:
+            try:
+                shell.communicate()
+            except KeyboardInterrupt:
+                continue
+            break
 
     # Checks if db exists.
     # Note that this will return false if ysqlsh can't connect, even if db exists.
@@ -4256,7 +4275,12 @@ class YcqlProxy(object):
         if self.keyspace is not None:
             cmd.extend(["-k", self.keyspace])
         shell = subprocess.Popen(cmd)
-        shell.communicate()
+        while True:
+            try:
+                shell.communicate()
+            except KeyboardInterrupt:
+                continue
+            break
 
     # Check user exists
     # Note that this will return false if ycqlsh can't connect, even if user exists.

build-support/common-build-env.sh

@@ -2171,10 +2171,13 @@ run_shellcheck() {
     build-support/common-build-env.sh
     build-support/common-cli-env.sh
     build-support/common-test-env.sh
+    build-support/jenkins/common-lto.sh
     build-support/compiler-wrappers/compiler-wrapper.sh
     build-support/find_linuxbrew.sh
-    build-support/jenkins/build-and-test.sh
+    build-support/jenkins/build.sh
+    build-support/jenkins/test.sh
     build-support/jenkins/yb-jenkins-build.sh
+    build-support/jenkins/yb-jenkins-test.sh
     build-support/run-test.sh
     yb_build.sh
   )