Migrate SCons build scripts to Python 3 (#1505)

* Decouple local SCons from custom MSVC tools

* Remove some unused python imports

* Fix encoding when writing the testsuite report

* Use zip in Python 3

* Remove unused imports

* Fix abuild bash script

* Fix abuild bash script

* Use print function instead of the Python 2 statement

* Missing ported next()

* Remove unused stuff and other Python 3 fixes

* Update bottle to a Python 2/3 compatible version

* SCons envs cannot be cloned inside actions

* Do not write valgrind reports, since it is not supported

* More ports of isinstance(_, basestring)

* Do not write testuite report as binary data

* Fix testsuite report generation

* Fix testsuite report generation

* More Python 3 fixes for the doxygen builder

* Fix dict "iteritems" comming from the master branch

SConstruct

@@ -12,11 +12,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import glob
-import re
 import shutil
 import sys, os
-import platform
 from SCons.Script import PathVariable
 import SCons
 

abuild

@@ -1,4 +1,10 @@
 #!/bin/sh
 
-## invokes a local install of scons (forwarding all arguments)
-python -B tools/scons/scons.py --site-dir=tools/scons-custom "$@"
+SOURCE="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
+
+SCONS_PATH=$SOURCE/tools/scons/scons.py
+SCONS_TOOLS=$SOURCE/tools/scons-custom
+SCONS="python -B $SCONS_PATH"
+
+export PYTHONPATH=$SCONS_TOOLS
+$SCONS --site-dir=$SCONS_TOOLS $*

abuild.bat

@@ -1,3 +1,9 @@
-@REM invokes a local install of scons (forwarding all arguments)
+@SETLOCAL ENABLEEXTENSIONS
+@ECHO OFF
 
-@python -B tools\scons\scons.py --site-dir=tools\scons-custom %*
+SET SCONS_PATH="%~dp0\tools\scons\scons.py"
+SET SCONS_TOOLS="%~dp0\tools\scons-custom"
+SET SCONS=python -B %SCONS_PATH%
+
+SET PYTHONPATH=%SCONS_TOOLS%
+%SCONS% --site-dir=%SCONS_TOOLS% %*

cmd/SConscript

@@ -12,8 +12,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from utils import system, dependencies
-from utils.build_tools import find_files_recursive, link_usd_libraries
+from utils import dependencies
+from utils.build_tools import find_files_recursive
 import os
 import os.path
 

ndr/SConscript

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import os
-from utils import system, build_tools, dependencies
+from utils import system, dependencies
 
 Import('env')
 local_env = env.Clone()

procedural/SConscript

@@ -14,7 +14,7 @@
 # limitations under the License.
 ## load our own python modules
 from utils import system
-from utils.build_tools import find_files_recursive, link_usd_libraries
+from utils.build_tools  import link_usd_libraries
 from utils.dependencies import get_tbb_lib
 
 import os

render_delegate/SConscript

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import os
-from utils import system, build_tools, dependencies
+from utils import system, dependencies
 
 Import('env')
 local_env = env.Clone()

scene_delegate/SConscript

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import os
-from utils import system, build_tools, dependencies
+from utils import system, dependencies
 
 Import('env')
 local_env = env.Clone()

schemas/SConscript

@@ -13,15 +13,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 ## load our own python modules
-from utils import system
-from utils.build_tools import find_files_recursive, link_usd_libraries
 from utils.system import IS_WINDOWS, IS_DARWIN, IS_LINUX
-from utils.dependencies import get_tbb_lib
 
 import os
-import platform
 import sys
-import shutil
 import filecmp
 
 # import build env
@@ -109,12 +104,12 @@ elif os.path.exists(schemaPreviousFile) and os.path.exists(generatedSchema):
 
 if generateSchemaFiles:
     if os.path.exists(schemasBuildFolder):
-        print 'Cleaning existing schema files'
+        print('Cleaning existing schema files')
         prevFiles = os.listdir(schemasBuildFolder)
         for f in prevFiles:
             os.remove(os.path.join(schemasBuildFolder, f))
 
-    print 'Now running usdGenSchema to create the C++ files...'
+    print('Now running usdGenSchema to create the C++ files...')
     cmd = '{genSchema} "{schemaFile}" "{output}"'.format(**{
         'genSchema': genSchema,
         'schemaFile': schemaFile,

testsuite/SConscript

@@ -1,6 +1,6 @@
 # vim: filetype=python
 
-import sys, os, glob, shutil, time, subprocess
+import sys, os, glob, shutil, time
 
 ## load third-party python modules
 from utils.contrib.bottle import SimpleTemplate
@@ -12,7 +12,6 @@ from utils.regression_test import Test
 from utils.system          import print_safe, IS_WINDOWS, IS_LINUX, IS_DARWIN
 from utils.test_stats      import *
 from utils.testsuite       import *
-from utils.valgrind        import *
 
 Import('env')
 
@@ -131,7 +130,7 @@ def run_test(target, source, env):
 
    test_dir = os.path.dirname(target[0].get_abspath())
 
-   test_env = env.Clone()
+   test_env = env
    test_env.AppendENVPath('ARNOLD_PLUGIN_PATH', os.path.dirname(test_env['USD_PROCEDURAL_PATH']))
    test_env.AppendENVPath('PXR_PLUGINPATH_NAME', test_env['PREFIX_RENDER_DELEGATE'])
 
@@ -141,7 +140,7 @@ def run_test(target, source, env):
    # subprocess.Popen (only string values allowed)
    local_env = {}
 
-   for k, v in test_env['ENV'].iteritems():
+   for k, v in test_env['ENV'].items():
       local_env[k] = str(v)
 
 
@@ -219,8 +218,8 @@ def run_test(target, source, env):
       # NOTE #5079: Execute the test printing the logs to the standard output
       # if requested (parameter verbose), and always write them to a .log file.
 
-      print '====Exec %s' % cmd
-      print os.getenv('PATH')
+      print('====Exec %s' % cmd)
+      print(os.getenv('PATH'))
 
       retcode, out = sa.system.execute(cmd, cwd=test_dir, env=local_env, shell=use_shell, timeout=test_env.get('TIMELIMIT', 0), verbose=show_test_output)
       with open(os.path.join(test_dir, "%s.log") % TEST_NAME, 'w') as f:
@@ -274,8 +273,8 @@ def run_test(target, source, env):
       cmd = oiiotool_path + ' -v --info ' + reference_image
       retcode, info = sa.system.execute(cmd, cwd=test_dir, env=local_env, shell=use_shell)
 
-      print "Exec %s" % cmd
-      print "Info %s" % info
+      print("Exec %s" % cmd)
+      print("Info %s" % info)
 
       is_deep = 'deep' in info[1].split(',')[-1]
       channel_names = info[2].split(': ')[1].split(',')
@@ -409,8 +408,6 @@ def run_test(target, source, env):
       }
       f.write(html_template.format(**params))
 
-      vg_write_report(f, os.path.join(test_dir, "%s.vg.xml") % TEST_NAME, test_env['USE_VALGRIND'])
-
    ## always succeeds (unless some of the functions above throw an error)
    return None
 
@@ -551,7 +548,13 @@ def process_testsuite(target, source, env):
    ## Render report to HTML file using template
    with open(os.path.join(testsuite_common, 'testsuite.html.template'), 'r') as template_file:
       with open(str(target[0]), 'w') as report_file:
-         report_file.write(SimpleTemplate(template_file.read()).render(report_params).encode('utf8'))
+         report = SimpleTemplate(template_file.read()).render(report_params)
+         try:
+            report = str(report)
+         except UnicodeEncodeError:
+            # This with be catched in Python 2
+            report = report.encode('utf-8')
+         report_file.write(report)
 
    ## Report results in console
 

testsuite/hydra_test/SConscript

@@ -11,8 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from utils import system, dependencies
-from utils.build_tools import find_files_recursive, link_usd_libraries
+from utils import dependencies
 import os
 import os.path
 

tools/scons-custom/site_tools/MSCommon/__init__.py

@@ -0,0 +1,57 @@
+#
+# Copyright (c) 2001 - 2019 The SCons Foundation
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
+# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+
+__revision__ = "src/engine/SCons/Tool/MSCommon/__init__.py bee7caf9defd6e108fc2998a2520ddb36a967691 2019-12-17 02:07:09 bdeegan"
+
+__doc__ = """
+Common functions for Microsoft Visual Studio and Visual C/C++.
+"""
+
+import copy
+import os
+import re
+import subprocess
+
+import SCons.Errors
+import SCons.Platform.win32
+import SCons.Util
+
+from .sdk import mssdk_exists, \
+                                    mssdk_setup_env
+
+from .vc import msvc_exists, \
+                                   msvc_setup_env, \
+                                   msvc_setup_env_once, \
+                                   msvc_version_to_maj_min
+
+from .vs import get_default_version, \
+                                   get_vs_by_version, \
+                                   merge_default_version, \
+                                   msvs_exists, \
+                                   query_versions
+
+# Local Variables:
+# tab-width:4
+# indent-tabs-mode:nil
+# End:
+# vim: set expandtab tabstop=4 shiftwidth=4:

tools/scons-custom/site_tools/MSCommon/arch.py

@@ -0,0 +1,67 @@
+#
+# Copyright (c) 2001 - 2019 The SCons Foundation
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
+# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+
+__revision__ = "src/engine/SCons/Tool/MSCommon/arch.py bee7caf9defd6e108fc2998a2520ddb36a967691 2019-12-17 02:07:09 bdeegan"
+
+__doc__ = """Module to define supported Windows chip architectures.
+"""
+
+import os
+
+class ArchDefinition(object):
+    """
+    A class for defining architecture-specific settings and logic.
+    """
+    def __init__(self, arch, synonyms=[]):
+        self.arch = arch
+        self.synonyms = synonyms
+
+SupportedArchitectureList = [
+    ArchDefinition(
+        'x86',
+        ['i386', 'i486', 'i586', 'i686'],
+    ),
+
+    ArchDefinition(
+        'x86_64',
+        ['AMD64', 'amd64', 'em64t', 'EM64T', 'x86_64'],
+    ),
+
+    ArchDefinition(
+        'ia64',
+        ['IA64'],
+    ),
+
+    ArchDefinition(
+        'arm',
+        ['ARM'],
+    ),
+
+]
+
+SupportedArchitectureMap = {}
+for a in SupportedArchitectureList:
+    SupportedArchitectureMap[a.arch] = a
+    for s in a.synonyms:
+        SupportedArchitectureMap[s] = a
+