Merge pull request #255 from sit23/sh_fix

Fix for Sh > 2.* issue, and new environment files and template for running on maths2 (Exeter). Also fixing bug that stopped the automated tests from working. This seems to also have been related to sh, but with the ubuntu system that github actions is running on not liking `sh.cd`.

src/extra/env/maths2

@@ -0,0 +1,6 @@
+echo loadmodules for maths2 machines
+
+export F90=mpiifort
+export CC=mpiicc
+
+export GFDL_MKMF_TEMPLATE=maths2

src/extra/python/isca/codebase.py

@@ -7,7 +7,7 @@
 
 from isca import GFDL_WORK, GFDL_BASE, GFDL_SOC, _module_directory, get_env_file
 from .loghandler import Logger
-from .helpers import url_to_folder, destructive, useworkdir, mkdir, cd, git, P, git_run_in_directory
+from .helpers import url_to_folder, destructive, useworkdir, mkdir, git, P, git_run_in_directory, check_for_sh_stdout
 
 import pdb
 
@@ -137,7 +137,7 @@ def is_clean(self):
 
     @property
     def git_commit(self):
-        return self.git.log('-1', '--format="%H"').stdout.decode('utf8')
+        return check_for_sh_stdout(self.git.log('-1', '--format="%H"'))
 
     # @property
     # def git_diff(self):
@@ -162,11 +162,12 @@ def write_source_control_status(self, outfile):
 
             # write out the git commit id of GFDL_BASE
             file.write("\n\n*---commit hash used for code in GFDL_BASE, including this python module---*:\n")
-            file.write(gfdl_git.log('-1', '--format="%H"').stdout.decode('utf8'))
+            gfdl_git_out = check_for_sh_stdout(gfdl_git.log('-1', '--format="%H"'))
+            file.write(gfdl_git_out)
 
             # if there are any uncommited changes in the working directory,
             # add those to the file too
-            source_status = self.git.status("-b", "--porcelain").stdout.decode('utf8')
+            source_status = check_for_sh_stdout(self.git.status("-b", "--porcelain"))
             # filter the source status for changes in specific files
             filetypes = ('.f90', '.inc', '.c')
             source_status = [line for line in source_status.split('\n')
@@ -178,7 +179,7 @@ def write_source_control_status(self, outfile):
                 file.write("*---git status output (only f90 and inc files)---*:\n")
                 file.write('\n'.join(source_status))
                 file.write('\n\n*---git diff output---*\n')
-                source_diff = self.git.diff('--no-color').stdout.decode('utf8')
+                source_diff = check_for_sh_stdout(self.git.diff('--no-color'))
                 file.write(source_diff)
 
     def read_path_names(self, path_names_file):

src/extra/python/isca/helpers.py

@@ -4,7 +4,7 @@
 import sh
 
 mkdir = sh.mkdir.bake('-p')
-cd = sh.cd
+# cd = sh.cd
 git = sh.git.bake('--no-pager')
 
 P = os.path.join
@@ -66,11 +66,19 @@ def git_run_in_directory(GFDL_BASE_DIR, dir_in):
 
     try:
         codedir_git = git.bake('-C', GFDL_BASE_DIR)        
-        git_test = codedir_git.log('-1', '--format="%H"').stdout
+        git_test = check_for_sh_stdout(codedir_git.log('-1', '--format="%H"'))
         baked_git_fn = git.bake('-C', dir_in)
     except:
         codedir_git = git.bake('--git-dir='+GFDL_BASE_DIR+'/.git', '--work-tree='+GFDL_BASE_DIR)
-        git_test = codedir_git.log('-1', '--format="%H"').stdout
+        git_test = check_for_sh_stdout(codedir_git.log('-1', '--format="%H"'))
         baked_git_fn = git.bake('--git-dir='+dir_in+'/.git', '--work-tree='+dir_in)        
 
-    return baked_git_fn        
+    return baked_git_fn       
+
+def check_for_sh_stdout(input_exp):
+    """Versions of sh>2.* have started returning str types rather than a sh.RunningCommand type. To distinguish these possibilites, this function looks at the output of a sh expression, and asks for stdout and decodes it only if the type is a sh.RunningCommand."""
+
+    if type(input_exp)==sh.RunningCommand:     
+        input_exp=input_exp.stdout.decode('utf8')
+
+    return input_exp

src/extra/python/isca/templates/mkmf.template.maths2

@@ -0,0 +1,28 @@
+# template for the Intel fortran compiler
+# typical use with mkmf
+# mkmf -t template.ifc -c"-Duse_libMPI -Duse_netCDF" path_names /usr/local/include
+CPPFLAGS = `nc-config --cflags`
+NETCDF_LIBS = `nc-config --libs`
+
+# FFLAGS:
+#  -fpp: Use the fortran preprocessor
+#  -stack_temps:  Put temporary runtime arrays on the stack, not heap.
+#  -safe_cray_ptr: Cray pointers don't alias other variables.
+#  -ftz: Denormal numbers are flushed to zero.
+#  -assume byterecl: Specifies the units for the OPEN statement as bytes.
+#  -shared-intel:  Load intel libraries dynamically
+#  -i4: 4 byte integers
+#  -r8: 8 byte reals
+#  -g: Generate symbolic debugging info in code
+#  -O2: Level 2 speed optimisations
+#  -diag-disable 6843:
+#       This suppresses the warning: `warning #6843: A dummy argument with an explicit INTENT(OUT) declaration is not given an explicit value.` of which
+#       there are a lot of instances in the GFDL codebase.
+FFLAGS = $(CPPFLAGS) -fpp -stack_temps -safe_cray_ptr -ftz -assume byterecl -shared-intel -i4 -r8 -g -O2 -diag-disable 6843 -mcmodel large
+#FFLAGS = $(CPPFLAGS) -fltconsistency -stack_temps -safe_cray_ptr -ftz -shared-intel -assume byterecl -g -O0 -i4 -r8 -check -warn -warn noerrors -debug variable_locations -inline_debug_info -traceback
+FC = $(F90)
+LD = $(F90) $(NETCDF_LIBS)
+#CC = mpicc
+
+LDFLAGS = -lnetcdff -lnetcdf -lmpi -shared-intel -lhdf5_hl -lhdf5 -lm -lz -lsz -lbz2 -lxml2 -lcurl
+CFLAGS = -D__IFC

src/extra/python/requirements.txt

@@ -1,7 +1,7 @@
 sh
 jinja2
-git+git://github.com/marshallward/f90nml.git#egg=f90nml
+f90nml
 numpy
 pandas
 xarray
-tqdm
+tqdm