ci: update WATS plots & other minor fixes

WRF-CMake · Sep 5, 2019 · 71bed2f · 71bed2f
1 parent b61aed3
commit 71bed2f
Show file tree

Hide file tree

Showing 4 changed files with 53 additions and 35 deletions.
diff --git a/.ci/azure-pipelines/wats_plots.yml b/.ci/azure-pipelines/wats_plots.yml
@@ -3,7 +3,7 @@
 
 parameters:
   DOMAINS: [d01, d02]
-  TIMES: [0, 3, 6]
+  TIMES: [0, 6]
 
 jobs:
 - job: wats_plots
@@ -62,13 +62,6 @@ jobs:
                 itemPattern: 'wats_${{ OS_NAME }}_${{ BUILD_SYSTEM }}_${{ BUILD_TYPE }}_${{ MODE }}/**'
 
             - ${{ each DOMAIN in parameters.DOMAINS }}:
-              - script: |
-                  set -ex
-                  O=wats_outputs
-                  W="python wats/wats/plots.py compute --stats-dir wats_stats/${{ DOMAIN }}/TALL --filter ${{ DOMAIN }} --append"
-                  $W $O/wats_Linux_Make_Debug_serial $O/wats_${{ OS_NAME }}_${{ BUILD_SYSTEM }}_${{ BUILD_TYPE }}_${{ MODE }}
-                displayName: 'Run WATS: compute plot stats Linux/Make/Debug/serial vs ${{ OS_NAME }}/${{ BUILD_SYSTEM }}/${{ BUILD_TYPE }}/${{ MODE }} for domain ${{ DOMAIN }}'
-              
               - ${{ each TIME in parameters.TIMES }}:
                 - script: |
                     set -ex
@@ -85,32 +78,50 @@ jobs:
       rm -rf wats_outputs/wats_Linux_Make_Debug_serial
     displayName: Remove reference Linux/Make/Debug/serial WATS outputs 
 
-  - task: PublishBuildArtifacts@1
-    displayName: Store WATS plot stats
-    inputs:
-      pathtoPublish: wats_stats
-      artifactName: wats_stats
-
-  - ${{ each DOMAIN in parameters.DOMAINS }}:
-    - script: |
-        python wats/wats/plots.py plot --stats-dir wats_stats/${{ DOMAIN }}/TALL --plots-dir wats_plots/${{ DOMAIN }}/TALL/OSALL
-      displayName: 'Run WATS: create plots for domain ${{ DOMAIN }}'
-    
-    - ${{ each OS_NAME in parameters.OS_NAMES }}:
+  - ${{ each DOMAIN in parameters.DOMAINS }}:   
+    - ${{ each TIME in parameters.TIMES }}:
       - script: |
-          python wats/wats/plots.py plot --stats-dir wats_stats/${{ DOMAIN }}/TALL --plots-dir wats_plots/${{ DOMAIN }}/TALL/${{ OS_NAME }} --filter ${{ OS_NAME }}
-        displayName: 'Run WATS: create plots for domain ${{ DOMAIN }} - ${{ OS_NAME }}'
+          python wats/wats/plots.py plot --stats-dir wats_stats/${{ DOMAIN }}/T${{ TIME }} --plots-dir wats_plots/single_ref/${{ DOMAIN }}/T${{ TIME }}
+        displayName: 'Run WATS: create Linux/Make/Debug/serial reference plots for domain ${{ DOMAIN }} - T${{ TIME }}'
 
+  - script: |
+      rm -rf wats_stats/
+    displayName: Remove WATS stats files
+  
+  # Plot: CMake vs Make
+
+  - ${{ each OS_NAME in parameters.OS_NAMES }}:
+    - ${{ each BUILD_TYPE in parameters.BUILD_TYPES }}:
+      - ${{ each MODE in parameters.MODES }}:
+        # For Windows, since there is no Make-based build, skip that combination.
+        - ${{ if not(eq(OS_NAME, 'Windows')) }}:
+
+          - task: DownloadBuildArtifacts@0
+            displayName: Retrieve ${{ OS_NAME }}/{Make,CMake}/${{ BUILD_TYPE }}/${{ MODE }} WATS outputs
+            inputs:
+              downloadPath: wats_outputs
+              downloadType: specific
+              itemPattern: 'wats_${{ OS_NAME }}_*_${{ BUILD_TYPE }}_${{ MODE }}/**'
+
+          - ${{ each DOMAIN in parameters.DOMAINS }}:
+            - ${{ each TIME in parameters.TIMES }}:
+              - script: |
+                  set -ex
+                  O=wats_outputs
+                  W="python wats/wats/plots.py compute --stats-dir wats_stats/${{ DOMAIN }}/T${{ TIME }} --filter ${{ DOMAIN }} --time-idx ${{ TIME }} --append"
+                  $W $O/wats_${{ OS_NAME }}_Make_${{ BUILD_TYPE }}_${{ MODE }} $O/wats_${{ OS_NAME }}_CMake_${{ BUILD_TYPE }}_${{ MODE }}
+                displayName: 'Run WATS: compute plot stats ${{ OS_NAME }}/Make/${{ BUILD_TYPE }}/${{ MODE }} vs ${{ OS_NAME }}/CMake/${{ BUILD_TYPE }}/${{ MODE }} for domain ${{ DOMAIN }} - T${{ TIME }}'
+
+          - script: |
+              rm -rf wats_outputs/wats_${{ OS_NAME }}_*_${{ BUILD_TYPE }}_${{ MODE }}
+            displayName: Remove ${{ OS_NAME }}/{Make,CMake}/${{ BUILD_TYPE }}/${{ MODE }} WATS outputs 
+
+  - ${{ each DOMAIN in parameters.DOMAINS }}:   
     - ${{ each TIME in parameters.TIMES }}:
       - script: |
-          python wats/wats/plots.py plot --stats-dir wats_stats/${{ DOMAIN }}/T${{ TIME }} --plots-dir wats_plots/${{ DOMAIN }}/T${{ TIME }}/OSALL
-        displayName: 'Run WATS: create plots for domain ${{ DOMAIN }} - T${{ TIME }}'
-      
-      - ${{ each OS_NAME in parameters.OS_NAMES }}:
-        - script: |
-            python wats/wats/plots.py plot --stats-dir wats_stats/${{ DOMAIN }}/T${{ TIME }} --plots-dir wats_plots/${{ DOMAIN }}/T${{ TIME }}/${{ OS_NAME }} --filter ${{ OS_NAME }}
-          displayName: 'Run WATS: create plots for domain ${{ DOMAIN }} - T${{ TIME }} - ${{ OS_NAME }}'
-  
+          python wats/wats/plots.py plot --stats-dir wats_stats/${{ DOMAIN }}/T${{ TIME }} --plots-dir wats_plots/make_cmake/${{ DOMAIN }}/T${{ TIME }}
+        displayName: 'Run WATS: create Make vs CMake plots for domain ${{ DOMAIN }} - T${{ TIME }}'
+
   - task: PublishBuildArtifacts@1
     displayName: Store WATS plots
     inputs:

diff --git a/.ci/azure-pipelines/wats_run.yml b/.ci/azure-pipelines/wats_run.yml
@@ -29,6 +29,13 @@ steps:
 - bash: |
     set -ex
     if [[ $MODE == dm* ]]; then mpi_flag="--mpi"; fi
+    if [[ $OS_NAME == macOS ]]; then
+      # Work around Open MPI issue
+      # https://github.com/open-mpi/ompi/issues/6518
+      # https://github.com/open-mpi/ompi/issues/5798
+      # https://www.mail-archive.com/[email protected]/msg20760.html
+      export OMPI_MCA_btl=self,tcp
+    fi
     if [[ $BUILD_SYSTEM == "CMake" ]]; then dir_suffix="build/install"; fi
     python wats/wats/main.py run --mode $WATS_MODE --wrf-dir ./$dir_suffix --wps-dir ../WPS/$dir_suffix --wps-case-output-dir wats_wps_outputs/00 --work-dir wats_work $mpi_flag
   displayName: Run WATS

diff --git a/.ci/unix/setup-dependencies.sh b/.ci/unix/setup-dependencies.sh
@@ -49,15 +49,15 @@ if [ "$(uname)" == "Linux" ]; then
         else
             # From bionic onwards, libjasper is not available via apt-get.
             cd /tmp
-            curl --retry ${HTTP_RETRIES} https://www.ece.uvic.ca/~frodo/jasper/software/jasper-2.0.14.tar.gz | tar xz
-            cd jasper-2.0.14/build/
+            curl -L --retry ${HTTP_RETRIES} https://github.com/mdadams/jasper/archive/version-2.0.14.tar.gz | tar xz
+            cd jasper-version-2.0.14/build/
             cmake -DCMAKE_INSTALL_PREFIX=/usr ..
             sudo make install
         fi
 
         # Need to build netcdf-fortran manually as the Fortran compiler versions have to match.
         cd /tmp
-        curl --retry ${HTTP_RETRIES} https://www.unidata.ucar.edu/downloads/netcdf/ftp/netcdf-fortran-4.4.4.tar.gz | tar xz
+        curl -L --retry ${HTTP_RETRIES} https://github.com/Unidata/netcdf-fortran/archive/v4.4.4.tar.gz | tar xz
         cd netcdf-fortran-4.4.4
         sed -i 's/ADD_SUBDIRECTORY(examples)/#ADD_SUBDIRECTORY(examples)/' CMakeLists.txt
         mkdir build && cd build
@@ -77,7 +77,7 @@ if [ "$(uname)" == "Linux" ]; then
             # Need to build mpich manually as the Fortran compiler versions have to match.
             MPICH_VERSION=3.2.1
             cd /tmp
-            curl --retry ${HTTP_RETRIES} http://www.mpich.org/static/downloads/${MPICH_VERSION}/mpich-${MPICH_VERSION}.tar.gz | tar xz
+            curl -L --retry ${HTTP_RETRIES} https://www.mpich.org/static/downloads/${MPICH_VERSION}/mpich-${MPICH_VERSION}.tar.gz | tar xz
             cd mpich-${MPICH_VERSION}
             ./configure --prefix=/usr
             make -j 4

diff --git a/.ci/windows/install-netcdf.sh b/.ci/windows/install-netcdf.sh
@@ -23,7 +23,7 @@ rm -rf $MINGW_PREFIX/lib/cmake/netCDF # breaks for some reason otherwise in netc
 rm -rf * # avoid cmake cache using this directly in netcdf-fortran
 
 cd /tmp
-curl --retry ${HTTP_RETRIES} https://www.unidata.ucar.edu/downloads/netcdf/ftp/netcdf-fortran-4.4.4.tar.gz | tar xz
+curl -L --retry ${HTTP_RETRIES} https://github.com/Unidata/netcdf-fortran/archive/v4.4.4.tar.gz | tar xz
 cd netcdf-fortran-4.4.4
 sed -i 's/ADD_SUBDIRECTORY(examples)/#ADD_SUBDIRECTORY(examples)/' CMakeLists.txt # patch CMakeLists.txt and comment out example building
 mkdir build && cd build