Merge pull request #45 from nasa/issue_13

Issue 13

cluster_provisioning/dev-e2e/main.tf

@@ -25,7 +25,6 @@ module "common" {
   jenkins_api_user                        = var.jenkins_api_user
   keypair_name                            = var.keypair_name
   jenkins_api_key                         = var.jenkins_api_key
-  artifactory_fn_api_key                  = var.artifactory_fn_api_key
   ops_password                            = var.ops_password
   shared_credentials_file                 = var.shared_credentials_file
   profile                                 = var.profile
@@ -86,6 +85,8 @@ module "common" {
   docker_registry_bucket                  = var.docker_registry_bucket
   use_s3_uri_structure                    = var.use_s3_uri_structure
   inactivity_threshold                    = var.inactivity_threshold
+  artifactory_fn_user                     = var.artifactory_fn_user
+  artifactory_fn_api_key                  = var.artifactory_fn_api_key
   earthdata_user                          = var.earthdata_user
   earthdata_pass                          = var.earthdata_pass
   purge_es_snapshot                       = var.purge_es_snapshot
@@ -99,8 +100,8 @@ locals {
   daac_proxy_cnm_r_arn     = "arn:aws:sns:${var.region}:${var.aws_account_id}:${var.project}-${var.venue}-${module.common.counter}-daac-proxy-cnm-response"
   source_event_arn         = local.default_source_event_arn
   grq_es_url               = "${var.grq_aws_es ? "https" : "http"}://${var.grq_aws_es ? var.grq_aws_es_host : module.common.grq.private_ip}:${var.grq_aws_es ? var.grq_aws_es_port : 9200}"
-  #lambda_repo              = "${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/nisar/sds/pcm/lambda"
   lambda_repo              = "${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/${var.project}/sds/pcm/lambda"
+  #lambda_repo              = "${var.artifactory_base_url}/${var.artifactory_repo}/gov/nasa/jpl/nisar/sds/pcm/lambda"
   crid                     = lower(var.crid)
 }
 

cluster_provisioning/dev-e2e/variables.tf

@@ -44,15 +44,17 @@ variable "bach_api_repo" {
 }
 
 variable "bach_api_branch" {
-  default = "develop"
+#  default = "develop"
+  default = "main"
 }
 
 variable "bach_ui_repo" {
   default = "github.com/nasa/opera-sds-bach-ui.git"
 }
 
 variable "bach_ui_branch" {
-  default = "develop"
+#  default = "develop"
+  default = "main"
 }
 
 variable "product_delivery_branch" {
@@ -86,6 +88,10 @@ variable "jenkins_api_key" {
 variable "artifactory_fn_api_key" {
 }
 
+variable "artifactory_fn_user" {
+  default = ""
+}
+
 variable "ops_password" {
   default = "hysdsops"
 }

cluster_provisioning/int/variables.tf

@@ -97,6 +97,10 @@ variable "artifactory_fn_api_key" {
   default = ""
 }
 
+variable "artifactory_fn_user" {
+  default = ""
+}
+
 variable "keypair_name" {
   default = ""
 }

cluster_provisioning/run_smoke_test.sh

@@ -2,7 +2,7 @@
 source $HOME/.bash_profile
 
 # check args
-if [ "$#" -eq 20 ]; then
+if [ "$#" -eq 19 ]; then
   project=${1}
   environment=${2}
   venue=${3}
@@ -15,14 +15,13 @@ if [ "$#" -eq 20 ]; then
   pcm_branch=${10}
   product_delivery_repo=${11}
   product_delivery_branch=${12}
-  delete_old_job_catalog=${13}
-  mozart_private_ip=${14}
-  isl_bucket=${15}
-  source_event_arn=${16}
-  daac_delivery_proxy=${17}
-  use_daac_cnm=${18}
-  crid=${19}
-  cluster_type=${20}
+  mozart_private_ip=${13}
+  isl_bucket=${14}
+  source_event_arn=${15}
+  daac_delivery_proxy=${16}
+  use_daac_cnm=${17}
+  crid=${18}
+  cluster_type=${19}
 else
   echo "Invalid number or arguments ($#) $*" 1>&2
   exit 1
@@ -48,7 +47,6 @@ cnm_datasets=L3_DSWx_HLS
 #fi
 
 # build/import CNM product delivery
-# TODO chrisjrd: uncomment
 if [ "${use_artifactory}" = true ]; then
   ~/download_artifact.sh -m ${artifactory_mirror_url} -b ${artifactory_base_url} "${artifactory_base_url}/${artifactory_repo}/gov/nasa/jpl/nisar/sds/pcm/hysds_pkgs/container-iems-sds_cnm_product_delivery-${product_delivery_branch}.sdspkg.tar"
   sds pkg import container-iems-sds_cnm_product_delivery-${product_delivery_branch}.sdspkg.tar
@@ -72,8 +70,6 @@ cd ~/.sds/files
 # TODO chrisjrd: uncomment
 #~/mozart/ops/opera-pcm/conf/sds/files/test/update_asg.py ${project}-${venue}-${counter}-opera-job_worker-gpu --desired-capacity 1
 ~/mozart/ops/opera-pcm/conf/sds/files/test/update_asg.py ${project}-${venue}-${counter}-opera-job_worker-small --desired-capacity 7
-#~/mozart/ops/opera-pcm/conf/sds/files/test/update_asg.py ${project}-${venue}-${counter}-opera-job_worker-sciflo-l3_dswx_hls --desired-capacity 7
-#~/mozart/ops/nisar-pcm/conf/sds/files/test/update_asg.py ${project}-${venue}-${counter}-opera-job_worker-dswx-hls-acct --desired-capacity 5
 ~/mozart/ops/opera-pcm/conf/sds/files/test/update_asg.py ${project}-${venue}-${counter}-opera-job_worker-send_cnm_notify --desired-capacity 7
 ~/mozart/ops/opera-pcm/conf/sds/files/test/update_asg.py ${project}-${venue}-${counter}-opera-job_worker-rcv_cnm_notify --desired-capacity 7
 #~/mozart/ops/opera-pcm/conf/sds/files/test/update_asg.py ${project}-${venue}-${counter}-opera-job_worker-timer --desired-capacity 1
@@ -98,11 +94,11 @@ else
   sds -d ci remove_job -b ${pcm_branch} https://${pcm_repo}
 fi
 
-if [ "${delete_old_job_catalog}" = true ]; then
-  python ~/mozart/ops/opera-pcm/job_accountability/create_job_accountability_catalog.py --delete_old_catalog
-else
-  python ~/mozart/ops/opera-pcm/job_accountability/create_job_accountability_catalog.py
-fi
+#if [ "${delete_old_job_catalog}" = true ]; then
+#  python ~/mozart/ops/opera-pcm/job_accountability/create_job_accountability_catalog.py --delete_old_catalog
+#else
+#  python ~/mozart/ops/opera-pcm/job_accountability/create_job_accountability_catalog.py
+#fi
 
 # ingest Sacramento AOI to test ingest
 ~/mozart/ops/hysds/scripts/ingest_dataset.py AOI_sacramento_valley ~/mozart/etc/datasets.json --force
@@ -128,24 +124,21 @@ fab -f ~/.sds/cluster.py -R mozart,grq create_all_user_rules_index
 # verify number of ingested ancillary/auxiliary products
 #~/mozart/ops/opera-pcm/conf/sds/files/test/check_datasets_file.py --crid=${crid} datasets_e2e.json 1,2 /tmp/datasets.txt
 
-# stage met_required files to ISL
-#./stage_met_required_files_to_s3.sh ${isl_bucket}
-
-# verify number of ingested ancillary/auxiliary, NEN, and LDF
-#~/mozart/ops/opera-pcm/conf/sds/files/test/check_datasets_file.py --max_time=2700 --crid=${crid} datasets_e2e.json 1,2,3 /tmp/datasets.txt
+# stage L2_HLS_L30 & L2_HLS_S30 files to ISL
+./stage_l2_hls_to_s3.sh ${isl_bucket}
 
 # verify accountability table counts
 #python ~/mozart/ops/opera-pcm/conf/sds/files/test/check_accountability.py --max_time=2700 pass_accountability_catalog 12 /tmp/pass_accountability_catalog.txt
 
 # simulate reception of CNM-R messages from the DAAC and submit jobs for stamping their response on the dataset
-if [ "${use_daac_cnm}" = false ]; then
-  python ~/mozart/ops/opera-pcm/conf/sds/files/test/submit_cnm_r_msg.py --datasets ${cnm_datasets} ${source_event_arn} /tmp/cnm_r_stamped_dataset.json
-else
-  python ~/mozart/ops/opera-pcm/conf/sds/files/test/submit_cnm_r_msg.py --datasets ${cnm_datasets} --no_simulation  ${source_event_arn} /tmp/cnm_r_stamped_dataset.json
-fi
+#if [ "${use_daac_cnm}" = false ]; then
+#  python ~/mozart/ops/opera-pcm/conf/sds/files/test/submit_cnm_r_msg.py --datasets ${cnm_datasets} ${source_event_arn} /tmp/cnm_r_stamped_dataset.json
+#else
+#  python ~/mozart/ops/opera-pcm/conf/sds/files/test/submit_cnm_r_msg.py --datasets ${cnm_datasets} --no_simulation  ${source_event_arn} /tmp/cnm_r_stamped_dataset.json
+#fi
 
 # check that the datasets got stamped
-python ~/mozart/ops/opera-pcm/conf/sds/files/test/check_stamped_dataset.py /tmp/cnm_r_stamped_dataset.json daac_delivery_status /tmp/check_stamped_dataset_result.txt
+#python ~/mozart/ops/opera-pcm/conf/sds/files/test/check_stamped_dataset.py /tmp/cnm_r_stamped_dataset.json daac_delivery_status /tmp/check_stamped_dataset_result.txt
 
 # check that the ISL was cleaned out by the trigger rule
 python ~/mozart/ops/opera-pcm/conf/sds/files/test/check_empty_isl.py ${isl_bucket} /tmp/check_empty_isl_result.txt

opera_chimera/wf_xml/L3_HLS.sf.xml

@@ -51,6 +51,23 @@
           </sf:op>
         </sf:operator>
       </sf:process>
+
+      <!--sf:process id="post_L3_HLS">
+        <sf:inputs>
+          <sf_context/>
+          <job_id_PGE_L0B_Prime from="@#previous"/>
+          <pge_config_file>/home/ops/verdi/ops/opera-pcm/opera_chimera/configs/pge_configs/PGE_L3_HLS.yaml</pge_config_file>
+          <settings_file>/home/ops/verdi/ops/opera-pcm/conf/settings.yaml</settings_file>
+          <chimera_config_file>/home/ops/verdi/ops/opera-pcm/opera_chimera/configs/chimera_config.yaml</chimera_config_file>
+        </sf:inputs>
+        <sf:outputs/>
+        <sf:operator>
+          <sf:description>Postprocessing step for PGE L3_HLS</sf:description>
+          <sf:op>
+            <sf:binding>python:/home/ops/verdi/ops/chimera/chimera/post_processor.py?post_processor.post_process</sf:binding>
+          </sf:op>
+        </sf:operator>
+      </sf:process-->
     </sf:processes>
   </sf:flow>
 </sf:sciflo>