Merge pull request #520 from nasa/v1.9.1-release-lf

V1.9.1 release lf

Author: laurenfrederick

CHANGELOG.md

@@ -6,14 +6,23 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## [Unreleased]
 
+## [v1.9.1] - 2018-08-22
+
+**Please Note** To take advantage of the added granule tracking API functionality, updates are required for the message adapter and its libraries. You should be on the following versions:
+- `cumulus-message-adapter` 1.0.9+
+- `cumulus-message-adapter-js` 1.0.4+
+- `cumulus-message-adapter-java` 1.2.7+
+- `cumulus-message-adapter-python` 1.0.5+
+
 ### Added
 
-- **CUMULUS-687** Added logs endpoint to search for logs from a specific workflow execution. Added integration test
+- **CUMULUS-687** Added logs endpoint to search for logs from a specific workflow execution in `@cumulus/api`. Added integration test.
 - **CUMULUS-836** - `@cumulus/deployment` supports a configurable docker storage driver for ECS. ECS can be configured with either `devicemapper` (the default storage driver for AWS ECS-optimized AMIs) or `overlay2` (the storage driver used by the NGAP 2.0 AMI). The storage driver can be configured in `app/config.yml` with `ecs.docker.storageDriver: overlay2 | devicemapper`. The default is `overlay2`.
   - To support this configuration, a [Handlebars](https://handlebarsjs.com/) helper `ifEquals` was added to `packages/deployment/lib/kes.js`.
 - **CUMULUS-836** - `@cumulus/api` added IAM roles required by the NGAP 2.0 AMI. The NGAP 2.0 AMI runs a script `register_instances_with_ssm.py` which requires the ECS IAM role to include `ec2:DescribeInstances` and `ssm:GetParameter` permissions.
 
 ### Fixed
+
 - **CUMULUS-836** - `@cumulus/deployment` uses `overlay2` driver by default and does not attempt to write `--storage-opt dm.basesize` to fix [this error](https://github.com/moby/moby/issues/37039).
 - **CUMULUS-413** Kinesis processing now captures all errrors.
   - Added kinesis fallback mechanism when errors occur during record processing.
@@ -25,15 +34,14 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - **CUMULUS-855** Adds integration test for kinesis' error path.
 - **CUMULUS-686** Added workflow task name and version tracking via `@cumulus/api` executions endpoint under new `tasks` property, and under `workflow_tasks` in step input/output.
   - Depends on `cumulus-message-adapter` 1.0.9+, `cumulus-message-adapter-js` 1.0.4+, `cumulus-message-adapter-java` 1.2.7+ and `cumulus-message-adapter-python` 1.0.5+
-
-### Fixed
 - **CUMULUS-771**
   - Updated sync-granule to stream the remote file to s3
   - Added integration test for ingesting granules from ftp provider
   - Updated http/https integration tests for ingesting granules from http/https providers
 - **CUMULUS-862** Updated `@cumulus/integration-tests` to handle remote lambda output
 
 ### Changed
+
 - In `@cumulus/deployment`, changed the example app config.yml to have additional IAM roles
 
 ## [v1.9.0] - 2018-08-06
@@ -444,7 +452,8 @@ We may need to update the api documentation to reflect this.
 
 ## [v1.0.0] - 2018-02-23
 
-[Unreleased]: https://github.com/nasa/cumulus/compare/v1.9.0...HEAD
+[Unreleased]: https://github.com/nasa/cumulus/compare/v1.9.1...HEAD
+[v1.9.1]: https://github.com/nasa/cumulus/compare/v1.9.0...v1.9.1
 [v1.9.0]: https://github.com/nasa/cumulus/compare/v1.8.1...v1.9.0
 [v1.8.1]: https://github.com/nasa/cumulus/compare/v1.8.0...v1.8.1
 [v1.8.0]: https://github.com/nasa/cumulus/compare/v1.7.0...v1.8.0

example/package.json

@@ -23,23 +23,23 @@
     ]
   },
   "dependencies": {
-    "@cumulus/api": "^1.9.0",
-    "@cumulus/common": "^1.9.0",
-    "@cumulus/deployment": "^1.9.0",
-    "@cumulus/discover-granules": "^1.9.0",
-    "@cumulus/discover-pdrs": "^1.9.0",
-    "@cumulus/hello-world": "^1.9.0",
-    "@cumulus/integration-tests": "^1.9.0",
-    "@cumulus/move-granules": "^1.9.0",
-    "@cumulus/parse-pdr": "^1.9.0",
-    "@cumulus/pdr-status-check": "^1.9.0",
-    "@cumulus/post-to-cmr": "^1.9.0",
-    "@cumulus/queue-granules": "^1.9.0",
-    "@cumulus/queue-pdrs": "^1.9.0",
-    "@cumulus/sf-sns-report": "^1.9.0",
-    "@cumulus/sync-granule": "^1.9.0",
-    "@cumulus/test-data": "^1.9.0",
-    "@cumulus/test-processing": "^1.9.0",
+    "@cumulus/api": "^1.9.1",
+    "@cumulus/common": "^1.9.1",
+    "@cumulus/deployment": "^1.9.1",
+    "@cumulus/discover-granules": "^1.9.1",
+    "@cumulus/discover-pdrs": "^1.9.1",
+    "@cumulus/hello-world": "^1.9.1",
+    "@cumulus/integration-tests": "^1.9.1",
+    "@cumulus/move-granules": "^1.9.1",
+    "@cumulus/parse-pdr": "^1.9.1",
+    "@cumulus/pdr-status-check": "^1.9.1",
+    "@cumulus/post-to-cmr": "^1.9.1",
+    "@cumulus/queue-granules": "^1.9.1",
+    "@cumulus/queue-pdrs": "^1.9.1",
+    "@cumulus/sf-sns-report": "^1.9.1",
+    "@cumulus/sync-granule": "^1.9.1",
+    "@cumulus/test-data": "^1.9.1",
+    "@cumulus/test-processing": "^1.9.1",
     "aws-sdk": "^2.227.1",
     "child-process-promise": "^2.2.1",
     "sleep-promise": "^8.0.1"

example/spec/helpers/kinesisHelpers.js

@@ -226,7 +226,7 @@ function kinesisEventFromSqsMessage(message) {
   catch (error) {
     console.log('Error parsing KinesisEventFromSqsMessage(message)', JSON.stringify(message));
     console.log(error);
-    kinesisEvent = { identifier: 'Bad Message' };
+    kinesisEvent = { identifier: 'Fake Wrong Message' };
   }
   return kinesisEvent;
 }
@@ -244,29 +244,28 @@ function isTargetMessage(message, recordIdentifier) {
 }
 
 /**
- * Wait until a kinesisRecord appears in an SQS message who's identifier matches the input recordIdentifier.  Wait up to 5 minutes.
+ * Wait until a kinesisRecord appears in an SQS message who's identifier matches the input recordIdentifier.  Wait up to 10 minutes.
  *
  * @param {string} recordIdentifier - random string to match found messages against.
  * @param {string} queueUrl - kinesisFailure SQS url
- * @param {number} maxNumberElapsedPeriods - number of timeout intervals to wait.
+ * @param {number} maxNumberElapsedPeriods - number of timeout intervals (5 seconds) to wait.
  * @returns {Object} - matched Message from SQS.
  */
-async function waitForQueuedRecord(recordIdentifier, queueUrl, maxNumberElapsedPeriods = 60) {
+async function waitForQueuedRecord(recordIdentifier, queueUrl, maxNumberElapsedPeriods = 120) {
   const timeoutInterval = 5000;
   let queuedRecord;
   let elapsedPeriods = 0;
 
   while (!queuedRecord && elapsedPeriods < maxNumberElapsedPeriods) {
     const messages = await receiveSQSMessages(queueUrl);
     if (messages.length > 0) {
-
       const targetMessage = messages.find((message) => isTargetMessage(message, recordIdentifier));
       if (targetMessage) return targetMessage;
     }
     await timeout(timeoutInterval);
     elapsedPeriods += 1;
   }
-  return {};
+  return { waitForQueuedRecord: 'never found record on queue' };
 }
 
 module.exports = {

example/spec/kinesisTests/KinesisTestErrorSpec.js

@@ -36,6 +36,7 @@ describe('The kinesisConsumer receives a bad record.', () => {
   beforeAll(async () => {
     this.defaultTimeout = jasmine.DEFAULT_TIMEOUT_INTERVAL;
     jasmine.DEFAULT_TIMEOUT_INTERVAL = 10 * 60 * 1000;
+    this.maxNumberElapsedPeriods = jasmine.DEFAULT_TIMEOUT_INTERVAL / 5000;
     await tryCatchExit(async () => {
       await createOrUseTestStream(streamName);
       console.log(`\nWaiting for active streams: '${streamName}'.`);
@@ -56,7 +57,7 @@ describe('The kinesisConsumer receives a bad record.', () => {
   it('Eventually puts the bad record on the failure queue.', async () => {
     console.log('\nWait for minimum duration of failure process ~3min');
     console.log('\nWait for record on:', failureSqsUrl);
-    const queuedRecord = await waitForQueuedRecord(testRecordIdentifier, failureSqsUrl);
+    const queuedRecord = await waitForQueuedRecord(testRecordIdentifier, failureSqsUrl, this.maxNumberElapsedPeriods);
     this.ReceiptHandle = queuedRecord.ReceiptHandle;
     const queuedKinesisEvent = kinesisEventFromSqsMessage(queuedRecord);
     expect(queuedKinesisEvent).toEqual(badRecord);

example/workflows.yml

@@ -446,6 +446,11 @@ IngestGranule:
               destination: '{$.cumulus_meta.process}'
       Type: Task
       Resource: ${SyncGranuleLambdaFunction.Arn}
+      Retry:
+          - ErrorEquals:
+              - States.ALL
+            IntervalSeconds: 2
+            MaxAttempts: 3
       Catch:
         - ErrorEquals:
           - States.ALL
@@ -470,6 +475,11 @@ IngestGranule:
               destination: '{$.cumulus_meta.process}'
       Type: Task
       Resource: ${SyncGranuleNoVpcLambdaFunction.Arn}
+      Retry:
+          - ErrorEquals:
+              - States.ALL
+            IntervalSeconds: 2
+            MaxAttempts: 3
       Catch:
         - ErrorEquals:
           - States.ALL
@@ -624,11 +634,11 @@ KinesisTriggerTest:
           Next: StopStatus
       Next: StopStatus
     CnmResponseFail:
-      <<: *CnmResponseDefaults  
+      <<: *CnmResponseDefaults
       Catch:
         - <<: *ErrorEqualDefaults
-          Next: StopStatusFail          
-      Next: StopStatusFail     
+          Next: StopStatusFail
+      Next: StopStatusFail
     StopStatus: &StopStatusDefaults
       Type: Task
       Resource: ${sf2snsEndLambdaFunction.Arn}

lerna.json

@@ -1,6 +1,6 @@
 {
   "lerna": "2.9.0",
-  "version": "1.9.0",
+  "version": "1.9.1",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

packages/api/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cumulus/api",
-  "version": "1.9.0",
+  "version": "1.9.1",
   "description": "Lambda functions for handling all daac's API operations",
   "main": "index.js",
   "engine": {
@@ -34,10 +34,10 @@
   "author": "Cumulus Authors",
   "license": "Apache-2.0",
   "dependencies": {
-    "@cumulus/cmrjs": "^1.9.0",
-    "@cumulus/common": "^1.9.0",
-    "@cumulus/ingest": "^1.9.0",
-    "@cumulus/pvl": "^1.9.0",
+    "@cumulus/cmrjs": "^1.9.1",
+    "@cumulus/common": "^1.9.1",
+    "@cumulus/ingest": "^1.9.1",
+    "@cumulus/pvl": "^1.9.1",
     "@mapbox/dyno": "^1.4.1",
     "ajv": "^5.2.2",
     "archiver": "^2.1.1",

packages/cmrjs/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cumulus/cmrjs",
-  "version": "1.9.0",
+  "version": "1.9.1",
   "description": "A node SDK for CMR",
   "engine": {
     "node": ">=8.10.0"
@@ -31,7 +31,7 @@
   "author": "Cumulus Authors",
   "license": "Apache-2.0",
   "dependencies": {
-    "@cumulus/common": "^1.9.0",
+    "@cumulus/common": "^1.9.1",
     "got": "^8.3.0",
     "lodash.property": "^4.4.2",
     "public-ip": "^2.3.5",

packages/common/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cumulus/common",
-  "version": "1.9.0",
+  "version": "1.9.1",
   "description": "Common utilities used across tasks",
   "keywords": [
     "GIBS",
@@ -40,7 +40,7 @@
   "author": "Cumulus Authors",
   "license": "Apache-2.0",
   "dependencies": {
-    "@cumulus/test-data": "^1.9.0",
+    "@cumulus/test-data": "^1.9.1",
     "ajv": "^5.2.2",
     "async": "^2.0.0",
     "aws-sdk": "^2.238.1",

packages/deployment/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cumulus/deployment",
-  "version": "1.9.0",
+  "version": "1.9.1",
   "description": "Deployment templates for cumulus",
   "scripts": {
     "test": "ava",