Merge branch 'master' into CUMULUS-527---Parse-pdr-queues-up-selected…

…-granules

CHANGELOG.md

@@ -22,10 +22,18 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - **CUMULUS-533** Added fields to granule indexer to support EMS ingest and archive record creation
 - You can now deploy cumulus without ElasticSearch. Just add `es: null` to your `app/config.yml` file. This is only useful for debugging purposes. Cumulus still requires ElasticSearch to properly operate.
 - `@cumulus/integration-tests` includes and exports the `addRules` function, which seeds rules into the DynamoDB table.
+- **CUMULUS-534** Track deleted granules
+  - added `deletedgranule` type to `cumulus` index.
+  - **Important Note:** Force custom bootstrap to re-run by adding this to app/config.yml
+  `es: 
+      elasticSearchMapping: 7`
 - Added capability to support EFS in cloud formation template. Also added optional capability to ssh to your instance and privileged lambda functions.
 - Added support to force discovery of PDRs that have already been processed and filtering of selected data types
 - `@cumulus/cmrjs` uses an environment variable `USER_IP_ADDRESS` or fallback IP address of `10.0.0.0` when a public IP address is not available. This supports lambda functions deployed into a VPC's private subnet, where no public IP address is available.
 
+### Changed
+- CUMULUS-550 Custom bootstrap automatically adds new types to index on deployment
+
 ## [v1.5.1] - 2018-04-23
 ### Fixed
 - add the missing dist folder to the hello-world task

packages/api/es/indexer.js

@@ -456,16 +456,26 @@ async function granule(esClient, payload, index = defaultIndexAlias, type = 'gra
         }
       }
 
-      return esClient.update({
+      // If the granule exists in 'deletedgranule', delete it first before inserting the granule
+      // into ES.  Ignore 404 error, so the deletion still succeeds if the record doesn't exist.
+      const delGranParams = {
         index,
-        type,
+        type: 'deletedgranule',
         id: doc.granuleId,
         parent: collectionId,
-        body: {
-          doc,
-          doc_as_upsert: true
-        }
-      });
+        ignore: [404]
+      };
+      return esClient.delete(delGranParams)
+        .then(() => esClient.update({
+          index,
+          type,
+          id: doc.granuleId,
+          parent: collectionId,
+          body: {
+            doc,
+            doc_as_upsert: true
+          }
+        }));
     }
     return Promise.resolve();
   });
@@ -496,8 +506,29 @@ async function deleteRecord(esClient, id, type, parent, index = defaultIndexAlia
   if (parent) {
     params.parent = parent;
   }
-
-  return esClient.delete(params);
+  const result = await esClient.get(params);
+  return esClient.delete(params)
+    .then(async (response) => {
+      if (type === 'granule' && result.found) {
+        const doc = result._source;
+        doc.timestamp = Date.now();
+        doc.deletedAt = Date.now();
+
+        // When a 'granule' record is deleted, the record is added to 'deletedgranule'
+        // type for EMS report purpose.
+        await esClient.update({
+          index,
+          type: 'deletedgranule',
+          id: doc.granuleId,
+          parent: parent,
+          body: {
+            doc,
+            doc_as_upsert: true
+          }
+        });
+      }
+      return response;
+    });
 }
 
 /**

packages/api/lambdas/bootstrap.js

@@ -14,7 +14,6 @@
 'use strict';
 
 const got = require('got');
-const url = require('url');
 const get = require('lodash.get');
 const log = require('@cumulus/common/log');
 const { DefaultProvider } = require('@cumulus/ingest/crypto');
@@ -24,6 +23,27 @@ const { Search, defaultIndexAlias } = require('../es/search');
 const mappings = require('../models/mappings.json');
 const physicalId = 'cumulus-bootstraping-daac-ops-api-deployment';
 
+/**
+ * Check the index to see if mappings have been added since the index
+ * was last updated. Return any missing types from the mapping.
+ *
+ * @param {Object} esClient - elasticsearch client instance
+ * @param {string} index - index name (cannot be alias)
+ * @param {Array<string>} types - list of types to check against
+ * @returns {Array<string>} - list of missing indices
+ */
+async function findMissingMappings(esClient, index, types) {
+  const typesResponse = await esClient.indices.getMapping({
+    index
+  });
+
+  const indexMappings = get(typesResponse, index);
+
+  const indexTypes = Object.keys(indexMappings.mappings);
+
+  return types.filter((x) => !indexTypes.includes(x));
+}
+
 /**
  * Initialize elastic search. If the index does not exist, create it with an alias.
  * If an index exists but is not aliased, alias the index.
@@ -58,6 +78,10 @@ async function bootstrapElasticSearch(host, index = 'cumulus', alias = defaultIn
     log.info(`index ${index} created with alias ${alias} and mappings added.`);
   }
   else {
+    log.info(`index ${index} already exists`);
+
+    let aliasedIndex = index;
+
     const aliasExists = await esClient.indices.existsAlias({
       name: alias
     });
@@ -70,8 +94,30 @@ async function bootstrapElasticSearch(host, index = 'cumulus', alias = defaultIn
 
       log.info(`Created alias ${alias} for index ${index}`);
     }
+    else {
+      const indices = await esClient.indices.getAlias({ name: alias });
 
-    log.info(`index ${index} already exists`);
+      aliasedIndex = Object.keys(indices)[0];
+
+      if (indices.length > 1) {
+        log.info(`Multiple indices found for alias ${alias}, using index ${index}.`);
+      }
+    }
+
+    const missingTypes = await findMissingMappings(esClient, aliasedIndex, Object.keys(mappings));
+
+    if (missingTypes.length > 0) {
+      const addMissingTypesPromises = missingTypes.map((type) =>
+        esClient.indices.putMapping({
+          index: aliasedIndex,
+          type,
+          body: get(mappings, type)
+        }));
+
+      await Promise.all(addMissingTypesPromises);
+
+      log.info(`Added missing types to index ${aliasedIndex}: ${missingTypes}`);
+    }
   }
 }
 
@@ -187,7 +233,9 @@ function handler(event, context, cb) {
 
 module.exports = {
   handler,
-  bootstrapElasticSearch
+  bootstrapElasticSearch,
+  // for testing
+  findMissingMappings 
 };
 
 justLocalRun(() => {

packages/api/models/mappings.json

@@ -166,6 +166,57 @@
       }
     }
   },
+  "deletedgranule": {
+    "dynamic_templates": [
+      {
+        "all_date_fileds": {
+          "match": "*At",
+          "match_mapping_type": "long",
+          "mapping": {
+            "type": "date"
+          }
+        }
+      }
+    ],
+    "_parent": {
+      "type": "collection"
+    },
+    "properties": {
+      "execution": {
+        "type": "keyword"
+      },
+      "granuleId": {
+        "type": "keyword"
+      },
+      "pdrName": {
+        "type": "keyword"
+      },
+      "collectionId": {
+        "type": "keyword"
+      },
+      "status": {
+        "type": "keyword"
+      },
+      "provider": {
+        "type": "keyword"
+      },
+      "cmrLink": {
+        "type": "keyword"
+      },
+      "published": {
+        "type": "boolean"
+      },
+      "duration": {
+        "type": "float"
+      },
+      "timestamp": {
+        "type": "date"
+      },
+      "deletedAt": {
+        "type": "date"
+      }
+    }
+  },
   "provider": {
     "dynamic_templates": [
       {

packages/api/package.json

@@ -64,6 +64,7 @@
   },
   "devDependencies": {
     "ava": "^0.25.0",
+    "delay": "^2.0.0",
     "nyc": "^11.6.0",
     "sinon": "^4.5.0",
     "webpack": "~4.5.0",