CJ-165 | Add support for bulk in data models

common/src/main/java/org/apache/atlas/repository/Constants.java

@@ -203,6 +203,49 @@ public final class Constants {
 
     public static String[] PROCESS_EDGE_LABELS = {PROCESS_OUTPUTS, PROCESS_INPUTS};
 
+    /***
+     * DataModel
+     */
+    public static final String MODEL_ENTITY = "ModelEntity";
+    public static final String MODEL_ATTRIBUTE = "ModelAttribute";
+    public static final String MODEL_DATA_MODEL = "ModelDataModel";
+    public static final String MODEL_VERSION = "ModelVersion";
+    public static final String MODEL_ENTITY_ASSOCIATION = "ModelEntityAssociation";
+    public static final String MODEL_ATTRIBUTE_ASSOCIATION = "ModelAttributeAssociation";
+    public static final String MODEL_QUALIFIED_NAME_PATTERN = "modelVersionAgnosticQualifiedName";
+    public static final String MODEL_NAMESPACE = "modelNamespace";
+    public static final String MODEL_EXPIRED_AT_SYSTEM_DATE = "modelExpiredAtSystemDate";
+    public static final String MODEL_EXPIRED_AT_BUSINESS_DATE = "modelExpiredAtBusinessDate";
+    public static final String MODEL_SYSTEM_DATE = "modelSystemDate";
+    public static final String MODEL_BUSINESS_DATE = "modelBusinessDate";
+    public static final String MODEL_MODEL_VERSION_RELATION = "model_data_model_model_versions";
+    public static final String RELATED_DATA_MODEL = "modelDataModel";
+    public static final String RELATED_MODEL_VERSIONS = "modelVersions";
+    public static final String MODEL_VERSIONS_MODEL_VERSION_ENTITY_RELATION = "model_versions_model_version_entities";
+    public static final String MODEL_VERSION_ENTITIES = "modelVersionEntities";
+    public static final String MODEL_ENTITY_MAPPED_FROM_ENTITIES = "modelEntityMappedFromEntities";
+    public static final String MODEL_ENTITY_MAPPED_TO_ENTITIES = "modelEntityMappedToEntities";
+    public static final String MODEL_ENTITY_MODEL_ATTRIBUTES_RELATION = "model_attribute_entities_model_entity_attributes";
+    public static final String MODEL_ENTITY_ATTRIBUTES = "modelEntityAttributes";
+    public static final String MODEL_ATTRIBUTE_ENTITIES = "modelAttributeEntities";
+    public static final String MODEL_ATTRIBUTE_MAPPED_TO_ATTRIBUTES = "modelAttributeMappedToAttributes";
+    public static final String MODEL_ATTRIBUTE_MAPPED_FROM_ATTRIBUTES = "modelAttributeMappedFromAttributes";
+    public static final String MODEL_ENTITY_RELATED_FROM_ENTITIES = "modelEntityRelatedFromEntities";
+    public static final String MODEL_ENTITY_RELATED_TO_ENTITIES = "modelEntityRelatedToEntities";
+    public static final String MODEL_ATTRIBUTE_RELATED_FROM_ATTRIBUTES = "modelAttributeRelatedFromAttributes";
+    public static final String MODEL_ATTRIBUTE_RELATED_TO_ATTRIBUTES = "modelAttributeRelatedToAttributes";
+    public static String MODEL_ENTITY_ASSOCIATION_TO = "modelEntityAssociationTo";
+    public static String MODEL_ENTITY_ASSOCIATION_FROM = "modelEntityAssociationFrom";
+    public static  String MODEL_ATTRIBUTE_ASSOCIATION_FROM = "modelAttributeAssociationFrom";
+    public static String MODEL_ATTRIBUTE_ASSOCIATION_TO = "modelAttributeAssociationTo";
+
+
+
+
+
+
+
+
     /**
      * The homeId field is used when saving into Atlas a copy of an object that is being imported from another
      * repository. The homeId will be set to a String that identifies the other repository. The specific format
@@ -376,6 +419,8 @@ public final class Constants {
     public static final String NON_COMPLIANT_ASSET_POLICY_GUIDS  = "nonCompliantAssetPolicyGUIDs";
     public static final String ASSET_POLICIES_COUNT  = "assetPoliciesCount";
 
+
+
     /*
      * All supported file-format extensions for Bulk Imports through file upload
      */

intg/src/main/java/org/apache/atlas/AtlasErrorCode.java

@@ -291,8 +291,15 @@ public enum AtlasErrorCode {
     TASK_TYPE_NOT_SUPPORTED(400, "ATLAS-400-00-112", "Task type {0} is not supported"),
 
     PERSONA_POLICY_ASSETS_LIMIT_EXCEEDED(400, "ATLAS-400-00-113", "Exceeded limit of maximum allowed assets across policies for a Persona: Limit: {0}, assets: {1}"),
-    ADMIN_LIST_SHOULD_NOT_BE_EMPTY(400, "ATLAS-400-00-114", "Admin list should not be empty for type {0}");
-
+    ADMIN_LIST_SHOULD_NOT_BE_EMPTY(400, "ATLAS-400-00-114", "Admin list should not be empty for type {0}"),
+    DATA_MODEL_VERSION_NOT_EXIST(400, "ATLAS-400-00-115", "Model version {0} does not exist"),
+    DATA_ENTITY_NOT_EXIST(400, "ATLAS-400-00-116", "modelEntity {0} does not exist"),
+    DATA_MODEL_NOT_EXIST(400, "ATLAS-400-00-117", "DataModel {0} does not exist"),
+    QUALIFIED_NAME_PREFIX_NOT_EXIST(400, "ATLAS-400-00-118", "modelAgnosticQualifiedName is mandatory for modelEntity/modelAttribute"),
+    NO_TYPE_EXISTS_FOR_QUALIFIED_NAME_PREFIX (400,"ATLAS-400-00-119", "No modelEntity/modelAttribute exists for modelAgnosticQualifiedName : {0}"),
+    NAME_NAMESPACE_NOT_EXIST (400, "ATLAS-400-00-120", "name/namespace are mandatory for modelEntity/modelAttribute"),
+    QUALIFIED_NAME_PREFIX_OR_TYPE_NOT_FOUND(400, "ATLAS-400-00-121", "qualifiedName/entityType are mandatory"),
+    INVALID_ENTITY_TYPE(400, "ATLAS-400-00-122", "Invalid entity type");
     private String errorCode;
     private String errorMessage;
     private Response.Status httpCode;

repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityGraphDiscoveryV2.java

@@ -24,7 +24,9 @@
 import org.apache.atlas.model.instance.AtlasEntity;
 import org.apache.atlas.model.instance.AtlasObjectId;
 import org.apache.atlas.model.instance.AtlasStruct;
+import org.apache.atlas.repository.Constants;
 import org.apache.atlas.repository.graphdb.AtlasGraph;
+import org.apache.atlas.repository.graphdb.AtlasVertex;
 import org.apache.atlas.repository.store.graph.EntityGraphDiscovery;
 import org.apache.atlas.repository.store.graph.EntityGraphDiscoveryContext;
 import org.apache.atlas.repository.store.graph.EntityResolver;
@@ -103,8 +105,36 @@ public void validateAndNormalize(AtlasEntity entity) throws AtlasBaseException {
 
         validateLabels(entity.getLabels());
 
+
         type.validateValue(entity, entity.getTypeName(), messages);
 
+
+
+        // DMEntity and DMAttributeType are requested for update
+        // by dMQualifiedNamePrefix which is not a unique attribute
+        // This can return multiple entity/attribute that match this prefix vale
+      //   we have to return latest entity/attribute
+        if (entity.getTypeName().equals(Constants.MODEL_ENTITY) ||
+                entity.getTypeName().equals(Constants.MODEL_ATTRIBUTE)){
+
+            String qualifiedNamePrefix = (String) entity.getAttributes().get(Constants.MODEL_QUALIFIED_NAME_PATTERN);
+            if (qualifiedNamePrefix.isEmpty()){
+                throw new AtlasBaseException(AtlasErrorCode.QUALIFIED_NAME_PREFIX_NOT_EXIST);
+            }
+           // AtlasVertex vertex = AtlasGraphUtilsV2.findLatestEntityAttributeVerticesByType(entity.getTypeName(), qualifiedNamePrefix);
+            AtlasVertex vertex= discoveryContext.getResolvedEntityVertex(entity.getGuid());
+            if (vertex == null) {
+                // no entity exists with this qualifiedName, set qualifiedName and let entity be created
+               entity.setAttribute(Constants.QUALIFIED_NAME, qualifiedNamePrefix + "_" + RequestContext.get().getRequestTime());
+                return;
+            }
+
+         //   if guidFromVertex is found let entity be updated
+            entity.setGuid(AtlasGraphUtilsV2.getIdFromVertex(vertex));
+            type.getNormalizedValue(entity);
+            return;
+        }
+
         if (!messages.isEmpty()) {
             throw new AtlasBaseException(AtlasErrorCode.INSTANCE_CRUD_INVALID_PARAMS, messages);
         }
@@ -161,6 +191,8 @@ protected void discover() throws AtlasBaseException {
                     throw new AtlasBaseException(AtlasErrorCode.INVALID_PARAMETERS, "found null entity");
                 }
 
+                validateAttributesForDataModel(entity);
+
                 processDynamicAttributes(entity);
 
                 walkEntityGraph(entity);
@@ -485,4 +517,15 @@ private void processDynamicAttributes(AtlasEntity entity) throws AtlasBaseExcept
             }
         }
     }
+
+    private void validateAttributesForDataModel(AtlasEntity entity) throws AtlasBaseException {
+        if (entity.getTypeName().equals(Constants.MODEL_ENTITY) ||
+                entity.getTypeName().equals(Constants.MODEL_ATTRIBUTE)) {
+            if (entity.getAttributes().get(Constants.MODEL_QUALIFIED_NAME_PATTERN) == null ||
+                    entity.getAttributes().get(Constants.MODEL_QUALIFIED_NAME_PATTERN) == "") {
+              throw new AtlasBaseException(AtlasErrorCode.QUALIFIED_NAME_PREFIX_NOT_EXIST);
+            }
+        }
+
+    }
 }