Skip to content

Commit

Permalink
Merge pull request #62 from devgateway/v0.0.19-pre-release
Browse files Browse the repository at this point in the history 
Release v0.0.19
  • Loading branch information
@gmutuhu
gmutuhu authored Aug 31, 2018
2 parents 33bdb36 + f12093f commit c5fac30
Show file tree
Hide file tree
Showing 29 changed files with 6,063 additions and 1,037 deletions.
2 changes: 1 addition & 1 deletion import-core/import-rest/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
<parent>
<groupId>org.devgateway.importtool</groupId>
<artifactId>importtool</artifactId>
<version>0.0.18-SNAPSHOT</version>
<version>0.0.19-SNAPSHOT</version>
</parent>
<artifactId>import-rest</artifactId>
<dependencies>
Expand Down
8 changes: 6 additions & 2 deletions import-core/import-services/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@

<parent>
<groupId>org.devgateway.importtool</groupId>
<version>0.0.18-SNAPSHOT</version>
<version>0.0.19-SNAPSHOT</version>
<artifactId>importtool</artifactId>
</parent>
<artifactId>import-services</artifactId>
Expand All @@ -25,7 +25,11 @@
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
</dependency>

<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-text</artifactId>
<version>1.1</version>
</dependency>
<dependency>
<groupId>org.springframework.hateoas</groupId>
<artifactId>spring-hateoas</artifactId>
Expand Down
114 changes: 79 additions & 35 deletions ...rvices/src/main/java/org/devgateway/importtool/services/processor/AMPStaticProcessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,20 +39,15 @@
// TODO: Better error handling to the end user. Friendlier user messages, specially when referencing a missing dependency

public class AMPStaticProcessor implements IDestinationProcessor {
private static final String DEFAULT_LANGUAGE = "en";
private String descriptiveName = "AMP";

static final String BASEURL_PROPERTY = "AMPStaticProcessor.baseURL";
static final String BASEURL_DEFAULT_VALUE = "http://localhost:8081";

static final String AMP_IATI_ID_FIELD_PROPERTY = "AMPStaticProcessor.ampIatiIdField";
static final String AMP_IATI_ID_FIELD_DEFAULT_VALUE = "project_code";

static final String AMP_IMPLEMENTATION_LEVEL_ID_FIELD_PROPERTY = "AMPStaticProcessor.implementationLevel";
static final Integer AMP_IMPLEMENTATION_LEVEL_ID_DEFAULT_VALUE = 70; // Coming form common AMP configuration

private Log log = LogFactory.getLog(getClass());

private static final String DEFAULT_LANGUAGE_CODE = "en";
static final String AMP_IMPLEMENTATION_LEVEL_ID_FIELD_PROPERTY= "AMPStaticProcessor.implementationLevel";
static final Integer AMP_IMPLEMENTATION_LEVEL_ID_DEFAULT_VALUE = 70; //Coming form common AMP configuration
private Log log = LogFactory.getLog(getClass());
// AMP Configuration Details
private String DEFAULT_ID_FIELD = "amp-identifier";
private String DEFAULT_TITLE_FIELD = "project_title";
Expand Down Expand Up @@ -197,8 +192,9 @@ private Map<String, String> extractMultilanguageText(JsonNode jsonNode) {
Entry<String, JsonNode> entry = it.next();
languages.put(entry.getKey(), entry.getValue().asText());
}
if (languages.size() == 0) {
languages.put(DEFAULT_LANGUAGE, jsonNode.asText());

if (languages.isEmpty()) {
languages.put(DEFAULT_LANGUAGE_CODE, jsonNode.asText());
}
return languages;
}
Expand Down Expand Up @@ -256,8 +252,12 @@ private void updateProject(JsonBean project, InternalDocument source, List<Field
Optional<FieldValueMapping> optValueMappingLocation = valueMappings.stream().filter(n -> {
return n.getSourceField().getFieldName().equals(mapping.getSourceField().getFieldName());
}).findFirst();
project.set(destinationField.getFieldName(),
getCodesFromList(source, optValueMappingLocation.get(), false));

List<JsonBean> locations = getCodesFromList(source, optValueMappingLocation.get(), false);
if (locations != null) {
project.set(destinationField.getFieldName(), locations);
}

Properties props = getExtraInfo(source, optValueMappingLocation.get(), false);
if (props != null) {
@SuppressWarnings("unchecked")
Expand All @@ -275,9 +275,17 @@ private void updateProject(JsonBean project, InternalDocument source, List<Field
return n.getSourceField().getFieldName().equals(mapping.getSourceField().getFieldName());
}).findFirst();
if (optValueMapping.isPresent() && sourceField.isMultiple()) {
project.set(destinationField.getFieldName(), getCodesFromList(source, optValueMapping.get()));
List<JsonBean> values = getCodesFromList(source, optValueMapping.get());
if (values != null) {
project.set(destinationField.getFieldName(), values);
}

} else {
project.set(destinationField.getFieldName(), getCodeFromList(source, optValueMapping.get()));
Integer value = getCodeFromList(source, optValueMapping.get());
if (value != null) {
project.set(destinationField.getFieldName(), value);
}

}
}
break;
Expand Down Expand Up @@ -520,8 +528,12 @@ private JsonBean transformProject(InternalDocument source, List<FieldMapping> fi
Optional<FieldValueMapping> optValueMappingLocation = valueMappings.stream().filter(n -> {
return n.getSourceField().getFieldName().equals(mapping.getSourceField().getFieldName());
}).findFirst();
project.set(destinationField.getFieldName(),
getCodesFromList(source, optValueMappingLocation.get(), false));

List<JsonBean> locations = getCodesFromList(source, optValueMappingLocation.get(), false);
if (locations != null) {
project.set(destinationField.getFieldName(), locations);
}

Properties props = getExtraInfo(source, optValueMappingLocation.get(), false);
if (props != null) {
@SuppressWarnings("unchecked")
Expand All @@ -539,9 +551,15 @@ private JsonBean transformProject(InternalDocument source, List<FieldMapping> fi
return n.getSourceField().getFieldName().equals(mapping.getSourceField().getFieldName());
}).findFirst();
if (optValueMapping.isPresent() && sourceField.isMultiple()) {
project.set(destinationField.getFieldName(), getCodesFromList(source, optValueMapping.get()));
List<JsonBean> values = getCodesFromList(source, optValueMapping.get());
if (values != null) {
project.set(destinationField.getFieldName(), values);
}
} else {
project.set(destinationField.getFieldName(), getCodeFromList(source, optValueMapping.get()));
Integer value = getCodeFromList(source, optValueMapping.get());
if (value != null) {
project.set(destinationField.getFieldName(), value);
}
}
}
break;
Expand Down Expand Up @@ -682,15 +700,22 @@ private List<JsonBean> getSourceFundings(InternalDocument source, List<FieldMapp
List<JsonBean> fundingDetails = providerFundingDetails.get(entry.getValue().get("value"));
if (fundingDetails != null) {
JsonBean funding = new JsonBean();
int donorId = getIdFromList(entry.getValue().get("value"), "participating-org", fieldMappings,
Integer donorId = getIdFromList(entry.getValue().get("value"), "participating-org", fieldMappings,
valueMappings, false);
funding.set("donor_organization_id", donorId);

if (donorId != null) {
funding.set("donor_organization_id", donorId);
}

try {
String typeOfAssistance = source.getStringFields().get("default-finance-type");
if (typeOfAssistance != null) {
funding.set("type_of_assistance", getIdFromList(typeOfAssistance, "default-finance-type",
fieldMappings, valueMappings, true));
Integer typeOfAssistanceValue = getIdFromList(typeOfAssistance, "default-finance-type",
fieldMappings, valueMappings, true);
if (typeOfAssistanceValue != null) {
funding.set("type_of_assistance", typeOfAssistanceValue);
}

}
} catch (ValueMappingException e) {
log.debug("Dependent field not loaded: default-finance-type");
Expand All @@ -699,8 +724,11 @@ private List<JsonBean> getSourceFundings(InternalDocument source, List<FieldMapp
try {
if (source.getStringFields().get("default-aid-type") != null) {
String financingInstrument = source.getStringFields().get("default-aid-type");
funding.set("financing_instrument", getIdFromList(financingInstrument, "default-aid-type",
fieldMappings, valueMappings, true));
Integer financingInstrumentValue = getIdFromList(financingInstrument, "default-aid-type",
fieldMappings, valueMappings, true);
if (financingInstrumentValue != null) {
funding.set("financing_instrument", financingInstrumentValue);
}
}
} catch (ValueMappingException e) {
log.debug("Dependent field not loaded: default-aid-type");
Expand Down Expand Up @@ -878,7 +906,7 @@ private String getCurrencyId(String currencyCode) {
}).findFirst().get();

Optional<FieldValue> foundCurrency = currency.getPossibleValues().stream().filter(n -> {
return n.getValue().equals(currencyCode);
return n.getValue().equalsIgnoreCase(currencyCode);
}).findFirst();

FieldValue currencyValue;
Expand All @@ -895,17 +923,17 @@ private Object getTransactionAmount(String amount, Double percentage) {
return (amountValue * percentage) / 100;
}

private int getIdFromList(String fieldValue, String sourceField, List<FieldMapping> fieldMappings,
private Integer getIdFromList(String fieldValue, String sourceField, List<FieldMapping> fieldMappings,
List<FieldValueMapping> valueMappings, Boolean useCode) throws ValueMappingException {
Optional<FieldValueMapping> optVm = valueMappings.stream().filter(n -> {
return n.getSourceField().getFieldName().equals(sourceField);
}).findFirst();

if (!optVm.isPresent()) {
throw new ValueMappingException(sourceField + " not found.");
if ((!optVm.isPresent()) || (optVm.get().getSourceField().getPossibleValues() == null)) {
throw new ValueMappingException("The mapping for " + sourceField + " is invalid. No source values were found." );
}

FieldValueMapping vm = optVm.get();

FieldValue fvs = vm.getSourceField().getPossibleValues().stream().filter(n -> {
if (useCode) {
return n.getCode().equals(fieldValue);
Expand All @@ -916,7 +944,7 @@ private int getIdFromList(String fieldValue, String sourceField, List<FieldMappi
Integer sourceValueIndex = fvs.getIndex();
Integer destinationValueIndex = vm.getValueIndexMapping().get(sourceValueIndex);
FieldValue fvd = vm.getDestinationField().getPossibleValues().stream().filter(n -> {
return n.getIndex() == destinationValueIndex;
return destinationValueIndex != null && n.getIndex() == destinationValueIndex;
}).findFirst().get();
return Integer.parseInt(fvd.getCode());
}
Expand Down Expand Up @@ -978,7 +1006,7 @@ private Object getMapFromString(InternalDocument source, String destinationField
return langString;
}

private List<JsonBean> getCodesFromList(InternalDocument source, FieldValueMapping mapping) {
private List<JsonBean> getCodesFromList(InternalDocument source, FieldValueMapping mapping) throws ValueMappingException {
return getCodesFromList(source, mapping, true);
}

Expand All @@ -987,7 +1015,6 @@ private Properties getExtraInfo(InternalDocument source, FieldValueMapping mappi
Map<Integer, Integer> valueMapIndex = mapping.getValueIndexMapping();
List<FieldValue> sourcePossibleValues = mapping.getSourceField().getPossibleValues();
String[] stringValues = (String[]) value;
HashMap<Integer, Integer> uniqueValues = new HashMap<Integer, Integer>();

for (int i = 0; i < stringValues.length; i++) {
final int stringValueIndex = i;
Expand All @@ -1003,13 +1030,21 @@ private Properties getExtraInfo(InternalDocument source, FieldValueMapping mappi
return null;
}

private List<JsonBean> getCodesFromList(InternalDocument source, FieldValueMapping mapping, Boolean suffix) {
private List<JsonBean> getCodesFromList(InternalDocument source, FieldValueMapping mapping, Boolean suffix) throws ValueMappingException {
Object value = source.getStringMultiFields().get(mapping.getSourceField().getFieldName());
Map<Integer, Integer> valueMapIndex = mapping.getValueIndexMapping();
List<FieldValue> sourcePossibleValues = mapping.getSourceField().getPossibleValues();
String[] stringValues = (String[]) value;
HashMap<Integer, Integer> uniqueValues = new HashMap<Integer, Integer>();

if (stringValues == null) {
if (mapping.getDestinationField().isRequired()) {
throw new ValueMappingException("The mapping for " + mapping.getSourceField().getDisplayName() + " is invalid. No source values were found." );
} else {
return null;
}
}

for (int i = 0; i < stringValues.length; i++) {
final int stringValueIndex = i;
Optional<FieldValue> optSourceValueIndex = sourcePossibleValues.stream().filter(n -> {
Expand Down Expand Up @@ -1045,14 +1080,23 @@ private List<JsonBean> getCodesFromList(InternalDocument source, FieldValueMappi
return beanList;
}

private Integer getCodeFromList(InternalDocument source, FieldValueMapping mapping) {
private Integer getCodeFromList(InternalDocument source, FieldValueMapping mapping) throws ValueMappingException {
Object value = source.getStringFields().get(mapping.getSourceField().getFieldName());
Map<Integer, Integer> valueMapIndex = mapping.getValueIndexMapping();
List<FieldValue> sourcePossibleValues = mapping.getSourceField().getPossibleValues();
String stringValue = (String) value;
Optional<FieldValue> optSourceValueIndex = sourcePossibleValues.stream().filter(n -> {
return n.getCode().equals(stringValue);
}).findAny();

if (!optSourceValueIndex.isPresent()) {
if (mapping.getDestinationField().isRequired()) {
throw new ValueMappingException("The mapping for " + mapping.getSourceField().getDisplayName() + " is invalid. No source values were found." );
} else {
return null;
}
}

Integer sourceValueIndex = optSourceValueIndex.get().getIndex();
Integer destinationValueIndex = valueMapIndex.get(sourceValueIndex);
List<FieldValue> destinationPossibleValues = mapping.getDestinationField().getPossibleValues();
Expand Down
65 changes: 39 additions & 26 deletions ...-services/src/main/java/org/devgateway/importtool/services/processor/IATI1XProcessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
import org.devgateway.importtool.services.processor.helper.InternalDocument;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;
import org.springframework.util.DigestUtils;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
Expand Down Expand Up @@ -309,6 +310,9 @@ private List<FieldValue> getCodeListValues(String codeListName, Boolean concaten
DocumentBuilder builder = factory.newDocumentBuilder();
Document doc = builder.parse(is);
NodeList nodeList = doc.getElementsByTagName(standardFieldName);
if (nodeList.getLength() == 0) {
nodeList = doc.getElementsByTagName("codelist-item");
}
int index = 0;
for (int i = 0; i < nodeList.getLength(); i++) {
Node node = nodeList.item(i);
Expand Down Expand Up @@ -369,7 +373,6 @@ private NodeList getActivities() throws XPathExpressionException{
}
//Some filters need relative paths
String fieldName = getFieldName(field.getFieldName());

query.append(fieldName + "[");
for (int i = 0;i < filter.getFilters().size(); i++) {
String value = filter.getFilters().get(i);
Expand All @@ -389,6 +392,7 @@ private NodeList getActivities() throws XPathExpressionException{
}else{
query.setLength(query.length() - 1);
}

NodeList activities = (NodeList)xPath.compile(query.toString()).evaluate(this.getDoc(), XPathConstants.NODESET);
return activities;
}
Expand All @@ -397,7 +401,7 @@ private String getFieldName(String fieldName) {
String name = fieldName;
switch(fieldName) {
case "sector":
name = "transaction/sector";
name = "//sector";
break;
}
return name;
Expand Down Expand Up @@ -534,39 +538,48 @@ private List<InternalDocument> extractDocuments(Document doc) throws Exception {
if (fieldElement.getAttribute("role").equals(field.getSubType())) {
final String stringOrgValue = fieldElement.getTextContent();
final String ref = fieldElement.getAttribute("ref");
Map<String, String> orgFields = new HashMap<String, String>();
orgFields.put("value", stringOrgValue);
orgFields.put("role", field.getSubType());
orgFields.put("ref", ref);
orgFields.put("type", fieldElement.getAttribute("type"));
FieldValue fv = new FieldValue();
if(stringOrgValue != null && !stringOrgValue.isEmpty() ){
fv.setCode(stringOrgValue);
fv.setValue(stringOrgValue);
}else{
fv.setCode(ref);
fv.setValue(ref);
}
fv.setSelected(true);
int index = field.getPossibleValues() == null ? 0 : field.getPossibleValues().size();
fv.setIndex(index);
if (field.getPossibleValues() == null) {
field.setPossibleValues(new ArrayList<FieldValue>());
}
if(!field.getPossibleValues().stream().anyMatch(n->{ return n.getCode().equals(stringOrgValue);})) {
field.getPossibleValues().add(fv);
if ((stringOrgValue != null && !stringOrgValue.trim().isEmpty())
|| (ref != null && !ref.trim().isEmpty())) {
Map<String, String> orgFields = new HashMap<String, String>();
orgFields.put("value", stringOrgValue);
orgFields.put("role", field.getSubType());
orgFields.put("ref", ref);
orgFields.put("type", fieldElement.getAttribute("type"));
FieldValue fv = new FieldValue();
if (stringOrgValue != null && !stringOrgValue.isEmpty()) {
fv.setCode(stringOrgValue);
fv.setValue(stringOrgValue);
} else {
fv.setCode(ref);
fv.setValue(ref);
}
fv.setSelected(true);
int index = field.getPossibleValues() == null ? 0
: field.getPossibleValues().size();
fv.setIndex(index);

if (field.getPossibleValues() == null) {
field.setPossibleValues(new ArrayList<FieldValue>());
}

if (!field.getPossibleValues().stream().anyMatch(n -> {
return n.getCode().equals(stringOrgValue);
})) {
field.getPossibleValues().add(fv);
}

document.addOrganizationField(field.getFieldName() + "_" + field.getSubType() + "_"
+ DigestUtils.md5DigestAsHex(fv.getValue().getBytes()), orgFields);
}


document.addOrganizationField(field.getFieldName() + "_" + field.getSubType() + "_" + index, orgFields);
}
}
}

break;
case MULTILANG_STRING:
Map<String, String> mlv = new HashMap<String, String>();
fieldNodeList = element.getElementsByTagName(field.getFieldName());
fieldNodeList = (NodeList) xPath.evaluate(field.getFieldName(), element, XPathConstants.NODESET);
for (int k = 0; k < fieldNodeList.getLength(); ++k) {
Element fieldElement = (Element) fieldNodeList.item(k);
if (fieldElement.getChildNodes().getLength() == 1) {
Expand Down
Loading

0 comments on commit c5fac30

Please sign in to comment.