mycoSORT src update

config-sample.cfg

@@ -12,6 +12,18 @@ HOME_DIR=/.
 # corpus directory
 CORPUS_DIR=corpus/
 #
+# source documents directory
+SOURCE_DIR=src/
+#
+# duplicate documents directory 
+DUP_DIR=test/
+#
+# positive instances directory
+POS_DIR=positives/
+#
+# negative instances directory 
+NEG_DIR=negatives/
+#
 # train directory
 TRAIN_DIR=train/
 #
@@ -61,6 +73,9 @@ NGRAM_FEATURES=ngrams_features.txt
 # Paper title n-grams feature list
 TITLE_NGRAMS=titleGrams.txt
 #
+# Paper ID and class
+DOC_IDS=docIDs.txt
+#
 ###################################################
 ########################## FEATURE SETUP ##########
 # Extract size of abstract and title 
@@ -78,6 +93,9 @@ FEATURE_MIN_FREQ=2
 # minimum length (in chars) to consider a feature
 FEATURE_MIN_LENGTH=3
 #
+# extract document IDs
+USE_DOC_ID=true
+#
 #############################
 ######### ANNOTATIONS #######
 # Extract annotation content
@@ -109,6 +127,20 @@ NGRAM_SIZE=1
 # Define weight of features
 #WEIGHT=3
 #
+###################################################
+########################## FEATURE SELECTION SETUP ##########
+# Enable Odds Ratio (OR) filtering
+USE_ODDS_RATIO=false
+#
+# Define minimum OR threshold to keep attribute 
+OR_THRESHOLD=1
+#
+# Enable inverted document frequency (idf) filtering
+USE_IDF=false
+#
+# Define minimum OR threshold to keep attribute
+IDF_THRESHOLD=1
+#
 #################################################
 ########################### TASK SETUP ##########
 # experiment type : train = 0 / test = 1

src/.gitignore

@@ -0,0 +1,3 @@
+*.classpath
+*.project
+*.*~

src/analyse/Extractor.java

@@ -51,7 +51,7 @@ public class Extractor {
 
 	//String pathFile;
 	String id;
-	String endId;
+	protected String endId;
 	String openFile;	
 	String endFile;
 	String openAbst;
@@ -95,7 +95,8 @@ public String removeSpecialChar(String str){
 		str = str.replace(")", "");
 		str = str.replace("(", "");
 		str = str.replace("\t\t", "\t");
-		str = str.replace("-", "");
+		//losing ngrams because of hifen between names 
+		str = str.replace("-", " ");
 		str = str.replace("  ", "");
 
 		return str;
@@ -154,15 +155,15 @@ public String removeAbstractTags(String str){
 		//this order of removing tags matters to 
 		//exclude the first tag from the abstracts.
 
-		str = str.replace("<AbstractText>", "");
-		str = str.replace("<AbstractText", "");
-		str = str.replace("<CopyrightInformation>", "");
-		str = str.replace("</CopyrightInformation>", "");
-		str = str.replace("Copyright", "");		
-		str = str.replace("</AbstractText>", "");
-		str = str.replace("<Abstract>", "");
-		str = str.replace("</Abstract>", "");
-		str = str.replace("<AbstractText.*?>", "");		
+		str = str.replace("<abstracttext>", "");
+		str = str.replace("<abstracttext", "");
+		str = str.replace("<copyrightinformation>", "");
+		str = str.replace("</copyrightinformation>", "");
+		str = str.replace("copyright", "");		
+		str = str.replace("</abstractText>", "");
+		str = str.replace("<abstract>", "");
+		str = str.replace("</abstract>", "");
+		str = str.replace("<abstracttext.*?>", "");	
 
 		return str;
 	}