Initial commit

Author: arnim

.gitignore

@@ -0,0 +1,25 @@
+syntax: glob
+*.class
+.metadata
+.directory
+.keep
+.DS_Store
+*/\${builder}
+
+**/*.log*
+
+**/*.jar
+
+*/lib/*jar
+**/target
+**/.settings
+target
+bin
+*~
+*.orig
+.classpath
+.project
+.settings
+.cproject
+.Rhistory
+.Rapp.history

README.txt

@@ -0,0 +1,10 @@
+Hierarchical Dirichlet Process Gibbs sampling 
+=============================================
+
+
+(Re-)Implementation attempt of:
+Hierarchical Bayesian Nonparametric Models with Applications. 
+Y.W. Teh and M.I. Jordan. Bayesian Nonparametrics, 2010. Cambridge University Press.
+http://www.gatsby.ucl.ac.uk/~ywteh/research/npbayes/TehJor2010a.pdf
+
+

pom.xml

@@ -0,0 +1,26 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+	<modelVersion>4.0.0</modelVersion>
+	<groupId>de.uni-leipzig.informatik.asv</groupId>
+	<artifactId>hdp</artifactId>
+	<properties>
+		<maven.compiler.source>1.6</maven.compiler.source>
+		<maven.compiler.target>1.6</maven.compiler.target>
+		<encoding>UTF-8</encoding>
+	</properties>
+
+	<name>Hierarchical Dirichlet Processes</name>
+	<version>0.0.1-SNAPSHOT</version>
+	<description>Hierarchical Dirichlet Processes with Gibbs Sampling</description>
+	<dependencies>
+	</dependencies>
+
+	<build>
+		<plugins>
+		</plugins>
+	</build>
+	<repositories>
+	</repositories>
+</project>
+

src/main/java/de/uni_leipzig/informatik/asv/hdp/Corpus.java

@@ -0,0 +1,53 @@
+package de.uni_leipzig.informatik.asv.hdp;
+
+import java.io.BufferedReader;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+
+public class Corpus {
+
+	public int sizeVocabulary = 0;
+	public int totalNumberOfWords = 0;
+	public ArrayList<Document> docs;
+
+	public void read(String filename) throws FileNotFoundException {
+
+		InputStream is = new FileInputStream(filename);
+		int length, word;
+		Document d;
+
+		try {
+			docs = new ArrayList<Document>();
+			BufferedReader br = new BufferedReader(new InputStreamReader(is,
+					"UTF-8"));
+			String line = null;
+			while ((line = br.readLine()) != null) {
+				try {
+					String[] fields = line.split(" ");
+					length = Integer.parseInt(fields[0]);
+					d = new Document(length);
+					for (int n = 0; n < length; n++) {
+						String[] wordCounts = fields[n + 1].split(":");
+						word = Integer.parseInt(wordCounts[0]);
+						d.words[n] = word;
+						d.counts[n] = Integer.parseInt(wordCounts[1]);
+						d.total += Integer.parseInt(wordCounts[1]);
+						if (word >= sizeVocabulary)
+							sizeVocabulary = word + 1;
+					}
+					totalNumberOfWords += d.total;
+					docs.add(d);
+				} catch (Exception e) {
+					System.err.println(e.getMessage() + "\n");
+				}
+			}
+		} catch (Exception e) {
+			e.printStackTrace();
+		}
+
+	}
+
+}

src/main/java/de/uni_leipzig/informatik/asv/hdp/DOCState.java

@@ -0,0 +1,57 @@
+package de.uni_leipzig.informatik.asv.hdp;
+
+import java.util.ArrayList;
+import java.util.Collections;
+
+public class DOCState {
+	
+	static int idCounter = 0;
+	
+	int docID, documentLength, numberOfTables;
+
+    ArrayList<Integer> tableToTopic = new ArrayList<Integer>(); 
+    ArrayList<Integer> wordCountByTable = new ArrayList<Integer>(); 
+	WordInfo[] words;
+
+	
+	public DOCState(Document doc){  
+		docID = idCounter++;
+	    numberOfTables = 0;  
+	    documentLength = doc.total;
+	    words = new WordInfo[documentLength];	
+		for (int k = 0; k < 2; k++){
+			tableToTopic.add(null);
+			wordCountByTable.add(0);
+		}
+	    int word, count, m = 0;
+	    for (int n = 0; n < doc.numberOfUniquTerms; n++) {
+	        word  = doc.words[n];
+	        count = doc.counts[n];
+	        for (int j = 0; j < count; j++) {
+	            words[m] = new WordInfo(word, -1);
+	            m++;
+	        }
+	    }
+	}
+
+	
+	public void defragment(int[] kOldToKNew) {
+	    int[] tOldToTNew = new int[numberOfTables];
+	    int t, newNumberOfTables;
+	    for (t = 0, newNumberOfTables = 0; t < numberOfTables; t++){
+	        if (wordCountByTable.get(t) > 0){
+	            tOldToTNew[t] = newNumberOfTables;
+	            tableToTopic.set(newNumberOfTables, kOldToKNew[tableToTopic.get(t)]);
+	            Collections.swap(tableToTopic, newNumberOfTables, t);
+	            newNumberOfTables ++;
+	        } else
+	        	tableToTopic.set(t, -1);
+	    }
+	    numberOfTables = newNumberOfTables;
+	    for (int i = 0; i < documentLength; i++)
+	        words[i].tableAssignment = tOldToTNew[words[i].tableAssignment];
+	}
+
+	
+	
+}

src/main/java/de/uni_leipzig/informatik/asv/hdp/Document.java

@@ -0,0 +1,16 @@
+package de.uni_leipzig.informatik.asv.hdp;
+
+public class Document {
+
+	public int[] words = null;
+	public int[] counts = null;
+	public int numberOfUniquTerms = 0;
+	public int total = 0;
+
+	public Document(int len) {
+		numberOfUniquTerms = len;
+		words = new int[numberOfUniquTerms];
+		counts = new int[numberOfUniquTerms];
+	}
+
+}