build.xml

<!--======================================================================
Build file for OpenSextantToolbox.  
=======================================================================-->
<project name="OpenSextantToolbox" basedir="." default="release" xmlns:ivy="antlib:org.apache.ivy.ant">

	<!-- include properties -->
	<property file="build.properties" />

	<!--set the time stamps -->
	<tstamp />

	<!-- Project level properties -->
	<property name="project.name" value="OpenSextantToolbox" />
	<property name="version" value="3.0" />
	<property name="releaseStatus" value="integration" />
	<property name="java_target_version" value="1.8" />

	<!-- Source directory - contains the Java source files for this project -->
	<property name="src.dir" location="src" />

	<!-- directory to contain the created jars -->
	<property name="build.dir" location="build" />

	<!-- documentation directories -->
	<property name="doc.dir" location="${basedir}/doc" />
	<property name="javadoc.dir" location="${doc.dir}/javadoc" />

	<!-- lib directories  -->
	<property name="lib.dir" location="lib" />

	<!-- directory to contain the release -->
	<property name="release.dir" location="${basedir}/release" />

	<!-- location of the Language Resources directory -->
	<property name="resources.home" location="./LanguageResources" />

	<!-- location for the generated JAR files -->
	<property name="core.jar.location" location="${build.dir}/opensextant.jar" />
	<property name="toolbox.jar.location" location="${build.dir}/opensextant-toolbox.jar" />

	<!-- location of the license file -->
	<property name="license.location" location="../LICENSE.txt" />

	<!-- Classpath for compiling -->
	<path id="compile.classpath">
		<fileset dir="${lib.dir}/GATE" includes="**/*.jar" />
		<fileset dir="${lib.dir}/Standalone" includes="**/*.jar" />
	</path>

	<!--Gazetteer URL and  version as tagged in Gazettteer Release -->
	<property name="gazetteerVersion" value="v20160717" />
	<property name="gaz.file.name" value="opensextant-gazetteer.zip" />
	<property name="gazetteer.url" value="https://github.com/OpenSextant/Gazetteer/releases/download/${gazetteerVersion}/${gaz.file.name}" />

	<!-- Classpath to run solr gazetteer loader -->
	<path id="solr.runtime.classpath">
		<fileset dir="${build.dir}" includes="opensextant.jar" />
		<fileset dir="${lib.dir}/Standalone" includes="**/*.jar" />
	</path>

	<!-- Properties for creating/loading the Solr gazetteer -->
	<property name="solr.home" location="${basedir}/build/solr" />

	<!-- For testing we can use "MergedGazetteer_SMALL.txt", for production use MergedGazetteer.txt -->
	<!--property name="csv.gaz" value="${resources.home}/resources/geoGazetteer/MergedGazetteer.txt" /-->
	<property name="csv.gaz" value="${resources.home}/resources/geoGazetteer/MergedGazetteer_SMALL.txt" />


	<!--property name="fieldnames.gaz" value="id,place_id,name,lat,lon,feat_class,feat_code,FIPS_cc,cc,ISO3_cc,adm1,adm2,adm3,adm4,adm5,source,src_place_id,src_name_id,script,conflate_key,name_bias,id_bias,name_type,name_type_system,partition" /-->
	<property name="index.vocab" value="${resources.home}/resources/Vocabularies/vocab.def" />
	<!--property name="fieldnames.vocab" value="id,phrase,category,taxonomy" /-->

	<!-- Properties for example code -->

	<!-- Classpath to run example code which uses GATE -->
	<path id="examples.runtime.classpath">
		<fileset dir="${release.dir}/lib" includes="**/*.jar" />
	</path>

	<!-- classpath to run the components that don't need Gate -->
	<path id="examples.runtime.standalone.classpath">
		<fileset dir="${release.dir}/lib" includes="*.jar" />
	</path>


	<!-- location of the test data -->
	<property name="test.testdocs.dir" location="${release.dir}/LanguageResources/TestData/testDocs" />
	<property name="test.matcher.file" location="${release.dir}/LanguageResources/TestData/MatcherTestData/matcherTest.txt" />
	<property name="test.dateTest.file" location="${release.dir}/LanguageResources/TestData/RegexTestData/dateTime_test.csv" />
	<property name="test.geoTest.file" location="${release.dir}/LanguageResources/TestData/RegexTestData/geocoord_test.csv" />


	<!-- location of the GAPP and pattern files -->
	<property name="test.gapp.geotagger" location="${release.dir}/LanguageResources/GAPPs/OpenSextant_Geotagger.gapp" />
	<property name="test.gapp.generalPurpose" location="${release.dir}/LanguageResources/GAPPs/OpenSextant_GeneralPurpose.gapp" />

	<property name="test.patterns.date" location="${release.dir}/LanguageResources/resources/patterns/datetime_patterns2.cfg" />
	<property name="test.patterns.geo" location="${release.dir}/LanguageResources/resources/patterns/geocoord_regex.cfg" />

	<property name="test.patterns.results" location="${release.dir}/LanguageResources/TestData/results" />



	<!-- Properties for local GATE install -->
	<property name="gate.plugin.dir" location="${gate.home}/plugins" />
	<property name="toolbox.plugin.dir" location="${gate.plugin.dir}/OpenSextantToolbox" />


	<!-- properties for ivy config -->
	<property name="ivy.install.version" value="2.1.0-rc2" />
	<condition property="ivy.home" value="${env.IVY_HOME}">
		<isset property="env.IVY_HOME" />
	</condition>
	<property name="ivy.home" value="${user.home}/.ant" />
	<property name="ivy.jar.dir" value="${ivy.home}/lib" />
	<property name="ivy.jar.file" value="${ivy.jar.dir}/ivy.jar" />



	<!--====================== Targets ============================-->
	<target name="setup">
		<mkdir dir="${build.dir}" />
		<antcall target="setProxy" />
	</target>

	<!--Make sure ivy is installed -->
	<target name="download-ivy" unless="offline">

		<mkdir dir="${ivy.jar.dir}" />
		<get src="http://repo2.maven.org/maven2/org/apache/ivy/ivy/${ivy.install.version}/ivy-${ivy.install.version}.jar" dest="${ivy.jar.file}" usetimestamp="true" />
	</target>

	<target name="init-ivy" depends="download-ivy">
		<path id="ivy.lib.path">
			<fileset dir="${ivy.jar.dir}" includes="*.jar" />

		</path>
		<taskdef resource="org/apache/ivy/ant/antlib.xml" uri="antlib:org.apache.ivy.ant" classpathref="ivy.lib.path" />
	</target>



	<!-- delete created jars,retrieved dependencies,javadocs and previous releases-->
	<!-- NOTE: this leaves the created Gazetteer (see clean.gaz target) -->
	<target name="clean" depends="setup">
		<delete includeEmptyDirs="true">
			<fileset dir="${build.dir}" excludes="solr/**,opensextant-gazetteer.zip" />
		</delete>

		<delete dir="${release.dir}" />
		<delete dir="${lib.dir}" />
		<delete dir="${javadoc.dir}" />

		<mkdir dir="${build.dir}" />
		<mkdir dir="${release.dir}" />
		<mkdir dir="${lib.dir}" />
		<mkdir dir="${javadoc.dir}" />
	</target>

	<!-- delete the created Solr gazetteer -->
	<!-- NOTE: creating (loading) the gazetteer takes a while so don't delete if even mildly impatient -->
	<target name="clean.gaz" depends="setup">
		<delete includeEmptyDirs="true" quiet="true">
			<fileset dir="${solr.home}" />
		</delete>
	</target>


	<!-- set the proxy if behind a firewall (needed for ivy) -->
	<!-- the host and proxy values are set in the build.properties file -->
	<!-- omit from build.properties if no proxy -->
	<target name="setProxy">
		<property name="proxy.host" value="" />
		<property name="proxy.port" value="80" />
		<setproxy proxyhost="${proxy.host}" proxyport="${proxy.port}" />
	</target>

	<!-- get all the ivy defined dependencies and the gazetteer -->
	<target name="getDependencies" depends="clean">
		<ivy:retrieve pattern="${ivy.lib.dir}/[conf]/[artifact].[ext]" />
		<ivy:report todir="${build.dir}/ivy-report" graph="true" />
		<antcall target="getGazetteer" />
	</target>

	<target name="check-gazetteer">
		<available file="${build.dir}/${gaz.file.name}" property="gaz.present" />
	</target>

	<target name="getGazetteer" depends="check-gazetteer" unless="gaz.present">
		<get src="${gazetteer.url}" dest="${build.dir}" />
		<unzip src="${build.dir}/${gaz.file.name}" dest="${resources.home}/resources/geoGazetteer" />
	</target>


	<!-- compile the source  -->
	<target name="compile" depends="setup,getDependencies">
		<javac classpathref="compile.classpath" srcdir="${src.dir}" destdir="${build.dir}" debug="true" debuglevel="lines,source" source="${java_target_version}" encoding="utf-8" />
	</target>

	<!-- create the JAR files -->
	<target name="jar" depends="compile">

		<jar destfile="${core.jar.location}" update="false" basedir="${build.dir}">
			<include name="org/opensextant/**" />
			<exclude name="org/opensextant/toolbox/**" />
		</jar>

		<jar destfile="${toolbox.jar.location}" update="false" basedir="${build.dir}">
			<include name="org/opensextant/toolbox/**" />
		</jar>

	</target>

	<target name="plugin" depends="jar">
		<!-- zip up the toolbox jar and the creole.xml  -->
		<zip destfile="${build.dir}/opensextant-toolbox-plugin.zip">
			<zipfileset file="${basedir}/gate/creole.xml" prefix="OpenSextantToolbox" />
			<zipfileset file="${build.dir}/opensextant-toolbox.jar" prefix="OpenSextantToolbox/lib" />
		</zip>
	</target>

	<!-- Build and populate the Solr gazetteer -->
	<target name="build.gaz">
		<antcall target="clean.gaz" />
		<antcall target="create.solr.home" />
		<antcall target="load.gazetteer" />
		<!--antcall target="load.vocabulary.index" /-->
	</target>

	<!-- create the empty solr home directory -->
	<target name="create.solr.home">
		<copy todir="${solr.home}">
			<fileset dir="${basedir}/solr">
				<include name="**/*" />
			</fileset>
		</copy>
	</target>

	<!-- Load the gazetteer data into Solr -->
	<!-- NOTE: this is the step that takes a long time -->
	<target name="load.gazetteer">
		<java classname="org.opensextant.matching.DataLoader" classpathref="solr.runtime.classpath" fork="yes" dir="${basedir}">
			<jvmarg value="-Xmx3G" />
			<jvmarg value="-Dlog4j.configuration=file:${basedir}/etc/log4j.properties" />
			<jvmarg value="-Dsolr.home=${solr.home}" />
			<arg value="gazetteer" />
			<arg value="CSV" />
			<arg value="${csv.gaz}" />
		</java>
	</target>

	<!-- Load the vocabulary data into Solr (from a vocab index file)-->
	<!--target name="load.vocabulary.index">
		<java classname="org.opensextant.matching.DataLoader" classpathref="solr.runtime.classpath" fork="yes" dir="${basedir}">
			<jvmarg value="-Xmx3G" />
			<jvmarg value="-XX:MaxPermSize=256m" />
			<jvmarg value="-Dlog4j.configuration=file:${basedir}/etc/log4j.properties" />
			<jvmarg value="-Dsolr.home=${solr.home}" />
			<arg value="vocabulary" />
			<arg value="INDEX" />
			<arg value="${index.vocab}" />
		</java>
	</target-->

	<!-- create JavaDoc directory -->
	<target name="javadoc.prepare">
		<delete dir="${javadoc.dir}" />
		<mkdir dir="${javadoc.dir}" />
	</target>

	<!-- Generate JavaDoc documentation for this project -->
	<target name="javadoc" depends="javadoc.prepare">
		<javadoc destdir="${javadoc.dir}" packagenames="*" classpathref="compile.classpath" encoding="utf-8" access="public" windowtitle="OpenSextant JavaDoc">
			<sourcepath>
				<pathelement location="${src.dir}" />
			</sourcepath>
		</javadoc>
	</target>

	<!-- Main target: Build the whole thing (except for loading the gazetteer -->
	<target name="build" depends="plugin" />

	<!-- Build the jars and load the gazetteer (which takes a while)  -->
	<target name="buildWithGaz" depends="plugin,build.gaz" />

	<!-- create release directory structure -->
	<target name="release.prepare">
		<delete dir="${release.dir}" />

		<mkdir dir="${release.dir}" />
		<mkdir dir="${release.dir}/lib" />
		<mkdir dir="${release.dir}/plugins" />
		<mkdir dir="${release.dir}/LanguageResources" />
		<mkdir dir="${release.dir}/solr" />
		<mkdir dir="${release.dir}/etc" />
	</target>

	<!-- copy over jars to the release directory -->
	<target name="release.libs" depends="">

		<copy todir="${release.dir}/lib">
			<fileset file="${core.jar.location}" />
			<fileset dir="${basedir}/lib/Standalone" />
		</copy>

		<copy todir="${release.dir}/lib/GATE">
			<fileset dir="${basedir}/lib/GATE">
				<include name="*.jar" />
			</fileset>
		</copy>

	</target>

	<!-- copy over the plugin to the release directory -->
	<target name="release.plugin" depends="">
		<copy file="${basedir}/gate/creole.xml" tofile="${release.dir}/plugins/OpenSextantToolbox/creole.xml" />
		<copy todir="${release.dir}/plugins/OpenSextantToolbox/lib">
			<fileset file="${toolbox.jar.location}" />
		</copy>
		<copy todir="${release.dir}/plugins/ANNIE">
			<fileset dir="${basedir}/gate/plugins/ANNIE" />
		</copy>
	</target>


	<!-- Copy over the resource files -->
	<target name="release.resources">
		<copy todir="${release.dir}/LanguageResources">
			<fileset dir="${resources.home}">
				<exclude name="**/geoGazetteer/" />
			</fileset>
		</copy>
	</target>

	<!-- Copy over the some misc config stuff -->
	<target name="release.misc">
		<copy todir="${release.dir}/etc" file="${basedir}/etc/log4j.properties" />
		<copy todir="${release.dir}/etc" file="${basedir}/etc/restletlog.properties" />
		<copy todir="${release.dir}/etc" file="${basedir}/etc/service-config.properties" />
		<copy todir="${release.dir}" file="${basedir}/gate/gate.xml" />
		<copy todir="${release.dir}" file="${basedir}/gate/user-gate.xml" />
		<copy todir="${release.dir}" file="${doc.dir}/README_DISTRO.txt" />
	</target>

	<!-- Copy over the scritps -->
	<target name="release.scripts">
		<copy todir="${release.dir}/scripts">
			<fileset dir="${basedir}/scripts" />
		</copy>
	</target>

	<!-- Copy the gaz to the release folder-->
	<target name="release.gaz">
		<copy todir="${release.dir}/solr">
			<fileset dir="${build.dir}/solr" />
		</copy>
	</target>


	<!-- Build a full release in the "release" directory -->
	<target name="release" depends="buildWithGaz">
		<antcall target="release.prepare" />
		<antcall target="release.libs" />
		<antcall target="release.plugin" />
		<antcall target="release.resources" />
		<antcall target="release.gaz" />
		<antcall target="release.misc" />
		<antcall target="release.scripts" />

		<!-- delete existing release zip if there -->
		<delete quiet="true" file="${basedir}/opensextant-toolbox-${version}.zip" />

		<!-- zip up the finished release -->
		<zip destfile="${basedir}/opensextant-toolbox-${version}.zip">
			<zipfileset dir="${release.dir}" prefix="opensextant-toolbox-${version}" />
		</zip>
	</target>

	<!--  Build a release without the gazetteer in the "release" directory for cloud foundry -->
	<target name="release-pcf" depends="build">
		<antcall target="release.libs" />
		<antcall target="release.plugin" />
		<antcall target="release.resources" />
		<antcall target="release.misc" />
		<copy todir="${release.dir}/bin" file="${basedir}/scripts/opensextant-toolbox" />

		<!-- delete existing release zip if there -->
		<delete quiet="true" file="${basedir}/opensextant-toolbox-${version}-pcf.zip" />

		<!-- zip up the finished release -->
		<zip destfile="${basedir}/opensextant-toolbox-${version}-pcf.zip">
			<zipfileset dir="${release.dir}" prefix="opensextant-toolbox-${version}-pcf" />
		</zip>
	</target>

	<target name="test" depends="release,run.examples" />

	<!-- the following targets asssume a release has been built but doesn't trigger one if it is not -->

	<!-- run some simple examples -->
	<target name="run.examples">
		<antcall target="example.geotagger" />
		<antcall target="example.generalPurposeTagger" />
		<antcall target="example.placeNameMatcher" />
		<!--antcall target="example.vocabMatcher" /-->
		<antcall target="example.placeNameSearch" />
		<antcall target="example.dateRegexMatcher" />
		<antcall target="example.geoRegexMatcher" />
	</target>

	<target name="example.geotagger">
		<echo>-----------------------------------------------</echo>
		<echo>Running the Geotagger Example</echo>
		<echo>-----------------------------------------------</echo>
		<!-- run the geotagger example-->
		<java classname="org.opensextant.examples.GeotaggerExample" classpathref="examples.runtime.classpath" fork="yes" dir="${basedir}">
			<jvmarg value="-Xmx2G" />
			<jvmarg value="-Dgate.home=${release.dir}" />
			<jvmarg value="-Dlog4j.configuration=file:${release.dir}/etc/log4j.properties" />
			<env key="solr.home" value="${release.dir}/solr" />
			<arg value="${test.gapp.geotagger}" />
			<arg value="${test.testdocs.dir}" />
		</java>
	</target>

	<target name="example.generalPurposeTagger">
		<echo>-------------------------------------------------------------------------------</echo>
		<echo>Running the General Purpose Entity Extractor Example</echo>
		<echo>-------------------------------------------------------------------------------</echo>
		<!-- run the general purpose tagger example-->
		<java classname="org.opensextant.examples.GeneralPurposeTaggerExample" classpathref="examples.runtime.classpath" fork="yes" dir="${basedir}">
			<jvmarg value="-Xmx2G" />
			<jvmarg value="-Dgate.home=${release.dir}" />
			<jvmarg value="-Dlog4j.configuration=file:${release.dir}/etc/log4j.properties" />
			<env key="solr.home" value="${release.dir}/solr" />
			<arg value="${test.gapp.generalPurpose}" />
			<arg value="${test.testdocs.dir}" />
		</java>
	</target>

	<target name="example.placeNameMatcher">
		<echo>--------------------------------------------------</echo>
		<echo>Running the place name Matcher Example</echo>
		<echo>--------------------------------------------------</echo>
		<!-- run the matcher example -->
		<java classname="org.opensextant.examples.MatcherTest" classpathref="examples.runtime.standalone.classpath" fork="yes" dir="${basedir}">
			<jvmarg value="-Xmx500M" />
			<jvmarg value="-Dlog4j.configuration=file:${release.dir}/etc/log4j.properties" />
			<env key="solr.home" value="${release.dir}/solr" />
			<!--arg value="${release.dir}/Gazetteer/solr" /-->
			<arg value="${test.matcher.file}" />
		</java>
	</target>

	<target name="example.vocabMatcher">
		<echo>--------------------------------------------------</echo>
		<echo>Running the vocabulary  Matcher Example</echo>
		<echo>--------------------------------------------------</echo>
		<!-- run the matcher example -->
		<java classname="org.opensextant.examples.VocabMatcherTest" classpathref="examples.runtime.standalone.classpath" fork="yes" dir="${basedir}">
			<jvmarg value="-Xmx500M" />
			<jvmarg value="-Dlog4j.configuration=file:${release.dir}/etc/log4j.properties" />
			<!--jvmarg value="-Dsolr.home=${release.dir}/solr" /-->
			<env key="solr.home" value="${release.dir}/solr" />
			<arg value="${test.testdocs.dir}/test.txt" />
			<!--arg value="${release.dir}/solr" /-->
		</java>
	</target>

	<target name="example.placeNameSearch">
		<echo>--------------------------------------------------</echo>
		<echo>Running the Solr Searcher Example</echo>
		<echo>--------------------------------------------------</echo>
		<!-- run the searcher example -->
		<java classname="org.opensextant.examples.SearcherTest" classpathref="examples.runtime.standalone.classpath" fork="yes" dir="${basedir}">
			<jvmarg value="-Xmx500M" />
			<jvmarg value="-Dlog4j.configuration=file:${release.dir}/etc/log4j.properties" />
			<env key="solr.home" value="${release.dir}/solr" />
			<!--arg value="${release.dir}/Gazetteer/solr" /-->
		</java>
	</target>
	<target name="example.dateRegexMatcher">
		<echo>--------------------------------------------------</echo>
		<echo>Running the Date Regex Matcher Example</echo>
		<echo>--------------------------------------------------</echo>
		<!-- run the date regex example -->
		<java classname="org.opensextant.examples.RegexTest" classpathref="examples.runtime.standalone.classpath" fork="yes" dir="${basedir}">
			<jvmarg value="-Xmx500M" />
			<jvmarg value="-Dlog4j.configuration=file:${release.dir}/etc/log4j.properties" />
			<arg value="${test.dateTest.file}" />
			<arg value="${test.patterns.date}" />
			<arg value="${test.patterns.results}/dateResults.txt" />
		</java>
	</target>
	<target name="example.geoRegexMatcher">
		<echo>--------------------------------------------------</echo>
		<echo>Running the Geocoord Regex Matcher Example</echo>
		<echo>--------------------------------------------------</echo>
		<!-- run the geocoord regex example -->
		<java classname="org.opensextant.examples.RegexTest" classpathref="examples.runtime.standalone.classpath" fork="yes" dir="${basedir}">
			<jvmarg value="-Xmx500M" />
			<jvmarg value="-Dlog4j.configuration=file:${release.dir}/etc/log4j.properties" />
			<arg value="${test.geoTest.file}" />
			<arg value="${test.patterns.geo}" />
			<arg value="${test.patterns.results}/geoResults.txt" />
		</java>
	</target>


	<target name="example.serviceClient">
		<echo />
		<echo>--------------------------------------------------</echo>
		<echo>Running the Service Client Example</echo>
		<echo>--------------------------------------------------</echo>
		<!-- run the geocoord regex example -->
		<java classname="org.opensextant.service.processing.ServiceDriver" classpathref="examples.runtime.standalone.classpath" fork="yes" dir="${basedir}">
			<!-- add some memory -->
			<jvmarg value="-Xmx500M" />
			<!-- point to the log config file -->
			<jvmarg value="-Dlog4j.configuration=file:${release.dir}/etc/log4j.properties" />
			<!-- directory containing the test doc -->
			<arg value="${test.testdocs.dir}" />
			<!-- host of the extractor service -->
			<arg value="localhost" />
			<!-- number of submitter threads-->
			<arg value="12" />
		</java>

	</target>




	<!-- deploy OpenSextantToolbox to a local GATE installation to test -->
	<!-- NOTE: this target presumes a release has been built but doesnt trigger one -->
	<target name="toolbox.deploy" depends="">
		<delete dir="${toolbox.plugin.dir}" />
		<!-- unzip the toolbox plugin zip into GATE_HOME/plugins -->
		<unzip src="${build.dir}/opensextant-toolbox-plugin.zip" dest="${gate.plugin.dir}" />

		<!-- copy over the other jars libs from release -->
		<copy todir="${toolbox.plugin.dir}/lib_extra">
			<fileset dir="${lib.dir}/Standalone">
				<include name="*.jar" />
				<exclude name="**/GATE/**" />
			</fileset>
			<fileset file="${build.dir}/opensextant.jar" />
		</copy>

		<echo> Remember to add -Dgate.class.path=GATE_HOME/plugins/OpensextantToolbox/lib_extra/*  and -Dsolr.home=WHERE.YOU.UNZIPPED.THE.RELEASE/solr to your gate.l4j.ini or other start up script </echo>

	</target>
</project>