Tika grpc mv take3 (#1854)

* TIKA-4181 - grpc server and client

* latest updates - wip

* code formatting

* fix the issues with deps and such

* add bidi streaming

* clean up wording

* make it as a stale connection expiring store

* clean

* delete dead code

* add closeable

* add closeable

* add closeable

* fixed issues related to proto lint

* example docker file

* example docker file and install docker file

* fix the rm

* more info

* --platform linux/arm64

* --platform linux/arm64

* add work around comment for future

* add comment

* fix issues with files in wrong dir

* add quotes

* add a few more cleanups

* add logging
add reflection api for grpc so you can use grpc curl

* fix issues and add tests for get, delete

* push latest fixes

* add start of a test scenario

* add a fully functional http test case

* add mtls as an option

* add mtls in the example

* add some robustness and add an exec

* Fix issues with the fetch metadata

* TIKA-4252: fix metadata issue

* TIKA-4252: fix issue with config param serialization.

* TIKA-4252: add error path

* TIKA-4252: add protection against null metadata

* TIKA-4252: fix merge conflicts from main

* TIKA-4252: add json schema methods

* TIKA-4252: fix broken tests, useless method

* TIKA-4252: remove stupid sleep from test

* TIKA-4252: fix checkstyle issue

* TIKA-4252: log violations to console

* TIKA-4252: log violations to console

* TIKA-4252: exclude generated sources

* TIKA-4252: if tika config is read-only, use a tmp file for tika server so that it can be modified

* TIKA-4252: if tika config is read-only, use a tmp file for tika server so that it can be modified

* add a health check

* stop sending "metadata json" instead send a copy of the fetch configuration json that can add or override the main fetcher config

* fix merge conflicts

* fix issue with additional headers

* move tika-grpc where it belongs inside the tika-server, not in the pipes project.

* make params align with tika server and make them optional

* merge TIKA-4181 and mv tika-grpc to higher level module

* expand bounds in rtf parser test to deal with locally installed tesseract and exiftool

---------

Co-authored-by: Nicholas DiPiazza <[email protected]>

Author: tballison

pom.xml

@@ -45,6 +45,7 @@
     <module>tika-batch</module>
     <module>tika-langdetect</module>
     <module>tika-pipes</module>
+    <module>tika-grpc</module>
     <module>tika-app</module>
     <module>tika-server</module>
     <module>tika-integration-tests</module>

tika-pipes/tika-grpc/README.md renamed to tika-grpc/README.md

@@ -29,9 +29,9 @@
 
   <parent>
     <groupId>org.apache.tika</groupId>
-    <artifactId>tika-pipes</artifactId>
+    <artifactId>tika-parent</artifactId>
     <version>3.0.0-SNAPSHOT</version>
-    <relativePath>../pom.xml</relativePath>
+    <relativePath>../tika-parent/pom.xml</relativePath>
   </parent>
 
   <properties>
@@ -353,6 +353,99 @@
           <mainClass>org.apache.tika.pipes.grpc.TikaGrpcServer</mainClass>
         </configuration>
       </plugin>
+      <plugin>
+        <artifactId>maven-shade-plugin</artifactId>
+        <version>${maven.shade.version}</version>
+        <executions>
+          <execution>
+            <phase>package</phase>
+            <goals>
+              <goal>shade</goal>
+            </goals>
+            <configuration>
+              <createDependencyReducedPom>
+                false
+              </createDependencyReducedPom>
+              <artifactSet>
+                <excludes>
+                </excludes>
+              </artifactSet>
+              <filters>
+                <filter>
+                  <artifact>*:*</artifact>
+                  <excludes>
+                    <exclude>module-info.class</exclude>
+                    <exclude>META-INF/versions/9/module-info.class</exclude>
+                    <exclude>META-INF/*.SF</exclude>
+                    <exclude>META-INF/*.DSA</exclude>
+                    <exclude>META-INF/*.RSA</exclude>
+                    <exclude>META-INF/*.txt</exclude>
+                    <exclude>META-INF/ASL2.0</exclude>
+                    <exclude>META-INF/DEPENDENCIES</exclude>
+                    <exclude>META-INF/LICENSE</exclude>
+                    <exclude>META-INF/NOTICE</exclude>
+                    <exclude>META-INF/README</exclude>
+                    <exclude>META-INF/MANIFEST.MF</exclude>
+                    <exclude>LICENSE.txt</exclude>
+                    <exclude>NOTICE.txt</exclude>
+                    <exclude>CHANGES</exclude>
+                    <exclude>README</exclude>
+                    <exclude>builddef.lst</exclude>
+                  </excludes>
+                </filter>
+              </filters>
+              <transformers>
+                <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+                  <mainClass>org.apache.tika.pipes.grpc.TikaGrpcServer</mainClass>
+                  <manifestEntries>
+                    <Multi-Release>true</Multi-Release>
+                  </manifestEntries>
+                </transformer>
+                <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
+                  <resource>META-INF/LICENSE</resource>
+                  <file>target/classes/META-INF/LICENSE</file>
+                </transformer>
+                <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
+                  <resource>META-INF/NOTICE</resource>
+                  <file>target/classes/META-INF/NOTICE</file>
+                </transformer>
+              </transformers>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-checkstyle-plugin</artifactId>
+        <version>${checkstyle.plugin.version}</version>
+        <dependencies>
+          <dependency>
+            <groupId>com.puppycrawl.tools</groupId>
+            <artifactId>checkstyle</artifactId>
+            <version>${puppycrawl.version}</version>
+          </dependency>
+        </dependencies>
+        <executions>
+          <execution>
+            <id>validate</id>
+            <phase>validate</phase>
+            <configuration>
+              <configLocation>checkstyle.xml</configLocation>
+              <inputEncoding>UTF-8</inputEncoding>
+              <consoleOutput>true</consoleOutput>
+              <logViolationsToConsole>true</logViolationsToConsole>
+              <includeTestSourceDirectory>true</includeTestSourceDirectory>
+              <testSourceDirectories>${project.basedir}/src/test/java</testSourceDirectories>
+              <violationSeverity>error</violationSeverity>
+              <failOnViolation>true</failOnViolation>
+              <excludeGeneratedSources>true</excludeGeneratedSources>
+            </configuration>
+            <goals>
+              <goal>check</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
     </plugins>
   </build>
 </project>

tika-pipes/tika-grpc/pom.xml renamed to tika-grpc/pom.xml

@@ -19,6 +19,8 @@
 import static io.grpc.health.v1.HealthCheckResponse.ServingStatus;
 
 import java.io.File;
+import java.io.FileWriter;
+import java.nio.charset.StandardCharsets;
 import java.util.concurrent.TimeUnit;
 
 import com.beust.jcommander.JCommander;
@@ -33,16 +35,20 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import org.apache.tika.config.TikaConfig;
+import org.apache.tika.config.TikaConfigSerializer;
+
 /**
  * Server that manages startup/shutdown of the GRPC Tika server.
  */
 public class TikaGrpcServer {
     private static final Logger LOGGER = LoggerFactory.getLogger(TikaGrpcServer.class);
+    public static final int TIKA_SERVER_GRPC_DEFAULT_PORT = 50052;
     private Server server;
-    @Parameter(names = {"-p", "--port"}, description = "The grpc server port", help = true, required = true)
-    private Integer port;
+    @Parameter(names = {"-p", "--port"}, description = "The grpc server port", help = true)
+    private Integer port = TIKA_SERVER_GRPC_DEFAULT_PORT;
 
-    @Parameter(names = {"-t", "--tika-config"}, description = "The grpc server port", help = true, required = true)
+    @Parameter(names = {"-c", "--config"}, description = "The grpc server port", help = true)
     private File tikaConfigXml;
 
     @Parameter(names = {"-s", "--secure"}, description = "Enable credentials required to access this grpc server")
@@ -82,13 +88,20 @@ public void start() throws Exception {
         } else {
             creds = InsecureServerCredentials.create();
         }
+        if (tikaConfigXml == null) {
+            // Create a default tika config
+            tikaConfigXml = File.createTempFile("tika-config", ".xml");
+            try (FileWriter fw = new FileWriter(tikaConfigXml, StandardCharsets.UTF_8)) {
+                TikaConfigSerializer.serialize(new TikaConfig(), TikaConfigSerializer.Mode.STATIC_FULL, fw, StandardCharsets.UTF_8);
+            }
+        }
         File tikaConfigFile = new File(tikaConfigXml.getAbsolutePath());
         healthStatusManager.setStatus(TikaGrpcServer.class.getSimpleName(), ServingStatus.SERVING);
         server = Grpc
                 .newServerBuilderForPort(port, creds)
                 .addService(new TikaGrpcServerImpl(tikaConfigFile.getAbsolutePath()))
                 .addService(healthStatusManager.getHealthService())
-                .addService(ProtoReflectionService.newInstance()) // Enable reflection
+                .addService(ProtoReflectionService.newInstance())
                 .build()
                 .start();
         LOGGER.info("Server started, listening on " + port);

tika-pipes/tika-grpc/src/main/java/org/apache/tika/pipes/grpc/ExpiringFetcherStore.java renamed to tika-grpc/src/main/java/org/apache/tika/pipes/grpc/ExpiringFetcherStore.java

@@ -66,7 +66,6 @@
 import org.apache.tika.TikaGrpc;
 import org.apache.tika.config.Initializable;
 import org.apache.tika.config.Param;
-import org.apache.tika.config.TikaConfigSerializer;
 import org.apache.tika.exception.TikaConfigException;
 import org.apache.tika.metadata.Metadata;
 import org.apache.tika.parser.ParseContext;
@@ -81,7 +80,7 @@
 import org.apache.tika.pipes.fetcher.config.FetcherConfigContainer;
 
 class TikaGrpcServerImpl extends TikaGrpc.TikaImplBase {
-    private static final Logger LOG = LoggerFactory.getLogger(TikaConfigSerializer.class);
+    private static final Logger LOG = LoggerFactory.getLogger(TikaGrpcServerImpl.class);
     private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
     static {
         OBJECT_MAPPER.setSerializationInclusion(JsonInclude.Include.NON_NULL);
@@ -125,6 +124,10 @@ private void updateTikaConfig()
                 DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(tikaConfigPath);
 
         Element fetchersElement = (Element) tikaConfigDoc.getElementsByTagName("fetchers").item(0);
+        if (fetchersElement == null) {
+            fetchersElement = tikaConfigDoc.createElement("fetchers");
+            tikaConfigDoc.getDocumentElement().appendChild(fetchersElement);
+        }
         for (int i = 0; i < fetchersElement.getChildNodes().getLength(); ++i) {
             fetchersElement.removeChild(fetchersElement.getChildNodes().item(i));
         }