TIKA-4272 add pf4j to tika pipes, decouple Fetcher with Fetcher Config, stop using shade plugin for fetchers

pom.xml

@@ -38,7 +38,6 @@
     <module>tika-parent</module>
     <module>tika-bom</module>
     <module>tika-core</module>
-    <module>tika-serialization</module>
     <module>tika-parsers</module>
     <module>tika-bundles</module>
     <module>tika-xmp</module>
@@ -114,6 +113,14 @@
                         <include name="tika-server/tika-server-standard/target/tika-server-standard-${project.version}-bin.tgz*" />
                         <include name="tika-server/tika-server-standard/target/tika-server-standard-${project.version}-bin.zip*" />
                         <include name="tika-eval/tika-eval-app/target/tika-eval-app-${project.version}.jar*" />
+                        <include name="tika-grpc/target/tika-grpc-${project.version}.zip*" />
+                        <include name="tika-pipes/tika-fetchers/tika-fetcher-microsoft-graph/target/tika-fetcher-microsoft-graph-${project.version}.zip*" />
+                        <include name="tika-pipes/tika-fetchers/tika-fetcher-s3/target/tika-fetcher-s3-${project.version}.zip*" />
+                        <include name="tika-pipes/tika-fetchers/tika-fetcher-az-blob/target/tika-fetcher-az-blob-${project.version}.zip*" />
+                        <include name="tika-pipes/tika-fetchers/tika-fetcher-gcs/target/tika-fetcher-gcs-${project.version}.zip*" />
+                        <include name="tika-pipes/tika-fetchers/tika-fetcher-file-system/target/tika-fetcher-file-system-${project.version}.zip*" />
+                        <include name="tika-pipes/tika-fetchers/tika-fetcher-url/target/tika-fetcher-url-${project.version}.zip*" />
+                        <include name="tika-pipes/tika-fetchers/tika-fetcher-http/target/tika-fetcher-http-${project.version}.zip*" />
                       </fileset>
                     </copy>
                     <checksum algorithm="SHA-512" fileext=".sha512">

tika-core/pom.xml

@@ -19,7 +19,8 @@
   under the License.
 -->
 
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
   <modelVersion>4.0.0</modelVersion>
 
   <parent>
@@ -40,6 +41,12 @@
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-api</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.pf4j</groupId>
+      <artifactId>pf4j</artifactId>
+      <!-- !!! VERY IMPORTANT -->
+      <scope>provided</scope>
+    </dependency>
     <dependency>
       <groupId>commons-io</groupId>
       <artifactId>commons-io</artifactId>
@@ -255,9 +262,9 @@
           <jvmArgs>-Xmx256m</jvmArgs>
           <timeout>240000</timeout>
           <effort>max</effort>
-        <failOnError>true</failOnError>
-      </configuration>
-    </plugin>
+          <failOnError>true</failOnError>
+        </configuration>
+      </plugin>
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-project-info-reports-plugin</artifactId>

tika-core/src/main/java/org/apache/tika/extractor/AbstractEmbeddedDocumentBytesHandler.java

@@ -25,7 +25,6 @@
 import org.apache.tika.io.FilenameUtils;
 import org.apache.tika.metadata.Metadata;
 import org.apache.tika.metadata.TikaCoreProperties;
-import org.apache.tika.pipes.extractor.EmbeddedDocumentBytesConfig;
 import org.apache.tika.utils.StringUtils;
 
 public abstract class AbstractEmbeddedDocumentBytesHandler implements EmbeddedDocumentBytesHandler {

tika-core/src/main/java/org/apache/tika/extractor/BasicEmbeddedDocumentBytesHandler.java

@@ -25,7 +25,6 @@
 import org.apache.commons.io.input.UnsynchronizedBufferedInputStream;
 
 import org.apache.tika.metadata.Metadata;
-import org.apache.tika.pipes.extractor.EmbeddedDocumentBytesConfig;
 
 /**
  * For now, this is an in-memory EmbeddedDocumentBytesHandler that stores

tika-core/src/main/java/org/apache/tika/pipes/extractor/EmbeddedDocumentBytesConfig.java → tika-core/src/main/java/org/apache/tika/extractor/EmbeddedDocumentBytesConfig.java

@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.tika.pipes.extractor;
+package org.apache.tika.extractor;
 
 import java.io.Serializable;
 import java.util.Objects;

tika-fuzzing/pom.xml

@@ -39,6 +39,11 @@
       <version>${project.version}</version>
       <scope>provided</scope>
     </dependency>
+    <dependency>
+      <groupId>org.apache.tika</groupId>
+      <artifactId>tika-pipes-core</artifactId>
+      <version>${project.version}</version>
+    </dependency>
     <dependency>
       <groupId>${project.groupId}</groupId>
       <artifactId>tika-serialization</artifactId>
@@ -87,6 +92,12 @@
       <type>test-jar</type>
       <scope>test</scope>
     </dependency>
+
+    <dependency>
+      <groupId>org.pf4j</groupId>
+      <artifactId>pf4j</artifactId>
+      <scope>provided</scope>
+    </dependency>
   </dependencies>
   <build>
     <plugins>
@@ -133,4 +144,4 @@
       </plugin>
     </plugins>
   </build>
-</project>
+</project>

tika-fuzzing/src/main/java/org/apache/tika/fuzzing/cli/FuzzingCLI.java

@@ -77,7 +77,7 @@ private void execute(FuzzingCLIConfig config) throws Exception {
         ArrayBlockingQueue<FetchEmitTuple> q = new ArrayBlockingQueue(10000);
 
         PipesConfig pipesConfig = PipesConfig.load(config.getTikaConfig());
-        FetcherManager fetcherManager = FetcherManager.load(config.getTikaConfig());
+        FetcherManager fetcherManager = new FetcherManager();
 
         int totalThreads = pipesConfig.getNumClients() + 1;