diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index bcc865847..99bd9bdf9 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -27,10 +27,10 @@ jobs:
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- - name: Set up JDK 17
+ - name: Set up JDK 25
uses: actions/setup-java@03ad4de0992f5dab5e18fcb136590ce7c4a0ac95 # v5.6.0
with:
- java-version: '17'
+ java-version: '25'
distribution: 'temurin'
- name: Cache Maven packages
diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml
index 7da81f4d9..29af1bdf9 100644
--- a/.github/workflows/maven.yml
+++ b/.github/workflows/maven.yml
@@ -41,7 +41,7 @@ jobs:
uses: actions/setup-java@03ad4de0992f5dab5e18fcb136590ce7c4a0ac95 # v5.6.0
with:
distribution: adopt
- java-version: 17
+ java-version: 25
- name: Build with Maven
run: mvn -B --no-transfer-progress -Prat -DskipTests verify -Dskip.format.code=false
@@ -52,7 +52,7 @@ jobs:
strategy:
matrix:
os: [ubuntu-latest]
- java: [ 17 ]
+ java: [ 25 ]
experimental: [false]
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
diff --git a/README.md b/README.md
index 7f5885cd5..672e2c112 100644
--- a/README.md
+++ b/README.md
@@ -9,9 +9,9 @@ Apache StormCrawler is an open source collection of resources for building low-l
## Quickstart
-NOTE: These instructions assume that you have [Apache Maven](https://maven.apache.org/install.html) installed. You will need to install [Apache Storm 2.8.8](http://storm.apache.org/) to run the crawler.
+NOTE: These instructions assume that you have [Apache Maven](https://maven.apache.org/install.html) installed. You will need to install [Apache Storm 3.0.0](http://storm.apache.org/) to run the crawler.
-StormCrawler requires Java 17 or above. To execute tests, it requires you to have a locally installed and working Docker environment.
+StormCrawler requires Java 25 or above. To execute tests, it requires you to have a locally installed and working Docker environment.
Once Storm is installed, the easiest way to get started is to generate a new StormCrawler project following the instructions below:
@@ -53,7 +53,7 @@ mvn clean install -Dskip.format.code=false
The requirements for building from source are as follows
-- JDK 17+
+- JDK 25+
- Apache Maven 3
- Docker (if you want to run tests)
diff --git a/THIRD-PARTY.txt b/THIRD-PARTY.txt
index 5cbae541d..be623d906 100644
--- a/THIRD-PARTY.txt
+++ b/THIRD-PARTY.txt
@@ -14,7 +14,7 @@ List of third-party dependencies grouped by their license type.
AL 2.0, GPL v2, MPL 2.0
- * RabbitMQ Java Client (com.rabbitmq:amqp-client:5.28.0 - https://www.rabbitmq.com)
+ * RabbitMQ Java Client (com.rabbitmq:amqp-client:5.30.0 - https://www.rabbitmq.com)
Apache License, Version 2.0
@@ -28,6 +28,7 @@ List of third-party dependencies grouped by their license type.
* Apache Commons CSV (org.apache.commons:commons-csv:1.14.1 - https://commons.apache.org/proper/commons-csv/)
* Apache Commons Exec (org.apache.commons:commons-exec:1.6.0 - https://commons.apache.org/proper/commons-exec/)
* Apache Commons IO (commons-io:commons-io:2.17.0 - https://commons.apache.org/proper/commons-io/)
+ * Apache Commons IO (commons-io:commons-io:2.20.0 - https://commons.apache.org/proper/commons-io/)
* Apache Commons IO (commons-io:commons-io:2.22.0 - https://commons.apache.org/proper/commons-io/)
* Apache Commons Lang (org.apache.commons:commons-lang3:3.20.0 - https://commons.apache.org/proper/commons-lang/)
* Apache Commons Logging (commons-logging:commons-logging:1.2 - http://commons.apache.org/proper/commons-logging/)
@@ -36,19 +37,19 @@ List of third-party dependencies grouped by their license type.
* Apache FontBox (org.apache.pdfbox:fontbox:3.0.7 - http://pdfbox.apache.org/)
* Apache Hadoop Client API (org.apache.hadoop:hadoop-client-api:3.5.0 - no url defined)
* Apache Hadoop Client Runtime (org.apache.hadoop:hadoop-client-runtime:3.5.0 - no url defined)
- * Apache HBase - Client (org.apache.hbase:hbase-client:2.6.5-hadoop3 - https://hbase.apache.org/hbase-build-configuration/hbase-client)
- * Apache HBase - Common (org.apache.hbase:hbase-common:2.6.5-hadoop3 - https://hbase.apache.org/hbase-build-configuration/hbase-common)
- * Apache HBase - Hadoop Compatibility (org.apache.hbase:hbase-hadoop-compat:2.6.5-hadoop3 - https://hbase.apache.org/hbase-build-configuration/hbase-hadoop-compat)
- * Apache HBase - Hadoop Two Compatibility (org.apache.hbase:hbase-hadoop2-compat:2.6.5-hadoop3 - https://hbase.apache.org/hbase-build-configuration/hbase-hadoop2-compat)
- * Apache HBase - Logging (org.apache.hbase:hbase-logging:2.6.5-hadoop3 - https://hbase.apache.org/hbase-build-configuration/hbase-logging)
- * Apache HBase - Metrics API (org.apache.hbase:hbase-metrics-api:2.6.5-hadoop3 - https://hbase.apache.org/hbase-build-configuration/hbase-metrics-api)
- * Apache HBase - Metrics Implementation (org.apache.hbase:hbase-metrics:2.6.5-hadoop3 - https://hbase.apache.org/hbase-build-configuration/hbase-metrics)
+ * Apache HBase - Client (org.apache.hbase:hbase-client:2.6.6-hadoop3 - https://hbase.apache.org/hbase-build-configuration/hbase-client)
+ * Apache HBase - Common (org.apache.hbase:hbase-common:2.6.6-hadoop3 - https://hbase.apache.org/hbase-build-configuration/hbase-common)
+ * Apache HBase - Hadoop Compatibility (org.apache.hbase:hbase-hadoop-compat:2.6.6-hadoop3 - https://hbase.apache.org/hbase-build-configuration/hbase-hadoop-compat)
+ * Apache HBase - Hadoop Two Compatibility (org.apache.hbase:hbase-hadoop2-compat:2.6.6-hadoop3 - https://hbase.apache.org/hbase-build-configuration/hbase-hadoop2-compat)
+ * Apache HBase - Logging (org.apache.hbase:hbase-logging:2.6.6-hadoop3 - https://hbase.apache.org/hbase-build-configuration/hbase-logging)
+ * Apache HBase - Metrics API (org.apache.hbase:hbase-metrics-api:2.6.6-hadoop3 - https://hbase.apache.org/hbase-build-configuration/hbase-metrics-api)
+ * Apache HBase - Metrics Implementation (org.apache.hbase:hbase-metrics:2.6.6-hadoop3 - https://hbase.apache.org/hbase-build-configuration/hbase-metrics)
* Apache HBase Patched and Relocated (Shaded) Protobuf (org.apache.hbase.thirdparty:hbase-shaded-protobuf:4.1.13 - https://hbase.apache.org/hbase-shaded-protobuf)
- * Apache HBase - Protocol (org.apache.hbase:hbase-protocol:2.6.5-hadoop3 - https://hbase.apache.org/hbase-build-configuration/hbase-protocol)
+ * Apache HBase - Protocol (org.apache.hbase:hbase-protocol:2.6.6-hadoop3 - https://hbase.apache.org/hbase-build-configuration/hbase-protocol)
* Apache HBase Relocated (Shaded) GSON Libs (org.apache.hbase.thirdparty:hbase-shaded-gson:4.1.13 - https://hbase.apache.org/hbase-shaded-gson)
* Apache HBase Relocated (Shaded) Netty Libs (org.apache.hbase.thirdparty:hbase-shaded-netty:4.1.13 - https://hbase.apache.org/hbase-shaded-netty)
* Apache HBase Relocated (Shaded) Third-party Miscellaneous Libs (org.apache.hbase.thirdparty:hbase-shaded-miscellaneous:4.1.13 - https://hbase.apache.org/hbase-shaded-miscellaneous)
- * Apache HBase - Shaded Protocol (org.apache.hbase:hbase-protocol-shaded:2.6.5-hadoop3 - https://hbase.apache.org/hbase-build-configuration/hbase-protocol-shaded)
+ * Apache HBase - Shaded Protocol (org.apache.hbase:hbase-protocol-shaded:2.6.6-hadoop3 - https://hbase.apache.org/hbase-build-configuration/hbase-protocol-shaded)
* Apache HBase Unsafe Wrapper (org.apache.hbase.thirdparty:hbase-unsafe:4.1.13 - https://hbase.apache.org/hbase-unsafe)
* Apache HttpAsyncClient (org.apache.httpcomponents:httpasyncclient:4.1.5 - http://hc.apache.org/httpcomponents-asyncclient)
* Apache HttpClient (org.apache.httpcomponents:httpclient:4.5.14 - http://hc.apache.org/httpcomponents-client-ga)
@@ -63,8 +64,8 @@ List of third-party dependencies grouped by their license type.
* Apache James :: Mime4j :: Core (org.apache.james:apache-mime4j-core:0.8.14 - http://james.apache.org/mime4j/apache-mime4j-core)
* Apache James :: Mime4j :: DOM (org.apache.james:apache-mime4j-dom:0.8.14 - http://james.apache.org/mime4j/apache-mime4j-dom)
* Apache JempBox (org.apache.pdfbox:jempbox:1.8.17 - http://www.apache.org/pdfbox-parent/jempbox/)
- * Apache Log4j API (org.apache.logging.log4j:log4j-api:2.25.4 - https://logging.apache.org/log4j/2.x/)
- * Apache Log4j Core (org.apache.logging.log4j:log4j-core:2.25.4 - https://logging.apache.org/log4j/2.x/)
+ * Apache Log4j API (org.apache.logging.log4j:log4j-api:2.26.0 - https://logging.apache.org/log4j/2.x/)
+ * Apache Log4j Core (org.apache.logging.log4j:log4j-core:2.26.0 - https://logging.apache.org/log4j/2.x/)
* Apache Log4j JUL Adapter (org.apache.logging.log4j:log4j-jul:2.25.4 - https://logging.apache.org/log4j/2.x/)
* Apache Lucene (module: backward-codecs) (org.apache.lucene:lucene-backward-codecs:9.12.3 - https://lucene.apache.org/)
* Apache Lucene (module: common) (org.apache.lucene:lucene-analysis-common:9.12.3 - https://lucene.apache.org/)
@@ -168,7 +169,7 @@ List of third-party dependencies grouped by their license type.
* error-prone annotations (com.google.errorprone:error_prone_annotations:2.49.0 - https://errorprone.info/error_prone_annotations)
* FindBugs-jsr305 (com.google.code.findbugs:jsr305:3.0.2 - http://findbugs.sourceforge.net/)
* Google Android Annotations Library (com.google.android:annotations:4.1.1.4 - http://source.android.com/)
- * Graphite Integration for Metrics (io.dropwizard.metrics:metrics-graphite:4.2.38 - https://metrics.dropwizard.io/metrics-graphite)
+ * Graphite Integration for Metrics (io.dropwizard.metrics:metrics-graphite:4.2.39 - https://metrics.dropwizard.io/metrics-graphite)
* Gson (com.google.code.gson:gson:2.11.0 - https://github.com/google/gson)
* Gson (com.google.code.gson:gson:2.14.0 - https://github.com/google/gson)
* Guava: Google Core Libraries for Java (com.google.guava:guava:18.0 - http://code.google.com/p/guava-libraries/guava)
@@ -201,7 +202,7 @@ List of third-party dependencies grouped by their license type.
* Joda-Time (joda-time:joda-time:2.12.7 - https://www.joda.org/joda-time/)
* jsonic (net.arnx:jsonic:1.2.11 - http://jsonic.sourceforge.jp/)
* JSpecify annotations (org.jspecify:jspecify:1.0.0 - http://jspecify.org/)
- * JVM Integration for Metrics (io.dropwizard.metrics:metrics-jvm:4.2.38 - https://metrics.dropwizard.io/metrics-jvm)
+ * JVM Integration for Metrics (io.dropwizard.metrics:metrics-jvm:4.2.39 - https://metrics.dropwizard.io/metrics-jvm)
* jwarc (org.netpreserve:jwarc:0.36.0 - https://github.com/iipc/jwarc)
* Kotlin Stdlib (org.jetbrains.kotlin:kotlin-stdlib:2.1.21 - https://kotlinlang.org/)
* LangChain4j :: Core (dev.langchain4j:langchain4j-core:1.18.0 - https://github.com/langchain4j/langchain4j/tree/main/langchain4j-core)
@@ -212,29 +213,29 @@ List of third-party dependencies grouped by their license type.
* lang-mustache (org.opensearch.plugin:lang-mustache-client:2.19.6 - https://github.com/opensearch-project/OpenSearch.git)
* language-detector (com.optimaize.languagedetector:language-detector:0.6 - https://github.com/optimaize/language-detector)
* mapper-extras (org.opensearch.plugin:mapper-extras-client:2.19.6 - https://github.com/opensearch-project/OpenSearch.git)
- * Metrics Core (io.dropwizard.metrics:metrics-core:4.2.38 - https://metrics.dropwizard.io/metrics-core)
- * Metrics Integration with JMX (io.dropwizard.metrics:metrics-jmx:4.2.38 - https://metrics.dropwizard.io/metrics-jmx)
+ * Metrics Core (io.dropwizard.metrics:metrics-core:4.2.39 - https://metrics.dropwizard.io/metrics-core)
+ * Metrics Integration with JMX (io.dropwizard.metrics:metrics-jmx:4.2.39 - https://metrics.dropwizard.io/metrics-jmx)
* Netty/Buffer (io.netty:netty-buffer:4.1.119.Final - https://netty.io/netty-buffer/)
* Netty/Buffer (io.netty:netty-buffer:4.1.135.Final - https://netty.io/netty-buffer/)
- * Netty/Buffer (io.netty:netty-buffer:4.2.7.Final - https://netty.io/netty-buffer/)
- * Netty/Codec/Base (io.netty:netty-codec-base:4.2.7.Final - https://netty.io/netty-codec-base/)
- * Netty/Codec/Compression (io.netty:netty-codec-compression:4.2.7.Final - https://netty.io/netty-codec-compression/)
+ * Netty/Buffer (io.netty:netty-buffer:4.2.12.Final - https://netty.io/netty-buffer/)
+ * Netty/Codec/Base (io.netty:netty-codec-base:4.2.12.Final - https://netty.io/netty-codec-base/)
+ * Netty/Codec/Compression (io.netty:netty-codec-compression:4.2.12.Final - https://netty.io/netty-codec-compression/)
* Netty/Codec/HTTP (io.netty:netty-codec-http:4.1.135.Final - https://netty.io/netty-codec-http/)
* Netty/Codec/HTTP2 (io.netty:netty-codec-http2:4.1.135.Final - https://netty.io/netty-codec-http2/)
- * Netty/Codec/Marshalling (io.netty:netty-codec-marshalling:4.2.7.Final - https://netty.io/netty-codec-marshalling/)
- * Netty/Codec/Protobuf (io.netty:netty-codec-protobuf:4.2.7.Final - https://netty.io/netty-codec-protobuf/)
+ * Netty/Codec/Marshalling (io.netty:netty-codec-marshalling:4.2.12.Final - https://netty.io/netty-codec-marshalling/)
+ * Netty/Codec/Protobuf (io.netty:netty-codec-protobuf:4.2.12.Final - https://netty.io/netty-codec-protobuf/)
* Netty/Codec (io.netty:netty-codec:4.1.119.Final - https://netty.io/netty-codec/)
* Netty/Codec (io.netty:netty-codec:4.1.135.Final - https://netty.io/netty-codec/)
- * Netty/Codec (io.netty:netty-codec:4.2.7.Final - https://netty.io/netty-codec/)
+ * Netty/Codec (io.netty:netty-codec:4.2.12.Final - https://netty.io/netty-codec/)
* Netty/Common (io.netty:netty-common:4.1.119.Final - https://netty.io/netty-common/)
* Netty/Common (io.netty:netty-common:4.1.135.Final - https://netty.io/netty-common/)
- * Netty/Common (io.netty:netty-common:4.2.7.Final - https://netty.io/netty-common/)
+ * Netty/Common (io.netty:netty-common:4.2.12.Final - https://netty.io/netty-common/)
* Netty/Handler (io.netty:netty-handler:4.1.119.Final - https://netty.io/netty-handler/)
* Netty/Handler (io.netty:netty-handler:4.1.135.Final - https://netty.io/netty-handler/)
- * Netty/Handler (io.netty:netty-handler:4.2.7.Final - https://netty.io/netty-handler/)
+ * Netty/Handler (io.netty:netty-handler:4.2.12.Final - https://netty.io/netty-handler/)
* Netty/Resolver (io.netty:netty-resolver:4.1.119.Final - https://netty.io/netty-resolver/)
* Netty/Resolver (io.netty:netty-resolver:4.1.135.Final - https://netty.io/netty-resolver/)
- * Netty/Resolver (io.netty:netty-resolver:4.2.7.Final - https://netty.io/netty-resolver/)
+ * Netty/Resolver (io.netty:netty-resolver:4.2.12.Final - https://netty.io/netty-resolver/)
* Netty/TomcatNative [BoringSSL - Static] (io.netty:netty-tcnative-boringssl-static:2.0.70.Final - https://github.com/netty/netty-tcnative/netty-tcnative-boringssl-static/)
* Netty/TomcatNative [OpenSSL - Classes] (io.netty:netty-tcnative-classes:2.0.70.Final - https://github.com/netty/netty-tcnative/netty-tcnative-classes/)
* Netty/Transport/Classes/Epoll (io.netty:netty-transport-classes-epoll:4.1.119.Final - https://netty.io/netty-transport-classes-epoll/)
@@ -242,10 +243,10 @@ List of third-party dependencies grouped by their license type.
* Netty/Transport/Native/Epoll (io.netty:netty-transport-native-epoll:4.1.119.Final - https://netty.io/netty-transport-native-epoll/)
* Netty/Transport/Native/Unix/Common (io.netty:netty-transport-native-unix-common:4.1.119.Final - https://netty.io/netty-transport-native-unix-common/)
* Netty/Transport/Native/Unix/Common (io.netty:netty-transport-native-unix-common:4.1.135.Final - https://netty.io/netty-transport-native-unix-common/)
- * Netty/Transport/Native/Unix/Common (io.netty:netty-transport-native-unix-common:4.2.7.Final - https://netty.io/netty-transport-native-unix-common/)
+ * Netty/Transport/Native/Unix/Common (io.netty:netty-transport-native-unix-common:4.2.12.Final - https://netty.io/netty-transport-native-unix-common/)
* Netty/Transport (io.netty:netty-transport:4.1.119.Final - https://netty.io/netty-transport/)
* Netty/Transport (io.netty:netty-transport:4.1.135.Final - https://netty.io/netty-transport/)
- * Netty/Transport (io.netty:netty-transport:4.2.7.Final - https://netty.io/netty-transport/)
+ * Netty/Transport (io.netty:netty-transport:4.2.12.Final - https://netty.io/netty-transport/)
* Non-Blocking Reactive Foundation for the JVM (io.projectreactor:reactor-core:3.7.19 - https://github.com/reactor/reactor-core)
* Objenesis (org.objenesis:objenesis:3.3 - http://objenesis.org/objenesis)
* Ogg and Vorbis for Java, Core (org.gagravarr:vorbis-java-core:0.8 - https://github.com/Gagravarr/VorbisJava)
@@ -302,14 +303,14 @@ List of third-party dependencies grouped by their license type.
* rome (com.rometools:rome:2.1.0 - http://rometools.com/rome)
* rome-utils (com.rometools:rome-utils:2.1.0 - http://rometools.com/rome-utils)
* server (org.opensearch:opensearch:2.19.6 - https://github.com/opensearch-project/OpenSearch.git)
- * Shaded Deps for Storm Client (org.apache.storm:storm-shaded-deps:2.8.8 - https://storm.apache.org/storm-shaded-deps)
* SnakeYAML (org.yaml:snakeyaml:2.6 - https://bitbucket.org/snakeyaml/snakeyaml)
* snappy-java (org.xerial.snappy:snappy-java:1.1.10.4 - https://github.com/xerial/snappy-java)
* sniffer (org.opensearch.client:opensearch-rest-client-sniffer:2.19.6 - https://github.com/opensearch-project/OpenSearch.git)
* SparseBitSet (com.zaxxer:SparseBitSet:1.3 - https://github.com/brettwooldridge/SparseBitSet)
- * storm-autocreds (org.apache.storm:storm-autocreds:2.8.8 - https://storm.apache.org/external/storm-autocreds)
- * Storm Client (org.apache.storm:storm-client:2.8.8 - https://storm.apache.org/storm-client)
- * storm-hdfs (org.apache.storm:storm-hdfs:2.8.8 - https://storm.apache.org/external/storm-hdfs)
+ * Storm Autocreds (org.apache.storm:storm-autocreds:3.0.0 - https://storm.apache.org/storm-external/storm-autocreds)
+ * Storm Client (org.apache.storm:storm-client:3.0.0 - https://storm.apache.org/storm-client)
+ * Storm HDFS (org.apache.storm:storm-hdfs:3.0.0 - https://storm.apache.org/storm-external/storm-hdfs)
+ * Storm Shaded Deps (org.apache.storm:storm-shaded-deps:3.0.0 - https://storm.apache.org/storm-shaded-deps)
* swagger-annotations-jakarta (io.swagger.core.v3:swagger-annotations-jakarta:2.2.22 - https://github.com/swagger-api/swagger-core/modules/swagger-annotations-jakarta)
* T-Digest (com.tdunning:t-digest:3.2 - https://github.com/tdunning/t-digest)
* urlfrontier-API (com.github.crawler-commons:urlfrontier-API:2.5 - https://github.com/crawler-commons/url-frontier/urlfrontier-API)
@@ -348,7 +349,7 @@ List of third-party dependencies grouped by their license type.
BSD 2-Clause License
- * zstd-jni (com.github.luben:zstd-jni:1.5.5-5 - https://github.com/luben/zstd-jni)
+ * zstd-jni (com.github.luben:zstd-jni:1.5.7-11 - https://github.com/luben/zstd-jni)
BSD 3-Clause License
diff --git a/archetype/src/main/resources/archetype-resources/README.md b/archetype/src/main/resources/archetype-resources/README.md
index 047711bec..92f096b7f 100644
--- a/archetype/src/main/resources/archetype-resources/README.md
+++ b/archetype/src/main/resources/archetype-resources/README.md
@@ -5,7 +5,7 @@ Have a look at the code and resources and modify them to your heart's content.
## Native
-You need to install Apache Storm. The instructions on [setting up a Storm cluster](https://storm.apache.org/releases/2.8.8/Setting-up-a-Storm-cluster.html) should help.
+You need to install Apache Storm. The instructions on [setting up a Storm cluster](https://storm.apache.org/releases/3.0.0/Setting-up-a-Storm-cluster.html) should help.
You also need to have an instance of URLFrontier running. See [the URLFrontier README](https://github.com/crawler-commons/url-frontier/tree/master/service); the easiest way is to use Docker, like so:
```
diff --git a/archetype/src/main/resources/archetype-resources/pom.xml b/archetype/src/main/resources/archetype-resources/pom.xml
index 804c88099..c57c09b92 100644
--- a/archetype/src/main/resources/archetype-resources/pom.xml
+++ b/archetype/src/main/resources/archetype-resources/pom.xml
@@ -32,7 +32,7 @@ under the License.
UTF-8
${project.version}
- 2.8.8
+ 3.0.0
2.4
@@ -43,8 +43,8 @@ under the License.
maven-compiler-plugin
3.11.0
- 17
- 17
+ 25
+ 25
diff --git a/docs/src/main/asciidoc/quick-start.adoc b/docs/src/main/asciidoc/quick-start.adoc
index ce780f10d..e42b27eb3 100644
--- a/docs/src/main/asciidoc/quick-start.adoc
+++ b/docs/src/main/asciidoc/quick-start.adoc
@@ -10,7 +10,7 @@ These instructions should help you get Apache StormCrawler up and running in 5 t
=== Prerequisites
-To run StormCrawler, you will need Java SE 17 or later.
+To run StormCrawler, you will need Java SE 25 or later.
Additionally, since we'll be running the required Apache Storm cluster using Docker Compose,
make sure Docker is installed on your operating system.
diff --git a/external/opensearch/archetype/src/main/resources/archetype-resources/pom.xml b/external/opensearch/archetype/src/main/resources/archetype-resources/pom.xml
index d58ed4784..eff9791d4 100644
--- a/external/opensearch/archetype/src/main/resources/archetype-resources/pom.xml
+++ b/external/opensearch/archetype/src/main/resources/archetype-resources/pom.xml
@@ -34,7 +34,7 @@ under the License.
UTF-8
${StormCrawlerVersion}
- 2.8.8
+ 3.0.0
@@ -44,8 +44,8 @@ under the License.
maven-compiler-plugin
3.11.0
- 17
- 17
+ 25
+ 25
diff --git a/external/solr/README.md b/external/solr/README.md
index 626eeda65..226db5fbb 100644
--- a/external/solr/README.md
+++ b/external/solr/README.md
@@ -14,7 +14,7 @@ You'll be asked to enter a groupId (e.g. com.mycompany.crawler), an artefactId (
This will not only create a fully formed project containing a POM with the dependency above but also a set of resources, configuration files and sample topology classes. Enter the directory you just created (should be the same as the artefactId you specified earlier) and follow the instructions on the README file.
-You will of course need to have both Apache Storm (2.8.8) and Apache Solr (10.0.0) installed.
+You will of course need to have both Apache Storm (3.0.0) and Apache Solr (10.0.0) installed.
Official references:
* [Apache Storm: Setting Up a Development Environment](https://storm.apache.org/releases/current/Setting-up-development-environment.html)
diff --git a/external/solr/archetype/src/main/resources/archetype-resources/README.md b/external/solr/archetype/src/main/resources/archetype-resources/README.md
index dbae9f23b..d6439caa6 100644
--- a/external/solr/archetype/src/main/resources/archetype-resources/README.md
+++ b/external/solr/archetype/src/main/resources/archetype-resources/README.md
@@ -1,7 +1,7 @@
This has been generated by the StormCrawler Maven Archetype as a starting point for building your own crawler with [Apache Solr](https://solr.apache.org/) as a backend.
Have a look at the code and resources and modify them to your heart's content.
-You need to have Apache Storm (2.8.8) installed, as well as a running instance of Apache Solr (9.8.0).
+You need to have Apache Storm (3.0.0) installed, as well as a running instance of Apache Solr (9.8.0).
## Generated resources
diff --git a/external/solr/archetype/src/main/resources/archetype-resources/pom.xml b/external/solr/archetype/src/main/resources/archetype-resources/pom.xml
index 862c24ece..b0da46fd1 100644
--- a/external/solr/archetype/src/main/resources/archetype-resources/pom.xml
+++ b/external/solr/archetype/src/main/resources/archetype-resources/pom.xml
@@ -34,7 +34,7 @@ under the License.
UTF-8
${StormCrawlerVersion}
- 2.8.8
+ 3.0.0
@@ -44,8 +44,8 @@ under the License.
maven-compiler-plugin
3.11.0
- 17
- 17
+ 25
+ 25
diff --git a/external/warc/README.md b/external/warc/README.md
index b1b0466f4..8f22aa117 100644
--- a/external/warc/README.md
+++ b/external/warc/README.md
@@ -185,14 +185,14 @@ Please note that in order to access WARC files on AWS S3, you will need to add t
org.apache.hadoop
hadoop-aws
- 2.10.1
+ 3.5.0
```
where the version should match the one used by Apache Storm. In doubt, you can check with
```
-mvn dependency:tree | grep "org.apache.hadoop:hadoop-hdfs:jar"
+mvn dependency:tree | grep "org.apache.hadoop:hadoop-client-api:jar"
```
diff --git a/external/warc/pom.xml b/external/warc/pom.xml
index 32bc2c35f..5e6edb440 100644
--- a/external/warc/pom.xml
+++ b/external/warc/pom.xml
@@ -63,22 +63,9 @@ under the License.
${storm-client.version}
-
- jdk.tools
- jdk.tools
-
-
-
- org.apache.hive.hcatalog
- hive-webhcat-java-client
-
-
+
org.apache.hbase
- hbase-server
-
-
- org.apache.hive.hcatalog
- hive-hcatalog-streaming
+ hbase-client
diff --git a/external/warc/src/main/java/org/apache/stormcrawler/warc/GzipHdfsBolt.java b/external/warc/src/main/java/org/apache/stormcrawler/warc/GzipHdfsBolt.java
index 8342e7d91..a2d70d83b 100644
--- a/external/warc/src/main/java/org/apache/stormcrawler/warc/GzipHdfsBolt.java
+++ b/external/warc/src/main/java/org/apache/stormcrawler/warc/GzipHdfsBolt.java
@@ -15,9 +15,11 @@
package org.apache.stormcrawler.warc;
+import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import java.util.zip.GZIPOutputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.Path;
import org.apache.storm.hdfs.bolt.HdfsBolt;
@@ -25,7 +27,6 @@
import org.apache.storm.hdfs.common.AbstractHDFSWriter;
import org.apache.storm.hdfs.common.HDFSWriter;
import org.apache.storm.tuple.Tuple;
-import org.apache.storm.utils.Utils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -34,6 +35,15 @@ public class GzipHdfsBolt extends HdfsBolt {
private static final Logger LOG = LoggerFactory.getLogger(GzipHdfsBolt.class);
+ /** Compresses the given bytes into a standalone gzip container. */
+ protected static byte[] gzip(byte[] bytes) throws IOException {
+ ByteArrayOutputStream bos = new ByteArrayOutputStream(bytes.length / 2 + 64);
+ try (GZIPOutputStream gzos = new GZIPOutputStream(bos)) {
+ gzos.write(bytes);
+ }
+ return bos.toByteArray();
+ }
+
protected transient FSDataOutputStream out = null;
protected RecordFormat format;
@@ -59,7 +69,7 @@ public byte[] format(Tuple tuple) {
if (bytes.length == 0 && !compressEmpty) {
return new byte[0];
}
- return Utils.gzip(bytes);
+ return gzip(bytes);
} catch (Exception e) {
LOG.error("Exception caught when formatting - skipping the whole tuple");
return new byte[0];
diff --git a/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCHdfsBolt.java b/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCHdfsBolt.java
index 5b77a7289..48dcf32d9 100644
--- a/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCHdfsBolt.java
+++ b/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCHdfsBolt.java
@@ -30,7 +30,6 @@
import org.apache.storm.task.OutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.tuple.Tuple;
-import org.apache.storm.utils.Utils;
import org.apache.stormcrawler.protocol.ProtocolResponse;
import org.apache.stormcrawler.util.ConfUtils;
import org.slf4j.Logger;
@@ -102,7 +101,7 @@ protected AbstractHDFSWriter makeNewWriter(Path path, Tuple tuple) throws IOExce
// write the header at the beginning of the file
if (header != null && header.length > 0) {
- super.out.write(Utils.gzip(header));
+ super.out.write(gzip(header));
}
return writer;
diff --git a/pom.xml b/pom.xml
index 0acc77fda..62ae9268b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -65,7 +65,7 @@ under the License.
-Xdoclint:none
6.1.2
- 2.8.8
+ 3.0.0
2.21
2.21.3
@@ -95,8 +95,8 @@ under the License.
true
- 17
- 17
+ 25
+ 25
1.7
@@ -117,9 +117,9 @@ under the License.
maven-compiler-plugin
3.15.0
- 17
- 17
- 17
+ 25
+ 25
+ 25
@@ -128,7 +128,7 @@ under the License.
3.12.0
none
- 17
+ 25
src/main/java