From b14a57412f41e0912ccf54b7f768606cd1666582 Mon Sep 17 00:00:00 2001 From: Julien Nioche Date: Thu, 23 Jul 2026 15:01:29 +0100 Subject: [PATCH 1/2] Mark HTTPClient as deprecated Signed-off-by: Julien Nioche --- .../stormcrawler/protocol/DelegatorProtocol.java | 4 ++-- .../stormcrawler/protocol/ProtocolFactory.java | 4 ++-- .../protocol/httpclient/HttpProtocol.java | 9 ++++++++- .../apache/stormcrawler/proxy/MultiProxyManager.java | 3 +-- core/src/main/resources/crawler-default.yaml | 7 +++++-- .../HttpClientProtocolProxyManagerTest.java | 1 + docs/src/main/asciidoc/configuration.adoc | 12 ++++++++---- docs/src/main/asciidoc/internals.adoc | 6 +++--- 8 files changed, 30 insertions(+), 16 deletions(-) diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/DelegatorProtocol.java b/core/src/main/java/org/apache/stormcrawler/protocol/DelegatorProtocol.java index 087bc70b2..b19760b93 100644 --- a/core/src/main/java/org/apache/stormcrawler/protocol/DelegatorProtocol.java +++ b/core/src/main/java/org/apache/stormcrawler/protocol/DelegatorProtocol.java @@ -38,7 +38,7 @@ * * 
  * protocol.delegator.config:
- * - className: "org.apache.stormcrawler.protocol.httpclient.HttpProtocol"
+ * - className: "org.apache.stormcrawler.protocol.okhttp.HttpProtocol"
  *   filters:
  *     domain: "example.com"
  *     depth: "3"
@@ -50,7 +50,7 @@
  *   regex:
  *    - \.pdf
  *    - \.doc
- * - className: "org.apache.stormcrawler.protocol.selenium.SeleniumProtocol"
+ * - className: "org.apache.stormcrawler.protocol.playwright.HttpProtocol"
  *
* * Typically, the last one in the list must not have filters as it is used as a default value. The diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/ProtocolFactory.java b/core/src/main/java/org/apache/stormcrawler/protocol/ProtocolFactory.java index 5b870fbdc..10619f8b0 100644 --- a/core/src/main/java/org/apache/stormcrawler/protocol/ProtocolFactory.java +++ b/core/src/main/java/org/apache/stormcrawler/protocol/ProtocolFactory.java @@ -72,10 +72,10 @@ private void configure(Config conf) { // set the default values if (protocol.equalsIgnoreCase("http")) { protocolimplementation = - "org.apache.stormcrawler.protocol.httpclient.HttpProtocol"; + "org.apache.stormcrawler.protocol.okhttp.HttpProtocol"; } else if (protocol.equalsIgnoreCase("https")) { protocolimplementation = - "org.apache.stormcrawler.protocol.httpclient.HttpProtocol"; + "org.apache.stormcrawler.protocol.okhttp.HttpProtocol"; } else { throw new RuntimeException(paramName + "should not have an empty value"); } diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/httpclient/HttpProtocol.java b/core/src/main/java/org/apache/stormcrawler/protocol/httpclient/HttpProtocol.java index 0c67b1841..9ea6a3321 100644 --- a/core/src/main/java/org/apache/stormcrawler/protocol/httpclient/HttpProtocol.java +++ b/core/src/main/java/org/apache/stormcrawler/protocol/httpclient/HttpProtocol.java @@ -68,7 +68,14 @@ import org.apache.stormcrawler.util.CookieConverter; import org.slf4j.LoggerFactory; -/** Uses Apache httpclient to handle http and https. */ +/** + * Uses Apache httpclient to handle http and https. + * + * @deprecated The Apache HttpClient protocol implementation is deprecated and will be removed in the + * next major release of StormCrawler. Use the OkHttp implementation ({@link + * org.apache.stormcrawler.protocol.okhttp.HttpProtocol}) instead. + */ +@Deprecated public class HttpProtocol extends AbstractHttpProtocol implements ResponseHandler { diff --git a/core/src/main/java/org/apache/stormcrawler/proxy/MultiProxyManager.java b/core/src/main/java/org/apache/stormcrawler/proxy/MultiProxyManager.java index 68a600d46..b65392c3c 100644 --- a/core/src/main/java/org/apache/stormcrawler/proxy/MultiProxyManager.java +++ b/core/src/main/java/org/apache/stormcrawler/proxy/MultiProxyManager.java @@ -30,7 +30,6 @@ import org.apache.commons.lang3.ArrayUtils; import org.apache.storm.Config; import org.apache.stormcrawler.Metadata; -import org.apache.stormcrawler.protocol.httpclient.HttpProtocol; import org.apache.stormcrawler.util.ConfUtils; import org.slf4j.LoggerFactory; @@ -48,7 +47,7 @@ public enum ProxyRotation { private final AtomicInteger lastAccessedIndex = new AtomicInteger(0); private Map proxyLookupMap; - private static final org.slf4j.Logger LOG = LoggerFactory.getLogger(HttpProtocol.class); + private static final org.slf4j.Logger LOG = LoggerFactory.getLogger(MultiProxyManager.class); /** Default constructor for setting up the proxy manager. */ private void init(ProxyRotation rotation) { diff --git a/core/src/main/resources/crawler-default.yaml b/core/src/main/resources/crawler-default.yaml index 9961320bd..f5c55104f 100644 --- a/core/src/main/resources/crawler-default.yaml +++ b/core/src/main/resources/crawler-default.yaml @@ -194,8 +194,11 @@ config: robots.error.cache.spec: "maximumSize=10000,expireAfterWrite=1h" protocols: "http,https,file" - http.protocol.implementation: "org.apache.stormcrawler.protocol.httpclient.HttpProtocol" - https.protocol.implementation: "org.apache.stormcrawler.protocol.httpclient.HttpProtocol" + # NOTE: the Apache HttpClient protocol implementation + # (org.apache.stormcrawler.protocol.httpclient.HttpProtocol) is deprecated and will be removed in + # the next major release of StormCrawler. OkHttp is now the default. + http.protocol.implementation: "org.apache.stormcrawler.protocol.okhttp.HttpProtocol" + https.protocol.implementation: "org.apache.stormcrawler.protocol.okhttp.HttpProtocol" file.protocol.implementation: "org.apache.stormcrawler.protocol.file.FileProtocol" # number of instances for each protocol implementation diff --git a/core/src/test/java/org/apache/stormcrawler/protocol/httpclient/HttpClientProtocolProxyManagerTest.java b/core/src/test/java/org/apache/stormcrawler/protocol/httpclient/HttpClientProtocolProxyManagerTest.java index d67de9368..d350e8f80 100644 --- a/core/src/test/java/org/apache/stormcrawler/protocol/httpclient/HttpClientProtocolProxyManagerTest.java +++ b/core/src/test/java/org/apache/stormcrawler/protocol/httpclient/HttpClientProtocolProxyManagerTest.java @@ -26,6 +26,7 @@ import org.apache.stormcrawler.proxy.SingleProxyManager; import org.junit.jupiter.api.Test; +@SuppressWarnings("deprecation") class HttpClientProtocolProxyManagerTest { @Test diff --git a/docs/src/main/asciidoc/configuration.adoc b/docs/src/main/asciidoc/configuration.adoc index fd9e52b61..1b3075f5c 100644 --- a/docs/src/main/asciidoc/configuration.adoc +++ b/docs/src/main/asciidoc/configuration.adoc @@ -201,8 +201,10 @@ is defined. | http.basicauth.password | - | Password for http.basicauth.user. | http.basicauth.user | - | Username for Basic Authentication. | http.content.limit | -1 | Maximum HTTP response body size (bytes). Default: no limit. -| http.protocol.implementation | org.apache.stormcrawler.protocol.httpclient.HttpProtocol | HTTP Protocol -implementation. +| http.protocol.implementation | org.apache.stormcrawler.protocol.okhttp.HttpProtocol | HTTP Protocol +implementation. Note: the Apache HttpClient implementation +(`org.apache.stormcrawler.protocol.httpclient.HttpProtocol`) is *deprecated* and will be removed in +the next major release. | http.proxy | - | Full **SCProxy** connection string read by SingleProxyManager when that manager is selected, e.g. `http://user:pass@proxy.example.com:8080`. | http.proxy.file | - | Proxy connection-string file read by MultiProxyManager when that manager is selected. | http.proxy.host | - | SingleProxyManager component proxy host. @@ -225,8 +227,10 @@ implementation. | http.store.headers | false | Whether to store response headers. | http.timeout | 10000 | Connection timeout (ms). | http.use.cookies | false | Use cookies in subsequent requests. -| https.protocol.implementation | org.apache.stormcrawler.protocol.httpclient.HttpProtocol | HTTPS Protocol -implementation. +| https.protocol.implementation | org.apache.stormcrawler.protocol.okhttp.HttpProtocol | HTTPS Protocol +implementation. Note: the Apache HttpClient implementation +(`org.apache.stormcrawler.protocol.httpclient.HttpProtocol`) is *deprecated* and will be removed in +the next major release. | partition.url.mode | byHost | Defines how URLs are partitioned: byHost, byDomain, or byIP. | protocols | http,https,file | Supported protocols. | redirections.allowed | true | If true, emit redirect target URLs as "outlinks" to the status stream. If false, do not follow redirects. See also `http.allow.redirects`. diff --git a/docs/src/main/asciidoc/internals.adoc b/docs/src/main/asciidoc/internals.adoc index 7dc32d5c5..0b008a326 100644 --- a/docs/src/main/asciidoc/internals.adoc +++ b/docs/src/main/asciidoc/internals.adoc @@ -395,13 +395,13 @@ The following network protocols are implemented in StormCrawler: * link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/protocol/file/FileProtocol.java[FileProtocol] ===== DelegatorProtocol -The link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/protocol/DelegatorProtocol.java[DelegatorProtocol] enables selection from a collection of sub-protocols using filters based on metadata and URL patterns. The last protocol in the list acts as the default. This is useful when certain URLs need to be fetched with a browser-based protocol (e.g., Selenium) while the majority use a standard HTTP client. +The link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/protocol/DelegatorProtocol.java[DelegatorProtocol] enables selection from a collection of sub-protocols using filters based on metadata and URL patterns. The last protocol in the list acts as the default. This is useful when certain URLs need to be fetched with a browser-based protocol (e.g., Playwright) while the majority use a standard HTTP client. [source,yaml] ---- http.protocol.implementation: "org.apache.stormcrawler.protocol.DelegatorProtocol" protocol.delegator.config: - - className: "org.apache.stormcrawler.protocol.httpclient.HttpProtocol" + - className: "org.apache.stormcrawler.protocol.playwright.HttpProtocol" filters: domain: "example.com" - className: "org.apache.stormcrawler.protocol.okhttp.HttpProtocol" @@ -421,7 +421,7 @@ http.protocol.implementation: "org.apache.stormcrawler.protocol.okhttp.HttpProto https.protocol.implementation: "org.apache.stormcrawler.protocol.okhttp.HttpProtocol" ---- -* link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/protocol/httpclient/HttpProtocol.java[HttpClient] +* link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/protocol/httpclient/HttpProtocol.java[HttpClient] (*deprecated* — will be removed in the next major release; use OKHttp instead) * link:https://github.com/apache/stormcrawler/blob/main/external/selenium/src/main/java/org/apache/stormcrawler/protocol/selenium/SeleniumProtocol.java[Selenium] * link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java[OKHttp] From db0f9b524eceed488e16fd552ddb83be849d5c61 Mon Sep 17 00:00:00 2001 From: Julien Nioche Date: Thu, 23 Jul 2026 15:05:21 +0100 Subject: [PATCH 2/2] Formatting! Signed-off-by: Julien Nioche --- .../org/apache/stormcrawler/protocol/ProtocolFactory.java | 6 ++---- .../stormcrawler/protocol/httpclient/HttpProtocol.java | 4 ++-- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/ProtocolFactory.java b/core/src/main/java/org/apache/stormcrawler/protocol/ProtocolFactory.java index 10619f8b0..3f019bee1 100644 --- a/core/src/main/java/org/apache/stormcrawler/protocol/ProtocolFactory.java +++ b/core/src/main/java/org/apache/stormcrawler/protocol/ProtocolFactory.java @@ -71,11 +71,9 @@ private void configure(Config conf) { if (StringUtils.isBlank(protocolimplementation)) { // set the default values if (protocol.equalsIgnoreCase("http")) { - protocolimplementation = - "org.apache.stormcrawler.protocol.okhttp.HttpProtocol"; + protocolimplementation = "org.apache.stormcrawler.protocol.okhttp.HttpProtocol"; } else if (protocol.equalsIgnoreCase("https")) { - protocolimplementation = - "org.apache.stormcrawler.protocol.okhttp.HttpProtocol"; + protocolimplementation = "org.apache.stormcrawler.protocol.okhttp.HttpProtocol"; } else { throw new RuntimeException(paramName + "should not have an empty value"); } diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/httpclient/HttpProtocol.java b/core/src/main/java/org/apache/stormcrawler/protocol/httpclient/HttpProtocol.java index 9ea6a3321..6c4a137d2 100644 --- a/core/src/main/java/org/apache/stormcrawler/protocol/httpclient/HttpProtocol.java +++ b/core/src/main/java/org/apache/stormcrawler/protocol/httpclient/HttpProtocol.java @@ -71,8 +71,8 @@ /** * Uses Apache httpclient to handle http and https. * - * @deprecated The Apache HttpClient protocol implementation is deprecated and will be removed in the - * next major release of StormCrawler. Use the OkHttp implementation ({@link + * @deprecated The Apache HttpClient protocol implementation is deprecated and will be removed in + * the next major release of StormCrawler. Use the OkHttp implementation ({@link * org.apache.stormcrawler.protocol.okhttp.HttpProtocol}) instead. */ @Deprecated