diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/DelegatorProtocol.java b/core/src/main/java/org/apache/stormcrawler/protocol/DelegatorProtocol.java
index 087bc70b2..b19760b93 100644
--- a/core/src/main/java/org/apache/stormcrawler/protocol/DelegatorProtocol.java
+++ b/core/src/main/java/org/apache/stormcrawler/protocol/DelegatorProtocol.java
@@ -38,7 +38,7 @@
*
*
* protocol.delegator.config:
- * - className: "org.apache.stormcrawler.protocol.httpclient.HttpProtocol"
+ * - className: "org.apache.stormcrawler.protocol.okhttp.HttpProtocol"
* filters:
* domain: "example.com"
* depth: "3"
@@ -50,7 +50,7 @@
* regex:
* - \.pdf
* - \.doc
- * - className: "org.apache.stormcrawler.protocol.selenium.SeleniumProtocol"
+ * - className: "org.apache.stormcrawler.protocol.playwright.HttpProtocol"
*
*
* Typically, the last one in the list must not have filters as it is used as a default value. The
diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/ProtocolFactory.java b/core/src/main/java/org/apache/stormcrawler/protocol/ProtocolFactory.java
index 5b870fbdc..3f019bee1 100644
--- a/core/src/main/java/org/apache/stormcrawler/protocol/ProtocolFactory.java
+++ b/core/src/main/java/org/apache/stormcrawler/protocol/ProtocolFactory.java
@@ -71,11 +71,9 @@ private void configure(Config conf) {
if (StringUtils.isBlank(protocolimplementation)) {
// set the default values
if (protocol.equalsIgnoreCase("http")) {
- protocolimplementation =
- "org.apache.stormcrawler.protocol.httpclient.HttpProtocol";
+ protocolimplementation = "org.apache.stormcrawler.protocol.okhttp.HttpProtocol";
} else if (protocol.equalsIgnoreCase("https")) {
- protocolimplementation =
- "org.apache.stormcrawler.protocol.httpclient.HttpProtocol";
+ protocolimplementation = "org.apache.stormcrawler.protocol.okhttp.HttpProtocol";
} else {
throw new RuntimeException(paramName + "should not have an empty value");
}
diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/httpclient/HttpProtocol.java b/core/src/main/java/org/apache/stormcrawler/protocol/httpclient/HttpProtocol.java
index 0c67b1841..6c4a137d2 100644
--- a/core/src/main/java/org/apache/stormcrawler/protocol/httpclient/HttpProtocol.java
+++ b/core/src/main/java/org/apache/stormcrawler/protocol/httpclient/HttpProtocol.java
@@ -68,7 +68,14 @@
import org.apache.stormcrawler.util.CookieConverter;
import org.slf4j.LoggerFactory;
-/** Uses Apache httpclient to handle http and https. */
+/**
+ * Uses Apache httpclient to handle http and https.
+ *
+ * @deprecated The Apache HttpClient protocol implementation is deprecated and will be removed in
+ * the next major release of StormCrawler. Use the OkHttp implementation ({@link
+ * org.apache.stormcrawler.protocol.okhttp.HttpProtocol}) instead.
+ */
+@Deprecated
public class HttpProtocol extends AbstractHttpProtocol
implements ResponseHandler {
diff --git a/core/src/main/java/org/apache/stormcrawler/proxy/MultiProxyManager.java b/core/src/main/java/org/apache/stormcrawler/proxy/MultiProxyManager.java
index 68a600d46..b65392c3c 100644
--- a/core/src/main/java/org/apache/stormcrawler/proxy/MultiProxyManager.java
+++ b/core/src/main/java/org/apache/stormcrawler/proxy/MultiProxyManager.java
@@ -30,7 +30,6 @@
import org.apache.commons.lang3.ArrayUtils;
import org.apache.storm.Config;
import org.apache.stormcrawler.Metadata;
-import org.apache.stormcrawler.protocol.httpclient.HttpProtocol;
import org.apache.stormcrawler.util.ConfUtils;
import org.slf4j.LoggerFactory;
@@ -48,7 +47,7 @@ public enum ProxyRotation {
private final AtomicInteger lastAccessedIndex = new AtomicInteger(0);
private Map proxyLookupMap;
- private static final org.slf4j.Logger LOG = LoggerFactory.getLogger(HttpProtocol.class);
+ private static final org.slf4j.Logger LOG = LoggerFactory.getLogger(MultiProxyManager.class);
/** Default constructor for setting up the proxy manager. */
private void init(ProxyRotation rotation) {
diff --git a/core/src/main/resources/crawler-default.yaml b/core/src/main/resources/crawler-default.yaml
index 9961320bd..f5c55104f 100644
--- a/core/src/main/resources/crawler-default.yaml
+++ b/core/src/main/resources/crawler-default.yaml
@@ -194,8 +194,11 @@ config:
robots.error.cache.spec: "maximumSize=10000,expireAfterWrite=1h"
protocols: "http,https,file"
- http.protocol.implementation: "org.apache.stormcrawler.protocol.httpclient.HttpProtocol"
- https.protocol.implementation: "org.apache.stormcrawler.protocol.httpclient.HttpProtocol"
+ # NOTE: the Apache HttpClient protocol implementation
+ # (org.apache.stormcrawler.protocol.httpclient.HttpProtocol) is deprecated and will be removed in
+ # the next major release of StormCrawler. OkHttp is now the default.
+ http.protocol.implementation: "org.apache.stormcrawler.protocol.okhttp.HttpProtocol"
+ https.protocol.implementation: "org.apache.stormcrawler.protocol.okhttp.HttpProtocol"
file.protocol.implementation: "org.apache.stormcrawler.protocol.file.FileProtocol"
# number of instances for each protocol implementation
diff --git a/core/src/test/java/org/apache/stormcrawler/protocol/httpclient/HttpClientProtocolProxyManagerTest.java b/core/src/test/java/org/apache/stormcrawler/protocol/httpclient/HttpClientProtocolProxyManagerTest.java
index d67de9368..d350e8f80 100644
--- a/core/src/test/java/org/apache/stormcrawler/protocol/httpclient/HttpClientProtocolProxyManagerTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/protocol/httpclient/HttpClientProtocolProxyManagerTest.java
@@ -26,6 +26,7 @@
import org.apache.stormcrawler.proxy.SingleProxyManager;
import org.junit.jupiter.api.Test;
+@SuppressWarnings("deprecation")
class HttpClientProtocolProxyManagerTest {
@Test
diff --git a/docs/src/main/asciidoc/configuration.adoc b/docs/src/main/asciidoc/configuration.adoc
index fd9e52b61..1b3075f5c 100644
--- a/docs/src/main/asciidoc/configuration.adoc
+++ b/docs/src/main/asciidoc/configuration.adoc
@@ -201,8 +201,10 @@ is defined.
| http.basicauth.password | - | Password for http.basicauth.user.
| http.basicauth.user | - | Username for Basic Authentication.
| http.content.limit | -1 | Maximum HTTP response body size (bytes). Default: no limit.
-| http.protocol.implementation | org.apache.stormcrawler.protocol.httpclient.HttpProtocol | HTTP Protocol
-implementation.
+| http.protocol.implementation | org.apache.stormcrawler.protocol.okhttp.HttpProtocol | HTTP Protocol
+implementation. Note: the Apache HttpClient implementation
+(`org.apache.stormcrawler.protocol.httpclient.HttpProtocol`) is *deprecated* and will be removed in
+the next major release.
| http.proxy | - | Full **SCProxy** connection string read by SingleProxyManager when that manager is selected, e.g. `http://user:pass@proxy.example.com:8080`.
| http.proxy.file | - | Proxy connection-string file read by MultiProxyManager when that manager is selected.
| http.proxy.host | - | SingleProxyManager component proxy host.
@@ -225,8 +227,10 @@ implementation.
| http.store.headers | false | Whether to store response headers.
| http.timeout | 10000 | Connection timeout (ms).
| http.use.cookies | false | Use cookies in subsequent requests.
-| https.protocol.implementation | org.apache.stormcrawler.protocol.httpclient.HttpProtocol | HTTPS Protocol
-implementation.
+| https.protocol.implementation | org.apache.stormcrawler.protocol.okhttp.HttpProtocol | HTTPS Protocol
+implementation. Note: the Apache HttpClient implementation
+(`org.apache.stormcrawler.protocol.httpclient.HttpProtocol`) is *deprecated* and will be removed in
+the next major release.
| partition.url.mode | byHost | Defines how URLs are partitioned: byHost, byDomain, or byIP.
| protocols | http,https,file | Supported protocols.
| redirections.allowed | true | If true, emit redirect target URLs as "outlinks" to the status stream. If false, do not follow redirects. See also `http.allow.redirects`.
diff --git a/docs/src/main/asciidoc/internals.adoc b/docs/src/main/asciidoc/internals.adoc
index 7dc32d5c5..0b008a326 100644
--- a/docs/src/main/asciidoc/internals.adoc
+++ b/docs/src/main/asciidoc/internals.adoc
@@ -395,13 +395,13 @@ The following network protocols are implemented in StormCrawler:
* link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/protocol/file/FileProtocol.java[FileProtocol]
===== DelegatorProtocol
-The link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/protocol/DelegatorProtocol.java[DelegatorProtocol] enables selection from a collection of sub-protocols using filters based on metadata and URL patterns. The last protocol in the list acts as the default. This is useful when certain URLs need to be fetched with a browser-based protocol (e.g., Selenium) while the majority use a standard HTTP client.
+The link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/protocol/DelegatorProtocol.java[DelegatorProtocol] enables selection from a collection of sub-protocols using filters based on metadata and URL patterns. The last protocol in the list acts as the default. This is useful when certain URLs need to be fetched with a browser-based protocol (e.g., Playwright) while the majority use a standard HTTP client.
[source,yaml]
----
http.protocol.implementation: "org.apache.stormcrawler.protocol.DelegatorProtocol"
protocol.delegator.config:
- - className: "org.apache.stormcrawler.protocol.httpclient.HttpProtocol"
+ - className: "org.apache.stormcrawler.protocol.playwright.HttpProtocol"
filters:
domain: "example.com"
- className: "org.apache.stormcrawler.protocol.okhttp.HttpProtocol"
@@ -421,7 +421,7 @@ http.protocol.implementation: "org.apache.stormcrawler.protocol.okhttp.HttpProto
https.protocol.implementation: "org.apache.stormcrawler.protocol.okhttp.HttpProtocol"
----
-* link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/protocol/httpclient/HttpProtocol.java[HttpClient]
+* link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/protocol/httpclient/HttpProtocol.java[HttpClient] (*deprecated* — will be removed in the next major release; use OKHttp instead)
* link:https://github.com/apache/stormcrawler/blob/main/external/selenium/src/main/java/org/apache/stormcrawler/protocol/selenium/SeleniumProtocol.java[Selenium]
* link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java[OKHttp]