From d4f55761e4b663b1872b95bb91275519327a1274 Mon Sep 17 00:00:00 2001 From: Julien Nioche Date: Thu, 23 Jul 2026 12:35:29 +0100 Subject: [PATCH] Mark CloudSearch as deprecated. Fixes #2016 Signed-off-by: Julien Nioche --- docs/src/main/asciidoc/configuration.adoc | 2 ++ docs/src/main/asciidoc/internals.adoc | 2 +- external/aws/README.md | 4 +++- external/aws/aws-conf.yaml | 3 +++ .../stormcrawler/aws/bolt/CloudSearchConstants.java | 5 +++++ .../stormcrawler/aws/bolt/CloudSearchIndexerBolt.java | 8 +++++++- .../apache/stormcrawler/aws/bolt/CloudSearchUtils.java | 5 +++++ 7 files changed, 26 insertions(+), 3 deletions(-) diff --git a/docs/src/main/asciidoc/configuration.adoc b/docs/src/main/asciidoc/configuration.adoc index fd9e52b61..97aee4dd7 100644 --- a/docs/src/main/asciidoc/configuration.adoc +++ b/docs/src/main/asciidoc/configuration.adoc @@ -517,6 +517,8 @@ NOTE: When using the Tika `ParserBolt` alongside `JSoupParserBolt`, set `jsoup.t Integration with AWS services: CloudSearch for indexing and S3 for content caching. See the link:https://github.com/apache/stormcrawler/tree/main/external/aws[aws module] for full details. +NOTE: AWS CloudSearch support (the `cloudsearch.*` settings below) is deprecated and will be removed in the next major release of StormCrawler. + [cols="1,1,3", options="header"] |=== | key | default value | description diff --git a/docs/src/main/asciidoc/internals.adoc b/docs/src/main/asciidoc/internals.adoc index 7dc32d5c5..74bf53cf9 100644 --- a/docs/src/main/asciidoc/internals.adoc +++ b/docs/src/main/asciidoc/internals.adoc @@ -103,7 +103,7 @@ The purpose of crawlers is often to index web pages to make them searchable. The * link:https://github.com/apache/stormcrawler/blob/main/external/solr/src/main/java/org/apache/stormcrawler/solr/bolt/IndexerBolt.java[Apache SOLR] * link:https://github.com/apache/stormcrawler/blob/main/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/bolt/IndexerBolt.java[OpenSearch] -* link:https://github.com/apache/stormcrawler/blob/main/external/aws/src/main/java/org/apache/stormcrawler/aws/bolt/CloudSearchIndexerBolt.java[AWS CloudSearch] +* link:https://github.com/apache/stormcrawler/blob/main/external/aws/src/main/java/org/apache/stormcrawler/aws/bolt/CloudSearchIndexerBolt.java[AWS CloudSearch] (deprecated, will be removed in the next major release) All of these extend the class link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/indexing/AbstractIndexerBolt.java[AbstractIndexerBolt]. diff --git a/external/aws/README.md b/external/aws/README.md index 6c53ee8c1..00cf2fc8b 100644 --- a/external/aws/README.md +++ b/external/aws/README.md @@ -1,7 +1,7 @@ # stormcrawler-aws ================================ -AWS resources for StormCrawler, currently contains an indexer bolt for [CloudSearch](https://aws.amazon.com/cloudsearch/) and another bolt for storing and retrieving web pages to/from [S3](https://aws.amazon.com/s3/). +AWS resources for StormCrawler, currently contains an indexer bolt for [CloudSearch](https://aws.amazon.com/cloudsearch/) (**deprecated**, see below) and another bolt for storing and retrieving web pages to/from [S3](https://aws.amazon.com/s3/). This module is built on the [AWS SDK for Java 2.x](https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/home.html). Credentials are resolved through the SDK's [default credentials provider chain](https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/credentials-chain.html) (environment variables, Java system properties, `~/.aws/credentials`, container/instance profiles, etc.). @@ -21,6 +21,8 @@ Edit `~/.aws/credentials`, see [http://docs.aws.amazon.com/cli/latest/userguide/ ## CloudSearch +> **Deprecated:** AWS CloudSearch support is deprecated and will be removed in the next major release of StormCrawler. + * How to use? Add stormcrawler-aws as a Maven dependency, use the class CloudSearchIndexWriter in your Storm topology alongside the core StormCrawler components and create a yaml configuration file (see below). diff --git a/external/aws/aws-conf.yaml b/external/aws/aws-conf.yaml index 67618b0f1..aff5da87f 100644 --- a/external/aws/aws-conf.yaml +++ b/external/aws/aws-conf.yaml @@ -17,6 +17,9 @@ # CloudSearch Configuration # ############################### +# NOTE: AWS CloudSearch support is deprecated and will be removed +# in the next major release of StormCrawler. + # max number of docs to keep in buffer # before sending to CS irrespective of size limits cloudsearch.batch.maxSize: 100 diff --git a/external/aws/src/main/java/org/apache/stormcrawler/aws/bolt/CloudSearchConstants.java b/external/aws/src/main/java/org/apache/stormcrawler/aws/bolt/CloudSearchConstants.java index fcf234160..3ffba04b2 100644 --- a/external/aws/src/main/java/org/apache/stormcrawler/aws/bolt/CloudSearchConstants.java +++ b/external/aws/src/main/java/org/apache/stormcrawler/aws/bolt/CloudSearchConstants.java @@ -17,6 +17,11 @@ package org.apache.stormcrawler.aws.bolt; +/** + * @deprecated AWS CloudSearch support is deprecated and will be removed in the next major release + * of StormCrawler. + */ +@Deprecated public interface CloudSearchConstants { public static final String CLOUDSEARCH_PREFIX = "cloudsearch."; public static final String ENDPOINT = CLOUDSEARCH_PREFIX + "endpoint"; diff --git a/external/aws/src/main/java/org/apache/stormcrawler/aws/bolt/CloudSearchIndexerBolt.java b/external/aws/src/main/java/org/apache/stormcrawler/aws/bolt/CloudSearchIndexerBolt.java index 26c25b1b7..f7e4f3878 100644 --- a/external/aws/src/main/java/org/apache/stormcrawler/aws/bolt/CloudSearchIndexerBolt.java +++ b/external/aws/src/main/java/org/apache/stormcrawler/aws/bolt/CloudSearchIndexerBolt.java @@ -66,7 +66,13 @@ import software.amazon.awssdk.services.cloudsearchdomain.model.UploadDocumentsRequest; import software.amazon.awssdk.services.cloudsearchdomain.model.UploadDocumentsResponse; -/** Writes documents to CloudSearch. */ +/** + * Writes documents to CloudSearch. + * + * @deprecated AWS CloudSearch support is deprecated and will be removed in the next major release + * of StormCrawler. + */ +@Deprecated public class CloudSearchIndexerBolt extends AbstractIndexerBolt { public static final Logger LOG = LoggerFactory.getLogger(CloudSearchIndexerBolt.class); diff --git a/external/aws/src/main/java/org/apache/stormcrawler/aws/bolt/CloudSearchUtils.java b/external/aws/src/main/java/org/apache/stormcrawler/aws/bolt/CloudSearchUtils.java index 2a2fb442c..51c1c3f05 100644 --- a/external/aws/src/main/java/org/apache/stormcrawler/aws/bolt/CloudSearchUtils.java +++ b/external/aws/src/main/java/org/apache/stormcrawler/aws/bolt/CloudSearchUtils.java @@ -22,6 +22,11 @@ import java.util.regex.Pattern; import org.apache.commons.codec.digest.DigestUtils; +/** + * @deprecated AWS CloudSearch support is deprecated and will be removed in the next major release + * of StormCrawler. + */ +@Deprecated public class CloudSearchUtils { private static final Pattern INVALID_XML_CHARS =