From 7ed1bb32684b4b3cb5b244541905015ba90f2227 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Szigecs=C3=A1n?= Date: Wed, 22 Jul 2026 22:48:14 +0200 Subject: [PATCH 1/6] Allow indexer bolts to use a metadata value as the document ID --- .../indexing/AbstractIndexerBolt.java | 35 +++++++++++++++++- .../indexer/BasicIndexingTest.java | 37 +++++++++++++++++++ .../stormcrawler/indexer/DummyIndexer.java | 4 ++ docs/src/main/asciidoc/configuration.adoc | 1 + .../aws/bolt/CloudSearchIndexerBolt.java | 4 ++ .../opensearch/bolt/DeletionBolt.java | 21 ++++++++++- .../opensearch/bolt/DeletionBolt.java | 21 ++++++++++- 7 files changed, 118 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/org/apache/stormcrawler/indexing/AbstractIndexerBolt.java b/core/src/main/java/org/apache/stormcrawler/indexing/AbstractIndexerBolt.java index f6f7dceaf..38e5d2314 100644 --- a/core/src/main/java/org/apache/stormcrawler/indexing/AbstractIndexerBolt.java +++ b/core/src/main/java/org/apache/stormcrawler/indexing/AbstractIndexerBolt.java @@ -75,8 +75,19 @@ public abstract class AbstractIndexerBolt extends BaseRichBolt { /** Indicates that empty field values should not be emitted at all. */ public static final String ignoreEmptyFieldValueParamName = "indexer.ignore.empty.fields"; + /** + * Metadata key whose value should be used as the document ID, if present. This can be used e.g. + * by a {@link org.apache.stormcrawler.parse.ParseFilter} generating a hash of the content so + * that duplicate content ends up under the same document ID regardless of the URL(s) it was + * found at. Falls back to the default (a SHA-256 digest of the URL) if not set or if the + * metadata does not contain a value for the configured key. + */ + public static final String DOC_ID_METADATA_PARAM_NAME = "indexer.md.docid"; + private String[] filterKeyValue = null; + private String docIdMetadataKey = null; + private final List metadata2field = new ArrayList<>(); private String fieldNameForText = null; @@ -157,6 +168,8 @@ public void prepare( canonicalMetadataName = ConfUtils.getString(conf, canonicalMetadataParamName); + docIdMetadataKey = ConfUtils.getString(conf, DOC_ID_METADATA_PARAM_NAME); + final Pattern indexValuePattern = Pattern.compile("\\[(\\d+)\\]"); for (String mapping : ConfUtils.loadListFromConf(metadata2fieldParamName, conf)) { @@ -259,12 +272,32 @@ protected Map filterMetadata(Metadata meta) { * * @param metadata The {@link Metadata}. * @param normalisedUrl The normalised url. - * @return Return the normalised url SHA-256 digest as String. + * @return The value found in the metadata for the key configured with {@link + * #DOC_ID_METADATA_PARAM_NAME}, if any, otherwise the normalised url SHA-256 digest as + * String. */ protected String getDocumentID(Metadata metadata, String normalisedUrl) { + final String fromMetadata = getDocumentIDFromMetadata(metadata); + if (fromMetadata != null) { + return fromMetadata; + } return org.apache.commons.codec.digest.DigestUtils.sha256Hex(normalisedUrl); } + /** + * Returns the value found in the metadata for the key configured with {@link + * #DOC_ID_METADATA_PARAM_NAME}, or null if the parameter is not set or has no matching value in + * the metadata. Subclasses which override {@link #getDocumentID} entirely (e.g. to hash or + * sanitize the ID for a specific backend) should call this first so that they still honor the + * configured metadata-based override. + */ + protected final String getDocumentIDFromMetadata(Metadata metadata) { + if (docIdMetadataKey == null) { + return null; + } + return StringUtils.trimToNull(metadata.getFirstValue(docIdMetadataKey)); + } + /** * Returns the value to be used as the URL for indexing purposes. If present the canonical value * is used instead. diff --git a/core/src/test/java/org/apache/stormcrawler/indexer/BasicIndexingTest.java b/core/src/test/java/org/apache/stormcrawler/indexer/BasicIndexingTest.java index cf8cde88c..74c568bbd 100644 --- a/core/src/test/java/org/apache/stormcrawler/indexer/BasicIndexingTest.java +++ b/core/src/test/java/org/apache/stormcrawler/indexer/BasicIndexingTest.java @@ -197,6 +197,43 @@ void testEmptyFilterMetadata() throws Exception { "Index only the URL if no mapping is provided"); } + @Test + void testDocumentIDFromMetadata() throws Exception { + Map config = new HashMap<>(); + config.put(AbstractIndexerBolt.DOC_ID_METADATA_PARAM_NAME, "content.hash"); + Metadata metadata = new Metadata(); + metadata.setValue("content.hash", "abcdef123456"); + prepareIndexerBolt(config); + String docId = ((DummyIndexer) bolt).docId(metadata, URL); + Assertions.assertEquals( + "abcdef123456", + docId, + "The value found in the metadata should be used as the document ID"); + } + + @Test + void testDocumentIDFromMetadataMissingValue() throws Exception { + Map config = new HashMap<>(); + config.put(AbstractIndexerBolt.DOC_ID_METADATA_PARAM_NAME, "content.hash"); + prepareIndexerBolt(config); + String docId = ((DummyIndexer) bolt).docId(new Metadata(), URL); + Assertions.assertEquals( + org.apache.commons.codec.digest.DigestUtils.sha256Hex(URL), + docId, + "Should fall back to the URL digest if the metadata key has no value"); + } + + @Test + void testDocumentIDDefault() throws Exception { + Map config = new HashMap<>(); + prepareIndexerBolt(config); + String docId = ((DummyIndexer) bolt).docId(new Metadata(), URL); + Assertions.assertEquals( + org.apache.commons.codec.digest.DigestUtils.sha256Hex(URL), + docId, + "Should default to the URL digest if no metadata key is configured"); + } + @Test void testGlobFilterMetadata() throws Exception { Map config = new HashMap<>(); diff --git a/core/src/test/java/org/apache/stormcrawler/indexer/DummyIndexer.java b/core/src/test/java/org/apache/stormcrawler/indexer/DummyIndexer.java index 55d5d2925..659f542f6 100644 --- a/core/src/test/java/org/apache/stormcrawler/indexer/DummyIndexer.java +++ b/core/src/test/java/org/apache/stormcrawler/indexer/DummyIndexer.java @@ -86,4 +86,8 @@ private String trimValue(String value) { public Map returnFields() { return fields; } + + public String docId(Metadata metadata, String url) { + return getDocumentID(metadata, url); + } } diff --git a/docs/src/main/asciidoc/configuration.adoc b/docs/src/main/asciidoc/configuration.adoc index fd9e52b61..c629e13ae 100644 --- a/docs/src/main/asciidoc/configuration.adoc +++ b/docs/src/main/asciidoc/configuration.adoc @@ -318,6 +318,7 @@ The values below are used by sub-classes of `AbstractIndexerBolt`. | indexer.canonical.name | canonical | Metadata key for the canonical URL. Used to replace the URL with its canonical form before indexing. | indexer.ignore.empty.fields | false | If true, skip fields with empty values when indexing. +| indexer.md.docid | - | Metadata key whose value, if present, is used as the document ID instead of the default SHA-256 digest of the URL. Useful e.g. for content-based deduplication when a `ParseFilter` stores a content hash in the metadata. | indexer.md.filter | - | YAML list of key=value filters for metadata-based indexing. | indexer.md.mapping | - | YAML mapping from metadata fields to persistence layer fields. | indexer.text.fieldname | content | Field name for indexed HTML body text. diff --git a/external/aws/src/main/java/org/apache/stormcrawler/aws/bolt/CloudSearchIndexerBolt.java b/external/aws/src/main/java/org/apache/stormcrawler/aws/bolt/CloudSearchIndexerBolt.java index 26c25b1b7..518007acc 100644 --- a/external/aws/src/main/java/org/apache/stormcrawler/aws/bolt/CloudSearchIndexerBolt.java +++ b/external/aws/src/main/java/org/apache/stormcrawler/aws/bolt/CloudSearchIndexerBolt.java @@ -174,6 +174,10 @@ public void prepare( @Override protected String getDocumentID(Metadata metadata, String normalisedUrl) { + final String fromMetadata = getDocumentIDFromMetadata(metadata); + if (fromMetadata != null) { + return CloudSearchUtils.getID(fromMetadata); + } return CloudSearchUtils.getID(normalisedUrl); } diff --git a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/bolt/DeletionBolt.java b/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/bolt/DeletionBolt.java index 0c10dd3c2..e6ea14dcc 100644 --- a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/bolt/DeletionBolt.java +++ b/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/bolt/DeletionBolt.java @@ -19,12 +19,14 @@ import java.lang.invoke.MethodHandles; import java.util.Map; +import org.apache.commons.lang3.StringUtils; import org.apache.storm.task.OutputCollector; import org.apache.storm.task.TopologyContext; import org.apache.storm.topology.OutputFieldsDeclarer; import org.apache.storm.topology.base.BaseRichBolt; import org.apache.storm.tuple.Tuple; import org.apache.stormcrawler.Metadata; +import org.apache.stormcrawler.indexing.AbstractIndexerBolt; import org.apache.stormcrawler.metrics.CrawlerMetrics; import org.apache.stormcrawler.opensearch.AsyncBulkProcessor; import org.apache.stormcrawler.opensearch.OpenSearchConnection; @@ -56,6 +58,8 @@ public class DeletionBolt extends BaseRichBolt implements AsyncBulkProcessor.Lis private WaitAckCache waitAck; + private String docIdMetadataKey; + public DeletionBolt() {} /** Sets the index name instead of taking it from the configuration. * */ @@ -71,6 +75,9 @@ public void prepare( indexName = ConfUtils.getString(conf, IndexerBolt.OSIndexNameParamName, "content"); } + docIdMetadataKey = + ConfUtils.getString(conf, AbstractIndexerBolt.DOC_ID_METADATA_PARAM_NAME); + try { connection = OpenSearchConnection.getConnection(conf, BOLT_TYPE, this); } catch (Exception e1) { @@ -121,13 +128,23 @@ protected String getIndexName(Metadata m) { } /** - * Get the document id. + * Get the document id. Uses the value found in the metadata for the key configured with {@link + * AbstractIndexerBolt#DOC_ID_METADATA_PARAM_NAME} if present, so that the ID matches the one + * generated by the indexer for the same document. * * @param metadata The {@link Metadata}. * @param url The normalised url. - * @return Return the normalised url SHA-256 digest as String. + * @return Return the value found in the metadata, or the normalised url SHA-256 digest as + * String. */ protected String getDocumentID(Metadata metadata, String url) { + if (docIdMetadataKey != null) { + final String fromMetadata = + StringUtils.trimToNull(metadata.getFirstValue(docIdMetadataKey)); + if (fromMetadata != null) { + return fromMetadata; + } + } return org.apache.commons.codec.digest.DigestUtils.sha256Hex(url); } diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/bolt/DeletionBolt.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/bolt/DeletionBolt.java index 9295aa15f..51e2c003b 100644 --- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/bolt/DeletionBolt.java +++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/bolt/DeletionBolt.java @@ -19,12 +19,14 @@ import java.lang.invoke.MethodHandles; import java.util.Map; +import org.apache.commons.lang3.StringUtils; import org.apache.storm.task.OutputCollector; import org.apache.storm.task.TopologyContext; import org.apache.storm.topology.OutputFieldsDeclarer; import org.apache.storm.topology.base.BaseRichBolt; import org.apache.storm.tuple.Tuple; import org.apache.stormcrawler.Metadata; +import org.apache.stormcrawler.indexing.AbstractIndexerBolt; import org.apache.stormcrawler.metrics.CrawlerMetrics; import org.apache.stormcrawler.opensearch.OpenSearchConnection; import org.apache.stormcrawler.opensearch.WaitAckCache; @@ -56,6 +58,8 @@ public class DeletionBolt extends BaseRichBolt implements Listener { private WaitAckCache waitAck; + private String docIdMetadataKey; + public DeletionBolt() {} /** Sets the index name instead of taking it from the configuration. * */ @@ -71,6 +75,9 @@ public void prepare( indexName = ConfUtils.getString(conf, IndexerBolt.OSIndexNameParamName, "content"); } + docIdMetadataKey = + ConfUtils.getString(conf, AbstractIndexerBolt.DOC_ID_METADATA_PARAM_NAME); + try { connection = OpenSearchConnection.getConnection(conf, BOLT_TYPE, this); } catch (Exception e1) { @@ -119,13 +126,23 @@ protected String getIndexName(Metadata m) { } /** - * Get the document id. + * Get the document id. Uses the value found in the metadata for the key configured with {@link + * AbstractIndexerBolt#DOC_ID_METADATA_PARAM_NAME} if present, so that the ID matches the one + * generated by the indexer for the same document. * * @param metadata The {@link Metadata}. * @param url The normalised url. - * @return Return the normalised url SHA-256 digest as String. + * @return Return the value found in the metadata, or the normalised url SHA-256 digest as + * String. */ protected String getDocumentID(Metadata metadata, String url) { + if (docIdMetadataKey != null) { + final String fromMetadata = + StringUtils.trimToNull(metadata.getFirstValue(docIdMetadataKey)); + if (fromMetadata != null) { + return fromMetadata; + } + } return org.apache.commons.codec.digest.DigestUtils.sha256Hex(url); } From 7a6d5a40171d58f1722c0ecf4b0437e04592ad9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Szigecs=C3=A1n?= Date: Thu, 23 Jul 2026 09:52:53 +0200 Subject: [PATCH 2/6] Use the SHA-256 digest of the metadata value --- .../indexing/AbstractIndexerBolt.java | 23 +++++++++---------- .../indexer/BasicIndexingTest.java | 10 ++++---- .../opensearch/bolt/DeletionBolt.java | 15 ++++++------ .../opensearch/bolt/DeletionBolt.java | 15 ++++++------ 4 files changed, 33 insertions(+), 30 deletions(-) diff --git a/core/src/main/java/org/apache/stormcrawler/indexing/AbstractIndexerBolt.java b/core/src/main/java/org/apache/stormcrawler/indexing/AbstractIndexerBolt.java index 38e5d2314..5d75ef104 100644 --- a/core/src/main/java/org/apache/stormcrawler/indexing/AbstractIndexerBolt.java +++ b/core/src/main/java/org/apache/stormcrawler/indexing/AbstractIndexerBolt.java @@ -76,11 +76,12 @@ public abstract class AbstractIndexerBolt extends BaseRichBolt { public static final String ignoreEmptyFieldValueParamName = "indexer.ignore.empty.fields"; /** - * Metadata key whose value should be used as the document ID, if present. This can be used e.g. - * by a {@link org.apache.stormcrawler.parse.ParseFilter} generating a hash of the content so - * that duplicate content ends up under the same document ID regardless of the URL(s) it was - * found at. Falls back to the default (a SHA-256 digest of the URL) if not set or if the - * metadata does not contain a value for the configured key. + * Metadata key whose value should be used to derive the document ID, if present. The value is + * passed through the same SHA-256 digest used for the URL-based ID, so this can be used e.g. by + * a {@link org.apache.stormcrawler.parse.ParseFilter} generating a hash of the content so that + * duplicate content ends up under the same document ID regardless of the URL(s) it was found + * at. Falls back to the default (a SHA-256 digest of the URL) if not set or if the metadata + * does not contain a value for the configured key. */ public static final String DOC_ID_METADATA_PARAM_NAME = "indexer.md.docid"; @@ -272,16 +273,14 @@ protected Map filterMetadata(Metadata meta) { * * @param metadata The {@link Metadata}. * @param normalisedUrl The normalised url. - * @return The value found in the metadata for the key configured with {@link - * #DOC_ID_METADATA_PARAM_NAME}, if any, otherwise the normalised url SHA-256 digest as - * String. + * @return The SHA-256 digest of the value found in the metadata for the key configured with + * {@link #DOC_ID_METADATA_PARAM_NAME}, if any, otherwise the SHA-256 digest of the + * normalised url. */ protected String getDocumentID(Metadata metadata, String normalisedUrl) { final String fromMetadata = getDocumentIDFromMetadata(metadata); - if (fromMetadata != null) { - return fromMetadata; - } - return org.apache.commons.codec.digest.DigestUtils.sha256Hex(normalisedUrl); + final String source = fromMetadata != null ? fromMetadata : normalisedUrl; + return org.apache.commons.codec.digest.DigestUtils.sha256Hex(source); } /** diff --git a/core/src/test/java/org/apache/stormcrawler/indexer/BasicIndexingTest.java b/core/src/test/java/org/apache/stormcrawler/indexer/BasicIndexingTest.java index 74c568bbd..b6a65145e 100644 --- a/core/src/test/java/org/apache/stormcrawler/indexer/BasicIndexingTest.java +++ b/core/src/test/java/org/apache/stormcrawler/indexer/BasicIndexingTest.java @@ -22,6 +22,7 @@ import java.util.List; import java.util.Map; import java.util.TreeSet; +import org.apache.commons.codec.digest.DigestUtils; import org.apache.stormcrawler.Metadata; import org.apache.stormcrawler.indexing.AbstractIndexerBolt; import org.junit.jupiter.api.Assertions; @@ -206,9 +207,10 @@ void testDocumentIDFromMetadata() throws Exception { prepareIndexerBolt(config); String docId = ((DummyIndexer) bolt).docId(metadata, URL); Assertions.assertEquals( - "abcdef123456", + DigestUtils.sha256Hex("abcdef123456"), docId, - "The value found in the metadata should be used as the document ID"); + "The SHA-256 digest of the value found in the metadata should be used as the" + + " document ID"); } @Test @@ -218,7 +220,7 @@ void testDocumentIDFromMetadataMissingValue() throws Exception { prepareIndexerBolt(config); String docId = ((DummyIndexer) bolt).docId(new Metadata(), URL); Assertions.assertEquals( - org.apache.commons.codec.digest.DigestUtils.sha256Hex(URL), + DigestUtils.sha256Hex(URL), docId, "Should fall back to the URL digest if the metadata key has no value"); } @@ -229,7 +231,7 @@ void testDocumentIDDefault() throws Exception { prepareIndexerBolt(config); String docId = ((DummyIndexer) bolt).docId(new Metadata(), URL); Assertions.assertEquals( - org.apache.commons.codec.digest.DigestUtils.sha256Hex(URL), + DigestUtils.sha256Hex(URL), docId, "Should default to the URL digest if no metadata key is configured"); } diff --git a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/bolt/DeletionBolt.java b/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/bolt/DeletionBolt.java index e6ea14dcc..f77662ea6 100644 --- a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/bolt/DeletionBolt.java +++ b/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/bolt/DeletionBolt.java @@ -128,24 +128,25 @@ protected String getIndexName(Metadata m) { } /** - * Get the document id. Uses the value found in the metadata for the key configured with {@link - * AbstractIndexerBolt#DOC_ID_METADATA_PARAM_NAME} if present, so that the ID matches the one - * generated by the indexer for the same document. + * Get the document id. Uses the SHA-256 digest of the value found in the metadata for the key + * configured with {@link AbstractIndexerBolt#DOC_ID_METADATA_PARAM_NAME} if present, so that + * the ID matches the one generated by the indexer for the same document. * * @param metadata The {@link Metadata}. * @param url The normalised url. - * @return Return the value found in the metadata, or the normalised url SHA-256 digest as - * String. + * @return Return the SHA-256 digest of the value found in the metadata, or of the normalised + * url if not present. */ protected String getDocumentID(Metadata metadata, String url) { + String source = url; if (docIdMetadataKey != null) { final String fromMetadata = StringUtils.trimToNull(metadata.getFirstValue(docIdMetadataKey)); if (fromMetadata != null) { - return fromMetadata; + source = fromMetadata; } } - return org.apache.commons.codec.digest.DigestUtils.sha256Hex(url); + return org.apache.commons.codec.digest.DigestUtils.sha256Hex(source); } @Override diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/bolt/DeletionBolt.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/bolt/DeletionBolt.java index 51e2c003b..16b3a0059 100644 --- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/bolt/DeletionBolt.java +++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/bolt/DeletionBolt.java @@ -126,24 +126,25 @@ protected String getIndexName(Metadata m) { } /** - * Get the document id. Uses the value found in the metadata for the key configured with {@link - * AbstractIndexerBolt#DOC_ID_METADATA_PARAM_NAME} if present, so that the ID matches the one - * generated by the indexer for the same document. + * Get the document id. Uses the SHA-256 digest of the value found in the metadata for the key + * configured with {@link AbstractIndexerBolt#DOC_ID_METADATA_PARAM_NAME} if present, so that + * the ID matches the one generated by the indexer for the same document. * * @param metadata The {@link Metadata}. * @param url The normalised url. - * @return Return the value found in the metadata, or the normalised url SHA-256 digest as - * String. + * @return Return the SHA-256 digest of the value found in the metadata, or of the normalised + * url if not present. */ protected String getDocumentID(Metadata metadata, String url) { + String source = url; if (docIdMetadataKey != null) { final String fromMetadata = StringUtils.trimToNull(metadata.getFirstValue(docIdMetadataKey)); if (fromMetadata != null) { - return fromMetadata; + source = fromMetadata; } } - return org.apache.commons.codec.digest.DigestUtils.sha256Hex(url); + return org.apache.commons.codec.digest.DigestUtils.sha256Hex(source); } @Override From 7caf1fc699a6d2b597980931deb19cece2f71ebc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Szigecs=C3=A1n?= Date: Thu, 23 Jul 2026 16:15:57 +0200 Subject: [PATCH 3/6] Fix missing "SHA-256 digested" from the description --- docs/src/main/asciidoc/configuration.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/main/asciidoc/configuration.adoc b/docs/src/main/asciidoc/configuration.adoc index c629e13ae..6f64ccb2f 100644 --- a/docs/src/main/asciidoc/configuration.adoc +++ b/docs/src/main/asciidoc/configuration.adoc @@ -318,7 +318,7 @@ The values below are used by sub-classes of `AbstractIndexerBolt`. | indexer.canonical.name | canonical | Metadata key for the canonical URL. Used to replace the URL with its canonical form before indexing. | indexer.ignore.empty.fields | false | If true, skip fields with empty values when indexing. -| indexer.md.docid | - | Metadata key whose value, if present, is used as the document ID instead of the default SHA-256 digest of the URL. Useful e.g. for content-based deduplication when a `ParseFilter` stores a content hash in the metadata. +| indexer.md.docid | - | Metadata key whose value, if present, is SHA-256 digested and used as the document ID instead of the default SHA-256 digest of the URL. Useful e.g. for content-based deduplication when a `ParseFilter` stores a content hash in the metadata. | indexer.md.filter | - | YAML list of key=value filters for metadata-based indexing. | indexer.md.mapping | - | YAML mapping from metadata fields to persistence layer fields. | indexer.text.fieldname | content | Field name for indexed HTML body text. From 1df37d40dcddfd807e4b87bd8855ff1895381365 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Szigecs=C3=A1n?= Date: Thu, 23 Jul 2026 16:46:56 +0200 Subject: [PATCH 4/6] Add Note to the documentation --- docs/src/main/asciidoc/configuration.adoc | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/src/main/asciidoc/configuration.adoc b/docs/src/main/asciidoc/configuration.adoc index 6f64ccb2f..2f0361884 100644 --- a/docs/src/main/asciidoc/configuration.adoc +++ b/docs/src/main/asciidoc/configuration.adoc @@ -326,6 +326,15 @@ The values below are used by sub-classes of `AbstractIndexerBolt`. | indexer.url.fieldname | url | Field name for indexed URL. |=== +Note: `indexer.md.docid` is never used raw: the matching value is passed through SHA-256 +(the same digest used for the default URL-based ID), except for CloudSearch, whose +`CloudSearchIndexerBolt` re-digests it with SHA-512 via `CloudSearchUtils.getID` to satisfy +CloudSearch's ID constraints. It is only honoured by backends whose bolts call +`AbstractIndexerBolt#getDocumentID` under the hood: the OpenSearch and OpenSearch-Java +`IndexerBolt`/`DeletionBolt` pairs, and the CloudSearch `CloudSearchIndexerBolt`. The Solr and +SQL indexers build their document/row key directly from the URL and never call +`getDocumentID()`, so this setting is silently ignored for those two backends. + ==== Status Persistence This refers to persisting the status of a URL (e.g. ERROR, DISCOVERED etc.) along with something like a `nextFetchDate` From cf71395adc6240b5b160f01348319f20f518fde2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Szigecs=C3=A1n?= Date: Thu, 23 Jul 2026 16:27:37 +0200 Subject: [PATCH 5/6] Extend Javadoc with the caveats --- .../stormcrawler/indexing/AbstractIndexerBolt.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/core/src/main/java/org/apache/stormcrawler/indexing/AbstractIndexerBolt.java b/core/src/main/java/org/apache/stormcrawler/indexing/AbstractIndexerBolt.java index 5d75ef104..a121af11c 100644 --- a/core/src/main/java/org/apache/stormcrawler/indexing/AbstractIndexerBolt.java +++ b/core/src/main/java/org/apache/stormcrawler/indexing/AbstractIndexerBolt.java @@ -82,6 +82,18 @@ public abstract class AbstractIndexerBolt extends BaseRichBolt { * duplicate content ends up under the same document ID regardless of the URL(s) it was found * at. Falls back to the default (a SHA-256 digest of the URL) if not set or if the metadata * does not contain a value for the configured key. + * + *

Deletion path caveat: a {@code DeletionBolt} handling a "gone" page does + * not reparse it, so it only sees this key if it was included in {@code metadata.persist} (see + * {@code MetadataTransfer#metadataPersistParamName}) and therefore round-tripped through the + * status store. If the key is missing from {@code metadata.persist}, the deletion falls back to + * {@code sha256(url)}, which will not match the ID the document was indexed under, silently + * orphaning it. + * + *

Content-dedup caveat: when several URLs resolve to the same + * metadata-based ID (e.g. a content hash shared across duplicate pages), deleting any one of + * those URLs deletes the shared document, even though the content may still be live under the + * other URLs, until one of them is re-fetched and re-indexes it. */ public static final String DOC_ID_METADATA_PARAM_NAME = "indexer.md.docid"; From 78bd752951e02088305db4c45968d9916a9b1855 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Szigecs=C3=A1n?= Date: Thu, 23 Jul 2026 16:07:33 +0200 Subject: [PATCH 6/6] Add default empty string to the crawler-default.yaml --- core/src/main/resources/crawler-default.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/core/src/main/resources/crawler-default.yaml b/core/src/main/resources/crawler-default.yaml index 9961320bd..696e97403 100644 --- a/core/src/main/resources/crawler-default.yaml +++ b/core/src/main/resources/crawler-default.yaml @@ -303,6 +303,7 @@ config: # configuration for the classes extending AbstractIndexerBolt # indexer.md.filter: "someKey=aValue" + indexer.md.docid: "" indexer.ignore.empty.fields: false indexer.url.fieldname: "url" indexer.text.fieldname: "content"