From f96e04947f83e753e2d149c8c2955497008eef59 Mon Sep 17 00:00:00 2001 From: "Piotr P. Karwasz" Date: Tue, 28 Jul 2026 21:23:52 +0200 Subject: [PATCH 1/6] Build with the `logging-parent` reusable workflows Flume no longer maintains its own checkout, JDK and Maven steps, and gains snapshot/release deployment and reproducibility verification in exchange. Assisted-By: Claude Fable 5 --- .github/workflows/build.yaml | 90 ++++++++++++++++++++++++++++++++++++ .github/workflows/build.yml | 77 ------------------------------ 2 files changed, 90 insertions(+), 77 deletions(-) create mode 100644 .github/workflows/build.yaml delete mode 100644 .github/workflows/build.yml diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml new file mode 100644 index 0000000000..a60a31f897 --- /dev/null +++ b/.github/workflows/build.yaml @@ -0,0 +1,90 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to you under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: build + +on: + push: + branches: + - "trunk" + - "release/*" + pull_request: + +# Disable all permissions by defaults: +# Permissions are enabled on a per-job basis. +permissions: { } + +concurrency: + # One group per PR, or per ref for branch pushes. + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + # Cancel in-progress runs for PRs only, so release branch builds always complete. + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + + build: + uses: apache/logging-parent/.github/workflows/build-reusable.yaml@gha/v0 + with: + java-version: 17 + site-enabled: true + reproducibility-check-enabled: false + + deploy-snapshot: + needs: build + if: github.repository == 'apache/logging-flume' && github.ref_name == 'trunk' + uses: apache/logging-parent/.github/workflows/deploy-snapshot-reusable.yaml@gha/v0 + # Secrets for deployments + secrets: + NEXUS_USERNAME: ${{ secrets.LOGGING_STAGE_DEPLOYER_USER }} + NEXUS_PASSWORD: ${{ secrets.LOGGING_STAGE_DEPLOYER_PW }} + with: + java-version: 17 + + deploy-release: + needs: build + if: github.repository == 'apache/logging-flume' && startsWith(github.ref_name, 'release/') + uses: apache/logging-parent/.github/workflows/deploy-release-reusable.yaml@gha/v0 + # Secrets for deployments + secrets: + GPG_SECRET_KEY: ${{ secrets.LOGGING_GPG_SECRET_KEY }} + NEXUS_USERNAME: ${{ secrets.LOGGING_STAGE_DEPLOYER_USER }} + NEXUS_PASSWORD: ${{ secrets.LOGGING_STAGE_DEPLOYER_PW }} + SVN_USERNAME: ${{ secrets.LOGGING_SVN_DEV_USERNAME }} + SVN_PASSWORD: ${{ secrets.LOGGING_SVN_DEV_PASSWORD }} + # Write permissions to allow the Maven `revision` property update, changelog release, etc. + permissions: + contents: write + with: + java-version: 17 + project-id: flume + + verify-reproducibility-snapshot: + needs: deploy-snapshot + uses: apache/logging-parent/.github/workflows/verify-reproducibility-reusable.yaml@gha/v0 + with: + java-version: 17 + # Compare against the repository `deploy-snapshot` uploaded as a run artifact, so that + # the check does not depend on the snapshot having propagated to the Nexus group repository. + reference-artifact-name: ${{ needs.deploy-snapshot.outputs.repository-artifact-name }} + + verify-reproducibility-release: + needs: deploy-release + uses: apache/logging-parent/.github/workflows/verify-reproducibility-reusable.yaml@gha/v0 + with: + java-version: 17 + # `deploy-release` publishes no run artifact, so the staging repository is the reference. + nexus-url: ${{ needs.deploy-release.outputs.nexus-url }} diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml deleted file mode 100644 index afbb2120ae..0000000000 --- a/.github/workflows/build.yml +++ /dev/null @@ -1,77 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to you under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -name: build - -on: - push: - branches: - - trunk - - build-dev - pull_request: - -# Remove all permissions by default. They will be added at a job level. -permissions: {} - -concurrency: - # One group per PR, or per ref for branch pushes. - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - # Cancel in-progress runs for PRs only, so release branch builds always complete. - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - -jobs: - build: - - runs-on: ${{ matrix.os }} - name: build (${{ matrix.os }}) - - strategy: - # Don't cancel the remaining OS builds when one fails - fail-fast: false - matrix: - os: [ ubuntu-latest ] - java-distribution: [ temurin ] - # - # There is no protobuf 2.x version for `aarch64`. - # Reenable the macOS build after the dependency has been upgraded. - # - #include: - # # There is no Temurin JDK 8 for ARM - # - os: macos-latest - # java-distribution: zulu - - steps: - - - name: Checkout repository - uses: actions/checkout@v6 - - # JDK 8 is needed for the build, and it is the primary bytecode target. - - name: Setup JDK 17 - uses: actions/setup-java@v5 - with: - distribution: ${{ matrix.java-distribution }} - java-version: 17 - cache: maven - - - name: Build with Maven - timeout-minutes: 120 - shell: bash - run: | - ./mvnw clean verify -DredirectTestOutput=true \ - --show-version --batch-mode --errors --no-transfer-progress \ - -DtrimStackTrace=false \ - -Dsurefire.rerunFailingTestsCount=2 From e3012af02c18522602f4a33998ce173ea3a1defa Mon Sep 17 00:00:00 2001 From: "Piotr P. Karwasz" Date: Wed, 29 Jul 2026 08:08:20 +0200 Subject: [PATCH 2/6] Skip Taildir tests on non-POSIX file systems TaildirSource tracks files by inode, so its tests cannot pass when the default file system lacks a `unix` attribute view (e.g. on Windows). Document that requirement in the user guide. Assisted-By: Claude Fable 5 --- .../flume/source/taildir/TestTaildirEventReader.java | 10 ++++++++++ .../apache/flume/source/taildir/TestTaildirSource.java | 10 ++++++++++ src/site/antora/modules/ROOT/pages/FlumeUserGuide.adoc | 5 ++++- 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/flume-ng-sources/flume-taildir-source/src/test/java/org/apache/flume/source/taildir/TestTaildirEventReader.java b/flume-ng-sources/flume-taildir-source/src/test/java/org/apache/flume/source/taildir/TestTaildirEventReader.java index bb1986400d..7db3ad1000 100644 --- a/flume-ng-sources/flume-taildir-source/src/test/java/org/apache/flume/source/taildir/TestTaildirEventReader.java +++ b/flume-ng-sources/flume-taildir-source/src/test/java/org/apache/flume/source/taildir/TestTaildirEventReader.java @@ -31,18 +31,28 @@ import com.google.common.io.Files; import java.io.File; import java.io.IOException; +import java.nio.file.FileSystems; import java.util.List; import java.util.Map; import java.util.Set; import org.apache.flume.Event; import org.junit.After; +import org.junit.Assume; import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; public class TestTaildirEventReader { private File tmpDir; private String posFilePath; + @BeforeClass + public static void requireUnixFileAttributeView() { + // ReliableTaildirEventReader tracks files by inode via the `unix:ino` file attribute + Assume.assumeTrue("The default file system does not provide a `unix` file attribute view", + FileSystems.getDefault().supportedFileAttributeViews().contains("unix")); + } + public static String bodyAsString(Event event) { return new String(event.getBody()); } diff --git a/flume-ng-sources/flume-taildir-source/src/test/java/org/apache/flume/source/taildir/TestTaildirSource.java b/flume-ng-sources/flume-taildir-source/src/test/java/org/apache/flume/source/taildir/TestTaildirSource.java index 2991bfb721..eb97f69593 100644 --- a/flume-ng-sources/flume-taildir-source/src/test/java/org/apache/flume/source/taildir/TestTaildirSource.java +++ b/flume-ng-sources/flume-taildir-source/src/test/java/org/apache/flume/source/taildir/TestTaildirSource.java @@ -39,6 +39,7 @@ import com.google.common.io.Files; import java.io.File; import java.io.IOException; +import java.nio.file.FileSystems; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -58,7 +59,9 @@ import org.apache.flume.lifecycle.LifecycleState; import org.apache.flume.util.Whitebox; import org.junit.After; +import org.junit.Assume; import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; import org.mockito.Mockito; @@ -68,6 +71,13 @@ public class TestTaildirSource { private File tmpDir; private String posFilePath; + @BeforeClass + public static void requireUnixFileAttributeView() { + // TaildirSource tracks files by inode via the `unix:ino` file attribute + Assume.assumeTrue("The default file system does not provide a `unix` file attribute view", + FileSystems.getDefault().supportedFileAttributeViews().contains("unix")); + } + @Before public void setUp() { source = new TaildirSource(); diff --git a/src/site/antora/modules/ROOT/pages/FlumeUserGuide.adoc b/src/site/antora/modules/ROOT/pages/FlumeUserGuide.adoc index 72d2428cdd..397c802724 100644 --- a/src/site/antora/modules/ROOT/pages/FlumeUserGuide.adoc +++ b/src/site/antora/modules/ROOT/pages/FlumeUserGuide.adoc @@ -1680,7 +1680,10 @@ This deserializer reads a Binary Large Object (BLOB) per event, typically one BL [NOTE] ==== -*This source is provided as a preview feature. It does not work on Windows.* +*This source is provided as a preview feature.* + +This source tracks files by their inode number and therefore requires a file system that provides the `unix` file attribute view. +In particular, it does not work on Windows. ==== Watch the specified files, and tail them in nearly real-time once detected new lines appended to the each files. From 2589484103e5e9d245101a6952673571158f3edc Mon Sep 17 00:00:00 2001 From: "Piotr P. Karwasz" Date: Wed, 29 Jul 2026 08:09:30 +0200 Subject: [PATCH 3/6] Formatting --- .../apache/flume/source/taildir/TestTaildirEventReader.java | 3 ++- .../org/apache/flume/source/taildir/TestTaildirSource.java | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/flume-ng-sources/flume-taildir-source/src/test/java/org/apache/flume/source/taildir/TestTaildirEventReader.java b/flume-ng-sources/flume-taildir-source/src/test/java/org/apache/flume/source/taildir/TestTaildirEventReader.java index 7db3ad1000..54d5738039 100644 --- a/flume-ng-sources/flume-taildir-source/src/test/java/org/apache/flume/source/taildir/TestTaildirEventReader.java +++ b/flume-ng-sources/flume-taildir-source/src/test/java/org/apache/flume/source/taildir/TestTaildirEventReader.java @@ -49,7 +49,8 @@ public class TestTaildirEventReader { @BeforeClass public static void requireUnixFileAttributeView() { // ReliableTaildirEventReader tracks files by inode via the `unix:ino` file attribute - Assume.assumeTrue("The default file system does not provide a `unix` file attribute view", + Assume.assumeTrue( + "The default file system does not provide a `unix` file attribute view", FileSystems.getDefault().supportedFileAttributeViews().contains("unix")); } diff --git a/flume-ng-sources/flume-taildir-source/src/test/java/org/apache/flume/source/taildir/TestTaildirSource.java b/flume-ng-sources/flume-taildir-source/src/test/java/org/apache/flume/source/taildir/TestTaildirSource.java index eb97f69593..4c8ec6a6b4 100644 --- a/flume-ng-sources/flume-taildir-source/src/test/java/org/apache/flume/source/taildir/TestTaildirSource.java +++ b/flume-ng-sources/flume-taildir-source/src/test/java/org/apache/flume/source/taildir/TestTaildirSource.java @@ -74,7 +74,8 @@ public class TestTaildirSource { @BeforeClass public static void requireUnixFileAttributeView() { // TaildirSource tracks files by inode via the `unix:ino` file attribute - Assume.assumeTrue("The default file system does not provide a `unix` file attribute view", + Assume.assumeTrue( + "The default file system does not provide a `unix` file attribute view", FileSystems.getDefault().supportedFileAttributeViews().contains("unix")); } From 5b511a7005ea69e399081869cda80c1af47b6586 Mon Sep 17 00:00:00 2001 From: "Piotr P. Karwasz" Date: Thu, 30 Jul 2026 09:08:15 +0200 Subject: [PATCH 4/6] Fix race condition in TestSyslogTcpSource.testSSLMessages The test read from the channel immediately after closing the client socket, but the source delivers events asynchronously, so the take could return null on slow machines (first seen on Windows CI). Wait for the source counter with Awaitility and close the TLS socket gracefully, flushing before close. Assisted-By: Claude Fable 5 --- flume-ng-sources/flume-syslog-source/pom.xml | 6 ++++++ .../source/syslog/TestSyslogTcpSource.java | 19 ++++++++++++------- flume-parent/pom.xml | 8 ++++++++ 3 files changed, 26 insertions(+), 7 deletions(-) diff --git a/flume-ng-sources/flume-syslog-source/pom.xml b/flume-ng-sources/flume-syslog-source/pom.xml index f4bf49c18a..d169d16458 100644 --- a/flume-ng-sources/flume-syslog-source/pom.xml +++ b/flume-ng-sources/flume-syslog-source/pom.xml @@ -74,6 +74,12 @@ test + + org.awaitility + awaitility + test + + org.mockito mockito-core diff --git a/flume-ng-sources/flume-syslog-source/src/test/java/org/apache/flume/source/syslog/TestSyslogTcpSource.java b/flume-ng-sources/flume-syslog-source/src/test/java/org/apache/flume/source/syslog/TestSyslogTcpSource.java index cbc6f40446..ada3a3f6dc 100644 --- a/flume-ng-sources/flume-syslog-source/src/test/java/org/apache/flume/source/syslog/TestSyslogTcpSource.java +++ b/flume-ng-sources/flume-syslog-source/src/test/java/org/apache/flume/source/syslog/TestSyslogTcpSource.java @@ -16,7 +16,9 @@ */ package org.apache.flume.source.syslog; +import static org.awaitility.Awaitility.await; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.mockito.ArgumentMatchers.any; @@ -269,17 +271,20 @@ public X509Certificate[] getAcceptedIssuers() { }, null); SocketFactory socketFactory = sslContext.getSocketFactory(); - Socket socket = socketFactory.createSocket(); - socket.connect(address); - OutputStream outputStream = socket.getOutputStream(); - outputStream.write(bodyWithTandH.getBytes()); - socket.close(); - // Thread.sleep(100); + try (Socket socket = socketFactory.createSocket()) { + socket.connect(address); + OutputStream outputStream = socket.getOutputStream(); + outputStream.write(bodyWithTandH.getBytes()); + outputStream.flush(); + } + // The counter is incremented after the event is committed to the channel + await().until(() -> source.getSourceCounter().getEventAcceptedCount() >= 1); Transaction transaction = channel.getTransaction(); transaction.begin(); Event event = channel.take(); - assertEquals(new String(event.getBody()), data1); + assertNotNull("The source accepted an event, but the channel did not deliver one.", event); + assertEquals(data1, new String(event.getBody())); transaction.commit(); transaction.close(); } diff --git a/flume-parent/pom.xml b/flume-parent/pom.xml index 56a3f80b89..00c1969ad4 100644 --- a/flume-parent/pom.xml +++ b/flume-parent/pom.xml @@ -95,6 +95,7 @@ **/Test*.java true + 4.3.0 5.9.0 1.4.1 3.5.0 @@ -118,6 +119,13 @@ import + + org.awaitility + awaitility + ${awaitility.version} + test + + junit junit From f0850624df7e5aeb29062c65f7f0a3b639e3c4a2 Mon Sep 17 00:00:00 2001 From: "Piotr P. Karwasz" Date: Thu, 30 Jul 2026 09:47:43 +0200 Subject: [PATCH 5/6] Keep the client socket open until the syslog event is committed The Windows CI logs show the TLS handshake succeeding and the connection aborting one millisecond later: the client never reads the TLS 1.3 session tickets sent by the server, so close() aborts the connection with a TCP RST, which on Windows discards the syslog record before the server reads it. Assisted-By: Claude Fable 5 --- .../apache/flume/source/syslog/TestSyslogTcpSource.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/flume-ng-sources/flume-syslog-source/src/test/java/org/apache/flume/source/syslog/TestSyslogTcpSource.java b/flume-ng-sources/flume-syslog-source/src/test/java/org/apache/flume/source/syslog/TestSyslogTcpSource.java index ada3a3f6dc..e51b379957 100644 --- a/flume-ng-sources/flume-syslog-source/src/test/java/org/apache/flume/source/syslog/TestSyslogTcpSource.java +++ b/flume-ng-sources/flume-syslog-source/src/test/java/org/apache/flume/source/syslog/TestSyslogTcpSource.java @@ -276,9 +276,12 @@ public X509Certificate[] getAcceptedIssuers() { OutputStream outputStream = socket.getOutputStream(); outputStream.write(bodyWithTandH.getBytes()); outputStream.flush(); + // Close the socket only after the server has committed the event to the channel + // (the counter is incremented after the commit). The client never reads the TLS 1.3 + // session tickets sent by the server, so an earlier close() aborts the connection + // with a TCP RST, which on Windows can discard the data before the server reads it. + await().until(() -> source.getSourceCounter().getEventAcceptedCount() >= 1); } - // The counter is incremented after the event is committed to the channel - await().until(() -> source.getSourceCounter().getEventAcceptedCount() >= 1); Transaction transaction = channel.getTransaction(); transaction.begin(); From 3cb7287b3ce9461983eb182e3b3989d56d46c502 Mon Sep 17 00:00:00 2001 From: "Piotr P. Karwasz" Date: Thu, 30 Jul 2026 09:51:10 +0200 Subject: [PATCH 6/6] Reflow comment --- .../apache/flume/source/syslog/TestSyslogTcpSource.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/flume-ng-sources/flume-syslog-source/src/test/java/org/apache/flume/source/syslog/TestSyslogTcpSource.java b/flume-ng-sources/flume-syslog-source/src/test/java/org/apache/flume/source/syslog/TestSyslogTcpSource.java index e51b379957..d6cf09a49f 100644 --- a/flume-ng-sources/flume-syslog-source/src/test/java/org/apache/flume/source/syslog/TestSyslogTcpSource.java +++ b/flume-ng-sources/flume-syslog-source/src/test/java/org/apache/flume/source/syslog/TestSyslogTcpSource.java @@ -276,10 +276,10 @@ public X509Certificate[] getAcceptedIssuers() { OutputStream outputStream = socket.getOutputStream(); outputStream.write(bodyWithTandH.getBytes()); outputStream.flush(); - // Close the socket only after the server has committed the event to the channel - // (the counter is incremented after the commit). The client never reads the TLS 1.3 - // session tickets sent by the server, so an earlier close() aborts the connection - // with a TCP RST, which on Windows can discard the data before the server reads it. + // Close the socket only after the server has committed the event to the channel. + // The client never reads the TLS 1.3 session tickets sent by the server, + // so an earlier close() aborts the connection with a TCP RST, + // which on Windows can discard the data before the server reads it. await().until(() -> source.getSourceCounter().getEventAcceptedCount() >= 1); } Transaction transaction = channel.getTransaction();