diff --git a/.brazil.json b/.brazil.json index 69638fc11a0a..ad498cefb483 100644 --- a/.brazil.json +++ b/.brazil.json @@ -117,7 +117,8 @@ "crt-unavailable-tests": { "skipImport": true }, "bundle-shading-tests": { "skipImport": true }, "architecture-tests": {"skipImport": true}, - "s3-tests": {"skipImport": true} + "s3-tests": {"skipImport": true}, + "warmup-tests": { "skipImport": true } }, "dependencies": { diff --git a/.github/workflows/dependency-cve-monitor.yml b/.github/workflows/dependency-cve-monitor.yml index 22868167c539..69b95ec86c9a 100644 --- a/.github/workflows/dependency-cve-monitor.yml +++ b/.github/workflows/dependency-cve-monitor.yml @@ -21,7 +21,7 @@ jobs: maven-args: >- -DtransitiveExcludes=*:* -DclasspathScope=runtime - -pl !build-tools,!release-scripts,!archetypes,!test/test-utils,!test/sdk-benchmarks,!test/http-client-tests,!test/http-client-benchmarks,!test/s3-benchmarks,!test/protocol-tests-core,!test/ruleset-testing-core,!test/protocol-tests,!test/service-test-utils,!test/codegen-generated-classes-test,!test/sdk-standard-benchmarks,!test/module-path-tests,!test/tests-coverage-reporting,!test/stability-tests,!test/sdk-native-image-test,!test/auth-tests,!test/region-testing,!test/old-client-version-compatibility-test,!test/bundle-logging-bridge-binding-test,!test/v2-migration-tests,!test/bundle-shading-tests,!test/crt-unavailable-tests,!test/architecture-tests,!test/s3-tests + -pl !build-tools,!release-scripts,!archetypes,!test/test-utils,!test/sdk-benchmarks,!test/http-client-tests,!test/http-client-benchmarks,!test/s3-benchmarks,!test/protocol-tests-core,!test/ruleset-testing-core,!test/protocol-tests,!test/service-test-utils,!test/codegen-generated-classes-test,!test/sdk-standard-benchmarks,!test/module-path-tests,!test/tests-coverage-reporting,!test/stability-tests,!test/sdk-native-image-test,!test/auth-tests,!test/region-testing,!test/old-client-version-compatibility-test,!test/bundle-logging-bridge-binding-test,!test/v2-migration-tests,!test/bundle-shading-tests,!test/crt-unavailable-tests,!test/architecture-tests,!test/s3-tests,!test/warmup-tests notify-alerts: if: github.repository == 'aws/aws-sdk-java-v2' diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/crac/SdkWarmUp.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/crac/SdkWarmUp.java index 3ab09cfcf437..9bd34cab3122 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/crac/SdkWarmUp.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/crac/SdkWarmUp.java @@ -19,6 +19,7 @@ import software.amazon.awssdk.annotations.SdkPublicApi; import software.amazon.awssdk.annotations.ThreadSafe; import software.amazon.awssdk.core.internal.crac.ClasspathWarmUpInvoker; +import software.amazon.awssdk.core.internal.http.loader.ClasspathHttpWarmupInvoker; /** * Entry point for warming up SDK service request paths before a Coordinated Restore at Checkpoint (CRaC) @@ -66,6 +67,7 @@ public static void prime() { } // Set primed only after invokeAll() succeeds, so a failed run leaves primed false and a later call retries. ClasspathWarmUpInvoker.create().invokeAll(); + ClasspathHttpWarmupInvoker.create().invokeAll(); primed = true; } } diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/crac/ClasspathWarmUpInvoker.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/crac/ClasspathWarmUpInvoker.java index 4624b4fdeb14..9f215505b8d9 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/crac/ClasspathWarmUpInvoker.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/crac/ClasspathWarmUpInvoker.java @@ -15,13 +15,10 @@ package software.amazon.awssdk.core.internal.crac; -import java.util.Iterator; -import java.util.ServiceConfigurationError; import java.util.ServiceLoader; import software.amazon.awssdk.annotations.SdkInternalApi; import software.amazon.awssdk.annotations.SdkTestInternalApi; import software.amazon.awssdk.core.crac.SdkWarmUpProvider; -import software.amazon.awssdk.utils.Logger; /** * {@link WarmUpInvoker} implementation that uses {@link ServiceLoader} to find {@link SdkWarmUpProvider} @@ -30,8 +27,6 @@ @SdkInternalApi public final class ClasspathWarmUpInvoker implements WarmUpInvoker { - private static final Logger log = Logger.loggerFor(ClasspathWarmUpInvoker.class); - private final WarmUpServiceLoader serviceLoader; @SdkTestInternalApi @@ -41,30 +36,7 @@ public final class ClasspathWarmUpInvoker implements WarmUpInvoker { @Override public void invokeAll() { - Iterator iterator = serviceLoader.loadProviders(); - boolean invokedAny = false; - - while (iterator.hasNext()) { - SdkWarmUpProvider provider; - try { - provider = iterator.next(); - } catch (ServiceConfigurationError e) { - // next() has already advanced past the bad provider, so it is safe to continue to the next one. - log.warn(() -> "Skipping an SdkWarmUpProvider that could not be loaded.", e); - continue; - } - - invokedAny = true; - try { - provider.warmUp(); - } catch (RuntimeException e) { - log.warn(() -> "An SdkWarmUpProvider failed during warmUp() and was skipped.", e); - } - } - - if (!invokedAny) { - log.debug(() -> "No SdkWarmUpProvider implementations were discovered on the classpath."); - } + WarmUpDiscovery.forEachDiscovered(serviceLoader.loadProviders(), SdkWarmUpProvider::warmUp); } /** diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/crac/RegionEndpointResolver.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/crac/RegionEndpointResolver.java new file mode 100644 index 000000000000..fe7a71f5c3a8 --- /dev/null +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/crac/RegionEndpointResolver.java @@ -0,0 +1,77 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.core.internal.crac; + +import java.net.URI; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.core.SdkSystemSetting; +import software.amazon.awssdk.utils.HostnameValidator; +import software.amazon.awssdk.utils.Logger; +import software.amazon.awssdk.utils.StringUtils; + +/** + * Resolves the endpoint used by the CRaC HTTP-client warm-up. The warm-up targets a single predetermined service; that service + * is currently STS, so this builds the regional STS endpoint ({@code https://sts..amazonaws.com/}). + * + *

The region is taken from {@link SdkSystemSetting#AWS_REGION} (the {@code aws.region} system property or {@code AWS_REGION} + * environment variable), falling back to {@value #DEFAULT_REGION} when it is unset or not a valid hostname component. This + * matches how the SDK resolves a region from system settings. + * + *

Only system properties and environment variables are read. The full SDK region-resolution chain (IMDS, profile file) is + * avoided during priming because those add network or filesystem calls that may fail or time out. The endpoint host always + * uses the {@code amazonaws.com} suffix, which is incorrect for the China, GovCloud, and ISO partitions; in those partitions + * the warm-up request simply fails and is ignored, since it is best-effort. + */ +@SdkInternalApi +public final class RegionEndpointResolver { + + static final String DEFAULT_REGION = "us-east-1"; + + private static final Logger log = Logger.loggerFor(RegionEndpointResolver.class); + + private RegionEndpointResolver() { + } + + public static RegionEndpointResolver create() { + return new RegionEndpointResolver(); + } + + /** + * @return the regional STS endpoint URI for the resolved region; never null. + */ + public URI endpoint() { + return URI.create("https://sts." + resolveRegion() + ".amazonaws.com/"); + } + + private String resolveRegion() { + // trimToNull turns blank/empty into null so a blank AWS_REGION falls through to the default. + String awsRegion = SdkSystemSetting.AWS_REGION.getStringValue() + .map(StringUtils::trimToNull) + .orElse(null); + if (awsRegion == null) { + return DEFAULT_REGION; + } + // A real region is a hostname-compliant token. Reject anything else so it cannot alter the endpoint host, and fall + // back to the default so the best-effort warm-up still runs. + try { + HostnameValidator.validateHostnameCompliant(awsRegion, "region", "AWS_REGION"); + return awsRegion; + } catch (IllegalArgumentException e) { + log.debug(() -> "Configured region is not a valid hostname component; using " + DEFAULT_REGION + " for warm-up.", e); + return DEFAULT_REGION; + } + } +} diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/crac/WarmUpDiscovery.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/crac/WarmUpDiscovery.java new file mode 100644 index 000000000000..f97a13dcd613 --- /dev/null +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/crac/WarmUpDiscovery.java @@ -0,0 +1,66 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.core.internal.crac; + +import java.util.Iterator; +import java.util.ServiceConfigurationError; +import java.util.function.Consumer; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.utils.Logger; + +/** + * Shared best-effort {@link java.util.ServiceLoader} iteration for the CRaC warm-up paths. + */ +@SdkInternalApi +public final class WarmUpDiscovery { + + private static final Logger log = Logger.loggerFor(WarmUpDiscovery.class); + + private WarmUpDiscovery() { + } + + /** + * Applies {@code action} to every discovered element, skipping (and logging at warn) any element that fails to load or + * whose action throws, so the rest still run. Logs at debug when nothing is discovered. + */ + public static void forEachDiscovered(Iterator iterator, Consumer action) { + boolean discoveredAny = false; + while (iterator.hasNext()) { + T element; + try { + element = iterator.next(); + } catch (ServiceConfigurationError e) { + // next() has already advanced past the bad element, so it is safe to continue to the next one. + log.warn(() -> "Skipping a warm-up task that could not be loaded.", e); + continue; + } + + discoveredAny = true; + T discovered = element; + try { + action.accept(discovered); + } catch (RuntimeException | LinkageError e) { + // LinkageError because a discovered element can fail to link (missing deps/native lib, failed static init), + // which is an Error, not an Exception. Skip it to keep warm-up best-effort; fatal Errors still propagate. + log.warn(() -> "Warm-up failed for " + discovered.getClass().getName() + " and was skipped.", e); + } + } + + if (!discoveredAny) { + log.debug(() -> "No warm-up tasks were discovered on the classpath."); + } + } +} diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/loader/ClasspathHttpWarmupInvoker.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/loader/ClasspathHttpWarmupInvoker.java new file mode 100644 index 000000000000..3cc15194e7c3 --- /dev/null +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/loader/ClasspathHttpWarmupInvoker.java @@ -0,0 +1,49 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.core.internal.http.loader; + +import java.util.Collections; +import java.util.List; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.annotations.SdkTestInternalApi; + +/** + * {@link HttpWarmupInvoker} that owns the set of {@link HttpClientWarmer}s and invokes each. + */ +@SdkInternalApi +public final class ClasspathHttpWarmupInvoker implements HttpWarmupInvoker { + + private final List warmers; + + @SdkTestInternalApi + ClasspathHttpWarmupInvoker(List warmers) { + this.warmers = warmers; + } + + /** + * @return an invoker over the HTTP-client warmers on the classpath. + */ + public static HttpWarmupInvoker create() { + return new ClasspathHttpWarmupInvoker(Collections.singletonList(SyncHttpClientWarmer.create())); + } + + @Override + public void invokeAll() { + for (HttpClientWarmer warmer : warmers) { + warmer.warmAll(); + } + } +} diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/loader/HttpClientWarmer.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/loader/HttpClientWarmer.java new file mode 100644 index 000000000000..e63b7d62dec2 --- /dev/null +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/loader/HttpClientWarmer.java @@ -0,0 +1,30 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.core.internal.http.loader; + +import software.amazon.awssdk.annotations.SdkInternalApi; + +/** + * Warms the sync or async HTTP clients on the classpath for CRaC priming. + */ +@SdkInternalApi +public interface HttpClientWarmer { + + /** + * Warms every HTTP client found on the classpath. Best-effort; never throws. + */ + void warmAll(); +} diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/loader/HttpWarmupInvoker.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/loader/HttpWarmupInvoker.java new file mode 100644 index 000000000000..c7fc72c19fd1 --- /dev/null +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/loader/HttpWarmupInvoker.java @@ -0,0 +1,30 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.core.internal.http.loader; + +import software.amazon.awssdk.annotations.SdkInternalApi; + +/** + * Warms every HTTP client on the classpath for CRaC priming by invoking each {@link HttpClientWarmer}. + */ +@SdkInternalApi +public interface HttpWarmupInvoker { + + /** + * Invokes {@link HttpClientWarmer#warmAll()} on every warmer. Best-effort; never throws. + */ + void invokeAll(); +} diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/loader/SyncHttpClientWarmer.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/loader/SyncHttpClientWarmer.java new file mode 100644 index 000000000000..9cac7044cb8b --- /dev/null +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/loader/SyncHttpClientWarmer.java @@ -0,0 +1,116 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.core.internal.http.loader; + +import java.net.URI; +import java.util.Collections; +import java.util.function.Supplier; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.annotations.SdkTestInternalApi; +import software.amazon.awssdk.core.internal.crac.RegionEndpointResolver; +import software.amazon.awssdk.core.internal.crac.WarmUpDiscovery; +import software.amazon.awssdk.http.ExecutableHttpRequest; +import software.amazon.awssdk.http.HttpExecuteRequest; +import software.amazon.awssdk.http.HttpExecuteResponse; +import software.amazon.awssdk.http.SdkHttpClient; +import software.amazon.awssdk.http.SdkHttpMethod; +import software.amazon.awssdk.http.SdkHttpRequest; +import software.amazon.awssdk.http.SdkHttpService; +import software.amazon.awssdk.utils.AttributeMap; +import software.amazon.awssdk.utils.IoUtils; +import software.amazon.awssdk.utils.Logger; + +/** + * Warms every sync {@link SdkHttpService} on the classpath for CRaC priming: builds each client and sends a best-effort + * {@code GET} to the resolved STS endpoint, draining the response body, so the HTTP/DNS/TLS/cert-chain code is JIT-compiled + * into the snapshot. + */ +@SdkInternalApi +public final class SyncHttpClientWarmer implements HttpClientWarmer { + + private static final Logger log = Logger.loggerFor(SyncHttpClientWarmer.class); + + private final Iterable services; + private final Supplier endpointProvider; + + @SdkTestInternalApi + SyncHttpClientWarmer(Iterable services, Supplier endpointProvider) { + this.services = services; + this.endpointProvider = endpointProvider; + } + + /** + * Warms a single {@code service} against {@code endpointProvider}. + */ + @SdkTestInternalApi + public static SyncHttpClientWarmer forService(Supplier endpointProvider, SdkHttpService service) { + return new SyncHttpClientWarmer(Collections.singletonList(service), endpointProvider); + } + + public static SyncHttpClientWarmer create() { + return new SyncHttpClientWarmer(discoverServices(), () -> RegionEndpointResolver.create().endpoint()); + } + + /** + * Like {@link #create()}, but warms against {@code endpointProvider}. + */ + @SdkTestInternalApi + public static SyncHttpClientWarmer create(Supplier endpointProvider) { + return new SyncHttpClientWarmer(discoverServices(), endpointProvider); + } + + private static Iterable discoverServices() { + return () -> SdkServiceLoader.INSTANCE.loadServices(SdkHttpService.class); + } + + @Override + public void warmAll() { + URI endpoint = endpointProvider.get(); + WarmUpDiscovery.forEachDiscovered(services.iterator(), service -> { + SdkHttpClient client = service.createHttpClientBuilder().buildWithDefaults(AttributeMap.empty()); + warmClient(client, endpoint); + }); + } + + /** + * Sends the warm-up {@code GET} to {@code endpoint}, drains the response body, and closes the client. Best-effort: the + * goal is JIT compilation, not a successful request, so any failure is logged and swallowed. + */ + private void warmClient(SdkHttpClient client, URI endpoint) { + try { + SdkHttpRequest httpRequest = SdkHttpRequest.builder() + .method(SdkHttpMethod.GET) + .uri(endpoint) + .build(); + HttpExecuteRequest request = HttpExecuteRequest.builder() + .request(httpRequest) + .build(); + ExecutableHttpRequest executableRequest = client.prepareRequest(request); + HttpExecuteResponse response = executableRequest.call(); + response.responseBody().ifPresent(body -> { + try { + IoUtils.drainInputStream(body); + } finally { + IoUtils.closeQuietlyV2(body, log); + } + }); + } catch (Exception e) { + log.debug(() -> "Sync HTTP client warm-up call failed (ignored).", e); + } finally { + IoUtils.closeQuietlyV2(client, log); + } + } +} diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/crac/SdkWarmUpTest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/crac/SdkWarmUpTest.java index 76d0dfbb2c3e..e21cdf3dd657 100644 --- a/core/sdk-core/src/test/java/software/amazon/awssdk/core/crac/SdkWarmUpTest.java +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/crac/SdkWarmUpTest.java @@ -20,6 +20,8 @@ import java.util.ArrayList; import java.util.List; import java.util.concurrent.CountDownLatch; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; /** @@ -29,6 +31,24 @@ */ class SdkWarmUpTest { + private String savedRegionProperty; + + @BeforeEach + void setup() { + // Dummy region so prime()'s HTTP warm-up resolves a non-existent STS host and fails DNS immediately, keeping the test offline. + savedRegionProperty = System.getProperty("aws.region"); + System.setProperty("aws.region", "warmup-unit-test"); + } + + @AfterEach + void teardown() { + if (savedRegionProperty != null) { + System.setProperty("aws.region", savedRegionProperty); + } else { + System.clearProperty("aws.region"); + } + } + @Test void prime_concurrentCalls_invokeRegisteredProviderExactlyOnce() throws InterruptedException { RegisteredWarmUpProvider.INVOCATIONS.set(0); diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/crac/ClasspathWarmUpInvokerTest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/crac/ClasspathWarmUpInvokerTest.java index c7d05fb7740c..284aa28cd582 100644 --- a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/crac/ClasspathWarmUpInvokerTest.java +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/crac/ClasspathWarmUpInvokerTest.java @@ -101,7 +101,7 @@ void invokeAll_whenProviderThrows_logsAtWarn() { assertThat(logCaptor.loggedEvents()) .filteredOn(loggedFromInvoker()) .anyMatch(event -> event.getLevel() == Level.WARN - && event.getMessage().getFormattedMessage().contains("failed during warmUp()")); + && event.getMessage().getFormattedMessage().contains("Warm-up failed for")); } } @@ -127,12 +127,12 @@ void invokeAll_whenNoProviders_logsAtDebug() { assertThat(logCaptor.loggedEvents()) .filteredOn(loggedFromInvoker()) .anyMatch(event -> event.getLevel() == Level.DEBUG - && event.getMessage().getFormattedMessage().contains("No SdkWarmUpProvider")); + && event.getMessage().getFormattedMessage().contains("No warm-up tasks")); } } private static Predicate loggedFromInvoker() { - return event -> ClasspathWarmUpInvoker.class.getName().equals(event.getLoggerName()); + return event -> WarmUpDiscovery.class.getName().equals(event.getLoggerName()); } private WarmUpInvoker invokerLoading(SdkWarmUpProvider... providers) { diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/crac/RegionEndpointResolverTest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/crac/RegionEndpointResolverTest.java new file mode 100644 index 000000000000..625093474a9c --- /dev/null +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/crac/RegionEndpointResolverTest.java @@ -0,0 +1,134 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.core.internal.crac; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.params.provider.Arguments.arguments; + +import java.net.URI; +import java.util.stream.Stream; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import software.amazon.awssdk.testutils.EnvironmentVariableHelper; + +class RegionEndpointResolverTest { + + private static final String REGION_PROPERTY = "aws.region"; + private static final String AWS_REGION_ENV = "AWS_REGION"; + private static final String AWS_DEFAULT_REGION_ENV = "AWS_DEFAULT_REGION"; + + private static final EnvironmentVariableHelper ENV = new EnvironmentVariableHelper(); + + @BeforeEach + void clearSettings() { + ENV.reset(); + System.clearProperty(REGION_PROPERTY); + } + + @AfterEach + void restoreSettings() { + ENV.reset(); + System.clearProperty(REGION_PROPERTY); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("regionResolutionCases") + void endpoint_whenRegionConfigured_resolvesExpectedEndpoint(String description, String sysprop, String awsRegion, + String expectedRegion) { + applyRegionSettings(sysprop, awsRegion); + + assertThat(RegionEndpointResolver.create().endpoint()) + .isEqualTo(URI.create("https://sts." + expectedRegion + ".amazonaws.com/")); + } + + private static Stream regionResolutionCases() { + return Stream.of( + // description aws.region AWS_REGION expected + arguments("nothing set -> default region", null, null, "us-east-1"), + arguments("system property wins over AWS_REGION", "eu-west-1", "ap-south-1", "eu-west-1"), + arguments("AWS_REGION used when no system property", null, "ap-south-1", "ap-south-1"), + arguments("blank system property falls through to AWS_REGION", " ", "ap-south-1", "ap-south-1"), + arguments("blank AWS_REGION falls through to default", null, " ", "us-east-1"), + arguments("chosen value is trimmed", " eu-central-1 ", null, "eu-central-1") + ); + } + + @Test + void endpoint_whenOnlyAwsDefaultRegionSet_ignoresItAndUsesDefault() { + ENV.set(AWS_DEFAULT_REGION_ENV, "us-west-2"); + + assertThat(RegionEndpointResolver.create().endpoint()) + .isEqualTo(URI.create("https://sts.us-east-1.amazonaws.com/")); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("untrustedRegionCases") + void endpoint_whenRegionNotHostnameCompliant_fallsBackToDefault(String description, String region) { + System.setProperty(REGION_PROPERTY, region); + + URI endpoint = RegionEndpointResolver.create().endpoint(); + + // A non-compliant region cannot reach the host; it is rejected and the default region is used instead. + assertThat(endpoint).isEqualTo(URI.create("https://sts.us-east-1.amazonaws.com/")); + } + + private static Stream untrustedRegionCases() { + return Stream.of( + arguments("newline is rejected", "us-east-1\nfoo"), + arguments("carriage return is rejected", "us-east-1\rfoo"), + arguments("forward slash is rejected", "us-east-1/foo"), + arguments("space is rejected", "us-east-1 foo"), + arguments("at sign is rejected", "evil@host"), + arguments("hash is rejected", "host#fragment"), + arguments("dot is rejected", "sts.evil.com") + ); + } + + @Test + void endpoint_whenImdsUnreachable_resolvesWithoutCallingImds() { + // Point IMDS at a non-routable address. A resolver that (incorrectly) called IMDS would block here; + // the correct resolver ignores IMDS entirely and returns the default region immediately. + String savedImdsEndpoint = System.getProperty("aws.ec2MetadataServiceEndpoint"); + System.setProperty("aws.ec2MetadataServiceEndpoint", "http://10.255.255.1"); + try { + long startNanos = System.nanoTime(); + URI endpoint = RegionEndpointResolver.create().endpoint(); + long elapsedMillis = (System.nanoTime() - startNanos) / 1_000_000; + + assertThat(endpoint).isEqualTo(URI.create("https://sts.us-east-1.amazonaws.com/")); + assertThat(elapsedMillis).isLessThan(1_000L); + } finally { + if (savedImdsEndpoint != null) { + System.setProperty("aws.ec2MetadataServiceEndpoint", savedImdsEndpoint); + } else { + System.clearProperty("aws.ec2MetadataServiceEndpoint"); + } + } + } + + private static void applyRegionSettings(String sysprop, String awsRegion) { + if (sysprop != null) { + System.setProperty(REGION_PROPERTY, sysprop); + } + if (awsRegion != null) { + ENV.set(AWS_REGION_ENV, awsRegion); + } + } +} diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/loader/ClasspathHttpWarmupInvokerTest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/loader/ClasspathHttpWarmupInvokerTest.java new file mode 100644 index 000000000000..7fd1240b8b73 --- /dev/null +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/loader/ClasspathHttpWarmupInvokerTest.java @@ -0,0 +1,57 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.core.internal.http.loader; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; + +import java.util.Arrays; +import java.util.Collections; +import java.util.concurrent.atomic.AtomicInteger; +import org.junit.jupiter.api.Test; + +class ClasspathHttpWarmupInvokerTest { + + @Test + void invokeAll_whenMultipleWarmers_invokesEachOnce() { + CountingWarmer first = new CountingWarmer(); + CountingWarmer second = new CountingWarmer(); + + new ClasspathHttpWarmupInvoker(Arrays.asList(first, second)).invokeAll(); + + assertThat(first.invocations()).isEqualTo(1); + assertThat(second.invocations()).isEqualTo(1); + } + + @Test + void invokeAll_whenNoWarmers_isNoOp() { + assertThatCode(() -> new ClasspathHttpWarmupInvoker(Collections.emptyList()).invokeAll()) + .doesNotThrowAnyException(); + } + + private static final class CountingWarmer implements HttpClientWarmer { + private final AtomicInteger invocations = new AtomicInteger(); + + @Override + public void warmAll() { + invocations.incrementAndGet(); + } + + int invocations() { + return invocations.get(); + } + } +} diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/loader/SyncHttpClientWarmerTest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/loader/SyncHttpClientWarmerTest.java new file mode 100644 index 000000000000..438d5beb9760 --- /dev/null +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/loader/SyncHttpClientWarmerTest.java @@ -0,0 +1,159 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.core.internal.http.loader; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.util.Arrays; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import software.amazon.awssdk.http.AbortableInputStream; +import software.amazon.awssdk.http.ExecutableHttpRequest; +import software.amazon.awssdk.http.HttpExecuteRequest; +import software.amazon.awssdk.http.HttpExecuteResponse; +import software.amazon.awssdk.http.SdkHttpClient; +import software.amazon.awssdk.http.SdkHttpMethod; +import software.amazon.awssdk.http.SdkHttpResponse; +import software.amazon.awssdk.http.SdkHttpService; + +/** + * Unit tests for {@link SyncHttpClientWarmer}. Every test drives the real {@link SyncHttpClientWarmer#warmAll()} with an + * injected list of stub {@link SdkHttpService}s and a fixed endpoint. + */ +class SyncHttpClientWarmerTest { + + private static final URI ENDPOINT = URI.create("https://sts.us-east-1.amazonaws.com/"); + + @Test + void warmAll_whenResponseHasBody_drainsAndClosesIt() throws IOException { + InputStream body = spy(new ByteArrayInputStream("denied".getBytes())); + SdkHttpClient client = stubClient(respondingWith(403, body)); + + warmer(serviceFor(client)).warmAll(); + + verify(body, atLeastOnce()).read(); // drained to EOF + verify(body).close(); + verify(client).close(); + } + + @Test + void warmAll_whenInvoked_issuesGetToResolvedEndpoint() { + SdkHttpClient client = stubClient(respondingWith(403, emptyBody())); + ArgumentCaptor request = ArgumentCaptor.forClass(HttpExecuteRequest.class); + + warmer(serviceFor(client)).warmAll(); + + verify(client).prepareRequest(request.capture()); + assertThat(request.getValue().httpRequest().method()).isEqualTo(SdkHttpMethod.GET); + assertThat(request.getValue().httpRequest().getUri()).isEqualTo(ENDPOINT); + } + + @Test + void warmAll_whenRequestFails_swallowsAndStillClosesClient() throws IOException { + SdkHttpClient client = mock(SdkHttpClient.class); + ExecutableHttpRequest request = mock(ExecutableHttpRequest.class); + when(client.prepareRequest(any(HttpExecuteRequest.class))).thenReturn(request); + when(request.call()).thenThrow(new IOException("offline")); + + assertThatCode(() -> warmer(serviceFor(client)).warmAll()).doesNotThrowAnyException(); + verify(client).close(); + } + + @Test + void warmAll_whenNoResponseBody_stillClosesClient() { + SdkHttpClient client = stubClient(respondingWith(403, null)); + + assertThatCode(() -> warmer(serviceFor(client)).warmAll()).doesNotThrowAnyException(); + verify(client).close(); + } + + @Test + void warmAll_whenMultipleServicesDiscovered_warmsEach() { + SdkHttpClient first = stubClient(respondingWith(403, emptyBody())); + SdkHttpClient second = stubClient(respondingWith(403, emptyBody())); + + warmer(serviceFor(first), serviceFor(second)).warmAll(); + + verify(first).prepareRequest(any(HttpExecuteRequest.class)); + verify(second).prepareRequest(any(HttpExecuteRequest.class)); + } + + @Test + void warmAll_whenOneServiceFailsToBuild_stillWarmsOthers() { + SdkHttpService failing = mock(SdkHttpService.class); + when(failing.createHttpClientBuilder()).thenThrow(new RuntimeException("bad service")); + SdkHttpClient healthy = stubClient(respondingWith(403, emptyBody())); + + warmer(failing, serviceFor(healthy)).warmAll(); + + verify(healthy).prepareRequest(any(HttpExecuteRequest.class)); + } + + @Test + void warmAll_whenNoServices_isNoOp() { + assertThatCode(() -> warmer().warmAll()).doesNotThrowAnyException(); + } + + private static SyncHttpClientWarmer warmer(SdkHttpService... services) { + return new SyncHttpClientWarmer(Arrays.asList(services), () -> ENDPOINT); + } + + /** A service whose builder yields the given client. */ + private static SdkHttpService serviceFor(SdkHttpClient client) { + SdkHttpClient.Builder builder = mock(SdkHttpClient.Builder.class); + when(builder.buildWithDefaults(any())).thenReturn(client); + + SdkHttpService service = mock(SdkHttpService.class); + when(service.createHttpClientBuilder()).thenReturn(builder); + return service; + } + + /** A client whose single request returns the given response. */ + private static SdkHttpClient stubClient(HttpExecuteResponse response) { + try { + ExecutableHttpRequest request = mock(ExecutableHttpRequest.class); + when(request.call()).thenReturn(response); + + SdkHttpClient client = mock(SdkHttpClient.class); + when(client.prepareRequest(any(HttpExecuteRequest.class))).thenReturn(request); + return client; + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private static HttpExecuteResponse respondingWith(int statusCode, InputStream body) { + return HttpExecuteResponse.builder() + .response(SdkHttpResponse.builder().statusCode(statusCode).build()) + .responseBody(body == null ? null : AbortableInputStream.create(body)) + .build(); + } + + private static InputStream emptyBody() { + return new ByteArrayInputStream(new byte[0]); + } +} diff --git a/pom.xml b/pom.xml index f93fdc49fe26..69b309991d10 100644 --- a/pom.xml +++ b/pom.xml @@ -74,6 +74,7 @@ third-party v2-migration test/http-client-tests + test/warmup-tests test/protocol-tests test/protocol-tests-core test/service-test-utils diff --git a/test/architecture-tests/src/test/java/software/amazon/awssdk/archtests/CodingConventionWithSuppressionTest.java b/test/architecture-tests/src/test/java/software/amazon/awssdk/archtests/CodingConventionWithSuppressionTest.java index c942eaf9c8b8..edfff3c08bf5 100644 --- a/test/architecture-tests/src/test/java/software/amazon/awssdk/archtests/CodingConventionWithSuppressionTest.java +++ b/test/architecture-tests/src/test/java/software/amazon/awssdk/archtests/CodingConventionWithSuppressionTest.java @@ -61,7 +61,7 @@ public class CodingConventionWithSuppressionTest { ArchUtils.classNameToPattern(KnownContentLengthAsyncRequestBodySubscriber.class), ArchUtils.classNameToPattern(UnknownContentLengthAsyncRequestBodySubscriber.class), ArchUtils.classNameToPattern(CopyObjectHelper.class), - ArchUtils.classNameToPattern("software.amazon.awssdk.core.internal.crac.ClasspathWarmUpInvoker"))); + ArchUtils.classNameToPattern("software.amazon.awssdk.core.internal.crac.WarmUpDiscovery"))); private static final Set ALLOWED_ERROR_LOG_SUPPRESSION = new HashSet<>( Arrays.asList( diff --git a/test/warmup-tests/pom.xml b/test/warmup-tests/pom.xml new file mode 100644 index 000000000000..1f898dd20ae4 --- /dev/null +++ b/test/warmup-tests/pom.xml @@ -0,0 +1,113 @@ + + + + + 4.0.0 + + aws-sdk-java-pom + software.amazon.awssdk + 2.46.18-SNAPSHOT + ../../pom.xml + + warmup-tests + AWS Java SDK :: Test :: Warm-up Tests + Centralized tests for the CRaC HTTP-client warm-up, run against every sync HTTP client so each client does not + need its own warm-up test. This is a leaf test module: nothing depends on it, so depending on the HTTP clients here + does not create a dependency cycle. + https://aws.amazon.com/sdkforjava + + + + + software.amazon.awssdk + bom-internal + ${project.version} + pom + import + + + + + + + software.amazon.awssdk + sdk-core + ${awsjavasdk.version} + test + + + software.amazon.awssdk + apache-client + ${awsjavasdk.version} + test + + + software.amazon.awssdk + apache5-client + ${awsjavasdk.version} + test + + + software.amazon.awssdk + aws-crt-client + ${awsjavasdk.version} + test + + + software.amazon.awssdk + url-connection-client + ${awsjavasdk.version} + test + + + org.junit.jupiter + junit-jupiter + test + + + org.assertj + assertj-core + test + + + com.github.tomakehurst + wiremock-jre8-standalone + test + + + + + + + + com.github.spotbugs + spotbugs-maven-plugin + + true + + + + org.apache.maven.plugins + maven-dependency-plugin + + true + + + + + diff --git a/test/warmup-tests/src/test/java/software/amazon/awssdk/http/warmup/SyncHttpClientWarmUpTest.java b/test/warmup-tests/src/test/java/software/amazon/awssdk/http/warmup/SyncHttpClientWarmUpTest.java new file mode 100644 index 000000000000..2f62733bad83 --- /dev/null +++ b/test/warmup-tests/src/test/java/software/amazon/awssdk/http/warmup/SyncHttpClientWarmUpTest.java @@ -0,0 +1,113 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.http.warmup; + +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.any; +import static com.github.tomakehurst.wiremock.client.WireMock.anyRequestedFor; +import static com.github.tomakehurst.wiremock.client.WireMock.anyUrl; +import static com.github.tomakehurst.wiremock.client.WireMock.getRequestedFor; +import static com.github.tomakehurst.wiremock.client.WireMock.urlPathEqualTo; +import static org.assertj.core.api.Assertions.assertThat; + +import com.github.tomakehurst.wiremock.WireMockServer; +import com.github.tomakehurst.wiremock.core.WireMockConfiguration; +import java.net.URI; +import java.util.ServiceLoader; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; +import software.amazon.awssdk.core.internal.http.loader.SyncHttpClientWarmer; +import software.amazon.awssdk.http.SdkHttpService; +import software.amazon.awssdk.http.apache.ApacheSdkHttpService; +import software.amazon.awssdk.http.apache5.Apache5SdkHttpService; +import software.amazon.awssdk.http.crt.AwsCrtSdkHttpService; +import software.amazon.awssdk.http.urlconnection.UrlConnectionSdkHttpService; + +/** + * Verifies the CRaC sync warm-up sends its GET through each real sync HTTP client. Every client is exercised here, in one + * place, instead of a separate test in each client module. The stub mirrors a real STS {@code GET} (302 redirect, empty + * body); the warm-up must not follow the redirect, so exactly one request is expected. + */ +class SyncHttpClientWarmUpTest { + + // The sync HTTP clients on this module's classpath: apache, apache5, aws-crt, url-connection. Hardcoded so a broken + // ServiceLoader that discovers nothing fails the test instead of passing a trivial verify(0). + private static final int SYNC_CLIENT_COUNT = 4; + + private WireMockServer mockServer; + + static SdkHttpService[] syncClients() { + return new SdkHttpService[] { + new Apache5SdkHttpService(), + new ApacheSdkHttpService(), + new AwsCrtSdkHttpService(), + new UrlConnectionSdkHttpService() + }; + } + + @BeforeEach + void setUp() { + mockServer = new WireMockServer(WireMockConfiguration.options().dynamicPort()); + mockServer.start(); + } + + @AfterEach + void tearDown() { + mockServer.stop(); + } + + @ParameterizedTest + @MethodSource("syncClients") + void warmAll_sendsWarmUpGetThroughClient(SdkHttpService service) { + mockServer.stubFor(any(anyUrl()).willReturn(aResponse() + .withStatus(302) + .withHeader("Location", "https://aws.amazon.com/iam"))); + + URI endpoint = URI.create("http://localhost:" + mockServer.port() + "/"); + SyncHttpClientWarmer.forService(() -> endpoint, service).warmAll(); + + mockServer.verify(1, getRequestedFor(urlPathEqualTo("/"))); + mockServer.verify(1, anyRequestedFor(anyUrl())); + } + + /** + * Exercises the real classpath-discovery path used by {@code prime()}: {@code warmAll()} discovers every sync + * {@link SdkHttpService} on the classpath via {@link ServiceLoader} and warms each. Confirms discovery finds all + * {@value #SYNC_CLIENT_COUNT} clients and that each receives exactly one warm-up GET. + */ + @Test + void warmAll_whenDiscoveringFromClasspath_warmsEverySyncClient() { + mockServer.stubFor(any(anyUrl()).willReturn(aResponse() + .withStatus(302) + .withHeader("Location", "https://aws.amazon.com/iam"))); + + int discoveredClients = 0; + for (SdkHttpService ignored : ServiceLoader.load(SdkHttpService.class)) { + discoveredClients++; + } + // Guard against a broken ServiceLoader: if discovery silently finds 0, verify(0) below would pass trivially. + assertThat(discoveredClients).isEqualTo(SYNC_CLIENT_COUNT); + + URI endpoint = URI.create("http://localhost:" + mockServer.port() + "/"); + SyncHttpClientWarmer.create(() -> endpoint).warmAll(); + + mockServer.verify(SYNC_CLIENT_COUNT, getRequestedFor(urlPathEqualTo("/"))); + mockServer.verify(SYNC_CLIENT_COUNT, anyRequestedFor(anyUrl())); + } +}