From 6f303a3c13344b2e43fe634017483db648fd9568 Mon Sep 17 00:00:00 2001 From: Vladimir Aseev Date: Tue, 7 Jul 2026 14:53:35 +0200 Subject: [PATCH 01/26] add realtime text to speech client, WIP --- orchestration/pom.xml | 5 + .../orchestration/OrchestrationClient.java | 165 ++++++++++++++- .../orchestration/RealtimeClientFactory.java | 200 ++++++++++++++++++ .../TextToSpeechRealtimeClient.java | 53 +++++ .../WSSOpenAIRealtimeClient.java | 157 ++++++++++++++ .../orchestration/WebsocketListenerGlue.java | 31 +++ .../java/com/sap/ai/sdk/app/Application.java | 25 ++- .../app/services/OrchestrationService.java | 9 + 8 files changed, 640 insertions(+), 5 deletions(-) create mode 100644 orchestration/src/main/java/com/sap/ai/sdk/orchestration/RealtimeClientFactory.java create mode 100644 orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextToSpeechRealtimeClient.java create mode 100644 orchestration/src/main/java/com/sap/ai/sdk/orchestration/WSSOpenAIRealtimeClient.java create mode 100644 orchestration/src/main/java/com/sap/ai/sdk/orchestration/WebsocketListenerGlue.java diff --git a/orchestration/pom.xml b/orchestration/pom.xml index f8027fe4a..a1ef764f1 100644 --- a/orchestration/pom.xml +++ b/orchestration/pom.xml @@ -122,6 +122,11 @@ com.fasterxml.jackson.dataformat jackson-dataformat-yaml + + com.openai + openai-java + 4.41.0 + org.projectlombok diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationClient.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationClient.java index 73c7161e0..4e901e966 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationClient.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationClient.java @@ -7,6 +7,10 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.annotations.Beta; +import com.openai.client.OpenAIClient; +import com.openai.client.okhttp.OpenAIOkHttpClient; +import com.openai.models.beta.realtime.sessions.Session; +import com.openai.models.realtime.SessionUpdateEvent; import com.sap.ai.sdk.core.AiCoreService; import com.sap.ai.sdk.orchestration.model.CompletionPostRequest; import com.sap.ai.sdk.orchestration.model.CompletionPostResponse; @@ -17,16 +21,27 @@ import com.sap.ai.sdk.orchestration.model.OrchestrationConfig; import com.sap.cloud.sdk.cloudplatform.connectivity.Header; import com.sap.cloud.sdk.cloudplatform.connectivity.HttpDestination; +import io.vavr.collection.Array; import io.vavr.control.Try; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; + +import java.io.ByteArrayOutputStream; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.WebSocket; +import java.nio.ByteBuffer; +import java.util.*; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.function.Consumer; import java.util.function.Supplier; import java.util.stream.Stream; +import java.util.stream.StreamSupport; import javax.annotation.Nonnull; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import lombok.val; +import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; +import org.apache.hc.client5.http.impl.classic.HttpClients; /** Client to execute requests to the orchestration service. */ @Slf4j @@ -34,9 +49,12 @@ public class OrchestrationClient { private static final String DEFAULT_SCENARIO = "orchestration"; private static final String COMPLETION_ENDPOINT = "/v2/completion"; + private static final String REALTIME_API_PATH = + "wss://realtime.ai.intprod-eu12.eu-central-1.aws.ml.hana.ondemand.com/v2/inference/deployments/ddb5a959fdb51ffe/v1/realtime"; static final ObjectMapper JACKSON = getOrchestrationObjectMapper(); + private final OrchestrationHttpExecutor executor; private final List
customHeaders = new ArrayList<>(); @@ -103,6 +121,147 @@ public OrchestrationChatResponse chatCompletion( return new OrchestrationChatResponse(response); } + /** + * Voices input text into output audio + * + * @param input text to voice + * @return byte chunks output stream in PCM16 format, 16-bit, mono, 24000 Hz, little-endian + */ + public Stream speech(String input) { + + var deploymentID = ""; + final List result = new ArrayList<>(); + try { + HttpClient client = HttpClient.newHttpClient(); // TODO: fix local setup and use autocloseable + WebSocket ws = + client + .newWebSocketBuilder() + .header( + "Authorization", + "Bearer .......") + .header("AI-Resource-Group", "default") + .buildAsync( + URI.create(REALTIME_API_PATH + deploymentID), + new WebSocket.Listener() { + + @Override + public void onOpen(WebSocket webSocket) { + log.warn("Websocket connection opened"); + String sessionUpdateJson = + """ + { + "type": "session.update", + "session": { + "type": "realtime", + "output_modalities": ["audio"], + "instructions": "you are a speaker and your role is to read (produce audio) of the user input speech. voice user text input", + "audio": { + "input": { "format": { "type": "audio/pcm", "rate": 24000 } }, + "output": { "format": { "type": "audio/pcm", "rate": 24000 }, "voice": "alloy" } + } + } + } + """; + + // String sessionUpdateJson = "{\"type\": \"session.update\", \"session\": + // {\"modalities\": [\"audio\", \"text\"]}}"; + webSocket.sendText(sessionUpdateJson, true); + webSocket.request(1); + + String textMessage = + """ +{ + "type": "conversation.item.create", + "item": { + "type": "message", + "role": "system", + "content": [ + { + "type": "input_text", + "text": "you are a speaker and your role is to read (produce audio) of the user input speech. voice user text input" + } + ] + } +} +"""; + //.replaceFirst("%VALUE%", input); // TODO: sanitize or/and escape input + log.warn("Sending message to websocket: {}", textMessage); + webSocket.sendText(textMessage, true); + webSocket.request(1); + + String voiceMessage = + """ + { + "type": "conversation.item.create", + "item": { + "type": "message", + "role": "user", + "content": [ + { + "type": "input_text", + "text": "Eat more of these fresh french baguettes" + } + ] + } + } + """; + + webSocket.sendText(voiceMessage, true); + webSocket.request(1); + + + + + String askForResponse = "{ \"type\": \"response.create\" }"; + webSocket.sendText(askForResponse, true); + + webSocket.request(80); + } + + @Override + public CompletionStage onText( + WebSocket webSocket, CharSequence data, boolean last) { + try { + Thread.sleep(500); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + log.warn("text received: {}", data); + final JsonNode event; + try { + event = JACKSON.readTree(data.toString()); + } catch (JsonProcessingException e) { + throw new OrchestrationClientException( + "Error parsing JSON response from speech API", e); + } + String type = event.get("type").asText(); + if ("response.output_audio.delta".equals(type)) { + String base64Audio = event.get("delta").asText(); + byte[] audio = Base64.getDecoder().decode(base64Audio); + result.add(audio); + } else if ("response.audio_transcript.delta".equals(type)) { + log.warn("RECEIVED TRANSCRIPT: {}", event.get("delta").asText()); + } else if ("response.output_audio.done".equals(type)) { + // TODO: we should not wait last ack, we should return stream immediately, + // this requires + // TODO: thread management, factory and thread safety of all structs + webSocket.sendClose(1000, "done"); + return null; + } else { + log.warn("Unhandled event type: {}", data); + } + return CompletableFuture.completedStage(null); + } + }) + .join(); + Thread.sleep(25000); + return result.stream(); + } catch (Exception e) { + //log.error(e.getMessage(), e); + throw new OrchestrationClientException("Failed to call realtime API", e); + } + } + /** * Generate a completion for the given prompt. * diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/RealtimeClientFactory.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/RealtimeClientFactory.java new file mode 100644 index 000000000..d48dd7b9f --- /dev/null +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/RealtimeClientFactory.java @@ -0,0 +1,200 @@ +package com.sap.ai.sdk.orchestration; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.openai.models.responses.Response; +import com.sap.cloud.sdk.cloudplatform.connectivity.Header; +import com.sap.cloud.sdk.cloudplatform.connectivity.HttpDestination; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import javax.annotation.Nonnull; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.WebSocket; +import java.util.*; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.Flow; +import java.util.function.Consumer; +import java.util.function.Supplier; +import java.util.stream.Stream; + +@Slf4j +@RequiredArgsConstructor(access = lombok.AccessLevel.PRIVATE) +public class RealtimeClientFactory { + private static final String AI_RESOURCE_GROUP_HEADER_NAME = "AI-Resource-Group"; + private static final String AI_RESOURCE_GROUP_HEADER_VALUE = "default"; + private static final String REALTIME_API_PATH_SUFFIX = "/v2/inference/deployments/%s/v1/realtime"; + private static final int PATH_BUFFER_SIZE = 400; // existing URLs are ~120 symbols long, 400 has reasonable margin + + + private static final String REALTIME_API_PATH = + "wss://realtime.ai.intprod-eu12.eu-central-1.aws.ml.hana.ondemand.com/v2/inference/deployments/ddb5a959fdb51ffe/v1/realtime"; + + private final Supplier destinationSupplier; + private final Supplier deploymentIdSupplier; + + /** + * Voices input text into output audio + * + * @param input text to voice + * @return byte chunks output stream in PCM16 format, 16-bit, mono, 24000 Hz, little-endian + */ + public Stream textToSpeech(@Nonnull final Consumer audioOutputConsumer) { + var destination = destinationSupplier.get(); + + var extraHeaders = destination.getHeaders(); + var headers = new HashMap(extraHeaders.size() + 1); + headers.put(AI_RESOURCE_GROUP_HEADER_NAME, AI_RESOURCE_GROUP_HEADER_VALUE); + for (Header header : extraHeaders) { + headers.put(header.getName(), header.getValue()); + } + + var deploymentId = deploymentIdSupplier.get(); + var sb = new StringBuilder(PATH_BUFFER_SIZE); + sb.append("wss://"); + sb.append(destination.getUri().getHost()); + // set port explicitly if defined in the uri + if (destination.getUri().getPort() != -1) { + sb.append(":").append(destination.getUri().getPort()); + } + sb.append(String.format(REALTIME_API_PATH_SUFFIX, deploymentId)); + var endpoint = sb.toString(); + + + final List result = new ArrayList<>(); + try { + HttpClient client = HttpClient.newHttpClient(); // TODO: fix local setup and use autocloseable + WebSocket ws = + client + .newWebSocketBuilder() + .header( + "Authorization", + "Bearer ....") + .header("AI-Resource-Group", "default") + .buildAsync( + URI.create(endpoint), + new WebSocket.Listener() { + + @Override + public void onOpen(WebSocket webSocket) { + log.warn("Websocket connection opened"); + String sessionUpdateJson = + """ + { + "type": "session.update", + "session": { + "type": "realtime", + "output_modalities": ["audio"], + "instructions": "you are a speaker and your role is to read (produce audio) of the user input speech. voice user text input", + "audio": { + "input": { "format": { "type": "audio/pcm", "rate": 24000 } }, + "output": { "format": { "type": "audio/pcm", "rate": 24000 }, "voice": "alloy" } + } + } + } + """; + + // String sessionUpdateJson = "{\"type\": \"session.update\", \"session\": + // {\"modalities\": [\"audio\", \"text\"]}}"; + webSocket.sendText(sessionUpdateJson, true); + webSocket.request(1); + + String textMessage = + """ + { + "type": "conversation.item.create", + "item": { + "type": "message", + "role": "system", + "content": [ + { + "type": "input_text", + "text": "you are a speaker and your role is to read (produce audio) of the user input speech. voice user text input" + } + ] + } + } + """; + //.replaceFirst("%VALUE%", input); // TODO: sanitize or/and escape input + log.warn("Sending message to websocket: {}", textMessage); + webSocket.sendText(textMessage, true); + webSocket.request(1); + + String voiceMessage = + """ + { + "type": "conversation.item.create", + "item": { + "type": "message", + "role": "user", + "content": [ + { + "type": "input_text", + "text": "Eat more of these fresh french baguettes" + } + ] + } + } + """; + + webSocket.sendText(voiceMessage, true); + webSocket.request(1); + + + + + String askForResponse = "{ \"type\": \"response.create\" }"; + webSocket.sendText(askForResponse, true); + + webSocket.request(80); + } + + @Override + public CompletionStage onText( + WebSocket webSocket, CharSequence data, boolean last) { + try { + Thread.sleep(500); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + log.warn("text received: {}", data); + final JsonNode event; + try { + event = JACKSON.readTree(data.toString()); + } catch (JsonProcessingException e) { + throw new OrchestrationClientException( + "Error parsing JSON response from speech API", e); + } + Response r = null; + + String type = event.get("type").asText(); + if ("response.output_audio.delta".equals(type)) { + String base64Audio = event.get("delta").asText(); + byte[] audio = Base64.getDecoder().decode(base64Audio); + result.add(audio); + } else if ("response.audio_transcript.delta".equals(type)) { + log.warn("RECEIVED TRANSCRIPT: {}", event.get("delta").asText()); + } else if ("response.output_audio.done".equals(type)) { + // TODO: we should not wait last ack, we should return stream immediately, + // this requires + // TODO: thread management, factory and thread safety of all structs + webSocket.sendClose(1000, "done"); + return null; + } else { + log.warn("Unhandled event type: {}", data); + } + return CompletableFuture.completedStage(null); + } + }) + .join(); + Thread.sleep(25000); + return result.stream(); + } catch (Exception e) { + //log.error(e.getMessage(), e); + throw new OrchestrationClientException("Failed to call realtime API", e); + } + } + +} diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextToSpeechRealtimeClient.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextToSpeechRealtimeClient.java new file mode 100644 index 000000000..7df1881da --- /dev/null +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextToSpeechRealtimeClient.java @@ -0,0 +1,53 @@ +package com.sap.ai.sdk.orchestration; + +import com.fasterxml.jackson.databind.JsonNode; +import com.openai.models.realtime.RealtimeSessionCreateRequest; +import io.vavr.collection.Array; + +import java.util.*; +import java.util.function.BiConsumer; + +public class TextToSpeechRealtimeClient extends WSSOpenAIRealtimeClient { + + private static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; + + private static final Set HANDLED_RESPONSE_TYPES = Set.of( + "response.output_audio.delta", "response.output_audio.done" + ); + + private static final List OUTPUT_MODALITIES = List.of( + RealtimeSessionCreateRequest.OutputModality.AUDIO + ); + + private static final String TASK = + "you are a speaker and your role is to read (produce audio) of the user input speech. voice user text input"; + + + private final BiConsumer outputConsumer; + + public TextToSpeechRealtimeClient(String url, Map httpHeaders, BiConsumer outputConsumer) { + super(url, httpHeaders, HANDLED_RESPONSE_TYPES); + this.outputConsumer = outputConsumer; + } + + @Override + protected String getSystemPrompt() { + return TASK; + } + + @Override + protected List getOutputModalities() { + return OUTPUT_MODALITIES; + } + + @Override + protected void onResponse(String eventType, JsonNode event) { + if ("response.output_audio.delta".equals(eventType)) { + var base64Audio = event.get("delta").asText(); + byte[] audio = Base64.getDecoder().decode(base64Audio); + this.outputConsumer.accept(audio, Boolean.FALSE); + } else if ("response.output_audio.done".equals(eventType)) { + this.outputConsumer.accept(EMPTY_BYTE_ARRAY, Boolean.TRUE); + } + } +} diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/WSSOpenAIRealtimeClient.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/WSSOpenAIRealtimeClient.java new file mode 100644 index 000000000..588e5853b --- /dev/null +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/WSSOpenAIRealtimeClient.java @@ -0,0 +1,157 @@ +package com.sap.ai.sdk.orchestration; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.openai.models.realtime.*; +import com.openai.models.realtime.clientsecrets.ClientSecretCreateParams; +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.WebSocket; +import java.nio.ByteBuffer; +import java.util.*; +import java.util.concurrent.CompletableFuture; + +import static com.sap.ai.sdk.orchestration.OrchestrationJacksonConfiguration.getOrchestrationObjectMapper; + +@Slf4j +@AllArgsConstructor(access = AccessLevel.PACKAGE) +public abstract class WSSOpenAIRealtimeClient implements AutoCloseable { + + private static final int SUCCESS_FINISH_WSS_CODE = 1000; + private static final ObjectMapper JACKSON = getOrchestrationObjectMapper(); + + /** + * WebSocket keeps TCP connection to the server. If connection idles, depending on the provider + * in cloud environments, cloud provider sends hard TCP RST (reset) after 60-300 seconds of inactivity, + * this is normally not configurable and would leave realtime conversation unusable on a pause. + *

+ * In mobile networks this period is even shorter and typical CGNAT session can only idle for 10-15 seconds + * before it gets terminated + *

+ * In realtime client we rely on explicit lifetime management (create/close), heartbeat mechanism is required + * to prevent unintended closing of connections while conversation is still used + */ + private static final int HEARTBEAT_INTERVAL_MILLIS = 4500; + private static final String HEARTBEAT_TIMER_NAME = "wss_realtime_heartbeat"; + + private final HttpClient client; + private final CompletableFuture ws; + private final Timer heartbeatTimer; + private final Set handleMessageTypes; + + public WSSOpenAIRealtimeClient(String url, Map httpHeaders, Set handleMessageTypes) { + this.client = HttpClient.newHttpClient(); + var wsBuilder = this.client.newWebSocketBuilder(); + for (Map.Entry entry : httpHeaders.entrySet()) { + wsBuilder = wsBuilder.header(entry.getKey(), entry.getValue()); + } + this.ws = wsBuilder.buildAsync(URI.create(url), new WebsocketListenerGlue(this::onSocketOpen, this::onText)); + this.handleMessageTypes = handleMessageTypes; + this.heartbeatTimer = new Timer(HEARTBEAT_TIMER_NAME,true); + } + + public synchronized void askForResponse() { + WebSocket ws = this.ws.join(); + ws.sendText(JACKSON.valueToTree(ResponseCreateEvent.builder().build()), true); + ws.request(1); + } + + @Override + public synchronized void close() { + WebSocket ws = this.ws.join(); + heartbeatTimer.cancel(); + try { + ws.sendClose(SUCCESS_FINISH_WSS_CODE, "done").join(); + } catch (Exception e) { + log.error("Error while closing WebSocket", e); + } + this.client.close(); + } + + protected abstract String getSystemPrompt(); + + protected abstract List getOutputModalities(); + + protected abstract void onResponse(String eventType, JsonNode event); + + protected synchronized void sendMessage(RealtimeConversationItemUserMessage message) { + WebSocket ws = this.ws.join(); + ws.sendText(JACKSON.valueToTree(message), true); + ws.request(1); + } + + private synchronized void onSocketOpen(WebSocket ws) { + configureSession(ws); + configureConversation(ws); + scheduleHeartbeat(ws); + } + + private void onText(WebSocket webSocket, CharSequence data) { + final JsonNode event; + try { + event = JACKSON.readTree(data.toString()); + } catch (JsonProcessingException e) { + throw new OrchestrationClientException( + "Error parsing JSON response from speech API", e); + } + var eventType = event.get("type").asText(); + if (handleMessageTypes.contains(eventType)) { + onResponse(eventType, event); + } + + webSocket.request(1); + } + + private synchronized void sendPing(WebSocket ws) { + if (ws.isInputClosed()) { + return; + } + ws.sendPing(ByteBuffer.wrap("ping".getBytes())).join(); + ws.request(1); + } + + private void configureSession(WebSocket ws) { + SessionUpdateEvent sue = SessionUpdateEvent.builder() + .session(ClientSecretCreateParams.Session.ofRealtime( + RealtimeSessionCreateRequest.builder() + .outputModalities(getOutputModalities()) + .audio(RealtimeAudioConfig.builder() + .input(RealtimeAudioConfigInput.builder() + .format(RealtimeAudioFormats.AudioPcm.builder().build()) + .build()) + .output(RealtimeAudioConfigOutput.builder() + .format(RealtimeAudioFormats.AudioPcm.builder().build()) + .voice(RealtimeAudioConfigOutput.Voice.UnionMember1.ALLOY) + .build()) + .build()) + .build()) + .asRealtime()) + .build(); + ws.sendText(JACKSON.valueToTree(sue), true); + ws.request(1); + } + + private void configureConversation(WebSocket ws) { + var systemPrompt = getSystemPrompt(); + var message = new RealtimeConversationItemSystemMessage.Builder(). + addContent(RealtimeConversationItemSystemMessage.Content.builder().text(systemPrompt).build()); + ws.sendText(JACKSON.valueToTree(message), true); + ws.request(1); + } + + private void scheduleHeartbeat(WebSocket ws) { + TimerTask task = new TimerTask(){ + @Override + public void run() { + sendPing(ws); + } + + }; + heartbeatTimer.scheduleAtFixedRate(task, 0, HEARTBEAT_INTERVAL_MILLIS); + } +} diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/WebsocketListenerGlue.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/WebsocketListenerGlue.java new file mode 100644 index 000000000..28bb102ec --- /dev/null +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/WebsocketListenerGlue.java @@ -0,0 +1,31 @@ +package com.sap.ai.sdk.orchestration; + +import java.net.http.WebSocket; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.function.BiConsumer; +import java.util.function.Consumer; + +public class WebsocketListenerGlue implements WebSocket.Listener { + + private final Consumer onOpen; + private final BiConsumer onText; + + public WebsocketListenerGlue(Consumer onOpen, BiConsumer onText) { + this.onOpen = onOpen; + this.onText = onText; + } + + @Override + public void onOpen(WebSocket webSocket) { + this.onOpen.accept(webSocket); + webSocket.request(1); + } + + @Override + public CompletionStage onText(WebSocket webSocket, CharSequence data, boolean last) { + this.onText.accept(webSocket, data); + webSocket.request(1); + return CompletableFuture.completedStage(null); + } +} diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/Application.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/Application.java index 0f4ed8348..01269478d 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/Application.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/Application.java @@ -6,6 +6,9 @@ import com.fasterxml.jackson.annotation.PropertyAccessor; import com.fasterxml.jackson.databind.ObjectMapper; import javax.annotation.Nonnull; + +import com.sap.ai.sdk.app.services.OrchestrationService; +import com.sap.ai.sdk.core.AiCoreService; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.web.servlet.ServletComponentScan; @@ -13,6 +16,10 @@ import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Primary; +import java.io.*; +import java.nio.file.Files; +import java.util.stream.Collectors; + /** Main class to start the Spring Boot application. */ @SpringBootApplication @ComponentScan({"com.sap.cloud.sdk", "com.sap.ai.sdk.app"}) @@ -37,7 +44,21 @@ public ObjectMapper objectMapper() { * * @param args Command line arguments. */ - public static void main(final String[] args) { - SpringApplication.run(Application.class, args); + public static void main(final String[] args) throws IOException { + var ctx = SpringApplication.run(Application.class, args); + var svc = ctx.getBean(OrchestrationService.class); + var res = svc.speech("Eat more of these fresh french baguettes").toList(); + String path = System.getProperty("user.home") + File.separator + "Documents" + File.separator + System.currentTimeMillis() + ".pcm"; + File file = new File(path); + try (FileOutputStream in = new FileOutputStream(file)) { + var ttl = 0; + for (byte[] chunk: res) { + System.out.println("Writing bytes: " + chunk.length); + in.write(chunk); + ttl += chunk.length; + } + System.out.println("Written bytes: " + ttl); + } + } } diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OrchestrationService.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OrchestrationService.java index bf8d82b9d..4ef188372 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OrchestrationService.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OrchestrationService.java @@ -82,6 +82,15 @@ public OrchestrationChatResponse completion(@Nonnull final String famousPhrase) return client.chatCompletion(prompt, config); } + /** + * TODO: document + * @param input + * @return + */ + public Stream speech(@Nonnull final String input) { + return client.speech(input); + } + /** * Chat request to OpenAI through the Orchestration service with an image. * From ecd953e472948e64b9ccd96176ea7ede8f7142a5 Mon Sep 17 00:00:00 2001 From: Vladimir Aseev Date: Tue, 7 Jul 2026 19:42:08 +0200 Subject: [PATCH 02/26] polish minimap text-to-speech realtime poc --- .../foundationmodels/openai/OpenAiModel.java | 2 +- .../orchestration/OrchestrationAiModel.java | 3 + .../orchestration/OrchestrationClient.java | 152 ++----------- .../orchestration/RealtimeChannelFactory.java | 58 +++++ .../orchestration/RealtimeClientFactory.java | 200 ------------------ .../sdk/orchestration/TextInputChannel.java | 5 + .../TextToSpeechRealtimeClient.java | 22 +- .../WSSOpenAIRealtimeClient.java | 121 +++++++---- .../java/com/sap/ai/sdk/app/Application.java | 28 ++- .../app/services/OrchestrationService.java | 34 +-- 10 files changed, 225 insertions(+), 400 deletions(-) create mode 100644 orchestration/src/main/java/com/sap/ai/sdk/orchestration/RealtimeChannelFactory.java delete mode 100644 orchestration/src/main/java/com/sap/ai/sdk/orchestration/RealtimeClientFactory.java create mode 100644 orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextInputChannel.java diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiModel.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiModel.java index 1926f56ce..62c435bcc 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiModel.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiModel.java @@ -121,7 +121,7 @@ public record OpenAiModel(@Nonnull String name, @Nullable String version) implem /** Azure OpenAI GPT-5-nano model */ public static final OpenAiModel GPT_5_NANO = new OpenAiModel("gpt-5-nano", null); - /** Azure OpenAI GPT-5-nano model */ + /** Azure OpenAI GPT-realtime model */ public static final OpenAiModel GPT_REALTIME = new OpenAiModel("gpt-realtime", null); /** Azure OpenAI GPT-5.2 model */ diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationAiModel.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationAiModel.java index 06b6beb08..1c6cdf7b3 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationAiModel.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationAiModel.java @@ -330,6 +330,9 @@ public class OrchestrationAiModel { /** Azure OpenAI GPT-5-mini model */ public static final OrchestrationAiModel GPT_5_MINI = new OrchestrationAiModel("gpt-5-mini"); + /** Azure OpenAI GPT-realtime model */ + public static final OrchestrationAiModel GPT_REALTIME = new OrchestrationAiModel("gpt-realtime"); + /** Azure OpenAI GPT-5-nano model */ public static final OrchestrationAiModel GPT_5_NANO = new OrchestrationAiModel("gpt-5-nano"); diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationClient.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationClient.java index 4e901e966..ad082065d 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationClient.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationClient.java @@ -12,6 +12,7 @@ import com.openai.models.beta.realtime.sessions.Session; import com.openai.models.realtime.SessionUpdateEvent; import com.sap.ai.sdk.core.AiCoreService; +import com.sap.ai.sdk.core.AiModel; import com.sap.ai.sdk.orchestration.model.CompletionPostRequest; import com.sap.ai.sdk.orchestration.model.CompletionPostResponse; import com.sap.ai.sdk.orchestration.model.CompletionRequestConfiguration; @@ -32,6 +33,7 @@ import java.util.*; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; +import java.util.function.BiConsumer; import java.util.function.Consumer; import java.util.function.Supplier; import java.util.stream.Stream; @@ -42,6 +44,8 @@ import lombok.val; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; import org.apache.hc.client5.http.impl.classic.HttpClients; +import org.jspecify.annotations.NonNull; +import org.jspecify.annotations.Nullable; /** Client to execute requests to the orchestration service. */ @Slf4j @@ -124,142 +128,26 @@ public OrchestrationChatResponse chatCompletion( /** * Voices input text into output audio * - * @param input text to voice - * @return byte chunks output stream in PCM16 format, 16-bit, mono, 24000 Hz, little-endian + * @param output byte chunks output consumer in PCM16 format, 16-bit, mono, 24000 Hz, little-endian, + * boolean flag signifies end of the current reply (only true for last chunk of current response) + * @return text input channel to supply text to voice, should be closed when not needed anymore */ - public Stream speech(String input) { - - var deploymentID = ""; - final List result = new ArrayList<>(); - try { - HttpClient client = HttpClient.newHttpClient(); // TODO: fix local setup and use autocloseable - WebSocket ws = - client - .newWebSocketBuilder() - .header( - "Authorization", - "Bearer .......") - .header("AI-Resource-Group", "default") - .buildAsync( - URI.create(REALTIME_API_PATH + deploymentID), - new WebSocket.Listener() { - - @Override - public void onOpen(WebSocket webSocket) { - log.warn("Websocket connection opened"); - String sessionUpdateJson = - """ - { - "type": "session.update", - "session": { - "type": "realtime", - "output_modalities": ["audio"], - "instructions": "you are a speaker and your role is to read (produce audio) of the user input speech. voice user text input", - "audio": { - "input": { "format": { "type": "audio/pcm", "rate": 24000 } }, - "output": { "format": { "type": "audio/pcm", "rate": 24000 }, "voice": "alloy" } - } - } - } - """; - - // String sessionUpdateJson = "{\"type\": \"session.update\", \"session\": - // {\"modalities\": [\"audio\", \"text\"]}}"; - webSocket.sendText(sessionUpdateJson, true); - webSocket.request(1); - - String textMessage = - """ -{ - "type": "conversation.item.create", - "item": { - "type": "message", - "role": "system", - "content": [ - { - "type": "input_text", - "text": "you are a speaker and your role is to read (produce audio) of the user input speech. voice user text input" - } - ] - } -} -"""; - //.replaceFirst("%VALUE%", input); // TODO: sanitize or/and escape input - log.warn("Sending message to websocket: {}", textMessage); - webSocket.sendText(textMessage, true); - webSocket.request(1); - - String voiceMessage = - """ - { - "type": "conversation.item.create", - "item": { - "type": "message", - "role": "user", - "content": [ - { - "type": "input_text", - "text": "Eat more of these fresh french baguettes" - } - ] - } - } - """; - - webSocket.sendText(voiceMessage, true); - webSocket.request(1); - - - + public TextInputChannel textToSpeech(BiConsumer output) { + var destination = new AiCoreService().getInferenceDestination().forModel(new AiModel() { + @Override + public @NonNull String name() { + return OrchestrationAiModel.GPT_REALTIME.getName(); + } - String askForResponse = "{ \"type\": \"response.create\" }"; - webSocket.sendText(askForResponse, true); + @Override + public @Nullable String version() { + return OrchestrationAiModel.GPT_REALTIME.getVersion(); + } + }); - webSocket.request(80); - } + var factory = new RealtimeChannelFactory(() -> destination); - @Override - public CompletionStage onText( - WebSocket webSocket, CharSequence data, boolean last) { - try { - Thread.sleep(500); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - log.warn("text received: {}", data); - final JsonNode event; - try { - event = JACKSON.readTree(data.toString()); - } catch (JsonProcessingException e) { - throw new OrchestrationClientException( - "Error parsing JSON response from speech API", e); - } - String type = event.get("type").asText(); - if ("response.output_audio.delta".equals(type)) { - String base64Audio = event.get("delta").asText(); - byte[] audio = Base64.getDecoder().decode(base64Audio); - result.add(audio); - } else if ("response.audio_transcript.delta".equals(type)) { - log.warn("RECEIVED TRANSCRIPT: {}", event.get("delta").asText()); - } else if ("response.output_audio.done".equals(type)) { - // TODO: we should not wait last ack, we should return stream immediately, - // this requires - // TODO: thread management, factory and thread safety of all structs - webSocket.sendClose(1000, "done"); - return null; - } else { - log.warn("Unhandled event type: {}", data); - } - return CompletableFuture.completedStage(null); - } - }) - .join(); - Thread.sleep(25000); - return result.stream(); - } catch (Exception e) { - //log.error(e.getMessage(), e); - throw new OrchestrationClientException("Failed to call realtime API", e); - } + return factory.textToSpeech(output); } /** diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/RealtimeChannelFactory.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/RealtimeChannelFactory.java new file mode 100644 index 000000000..cffba0e1f --- /dev/null +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/RealtimeChannelFactory.java @@ -0,0 +1,58 @@ +package com.sap.ai.sdk.orchestration; + +import com.sap.cloud.sdk.cloudplatform.connectivity.Header; +import com.sap.cloud.sdk.cloudplatform.connectivity.HttpDestination; +import lombok.AccessLevel; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import javax.annotation.Nonnull; +import java.util.*; +import java.util.function.BiConsumer; +import java.util.function.Supplier; + +@Slf4j +@RequiredArgsConstructor(access = AccessLevel.PACKAGE) +public class RealtimeChannelFactory { + private static final String AI_RESOURCE_GROUP_HEADER_NAME = "AI-Resource-Group"; + private static final String AI_RESOURCE_GROUP_HEADER_VALUE = "default"; + private static final String REALTIME_API_PATH_SUFFIX = "/v2/inference/deployments/%s/v1/realtime"; + private static final int PATH_BUFFER_SIZE = 400; // existing URLs are ~120 symbols long, 400 has reasonable margin + + + private static final String REALTIME_API_PATH = + "wss://realtime.ai.intprod-eu12.eu-central-1.aws.ml.hana.ondemand.com/v2/inference/deployments/ddb5a959fdb51ffe/v1/realtime"; + + private final Supplier destinationSupplier; + + /** + * Creates realtime channel allowing to input text and voice it (receive audio output) + * + * @param audioOutputConsumer - audio bytes consumer, where byte[] - chunks of raw PCM mono 24000 Hz output and + * Boolean - marker if current audio reply has been finished (current response ended) + * @return input channel, allowing for text input + */ + public TextInputChannel textToSpeech(@Nonnull final BiConsumer audioOutputConsumer) { + var destination = destinationSupplier.get(); + + var extraHeaders = destination.getHeaders(); + var headers = new HashMap(extraHeaders.size() + 1); + headers.put(AI_RESOURCE_GROUP_HEADER_NAME, AI_RESOURCE_GROUP_HEADER_VALUE); + for (Header header : extraHeaders) { + headers.put(header.getName(), header.getValue()); + } + + var sb = new StringBuilder(PATH_BUFFER_SIZE); + sb.append("wss://"); + var pathParts = destination.getUri().toString().split("//"); + if (pathParts.length != 2) { + throw new IllegalArgumentException("Invalid destination URI: " + destination.getUri()); + } + sb.append(pathParts[1].replaceFirst("^api\\.", "realtime.")); + sb.append("v1/realtime"); + var endpoint = sb.toString(); + + return new TextToSpeechRealtimeClient(endpoint, headers, audioOutputConsumer); + } + +} diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/RealtimeClientFactory.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/RealtimeClientFactory.java deleted file mode 100644 index d48dd7b9f..000000000 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/RealtimeClientFactory.java +++ /dev/null @@ -1,200 +0,0 @@ -package com.sap.ai.sdk.orchestration; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.JsonNode; -import com.openai.models.responses.Response; -import com.sap.cloud.sdk.cloudplatform.connectivity.Header; -import com.sap.cloud.sdk.cloudplatform.connectivity.HttpDestination; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; - -import javax.annotation.Nonnull; -import java.net.URI; -import java.net.http.HttpClient; -import java.net.http.WebSocket; -import java.util.*; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionStage; -import java.util.concurrent.Flow; -import java.util.function.Consumer; -import java.util.function.Supplier; -import java.util.stream.Stream; - -@Slf4j -@RequiredArgsConstructor(access = lombok.AccessLevel.PRIVATE) -public class RealtimeClientFactory { - private static final String AI_RESOURCE_GROUP_HEADER_NAME = "AI-Resource-Group"; - private static final String AI_RESOURCE_GROUP_HEADER_VALUE = "default"; - private static final String REALTIME_API_PATH_SUFFIX = "/v2/inference/deployments/%s/v1/realtime"; - private static final int PATH_BUFFER_SIZE = 400; // existing URLs are ~120 symbols long, 400 has reasonable margin - - - private static final String REALTIME_API_PATH = - "wss://realtime.ai.intprod-eu12.eu-central-1.aws.ml.hana.ondemand.com/v2/inference/deployments/ddb5a959fdb51ffe/v1/realtime"; - - private final Supplier destinationSupplier; - private final Supplier deploymentIdSupplier; - - /** - * Voices input text into output audio - * - * @param input text to voice - * @return byte chunks output stream in PCM16 format, 16-bit, mono, 24000 Hz, little-endian - */ - public Stream textToSpeech(@Nonnull final Consumer audioOutputConsumer) { - var destination = destinationSupplier.get(); - - var extraHeaders = destination.getHeaders(); - var headers = new HashMap(extraHeaders.size() + 1); - headers.put(AI_RESOURCE_GROUP_HEADER_NAME, AI_RESOURCE_GROUP_HEADER_VALUE); - for (Header header : extraHeaders) { - headers.put(header.getName(), header.getValue()); - } - - var deploymentId = deploymentIdSupplier.get(); - var sb = new StringBuilder(PATH_BUFFER_SIZE); - sb.append("wss://"); - sb.append(destination.getUri().getHost()); - // set port explicitly if defined in the uri - if (destination.getUri().getPort() != -1) { - sb.append(":").append(destination.getUri().getPort()); - } - sb.append(String.format(REALTIME_API_PATH_SUFFIX, deploymentId)); - var endpoint = sb.toString(); - - - final List result = new ArrayList<>(); - try { - HttpClient client = HttpClient.newHttpClient(); // TODO: fix local setup and use autocloseable - WebSocket ws = - client - .newWebSocketBuilder() - .header( - "Authorization", - "Bearer ....") - .header("AI-Resource-Group", "default") - .buildAsync( - URI.create(endpoint), - new WebSocket.Listener() { - - @Override - public void onOpen(WebSocket webSocket) { - log.warn("Websocket connection opened"); - String sessionUpdateJson = - """ - { - "type": "session.update", - "session": { - "type": "realtime", - "output_modalities": ["audio"], - "instructions": "you are a speaker and your role is to read (produce audio) of the user input speech. voice user text input", - "audio": { - "input": { "format": { "type": "audio/pcm", "rate": 24000 } }, - "output": { "format": { "type": "audio/pcm", "rate": 24000 }, "voice": "alloy" } - } - } - } - """; - - // String sessionUpdateJson = "{\"type\": \"session.update\", \"session\": - // {\"modalities\": [\"audio\", \"text\"]}}"; - webSocket.sendText(sessionUpdateJson, true); - webSocket.request(1); - - String textMessage = - """ - { - "type": "conversation.item.create", - "item": { - "type": "message", - "role": "system", - "content": [ - { - "type": "input_text", - "text": "you are a speaker and your role is to read (produce audio) of the user input speech. voice user text input" - } - ] - } - } - """; - //.replaceFirst("%VALUE%", input); // TODO: sanitize or/and escape input - log.warn("Sending message to websocket: {}", textMessage); - webSocket.sendText(textMessage, true); - webSocket.request(1); - - String voiceMessage = - """ - { - "type": "conversation.item.create", - "item": { - "type": "message", - "role": "user", - "content": [ - { - "type": "input_text", - "text": "Eat more of these fresh french baguettes" - } - ] - } - } - """; - - webSocket.sendText(voiceMessage, true); - webSocket.request(1); - - - - - String askForResponse = "{ \"type\": \"response.create\" }"; - webSocket.sendText(askForResponse, true); - - webSocket.request(80); - } - - @Override - public CompletionStage onText( - WebSocket webSocket, CharSequence data, boolean last) { - try { - Thread.sleep(500); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - log.warn("text received: {}", data); - final JsonNode event; - try { - event = JACKSON.readTree(data.toString()); - } catch (JsonProcessingException e) { - throw new OrchestrationClientException( - "Error parsing JSON response from speech API", e); - } - Response r = null; - - String type = event.get("type").asText(); - if ("response.output_audio.delta".equals(type)) { - String base64Audio = event.get("delta").asText(); - byte[] audio = Base64.getDecoder().decode(base64Audio); - result.add(audio); - } else if ("response.audio_transcript.delta".equals(type)) { - log.warn("RECEIVED TRANSCRIPT: {}", event.get("delta").asText()); - } else if ("response.output_audio.done".equals(type)) { - // TODO: we should not wait last ack, we should return stream immediately, - // this requires - // TODO: thread management, factory and thread safety of all structs - webSocket.sendClose(1000, "done"); - return null; - } else { - log.warn("Unhandled event type: {}", data); - } - return CompletableFuture.completedStage(null); - } - }) - .join(); - Thread.sleep(25000); - return result.stream(); - } catch (Exception e) { - //log.error(e.getMessage(), e); - throw new OrchestrationClientException("Failed to call realtime API", e); - } - } - -} diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextInputChannel.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextInputChannel.java new file mode 100644 index 000000000..20e6fcc66 --- /dev/null +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextInputChannel.java @@ -0,0 +1,5 @@ +package com.sap.ai.sdk.orchestration; + +public interface TextInputChannel extends AutoCloseable { + void sendText(String text); +} diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextToSpeechRealtimeClient.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextToSpeechRealtimeClient.java index 7df1881da..45ea9335a 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextToSpeechRealtimeClient.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextToSpeechRealtimeClient.java @@ -1,13 +1,14 @@ package com.sap.ai.sdk.orchestration; import com.fasterxml.jackson.databind.JsonNode; -import com.openai.models.realtime.RealtimeSessionCreateRequest; -import io.vavr.collection.Array; +import com.openai.models.realtime.*; +import lombok.extern.slf4j.Slf4j; import java.util.*; import java.util.function.BiConsumer; -public class TextToSpeechRealtimeClient extends WSSOpenAIRealtimeClient { +@Slf4j +public class TextToSpeechRealtimeClient extends WSSOpenAIRealtimeClient implements TextInputChannel { private static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; @@ -30,6 +31,18 @@ public TextToSpeechRealtimeClient(String url, Map httpHeaders, B this.outputConsumer = outputConsumer; } + public void sendText(String text) { + var message = ConversationItemCreateEvent.builder() + .item(ConversationItem.ofRealtimeConversationItemUserMessage( + RealtimeConversationItemUserMessage.builder() + .addContent(RealtimeConversationItemUserMessage.Content.builder() + .text(text) + .type(RealtimeConversationItemUserMessage.Content.Type.INPUT_TEXT).build()).build() + )).build(); + + super.sendMessage(message); + } + @Override protected String getSystemPrompt() { return TASK; @@ -46,8 +59,11 @@ protected void onResponse(String eventType, JsonNode event) { var base64Audio = event.get("delta").asText(); byte[] audio = Base64.getDecoder().decode(base64Audio); this.outputConsumer.accept(audio, Boolean.FALSE); + log.warn("Called consumer with base64 converted to bytes: {} (base64)", base64Audio); } else if ("response.output_audio.done".equals(eventType)) { this.outputConsumer.accept(EMPTY_BYTE_ARRAY, Boolean.TRUE); + } else { + log.warn("skipping message type: {}", eventType); } } } diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/WSSOpenAIRealtimeClient.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/WSSOpenAIRealtimeClient.java index 588e5853b..8770baf5f 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/WSSOpenAIRealtimeClient.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/WSSOpenAIRealtimeClient.java @@ -1,12 +1,16 @@ package com.sap.ai.sdk.orchestration; +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.PropertyAccessor; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.openai.models.conversations.Message; +import com.openai.models.conversations.TextContent; import com.openai.models.realtime.*; import com.openai.models.realtime.clientsecrets.ClientSecretCreateParams; import lombok.AccessLevel; -import lombok.AllArgsConstructor; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import java.net.URI; @@ -19,11 +23,12 @@ import static com.sap.ai.sdk.orchestration.OrchestrationJacksonConfiguration.getOrchestrationObjectMapper; @Slf4j -@AllArgsConstructor(access = AccessLevel.PACKAGE) +@RequiredArgsConstructor(access = AccessLevel.PACKAGE) public abstract class WSSOpenAIRealtimeClient implements AutoCloseable { private static final int SUCCESS_FINISH_WSS_CODE = 1000; - private static final ObjectMapper JACKSON = getOrchestrationObjectMapper(); + private static final ObjectMapper JACKSON = getOrchestrationObjectMapper() + .setVisibility(PropertyAccessor.IS_GETTER, JsonAutoDetect.Visibility.NONE); /** * WebSocket keeps TCP connection to the server. If connection idles, depending on the provider @@ -34,7 +39,7 @@ public abstract class WSSOpenAIRealtimeClient implements AutoCloseable { * before it gets terminated *

* In realtime client we rely on explicit lifetime management (create/close), heartbeat mechanism is required - * to prevent unintended closing of connections while conversation is still used + * to prevent unintended closing of connections while conversation is still expected to continue */ private static final int HEARTBEAT_INTERVAL_MILLIS = 4500; private static final String HEARTBEAT_TIMER_NAME = "wss_realtime_heartbeat"; @@ -55,22 +60,30 @@ public WSSOpenAIRealtimeClient(String url, Map httpHeaders, Set< this.heartbeatTimer = new Timer(HEARTBEAT_TIMER_NAME,true); } - public synchronized void askForResponse() { + public void askForResponse() { WebSocket ws = this.ws.join(); - ws.sendText(JACKSON.valueToTree(ResponseCreateEvent.builder().build()), true); - ws.request(1); + synchronized (this) { + try { + ws.sendText(JACKSON.writeValueAsString(ResponseCreateEvent.builder().build()), true); + } catch (JsonProcessingException e) { + throw new RuntimeException("Failed to serialize ask for response", e); + } + ws.request(1); + } } @Override - public synchronized void close() { + public void close() { WebSocket ws = this.ws.join(); - heartbeatTimer.cancel(); - try { - ws.sendClose(SUCCESS_FINISH_WSS_CODE, "done").join(); - } catch (Exception e) { - log.error("Error while closing WebSocket", e); + synchronized (this) { + heartbeatTimer.cancel(); + try { + ws.sendClose(SUCCESS_FINISH_WSS_CODE, "done").join(); + } catch (Exception e) { + log.error("Error while closing WebSocket", e); + } + // this.client.close(); // exists only since java 21 } - this.client.close(); } protected abstract String getSystemPrompt(); @@ -79,10 +92,19 @@ public synchronized void close() { protected abstract void onResponse(String eventType, JsonNode event); - protected synchronized void sendMessage(RealtimeConversationItemUserMessage message) { + protected void sendMessage(ConversationItemCreateEvent message) { WebSocket ws = this.ws.join(); - ws.sendText(JACKSON.valueToTree(message), true); - ws.request(1); + synchronized (this) { + try { + ws.sendText(JACKSON.writeValueAsString(message), true); + ws.request(1); + log.warn("WSS sent message: {}", JACKSON.writeValueAsString(message)); + askForResponse(); + } catch (JsonProcessingException e) { + throw new OrchestrationClientException("Failed to serialize message", e); + } + + } } private synchronized void onSocketOpen(WebSocket ws) { @@ -92,6 +114,7 @@ private synchronized void onSocketOpen(WebSocket ws) { } private void onText(WebSocket webSocket, CharSequence data) { + log.warn("WSS received message: {}", data); final JsonNode event; try { event = JACKSON.readTree(data.toString()); @@ -116,31 +139,57 @@ private synchronized void sendPing(WebSocket ws) { } private void configureSession(WebSocket ws) { - SessionUpdateEvent sue = SessionUpdateEvent.builder() - .session(ClientSecretCreateParams.Session.ofRealtime( + SessionUpdateEvent sue = + SessionUpdateEvent.builder() + .session( + ClientSecretCreateParams.Session.ofRealtime( RealtimeSessionCreateRequest.builder() - .outputModalities(getOutputModalities()) - .audio(RealtimeAudioConfig.builder() - .input(RealtimeAudioConfigInput.builder() - .format(RealtimeAudioFormats.AudioPcm.builder().build()) - .build()) - .output(RealtimeAudioConfigOutput.builder() - .format(RealtimeAudioFormats.AudioPcm.builder().build()) - .voice(RealtimeAudioConfigOutput.Voice.UnionMember1.ALLOY) - .build()) - .build()) - .build()) - .asRealtime()) - .build(); - ws.sendText(JACKSON.valueToTree(sue), true); + .outputModalities(getOutputModalities()) + .audio( + RealtimeAudioConfig.builder() + .input( + RealtimeAudioConfigInput.builder() + .format( + RealtimeAudioFormats.AudioPcm.builder() + .type(RealtimeAudioFormats.AudioPcm.Type.AUDIO_PCM) + .build()) + .build()) + .output( + RealtimeAudioConfigOutput.builder() + .format(RealtimeAudioFormats.AudioPcm.builder().build()) + .voice( + RealtimeAudioConfigOutput.Voice.UnionMember1.ALLOY) + .build()) + .build()) + .build()) + .asRealtime()) + .build(); + try { + ws.sendText(JACKSON.writeValueAsString(sue), true); + } catch (JsonProcessingException e) { + throw new OrchestrationClientException("Failed to serialize session request", e); + } + ws.request(1); } private void configureConversation(WebSocket ws) { var systemPrompt = getSystemPrompt(); - var message = new RealtimeConversationItemSystemMessage.Builder(). - addContent(RealtimeConversationItemSystemMessage.Content.builder().text(systemPrompt).build()); - ws.sendText(JACKSON.valueToTree(message), true); + var systemConversationItem = ConversationItemCreateEvent.builder() + .item(ConversationItem.ofRealtimeConversationItemSystemMessage( + RealtimeConversationItemSystemMessage.builder() + .addContent(RealtimeConversationItemSystemMessage.Content.builder() + .text(systemPrompt) + .type(RealtimeConversationItemSystemMessage.Content.Type.INPUT_TEXT).build()).build() + )).build(); + + try { + var json = JACKSON.writeValueAsString(systemConversationItem); + ws.sendText(json, true); + log.warn("WSS conversation config sent: {}", json); + } catch (JsonProcessingException e) { + throw new OrchestrationClientException("Failed to serialize message", e); + } ws.request(1); } diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/Application.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/Application.java index 01269478d..aced5c87e 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/Application.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/Application.java @@ -18,6 +18,8 @@ import java.io.*; import java.nio.file.Files; +import java.util.ArrayList; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; /** Main class to start the Spring Boot application. */ @@ -44,15 +46,35 @@ public ObjectMapper objectMapper() { * * @param args Command line arguments. */ - public static void main(final String[] args) throws IOException { + public static void main(final String[] args) throws Exception { var ctx = SpringApplication.run(Application.class, args); var svc = ctx.getBean(OrchestrationService.class); - var res = svc.speech("Eat more of these fresh french baguettes").toList(); + var outputBytes = new ArrayList(); + var done = new AtomicBoolean(false); + try (var channel = svc.textToSpeech((bytes, last) -> { + System.err.println("appended output bytes: " + bytes.length); + outputBytes.add(bytes); + if (last) { + done.set(true); + } + })) { + System.err.println("BEFORE SENDING TEXT"); + channel.sendText("Eat more of these fresh french baguettes and drink the tea!"); + System.err.println("AFTER SENDING TEXT"); + long timeout = System.currentTimeMillis() + 20000; + while (!done.get() && System.currentTimeMillis() < timeout) { + Thread.yield(); + System.err.println("Waiting for messages..."); + Thread.sleep(500); + } + } + + //var res = svc.textToSpeech("Eat more of these fresh french baguettes").toList(); String path = System.getProperty("user.home") + File.separator + "Documents" + File.separator + System.currentTimeMillis() + ".pcm"; File file = new File(path); try (FileOutputStream in = new FileOutputStream(file)) { var ttl = 0; - for (byte[] chunk: res) { + for (byte[] chunk: outputBytes) { System.out.println("Writing bytes: " + chunk.length); in.write(chunk); ttl += chunk.length; diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OrchestrationService.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OrchestrationService.java index 4ef188372..49fbaa594 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OrchestrationService.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OrchestrationService.java @@ -10,26 +10,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.sap.ai.sdk.core.AiCoreService; -import com.sap.ai.sdk.orchestration.AzureContentFilter; -import com.sap.ai.sdk.orchestration.AzureFilterThreshold; -import com.sap.ai.sdk.orchestration.DpiMasking; -import com.sap.ai.sdk.orchestration.Grounding; -import com.sap.ai.sdk.orchestration.ImageItem; -import com.sap.ai.sdk.orchestration.LlamaGuardFilter; -import com.sap.ai.sdk.orchestration.Message; -import com.sap.ai.sdk.orchestration.OrchestrationAiModel; -import com.sap.ai.sdk.orchestration.OrchestrationChatResponse; -import com.sap.ai.sdk.orchestration.OrchestrationClient; -import com.sap.ai.sdk.orchestration.OrchestrationClientException; -import com.sap.ai.sdk.orchestration.OrchestrationConfigReference; -import com.sap.ai.sdk.orchestration.OrchestrationEmbeddingRequest; -import com.sap.ai.sdk.orchestration.OrchestrationEmbeddingResponse; -import com.sap.ai.sdk.orchestration.OrchestrationModuleConfig; -import com.sap.ai.sdk.orchestration.OrchestrationPrompt; -import com.sap.ai.sdk.orchestration.ResponseJsonSchema; -import com.sap.ai.sdk.orchestration.SystemMessage; -import com.sap.ai.sdk.orchestration.TemplateConfig; -import com.sap.ai.sdk.orchestration.TranslationConfig; +import com.sap.ai.sdk.orchestration.*; import com.sap.ai.sdk.orchestration.model.DPIEntities; import com.sap.ai.sdk.orchestration.model.DataRepositoryType; import com.sap.ai.sdk.orchestration.model.DocumentGroundingFilter; @@ -52,6 +33,7 @@ import java.nio.file.Path; import java.util.List; import java.util.Map; +import java.util.function.BiConsumer; import java.util.stream.Stream; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -83,12 +65,14 @@ public OrchestrationChatResponse completion(@Nonnull final String famousPhrase) } /** - * TODO: document - * @param input - * @return + * Voices input text into output audio + * + * @param outputConsumer byte chunks output consumer in PCM16 format, 16-bit, mono, 24000 Hz, little-endian, + * boolean flag signifies end of the current reply (only true for last chunk of current response) + * @return text input channel to supply text to voice, should be closed when not needed anymore */ - public Stream speech(@Nonnull final String input) { - return client.speech(input); + public TextInputChannel textToSpeech(@Nonnull final BiConsumer outputConsumer) { + return client.textToSpeech(outputConsumer); } /** From 71d4a4cc2b5cd9db00edaf14546030dbdd0e7938 Mon Sep 17 00:00:00 2001 From: Vladimir Aseev Date: Tue, 7 Jul 2026 20:03:22 +0200 Subject: [PATCH 03/26] add incomplete interim packets buffering and glueing --- ...{WebsocketListenerGlue.java => BufferedWebSocketListener.java} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename orchestration/src/main/java/com/sap/ai/sdk/orchestration/{WebsocketListenerGlue.java => BufferedWebSocketListener.java} (100%) diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/WebsocketListenerGlue.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/BufferedWebSocketListener.java similarity index 100% rename from orchestration/src/main/java/com/sap/ai/sdk/orchestration/WebsocketListenerGlue.java rename to orchestration/src/main/java/com/sap/ai/sdk/orchestration/BufferedWebSocketListener.java From 232725d79a04f8b1b162b41b97a01cbca4da1eb0 Mon Sep 17 00:00:00 2001 From: Vladimir Aseev Date: Tue, 7 Jul 2026 20:06:44 +0200 Subject: [PATCH 04/26] remove obsolete debug constants --- .../orchestration/BufferedWebSocketListener.java | 14 +++++++++++--- .../ai/sdk/orchestration/OrchestrationClient.java | 2 -- .../sdk/orchestration/RealtimeChannelFactory.java | 5 ----- .../sdk/orchestration/WSSOpenAIRealtimeClient.java | 5 ++--- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/BufferedWebSocketListener.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/BufferedWebSocketListener.java index 28bb102ec..118b2d5ce 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/BufferedWebSocketListener.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/BufferedWebSocketListener.java @@ -6,14 +6,16 @@ import java.util.function.BiConsumer; import java.util.function.Consumer; -public class WebsocketListenerGlue implements WebSocket.Listener { +public class BufferedWebSocketListener implements WebSocket.Listener { private final Consumer onOpen; private final BiConsumer onText; + private final StringBuilder buffer; - public WebsocketListenerGlue(Consumer onOpen, BiConsumer onText) { + public BufferedWebSocketListener(Consumer onOpen, BiConsumer onText) { this.onOpen = onOpen; this.onText = onText; + this.buffer = new StringBuilder(1024 * 1024); } @Override @@ -24,8 +26,14 @@ public void onOpen(WebSocket webSocket) { @Override public CompletionStage onText(WebSocket webSocket, CharSequence data, boolean last) { - this.onText.accept(webSocket, data); + buffer.append(data); webSocket.request(1); + if (last) { + var completeMessage = buffer.toString(); + buffer.setLength(0); + this.onText.accept(webSocket, completeMessage); + } + return CompletableFuture.completedStage(null); } } diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationClient.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationClient.java index ad082065d..7c52d772b 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationClient.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationClient.java @@ -53,8 +53,6 @@ public class OrchestrationClient { private static final String DEFAULT_SCENARIO = "orchestration"; private static final String COMPLETION_ENDPOINT = "/v2/completion"; - private static final String REALTIME_API_PATH = - "wss://realtime.ai.intprod-eu12.eu-central-1.aws.ml.hana.ondemand.com/v2/inference/deployments/ddb5a959fdb51ffe/v1/realtime"; static final ObjectMapper JACKSON = getOrchestrationObjectMapper(); diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/RealtimeChannelFactory.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/RealtimeChannelFactory.java index cffba0e1f..28edd05c4 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/RealtimeChannelFactory.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/RealtimeChannelFactory.java @@ -16,13 +16,8 @@ public class RealtimeChannelFactory { private static final String AI_RESOURCE_GROUP_HEADER_NAME = "AI-Resource-Group"; private static final String AI_RESOURCE_GROUP_HEADER_VALUE = "default"; - private static final String REALTIME_API_PATH_SUFFIX = "/v2/inference/deployments/%s/v1/realtime"; private static final int PATH_BUFFER_SIZE = 400; // existing URLs are ~120 symbols long, 400 has reasonable margin - - private static final String REALTIME_API_PATH = - "wss://realtime.ai.intprod-eu12.eu-central-1.aws.ml.hana.ondemand.com/v2/inference/deployments/ddb5a959fdb51ffe/v1/realtime"; - private final Supplier destinationSupplier; /** diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/WSSOpenAIRealtimeClient.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/WSSOpenAIRealtimeClient.java index 8770baf5f..678bff93a 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/WSSOpenAIRealtimeClient.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/WSSOpenAIRealtimeClient.java @@ -5,8 +5,6 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import com.openai.models.conversations.Message; -import com.openai.models.conversations.TextContent; import com.openai.models.realtime.*; import com.openai.models.realtime.clientsecrets.ClientSecretCreateParams; import lombok.AccessLevel; @@ -55,7 +53,7 @@ public WSSOpenAIRealtimeClient(String url, Map httpHeaders, Set< for (Map.Entry entry : httpHeaders.entrySet()) { wsBuilder = wsBuilder.header(entry.getKey(), entry.getValue()); } - this.ws = wsBuilder.buildAsync(URI.create(url), new WebsocketListenerGlue(this::onSocketOpen, this::onText)); + this.ws = wsBuilder.buildAsync(URI.create(url), new BufferedWebSocketListener(this::onSocketOpen, this::onText)); this.handleMessageTypes = handleMessageTypes; this.heartbeatTimer = new Timer(HEARTBEAT_TIMER_NAME,true); } @@ -149,6 +147,7 @@ private void configureSession(WebSocket ws) { RealtimeAudioConfig.builder() .input( RealtimeAudioConfigInput.builder() + .turnDetection(Optional.empty()) .format( RealtimeAudioFormats.AudioPcm.builder() .type(RealtimeAudioFormats.AudioPcm.Type.AUDIO_PCM) From 6b872d4b76a02976b4f4ad98ef0606bc7681704f Mon Sep 17 00:00:00 2001 From: SAP Cloud SDK Bot Date: Tue, 7 Jul 2026 18:08:02 +0000 Subject: [PATCH 05/26] Formatting --- .../BufferedWebSocketListener.java | 49 +-- .../orchestration/OrchestrationClient.java | 65 ++-- .../orchestration/RealtimeChannelFactory.java | 77 ++--- .../sdk/orchestration/TextInputChannel.java | 2 +- .../TextToSpeechRealtimeClient.java | 99 +++--- .../WSSOpenAIRealtimeClient.java | 297 +++++++++--------- .../java/com/sap/ai/sdk/app/Application.java | 42 +-- .../app/services/OrchestrationService.java | 5 +- 8 files changed, 321 insertions(+), 315 deletions(-) diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/BufferedWebSocketListener.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/BufferedWebSocketListener.java index 118b2d5ce..6116baeea 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/BufferedWebSocketListener.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/BufferedWebSocketListener.java @@ -8,32 +8,33 @@ public class BufferedWebSocketListener implements WebSocket.Listener { - private final Consumer onOpen; - private final BiConsumer onText; - private final StringBuilder buffer; + private final Consumer onOpen; + private final BiConsumer onText; + private final StringBuilder buffer; - public BufferedWebSocketListener(Consumer onOpen, BiConsumer onText) { - this.onOpen = onOpen; - this.onText = onText; - this.buffer = new StringBuilder(1024 * 1024); - } - - @Override - public void onOpen(WebSocket webSocket) { - this.onOpen.accept(webSocket); - webSocket.request(1); - } + public BufferedWebSocketListener( + Consumer onOpen, BiConsumer onText) { + this.onOpen = onOpen; + this.onText = onText; + this.buffer = new StringBuilder(1024 * 1024); + } - @Override - public CompletionStage onText(WebSocket webSocket, CharSequence data, boolean last) { - buffer.append(data); - webSocket.request(1); - if (last) { - var completeMessage = buffer.toString(); - buffer.setLength(0); - this.onText.accept(webSocket, completeMessage); - } + @Override + public void onOpen(WebSocket webSocket) { + this.onOpen.accept(webSocket); + webSocket.request(1); + } - return CompletableFuture.completedStage(null); + @Override + public CompletionStage onText(WebSocket webSocket, CharSequence data, boolean last) { + buffer.append(data); + webSocket.request(1); + if (last) { + var completeMessage = buffer.toString(); + buffer.setLength(0); + this.onText.accept(webSocket, completeMessage); } + + return CompletableFuture.completedStage(null); + } } diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationClient.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationClient.java index 7c52d772b..b03548e49 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationClient.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationClient.java @@ -7,10 +7,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.annotations.Beta; -import com.openai.client.OpenAIClient; -import com.openai.client.okhttp.OpenAIOkHttpClient; -import com.openai.models.beta.realtime.sessions.Session; -import com.openai.models.realtime.SessionUpdateEvent; import com.sap.ai.sdk.core.AiCoreService; import com.sap.ai.sdk.core.AiModel; import com.sap.ai.sdk.orchestration.model.CompletionPostRequest; @@ -22,28 +18,15 @@ import com.sap.ai.sdk.orchestration.model.OrchestrationConfig; import com.sap.cloud.sdk.cloudplatform.connectivity.Header; import com.sap.cloud.sdk.cloudplatform.connectivity.HttpDestination; -import io.vavr.collection.Array; import io.vavr.control.Try; - -import java.io.ByteArrayOutputStream; -import java.net.URI; -import java.net.http.HttpClient; -import java.net.http.WebSocket; -import java.nio.ByteBuffer; import java.util.*; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionStage; import java.util.function.BiConsumer; -import java.util.function.Consumer; import java.util.function.Supplier; import java.util.stream.Stream; -import java.util.stream.StreamSupport; import javax.annotation.Nonnull; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import lombok.val; -import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; -import org.apache.hc.client5.http.impl.classic.HttpClients; import org.jspecify.annotations.NonNull; import org.jspecify.annotations.Nullable; @@ -56,7 +39,6 @@ public class OrchestrationClient { static final ObjectMapper JACKSON = getOrchestrationObjectMapper(); - private final OrchestrationHttpExecutor executor; private final List

customHeaders = new ArrayList<>(); @@ -123,30 +105,35 @@ public OrchestrationChatResponse chatCompletion( return new OrchestrationChatResponse(response); } - /** - * Voices input text into output audio - * - * @param output byte chunks output consumer in PCM16 format, 16-bit, mono, 24000 Hz, little-endian, - * boolean flag signifies end of the current reply (only true for last chunk of current response) - * @return text input channel to supply text to voice, should be closed when not needed anymore - */ - public TextInputChannel textToSpeech(BiConsumer output) { - var destination = new AiCoreService().getInferenceDestination().forModel(new AiModel() { - @Override - public @NonNull String name() { - return OrchestrationAiModel.GPT_REALTIME.getName(); - } + /** + * Voices input text into output audio + * + * @param output byte chunks output consumer in PCM16 format, 16-bit, mono, 24000 Hz, + * little-endian, boolean flag signifies end of the current reply (only true for last chunk of + * current response) + * @return text input channel to supply text to voice, should be closed when not needed anymore + */ + public TextInputChannel textToSpeech(BiConsumer output) { + var destination = + new AiCoreService() + .getInferenceDestination() + .forModel( + new AiModel() { + @Override + public @NonNull String name() { + return OrchestrationAiModel.GPT_REALTIME.getName(); + } - @Override - public @Nullable String version() { - return OrchestrationAiModel.GPT_REALTIME.getVersion(); - } - }); + @Override + public @Nullable String version() { + return OrchestrationAiModel.GPT_REALTIME.getVersion(); + } + }); - var factory = new RealtimeChannelFactory(() -> destination); + var factory = new RealtimeChannelFactory(() -> destination); - return factory.textToSpeech(output); - } + return factory.textToSpeech(output); + } /** * Generate a completion for the given prompt. diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/RealtimeChannelFactory.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/RealtimeChannelFactory.java index 28edd05c4..6eb076f12 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/RealtimeChannelFactory.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/RealtimeChannelFactory.java @@ -2,52 +2,53 @@ import com.sap.cloud.sdk.cloudplatform.connectivity.Header; import com.sap.cloud.sdk.cloudplatform.connectivity.HttpDestination; -import lombok.AccessLevel; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; - -import javax.annotation.Nonnull; import java.util.*; import java.util.function.BiConsumer; import java.util.function.Supplier; +import javax.annotation.Nonnull; +import lombok.AccessLevel; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; @Slf4j @RequiredArgsConstructor(access = AccessLevel.PACKAGE) public class RealtimeChannelFactory { - private static final String AI_RESOURCE_GROUP_HEADER_NAME = "AI-Resource-Group"; - private static final String AI_RESOURCE_GROUP_HEADER_VALUE = "default"; - private static final int PATH_BUFFER_SIZE = 400; // existing URLs are ~120 symbols long, 400 has reasonable margin - - private final Supplier destinationSupplier; - - /** - * Creates realtime channel allowing to input text and voice it (receive audio output) - * - * @param audioOutputConsumer - audio bytes consumer, where byte[] - chunks of raw PCM mono 24000 Hz output and - * Boolean - marker if current audio reply has been finished (current response ended) - * @return input channel, allowing for text input - */ - public TextInputChannel textToSpeech(@Nonnull final BiConsumer audioOutputConsumer) { - var destination = destinationSupplier.get(); - - var extraHeaders = destination.getHeaders(); - var headers = new HashMap(extraHeaders.size() + 1); - headers.put(AI_RESOURCE_GROUP_HEADER_NAME, AI_RESOURCE_GROUP_HEADER_VALUE); - for (Header header : extraHeaders) { - headers.put(header.getName(), header.getValue()); - } - - var sb = new StringBuilder(PATH_BUFFER_SIZE); - sb.append("wss://"); - var pathParts = destination.getUri().toString().split("//"); - if (pathParts.length != 2) { - throw new IllegalArgumentException("Invalid destination URI: " + destination.getUri()); - } - sb.append(pathParts[1].replaceFirst("^api\\.", "realtime.")); - sb.append("v1/realtime"); - var endpoint = sb.toString(); + private static final String AI_RESOURCE_GROUP_HEADER_NAME = "AI-Resource-Group"; + private static final String AI_RESOURCE_GROUP_HEADER_VALUE = "default"; + private static final int PATH_BUFFER_SIZE = + 400; // existing URLs are ~120 symbols long, 400 has reasonable margin + + private final Supplier destinationSupplier; + + /** + * Creates realtime channel allowing to input text and voice it (receive audio output) + * + * @param audioOutputConsumer - audio bytes consumer, where byte[] - chunks of raw PCM mono 24000 + * Hz output and Boolean - marker if current audio reply has been finished (current response + * ended) + * @return input channel, allowing for text input + */ + public TextInputChannel textToSpeech( + @Nonnull final BiConsumer audioOutputConsumer) { + var destination = destinationSupplier.get(); + + var extraHeaders = destination.getHeaders(); + var headers = new HashMap(extraHeaders.size() + 1); + headers.put(AI_RESOURCE_GROUP_HEADER_NAME, AI_RESOURCE_GROUP_HEADER_VALUE); + for (Header header : extraHeaders) { + headers.put(header.getName(), header.getValue()); + } - return new TextToSpeechRealtimeClient(endpoint, headers, audioOutputConsumer); + var sb = new StringBuilder(PATH_BUFFER_SIZE); + sb.append("wss://"); + var pathParts = destination.getUri().toString().split("//"); + if (pathParts.length != 2) { + throw new IllegalArgumentException("Invalid destination URI: " + destination.getUri()); } + sb.append(pathParts[1].replaceFirst("^api\\.", "realtime.")); + sb.append("v1/realtime"); + var endpoint = sb.toString(); + return new TextToSpeechRealtimeClient(endpoint, headers, audioOutputConsumer); + } } diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextInputChannel.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextInputChannel.java index 20e6fcc66..eff6a15df 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextInputChannel.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextInputChannel.java @@ -1,5 +1,5 @@ package com.sap.ai.sdk.orchestration; public interface TextInputChannel extends AutoCloseable { - void sendText(String text); + void sendText(String text); } diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextToSpeechRealtimeClient.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextToSpeechRealtimeClient.java index 45ea9335a..74552bb68 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextToSpeechRealtimeClient.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextToSpeechRealtimeClient.java @@ -2,68 +2,71 @@ import com.fasterxml.jackson.databind.JsonNode; import com.openai.models.realtime.*; -import lombok.extern.slf4j.Slf4j; - import java.util.*; import java.util.function.BiConsumer; +import lombok.extern.slf4j.Slf4j; @Slf4j -public class TextToSpeechRealtimeClient extends WSSOpenAIRealtimeClient implements TextInputChannel { - - private static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; +public class TextToSpeechRealtimeClient extends WSSOpenAIRealtimeClient + implements TextInputChannel { - private static final Set HANDLED_RESPONSE_TYPES = Set.of( - "response.output_audio.delta", "response.output_audio.done" - ); + private static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; - private static final List OUTPUT_MODALITIES = List.of( - RealtimeSessionCreateRequest.OutputModality.AUDIO - ); + private static final Set HANDLED_RESPONSE_TYPES = + Set.of("response.output_audio.delta", "response.output_audio.done"); - private static final String TASK = - "you are a speaker and your role is to read (produce audio) of the user input speech. voice user text input"; + private static final List OUTPUT_MODALITIES = + List.of(RealtimeSessionCreateRequest.OutputModality.AUDIO); + private static final String TASK = + "you are a speaker and your role is to read (produce audio) of the user input speech. voice user text input"; - private final BiConsumer outputConsumer; + private final BiConsumer outputConsumer; - public TextToSpeechRealtimeClient(String url, Map httpHeaders, BiConsumer outputConsumer) { - super(url, httpHeaders, HANDLED_RESPONSE_TYPES); - this.outputConsumer = outputConsumer; - } + public TextToSpeechRealtimeClient( + String url, Map httpHeaders, BiConsumer outputConsumer) { + super(url, httpHeaders, HANDLED_RESPONSE_TYPES); + this.outputConsumer = outputConsumer; + } - public void sendText(String text) { - var message = ConversationItemCreateEvent.builder() - .item(ConversationItem.ofRealtimeConversationItemUserMessage( - RealtimeConversationItemUserMessage.builder() - .addContent(RealtimeConversationItemUserMessage.Content.builder() - .text(text) - .type(RealtimeConversationItemUserMessage.Content.Type.INPUT_TEXT).build()).build() - )).build(); + public void sendText(String text) { + var message = + ConversationItemCreateEvent.builder() + .item( + ConversationItem.ofRealtimeConversationItemUserMessage( + RealtimeConversationItemUserMessage.builder() + .addContent( + RealtimeConversationItemUserMessage.Content.builder() + .text(text) + .type(RealtimeConversationItemUserMessage.Content.Type.INPUT_TEXT) + .build()) + .build())) + .build(); - super.sendMessage(message); - } + super.sendMessage(message); + } - @Override - protected String getSystemPrompt() { - return TASK; - } + @Override + protected String getSystemPrompt() { + return TASK; + } - @Override - protected List getOutputModalities() { - return OUTPUT_MODALITIES; - } + @Override + protected List getOutputModalities() { + return OUTPUT_MODALITIES; + } - @Override - protected void onResponse(String eventType, JsonNode event) { - if ("response.output_audio.delta".equals(eventType)) { - var base64Audio = event.get("delta").asText(); - byte[] audio = Base64.getDecoder().decode(base64Audio); - this.outputConsumer.accept(audio, Boolean.FALSE); - log.warn("Called consumer with base64 converted to bytes: {} (base64)", base64Audio); - } else if ("response.output_audio.done".equals(eventType)) { - this.outputConsumer.accept(EMPTY_BYTE_ARRAY, Boolean.TRUE); - } else { - log.warn("skipping message type: {}", eventType); - } + @Override + protected void onResponse(String eventType, JsonNode event) { + if ("response.output_audio.delta".equals(eventType)) { + var base64Audio = event.get("delta").asText(); + byte[] audio = Base64.getDecoder().decode(base64Audio); + this.outputConsumer.accept(audio, Boolean.FALSE); + log.warn("Called consumer with base64 converted to bytes: {} (base64)", base64Audio); + } else if ("response.output_audio.done".equals(eventType)) { + this.outputConsumer.accept(EMPTY_BYTE_ARRAY, Boolean.TRUE); + } else { + log.warn("skipping message type: {}", eventType); } + } } diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/WSSOpenAIRealtimeClient.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/WSSOpenAIRealtimeClient.java index 678bff93a..9415f1840 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/WSSOpenAIRealtimeClient.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/WSSOpenAIRealtimeClient.java @@ -1,5 +1,7 @@ package com.sap.ai.sdk.orchestration; +import static com.sap.ai.sdk.orchestration.OrchestrationJacksonConfiguration.getOrchestrationObjectMapper; + import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.PropertyAccessor; import com.fasterxml.jackson.core.JsonProcessingException; @@ -7,136 +9,138 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.openai.models.realtime.*; import com.openai.models.realtime.clientsecrets.ClientSecretCreateParams; -import lombok.AccessLevel; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; - import java.net.URI; import java.net.http.HttpClient; import java.net.http.WebSocket; import java.nio.ByteBuffer; import java.util.*; import java.util.concurrent.CompletableFuture; - -import static com.sap.ai.sdk.orchestration.OrchestrationJacksonConfiguration.getOrchestrationObjectMapper; +import lombok.AccessLevel; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; @Slf4j @RequiredArgsConstructor(access = AccessLevel.PACKAGE) public abstract class WSSOpenAIRealtimeClient implements AutoCloseable { - private static final int SUCCESS_FINISH_WSS_CODE = 1000; - private static final ObjectMapper JACKSON = getOrchestrationObjectMapper() - .setVisibility(PropertyAccessor.IS_GETTER, JsonAutoDetect.Visibility.NONE); - - /** - * WebSocket keeps TCP connection to the server. If connection idles, depending on the provider - * in cloud environments, cloud provider sends hard TCP RST (reset) after 60-300 seconds of inactivity, - * this is normally not configurable and would leave realtime conversation unusable on a pause. - *

- * In mobile networks this period is even shorter and typical CGNAT session can only idle for 10-15 seconds - * before it gets terminated - *

- * In realtime client we rely on explicit lifetime management (create/close), heartbeat mechanism is required - * to prevent unintended closing of connections while conversation is still expected to continue - */ - private static final int HEARTBEAT_INTERVAL_MILLIS = 4500; - private static final String HEARTBEAT_TIMER_NAME = "wss_realtime_heartbeat"; - - private final HttpClient client; - private final CompletableFuture ws; - private final Timer heartbeatTimer; - private final Set handleMessageTypes; - - public WSSOpenAIRealtimeClient(String url, Map httpHeaders, Set handleMessageTypes) { - this.client = HttpClient.newHttpClient(); - var wsBuilder = this.client.newWebSocketBuilder(); - for (Map.Entry entry : httpHeaders.entrySet()) { - wsBuilder = wsBuilder.header(entry.getKey(), entry.getValue()); - } - this.ws = wsBuilder.buildAsync(URI.create(url), new BufferedWebSocketListener(this::onSocketOpen, this::onText)); - this.handleMessageTypes = handleMessageTypes; - this.heartbeatTimer = new Timer(HEARTBEAT_TIMER_NAME,true); + private static final int SUCCESS_FINISH_WSS_CODE = 1000; + private static final ObjectMapper JACKSON = + getOrchestrationObjectMapper() + .setVisibility(PropertyAccessor.IS_GETTER, JsonAutoDetect.Visibility.NONE); + + /** + * WebSocket keeps TCP connection to the server. If connection idles, depending on the provider in + * cloud environments, cloud provider sends hard TCP RST (reset) after 60-300 seconds of + * inactivity, this is normally not configurable and would leave realtime conversation unusable on + * a pause. + * + *

In mobile networks this period is even shorter and typical CGNAT session can only idle for + * 10-15 seconds before it gets terminated + * + *

In realtime client we rely on explicit lifetime management (create/close), heartbeat + * mechanism is required to prevent unintended closing of connections while conversation is still + * expected to continue + */ + private static final int HEARTBEAT_INTERVAL_MILLIS = 4500; + + private static final String HEARTBEAT_TIMER_NAME = "wss_realtime_heartbeat"; + + private final HttpClient client; + private final CompletableFuture ws; + private final Timer heartbeatTimer; + private final Set handleMessageTypes; + + public WSSOpenAIRealtimeClient( + String url, Map httpHeaders, Set handleMessageTypes) { + this.client = HttpClient.newHttpClient(); + var wsBuilder = this.client.newWebSocketBuilder(); + for (Map.Entry entry : httpHeaders.entrySet()) { + wsBuilder = wsBuilder.header(entry.getKey(), entry.getValue()); } - - public void askForResponse() { - WebSocket ws = this.ws.join(); - synchronized (this) { - try { - ws.sendText(JACKSON.writeValueAsString(ResponseCreateEvent.builder().build()), true); - } catch (JsonProcessingException e) { - throw new RuntimeException("Failed to serialize ask for response", e); - } - ws.request(1); - } + this.ws = + wsBuilder.buildAsync( + URI.create(url), new BufferedWebSocketListener(this::onSocketOpen, this::onText)); + this.handleMessageTypes = handleMessageTypes; + this.heartbeatTimer = new Timer(HEARTBEAT_TIMER_NAME, true); + } + + public void askForResponse() { + WebSocket ws = this.ws.join(); + synchronized (this) { + try { + ws.sendText(JACKSON.writeValueAsString(ResponseCreateEvent.builder().build()), true); + } catch (JsonProcessingException e) { + throw new RuntimeException("Failed to serialize ask for response", e); + } + ws.request(1); } - - @Override - public void close() { - WebSocket ws = this.ws.join(); - synchronized (this) { - heartbeatTimer.cancel(); - try { - ws.sendClose(SUCCESS_FINISH_WSS_CODE, "done").join(); - } catch (Exception e) { - log.error("Error while closing WebSocket", e); - } - // this.client.close(); // exists only since java 21 - } + } + + @Override + public void close() { + WebSocket ws = this.ws.join(); + synchronized (this) { + heartbeatTimer.cancel(); + try { + ws.sendClose(SUCCESS_FINISH_WSS_CODE, "done").join(); + } catch (Exception e) { + log.error("Error while closing WebSocket", e); + } + // this.client.close(); // exists only since java 21 } + } - protected abstract String getSystemPrompt(); - - protected abstract List getOutputModalities(); + protected abstract String getSystemPrompt(); - protected abstract void onResponse(String eventType, JsonNode event); + protected abstract List getOutputModalities(); - protected void sendMessage(ConversationItemCreateEvent message) { - WebSocket ws = this.ws.join(); - synchronized (this) { - try { - ws.sendText(JACKSON.writeValueAsString(message), true); - ws.request(1); - log.warn("WSS sent message: {}", JACKSON.writeValueAsString(message)); - askForResponse(); - } catch (JsonProcessingException e) { - throw new OrchestrationClientException("Failed to serialize message", e); - } + protected abstract void onResponse(String eventType, JsonNode event); - } + protected void sendMessage(ConversationItemCreateEvent message) { + WebSocket ws = this.ws.join(); + synchronized (this) { + try { + ws.sendText(JACKSON.writeValueAsString(message), true); + ws.request(1); + log.warn("WSS sent message: {}", JACKSON.writeValueAsString(message)); + askForResponse(); + } catch (JsonProcessingException e) { + throw new OrchestrationClientException("Failed to serialize message", e); + } } - - private synchronized void onSocketOpen(WebSocket ws) { - configureSession(ws); - configureConversation(ws); - scheduleHeartbeat(ws); + } + + private synchronized void onSocketOpen(WebSocket ws) { + configureSession(ws); + configureConversation(ws); + scheduleHeartbeat(ws); + } + + private void onText(WebSocket webSocket, CharSequence data) { + log.warn("WSS received message: {}", data); + final JsonNode event; + try { + event = JACKSON.readTree(data.toString()); + } catch (JsonProcessingException e) { + throw new OrchestrationClientException("Error parsing JSON response from speech API", e); } - - private void onText(WebSocket webSocket, CharSequence data) { - log.warn("WSS received message: {}", data); - final JsonNode event; - try { - event = JACKSON.readTree(data.toString()); - } catch (JsonProcessingException e) { - throw new OrchestrationClientException( - "Error parsing JSON response from speech API", e); - } - var eventType = event.get("type").asText(); - if (handleMessageTypes.contains(eventType)) { - onResponse(eventType, event); - } - - webSocket.request(1); + var eventType = event.get("type").asText(); + if (handleMessageTypes.contains(eventType)) { + onResponse(eventType, event); } - private synchronized void sendPing(WebSocket ws) { - if (ws.isInputClosed()) { - return; - } - ws.sendPing(ByteBuffer.wrap("ping".getBytes())).join(); - ws.request(1); + webSocket.request(1); + } + + private synchronized void sendPing(WebSocket ws) { + if (ws.isInputClosed()) { + return; } + ws.sendPing(ByteBuffer.wrap("ping".getBytes())).join(); + ws.request(1); + } - private void configureSession(WebSocket ws) { + private void configureSession(WebSocket ws) { SessionUpdateEvent sue = SessionUpdateEvent.builder() .session( @@ -147,10 +151,12 @@ private void configureSession(WebSocket ws) { RealtimeAudioConfig.builder() .input( RealtimeAudioConfigInput.builder() - .turnDetection(Optional.empty()) + .turnDetection(Optional.empty()) .format( RealtimeAudioFormats.AudioPcm.builder() - .type(RealtimeAudioFormats.AudioPcm.Type.AUDIO_PCM) + .type( + RealtimeAudioFormats.AudioPcm.Type + .AUDIO_PCM) .build()) .build()) .output( @@ -163,43 +169,48 @@ private void configureSession(WebSocket ws) { .build()) .asRealtime()) .build(); - try { - ws.sendText(JACKSON.writeValueAsString(sue), true); - } catch (JsonProcessingException e) { - throw new OrchestrationClientException("Failed to serialize session request", e); - } - - ws.request(1); + try { + ws.sendText(JACKSON.writeValueAsString(sue), true); + } catch (JsonProcessingException e) { + throw new OrchestrationClientException("Failed to serialize session request", e); } - private void configureConversation(WebSocket ws) { - var systemPrompt = getSystemPrompt(); - var systemConversationItem = ConversationItemCreateEvent.builder() - .item(ConversationItem.ofRealtimeConversationItemSystemMessage( - RealtimeConversationItemSystemMessage.builder() - .addContent(RealtimeConversationItemSystemMessage.Content.builder() - .text(systemPrompt) - .type(RealtimeConversationItemSystemMessage.Content.Type.INPUT_TEXT).build()).build() - )).build(); - - try { - var json = JACKSON.writeValueAsString(systemConversationItem); - ws.sendText(json, true); - log.warn("WSS conversation config sent: {}", json); - } catch (JsonProcessingException e) { - throw new OrchestrationClientException("Failed to serialize message", e); - } - ws.request(1); - } - - private void scheduleHeartbeat(WebSocket ws) { - TimerTask task = new TimerTask(){ - @Override - public void run() { - sendPing(ws); - } + ws.request(1); + } + + private void configureConversation(WebSocket ws) { + var systemPrompt = getSystemPrompt(); + var systemConversationItem = + ConversationItemCreateEvent.builder() + .item( + ConversationItem.ofRealtimeConversationItemSystemMessage( + RealtimeConversationItemSystemMessage.builder() + .addContent( + RealtimeConversationItemSystemMessage.Content.builder() + .text(systemPrompt) + .type(RealtimeConversationItemSystemMessage.Content.Type.INPUT_TEXT) + .build()) + .build())) + .build(); - }; - heartbeatTimer.scheduleAtFixedRate(task, 0, HEARTBEAT_INTERVAL_MILLIS); + try { + var json = JACKSON.writeValueAsString(systemConversationItem); + ws.sendText(json, true); + log.warn("WSS conversation config sent: {}", json); + } catch (JsonProcessingException e) { + throw new OrchestrationClientException("Failed to serialize message", e); } + ws.request(1); + } + + private void scheduleHeartbeat(WebSocket ws) { + TimerTask task = + new TimerTask() { + @Override + public void run() { + sendPing(ws); + } + }; + heartbeatTimer.scheduleAtFixedRate(task, 0, HEARTBEAT_INTERVAL_MILLIS); + } } diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/Application.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/Application.java index aced5c87e..61b5be494 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/Application.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/Application.java @@ -5,10 +5,11 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; import com.fasterxml.jackson.annotation.PropertyAccessor; import com.fasterxml.jackson.databind.ObjectMapper; -import javax.annotation.Nonnull; - import com.sap.ai.sdk.app.services.OrchestrationService; -import com.sap.ai.sdk.core.AiCoreService; +import java.io.*; +import java.util.ArrayList; +import java.util.concurrent.atomic.AtomicBoolean; +import javax.annotation.Nonnull; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.web.servlet.ServletComponentScan; @@ -16,12 +17,6 @@ import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Primary; -import java.io.*; -import java.nio.file.Files; -import java.util.ArrayList; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.stream.Collectors; - /** Main class to start the Spring Boot application. */ @SpringBootApplication @ComponentScan({"com.sap.cloud.sdk", "com.sap.ai.sdk.app"}) @@ -51,13 +46,15 @@ public static void main(final String[] args) throws Exception { var svc = ctx.getBean(OrchestrationService.class); var outputBytes = new ArrayList(); var done = new AtomicBoolean(false); - try (var channel = svc.textToSpeech((bytes, last) -> { - System.err.println("appended output bytes: " + bytes.length); - outputBytes.add(bytes); - if (last) { - done.set(true); - } - })) { + try (var channel = + svc.textToSpeech( + (bytes, last) -> { + System.err.println("appended output bytes: " + bytes.length); + outputBytes.add(bytes); + if (last) { + done.set(true); + } + })) { System.err.println("BEFORE SENDING TEXT"); channel.sendText("Eat more of these fresh french baguettes and drink the tea!"); System.err.println("AFTER SENDING TEXT"); @@ -69,18 +66,23 @@ public static void main(final String[] args) throws Exception { } } - //var res = svc.textToSpeech("Eat more of these fresh french baguettes").toList(); - String path = System.getProperty("user.home") + File.separator + "Documents" + File.separator + System.currentTimeMillis() + ".pcm"; + // var res = svc.textToSpeech("Eat more of these fresh french baguettes").toList(); + String path = + System.getProperty("user.home") + + File.separator + + "Documents" + + File.separator + + System.currentTimeMillis() + + ".pcm"; File file = new File(path); try (FileOutputStream in = new FileOutputStream(file)) { var ttl = 0; - for (byte[] chunk: outputBytes) { + for (byte[] chunk : outputBytes) { System.out.println("Writing bytes: " + chunk.length); in.write(chunk); ttl += chunk.length; } System.out.println("Written bytes: " + ttl); } - } } diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OrchestrationService.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OrchestrationService.java index 49fbaa594..8a26d382b 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OrchestrationService.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OrchestrationService.java @@ -67,8 +67,9 @@ public OrchestrationChatResponse completion(@Nonnull final String famousPhrase) /** * Voices input text into output audio * - * @param outputConsumer byte chunks output consumer in PCM16 format, 16-bit, mono, 24000 Hz, little-endian, - * boolean flag signifies end of the current reply (only true for last chunk of current response) + * @param outputConsumer byte chunks output consumer in PCM16 format, 16-bit, mono, 24000 Hz, + * little-endian, boolean flag signifies end of the current reply (only true for last chunk of + * current response) * @return text input channel to supply text to voice, should be closed when not needed anymore */ public TextInputChannel textToSpeech(@Nonnull final BiConsumer outputConsumer) { From 54b06a55a28df501253efb0050e5608e91c3e3a2 Mon Sep 17 00:00:00 2001 From: Vladimir Aseev Date: Fri, 10 Jul 2026 14:56:26 +0200 Subject: [PATCH 06/26] add speech to speech realtime support and text to speech example --- foundation-models/openai/pom.xml | 4 + .../openai/AudioInputChannel.java | 17 +++ .../openai/AudioOutputChannel.java | 22 ++++ .../openai}/BufferedWebSocketListener.java | 17 ++- .../foundationmodels/openai/OpenAiClient.java | 73 ++++++++++++ .../openai/SpeechToSpeechRealtimeClient.java | 103 +++++++++++++++++ .../openai/SpeechToTextRealtimeClient.java | 93 +++++++++++++++ .../openai}/TextInputChannel.java | 2 +- .../openai/TextOutputChannel.java | 20 ++++ .../openai/TextToSpeechRealtimeClient.java | 100 ++++++++++++++++ .../openai}/WSSOpenAIRealtimeClient.java | 58 +++------- orchestration/pom.xml | 5 - .../orchestration/OrchestrationClient.java | 46 -------- .../orchestration/RealtimeChannelFactory.java | 53 --------- .../TextToSpeechRealtimeClient.java | 69 ----------- pom.xml | 6 + sample-code/spring-app/pom.xml | 4 + .../java/com/sap/ai/sdk/app/Application.java | 46 +------- .../com/sap/ai/sdk/app/WebsocketConfig.java | 23 ++++ .../TextToSpeechWebsocketHandler.java | 61 ++++++++++ .../ai/sdk/app/services/OpenAiService.java | 48 ++++++-- .../app/services/OrchestrationService.java | 12 -- .../src/main/resources/static/index.html | 107 ++++++++++++++++++ 23 files changed, 707 insertions(+), 282 deletions(-) create mode 100644 foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/AudioInputChannel.java create mode 100644 foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/AudioOutputChannel.java rename {orchestration/src/main/java/com/sap/ai/sdk/orchestration => foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai}/BufferedWebSocketListener.java (69%) create mode 100644 foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java create mode 100644 foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToTextRealtimeClient.java rename {orchestration/src/main/java/com/sap/ai/sdk/orchestration => foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai}/TextInputChannel.java (65%) create mode 100644 foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextOutputChannel.java create mode 100644 foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java rename {orchestration/src/main/java/com/sap/ai/sdk/orchestration => foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai}/WSSOpenAIRealtimeClient.java (70%) delete mode 100644 orchestration/src/main/java/com/sap/ai/sdk/orchestration/RealtimeChannelFactory.java delete mode 100644 orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextToSpeechRealtimeClient.java create mode 100644 sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java create mode 100644 sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java diff --git a/foundation-models/openai/pom.xml b/foundation-models/openai/pom.xml index 4e47adcd3..81ea4024f 100644 --- a/foundation-models/openai/pom.xml +++ b/foundation-models/openai/pom.xml @@ -66,6 +66,10 @@ com.sap.ai.sdk core + + com.openai + openai-java + com.google.code.findbugs jsr305 diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/AudioInputChannel.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/AudioInputChannel.java new file mode 100644 index 000000000..9b4862802 --- /dev/null +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/AudioInputChannel.java @@ -0,0 +1,17 @@ +package com.sap.ai.sdk.foundationmodels.openai; + +/** + * Functional interface representing audio input channel (audio data consumer) + *

+ * Should be closed by application (try-with-resources) when not needed anymore + */ +public interface AudioInputChannel extends AutoCloseable { + + /** + * This method is sequentially invoked by audio data provider to supply implementer (consumer) + * with the audio data. Exact audio format (encoding, sampling rate, etc.) depends on the usage context + * + * @param rawBytesChunk binary data in the depending on the use case format + */ + void inputAudio(byte[] rawBytesChunk); +} diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/AudioOutputChannel.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/AudioOutputChannel.java new file mode 100644 index 000000000..e62e3c4cd --- /dev/null +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/AudioOutputChannel.java @@ -0,0 +1,22 @@ +package com.sap.ai.sdk.foundationmodels.openai; + +import java.io.IOException; + +/** + * Functional interface representing audio output channel (audio data consumer) + *

+ * Should be closed by application (try-with-resources) when not needed anymore + */ +public interface AudioOutputChannel { + + /** + * This method is sequentially invoked by audio data provider to supply implementer (consumer) + * with the audio data. Exact audio format (encoding, sampling rate, etc.) depends on the usage context + * + * @param rawBytesChunk binary data in the depending on the use case format + * @param isLast true if this call logically concludes previous and this passed bytes data into a + * single logical entity (e.g. gets called at the end when all byte parts of a single + * message get passed) + */ + void outputAudio(byte[] rawBytesChunk, boolean isLast); +} diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/BufferedWebSocketListener.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/BufferedWebSocketListener.java similarity index 69% rename from orchestration/src/main/java/com/sap/ai/sdk/orchestration/BufferedWebSocketListener.java rename to foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/BufferedWebSocketListener.java index 118b2d5ce..8377a2889 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/BufferedWebSocketListener.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/BufferedWebSocketListener.java @@ -1,11 +1,15 @@ -package com.sap.ai.sdk.orchestration; +package com.sap.ai.sdk.foundationmodels.openai; + +import lombok.extern.slf4j.Slf4j; import java.net.http.WebSocket; +import java.nio.ByteBuffer; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; import java.util.function.BiConsumer; import java.util.function.Consumer; +@Slf4j public class BufferedWebSocketListener implements WebSocket.Listener { private final Consumer onOpen; @@ -36,4 +40,15 @@ public CompletionStage onText(WebSocket webSocket, CharSequence data, boolean return CompletableFuture.completedStage(null); } + + @Override + public void onError(WebSocket webSocket, Throwable error) { + log.error("Websocket error occurred during realtime communication", error); + } + + @Override + public CompletionStage onBinary(WebSocket webSocket, ByteBuffer data, boolean last) { + log.warn("Received unexpected binary bytes for WebSocket connection"); + return CompletableFuture.completedStage(null); + } } diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java index ce981fa2a..72e162e6a 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java @@ -33,6 +33,7 @@ import com.sap.cloud.sdk.cloudplatform.connectivity.HttpDestination; import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.stream.Stream; import javax.annotation.Nonnull; @@ -50,6 +51,8 @@ @RequiredArgsConstructor(access = AccessLevel.PRIVATE) public final class OpenAiClient { private static final String DEFAULT_API_VERSION = "2024-02-01"; + private static final int PATH_BUFFER_SIZE = 400; // existing URLs are ~120 symbols long, 400 has reasonable margin + private static final ObjectMapper JACKSON = getOpenAiObjectMapper(); @@ -260,6 +263,76 @@ public Stream streamChatCompletion(@Nonnull final String prompt) .map(OpenAiChatCompletionDelta::getDeltaContent); } + /** + * Creates realtime channel allowing to input text and voice it (receive audio output) + * + *

The input channel should be used with a try-with-resources block to ensure that the underlying + * connection is closed. + * + *

Example: + * + * 

{@code
+   * try (var textInputChannel = client.textToSpeech(audioOutputConsumer)) {
+   *       textInputChannel.sendText("...");
+   *       ....
+   * }
+   * }
+ * + * This API implements full duplex (input + output) communication channels. Application should logically + * synchronize their state and close input channel when it is appropriate (e.g. last part of the response + * has been received via output channel and application does not need to send any other input). When input + * channel is closed, output channel will be closed automatically and output consumer will not be called + * anymore. + * + * @param audioOutputConsumer - audio consumer of raw PCM mono 24000 Hz little endian output + * @return input channel, allowing for text input + */ + @Nonnull + public TextInputChannel textToSpeech(@Nonnull final AudioOutputChannel audioOutputConsumer) { + var extraHeaders = destination.asHttp().getHeaders(); + var headers = new HashMap(extraHeaders.size() + 1); + for (Header header : extraHeaders) { + headers.put(header.getName(), header.getValue()); + } + var endpoint = getRealtimeEndpoint(); + + return new TextToSpeechRealtimeClient(endpoint, headers, audioOutputConsumer); + } + + public AudioInputChannel speechToText(@Nonnull final TextOutputChannel textOutputConsumer) { + var extraHeaders = destination.asHttp().getHeaders(); + var headers = new HashMap(extraHeaders.size() + 1); + for (Header header : extraHeaders) { + headers.put(header.getName(), header.getValue()); + } + var endpoint = getRealtimeEndpoint() + "?intent=transcription"; + + return new SpeechToTextRealtimeClient(endpoint, headers, textOutputConsumer); + } + + public AudioInputChannel speechToSpeech(@Nonnull final AudioOutputChannel audioOutputChannel) { + var extraHeaders = destination.asHttp().getHeaders(); + var headers = new HashMap(extraHeaders.size() + 1); + for (Header header : extraHeaders) { + headers.put(header.getName(), header.getValue()); + } + var endpoint = getRealtimeEndpoint(); + + return new SpeechToSpeechRealtimeClient(endpoint, headers, audioOutputChannel); + } + + private String getRealtimeEndpoint() { + var sb = new StringBuilder(PATH_BUFFER_SIZE); + sb.append("wss://"); + var pathParts = destination.asHttp().getUri().toString().split("//"); + if (pathParts.length != 2) { + throw new IllegalArgumentException("Invalid destination URI: " + destination.asHttp().getUri()); + } + sb.append(pathParts[1].replaceFirst("^api\\.", "realtime.")); + sb.append("v1/realtime"); + return sb.toString(); + } + private static void throwOnContentFilter(@Nonnull final OpenAiChatCompletionDelta delta) { final String finishReason = delta.getFinishReason(); if (finishReason != null && finishReason.equals("content_filter")) { diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java new file mode 100644 index 000000000..3addfabfe --- /dev/null +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java @@ -0,0 +1,103 @@ +package com.sap.ai.sdk.foundationmodels.openai; + +import com.fasterxml.jackson.databind.JsonNode; +import com.openai.models.realtime.*; +import com.openai.models.realtime.clientsecrets.ClientSecretCreateParams; +import java.util.*; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class SpeechToSpeechRealtimeClient extends WSSOpenAIRealtimeClient implements AudioInputChannel { + + private static final int MAX_DATA_CHUNK_SIZE_BYTES = 8192; + private static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; + + private static final Set HANDLED_RESPONSE_TYPES = Set.of( + "response.output_audio.delta", "response.output_audio.done" + ); + + private static final List OUTPUT_MODALITIES = List.of( + RealtimeSessionCreateRequest.OutputModality.AUDIO + ); + + private static final String TASK = ""; + + + private final AudioOutputChannel outputConsumer; + + public SpeechToSpeechRealtimeClient(String url, Map httpHeaders, AudioOutputChannel outputConsumer) { + super(url, httpHeaders, HANDLED_RESPONSE_TYPES); + this.outputConsumer = outputConsumer; + } + + public void inputAudio(byte[] rawAudioChunk) { + if (rawAudioChunk.length == 0) { + return; + } + var cursorLeft = 0; + while (cursorLeft < rawAudioChunk.length) { + var cursorRight = Math.min(cursorLeft + MAX_DATA_CHUNK_SIZE_BYTES, rawAudioChunk.length); + var part = Arrays.copyOfRange(rawAudioChunk, cursorLeft, cursorRight); + var audioInputMessage = InputAudioBufferAppendEvent.builder() + .audio(Base64.getEncoder().encodeToString(part)).build(); + super.sendMessage(audioInputMessage); + cursorLeft += MAX_DATA_CHUNK_SIZE_BYTES; + } + + var commitAudioMessage = InputAudioBufferCommitEvent.builder().build(); + super.sendMessage(commitAudioMessage); + askForResponse(); + } + + @Override + protected String getSystemPrompt() { + return TASK; + } + + @Override + protected void onResponse(String eventType, JsonNode event) { + if ("response.output_audio.delta".equals(eventType)) { + var base64Audio = event.get("delta").asText(); + byte[] audio = Base64.getDecoder().decode(base64Audio); + this.outputConsumer.outputAudio(audio, Boolean.FALSE); + } else if ("response.output_audio.done".equals(eventType)) { + this.outputConsumer.outputAudio(EMPTY_BYTE_ARRAY, Boolean.TRUE); + } + } + + @Override + protected SessionUpdateEvent sessionConfiguration() { + SessionUpdateEvent sessionUpdateEvent = SessionUpdateEvent.builder() + .session( + ClientSecretCreateParams.Session.ofRealtime( + RealtimeSessionCreateRequest.builder() + .outputModalities(OUTPUT_MODALITIES) + .audio( + RealtimeAudioConfig.builder() + .input( + RealtimeAudioConfigInput.builder() + .turnDetection(Optional.empty()) + .format( + RealtimeAudioFormats.AudioPcm + .builder() + .type(RealtimeAudioFormats.AudioPcm.Type.AUDIO_PCM) + .rate(RealtimeAudioFormats.AudioPcm.Rate._24000) + .build()) + .build()) + .output( + RealtimeAudioConfigOutput.builder() + .format(RealtimeAudioFormats.AudioPcm + .builder() + .type(RealtimeAudioFormats.AudioPcm.Type.AUDIO_PCM) + .rate(RealtimeAudioFormats.AudioPcm.Rate._24000) + .build()) + .voice( + RealtimeAudioConfigOutput.Voice.UnionMember1.ALLOY) + .build()) + .build()) + .build()) + .asRealtime()) + .build(); + return sessionUpdateEvent; + } +} diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToTextRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToTextRealtimeClient.java new file mode 100644 index 000000000..6e56866fb --- /dev/null +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToTextRealtimeClient.java @@ -0,0 +1,93 @@ +package com.sap.ai.sdk.foundationmodels.openai; + +import com.fasterxml.jackson.databind.JsonNode; +import com.openai.models.realtime.*; +import java.util.*; + +import com.openai.models.realtime.clientsecrets.ClientSecretCreateParams; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class SpeechToTextRealtimeClient extends WSSOpenAIRealtimeClient implements AudioInputChannel { + + private static final int MAX_DATA_CHUNK_SIZE_BYTES = 8192; + private static final Set HANDLED_RESPONSE_TYPES = Set.of( + "response.output_audio_transcript.done" + ); + + private static final String TASK = + "you are a transcript writer and your role is to convert input speech to text as precise as possible (word to word exactly, if possible)"; + + + private final TextOutputChannel outputConsumer; + + public SpeechToTextRealtimeClient(String url, Map httpHeaders, TextOutputChannel outputConsumer) { + super(url, httpHeaders, HANDLED_RESPONSE_TYPES); + this.outputConsumer = outputConsumer; + } + + public void inputAudio(byte[] rawAudioChunk){ + if (rawAudioChunk.length == 0) { + return; + } + var cursorLeft = 0; + while (cursorLeft < rawAudioChunk.length) { + var cursorRight = Math.min(cursorLeft + MAX_DATA_CHUNK_SIZE_BYTES, rawAudioChunk.length); + var part = Arrays.copyOfRange(rawAudioChunk, cursorLeft, cursorRight); + var audioInputMessage = InputAudioBufferAppendEvent.builder() + .audio(Base64.getEncoder().encodeToString(part)).build(); + super.sendMessage(audioInputMessage); + cursorLeft += MAX_DATA_CHUNK_SIZE_BYTES; + } + + var commitAudioMessage = InputAudioBufferCommitEvent.builder().build(); + super.sendMessage(commitAudioMessage); + askForResponse(); + } + + @Override + protected String getSystemPrompt() { + return TASK; + } + + @Override + protected void onResponse(String eventType, JsonNode event) { + if ("response.output_audio_transcript.done".equals(eventType)) { + this.outputConsumer + .outputText(event.has("transcript") ? event.get("transcript").asText() : "", Boolean.TRUE); + } else { + log.trace("skipping message type: {}", eventType); + } + } + + @Override + protected SessionUpdateEvent sessionConfiguration() { + SessionUpdateEvent sessionUpdateEvent = + SessionUpdateEvent.builder() + .session( + ClientSecretCreateParams.Session.ofTranscription( + RealtimeTranscriptionSessionCreateRequest.builder() + .audio( + RealtimeTranscriptionSessionAudio.builder() + .input( + RealtimeTranscriptionSessionAudioInput.builder() + .turnDetection(Optional.empty()) + .format( + RealtimeAudioFormats.AudioPcm.builder() + .type( + RealtimeAudioFormats.AudioPcm.Type + .AUDIO_PCM) + .rate(RealtimeAudioFormats.AudioPcm.Rate._24000) + .build()) + .transcription( + AudioTranscription.builder() + .model(AudioTranscription.Model.GPT_4O_TRANSCRIBE) + .build()) + .build()) + .build()) + .build()) + .asTranscription()) + .build(); + return sessionUpdateEvent; + } +} diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextInputChannel.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextInputChannel.java similarity index 65% rename from orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextInputChannel.java rename to foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextInputChannel.java index 20e6fcc66..8e2372024 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextInputChannel.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextInputChannel.java @@ -1,4 +1,4 @@ -package com.sap.ai.sdk.orchestration; +package com.sap.ai.sdk.foundationmodels.openai; public interface TextInputChannel extends AutoCloseable { void sendText(String text); diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextOutputChannel.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextOutputChannel.java new file mode 100644 index 000000000..1c0425bb4 --- /dev/null +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextOutputChannel.java @@ -0,0 +1,20 @@ +package com.sap.ai.sdk.foundationmodels.openai; + +/** + * Functional interface representing test output channel (text data consumer) + *

+ * Should be closed by application (try-with-resources) when not needed anymore + */ +public interface TextOutputChannel { + + /** + * This method is sequentially invoked by test data provider to supply implementer (consumer) + * with the test data. + * + * @param textChunk chunk of the text (possibly partial content) + * @param isLast true if this call logically concludes previous and this passed text data into a + * single logical entity (e.g. gets called at the end when all parts of a single + * message get passed) + */ + void outputText(CharSequence textChunk, boolean isLast); +} diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java new file mode 100644 index 000000000..5e42e5ac4 --- /dev/null +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java @@ -0,0 +1,100 @@ +package com.sap.ai.sdk.foundationmodels.openai; + +import com.fasterxml.jackson.databind.JsonNode; +import com.openai.models.realtime.*; +import com.openai.models.realtime.clientsecrets.ClientSecretCreateParams; +import lombok.extern.slf4j.Slf4j; + +import java.util.*; + +@Slf4j +public class TextToSpeechRealtimeClient extends WSSOpenAIRealtimeClient implements TextInputChannel { + + private static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; + + private static final Set HANDLED_RESPONSE_TYPES = Set.of( + "response.output_audio.delta", "response.output_audio.done" + ); + + private static final List OUTPUT_MODALITIES = List.of( + RealtimeSessionCreateRequest.OutputModality.AUDIO + ); + + private static final String TASK = + "you are a speaker and your role is to read (produce audio) of the user input speech. voice user text input"; + + + private final AudioOutputChannel outputConsumer; + + public TextToSpeechRealtimeClient(String url, Map httpHeaders, AudioOutputChannel outputConsumer) { + super(url, httpHeaders, HANDLED_RESPONSE_TYPES); + this.outputConsumer = outputConsumer; + } + + public void sendText(String text) { + var message = ConversationItemCreateEvent.builder() + .item(ConversationItem.ofRealtimeConversationItemUserMessage( + RealtimeConversationItemUserMessage.builder() + .addContent(RealtimeConversationItemUserMessage.Content.builder() + .text(text) + .type(RealtimeConversationItemUserMessage.Content.Type.INPUT_TEXT).build()).build() + )).build(); + + super.sendMessage(message); + askForResponse(); + } + + @Override + protected String getSystemPrompt() { + return TASK; + } + + @Override + protected void onResponse(String eventType, JsonNode event) { + if ("response.output_audio.delta".equals(eventType)) { + var base64Audio = event.get("delta").asText(); + byte[] audio = Base64.getDecoder().decode(base64Audio); + this.outputConsumer.outputAudio(audio, Boolean.FALSE); + } else if ("response.output_audio.done".equals(eventType)) { + this.outputConsumer.outputAudio(EMPTY_BYTE_ARRAY, Boolean.TRUE); + } else { + log.warn("skipping message type: {}", eventType); + } + } + + @Override + protected SessionUpdateEvent sessionConfiguration() { + SessionUpdateEvent sessionUpdateEvent = SessionUpdateEvent.builder() + .session( + ClientSecretCreateParams.Session.ofRealtime( + RealtimeSessionCreateRequest.builder() + .outputModalities(OUTPUT_MODALITIES) + .audio( + RealtimeAudioConfig.builder() + .input( + RealtimeAudioConfigInput.builder() + .turnDetection(Optional.empty()) + .format( + RealtimeAudioFormats.AudioPcm + .builder() + .type(RealtimeAudioFormats.AudioPcm.Type.AUDIO_PCM) + .rate(RealtimeAudioFormats.AudioPcm.Rate._24000) + .build()) + .build()) + .output( + RealtimeAudioConfigOutput.builder() + .format(RealtimeAudioFormats.AudioPcm + .builder() + .type(RealtimeAudioFormats.AudioPcm.Type.AUDIO_PCM) + .rate(RealtimeAudioFormats.AudioPcm.Rate._24000) + .build()) + .voice( + RealtimeAudioConfigOutput.Voice.UnionMember1.ALLOY) + .build()) + .build()) + .build()) + .asRealtime()) + .build(); + return sessionUpdateEvent; + } +} diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/WSSOpenAIRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSSOpenAIRealtimeClient.java similarity index 70% rename from orchestration/src/main/java/com/sap/ai/sdk/orchestration/WSSOpenAIRealtimeClient.java rename to foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSSOpenAIRealtimeClient.java index 678bff93a..77e2d881b 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/WSSOpenAIRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSSOpenAIRealtimeClient.java @@ -1,4 +1,4 @@ -package com.sap.ai.sdk.orchestration; +package com.sap.ai.sdk.foundationmodels.openai; import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.PropertyAccessor; @@ -7,6 +7,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.openai.models.realtime.*; import com.openai.models.realtime.clientsecrets.ClientSecretCreateParams; +import com.sap.ai.sdk.core.common.ClientException; import lombok.AccessLevel; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -18,14 +19,12 @@ import java.util.*; import java.util.concurrent.CompletableFuture; -import static com.sap.ai.sdk.orchestration.OrchestrationJacksonConfiguration.getOrchestrationObjectMapper; - @Slf4j @RequiredArgsConstructor(access = AccessLevel.PACKAGE) public abstract class WSSOpenAIRealtimeClient implements AutoCloseable { private static final int SUCCESS_FINISH_WSS_CODE = 1000; - private static final ObjectMapper JACKSON = getOrchestrationObjectMapper() + private static final ObjectMapper JACKSON = new ObjectMapper() .setVisibility(PropertyAccessor.IS_GETTER, JsonAutoDetect.Visibility.NONE); /** @@ -86,20 +85,18 @@ public void close() { protected abstract String getSystemPrompt(); - protected abstract List getOutputModalities(); - protected abstract void onResponse(String eventType, JsonNode event); - protected void sendMessage(ConversationItemCreateEvent message) { + protected abstract SessionUpdateEvent sessionConfiguration(); + + protected void sendMessage(Object message) { WebSocket ws = this.ws.join(); synchronized (this) { try { ws.sendText(JACKSON.writeValueAsString(message), true); ws.request(1); - log.warn("WSS sent message: {}", JACKSON.writeValueAsString(message)); - askForResponse(); } catch (JsonProcessingException e) { - throw new OrchestrationClientException("Failed to serialize message", e); + throw new ClientException("Failed to serialize message", e); } } @@ -112,17 +109,17 @@ private synchronized void onSocketOpen(WebSocket ws) { } private void onText(WebSocket webSocket, CharSequence data) { - log.warn("WSS received message: {}", data); final JsonNode event; try { event = JACKSON.readTree(data.toString()); } catch (JsonProcessingException e) { - throw new OrchestrationClientException( - "Error parsing JSON response from speech API", e); + throw new ClientException("Error parsing JSON response from speech API", e); } var eventType = event.get("type").asText(); if (handleMessageTypes.contains(eventType)) { onResponse(eventType, event); + } else { + log.trace("Unhandled event type: {}", eventType); } webSocket.request(1); @@ -137,36 +134,11 @@ private synchronized void sendPing(WebSocket ws) { } private void configureSession(WebSocket ws) { - SessionUpdateEvent sue = - SessionUpdateEvent.builder() - .session( - ClientSecretCreateParams.Session.ofRealtime( - RealtimeSessionCreateRequest.builder() - .outputModalities(getOutputModalities()) - .audio( - RealtimeAudioConfig.builder() - .input( - RealtimeAudioConfigInput.builder() - .turnDetection(Optional.empty()) - .format( - RealtimeAudioFormats.AudioPcm.builder() - .type(RealtimeAudioFormats.AudioPcm.Type.AUDIO_PCM) - .build()) - .build()) - .output( - RealtimeAudioConfigOutput.builder() - .format(RealtimeAudioFormats.AudioPcm.builder().build()) - .voice( - RealtimeAudioConfigOutput.Voice.UnionMember1.ALLOY) - .build()) - .build()) - .build()) - .asRealtime()) - .build(); + SessionUpdateEvent sue = sessionConfiguration(); try { ws.sendText(JACKSON.writeValueAsString(sue), true); } catch (JsonProcessingException e) { - throw new OrchestrationClientException("Failed to serialize session request", e); + throw new ClientException("Failed to serialize session request", e); } ws.request(1); @@ -174,6 +146,9 @@ private void configureSession(WebSocket ws) { private void configureConversation(WebSocket ws) { var systemPrompt = getSystemPrompt(); + if (systemPrompt == null || systemPrompt.isEmpty()) { + return; + } var systemConversationItem = ConversationItemCreateEvent.builder() .item(ConversationItem.ofRealtimeConversationItemSystemMessage( RealtimeConversationItemSystemMessage.builder() @@ -185,9 +160,8 @@ private void configureConversation(WebSocket ws) { try { var json = JACKSON.writeValueAsString(systemConversationItem); ws.sendText(json, true); - log.warn("WSS conversation config sent: {}", json); } catch (JsonProcessingException e) { - throw new OrchestrationClientException("Failed to serialize message", e); + throw new ClientException("Failed to serialize message", e); } ws.request(1); } diff --git a/orchestration/pom.xml b/orchestration/pom.xml index a1ef764f1..f8027fe4a 100644 --- a/orchestration/pom.xml +++ b/orchestration/pom.xml @@ -122,11 +122,6 @@ com.fasterxml.jackson.dataformat jackson-dataformat-yaml - - com.openai - openai-java - 4.41.0 - org.projectlombok diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationClient.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationClient.java index 7c52d772b..78f347b56 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationClient.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationClient.java @@ -7,12 +7,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.annotations.Beta; -import com.openai.client.OpenAIClient; -import com.openai.client.okhttp.OpenAIOkHttpClient; -import com.openai.models.beta.realtime.sessions.Session; -import com.openai.models.realtime.SessionUpdateEvent; import com.sap.ai.sdk.core.AiCoreService; -import com.sap.ai.sdk.core.AiModel; import com.sap.ai.sdk.orchestration.model.CompletionPostRequest; import com.sap.ai.sdk.orchestration.model.CompletionPostResponse; import com.sap.ai.sdk.orchestration.model.CompletionRequestConfiguration; @@ -22,30 +17,15 @@ import com.sap.ai.sdk.orchestration.model.OrchestrationConfig; import com.sap.cloud.sdk.cloudplatform.connectivity.Header; import com.sap.cloud.sdk.cloudplatform.connectivity.HttpDestination; -import io.vavr.collection.Array; import io.vavr.control.Try; -import java.io.ByteArrayOutputStream; -import java.net.URI; -import java.net.http.HttpClient; -import java.net.http.WebSocket; -import java.nio.ByteBuffer; import java.util.*; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionStage; -import java.util.function.BiConsumer; -import java.util.function.Consumer; import java.util.function.Supplier; import java.util.stream.Stream; -import java.util.stream.StreamSupport; import javax.annotation.Nonnull; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import lombok.val; -import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; -import org.apache.hc.client5.http.impl.classic.HttpClients; -import org.jspecify.annotations.NonNull; -import org.jspecify.annotations.Nullable; /** Client to execute requests to the orchestration service. */ @Slf4j @@ -56,7 +36,6 @@ public class OrchestrationClient { static final ObjectMapper JACKSON = getOrchestrationObjectMapper(); - private final OrchestrationHttpExecutor executor; private final List

customHeaders = new ArrayList<>(); @@ -123,31 +102,6 @@ public OrchestrationChatResponse chatCompletion( return new OrchestrationChatResponse(response); } - /** - * Voices input text into output audio - * - * @param output byte chunks output consumer in PCM16 format, 16-bit, mono, 24000 Hz, little-endian, - * boolean flag signifies end of the current reply (only true for last chunk of current response) - * @return text input channel to supply text to voice, should be closed when not needed anymore - */ - public TextInputChannel textToSpeech(BiConsumer output) { - var destination = new AiCoreService().getInferenceDestination().forModel(new AiModel() { - @Override - public @NonNull String name() { - return OrchestrationAiModel.GPT_REALTIME.getName(); - } - - @Override - public @Nullable String version() { - return OrchestrationAiModel.GPT_REALTIME.getVersion(); - } - }); - - var factory = new RealtimeChannelFactory(() -> destination); - - return factory.textToSpeech(output); - } - /** * Generate a completion for the given prompt. * diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/RealtimeChannelFactory.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/RealtimeChannelFactory.java deleted file mode 100644 index 28edd05c4..000000000 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/RealtimeChannelFactory.java +++ /dev/null @@ -1,53 +0,0 @@ -package com.sap.ai.sdk.orchestration; - -import com.sap.cloud.sdk.cloudplatform.connectivity.Header; -import com.sap.cloud.sdk.cloudplatform.connectivity.HttpDestination; -import lombok.AccessLevel; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; - -import javax.annotation.Nonnull; -import java.util.*; -import java.util.function.BiConsumer; -import java.util.function.Supplier; - -@Slf4j -@RequiredArgsConstructor(access = AccessLevel.PACKAGE) -public class RealtimeChannelFactory { - private static final String AI_RESOURCE_GROUP_HEADER_NAME = "AI-Resource-Group"; - private static final String AI_RESOURCE_GROUP_HEADER_VALUE = "default"; - private static final int PATH_BUFFER_SIZE = 400; // existing URLs are ~120 symbols long, 400 has reasonable margin - - private final Supplier destinationSupplier; - - /** - * Creates realtime channel allowing to input text and voice it (receive audio output) - * - * @param audioOutputConsumer - audio bytes consumer, where byte[] - chunks of raw PCM mono 24000 Hz output and - * Boolean - marker if current audio reply has been finished (current response ended) - * @return input channel, allowing for text input - */ - public TextInputChannel textToSpeech(@Nonnull final BiConsumer audioOutputConsumer) { - var destination = destinationSupplier.get(); - - var extraHeaders = destination.getHeaders(); - var headers = new HashMap(extraHeaders.size() + 1); - headers.put(AI_RESOURCE_GROUP_HEADER_NAME, AI_RESOURCE_GROUP_HEADER_VALUE); - for (Header header : extraHeaders) { - headers.put(header.getName(), header.getValue()); - } - - var sb = new StringBuilder(PATH_BUFFER_SIZE); - sb.append("wss://"); - var pathParts = destination.getUri().toString().split("//"); - if (pathParts.length != 2) { - throw new IllegalArgumentException("Invalid destination URI: " + destination.getUri()); - } - sb.append(pathParts[1].replaceFirst("^api\\.", "realtime.")); - sb.append("v1/realtime"); - var endpoint = sb.toString(); - - return new TextToSpeechRealtimeClient(endpoint, headers, audioOutputConsumer); - } - -} diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextToSpeechRealtimeClient.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextToSpeechRealtimeClient.java deleted file mode 100644 index 45ea9335a..000000000 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextToSpeechRealtimeClient.java +++ /dev/null @@ -1,69 +0,0 @@ -package com.sap.ai.sdk.orchestration; - -import com.fasterxml.jackson.databind.JsonNode; -import com.openai.models.realtime.*; -import lombok.extern.slf4j.Slf4j; - -import java.util.*; -import java.util.function.BiConsumer; - -@Slf4j -public class TextToSpeechRealtimeClient extends WSSOpenAIRealtimeClient implements TextInputChannel { - - private static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; - - private static final Set HANDLED_RESPONSE_TYPES = Set.of( - "response.output_audio.delta", "response.output_audio.done" - ); - - private static final List OUTPUT_MODALITIES = List.of( - RealtimeSessionCreateRequest.OutputModality.AUDIO - ); - - private static final String TASK = - "you are a speaker and your role is to read (produce audio) of the user input speech. voice user text input"; - - - private final BiConsumer outputConsumer; - - public TextToSpeechRealtimeClient(String url, Map httpHeaders, BiConsumer outputConsumer) { - super(url, httpHeaders, HANDLED_RESPONSE_TYPES); - this.outputConsumer = outputConsumer; - } - - public void sendText(String text) { - var message = ConversationItemCreateEvent.builder() - .item(ConversationItem.ofRealtimeConversationItemUserMessage( - RealtimeConversationItemUserMessage.builder() - .addContent(RealtimeConversationItemUserMessage.Content.builder() - .text(text) - .type(RealtimeConversationItemUserMessage.Content.Type.INPUT_TEXT).build()).build() - )).build(); - - super.sendMessage(message); - } - - @Override - protected String getSystemPrompt() { - return TASK; - } - - @Override - protected List getOutputModalities() { - return OUTPUT_MODALITIES; - } - - @Override - protected void onResponse(String eventType, JsonNode event) { - if ("response.output_audio.delta".equals(eventType)) { - var base64Audio = event.get("delta").asText(); - byte[] audio = Base64.getDecoder().decode(base64Audio); - this.outputConsumer.accept(audio, Boolean.FALSE); - log.warn("Called consumer with base64 converted to bytes: {} (base64)", base64Audio); - } else if ("response.output_audio.done".equals(eventType)) { - this.outputConsumer.accept(EMPTY_BYTE_ARRAY, Boolean.TRUE); - } else { - log.warn("skipping message type: {}", eventType); - } - } -} diff --git a/pom.xml b/pom.xml index 6cf8692d1..03781bc31 100644 --- a/pom.xml +++ b/pom.xml @@ -66,6 +66,7 @@ 2.1.3 3.5.6 1.1.8 + 4.41.0 3.8.6 3.2.0 5.23.0 @@ -117,6 +118,11 @@ pom import + + com.openai + openai-java + ${com.openai.openai-java.version} + io.projectreactor diff --git a/sample-code/spring-app/pom.xml b/sample-code/spring-app/pom.xml index 41f069c14..c5ccd9c1f 100644 --- a/sample-code/spring-app/pom.xml +++ b/sample-code/spring-app/pom.xml @@ -130,6 +130,10 @@ org.springframework.ai spring-ai-commons + + org.springframework.boot + spring-boot-starter-websocket + org.springframework.ai spring-ai-model diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/Application.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/Application.java index aced5c87e..0f6b0e26c 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/Application.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/Application.java @@ -7,8 +7,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import javax.annotation.Nonnull; -import com.sap.ai.sdk.app.services.OrchestrationService; -import com.sap.ai.sdk.core.AiCoreService; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.web.servlet.ServletComponentScan; @@ -16,12 +14,6 @@ import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Primary; -import java.io.*; -import java.nio.file.Files; -import java.util.ArrayList; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.stream.Collectors; - /** Main class to start the Spring Boot application. */ @SpringBootApplication @ComponentScan({"com.sap.cloud.sdk", "com.sap.ai.sdk.app"}) @@ -46,41 +38,7 @@ public ObjectMapper objectMapper() { * * @param args Command line arguments. */ - public static void main(final String[] args) throws Exception { - var ctx = SpringApplication.run(Application.class, args); - var svc = ctx.getBean(OrchestrationService.class); - var outputBytes = new ArrayList(); - var done = new AtomicBoolean(false); - try (var channel = svc.textToSpeech((bytes, last) -> { - System.err.println("appended output bytes: " + bytes.length); - outputBytes.add(bytes); - if (last) { - done.set(true); - } - })) { - System.err.println("BEFORE SENDING TEXT"); - channel.sendText("Eat more of these fresh french baguettes and drink the tea!"); - System.err.println("AFTER SENDING TEXT"); - long timeout = System.currentTimeMillis() + 20000; - while (!done.get() && System.currentTimeMillis() < timeout) { - Thread.yield(); - System.err.println("Waiting for messages..."); - Thread.sleep(500); - } - } - - //var res = svc.textToSpeech("Eat more of these fresh french baguettes").toList(); - String path = System.getProperty("user.home") + File.separator + "Documents" + File.separator + System.currentTimeMillis() + ".pcm"; - File file = new File(path); - try (FileOutputStream in = new FileOutputStream(file)) { - var ttl = 0; - for (byte[] chunk: outputBytes) { - System.out.println("Writing bytes: " + chunk.length); - in.write(chunk); - ttl += chunk.length; - } - System.out.println("Written bytes: " + ttl); - } - + public static void main(final String[] args) { + SpringApplication.run(Application.class, args); } } diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java new file mode 100644 index 000000000..606b3bc4a --- /dev/null +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java @@ -0,0 +1,23 @@ +package com.sap.ai.sdk.app; + +import com.sap.ai.sdk.app.realtime.TextToSpeechWebsocketHandler; +import org.springframework.context.annotation.Configuration; +import org.springframework.web.socket.config.annotation.EnableWebSocket; +import org.springframework.web.socket.config.annotation.WebSocketConfigurer; +import org.springframework.web.socket.config.annotation.WebSocketHandlerRegistry; + +@Configuration +@EnableWebSocket +public class WebsocketConfig implements WebSocketConfigurer { + + private final TextToSpeechWebsocketHandler textToSpeech; + + public WebsocketConfig(TextToSpeechWebsocketHandler textToSpeech) { + this.textToSpeech = textToSpeech; + } + + @Override + public void registerWebSocketHandlers(WebSocketHandlerRegistry registry) { + registry.addHandler(textToSpeech, "/text-to-speech").setAllowedOrigins("*"); + } +} diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java new file mode 100644 index 000000000..27a667d3c --- /dev/null +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java @@ -0,0 +1,61 @@ +package com.sap.ai.sdk.app.realtime; + +import com.sap.ai.sdk.app.services.OpenAiService; +import com.sap.ai.sdk.foundationmodels.openai.TextInputChannel; +import lombok.extern.slf4j.Slf4j; +import org.jspecify.annotations.NonNull; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.springframework.web.socket.BinaryMessage; +import org.springframework.web.socket.CloseStatus; +import org.springframework.web.socket.WebSocketSession; +import org.springframework.web.socket.handler.BinaryWebSocketHandler; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +@Component +@Slf4j +public class TextToSpeechWebsocketHandler extends BinaryWebSocketHandler { + + private final OpenAiService service; + private final Map channels; + + @Autowired + public TextToSpeechWebsocketHandler(OpenAiService service) { + this.service = service; + channels = new ConcurrentHashMap<>(); + } + + @Override + protected void handleBinaryMessage(WebSocketSession session, BinaryMessage message) { + ByteBuffer payload = message.getPayload(); + byte[] textBytes = payload.array(); + TextInputChannel channel = channels.computeIfAbsent( + session.getId(), + sessionId -> service.textToSpeech((rawBytesChunk, isLast) -> { + try { + session.sendMessage(new BinaryMessage(rawBytesChunk, isLast)); + } catch (IOException e) { + log.error("failed to send text message to realtime api", e); + } + })); + channel.sendText(new String(textBytes)); + } + + @Override + public void afterConnectionClosed(WebSocketSession session, @NonNull CloseStatus status) throws Exception { + channels.computeIfPresent(session.getId(), (sessionId, inputChannel) -> { + try { + inputChannel.close(); + } catch (Exception e) { + log.warn("failed to close input channel for session {}", sessionId, e); + } + return null; + }); + super.afterConnectionClosed(session, status); + } + +} diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OpenAiService.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OpenAiService.java index 1b84e56f6..efb884097 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OpenAiService.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OpenAiService.java @@ -5,15 +5,8 @@ import static com.sap.ai.sdk.foundationmodels.openai.OpenAiModel.TEXT_EMBEDDING_3_SMALL; import com.sap.ai.sdk.core.AiCoreService; -import com.sap.ai.sdk.foundationmodels.openai.OpenAiChatCompletionDelta; -import com.sap.ai.sdk.foundationmodels.openai.OpenAiChatCompletionRequest; -import com.sap.ai.sdk.foundationmodels.openai.OpenAiChatCompletionResponse; -import com.sap.ai.sdk.foundationmodels.openai.OpenAiClient; -import com.sap.ai.sdk.foundationmodels.openai.OpenAiEmbeddingRequest; -import com.sap.ai.sdk.foundationmodels.openai.OpenAiEmbeddingResponse; -import com.sap.ai.sdk.foundationmodels.openai.OpenAiImageItem; -import com.sap.ai.sdk.foundationmodels.openai.OpenAiMessage; -import com.sap.ai.sdk.foundationmodels.openai.OpenAiTool; +import com.sap.ai.sdk.foundationmodels.openai.*; + import java.util.ArrayList; import java.util.List; import java.util.stream.Stream; @@ -74,6 +67,43 @@ public Stream streamChatCompletionDeltas( return OpenAiClient.forModel(GPT_5_MINI).streamChatCompletionDeltas(request); } + /** + * Creates realtime channel allowing to input text and voice it (receive audio output) + * + *

The input channel should be used with a try-with-resources block to ensure that the underlying + * connection is closed. + * + *

Example: + * + * 

{@code
+   * try (var textInputChannel = client.textToSpeech(audioOutputConsumer)) {
+   *       textInputChannel.sendText("...");
+   *       ....
+   * }
+   * }
+ * + * This API implements full duplex (input + output) communication channels. Application should logically + * synchronize their state and close input channel when it is appropriate (e.g. last part of the response + * has been received via output channel and application does not need to send any other input). When input + * channel is closed, output channel will be closed automatically and output consumer will not be called + * anymore. + * + * @param audioOutputConsumer - audio consumer of raw PCM mono 24000 Hz little endian output + * @return input channel, allowing for text input + */ + @Nonnull + public TextInputChannel textToSpeech(@Nonnull final AudioOutputChannel audioOutputConsumer) { + return OpenAiClient.forModel(OpenAiModel.GPT_REALTIME).textToSpeech(audioOutputConsumer); + } + + public AudioInputChannel speechToText(@Nonnull final TextOutputChannel textOutputConsumer) { + return OpenAiClient.forModel(OpenAiModel.GPT_REALTIME).speechToText(textOutputConsumer); + } + + public AudioInputChannel speechToSpeech(@Nonnull final AudioOutputChannel audioOutputConsumer) { + return OpenAiClient.forModel(OpenAiModel.GPT_REALTIME).speechToSpeech(audioOutputConsumer); + } + /** * Asynchronous stream of an OpenAI chat request * diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OrchestrationService.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OrchestrationService.java index 49fbaa594..f6fc24853 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OrchestrationService.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OrchestrationService.java @@ -33,7 +33,6 @@ import java.nio.file.Path; import java.util.List; import java.util.Map; -import java.util.function.BiConsumer; import java.util.stream.Stream; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -64,17 +63,6 @@ public OrchestrationChatResponse completion(@Nonnull final String famousPhrase) return client.chatCompletion(prompt, config); } - /** - * Voices input text into output audio - * - * @param outputConsumer byte chunks output consumer in PCM16 format, 16-bit, mono, 24000 Hz, little-endian, - * boolean flag signifies end of the current reply (only true for last chunk of current response) - * @return text input channel to supply text to voice, should be closed when not needed anymore - */ - public TextInputChannel textToSpeech(@Nonnull final BiConsumer outputConsumer) { - return client.textToSpeech(outputConsumer); - } - /** * Chat request to OpenAI through the Orchestration service with an image. * diff --git a/sample-code/spring-app/src/main/resources/static/index.html b/sample-code/spring-app/src/main/resources/static/index.html index cd874bc61..be0528612 100644 --- a/sample-code/spring-app/src/main/resources/static/index.html +++ b/sample-code/spring-app/src/main/resources/static/index.html @@ -1355,6 +1355,113 @@

📂 Batch API

+ +
+
+
+
+

⏰ Realtime API

+
+ Realtime API allows for various real time interactions TBD Documentation +
+
+
    +
  • +
    +

    Text to speech

    + disabled +
    + + + +
    +
    + +
    • +
+
+
+
From 80a5d3eecd598f90d110ce2abcfbac1e716a9b10 Mon Sep 17 00:00:00 2001 From: Vladimir Aseev Date: Fri, 10 Jul 2026 19:07:08 +0200 Subject: [PATCH 07/26] add speech to speech demo scenario --- .../openai/SpeechToSpeechRealtimeClient.java | 34 ++--- .../com/sap/ai/sdk/app/WebsocketConfig.java | 6 +- .../SpeechToSpeechWebsocketHandler.java | 61 +++++++++ .../src/main/resources/static/index.html | 91 ++------------ .../main/resources/static/speech-to-speech.js | 117 ++++++++++++++++++ .../main/resources/static/text-to-speech.js | 77 ++++++++++++ 6 files changed, 292 insertions(+), 94 deletions(-) create mode 100644 sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java create mode 100644 sample-code/spring-app/src/main/resources/static/speech-to-speech.js create mode 100644 sample-code/spring-app/src/main/resources/static/text-to-speech.js diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java index 3addfabfe..afbefea13 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java @@ -44,9 +44,10 @@ public void inputAudio(byte[] rawAudioChunk) { cursorLeft += MAX_DATA_CHUNK_SIZE_BYTES; } - var commitAudioMessage = InputAudioBufferCommitEvent.builder().build(); - super.sendMessage(commitAudioMessage); - askForResponse(); + // TODO: make turn detection configurable and uncomment for explicit turn control + // var commitAudioMessage = InputAudioBufferCommitEvent.builder().build(); + // super.sendMessage(commitAudioMessage); + // askForResponse(); } @Override @@ -67,7 +68,8 @@ protected void onResponse(String eventType, JsonNode event) { @Override protected SessionUpdateEvent sessionConfiguration() { - SessionUpdateEvent sessionUpdateEvent = SessionUpdateEvent.builder() + SessionUpdateEvent sessionUpdateEvent = + SessionUpdateEvent.builder() .session( ClientSecretCreateParams.Session.ofRealtime( RealtimeSessionCreateRequest.builder() @@ -76,21 +78,25 @@ protected SessionUpdateEvent sessionConfiguration() { RealtimeAudioConfig.builder() .input( RealtimeAudioConfigInput.builder() - .turnDetection(Optional.empty()) + .turnDetection( + RealtimeAudioInputTurnDetection.ofSemanticVad(RealtimeAudioInputTurnDetection.SemanticVad.builder().build())) .format( - RealtimeAudioFormats.AudioPcm - .builder() - .type(RealtimeAudioFormats.AudioPcm.Type.AUDIO_PCM) - .rate(RealtimeAudioFormats.AudioPcm.Rate._24000) - .build()) + RealtimeAudioFormats.AudioPcm.builder() + .type( + RealtimeAudioFormats.AudioPcm.Type + .AUDIO_PCM) + .rate(RealtimeAudioFormats.AudioPcm.Rate._24000) + .build()) .build()) .output( RealtimeAudioConfigOutput.builder() - .format(RealtimeAudioFormats.AudioPcm - .builder() - .type(RealtimeAudioFormats.AudioPcm.Type.AUDIO_PCM) + .format( + RealtimeAudioFormats.AudioPcm.builder() + .type( + RealtimeAudioFormats.AudioPcm.Type + .AUDIO_PCM) .rate(RealtimeAudioFormats.AudioPcm.Rate._24000) - .build()) + .build()) .voice( RealtimeAudioConfigOutput.Voice.UnionMember1.ALLOY) .build()) diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java index 606b3bc4a..ac7b07758 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java @@ -1,5 +1,6 @@ package com.sap.ai.sdk.app; +import com.sap.ai.sdk.app.realtime.SpeechToSpeechWebsocketHandler; import com.sap.ai.sdk.app.realtime.TextToSpeechWebsocketHandler; import org.springframework.context.annotation.Configuration; import org.springframework.web.socket.config.annotation.EnableWebSocket; @@ -11,13 +12,16 @@ public class WebsocketConfig implements WebSocketConfigurer { private final TextToSpeechWebsocketHandler textToSpeech; + private final SpeechToSpeechWebsocketHandler speechToSpeech; - public WebsocketConfig(TextToSpeechWebsocketHandler textToSpeech) { + public WebsocketConfig(TextToSpeechWebsocketHandler textToSpeech, SpeechToSpeechWebsocketHandler speechToSpeech) { this.textToSpeech = textToSpeech; + this.speechToSpeech = speechToSpeech; } @Override public void registerWebSocketHandlers(WebSocketHandlerRegistry registry) { registry.addHandler(textToSpeech, "/text-to-speech").setAllowedOrigins("*"); + registry.addHandler(speechToSpeech, "/speech-to-speech").setAllowedOrigins("*"); } } diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java new file mode 100644 index 000000000..72f27fab0 --- /dev/null +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java @@ -0,0 +1,61 @@ +package com.sap.ai.sdk.app.realtime; + +import com.sap.ai.sdk.app.services.OpenAiService; +import com.sap.ai.sdk.foundationmodels.openai.AudioInputChannel; +import com.sap.ai.sdk.foundationmodels.openai.TextInputChannel; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import lombok.extern.slf4j.Slf4j; +import org.jspecify.annotations.NonNull; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.springframework.web.socket.BinaryMessage; +import org.springframework.web.socket.CloseStatus; +import org.springframework.web.socket.WebSocketSession; +import org.springframework.web.socket.handler.BinaryWebSocketHandler; + +@Component +@Slf4j +public class SpeechToSpeechWebsocketHandler extends BinaryWebSocketHandler { + + private final OpenAiService service; + private final Map channels; + + @Autowired + public SpeechToSpeechWebsocketHandler(OpenAiService service) { + this.service = service; + channels = new ConcurrentHashMap<>(); + } + + @Override + protected void handleBinaryMessage(WebSocketSession session, BinaryMessage message) { + ByteBuffer payload = message.getPayload(); + byte[] chunkBytes = payload.array(); + AudioInputChannel channel = channels.computeIfAbsent( + session.getId(), + sessionId -> service.speechToSpeech((rawBytesChunk, isLast) -> { + try { + session.sendMessage(new BinaryMessage(rawBytesChunk, isLast)); + } catch (IOException e) { + log.error("failed to send audio data to realtime api", e); + } + })); + channel.inputAudio(chunkBytes); + } + + @Override + public void afterConnectionClosed(WebSocketSession session, @NonNull CloseStatus status) throws Exception { + channels.computeIfPresent(session.getId(), (sessionId, inputChannel) -> { + try { + inputChannel.close(); + } catch (Exception e) { + log.warn("failed to close input channel for session {}", sessionId, e); + } + return null; + }); + super.afterConnectionClosed(session, status); + } + +} diff --git a/sample-code/spring-app/src/main/resources/static/index.html b/sample-code/spring-app/src/main/resources/static/index.html index be0528612..a30067be7 100644 --- a/sample-code/spring-app/src/main/resources/static/index.html +++ b/sample-code/spring-app/src/main/resources/static/index.html @@ -1378,85 +1378,18 @@

Text to speech

- + + +
  • +
    +

    Speech to speech

    +
    disconnected
    +
    disabled
    +
    + +
    +
    +
    • diff --git a/sample-code/spring-app/src/main/resources/static/speech-to-speech.js b/sample-code/spring-app/src/main/resources/static/speech-to-speech.js new file mode 100644 index 000000000..e6cd3f9be --- /dev/null +++ b/sample-code/spring-app/src/main/resources/static/speech-to-speech.js @@ -0,0 +1,117 @@ +const SPEECH_TO_SPEECH_URL = 'ws://localhost:8080/speech-to-speech'; +const SPEECH_TO_SPEECH_SAMPLE_RATE = 24000; + +const wsStatusEl = document.getElementById('speech-to-speech-websocket-status') +const micStatusEl = document.getElementById('speech-to-speech-mic-status') +const speechBtn = document.getElementById('speech-to-speech-btn') + +let sts_ws = null; +let sts_audioCtx = null; +let sts_mediaStream = null; +let sts_processor = null; +let sts_nextStartTime = 0; +let sts_started = false; + +async function startSession() { + sts_ws = new WebSocket(SPEECH_TO_SPEECH_URL); + sts_ws.binaryType = 'arraybuffer'; + + sts_ws.onopen = async () => { + wsStatusEl.textContent = `connected to ${SPEECH_TO_SPEECH_URL}`; + wsStatusEl.style.color = 'green'; + + await startMicrophone(); + speechBtn.innerText = 'Stop'; + sts_started = true; + } + + sts_ws.onmessage = (event) => { + if (event.data instanceof ArrayBuffer) { + sts_playPcmAudio(event.data) + } + } + + sts_ws.onclose = () => stopSession(); +} + +async function startMicrophone() { + try { + sts_mediaStream = await navigator.mediaDevices.getUserMedia({audio: true}); + console.log("mediaStream is: ", sts_mediaStream) + micStatusEl.textContent = 'active'; + micStatusEl.style.color = 'green'; + + sts_audioCtx = new (window.AudioContext || window.webkitAudioContext)({sampleRate: SPEECH_TO_SPEECH_SAMPLE_RATE}); + + const source = sts_audioCtx.createMediaStreamSource(sts_mediaStream); + sts_processor = sts_audioCtx.createScriptProcessor(2048, 1, 1); + sts_processor.onaudioprocess = (e) => { + if (sts_ws && sts_ws.readyState === WebSocket.OPEN) { + const float32Input = e.inputBuffer.getChannelData(0); + const int16Buffer = new Int16Array(float32Input.length); + + for (let i = 0; i < float32Input.length; i++) { + let s = Math.max(-1, Math.min(1, float32Input[i])); + int16Buffer[i] = s < 0 ? s * 0x8000 : s * 0x7FFF; + } + sts_ws.send(int16Buffer.buffer); + } + }; + + const gainNode = sts_audioCtx.createGain(); + gainNode.gain.value = 0; + + source.connect(sts_processor); + sts_processor.connect(gainNode); + gainNode.connect(sts_audioCtx.destination) + + } catch (err) { + console.error('failed to find or bind microphone: ', err); + micStatusEl.innerText = 'mic binding error'; + micStatusEl.style.color = 'red'; + } +} + +function sts_playPcmAudio(arrayBuffer) { + if (!sts_audioCtx) return; + + const int16Data = new Int16Array(arrayBuffer); + const float32Data = new Float32Array(int16Data.length); + for (let i = 0; i < int16Data.length; i++) { + float32Data[i] = int16Data[i] / 32768.0; + } + + const audioBuffer = sts_audioCtx.createBuffer(1, float32Data.length, SPEECH_TO_SPEECH_SAMPLE_RATE); + audioBuffer.copyToChannel(float32Data, 0); + const source = sts_audioCtx.createBufferSource(); + source.buffer = audioBuffer; + source.connect(sts_audioCtx.destination); + + if (sts_nextStartTime < sts_audioCtx.currentTime) { + sts_nextStartTime = sts_audioCtx.currentTime; + } + + source.start(sts_nextStartTime); + sts_nextStartTime += audioBuffer.duration; +} + +function stopSession() { + if (sts_ws) sts_ws.close(); + if (sts_mediaStream) sts_mediaStream.getTracks().forEach(track => track.stop()); + if (sts_processor) sts_processor.disconnect(); + if (sts_audioCtx) { + sts_audioCtx.close(); + sts_audioCtx = null; + } + + wsStatusEl.textContent = 'disconnected'; + wsStatusEl.style.color = 'red'; + micStatusEl.textContent = 'disabled'; + micStatusEl.style.color = 'red'; + speechBtn.innerText = 'Start'; + sts_started = false; +} + +speechBtn.addEventListener('click', () => { + sts_started ? stopSession() : startSession(); +}) \ No newline at end of file diff --git a/sample-code/spring-app/src/main/resources/static/text-to-speech.js b/sample-code/spring-app/src/main/resources/static/text-to-speech.js new file mode 100644 index 000000000..80bd75841 --- /dev/null +++ b/sample-code/spring-app/src/main/resources/static/text-to-speech.js @@ -0,0 +1,77 @@ +const WS_URL = 'ws://localhost:8080/text-to-speech'; +const SAMPLE_RATE = 24000; + +const statusEl = document.getElementById('text-to-speech-status') +const textInput = document.getElementById('text-to-speech-input') +const sendBtn = document.getElementById('text-to-speech-send-btn') + +let audioCtx; +let nextStartTime = 0; + +let ws = new WebSocket(WS_URL); +ws.binaryType = 'arraybuffer'; + +ws.onopen = () => { + statusEl.textContent = `connected to ${WS_URL}`; + statusEl.style.color = 'green'; + textInput.disabled = false; + sendBtn.disabled = false; +}; + +ws.onclose = () => { + statusEl.textContent = 'disconnected'; + statusEl.style.color = 'red'; + textInput.disabled = true; + sendBtn.disabled = true; +} + +ws.onerror = (error) => { + console.error('text-to-speech WebSocket error:', error); + statusEl.textContent = 'connection error' + statusEl.style.color = 'red'; +} + +ws.onmessage = (event) => { + if (event.data instanceof ArrayBuffer) { + playPcmAudio(event.data); + } else { + console.log('text data received:', event.data) + } +} + +sendBtn.addEventListener('click', () => { + const text = textInput.value.trim(); + if (text && ws.readyState === WebSocket.OPEN) { + if (!audioCtx) { + audioCtx = new (window.AudioContext || window.webkitAudioContext)({ sampleRate: SAMPLE_RATE}); + } + + const encoder = new TextEncoder(); + const binaryData = encoder.encode(text); + ws.send(binaryData); + textInput.value = ''; + } +}) + +function playPcmAudio(arrayBuffer) { + if (!audioCtx) return; + + const int16Data = new Int16Array(arrayBuffer); + const float32Data = new Float32Array(int16Data.length); + for (let i = 0; i < int16Data.length; i++) { + float32Data[i] = int16Data[i] / 32768.0; + } + + const audioBuffer = audioCtx.createBuffer(1, float32Data.length, SAMPLE_RATE); + audioBuffer.copyToChannel(float32Data, 0); + const source = audioCtx.createBufferSource(); + source.buffer = audioBuffer; + source.connect(audioCtx.destination); + + if (nextStartTime < audioCtx.currentTime) { + nextStartTime = audioCtx.currentTime; + } + + source.start(nextStartTime); + nextStartTime += audioBuffer.duration; +} \ No newline at end of file From d6ad031d61916b33d828979636a9f7705c50898b Mon Sep 17 00:00:00 2001 From: Vladimir Aseev Date: Mon, 13 Jul 2026 17:34:57 +0200 Subject: [PATCH 08/26] add smoke test for realtime api and configurability --- .../ai/sdk/core/model/SpeechOutputParam.java | 14 +++ .../model/SpeechOutputParamTurnDetection.java | 35 ++++++ .../core/model/SpeechOutputParamVoice.java | 35 ++++++ .../foundationmodels/openai/OpenAiClient.java | 14 ++- .../openai/SpeechToSpeechRealtimeClient.java | 118 ++++++++++++------ .../openai/TextToSpeechRealtimeClient.java | 37 +++++- .../SpeechToSpeechWebsocketHandler.java | 4 +- .../ai/sdk/app/services/OpenAiService.java | 4 - .../sdk/app/controllers/RealtimeApiTest.java | 47 +++++++ 9 files changed, 253 insertions(+), 55 deletions(-) create mode 100644 core/src/main/java/com/sap/ai/sdk/core/model/SpeechOutputParam.java create mode 100644 core/src/main/java/com/sap/ai/sdk/core/model/SpeechOutputParamTurnDetection.java create mode 100644 core/src/main/java/com/sap/ai/sdk/core/model/SpeechOutputParamVoice.java create mode 100644 sample-code/spring-app/src/test/java/com/sap/ai/sdk/app/controllers/RealtimeApiTest.java diff --git a/core/src/main/java/com/sap/ai/sdk/core/model/SpeechOutputParam.java b/core/src/main/java/com/sap/ai/sdk/core/model/SpeechOutputParam.java new file mode 100644 index 000000000..55a7d08c1 --- /dev/null +++ b/core/src/main/java/com/sap/ai/sdk/core/model/SpeechOutputParam.java @@ -0,0 +1,14 @@ +package com.sap.ai.sdk.core.model; + +import javax.annotation.Nonnull; + +public interface SpeechOutputParam { + enum SpeechOutputParamName { + VOICE, + TURN_DETECTION, + } + + @Nonnull + SpeechOutputParamName getParamName(); + +} diff --git a/core/src/main/java/com/sap/ai/sdk/core/model/SpeechOutputParamTurnDetection.java b/core/src/main/java/com/sap/ai/sdk/core/model/SpeechOutputParamTurnDetection.java new file mode 100644 index 000000000..aea82e0d8 --- /dev/null +++ b/core/src/main/java/com/sap/ai/sdk/core/model/SpeechOutputParamTurnDetection.java @@ -0,0 +1,35 @@ +package com.sap.ai.sdk.core.model; + +import org.jspecify.annotations.NonNull; + +import java.util.Objects; + +public final class SpeechOutputParamTurnDetection implements SpeechOutputParam { + + public static final SpeechOutputParamTurnDetection BY_MODEL_AUTO = new SpeechOutputParamTurnDetection("BY_MODEL_AUTO"); + public static final SpeechOutputParamTurnDetection EACH_CALL_IS_A_TURN = + new SpeechOutputParamTurnDetection("EACH_CALL_IS_A_TURN"); + + private final String value; + + private SpeechOutputParamTurnDetection(String value) { + this.value = value; + } + + @Override + public @NonNull SpeechOutputParamName getParamName() { + return SpeechOutputParamName.TURN_DETECTION; + } + + @Override + public boolean equals(Object o) { + if (o == null || getClass() != o.getClass()) return false; + SpeechOutputParamTurnDetection that = (SpeechOutputParamTurnDetection) o; + return Objects.equals(value, that.value); + } + + @Override + public int hashCode() { + return Objects.hashCode(value); + } +} diff --git a/core/src/main/java/com/sap/ai/sdk/core/model/SpeechOutputParamVoice.java b/core/src/main/java/com/sap/ai/sdk/core/model/SpeechOutputParamVoice.java new file mode 100644 index 000000000..8f8c12473 --- /dev/null +++ b/core/src/main/java/com/sap/ai/sdk/core/model/SpeechOutputParamVoice.java @@ -0,0 +1,35 @@ +package com.sap.ai.sdk.core.model; + +import org.jspecify.annotations.NonNull; + +import javax.annotation.Nonnull; +import java.util.Objects; + +public final class SpeechOutputParamVoice implements SpeechOutputParam { + + public static final SpeechOutputParamVoice DEFAULT_MAN = new SpeechOutputParamVoice("DEFAULT_MAN"); + public static final SpeechOutputParamVoice DEFAULT_WOMAN = new SpeechOutputParamVoice("DEFAULT_WOMAN"); + + private final String voice; + + private SpeechOutputParamVoice(@Nonnull String voice) { + this.voice = voice; + } + + @Override + public @NonNull SpeechOutputParamName getParamName() { + return SpeechOutputParamName.VOICE; + } + + @Override + public boolean equals(Object o) { + if (o == null || getClass() != o.getClass()) return false; + SpeechOutputParamVoice that = (SpeechOutputParamVoice) o; + return Objects.equals(voice, that.voice); + } + + @Override + public int hashCode() { + return Objects.hashCode(voice); + } +} diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java index 72e162e6a..de19dbd71 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java @@ -15,6 +15,7 @@ import com.sap.ai.sdk.core.common.ClientStreamingHandler; import com.sap.ai.sdk.core.common.RequestLogContext; import com.sap.ai.sdk.core.common.StreamedDelta; +import com.sap.ai.sdk.core.model.SpeechOutputParam; import com.sap.ai.sdk.foundationmodels.openai.generated.model.ChatCompletionStreamOptions; import com.sap.ai.sdk.foundationmodels.openai.generated.model.CreateChatCompletionRequest; import com.sap.ai.sdk.foundationmodels.openai.generated.model.CreateChatCompletionResponse; @@ -288,7 +289,8 @@ public Stream streamChatCompletion(@Nonnull final String prompt) * @return input channel, allowing for text input */ @Nonnull - public TextInputChannel textToSpeech(@Nonnull final AudioOutputChannel audioOutputConsumer) { + public TextInputChannel textToSpeech(@Nonnull final AudioOutputChannel audioOutputConsumer, + SpeechOutputParam... params) { var extraHeaders = destination.asHttp().getHeaders(); var headers = new HashMap(extraHeaders.size() + 1); for (Header header : extraHeaders) { @@ -296,10 +298,11 @@ public TextInputChannel textToSpeech(@Nonnull final AudioOutputChannel audioOutp } var endpoint = getRealtimeEndpoint(); - return new TextToSpeechRealtimeClient(endpoint, headers, audioOutputConsumer); + return new TextToSpeechRealtimeClient(endpoint, headers, audioOutputConsumer, params); } - public AudioInputChannel speechToText(@Nonnull final TextOutputChannel textOutputConsumer) { + // unstable, requires further debugging and testing + AudioInputChannel speechToText(@Nonnull final TextOutputChannel textOutputConsumer) { var extraHeaders = destination.asHttp().getHeaders(); var headers = new HashMap(extraHeaders.size() + 1); for (Header header : extraHeaders) { @@ -310,7 +313,8 @@ public AudioInputChannel speechToText(@Nonnull final TextOutputChannel textOutpu return new SpeechToTextRealtimeClient(endpoint, headers, textOutputConsumer); } - public AudioInputChannel speechToSpeech(@Nonnull final AudioOutputChannel audioOutputChannel) { + public AudioInputChannel speechToSpeech(@Nonnull final AudioOutputChannel audioOutputChannel, + SpeechOutputParam... params) { var extraHeaders = destination.asHttp().getHeaders(); var headers = new HashMap(extraHeaders.size() + 1); for (Header header : extraHeaders) { @@ -318,7 +322,7 @@ public AudioInputChannel speechToSpeech(@Nonnull final AudioOutputChannel audioO } var endpoint = getRealtimeEndpoint(); - return new SpeechToSpeechRealtimeClient(endpoint, headers, audioOutputChannel); + return new SpeechToSpeechRealtimeClient(endpoint, headers, audioOutputChannel, params); } private String getRealtimeEndpoint() { diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java index afbefea13..3efbd96ad 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java @@ -4,6 +4,10 @@ import com.openai.models.realtime.*; import com.openai.models.realtime.clientsecrets.ClientSecretCreateParams; import java.util.*; + +import com.sap.ai.sdk.core.model.SpeechOutputParam; +import com.sap.ai.sdk.core.model.SpeechOutputParamTurnDetection; +import com.sap.ai.sdk.core.model.SpeechOutputParamVoice; import lombok.extern.slf4j.Slf4j; @Slf4j @@ -22,12 +26,37 @@ public class SpeechToSpeechRealtimeClient extends WSSOpenAIRealtimeClient implem private static final String TASK = ""; - private final AudioOutputChannel outputConsumer; + private final RealtimeAudioConfigOutput.Voice.UnionMember1 voice; + private final boolean eagerTurnDetection; - public SpeechToSpeechRealtimeClient(String url, Map httpHeaders, AudioOutputChannel outputConsumer) { + public SpeechToSpeechRealtimeClient(String url, Map httpHeaders, AudioOutputChannel outputConsumer, + SpeechOutputParam... params) { super(url, httpHeaders, HANDLED_RESPONSE_TYPES); this.outputConsumer = outputConsumer; + + var voice = RealtimeAudioConfigOutput.Voice.UnionMember1.MARIN; + var turnDetectionEager = false; + for (SpeechOutputParam param : params) { + switch (param.getParamName()) { + case VOICE -> { + if (SpeechOutputParamVoice.DEFAULT_WOMAN.equals(param)) { + voice = RealtimeAudioConfigOutput.Voice.UnionMember1.MARIN; + } else if (SpeechOutputParamVoice.DEFAULT_MAN.equals(param)) { + voice = RealtimeAudioConfigOutput.Voice.UnionMember1.ECHO; + } + } + case TURN_DETECTION -> { + if (SpeechOutputParamTurnDetection.EACH_CALL_IS_A_TURN.equals(param)) { + turnDetectionEager = true; + } else if (SpeechOutputParamTurnDetection.BY_MODEL_AUTO.equals(param)) { + turnDetectionEager = false; + } + } + } + } + this.voice = voice; + this.eagerTurnDetection = turnDetectionEager; } public void inputAudio(byte[] rawAudioChunk) { @@ -44,10 +73,11 @@ public void inputAudio(byte[] rawAudioChunk) { cursorLeft += MAX_DATA_CHUNK_SIZE_BYTES; } - // TODO: make turn detection configurable and uncomment for explicit turn control - // var commitAudioMessage = InputAudioBufferCommitEvent.builder().build(); - // super.sendMessage(commitAudioMessage); - // askForResponse(); + if (eagerTurnDetection) { + var commitAudioMessage = InputAudioBufferCommitEvent.builder().build(); + super.sendMessage(commitAudioMessage); + askForResponse(); + } } @Override @@ -68,42 +98,50 @@ protected void onResponse(String eventType, JsonNode event) { @Override protected SessionUpdateEvent sessionConfiguration() { - SessionUpdateEvent sessionUpdateEvent = - SessionUpdateEvent.builder() + RealtimeAudioInputTurnDetection turnDetection; + if (eagerTurnDetection) { + turnDetection = null; + } else { + turnDetection = RealtimeAudioInputTurnDetection.ofSemanticVad( + RealtimeAudioInputTurnDetection.SemanticVad.builder().build() + ); + } + + return SessionUpdateEvent.builder() .session( ClientSecretCreateParams.Session.ofRealtime( - RealtimeSessionCreateRequest.builder() + RealtimeSessionCreateRequest + .builder() .outputModalities(OUTPUT_MODALITIES) - .audio( - RealtimeAudioConfig.builder() - .input( - RealtimeAudioConfigInput.builder() - .turnDetection( - RealtimeAudioInputTurnDetection.ofSemanticVad(RealtimeAudioInputTurnDetection.SemanticVad.builder().build())) - .format( - RealtimeAudioFormats.AudioPcm.builder() - .type( - RealtimeAudioFormats.AudioPcm.Type - .AUDIO_PCM) - .rate(RealtimeAudioFormats.AudioPcm.Rate._24000) - .build()) - .build()) - .output( - RealtimeAudioConfigOutput.builder() - .format( - RealtimeAudioFormats.AudioPcm.builder() - .type( - RealtimeAudioFormats.AudioPcm.Type - .AUDIO_PCM) - .rate(RealtimeAudioFormats.AudioPcm.Rate._24000) - .build()) - .voice( - RealtimeAudioConfigOutput.Voice.UnionMember1.ALLOY) - .build()) - .build()) - .build()) - .asRealtime()) - .build(); - return sessionUpdateEvent; + .audio(RealtimeAudioConfig.builder() + .input( + RealtimeAudioConfigInput.builder() + .turnDetection(turnDetection) + .format( + RealtimeAudioFormats.AudioPcm.builder() + .type(RealtimeAudioFormats.AudioPcm.Type.AUDIO_PCM) + .rate(RealtimeAudioFormats.AudioPcm.Rate._24000) + .build() + ) + .build() + ) + .output( + RealtimeAudioConfigOutput.builder() + .format( + RealtimeAudioFormats.AudioPcm.builder() + .type(RealtimeAudioFormats.AudioPcm.Type.AUDIO_PCM) + .rate(RealtimeAudioFormats.AudioPcm.Rate._24000) + .build() + ) + .voice(voice) + .build() + ) + .build() + ) + .build() + ) + .asRealtime() + ) + .build(); } } diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java index 5e42e5ac4..d4e812cf0 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java @@ -3,6 +3,9 @@ import com.fasterxml.jackson.databind.JsonNode; import com.openai.models.realtime.*; import com.openai.models.realtime.clientsecrets.ClientSecretCreateParams; +import com.sap.ai.sdk.core.model.SpeechOutputParam; +import com.sap.ai.sdk.core.model.SpeechOutputParamTurnDetection; +import com.sap.ai.sdk.core.model.SpeechOutputParamVoice; import lombok.extern.slf4j.Slf4j; import java.util.*; @@ -25,10 +28,35 @@ public class TextToSpeechRealtimeClient extends WSSOpenAIRealtimeClient implemen private final AudioOutputChannel outputConsumer; + private final RealtimeAudioConfigOutput.Voice.UnionMember1 voice; + private final boolean eagerTurnDetection; - public TextToSpeechRealtimeClient(String url, Map httpHeaders, AudioOutputChannel outputConsumer) { + public TextToSpeechRealtimeClient(String url, Map httpHeaders, AudioOutputChannel outputConsumer, + SpeechOutputParam... params) { super(url, httpHeaders, HANDLED_RESPONSE_TYPES); this.outputConsumer = outputConsumer; + var voice = RealtimeAudioConfigOutput.Voice.UnionMember1.MARIN; + var turnDetectionEager = true; + for (SpeechOutputParam param : params) { + switch (param.getParamName()) { + case VOICE -> { + if (SpeechOutputParamVoice.DEFAULT_WOMAN.equals(param)) { + voice = RealtimeAudioConfigOutput.Voice.UnionMember1.MARIN; + } else if (SpeechOutputParamVoice.DEFAULT_MAN.equals(param)) { + voice = RealtimeAudioConfigOutput.Voice.UnionMember1.ECHO; + } + } + case TURN_DETECTION -> { + if (SpeechOutputParamTurnDetection.EACH_CALL_IS_A_TURN.equals(param)) { + turnDetectionEager = true; + } else if (SpeechOutputParamTurnDetection.BY_MODEL_AUTO.equals(param)) { + turnDetectionEager = false; + } + } + } + } + this.voice = voice; + this.eagerTurnDetection = turnDetectionEager; } public void sendText(String text) { @@ -41,7 +69,9 @@ public void sendText(String text) { )).build(); super.sendMessage(message); - askForResponse(); + if (eagerTurnDetection) { + askForResponse(); + } } @Override @@ -88,8 +118,7 @@ protected SessionUpdateEvent sessionConfiguration() { .type(RealtimeAudioFormats.AudioPcm.Type.AUDIO_PCM) .rate(RealtimeAudioFormats.AudioPcm.Rate._24000) .build()) - .voice( - RealtimeAudioConfigOutput.Voice.UnionMember1.ALLOY) + .voice(voice) .build()) .build()) .build()) diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java index 72f27fab0..e593fb651 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java @@ -2,7 +2,6 @@ import com.sap.ai.sdk.app.services.OpenAiService; import com.sap.ai.sdk.foundationmodels.openai.AudioInputChannel; -import com.sap.ai.sdk.foundationmodels.openai.TextInputChannel; import java.io.IOException; import java.nio.ByteBuffer; import java.util.Map; @@ -41,7 +40,8 @@ protected void handleBinaryMessage(WebSocketSession session, BinaryMessage messa } catch (IOException e) { log.error("failed to send audio data to realtime api", e); } - })); + }) + ); channel.inputAudio(chunkBytes); } diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OpenAiService.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OpenAiService.java index efb884097..ad25c15a5 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OpenAiService.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OpenAiService.java @@ -96,10 +96,6 @@ public TextInputChannel textToSpeech(@Nonnull final AudioOutputChannel audioOutp return OpenAiClient.forModel(OpenAiModel.GPT_REALTIME).textToSpeech(audioOutputConsumer); } - public AudioInputChannel speechToText(@Nonnull final TextOutputChannel textOutputConsumer) { - return OpenAiClient.forModel(OpenAiModel.GPT_REALTIME).speechToText(textOutputConsumer); - } - public AudioInputChannel speechToSpeech(@Nonnull final AudioOutputChannel audioOutputConsumer) { return OpenAiClient.forModel(OpenAiModel.GPT_REALTIME).speechToSpeech(audioOutputConsumer); } diff --git a/sample-code/spring-app/src/test/java/com/sap/ai/sdk/app/controllers/RealtimeApiTest.java b/sample-code/spring-app/src/test/java/com/sap/ai/sdk/app/controllers/RealtimeApiTest.java new file mode 100644 index 000000000..988ae7031 --- /dev/null +++ b/sample-code/spring-app/src/test/java/com/sap/ai/sdk/app/controllers/RealtimeApiTest.java @@ -0,0 +1,47 @@ +package com.sap.ai.sdk.app.controllers; + +import com.sap.ai.sdk.core.model.SpeechOutputParamTurnDetection; +import com.sap.ai.sdk.foundationmodels.openai.OpenAiClient; +import com.sap.ai.sdk.foundationmodels.openai.OpenAiModel; +import com.sap.ai.sdk.foundationmodels.openai.TextInputChannel; +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +import java.io.ByteArrayOutputStream; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.fail; + +@Slf4j +public class RealtimeApiTest { + + @Test + @Timeout(value = 30, unit = TimeUnit.SECONDS) + void testSpeechToSpeech() { + var outputBuffer = new ByteArrayOutputStream(300000); + var monitor = new CountDownLatch(1); + var client = OpenAiClient.forModel(OpenAiModel.GPT_REALTIME); + + try (TextInputChannel input = client.textToSpeech((byteChunk, last) -> { + outputBuffer.writeBytes(byteChunk); + if (last) { + monitor.countDown(); + } + }, SpeechOutputParamTurnDetection.EACH_CALL_IS_A_TURN)) { + input.sendText("Ordnung muss sein!"); + monitor.await(); + } catch (Exception e) { + if (!(e instanceof InterruptedException)) { + fail(e); + } + // do nothing, test has either been interrupted by user (intended) or by jupiter if timeout is reached + return; + } + assertThat(monitor.getCount()).isEqualTo(0); + assertThat(outputBuffer.size()).isGreaterThan(0); + } + +} \ No newline at end of file From 2ed77e0435e2c2c889004798403cb9235058e15e Mon Sep 17 00:00:00 2001 From: Vladimir Aseev Date: Mon, 13 Jul 2026 17:48:08 +0200 Subject: [PATCH 09/26] rework sample application so it no longer use deprecated browser microphone api --- .../main/resources/static/speech-to-speech.js | 47 ++++++++++++------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/sample-code/spring-app/src/main/resources/static/speech-to-speech.js b/sample-code/spring-app/src/main/resources/static/speech-to-speech.js index e6cd3f9be..71d224a9c 100644 --- a/sample-code/spring-app/src/main/resources/static/speech-to-speech.js +++ b/sample-code/spring-app/src/main/resources/static/speech-to-speech.js @@ -8,7 +8,7 @@ const speechBtn = document.getElementById('speech-to-speech-btn') let sts_ws = null; let sts_audioCtx = null; let sts_mediaStream = null; -let sts_processor = null; +let sts_workletNode = null; let sts_nextStartTime = 0; let sts_started = false; @@ -43,27 +43,38 @@ async function startMicrophone() { sts_audioCtx = new (window.AudioContext || window.webkitAudioContext)({sampleRate: SPEECH_TO_SPEECH_SAMPLE_RATE}); + const workletCode = ` + class MicProcessor extends AudioWorkletProcessor { + process(inputs) { + const input = inputs[0]; + if (input && input[0]) { + const float32Input = input[0]; + const int16Buffer = new Int16Array(float32Input.length); + for (let i = 0; i < float32Input.length; i++) { + const s = Math.max(-1, Math.min(1, float32Input[i])); + int16Buffer[i] = s < 0 ? s * 0x8000 : s * 0x7FFF; + } + this.port.postMessage(int16Buffer.buffer, [int16Buffer.buffer]); + } + return true; + } + } + registerProcessor('mic-processor', MicProcessor); + `; + const blob = new Blob([workletCode], {type: 'application/javascript'}); + const workletUrl = URL.createObjectURL(blob); + await sts_audioCtx.audioWorklet.addModule(workletUrl); + URL.revokeObjectURL(workletUrl); + const source = sts_audioCtx.createMediaStreamSource(sts_mediaStream); - sts_processor = sts_audioCtx.createScriptProcessor(2048, 1, 1); - sts_processor.onaudioprocess = (e) => { + sts_workletNode = new AudioWorkletNode(sts_audioCtx, 'mic-processor'); + sts_workletNode.port.onmessage = (e) => { if (sts_ws && sts_ws.readyState === WebSocket.OPEN) { - const float32Input = e.inputBuffer.getChannelData(0); - const int16Buffer = new Int16Array(float32Input.length); - - for (let i = 0; i < float32Input.length; i++) { - let s = Math.max(-1, Math.min(1, float32Input[i])); - int16Buffer[i] = s < 0 ? s * 0x8000 : s * 0x7FFF; - } - sts_ws.send(int16Buffer.buffer); + sts_ws.send(e.data); } }; - const gainNode = sts_audioCtx.createGain(); - gainNode.gain.value = 0; - - source.connect(sts_processor); - sts_processor.connect(gainNode); - gainNode.connect(sts_audioCtx.destination) + source.connect(sts_workletNode); } catch (err) { console.error('failed to find or bind microphone: ', err); @@ -98,7 +109,7 @@ function sts_playPcmAudio(arrayBuffer) { function stopSession() { if (sts_ws) sts_ws.close(); if (sts_mediaStream) sts_mediaStream.getTracks().forEach(track => track.stop()); - if (sts_processor) sts_processor.disconnect(); + if (sts_workletNode) sts_workletNode.disconnect(); if (sts_audioCtx) { sts_audioCtx.close(); sts_audioCtx = null; From c212c1e854c5c8dd06b1e3c0dc1b84260bd2c09f Mon Sep 17 00:00:00 2001 From: Vladimir Aseev Date: Mon, 13 Jul 2026 17:56:23 +0200 Subject: [PATCH 10/26] make realtime implementation package local --- .../foundationmodels/openai/SpeechToSpeechRealtimeClient.java | 2 +- .../sdk/foundationmodels/openai/SpeechToTextRealtimeClient.java | 2 +- .../sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java | 2 +- .../ai/sdk/foundationmodels/openai/WSSOpenAIRealtimeClient.java | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java index 3efbd96ad..8ee5280d6 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java @@ -11,7 +11,7 @@ import lombok.extern.slf4j.Slf4j; @Slf4j -public class SpeechToSpeechRealtimeClient extends WSSOpenAIRealtimeClient implements AudioInputChannel { +class SpeechToSpeechRealtimeClient extends WSSOpenAIRealtimeClient implements AudioInputChannel { private static final int MAX_DATA_CHUNK_SIZE_BYTES = 8192; private static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToTextRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToTextRealtimeClient.java index 6e56866fb..aacdf2a0a 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToTextRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToTextRealtimeClient.java @@ -8,7 +8,7 @@ import lombok.extern.slf4j.Slf4j; @Slf4j -public class SpeechToTextRealtimeClient extends WSSOpenAIRealtimeClient implements AudioInputChannel { +class SpeechToTextRealtimeClient extends WSSOpenAIRealtimeClient implements AudioInputChannel { private static final int MAX_DATA_CHUNK_SIZE_BYTES = 8192; private static final Set HANDLED_RESPONSE_TYPES = Set.of( diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java index d4e812cf0..b42731278 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java @@ -11,7 +11,7 @@ import java.util.*; @Slf4j -public class TextToSpeechRealtimeClient extends WSSOpenAIRealtimeClient implements TextInputChannel { +class TextToSpeechRealtimeClient extends WSSOpenAIRealtimeClient implements TextInputChannel { private static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSSOpenAIRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSSOpenAIRealtimeClient.java index 77e2d881b..5d8f571e6 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSSOpenAIRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSSOpenAIRealtimeClient.java @@ -21,7 +21,7 @@ @Slf4j @RequiredArgsConstructor(access = AccessLevel.PACKAGE) -public abstract class WSSOpenAIRealtimeClient implements AutoCloseable { +abstract class WSSOpenAIRealtimeClient implements AutoCloseable { private static final int SUCCESS_FINISH_WSS_CODE = 1000; private static final ObjectMapper JACKSON = new ObjectMapper() From 03d1e70a75fc0f04233ee53bdf73d3ec9c69eaad Mon Sep 17 00:00:00 2001 From: SAP Cloud SDK Bot Date: Mon, 13 Jul 2026 16:02:30 +0000 Subject: [PATCH 11/26] Formatting --- .../ai/sdk/core/model/SpeechOutputParam.java | 13 +- .../model/SpeechOutputParamTurnDetection.java | 56 ++-- .../core/model/SpeechOutputParamVoice.java | 57 ++-- .../openai/AudioInputChannel.java | 19 +- .../openai/AudioOutputChannel.java | 27 +- .../openai/BufferedWebSocketListener.java | 74 ++--- .../foundationmodels/openai/OpenAiClient.java | 29 +- .../openai/SpeechToSpeechRealtimeClient.java | 217 +++++++------ .../openai/SpeechToTextRealtimeClient.java | 97 +++--- .../openai/TextOutputChannel.java | 24 +- .../openai/TextToSpeechRealtimeClient.java | 170 ++++++----- .../openai/WSSOpenAIRealtimeClient.java | 288 +++++++++--------- .../java/com/sap/ai/sdk/app/Application.java | 1 - .../com/sap/ai/sdk/app/WebsocketConfig.java | 23 +- .../SpeechToSpeechWebsocketHandler.java | 70 +++-- .../TextToSpeechWebsocketHandler.java | 78 ++--- .../ai/sdk/app/services/OpenAiService.java | 15 +- .../sdk/app/controllers/RealtimeApiTest.java | 64 ++-- 18 files changed, 672 insertions(+), 650 deletions(-) diff --git a/core/src/main/java/com/sap/ai/sdk/core/model/SpeechOutputParam.java b/core/src/main/java/com/sap/ai/sdk/core/model/SpeechOutputParam.java index 55a7d08c1..d9c372cbd 100644 --- a/core/src/main/java/com/sap/ai/sdk/core/model/SpeechOutputParam.java +++ b/core/src/main/java/com/sap/ai/sdk/core/model/SpeechOutputParam.java @@ -3,12 +3,11 @@ import javax.annotation.Nonnull; public interface SpeechOutputParam { - enum SpeechOutputParamName { - VOICE, - TURN_DETECTION, - } - - @Nonnull - SpeechOutputParamName getParamName(); + enum SpeechOutputParamName { + VOICE, + TURN_DETECTION, + } + @Nonnull + SpeechOutputParamName getParamName(); } diff --git a/core/src/main/java/com/sap/ai/sdk/core/model/SpeechOutputParamTurnDetection.java b/core/src/main/java/com/sap/ai/sdk/core/model/SpeechOutputParamTurnDetection.java index aea82e0d8..1a0e60d27 100644 --- a/core/src/main/java/com/sap/ai/sdk/core/model/SpeechOutputParamTurnDetection.java +++ b/core/src/main/java/com/sap/ai/sdk/core/model/SpeechOutputParamTurnDetection.java @@ -1,35 +1,35 @@ package com.sap.ai.sdk.core.model; -import org.jspecify.annotations.NonNull; - import java.util.Objects; +import org.jspecify.annotations.NonNull; public final class SpeechOutputParamTurnDetection implements SpeechOutputParam { - public static final SpeechOutputParamTurnDetection BY_MODEL_AUTO = new SpeechOutputParamTurnDetection("BY_MODEL_AUTO"); - public static final SpeechOutputParamTurnDetection EACH_CALL_IS_A_TURN = - new SpeechOutputParamTurnDetection("EACH_CALL_IS_A_TURN"); - - private final String value; - - private SpeechOutputParamTurnDetection(String value) { - this.value = value; - } - - @Override - public @NonNull SpeechOutputParamName getParamName() { - return SpeechOutputParamName.TURN_DETECTION; - } - - @Override - public boolean equals(Object o) { - if (o == null || getClass() != o.getClass()) return false; - SpeechOutputParamTurnDetection that = (SpeechOutputParamTurnDetection) o; - return Objects.equals(value, that.value); - } - - @Override - public int hashCode() { - return Objects.hashCode(value); - } + public static final SpeechOutputParamTurnDetection BY_MODEL_AUTO = + new SpeechOutputParamTurnDetection("BY_MODEL_AUTO"); + public static final SpeechOutputParamTurnDetection EACH_CALL_IS_A_TURN = + new SpeechOutputParamTurnDetection("EACH_CALL_IS_A_TURN"); + + private final String value; + + private SpeechOutputParamTurnDetection(String value) { + this.value = value; + } + + @Override + public @NonNull SpeechOutputParamName getParamName() { + return SpeechOutputParamName.TURN_DETECTION; + } + + @Override + public boolean equals(Object o) { + if (o == null || getClass() != o.getClass()) return false; + SpeechOutputParamTurnDetection that = (SpeechOutputParamTurnDetection) o; + return Objects.equals(value, that.value); + } + + @Override + public int hashCode() { + return Objects.hashCode(value); + } } diff --git a/core/src/main/java/com/sap/ai/sdk/core/model/SpeechOutputParamVoice.java b/core/src/main/java/com/sap/ai/sdk/core/model/SpeechOutputParamVoice.java index 8f8c12473..35f4f5eb2 100644 --- a/core/src/main/java/com/sap/ai/sdk/core/model/SpeechOutputParamVoice.java +++ b/core/src/main/java/com/sap/ai/sdk/core/model/SpeechOutputParamVoice.java @@ -1,35 +1,36 @@ package com.sap.ai.sdk.core.model; -import org.jspecify.annotations.NonNull; - -import javax.annotation.Nonnull; import java.util.Objects; +import javax.annotation.Nonnull; +import org.jspecify.annotations.NonNull; public final class SpeechOutputParamVoice implements SpeechOutputParam { - public static final SpeechOutputParamVoice DEFAULT_MAN = new SpeechOutputParamVoice("DEFAULT_MAN"); - public static final SpeechOutputParamVoice DEFAULT_WOMAN = new SpeechOutputParamVoice("DEFAULT_WOMAN"); - - private final String voice; - - private SpeechOutputParamVoice(@Nonnull String voice) { - this.voice = voice; - } - - @Override - public @NonNull SpeechOutputParamName getParamName() { - return SpeechOutputParamName.VOICE; - } - - @Override - public boolean equals(Object o) { - if (o == null || getClass() != o.getClass()) return false; - SpeechOutputParamVoice that = (SpeechOutputParamVoice) o; - return Objects.equals(voice, that.voice); - } - - @Override - public int hashCode() { - return Objects.hashCode(voice); - } + public static final SpeechOutputParamVoice DEFAULT_MAN = + new SpeechOutputParamVoice("DEFAULT_MAN"); + public static final SpeechOutputParamVoice DEFAULT_WOMAN = + new SpeechOutputParamVoice("DEFAULT_WOMAN"); + + private final String voice; + + private SpeechOutputParamVoice(@Nonnull String voice) { + this.voice = voice; + } + + @Override + public @NonNull SpeechOutputParamName getParamName() { + return SpeechOutputParamName.VOICE; + } + + @Override + public boolean equals(Object o) { + if (o == null || getClass() != o.getClass()) return false; + SpeechOutputParamVoice that = (SpeechOutputParamVoice) o; + return Objects.equals(voice, that.voice); + } + + @Override + public int hashCode() { + return Objects.hashCode(voice); + } } diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/AudioInputChannel.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/AudioInputChannel.java index 9b4862802..50bea1504 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/AudioInputChannel.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/AudioInputChannel.java @@ -2,16 +2,17 @@ /** * Functional interface representing audio input channel (audio data consumer) - *

    - * Should be closed by application (try-with-resources) when not needed anymore + * + *

    Should be closed by application (try-with-resources) when not needed anymore */ public interface AudioInputChannel extends AutoCloseable { - /** - * This method is sequentially invoked by audio data provider to supply implementer (consumer) - * with the audio data. Exact audio format (encoding, sampling rate, etc.) depends on the usage context - * - * @param rawBytesChunk binary data in the depending on the use case format - */ - void inputAudio(byte[] rawBytesChunk); + /** + * This method is sequentially invoked by audio data provider to supply implementer (consumer) + * with the audio data. Exact audio format (encoding, sampling rate, etc.) depends on the usage + * context + * + * @param rawBytesChunk binary data in the depending on the use case format + */ + void inputAudio(byte[] rawBytesChunk); } diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/AudioOutputChannel.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/AudioOutputChannel.java index e62e3c4cd..b190fbda5 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/AudioOutputChannel.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/AudioOutputChannel.java @@ -1,22 +1,21 @@ package com.sap.ai.sdk.foundationmodels.openai; -import java.io.IOException; - /** * Functional interface representing audio output channel (audio data consumer) - *

    - * Should be closed by application (try-with-resources) when not needed anymore + * + *

    Should be closed by application (try-with-resources) when not needed anymore */ public interface AudioOutputChannel { - /** - * This method is sequentially invoked by audio data provider to supply implementer (consumer) - * with the audio data. Exact audio format (encoding, sampling rate, etc.) depends on the usage context - * - * @param rawBytesChunk binary data in the depending on the use case format - * @param isLast true if this call logically concludes previous and this passed bytes data into a - * single logical entity (e.g. gets called at the end when all byte parts of a single - * message get passed) - */ - void outputAudio(byte[] rawBytesChunk, boolean isLast); + /** + * This method is sequentially invoked by audio data provider to supply implementer (consumer) + * with the audio data. Exact audio format (encoding, sampling rate, etc.) depends on the usage + * context + * + * @param rawBytesChunk binary data in the depending on the use case format + * @param isLast true if this call logically concludes previous and this passed bytes data into a + * single logical entity (e.g. gets called at the end when all byte parts of a single message + * get passed) + */ + void outputAudio(byte[] rawBytesChunk, boolean isLast); } diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/BufferedWebSocketListener.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/BufferedWebSocketListener.java index 8377a2889..8d29cb358 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/BufferedWebSocketListener.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/BufferedWebSocketListener.java @@ -1,54 +1,54 @@ package com.sap.ai.sdk.foundationmodels.openai; -import lombok.extern.slf4j.Slf4j; - import java.net.http.WebSocket; import java.nio.ByteBuffer; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; import java.util.function.BiConsumer; import java.util.function.Consumer; +import lombok.extern.slf4j.Slf4j; @Slf4j public class BufferedWebSocketListener implements WebSocket.Listener { - private final Consumer onOpen; - private final BiConsumer onText; - private final StringBuilder buffer; - - public BufferedWebSocketListener(Consumer onOpen, BiConsumer onText) { - this.onOpen = onOpen; - this.onText = onText; - this.buffer = new StringBuilder(1024 * 1024); + private final Consumer onOpen; + private final BiConsumer onText; + private final StringBuilder buffer; + + public BufferedWebSocketListener( + Consumer onOpen, BiConsumer onText) { + this.onOpen = onOpen; + this.onText = onText; + this.buffer = new StringBuilder(1024 * 1024); + } + + @Override + public void onOpen(WebSocket webSocket) { + this.onOpen.accept(webSocket); + webSocket.request(1); + } + + @Override + public CompletionStage onText(WebSocket webSocket, CharSequence data, boolean last) { + buffer.append(data); + webSocket.request(1); + if (last) { + var completeMessage = buffer.toString(); + buffer.setLength(0); + this.onText.accept(webSocket, completeMessage); } - @Override - public void onOpen(WebSocket webSocket) { - this.onOpen.accept(webSocket); - webSocket.request(1); - } + return CompletableFuture.completedStage(null); + } - @Override - public CompletionStage onText(WebSocket webSocket, CharSequence data, boolean last) { - buffer.append(data); - webSocket.request(1); - if (last) { - var completeMessage = buffer.toString(); - buffer.setLength(0); - this.onText.accept(webSocket, completeMessage); - } - - return CompletableFuture.completedStage(null); - } - - @Override - public void onError(WebSocket webSocket, Throwable error) { - log.error("Websocket error occurred during realtime communication", error); - } + @Override + public void onError(WebSocket webSocket, Throwable error) { + log.error("Websocket error occurred during realtime communication", error); + } - @Override - public CompletionStage onBinary(WebSocket webSocket, ByteBuffer data, boolean last) { - log.warn("Received unexpected binary bytes for WebSocket connection"); - return CompletableFuture.completedStage(null); - } + @Override + public CompletionStage onBinary(WebSocket webSocket, ByteBuffer data, boolean last) { + log.warn("Received unexpected binary bytes for WebSocket connection"); + return CompletableFuture.completedStage(null); + } } diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java index de19dbd71..00f803d3d 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java @@ -52,8 +52,8 @@ @RequiredArgsConstructor(access = AccessLevel.PRIVATE) public final class OpenAiClient { private static final String DEFAULT_API_VERSION = "2024-02-01"; - private static final int PATH_BUFFER_SIZE = 400; // existing URLs are ~120 symbols long, 400 has reasonable margin - + private static final int PATH_BUFFER_SIZE = + 400; // existing URLs are ~120 symbols long, 400 has reasonable margin private static final ObjectMapper JACKSON = getOpenAiObjectMapper(); @@ -267,8 +267,8 @@ public Stream streamChatCompletion(@Nonnull final String prompt) /** * Creates realtime channel allowing to input text and voice it (receive audio output) * - *

    The input channel should be used with a try-with-resources block to ensure that the underlying - * connection is closed. + *

    The input channel should be used with a try-with-resources block to ensure that the + * underlying connection is closed. * *

    Example: * @@ -279,18 +279,18 @@ public Stream streamChatCompletion(@Nonnull final String prompt) * } * } * - * This API implements full duplex (input + output) communication channels. Application should logically - * synchronize their state and close input channel when it is appropriate (e.g. last part of the response - * has been received via output channel and application does not need to send any other input). When input - * channel is closed, output channel will be closed automatically and output consumer will not be called - * anymore. + * This API implements full duplex (input + output) communication channels. Application should + * logically synchronize their state and close input channel when it is appropriate (e.g. last + * part of the response has been received via output channel and application does not need to send + * any other input). When input channel is closed, output channel will be closed automatically and + * output consumer will not be called anymore. * * @param audioOutputConsumer - audio consumer of raw PCM mono 24000 Hz little endian output * @return input channel, allowing for text input */ @Nonnull - public TextInputChannel textToSpeech(@Nonnull final AudioOutputChannel audioOutputConsumer, - SpeechOutputParam... params) { + public TextInputChannel textToSpeech( + @Nonnull final AudioOutputChannel audioOutputConsumer, SpeechOutputParam... params) { var extraHeaders = destination.asHttp().getHeaders(); var headers = new HashMap(extraHeaders.size() + 1); for (Header header : extraHeaders) { @@ -313,8 +313,8 @@ AudioInputChannel speechToText(@Nonnull final TextOutputChannel textOutputConsum return new SpeechToTextRealtimeClient(endpoint, headers, textOutputConsumer); } - public AudioInputChannel speechToSpeech(@Nonnull final AudioOutputChannel audioOutputChannel, - SpeechOutputParam... params) { + public AudioInputChannel speechToSpeech( + @Nonnull final AudioOutputChannel audioOutputChannel, SpeechOutputParam... params) { var extraHeaders = destination.asHttp().getHeaders(); var headers = new HashMap(extraHeaders.size() + 1); for (Header header : extraHeaders) { @@ -330,7 +330,8 @@ private String getRealtimeEndpoint() { sb.append("wss://"); var pathParts = destination.asHttp().getUri().toString().split("//"); if (pathParts.length != 2) { - throw new IllegalArgumentException("Invalid destination URI: " + destination.asHttp().getUri()); + throw new IllegalArgumentException( + "Invalid destination URI: " + destination.asHttp().getUri()); } sb.append(pathParts[1].replaceFirst("^api\\.", "realtime.")); sb.append("v1/realtime"); diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java index 8ee5280d6..3202f1c1f 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java @@ -3,117 +3,119 @@ import com.fasterxml.jackson.databind.JsonNode; import com.openai.models.realtime.*; import com.openai.models.realtime.clientsecrets.ClientSecretCreateParams; -import java.util.*; - import com.sap.ai.sdk.core.model.SpeechOutputParam; import com.sap.ai.sdk.core.model.SpeechOutputParamTurnDetection; import com.sap.ai.sdk.core.model.SpeechOutputParamVoice; +import java.util.*; import lombok.extern.slf4j.Slf4j; @Slf4j class SpeechToSpeechRealtimeClient extends WSSOpenAIRealtimeClient implements AudioInputChannel { - private static final int MAX_DATA_CHUNK_SIZE_BYTES = 8192; - private static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; - - private static final Set HANDLED_RESPONSE_TYPES = Set.of( - "response.output_audio.delta", "response.output_audio.done" - ); - - private static final List OUTPUT_MODALITIES = List.of( - RealtimeSessionCreateRequest.OutputModality.AUDIO - ); - - private static final String TASK = ""; - - private final AudioOutputChannel outputConsumer; - private final RealtimeAudioConfigOutput.Voice.UnionMember1 voice; - private final boolean eagerTurnDetection; - - public SpeechToSpeechRealtimeClient(String url, Map httpHeaders, AudioOutputChannel outputConsumer, - SpeechOutputParam... params) { - super(url, httpHeaders, HANDLED_RESPONSE_TYPES); - this.outputConsumer = outputConsumer; - - var voice = RealtimeAudioConfigOutput.Voice.UnionMember1.MARIN; - var turnDetectionEager = false; - for (SpeechOutputParam param : params) { - switch (param.getParamName()) { - case VOICE -> { - if (SpeechOutputParamVoice.DEFAULT_WOMAN.equals(param)) { - voice = RealtimeAudioConfigOutput.Voice.UnionMember1.MARIN; - } else if (SpeechOutputParamVoice.DEFAULT_MAN.equals(param)) { - voice = RealtimeAudioConfigOutput.Voice.UnionMember1.ECHO; - } - } - case TURN_DETECTION -> { - if (SpeechOutputParamTurnDetection.EACH_CALL_IS_A_TURN.equals(param)) { - turnDetectionEager = true; - } else if (SpeechOutputParamTurnDetection.BY_MODEL_AUTO.equals(param)) { - turnDetectionEager = false; - } - } - } - } - this.voice = voice; - this.eagerTurnDetection = turnDetectionEager; - } - - public void inputAudio(byte[] rawAudioChunk) { - if (rawAudioChunk.length == 0) { - return; - } - var cursorLeft = 0; - while (cursorLeft < rawAudioChunk.length) { - var cursorRight = Math.min(cursorLeft + MAX_DATA_CHUNK_SIZE_BYTES, rawAudioChunk.length); - var part = Arrays.copyOfRange(rawAudioChunk, cursorLeft, cursorRight); - var audioInputMessage = InputAudioBufferAppendEvent.builder() - .audio(Base64.getEncoder().encodeToString(part)).build(); - super.sendMessage(audioInputMessage); - cursorLeft += MAX_DATA_CHUNK_SIZE_BYTES; + private static final int MAX_DATA_CHUNK_SIZE_BYTES = 8192; + private static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; + + private static final Set HANDLED_RESPONSE_TYPES = + Set.of("response.output_audio.delta", "response.output_audio.done"); + + private static final List OUTPUT_MODALITIES = + List.of(RealtimeSessionCreateRequest.OutputModality.AUDIO); + + private static final String TASK = ""; + + private final AudioOutputChannel outputConsumer; + private final RealtimeAudioConfigOutput.Voice.UnionMember1 voice; + private final boolean eagerTurnDetection; + + public SpeechToSpeechRealtimeClient( + String url, + Map httpHeaders, + AudioOutputChannel outputConsumer, + SpeechOutputParam... params) { + super(url, httpHeaders, HANDLED_RESPONSE_TYPES); + this.outputConsumer = outputConsumer; + + var voice = RealtimeAudioConfigOutput.Voice.UnionMember1.MARIN; + var turnDetectionEager = false; + for (SpeechOutputParam param : params) { + switch (param.getParamName()) { + case VOICE -> { + if (SpeechOutputParamVoice.DEFAULT_WOMAN.equals(param)) { + voice = RealtimeAudioConfigOutput.Voice.UnionMember1.MARIN; + } else if (SpeechOutputParamVoice.DEFAULT_MAN.equals(param)) { + voice = RealtimeAudioConfigOutput.Voice.UnionMember1.ECHO; + } } - - if (eagerTurnDetection) { - var commitAudioMessage = InputAudioBufferCommitEvent.builder().build(); - super.sendMessage(commitAudioMessage); - askForResponse(); + case TURN_DETECTION -> { + if (SpeechOutputParamTurnDetection.EACH_CALL_IS_A_TURN.equals(param)) { + turnDetectionEager = true; + } else if (SpeechOutputParamTurnDetection.BY_MODEL_AUTO.equals(param)) { + turnDetectionEager = false; + } } + } } + this.voice = voice; + this.eagerTurnDetection = turnDetectionEager; + } - @Override - protected String getSystemPrompt() { - return TASK; + public void inputAudio(byte[] rawAudioChunk) { + if (rawAudioChunk.length == 0) { + return; } - - @Override - protected void onResponse(String eventType, JsonNode event) { - if ("response.output_audio.delta".equals(eventType)) { - var base64Audio = event.get("delta").asText(); - byte[] audio = Base64.getDecoder().decode(base64Audio); - this.outputConsumer.outputAudio(audio, Boolean.FALSE); - } else if ("response.output_audio.done".equals(eventType)) { - this.outputConsumer.outputAudio(EMPTY_BYTE_ARRAY, Boolean.TRUE); - } + var cursorLeft = 0; + while (cursorLeft < rawAudioChunk.length) { + var cursorRight = Math.min(cursorLeft + MAX_DATA_CHUNK_SIZE_BYTES, rawAudioChunk.length); + var part = Arrays.copyOfRange(rawAudioChunk, cursorLeft, cursorRight); + var audioInputMessage = + InputAudioBufferAppendEvent.builder() + .audio(Base64.getEncoder().encodeToString(part)) + .build(); + super.sendMessage(audioInputMessage); + cursorLeft += MAX_DATA_CHUNK_SIZE_BYTES; } - @Override - protected SessionUpdateEvent sessionConfiguration() { - RealtimeAudioInputTurnDetection turnDetection; - if (eagerTurnDetection) { - turnDetection = null; - } else { - turnDetection = RealtimeAudioInputTurnDetection.ofSemanticVad( - RealtimeAudioInputTurnDetection.SemanticVad.builder().build() - ); - } + if (eagerTurnDetection) { + var commitAudioMessage = InputAudioBufferCommitEvent.builder().build(); + super.sendMessage(commitAudioMessage); + askForResponse(); + } + } + + @Override + protected String getSystemPrompt() { + return TASK; + } + + @Override + protected void onResponse(String eventType, JsonNode event) { + if ("response.output_audio.delta".equals(eventType)) { + var base64Audio = event.get("delta").asText(); + byte[] audio = Base64.getDecoder().decode(base64Audio); + this.outputConsumer.outputAudio(audio, Boolean.FALSE); + } else if ("response.output_audio.done".equals(eventType)) { + this.outputConsumer.outputAudio(EMPTY_BYTE_ARRAY, Boolean.TRUE); + } + } + + @Override + protected SessionUpdateEvent sessionConfiguration() { + RealtimeAudioInputTurnDetection turnDetection; + if (eagerTurnDetection) { + turnDetection = null; + } else { + turnDetection = + RealtimeAudioInputTurnDetection.ofSemanticVad( + RealtimeAudioInputTurnDetection.SemanticVad.builder().build()); + } - return SessionUpdateEvent.builder() - .session( - ClientSecretCreateParams.Session.ofRealtime( - RealtimeSessionCreateRequest - .builder() - .outputModalities(OUTPUT_MODALITIES) - .audio(RealtimeAudioConfig.builder() + return SessionUpdateEvent.builder() + .session( + ClientSecretCreateParams.Session.ofRealtime( + RealtimeSessionCreateRequest.builder() + .outputModalities(OUTPUT_MODALITIES) + .audio( + RealtimeAudioConfig.builder() .input( RealtimeAudioConfigInput.builder() .turnDetection(turnDetection) @@ -121,27 +123,20 @@ protected SessionUpdateEvent sessionConfiguration() { RealtimeAudioFormats.AudioPcm.builder() .type(RealtimeAudioFormats.AudioPcm.Type.AUDIO_PCM) .rate(RealtimeAudioFormats.AudioPcm.Rate._24000) - .build() - ) - .build() - ) + .build()) + .build()) .output( RealtimeAudioConfigOutput.builder() .format( RealtimeAudioFormats.AudioPcm.builder() - .type(RealtimeAudioFormats.AudioPcm.Type.AUDIO_PCM) + .type(RealtimeAudioFormats.AudioPcm.Type.AUDIO_PCM) .rate(RealtimeAudioFormats.AudioPcm.Rate._24000) - .build() - ) + .build()) .voice(voice) - .build() - ) - .build() - ) - .build() - ) - .asRealtime() - ) + .build()) + .build()) + .build()) + .asRealtime()) .build(); - } + } } diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToTextRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToTextRealtimeClient.java index aacdf2a0a..a6a606da3 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToTextRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToTextRealtimeClient.java @@ -2,66 +2,66 @@ import com.fasterxml.jackson.databind.JsonNode; import com.openai.models.realtime.*; -import java.util.*; - import com.openai.models.realtime.clientsecrets.ClientSecretCreateParams; +import java.util.*; import lombok.extern.slf4j.Slf4j; @Slf4j class SpeechToTextRealtimeClient extends WSSOpenAIRealtimeClient implements AudioInputChannel { - private static final int MAX_DATA_CHUNK_SIZE_BYTES = 8192; - private static final Set HANDLED_RESPONSE_TYPES = Set.of( - "response.output_audio_transcript.done" - ); + private static final int MAX_DATA_CHUNK_SIZE_BYTES = 8192; + private static final Set HANDLED_RESPONSE_TYPES = + Set.of("response.output_audio_transcript.done"); - private static final String TASK = - "you are a transcript writer and your role is to convert input speech to text as precise as possible (word to word exactly, if possible)"; + private static final String TASK = + "you are a transcript writer and your role is to convert input speech to text as precise as possible (word to word exactly, if possible)"; + private final TextOutputChannel outputConsumer; - private final TextOutputChannel outputConsumer; + public SpeechToTextRealtimeClient( + String url, Map httpHeaders, TextOutputChannel outputConsumer) { + super(url, httpHeaders, HANDLED_RESPONSE_TYPES); + this.outputConsumer = outputConsumer; + } - public SpeechToTextRealtimeClient(String url, Map httpHeaders, TextOutputChannel outputConsumer) { - super(url, httpHeaders, HANDLED_RESPONSE_TYPES); - this.outputConsumer = outputConsumer; + public void inputAudio(byte[] rawAudioChunk) { + if (rawAudioChunk.length == 0) { + return; } - - public void inputAudio(byte[] rawAudioChunk){ - if (rawAudioChunk.length == 0) { - return; - } - var cursorLeft = 0; - while (cursorLeft < rawAudioChunk.length) { - var cursorRight = Math.min(cursorLeft + MAX_DATA_CHUNK_SIZE_BYTES, rawAudioChunk.length); - var part = Arrays.copyOfRange(rawAudioChunk, cursorLeft, cursorRight); - var audioInputMessage = InputAudioBufferAppendEvent.builder() - .audio(Base64.getEncoder().encodeToString(part)).build(); - super.sendMessage(audioInputMessage); - cursorLeft += MAX_DATA_CHUNK_SIZE_BYTES; - } - - var commitAudioMessage = InputAudioBufferCommitEvent.builder().build(); - super.sendMessage(commitAudioMessage); - askForResponse(); + var cursorLeft = 0; + while (cursorLeft < rawAudioChunk.length) { + var cursorRight = Math.min(cursorLeft + MAX_DATA_CHUNK_SIZE_BYTES, rawAudioChunk.length); + var part = Arrays.copyOfRange(rawAudioChunk, cursorLeft, cursorRight); + var audioInputMessage = + InputAudioBufferAppendEvent.builder() + .audio(Base64.getEncoder().encodeToString(part)) + .build(); + super.sendMessage(audioInputMessage); + cursorLeft += MAX_DATA_CHUNK_SIZE_BYTES; } - @Override - protected String getSystemPrompt() { - return TASK; - } + var commitAudioMessage = InputAudioBufferCommitEvent.builder().build(); + super.sendMessage(commitAudioMessage); + askForResponse(); + } + + @Override + protected String getSystemPrompt() { + return TASK; + } - @Override - protected void onResponse(String eventType, JsonNode event) { - if ("response.output_audio_transcript.done".equals(eventType)) { - this.outputConsumer - .outputText(event.has("transcript") ? event.get("transcript").asText() : "", Boolean.TRUE); - } else { - log.trace("skipping message type: {}", eventType); - } + @Override + protected void onResponse(String eventType, JsonNode event) { + if ("response.output_audio_transcript.done".equals(eventType)) { + this.outputConsumer.outputText( + event.has("transcript") ? event.get("transcript").asText() : "", Boolean.TRUE); + } else { + log.trace("skipping message type: {}", eventType); } + } - @Override - protected SessionUpdateEvent sessionConfiguration() { + @Override + protected SessionUpdateEvent sessionConfiguration() { SessionUpdateEvent sessionUpdateEvent = SessionUpdateEvent.builder() .session( @@ -81,13 +81,14 @@ protected SessionUpdateEvent sessionConfiguration() { .build()) .transcription( AudioTranscription.builder() - .model(AudioTranscription.Model.GPT_4O_TRANSCRIBE) - .build()) + .model( + AudioTranscription.Model.GPT_4O_TRANSCRIBE) + .build()) .build()) .build()) .build()) .asTranscription()) .build(); - return sessionUpdateEvent; - } + return sessionUpdateEvent; + } } diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextOutputChannel.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextOutputChannel.java index 1c0425bb4..ff482d553 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextOutputChannel.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextOutputChannel.java @@ -2,19 +2,19 @@ /** * Functional interface representing test output channel (text data consumer) - *

    - * Should be closed by application (try-with-resources) when not needed anymore + * + *

    Should be closed by application (try-with-resources) when not needed anymore */ public interface TextOutputChannel { - /** - * This method is sequentially invoked by test data provider to supply implementer (consumer) - * with the test data. - * - * @param textChunk chunk of the text (possibly partial content) - * @param isLast true if this call logically concludes previous and this passed text data into a - * single logical entity (e.g. gets called at the end when all parts of a single - * message get passed) - */ - void outputText(CharSequence textChunk, boolean isLast); + /** + * This method is sequentially invoked by test data provider to supply implementer (consumer) with + * the test data. + * + * @param textChunk chunk of the text (possibly partial content) + * @param isLast true if this call logically concludes previous and this passed text data into a + * single logical entity (e.g. gets called at the end when all parts of a single message get + * passed) + */ + void outputText(CharSequence textChunk, boolean isLast); } diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java index b42731278..a94b957c5 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java @@ -6,95 +6,100 @@ import com.sap.ai.sdk.core.model.SpeechOutputParam; import com.sap.ai.sdk.core.model.SpeechOutputParamTurnDetection; import com.sap.ai.sdk.core.model.SpeechOutputParamVoice; -import lombok.extern.slf4j.Slf4j; - import java.util.*; +import lombok.extern.slf4j.Slf4j; @Slf4j class TextToSpeechRealtimeClient extends WSSOpenAIRealtimeClient implements TextInputChannel { - private static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; + private static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; - private static final Set HANDLED_RESPONSE_TYPES = Set.of( - "response.output_audio.delta", "response.output_audio.done" - ); + private static final Set HANDLED_RESPONSE_TYPES = + Set.of("response.output_audio.delta", "response.output_audio.done"); - private static final List OUTPUT_MODALITIES = List.of( - RealtimeSessionCreateRequest.OutputModality.AUDIO - ); + private static final List OUTPUT_MODALITIES = + List.of(RealtimeSessionCreateRequest.OutputModality.AUDIO); - private static final String TASK = - "you are a speaker and your role is to read (produce audio) of the user input speech. voice user text input"; + private static final String TASK = + "you are a speaker and your role is to read (produce audio) of the user input speech. voice user text input"; + private final AudioOutputChannel outputConsumer; + private final RealtimeAudioConfigOutput.Voice.UnionMember1 voice; + private final boolean eagerTurnDetection; - private final AudioOutputChannel outputConsumer; - private final RealtimeAudioConfigOutput.Voice.UnionMember1 voice; - private final boolean eagerTurnDetection; - - public TextToSpeechRealtimeClient(String url, Map httpHeaders, AudioOutputChannel outputConsumer, - SpeechOutputParam... params) { - super(url, httpHeaders, HANDLED_RESPONSE_TYPES); - this.outputConsumer = outputConsumer; - var voice = RealtimeAudioConfigOutput.Voice.UnionMember1.MARIN; - var turnDetectionEager = true; - for (SpeechOutputParam param : params) { - switch (param.getParamName()) { - case VOICE -> { - if (SpeechOutputParamVoice.DEFAULT_WOMAN.equals(param)) { - voice = RealtimeAudioConfigOutput.Voice.UnionMember1.MARIN; - } else if (SpeechOutputParamVoice.DEFAULT_MAN.equals(param)) { - voice = RealtimeAudioConfigOutput.Voice.UnionMember1.ECHO; - } - } - case TURN_DETECTION -> { - if (SpeechOutputParamTurnDetection.EACH_CALL_IS_A_TURN.equals(param)) { - turnDetectionEager = true; - } else if (SpeechOutputParamTurnDetection.BY_MODEL_AUTO.equals(param)) { - turnDetectionEager = false; - } - } - } + public TextToSpeechRealtimeClient( + String url, + Map httpHeaders, + AudioOutputChannel outputConsumer, + SpeechOutputParam... params) { + super(url, httpHeaders, HANDLED_RESPONSE_TYPES); + this.outputConsumer = outputConsumer; + var voice = RealtimeAudioConfigOutput.Voice.UnionMember1.MARIN; + var turnDetectionEager = true; + for (SpeechOutputParam param : params) { + switch (param.getParamName()) { + case VOICE -> { + if (SpeechOutputParamVoice.DEFAULT_WOMAN.equals(param)) { + voice = RealtimeAudioConfigOutput.Voice.UnionMember1.MARIN; + } else if (SpeechOutputParamVoice.DEFAULT_MAN.equals(param)) { + voice = RealtimeAudioConfigOutput.Voice.UnionMember1.ECHO; + } + } + case TURN_DETECTION -> { + if (SpeechOutputParamTurnDetection.EACH_CALL_IS_A_TURN.equals(param)) { + turnDetectionEager = true; + } else if (SpeechOutputParamTurnDetection.BY_MODEL_AUTO.equals(param)) { + turnDetectionEager = false; + } } - this.voice = voice; - this.eagerTurnDetection = turnDetectionEager; + } } + this.voice = voice; + this.eagerTurnDetection = turnDetectionEager; + } - public void sendText(String text) { - var message = ConversationItemCreateEvent.builder() - .item(ConversationItem.ofRealtimeConversationItemUserMessage( - RealtimeConversationItemUserMessage.builder() - .addContent(RealtimeConversationItemUserMessage.Content.builder() - .text(text) - .type(RealtimeConversationItemUserMessage.Content.Type.INPUT_TEXT).build()).build() - )).build(); + public void sendText(String text) { + var message = + ConversationItemCreateEvent.builder() + .item( + ConversationItem.ofRealtimeConversationItemUserMessage( + RealtimeConversationItemUserMessage.builder() + .addContent( + RealtimeConversationItemUserMessage.Content.builder() + .text(text) + .type(RealtimeConversationItemUserMessage.Content.Type.INPUT_TEXT) + .build()) + .build())) + .build(); - super.sendMessage(message); - if (eagerTurnDetection) { - askForResponse(); - } + super.sendMessage(message); + if (eagerTurnDetection) { + askForResponse(); } + } - @Override - protected String getSystemPrompt() { - return TASK; - } + @Override + protected String getSystemPrompt() { + return TASK; + } - @Override - protected void onResponse(String eventType, JsonNode event) { - if ("response.output_audio.delta".equals(eventType)) { - var base64Audio = event.get("delta").asText(); - byte[] audio = Base64.getDecoder().decode(base64Audio); - this.outputConsumer.outputAudio(audio, Boolean.FALSE); - } else if ("response.output_audio.done".equals(eventType)) { - this.outputConsumer.outputAudio(EMPTY_BYTE_ARRAY, Boolean.TRUE); - } else { - log.warn("skipping message type: {}", eventType); - } + @Override + protected void onResponse(String eventType, JsonNode event) { + if ("response.output_audio.delta".equals(eventType)) { + var base64Audio = event.get("delta").asText(); + byte[] audio = Base64.getDecoder().decode(base64Audio); + this.outputConsumer.outputAudio(audio, Boolean.FALSE); + } else if ("response.output_audio.done".equals(eventType)) { + this.outputConsumer.outputAudio(EMPTY_BYTE_ARRAY, Boolean.TRUE); + } else { + log.warn("skipping message type: {}", eventType); } + } - @Override - protected SessionUpdateEvent sessionConfiguration() { - SessionUpdateEvent sessionUpdateEvent = SessionUpdateEvent.builder() + @Override + protected SessionUpdateEvent sessionConfiguration() { + SessionUpdateEvent sessionUpdateEvent = + SessionUpdateEvent.builder() .session( ClientSecretCreateParams.Session.ofRealtime( RealtimeSessionCreateRequest.builder() @@ -105,25 +110,28 @@ protected SessionUpdateEvent sessionConfiguration() { RealtimeAudioConfigInput.builder() .turnDetection(Optional.empty()) .format( - RealtimeAudioFormats.AudioPcm - .builder() - .type(RealtimeAudioFormats.AudioPcm.Type.AUDIO_PCM) - .rate(RealtimeAudioFormats.AudioPcm.Rate._24000) - .build()) + RealtimeAudioFormats.AudioPcm.builder() + .type( + RealtimeAudioFormats.AudioPcm.Type + .AUDIO_PCM) + .rate(RealtimeAudioFormats.AudioPcm.Rate._24000) + .build()) .build()) .output( RealtimeAudioConfigOutput.builder() - .format(RealtimeAudioFormats.AudioPcm - .builder() - .type(RealtimeAudioFormats.AudioPcm.Type.AUDIO_PCM) + .format( + RealtimeAudioFormats.AudioPcm.builder() + .type( + RealtimeAudioFormats.AudioPcm.Type + .AUDIO_PCM) .rate(RealtimeAudioFormats.AudioPcm.Rate._24000) - .build()) + .build()) .voice(voice) .build()) .build()) .build()) .asRealtime()) .build(); - return sessionUpdateEvent; - } + return sessionUpdateEvent; + } } diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSSOpenAIRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSSOpenAIRealtimeClient.java index 5d8f571e6..a78f66d83 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSSOpenAIRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSSOpenAIRealtimeClient.java @@ -6,174 +6,182 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.openai.models.realtime.*; -import com.openai.models.realtime.clientsecrets.ClientSecretCreateParams; import com.sap.ai.sdk.core.common.ClientException; -import lombok.AccessLevel; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; - import java.net.URI; import java.net.http.HttpClient; import java.net.http.WebSocket; import java.nio.ByteBuffer; import java.util.*; import java.util.concurrent.CompletableFuture; +import lombok.AccessLevel; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; @Slf4j @RequiredArgsConstructor(access = AccessLevel.PACKAGE) abstract class WSSOpenAIRealtimeClient implements AutoCloseable { - private static final int SUCCESS_FINISH_WSS_CODE = 1000; - private static final ObjectMapper JACKSON = new ObjectMapper() - .setVisibility(PropertyAccessor.IS_GETTER, JsonAutoDetect.Visibility.NONE); - - /** - * WebSocket keeps TCP connection to the server. If connection idles, depending on the provider - * in cloud environments, cloud provider sends hard TCP RST (reset) after 60-300 seconds of inactivity, - * this is normally not configurable and would leave realtime conversation unusable on a pause. - *

    - * In mobile networks this period is even shorter and typical CGNAT session can only idle for 10-15 seconds - * before it gets terminated - *

    - * In realtime client we rely on explicit lifetime management (create/close), heartbeat mechanism is required - * to prevent unintended closing of connections while conversation is still expected to continue - */ - private static final int HEARTBEAT_INTERVAL_MILLIS = 4500; - private static final String HEARTBEAT_TIMER_NAME = "wss_realtime_heartbeat"; - - private final HttpClient client; - private final CompletableFuture ws; - private final Timer heartbeatTimer; - private final Set handleMessageTypes; - - public WSSOpenAIRealtimeClient(String url, Map httpHeaders, Set handleMessageTypes) { - this.client = HttpClient.newHttpClient(); - var wsBuilder = this.client.newWebSocketBuilder(); - for (Map.Entry entry : httpHeaders.entrySet()) { - wsBuilder = wsBuilder.header(entry.getKey(), entry.getValue()); - } - this.ws = wsBuilder.buildAsync(URI.create(url), new BufferedWebSocketListener(this::onSocketOpen, this::onText)); - this.handleMessageTypes = handleMessageTypes; - this.heartbeatTimer = new Timer(HEARTBEAT_TIMER_NAME,true); + private static final int SUCCESS_FINISH_WSS_CODE = 1000; + private static final ObjectMapper JACKSON = + new ObjectMapper().setVisibility(PropertyAccessor.IS_GETTER, JsonAutoDetect.Visibility.NONE); + + /** + * WebSocket keeps TCP connection to the server. If connection idles, depending on the provider in + * cloud environments, cloud provider sends hard TCP RST (reset) after 60-300 seconds of + * inactivity, this is normally not configurable and would leave realtime conversation unusable on + * a pause. + * + *

    In mobile networks this period is even shorter and typical CGNAT session can only idle for + * 10-15 seconds before it gets terminated + * + *

    In realtime client we rely on explicit lifetime management (create/close), heartbeat + * mechanism is required to prevent unintended closing of connections while conversation is still + * expected to continue + */ + private static final int HEARTBEAT_INTERVAL_MILLIS = 4500; + + private static final String HEARTBEAT_TIMER_NAME = "wss_realtime_heartbeat"; + + private final HttpClient client; + private final CompletableFuture ws; + private final Timer heartbeatTimer; + private final Set handleMessageTypes; + + public WSSOpenAIRealtimeClient( + String url, Map httpHeaders, Set handleMessageTypes) { + this.client = HttpClient.newHttpClient(); + var wsBuilder = this.client.newWebSocketBuilder(); + for (Map.Entry entry : httpHeaders.entrySet()) { + wsBuilder = wsBuilder.header(entry.getKey(), entry.getValue()); } - - public void askForResponse() { - WebSocket ws = this.ws.join(); - synchronized (this) { - try { - ws.sendText(JACKSON.writeValueAsString(ResponseCreateEvent.builder().build()), true); - } catch (JsonProcessingException e) { - throw new RuntimeException("Failed to serialize ask for response", e); - } - ws.request(1); - } + this.ws = + wsBuilder.buildAsync( + URI.create(url), new BufferedWebSocketListener(this::onSocketOpen, this::onText)); + this.handleMessageTypes = handleMessageTypes; + this.heartbeatTimer = new Timer(HEARTBEAT_TIMER_NAME, true); + } + + public void askForResponse() { + WebSocket ws = this.ws.join(); + synchronized (this) { + try { + ws.sendText(JACKSON.writeValueAsString(ResponseCreateEvent.builder().build()), true); + } catch (JsonProcessingException e) { + throw new RuntimeException("Failed to serialize ask for response", e); + } + ws.request(1); } - - @Override - public void close() { - WebSocket ws = this.ws.join(); - synchronized (this) { - heartbeatTimer.cancel(); - try { - ws.sendClose(SUCCESS_FINISH_WSS_CODE, "done").join(); - } catch (Exception e) { - log.error("Error while closing WebSocket", e); - } - // this.client.close(); // exists only since java 21 - } + } + + @Override + public void close() { + WebSocket ws = this.ws.join(); + synchronized (this) { + heartbeatTimer.cancel(); + try { + ws.sendClose(SUCCESS_FINISH_WSS_CODE, "done").join(); + } catch (Exception e) { + log.error("Error while closing WebSocket", e); + } + // this.client.close(); // exists only since java 21 } + } - protected abstract String getSystemPrompt(); + protected abstract String getSystemPrompt(); - protected abstract void onResponse(String eventType, JsonNode event); + protected abstract void onResponse(String eventType, JsonNode event); - protected abstract SessionUpdateEvent sessionConfiguration(); - - protected void sendMessage(Object message) { - WebSocket ws = this.ws.join(); - synchronized (this) { - try { - ws.sendText(JACKSON.writeValueAsString(message), true); - ws.request(1); - } catch (JsonProcessingException e) { - throw new ClientException("Failed to serialize message", e); - } - - } - } + protected abstract SessionUpdateEvent sessionConfiguration(); - private synchronized void onSocketOpen(WebSocket ws) { - configureSession(ws); - configureConversation(ws); - scheduleHeartbeat(ws); + protected void sendMessage(Object message) { + WebSocket ws = this.ws.join(); + synchronized (this) { + try { + ws.sendText(JACKSON.writeValueAsString(message), true); + ws.request(1); + } catch (JsonProcessingException e) { + throw new ClientException("Failed to serialize message", e); + } } - - private void onText(WebSocket webSocket, CharSequence data) { - final JsonNode event; - try { - event = JACKSON.readTree(data.toString()); - } catch (JsonProcessingException e) { - throw new ClientException("Error parsing JSON response from speech API", e); - } - var eventType = event.get("type").asText(); - if (handleMessageTypes.contains(eventType)) { - onResponse(eventType, event); - } else { - log.trace("Unhandled event type: {}", eventType); - } - - webSocket.request(1); + } + + private synchronized void onSocketOpen(WebSocket ws) { + configureSession(ws); + configureConversation(ws); + scheduleHeartbeat(ws); + } + + private void onText(WebSocket webSocket, CharSequence data) { + final JsonNode event; + try { + event = JACKSON.readTree(data.toString()); + } catch (JsonProcessingException e) { + throw new ClientException("Error parsing JSON response from speech API", e); } - - private synchronized void sendPing(WebSocket ws) { - if (ws.isInputClosed()) { - return; - } - ws.sendPing(ByteBuffer.wrap("ping".getBytes())).join(); - ws.request(1); + var eventType = event.get("type").asText(); + if (handleMessageTypes.contains(eventType)) { + onResponse(eventType, event); + } else { + log.trace("Unhandled event type: {}", eventType); } - private void configureSession(WebSocket ws) { - SessionUpdateEvent sue = sessionConfiguration(); - try { - ws.sendText(JACKSON.writeValueAsString(sue), true); - } catch (JsonProcessingException e) { - throw new ClientException("Failed to serialize session request", e); - } + webSocket.request(1); + } - ws.request(1); + private synchronized void sendPing(WebSocket ws) { + if (ws.isInputClosed()) { + return; } + ws.sendPing(ByteBuffer.wrap("ping".getBytes())).join(); + ws.request(1); + } - private void configureConversation(WebSocket ws) { - var systemPrompt = getSystemPrompt(); - if (systemPrompt == null || systemPrompt.isEmpty()) { - return; - } - var systemConversationItem = ConversationItemCreateEvent.builder() - .item(ConversationItem.ofRealtimeConversationItemSystemMessage( - RealtimeConversationItemSystemMessage.builder() - .addContent(RealtimeConversationItemSystemMessage.Content.builder() - .text(systemPrompt) - .type(RealtimeConversationItemSystemMessage.Content.Type.INPUT_TEXT).build()).build() - )).build(); - - try { - var json = JACKSON.writeValueAsString(systemConversationItem); - ws.sendText(json, true); - } catch (JsonProcessingException e) { - throw new ClientException("Failed to serialize message", e); - } - ws.request(1); + private void configureSession(WebSocket ws) { + SessionUpdateEvent sue = sessionConfiguration(); + try { + ws.sendText(JACKSON.writeValueAsString(sue), true); + } catch (JsonProcessingException e) { + throw new ClientException("Failed to serialize session request", e); } - private void scheduleHeartbeat(WebSocket ws) { - TimerTask task = new TimerTask(){ - @Override - public void run() { - sendPing(ws); - } + ws.request(1); + } - }; - heartbeatTimer.scheduleAtFixedRate(task, 0, HEARTBEAT_INTERVAL_MILLIS); + private void configureConversation(WebSocket ws) { + var systemPrompt = getSystemPrompt(); + if (systemPrompt == null || systemPrompt.isEmpty()) { + return; } + var systemConversationItem = + ConversationItemCreateEvent.builder() + .item( + ConversationItem.ofRealtimeConversationItemSystemMessage( + RealtimeConversationItemSystemMessage.builder() + .addContent( + RealtimeConversationItemSystemMessage.Content.builder() + .text(systemPrompt) + .type(RealtimeConversationItemSystemMessage.Content.Type.INPUT_TEXT) + .build()) + .build())) + .build(); + + try { + var json = JACKSON.writeValueAsString(systemConversationItem); + ws.sendText(json, true); + } catch (JsonProcessingException e) { + throw new ClientException("Failed to serialize message", e); + } + ws.request(1); + } + + private void scheduleHeartbeat(WebSocket ws) { + TimerTask task = + new TimerTask() { + @Override + public void run() { + sendPing(ws); + } + }; + heartbeatTimer.scheduleAtFixedRate(task, 0, HEARTBEAT_INTERVAL_MILLIS); + } } diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/Application.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/Application.java index 0f6b0e26c..0f4ed8348 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/Application.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/Application.java @@ -6,7 +6,6 @@ import com.fasterxml.jackson.annotation.PropertyAccessor; import com.fasterxml.jackson.databind.ObjectMapper; import javax.annotation.Nonnull; - import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.web.servlet.ServletComponentScan; diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java index ac7b07758..cff9b17c9 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java @@ -11,17 +11,18 @@ @EnableWebSocket public class WebsocketConfig implements WebSocketConfigurer { - private final TextToSpeechWebsocketHandler textToSpeech; - private final SpeechToSpeechWebsocketHandler speechToSpeech; + private final TextToSpeechWebsocketHandler textToSpeech; + private final SpeechToSpeechWebsocketHandler speechToSpeech; - public WebsocketConfig(TextToSpeechWebsocketHandler textToSpeech, SpeechToSpeechWebsocketHandler speechToSpeech) { - this.textToSpeech = textToSpeech; - this.speechToSpeech = speechToSpeech; - } + public WebsocketConfig( + TextToSpeechWebsocketHandler textToSpeech, SpeechToSpeechWebsocketHandler speechToSpeech) { + this.textToSpeech = textToSpeech; + this.speechToSpeech = speechToSpeech; + } - @Override - public void registerWebSocketHandlers(WebSocketHandlerRegistry registry) { - registry.addHandler(textToSpeech, "/text-to-speech").setAllowedOrigins("*"); - registry.addHandler(speechToSpeech, "/speech-to-speech").setAllowedOrigins("*"); - } + @Override + public void registerWebSocketHandlers(WebSocketHandlerRegistry registry) { + registry.addHandler(textToSpeech, "/text-to-speech").setAllowedOrigins("*"); + registry.addHandler(speechToSpeech, "/speech-to-speech").setAllowedOrigins("*"); + } } diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java index e593fb651..696d10fab 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java @@ -19,43 +19,47 @@ @Slf4j public class SpeechToSpeechWebsocketHandler extends BinaryWebSocketHandler { - private final OpenAiService service; - private final Map channels; + private final OpenAiService service; + private final Map channels; - @Autowired - public SpeechToSpeechWebsocketHandler(OpenAiService service) { - this.service = service; - channels = new ConcurrentHashMap<>(); - } + @Autowired + public SpeechToSpeechWebsocketHandler(OpenAiService service) { + this.service = service; + channels = new ConcurrentHashMap<>(); + } - @Override - protected void handleBinaryMessage(WebSocketSession session, BinaryMessage message) { - ByteBuffer payload = message.getPayload(); - byte[] chunkBytes = payload.array(); - AudioInputChannel channel = channels.computeIfAbsent( - session.getId(), - sessionId -> service.speechToSpeech((rawBytesChunk, isLast) -> { - try { + @Override + protected void handleBinaryMessage(WebSocketSession session, BinaryMessage message) { + ByteBuffer payload = message.getPayload(); + byte[] chunkBytes = payload.array(); + AudioInputChannel channel = + channels.computeIfAbsent( + session.getId(), + sessionId -> + service.speechToSpeech( + (rawBytesChunk, isLast) -> { + try { session.sendMessage(new BinaryMessage(rawBytesChunk, isLast)); - } catch (IOException e) { + } catch (IOException e) { log.error("failed to send audio data to realtime api", e); - } - }) - ); - channel.inputAudio(chunkBytes); - } + } + })); + channel.inputAudio(chunkBytes); + } - @Override - public void afterConnectionClosed(WebSocketSession session, @NonNull CloseStatus status) throws Exception { - channels.computeIfPresent(session.getId(), (sessionId, inputChannel) -> { - try { - inputChannel.close(); - } catch (Exception e) { - log.warn("failed to close input channel for session {}", sessionId, e); - } - return null; + @Override + public void afterConnectionClosed(WebSocketSession session, @NonNull CloseStatus status) + throws Exception { + channels.computeIfPresent( + session.getId(), + (sessionId, inputChannel) -> { + try { + inputChannel.close(); + } catch (Exception e) { + log.warn("failed to close input channel for session {}", sessionId, e); + } + return null; }); - super.afterConnectionClosed(session, status); - } - + super.afterConnectionClosed(session, status); + } } diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java index 27a667d3c..34469513f 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java @@ -2,6 +2,10 @@ import com.sap.ai.sdk.app.services.OpenAiService; import com.sap.ai.sdk.foundationmodels.openai.TextInputChannel; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import lombok.extern.slf4j.Slf4j; import org.jspecify.annotations.NonNull; import org.springframework.beans.factory.annotation.Autowired; @@ -11,51 +15,51 @@ import org.springframework.web.socket.WebSocketSession; import org.springframework.web.socket.handler.BinaryWebSocketHandler; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - @Component @Slf4j public class TextToSpeechWebsocketHandler extends BinaryWebSocketHandler { - private final OpenAiService service; - private final Map channels; + private final OpenAiService service; + private final Map channels; - @Autowired - public TextToSpeechWebsocketHandler(OpenAiService service) { - this.service = service; - channels = new ConcurrentHashMap<>(); - } + @Autowired + public TextToSpeechWebsocketHandler(OpenAiService service) { + this.service = service; + channels = new ConcurrentHashMap<>(); + } - @Override - protected void handleBinaryMessage(WebSocketSession session, BinaryMessage message) { - ByteBuffer payload = message.getPayload(); - byte[] textBytes = payload.array(); - TextInputChannel channel = channels.computeIfAbsent( - session.getId(), - sessionId -> service.textToSpeech((rawBytesChunk, isLast) -> { - try { + @Override + protected void handleBinaryMessage(WebSocketSession session, BinaryMessage message) { + ByteBuffer payload = message.getPayload(); + byte[] textBytes = payload.array(); + TextInputChannel channel = + channels.computeIfAbsent( + session.getId(), + sessionId -> + service.textToSpeech( + (rawBytesChunk, isLast) -> { + try { session.sendMessage(new BinaryMessage(rawBytesChunk, isLast)); - } catch (IOException e) { + } catch (IOException e) { log.error("failed to send text message to realtime api", e); - } - })); - channel.sendText(new String(textBytes)); - } + } + })); + channel.sendText(new String(textBytes)); + } - @Override - public void afterConnectionClosed(WebSocketSession session, @NonNull CloseStatus status) throws Exception { - channels.computeIfPresent(session.getId(), (sessionId, inputChannel) -> { - try { - inputChannel.close(); - } catch (Exception e) { - log.warn("failed to close input channel for session {}", sessionId, e); - } - return null; + @Override + public void afterConnectionClosed(WebSocketSession session, @NonNull CloseStatus status) + throws Exception { + channels.computeIfPresent( + session.getId(), + (sessionId, inputChannel) -> { + try { + inputChannel.close(); + } catch (Exception e) { + log.warn("failed to close input channel for session {}", sessionId, e); + } + return null; }); - super.afterConnectionClosed(session, status); - } - + super.afterConnectionClosed(session, status); + } } diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OpenAiService.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OpenAiService.java index ad25c15a5..4bc64655b 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OpenAiService.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OpenAiService.java @@ -6,7 +6,6 @@ import com.sap.ai.sdk.core.AiCoreService; import com.sap.ai.sdk.foundationmodels.openai.*; - import java.util.ArrayList; import java.util.List; import java.util.stream.Stream; @@ -70,8 +69,8 @@ public Stream streamChatCompletionDeltas( /** * Creates realtime channel allowing to input text and voice it (receive audio output) * - *

    The input channel should be used with a try-with-resources block to ensure that the underlying - * connection is closed. + *

    The input channel should be used with a try-with-resources block to ensure that the + * underlying connection is closed. * *

    Example: * @@ -82,11 +81,11 @@ public Stream streamChatCompletionDeltas( * } * } * - * This API implements full duplex (input + output) communication channels. Application should logically - * synchronize their state and close input channel when it is appropriate (e.g. last part of the response - * has been received via output channel and application does not need to send any other input). When input - * channel is closed, output channel will be closed automatically and output consumer will not be called - * anymore. + * This API implements full duplex (input + output) communication channels. Application should + * logically synchronize their state and close input channel when it is appropriate (e.g. last + * part of the response has been received via output channel and application does not need to send + * any other input). When input channel is closed, output channel will be closed automatically and + * output consumer will not be called anymore. * * @param audioOutputConsumer - audio consumer of raw PCM mono 24000 Hz little endian output * @return input channel, allowing for text input diff --git a/sample-code/spring-app/src/test/java/com/sap/ai/sdk/app/controllers/RealtimeApiTest.java b/sample-code/spring-app/src/test/java/com/sap/ai/sdk/app/controllers/RealtimeApiTest.java index 988ae7031..b9098335b 100644 --- a/sample-code/spring-app/src/test/java/com/sap/ai/sdk/app/controllers/RealtimeApiTest.java +++ b/sample-code/spring-app/src/test/java/com/sap/ai/sdk/app/controllers/RealtimeApiTest.java @@ -1,47 +1,49 @@ package com.sap.ai.sdk.app.controllers; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.fail; + import com.sap.ai.sdk.core.model.SpeechOutputParamTurnDetection; import com.sap.ai.sdk.foundationmodels.openai.OpenAiClient; import com.sap.ai.sdk.foundationmodels.openai.OpenAiModel; import com.sap.ai.sdk.foundationmodels.openai.TextInputChannel; -import lombok.extern.slf4j.Slf4j; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.Timeout; - import java.io.ByteArrayOutputStream; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.fail; +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; @Slf4j public class RealtimeApiTest { - @Test - @Timeout(value = 30, unit = TimeUnit.SECONDS) - void testSpeechToSpeech() { - var outputBuffer = new ByteArrayOutputStream(300000); - var monitor = new CountDownLatch(1); - var client = OpenAiClient.forModel(OpenAiModel.GPT_REALTIME); + @Test + @Timeout(value = 30, unit = TimeUnit.SECONDS) + void testSpeechToSpeech() { + var outputBuffer = new ByteArrayOutputStream(300000); + var monitor = new CountDownLatch(1); + var client = OpenAiClient.forModel(OpenAiModel.GPT_REALTIME); - try (TextInputChannel input = client.textToSpeech((byteChunk, last) -> { - outputBuffer.writeBytes(byteChunk); - if (last) { + try (TextInputChannel input = + client.textToSpeech( + (byteChunk, last) -> { + outputBuffer.writeBytes(byteChunk); + if (last) { monitor.countDown(); - } - }, SpeechOutputParamTurnDetection.EACH_CALL_IS_A_TURN)) { - input.sendText("Ordnung muss sein!"); - monitor.await(); - } catch (Exception e) { - if (!(e instanceof InterruptedException)) { - fail(e); - } - // do nothing, test has either been interrupted by user (intended) or by jupiter if timeout is reached - return; - } - assertThat(monitor.getCount()).isEqualTo(0); - assertThat(outputBuffer.size()).isGreaterThan(0); + } + }, + SpeechOutputParamTurnDetection.EACH_CALL_IS_A_TURN)) { + input.sendText("Ordnung muss sein!"); + monitor.await(); + } catch (Exception e) { + if (!(e instanceof InterruptedException)) { + fail(e); + } + // do nothing, test has either been interrupted by user (intended) or by jupiter if timeout is + // reached + return; } - -} \ No newline at end of file + assertThat(monitor.getCount()).isEqualTo(0); + assertThat(outputBuffer.size()).isGreaterThan(0); + } +} From 5415795a6c1292a147ffaf31ed5d6725cfd816b7 Mon Sep 17 00:00:00 2001 From: Vladimir Aseev Date: Fri, 17 Jul 2026 15:32:42 +0200 Subject: [PATCH 12/26] address pr feedback, add tests --- ...echOutputParam.java => RealtimeParam.java} | 7 +- .../sdk/core/RealtimeParamTurnDetection.java | 40 ++++ .../sap/ai/sdk/core/RealtimeParamVoice.java | 45 ++++ .../model/SpeechOutputParamTurnDetection.java | 35 --- .../core/model/SpeechOutputParamVoice.java | 36 ---- .../openai/BufferedWebSocketListener.java | 10 +- .../foundationmodels/openai/OpenAiClient.java | 88 +------- .../openai/OpenAiRealtimeClient.java | 113 ++++++++++ .../openai/SpeechToSpeechRealtimeClient.java | 20 +- .../openai/SpeechToTextRealtimeClient.java | 2 +- .../openai/TextOutputChannel.java | 6 +- .../openai/TextToSpeechRealtimeClient.java | 25 ++- .../openai/ToAudioRealtimeClient.java | 33 +++ .../openai/WSSOpenAiRealtimeClient.java | 204 ++++++++++++++++++ .../orchestration/OrchestrationAiModel.java | 3 - .../com/sap/ai/sdk/app/WebsocketConfig.java | 4 +- .../ai/sdk/app/services/OpenAiService.java | 41 +++- .../src/main/resources/static/index.html | 2 +- .../sdk/app/controllers/RealtimeApiTest.java | 115 +++++++++- .../src/test/resources/fixtures/question.pcm | Bin 0 -> 97200 bytes 20 files changed, 633 insertions(+), 196 deletions(-) rename core/src/main/java/com/sap/ai/sdk/core/{model/SpeechOutputParam.java => RealtimeParam.java} (60%) create mode 100644 core/src/main/java/com/sap/ai/sdk/core/RealtimeParamTurnDetection.java create mode 100644 core/src/main/java/com/sap/ai/sdk/core/RealtimeParamVoice.java delete mode 100644 core/src/main/java/com/sap/ai/sdk/core/model/SpeechOutputParamTurnDetection.java delete mode 100644 core/src/main/java/com/sap/ai/sdk/core/model/SpeechOutputParamVoice.java create mode 100644 foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiRealtimeClient.java create mode 100644 foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/ToAudioRealtimeClient.java create mode 100644 foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSSOpenAiRealtimeClient.java create mode 100644 sample-code/spring-app/src/test/resources/fixtures/question.pcm diff --git a/core/src/main/java/com/sap/ai/sdk/core/model/SpeechOutputParam.java b/core/src/main/java/com/sap/ai/sdk/core/RealtimeParam.java similarity index 60% rename from core/src/main/java/com/sap/ai/sdk/core/model/SpeechOutputParam.java rename to core/src/main/java/com/sap/ai/sdk/core/RealtimeParam.java index d9c372cbd..bb4cd79c4 100644 --- a/core/src/main/java/com/sap/ai/sdk/core/model/SpeechOutputParam.java +++ b/core/src/main/java/com/sap/ai/sdk/core/RealtimeParam.java @@ -1,8 +1,8 @@ -package com.sap.ai.sdk.core.model; +package com.sap.ai.sdk.core; import javax.annotation.Nonnull; -public interface SpeechOutputParam { +public interface RealtimeParam { enum SpeechOutputParamName { VOICE, TURN_DETECTION, @@ -10,4 +10,7 @@ enum SpeechOutputParamName { @Nonnull SpeechOutputParamName getParamName(); + + @Nonnull + String getValueAsString(); } diff --git a/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamTurnDetection.java b/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamTurnDetection.java new file mode 100644 index 000000000..60f736fd9 --- /dev/null +++ b/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamTurnDetection.java @@ -0,0 +1,40 @@ +package com.sap.ai.sdk.core; + +import java.util.Objects; +import org.jspecify.annotations.NonNull; + +public final class RealtimeParamTurnDetection implements RealtimeParam { + + public static final RealtimeParamTurnDetection BY_MODEL_AUTO = + new RealtimeParamTurnDetection("BY_MODEL_AUTO"); + public static final RealtimeParamTurnDetection EACH_CALL_IS_A_TURN = + new RealtimeParamTurnDetection("EACH_CALL_IS_A_TURN"); + + private final String turnDetectionKind; + + private RealtimeParamTurnDetection(String turnDetectionKind) { + this.turnDetectionKind = turnDetectionKind; + } + + @Override + public @NonNull SpeechOutputParamName getParamName() { + return SpeechOutputParamName.TURN_DETECTION; + } + + @Override + public @NonNull String getValueAsString() { + return turnDetectionKind; + } + + @Override + public boolean equals(Object o) { + if (o == null || getClass() != o.getClass()) return false; + RealtimeParamTurnDetection that = (RealtimeParamTurnDetection) o; + return Objects.equals(turnDetectionKind, that.turnDetectionKind); + } + + @Override + public int hashCode() { + return Objects.hashCode(turnDetectionKind); + } +} diff --git a/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamVoice.java b/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamVoice.java new file mode 100644 index 000000000..10fc818a8 --- /dev/null +++ b/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamVoice.java @@ -0,0 +1,45 @@ +package com.sap.ai.sdk.core; + +import java.util.Objects; +import javax.annotation.Nonnull; +import org.jspecify.annotations.NonNull; + +public final class RealtimeParamVoice implements RealtimeParam { + + public static final RealtimeParamVoice DEFAULT_1 = + new RealtimeParamVoice("DEFAULT_1"); + public static final RealtimeParamVoice DEFAULT_2 = + new RealtimeParamVoice("DEFAULT_1"); + + private final String voice; + + private RealtimeParamVoice(@Nonnull String voice) { + this.voice = voice; + } + + public static RealtimeParamVoice unsafeWithExplicitVoice(String voiceName) { + return new RealtimeParamVoice(voiceName); + } + + @Override + public @NonNull SpeechOutputParamName getParamName() { + return SpeechOutputParamName.VOICE; + } + + @Override + public @NonNull String getValueAsString() { + return voice; + } + + @Override + public boolean equals(Object o) { + if (o == null || getClass() != o.getClass()) return false; + RealtimeParamVoice that = (RealtimeParamVoice) o; + return Objects.equals(voice, that.voice); + } + + @Override + public int hashCode() { + return Objects.hashCode(voice); + } +} diff --git a/core/src/main/java/com/sap/ai/sdk/core/model/SpeechOutputParamTurnDetection.java b/core/src/main/java/com/sap/ai/sdk/core/model/SpeechOutputParamTurnDetection.java deleted file mode 100644 index 1a0e60d27..000000000 --- a/core/src/main/java/com/sap/ai/sdk/core/model/SpeechOutputParamTurnDetection.java +++ /dev/null @@ -1,35 +0,0 @@ -package com.sap.ai.sdk.core.model; - -import java.util.Objects; -import org.jspecify.annotations.NonNull; - -public final class SpeechOutputParamTurnDetection implements SpeechOutputParam { - - public static final SpeechOutputParamTurnDetection BY_MODEL_AUTO = - new SpeechOutputParamTurnDetection("BY_MODEL_AUTO"); - public static final SpeechOutputParamTurnDetection EACH_CALL_IS_A_TURN = - new SpeechOutputParamTurnDetection("EACH_CALL_IS_A_TURN"); - - private final String value; - - private SpeechOutputParamTurnDetection(String value) { - this.value = value; - } - - @Override - public @NonNull SpeechOutputParamName getParamName() { - return SpeechOutputParamName.TURN_DETECTION; - } - - @Override - public boolean equals(Object o) { - if (o == null || getClass() != o.getClass()) return false; - SpeechOutputParamTurnDetection that = (SpeechOutputParamTurnDetection) o; - return Objects.equals(value, that.value); - } - - @Override - public int hashCode() { - return Objects.hashCode(value); - } -} diff --git a/core/src/main/java/com/sap/ai/sdk/core/model/SpeechOutputParamVoice.java b/core/src/main/java/com/sap/ai/sdk/core/model/SpeechOutputParamVoice.java deleted file mode 100644 index 35f4f5eb2..000000000 --- a/core/src/main/java/com/sap/ai/sdk/core/model/SpeechOutputParamVoice.java +++ /dev/null @@ -1,36 +0,0 @@ -package com.sap.ai.sdk.core.model; - -import java.util.Objects; -import javax.annotation.Nonnull; -import org.jspecify.annotations.NonNull; - -public final class SpeechOutputParamVoice implements SpeechOutputParam { - - public static final SpeechOutputParamVoice DEFAULT_MAN = - new SpeechOutputParamVoice("DEFAULT_MAN"); - public static final SpeechOutputParamVoice DEFAULT_WOMAN = - new SpeechOutputParamVoice("DEFAULT_WOMAN"); - - private final String voice; - - private SpeechOutputParamVoice(@Nonnull String voice) { - this.voice = voice; - } - - @Override - public @NonNull SpeechOutputParamName getParamName() { - return SpeechOutputParamName.VOICE; - } - - @Override - public boolean equals(Object o) { - if (o == null || getClass() != o.getClass()) return false; - SpeechOutputParamVoice that = (SpeechOutputParamVoice) o; - return Objects.equals(voice, that.voice); - } - - @Override - public int hashCode() { - return Objects.hashCode(voice); - } -} diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/BufferedWebSocketListener.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/BufferedWebSocketListener.java index 8d29cb358..1eb449229 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/BufferedWebSocketListener.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/BufferedWebSocketListener.java @@ -9,13 +9,13 @@ import lombok.extern.slf4j.Slf4j; @Slf4j -public class BufferedWebSocketListener implements WebSocket.Listener { +class BufferedWebSocketListener implements WebSocket.Listener { private final Consumer onOpen; private final BiConsumer onText; private final StringBuilder buffer; - public BufferedWebSocketListener( + BufferedWebSocketListener( Consumer onOpen, BiConsumer onText) { this.onOpen = onOpen; this.onText = onText; @@ -29,10 +29,10 @@ public void onOpen(WebSocket webSocket) { } @Override - public CompletionStage onText(WebSocket webSocket, CharSequence data, boolean last) { + public CompletionStage onText(WebSocket webSocket, CharSequence data, boolean isLast) { buffer.append(data); webSocket.request(1); - if (last) { + if (isLast) { var completeMessage = buffer.toString(); buffer.setLength(0); this.onText.accept(webSocket, completeMessage); @@ -47,7 +47,7 @@ public void onError(WebSocket webSocket, Throwable error) { } @Override - public CompletionStage onBinary(WebSocket webSocket, ByteBuffer data, boolean last) { + public CompletionStage onBinary(WebSocket webSocket, ByteBuffer data, boolean isLast) { log.warn("Received unexpected binary bytes for WebSocket connection"); return CompletableFuture.completedStage(null); } diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java index 00f803d3d..3402e39d8 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java @@ -15,7 +15,6 @@ import com.sap.ai.sdk.core.common.ClientStreamingHandler; import com.sap.ai.sdk.core.common.RequestLogContext; import com.sap.ai.sdk.core.common.StreamedDelta; -import com.sap.ai.sdk.core.model.SpeechOutputParam; import com.sap.ai.sdk.foundationmodels.openai.generated.model.ChatCompletionStreamOptions; import com.sap.ai.sdk.foundationmodels.openai.generated.model.CreateChatCompletionRequest; import com.sap.ai.sdk.foundationmodels.openai.generated.model.CreateChatCompletionResponse; @@ -52,8 +51,6 @@ @RequiredArgsConstructor(access = AccessLevel.PRIVATE) public final class OpenAiClient { private static final String DEFAULT_API_VERSION = "2024-02-01"; - private static final int PATH_BUFFER_SIZE = - 400; // existing URLs are ~120 symbols long, 400 has reasonable margin private static final ObjectMapper JACKSON = getOpenAiObjectMapper(); @@ -79,6 +76,17 @@ public static OpenAiClient forModel(@Nonnull final OpenAiModel foundationModel) return client.withApiVersion(DEFAULT_API_VERSION); } + /** + * Creates and configures OpenAI Realtime API client + * + * @return created client + */ + @Beta + public static OpenAiRealtimeClient realtimeClient() { + var withResolvedDestination = OpenAiClient.forModel(OpenAiModel.GPT_REALTIME); + return new OpenAiRealtimeClient(withResolvedDestination.destination); + } + /** * Create a new OpenAI client targeting the specified API version. * @@ -264,80 +272,6 @@ public Stream streamChatCompletion(@Nonnull final String prompt) .map(OpenAiChatCompletionDelta::getDeltaContent); } - /** - * Creates realtime channel allowing to input text and voice it (receive audio output) - * - *

    The input channel should be used with a try-with-resources block to ensure that the - * underlying connection is closed. - * - *

    Example: - * - * 

    {@code
-   * try (var textInputChannel = client.textToSpeech(audioOutputConsumer)) {
-   *       textInputChannel.sendText("...");
-   *       ....
-   * }
-   * }
    - * - * This API implements full duplex (input + output) communication channels. Application should - * logically synchronize their state and close input channel when it is appropriate (e.g. last - * part of the response has been received via output channel and application does not need to send - * any other input). When input channel is closed, output channel will be closed automatically and - * output consumer will not be called anymore. - * - * @param audioOutputConsumer - audio consumer of raw PCM mono 24000 Hz little endian output - * @return input channel, allowing for text input - */ - @Nonnull - public TextInputChannel textToSpeech( - @Nonnull final AudioOutputChannel audioOutputConsumer, SpeechOutputParam... params) { - var extraHeaders = destination.asHttp().getHeaders(); - var headers = new HashMap(extraHeaders.size() + 1); - for (Header header : extraHeaders) { - headers.put(header.getName(), header.getValue()); - } - var endpoint = getRealtimeEndpoint(); - - return new TextToSpeechRealtimeClient(endpoint, headers, audioOutputConsumer, params); - } - - // unstable, requires further debugging and testing - AudioInputChannel speechToText(@Nonnull final TextOutputChannel textOutputConsumer) { - var extraHeaders = destination.asHttp().getHeaders(); - var headers = new HashMap(extraHeaders.size() + 1); - for (Header header : extraHeaders) { - headers.put(header.getName(), header.getValue()); - } - var endpoint = getRealtimeEndpoint() + "?intent=transcription"; - - return new SpeechToTextRealtimeClient(endpoint, headers, textOutputConsumer); - } - - public AudioInputChannel speechToSpeech( - @Nonnull final AudioOutputChannel audioOutputChannel, SpeechOutputParam... params) { - var extraHeaders = destination.asHttp().getHeaders(); - var headers = new HashMap(extraHeaders.size() + 1); - for (Header header : extraHeaders) { - headers.put(header.getName(), header.getValue()); - } - var endpoint = getRealtimeEndpoint(); - - return new SpeechToSpeechRealtimeClient(endpoint, headers, audioOutputChannel, params); - } - - private String getRealtimeEndpoint() { - var sb = new StringBuilder(PATH_BUFFER_SIZE); - sb.append("wss://"); - var pathParts = destination.asHttp().getUri().toString().split("//"); - if (pathParts.length != 2) { - throw new IllegalArgumentException( - "Invalid destination URI: " + destination.asHttp().getUri()); - } - sb.append(pathParts[1].replaceFirst("^api\\.", "realtime.")); - sb.append("v1/realtime"); - return sb.toString(); - } - private static void throwOnContentFilter(@Nonnull final OpenAiChatCompletionDelta delta) { final String finishReason = delta.getFinishReason(); if (finishReason != null && finishReason.equals("content_filter")) { diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiRealtimeClient.java new file mode 100644 index 000000000..f1923347e --- /dev/null +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiRealtimeClient.java @@ -0,0 +1,113 @@ +package com.sap.ai.sdk.foundationmodels.openai; + +import com.google.common.annotations.Beta; +import com.sap.ai.sdk.core.RealtimeParam; +import com.sap.cloud.sdk.cloudplatform.connectivity.Destination; +import com.sap.cloud.sdk.cloudplatform.connectivity.Header; + +import javax.annotation.Nonnull; +import java.util.HashMap; + +public class OpenAiRealtimeClient { + + private static final int PATH_BUFFER_SIZE = + 400; // existing URLs are ~120 symbols long, 400 has reasonable margin + + private final Destination destination; + + OpenAiRealtimeClient(Destination destination) { + this.destination = destination; + } + + /** + * Creates realtime channel allowing to input text and voice it (receive audio output) + * + *

    The input channel should be used with a try-with-resources block to ensure that the + * underlying connection is closed. + * + *

    Example: + * + * 

    {@code
+     * try (var textInputChannel = client.textToSpeech(audioOutputConsumer)) {
+     *       textInputChannel.sendText("...");
+     *       ....
+     * }
+     * }
    + * + * This API implements full duplex (input + output) communication channels. Application should + * logically synchronize their state and close input channel when it is appropriate (e.g. last + * part of the response has been received via output channel and application does not need to send + * any other input). When input channel is closed, output channel will be closed automatically and + * output consumer will not be called anymore. + * + * @param audioOutputConsumer - audio consumer of raw PCM mono 24000 Hz little endian output, 16 bit depth + * @return input channel, allowing for text input + */ + @Nonnull + @Beta + public TextInputChannel textToSpeech( + @Nonnull final AudioOutputChannel audioOutputConsumer, RealtimeParam... params) { + var extraHeaders = destination.asHttp().getHeaders(); + var headers = new HashMap(extraHeaders.size() + 1); + for (Header header : extraHeaders) { + headers.put(header.getName(), header.getValue()); + } + var endpoint = getRealtimeEndpoint(); + + return new TextToSpeechRealtimeClient(endpoint, headers, audioOutputConsumer, params); + } + + + /** + * Creates realtime channel allowing for audio conversation with a model + * + *

    The input channel should be used with a try-with-resources block to ensure that the + * underlying connection is closed. + * + *

    Example: + * + * 

    {@code
+     * try (var audioInputChannel = client.speechToSpeech(audioOutputConsumer)) {
+     *       audioInputChannel.inputAudio(audioBytesData);
+     *       ....
+     * }
+     * }
    + * + * This API implements full duplex (input + output) communication channels. Application should + * logically synchronize their state and close input channel when it is appropriate (e.g. last + * part of the response has been received via output channel and application does not need to send + * any other input). When input channel is closed, output channel will be closed automatically and + * output consumer will not be called anymore. + * + * @param audioOutputConsumer - audio consumer of raw PCM mono 24000 Hz little endian output, 16 bit depth + * @param params - optional configuration params + * @return input channel, allowing for audio data input (bytes, PCM mono 24000 Hz little endian 16 bit) + */ + @Nonnull + @Beta + public AudioInputChannel speechToSpeech( + @Nonnull final AudioOutputChannel audioOutputConsumer, RealtimeParam... params) { + var extraHeaders = destination.asHttp().getHeaders(); + var headers = new HashMap(extraHeaders.size() + 1); + for (Header header : extraHeaders) { + headers.put(header.getName(), header.getValue()); + } + var endpoint = getRealtimeEndpoint(); + + return new SpeechToSpeechRealtimeClient(endpoint, headers, audioOutputConsumer, params); + } + + private String getRealtimeEndpoint() { + var sb = new StringBuilder(PATH_BUFFER_SIZE); + sb.append("wss://"); + var pathParts = destination.asHttp().getUri().toString().split("//"); + if (pathParts.length != 2) { + throw new IllegalArgumentException( + "Invalid destination URI: " + destination.asHttp().getUri()); + } + sb.append(pathParts[1].replaceFirst("^api\\.", "realtime.")); + sb.append("v1/realtime"); + return sb.toString(); + } + +} diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java index 3202f1c1f..59fab1a02 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java @@ -3,14 +3,14 @@ import com.fasterxml.jackson.databind.JsonNode; import com.openai.models.realtime.*; import com.openai.models.realtime.clientsecrets.ClientSecretCreateParams; -import com.sap.ai.sdk.core.model.SpeechOutputParam; -import com.sap.ai.sdk.core.model.SpeechOutputParamTurnDetection; -import com.sap.ai.sdk.core.model.SpeechOutputParamVoice; +import com.sap.ai.sdk.core.RealtimeParam; +import com.sap.ai.sdk.core.RealtimeParamTurnDetection; +import com.sap.ai.sdk.core.RealtimeParamVoice; import java.util.*; import lombok.extern.slf4j.Slf4j; @Slf4j -class SpeechToSpeechRealtimeClient extends WSSOpenAIRealtimeClient implements AudioInputChannel { +class SpeechToSpeechRealtimeClient extends WSSOpenAiRealtimeClient implements AudioInputChannel { private static final int MAX_DATA_CHUNK_SIZE_BYTES = 8192; private static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; @@ -31,25 +31,25 @@ public SpeechToSpeechRealtimeClient( String url, Map httpHeaders, AudioOutputChannel outputConsumer, - SpeechOutputParam... params) { + RealtimeParam... params) { super(url, httpHeaders, HANDLED_RESPONSE_TYPES); this.outputConsumer = outputConsumer; var voice = RealtimeAudioConfigOutput.Voice.UnionMember1.MARIN; var turnDetectionEager = false; - for (SpeechOutputParam param : params) { + for (RealtimeParam param : params) { switch (param.getParamName()) { case VOICE -> { - if (SpeechOutputParamVoice.DEFAULT_WOMAN.equals(param)) { + if (RealtimeParamVoice.DEFAULT_2.equals(param)) { voice = RealtimeAudioConfigOutput.Voice.UnionMember1.MARIN; - } else if (SpeechOutputParamVoice.DEFAULT_MAN.equals(param)) { + } else if (RealtimeParamVoice.DEFAULT_1.equals(param)) { voice = RealtimeAudioConfigOutput.Voice.UnionMember1.ECHO; } } case TURN_DETECTION -> { - if (SpeechOutputParamTurnDetection.EACH_CALL_IS_A_TURN.equals(param)) { + if (RealtimeParamTurnDetection.EACH_CALL_IS_A_TURN.equals(param)) { turnDetectionEager = true; - } else if (SpeechOutputParamTurnDetection.BY_MODEL_AUTO.equals(param)) { + } else if (RealtimeParamTurnDetection.BY_MODEL_AUTO.equals(param)) { turnDetectionEager = false; } } diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToTextRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToTextRealtimeClient.java index a6a606da3..377e43625 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToTextRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToTextRealtimeClient.java @@ -7,7 +7,7 @@ import lombok.extern.slf4j.Slf4j; @Slf4j -class SpeechToTextRealtimeClient extends WSSOpenAIRealtimeClient implements AudioInputChannel { +class SpeechToTextRealtimeClient extends WSSOpenAiRealtimeClient implements AudioInputChannel { private static final int MAX_DATA_CHUNK_SIZE_BYTES = 8192; private static final Set HANDLED_RESPONSE_TYPES = diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextOutputChannel.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextOutputChannel.java index ff482d553..5896b85ab 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextOutputChannel.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextOutputChannel.java @@ -1,15 +1,15 @@ package com.sap.ai.sdk.foundationmodels.openai; /** - * Functional interface representing test output channel (text data consumer) + * Functional interface representing text output channel (text data consumer) * *

    Should be closed by application (try-with-resources) when not needed anymore */ public interface TextOutputChannel { /** - * This method is sequentially invoked by test data provider to supply implementer (consumer) with - * the test data. + * This method is sequentially invoked by text data provider to supply implementer (consumer) with + * the text data. * * @param textChunk chunk of the text (possibly partial content) * @param isLast true if this call logically concludes previous and this passed text data into a diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java index a94b957c5..4ad9f6918 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java @@ -3,14 +3,14 @@ import com.fasterxml.jackson.databind.JsonNode; import com.openai.models.realtime.*; import com.openai.models.realtime.clientsecrets.ClientSecretCreateParams; -import com.sap.ai.sdk.core.model.SpeechOutputParam; -import com.sap.ai.sdk.core.model.SpeechOutputParamTurnDetection; -import com.sap.ai.sdk.core.model.SpeechOutputParamVoice; +import com.sap.ai.sdk.core.RealtimeParam; +import com.sap.ai.sdk.core.RealtimeParamTurnDetection; +import com.sap.ai.sdk.core.RealtimeParamVoice; import java.util.*; import lombok.extern.slf4j.Slf4j; @Slf4j -class TextToSpeechRealtimeClient extends WSSOpenAIRealtimeClient implements TextInputChannel { +class TextToSpeechRealtimeClient extends WSSOpenAiRealtimeClient implements TextInputChannel { private static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; @@ -21,7 +21,8 @@ class TextToSpeechRealtimeClient extends WSSOpenAIRealtimeClient implements Text List.of(RealtimeSessionCreateRequest.OutputModality.AUDIO); private static final String TASK = - "you are a speaker and your role is to read (produce audio) of the user input speech. voice user text input"; + "you are a speaker and your role is to read (produce audio) of the user input speech. voice user text input, " + + "do not answer questions, just read them"; private final AudioOutputChannel outputConsumer; private final RealtimeAudioConfigOutput.Voice.UnionMember1 voice; @@ -31,24 +32,26 @@ public TextToSpeechRealtimeClient( String url, Map httpHeaders, AudioOutputChannel outputConsumer, - SpeechOutputParam... params) { + RealtimeParam... params) { super(url, httpHeaders, HANDLED_RESPONSE_TYPES); this.outputConsumer = outputConsumer; var voice = RealtimeAudioConfigOutput.Voice.UnionMember1.MARIN; var turnDetectionEager = true; - for (SpeechOutputParam param : params) { + for (RealtimeParam param : params) { switch (param.getParamName()) { case VOICE -> { - if (SpeechOutputParamVoice.DEFAULT_WOMAN.equals(param)) { + if (RealtimeParamVoice.DEFAULT_2.equals(param)) { voice = RealtimeAudioConfigOutput.Voice.UnionMember1.MARIN; - } else if (SpeechOutputParamVoice.DEFAULT_MAN.equals(param)) { + } else if (RealtimeParamVoice.DEFAULT_1.equals(param)) { voice = RealtimeAudioConfigOutput.Voice.UnionMember1.ECHO; + } else { + voice = RealtimeAudioConfigOutput.Voice.UnionMember1.of(param.getValueAsString().toLowerCase()); } } case TURN_DETECTION -> { - if (SpeechOutputParamTurnDetection.EACH_CALL_IS_A_TURN.equals(param)) { + if (RealtimeParamTurnDetection.EACH_CALL_IS_A_TURN.equals(param)) { turnDetectionEager = true; - } else if (SpeechOutputParamTurnDetection.BY_MODEL_AUTO.equals(param)) { + } else if (RealtimeParamTurnDetection.BY_MODEL_AUTO.equals(param)) { turnDetectionEager = false; } } diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/ToAudioRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/ToAudioRealtimeClient.java new file mode 100644 index 000000000..785414a0f --- /dev/null +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/ToAudioRealtimeClient.java @@ -0,0 +1,33 @@ +package com.sap.ai.sdk.foundationmodels.openai; + +import com.fasterxml.jackson.databind.JsonNode; +import com.openai.models.realtime.SessionUpdateEvent; + +import java.util.Map; +import java.util.Set; + +class ToAudioRealtimeClient extends WSSOpenAiRealtimeClient { + + private static final Set HANDLED_RESPONSE_TYPES = + Set.of("response.output_audio.delta", "response.output_audio.done"); + private static final String DEFAULT_SYSTEM_PROMPT = ""; + + public ToAudioRealtimeClient(String url, Map httpHeaders) { + super(url, httpHeaders, HANDLED_RESPONSE_TYPES); + } + + @Override + protected String getSystemPrompt() { + return ""; + } + + @Override + protected void onResponse(String eventType, JsonNode event) { + + } + + @Override + protected SessionUpdateEvent sessionConfiguration() { + return null; + } +} diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSSOpenAiRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSSOpenAiRealtimeClient.java new file mode 100644 index 000000000..213725452 --- /dev/null +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSSOpenAiRealtimeClient.java @@ -0,0 +1,204 @@ +package com.sap.ai.sdk.foundationmodels.openai; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.PropertyAccessor; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.openai.models.realtime.*; +import com.sap.ai.sdk.core.common.ClientException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.WebSocket; +import java.nio.ByteBuffer; +import java.util.*; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; + +import lombok.AccessLevel; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@RequiredArgsConstructor(access = AccessLevel.PACKAGE) +abstract class WSSOpenAiRealtimeClient implements AutoCloseable { + + private static final int SUCCESS_FINISH_WSS_CODE = 1000; + private static final ObjectMapper JACKSON = + new ObjectMapper().setVisibility(PropertyAccessor.IS_GETTER, JsonAutoDetect.Visibility.NONE); + + /** + * WebSocket keeps TCP connection to the server. If connection idles, depending on the provider in + * cloud environments, cloud provider sends hard TCP RST (reset) after 60-300 seconds of + * inactivity, this is normally not configurable and would leave realtime conversation unusable on + * a pause. + * + *

    In mobile networks this period is even shorter and typical CGNAT session can only idle for + * 10-15 seconds before it gets terminated + * + *

    In realtime client we rely on explicit lifetime management (create/close), heartbeat + * mechanism is required to prevent unintended closing of connections while conversation is still + * expected to continue + */ + private static final int HEARTBEAT_INTERVAL_MILLIS = 4500; + + private static final String HEARTBEAT_TIMER_NAME = "wss_realtime_heartbeat"; + + private final HttpClient client; + private final CompletableFuture ws; + private final Timer heartbeatTimer; + private final Set handleMessageTypes; + + public WSSOpenAiRealtimeClient( + String url, Map httpHeaders, Set handleMessageTypes) { + this.client = HttpClient.newHttpClient(); + var wsBuilder = this.client.newWebSocketBuilder(); + for (Map.Entry entry : httpHeaders.entrySet()) { + wsBuilder = wsBuilder.header(entry.getKey(), entry.getValue()); + } + this.ws = + wsBuilder.buildAsync( + URI.create(url), new BufferedWebSocketListener(this::onSocketOpen, this::onText)); + this.handleMessageTypes = handleMessageTypes; + this.heartbeatTimer = new Timer(HEARTBEAT_TIMER_NAME, true); + } + + public void askForResponse() { + WebSocket ws; + try { + ws = this.ws.join(); + } catch (CompletionException e) { + throw new ClientException("failed to establish web socket connection", e); + } + synchronized (this) { + try { + ws.sendText(JACKSON.writeValueAsString(ResponseCreateEvent.builder().build()), true); + } catch (JsonProcessingException e) { + throw new RuntimeException("Failed to serialize ask for response", e); + } + ws.request(1); + } + } + + @Override + public void close() { + WebSocket ws; + try { + ws = this.ws.join(); + } catch (CompletionException e) { + throw new ClientException("failed to establish web socket connection", e); + } + synchronized (this) { + heartbeatTimer.cancel(); + try { + ws.sendClose(SUCCESS_FINISH_WSS_CODE, "done").join(); + } catch (Exception e) { + log.error("Error while closing WebSocket", e); + } + // this.client.close(); // exists only since java 21 + } + } + + protected abstract String getSystemPrompt(); + + protected abstract void onResponse(String eventType, JsonNode event); + + protected abstract SessionUpdateEvent sessionConfiguration(); + + protected void sendMessage(Object message) { + WebSocket ws; + try { + ws = this.ws.join(); + } catch (CompletionException e) { + throw new ClientException("failed to establish web socket connection", e); + } + synchronized (this) { + try { + ws.sendText(JACKSON.writeValueAsString(message), true); + ws.request(1); + } catch (JsonProcessingException e) { + throw new ClientException("Failed to serialize message", e); + } + } + } + + private synchronized void onSocketOpen(WebSocket ws) { + configureSession(ws); + configureConversation(ws); + scheduleHeartbeat(ws); + } + + private void onText(WebSocket webSocket, CharSequence data) { + final JsonNode event; + try { + event = JACKSON.readTree(data.toString()); + } catch (JsonProcessingException e) { + throw new ClientException("Error parsing JSON response from speech API", e); + } + var eventType = event.get("type").asText(); + if (handleMessageTypes.contains(eventType)) { + onResponse(eventType, event); + } else { + log.trace("Unhandled event type: {}", eventType); + } + + webSocket.request(1); + } + + private synchronized void sendPing(WebSocket ws) { + if (ws.isInputClosed()) { + return; + } + ws.sendPing(ByteBuffer.wrap("ping".getBytes())).join(); + ws.request(1); + } + + private void configureSession(WebSocket ws) { + SessionUpdateEvent sue = sessionConfiguration(); + try { + ws.sendText(JACKSON.writeValueAsString(sue), true); + } catch (JsonProcessingException e) { + throw new ClientException("Failed to serialize session request", e); + } + + ws.request(1); + } + + private void configureConversation(WebSocket ws) { + var systemPrompt = getSystemPrompt(); + if (systemPrompt == null || systemPrompt.isEmpty()) { + return; + } + var systemConversationItem = + ConversationItemCreateEvent.builder() + .item( + ConversationItem.ofRealtimeConversationItemSystemMessage( + RealtimeConversationItemSystemMessage.builder() + .addContent( + RealtimeConversationItemSystemMessage.Content.builder() + .text(systemPrompt) + .type(RealtimeConversationItemSystemMessage.Content.Type.INPUT_TEXT) + .build()) + .build())) + .build(); + + try { + var json = JACKSON.writeValueAsString(systemConversationItem); + ws.sendText(json, true); + } catch (JsonProcessingException e) { + throw new ClientException("Failed to serialize message", e); + } + ws.request(1); + } + + private void scheduleHeartbeat(WebSocket ws) { + TimerTask task = + new TimerTask() { + @Override + public void run() { + sendPing(ws); + } + }; + heartbeatTimer.scheduleAtFixedRate(task, 0, HEARTBEAT_INTERVAL_MILLIS); + } +} diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationAiModel.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationAiModel.java index 15058cc5e..1345deaba 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationAiModel.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationAiModel.java @@ -346,9 +346,6 @@ public class OrchestrationAiModel { /** Azure OpenAI GPT-5-mini model */ public static final OrchestrationAiModel GPT_5_MINI = new OrchestrationAiModel("gpt-5-mini"); - /** Azure OpenAI GPT-realtime model */ - public static final OrchestrationAiModel GPT_REALTIME = new OrchestrationAiModel("gpt-realtime"); - /** Azure OpenAI GPT-5-nano model */ public static final OrchestrationAiModel GPT_5_NANO = new OrchestrationAiModel("gpt-5-nano"); diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java index cff9b17c9..9273342c8 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java @@ -22,7 +22,7 @@ public WebsocketConfig( @Override public void registerWebSocketHandlers(WebSocketHandlerRegistry registry) { - registry.addHandler(textToSpeech, "/text-to-speech").setAllowedOrigins("*"); - registry.addHandler(speechToSpeech, "/speech-to-speech").setAllowedOrigins("*"); + registry.addHandler(textToSpeech, "/text-to-speech"); + registry.addHandler(speechToSpeech, "/speech-to-speech"); } } diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OpenAiService.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OpenAiService.java index 4bc64655b..ad6c92c2a 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OpenAiService.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OpenAiService.java @@ -5,7 +5,18 @@ import static com.sap.ai.sdk.foundationmodels.openai.OpenAiModel.TEXT_EMBEDDING_3_SMALL; import com.sap.ai.sdk.core.AiCoreService; -import com.sap.ai.sdk.foundationmodels.openai.*; +import com.sap.ai.sdk.foundationmodels.openai.OpenAiChatCompletionDelta; +import com.sap.ai.sdk.foundationmodels.openai.OpenAiChatCompletionRequest; +import com.sap.ai.sdk.foundationmodels.openai.OpenAiChatCompletionResponse; +import com.sap.ai.sdk.foundationmodels.openai.OpenAiClient; +import com.sap.ai.sdk.foundationmodels.openai.OpenAiEmbeddingRequest; +import com.sap.ai.sdk.foundationmodels.openai.OpenAiEmbeddingResponse; +import com.sap.ai.sdk.foundationmodels.openai.OpenAiImageItem; +import com.sap.ai.sdk.foundationmodels.openai.OpenAiMessage; +import com.sap.ai.sdk.foundationmodels.openai.OpenAiTool; +import com.sap.ai.sdk.foundationmodels.openai.AudioInputChannel; +import com.sap.ai.sdk.foundationmodels.openai.AudioOutputChannel; +import com.sap.ai.sdk.foundationmodels.openai.TextInputChannel; import java.util.ArrayList; import java.util.List; import java.util.stream.Stream; @@ -92,11 +103,35 @@ public Stream streamChatCompletionDeltas( */ @Nonnull public TextInputChannel textToSpeech(@Nonnull final AudioOutputChannel audioOutputConsumer) { - return OpenAiClient.forModel(OpenAiModel.GPT_REALTIME).textToSpeech(audioOutputConsumer); + return OpenAiClient.realtimeClient().textToSpeech(audioOutputConsumer); } + /** + * Creates realtime channel allowing for audio conversation with a model + * + *

    The input channel should be used with a try-with-resources block to ensure that the + * underlying connection is closed. + * + *

    Example: + * + * 

    {@code
+   * try (var audioInputChannel = client.speechToSpeech(audioOutputConsumer)) {
+   *       audioInputChannel.inputAudio(audioBytesData);
+   *       ....
+   * }
+   * }
    + * + * This API implements full duplex (input + output) communication channels. Application should + * logically synchronize their state and close input channel when it is appropriate (e.g. last + * part of the response has been received via output channel and application does not need to send + * any other input). When input channel is closed, output channel will be closed automatically and + * output consumer will not be called anymore. + * + * @param audioOutputConsumer - audio consumer of raw PCM mono 24000 Hz little endian output, 16 bit depth + * @return input channel, allowing for audio data input (bytes, PCM mono 24000 Hz little endian 16 bit) + */ public AudioInputChannel speechToSpeech(@Nonnull final AudioOutputChannel audioOutputConsumer) { - return OpenAiClient.forModel(OpenAiModel.GPT_REALTIME).speechToSpeech(audioOutputConsumer); + return OpenAiClient.realtimeClient().speechToSpeech(audioOutputConsumer); } /** diff --git a/sample-code/spring-app/src/main/resources/static/index.html b/sample-code/spring-app/src/main/resources/static/index.html index 70a5ffe2e..014fddcf1 100644 --- a/sample-code/spring-app/src/main/resources/static/index.html +++ b/sample-code/spring-app/src/main/resources/static/index.html @@ -1453,7 +1453,7 @@

    📂 Batch API

    ⏰ Realtime API

    Realtime API allows for various real time interactions TBD Documentation
    diff --git a/sample-code/spring-app/src/test/java/com/sap/ai/sdk/app/controllers/RealtimeApiTest.java b/sample-code/spring-app/src/test/java/com/sap/ai/sdk/app/controllers/RealtimeApiTest.java index b9098335b..8b73f7c77 100644 --- a/sample-code/spring-app/src/test/java/com/sap/ai/sdk/app/controllers/RealtimeApiTest.java +++ b/sample-code/spring-app/src/test/java/com/sap/ai/sdk/app/controllers/RealtimeApiTest.java @@ -3,36 +3,54 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.fail; -import com.sap.ai.sdk.core.model.SpeechOutputParamTurnDetection; +import com.sap.ai.sdk.core.RealtimeParamTurnDetection; +import com.sap.ai.sdk.foundationmodels.openai.AudioInputChannel; import com.sap.ai.sdk.foundationmodels.openai.OpenAiClient; -import com.sap.ai.sdk.foundationmodels.openai.OpenAiModel; import com.sap.ai.sdk.foundationmodels.openai.TextInputChannel; import java.io.ByteArrayOutputStream; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.Arrays; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; @Slf4j public class RealtimeApiTest { + private static final double LOG_2 = Math.log(2.0); + + private static byte[] QUESTION_FIXTURE_PCM; + + @BeforeAll + public static void setUp() { + try (var fis = new FileInputStream( "src/test/resources/fixtures/question.pcm")) { + QUESTION_FIXTURE_PCM = fis.readAllBytes(); + } catch (IOException e) { + fail(e.getMessage()); + } + } + @Test @Timeout(value = 30, unit = TimeUnit.SECONDS) - void testSpeechToSpeech() { + void testTextToSpeech() { var outputBuffer = new ByteArrayOutputStream(300000); var monitor = new CountDownLatch(1); - var client = OpenAiClient.forModel(OpenAiModel.GPT_REALTIME); + var client = OpenAiClient.realtimeClient(); try (TextInputChannel input = client.textToSpeech( - (byteChunk, last) -> { + (byteChunk, isLast) -> { outputBuffer.writeBytes(byteChunk); - if (last) { + if (isLast) { monitor.countDown(); } }, - SpeechOutputParamTurnDetection.EACH_CALL_IS_A_TURN)) { + RealtimeParamTurnDetection.EACH_CALL_IS_A_TURN)) { input.sendText("Ordnung muss sein!"); monitor.await(); } catch (Exception e) { @@ -45,5 +63,88 @@ void testSpeechToSpeech() { } assertThat(monitor.getCount()).isEqualTo(0); assertThat(outputBuffer.size()).isGreaterThan(0); + + var metrics = pcm16AudioMetrics(outputBuffer.toByteArray()); + + // root mean squire (measures the deviation from an average value) + assertThat(metrics.rms).isGreaterThan(500d); + // asserts that variety of deviation is sufficient (not a trivial repeating pattern) + assertThat(metrics.entropy).isGreaterThan(4); + } + + @Test + @Timeout(value = 60, unit = TimeUnit.SECONDS) + void testSpeechToSpeech() { + var outputBuffer = new ByteArrayOutputStream(300000); + var monitor = new CountDownLatch(1); + var client = OpenAiClient.realtimeClient(); + + try (AudioInputChannel input = + client.speechToSpeech( + (byteChunk, isLast) -> { + outputBuffer.writeBytes(byteChunk); + if (isLast) { + monitor.countDown(); + } + }, + RealtimeParamTurnDetection.EACH_CALL_IS_A_TURN)) { + input.inputAudio(QUESTION_FIXTURE_PCM); + monitor.await(); + } catch (Exception e) { + if (!(e instanceof InterruptedException)) { + fail(e); + } + // do nothing, test has either been interrupted by user (intended) or by jupiter if timeout is + // reached + return; + } + + try (var fis = new FileOutputStream("/tmp/response.pcm")) { + fis.write(outputBuffer.toByteArray()); + } catch (IOException e) { + throw new RuntimeException(e); + } + + } + + private record AudioMetrics(double rms, double entropy){}; + + /** + * Audio quality metrics for signed 16-bit little-endian PCM, computed on the mean-subtracted + * (DC-offset-removed) signal so they reflect only the varying, audible part of the audio. + * @param pcm - The raw PCM audio buffer. + * @return The RMS amplitude and Shannon entropy (bits) of the zero-mean samples. + */ + AudioMetrics pcm16AudioMetrics(byte[] pcm) { + if (pcm.length < 1) { + return new AudioMetrics(0, 0); + } + var sum = 0L; + for (var i = 0; i < pcm.length; i+=2) { + sum += (pcm[i]) | ((pcm[i+1]) << 8); + } + var mean = sum / (pcm.length / 2); + + var sumOfSquares = 0d; + var counts = new char[Character.MAX_VALUE]; + for (var i = 0; i < pcm.length; i+= 2) { + var residual = ((pcm[i]) | ((pcm[i+1]) << 8)) - mean; + sumOfSquares += residual * residual; + var key = (int) (residual + Short.MAX_VALUE + 1); + counts[key]++; + } + + var rms = Math.sqrt(sumOfSquares / (double) ( pcm.length / 2 )); + var entropy = 0d; + for (var i = 0; i < counts.length; i++) { + var p = counts[i] / counts.length; + entropy -= p * log2(p); + } + + return new AudioMetrics(rms, entropy); + } + + private static double log2(double logNumber) { + return Math.log(logNumber) / LOG_2; } } diff --git a/sample-code/spring-app/src/test/resources/fixtures/question.pcm b/sample-code/spring-app/src/test/resources/fixtures/question.pcm new file mode 100644 index 0000000000000000000000000000000000000000..d0707295bf8098bee8e97e78b77d3ac79a22629b GIT binary patch literal 97200 zcmYhj1$Z1eur8|RVIH$%JC2h$%*@OT8|G}7nVE6J%&=kRWW&tNhL}A}&FZH~&bjA} zzHY(vNL8g$Nv*#$0N^M9pePu?Pz=!hPpl}i0+sORPyY9pzi$#M3;!bgZ8#3!|9t+W z|F0zc%Rk?SQ_}BWgzw>+!&mqo{v}i-l#)>TPuZW8jLO1PgntpPC){qhEZpv&(s1g3 zYx`5Li1Oju!uS6_UEx;#Q1ho8L-mFE28eqMwMqj5r73jB0EuD@l@sXtQxg8=pDQe# zaC!I}3f=!a_otpeyrO3RpFG0l;ryRZn42(%;nHv{|CVgH_3$@}BA_@dr7(8@{E-j; zn-@v>-#EE6n*HIl=$QhvgNf;s06%zd8LwYncAwkZvM^96zRQ3Cp#Ba*NwDAgyI1egb#r-?vD>+K8lBHxS+33nZ;ivLL zNk`!aN~fVXL-~c`U+9?}#i`^gF7!Lk|J(zpCjK{9f4Cq}n;2?Y^_vPcqD_m^24F&Y z6EFe`y3D|a(q>>qIUBm0P_UtNIB!QG0_DRw!|!K|=$A&6HlmsgD6Rd?vj){lA-;8p zUlnREtTzP8<0$^qH{5SI3krvNS5iWVN=^x&E2Q|41pFu+zPr&C4j~lS-yt0HD951? z9w%Yl`ZE?OB-d~sgo6sT748A%|K-DchkH2OUJ%tDZpVY9>_8zO$@UkL<9Fq=@=$j0KayOt2WN0BgV|upR6JN5Bbi0bB&v!CmkeJOKAm zxQ)V1RCXL3M9=&KmVy~zH0X`$sRK%&Uci7``KsJe&M14770L`{l+sISt29>1DG5rf zVp2#Y1Ou=T=D-a24gNsaXOzl>SuhtCKrakJ2{MXViBgi3@=6`0xza`XTbZUTQ8p+? zl^clHOvDid;z32w1oQ&q!9uVB(Q*kq1@Ay6a03Y#FdLSDrC`Oe(pWjHGFBd4<*+hX z1uPkbL@X9FV1?8}^ zNm;CnR|Y9Plx9i|B?Zx_RyaiU7x)z3gs0#^xE*eQtKlLz7tV$=;4C;FE{4nCdbkb# z3(vy4@ICwmS!h7K)m6GGVocII#`FK{sd$I4pf!$yg zSb%zWFzAdVRsnHkL0n}ct-6C+-;Q`1hG?m$#3_v8gfHMlMC}4N7WRhCU=^4Gqo5X< zM@e?expKCgA^(zp%HQNP6z8DelKpZ}hDhacXoX3z46F`Y!hvuq+z8LXSI`Z$sPDTg zGnIqN3#8d`pcT@Xqu?zdu&US)Y%SvG7pB20<2~?M_%8ek{t@7z4^!Js)PiL8(Vjij^6W@Vl-P-&u+P;^Qme1J4~8T=bI zhY1jadGd4ljJ#J~EKib0$X(?Ya(%guTnk;zP-r7}k^9QS<>~S=c^!J{uzXg&fx>P1 zKl#3VSH33SLh)@BUzhKpvb*vV`KA0`PM1AMTjF3tB&Ac(32P!6vXshTH5!Y}u|1fK zwZ*sLId}rbm{4J9$YN~-<3LwX1!#H>faZL z%k%O9d852k9xZp5>&iuBwOk-Qkq$@;rIAuQsg6`eij|BKC5fV6grZi8mdZAO?|$*=-E3`?MK*$G*)>6nB~!v%ae@sLO$r;^u6jx0-cq9#+TseRNj>NItYI!^7O z{-I`5BT;!xDw<-+m*iG*3|WuVlAnqF!~~)?K@iXJEl4XW;34cWwi4@x6~%JFRb(~C zfo5nfU_jn)3wIXT*PTcz^OO-vU!^^=*|n8qWa0AQeWa;Nkwogj25Zh3C^AVry^??}}tmkvu{QWKC*3 zbs0^pQFL{>DGCGVfpkkcg=VQQ)M07@Rf8%Zk0QNEMjCR67(x^y{P+ue2R<3EkE`$> z*m-1m2B8^20BPVWxCf3QJGmWM-+5>T7zPG`4rqoX!87HMGGFPfR8SzAtFFNva3LHA z`ysIBumnJ>8f;0Iw);H91WBjOGPDC z%n&b$OU0pLC6N@L2s?y@LN}qIP*Er=lo6^49fX;}R^gfu6v~N{#M@#;>5NnY+1rb7 zyRr}r#`@z8h$PZQuBD38`{^poM&<7oW2Ci`8{8B z0Bt}`5Dh%YTAV~P@NlJ*(gtaTMbRj^@E*K`tm6zMkCteJBtafojEl%#%t6}GTqflw z(lcqFv>tI^7x8Y8tP+ski5JC{;t;W>s1`p8`-GuFtdPxL;kWQ>_CO7xm07+$JM{oMom-A zD2-WjLETG@tBb^wzc5?)OMv_vel_2pPvtYX z3tU?+g8R&#Vb8Hk*rV)jc02o;EzRxW(z)*Z13p4{B6JXAB#(4XJ_5HYm%w$*OdKUQ zQtxP%`Ju{E@6i0Em9=Aa%XH;*RdfluOWM)e|1>|;)zxcNX^ewzOi!hzlaXW>Vja%l zqp=(`Lv{qWm2OH?^V+1XM`>o(~I=puC; zw7Hs_>g6gwvxiL z;*ceFz_w_Gl&MSyKF|;IVnfl&q%Cn8jlfx?j_OG*qkd8hJ(ylbPo;a%3KdUBAUVG$ zGm(~^M9S59Cyfw$35WT7t{(TC?ZWy(+e1A=rqIRUlHk~2yI`fDHTWVh zD=;-MI?yW6E>I>A9hex{9yk#w3``6D6^dkO?iM$aKOtm_ljMrZGcX)~MNFqQGyBvQ zU5;LD%rZ_f-83Jt+_E~XO)dYJ=bC)R6UGhtm)fgpH#3FaMAjr?ux?6gXpv>{s-PF< z^AWt8JIy8XrT9pGHh+iTAdC}hN&iVNqzUpuxLg?kqOf;Zd%ObCn5<4k)7R<7jFTCs zx~J-|ey!%!d1|#LNBxibih7{>u_{H?oVh@|sT1Ttq7-h!oT&G;a%J&7|B>TZnw=3e z1-kkjzW2U^z6L(Rm*uVLJLT)-|LHFh7!=qLs1$q@936TRS{phSl0x;_nQUve3|k+) zV9nt6aS6PYf6X@*&q%U-UO9;k!%veVn1gDYzJsxmDZ(7I6t~^7KaX_UKij_9Zdh|I z5#}62aa}ddXjL;Rnpg~W$Rnlu!W8ZclKc0-Q-7L&4|<8a6nGYh2_6mdp~GBBp@Fzj z%#)5lQ9-M2yfT54PpPgDcGk;F#yw;dtm6;9TkY?r!Tn?R(*W z8psN2*x}p_o)xBx)6o1dK>8((kS8Jwnk)}S^JqJ{47@3?fd+6_!ST(+C$cn?rmC)u z(+@L`vL3WV+h$mw*v3Uxj*PZdG2Jyh(Jj-SRF5K;gZc7aVKNsVdgcG&v3YK}S?44d z^Vu{N+SwbZu!GCwh0GuGC(*63ATsA2dqNQ&Lq zasJ~T(z!kVL(b%^LBIB7dNL|!PR;!MOPA9suSj7Hr|1s(e8D*0E)I}a!`8|>;Kn38 z8?TMm!JeSE`GLw;_#d(hb40!PO{guL6LyI$kd1F6Tb0&$KHXn+NPS*g&R8sJN!-`y zZ+3%ybHu5La#0qmQ(c*|V62qHp9y?%!@@Ip)~xB7s_anCmfV^7&kH8F5`!!F?$S{u zl^UU)Xnba>Yb;^rY&9c$MRl{UF;6okn_HT97=P&7>uzgV%|`WO^)HR6P1H5iw$v2W ztkX2nY*1%0|4}aq^gbecxCx=Hfrs8HuJr{{?%-S|@1MK}c}w%_7QS?@^i~db=F-Ft zFc)kg&Qexp0JDTPQ~5+!yaV_mZxp7o3Bkv{GoJJAsxHbo+BpX;nLB!BdQ`r@gXQ^t zat*u#5y{ll-O)d>!`RIDj)~Wc42kX)u_>-(T%Cwg=JRB0_)Yfks{*Ax0}CEyS+d(_ zoz6U&{Vk6y$afU?{Nu0Ay^}sE(}{~rSmQH(- zzU9}=oN)z}Jd*-{aUJDb$}HSP4WVz-ewwGt(X+{XObr)HL&Z@-jIe}#61e4i?928q z38sY}2lobgvhh5|+oh)P7CcTaVG1aRW|m>Laed^a=(e`Q=5EF`=3lZeZ&bH5BaEG`NfCAJea!7l@60bu_w_lt zmHM7W-n7WjPCwPu+}1H_WbDA$#<6o_Z^UemdK?+FS2ADLmLW;m#SZb`alJ3No!vCE zdFGQ$LpI23TCm9Z)ZdTKln!F)z=B~{-jg6N9%hA+8^p|w7n=ajdtXM~&d zI2|$03~x36bKktc3ZK#4(BIondK-k61Vqm`wm_^Ro2W&aPuhji)r)7P3`^xpI1=|2 zJ5eH`$Rb-^ZBye9>fj4EokI!G>Jm8+i?gb+>qH(#gn9*)s z9^tfJ)|F(MszzxX+ArDz>fVMsw(^m8<7cN#D}AJ*wo>nM8_V`9U$Jce;v?g_8zr)m zG}<#JEA_kjjstvy#DCWFaI20JNBZ`Te>e77a9j|20jHN{M9|U`|m<)L9X+H>z(JM;E`7o zZ%s8L=12Y&w=b5BQ>Sc99u}Qw-b8K{nh5U$Kk|;JKmSzat?To-uOGkk{t46X6_n*B zz-3r1X0-mMrCAIfJ1L@}y{2WKv7z~hVWUQ)+hn+Bd}#IBheypyD3(&XxGOcX)UxE> zNq1rwoBC;*5SfAc`76`jzK(p^>-CA}(VyraFTUK&p6l|2?rF+Ic8D#XxHdtE-V%E> zk&Lfz&thtGz5V&V&#p}ct8zbO-OhdFIO1C6E5{!bi^z|$1Y(UcRm=!>4{T=7aa3S~ zdy;FUe;#blRMwp`wY7}0{T*XSIGx~%>tX$m=*=DU>hnuxFrUi4Z20W9xj2_EiacO6FCVTccgA zA=SrM%t>_@T@X#`zr#1qnD5`8*ZQya<%lb-uYbRH_|3jgru1gc``kEv#nhsehLo>b zPF<{C5o^lzxO{UvZI9r*+}+uavkUXDxSF`9J9n_#iIemf)pw>cIe{6k>M5stlk+NM zq-2iF*^@Ugk0^ZWO%b)mjR|7$ywYt-uT33Q+Fa^PkLQf?vA^& z?cR@PjX%B0Fu9hpV?e5&wtceE*0Yu+wxjl&rW|G-wio}vOfw&kZV_WG5>;$q>ZGc5 z>({SUyK1|#FO#;!EHK+ZbLY7X`K5B(bv@(UhzpkgG9MKA(6X?P&m>Pa`;yz%nb)p- z(+)K*SJ_d%WQ9v9_tXYP=DbNXDNIB?T>K59mw%mxjx--g= z;9f5ktQ{=ro#^=P7#!*jzUc4RC)*C#cSH=aiiX|HE^(EoTUPF;d9T~NTJU1Pi&yVA ze4m^5I^Y(*!}WA6;~v|68>7__?Iki#Q+k8-AgGCDm zr$6~#>_gt0pCspX}1C6BZARKJx~-VLq=&OwDY z-HDzpzGlLI@D6cVV>DeiO*Pcjo*{ZEW2G2&eW1YC!F|&`(tl9cMNcuAqTa->N?4Xu zyI4We-|-Lat<{l2rNVkYroLM8r0avpw|m`=zkB`8oco23n|@s7DvIqOdx1&NB&Cyk zv=5kE^(1wada&`G%^1Ix(uhwV~pO>ffu~DPJTp!q}7eE;eH8g)VTP1ET_`#cCL>ZD47uKdoPG_?Ov2 zG*uFVG2Xg9pJTFv^BwXG;~eA)=CtKv-LIy195JP`qGQ z+L;fl?+v{&>Uz@UA(wvMxptp=tID2IFw@goIz{c${cW>HHnHD~YnpO1b!4>;)jL<( zP}Nj!U&@Ds52n*{f9LnKRj*n+aXq~7VB(t!KMrIbcHb8-F$bcSC+;r#EU7_4*=V!< zm8qFdukJ$Mr0dfesz%yt`Xn=HYig@w`lvZZpTj$WmCBURg1`ZP7ynS94j&5nq^j~6 z;MDiGbdOjZv&?qTUL=-`Efv$vuF}jW=7*@P3!f{!8TMk#;|Y&@J$mx^&D&C6p8R}U zkXbm#QOvctV7IG5@Q1jHaO$b3hY3+-n$~DocVN9ywQ|aADst8^L7G!gJ^imQ!{2@V zSo?F)ACCMkp-y0@KEhnqSjDnXzge|SeT{60FIN{;uV)BDl%bR5Yy6w|gVC2GzZmc8 z8tdce8uEO}$}bG+*$&=h_gUX0Uq62h@g-4Cb6?NtIvXoRSB|L`{UFL1)z?lKD&YD+ zUf#v*B0pPxtoe4(%bU+TK7aSV!-zjEh_ z?MpJnk4MGn7UM5MD(9oZ!v$;e{EmUHuAU!3Nw^}ORf;R~MT1xhtb|{&H0?xn7rQE9 zUexs16*0GM*P`o2p%sFzi-x3g!AC$TgNd>X$*uF=cTUJo%PZ?H>F5-4$fc!RVz92I z>8G_+#1J!XHtQNdPk{86$@`jfEcawuY+Cb-L+LxSXBFW2Gjg6}reti%+2`!$4+zWX zSNcOy^Aa8>?<(E6)XOqkN)(MvvGB}TsV5J4jI(k1!E-?$TTcAKXA3hz69eg?D7I*@ z3ZIL&qsQt-TCdp(W7{W8j2IVjJTlu}FY=jjt2U^qXYgv*;N$3n_(Nd;f5TZnzmcb( zBJ3CO3n(o0>?wW zd`aHjz89WJ{vWV|B9dcF0}PL?10%ke7h5LUmRK6vsz!+R?h#Y$r7ZW%y-Yz>QSz%$ z)mOf-Aa8}EmLtMHncd8nBOtLKZ%ZfRZ$KybEtKZ(;Gf{DRTwNt&99qfOW*hH`9-p|^|xKZC6 zUn)>yW&a}AS8p@ND5ouOKj7gm;L-GUHDw&5e`EM!Jz^{WiSHR|5X$7f2a5z>b2@KDUtwU8OY80K);i*IVshJM z-2SXjFZ{CSL*|c1pRx2e!5y%VVVCWONo6tF%g5}FRK+cemaTJQMn|8weMRf-W+u$M zUApS9Q3XY{iTK|@Ph)kxXaDTa!%#!ihJ z5mPc^is_TK3fUNIE#KzOvGJTPcreh)KgIpfUBy?*>2tR840d#Pe{&BgOm*tqW%8AXn4N*=7qk9Z!RQ>1$QE?a~R#Qch`n_M?}chan+%(zdn z)uSueE1J}*t@1{$h;O~Shp&p)9S~TJ;Fa!54dhMmmE2Rw0y6$wou*%FZyr?;)iquh zebQdZI#An<8VLG}=^=*wH&o#76#VM_U>wQEO&DLzu9AQ$oy8h!MquHDY@C% z=Y9n;#-x8j@35Ql-@5k%J`im*`y(499w~Az(GxSk-pqC^`g2^5WJjtuX-ndS=n-}? zs)OaZ@ip;H6uF(=;=yM@Q|OxblK;iuM>`)mTy6dz-YRUDc=(eTt39R9FrT(5X3jF) z@Jg4Y3o#~U1$G`*g3X0X-09#2x65&>ph!-woO?MXa}2r5v!imAoRs{id{<77+=T2h z*@^l43$8mx`-X>B@#U$V`o{Kc2_@rVA{*M(hJJ=3Q3Ioz#up@Pig^%QE_zS&@CdEt zvf&JsOCHAE$|md|w4;3)U`lI*VBM7@wDYiwo5J1XKZ_UTPgqH!HhzGZue_1VK@p}R z9N-ohs0>F)Weny*D8O^Mr?{2x$`uKT>TClF12F0)1T7RJ}}N*Ys6ks@wDwdJnaN+(`_?UMdq|UAe6kC)E=< zw1d}%{S_LLKNi-a-Jsjz9cie%L~e@^q$g5Sxd=kv z9>ZD63ZTQ@AfJT(i0+~Y#cYjL$1ftJFpg9sN=H$tRAp*3MbdWKNH?T8Dwldk4X1We zwW-eJabh0612bUT(Z2UFgn4HpJmauZ1{8oUSX-5yF^)JQyx0286qp0(XE78-R2zk91Iz=?(N8I)RzWBrqkINAy>^J2Qv5Nq3?D zqHd5WWIKd;13VIo2R9MU{2A($^9YG(jJ#TS1ygj8kh73HQ{@J7Q-sJimxoE0B%L%> z_{3*(YJNGtiZ=`Q`67HdVLZ|Yvou5QgAloQ@@1F?8=$@ay|5he6j_D5AF6^|ia}|G z_9-4BSG#(+XRS+h94x#g#VKIc2&qaPEB=9Rmz*#_J zO|hQHKVmY%a`Hhewg>b@`8L=DEE{<{oB$f^C-wt>fG;MyQ&;H=%ujkN!_n)=+0+Q4 z6&^u_2os|16}W|OBxfS5tgG@CdWp5*4zwXZgnUJYHh7tzDBqFha7+1xVzg-CdkZ?D zoFGX-xvk=oN(z<5?&4!fEBBZF5yy%jq=^WtD~4*AC-0Vm(hR6VUNuJSJo2*9fGgM$ zgg?DTb)Uy4;g`s0sti#Ye+&wdoq3F{!UZfIzl(Lp^N>_YVk~|NZ-BSMdSQpLRK!Oi zS%FL>HsaR-hONf8W5bmRa-{T2w2Fs?MZ#RZBR7=&J0x?PgcPv}LJkuV@-!Iv7$nN$ zrDWu3@HaxSCWBn?1|%V^+JhCt$ANr=el9|^wFPSAhx9k{Gr5QG!HvWU!c8nfDB>G( zKXD47$a}GgL}z*sRutTkV`M$4!6;A-`3Kxnm!Kh;z$_6DN*h35Wk1#sycPS)6;Xd} zkrm;!uwVWFb(mi1Ej|(t%Pol3u%wVJ-4nWV@6ir#6Cnv+0Lj=rG~%*A6uF+%;Emuz zj3HW4b~=Z26EBJNBe7K6VBM%-%t<2}h=R6S-d<6~TOAvK@s zMR|!@WHaI|b%vQl)67%q7TJ&LLCIteehX_&rXxG`0k4l$P>v&9v>CE5-K5^K37(es z$wL(f*aPaw{|OduO{g<_jQ8{RSxe|dsDPcoCbAW{Wqe~+W@qs0rC)Ff+znpJC4{2# zZsbeU0$jr=as`!2)}Z$>u}oL>Jyk_!5wnl!p|NRisTML;dLucPUd2qKZqYbhj2=Z! z0!>H~VW36uPsqPP07Jnp`~kKFp^7Txixdk({Bdy}!Y$RxQE4Sti*3iog|dR%f^&mw zf;$3-{5$;zgNbYjc4DX$J2*5%xGxnV6kjLxM_wc(yoR*hk6ovZkw2+*^a*;gX0+z6 zdbz$py-_<#r%|ufw9zH#Pv}o*(0;zYv2L*T3tfrk$iDbDoJUTYR^+!a1`I`X%#t69 zK)A=Iq4~AF*i_088}ru#CAn_GZ2k$io(qKbvA@`;AQ@~GY8E;g{ORxPU%(~^m4vQx z7x9Wv8+jW z>;KmKwEt+t7i2Z6>LP8x zSAZ0GyzG)5^3OSpO$xF6a;_ssgzorf1?T%}_@@Ml1&#+ysMi*9Z`pIflm6>&ue+7! zn1}S&@YV>lW4nfw;CwE`_m(=t8e&U%r7|18M$X0BBTtjF6r+8l9<8Eut&M9<+ssSM zRV^CpRO@N0*|Nhn-Tu{Tv7{SX7)I#x45jriRXfO?_&wqP_C?t%mE_9`ix8GSosDA? zxF+Z=YNGF*XT5K4pt!e0;2-}9@AW{rV1?jA?{sfv=cL^6&e5Lb?&#n!e};ElC_m&3 zw&q&$Npcaa9_Gh*hSVOxcc1_lTB_^)}|dLR29g--iZygfX%-CMo){ZDr?0fD5q)xwe5W zq5AwlXVnI^PxnSQ%kbKeqVI0)ZqrANjE=R>v<x%O1b~W-$ z^nP}(b*Fg#^2E9}IoG&WI|?183f|>B$?Nal9D2aL=4+u<`ZpHis=-RiSNtn=i2g;d zWM*iKY2ytV;|Jp`Q*Fy3yE~$5bnj@+-YsH_ov~H7uZf%$K}M~#-7`fQC+mmlW3>%b z?TLzD0sbCtQkqI5c_%+7u*LtA7?362>OLxp@G3AVziV{Q6^}cmkL`+O_q|2_e)r$~p zvC&(jszelvI$^0~DrKsrFQZFU)lxO59Qb$gDHw$L(EHpTP71yYz~B?s8#wBl@0;k2 z_FQzIbp3F5@s{&Qjy{e=XQjeT`4NsQSvztDJH{85_00FpX2%5D`Q4$$p4Dt|a8`~- zYq`^8Kl&Wgmr0{L>uwm{nu?m5nm^e!QK`{wqvzSJk+mY@qol|ok-MVvqXyVyv%xe* z@6k@u?PT&vnsh50@Qql!+(J6a7Z1e;L1=dn52Ocn1$KB6ytUjP-N!r`D7CndceHa{ z&g<*=J9mHH#5`^OSm#T}O8=u^ivS38^>p$(`B}mod5SU-w4#2Q6LdJc1+{7mj8o#2!b zB{vlgvsZ)p!9_tcw=LMuzs+C8Tf=?PEqgzEQamNxO&yC0Ul&fytDILSzf4Z^9B1C$ z{J&iR-)#Rj&pFS>K$LGus0aU%|AZZ&XzX8tVdhf1R3Fg1THUC(ZnRyCm~E#c`$Zj$ z#bT5QA+m1dfVdj*r6Yw1w{?nTrg^gNF&Yia)VG*<>BMJSx_hcd_n8H_IWRIUuSpDH9KlKU%TG%&0p$8x|U|GS!XF>pA&T|dQ8;%sF9J=Bj3i>j9ncQ zh`b-wIl8%Jg1L>+Xy|FWr|rlH)OqGPr6Z2udE!ZFkhqR7%0CHS<0i6=L$3l|ywm;l z5%Rdxy~S13ZE&hw9UbNJ`{XswZ<3dm9hV>J%rAWE9_c>raJa8}qk?LF58s`8&grH1 z2nqDl{n6Xj9r~-Pi~6j_X8zkU$8z3ovqjn}MW@BAjh-DfBGMc&E~-`}Zf_8I)Yit* z$kbQ!T|HXYU01+#Bxd1%BmeAD$~MU=ZV{>qbA!gvlb||Ojw=jg_-?uzIV-pqyXv?W zIF1xjjv%^kDcDe$=&0&ySy;Yck7K4Y*{cg33ziB+2(qNR zOXyEzNvuEC7~6#Wm!65c_}#)T;aTWmC^^(Ka5nG?&2!b=n>dJI&bhq(84L0UVp#1^@`P|i)ugI^!3t}&tfme~u8G_!gD%6E^ zCv@|*Jw4?IwvOh}hQ7uG-2|;mvr_$wUV)dx+faWI zYvBv|rtA>!@{ib`Y;lg~E{DRWWwA zOt#*!PB%3)?Y2%g&o^ALd@+yEM_XQ-cWYnkM;H%lTB&N%$#i9MIdPGCM6?7Ol?V7R zcw87DPZG7rJ97iyhMUPYM{Dn!Au+JpzcWzM@AiH5clGD{5Bq}N65ge*zuft5*^PM& zz81j=$g5~eupiriYrxLsONxD^{!nmB zpslZ0a3gz@`-@u?>LjGdcZAN8O<9cmYMbJXiEC6Xq7peq(?nZJwU_FzxofznyMw%L za;@j}PIWKid98t7po=jbQ*Tr~Q{7`OtJaexsoG=+EXEvAfRn&*vfB=&e{O~@E>vJ1J% zp?zrQa35bAof6@tA#xC%C~2Z>2KBHaI8P3zTM#$s6zyzP8LdwCn%A1j%trM_x))uaY(}f7&*=QhM|Aq7qx4nmB1elc>*li1zKT6g%NU7tQjmT zb;G-0$N2BULbT&iA9-fJ!CR9TvDV-8W}}e<7cfFPUnp6QG8&hd4xT7Uzq1pjrA$^z%Q2?m}^PN^plTgBudK>O17~ zh6V)3`5UtXg4cpyxznMK>?*XnKPj%317a3`UeOZGF*DlX97dD_la}6i- zrbz*4)hR>|xeI{w9eIpYk(7Z}=*f2^Ieb+pEwoEGgkxNRZwh!S*7jcGEO0%-H|hnR z!^V=%ZD1pn){33G?71eBm^GB^F92_$JMrXV+H&7;1gES3^XJ`lNFEW!HPG#ZE z2?M5w<;ZPx78gmDBj&RhIfCfJFBKT#D_9N32~(sBV3AUkjhFUdCAjlalDq=#iM{8L zMZ(K-J=hU=YiUiWwqlnSa26#=atNPbI=@XwBrda7b{~}{1J7Z)AN?h`iGHip<5Q^h zU@||42@)S+0nq?kPu#$l(l?-%*rD1l9S65nPVxobfYdQYA`$yrQ=NPU-Vl?BI`jm3 zvQz`q)(l5`I1S`&bP$Dvm`%)OG&DLW-+peyrTx+ByImLZzqOwfuu#L8kZ=#gWj zeK4MHD&3&B@kV(gv5RYuMHBz>1MxS=`{^S-mU_ibCCh48@scpvbeb6Gf1nz#o{sf| z-IOlcMUWE%*l6lKgTu-2qZU)DO6#;5X&<>0GYM`@PbObjC)HOKSKam-30n8Vds=E> z_6l3ryF>%kJ?;}culW*eCHU00#0y*}VgtOyzCgQgxt>pO5xF{O#x6)aI}_W5R}f^R z$rR>f^M#T43AjMg@-2vqc($+zZb6>uC14rhAJ|XxOlU10Rj=R!$~gQyeO=5F8|hb| zXBX>g!_I_}D61S%FXo*Li=W08l6mY(rXE%iTSl(uyBbsZZtm&{g3#7`PPbf%myTl9 zz%tq;eS$TZ)7&}buxehY3$UvSgr@Q{v=>%H?1C?a=g~Q>rJ<_uy`gBZaiFznH@qLH zLCuiO(oMEG)d{l&zmW@pMXE(Eg&zbHfhniaIglGfyeJ1g+l#`)JclWTY~)*|-i;UY zrfAO+h5iWDC-r#u4@_-4;XDZkYS#!qz!l9F{*$=L$_eebo4RB2Po!rA@isI^ZJ}EQ zEE*?XOmX0Kgf!|4xn4L&;4)44guTjQ-K*SF$Q!|H&u8g{t|oVa`#?v?YtZ?_r_v_;9SmS^r7E(GJ`1x%2Qd;` zC6r*!1bBSEYD4INdYUqitw+C8t@Fl^v(ytXO$*24Ubc$lCG- zA(Qq=Cxi7EfgIr(L+{YoJ^R&m{9Z5%uZmB^=lC<3E^>YLr>d2>13ydu6&!{E(-|K@ zE+89=Dl#6QB7PN4Y3g8@Cr?$5De6C?ey#Kq>oe7nN9;YKHf9(1ke|>@6^+fnv_KHg z5J&l)x~{>&;xvO>m=gOauM0jaq z|5jo!L-FH)iaEBL){Cl!*$Hs&=wh}N-C7N51&s3`B@=O z(-!aT8;vI#Q+<8mbGoZi4Bq448lDLo_&cga!VYoK097f=e0M$fKsh=CTJgp@Y7+)GR75kf}Zs`alks(}kZ*syvpPsTRZ! zVtcw+@UAvL1V}Tr*w;!sN#WQAnlB2$-chXwSNNescWNQm41YnF3GUTY6-EAxb)IuH znXO&!&88gcjb53KCKmBtbi{qK@C^i2t^6L^j9ur3YK93VILi21{EJ(mrTGkHJicE# z2piA|LM|9a9to|XQ?b*&4AiFJ>#VWysVcx96tC*--jZZK^(J%(AEQZjI_O;8a>sa0 zbEPqytmF`7WjwMe8imNjC=JY}7h|t>b&ToXgQu=CPFKpqV#Vce$n99y^b;Xc=~m ze(m3_(n$BPJcU*UkTW=eKI`wUTCZFTHZV^JCG%hPccn9d!TJXB7JeFaQXYWSR_cKY z=v?Did=VT5o(Vg2Nx{2ttnQ?52=!j>ihaQoxh1N9u|)QOCRzpQ1mA7MnA7v85#al8Weg1Qg7$K1Gsf-;Nl{mv^;H$Hy25ATJ4`^U$HU48bSiooF3VfsBHSd0pcm9p zUP+bI%K~cE3VDFM7oC7Eg;kU)keT2ra>|*9PAKtMS8R%yMNGr=VkFg$i&g)}Uj-k? zkHR#~=3tC^gftwT7wgPD*4zjjCV5aPNN6^5U6eueC24_@Nm%4Ws<>Pqn?+3JuTu?# zY+|%H6W4>|up)j^NT!n{lRTY#4mV?)#D7(r+3z@xNAn^e6b7u4Rg@i3rNdS#OQdw- zIy@v%*eR?$+^oz+C#EkV}ERpcK{`o%b&bvN4r7pI8ao;v0lyY787M1H2#H4pz!0G+u2?4fNhNhB^-p z3Imx_aIhp%F7Y7V1nwf&NL$bw-b`t*>Lb4#;d=9>NPLI%9A5*!07@=Oxwvl)!H%C!|xno9sd7`i=5L9E$4+we%88S9Zw@u^H%; z;&qsS&4qi=|6Pm+gl;;cS~yrvhyS>Q5s05gyqXhHNc$`JST8brEr zPwz&r*G>^zjXJ~=?Yyzr=n8Mje}oO$ctbokE)$RS(qsdymT6!xT!O!O4|&oUP1J zmG@7gFkG~?2p=&Eb@V5CNAh3d6VX$D4%Pw-SEzs!cT?ge_#Rt~Ys4brGtej1jK)wU zeT1f=HRulyhynUPB8gb?gfSGApE*Q(;<-_q$k4|Se;AL66}V5hOJsy`o~Q_o!b4*Y zMr)8>fc(#&$YCCw08>K3{rqN3G3KLke2z@j&w)15j#y*7CsOdLf8be8M64{vYx5Ir zjRf4&6CkuSCyqg#6F~%U{|lf#oMnu_&x$AB;!3+h4-`#&#(io$LB_1PE5kK=EVh)G{ zKM}Q^jQs=yPRlT2yHN}$kQYW-xUIj!ZM+cgqb_m7=!{W#0-}rxr&vLvKXKofibxqu zv<744A)a9^p@wQ>G=OSn6j1*{4KPD4{kI(OT5?pn6*y4R{~F|Qp9<9&bK3KdXOA=FMWx(xFfHi6&iuN^%|P2|42*C zvGbaWQJ9GL1r`$4ote;0><7=G6|`XM@g6$h^XrY9x?wCrjn!!sCJK`8@xNC!CKG9T zdtw(NC!Y8Xk==}ViYqORdw3ZmRtvu3xCeTBiSd9dHl8 z&%la|(R3m@esC37=(-C@; zKZqvATVx?KjPqDIqDUI|bri1tGG@*aeX!95EBOxS(k^J5phIf}YL9MQ)B9mY_CWN< zfmF~KtH=jLh(TtPM?nEfLUc93bMKN-&R7c0PFLbzj9)Fp&wb+-)HtoE=U^NxL}Uci zpT;KQfUy>9&||C>vy4Ny|CK>z$QA1FjbrHKW$qktQRWQSEBGN7+wmxD$vQ$OtHIarVW@kjjeO&b|s78n2 zcY|m_+#s#c^vOga{iQw=Yexl)P#>d#(UWMRKi4yj=XxFJ^zyWY;HZ4jn?W_y+1LYR zRSsU$29fT;Zs)XdS?>!aBWD~3ujmWr#%IL(CR{^X{Dj6>N2*|@eF56Q9;|C$5kIGm z%|;5|Uklhae zM9vAsaYdt}F%7dn)`$c1;3Rf~Rj^*982ynO-)h{}mqO)I7HZcT&=?9tBVrsP??18= zh4Iz#f3_z+KrQxx*o@DX!IdloF(KQqU_=fP4EYFi@)XgYyaKgbJ8)-8l7~qKF8C5q z8jT?nC)-2@vMw6$`~<-=Wa%R z&{;wt_QV?>^v~LC=mvw@S}m+v)v(-1sUeS%M}eWS6^w}WU@PRvK`0)Ngp0_jVM^W) z{o_(aRM%;(jL&*lkAU(pOcaK8@d5kG4#~i6>lNzALW|XMyJMSPF$} z$z|mH@*Bx4^^~B2k#a&srQYFg()D04xF=X9kQ2xXczwzK(*8rfXy0~!Rln-L=wIe* zLi8zLLmlka4)+9uk!ThhhlmM69m5$33zh==wn zQR%T#!qfO3iN_OdiP3R)?2Ap;%vRe6TU&dkx#R%C(paWpU*!)UPy%)Yt+}+DUr_;opBL1$vQ9L zQ9_ZVnnk!oS3FfDsqo@L1B?8dv@s@MOj~;yaS}hpyv1_bHqH8jS!}$~dBdunmQp-v zITsx#9XYuT-4(q%ec8bhffIS3omKKmxl0B+_n7l!UPN9XuQePGH9d}C3;zb`0=OWW zI4;H=-#*{dLg~p#F=vxD$Ip%VQrK1WRgtD8mKF==pIT5Y(XZ(DGZ|)|6B;O9lq|DQqMRF)dV^6@9;I83Ib{5I`mUSvK?38>1U|l{RPD; zpQ483aSX_9kYmjXWy%?*oK4=+Qb=y2eN!uGQ;Z@|ou{*p_#?tL%RxJwIk6vNx5oYx zUD@uo3^jEVcba>ccA3VB-G%a^kDjB8N<=W%*T`Ks?~e1=y!oh!7ja#26w3By*)j^G zmrk#n-Y|p9I_X&FYU&QS>$;qI&b*ww4W8N1vwZOP2ws#rhDT~Wj56AHraV87X>30k zbt-O1VY)!GcrtE2W~G|^un1WwE&u8w1M;^mP(S%~Qd+^pq?m-J5r3GL*j_~Wqk=YB zXvMUp50HWI1K;34LFroXxNlTmZte}w20!O5;JB9gPv*R=tvS_mJ~(FkV?*_%zogfJ zt-(^^6s41P0P48?+(_ZJCE0$)=CfV3yf7^mQiK_TMI2!%V6I~R!^8>I_(se;Vvtr% z)}=n7EdML00^9h%_=dQvHtT0*<*f4A<8voC&gR9q>f~+9>*4qn)%TaK zhrUGrsz83QX^u+$L*Jp<{agD?*J2n}F;$Mdo!BUUWODa-Dq?qZt=Mj{T7hZ>>KEx; zqI|(sNofg%^Zi z=|J15qO4iiE>NP*cMu*6pM)$>DOxbi=nv#LBcB#jcWTu^(%V*{%Ha^QgfmiM?<_Z~7H*wr#U2;&cSKsusOUzvMV1bh;kL%MGv-^S8RoUN z=T=eJP5r>`c@w-A*e~fTK#qHZYF#t=B`7~Pq|>M?o%Qpf%HcWTRH>xY0Dg`?Ll=W< zLmQqL`b~9Ctx**rF zTCVCt?LJdg)R3yH9Z@f};%smYPwM|TfFtyYAYVFgb{b2@fG{?ST1_pXr!g}bk{LzE z!`;#j4u@A5Phj~W0%s1up@f*O;naEybb~D?VQ_MEz zJUtqeyV~?mu=IIibg*PNF`rmXY%Y!$yNE@_okCT? ziC1jnck>JQn*3944>uTEZJpf=s+*r#%p@}V=^8W-7om;nMII+w;uO%^uo}yB2d>1W zj)#w;sQLg;;W$vmj)ALn2nvjepe&)R3O~iGuo5=OdEs~AyWzLtec^3z2{jG-r6ffh8B-NfE!Ke1K0Ra`bViND8de5_y-iU{?D5yE(3 zAfjNe@J6U1t`i@E-St{LEDjPIinYWTQ5QZ6mxXykE#VWi>_z!W+zYl6+n2cx=3yo^ z6=cJq5Q9`N;L z%WI+7@`O{uGs3@zi-vQg^KgOvA_YSGLK8xzLhj(J;CB3q2Z3r#Xc0bp7v!Zz(lqIf z)H?hg-j6GCb>*fq3Ugr?$ZzGqKl>dg;IUFxpR44iZZ3efnBQu5ZF%v-R z+r@5Yx3LGl_C+}I-!E(6IKj3v}!j0ipa@V+rTpDMFui!T4dgH?9LR!sPv*0Dur4~b)@2R5+9!JR0m7EC?k| z*Ay=q2qD7L*ho@j36-0b3mk-O26yEf`cH3nWd4ur9q@RdXn)Bn%D z+zi&1K{mmCd_e`M9`tEip$9VKm_guProd-ckR5}8C|bhaPc7-YnfY(cg>Mn+(tKs(-& zDFFZ6M|unW8!gb+sNs}>y0{-MyBDCtJpdKX3&LlTwnNRYZdZyax3R(o;nM1k`4g7z zO3S5QQhAA%euQp;T!&{BX3gc$jnM5-DwJs^sS4JTzoflbgYDst;dvMfM*dY^EuWGz zF*{q}-1bI^R)1BOVKpxeQsffwLcQ=|_JOy@sn}Q=Y$|IXI8#!zZ*|rrp(=B#_Utpd&u!^O!H`ssJnTW|U>{n(bc3W|9X`P~{)1~OA)O0Ez^@>~scb*A(fhEY% zMI*a<7|*PN##3;2yMll#=$YCbJZT4OwKNW6cST(fHs}borP>CX!C&!bWt{S5C9HTc zYjTw|B=WOgKjHFm95G$kTe%4OO$`X58bEyqpSg0axeIr=auJ5mSO{Sah$qM zy`;L-Sgk&O&OGpFMZFmauPORvf@khv0d~g8DobS^nc-LgyjB@*nN+ zJef(xQ!S~vaP8UY#`NFxB|4XmW12G45l`uG>sDi%vO~aRJqPmWYxW8v=pk02TZp4Q z7@<}8S_`%i>tr5cSJRV;W?s@e=_!b&65tuXLoD~F%2O)l{wmaBKt5j;(PM-C_x znYMvO5-KHgkje6Dr{RXJtYxW()G6vOs!zFqRkx?oNGT73m?1yEk>)P^~W4M z38L)>WY0>0`Z@&8%LlrM)nF=ee+;S;XDsI!uKOWlM>Xs1?Cli-;y2w!ALwKzz;1@Ld18mUI%&&t>d_EVRF z*Lp`y$7uMm@0Bnj@mdi$Y-@oG*%qY8_F7l18=e&-vFn;y|8YqcENU>S9e?${e$M9xemii6HQ2|mt;q)e8ino~oO-Ath_Q1_`EilvKyaa|WY z=Mjjn1$ahHLxjwrC*#lQ*ip^HXkb^3e-DA)u>qY7?Z%bx1j9)A9 zu>z-$rPLy7J~fA$P0gg{frdSn8isYR3ubUl>KCdwWv6uPR2}3;@)>yt&y%fSJdDPe z)kj>#L#>fU+yifT6)_E&*xwN8h2Yxtqr1UV)bMr~D~%bbO8m!vua3;5Xvk0~zXl)q z0y4fU!Frzt7H%uODX0d8|4)n#V3mFb_Vsztfm5)4|Eul5nPQW+9MQ87-%cZ<6k1{Z8ypq4N*^?hikb8JtBcTaed^c z7ZSU1pWpwV8eak)fF9&Pay&Au>p_S=NS-FIk&m%2d=IYpPx2f29o2&jjKELwD?axP ze|t{8A)kWmdJD7p4EW<)$!+-FWw;Ad@dW7$W^{9Kw9DZm7Av{|PWfx%F6xFUsQb-F zJ*+2k!?i)tjYYosBVzvm$o795T`^7-jFQMVV+&#^m>W6ptbRe9-31H(lD-vg)_I7x zF`&&)1~I-rJi~v1Cp-av9|0C^XT1kvvAP})0==%$V3G@n%b%JAih5Ce*FI$1({wlf z&p(i@FAE~?ed9P*_r|C_o+4x6q^^x>%M~#DcJpP})#~bCT-d3QB95 z>_j`GGIqcLXzI_Qx|OWA(l+QORP;*F&7pr8uf$QSm`n6hdIPaT>7-sX=3&qN${3-K z&`KLsP-BRvCs8tLhmS~u8cY|a=8%6<6Ueg0F}Q5olDQxyc7?j8FVylW;QVJB6!Zfh z(2XM3s7TB=BDKdxDe70O1WqCkGi51iw|xvN@m;H`zbAHJS0j+u^`=Ui)*7+9!WfI% zd^&VNUqEA@N0cWo5(~ARid|QU$(Y}5jA2SItrPhPHRTc1H)E}OP`^oZB+FBsjC7@{ z)`wnCAJ+;QgQ!YIo_yFC!p@*esQcAV=mHT#l%=LYsT(jl)!YQo5198ggOWq~NARDrc zZfJ~AN)eA)m9Bz_s;W;Qi;>HTJ9IfJ7_O<_68_-U=noW0y^M8iG1-+?jBDx&R5?ZC zAT6?zzDY?!mxBr68dP8InI5nFPWEJj`avz*IHT7hD^OL4Hrg}21+-HsL_E)s4Z?}~ zPpnA|K?ZIN@BL#k436_Gq7wY;l~4<+jU4tseF#kvoSu$5M?n$Zly;I+^c6VC652_! z0CYgb^n2tJ%+?X8+)gEQZ4Hz%N#q5+1QAa)*BYt>-G$kpuhUkN_o2O*O8WJk+A=uY zdmFPf+8`+wquGsqLQVunaU1c9vas1;Xm=s*s%N!6Oi{Kr>hv3luv&w_*@haSelQ4V zqN4T6Acpov_1{JYlw_likVhNp2yh}M5HINt{ft%A-F)ann>PYWNv~Th7## zH~C9zlUad3Nzi!5WCYL$cUq+3)MZWjBD8m*6)#)e9h&xEq{4RyFN4|D#mI)E(C zRWZ)W-LPh_QxDM#`Bg+W(6JCLTryjUYNVA=*XRed&rCQ!qN18%(bm+D`ioF9@^Kr5^(}tLbUB_OqI&^9awUxe0&X;GZWr($-b9^!ArQ)7xJU#mH~jqImQ!~Wtb)r9Pe{D=-d zNFDu*(T8>s9;ul!hutl1(8v1MYKz!vxXS882KENI>UH`cUsOLJM;XQShQ=l6K#MTF zhj9*24nCB2-pui@-Z<(G+aZ{# za(pW-6dXpBVRXH-u>xHfVw5MLM5QeuP_(HFb2)fescmdgWA*RhD;0d&$4c zd+1NXHlnY#LY=Ef$kW!-x-$_%f1pwT{_WR>nf}8@oy&}pI@1T}E$rXSYHFgkDYQY0XGc?sL|3qnO6j@AX6V+c3kl?7 zZ=CKhrPCecGN871_IrJE*{RV@xIRjRIy$f?u$%p8>B9B~t$e=pi1e}h`91Vn*)P>2 zE>eZa1<-^&U~#%otFRtEN1Yxztr+A>E`=FNy48WQhE5n}qpL8CE2J)$is)a7;pzjj zIvXMeDHTJN$UiJsv;~sN)-Z1Ra+Hwe8<&fY2WyGOOqzNr^qq|}zf)=jDjRRvf>aWd zMK@AsO04>e{zXeQ_lZcxn{q5*JVHlEzAdaMs#|y>2VoO(ldZ(jzucm_|u@5p9CrQm(HJWMk=KOe-Od zdg^`}xXo^5h7%U%4EM=+Eq#$ysD2EiF*Nfn;ZM1xj+4gf7uoYPlvPv{%}|C@1&HRz z$Nym7vW1Bv(i-_LBMGbJXMwWxUGX%1OwUlAS{3y*p)lh#R-R8kw-ixLp#thwbqtkc z=}ZMeUzGc}zKcdk??zrj+-+tYMiIRzR3dfdWuO7(P}`*6iR#Q^occPDCAryb*vK#U z3?3x?I9Y`Q&nFcW3q1qb~G6hBlH?yrH>X~BL5AsC! zY3Z7Ng;s=J=IeB1$QAfAlp&8{ms$VjdP1S`G~6xJR?VSj84=8F+Z=kOXMXTG6dspU zEAf<{$yL!VVI_R(y&H^Y62u$K1TKks;`jSSy^(O959pjSj7+1e8WCz6W1PMQs>}`4 z-_&nRAFWZanBIvgLX4m&a|x;=a+A}IAY(xnge>X;W^Px7mHt#l&~1<}@6TM&M@SLj zWFrRE1BHz>cT37ROJ_HbD|c_ z;oQB6br9nN<3g9AHJ&BxBX5W2s~1s;8ERA~8t8-7WaAcT}$ORJW+za1HOlv^krl`B>IDPRk{cz zY-M_>QHOd?&!v`Y*HkZgm=eh-7S@Vz5p4<6f{NFy^b5A3T1#$1HR1{r54D9vA?g>b zuRb+IwW3}cW60TT72{Z7O=v9jTv%oN1HVjJx*6(ILr5Dpfix*jWe9nISgi~q_Oo6# z!w{7?b*eI5O(zaf1=+q-CeB>t6;YOr0_gk`Nq3=#8U4dUp#LZZjekvYJ-wc(K+Kn# z1m=aEL}hL!|A_2NtkazG6!jIko~f=EC7QD@^(gtIRs&sWXs(ATNG2$RLTRzkKYd|l z3Ejy}sFj@6%w$!1jo{?0DkU`~4O-T`dLgDAcFU1em|TXK=|vBq4$`;jlg2^ys$PxU zZ9E|x3mbTiZbV&G@5rAGJ6Vc2K<9ApvDcfX{ZwgkBDBTRiOEbnaay&b(iN!b!5O8u6M%ezJanrdZ6X%DaHy)A-1cdl#$wYrH#ztZg*xMgZcST->19`EyL_9 zMt`Fh7=>~2V&qz(9r9}IT9V-xYi@j(YiP^W#h@@v)Hg$OKUA#_r$93KS=&KG(c9H5 zc`kV$T&LoCKDsG$gIq^{W{=aEL=k!++mro++Cj<8Gpu_L>05##+@yQZ>)8nA5cz_M z6&nb4E{R=7{RA;AhB-kWq+4(-`yN`W-=OMpkY&-is~JuW44&>~^hx9e>N0X8t&FzX zpXx%jHwaPNafV!|y+RkHOWIVuF*s)J^-;LqhUy)KSBfZhWUophQ*{C?fR+*it;#If z7hVaKPaynC8X5fQALD)CI^=B;yaku!0%R!&c|ec~9t@5PRgvl`rJ+Zh10B^_qMLrq zC`Tu-NvN%?M8BL$pe8*w?XX;cmwuc5x^0u?g!#RBo%xYTHeI!(o70 zO3_`FT%)O{f z5p(Pxtg9`TEE%?>h-LPDHrCQy$j4PrV8wVzq~ubK}Cp!*itrq4A+N!Rf(m zp*rDd;Z>-BWJ=S*!{s5$CH1^k9Gz2sk|(J4%pO)^JMhcI8s>r)+0k7$j5{8uM{%Yn(7SsM32OPaM+DU{5+Byg4KW$v}dys&~0(py#27K)rsT_jk`8 z_aXOnchJ?=#keTfN>`4nv|Df;aGc5w^HUxG4V|d^5sRC;44rI|csAZKmp?zS@5GvkIf|<6B0UKjM}i*C-4u$Ow^}M%lWl{ne_KV%J-FUa zvXkIaJRvj`T62G}McFu}6PVtgl)b1&Zj%Qo1LQgANwZFxBPB{tgYQ6v33`wCYWUOr z{R2Y-j|0DeYjDF?%NOY#?oM!>amG2P=8VZYoAD^UO2+W4#*VU{=fRKab!rsfR?Ohq z(VAXLGh-LrfgL3th{%XvlDw>N)xsAF%u1>i)z3Z+9ZsIuM@O8B-WiveR50JS#A`9h z)@uB9@|;#g=`5|1>PW4^2hb3-uAYRxE)MBi@J8@saA2@P;Dx_Us9$)EnyU{X@<7P@ z!qjG;GgnaiI8Ba3o@pi5SnOyTW+`c1WNmIWOrM2V?kIJi%q7cG>&YfWDzbR3puN1V zT~hLc(%D(=DbwLYp|Qcw!GA)drQ*`*(78}f=%@54yaDz5r0|?j#~>H%4Yf+1uZ=gK zr?Y#mtDSR7&WDT#>2K2OWeC|3j((nB!|_BW>k)qA`>}_}{_x_r)jAn5%w+RX`}Twz z1?gfJi>U?7Nqb}GN41KmW8H5#VmlPoDqyoc6ucz9!QxPO@cyZ=q7wVW#VR?q5}$uPr$1vrGA%?v;dU_JFKh(d4K z^87KOu=$OpsP&oYFfXundK=jtlmeT2TuD{)A-A7vn^?{jsmAj=%w11NH20S-;}s*5izD1Jf+K- zmfLPbO^H91^emxmOq6{Tzk_J6tn^iO{h2r2-7HvNj>7quC)ZL_p;%q2a#Dxjw?Lm@ z&(J8juvQHon0rhouD)RAQ`qtJBjncl5k*L-R*k>l)~W~n=xNk|;us3+$xb3d|0dUk zo}#2esuQ#=TB_av^~51it*_VLsY8|Sa)a<_xrZz&oSLGhX$8@nv7h!1bltC^UjjXR zbv?8z&e_k=H0N2?{H(n{CV#K~?b?@_->?2`kyFF-BfNr~Y?>S~#&*T_R_w|t%oSn@ zS&9_DR@0@gT0cFi)VWhQ(8>5?T;KCeItsF=xcE@pVWu` z-&~s=7aVt8Z@hy-ht$c$dwLnVUFByFQ?m_9`=Vy6ca>VGf)pZxbR<`Wf6r;0lXbF# znYzg7SkneDTzO3YjeW)* z_9ab`BaJ=ka%HP5$bIBRa;CBx^{Tn*UrLv7LMY8&#xvhpC+B+RxS#PqYJY3>CGa`o z>w<65>3y>uPCmF;pU*zC?2jxTofw&FZ6rSAHnMeD^sX}JMGefS7F%5=x?I&Vl}iQ- ztxMQs-^Pt5MxvMF0OAbuhj`E&ja9jou#Ks1%nUvE{F3);UT4oY|4~U%D^VUUSF8#} zT`|teOa`@c9G>d8jWJN;ry;A+ky$|(q$P40928?UtDFb!p36TcP&Du~a3a`SDy^K> zJ|bT?hA5A7{ScxP`inZzDe3GyZLDz+6_>%vTe*F>O*kiX zG1NKK0t6S*`_6UCSuFSWEbCA2H~mZK^PTPv6PS8YRVVgKOvCxIDS-g zEBg%7D82;4(|+!QwPZYie8r>6c*_c9Dir%OUvA_uVKAXcze3SqQmzvH7!$uw)J^Th zuWVi8mGmVb`%e0v_{)ZhDiqnB8!q-WvzFTCOQwcGTQ-g^L(VY1YNu7N;#cZIdE-O% z`ijyH)yE&`IrVQ~v_IQ-!sqeGzN&$Sp(miI^fNZnv)L1DWqvSMhv~%}Azo{9j9hh` zz8|^#-^k*0FQzJppU24J#9c5hvqEej#&^YA&wJjR>AmMY?>X*fTtghz?989m@3+6? zeeU*U-}h75d7dgtkXvED6@Rawz3{w(6Ozr*f~7E@M&*(VnS$a2`>Mn~MXHsmQEoz+ zQN^0%D<3_JziZqM4f34z^!FE&?-A44!Tf5|Aya$d9`jDSBdreJ3>*)33LjT%kR>N1gKk6;v zJMLxuDZ#|>YCV-&!tE4OOs&kdOpQSd`3?Mzy2@kOC;y4Hb0E2luE&mNUeklfm)c!< zROpl+1M3kygFJUUIi8!I>+Yg?g>q|TP5&ACKJxq3A5AjeJ12(v6Z_3wVt*ucEnKDO zi~M7gjz{vsV!DYjMw>z|;d8C?6H*I(DRI2afl|)GbrP%C!;C{2>A{|7QRI| z*gB^DmYLQZa|YjwvTNhQG19M6XStmAl^Do%G{xDbL`;v|9nr@2hdG9SPHoZeNo9R^ zT?3ubj#S4G=Pu_~&waeZ{^2X2`lLxY!REfo-UaTF?lfV8-^0YE6u-PT$+OmxFf$61wuKqbNXRb@~K9MRyFI4=$+Pr*P za{8~hj<^Q8Cwa1irPS8sM6r|oLR41F)u_d`qNdw)f-zej5sC;l3&qKuz>ocl9!L)* zd+Fokmw{QXjX5o{%49jSw}OTF+IK6IB!5xnsVC%jU@N8sOG(pk(%(;xWxtAkn>lh_ zlpb9+x_DIn$hY>^wj=g@5##K9RJSO1Wc8S|D674?t)O`%*1Rgz5U^Xy8GmW#;6y7= zJf<5m%g|4ECUcCb#YJ#E*$?atUKMtNgSyacvDAeJ;u|}PEUQ%V#Gjy=Kv7Z$ zxx_!nWa73KsmxST<*Dj)YAR>3oQ*mfYf5aATsF}Z?}%PyTW#vVC$Yzv_S{FlpsBZI zm~Ex~oB4`4lOKofzeV+ON?W}Js)C8^MsA<**kZD{%{MK(OvrSJAwCM4!xX*;n@;$Z zDE|QmlRo}q-FH)77kvBgr@a|pJ@eEG^aK7cQ(18#TUHPG2Rb9sQ(~5*j&FeSl(}h} z8UH37m9i)uc3u{=|8>Ia#jB4ghilN!G$F(UCwY>CJh zmL|d#aEyMC)j=1YFO;&jwa<&>$zWMXAXC{v4bT|AAaav5< zq}&Mo^uO_(bZ6$Raz;4U?>WclSrLT$NdNFQhDLX-3OeMBY*A?M<3S(-#2NkHihcR)fF$8o0+Sd_VCp~+p0@k z!*jhJd6?cWOtEP89#M(WYopdjHnoS$gTTaV4L0&Bp^&M8WvngHo@xJXduzFDqJ=27 zD!q){Pu8L`sfWx?b_!orI4SPMvsbW9wAQp$uoX6cH}5i4BM(Zm^LBrq^l|e0QmJKM ze|Vq$^+E0~X`nus;`wqItAg?o&whuIRVB-vbK84baWHdizsEF5uqQj?W20)Bn~>#| z^uRmMJ$DCBd0&@MVby}_aXZqcyOk;7)&A|C%6UI?8s#2wOm7?1=`ayjBR%lwFok$ce@rEo+sD{0vytmpp`Itdro z_7^pS-7Sm}AES?QD*Gp$LA@d05(Q}|Q=iXg{@3b&_iAiJwEbWEE6XR-U{jXxLhNIn zZ&_;{X4mYe?aS@|T1%U!3FE+QQpiNggzVQ&GK*f#Z4!E$qRg`Z1M3!fFJh_fgy}15 zH_C)o%$=6EEHBsBUK*kuA)lfie}wF96jrJQfAKtam~s#04tDl+ zkMpxqeQiAX3mY$-7EHoU{u7&z?ZY%hRiq@do~2vV4^L4$~{`Wh76v|2We3Ospf6&p?S^0G!Hg8{cQg(&x+m2}e zP<1rdCqj*Tl~gn7TkHn=Oa3`gK<54ZJ-yw%y&FP1mH+Dd-H=P13O2xwaEkw++v_Op zc;kr3OZM&wZjvV%PP!7_$$H@v*Oz<6G=h%m2$Ze&*cQSX@tP%UO#totu)Vtdv2CGk zoOP77j9s&4*nWvfw5Qo7pxe+obaG{QCv%@Z1Xa)!Xorf^zp`Vv!$JbO3oo)Qvi)g4 zXui*#rKZcZy-_)7pNY5K-dNr|`%w7Dnau0%rNMRTIHQ*y4vz>#xVGk=%{`iX##2nH zW>CV=h(WQH5|+mgiL7W|${1>4Db~NwQ`Kt*8LhoukS@(W1hH=y*-aY|F6Zy%`s^Hq zZkcDi=3r6zux^lDQLXO5Kj7|w{PmXE4K3y>?kV3_sBhY89%uexo^6iDudawii+x$d z&B%>WTGa8VMUe^ijuwaTo&B5cNnV1VKM9`Rb7WB_oz?hJrc35emT_>WG`D;e?=cDb zrm*0vpK~U?>*osZy&ugAx4dVg=M8kwNyxr@a|d#xoG){~8>Q`$xL2 zW;gyh{@dCwXTIM3F)p)&W1y>wXTGn1&+YxwSIvvgwcfkF2cg-W3!jDE^KVe7GT=}cq1+CY9Bx)ZpGe9&5do1iVUQmUZ1 z)JOUY@^_{KzrxhoGSb@Hy2x_Gv_Qya8`G7^D0p{0WCvCgqOAKP`bTe#oe_5~mWVkS zS;aQbd`@7vI5wGyh4QZ+9YT+j2%3h1!3M?X5~?UwgB)wL(q4xL1*ds$=WWStkTo-- z#?MCS?SGz!FR)SGY|kDLN%sefg>u3b(Q&&0oIWdzOxnZ#;HQhTglfVap(@{ld(Nz( z9>BSfLE7j{bUS`YSD^o)){xK*=>3%vVLdbmn*XceI&vemjeg504{Zq#4XcH@MVDa| zFsw7!Ezsg0gVsFAnpuH)1C8AYY70F8wc|L`WlM7~P6tHIjyeJc?-g8c-MEv zn~3ahKJP@I8kiYQR3nIj>|kN8Ni+91Efa?bC((WO7itS}7vBE-M02tyWl-no*R-EL z#Y~1qw*}~n*STN#uILTl9@X`Y^hL@=PAB_A`L>+0GsjttD=V19>Y^+x5b7bHGlOXe z>MttB)K2;|vlX4N8}sMk)#=Yq=PI&E3`?&kn#XAbv3OPOXH)s zI90TnM&o(IS-x7HSpT&>vrV(fRs)^kE1~AQzew3Wn}O&p$iVIM^yM2HnaJ`LFrM z1?B|bg%ZMqyhPqC-;$5W8S)3@Odsm+h<~Uu48?O%PmJ_Wpqmgsb% zGn2W?d|{!vKnNrG0dV}h;g_I3uvYA3dT9D$DsFCK?qwcj?r)xHW-N0og{`}-@2n}- zXlrFlhH0Sq86IVgxlP}MI{yuM74(Uw`X2S2Tn|-~X92(Ox~Ckn#EtUaJ7e>RJk41i z6y$E6SKgLB&3n?j#q-u(8qDn_uI8?~?%zEjU+>^S^lTdqzJ3oQhbVx^7=%vu!{Eg@ z&D4R9g5&OUllf!(E50EZ_fLeO;!x8R(_E9qv|j9rs9`X_N(feAl2BTlBJLLVi49Co zOzEbM<}T)6%w5d~&27v<({WRpcu>qQzT_)l_p=Eq`Utin6GNY%qNq5s5^+>Fq0W*Z zR|SwKhty4U0W#k1Bex!mp?_ZrV> zUx{ER>9yPuz1&lEhHOu@VuFazcZmms<8(o7(o+ zF4*#+gF-{4cQhIbY&@GsE+eU;EwtZa%ioe-CZq}V}-6>9S@ zSc3gVpQd_Jm&kEoDSO~HWV8k99HpfEQJN461&;ai`($?q^Z;&~t7cEirn3LaewkC< zA>=J|pZ2!!JN@PTpS_xAhNq>wzq_Glou{oYe{iqVT4|=&BleS}>8Xs3s|CvQU^p&m z(@Xq*Ps|f~3iaV6yD3&S&#>&bPOxR#ber9N-?q=%(h@e^5?{d&n=BmQX?`5nl}q6o z@h|x{nAxj@`$7{q&aMiAVB({2V*U+zh`*R8bV)iJ>fw#>m_0W7>GicfDl%ZQTN)m^ zg<9L+-hbUyT#93P&cN(BS!!lZmOZC$Zhz-0S3OUvHw8JL1Gp3UK(lTQ+V(ropT6$F zf#J#OFhhpcFhoyhOYrexinvK^g)?AC9BEo@ibVI3H71LBy=ATf1HvDCbI_gF*iS-yTl~*8*qP-1^xwvKnQPS-)hL&jpD; zFU_6dnd@DFj@{*5_w%Cjdgb|C@6a3de4v%|tI|+!3gzoT%Enw`H}l2Bb|Q<_roOmH zENuEF))X&^S4;yelWn){LnH1-yv9zsP{eurTw4`u6=WCUOc%vr;&~wiPuq7c3K8WJ zT8L}Js^T8pjXJ_&eg(IXeZ-7rHo`N|60W3P^bx8qHG`ZEcE@*pmo`Nmt5lKGrHP>> zfhE3Uo>#7PXK6>joU2*2vPxtv$f}sr($Ow&ncMHF?-jhWJk#A5Tpe9;ps^>oIj_^d z47K6wDr+1dc2EIk3fBhL@LJd>zBTPPcQmgBoBOWmi@CD(i}g2KmTf-nM6rk;cAqWD zw#3R>r&%H`HO(IJlkk)8#rMS%WFSrnhxjQ%RdFohj1nyTAk3+~oDaRoiZV55i8?@C zfl_iZXeo_RlmA`MMAc(wczS43@Th-_cbsQ}E8F4E?UHNCd6Atzr$A2C+*s$3yn(K_ zt^uw=u4H$4cOUlzcR9D>eu3-15UeKk51)}QtMFowZRk`+X6y1(g+1aWQz!FWd>l8A zu+*_$vWBc*t#7S$Y!7T*?9c4C?ZfQ@?B(rOZIy8^KAZo=lX9=nK)AyX;cfgroJGs> zo3Wn#E`AYH#d&xysa!tR&io0s-6YiE_iH*du6BK{_EG5}uMA%eH3{7EHS$h#KXvJO z-JQc6eRGTCyw2{K(a4NnEHsj_^Rv~`aZeNXs<7aW2T9AN*kp0 zQ&Z$TX;!EUC>)=CYrLP`hh1@~okck|r?3Eo35rbSYVaBcaGoS@#<{!_kk%vUIb{ua-eI#_DhcH2^|>6U^Pv*iV@zm8cq z)ij69lPrGA9P1jKGh(gr)|*zJb%ix-zKzV%2YB?t@CZ%k7V-_mdFG(yrPXG=XugQE z)e>epy#Obx!_d(mH6o1GdM~Y@nkG}>Q6VGn%13*7cTZ=l+!{HXvX^Gx$vT(yFza;I zn`~G1pq%Tu#~l-0%e@QzS%F!>-NDPjP;dsA^p(P;;R&AhKaS1u>Cv9|7j|2$k4Ua?nGLXped)*=_eR>nN!Rq&3_yB=)3Ei>P~3- zs%LPQnT5y)`hk4H9C{scg64CV_(kfFdI(<*)np%b7CeRB>8EgK{6!WcJ*xJh6TxZz zJ)R{(FK0%9KX*m;m8_>(s_e_zPqWKsj|AOdclHj@98Tqk`Gn9xya}dcBjzr3hzgS& z+-Zx+b#SqcCk~O97)pIm^M~fXcAY-SkYV~^_^R2%RiN*av#CLF&kTga?-sjMov6F6 zEmT+Mi!;yZemrQq^!vVmLjLrMb;qF8h*Q%yguD7`Vt0@?w7WhE?J$<6=P44M`@8g z%6qh|wyw61c9mu~9|valBC$W1{eEeZN}zVp`Ah@aM-C(Vf?%f(9}F*5#nAWo1Nz67 zT9JdI4n-cdjWmU{4fuh~bSR{!vcvf1npiENKdh^&+o5^E-{2}SG&wa?Jvb{+Q+lIx z2@%R$PnZ17>C2OwB|rLU_}wXUbJp=}Hvd}g$DFK;Ng1)(ck^bt&x_r{E%>9_a>lc! zUV59J)25&XJi<(3h9kjXuzHQ{avPtbGqRd-f-ir9`a(Q_ z7iz6)V0e6Jy=oGjq%LQv7xg>(&zK+9bp34g1^N)q3&nmC`*? zTe(LhlK+s%wiSx7R`dtbqjU`96lA1#|9LLyQ*wu2$J0;YGtbX!nbSVIQLdQV7b(E= zo!$Ip!oT=8`fK_Zy2aXUx~}^Ay7%e;6CyW~YEG|B*UvN;Gqu(?Qa7cahU_31{1yi) zkBJ#{Cw3D%2CBxpn5iEI(`u%?P9~_D46V~xyW7r2ca2h+FKY|wSfaA(G_jHDMjoVo zQ6{#Cb|aD-{?@NyHiX{!Ke*lg6-t8YAN;9_@DeQZ6y#~rE2cC{R{w6C5zH`WW#!Dw zev^ACZ(z=mTsq(8d>B{~?#Ncsgw+o;HMA-EJ%&n#%G!Ngb$%avN4-(EMt8*Y$~xP; zL)Vs7;J9cNnn_$F9ueQrpN*qeviZCkZ|5PfI1`jCr8~0i70pG{X6vh{2T@Hd6LgtO zKJ)=IXfRyEiO3zT!Hv}|(tk4S(~h8a$yWbC@0LJq~d%BujAM&XD3y$Yffn!o-I7tga zSEQTXdW8eC$EC0OUHoT4vYedsvu;L8ZvX7U^wqy-r=7{WD0~$^1P=!GNRi<#Oc@?CFZL*Wi@ z4o8LND`8a$PShPV^|D>IZL>|asmu@cahhrLHR8zs{03A8(jMv=+8DaWhUz0$jyG5TG)XZmLP9$KTWvv!7Ni(129hIag<@|T<>sY9LV zznC2OsC&V)*)^OP`l6~tFXg9cPZ(yKMwsFvq9WpKCoE?*>$&bsH!_1N%Z*c;bwv%| zbTj!>?gn#8l^s%0cW9s#=tm4vZBr@-wt8PWj~5WROEVgz4^Equ-aO;a)C#|x$y%Wtq;KscZ=9hjbVnWbM-gOJ0pHZe6+nXb=M4G)?r?+ zV`Mggf35av*6I%E+GsAp%^`sf=9MZaPob@B3QE@GU<;qh+rjU$e$OJ740n?=l5Iww`Q7t(iNdLLRgIxm(XXf+Vj=vL4Ty=%4{e-z ztL=sDv@uHaj@2=vxCuCe_U8^UZdOpg($&{*)I6e1VK4MEDRPu*2M7lF;9-I%fSg{F z|A0HjHL_63?VZ~s`+N4e{2}(~g5oYO>~Xv4EHh zW&(8?d7|r8VJekZ(&3L54o<+}@3*^4=UI4ipBNVaNX&$d?S~{PsM?gvIkE5jpSqM3q6y4$E6|h_B{OPAK3z) z*N#QjoTN2qy7E5m4&O;_(il-O-vF0pm-@ZBuI47rVzJzCW;8vB)-(5*KNx}PMLNMw ztsDAF*&(eKUqScZ&{Ni(CRp8V+#%t%>#}pGlW@*={&Lc;*RD!}S19M6?f&ig?j7bI z8OQ}U&xf`-B?|J-CNxN{TB3o&gjSKll3h2w`MvGPGk$!ZP5vs%-_bfCa`vT z7qtRuNngSPl^)W`z%Ji+uhHAqv&o$yJaA2PK6GT*E%w5~dxeh*$J)Q!L-uoytInRT zk?0AT+*RC9+|@kgy{~;)eqUgDZ~}5*o$%Y3Kzca@p8yA1;YRe}uG4*);b8l8LoetF zUrKXXJ3&9j;4+LdJ~iGo4l@=vN`^~@uwjVtKjR)_*jU+=Xsl)M>CU2}nxd_*y|4L8 zGYZP-q8x!fY#zJ{<%rA>A#W3V1eW`^`p$TNcm}yE3I|=SoRlNSzQ=x-Ss86{&k(@)c0vuHk#q))Y_f~C0SEKUVm%$>}QOdAZx^o{jf zbpv!obW1g})D^(kdWs$dMb{>;t3HKF$yI}{zy-g>|IAw#s=VFkeq41mUA2#16PLYDB#ZS&H;JH82iD)2MF1y6t{Gd4IocqTYo z`X-kN?GOJ$%pk|%DIUR;U~{;I=qqqe|BJi*eLj?)JI;d)CjKw2l@l?lAH+?bg#0kBN;e)OW`Xn7S*VL`}Gi)1XE;SGi)h(fO@(JlX zl%dQ03!p8k<5}z;E);R)BfYSg)!&uqdg>}8n1n)MqUWe* zo@cQ)**o2L#XrtJ!)Nvt@vZPp@%I9srbw_rvMG`9{5K4LR`n+9(H9wpbMm9L5A{va zgF0=>HFdNEFuA^LJ!xHIy=Q%5wOM;xDqAwmU(5~6g7LUvuima}r0c2Op*f*m%1`9p zF*%fs1gtb@;9JRugPp{dfqwp@zU$tho_6jY!VA|h*9Yfu=SrvG{MWS;|5o2M2wAXe zT^e-654v^>@$M{ltoMU=jjtl8Lq~jQ|Q{fpHq54TQ zrv}g#_6Gkzvs%~QFxvRi#97u_vT%>bTW4AOS?gH4Se}?&<{Wba%WHGcG!=PkZ4A5g z**a3!4|%Cs>P37f>@Lmdw$w2~3f~D8Rfb59p|#BmIQ^%6TaaYg*fYZY02=H8LM`EM zAbI)7c-wC9x znBYa}jl33=^76zkvN_$2eZ%KzX6tl@HOAAX<>p?NOO{)fbjx+iHcJ=FJ+s@~%`(c; z5BECVBpI6+&l+roqI#>Y1Nu~xu*3A=8e)Gm)7!`((!Rcd=~gK12(}UH1X}yA!uO!@ zHSj8)rs(x`cPHY>*y|Ep>8@%*31N$CvMW(2hO24e9^gLV4!Wb<2|^dwpRUU;tMC#U zbGQ4aXRSBNx6k*;-#~m7+#;6%8-G1P(a~%QH$wdonwu|rm*KNcaa z!g3J?sj%g##b{|^{*LaSXk2Xc8@d<@_5WbL=+#_TU*NxUpK;QSpsP_%A`4X8^PzZD z_?}=rBylGO_WNJ?=VJBqdNVx@-9e$T`;0J9=qpqZ8VjQYtFYCj6_yIa@NUKkznvAF zL-72!bnbR4&R)2-Qv&0D>3-+Q^zQN(29m^qk`{jT)MC$eyk`#3 zm#7Qm0pg~LR5d_9FHspL)ACwL34RT>!_z(+ddhsP+_9b(p1z)&?*H6IcQ+wXsE2iN zimQ!ln=|BC?)YF|ZjW$mM|$8q=XLb<&bcNDz1{!eO}*#SgBP2QT)uxpH&ri5JH3xR zjikLN>Mt5mtHYj`X?S4VVEV)S&D_Vb+j0`KwRp>5bH3@Ysj4Z}IM%qxa7%C0m(xu^ z`l3qnng7l0XIn6}=y9OO>Bxgf3|^s9gWtCWzRFQbo;*%gq?zz6Y!Ukf4*4qMJ$>z7 z>~88#7d8o9h1Eh0VVmoMYqZPf+~wq)^Blb#=NyADWoVCQpA|j`d)yN}y}cuSz5Fu* zYsJYyWPzhwXe2(6J?ImR$adqKs%vQmX`ARotb~n?Z;UHVt;{#gW=j)`%e>m$%KX@* zHf=ZlHe80nvORK1b2KrUzp$F9z^-h|u4dlQ)#*J{5o#iQ%#*>OSPQRr^KcJ%^(rcQ zSqTmZ#*5Sa@xED}wSwSkATUC2cqSr*R_Htyb^Szs>3ru%=MiU`qnqQ9Lv+-0w#90B zQ`iju##HZR@HF%M&#?mykhaMWFh`CgzL5v%vg{tNg}SV!l(xREvc9WfnDJkeVA^D^ z0Il6*OMgq2`ET<_(=X#7V-I6r!xw#Z{abC8CJucvBfkO+zcg^9e6)(bLUp2&$eSP` zkHnmq_MJvIrUv{b^^pRcsc@KgRw3f3g~-j@#-COH(45n4 z(2q0xGJZ4NH($5ZwtcXjwY87<&$hwxw>iRS(%00Pv>!ANHCD}R^-rGQ|Kv8Yv!M+9 zo9V`EqMZ~?{|n~KS?V{qKC9rftVuLgT?s!$KR7};DSrfS=+9s=v6a7<_otBUs9e}G z?@M+<_8-}2vZcJ|_L{DS?ggHXU>j@PLxoeAvF&x%bJlhaadj2ic#M8C(q^;ejF6&g zNd00|yhFV~yH+<$-^SoE%rG*h!KNLid!{O8gGI0$wzjoRvmLPp%=Jv9;fnUQ`Uj_F z=h7Cc7@moBAeY<&2b{%OcY*2-6iJVXv*a6)_|}r8NaQi%sVPR(2Tf)-curl3y2y#h z2+da3$-MM1@VC$5UhK-ZUxRljEw^g!*IaLI(}LsnSl3qfNzX9v8qZwMa_dt6Y)eE<^s3C;8_IyLJHXO3HX#I@(1ZBtKmcy z0Ut$k%;@`p!tyt93)Gc$Nc(sh`mPAloM1!I1TWG|Omprzwn4dexnDArZGFHXE2V8o@t=TdFO559$(s>2-ROKSS#^A|S`F=d$! zn~$MVX>Zl8J5w92Evk91o`IFH2ltvC!EB^m)DfKW-oeYqg^MCH z1PK<+;7I6c*IA4|N`Jt#l9dHtd1D17rwLJ&Yq*fE!P=mOUGlor!ceNS-~Rck}Q}wKezXCMhnN?v%T~D z7XmlLcG7uiuiOnhm%6IfgiNd;3*oDr&#Yw+^EK2>H8XXC^lHOLqZZFlO>;-{H&X@k za?@<%Qo~k#qApo8Ruiwj#VrF9a}Zd5v*;TjL{$XedmMDH>%m=*0;j(ar`ruk1-XIm zBnyn08sUb>eGrg$P!mj#1h9I3f%kL-&YQ8yelXVZf_uapfj$29zMciSbx*O;cH8h+ztj7$hwXvyjlVOeio9>17IQZ)Md<%Xm-21~(56B?- z6{0thN*zQ$e=tr%qw)6@BTo_cLA9z$5TL5K!&AaK^boFsgp?RA3{?%gLoGu#Fj2}Y zujKx6HaKWg(bakca&8U(EZ;is1y7_W(LDxLLpj$LXNh4JR{AQGnEpc4_yLLX|(D%!H`ABLu64Hvu-ou{$ zo+IuWsEP&(3Bo(q23JGZYbTHI%In$ynEF4R5JYYlY_I;c>a7zP>6=|}56L8I^zJ&e`7 zmz&6qJNpiNK91Q- zHv|Xs3)O;(0$Xi2*&E8gzsWzyCeUN71r2kRN~hWf@|OV5d4teX*T@7|9}s)JkQz9wuI&I>C9r?B1K#xu&>!B^K` zHSjR-LVN_3&}w-eSh9JcqhL3VSFywbVmNwu8PqwtBe;}9SO>cX?Brd15$Jk;s1vaY zjMkjh#A(ZE`$H8oLEAuEQmex^_EE!V^E6vDU7;><^CS6#c)L2Y_m~#U1G*828msUO zhDo^R$qMkZy1_b|tcpc{I2joe%fhw77eL->7<#NsR+=jTTwhVyCryz)f+uVVCW7~8rag{C52fPNC&wlr2_Zzp~)6+wES9mM=6yH04_kdPR6{iMsgD0g9 z@;*6Md7@Mg?E$+gHQWz*8cD=8atH-(By*h^#}?-vaLxGle1G)=b+l#~^fIX$32vF% z+G5&?@y_Mfmw% z;W|%;8h{$NOjR4)!bxNVNVoOrOnMUIWR^li)qy(%{ZJIvzuEi^{CtFxXbJex4WW%V z$4%fGaZb$SN3+%0Oy(>zj%mWMOfFprO0I{_qkn=WxtbnBccNwL0yU57M#WHCN&vT^ zfcy;V-4L+u9K<$aEK#WX2aLp<;8MA9#qpt|VEN_9i)9hdZ82#A-pAH(ZF~G9{P%qI zd^z4l-p1ZK-l5*5-Y?!#zA3&7J{kR_nf`Qt%fMogfBzNFh_W~qglQxE?gyk_QcZa~ z*x(bDGNEf=OL)UOP<{0Ui-#avko!muD|!^Yk(TKR%rB6STd-T%FKh%?0j_%j6Y#6- z5Vjg?WWRwhYK3!lEFFcXGzAl^C)96r1SvWe`!u3m=>)KW%yc4MiH--Gv@tyrM9JIW zlU{||?99)j*{9Obt6gvCUHUgQR*KM1f?fsq{2B zEtoDY7wd~=F%87tXMwbUMro=XCMo{K*CWuRU_nrC4;Tk5d6Kvs0|j9Nn|7HD)!XFbUt01`N_~& z<&s!07||c`Yd1RsTE?aL-4O7PIX0Quz?5VBxGxFxDXJm$8BESf;6nBw^1y?QLIPWz z@K?}0O`#l63GXQfKxMpwyK(}V@);lw`jkRk%YLOlNSDPGBiNu0ISZ7$J7Dv6k!xZN zx5#SQC~Gj4TLA9rLg}D%9K_W_(ln4r`+^R>UP_i)%G2aEAlH6|bM=z^T)r=_kf(#c zIT1|TMA<680*m`ka7$Gnmi~_$-3N@@yULkRb=-v|sbh+edvKiil z14K9E}^C?_S2sEbKVEvwz-^okl2hi9(k>}$sJ;oI-kk^7YdIMA1yP#~V zutRvI4L`8cT`b z`<^8yk`1vp2eAU}Kn3*#S6x=+LsFFny^H_lG_6u?L()|Nm}@NY6r}K(|IfSBq0V|8 zJ^|K#OVp3S5RbFcHBc2Nfzmh@?B2m}R%yT~o3B(s_h23<#Y^OW zkNklm?@c376`SYB^Gs;!zWy0F$yI7`Ba3;Wr7-#+&CvcGfB+BdNi$ z?Fw4v7Vs|LLJR!@gv?vSc98u4BUHFE-Bf9KueX3j=~G@Pua!HX%N_$W?=~o&r@?5v ziTCI}eqT{Z0o(f+II|hj6KS4glm-PwF%sW+zW+M)PP_gAZ zPrDfR98Yy$nLyQGQ~7r2l-POoCe^UsjPbRV^7U;Ss@5&a+fO}Z_*2(4RFS~Ev& zR$oJXP>ov*^0~~c#V0JsK)*+K#HXAF_0M3^NmM4fVuf>r>WA7Z_vOCw3u);8Gm%YL zwKw2t>;#(cNo5~u{LMIQd@ud`E5&&bD>w%G7(z ze}^E^Z=upxzAU+d2ZOtU<%0wgLB5GKgX&;x@Qk=ryelpQ=e=Lxn*S-jw7OR9kaeo2Z#14W31*wMm-V^j zAL~7{V7g=aY}l>uq%&$iqyC=5Nno1yXSRW&Jd}PxC1XduK*gg@Jxn$rvxt0rKfjTK zd|VzS{S~Y#E(z2MjQ7X;OX6p%AM`VEN3gye2j(MB{-)N!!Ih|fsj05B>96QLP&)pt zpQj_Vcho)kJ8UCnDD@WZs5zloO1#`n+7m2>%I~x2M1n-F=n+q0_b(9liC2*wF*1;f zEu*CGu4k6Jfe>_7cV4wGE8K$9JC%PgZ*Shly#4vD3SZdoxN& zcN)i=%iEZUN|BQyM@IM~thNT$j^9cZ9HG}%Xu>25B zuC{|-Hk371hqtM%rQA9=?*EZ9*mCZ>QGz1e4;b!XeB` z?7rP%8+n7$L3NKjPEY54@-doP+Gt&}ZlX@DYoQ&XNm0M%2XQ6XTFeys9BRcluv$kG zors^vhD!|(fVS2RLi&AhCk~3K{`_IZVU>_wazj}xw3xjUkJ?4PmsVg|*o zkDea2EJ7W5$+{lO;5G)AUeZ6;rlX^zXd0obu}0mJTYy@o484e07Df(0aA9D(FTpLj zhPza_+f5w~`)21f!RxNz>xt_nLV98z6{bgU>HIu3L?)Vm=3h-K4JO&#WUeGz6lc`e z)IG|Cn$bl2s75#e)uB&QKTr`}Mq1@wWr4I`tmNP3S&07o>a44Hj(rd&jyh_%-B(B7BQ_0pZz#p-kQ zjSQnK=dHD^e@29?<*n_lPmE1;wbkdD1;iHleIV8oa&~bPK!N*L{)hb6h25OrggozK zv5itO+=TFwdyu3OPya!!#M9bE;?1@D(+eG%d6rHTR)i zD8^Q%BZ>FQN@-DGi!aZUC5(pGroB+!UD6W)4o8bXNPHbkfp7U1&MoJqH^J&6m#xvuSO4vo*6AiRg2skQQC4ruhJZ6_mYRe5&h;X z>Y3)5>|&kooPC_tTzWSJ=fhQhGh`Q9aB}Pkcl1tlrsqm!g1^KP;&ria@Ox0Nya=VJ zYEoYM5oY(-)Nb_`HK`fN2iW1vIkGe|ska5U1~&SdfsE7KqwyAc|MUL}v_qG-fqWRd z|8ubHyI@B3L#zVryxkk}jPm4zFR?`!>nd;#7IL7I%5c@QXJBfdQLx*-*D=Y}-K7s~ zm5+xq*hAcSZ86I$+uF#eXkW~mn7i>m;>W}^jjI*IL=TMVXp4%tV*6#O8qvji-}1=z z*80M--CV`^kM61Z3EPQKg$kte{s>>J=YNyZIhaphaLp2`dFpzT{H4VY;ui2kx5!yI zQCWgx11|yxe2;xe{)PSzVtKi4s0hJRqnTaob?yScfs5eIG4XU!l2WNdb0ll}J{JvO`%d3Z}ZY29^Zp&?~?|8yovi?7p~^;-!j-3BM9X#P5zR z9a|8&-B!zb$|9O~S*lo?ShbOhA}2=wjA{|pGb&_jV{WNi!%ZT~ggS_oJ=a~k94#F6 z9Mv3`oSZPiljsZiUIto-k-;YTE-s5r@wQofxt{%=rRbmR@?7;!3A}|q>Ak9e)Uc^+ zU#>o`SWP>K9pTZ+aCjfWfjNO9;;%rLU`?qrXw8ApAoRliK~Hf%lz_cM|H)r41HS2Z z_!{^V{Kx!yaf>)xDyuZd`Q?iI6P1uQaM9gTs8l#Jzim!bTHo|OX{$45=h_N;dM8SD zhSK-9436m@U!~ZTVtW&^i>`_%;%dg!K%(8S$dxuFLK|s}OpD$hQ!RE*Y))*e*po3Y zqW-kjHa*cyVBV-U23g-g;kDz3UF*;|>$o!91AWZ`!-9*0rGkxu(P9^WYfM{i3ipJ$ zt`DvPOyU0WV73oWzyL`VYN1LYGRP1r+2!O<)$mY5&?Dml=X_bd=DwhRzgS#KQx=7L z6F15E)N=GZ%9Dl2Z9N8h=7V5Vuu8C-*ew_iUXesODnugns5((nwFanSPwktYl)XMT))^7}rdrMYV~nBVmfS zZs2!dw{!-{V(a9o!Q+8rKAUHmu+llx@zZezTBF;-JGa|g75x-@a5Pk|i;y;PM>+uB zQJp|}^kVLNUEa06v;L*x-?%T|!Y#>mNRl$M>zKw279+AD{N2|=$;b~d;AB=3>D*Nb z4Nki~Nh>*)C{1n-cR{WGT_RAYjrCq~x`p=nU$gSEys4wpzNa5aublO-vx~P{$j0^8 zOt;;MnH1ltNM!uqan{)C(W@dn<~&PN+e+Id_-uE?vPEVU)g-hi5?ORrT&!|d(!4*=iM$W@Zasx9Ty;2<$jYN=gs@lk+m@a<@88IPX7ZTkb z`>C9vIp5NcrI${7mD(k1WnNeSL7Hfz>0zoJO~$6je~CX9+bLEZ$=h6(mFC@+PS$(2 zebLq9Y8Gvg@H`S-3|?xeuyms0pAs0SO09kD_Bu(smux= z1z)ffw79(F5480g-RoR;9Tgon9qEq#u17AP$LF2kGY1ACQL84N#A?DK*Ko&GM^{u= zdtEo(d7j1KEDe_Lg>>lrT&68J;kt0b8^ySg6?2AcM`n_RR3oSkYeMnY96j9xhDExH zfoiWx2<=5*@0(;*T8kagalPW7=PfO)DLk8-mYth^GnG#3k&%{j&vDV$BNSp+7}Fv~ z#mtXi5uX&-CboLyP)mgAtg)QYZv16#Z%d7K#hZ&ZOc-CRR02`7MQoD@pD9{@lYdMl zs&2}mV8!68fD)Jl56?3BwxWcpg|8ypYhY-noD?h>_~L8qo$FpIqzLPUlAg72L!b6G z^p5l%^EgmT>YVHBF@+5Zhhs9(k>39{T5Zvd<9RIJ?2P+E8})CsSUV)mN)M7hBo zJx7ma6?pd#a1`&t)KTJFbI-vn$iuX&8etB=I>WQO_#f&T>f-Pb4b{eJkE& zD%b~U8x1k9iY3OA4l;qd94;3gD8CN22?o8VyxV+_y^RBZiE{%!WWf03A@W%9vA>09 zh-*;(@Z1xbzKk`Q2QvQ4X_2?wK2ezODdC@mKC4-DUM}U>8}!%pTXb4&cddvF!0vo3YIz;o6O`0zbS4>1O`+>kO-T>=hiY2*yV5)m=gD^r z$uF6`Br_>>TH4v{2Kki*Li(h7$_Gr#EY~A`+ge-78h2=1+-eHP z{8zkMNv>4ak{gSDjy+<_*QfCv>EX!c9;xg_7DHE6TkN-Vx`WZdDSj8uP-`4F3-=Xvwx1K~`O~BwK5$3bP{_Q~Cz`!@GW-fH_!*C+Sx;9Q~z)d23~K3orNh_^Az7%lmpc*ZP-a{s-dxn*(m zqPVOghJ^P;ql=u2SsHQPv|QJoAIA2h4^j%IV^x^$Yy~z0D{nq>@CK=7DsQDjAQ*N) z4c;YK7MjDL>KT|LT=%S>_KTAapeg~i9buHKhGXpc--~TdsN&D z-|#%&c;Se1RzX^Bi@Y~EZ|ozzZ^Rx%Gxd4>TWgJ|2R3cQ42w&@OLK!;K@Wu5bswF; z`7~E_Zwz%!8;zF?J9L-%kznN|$bX2(y{84yHPSi9@e<_Q0geT(-7d3W@qY3x5_9By zc*4~&5$#A{Ktk?8?kgX!U27O)t{CwsDi){kDY27c#z&>u%369Dzu`Pesei+9 zUz3~1CP0V0hRLAIP5qPmO4*wnPHOkFL-y{RL}9d#@g0*HiH1j`s)NW+&n<<7;p<4{PWJCtP35}j z$68BAHjiSXOy+BbC+b@4ztnl+4^S>1@(nZ}^|j3V&66w|b0z&=?mYEM=@MAvS?F{X zj<)Xujq{+ufkm(k-IxW)T3i(<6R3_H!jfX5q?bp8suM@3YHVrsY%OhQZklVpY3XOJ zW{a_DZLM)Sv05gZHD(zav|omL;J@_Nximl2YQ83$Ne@6zBvMrd>Xine&B&Wqkjyv+ zN}NGZ$OP#jOcvXi3$c~ienfxJw~u;~3cqAe{+*OOqHXBuVxgb=y-(hf7@iy?^L+W)X*a%7oPq2Pq49)|s z{Q%AIskC*d!zb3Z*`x)Bnyr%bu8DHT!<%x~#={Yn@!cs~V-= zVDd(Eif$Mkh^T8Btl!0br}B|L_6y39#_S>WdfgCHd-E8hRo{v4PVETU13QJ94hI+@ z=k4WCH&*gy_+JKhOY`J=n7s9u`Uj5rZ~2D%t#IW!q|u?n)L}Nrdv#9?DW-Rpf#yQf z0yAZrXPjsp4jRCp<~rtnmLJwA>%Z2r)=Q?5#zDFrYCC(D-b*zkNMfE!Lp&qqL32`- zoQ4T`HBeA8$u`t;>VGcL7NB^RH5?0uXk^BJhJB%?z4Y(Cb$Rq){9xt4iClDZIx6qcrUQa z&mgh4Z}2k`kY*4aX@wi3DWPj&7--~8cB9|8*H9lMhj@KE<62`q(`ZX$YY|)Xh@-Zv zmYL>ShK9PXd}UTcH9(f$t#Bz~m#Q3f8(9J4>2$Iwy^s8gSx6RUF>%y(ss_CP^S5-$ z85$fM=1p=8%AJz>Df#GkZF2LUtyAA-l`Ht_9uRPg!^G~s>28bTdhVmlxf!0!v$@4x zm;8f>8QLUMg{WY3Qq<%~(fm`pj_r&5p&*g~R$`srr7dj?n-WZG^d5C>wxcRDc*&dO zjCGnEwH#ZVvxQ^ceE}NN*3t4F=m(}qesPI7GO$jp44&*HkVEa{9p(xfr?u#c8D1K@ zqoWuwjMER&y0lHS=e4CxS;i>Sp$MO~Pt?Q6s@4zY5r$~(F?K2a6;t6|&^(MG&x5*I zot2Yw>AIX3gAEG#a@Me3twK5iE3NUCa3wmAI6gpGTh6n^=Llqghy6+FuGGe)y-G06 ze;x|y80n6@B~+DML;vRXawD{BwVeL2ewlWn?xn`5sRQ@UZp}CCNxjQxwKTO>L?Zhe z+iojsE@5b|`M?!`^mPGqk_O})>IGezeFeA7FSZ|>OSLDL5L%>h1>7C-cE-cJHm2J=XoYpL>O2IbQ zr(k_XtN&^9$CQlyH)e5U8?#YYiqA#1p@#0vR8;?=KW^-48D(y1IHGILohG8?KYWda zKHyl-aqM>{x*vPxz?WbRyyt~V`_O5mRUVaE!++BZJtAjFMd+yF>|qf5dEFA-MO_bc z%-d=*`BB_zb`APeSJiX0M-27NMQlwYzC^GQ6)Z1J>AJq^tE`urM{Xm#Vp2MX+svQi zduy)p)3}Gse`HV9zEE`~Bv(cnOqtM5)n(Od=p`lwJ9#fVUgoY$J(6tF%RC(ht*xxS(>fz2@Wz( zwk)vJH$OI4HB~pv)Ll^bhtErbO5i3E*oX36)f4%1Tp~B0DMpD{S_y z^E%l7@%1MDW1HzI>x77QQHLTrSneB4x@b*9%{$E{-BYL-i`#bDHe0uvFX*zlm83(i zCtmkT!a_JHx(dZT&EQq&D7Fl~6lW=tyjrOlJ{js9KB)2#H;5I;<*`r|nK9s*WKt1G zZyZYvWH#~bHQn_~jAh_ExMHepXbo*=Kdu?O4D)fjrn6CPZeraOan-7rCmM6q71%e_ z9#s%obaz479f13_nUB&e~L!Ecl591 zq*p1se~tg$KC?y6+uUh|j>1XyyAGdotuxv6#Qm@DozzwJlitkDfLGUQxoQ2&`pmM) z(%)*Zw6QETKQj$CH!y8BR@Hyg)aENO*Km?F%l`$Y_*|ZJ_(Ki~Q{2Zr)qMQ}cg3U9 zFlBmpE%Kl5QHwE|Od)+_PogX|!5-;>T%ha=Kcz*ky#9shT7(oC8`;gW!7x=5#T92N zk$tFRtU*7+8WHm-P91+O{#{h5i0Zm|$OwBc^$Mkhju8FW-D-z!tf7y}n%DIA(DlG@PjP33!mR~k3qA19jdVpi{e|=Gm2x^{)=yoN z6q_RbXp_A5r~E5DBPM4`!5(`Fm*3UW&47Sa$NSh@(QlJ>DznJ{xEtCi((0$)B==2%dXIyV4A0)qP!Fm4M-lm?%9-DWxZ@OO;_Xb-i z*Fx2iGCY|m3Cf^@ybM3{2c;|A0H*^-0-MBMax;23m!n;1X=~1~bTmKGT;)zt-;qN) znOH$@(-d0%js6`!zv!O$TCwk}Jq-`Jmt;qH3ka$^r_~KJUN!eK~G>TY<9DmPRKe$rdwnLvC%jn1t{ z8blxoUj2vRm*KRbmi90^pNI?3fYxGwsxGyMpJ{kw{TcN(YJTKN%P4($^?b|;+Y=1P znu(fvhDjzB+=_&5FPobb8N> zMVZ=cCUS2D z&!_8DrgU?AQfS|Zo3xFpuM_1-rmGf&87A1^fAF` z8)HYEU=WalCq@s(_iEKISyC1pr zoSK`+G(nQbL1^A=$joj|-3D{)3213qj6ye}{vj@hvz1wL19<~xQWwJc#B#7-S#}~b zlujf);VYQu-;gTGHA7oe6{sYp33r~W$bDsgBCWNq>M0VIzlSWU5?}^BXPUBA*a}QM zRg-uga-%|sk=i09oPa9jFqHdsLcH>)gnSx*J>OdIXm7UnyD!@>2TqHZ#1rE002$~7 zN8%#iQQu(y(Lk=)N~!=2+6w62Dk`a{HQ&ey%4g+wC{`7RQ)vKM^UaXcvyLoFz9u{b zW}oCysvn(9cZCvqGcwTIFh&07^Z^a>G}8k*g?${ucjB9XvHFc$%(dc*K)I8}USe0W zwV~cQ2%5)ldInt^s_+}Au}4zT(C@QkS)w_n(w9SR!9|Ql0&0ECIo~RO;}eZQqWlje z3>CwjivgXYWC;0Tp^l*-G)IcO8cFF@<;u`tEku=k1^TmYZ~_jLo&}2rw~JLo0q)wY zfK6;Fju&@`=b_!$Ev^(ti@n7W;yUp=m`0a_e<2mB4HSDrl<_J7cU`ZN#xqeobc@`BkxZ?L%31?t0W>2GNG{y_TjFQhfq#y$Nv zG!?1DN1y@Z!#$wQte{$meBVl7!R|n^>u2O+Im1oGEpefVKvQ&VCg}Kn95Yi>MrP~F|1kJ%X zOuniF%R!kG8O#;$i3gFy)L$GUZWmvQdL+E<4892Zf)U6Xse)YiaZqg9@m*hn9(y7b z*#t7C+dvaK3HspOpR4pary_x~3KFHeKrgb8*awUqimow`}tCtblE zJSCl!E=w2j`yWyW9{K;k=uDNPlrFd<8q-(h z8^r%k5m+zslu!8YxAD(ZFjv(f3w~sD_dbA-wj23A`|$T2g1+$!{v?5QsC1}a=%3IC z@dWs(I?(V@Y z5F}`D4H6sz1Pcy<;1E2xOK^90hr^xYIJdQt?(gm8_kTX-*=28cXLh=}y1Kght+x&U zDczMm2JW&gokNz0JNzbU&X>k0JVvguj#@eUD|3qmum& z>p%Wf>>yK68{J8MCR1N3q~zDKM;6lAFq4OD45kg|V(`S3hjsI*sWsiU+6 zWYgxPvw`6*0=Y~?9qOqpT?thSDxI(|(SR^&rvu0mau|D(eo__uHP3;U+`>Pha{D?| zCBF?E?gixM3?L~)p*iR+{6b~q(Uqe6MH^I$9tmXmJdDV$7@d@7fwm_Y|5(GoP4gOA zSMj_CnT{RcnIbdvVK?xZkNJ6e&=54RzaX7+L3#A@EXt2p+!*{0ezODE(wB z^NOdwxD*mKuw2Nv?B5j?vVU^B(g~gU$qsvl+o z^0FkVIQ76>mD_k}Mj3{=-g;rAnoE#_c~RepEY_!}dYcU>g~n!Q<1t?>a#{`f0pQU8 z0+(+!9&L>=Q_KMJzFLDhWFeo*H;4${kHip?yqCJDKudb-Gl{VT4bs?DOduXq*CImMTqEp$lrgku# zE43Asbq8PGR7AlVsh8Bp>=3dVTPPcmr?8AYCd*k{r5QcKDxm)I7}lNc(@ zxIO|A-UMSiH_ctJaZ`*FW{NQy`%NzcHOy!|uQ`vdu;ew@85_-#JPP>Uwq`we1}m&u zn6>x9a{X)7K;^+L!bvBRLa64N!ng6+sJ;DxRl+!}rJ%wiSwL5ai>S#u3Mlz#L=tnt zXRWOY@Eq$(M^q`iLUxfNG#y+rp}=G$C=b-F>^OE`PvJVtNqOK-h5~so0Qt4)A{toh zwb+$UAq9m(e^CAtlYqnVfJJwnIE>hWPG-^n#91yHlPP+ z{g+lRWY8T%<+)F=RmrHrG>512Z)PdfuY0dIMD4CpbO*VN8k3v28|c6C{1j&U>3jwD zuUw=IYsfcQeMm)eo`fS;0u^C-Q?i_wpqs5HxRc4e1gS+b)3#&;#%@7ijJA^Ps8aZW zbRsQiRp7}-();`&M!bnC?7wjv_G0EE!|x6%x*rph!~G;P*0GW;Pj!8VVbyyT*td$M$SP>qJ~}rvYC`4$zmTWbvC1MB81KXrhlI}NBWcZ z#xmr=mm(*`OCC)x0ju4IHo!c3ffl3tU?GN!2;@Coh76C0AtXvvBppOE@e_T8tgjT> z9HaDSuyl;zgRK?xExV4k?y#7^;*E-?dSoBG-sO6p@) zsLq?C7dYxjQB(9sr7v<{G%^5rvH37x z#gdUc9KQBY)SnHbV?=*?Li|W`(I3Q8T9$WWMM)&2Tb2AyX90l}N^8&?{0qH}%B}U- zOT>*wi%4!4Gck(tQwL%d+xRKEm@IyV@DW?xWoG!Zd8VI`5Z7&X!4G7zbuR0m3KhB!zE@gb;$ z8wWjlKzfonOt-=?^Q?sYXVSsQfT=-i0tb8$Sb{yg1@32-2t%G@HY;A~YPQwRi1GRf zRIy93GDI>kqzA=x zMS{q06(wDfJ^44OCHjbm$mG3%TCn%|K@w~JOLmAV*m+tYGWbsAuK7u{An$o3RzpuQ zmKww7j<#wbBDg`kvJ9F}EE0RgW>mNL!lOBYIbbEML2mLI^HE)rm&{a7A%CYR8IIXL zFY7CYTFuBTvJbzhfjcXjgs^wKzF3JJub+_(Qk{iUSrPe-*ukq{UKy^<7OOF)JCGT` z^z2q%^W(^%>xta<$>ccf>T+b#?I9Ynl}2FBO$8IhWRX*IThB>*QOoMiGMiDf1ew4G zSm)p?{RE%w0{ump$nd`^{^gG_C;Wr#Y%kp|;&^BBkF}f56t6`R^@=q(fHLKq$tbmo zF^7#37sV*i548>QLXumtF275+AY!(i3=;?Ohg?U}D57UGWDsoOCjNluWF^f9^fY}d{s13Y4r1{ym_iY+ z!HqLjGzC}MQD9*%iw>d!`DCGFm}o<1@ol7&7=_VYk<1s_k+*XR+EG;$A>-KJ)&zEr zjW=2-fAg|*FrCbk;G>KZ2|(&x2Dkk)@XoElbw86uGz+r4CeSCMlhp}XL1%e)__KBC zpZu9v4qmSe@(@)d3LwkBhgE_w*kn6$%_sB0WDU6pe-!+BVk`ekxx=d>=GKglrF)S- zwoe>`G>(v&;wt}&G1UzBR)#EMi+OHZ9M<{Mttk+qU1%LXQK^iXXflZt@UGE!pG0qH#Bbstd?+8XSyu3Pu+6m$~eTNL2#&o6FPi{b$y5kzx#Vg(bvbYRJ-GIVXfD1r6KH_L#;{>6A-B4SaWvMl)< z`uGdtSWRdRD>pRqFEI^%{wdPJT7y+=CD9gpjQcP$U%?{($&11Y6vLd|3VEN&WH#(q z4jRj=iE7{-JSJY#_53jAw@+B-O`-pQ@%aMl3XX;akXkKNmgtNtoxzHs4!MB&WddZh z7@qzQkmOgg5_4}D3B|l|2DAG>k&_l85kjyt<`v>(&0!TkBjag2qBsho8OtE0i~KjX z0hx~D;hj`Mp7eIGFm(Z{tsnW1|Dx_hE$(@kInF^sjcFq4x*kM-7KX0;O#1OE^bdI8 z9q0#9h_u8W>N=d&No*tKF(L=jQdr-7#;Xm$5Ix{`L@JriOTlZZ1`llxRWVa^BX9U! zWYmGU7#1iF7%L63^P&Fde$gB@v?glewtzR31OAqS)Dy2@x!f4J)#*!~jP++nXiXt{ zggmyEir>jpI^U{F--#Yz6qUAQ3we%t_7eE8j*H=FXEAbueBghP#>g!lNeHsTmJtg1 zwZXb`7IqV7@H~XWO0*((_%!+&E4gd%HZs9OX@T0unn_56E)$u=2HIP+qph*34X2-B+vf?Bj477$LuGUEW z3a|SPaxOR14wz$2%LcznK^{Gl{>+ z8PbAhXYCN1*ht1%ud#lQL2X=)J{wD#ppHm(^g=v7%?EFCy0|UgQw6;VATv-k#ndt2 zAz4em@uTqM60Nnui#Ss$zJm6{Oj4J2z>Jq$6on?d;xoxfaF|5W(&Bes3;WZjL=@dX zZt;`w{S;W2OS}N)`(}JFo6CQsFZe2;;6K7n&V-LWS|o|~KqxoTmT;Sx!>;rHFpmv{ zO?-;EZH{P9;l&e`))fE1Ix<=cBfc0j+aywp`bBNJl3YT6426cSfcLC`@xwu8V8wMA zna0n_RWaL4hOa*wmefrTi)Ew|^u9PNWDTKpz^42ww0)L1VV&d&#I&}e6b^yctSW z>nS_V3*#Q%iM^=F&;%S3^}r>zl@%Ag(2vc@6p;>(d=)&t?U3v(^vfpL=!N2%63Km7 z5mW)MeMii33(RvQ40}SWNeFV%f5)E2a?JRZA=%0pjYGi0@|V?w9<$<6C$25D={@Y- z1o03f;}Y>&(HKKVEk;L@Vq%k6fcTV8lozA<0Cgz#v+Kg{JmK#!D_5bFNK^RGHxNhK zjsDICy*U9%^`s6_4OS#4*^OGf;aG2-q-U(Qbey>z`d7hfgfUkD7QH@vse-H%zelTC z%P@-2B$cpN?7wKB$?&YV; z`CC|&8saV8!jdtX+_X9$O;(u>nuhhQgS_C!t?A%RxWK2Qo?0@WN7iEwF9YwnI$evn zbONISc7)AFMbDj(ug5Awx+)sxCPt%hb|1*&3SwKOX>PESUc*e1gBG*4(s802c~9Hv z%Yd_5Vk*Gi)E5be<@_S9h}Wh^?8aTZ7E$1osv&BDrFbdELP0AhJ1cGl2Eya9cm~)5 zvXgeS5l~VOX%4Y~tP|tJ9Px4tK?)wV3X`*1&PPXxgg?T zbI46&5>P2I$V5-EX3=}1B zpP$BBDw6jV9f1j4%FDt}8w;PVIA)^Om`#VnI+muDQ32Y8SkoOc3Ra@87=_QGMJhdv zm0~E?uvf`lcApgD)gbd~q=A^tf5z_gA6QRhBfr5%vqU}E^$39a$cn4Zndu;Tl~<{=s_q zUqm4ai)q+Fm@mG-r~MD>2pgFSCY1lMMwjdTHzXAG&)$)&tRbxee>)%Th8SK8jKfg6 z0~TZjqIe@jZg{2d#ZAN%n*yPBpS;5SehJs@30qJG>$VtC271#BEN?f!7*UPv1b_4l z%v=}2?LQGL*jo|p)X6aV8S$pOh*{?*_b@lPzyNUq9>@c}T9m*p!%7i`7<7HiS9h^u zstjJ3{8-<$7KIUa`9mBh6=)P<24}$*-<-xmes9rF55)no9Dbz=&o&2bjZtwBYy%Tu zcej&F(6JTp=XyZXcEctQB581+%%BPQwSMq(hhWv+gcKlKz-99o^H&mPzdoWEY||)F6tmDv%nS>$#x}9O zAmTo62bP0PVhQeI2wFD{{`Ml|_dbH(+>bv%47@P*QbzG_ydr%5lEgs#c@B1Yj$pRh zg>_Ol%u>amnPbUwG8ftr#(rlOn9qu^Rdg3BSw5wHnw!;NCDmf~zH%?FQa2v*y#p&_kA1+keg<0la7 z-)gorFM?rZ0rKMN2ezT|@j%~l?;cMz7P+wJ0u<4-s#2DkECBkF6Z2gdEjUs z?pfjK?rr01;9n9rrr$6R@%La9@1uRN=XdROpAXp?dL3ES#oT3_2kqBUpXw5O3TF4V zY#6FYu25fT+;-ny+)>fd%TWOJ@b23p!7h-bRk5XNhirZ9x9qzdQK;#4(HTI^DX(Kc z7{037FQUpxoO(@duM}5iu*qPCTLS)w$6$_V1k}zu)&{%94HQi&4(7orSb5YIlemT* zh#6?*bVRp{n7<;58ww0rvS)UNCw+SQ7vPy+WJGwU`D*zM-)>)X&#JW4Bg*|uM93kojnuo_4%k}Zdw%o2yFn%`;>j~!GA>cZR=w?sR za%t~TscX~Xh(7${ z{hJ>?DzOI=hh#L@ABa$({5OJS^CayIe{}=DNYYWMuaz@L=%w(dnJZ@Lm3e*Ue32!? zcDVA|JF4SY7OV_!BX6S)_BSF~9CkJHAaaKp$pX_4nD85b0jiGwgMVHimjTucs}H#h zWR^#(VxQ~qp~i9+_f&UTcSSIjHg$36$5dyUV?AmSg`>`x7kGypT3@w@GMOd74&Q~f zD2mw519*Ao=_KrrRZ;&|h3Zk0)plS!3R9b)-pvirXBl?m9XEl z&#KK!^Lb)Ft)=v~{pV=q?h%?de0_L)c)jqiV3IrRvN`wLm)QmbBSFC#`5t3OGW0xB z)-jbu!29hZ7W1KC%*urMrZ(M-`XT+m2=;?rx4RsNP}}Ku@IVx|-2zAAC0l*R80U4@ z2=^EFV0Tm3O~-l7!?0UyrUXX&274=bkE1U2`@jg}mU#`e9QGSqeJ9e2Czpu*@-g%+ zdvW^N+2|S3hh9$l(D`%0#0u#Jyxq+=R%L#cuQbo5H;demMhtc zMLo@4H(S5Rd7%fK^EIlTr2kK;5X~WYuqe@rN zz(HfE^#gWdN?}$3&l2{+I%8L{1R@UQ)Nl4tz@{|`cSRJ9SQ;J(9Tig4RoxL`pRW0| zcHs2MZ|h|1X4{Y6|3h7(00#-5eKN1e$M9TO8xNsfl?JM*9k-=|7igjFv6ik5P^T&j zz!cR+%>!<+e9nBX|J;+^PULr7aNM=kP#40hs*T*>IP*5LQvb8|TdCL`_F~WJqQ9*t zGQB|ZFA2%t#(nJ>bLvZhFXg_B|1#-|FXnE{xtNb%#n%PjrYDw9UX#{6z>FR&QCn@h z;9THr>}X(DY_W8&+J5RPhc0V|?E9CZ6ryOnL(=g0y2p)ce2*7uSSx0vwRFfch+oDz|$a zqg^2(5jH)2M|wEwc@OEjp&uc6X*;iNZd z?Yt`jMG=d62L7&&^eruonBop`0y{Uo?H`;YL(4{nXZ}5Fi!8roz8StC^uDXRql0ad zdIlV6zt~4QHaq4zR@%dC50r8w-n<_u=s)J|ajjS?ZO8oZXdIZSYauh`A}gwFP<{e)#!0YPR#MkuKXfYl8#s+T zKqGF#3=$2rkPbN=GHV*4`eXk_@U?$Qk5Ai~Iw9q7@;KC4sGe9Mp=W&AxclEO$MV?e zU{z=mS2tcy%$oWxJ;|HXXkr#Xe3+uo%3wWq80)>8?2UTQmd*V~$bj(s5j65)M9uIM z;K|74MA>)8EcQDMD}3!tj*t!%toh4$M_x z_FN_yb^~8wjwyqh^Ib_ZF%^i%NHdF36+X@y@L@goPxQz6`uaw}&-p83fBIAKZKtFb zNDE6_ou;RqNL!qqExi!fBYnsfY>nCu%YD&-GUjTmEt{|v;3l=Jdz3%fCpMbRRZeT4 zZI2yUUF%&6*b+B5pP?qFSGxlA34s<2vPav0aICQ(u{G5OE4f))vKcIhrLb#rocsi= zUR{=s>bJMB!9??teTF$`!;6;9q=fM6~0c2;-z^d@2SS!M@0^5Z5iy;n^ zj(wG@*u}et*lc#h#$zyhx8enWJjub!z&l9;8s~yF31czLdTNdY&c^^##C)TrQQOF5 zXhuPxp*kD=j2=MxGy~sLvi?{6k!?m?p4fdyy_ zMB^*qer5sJG6)Eu)@o%nhnfUWcL2~YfSFHs6n;@K^>4t{bc_`!9- zD_V!uV^!ca`@vPc2;3%idK-JD!?5lsi1kt`X2P>rK`a&15%m~_8L}<>$buqN7(4;H zesSP^5IhQToN|bJ)IiLmJk|lFurjKQI+HEnS+@}lMQP#2uKz8}tP8QvJpy~=O(9c- zzXI-S8&D)ut+C)StqbTlT8s(~z3XZKM-CMsBF8!EX_ zt$07!bBnRJKui4$hR@50T(?GiCIR}o1emp=h{j#z>##f79IedDvw|7eg~PzTe+91b zF*pw&;=rzgMSzDYfbm<8x8WVZXg8Gq&S&!F*sa^iPeL<4K{sruu~QV1tqK`egk($L zC&o=86)sfY<0SQonRc|okQt7A3X8LY~GVlBLz+yDm84}4T9is&fq zM@NDkb_V?$a$Q1~1B0~~|Nlj2(bVw`tDOra&Pmd&yEQyuzKNtA^S6%QU|fkAAFm|%BA0R|%C^9L{tQ*q1( z9&RJ{U$0`%{}W=YK4Bwn;F<_1L|N0*3PTZ+680+SQ>jGlJOI+QUPnxY$Ochfn#<45yx|^ zz8_)V=Q<=TcT}(9@0Ylb6vPX&qm>0o*&qXPRbUxvLyk2OPiTO6Vn@ic6aH^W8bU`) zBWj!#cTKSVC5S7gVFmsPagN*K0`?7dg1LA-*3WBTwYFj2md6%EGdJPyg=qf_U_B;7 z8dJdZIUDQnMey&J;CZq58)q%Y^UC0{1<~1^sNZ=Saf|D?(+l|j5}wcDozvj_Jch?r zd~y?#yaRo?j##ez#wC1u5yut$eK+{YQ$%B4<9LkYF1~*Y-@PO*2hYBbk@pmmd5hhl zbo2%z03xG5GC|V{V)aoTc*>uGeK83=un02V1VryPpaC|L4X}Gla4bUn?{Dq>Heh z)Gy(lu0e8-AjN3xA%4TyI$5BvQe!E!3q_ZpUvl?a1V^`qcLjb^zK4Q+ z_{Y#!Ahm=pI`QoQ?&%Zm?lxq10UCZ5cYYand=*kWk9W?Z6<5JiDy8xTxQi6@crxS= zgR?$i9K3)QybHdc0ckPBx*2{$$M@pU-sjMw+c^I+t|T89aph}}$>pF_uHpY1xb{O_ zQ3n4PhL+_+%X7oNMPamN$8%Olz>Z(% zIKz)KdGK0(T!{tSZQvar-jllH!F4mx4p2wpm6Q#GEL_mu?6^)IT)hA!UKr2P!sNs| zS@3>H@H>hWVbGFh;QWl>kp^u{!HAZBdGNoEXE`o>XpOYDCVs=jYl=I>h9In$6Tc{N zlU|(h6+Q44l8lD5UP1rl2)qla+<@d{zetUIiqBp{-ibl258#)-_o%erHdy;yI0{1_ ze!x*1mar~HPjx(MKwHXSJeGz1DTSjn{wj$>{#6V|L9{Io&X*R?4$O%iBbh>8^1jp1 z%Srgfm|#rx3-0p^eqHu|AlOS5TAhZ!5?~3wKnLC+cKQze_zwO42G1XH$1l(>V3W{- zCl~|wF=8IzSz4Y)c)p3&J%ENh#5Z1`?H_{L^yz=+zQ zN~?SuXFtI42%kR1@e1F66+Ghu?j;U)rw8R9f;Q$1&K9L0gJO_K<)DnJLShXunrh>y zfpJv>uch4N2&;|f+E~xk!gs6Sj8d>h`61^V(6=Z^Eju3JkeV%M|3F@VUXi1ZV;s2A zGj@F9z_SH?kXkL#Owy`I+x7kbM7*a%%CUGyO7bgSXW;*D_+NfYS|TZTInKTX z$7lRXEUa51`qTrx#AZG;fuRLb!!$gH;8}&{M&h+>Ul@r%+ohKwM@SKj%c2-D#h_co zA){hJO)DIn4T?fL^I=34fV48>+nI0$d43qK=EPr_AtU))0>7LbZ1*SJO(KquXmcDS zAlvvF(s+%w#X?f?I1+-NNZpe&(MNnsqBNvjzU!=y%G(ZC}ecgLY4I&>Z-lb{#UBMU>T!h&8v zBrIkWdRp3MIYZ^cqaY5cDY=5S)D`@;9nYB|jV$OImf+0Z@)&%Ws5(db;)Q& z23mx@DYRZnI3ALSN81zdjt6a)BTmjt?}Ad3)=i!#tzsIk>qG0M-T!{xr=X6=(Hb9= zqa4Mt_`bA)Qn%jWO7DWYBW<^|6j-34x1|QjIZlNXooIpd)1_|6hqNDd{O!i$`(cN4 zvg45BsVK%+DM+Vea3+*8iNYsQ_&+}+QwX0*nHR%1bKsD+N{+OgIC2H0oddrhWhQ4N zsl(s>1{;3aiBDudO3NniQqBn9?^)`yoZa-Gbr5KSJWI~m>5xi1WF@sqjv48#$vM#% z?1MDCmSa|qN2zb}h{YKhkYExdD5WSri9s(t$Kxsb>J=VOf|CFL5|lFh0;zt%m13Zy zaX3#}O3-uQkRy^}R+Xbrw&=Trkuvzdai!sZ>8nZ|QE=BL&XFF18PsEWj*h=%i{$4j zWTW8W4)%iV7bj%nhE(K3N=Wt*P*C_kA}HOE;A|j24GA6*_*?d&4cGg>f8Xte^akWR z|M$L}Z>6OE&mWOi1clUarkp?Iok;sD`(FAzB`_Pw`KbYE1SCXt%SkFv2 z`0uWRM`D#yKxqN&WoPU+t_4zbr8&(=1H*M6^wz_aLI!N z4t0!DWcw@tKIWcz%_s|$j~}s^=fJ#VL0(Qipeu{era+|Hk%!V3XuVM^5o{Wpffs5C zMC32ZYw8Blqk>Wym2xVvCg7QWNZSB!XsVsGt;m27ybyL`UtUrF(geHJ(;cAdm2x$N2n;})o{_kJOKR3cvy#KY$6b8orTAI zNdu0@tf*C91b81H=a#AoN-aXIuf{0;RVl(MSiAW`?ABEQf~5yynN_U_^A+$DaoA}e zOc$yrm94ytVH(>h({_v72C|5ZAB2b9K}2G_RV#3mjzrvkB_h|SiE7ScZJjq+ezPgJ zSRFN!Xo`qoKl6*YicGV?w7j}XD8_Q@ckFB}VkzW{6=LK;K22@Xm|Y9edAX!){tj#u zDs((&v4{x$Y4)@_Ba3V|?qdpFXEZUEkQDF&?F9}hgu0MbyO!8(SGDrA7(X3YWE^0v zZF%TZy(f^)hmd8p6wEjm8_#cNF6b!;5=Y%NLOGf&#MKEAFSufW#C>5nca-u zq^H%3iMdU#L$-KLNfbu(I4dwy5RlA^OLyhuHd{V$A zKDj5z9r^8lAsSfL%I+QJukLJ_xfZJz z=!eonF(IRdq; zx@t>aRz0d&dLB<6@>V-(7Bw>|H?jYhLm8`7<(t4{*xTC5DmjWN)2x=pT|FCC&3`E4 z>~q;gE>P}ivHdsf#tlbcGNb7&9?s3CkIYgXI$$Y+srn;GWuh>OL2fnVZV9WD#hZEu{fQE zo7JskEeFfSS0kRc-K=01(|&d|=5D=^l?F>|Lr(f+YZC9RR28NDaeSWkP`d?$VjfG& z;Iu<{w7Sz<-^j+sVHA{7kJ6s{E~BK?nuI$`+ecb$GivH7_H$Z(>k`RtPY<+6RF#fd zu2AmpLsP7kdQmz~+XFrKQ%l_&$ZEZ3|MER%5&LA={^0Y$nREPHY{Wh-HII8?7R$i%+4!~L`nbMz;5hEHbXS* z6rbo*d?_|h=v$f#737ZdlV&laxOUKx-2+%9+4+ew&`Rw)pb8>W7@R-UqVyZkq2{7qMQO>&T=J@l~

    PIZGsY`9%r%UBoXCRX5X3h%~8m!y5ehTw9t|qE8KT&%~G1AtrvN;yLL;-4QAQXzE?iZHd2og zWiLpEcuSdW>^lF^_%-y2D?a0P+9K-@c1D?D>(57<<5*`pmKQLJSv!<0+ET^Dd(YHu91U(aedKl-ox>-ySz zqiJWyy714A%$~A|_4INfSDj;!!P3RLR zV~kS|S!#qQ^HQyc|QcSOvpl3W( z+lJn9zr&1lz#Ep_Rea9e(>}l)AS$N);+q#fD=dT8NX@Jlhn;K4KG>(R_udn}#jLRV zkiDT*DdTJ)MzguMi*r^*M;_~7Mg@A@G2J?96j8eCSM2HTKm6s>I{J^=&t@v=9_N3W zJRvQ^?CqMKv#v``Y4s&NwX1zxmU@{CH8rDET-EqC&V@xwx?SK`PWP3xW@^u!b?6D~ zXY5lx5Z(CK_+~_?heZV|(NG;EuOfLg1TG;&J-Dh<~X3#>%a=yxQioA68HT(F}sAg}W z#E?v8r3|k=hgPt44SAsr3e2;%ic=Y3fk;+R6Q!oZ((aP7WsQ1ziI-q zpSsXQCQ#qGx0JhXlafqwj9PrufnKvsukm3ZoRUXS9_W51GCJh zN*?VOGuf!k2N*fDT_JglM+t|-BS#KeflgG%xnhmx>3NW`lU-e|tP?Yc;@Cq<`>OEL zwrN@iATx{k`&q}}Uk(R@>-Dsr2?fcMYz^HPtVGYgj9tLpY*eObKZ{1*6?8&qIA54{ z*{r4vbCz+P(0}(1(MPi!u3Fj$WNFz0lf2=2?~q2B-ZEce4&N|$RcoZ@E2|c=Mp@)P z?^}y~h2hpPeoJ#}y04txL7QeKCyldx$QS1*^|e?00QE21OC<*Ar3dEHj48fzVzn)Y z^C4;A>BtRRT{A3@pH)X4p;(SJFW(CE<5tV>%IwH%%s?H&GOp9%XUV@tQJz_U4c@u+ z?7btW*3sCR{DJSX1+a;C;4ZRkYkRspFeM)oS5y5;CT|iS@RcCk*3PqPeg46F!h3_iQvP$jb8RqRr9JV2v%wafskvG!wY@LZHe7R9Kbte@ zKz`6zZ(Br$nu{!_Z9CU}5g}fCB{MeA!8bSEsdULVUYnfK6LVTOI^DLCX;y9JGp}nj zpjFjX$|`Z+ybGLYUeu>L>Ua~@#dLa#>X!s_WOr<$EA=)a9C_5gD5>PE)mG$l{Hav& z&H?XIa$uu9Hd8$wl2#i05UGJG%E6Fc+I-!Xwwu>=jKQiEmB{R`_*b(!Kc$xwozxuU z9lI2gES4txYkbQ3P8sl}skcPLuuK<7Ay2QgeWt_Z4#}ylwP5v9n}5B`zGQD0elERB z@(B^I7SR(DBh~f=XN9^`ZhDgV3I9m{IeV6zd%`w*c7HpOa$b3Ae`51HzR;h%Zg`%9 zNjdGTM42kD-Kw=K*|PE|{mCOob(f1b_ub4m>Rm=Uh1PMGU>B_E8ErBS@aC=y+8|m_ zsV8oFXQbUQ4o0Y%Z9 zou!3!Q^t6TXY@9XsV&{%_98}x&!$K6T1pewpY%q?8hRwGhHr7ApH0dTEVcp5C-f4Kr2Nuf!Zoh$SJRW$nAQr>-#n(}WPCPKe!+pF9C(vVz#84HG93 z1emD(N@`mP$ekWcyyOg@seN@`Q40s&rA7zZJI013+2@m%=@A~?YA7ZNo}f!*J=@P%WB!e--${Bu{uFB_t7+91A+|17e}ky|k-1o3>0pGXwqg~s{AK%^@yXYfFQX$| zn;l2>r>Rr*xoo_>VrVH>thdJ30lxfUpR$ZlZ7Gqy1!9h|m6gyAv%-oU{9vPun*McG zK{iUur`A`ti<h75Y=u-W@pe$;fW* zVa3s0;2>?teW*52lpfKK_^$A};LMIxsDG@tC|%@u;CRpcMr}_&ucj7v=Cr2*)q2Iu zW%VO%l{~h?#*gXGmEz$|*-3K)@~AJFQd!5xxN?&6M8BqLYLYj#Dk2ro&%yx zXaz?%{?v0J?W@`Y~QuYdDFuc^iz+N;)6Z+?r2jaQAxNE$zEWL&Ci!zWeFmAG*?)CO>zC+1J{AWpJ z+e=3-dr2$9zXmKVku1qR)4oF#(NnCfzHh!#S}kWT*D%}9=Gv4IX>R}TbW-+_`PUoc zJ`G6Y<^WpI#EYDq`H*fP~u0YZy(eYxKRy zs-4oRkgj@&zJgy+z4k<<3F~C?v=IL$+W}{F7Ht;64(JD~GgzZukss|vwOu4P9cLBx z7Bf$3W7X=&f;bCK^2z#by^)qnJuWEf^SlEaM^<&5`ofAgpOC`ZPOLfC>qA8a1mk~K zitwaBH~8yaolUbBVlgS*%xua~)Eu8{ucbxtzx00mg#w%tvK|Vujkwk-SmjgZEb>iz zsruNjRv84DCy}3B{oLtR0pIU|4Bn46wO4fyQuCQN{3rBh{>xT9XF_N{*4hk3g}W!b zpf$|gD*gjgZ%cL+tMw7SI{M#^n{Lf`;+=0EGTxKN?6h)^)fL0^6Z#M0sI`|52I^w8 z)>s>7wn(pKIvnTihqb-RS~5)6GA5ZP?5ADBoJYtq@4&R`)=T?K`yF)xi!|;2X;u^& zt4u?M>7J}T`N1me8?2|Z0nXdn5NnVBk*}NiCvD@%X1_yv@do~i{+)D-?Hq7+ca^M0 z_rPvs(w49WYqcFotQbF}kJKmeN!k$GUNRBvJE)~7EHLeE&>EQ;iRHZTWSl+Ji^%NW z%1>Jz%%S`hVjC4u4V{sF+z@T)R5FOpM4o0bYqZq_v9%@ij?$dWK~?JW#zL#P(!sGp z%d7ljdOgSW<4OzrHf1}n2p*Cq$ZBiCN+U{Fi*+FDjM=)I*P$KN1-2Q=Xlsj6)7l`0 z0CD(G8A#U~c5@DS$mXk)l?$RWs@$EoN`tfaxYmqa=G(=5>z;9zABD|{RIiB6=5YNK ze@bs6wp@jEpy8sJRUI|*&hThut#(#C0sA{5hu|$cpgtg{fmd3AOz5A~ENVI;!s|#) z{*zgXzY$&8G$4l~&8r@#c|faW%g#>nqGndTi8)=Qu)ov=j3eXnS1_vdwh)I_j%&ZE zE4c#3w+Jx4998SG#=O7zfgb`cau*Q(3i*SVMLe*%)!SGv_G{&ADYS!`;kO%ZniG{( zBj_of3XUKRd9$yOL0*>E1Zrdqc-r#u4`iw`Tb;ss8I`bZYDV9K@hLO8WrcHHtOU|z zJgEU(dkO5gmla0{JV^Rh2;Ru51+JHEG@II&O+aOcUdVJ!hv#|&`JY3870yg~>%DO53qAgihIuRJPjR?F$nxdtqRsqz6v?s z4Z-5k3;FRCd4_ll#M?IB5OwPtie12?^c9~G-^>jJ;XC9nnm{TP;7fs({e|xYM&T5{ zW8Jk{Sqsc!Ru5hP=*L>TkM)z;&HTq|kKCtjVgkQ}ESf=x-RZzIh9IuI0QFoy(y@qc z^h0&ZLDWQGd@8%CTtS^yZ~_Ad-2yd`qfps(md#=B4%B8CauO_fs{@g3a0k^MCjjA_ zuI;xi&_*fG=qModegF^MSk{gmCl$fgFbH>%gP#LJuCO)UynsritAL1`ZPYVHp@wZo zy%(@>CG^9Ah2VQzjf%H_2b`#2_&U%`PXkZdSO2eIQ2*-x!Qa}m1sKGJ8ArVLJin#i zO{tmeNLi7V$#+ISV%|2t83Cg?uSNvxp-#3nah7mjaVNXqgj^1}7dkO)V_56(^zgh9 zd&94XrG%=Xe)kynJok9_LDxG+A^R+C6h_f*R+~0K24pgS3K_)%Sys_>19R4gcgOA- zL7w1MM4f>?VZ~AZq^?#(Ev6JiP2YUl2HRMB8T)?iDe%BvAX1+r%U~rbAlh-=OgG-@ zO#`YguV-<3YTDzW zC#_ks5q~G~VHypThjqE3wxr3>=i2m*-5FXA4Dq|e&W9fb-gwe?=Ir%0>po3M_0#FAW*YA|8?$khS;0fE~t^pHDupB zq2qz*nultut-;=wRqu_8q6blJr?R)NC)#t~^U9OmGuYe3Gs?HgTiJKh)5^Ee``ViT z#QyvAgK3@8=O#Z&G~?UHlf=Ih)8p61=SrIIdC2c;P1tLwQUuhGu;tk5urI!sh!w!@zH8weL^M$Fo%GF4-JSMPujT5K zqf4IOvqy$ZM7_q4h)bD1MGnn;BkJeeN26AUb+OO0odK(~VlM~};4a&y1c={yst0@x zJT*{bvqYf0`3C6fS^C_-#XzW0$Xvy*vCZ~g&hgFx+9PF@cHN%ph;@u~4t7lrxf62Q z)y%Hh`f01d-Wj2-QS;e`+1jXUz$#Q7?0uiXfKZg3WY3g1ii}Y!VBWUkfUdr5jbWX! z*Ye3YuU{~mS#7LAW>!Smbza;orVlbI0h3z7B6AEb)I*uvE|Wv*v{J$Y^$|$>UK63943u~O?<=3buo6N=5u0J0G>OW zIn^2tG&tp#5nJ7?>w$5`Tk{=w3uEDpWd<&J8S0jAMXxRg8{KwmFzS_80z&+yfmtu` zyD!F5&XdcN-=ldNryodF!1g>mxl~HKj6wQNV?o3N*0mqJp(puWd;#a7iuF<;NO-PL+1edu|h z23BJKc^7c3^;s3{OAZp-xsUGN)PUUUx0k9Te( z$6zejkCu~uU{o7K%7Jy}AE3c6i+=ouCAnU6^GK_-H4s`?7Tn77EWg>Er}Fk<4)%z* zV25ulGN=Y1PkJ(T0#}2NXSMavENxaXZt0`+Lq-W>p!piib4vq1`19#^z@b_MJEm6P zR^V-53_QoPV7@(Ws>lXP0_*;DpyIm#wYD9*z|(j(WCA!aGRFeBK81}(%VZC4!9HL! z?2sLRH`5Zi=CkNm`i*U77h${FVQ;e^s|c+7Ds0{FU@O^W)&fj=W5Aj>5UB4I_L-%F zA-NLjlt(i~i2(EKF}4F&Jb`mQQ_0iU4osUb#0GE#4Ir`bH7^Ng z5ZAO8xZ<6_T_Cd#JVepJ4G%^h>sYXvU@sDTqyLJm;M5v{Yy&S(F zZQxNq!Dlxi@%cdaJOiG>1d?tYaKih5IbDst_87c-5Z@UG-{vj$@Ck4rx3R0A0Bk`W z$tDQ|?kphwyW@_xgD=trgm(lu#Tez_9VrVIbDGkB-#v1j~+TtkkG z{g44@uv;VSzexnsEAT_6$m;eI4);TG=Yn~<0e zcMxNg8(6i{K>2k+7D0R9DXQYp7{AjVxW8t2RSDR?96)|ZM7Z zNW6{2%7J+Zk6XB-J9tP0QvyCo58`4%(B2Zr`ECS>wn2Zk#$M^qXnRk{wObH<-3a5b zI`DiF&z%F96DiRoT=fyIcmvvZ9aoij@YA@%lh8?tzkV2$hD1zBTn}n@;f}N*Vn!nT zB+AW&D_KDdN_r6G@&)JJ2+B#KLa*X_62tijl9EWZL}&|z#2q**587DY(K{ zfO(Rb=~B?`+~~usI9%v+*%uNG`4U%>SYvsd4<3*3$s@cUji{kS-%C`tALt;7>Xm4` zytq<6;87$#nBg~Vzz`+l9%NriJ(6g7i7`Eco;v_elH<^-13-H3#Q$f4D3{}SzKUOe zh4xD%*;gU)uv(GL~GI)XZc@DYVL3**g+9Mxn)0lO-ZEfLBtpB%V!L0oi(q zdX`$o@vK3^B=T0~927(wC8DxY@Q|abCMD9NC{c;@OT%4#$G1tnmPl5A@Q$Q3Wm}~jV$t%qkbp$n zO01xiuS6HhPo#Y06(!DG;`n7QgAbn@u-noOOJtwCvPA8pvJ+Y_rSd&jzzXV+M6pY3 zs>Dx9^qfR#JMfSg@NCe9T97^#Gr%{gRPRiE;C3ZJ~9FC$D_?sauSCsH7gEn zmWX4CLw=9 zDR_N}j+Eb#sAh?El-3i)r647#a}rz1F&D@j6p3h*_V0Ud%J(JiRcfsFe}~i?nKvOZ z@@aTZ#OpLjNurOX1ivFjrJOVHzWhvnI~A?>WS_erYuQSP9}UHK-8jnt{@jM|%g+@& zJ8@2UuwTB96PZ6GuOssU{TSi$n-cjeano`{I$%9Y0!^F=961?i=L_7OiEGNmjpk2kw?) z+&#q1p~IF=#@L;QY``?E#TtR}X%+voe_~S&PoC|$0hAI#;w(XwcbL;C}p<&5Ng=G0bg|k z{#b+}qcK7OHVVusx3JE<#=BUWsK`f|Z>^!$H*kYj2CL3QWdT?R{|4fAzQ&l(_{ay zvh#qm>FWCbIp^M)J53p)MIRA8gdn1h8U)cNLX;o~qt}=R!)PIDFoP(;C<$UPO7va^ z5k!w*^fH)Cx%ZU+cir>e&yeSNKL7Wf^O@UD+h^^y*Is3>-?E13s=3^3u3S@lfwfx0 zxapY!J}l@qAbw5I4|@g~i$Pn7H(Q`1Ct5GTgZ5}m{ab^mokVW<3NYGM83#0q4!d22D2;4R~UJ3E+B`IS4YUn~wPRd*vhc1*k(rt|=XBcD0bED5T1 z2FOJbpq-|fjjV!Tn+(NrXsp&^jnpw}vP@8Ou$ENtmEKl^z>sRAE@OrqVx_X$@fWad zn_Ah-)=C$!9d}wq;j_}Kx^$qqlYg@zTrery5iX3Dh!?;7QDGbFB?ZE$+O^TEBWnjZNej`DEVDVug$5x=%kQxE8lnm1#@=hHo`7dMBiTL?BrI}{_YrIOjeZ1S5IgrIXu1cn zIX5DgW?@U#Mnh{s=5RhF&kx|Ur!)GKzyeQXb@whbw+3Y4TTp8p((G%jwsXjZIG!ZN z+y@>2c&*T}t}=ma)?=)-7>W$1nEZ;w6O4=eBp z*WZl)VmR69+p#FW;JWeDF_>jOG0K?rp+Y^@3OpdfG6M{%n`SEOtUH^D)N_P7A3Oc2>ap8-+;NP|{TIk5O`rqhIreqTK(IXn%mT&^&v)+r?)vUJ?pyAXRR8`fBTvTB^rh+V zQO9;vX7S7@YO9wdHq+W2OwHtYcQvM*09II5HO}kBjBupmCG!+@=KrFa!A`tIe}YRf z$`S48>sX|f0GVN|HV4RJ2kWyYD0fX$FRRb=gb@dOWDGN+^~WBkUf4({3Y?P}Baqe8 zg|vat5 z==T6WzyJN=8&>-Hdx_ z5f~{^X{R$Dy4!g^_w@5r@x;38>ix~F;HW>jBrOQuopf7 z(K4sGP5+W=FK&ImvB-SK_(~tM6LiPDU(k^QI8r*a@Q-UDT@ z0%&`yS$#JPM66Kfb1-D?;|E)yy+`eg_KyC(aNy`eIeM^hs*^= zH++3DM3H*n&0Oc{;#uhl)hikYjTwxYyNsq@f*VYx^BmSsA21E$EBIm#$Zo^*hMsDk zW}aPCxH$;=h31KNZ_jK*4WP^JA)d#cbk9Z4aqu#R>c#bu`V0N2KHTu54oo?twz=P! zYP9?4n@km^BKj8m3? z3YcPcGXFp>G|_T_4EMeEi#l3e$x5UN(9wwwu?)HY2po@Jbbq~)LER5y8`u<`(JTD1 zfj$PyFG0KKsOUG#@3Q|L*8POWKS`aibsCm$(fPjETuG#(_{Nw#DfSEN& zE1^~bl{nHoV-&>mo2-M?13yj0_c>m#1;)c-&`x)=0^>0?n~tj0z>jci55eK6qfJw9 zAjj%~kCAK)r|MW`R{488D?qS0lsO|~U&c@>Ms)!x?MKf-&su$jJ{07UCdOf-nDr5B zc|!Sg-i*TZ8$#5trZvO7W29Iy%$!%jDx7OwLMmyj@F+_an;%)B(9ft&4EcMi49%uO z(QVeHCm1)8cWu!cy5Yx8)rYXQCIl&18^3;WW52HJTY080(03}EEv&AVi&?5aT5~Mb zYFK+jW=nT%q&fzikrd6(Sx(ijdvX_ za8-QiSniksmQghjL06&y1~OWkdA7UHxXXCTc>+8iP>btS=84RM?r-!x)L{F}d~Wmy zhbNNxvj;dK>&*pZ<#ady1xkDqqB!}CJ9;!!_}HjMjzJ?Mo};Q-z3EHU9Av{ zvz{An{U#i+gbGZZKn`qde5O++mzdB8Mqk$V-7_juv1N^U-u#-lfP;E*b@64lLOX8aJu&qLgm+APX_YPmE9 z3-kvpm%31LuHd?Y@7A2QtkpXriQ4FGi752ZqdT1be14)!OE(`>}A2@+=pK6RkrsY%xwzRca8ejtBYWIO`>*vR35; zt2KT{!#Rzn(F!f+H@5Hb1m~rCK?1V1xLMN((T7ln_@d{${=N}l)YngX+IwO>jSQFh z5xOXN)u3Xp1Vw9s<0LB<3OIrt8yqX(p?K#vj$}2)8VN;<~c#gRL$ZYDK z>~895ssBmk9u>_ogNiY$$w@m6UDuhT@i^qg8=lLmYqlcpneO>iZ)VJ^Vy&J3h7 z(hg%PGkI$)zVgVP+FFd3sy)+=QdR7XhLmPqNk8Wmv>MHsgIYzuf?%}{Eg=bX&J^{Y zdL2B^3+giHSVf(x{>t|Xnp=IQf`qFU0`=5VOKAD9c6Ni|lT!`DTIfxTA`DDj1$!?N ze@#&$R0*_t7sx*+u$b;6;U^L!8AtYFIV#wdWj6cRs)*)yj{0P(5r%{tP5fpk7)908 zNXI?L&#q0bOa6ePhGU(h8&z-nSgp+2RuR)-BzbmZj7rzj&ZQY?%QH4*b^*ny zt*4l~vAeXVxxUd@z{pNB^Ma6bo?2EewKVk?cA}5h2Lrc;nxfWM!{Mf})c=Y$kAgN; z2CXs|GgN(SuFut6>RMuUr=<$3R?Be{oXSRyQRLXgFmH`xe-~VK2KjbQ=#f($j5XW_ zETFX@=l)Dg#zfm0$EH!s^nJAR%Xo~RlW~`b4y-6wu)ro$iKsH~LFB;2QN!a5t=P-R zXwCmtvl2B(R-$=GeZ>r`Dv?8f@Noh_HByNbP6K5ySQ|?}M3GT59*WOLe_BJ!np3BK zs@g}*uZ3v&HF5<&RPrZy^#W}q+;Kx&PNrIQkh7@=+WR1k#eKJZjev!mIb2w04{&1h$=*WV$soIuUIQl566Cz+KpBQmOI zBxQ`xh)X}0HZc8``xG$J zgN&viA1A^mqwpepYc8k0Ks43#LRb@(hA#e_*4Ob7b;vapd9PH}&Ny~DKL%s)64>l>vQP^>JyyT@z~8uy3H@+Z|0br^QpJrK*YQ?2j^wUT-P{p5k- zhZfpIy-q}>BGyT3(B?9g7v${Z!GDrT#evVT%X=ydtbEK0yNIer&~IN`;mGV6*e}EJ z;AhfPxu{F{2WZAWVWaK@Wx6e^(LN?B*ao}1EttCfk+x@uL~o{-SJKzZLE6{Jiui;0 z*f-cmi^12qPRwrvxK&A1HK?FgSCc`l?tzu}6MYoJ-TvebDSTdN1#so}$h3vsLQouTo1m)Ph20RSIur2Q@q0smtQfHg?SbM3hM(f+g3_XIiX=3>#BM%y(wSc$JTCZ8e=SR@0 zwNy|UgjXpD|9&@mumE|n4)uVtg{nr+mEvl7>~$ACo`_933}3iH1oI?Rky;nypkN}_ z?NTx<{{RmwoG5W0+Av&Qr7luq!NrPTtzQ_moZ6|~!NTf+6%<6hr@!!iJm&v6xakMz zxhwR-ed@xjf;ZZM(;WjM(njp=-f%`+wvKSgUtUD9a1g#GqH_tB^$4E-SNeD;&#)H> z91o)MT#)}ZVABkZJBb660n8Izp#4p_ zesgOFqgFH5f|1vf)s$7y<|`RtAh^BIFHzC;G^>N|>6v<%QQ!ECt)uY;RW6sJDIYan z8l{McPe4nWPsBT(C;J)A^^sWxp8px&C)FTm>p?{j)H|z7!S@T(T2ke1HZ|~mp@!UY ztiM=#J^`$|tJ)JSh=_hMM`1@O)pBoX2U*cV6(g++`gT4ojT&0>)UU7(i>W$M?Bnq(f90TfcV@n5{D6^2qKUkJi!Wp*o~!8~ zYc0pl{T7PNq;KLG*_Yrt{B^u0!EYyt+aE>aI!%A3Qga{=bhr!h`CYs%x1h11-3n zFwb4yDuM4QCzgXBIpr$Z>N)YJ}ATm*P;{kdK& z6#5I#F=+Ql%>ihMD;c8~!D)Pj@2V8zv>dPR;JYdaqW4FPwI%ow&Y(5KFba!MmuutF84plF!;Hx-}ce4%s zy~+H|ykc%g8`{Sww+WWEwHK*Y4j)v1wjt0v4zFo4IR9Oc0*BOV_(Xok%X3UkpiOaX z3(*#5;jx(mX5M(U8?$+c`V4=>GQ3`Wko}GEmPs^Ia2ph6gxm1M75dwON3J??`7X>2 zU3iyO4|VbRwcz_Co^CzsQucD4OJE&7KzhBv19FZ2UCnGY8SWTO>jv>2g(T~PhTnv! zW_4oag8Ed9sA(Z0iKY0L7^lVjB1l138FeZVusR^Gb)?VBq7_hSgSnv> zEw6boBtyV63`g2Tuytmx>xM6F0@37Op+W%Cr!`m+bCH|Jp|AsOyDsz_hrB(B?wE@e zKvCLeESy(bFm~d5jQvuMvTT*u-bEIe)D%s`Bh^)_$$FrYS}5r7`_Nv#MCUB1J_j#h z662`?-pCQuI!d9A<5RFLHMnvSGN1YeaEl8+U^%NGRdVX^eL9xjV{DiMjPM0!1oK{~ z`5t!A0-kIOb2mOOo^K#|8l$KQ@+~W;mV)CL$vTdeTr(Qp?reRIt+AJR>u_SBpcc52reCB}Q6B^uk*B z9P2WIK4T;fWh_RZadpDm+lmo29({N}_q&JHnuGq%O;ji!ecOmRa4@{G7U^(@9QgCd z_2-PS3wTk_z)c4J>OfXk6`)d~s=i?M}wTg`yBNf5F9409yy%=ckU;0 za+X!zPq4ysBWGGbg??OlEPXH^PtZ!_>_uW%5}_)KPq`y=Uk_&BQEj_ls zK1S8zyVStd@R>9=B8?TsX}ou3nfs)U=@O*mA-qn1vu00p*HzwG%LTs`M%Dyl*%#n$ zHIZMD=z?3BWws-g*Fx>J$oA84R%xv20q9gUkm)Ac%50ukkcX&WOFT6d|C7aQ5;OGo z@N|F1(Rlp%G3Yk?kjg*N2Ls^xDva3>D?g*Q1F@xVnW5Iumet;MRxyk`G6R`cuOsmk zBA+2}Kq&9|k%^@laqSrs(a7}WNS$5cSD{`aS*En;m2PBr$)k;oPar(X-+68EAPi%*?U$WLHMF&UoBL&(EdjH`AVj zXn9l7nC&I*GzwYX9XuM`ql5)Ie=Ogj!^KRzXD^e#QrkU6P0 zbH^mOY%|aQJK8F>wHUwE8SkAK>HQe-eL;a8tqxKLVx5116llfh??iT1xLO%q(ZRaG z`^qyU#%=W9n?!g{L5pQjVGQw**3em2HVG=Mf;Az!&t~|3GGp`;G=j49ULG_C1bE7|7J!;6?%me8UY`g%KK`#hTY12m%y+I<6llwY1B0RpkU3)060c&cQc z<`Mg!FdyCH(<_b^iFp+c*+q{}$42T22leo-qLbI&XirVy`m)G$S-Dgl{_g+>3`Xyc z^y*eU&~HQvwq&2IMas=8%5%_T60$qQ+RW$*Fi%oheWo!A9eX;fwI^d8tHwTki9Ssq zi$`o4{U!329K#pCb=KKL;JvHcKj2U=iW1#m! zP^yoy|2+Fxan4%*g5Ww=;Ox#=lM|WY*P{nqg$Ey^8;0OnYKSzCM#3Cnbf=)dx@eQ2 zA=XB&^uh|Ck5oBFOd>auqZTt}l=>t7luK%+S_Y{cN&IZIHdUKJ{9=Q)mZ|}3&~FZF zr|_Na)lO&+Gy^N=miD{0hdmRt)_BvhBg_4?oZzSL!{4!pOr0jg+-1C$hi8kR(FgH= z7kc$a4nM*6PNaR)sJ%E6eX9ne>=COeuHhM*&n(r9*>*C%-G%rlJK{YpK!ohPvB%hj zU4EI&noDSSe$2WdQ12SsWy4MW+;g-%i1}&2CorwzQCtMG>2ex zi2T?s+0iPt;&VBUjxhvZ(@A*r6~67fS|u#R=2|;#CNZ)f@JN={iedlt!AAU*m|+(x zZgnIMbDFi@GuU$y+9#;{siAWpDyH)(P92F%n@`U)!25Pxsg4#e)n+?E)5CCTW31-J z$X5Ik#0s{+G4qJ8lsA6T2k4W@B(24I?rugOs_o2&yE2ULjb(7s7seVRidF7M(TjH* zXP7(N@Ova@cEtxgj8R$&nlEPtjz`*UMN6K}_f15n6*RdhBwIYO{mN*|3ieVj_&E%# zavK#E7cefyAUjGjdb+^*`SFQfVExcWcz+XC)K%t}xy+7fRJsemmuO<|J-{XkRnIXR zUC8n|WCip_YJUobesg5}-}q*>siry<`b~!aVzdTm&bhU((6nnhK6E^Xa$gZeSO$$- zXm4wmkp*2j?g~_GNv`BBd?6c=6_fFe?}uWi@iZ-H5se4(^nK zO>-ODCKb#7ETd{0TIf?W(9*PH3KIPh7S=iBLI@h%OmzDk%2f2l7qq`7cMXQeA1d#` z**)?67F16lH6zuTAlHwkvO}0Ap7Jm%l&zrU-!Z3-Q7@};tRN_@jZK>#^__v994by5>M6Ye6D#w|KIq zXvgm}Bke*?-=!|W7`#4zGE)bzGG(=yZ1zJ(3AWY|)rf`rS25NNB*ASWVLi|^Dq1(N ziAK_E1C2N&K#=(xqof=$xB5ubKe4QJq85GO?)LC?ICpGeo@F$wBHk1aSLmTsY++~Z!+G@+UhX% zsoDxjdX-T%iij&U~^D={fFcH+`lL^JAy2$XWd1<6vXCvfe*YS?JB{TJc~A0 z2Ay*x()9zp@n@M8)?@3;hZc*GHqH)uQkZjFyfHcp;rTBv&+gLmi-C6lZgMK9eV6|XqQGW{ebVN zDg9g%3QGQuWYwHOLQZ1l>cQ&HgVgzKL62{MU!JgXD+BLqC8~e!U`7zc(5tK-E6g0W z5NW&`@74~spP_9z)^a-deI}mHfsC(y_&ukQr}7@xcX7XqtR`%WE|p*11RbifHKq-f z(F4oSzP|MB04TK^>wFHfcObfiKjZlbJvjm0e+Ye=1JuZ~_#2({dn!FM3(cr5PfLAn zD9{ayMB)fvz>7c9*6GX#gQ4GjtkLIKk*P#KJ23NVtOgf<`+W8{#Ii4nt<@9G?t&#M zTv!|1CW2OMf-)npH40%_+@Kvx;j?^5jmON2JD_7Uz0!}ip2S0O7Oz|Yy6{k*p%**-z2>S0-N zeF@6;Cwkivt9Af({Ve9h)A-6Su;TazJ+c5DW()d!K72KiaL6oVNlRK13jc3_YLA(( zqzXbQs1OYIorTJ=jDr;GbJ}?cYdj7cp#~IE&_TANADqJauR_1Izz!jClZ zhV({nuNPn>y47qvEl*SSH&^`xOJy8<*%up5P^}BG zj=(`IMXCWv2HII@zSry1`G`57GA$Qmd0BC}3+Xo!%Xb^Rq#$YXqZd>}o2lpZ307c~ zSH#cM5r5NHctc9SV>h7WQ`$cZYPIFu4c-{fX?zIh@Q{@BdXpuyp%9WI3^^g619%3h zjv>`L_M#guN8{QA)l;F|Iq&MT0K7ADykx-$K1pSGtqkAt!*7E8E@;DV^Gpxm$lIKC z+nd*O0*buy<{RXM_L3hb9GVyY6u+h)er3sQk*o~KdeRuL7JVY9>5^X{>-4j6<#(Bz znxTum1y@UEg~3^}dR|b^4Y*E)PN_&rli!|jm6F`45LcHhJi(_Hzq#OhyS*}pY;8u) z?>yak`bXipvarJK+6K~w`}9p9ZKYN{@$W12PXaPXGL;TN5f3Y7#aC6y8>79*xH$$l z%If#O(cTJS%~zpcigI^5r$%yNB&$LE_kVKl6P$4nFZNcrWIZ$TN!lX$Xb*XcTu@c; z@x|kxn{!^#mRq#oBFD%k)j_0()4lv4xqjk7lNtk(O<`lJOD3LVaY*))jpQvEDmkHi zL7qu^T`FTp=1LA)_=q-1g$}7SU?DN&ibc7iV0ufwjAR1@@GADI2ii!6iDX(xhJ*!` zB&#l%UNHEUiDXZQJ|_QmE{t6RM4myaEIjnKTQU+PKSwqf_mD~(g0C*f)`EsE*+P=% zBiQPanIkyal2amCXOdT`A%`VX$ga&HSralgWL1-cca65lcl!*fhGP2|3vmT`B9&Kp zmp{~R?jRXNlHnnFMDnX-P#Ely%m_JN&JeofWwZ;9xxtYyyxAD9_+)3INdAtb zT$+Q{=7;{WNv@FO!bt9gv`SXW%f38(mTWp3-Cz1ldQ8ym1@~X_uyS*Z;N43WTM^C) z<>>|EU9uMXFw!MIMrbMdGEyZ#@>N8JODiN>MslBI zlOv=mh#V(*B9fQmW|Kd;n>?ZPlWcNV8MAm8O~vf1zQ zt9@mgvi8w(yzfu;3w`b9_WkO6xB1gqSiAIYD6KRee)_*9M)PWJVhe6st~zOtMn z{V7i-82qvcciYb?DEQJMyRG)U?cSHGN`L(GPwsEiK%O(}&%T0vE&0{gM_JFCb*z0q zn~tyVmve-IHf3bL-8-)zf{w357;NA^R8m zRerJQD!=>o$!}Rl$?rCG?X%@5o3b{AWo%|`QV&GF*?l1Il7A`LZPGWsk>oo@ezC7& z-_dS^+`;Bm`;O8|-_y#we7D=0^%Sq~w?7MgZJOG>=^IDF*Ftl76@K<{w2!BK`)z5E zwO7u`I!2E2^`P{UuaA73Yv0{&!J9p6Kf%AZSDwm#p4a#Je*I5RDpZmEZ~nc0w(qmu zf5J0ztnb&XvweH)E6MM6f7m_YyN~>4k33&%vUj8E zf8O=~_g$_llzsEK|8zGWo!>mxcjTMzK05fWDtrIgf3KgB#mn-moaNi=yF%8x?})5l c{_mf%X#amdH|vc59p$CH_OoZ5`QJbNKf%7=hyVZp literal 0 HcmV?d00001 From a0cd474b62e90871067b847e4b67bf7375ef83ce Mon Sep 17 00:00:00 2001 From: SAP Cloud SDK Bot Date: Fri, 17 Jul 2026 13:33:35 +0000 Subject: [PATCH 13/26] Formatting --- .../sap/ai/sdk/core/RealtimeParamVoice.java | 6 +- .../foundationmodels/openai/OpenAiClient.java | 1 - .../openai/OpenAiRealtimeClient.java | 186 +++++++++--------- .../openai/TextToSpeechRealtimeClient.java | 8 +- .../openai/ToAudioRealtimeClient.java | 35 ++-- .../openai/WSSOpenAiRealtimeClient.java | 1 - .../ai/sdk/app/services/OpenAiService.java | 10 +- .../sdk/app/controllers/RealtimeApiTest.java | 18 +- 8 files changed, 131 insertions(+), 134 deletions(-) diff --git a/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamVoice.java b/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamVoice.java index 10fc818a8..3746a2f3f 100644 --- a/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamVoice.java +++ b/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamVoice.java @@ -6,10 +6,8 @@ public final class RealtimeParamVoice implements RealtimeParam { - public static final RealtimeParamVoice DEFAULT_1 = - new RealtimeParamVoice("DEFAULT_1"); - public static final RealtimeParamVoice DEFAULT_2 = - new RealtimeParamVoice("DEFAULT_1"); + public static final RealtimeParamVoice DEFAULT_1 = new RealtimeParamVoice("DEFAULT_1"); + public static final RealtimeParamVoice DEFAULT_2 = new RealtimeParamVoice("DEFAULT_1"); private final String voice; diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java index 3402e39d8..adb1df725 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java @@ -33,7 +33,6 @@ import com.sap.cloud.sdk.cloudplatform.connectivity.HttpDestination; import java.io.IOException; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.stream.Stream; import javax.annotation.Nonnull; diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiRealtimeClient.java index f1923347e..4ccbbd12c 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiRealtimeClient.java @@ -4,110 +4,110 @@ import com.sap.ai.sdk.core.RealtimeParam; import com.sap.cloud.sdk.cloudplatform.connectivity.Destination; import com.sap.cloud.sdk.cloudplatform.connectivity.Header; - -import javax.annotation.Nonnull; import java.util.HashMap; +import javax.annotation.Nonnull; public class OpenAiRealtimeClient { - private static final int PATH_BUFFER_SIZE = - 400; // existing URLs are ~120 symbols long, 400 has reasonable margin - - private final Destination destination; + private static final int PATH_BUFFER_SIZE = + 400; // existing URLs are ~120 symbols long, 400 has reasonable margin - OpenAiRealtimeClient(Destination destination) { - this.destination = destination; - } + private final Destination destination; - /** - * Creates realtime channel allowing to input text and voice it (receive audio output) - * - *

    The input channel should be used with a try-with-resources block to ensure that the - * underlying connection is closed. - * - *

    Example: - * - * 

    {@code
-     * try (var textInputChannel = client.textToSpeech(audioOutputConsumer)) {
-     *       textInputChannel.sendText("...");
-     *       ....
-     * }
-     * }
    - * - * This API implements full duplex (input + output) communication channels. Application should - * logically synchronize their state and close input channel when it is appropriate (e.g. last - * part of the response has been received via output channel and application does not need to send - * any other input). When input channel is closed, output channel will be closed automatically and - * output consumer will not be called anymore. - * - * @param audioOutputConsumer - audio consumer of raw PCM mono 24000 Hz little endian output, 16 bit depth - * @return input channel, allowing for text input - */ - @Nonnull - @Beta - public TextInputChannel textToSpeech( - @Nonnull final AudioOutputChannel audioOutputConsumer, RealtimeParam... params) { - var extraHeaders = destination.asHttp().getHeaders(); - var headers = new HashMap(extraHeaders.size() + 1); - for (Header header : extraHeaders) { - headers.put(header.getName(), header.getValue()); - } - var endpoint = getRealtimeEndpoint(); + OpenAiRealtimeClient(Destination destination) { + this.destination = destination; + } - return new TextToSpeechRealtimeClient(endpoint, headers, audioOutputConsumer, params); + /** + * Creates realtime channel allowing to input text and voice it (receive audio output) + * + *

    The input channel should be used with a try-with-resources block to ensure that the + * underlying connection is closed. + * + *

    Example: + * + * 

    {@code
+   * try (var textInputChannel = client.textToSpeech(audioOutputConsumer)) {
+   *       textInputChannel.sendText("...");
+   *       ....
+   * }
+   * }
    + * + * This API implements full duplex (input + output) communication channels. Application should + * logically synchronize their state and close input channel when it is appropriate (e.g. last + * part of the response has been received via output channel and application does not need to send + * any other input). When input channel is closed, output channel will be closed automatically and + * output consumer will not be called anymore. + * + * @param audioOutputConsumer - audio consumer of raw PCM mono 24000 Hz little endian output, 16 + * bit depth + * @return input channel, allowing for text input + */ + @Nonnull + @Beta + public TextInputChannel textToSpeech( + @Nonnull final AudioOutputChannel audioOutputConsumer, RealtimeParam... params) { + var extraHeaders = destination.asHttp().getHeaders(); + var headers = new HashMap(extraHeaders.size() + 1); + for (Header header : extraHeaders) { + headers.put(header.getName(), header.getValue()); } + var endpoint = getRealtimeEndpoint(); + return new TextToSpeechRealtimeClient(endpoint, headers, audioOutputConsumer, params); + } - /** - * Creates realtime channel allowing for audio conversation with a model - * - *

    The input channel should be used with a try-with-resources block to ensure that the - * underlying connection is closed. - * - *

    Example: - * - * 

    {@code
-     * try (var audioInputChannel = client.speechToSpeech(audioOutputConsumer)) {
-     *       audioInputChannel.inputAudio(audioBytesData);
-     *       ....
-     * }
-     * }
    - * - * This API implements full duplex (input + output) communication channels. Application should - * logically synchronize their state and close input channel when it is appropriate (e.g. last - * part of the response has been received via output channel and application does not need to send - * any other input). When input channel is closed, output channel will be closed automatically and - * output consumer will not be called anymore. - * - * @param audioOutputConsumer - audio consumer of raw PCM mono 24000 Hz little endian output, 16 bit depth - * @param params - optional configuration params - * @return input channel, allowing for audio data input (bytes, PCM mono 24000 Hz little endian 16 bit) - */ - @Nonnull - @Beta - public AudioInputChannel speechToSpeech( - @Nonnull final AudioOutputChannel audioOutputConsumer, RealtimeParam... params) { - var extraHeaders = destination.asHttp().getHeaders(); - var headers = new HashMap(extraHeaders.size() + 1); - for (Header header : extraHeaders) { - headers.put(header.getName(), header.getValue()); - } - var endpoint = getRealtimeEndpoint(); - - return new SpeechToSpeechRealtimeClient(endpoint, headers, audioOutputConsumer, params); + /** + * Creates realtime channel allowing for audio conversation with a model + * + *

    The input channel should be used with a try-with-resources block to ensure that the + * underlying connection is closed. + * + *

    Example: + * + * 

    {@code
+   * try (var audioInputChannel = client.speechToSpeech(audioOutputConsumer)) {
+   *       audioInputChannel.inputAudio(audioBytesData);
+   *       ....
+   * }
+   * }
    + * + * This API implements full duplex (input + output) communication channels. Application should + * logically synchronize their state and close input channel when it is appropriate (e.g. last + * part of the response has been received via output channel and application does not need to send + * any other input). When input channel is closed, output channel will be closed automatically and + * output consumer will not be called anymore. + * + * @param audioOutputConsumer - audio consumer of raw PCM mono 24000 Hz little endian output, 16 + * bit depth + * @param params - optional configuration params + * @return input channel, allowing for audio data input (bytes, PCM mono 24000 Hz little endian 16 + * bit) + */ + @Nonnull + @Beta + public AudioInputChannel speechToSpeech( + @Nonnull final AudioOutputChannel audioOutputConsumer, RealtimeParam... params) { + var extraHeaders = destination.asHttp().getHeaders(); + var headers = new HashMap(extraHeaders.size() + 1); + for (Header header : extraHeaders) { + headers.put(header.getName(), header.getValue()); } + var endpoint = getRealtimeEndpoint(); - private String getRealtimeEndpoint() { - var sb = new StringBuilder(PATH_BUFFER_SIZE); - sb.append("wss://"); - var pathParts = destination.asHttp().getUri().toString().split("//"); - if (pathParts.length != 2) { - throw new IllegalArgumentException( - "Invalid destination URI: " + destination.asHttp().getUri()); - } - sb.append(pathParts[1].replaceFirst("^api\\.", "realtime.")); - sb.append("v1/realtime"); - return sb.toString(); - } + return new SpeechToSpeechRealtimeClient(endpoint, headers, audioOutputConsumer, params); + } + private String getRealtimeEndpoint() { + var sb = new StringBuilder(PATH_BUFFER_SIZE); + sb.append("wss://"); + var pathParts = destination.asHttp().getUri().toString().split("//"); + if (pathParts.length != 2) { + throw new IllegalArgumentException( + "Invalid destination URI: " + destination.asHttp().getUri()); + } + sb.append(pathParts[1].replaceFirst("^api\\.", "realtime.")); + sb.append("v1/realtime"); + return sb.toString(); + } } diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java index 4ad9f6918..d8f12aa66 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java @@ -21,8 +21,8 @@ class TextToSpeechRealtimeClient extends WSSOpenAiRealtimeClient implements Text List.of(RealtimeSessionCreateRequest.OutputModality.AUDIO); private static final String TASK = - "you are a speaker and your role is to read (produce audio) of the user input speech. voice user text input, " + - "do not answer questions, just read them"; + "you are a speaker and your role is to read (produce audio) of the user input speech. voice user text input, " + + "do not answer questions, just read them"; private final AudioOutputChannel outputConsumer; private final RealtimeAudioConfigOutput.Voice.UnionMember1 voice; @@ -45,7 +45,9 @@ public TextToSpeechRealtimeClient( } else if (RealtimeParamVoice.DEFAULT_1.equals(param)) { voice = RealtimeAudioConfigOutput.Voice.UnionMember1.ECHO; } else { - voice = RealtimeAudioConfigOutput.Voice.UnionMember1.of(param.getValueAsString().toLowerCase()); + voice = + RealtimeAudioConfigOutput.Voice.UnionMember1.of( + param.getValueAsString().toLowerCase()); } } case TURN_DETECTION -> { diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/ToAudioRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/ToAudioRealtimeClient.java index 785414a0f..84038bcea 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/ToAudioRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/ToAudioRealtimeClient.java @@ -2,32 +2,29 @@ import com.fasterxml.jackson.databind.JsonNode; import com.openai.models.realtime.SessionUpdateEvent; - import java.util.Map; import java.util.Set; class ToAudioRealtimeClient extends WSSOpenAiRealtimeClient { - private static final Set HANDLED_RESPONSE_TYPES = - Set.of("response.output_audio.delta", "response.output_audio.done"); - private static final String DEFAULT_SYSTEM_PROMPT = ""; - - public ToAudioRealtimeClient(String url, Map httpHeaders) { - super(url, httpHeaders, HANDLED_RESPONSE_TYPES); - } + private static final Set HANDLED_RESPONSE_TYPES = + Set.of("response.output_audio.delta", "response.output_audio.done"); + private static final String DEFAULT_SYSTEM_PROMPT = ""; - @Override - protected String getSystemPrompt() { - return ""; - } + public ToAudioRealtimeClient(String url, Map httpHeaders) { + super(url, httpHeaders, HANDLED_RESPONSE_TYPES); + } - @Override - protected void onResponse(String eventType, JsonNode event) { + @Override + protected String getSystemPrompt() { + return ""; + } - } + @Override + protected void onResponse(String eventType, JsonNode event) {} - @Override - protected SessionUpdateEvent sessionConfiguration() { - return null; - } + @Override + protected SessionUpdateEvent sessionConfiguration() { + return null; + } } diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSSOpenAiRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSSOpenAiRealtimeClient.java index 213725452..a3f9383e1 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSSOpenAiRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSSOpenAiRealtimeClient.java @@ -14,7 +14,6 @@ import java.util.*; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; - import lombok.AccessLevel; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OpenAiService.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OpenAiService.java index ad6c92c2a..af6554451 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OpenAiService.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OpenAiService.java @@ -5,6 +5,8 @@ import static com.sap.ai.sdk.foundationmodels.openai.OpenAiModel.TEXT_EMBEDDING_3_SMALL; import com.sap.ai.sdk.core.AiCoreService; +import com.sap.ai.sdk.foundationmodels.openai.AudioInputChannel; +import com.sap.ai.sdk.foundationmodels.openai.AudioOutputChannel; import com.sap.ai.sdk.foundationmodels.openai.OpenAiChatCompletionDelta; import com.sap.ai.sdk.foundationmodels.openai.OpenAiChatCompletionRequest; import com.sap.ai.sdk.foundationmodels.openai.OpenAiChatCompletionResponse; @@ -14,8 +16,6 @@ import com.sap.ai.sdk.foundationmodels.openai.OpenAiImageItem; import com.sap.ai.sdk.foundationmodels.openai.OpenAiMessage; import com.sap.ai.sdk.foundationmodels.openai.OpenAiTool; -import com.sap.ai.sdk.foundationmodels.openai.AudioInputChannel; -import com.sap.ai.sdk.foundationmodels.openai.AudioOutputChannel; import com.sap.ai.sdk.foundationmodels.openai.TextInputChannel; import java.util.ArrayList; import java.util.List; @@ -127,8 +127,10 @@ public TextInputChannel textToSpeech(@Nonnull final AudioOutputChannel audioOutp * any other input). When input channel is closed, output channel will be closed automatically and * output consumer will not be called anymore. * - * @param audioOutputConsumer - audio consumer of raw PCM mono 24000 Hz little endian output, 16 bit depth - * @return input channel, allowing for audio data input (bytes, PCM mono 24000 Hz little endian 16 bit) + * @param audioOutputConsumer - audio consumer of raw PCM mono 24000 Hz little endian output, 16 + * bit depth + * @return input channel, allowing for audio data input (bytes, PCM mono 24000 Hz little endian 16 + * bit) */ public AudioInputChannel speechToSpeech(@Nonnull final AudioOutputChannel audioOutputConsumer) { return OpenAiClient.realtimeClient().speechToSpeech(audioOutputConsumer); diff --git a/sample-code/spring-app/src/test/java/com/sap/ai/sdk/app/controllers/RealtimeApiTest.java b/sample-code/spring-app/src/test/java/com/sap/ai/sdk/app/controllers/RealtimeApiTest.java index 8b73f7c77..177d8d208 100644 --- a/sample-code/spring-app/src/test/java/com/sap/ai/sdk/app/controllers/RealtimeApiTest.java +++ b/sample-code/spring-app/src/test/java/com/sap/ai/sdk/app/controllers/RealtimeApiTest.java @@ -11,7 +11,6 @@ import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; -import java.util.Arrays; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; @@ -28,7 +27,7 @@ public class RealtimeApiTest { @BeforeAll public static void setUp() { - try (var fis = new FileInputStream( "src/test/resources/fixtures/question.pcm")) { + try (var fis = new FileInputStream("src/test/resources/fixtures/question.pcm")) { QUESTION_FIXTURE_PCM = fis.readAllBytes(); } catch (IOException e) { fail(e.getMessage()); @@ -104,14 +103,15 @@ void testSpeechToSpeech() { } catch (IOException e) { throw new RuntimeException(e); } - } - private record AudioMetrics(double rms, double entropy){}; + private record AudioMetrics(double rms, double entropy) {} + ; /** * Audio quality metrics for signed 16-bit little-endian PCM, computed on the mean-subtracted * (DC-offset-removed) signal so they reflect only the varying, audible part of the audio. + * * @param pcm - The raw PCM audio buffer. * @return The RMS amplitude and Shannon entropy (bits) of the zero-mean samples. */ @@ -120,21 +120,21 @@ AudioMetrics pcm16AudioMetrics(byte[] pcm) { return new AudioMetrics(0, 0); } var sum = 0L; - for (var i = 0; i < pcm.length; i+=2) { - sum += (pcm[i]) | ((pcm[i+1]) << 8); + for (var i = 0; i < pcm.length; i += 2) { + sum += (pcm[i]) | ((pcm[i + 1]) << 8); } var mean = sum / (pcm.length / 2); var sumOfSquares = 0d; var counts = new char[Character.MAX_VALUE]; - for (var i = 0; i < pcm.length; i+= 2) { - var residual = ((pcm[i]) | ((pcm[i+1]) << 8)) - mean; + for (var i = 0; i < pcm.length; i += 2) { + var residual = ((pcm[i]) | ((pcm[i + 1]) << 8)) - mean; sumOfSquares += residual * residual; var key = (int) (residual + Short.MAX_VALUE + 1); counts[key]++; } - var rms = Math.sqrt(sumOfSquares / (double) ( pcm.length / 2 )); + var rms = Math.sqrt(sumOfSquares / (double) (pcm.length / 2)); var entropy = 0d; for (var i = 0; i < counts.length; i++) { var p = counts[i] / counts.length; From 8dc8ae60a9db58993acee429f82c87933bb0bccf Mon Sep 17 00:00:00 2001 From: Vladimir Aseev Date: Fri, 17 Jul 2026 15:40:21 +0200 Subject: [PATCH 14/26] remove duplicated dependency --- pom.xml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/pom.xml b/pom.xml index b6aaf7cca..ec3bb4f6d 100644 --- a/pom.xml +++ b/pom.xml @@ -280,11 +280,6 @@ openai-java-core ${openai-java.version} - - com.openai - openai-java - ${openai-java.version} - From 42409100a5598e2325c36ebc1f10c423d8ef43be Mon Sep 17 00:00:00 2001 From: Vladimir Aseev Date: Fri, 17 Jul 2026 15:50:20 +0200 Subject: [PATCH 15/26] improve code style --- .../openai/WSSOpenAiRealtimeClient.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSSOpenAiRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSSOpenAiRealtimeClient.java index a3f9383e1..950ce5b57 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSSOpenAiRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSSOpenAiRealtimeClient.java @@ -5,15 +5,23 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import com.openai.models.realtime.*; +import com.openai.models.realtime.RealtimeConversationItemSystemMessage; import com.sap.ai.sdk.core.common.ClientException; +import com.openai.models.realtime.ResponseCreateEvent; +import com.openai.models.realtime.SessionUpdateEvent; +import com.openai.models.realtime.ConversationItemCreateEvent; +import com.openai.models.realtime.ConversationItem; + import java.net.URI; import java.net.http.HttpClient; import java.net.http.WebSocket; import java.nio.ByteBuffer; -import java.util.*; +import java.util.TimerTask; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; +import java.util.Map; +import java.util.Set; +import java.util.Timer; import lombok.AccessLevel; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; From 0603218aae981e331b2f307d868bbaeb96373ed0 Mon Sep 17 00:00:00 2001 From: Vladimir Aseev Date: Fri, 17 Jul 2026 15:50:59 +0200 Subject: [PATCH 16/26] improve code style --- .../openai/SpeechToSpeechRealtimeClient.java | 2 +- .../openai/SpeechToTextRealtimeClient.java | 2 +- .../openai/TextToSpeechRealtimeClient.java | 2 +- .../openai/ToAudioRealtimeClient.java | 2 +- ...lient.java => WSOpenAiRealtimeClient.java} | 4 +- .../openai/WSSOpenAIRealtimeClient.java | 187 ------------------ 6 files changed, 6 insertions(+), 193 deletions(-) rename foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/{WSSOpenAiRealtimeClient.java => WSOpenAiRealtimeClient.java} (98%) delete mode 100644 foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSSOpenAIRealtimeClient.java diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java index 59fab1a02..3f3085e9d 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java @@ -10,7 +10,7 @@ import lombok.extern.slf4j.Slf4j; @Slf4j -class SpeechToSpeechRealtimeClient extends WSSOpenAiRealtimeClient implements AudioInputChannel { +class SpeechToSpeechRealtimeClient extends WSOpenAiRealtimeClient implements AudioInputChannel { private static final int MAX_DATA_CHUNK_SIZE_BYTES = 8192; private static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToTextRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToTextRealtimeClient.java index 377e43625..e8ed82610 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToTextRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToTextRealtimeClient.java @@ -7,7 +7,7 @@ import lombok.extern.slf4j.Slf4j; @Slf4j -class SpeechToTextRealtimeClient extends WSSOpenAiRealtimeClient implements AudioInputChannel { +class SpeechToTextRealtimeClient extends WSOpenAiRealtimeClient implements AudioInputChannel { private static final int MAX_DATA_CHUNK_SIZE_BYTES = 8192; private static final Set HANDLED_RESPONSE_TYPES = diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java index d8f12aa66..5734e284d 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java @@ -10,7 +10,7 @@ import lombok.extern.slf4j.Slf4j; @Slf4j -class TextToSpeechRealtimeClient extends WSSOpenAiRealtimeClient implements TextInputChannel { +class TextToSpeechRealtimeClient extends WSOpenAiRealtimeClient implements TextInputChannel { private static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/ToAudioRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/ToAudioRealtimeClient.java index 84038bcea..7033cfec7 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/ToAudioRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/ToAudioRealtimeClient.java @@ -5,7 +5,7 @@ import java.util.Map; import java.util.Set; -class ToAudioRealtimeClient extends WSSOpenAiRealtimeClient { +class ToAudioRealtimeClient extends WSOpenAiRealtimeClient { private static final Set HANDLED_RESPONSE_TYPES = Set.of("response.output_audio.delta", "response.output_audio.done"); diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSSOpenAiRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSOpenAiRealtimeClient.java similarity index 98% rename from foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSSOpenAiRealtimeClient.java rename to foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSOpenAiRealtimeClient.java index 950ce5b57..db13f5030 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSSOpenAiRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSOpenAiRealtimeClient.java @@ -28,7 +28,7 @@ @Slf4j @RequiredArgsConstructor(access = AccessLevel.PACKAGE) -abstract class WSSOpenAiRealtimeClient implements AutoCloseable { +abstract class WSOpenAiRealtimeClient implements AutoCloseable { private static final int SUCCESS_FINISH_WSS_CODE = 1000; private static final ObjectMapper JACKSON = @@ -56,7 +56,7 @@ abstract class WSSOpenAiRealtimeClient implements AutoCloseable { private final Timer heartbeatTimer; private final Set handleMessageTypes; - public WSSOpenAiRealtimeClient( + public WSOpenAiRealtimeClient( String url, Map httpHeaders, Set handleMessageTypes) { this.client = HttpClient.newHttpClient(); var wsBuilder = this.client.newWebSocketBuilder(); diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSSOpenAIRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSSOpenAIRealtimeClient.java deleted file mode 100644 index a78f66d83..000000000 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSSOpenAIRealtimeClient.java +++ /dev/null @@ -1,187 +0,0 @@ -package com.sap.ai.sdk.foundationmodels.openai; - -import com.fasterxml.jackson.annotation.JsonAutoDetect; -import com.fasterxml.jackson.annotation.PropertyAccessor; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.openai.models.realtime.*; -import com.sap.ai.sdk.core.common.ClientException; -import java.net.URI; -import java.net.http.HttpClient; -import java.net.http.WebSocket; -import java.nio.ByteBuffer; -import java.util.*; -import java.util.concurrent.CompletableFuture; -import lombok.AccessLevel; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -@RequiredArgsConstructor(access = AccessLevel.PACKAGE) -abstract class WSSOpenAIRealtimeClient implements AutoCloseable { - - private static final int SUCCESS_FINISH_WSS_CODE = 1000; - private static final ObjectMapper JACKSON = - new ObjectMapper().setVisibility(PropertyAccessor.IS_GETTER, JsonAutoDetect.Visibility.NONE); - - /** - * WebSocket keeps TCP connection to the server. If connection idles, depending on the provider in - * cloud environments, cloud provider sends hard TCP RST (reset) after 60-300 seconds of - * inactivity, this is normally not configurable and would leave realtime conversation unusable on - * a pause. - * - *

    In mobile networks this period is even shorter and typical CGNAT session can only idle for - * 10-15 seconds before it gets terminated - * - *

    In realtime client we rely on explicit lifetime management (create/close), heartbeat - * mechanism is required to prevent unintended closing of connections while conversation is still - * expected to continue - */ - private static final int HEARTBEAT_INTERVAL_MILLIS = 4500; - - private static final String HEARTBEAT_TIMER_NAME = "wss_realtime_heartbeat"; - - private final HttpClient client; - private final CompletableFuture ws; - private final Timer heartbeatTimer; - private final Set handleMessageTypes; - - public WSSOpenAIRealtimeClient( - String url, Map httpHeaders, Set handleMessageTypes) { - this.client = HttpClient.newHttpClient(); - var wsBuilder = this.client.newWebSocketBuilder(); - for (Map.Entry entry : httpHeaders.entrySet()) { - wsBuilder = wsBuilder.header(entry.getKey(), entry.getValue()); - } - this.ws = - wsBuilder.buildAsync( - URI.create(url), new BufferedWebSocketListener(this::onSocketOpen, this::onText)); - this.handleMessageTypes = handleMessageTypes; - this.heartbeatTimer = new Timer(HEARTBEAT_TIMER_NAME, true); - } - - public void askForResponse() { - WebSocket ws = this.ws.join(); - synchronized (this) { - try { - ws.sendText(JACKSON.writeValueAsString(ResponseCreateEvent.builder().build()), true); - } catch (JsonProcessingException e) { - throw new RuntimeException("Failed to serialize ask for response", e); - } - ws.request(1); - } - } - - @Override - public void close() { - WebSocket ws = this.ws.join(); - synchronized (this) { - heartbeatTimer.cancel(); - try { - ws.sendClose(SUCCESS_FINISH_WSS_CODE, "done").join(); - } catch (Exception e) { - log.error("Error while closing WebSocket", e); - } - // this.client.close(); // exists only since java 21 - } - } - - protected abstract String getSystemPrompt(); - - protected abstract void onResponse(String eventType, JsonNode event); - - protected abstract SessionUpdateEvent sessionConfiguration(); - - protected void sendMessage(Object message) { - WebSocket ws = this.ws.join(); - synchronized (this) { - try { - ws.sendText(JACKSON.writeValueAsString(message), true); - ws.request(1); - } catch (JsonProcessingException e) { - throw new ClientException("Failed to serialize message", e); - } - } - } - - private synchronized void onSocketOpen(WebSocket ws) { - configureSession(ws); - configureConversation(ws); - scheduleHeartbeat(ws); - } - - private void onText(WebSocket webSocket, CharSequence data) { - final JsonNode event; - try { - event = JACKSON.readTree(data.toString()); - } catch (JsonProcessingException e) { - throw new ClientException("Error parsing JSON response from speech API", e); - } - var eventType = event.get("type").asText(); - if (handleMessageTypes.contains(eventType)) { - onResponse(eventType, event); - } else { - log.trace("Unhandled event type: {}", eventType); - } - - webSocket.request(1); - } - - private synchronized void sendPing(WebSocket ws) { - if (ws.isInputClosed()) { - return; - } - ws.sendPing(ByteBuffer.wrap("ping".getBytes())).join(); - ws.request(1); - } - - private void configureSession(WebSocket ws) { - SessionUpdateEvent sue = sessionConfiguration(); - try { - ws.sendText(JACKSON.writeValueAsString(sue), true); - } catch (JsonProcessingException e) { - throw new ClientException("Failed to serialize session request", e); - } - - ws.request(1); - } - - private void configureConversation(WebSocket ws) { - var systemPrompt = getSystemPrompt(); - if (systemPrompt == null || systemPrompt.isEmpty()) { - return; - } - var systemConversationItem = - ConversationItemCreateEvent.builder() - .item( - ConversationItem.ofRealtimeConversationItemSystemMessage( - RealtimeConversationItemSystemMessage.builder() - .addContent( - RealtimeConversationItemSystemMessage.Content.builder() - .text(systemPrompt) - .type(RealtimeConversationItemSystemMessage.Content.Type.INPUT_TEXT) - .build()) - .build())) - .build(); - - try { - var json = JACKSON.writeValueAsString(systemConversationItem); - ws.sendText(json, true); - } catch (JsonProcessingException e) { - throw new ClientException("Failed to serialize message", e); - } - ws.request(1); - } - - private void scheduleHeartbeat(WebSocket ws) { - TimerTask task = - new TimerTask() { - @Override - public void run() { - sendPing(ws); - } - }; - heartbeatTimer.scheduleAtFixedRate(task, 0, HEARTBEAT_INTERVAL_MILLIS); - } -} From fcebe1e6276a768267de2aa4f2d6850fc4142cf7 Mon Sep 17 00:00:00 2001 From: SAP Cloud SDK Bot Date: Fri, 17 Jul 2026 13:51:44 +0000 Subject: [PATCH 17/26] Formatting --- .../openai/WSOpenAiRealtimeClient.java | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSOpenAiRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSOpenAiRealtimeClient.java index db13f5030..90cf63bbb 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSOpenAiRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSOpenAiRealtimeClient.java @@ -5,23 +5,22 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.openai.models.realtime.ConversationItem; +import com.openai.models.realtime.ConversationItemCreateEvent; import com.openai.models.realtime.RealtimeConversationItemSystemMessage; -import com.sap.ai.sdk.core.common.ClientException; import com.openai.models.realtime.ResponseCreateEvent; import com.openai.models.realtime.SessionUpdateEvent; -import com.openai.models.realtime.ConversationItemCreateEvent; -import com.openai.models.realtime.ConversationItem; - +import com.sap.ai.sdk.core.common.ClientException; import java.net.URI; import java.net.http.HttpClient; import java.net.http.WebSocket; import java.nio.ByteBuffer; -import java.util.TimerTask; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionException; import java.util.Map; import java.util.Set; import java.util.Timer; +import java.util.TimerTask; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; import lombok.AccessLevel; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; From 9a6d251b3b6ba809a650a18295bbb8a063d8faf2 Mon Sep 17 00:00:00 2001 From: Vladimir Aseev Date: Fri, 17 Jul 2026 16:23:13 +0200 Subject: [PATCH 18/26] extract common realtime api functionality to a superclass --- .../openai/SpeechToSpeechRealtimeClient.java | 124 ++++++------------ .../openai/SpeechToTextRealtimeClient.java | 94 ------------- .../openai/TextToSpeechRealtimeClient.java | 110 ++++------------ .../openai/ToAudioRealtimeClient.java | 80 +++++++++-- 4 files changed, 131 insertions(+), 277 deletions(-) delete mode 100644 foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToTextRealtimeClient.java diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java index 3f3085e9d..31b8611ad 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java @@ -1,30 +1,26 @@ package com.sap.ai.sdk.foundationmodels.openai; -import com.fasterxml.jackson.databind.JsonNode; -import com.openai.models.realtime.*; -import com.openai.models.realtime.clientsecrets.ClientSecretCreateParams; +import com.openai.models.realtime.InputAudioBufferAppendEvent; +import com.openai.models.realtime.InputAudioBufferCommitEvent; +import com.openai.models.realtime.RealtimeAudioConfigInput; +import com.openai.models.realtime.RealtimeAudioFormats; +import com.openai.models.realtime.RealtimeAudioInputTurnDetection; import com.sap.ai.sdk.core.RealtimeParam; import com.sap.ai.sdk.core.RealtimeParamTurnDetection; -import com.sap.ai.sdk.core.RealtimeParamVoice; -import java.util.*; + import lombok.extern.slf4j.Slf4j; +import java.util.Arrays; +import java.util.Base64; +import java.util.Map; + @Slf4j -class SpeechToSpeechRealtimeClient extends WSOpenAiRealtimeClient implements AudioInputChannel { +class SpeechToSpeechRealtimeClient extends ToAudioRealtimeClient implements AudioInputChannel { private static final int MAX_DATA_CHUNK_SIZE_BYTES = 8192; - private static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; - - private static final Set HANDLED_RESPONSE_TYPES = - Set.of("response.output_audio.delta", "response.output_audio.done"); - - private static final List OUTPUT_MODALITIES = - List.of(RealtimeSessionCreateRequest.OutputModality.AUDIO); private static final String TASK = ""; - private final AudioOutputChannel outputConsumer; - private final RealtimeAudioConfigOutput.Voice.UnionMember1 voice; private final boolean eagerTurnDetection; public SpeechToSpeechRealtimeClient( @@ -32,33 +28,41 @@ public SpeechToSpeechRealtimeClient( Map httpHeaders, AudioOutputChannel outputConsumer, RealtimeParam... params) { - super(url, httpHeaders, HANDLED_RESPONSE_TYPES); - this.outputConsumer = outputConsumer; + super(url, httpHeaders, outputConsumer, params); - var voice = RealtimeAudioConfigOutput.Voice.UnionMember1.MARIN; var turnDetectionEager = false; for (RealtimeParam param : params) { - switch (param.getParamName()) { - case VOICE -> { - if (RealtimeParamVoice.DEFAULT_2.equals(param)) { - voice = RealtimeAudioConfigOutput.Voice.UnionMember1.MARIN; - } else if (RealtimeParamVoice.DEFAULT_1.equals(param)) { - voice = RealtimeAudioConfigOutput.Voice.UnionMember1.ECHO; - } + if (param.getParamName() == RealtimeParam.SpeechOutputParamName.TURN_DETECTION) { + if (RealtimeParamTurnDetection.EACH_CALL_IS_A_TURN.equals(param)) { + turnDetectionEager = true; + } else if (RealtimeParamTurnDetection.BY_MODEL_AUTO.equals(param)) { + turnDetectionEager = false; + } } - case TURN_DETECTION -> { - if (RealtimeParamTurnDetection.EACH_CALL_IS_A_TURN.equals(param)) { - turnDetectionEager = true; - } else if (RealtimeParamTurnDetection.BY_MODEL_AUTO.equals(param)) { - turnDetectionEager = false; - } - } - } } - this.voice = voice; this.eagerTurnDetection = turnDetectionEager; } + @Override + protected RealtimeAudioConfigInput inputConfig() { + RealtimeAudioInputTurnDetection turnDetection; + if (eagerTurnDetection) { + turnDetection = null; + } else { + turnDetection = + RealtimeAudioInputTurnDetection.ofSemanticVad( + RealtimeAudioInputTurnDetection.SemanticVad.builder().build()); + } + + return RealtimeAudioConfigInput.builder() + .turnDetection(turnDetection) + .format( + RealtimeAudioFormats.AudioPcm.builder() + .type(RealtimeAudioFormats.AudioPcm.Type.AUDIO_PCM) + .rate(RealtimeAudioFormats.AudioPcm.Rate._24000) + .build()).build(); + } + public void inputAudio(byte[] rawAudioChunk) { if (rawAudioChunk.length == 0) { return; @@ -87,56 +91,4 @@ protected String getSystemPrompt() { return TASK; } - @Override - protected void onResponse(String eventType, JsonNode event) { - if ("response.output_audio.delta".equals(eventType)) { - var base64Audio = event.get("delta").asText(); - byte[] audio = Base64.getDecoder().decode(base64Audio); - this.outputConsumer.outputAudio(audio, Boolean.FALSE); - } else if ("response.output_audio.done".equals(eventType)) { - this.outputConsumer.outputAudio(EMPTY_BYTE_ARRAY, Boolean.TRUE); - } - } - - @Override - protected SessionUpdateEvent sessionConfiguration() { - RealtimeAudioInputTurnDetection turnDetection; - if (eagerTurnDetection) { - turnDetection = null; - } else { - turnDetection = - RealtimeAudioInputTurnDetection.ofSemanticVad( - RealtimeAudioInputTurnDetection.SemanticVad.builder().build()); - } - - return SessionUpdateEvent.builder() - .session( - ClientSecretCreateParams.Session.ofRealtime( - RealtimeSessionCreateRequest.builder() - .outputModalities(OUTPUT_MODALITIES) - .audio( - RealtimeAudioConfig.builder() - .input( - RealtimeAudioConfigInput.builder() - .turnDetection(turnDetection) - .format( - RealtimeAudioFormats.AudioPcm.builder() - .type(RealtimeAudioFormats.AudioPcm.Type.AUDIO_PCM) - .rate(RealtimeAudioFormats.AudioPcm.Rate._24000) - .build()) - .build()) - .output( - RealtimeAudioConfigOutput.builder() - .format( - RealtimeAudioFormats.AudioPcm.builder() - .type(RealtimeAudioFormats.AudioPcm.Type.AUDIO_PCM) - .rate(RealtimeAudioFormats.AudioPcm.Rate._24000) - .build()) - .voice(voice) - .build()) - .build()) - .build()) - .asRealtime()) - .build(); - } } diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToTextRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToTextRealtimeClient.java deleted file mode 100644 index e8ed82610..000000000 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToTextRealtimeClient.java +++ /dev/null @@ -1,94 +0,0 @@ -package com.sap.ai.sdk.foundationmodels.openai; - -import com.fasterxml.jackson.databind.JsonNode; -import com.openai.models.realtime.*; -import com.openai.models.realtime.clientsecrets.ClientSecretCreateParams; -import java.util.*; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -class SpeechToTextRealtimeClient extends WSOpenAiRealtimeClient implements AudioInputChannel { - - private static final int MAX_DATA_CHUNK_SIZE_BYTES = 8192; - private static final Set HANDLED_RESPONSE_TYPES = - Set.of("response.output_audio_transcript.done"); - - private static final String TASK = - "you are a transcript writer and your role is to convert input speech to text as precise as possible (word to word exactly, if possible)"; - - private final TextOutputChannel outputConsumer; - - public SpeechToTextRealtimeClient( - String url, Map httpHeaders, TextOutputChannel outputConsumer) { - super(url, httpHeaders, HANDLED_RESPONSE_TYPES); - this.outputConsumer = outputConsumer; - } - - public void inputAudio(byte[] rawAudioChunk) { - if (rawAudioChunk.length == 0) { - return; - } - var cursorLeft = 0; - while (cursorLeft < rawAudioChunk.length) { - var cursorRight = Math.min(cursorLeft + MAX_DATA_CHUNK_SIZE_BYTES, rawAudioChunk.length); - var part = Arrays.copyOfRange(rawAudioChunk, cursorLeft, cursorRight); - var audioInputMessage = - InputAudioBufferAppendEvent.builder() - .audio(Base64.getEncoder().encodeToString(part)) - .build(); - super.sendMessage(audioInputMessage); - cursorLeft += MAX_DATA_CHUNK_SIZE_BYTES; - } - - var commitAudioMessage = InputAudioBufferCommitEvent.builder().build(); - super.sendMessage(commitAudioMessage); - askForResponse(); - } - - @Override - protected String getSystemPrompt() { - return TASK; - } - - @Override - protected void onResponse(String eventType, JsonNode event) { - if ("response.output_audio_transcript.done".equals(eventType)) { - this.outputConsumer.outputText( - event.has("transcript") ? event.get("transcript").asText() : "", Boolean.TRUE); - } else { - log.trace("skipping message type: {}", eventType); - } - } - - @Override - protected SessionUpdateEvent sessionConfiguration() { - SessionUpdateEvent sessionUpdateEvent = - SessionUpdateEvent.builder() - .session( - ClientSecretCreateParams.Session.ofTranscription( - RealtimeTranscriptionSessionCreateRequest.builder() - .audio( - RealtimeTranscriptionSessionAudio.builder() - .input( - RealtimeTranscriptionSessionAudioInput.builder() - .turnDetection(Optional.empty()) - .format( - RealtimeAudioFormats.AudioPcm.builder() - .type( - RealtimeAudioFormats.AudioPcm.Type - .AUDIO_PCM) - .rate(RealtimeAudioFormats.AudioPcm.Rate._24000) - .build()) - .transcription( - AudioTranscription.builder() - .model( - AudioTranscription.Model.GPT_4O_TRANSCRIBE) - .build()) - .build()) - .build()) - .build()) - .asTranscription()) - .build(); - return sessionUpdateEvent; - } -} diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java index 5734e284d..d2fc08a6a 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java @@ -1,31 +1,18 @@ package com.sap.ai.sdk.foundationmodels.openai; -import com.fasterxml.jackson.databind.JsonNode; import com.openai.models.realtime.*; -import com.openai.models.realtime.clientsecrets.ClientSecretCreateParams; import com.sap.ai.sdk.core.RealtimeParam; import com.sap.ai.sdk.core.RealtimeParamTurnDetection; -import com.sap.ai.sdk.core.RealtimeParamVoice; import java.util.*; import lombok.extern.slf4j.Slf4j; @Slf4j -class TextToSpeechRealtimeClient extends WSOpenAiRealtimeClient implements TextInputChannel { - - private static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; - - private static final Set HANDLED_RESPONSE_TYPES = - Set.of("response.output_audio.delta", "response.output_audio.done"); - - private static final List OUTPUT_MODALITIES = - List.of(RealtimeSessionCreateRequest.OutputModality.AUDIO); +class TextToSpeechRealtimeClient extends ToAudioRealtimeClient implements TextInputChannel { private static final String TASK = "you are a speaker and your role is to read (produce audio) of the user input speech. voice user text input, " + "do not answer questions, just read them"; - private final AudioOutputChannel outputConsumer; - private final RealtimeAudioConfigOutput.Voice.UnionMember1 voice; private final boolean eagerTurnDetection; public TextToSpeechRealtimeClient( @@ -33,36 +20,34 @@ public TextToSpeechRealtimeClient( Map httpHeaders, AudioOutputChannel outputConsumer, RealtimeParam... params) { - super(url, httpHeaders, HANDLED_RESPONSE_TYPES); - this.outputConsumer = outputConsumer; - var voice = RealtimeAudioConfigOutput.Voice.UnionMember1.MARIN; + super(url, httpHeaders, outputConsumer, params); var turnDetectionEager = true; for (RealtimeParam param : params) { - switch (param.getParamName()) { - case VOICE -> { - if (RealtimeParamVoice.DEFAULT_2.equals(param)) { - voice = RealtimeAudioConfigOutput.Voice.UnionMember1.MARIN; - } else if (RealtimeParamVoice.DEFAULT_1.equals(param)) { - voice = RealtimeAudioConfigOutput.Voice.UnionMember1.ECHO; - } else { - voice = - RealtimeAudioConfigOutput.Voice.UnionMember1.of( - param.getValueAsString().toLowerCase()); - } + if (param.getParamName() == RealtimeParam.SpeechOutputParamName.TURN_DETECTION) { + if (RealtimeParamTurnDetection.EACH_CALL_IS_A_TURN.equals(param)) { + turnDetectionEager = true; + } else if (RealtimeParamTurnDetection.BY_MODEL_AUTO.equals(param)) { + turnDetectionEager = false; + } } - case TURN_DETECTION -> { - if (RealtimeParamTurnDetection.EACH_CALL_IS_A_TURN.equals(param)) { - turnDetectionEager = true; - } else if (RealtimeParamTurnDetection.BY_MODEL_AUTO.equals(param)) { - turnDetectionEager = false; - } - } - } } - this.voice = voice; this.eagerTurnDetection = turnDetectionEager; } + @Override + protected RealtimeAudioConfigInput inputConfig() { + return RealtimeAudioConfigInput.builder() + .turnDetection(Optional.empty()) + .format( + RealtimeAudioFormats.AudioPcm.builder() + .type( + RealtimeAudioFormats.AudioPcm.Type + .AUDIO_PCM) + .rate(RealtimeAudioFormats.AudioPcm.Rate._24000) + .build()) + .build(); + } + public void sendText(String text) { var message = ConversationItemCreateEvent.builder() @@ -88,55 +73,4 @@ protected String getSystemPrompt() { return TASK; } - @Override - protected void onResponse(String eventType, JsonNode event) { - if ("response.output_audio.delta".equals(eventType)) { - var base64Audio = event.get("delta").asText(); - byte[] audio = Base64.getDecoder().decode(base64Audio); - this.outputConsumer.outputAudio(audio, Boolean.FALSE); - } else if ("response.output_audio.done".equals(eventType)) { - this.outputConsumer.outputAudio(EMPTY_BYTE_ARRAY, Boolean.TRUE); - } else { - log.warn("skipping message type: {}", eventType); - } - } - - @Override - protected SessionUpdateEvent sessionConfiguration() { - SessionUpdateEvent sessionUpdateEvent = - SessionUpdateEvent.builder() - .session( - ClientSecretCreateParams.Session.ofRealtime( - RealtimeSessionCreateRequest.builder() - .outputModalities(OUTPUT_MODALITIES) - .audio( - RealtimeAudioConfig.builder() - .input( - RealtimeAudioConfigInput.builder() - .turnDetection(Optional.empty()) - .format( - RealtimeAudioFormats.AudioPcm.builder() - .type( - RealtimeAudioFormats.AudioPcm.Type - .AUDIO_PCM) - .rate(RealtimeAudioFormats.AudioPcm.Rate._24000) - .build()) - .build()) - .output( - RealtimeAudioConfigOutput.builder() - .format( - RealtimeAudioFormats.AudioPcm.builder() - .type( - RealtimeAudioFormats.AudioPcm.Type - .AUDIO_PCM) - .rate(RealtimeAudioFormats.AudioPcm.Rate._24000) - .build()) - .voice(voice) - .build()) - .build()) - .build()) - .asRealtime()) - .build(); - return sessionUpdateEvent; - } } diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/ToAudioRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/ToAudioRealtimeClient.java index 7033cfec7..50b4db464 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/ToAudioRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/ToAudioRealtimeClient.java @@ -1,30 +1,92 @@ package com.sap.ai.sdk.foundationmodels.openai; import com.fasterxml.jackson.databind.JsonNode; +import com.openai.models.realtime.RealtimeAudioConfig; +import com.openai.models.realtime.RealtimeAudioConfigInput; +import com.openai.models.realtime.RealtimeAudioConfigOutput; +import com.openai.models.realtime.RealtimeAudioFormats; +import com.openai.models.realtime.RealtimeSessionCreateRequest; import com.openai.models.realtime.SessionUpdateEvent; +import com.openai.models.realtime.clientsecrets.ClientSecretCreateParams; +import com.sap.ai.sdk.core.RealtimeParam; +import com.sap.ai.sdk.core.RealtimeParamVoice; +import lombok.extern.slf4j.Slf4j; + +import java.util.Base64; +import java.util.List; import java.util.Map; import java.util.Set; -class ToAudioRealtimeClient extends WSOpenAiRealtimeClient { +@Slf4j +abstract class ToAudioRealtimeClient extends WSOpenAiRealtimeClient { private static final Set HANDLED_RESPONSE_TYPES = Set.of("response.output_audio.delta", "response.output_audio.done"); - private static final String DEFAULT_SYSTEM_PROMPT = ""; + private static final List OUTPUT_MODALITIES = + List.of(RealtimeSessionCreateRequest.OutputModality.AUDIO); + private static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; + + private final AudioOutputChannel outputConsumer; + private final RealtimeAudioConfigOutput.Voice.UnionMember1 voice; - public ToAudioRealtimeClient(String url, Map httpHeaders) { + public ToAudioRealtimeClient(String url, + Map httpHeaders, + AudioOutputChannel outputConsumer, + RealtimeParam... params) { super(url, httpHeaders, HANDLED_RESPONSE_TYPES); + var voice = RealtimeAudioConfigOutput.Voice.UnionMember1.MARIN; + for (RealtimeParam param : params) { + switch (param.getParamName()) { + case VOICE -> { + if (RealtimeParamVoice.DEFAULT_2.equals(param)) { + voice = RealtimeAudioConfigOutput.Voice.UnionMember1.MARIN; + } else if (RealtimeParamVoice.DEFAULT_1.equals(param)) { + voice = RealtimeAudioConfigOutput.Voice.UnionMember1.ECHO; + } + } + } + } + this.outputConsumer = outputConsumer; + this.voice = voice; } - @Override - protected String getSystemPrompt() { - return ""; - } + protected abstract RealtimeAudioConfigInput inputConfig(); @Override - protected void onResponse(String eventType, JsonNode event) {} + protected void onResponse(String eventType, JsonNode event) { + if ("response.output_audio.delta".equals(eventType)) { + var base64Audio = event.get("delta").asText(); + byte[] audio = Base64.getDecoder().decode(base64Audio); + this.outputConsumer.outputAudio(audio, Boolean.FALSE); + } else if ("response.output_audio.done".equals(eventType)) { + this.outputConsumer.outputAudio(EMPTY_BYTE_ARRAY, Boolean.TRUE); + } else { + log.warn("skipping message type: {}", eventType); + } + } @Override protected SessionUpdateEvent sessionConfiguration() { - return null; + return SessionUpdateEvent.builder() + .session( + ClientSecretCreateParams.Session.ofRealtime( + RealtimeSessionCreateRequest.builder() + .outputModalities(OUTPUT_MODALITIES) + .audio( + RealtimeAudioConfig.builder() + .input(inputConfig()) + .output( + RealtimeAudioConfigOutput.builder() + .format( + RealtimeAudioFormats.AudioPcm.builder() + .type(RealtimeAudioFormats.AudioPcm.Type.AUDIO_PCM) + .rate(RealtimeAudioFormats.AudioPcm.Rate._24000) + .build()) + .voice(voice) + .build()) + .build()) + .build()) + .asRealtime()) + .build(); } } From 0f2cd624843813e9d8f40209615fbeefc1edff39 Mon Sep 17 00:00:00 2001 From: SAP Cloud SDK Bot Date: Fri, 17 Jul 2026 14:24:26 +0000 Subject: [PATCH 19/26] Formatting --- .../openai/SpeechToSpeechRealtimeClient.java | 34 +++++++++---------- .../openai/TextToSpeechRealtimeClient.java | 29 +++++++--------- .../openai/ToAudioRealtimeClient.java | 14 ++++---- 3 files changed, 36 insertions(+), 41 deletions(-) diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java index 31b8611ad..9de369923 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java @@ -7,12 +7,10 @@ import com.openai.models.realtime.RealtimeAudioInputTurnDetection; import com.sap.ai.sdk.core.RealtimeParam; import com.sap.ai.sdk.core.RealtimeParamTurnDetection; - -import lombok.extern.slf4j.Slf4j; - import java.util.Arrays; import java.util.Base64; import java.util.Map; +import lombok.extern.slf4j.Slf4j; @Slf4j class SpeechToSpeechRealtimeClient extends ToAudioRealtimeClient implements AudioInputChannel { @@ -32,13 +30,13 @@ public SpeechToSpeechRealtimeClient( var turnDetectionEager = false; for (RealtimeParam param : params) { - if (param.getParamName() == RealtimeParam.SpeechOutputParamName.TURN_DETECTION) { - if (RealtimeParamTurnDetection.EACH_CALL_IS_A_TURN.equals(param)) { - turnDetectionEager = true; - } else if (RealtimeParamTurnDetection.BY_MODEL_AUTO.equals(param)) { - turnDetectionEager = false; - } + if (param.getParamName() == RealtimeParam.SpeechOutputParamName.TURN_DETECTION) { + if (RealtimeParamTurnDetection.EACH_CALL_IS_A_TURN.equals(param)) { + turnDetectionEager = true; + } else if (RealtimeParamTurnDetection.BY_MODEL_AUTO.equals(param)) { + turnDetectionEager = false; } + } } this.eagerTurnDetection = turnDetectionEager; } @@ -50,17 +48,18 @@ protected RealtimeAudioConfigInput inputConfig() { turnDetection = null; } else { turnDetection = - RealtimeAudioInputTurnDetection.ofSemanticVad( - RealtimeAudioInputTurnDetection.SemanticVad.builder().build()); + RealtimeAudioInputTurnDetection.ofSemanticVad( + RealtimeAudioInputTurnDetection.SemanticVad.builder().build()); } return RealtimeAudioConfigInput.builder() - .turnDetection(turnDetection) - .format( - RealtimeAudioFormats.AudioPcm.builder() - .type(RealtimeAudioFormats.AudioPcm.Type.AUDIO_PCM) - .rate(RealtimeAudioFormats.AudioPcm.Rate._24000) - .build()).build(); + .turnDetection(turnDetection) + .format( + RealtimeAudioFormats.AudioPcm.builder() + .type(RealtimeAudioFormats.AudioPcm.Type.AUDIO_PCM) + .rate(RealtimeAudioFormats.AudioPcm.Rate._24000) + .build()) + .build(); } public void inputAudio(byte[] rawAudioChunk) { @@ -90,5 +89,4 @@ public void inputAudio(byte[] rawAudioChunk) { protected String getSystemPrompt() { return TASK; } - } diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java index d2fc08a6a..6bde180ff 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java @@ -23,13 +23,13 @@ public TextToSpeechRealtimeClient( super(url, httpHeaders, outputConsumer, params); var turnDetectionEager = true; for (RealtimeParam param : params) { - if (param.getParamName() == RealtimeParam.SpeechOutputParamName.TURN_DETECTION) { - if (RealtimeParamTurnDetection.EACH_CALL_IS_A_TURN.equals(param)) { - turnDetectionEager = true; - } else if (RealtimeParamTurnDetection.BY_MODEL_AUTO.equals(param)) { - turnDetectionEager = false; - } + if (param.getParamName() == RealtimeParam.SpeechOutputParamName.TURN_DETECTION) { + if (RealtimeParamTurnDetection.EACH_CALL_IS_A_TURN.equals(param)) { + turnDetectionEager = true; + } else if (RealtimeParamTurnDetection.BY_MODEL_AUTO.equals(param)) { + turnDetectionEager = false; } + } } this.eagerTurnDetection = turnDetectionEager; } @@ -37,15 +37,13 @@ public TextToSpeechRealtimeClient( @Override protected RealtimeAudioConfigInput inputConfig() { return RealtimeAudioConfigInput.builder() - .turnDetection(Optional.empty()) - .format( - RealtimeAudioFormats.AudioPcm.builder() - .type( - RealtimeAudioFormats.AudioPcm.Type - .AUDIO_PCM) - .rate(RealtimeAudioFormats.AudioPcm.Rate._24000) - .build()) - .build(); + .turnDetection(Optional.empty()) + .format( + RealtimeAudioFormats.AudioPcm.builder() + .type(RealtimeAudioFormats.AudioPcm.Type.AUDIO_PCM) + .rate(RealtimeAudioFormats.AudioPcm.Rate._24000) + .build()) + .build(); } public void sendText(String text) { @@ -72,5 +70,4 @@ public void sendText(String text) { protected String getSystemPrompt() { return TASK; } - } diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/ToAudioRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/ToAudioRealtimeClient.java index 50b4db464..7279ba465 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/ToAudioRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/ToAudioRealtimeClient.java @@ -10,12 +10,11 @@ import com.openai.models.realtime.clientsecrets.ClientSecretCreateParams; import com.sap.ai.sdk.core.RealtimeParam; import com.sap.ai.sdk.core.RealtimeParamVoice; -import lombok.extern.slf4j.Slf4j; - import java.util.Base64; import java.util.List; import java.util.Map; import java.util.Set; +import lombok.extern.slf4j.Slf4j; @Slf4j abstract class ToAudioRealtimeClient extends WSOpenAiRealtimeClient { @@ -23,16 +22,17 @@ abstract class ToAudioRealtimeClient extends WSOpenAiRealtimeClient { private static final Set HANDLED_RESPONSE_TYPES = Set.of("response.output_audio.delta", "response.output_audio.done"); private static final List OUTPUT_MODALITIES = - List.of(RealtimeSessionCreateRequest.OutputModality.AUDIO); + List.of(RealtimeSessionCreateRequest.OutputModality.AUDIO); private static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; private final AudioOutputChannel outputConsumer; private final RealtimeAudioConfigOutput.Voice.UnionMember1 voice; - public ToAudioRealtimeClient(String url, - Map httpHeaders, - AudioOutputChannel outputConsumer, - RealtimeParam... params) { + public ToAudioRealtimeClient( + String url, + Map httpHeaders, + AudioOutputChannel outputConsumer, + RealtimeParam... params) { super(url, httpHeaders, HANDLED_RESPONSE_TYPES); var voice = RealtimeAudioConfigOutput.Voice.UnionMember1.MARIN; for (RealtimeParam param : params) { From 6082105816f437b0c339b5e18f23eb4a3395f0bf Mon Sep 17 00:00:00 2001 From: Vladimir Aseev Date: Fri, 17 Jul 2026 16:34:53 +0200 Subject: [PATCH 20/26] fix misleading javadocs --- .../sap/ai/sdk/foundationmodels/openai/AudioOutputChannel.java | 2 -- .../sap/ai/sdk/foundationmodels/openai/TextOutputChannel.java | 2 -- 2 files changed, 4 deletions(-) diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/AudioOutputChannel.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/AudioOutputChannel.java index b190fbda5..598fddabf 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/AudioOutputChannel.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/AudioOutputChannel.java @@ -2,8 +2,6 @@ /** * Functional interface representing audio output channel (audio data consumer) - * - *

    Should be closed by application (try-with-resources) when not needed anymore */ public interface AudioOutputChannel { diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextOutputChannel.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextOutputChannel.java index 5896b85ab..0d67d7af0 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextOutputChannel.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextOutputChannel.java @@ -2,8 +2,6 @@ /** * Functional interface representing text output channel (text data consumer) - * - *

    Should be closed by application (try-with-resources) when not needed anymore */ public interface TextOutputChannel { From 537203d538675ce5ce687c39de3d8ad224ab7143 Mon Sep 17 00:00:00 2001 From: SAP Cloud SDK Bot Date: Fri, 17 Jul 2026 14:36:07 +0000 Subject: [PATCH 21/26] Formatting --- .../ai/sdk/foundationmodels/openai/AudioOutputChannel.java | 4 +--- .../sap/ai/sdk/foundationmodels/openai/TextOutputChannel.java | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/AudioOutputChannel.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/AudioOutputChannel.java index 598fddabf..81217d100 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/AudioOutputChannel.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/AudioOutputChannel.java @@ -1,8 +1,6 @@ package com.sap.ai.sdk.foundationmodels.openai; -/** - * Functional interface representing audio output channel (audio data consumer) - */ +/** Functional interface representing audio output channel (audio data consumer) */ public interface AudioOutputChannel { /** diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextOutputChannel.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextOutputChannel.java index 0d67d7af0..add9c2460 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextOutputChannel.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextOutputChannel.java @@ -1,8 +1,6 @@ package com.sap.ai.sdk.foundationmodels.openai; -/** - * Functional interface representing text output channel (text data consumer) - */ +/** Functional interface representing text output channel (text data consumer) */ public interface TextOutputChannel { /** From 7a5220eb8a7f0e161c1dec21259e6aaaaa233f2c Mon Sep 17 00:00:00 2001 From: Vladimir Aseev Date: Fri, 17 Jul 2026 17:16:37 +0200 Subject: [PATCH 22/26] replace debug output in test with actual assertions --- .../ai/sdk/app/controllers/RealtimeApiTest.java | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/sample-code/spring-app/src/test/java/com/sap/ai/sdk/app/controllers/RealtimeApiTest.java b/sample-code/spring-app/src/test/java/com/sap/ai/sdk/app/controllers/RealtimeApiTest.java index 177d8d208..10f6027e4 100644 --- a/sample-code/spring-app/src/test/java/com/sap/ai/sdk/app/controllers/RealtimeApiTest.java +++ b/sample-code/spring-app/src/test/java/com/sap/ai/sdk/app/controllers/RealtimeApiTest.java @@ -9,7 +9,6 @@ import com.sap.ai.sdk.foundationmodels.openai.TextInputChannel; import java.io.ByteArrayOutputStream; import java.io.FileInputStream; -import java.io.FileOutputStream; import java.io.IOException; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -98,11 +97,15 @@ void testSpeechToSpeech() { return; } - try (var fis = new FileOutputStream("/tmp/response.pcm")) { - fis.write(outputBuffer.toByteArray()); - } catch (IOException e) { - throw new RuntimeException(e); - } + assertThat(monitor.getCount()).isEqualTo(0); + assertThat(outputBuffer.size()).isGreaterThan(0); + + var metrics = pcm16AudioMetrics(outputBuffer.toByteArray()); + + // root mean squire (measures the deviation from an average value) + assertThat(metrics.rms).isGreaterThan(500d); + // asserts that variety of deviation is sufficient (not a trivial repeating pattern) + assertThat(metrics.entropy).isGreaterThan(4); } private record AudioMetrics(double rms, double entropy) {} From e7d844d631d749d6a39639d28c6455145b2c6e13 Mon Sep 17 00:00:00 2001 From: Vladimir Aseev Date: Mon, 20 Jul 2026 11:42:45 +0200 Subject: [PATCH 23/26] fix annotations from invalid package --- .../com/sap/ai/sdk/core/RealtimeParamTurnDetection.java | 6 +++--- .../main/java/com/sap/ai/sdk/core/RealtimeParamVoice.java | 5 ++--- .../ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java | 5 +++-- .../ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java | 5 +++-- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamTurnDetection.java b/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamTurnDetection.java index 60f736fd9..4ace8ce1e 100644 --- a/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamTurnDetection.java +++ b/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamTurnDetection.java @@ -1,7 +1,7 @@ package com.sap.ai.sdk.core; +import javax.annotation.Nonnull; import java.util.Objects; -import org.jspecify.annotations.NonNull; public final class RealtimeParamTurnDetection implements RealtimeParam { @@ -17,12 +17,12 @@ private RealtimeParamTurnDetection(String turnDetectionKind) { } @Override - public @NonNull SpeechOutputParamName getParamName() { + public @Nonnull SpeechOutputParamName getParamName() { return SpeechOutputParamName.TURN_DETECTION; } @Override - public @NonNull String getValueAsString() { + public @Nonnull String getValueAsString() { return turnDetectionKind; } diff --git a/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamVoice.java b/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamVoice.java index 3746a2f3f..2a4178a2e 100644 --- a/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamVoice.java +++ b/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamVoice.java @@ -2,7 +2,6 @@ import java.util.Objects; import javax.annotation.Nonnull; -import org.jspecify.annotations.NonNull; public final class RealtimeParamVoice implements RealtimeParam { @@ -20,12 +19,12 @@ public static RealtimeParamVoice unsafeWithExplicitVoice(String voiceName) { } @Override - public @NonNull SpeechOutputParamName getParamName() { + public @Nonnull SpeechOutputParamName getParamName() { return SpeechOutputParamName.VOICE; } @Override - public @NonNull String getValueAsString() { + public @Nonnull String getValueAsString() { return voice; } diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java index 696d10fab..7ed9ad336 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java @@ -7,7 +7,6 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import lombok.extern.slf4j.Slf4j; -import org.jspecify.annotations.NonNull; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.springframework.web.socket.BinaryMessage; @@ -15,6 +14,8 @@ import org.springframework.web.socket.WebSocketSession; import org.springframework.web.socket.handler.BinaryWebSocketHandler; +import javax.annotation.Nonnull; + @Component @Slf4j public class SpeechToSpeechWebsocketHandler extends BinaryWebSocketHandler { @@ -48,7 +49,7 @@ protected void handleBinaryMessage(WebSocketSession session, BinaryMessage messa } @Override - public void afterConnectionClosed(WebSocketSession session, @NonNull CloseStatus status) + public void afterConnectionClosed(WebSocketSession session, @Nonnull CloseStatus status) throws Exception { channels.computeIfPresent( session.getId(), diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java index 34469513f..cede97963 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java @@ -7,7 +7,6 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import lombok.extern.slf4j.Slf4j; -import org.jspecify.annotations.NonNull; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.springframework.web.socket.BinaryMessage; @@ -15,6 +14,8 @@ import org.springframework.web.socket.WebSocketSession; import org.springframework.web.socket.handler.BinaryWebSocketHandler; +import javax.annotation.Nonnull; + @Component @Slf4j public class TextToSpeechWebsocketHandler extends BinaryWebSocketHandler { @@ -48,7 +49,7 @@ protected void handleBinaryMessage(WebSocketSession session, BinaryMessage messa } @Override - public void afterConnectionClosed(WebSocketSession session, @NonNull CloseStatus status) + public void afterConnectionClosed(WebSocketSession session, @Nonnull CloseStatus status) throws Exception { channels.computeIfPresent( session.getId(), From 7c6dced7f54fa2f3bb2d7e73cd73158f9e01e063 Mon Sep 17 00:00:00 2001 From: SAP Cloud SDK Bot Date: Mon, 20 Jul 2026 09:43:55 +0000 Subject: [PATCH 24/26] Formatting --- .../java/com/sap/ai/sdk/core/RealtimeParamTurnDetection.java | 2 +- .../ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java | 3 +-- .../sap/ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java | 3 +-- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamTurnDetection.java b/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamTurnDetection.java index 4ace8ce1e..a7b247546 100644 --- a/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamTurnDetection.java +++ b/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamTurnDetection.java @@ -1,7 +1,7 @@ package com.sap.ai.sdk.core; -import javax.annotation.Nonnull; import java.util.Objects; +import javax.annotation.Nonnull; public final class RealtimeParamTurnDetection implements RealtimeParam { diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java index 7ed9ad336..64911e08b 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java @@ -6,6 +6,7 @@ import java.nio.ByteBuffer; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import javax.annotation.Nonnull; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -14,8 +15,6 @@ import org.springframework.web.socket.WebSocketSession; import org.springframework.web.socket.handler.BinaryWebSocketHandler; -import javax.annotation.Nonnull; - @Component @Slf4j public class SpeechToSpeechWebsocketHandler extends BinaryWebSocketHandler { diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java index cede97963..88ce6109b 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java @@ -6,6 +6,7 @@ import java.nio.ByteBuffer; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import javax.annotation.Nonnull; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -14,8 +15,6 @@ import org.springframework.web.socket.WebSocketSession; import org.springframework.web.socket.handler.BinaryWebSocketHandler; -import javax.annotation.Nonnull; - @Component @Slf4j public class TextToSpeechWebsocketHandler extends BinaryWebSocketHandler { From 671f96078204e1ec02963e6ab6fa8d0140d671cd Mon Sep 17 00:00:00 2001 From: Vladimir Aseev Date: Mon, 20 Jul 2026 11:59:12 +0200 Subject: [PATCH 25/26] add missing javadoc comments --- .../com/sap/ai/sdk/core/RealtimeParam.java | 11 +++++++++++ .../sdk/core/RealtimeParamTurnDetection.java | 12 ++++++++++++ .../sap/ai/sdk/core/RealtimeParamVoice.java | 18 +++++++++++++++++- 3 files changed, 40 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/com/sap/ai/sdk/core/RealtimeParam.java b/core/src/main/java/com/sap/ai/sdk/core/RealtimeParam.java index bb4cd79c4..5c03cba92 100644 --- a/core/src/main/java/com/sap/ai/sdk/core/RealtimeParam.java +++ b/core/src/main/java/com/sap/ai/sdk/core/RealtimeParam.java @@ -2,15 +2,26 @@ import javax.annotation.Nonnull; +/** + * Represents possible configuration params of realtime client + */ public interface RealtimeParam { enum SpeechOutputParamName { VOICE, TURN_DETECTION, } + /** + * Returns param name + * @return name + */ @Nonnull SpeechOutputParamName getParamName(); + /** + * Returns string value representation of the param + * @return string value + */ @Nonnull String getValueAsString(); } diff --git a/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamTurnDetection.java b/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamTurnDetection.java index 4ace8ce1e..cadbbf32b 100644 --- a/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamTurnDetection.java +++ b/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamTurnDetection.java @@ -3,10 +3,22 @@ import javax.annotation.Nonnull; import java.util.Objects; +/** + * Allows to configure turn detection (how model responds). + */ public final class RealtimeParamTurnDetection implements RealtimeParam { + /** + * Model tries to recognize if/when it should respond automatically + */ public static final RealtimeParamTurnDetection BY_MODEL_AUTO = new RealtimeParamTurnDetection("BY_MODEL_AUTO"); + + /** + * Each call to the provided realtime client is considered a turn (eager explicit turn detection). + * Less convenient than the automatic option but may give lower latency in some cases (model does not need + * to perform additional turn detection analysis). + */ public static final RealtimeParamTurnDetection EACH_CALL_IS_A_TURN = new RealtimeParamTurnDetection("EACH_CALL_IS_A_TURN"); diff --git a/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamVoice.java b/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamVoice.java index 2a4178a2e..1c48040a1 100644 --- a/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamVoice.java +++ b/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamVoice.java @@ -3,10 +3,20 @@ import java.util.Objects; import javax.annotation.Nonnull; +/** + * Allows to configure model output voice + */ public final class RealtimeParamVoice implements RealtimeParam { + /** + * Standard voice 1 + */ public static final RealtimeParamVoice DEFAULT_1 = new RealtimeParamVoice("DEFAULT_1"); - public static final RealtimeParamVoice DEFAULT_2 = new RealtimeParamVoice("DEFAULT_1"); + + /** + * Standard voice 2 + */ + public static final RealtimeParamVoice DEFAULT_2 = new RealtimeParamVoice("DEFAULT_2"); private final String voice; @@ -14,6 +24,12 @@ private RealtimeParamVoice(@Nonnull String voice) { this.voice = voice; } + /** + * Allows to configure raw voice name as named by model provider. Unsafe because SDK cannot verify + * in advance if the provided voice name is correct and supported by the chosen model and use case + * @param voiceName as named by model provider + * @return typed voice client configuration param + */ public static RealtimeParamVoice unsafeWithExplicitVoice(String voiceName) { return new RealtimeParamVoice(voiceName); } From b4ba4715be8dcf68f870be52e92732d1cd326b9b Mon Sep 17 00:00:00 2001 From: SAP Cloud SDK Bot Date: Mon, 20 Jul 2026 10:00:27 +0000 Subject: [PATCH 26/26] Formatting --- .../java/com/sap/ai/sdk/core/RealtimeParam.java | 6 +++--- .../sap/ai/sdk/core/RealtimeParamTurnDetection.java | 12 ++++-------- .../com/sap/ai/sdk/core/RealtimeParamVoice.java | 13 ++++--------- 3 files changed, 11 insertions(+), 20 deletions(-) diff --git a/core/src/main/java/com/sap/ai/sdk/core/RealtimeParam.java b/core/src/main/java/com/sap/ai/sdk/core/RealtimeParam.java index 5c03cba92..9c1e4d57e 100644 --- a/core/src/main/java/com/sap/ai/sdk/core/RealtimeParam.java +++ b/core/src/main/java/com/sap/ai/sdk/core/RealtimeParam.java @@ -2,9 +2,7 @@ import javax.annotation.Nonnull; -/** - * Represents possible configuration params of realtime client - */ +/** Represents possible configuration params of realtime client */ public interface RealtimeParam { enum SpeechOutputParamName { VOICE, @@ -13,6 +11,7 @@ enum SpeechOutputParamName { /** * Returns param name + * * @return name */ @Nonnull @@ -20,6 +19,7 @@ enum SpeechOutputParamName { /** * Returns string value representation of the param + * * @return string value */ @Nonnull diff --git a/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamTurnDetection.java b/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamTurnDetection.java index 3327f92a2..78fbdfa3a 100644 --- a/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamTurnDetection.java +++ b/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamTurnDetection.java @@ -3,21 +3,17 @@ import java.util.Objects; import javax.annotation.Nonnull; -/** - * Allows to configure turn detection (how model responds). - */ +/** Allows to configure turn detection (how model responds). */ public final class RealtimeParamTurnDetection implements RealtimeParam { - /** - * Model tries to recognize if/when it should respond automatically - */ + /** Model tries to recognize if/when it should respond automatically */ public static final RealtimeParamTurnDetection BY_MODEL_AUTO = new RealtimeParamTurnDetection("BY_MODEL_AUTO"); /** * Each call to the provided realtime client is considered a turn (eager explicit turn detection). - * Less convenient than the automatic option but may give lower latency in some cases (model does not need - * to perform additional turn detection analysis). + * Less convenient than the automatic option but may give lower latency in some cases (model does + * not need to perform additional turn detection analysis). */ public static final RealtimeParamTurnDetection EACH_CALL_IS_A_TURN = new RealtimeParamTurnDetection("EACH_CALL_IS_A_TURN"); diff --git a/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamVoice.java b/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamVoice.java index 1c48040a1..1743430b0 100644 --- a/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamVoice.java +++ b/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamVoice.java @@ -3,19 +3,13 @@ import java.util.Objects; import javax.annotation.Nonnull; -/** - * Allows to configure model output voice - */ +/** Allows to configure model output voice */ public final class RealtimeParamVoice implements RealtimeParam { - /** - * Standard voice 1 - */ + /** Standard voice 1 */ public static final RealtimeParamVoice DEFAULT_1 = new RealtimeParamVoice("DEFAULT_1"); - /** - * Standard voice 2 - */ + /** Standard voice 2 */ public static final RealtimeParamVoice DEFAULT_2 = new RealtimeParamVoice("DEFAULT_2"); private final String voice; @@ -27,6 +21,7 @@ private RealtimeParamVoice(@Nonnull String voice) { /** * Allows to configure raw voice name as named by model provider. Unsafe because SDK cannot verify * in advance if the provided voice name is correct and supported by the chosen model and use case + * * @param voiceName as named by model provider * @return typed voice client configuration param */