com.openai.unity 8.8.0 (#391)

- Improved RealtimeSession websocket support for proxies
  - Proxy no longer handles the websocket connection directly, but instead initiates the connection to the OpenAI api directly using the ephemeral api key

Author: StephenHodgson

OpenAI/Packages/com.openai.unity/Runtime/Authentication/OpenAISettingsInfo.cs

@@ -8,7 +8,6 @@ namespace OpenAI
 {
     public sealed class OpenAISettingsInfo : ISettingsInfo
     {
-        internal const string WS = "ws://";
         internal const string WSS = "wss://";
         internal const string Http = "http://";
         internal const string Https = "https://";
@@ -72,9 +71,7 @@ public OpenAISettingsInfo(string domain, string apiVersion = DefaultOpenAIApiVer
             DeploymentId = string.Empty;
             BaseRequest = $"/{ApiVersion}/";
             BaseRequestUrlFormat = $"{ResourceName}{BaseRequest}{{0}}";
-            BaseWebSocketUrlFormat = ResourceName.Contains(Https)
-                ? $"{WSS}{domain}{BaseRequest}{{0}}"
-                : $"{WS}{domain}{BaseRequest}{{0}}";
+            BaseWebSocketUrlFormat = $"{WSS}{OpenAIDomain}{BaseRequest}{{0}}";
             UseOAuthAuthentication = true;
         }
 

OpenAI/Packages/com.openai.unity/Runtime/Common/OpenAIBaseEndpoint.cs

@@ -21,17 +21,18 @@ protected OpenAIBaseEndpoint(OpenAIClient client) : base(client) { }
         /// </remarks>
         protected virtual bool? IsAzureDeployment => null;
 
-        /// <summary>
-        /// Indicates if the endpoint is for a WebSocket.
-        /// </summary>
-        protected virtual bool? IsWebSocketEndpoint => null;
-
         /// <summary>
         /// Gets the full formatted url for the API endpoint.
         /// </summary>
         /// <param name="endpoint">The endpoint url.</param>
         /// <param name="queryParameters">Optional, parameters to add to the endpoint.</param>
         protected override string GetUrl(string endpoint = "", Dictionary<string, string> queryParameters = null)
+            => GetEndpoint(client.Settings.Info.BaseRequestUrlFormat, endpoint, queryParameters);
+
+        protected string GetWebsocketUri(string endpoint = "", Dictionary<string, string> queryParameters = null)
+            => GetEndpoint(client.Settings.Info.BaseWebSocketUrlFormat, endpoint, queryParameters);
+
+        private string GetEndpoint(string baseUrlFormat, string endpoint = "", Dictionary<string, string> queryParameters = null)
         {
             string route;
 
@@ -49,14 +50,11 @@ protected override string GetUrl(string endpoint = "", Dictionary<string, string
                 route = $"{Root}{endpoint}";
             }
 
-            var baseUrlFormat = IsWebSocketEndpoint == true
-                ? client.Settings.Info.BaseWebSocketUrlFormat
-                : client.Settings.Info.BaseRequestUrlFormat;
             var url = string.Format(baseUrlFormat, route);
 
             foreach (var defaultQueryParameter in client.Settings.Info.DefaultQueryParameters)
             {
-                queryParameters ??= new Dictionary<string, string>();
+                queryParameters ??= new();
                 queryParameters.Add(defaultQueryParameter.Key, defaultQueryParameter.Value);
             }
 

OpenAI/Packages/com.openai.unity/Runtime/Extensions/VoiceActivityDetectionSettingsConverter.cs

@@ -17,6 +17,11 @@ internal class VoiceActivityDetectionSettingsConverter : JsonConverter
         [Preserve]
         public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
         {
+            if (reader.TokenType == JsonToken.Null)
+            {
+                return null;
+            }
+
             var jObject = JObject.Load(reader);
             var type = jObject["type"]?.Value<string>() ?? "disabled";
 

OpenAI/Packages/com.openai.unity/Runtime/OpenAIClient.cs

@@ -20,7 +20,6 @@
 using System.Collections.Generic;
 using System.Security.Authentication;
 using Utilities.WebRequestRest;
-using Utilities.WebSockets;
 
 namespace OpenAI
 {
@@ -237,24 +236,5 @@ protected override void ValidateAuthentication()
         public ResponsesEndpoint ResponsesEndpoint { get; }
 
         #endregion Endpoints
-
-        internal WebSocket CreateWebSocket(string url)
-        {
-            return new WebSocket(url, new Dictionary<string, string>
-            {
-#if !PLATFORM_WEBGL
-                { "User-Agent", "OpenAI-DotNet" },
-                { "OpenAI-Beta", "realtime=v1" },
-                { "Authorization", $"Bearer {Authentication.Info.ApiKey}" }
-#endif
-            }, new List<string>
-            {
-#if PLATFORM_WEBGL // Web browsers do not support headers. https://github.com/openai/openai-realtime-api-beta/blob/339e9553a757ef1cf8c767272fc750c1e62effbb/lib/api.js#L76-L80
-                "realtime",
-                $"openai-insecure-api-key.{Authentication.Info.ApiKey}",
-                "openai-beta.realtime-v1"
-#endif
-            });
-        }
     }
 }

OpenAI/Packages/com.openai.unity/Runtime/Realtime/ClientSecret.cs

@@ -0,0 +1,51 @@
+// Licensed under the MIT License. See LICENSE in the project root for license information.
+
+using Newtonsoft.Json;
+using System;
+using UnityEngine.Scripting;
+
+namespace OpenAI.Realtime
+{
+    [Preserve]
+    public sealed class ClientSecret
+    {
+        [Preserve]
+        public ClientSecret(int? expiresAfter = null)
+        {
+            ExpiresAfter = expiresAfter ?? 600;
+        }
+
+        [Preserve]
+        [JsonConstructor]
+        internal ClientSecret(
+            [JsonProperty("value")] string ephemeralApiKey,
+            [JsonProperty("expires_at")] int? expiresAtUnixTimeSeconds = null,
+            [JsonProperty("expires_after")] ExpiresAfter expiresAfter = null)
+        {
+            EphemeralApiKey = ephemeralApiKey;
+            ExpiresAtUnixTimeSeconds = expiresAtUnixTimeSeconds;
+            ExpiresAfter = expiresAfter;
+        }
+
+        [Preserve]
+        [JsonProperty("value", DefaultValueHandling = DefaultValueHandling.Ignore)]
+        public string EphemeralApiKey { get; }
+
+        [Preserve]
+        [JsonProperty("expires_at", DefaultValueHandling = DefaultValueHandling.Ignore)]
+        public int? ExpiresAtUnixTimeSeconds { get; }
+
+        [Preserve]
+        [JsonIgnore]
+        public DateTime? ExpiresAt => ExpiresAtUnixTimeSeconds.HasValue
+            ? DateTimeOffset.FromUnixTimeSeconds(ExpiresAtUnixTimeSeconds.Value).UtcDateTime
+            : null;
+
+        [Preserve]
+        [JsonProperty("expires_after", DefaultValueHandling = DefaultValueHandling.Ignore)]
+        public ExpiresAfter ExpiresAfter { get; }
+
+        [Preserve]
+        public static implicit operator string(ClientSecret clientSecret) => clientSecret?.EphemeralApiKey;
+    }
+}

OpenAI/Packages/com.openai.unity/Runtime/Realtime/ClientSecret.cs.meta

@@ -12,4 +12,4 @@ public sealed class DisabledVAD : IVoiceActivityDetectionSettings
 
         public bool InterruptResponse => false;
     }
-}
+}

OpenAI/Packages/com.openai.unity/Runtime/Realtime/DisabledVAD.cs

@@ -0,0 +1,37 @@
+// Licensed under the MIT License. See LICENSE in the project root for license information.
+
+using Newtonsoft.Json;
+using UnityEngine.Scripting;
+
+namespace OpenAI.Realtime
+{
+    [Preserve]
+    public sealed class ExpiresAfter
+    {
+        [Preserve]
+        public ExpiresAfter(int seconds = 600)
+        {
+            Seconds = seconds;
+        }
+
+        [Preserve]
+        [JsonConstructor]
+        internal ExpiresAfter(string anchor, int seconds)
+        {
+            Anchor = anchor;
+            Seconds = seconds;
+        }
+
+        [Preserve]
+        [JsonProperty("anchor")]
+        public string Anchor { get; } = "created_at";
+
+        [Preserve]
+        [JsonProperty("seconds")]
+        public int Seconds { get; }
+
+        [Preserve]
+        public static implicit operator ExpiresAfter(int seconds)
+            => new(seconds);
+    }
+}

OpenAI/Packages/com.openai.unity/Runtime/Realtime/ExpiresAfter.cs

@@ -1,5 +1,7 @@
 // Licensed under the MIT License. See LICENSE in the project root for license information.
 
+using Newtonsoft.Json;
+using OpenAI.Extensions;
 using OpenAI.Models;
 using System;
 using System.Collections.Generic;
@@ -8,6 +10,8 @@
 using System.Threading.Tasks;
 using UnityEngine;
 using Utilities.Async;
+using Utilities.WebRequestRest;
+using Utilities.WebSockets;
 
 namespace OpenAI.Realtime
 {
@@ -17,8 +21,6 @@ public RealtimeEndpoint(OpenAIClient client) : base(client) { }
 
         protected override string Root => "realtime";
 
-        protected override bool? IsWebSocketEndpoint => true;
-
         /// <summary>
         /// Creates a new realtime session with the provided <see cref="SessionConfiguration"/> options.
         /// </summary>
@@ -39,7 +41,33 @@ public async Task<RealtimeSession> CreateSessionAsync(SessionConfiguration confi
                 queryParameters["model"] = model;
             }
 
-            var session = new RealtimeSession(client.CreateWebSocket(GetUrl(queryParameters: queryParameters)), EnableDebug);
+            var payload = JsonConvert.SerializeObject(configuration, OpenAIClient.JsonSerializationOptions);
+            var createSessionResponse = await Rest.PostAsync(GetUrl("/sessions"), payload, new RestParameters(client.DefaultRequestHeaders), cancellationToken);
+            createSessionResponse.Validate(EnableDebug);
+            var createSession = createSessionResponse.Deserialize<SessionConfiguration>(client);
+
+            if (createSession == null ||
+                string.IsNullOrWhiteSpace(createSession.ClientSecret?.EphemeralApiKey))
+            {
+                throw new InvalidOperationException("Failed to create a session. Ensure the configuration is valid and the API key is set.");
+            }
+
+            var websocket = new WebSocket(GetWebsocketUri(queryParameters: queryParameters), new Dictionary<string, string>
+            {
+#if !PLATFORM_WEBGL
+                { "User-Agent", "OpenAI-DotNet" },
+                { "OpenAI-Beta", "realtime=v1" },
+                { "Authorization", $"Bearer {createSession.ClientSecret!.EphemeralApiKey}" }
+#endif
+            }, new List<string>
+            {
+#if PLATFORM_WEBGL // Web browsers do not support headers. https://github.com/openai/openai-realtime-api-beta/blob/339e9553a757ef1cf8c767272fc750c1e62effbb/lib/api.js#L76-L80
+                "realtime",
+                $"openai-insecure-api-key.{createSession.ClientSecret!.EphemeralApiKey}",
+                "openai-beta.realtime-v1"
+#endif
+            });
+            var session = new RealtimeSession(websocket, EnableDebug);
             var sessionCreatedTcs = new TaskCompletionSource<SessionResponse>();
 
             try
@@ -49,7 +77,6 @@ public async Task<RealtimeSession> CreateSessionAsync(SessionConfiguration confi
                 await session.ConnectAsync(cancellationToken).ConfigureAwait(true);
                 var sessionResponse = await sessionCreatedTcs.Task.WithCancellation(cancellationToken).ConfigureAwait(true);
                 session.Configuration = sessionResponse.SessionConfiguration;
-                await session.SendAsync(new UpdateSessionRequest(configuration), cancellationToken: cancellationToken).ConfigureAwait(true);
             }
             finally
             {