diff --git a/dotnet/src/Connectors/Connectors.AzureOpenAI.UnitTests/Extensions/AzureOpenAIKernelBuilderExtensionsTests.cs b/dotnet/src/Connectors/Connectors.AzureOpenAI.UnitTests/Extensions/AzureOpenAIKernelBuilderExtensionsTests.cs index bcdc37c208d9..af673e4ae004 100644 --- a/dotnet/src/Connectors/Connectors.AzureOpenAI.UnitTests/Extensions/AzureOpenAIKernelBuilderExtensionsTests.cs +++ b/dotnet/src/Connectors/Connectors.AzureOpenAI.UnitTests/Extensions/AzureOpenAIKernelBuilderExtensionsTests.cs @@ -15,6 +15,8 @@ using Microsoft.SemanticKernel.TextToAudio; using Microsoft.SemanticKernel.TextToImage; +#pragma warning disable MEAI001 // Image generation APIs are experimental. + namespace SemanticKernel.Connectors.AzureOpenAI.UnitTests.Extensions; /// @@ -185,6 +187,50 @@ public void KernelBuilderExtensionsAddAzureOpenAITextToImageService(Initializati #endregion + #region Image Generation + + [Theory] + [InlineData(InitializationType.ApiKey, null)] + [InlineData(InitializationType.ApiKey, "images")] + [InlineData(InitializationType.TokenCredential, null)] + [InlineData(InitializationType.TokenCredential, "images")] + [InlineData(InitializationType.ClientInline, null)] + [InlineData(InitializationType.ClientInline, "images")] + [InlineData(InitializationType.ClientInServiceProvider, null)] + [InlineData(InitializationType.ClientInServiceProvider, "images")] + [InlineData(InitializationType.ApiVersion, null)] + [InlineData(InitializationType.ApiVersion, "images")] + public void KernelBuilderAddAzureOpenAIImageGeneratorAddsValidService(InitializationType type, string? serviceId) + { + // Arrange + var credentials = DelegatedTokenCredential.Create((_, _) => new AccessToken()); + var client = new AzureOpenAIClient(new Uri("https://localhost"), new ApiKeyCredential("key")); + var builder = Kernel.CreateBuilder(); + bool telemetryConfigured = false; + void configureTelemetry(OpenTelemetryImageGenerator _) => telemetryConfigured = true; + builder.Services.AddSingleton(client); + + // Act + builder = type switch + { + InitializationType.ApiKey => builder.AddAzureOpenAIImageGenerator("deployment-name", "https://endpoint", "api-key", serviceId, openTelemetryConfig: configureTelemetry), + InitializationType.TokenCredential => builder.AddAzureOpenAIImageGenerator("deployment-name", "https://endpoint", credentials, serviceId, openTelemetryConfig: configureTelemetry), + InitializationType.ClientInline => builder.AddAzureOpenAIImageGenerator("deployment-name", client, serviceId, openTelemetryConfig: configureTelemetry), + InitializationType.ClientInServiceProvider => builder.AddAzureOpenAIImageGenerator("deployment-name", serviceId: serviceId, openTelemetryConfig: configureTelemetry), + InitializationType.ApiVersion => builder.AddAzureOpenAIImageGenerator("deployment-name", "https://endpoint", "api-key", serviceId, apiVersion: "2025-04-01-preview", openTelemetryConfig: configureTelemetry), + _ => throw new ArgumentOutOfRangeException(nameof(type)) + }; + + // Assert + var kernel = builder.Build(); + var service = kernel.GetRequiredService(serviceId); + Assert.Equal("deployment-name", service.GetService()!.DefaultModelId); + Assert.Same(service, kernel.GetRequiredService(serviceId)); + Assert.True(telemetryConfigured); + } + + #endregion + #region Audio to text [Theory] diff --git a/dotnet/src/Connectors/Connectors.AzureOpenAI.UnitTests/Extensions/AzureOpenAIServiceCollectionExtensionsTests.cs b/dotnet/src/Connectors/Connectors.AzureOpenAI.UnitTests/Extensions/AzureOpenAIServiceCollectionExtensionsTests.cs index 4d30df551507..2e4285535297 100644 --- a/dotnet/src/Connectors/Connectors.AzureOpenAI.UnitTests/Extensions/AzureOpenAIServiceCollectionExtensionsTests.cs +++ b/dotnet/src/Connectors/Connectors.AzureOpenAI.UnitTests/Extensions/AzureOpenAIServiceCollectionExtensionsTests.cs @@ -2,15 +2,19 @@ using System; using System.ClientModel; +using System.Diagnostics; using System.IO; using System.Net; using System.Net.Http; +using System.Text.Json; using System.Threading; using System.Threading.Tasks; using Azure.AI.OpenAI; using Azure.Core; using Microsoft.Extensions.AI; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; using Microsoft.SemanticKernel; using Microsoft.SemanticKernel.AudioToText; using Microsoft.SemanticKernel.ChatCompletion; @@ -22,6 +26,8 @@ using Moq; using Moq.Protected; +#pragma warning disable MEAI001 // Image generation APIs are experimental. + namespace SemanticKernel.Connectors.AzureOpenAI.UnitTests.Extensions; /// /// Unit tests for the service collection extensions in the class. @@ -226,6 +232,141 @@ public void ServiceCollectionExtensionsAddAzureOpenAITextToImageService(Initiali #endregion + #region Image Generation + + [Theory] + [InlineData(InitializationType.ApiKey, null)] + [InlineData(InitializationType.ApiKey, "images")] + [InlineData(InitializationType.TokenCredential, null)] + [InlineData(InitializationType.TokenCredential, "images")] + [InlineData(InitializationType.ClientInline, null)] + [InlineData(InitializationType.ClientInline, "images")] + [InlineData(InitializationType.ClientInServiceProvider, null)] + [InlineData(InitializationType.ClientInServiceProvider, "images")] + [InlineData(InitializationType.ApiVersion, null)] + [InlineData(InitializationType.ApiVersion, "images")] + [InlineData(InitializationType.HttpClient, null)] + [InlineData(InitializationType.HttpClient, "images")] + public async Task ServiceCollectionAddAzureOpenAIImageGeneratorAddsValidServiceAsync(InitializationType type, string? serviceId) + { + // Arrange + using var responseMessage = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(File.ReadAllText("./TestData/text-to-image-b64_json-format-response.json")) + }; + using var messageHandler = new HttpMessageHandlerStub { ResponseToReturn = responseMessage }; + using var httpClient = new HttpClient(messageHandler, false); + var credentials = DelegatedTokenCredential.Create((_, _) => new AccessToken("test-token", DateTimeOffset.UtcNow.AddHours(1))); + var client = new AzureOpenAIClient( + new Uri("https://endpoint"), + new ApiKeyCredential("key"), + AzureClientCore.GetAzureOpenAIClientOptions(httpClient)); + var builder = Kernel.CreateBuilder(); + builder.Services.AddSingleton(client); + builder.Services.AddSingleton(httpClient); + builder.Services.AddSingleton(NullLoggerFactory.Instance); + bool telemetryConfigured = false; + void configureTelemetry(OpenTelemetryImageGenerator _) => telemetryConfigured = true; + string sourceName = $"AzureOpenAIImages.{Guid.NewGuid():N}"; + Activity? imageActivity = null; + using var listener = new ActivityListener + { + ShouldListenTo = source => source.Name == sourceName, + Sample = (ref ActivityCreationOptions _) => ActivitySamplingResult.AllData, + ActivityStopped = activity => imageActivity = activity + }; + ActivitySource.AddActivityListener(listener); + + // Act + IServiceCollection collection = type switch + { + InitializationType.ApiKey => builder.Services.AddAzureOpenAIImageGenerator("deployment-name", "https://endpoint", "api-key", serviceId, httpClient: httpClient, openTelemetrySourceName: sourceName, openTelemetryConfig: configureTelemetry), + InitializationType.TokenCredential => builder.Services.AddAzureOpenAIImageGenerator("deployment-name", "https://endpoint", credentials, serviceId, httpClient: httpClient, openTelemetrySourceName: sourceName, openTelemetryConfig: configureTelemetry), + InitializationType.ClientInline => builder.Services.AddAzureOpenAIImageGenerator("deployment-name", client, serviceId, sourceName, configureTelemetry), + InitializationType.ClientInServiceProvider => builder.Services.AddAzureOpenAIImageGenerator("deployment-name", serviceId: serviceId, openTelemetrySourceName: sourceName, openTelemetryConfig: configureTelemetry), + InitializationType.ApiVersion => builder.Services.AddAzureOpenAIImageGenerator("deployment-name", "https://endpoint", "api-key", serviceId, apiVersion: "2025-04-01-preview", httpClient: httpClient, openTelemetrySourceName: sourceName, openTelemetryConfig: configureTelemetry), + InitializationType.HttpClient => builder.Services.AddAzureOpenAIImageGenerator("deployment-name", "https://endpoint", "api-key", serviceId, openTelemetrySourceName: sourceName, openTelemetryConfig: configureTelemetry), + _ => throw new ArgumentOutOfRangeException(nameof(type)) + }; + using var provider = collection.BuildServiceProvider(); + var service = provider.GetRequiredKeyedService(serviceId); + var response = await service.GenerateImagesAsync("A sea otter", new ImageGenerationOptions + { + ImageSize = new System.Drawing.Size(1024, 1024) + }); + + // Assert + Assert.Same(builder.Services, collection); + Assert.Equal("deployment-name", service.GetService()!.DefaultModelId); + Assert.Same(service, provider.GetRequiredKeyedService(serviceId)); + var image = Assert.IsType(Assert.Single(response.Contents)); + Assert.Equal(Convert.FromBase64String("iVBORw0KGgoAAA=="), image.Data.ToArray()); + Assert.Equal("image/png", image.MediaType); + Assert.True(telemetryConfigured); + Assert.NotNull(imageActivity); + Assert.Equal(sourceName, imageActivity.Source.Name); + Assert.Equal("endpoint", messageHandler.RequestUri!.Host); + + using var request = JsonDocument.Parse(messageHandler.RequestContent!); + Assert.Equal("deployment-name", request.RootElement.GetProperty("model").GetString()); + Assert.Equal("A sea otter", request.RootElement.GetProperty("prompt").GetString()); + Assert.Equal("1024x1024", request.RootElement.GetProperty("size").GetString()); + Assert.Equal(1, request.RootElement.GetProperty("n").GetInt32()); + if (type == InitializationType.ApiVersion) + { + Assert.Contains("api-version=2025-04-01-preview", messageHandler.RequestUri.Query); + } + if (type == InitializationType.TokenCredential) + { + Assert.Equal("Bearer", messageHandler.RequestHeaders!.Authorization!.Scheme); + Assert.Equal("test-token", messageHandler.RequestHeaders.Authorization.Parameter); + } + } + + [Fact] + public async Task ImageGeneratorCanEditImagesAsync() + { + // Arrange + using var responseMessage = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent("""{"created":1726234481,"data":[{"b64_json":"AQID"}]}""") + }; + using var messageHandler = new HttpMessageHandlerStub { ResponseToReturn = responseMessage }; + using var httpClient = new HttpClient(messageHandler, false); + using var provider = new ServiceCollection() + .AddAzureOpenAIImageGenerator("deployment-name", "https://endpoint", "api-key", apiVersion: "2025-04-01-preview", httpClient: httpClient) + .BuildServiceProvider(); + var service = provider.GetRequiredService(); + + // Act + var response = await service.EditImageAsync( + new DataContent(new byte[] { 1, 2, 3 }, "image/png") { Name = "input.png" }, + "Make the background red"); + + // Assert + var image = Assert.IsType(Assert.Single(response.Contents)); + Assert.Equal(new byte[] { 1, 2, 3 }, image.Data.ToArray()); + Assert.Equal("image/png", image.MediaType); + Assert.Equal("endpoint", messageHandler.RequestUri!.Host); + Assert.EndsWith("/images/edits", messageHandler.RequestUri.AbsolutePath); + Assert.Contains("api-version=2025-04-01-preview", messageHandler.RequestUri.Query); + Assert.Equal("multipart/form-data", messageHandler.ContentHeaders!.ContentType!.MediaType); + } + + [Fact] + public void ImageGeneratorRequiresRegisteredAzureOpenAIClient() + { + // Arrange + using var provider = new ServiceCollection() + .AddAzureOpenAIImageGenerator("deployment-name") + .BuildServiceProvider(); + + // Act & Assert + Assert.Throws(() => provider.GetRequiredService()); + } + + #endregion + #region Audio to text [Theory] diff --git a/dotnet/src/Connectors/Connectors.AzureOpenAI.UnitTests/Services/AzureOpenAITextToImageServiceTests.cs b/dotnet/src/Connectors/Connectors.AzureOpenAI.UnitTests/Services/AzureOpenAITextToImageServiceTests.cs index dd2381598ba0..53085e6e7747 100644 --- a/dotnet/src/Connectors/Connectors.AzureOpenAI.UnitTests/Services/AzureOpenAITextToImageServiceTests.cs +++ b/dotnet/src/Connectors/Connectors.AzureOpenAI.UnitTests/Services/AzureOpenAITextToImageServiceTests.cs @@ -3,6 +3,7 @@ using System; using System.ClientModel; using System.IO; +using System.Net; using System.Net.Http; using System.Text; using System.Text.Json; @@ -195,6 +196,9 @@ public async Task GetUriImageContentsResponseFormatRequestWorksCorrectlyAsync(st [InlineData("hd", "hd")] [InlineData("high", "hd")] [InlineData("standard", "standard")] + [InlineData("medium", "medium")] + [InlineData("low", "low")] + [InlineData("auto", "auto")] public async Task GetUriImageContentsImageQualityRequestWorksCorrectlyAsync(string? quality, string? expectedQuality) { // Arrange @@ -335,6 +339,65 @@ public async Task GetUrlImageContentsResponseWorksCorrectlyAsync() Assert.Equal("my prompt", breakingGlass!.RevisedPrompt); } + [Fact] + public async Task GenerateGptImageReturnsDataUriAsync() + { + // Arrange + using var responseMessage = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(File.ReadAllText("./TestData/text-to-image-b64_json-format-response.json")) + }; + this._messageHandlerStub.ResponseToReturn = responseMessage; + var sut = new AzureOpenAITextToImageService("image-deployment", "https://api-host", "api-key", "gpt-image-1", this._httpClient); + + // Act + var result = await sut.GenerateImageAsync("A sea otter", 1024, 1024); + + // Assert + Assert.Equal("data:image/png;base64,iVBORw0KGgoAAA==", result); + } + + [Fact] + public async Task GetGptImageContentsPreservesSettingsWithProvidedClientAsync() + { + // Arrange + using var responseMessage = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(File.ReadAllText("./TestData/text-to-image-b64_json-format-response.json")) + }; + this._messageHandlerStub.ResponseToReturn = responseMessage; + var client = new AzureOpenAIClient( + new Uri("https://api-host"), + new ApiKeyCredential("api-key"), + AzureClientCore.GetAzureOpenAIClientOptions(this._httpClient, "2025-04-01-preview")); + var sut = new AzureOpenAITextToImageService("image-deployment", client, "gpt-image-1"); + + // Act + var result = await sut.GetImageContentsAsync("A sea otter", new OpenAITextToImageExecutionSettings + { + Size = (1536, 1024), + Quality = "low", + EndUserId = "test-user" + }); + + // Assert + var image = Assert.Single(result); + Assert.Equal(Convert.FromBase64String("iVBORw0KGgoAAA=="), image.Data!.Value.ToArray()); + Assert.Equal("image/png", image.MimeType); + Assert.Equal("my prompt", Assert.IsType(image.InnerContent).RevisedPrompt); + Assert.Equal("gpt-image-1", sut.Attributes[AIServiceExtensions.ModelIdKey]); + Assert.Equal("image-deployment", sut.Attributes[AzureClientCore.DeploymentNameKey]); + using var request = JsonDocument.Parse(this._messageHandlerStub.RequestContent!); + Assert.Equal("image-deployment", request.RootElement.GetProperty("model").GetString()); + Assert.Equal("A sea otter", request.RootElement.GetProperty("prompt").GetString()); + Assert.Equal("1536x1024", request.RootElement.GetProperty("size").GetString()); + Assert.Equal("low", request.RootElement.GetProperty("quality").GetString()); + Assert.Equal("test-user", request.RootElement.GetProperty("user").GetString()); + Assert.Equal(1, request.RootElement.GetProperty("n").GetInt32()); + Assert.False(request.RootElement.TryGetProperty("response_format", out _)); + Assert.Contains("api-version=2025-04-01-preview", this._messageHandlerStub.RequestUri!.Query); + } + [Theory] [MemberData(nameof(Versions))] public async Task ItTargetsApiVersionAsExpected(string? apiVersion, string? expectedVersion = null) diff --git a/dotnet/src/Connectors/Connectors.AzureOpenAI/Extensions/AzureOpenAIKernelBuilderExtensions.cs b/dotnet/src/Connectors/Connectors.AzureOpenAI/Extensions/AzureOpenAIKernelBuilderExtensions.cs index a370497e3696..f94369fddc6b 100644 --- a/dotnet/src/Connectors/Connectors.AzureOpenAI/Extensions/AzureOpenAIKernelBuilderExtensions.cs +++ b/dotnet/src/Connectors/Connectors.AzureOpenAI/Extensions/AzureOpenAIKernelBuilderExtensions.cs @@ -832,6 +832,121 @@ public static IKernelBuilder AddAzureOpenAIAudioToText( #endregion + #region Image Generation + + /// + /// Adds an Azure OpenAI to the . + /// + /// The instance to augment. + /// Azure OpenAI deployment name, see https://learn.microsoft.com/azure/cognitive-services/openai/how-to/create-resource + /// Azure OpenAI deployment URL, see https://learn.microsoft.com/azure/cognitive-services/openai/quickstart + /// Azure OpenAI API key, see https://learn.microsoft.com/azure/cognitive-services/openai/quickstart + /// A local identifier for the given AI service + /// Optional Azure OpenAI API version, see available here + /// The HttpClient to use with this service. + /// An optional name for the OpenTelemetry source. + /// An optional callback that can be used to configure the instance. + /// The same instance as . + [Experimental("SKEXP0010")] + public static IKernelBuilder AddAzureOpenAIImageGenerator( + this IKernelBuilder builder, + string deploymentName, + string endpoint, + string apiKey, + string? serviceId = null, + string? apiVersion = null, + HttpClient? httpClient = null, + string? openTelemetrySourceName = null, + Action? openTelemetryConfig = null) + { + Verify.NotNull(builder); + + builder.Services.AddAzureOpenAIImageGenerator( + deploymentName, + endpoint, + apiKey, + serviceId, + apiVersion, + httpClient, + openTelemetrySourceName, + openTelemetryConfig); + + return builder; + } + + /// + /// Adds an Azure OpenAI to the . + /// + /// The instance to augment. + /// Azure OpenAI deployment name, see https://learn.microsoft.com/azure/cognitive-services/openai/how-to/create-resource + /// Azure OpenAI deployment URL, see https://learn.microsoft.com/azure/cognitive-services/openai/quickstart + /// Token credentials, e.g. DefaultAzureCredential, ManagedIdentityCredential, EnvironmentCredential, etc. + /// A local identifier for the given AI service + /// Optional Azure OpenAI API version, see available here + /// The HttpClient to use with this service. + /// An optional name for the OpenTelemetry source. + /// An optional callback that can be used to configure the instance. + /// The same instance as . + [Experimental("SKEXP0010")] + public static IKernelBuilder AddAzureOpenAIImageGenerator( + this IKernelBuilder builder, + string deploymentName, + string endpoint, + TokenCredential credentials, + string? serviceId = null, + string? apiVersion = null, + HttpClient? httpClient = null, + string? openTelemetrySourceName = null, + Action? openTelemetryConfig = null) + { + Verify.NotNull(builder); + + builder.Services.AddAzureOpenAIImageGenerator( + deploymentName, + endpoint, + credentials, + serviceId, + apiVersion, + httpClient, + openTelemetrySourceName, + openTelemetryConfig); + + return builder; + } + + /// + /// Adds an Azure OpenAI to the . + /// + /// The instance to augment. + /// Azure OpenAI deployment name, see https://learn.microsoft.com/azure/cognitive-services/openai/how-to/create-resource + /// to use for the service. If null, one must be available in the service provider when this service is resolved. + /// A local identifier for the given AI service + /// An optional name for the OpenTelemetry source. + /// An optional callback that can be used to configure the instance. + /// The same instance as . + [Experimental("SKEXP0010")] + public static IKernelBuilder AddAzureOpenAIImageGenerator( + this IKernelBuilder builder, + string deploymentName, + AzureOpenAIClient? azureOpenAIClient = null, + string? serviceId = null, + string? openTelemetrySourceName = null, + Action? openTelemetryConfig = null) + { + Verify.NotNull(builder); + + builder.Services.AddAzureOpenAIImageGenerator( + deploymentName, + azureOpenAIClient, + serviceId, + openTelemetrySourceName, + openTelemetryConfig); + + return builder; + } + + #endregion + private static AzureOpenAIClient CreateAzureOpenAIClient(string endpoint, ApiKeyCredential credentials, HttpClient? httpClient, string? apiVersion) => new(new Uri(endpoint), credentials, AzureClientCore.GetAzureOpenAIClientOptions(httpClient, apiVersion)); diff --git a/dotnet/src/Connectors/Connectors.AzureOpenAI/Extensions/AzureOpenAIServiceCollectionExtensions.DependencyInjection.cs b/dotnet/src/Connectors/Connectors.AzureOpenAI/Extensions/AzureOpenAIServiceCollectionExtensions.DependencyInjection.cs index 229bef7163a4..42cacb206cf3 100644 --- a/dotnet/src/Connectors/Connectors.AzureOpenAI/Extensions/AzureOpenAIServiceCollectionExtensions.DependencyInjection.cs +++ b/dotnet/src/Connectors/Connectors.AzureOpenAI/Extensions/AzureOpenAIServiceCollectionExtensions.DependencyInjection.cs @@ -349,4 +349,157 @@ public static IServiceCollection AddAzureOpenAIEmbeddingGenerator( } #endregion + + #region Image Generation + + /// + /// Adds the to the . + /// + /// The instance to augment. + /// Azure OpenAI deployment name, see https://learn.microsoft.com/azure/cognitive-services/openai/how-to/create-resource + /// Azure OpenAI deployment URL, see https://learn.microsoft.com/azure/cognitive-services/openai/quickstart + /// Azure OpenAI API key, see https://learn.microsoft.com/azure/cognitive-services/openai/quickstart + /// A local identifier for the given AI service + /// Optional Azure OpenAI API version, see available here + /// The HttpClient to use with this service. + /// An optional name for the OpenTelemetry source. + /// An optional callback that can be used to configure the instance. + /// The same instance as . + [Experimental("SKEXP0010")] + public static IServiceCollection AddAzureOpenAIImageGenerator( + this IServiceCollection services, + string deploymentName, + string endpoint, + string apiKey, + string? serviceId = null, + string? apiVersion = null, + HttpClient? httpClient = null, + string? openTelemetrySourceName = null, + Action? openTelemetryConfig = null) + { + Verify.NotNull(services); + Verify.NotNullOrWhiteSpace(deploymentName); + Verify.NotNullOrWhiteSpace(endpoint); + Verify.NotNullOrWhiteSpace(apiKey); + + return services.AddKeyedSingleton(serviceId, (serviceProvider, _) => + { + var loggerFactory = serviceProvider.GetService(); + + AzureOpenAIClient client = Microsoft.SemanticKernel.AzureOpenAIServiceCollectionExtensions.CreateAzureOpenAIClient( + endpoint, + new ApiKeyCredential(apiKey), + HttpClientProvider.GetHttpClient(httpClient, serviceProvider), + apiVersion); + + var builder = client.GetImageClient(deploymentName) + .AsIImageGenerator() + .AsBuilder() + .UseOpenTelemetry(loggerFactory, openTelemetrySourceName, openTelemetryConfig); + + if (loggerFactory is not null) + { + builder.UseLogging(loggerFactory); + } + + return builder.Build(); + }); + } + + /// + /// Adds the to the . + /// + /// The instance to augment. + /// Azure OpenAI deployment name, see https://learn.microsoft.com/azure/cognitive-services/openai/how-to/create-resource + /// Azure OpenAI deployment URL, see https://learn.microsoft.com/azure/cognitive-services/openai/quickstart + /// Token credentials, e.g. DefaultAzureCredential, ManagedIdentityCredential, EnvironmentCredential, etc. + /// A local identifier for the given AI service + /// Optional Azure OpenAI API version, see available here + /// The HttpClient to use with this service. + /// An optional name for the OpenTelemetry source. + /// An optional callback that can be used to configure the instance. + /// The same instance as . + [Experimental("SKEXP0010")] + public static IServiceCollection AddAzureOpenAIImageGenerator( + this IServiceCollection services, + string deploymentName, + string endpoint, + TokenCredential credentials, + string? serviceId = null, + string? apiVersion = null, + HttpClient? httpClient = null, + string? openTelemetrySourceName = null, + Action? openTelemetryConfig = null) + { + Verify.NotNull(services); + Verify.NotNullOrWhiteSpace(deploymentName); + Verify.NotNullOrWhiteSpace(endpoint); + Verify.NotNull(credentials); + + return services.AddKeyedSingleton(serviceId, (serviceProvider, _) => + { + var loggerFactory = serviceProvider.GetService(); + + AzureOpenAIClient client = Microsoft.SemanticKernel.AzureOpenAIServiceCollectionExtensions.CreateAzureOpenAIClient( + endpoint, + credentials, + HttpClientProvider.GetHttpClient(httpClient, serviceProvider), + apiVersion); + + var builder = client.GetImageClient(deploymentName) + .AsIImageGenerator() + .AsBuilder() + .UseOpenTelemetry(loggerFactory, openTelemetrySourceName, openTelemetryConfig); + + if (loggerFactory is not null) + { + builder.UseLogging(loggerFactory); + } + + return builder.Build(); + }); + } + + /// + /// Adds the to the . + /// + /// The instance to augment. + /// Azure OpenAI deployment name, see https://learn.microsoft.com/azure/cognitive-services/openai/how-to/create-resource + /// to use for the service. If null, one must be available in the service provider when this service is resolved. + /// A local identifier for the given AI service + /// An optional name for the OpenTelemetry source. + /// An optional callback that can be used to configure the instance. + /// The same instance as . + [Experimental("SKEXP0010")] + public static IServiceCollection AddAzureOpenAIImageGenerator( + this IServiceCollection services, + string deploymentName, + AzureOpenAIClient? azureOpenAIClient = null, + string? serviceId = null, + string? openTelemetrySourceName = null, + Action? openTelemetryConfig = null) + { + Verify.NotNull(services); + Verify.NotNullOrWhiteSpace(deploymentName); + + return services.AddKeyedSingleton(serviceId, (serviceProvider, _) => + { + var loggerFactory = serviceProvider.GetService(); + var client = azureOpenAIClient ?? serviceProvider.GetRequiredService(); + + var builder = client.GetImageClient(deploymentName) + .AsIImageGenerator() + .AsBuilder() + .UseOpenTelemetry(loggerFactory, openTelemetrySourceName, openTelemetryConfig); + + if (loggerFactory is not null) + { + builder.UseLogging(loggerFactory); + } + + return builder.Build(); + }); + } + + #endregion } diff --git a/dotnet/src/Connectors/Connectors.AzureOpenAI/Services/AzureOpenAITextToImageService.cs b/dotnet/src/Connectors/Connectors.AzureOpenAI/Services/AzureOpenAITextToImageService.cs index 6fbaeeea49d1..36bacc7e9c3a 100644 --- a/dotnet/src/Connectors/Connectors.AzureOpenAI/Services/AzureOpenAITextToImageService.cs +++ b/dotnet/src/Connectors/Connectors.AzureOpenAI/Services/AzureOpenAITextToImageService.cs @@ -18,6 +18,10 @@ namespace Microsoft.SemanticKernel.Connectors.AzureOpenAI; /// /// Azure OpenAI text to image service. /// +/// +/// Image generation delegates to while preserving Semantic Kernel content and settings. +/// New applications can use AddAzureOpenAIImageGenerator to access image generation and editing directly. +/// [Experimental("SKEXP0010")] public class AzureOpenAITextToImageService : ITextToImageService { diff --git a/dotnet/src/Connectors/Connectors.OpenAI.UnitTests/Extensions/KernelBuilderExtensionsTests.cs b/dotnet/src/Connectors/Connectors.OpenAI.UnitTests/Extensions/KernelBuilderExtensionsTests.cs index 972cdb62b2ac..498e896bad2e 100644 --- a/dotnet/src/Connectors/Connectors.OpenAI.UnitTests/Extensions/KernelBuilderExtensionsTests.cs +++ b/dotnet/src/Connectors/Connectors.OpenAI.UnitTests/Extensions/KernelBuilderExtensionsTests.cs @@ -16,6 +16,8 @@ using OpenAI; using Xunit; +#pragma warning disable MEAI001 // Image generation APIs are experimental. + namespace SemanticKernel.Connectors.OpenAI.UnitTests.Extensions; public class KernelBuilderExtensionsTests @@ -99,6 +101,39 @@ public void ItCanAddTextToImageService() Assert.Equal("model", service.Attributes[AIServiceExtensions.ModelIdKey]); } + [Theory] + [InlineData(InitializationType.ApiKey, null)] + [InlineData(InitializationType.ApiKey, "images")] + [InlineData(InitializationType.OpenAIClientInline, null)] + [InlineData(InitializationType.OpenAIClientInline, "images")] + [InlineData(InitializationType.OpenAIClientInServiceProvider, null)] + [InlineData(InitializationType.OpenAIClientInServiceProvider, "images")] + public void KernelBuilderAddOpenAIImageGeneratorAddsValidService(InitializationType initializationType, string? serviceId) + { + // Arrange + var builder = Kernel.CreateBuilder(); + var client = new OpenAIClient(new ApiKeyCredential("api-key")); + bool telemetryConfigured = false; + void configureTelemetry(OpenTelemetryImageGenerator _) => telemetryConfigured = true; + builder.Services.AddSingleton(client); + + // Act + builder = initializationType switch + { + InitializationType.ApiKey => builder.AddOpenAIImageGenerator("gpt-image-1", "api-key", serviceId: serviceId, openTelemetryConfig: configureTelemetry), + InitializationType.OpenAIClientInline => builder.AddOpenAIImageGenerator("gpt-image-1", client, serviceId, openTelemetryConfig: configureTelemetry), + InitializationType.OpenAIClientInServiceProvider => builder.AddOpenAIImageGenerator("gpt-image-1", serviceId: serviceId, openTelemetryConfig: configureTelemetry), + _ => throw new ArgumentOutOfRangeException(nameof(initializationType)) + }; + + // Assert + var kernel = builder.Build(); + var service = kernel.GetRequiredService(serviceId); + Assert.Equal("gpt-image-1", service.GetService()!.DefaultModelId); + Assert.Same(service, kernel.GetRequiredService(serviceId)); + Assert.True(telemetryConfigured); + } + [Fact] public void ItCanAddTextToAudioService() { diff --git a/dotnet/src/Connectors/Connectors.OpenAI.UnitTests/Extensions/ServiceCollectionExtensionsTests.cs b/dotnet/src/Connectors/Connectors.OpenAI.UnitTests/Extensions/ServiceCollectionExtensionsTests.cs index cdedc54a90d5..ae1d34636be9 100644 --- a/dotnet/src/Connectors/Connectors.OpenAI.UnitTests/Extensions/ServiceCollectionExtensionsTests.cs +++ b/dotnet/src/Connectors/Connectors.OpenAI.UnitTests/Extensions/ServiceCollectionExtensionsTests.cs @@ -2,8 +2,17 @@ using System; using System.ClientModel; +using System.Diagnostics; +using System.IO; +using System.Net; +using System.Net.Http; +using System.Text; +using System.Text.Json; +using System.Threading.Tasks; using Microsoft.Extensions.AI; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; using Microsoft.SemanticKernel; using Microsoft.SemanticKernel.AudioToText; using Microsoft.SemanticKernel.ChatCompletion; @@ -14,8 +23,11 @@ using Microsoft.SemanticKernel.TextToAudio; using Microsoft.SemanticKernel.TextToImage; using OpenAI; +using OpenAI.Images; using Xunit; +#pragma warning disable MEAI001 // Image generation APIs are experimental. + namespace SemanticKernel.Connectors.OpenAI.UnitTests.Extensions; public class ServiceCollectionExtensionsTests @@ -185,6 +197,129 @@ public void ItCanAddFileService() .GetRequiredService(); } + #region Image Generation + + [Theory] + [InlineData(InitializationType.ApiKey, null)] + [InlineData(InitializationType.ApiKey, "images")] + [InlineData(InitializationType.ClientInline, null)] + [InlineData(InitializationType.ClientInline, "images")] + [InlineData(InitializationType.ClientInServiceProvider, null)] + [InlineData(InitializationType.ClientInServiceProvider, "images")] + public async Task ItCanAddImageGeneratorServiceAsync(InitializationType type, string? serviceId) + { + // Arrange + using var responseMessage = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(File.ReadAllText("./TestData/text-to-image-b64_json-format-response.json")) + }; + using var messageHandler = new HttpMessageHandlerStub { ResponseToReturn = responseMessage }; + using var httpClient = new HttpClient(messageHandler, false); + var client = new OpenAIClient(new ApiKeyCredential("key"), ClientCore.GetOpenAIClientOptions(httpClient)); + var builder = Kernel.CreateBuilder(); + builder.Services.AddSingleton(client); + builder.Services.AddSingleton(NullLoggerFactory.Instance); + bool telemetryConfigured = false; + void configureTelemetry(OpenTelemetryImageGenerator _) => telemetryConfigured = true; + string sourceName = $"OpenAIImages.{Guid.NewGuid():N}"; + Activity? imageActivity = null; + using var listener = new ActivityListener + { + ShouldListenTo = source => source.Name == sourceName, + Sample = (ref ActivityCreationOptions _) => ActivitySamplingResult.AllData, + ActivityStopped = activity => imageActivity = activity + }; + ActivitySource.AddActivityListener(listener); + + // Act + IServiceCollection collection = type switch + { + InitializationType.ApiKey => builder.Services.AddOpenAIImageGenerator("gpt-image-1", "api-key", organizationId: "organization", serviceId: serviceId, httpClient: httpClient, openTelemetrySourceName: sourceName, openTelemetryConfig: configureTelemetry), + InitializationType.ClientInline => builder.Services.AddOpenAIImageGenerator("gpt-image-1", client, serviceId, sourceName, configureTelemetry), + InitializationType.ClientInServiceProvider => builder.Services.AddOpenAIImageGenerator("gpt-image-1", serviceId: serviceId, openTelemetrySourceName: sourceName, openTelemetryConfig: configureTelemetry), + _ => throw new ArgumentOutOfRangeException(nameof(type)) + }; + using var provider = collection.BuildServiceProvider(); + var service = provider.GetRequiredKeyedService(serviceId); + var response = await service.GenerateImagesAsync("A sea otter", new Microsoft.Extensions.AI.ImageGenerationOptions + { + ImageSize = new System.Drawing.Size(1024, 1024) + }); + + // Assert + Assert.Same(builder.Services, collection); + Assert.Equal("gpt-image-1", service.GetService()!.DefaultModelId); + Assert.Same(service, provider.GetRequiredKeyedService(serviceId)); + var image = Assert.IsType(Assert.Single(response.Contents)); + Assert.Equal(Convert.FromBase64String("iVBORw0KGgoAAA=="), image.Data.ToArray()); + Assert.Equal("image/png", image.MediaType); + Assert.True(telemetryConfigured); + Assert.NotNull(imageActivity); + Assert.Equal(sourceName, imageActivity.Source.Name); + + using var request = JsonDocument.Parse(messageHandler.RequestContent!); + Assert.Equal("gpt-image-1", request.RootElement.GetProperty("model").GetString()); + Assert.Equal("A sea otter", request.RootElement.GetProperty("prompt").GetString()); + Assert.Equal("1024x1024", request.RootElement.GetProperty("size").GetString()); + Assert.Equal(1, request.RootElement.GetProperty("n").GetInt32()); + if (type == InitializationType.ApiKey) + { + Assert.Equal("organization", Assert.Single(messageHandler.RequestHeaders!.GetValues("OpenAI-Organization"))); + } + } + + [Fact] + public async Task ImageGeneratorCanEditImagesAsync() + { + // Arrange + using var responseMessage = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent("""{"created":1726234481,"output_format":"jpeg","data":[{"b64_json":"AQID"}]}""") + }; + using var messageHandler = new HttpMessageHandlerStub { ResponseToReturn = responseMessage }; + using var httpClient = new HttpClient(messageHandler, false); + using var provider = new ServiceCollection() + .AddOpenAIImageGenerator("gpt-image-1", "api-key", httpClient: httpClient) + .BuildServiceProvider(); + var service = provider.GetRequiredService(); + + // Act + var response = await service.EditImageAsync( + new DataContent(new byte[] { 1, 2, 3 }, "image/png") { Name = "input.png" }, + "Make the background red", + new Microsoft.Extensions.AI.ImageGenerationOptions + { + MediaType = "image/jpeg", + RawRepresentationFactory = _ => new ImageEditOptions { Quality = GeneratedImageQuality.LowQuality } + }); + + // Assert + var image = Assert.IsType(Assert.Single(response.Contents)); + Assert.Equal(new byte[] { 1, 2, 3 }, image.Data.ToArray()); + Assert.Equal("image/jpeg", image.MediaType); + Assert.EndsWith("/images/edits", messageHandler.RequestUri!.AbsolutePath); + Assert.Equal("multipart/form-data", messageHandler.ContentHeaders!.ContentType!.MediaType); + var request = Encoding.UTF8.GetString(messageHandler.RequestContent!); + Assert.Contains("input.png", request); + Assert.Contains("Make the background red", request); + Assert.Matches("name=\"?quality\"?\\r\\n(?:[^\\r\\n]+\\r\\n)*\\r\\nlow\\r\\n", request); + Assert.Matches("name=\"?output_format\"?\\r\\n(?:[^\\r\\n]+\\r\\n)*\\r\\njpeg\\r\\n", request); + } + + [Fact] + public void ImageGeneratorRequiresRegisteredOpenAIClient() + { + // Arrange + using var provider = new ServiceCollection() + .AddOpenAIImageGenerator("gpt-image-1") + .BuildServiceProvider(); + + // Act & Assert + Assert.Throws(() => provider.GetRequiredService()); + } + + #endregion + public enum InitializationType { ApiKey, diff --git a/dotnet/src/Connectors/Connectors.OpenAI.UnitTests/Services/OpenAITextToImageServiceTests.cs b/dotnet/src/Connectors/Connectors.OpenAI.UnitTests/Services/OpenAITextToImageServiceTests.cs index bd40dfe997a3..0380f6a097e6 100644 --- a/dotnet/src/Connectors/Connectors.OpenAI.UnitTests/Services/OpenAITextToImageServiceTests.cs +++ b/dotnet/src/Connectors/Connectors.OpenAI.UnitTests/Services/OpenAITextToImageServiceTests.cs @@ -2,10 +2,14 @@ using System; using System.IO; +using System.Net; using System.Net.Http; using System.Text; +using System.Text.Json; +using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; +using Microsoft.SemanticKernel; using Microsoft.SemanticKernel.Connectors.OpenAI; using Microsoft.SemanticKernel.Services; using Microsoft.SemanticKernel.TextToImage; @@ -117,6 +121,9 @@ public async Task GetUriImageContentsResponseFormatRequestWorksCorrectlyAsync(st [InlineData("hd", "hd")] [InlineData("high", "hd")] [InlineData("standard", "standard")] + [InlineData("medium", "medium")] + [InlineData("low", "low")] + [InlineData("auto", "auto")] public async Task GetUriImageContentsImageQualityRequestWorksCorrectlyAsync(string? quality, string? expectedQuality) { // Arrange @@ -252,6 +259,101 @@ public async Task GetUrlImageContentsResponseWorksCorrectlyAsync() Assert.Equal("my prompt", breakingGlass!.RevisedPrompt); } + [Fact] + public async Task GenerateGptImageReturnsDataUriAsync() + { + // Arrange + using var responseMessage = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(File.ReadAllText("./TestData/text-to-image-b64_json-format-response.json")) + }; + this._messageHandlerStub.ResponseToReturn = responseMessage; + var sut = new OpenAITextToImageService("api-key", httpClient: this._httpClient); + + // Act + var result = await sut.GenerateImageAsync("A sea otter", 1024, 1024); + + // Assert + Assert.Equal("data:image/png;base64,iVBORw0KGgoAAA==", result); + } + + [Fact] + public async Task GetGptImageContentsPreservesSettingsAsync() + { + // Arrange + using var responseMessage = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(File.ReadAllText("./TestData/text-to-image-b64_json-format-response.json")) + }; + this._messageHandlerStub.ResponseToReturn = responseMessage; + var sut = new OpenAITextToImageService("api-key", httpClient: this._httpClient); + + // Act + var result = await sut.GetImageContentsAsync("A sea otter", new OpenAITextToImageExecutionSettings + { + Size = (1536, 1024), + Quality = "low", + EndUserId = "test-user" + }); + + // Assert + var image = Assert.Single(result); + Assert.Equal(Convert.FromBase64String("iVBORw0KGgoAAA=="), image.Data!.Value.ToArray()); + Assert.Equal("image/png", image.MimeType); + Assert.Equal("my prompt", Assert.IsType(image.InnerContent).RevisedPrompt); + using var request = JsonDocument.Parse(this._messageHandlerStub.RequestContent!); + Assert.Equal("gpt-image-1", request.RootElement.GetProperty("model").GetString()); + Assert.Equal("A sea otter", request.RootElement.GetProperty("prompt").GetString()); + Assert.Equal("1536x1024", request.RootElement.GetProperty("size").GetString()); + Assert.Equal("low", request.RootElement.GetProperty("quality").GetString()); + Assert.Equal("test-user", request.RootElement.GetProperty("user").GetString()); + Assert.Equal(1, request.RootElement.GetProperty("n").GetInt32()); + Assert.False(request.RootElement.TryGetProperty("response_format", out _)); + } + + [Fact] + public async Task GetImageContentsPreservesHttpErrorsAsync() + { + // Arrange + using var responseMessage = new HttpResponseMessage(HttpStatusCode.BadRequest) + { + Content = new StringContent("""{"error":{"message":"Invalid image request","type":"invalid_request_error","code":"invalid_parameter"}}""") + }; + this._messageHandlerStub.ResponseToReturn = responseMessage; + var sut = new OpenAITextToImageService("api-key", httpClient: this._httpClient); + + // Act & Assert + var exception = await Assert.ThrowsAsync(() => sut.GetImageContentsAsync("A sea otter")); + Assert.Equal(HttpStatusCode.BadRequest, exception.StatusCode); + Assert.Contains("Invalid image request", exception.Message); + } + + [Fact] + public async Task GetImageContentsPropagatesCancellationAsync() + { + // Arrange + var sut = new OpenAITextToImageService("api-key", httpClient: this._httpClient); + var cancellationToken = new CancellationToken(canceled: true); + + // Act & Assert + await Assert.ThrowsAnyAsync(() => sut.GetImageContentsAsync("A sea otter", cancellationToken: cancellationToken)); + } + + [Fact] + public async Task GetImageContentsRejectsEmptyResponsesAsync() + { + // Arrange + using var responseMessage = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent("""{"created":1726234481,"data":[]}""") + }; + this._messageHandlerStub.ResponseToReturn = responseMessage; + var sut = new OpenAITextToImageService("api-key", httpClient: this._httpClient); + + // Act & Assert + await Assert.ThrowsAsync(() => sut.GetImageContentsAsync("A sea otter")); + } + public void Dispose() { this._httpClient.Dispose(); diff --git a/dotnet/src/Connectors/Connectors.OpenAI/Core/ClientCore.TextToImage.cs b/dotnet/src/Connectors/Connectors.OpenAI/Core/ClientCore.TextToImage.cs index 049f31b8c883..e9f89135e723 100644 --- a/dotnet/src/Connectors/Connectors.OpenAI/Core/ClientCore.TextToImage.cs +++ b/dotnet/src/Connectors/Connectors.OpenAI/Core/ClientCore.TextToImage.cs @@ -1,11 +1,14 @@ // Copyright (c) Microsoft. All rights reserved. using System; -using System.ClientModel; using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; +using Microsoft.Extensions.AI; using OpenAI.Images; +using OpenAIImageGenerationOptions = OpenAI.Images.ImageGenerationOptions; + +#pragma warning disable MEAI001 // Image generation APIs are experimental. namespace Microsoft.SemanticKernel.Connectors.OpenAI; @@ -14,51 +17,6 @@ namespace Microsoft.SemanticKernel.Connectors.OpenAI; /// internal partial class ClientCore { - /// - /// Generates an image with the provided configuration. - /// - /// Model identifier - /// Prompt to generate the image - /// Width of the image - /// Height of the image - /// The to monitor for cancellation requests. The default is . - /// Url of the generated image - internal async Task GenerateImageAsync( - string? targetModel, - string prompt, - int width, - int height, - CancellationToken cancellationToken) - { - Verify.NotNullOrWhiteSpace(prompt); - - var size = new GeneratedImageSize(width, height); - - var imageOptions = new ImageGenerationOptions() - { - Size = size, - }; - - // The model is not required by the OpenAI API and defaults to the DALL-E 2 server-side - https://platform.openai.com/docs/api-reference/images/create#images-create-model. - // However, considering that the model is required by the OpenAI SDK and the ModelId property is optional, it defaults to gpt-image-1 in the line below. - targetModel = string.IsNullOrEmpty(targetModel) ? "gpt-image-1" : targetModel!; - - ClientResult response = await RunRequestAsync(() => this.Client!.GetImageClient(targetModel).GenerateImageAsync(prompt, imageOptions, cancellationToken)).ConfigureAwait(false); - var generatedImage = response.Value; - - if (generatedImage.ImageUri is not null) - { - return generatedImage.ImageUri.ToString(); - } - - if (generatedImage.ImageBytes is not null) - { - return $"data:image/png;base64,{Convert.ToBase64String(generatedImage.ImageBytes.ToArray())}"; - } - - throw new KernelException("The generated image has no valid content."); - } - /// /// Generates an image with the provided configuration. /// @@ -75,13 +33,13 @@ internal async Task> GetImageContentsAsync( Kernel? kernel = null, CancellationToken cancellationToken = default) { - // Ensure the input is valid Verify.NotNull(input); + string? prompt = input.Text; + Verify.NotNull(prompt); - // Convert the generic execution settings to OpenAI-specific settings var imageSettings = OpenAITextToImageExecutionSettings.FromExecutionSettings(executionSettings); - var imageGenerationOptions = new ImageGenerationOptions() + var imageGenerationOptions = new OpenAIImageGenerationOptions() { Size = GetGeneratedImageSize(imageSettings.Size), ResponseFormat = GetResponseFormat(imageSettings.ResponseFormat), @@ -90,17 +48,34 @@ internal async Task> GetImageContentsAsync( EndUserId = imageSettings.EndUserId, }; - ClientResult response = await RunRequestAsync(() => this.Client!.GetImageClient(targetModel).GenerateImageAsync(input.Text, imageGenerationOptions, cancellationToken)).ConfigureAwait(false); - var generatedImage = response.Value; + var options = new Microsoft.Extensions.AI.ImageGenerationOptions + { + Count = 1, + RawRepresentationFactory = _ => imageGenerationOptions + }; - List result = []; - if (generatedImage.ImageUri is not null) + using var generator = this.Client!.GetImageClient(targetModel).AsIImageGenerator(); + var response = await RunRequestAsync(() => generator.GenerateAsync(new ImageGenerationRequest(prompt), options, cancellationToken)).ConfigureAwait(false); + if (response.Contents.Count == 0) { - result.Add(new ImageContent(uri: generatedImage.ImageUri) { InnerContent = generatedImage }); + throw new KernelException("The generated image has no valid content."); } - else + + var generatedImages = response.RawRepresentation as GeneratedImageCollection; + + List result = new(response.Contents.Count); + for (int i = 0; i < response.Contents.Count; i++) { - result.Add(new ImageContent(generatedImage.ImageBytes, "image/png") { InnerContent = generatedImage }); + var content = response.Contents[i]; + ImageContent image = content switch + { + DataContent data => new(data.Data, data.MediaType), + UriContent uri => new(uri.Uri) { MimeType = uri.MediaType }, + _ => throw new KernelException("The generated image has no valid content.") + }; + + image.InnerContent = generatedImages is not null ? generatedImages[i] : content.RawRepresentation; + result.Add(image); } return result; diff --git a/dotnet/src/Connectors/Connectors.OpenAI/Extensions/OpenAIKernelBuilderExtensions.cs b/dotnet/src/Connectors/Connectors.OpenAI/Extensions/OpenAIKernelBuilderExtensions.cs index 1d6f79766974..74400cc768b6 100644 --- a/dotnet/src/Connectors/Connectors.OpenAI/Extensions/OpenAIKernelBuilderExtensions.cs +++ b/dotnet/src/Connectors/Connectors.OpenAI/Extensions/OpenAIKernelBuilderExtensions.cs @@ -435,4 +435,76 @@ OpenAIChatCompletionService Factory(IServiceProvider serviceProvider, object? _) } #endregion + + #region Image Generation + + /// + /// Adds an OpenAI to the . + /// + /// The instance to augment. + /// Model identifier, for example, 'gpt-image-1'. + /// OpenAI API key + /// OpenAI organization ID (optional) + /// A local identifier for the given AI service + /// The HttpClient to use with this service. + /// An optional name for the OpenTelemetry source. + /// An optional callback that can be used to configure the instance. + /// The same instance as . + [Experimental("SKEXP0010")] + public static IKernelBuilder AddOpenAIImageGenerator( + this IKernelBuilder builder, + string modelId, + string apiKey, + string? organizationId = null, + string? serviceId = null, + HttpClient? httpClient = null, + string? openTelemetrySourceName = null, + Action? openTelemetryConfig = null) + { + Verify.NotNull(builder); + + builder.Services.AddOpenAIImageGenerator( + modelId, + apiKey, + organizationId, + serviceId, + httpClient, + openTelemetrySourceName, + openTelemetryConfig); + + return builder; + } + + /// + /// Adds an OpenAI to the . + /// + /// The instance to augment. + /// Model identifier, for example, 'gpt-image-1'. + /// to use for the service. If null, one must be available in the service provider when this service is resolved. + /// A local identifier for the given AI service + /// An optional name for the OpenTelemetry source. + /// An optional callback that can be used to configure the instance. + /// The same instance as . + [Experimental("SKEXP0010")] + public static IKernelBuilder AddOpenAIImageGenerator( + this IKernelBuilder builder, + string modelId, + OpenAIClient? openAIClient = null, + string? serviceId = null, + string? openTelemetrySourceName = null, + Action? openTelemetryConfig = null) + { + Verify.NotNull(builder); + + builder.Services.AddOpenAIImageGenerator( + modelId, + openAIClient, + serviceId, + openTelemetrySourceName, + openTelemetryConfig); + + return builder; + } + + #endregion } diff --git a/dotnet/src/Connectors/Connectors.OpenAI/Extensions/OpenAIServiceCollectionExtensions.DependencyInjection.cs b/dotnet/src/Connectors/Connectors.OpenAI/Extensions/OpenAIServiceCollectionExtensions.DependencyInjection.cs index 4a7adba2ac6a..2d079749f3a7 100644 --- a/dotnet/src/Connectors/Connectors.OpenAI/Extensions/OpenAIServiceCollectionExtensions.DependencyInjection.cs +++ b/dotnet/src/Connectors/Connectors.OpenAI/Extensions/OpenAIServiceCollectionExtensions.DependencyInjection.cs @@ -296,4 +296,99 @@ public static IServiceCollection AddOpenAIEmbeddingGenerator(this IServiceCollec }); } #endregion + + #region Image Generation + + /// + /// Adds the to the . + /// + /// The instance to augment. + /// Model identifier, for example, 'gpt-image-1'. + /// OpenAI API key + /// OpenAI organization ID (optional) + /// A local identifier for the given AI service + /// The HttpClient to use with this service. + /// An optional name for the OpenTelemetry source. + /// An optional callback that can be used to configure the instance. + /// The same instance as . + [Experimental("SKEXP0010")] + public static IServiceCollection AddOpenAIImageGenerator( + this IServiceCollection services, + string modelId, + string apiKey, + string? organizationId = null, + string? serviceId = null, + HttpClient? httpClient = null, + string? openTelemetrySourceName = null, + Action? openTelemetryConfig = null) + { + Verify.NotNull(services); + Verify.NotNullOrWhiteSpace(modelId); + Verify.NotNullOrWhiteSpace(apiKey); + + return services.AddKeyedSingleton(serviceId, (serviceProvider, _) => + { + var loggerFactory = serviceProvider.GetService(); + + var builder = new OpenAIClient( + credential: new ApiKeyCredential(apiKey), + options: ClientCore.GetOpenAIClientOptions( + httpClient: HttpClientProvider.GetHttpClient(httpClient, serviceProvider), + orgId: organizationId)) + .GetImageClient(modelId) + .AsIImageGenerator() + .AsBuilder() + .UseOpenTelemetry(loggerFactory, openTelemetrySourceName, openTelemetryConfig); + + if (loggerFactory is not null) + { + builder.UseLogging(loggerFactory); + } + + return builder.Build(); + }); + } + + /// + /// Adds the to the . + /// + /// The instance to augment. + /// Model identifier, for example, 'gpt-image-1'. + /// to use for the service. If null, one must be available in the service provider when this service is resolved. + /// A local identifier for the given AI service + /// An optional name for the OpenTelemetry source. + /// An optional callback that can be used to configure the instance. + /// The same instance as . + [Experimental("SKEXP0010")] + public static IServiceCollection AddOpenAIImageGenerator( + this IServiceCollection services, + string modelId, + OpenAIClient? openAIClient = null, + string? serviceId = null, + string? openTelemetrySourceName = null, + Action? openTelemetryConfig = null) + { + Verify.NotNull(services); + Verify.NotNullOrWhiteSpace(modelId); + + return services.AddKeyedSingleton(serviceId, (serviceProvider, _) => + { + var loggerFactory = serviceProvider.GetService(); + var client = openAIClient ?? serviceProvider.GetRequiredService(); + + var builder = client.GetImageClient(modelId) + .AsIImageGenerator() + .AsBuilder() + .UseOpenTelemetry(loggerFactory, openTelemetrySourceName, openTelemetryConfig); + + if (loggerFactory is not null) + { + builder.UseLogging(loggerFactory); + } + + return builder.Build(); + }); + } + + #endregion } diff --git a/dotnet/src/Connectors/Connectors.OpenAI/Services/OpenAITextToImageService.cs b/dotnet/src/Connectors/Connectors.OpenAI/Services/OpenAITextToImageService.cs index 9e3dce140102..6e4338e2aef5 100644 --- a/dotnet/src/Connectors/Connectors.OpenAI/Services/OpenAITextToImageService.cs +++ b/dotnet/src/Connectors/Connectors.OpenAI/Services/OpenAITextToImageService.cs @@ -13,6 +13,10 @@ namespace Microsoft.SemanticKernel.Connectors.OpenAI; /// /// OpenAI text to image service. /// +/// +/// Image generation delegates to while preserving Semantic Kernel content and settings. +/// New applications can use AddOpenAIImageGenerator to access image generation and editing directly. +/// [Experimental("SKEXP0010")] public class OpenAITextToImageService : ITextToImageService { diff --git a/dotnet/src/SemanticKernel.Abstractions/AI/TextToImage/TextToImageServiceExtensions.cs b/dotnet/src/SemanticKernel.Abstractions/AI/TextToImage/TextToImageServiceExtensions.cs index 1c9153e00f08..11e762c84ff0 100644 --- a/dotnet/src/SemanticKernel.Abstractions/AI/TextToImage/TextToImageServiceExtensions.cs +++ b/dotnet/src/SemanticKernel.Abstractions/AI/TextToImage/TextToImageServiceExtensions.cs @@ -20,7 +20,7 @@ public static class TextToImageServiceExtensions /// Image height in pixels /// The containing services, plugins, and other state for use throughout the operation. /// The to monitor for cancellation requests. The default is . - /// Generated image in base64 format or image URL + /// Generated image as a data URI or an image URL. public static async Task GenerateImageAsync(this ITextToImageService service, string description, int width, @@ -39,6 +39,7 @@ public static async Task GenerateImageAsync(this ITextToImageService ser var result = await service.GetImageContentsAsync(new TextContent(description), executionSettings, kernel, cancellationToken).ConfigureAwait(false); - return result[0].Uri!.ToString(); + var image = result[0]; + return image.Uri?.ToString() ?? image.DataUri ?? throw new KernelException("The generated image has no valid content."); } }