Adds logging LLM usage information using OpenTelemetry. Closes #1067

dev-proxy-abstractions/LanguageModel/OpenAIModels.cs

@@ -143,3 +143,106 @@ public class OpenAIChatCompletionResponseChoiceMessage
     public string Content { get; set; } = string.Empty;
     public string Role { get; set; } = string.Empty;
 }
+
+public class OpenAIAudioRequest : OpenAIRequest
+{
+    public string File { get; set; } = string.Empty;
+    [JsonPropertyName("response_format")]
+    public string? ResponseFormat { get; set; }
+    public string? Prompt { get; set; }
+    public string? Language { get; set; }
+}
+
+public class OpenAIAudioSpeechRequest : OpenAIRequest
+{
+    public string Input { get; set; } = string.Empty;
+    public string Voice { get; set; } = string.Empty;
+    [JsonPropertyName("response_format")]
+    public string? ResponseFormat { get; set; }
+    public double? Speed { get; set; }
+}
+
+public class OpenAIAudioTranscriptionResponse : OpenAIResponse
+{
+    public string Text { get; set; } = string.Empty;
+    public override string? Response => Text;
+}
+
+public class OpenAIEmbeddingRequest : OpenAIRequest
+{
+    public string? Input { get; set; }
+    [JsonPropertyName("encoding_format")]
+    public string? EncodingFormat { get; set; }
+    public int? Dimensions { get; set; }
+}
+
+public class OpenAIEmbeddingResponse : OpenAIResponse
+{
+    public OpenAIEmbeddingData[]? Data { get; set; }
+    public override string? Response => null; // Embeddings don't have a text response
+}
+
+public class OpenAIEmbeddingData
+{
+    public float[]? Embedding { get; set; }
+    public int Index { get; set; }
+    public string? Object { get; set; }
+}
+
+public class OpenAIFineTuneRequest : OpenAIRequest
+{
+    [JsonPropertyName("training_file")]
+    public string TrainingFile { get; set; } = string.Empty;
+    [JsonPropertyName("validation_file")]
+    public string? ValidationFile { get; set; }
+    public int? Epochs { get; set; }
+    [JsonPropertyName("batch_size")]
+    public int? BatchSize { get; set; }
+    [JsonPropertyName("learning_rate_multiplier")]
+    public double? LearningRateMultiplier { get; set; }
+    public string? Suffix { get; set; }
+}
+
+public class OpenAIFineTuneResponse : OpenAIResponse
+{    
+    [JsonPropertyName("fine_tuned_model")]
+    public string? FineTunedModel { get; set; }   
+    public string Status { get; set; } = string.Empty;    
+    public string? Organization { get; set; }    
+    public long CreatedAt { get; set; }    
+    public long UpdatedAt { get; set; }    
+    [JsonPropertyName("training_file")]
+    public string TrainingFile { get; set; } = string.Empty;
+    [JsonPropertyName("validation_file")]
+    public string? ValidationFile { get; set; }
+    [JsonPropertyName("result_files")]
+    public object[]? ResultFiles { get; set; }
+    public override string? Response => FineTunedModel;
+}
+
+public class OpenAIImageRequest : OpenAIRequest
+{
+    public string Prompt { get; set; } = string.Empty;
+    public int? N { get; set; }
+    public string? Size { get; set; }
+    [JsonPropertyName("response_format")]
+    public string? ResponseFormat { get; set; }
+    public string? User { get; set; }
+    public string? Quality { get; set; }
+    public string? Style { get; set; }
+}
+
+public class OpenAIImageResponse : OpenAIResponse
+{    
+    public OpenAIImageData[]? Data { get; set; }   
+    public override string? Response => null; // Image responses don't have a text response
+}
+
+public class OpenAIImageData
+{
+    public string? Url { get; set; }    
+    [JsonPropertyName("b64_json")]
+    public string? Base64Json { get; set; }    
+    [JsonPropertyName("revised_prompt")]
+    public string? RevisedPrompt { get; set; }
+}

dev-proxy-abstractions/LanguageModel/PricesData.cs

@@ -0,0 +1,62 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System.Diagnostics;
+
+namespace DevProxy.Abstractions.LanguageModel;
+
+public class PricesData: Dictionary<string, ModelPrices>
+{
+    public bool TryGetModelPrices(string modelName, out ModelPrices? prices)
+    {
+        prices = new ModelPrices();
+
+        if (string.IsNullOrEmpty(modelName))
+        {
+            return false;
+        }
+
+        // Try exact match first
+        if (TryGetValue(modelName, out prices))
+        {
+            return true;
+        }
+
+        // Try to find a matching prefix
+        // This handles cases like "gpt-4-turbo-2024-04-09" matching with "gpt-4"
+        var matchingModel = Keys
+            .Where(k => modelName.StartsWith(k, StringComparison.OrdinalIgnoreCase))
+            .OrderByDescending(k => k.Length)
+            .FirstOrDefault();
+
+        if (matchingModel != null && TryGetValue(matchingModel, out prices))
+        {
+            return true;
+        }
+
+        return false;
+    }
+
+    public (double Input, double Output) CalculateCost(string modelName, long inputTokens, long outputTokens)
+    {
+        if (!TryGetModelPrices(modelName, out var prices))
+        {
+            return (0, 0);
+        }
+
+        Debug.Assert(prices != null, "Prices data should not be null here.");
+
+        // Prices in the data are per 1M tokens
+        var inputCost = prices.Input * (inputTokens / 1_000_000.0);
+        var outputCost = prices.Output * (outputTokens / 1_000_000.0);
+
+        return (inputCost, outputCost);
+    }
+}
+
+public class ModelPrices
+{
+    public double Input { get; set; }
+    public double Output { get; set; }
+}

dev-proxy-abstractions/OpenTelemetry/SemanticConvention.cs

@@ -0,0 +1,86 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+namespace DevProxy.Abstractions.OpenTelemetry;
+
+public static class SemanticConvention
+{
+    // GenAI General
+    public const string GEN_AI_ENDPOINT = "gen_ai.endpoint";
+    public const string GEN_AI_SYSTEM = "gen_ai.system";
+    public const string GEN_AI_ENVIRONMENT = "gen_ai.environment";
+    public const string GEN_AI_APPLICATION_NAME = "gen_ai.application_name";
+    public const string GEN_AI_OPERATION = "gen_ai.type";
+    public const string GEN_AI_OPERATION_NAME = "gen_ai.operation.name";
+    public const string GEN_AI_HUB_OWNER = "gen_ai.hub.owner";
+    public const string GEN_AI_HUB_REPO = "gen_ai.hub.repo";
+    public const string GEN_AI_RETRIEVAL_SOURCE = "gen_ai.retrieval.source";
+    public const string GEN_AI_REQUESTS = "gen_ai.total.requests";
+
+    // GenAI Request
+    public const string GEN_AI_REQUEST_MODEL = "gen_ai.request.model";
+    public const string GEN_AI_REQUEST_TEMPERATURE = "gen_ai.request.temperature";
+    public const string GEN_AI_REQUEST_TOP_P = "gen_ai.request.top_p";
+    public const string GEN_AI_REQUEST_TOP_K = "gen_ai.request.top_k";
+    public const string GEN_AI_REQUEST_MAX_TOKENS = "gen_ai.request.max_tokens";
+    public const string GEN_AI_REQUEST_IS_STREAM = "gen_ai.request.is_stream";
+    public const string GEN_AI_REQUEST_USER = "gen_ai.request.user";
+    public const string GEN_AI_REQUEST_SEED = "gen_ai.request.seed";
+    public const string GEN_AI_REQUEST_FREQUENCY_PENALTY = "gen_ai.request.frequency_penalty";
+    public const string GEN_AI_REQUEST_PRESENCE_PENALTY = "gen_ai.request.presence_penalty";
+    public const string GEN_AI_REQUEST_ENCODING_FORMATS = "gen_ai.request.embedding_format";
+    public const string GEN_AI_REQUEST_EMBEDDING_DIMENSION = "gen_ai.request.embedding_dimension";
+    public const string GEN_AI_REQUEST_TOOL_CHOICE = "gen_ai.request.tool_choice";
+    public const string GEN_AI_REQUEST_AUDIO_VOICE = "gen_ai.request.audio_voice";
+    public const string GEN_AI_REQUEST_AUDIO_RESPONSE_FORMAT = "gen_ai.request.audio_response_format";
+    public const string GEN_AI_REQUEST_AUDIO_SPEED = "gen_ai.request.audio_speed";
+    public const string GEN_AI_REQUEST_FINETUNE_STATUS = "gen_ai.request.fine_tune_status";
+    public const string GEN_AI_REQUEST_FINETUNE_MODEL_SUFFIX = "gen_ai.request.fine_tune_model_suffix";
+    public const string GEN_AI_REQUEST_FINETUNE_MODEL_EPOCHS = "gen_ai.request.fine_tune_n_epochs";
+    public const string GEN_AI_REQUEST_FINETUNE_MODEL_LRM = "gen_ai.request.learning_rate_multiplier";
+    public const string GEN_AI_REQUEST_FINETUNE_BATCH_SIZE = "gen_ai.request.fine_tune_batch_size";
+    public const string GEN_AI_REQUEST_VALIDATION_FILE = "gen_ai.request.validation_file";
+    public const string GEN_AI_REQUEST_TRAINING_FILE = "gen_ai.request.training_file";
+    public const string GEN_AI_REQUEST_IMAGE_SIZE = "gen_ai.request.image_size";
+    public const string GEN_AI_REQUEST_IMAGE_QUALITY = "gen_ai.request.image_quality";
+    public const string GEN_AI_REQUEST_IMAGE_STYLE = "gen_ai.request.image_style";
+
+    // GenAI Usage
+    public const string GEN_AI_USAGE_INPUT_TOKENS = "gen_ai.usage.input_tokens";
+    public const string GEN_AI_USAGE_OUTPUT_TOKENS = "gen_ai.usage.output_tokens";
+    // OpenLIT
+    public const string GEN_AI_USAGE_TOTAL_TOKENS = "gen_ai.usage.total_tokens";
+    public const string GEN_AI_USAGE_COST = "gen_ai.usage.cost";
+    public const string GEN_AI_USAGE_TOTAL_COST = "gen_ai.usage.total_cost";
+
+    // GenAI Response
+    public const string GEN_AI_RESPONSE_ID = "gen_ai.response.id";
+    public const string GEN_AI_RESPONSE_MODEL = "gen_ai.response.model";
+    public const string GEN_AI_RESPONSE_FINISH_REASON = "gen_ai.response.finish_reason";
+    public const string GEN_AI_RESPONSE_IMAGE = "gen_ai.response.image";
+    public const string GEN_AI_RESPONSE_IMAGE_SIZE = "gen_ai.request.image_size";
+    public const string GEN_AI_RESPONSE_IMAGE_QUALITY = "gen_ai.request.image_quality"; 
+    public const string GEN_AI_RESPONSE_IMAGE_STYLE = "gen_ai.request.image_style";
+
+    // GenAI Content
+    public const string GEN_AI_CONTENT_PROMPT = "gen_ai.content.prompt";
+    public const string GEN_AI_CONTENT_COMPLETION = "gen_ai.completion";
+    public const string GEN_AI_CONTENT_REVISED_PROMPT = "gen_ai.content.revised_prompt";
+
+    // Operation Types
+    public const string GEN_AI_OPERATION_TYPE_CHAT = "chat";
+    public const string GEN_AI_OPERATION_TYPE_EMBEDDING = "embedding";
+    public const string GEN_AI_OPERATION_TYPE_IMAGE = "image";
+    public const string GEN_AI_OPERATION_TYPE_AUDIO = "audio";
+    public const string GEN_AI_OPERATION_TYPE_FINETUNING = "fine_tuning";
+    public const string GEN_AI_OPERATION_TYPE_VECTORDB = "vectordb";
+    public const string GEN_AI_OPERATION_TYPE_FRAMEWORK = "framework";
+
+    // Metrics
+    public const string GEN_AI_METRIC_CLIENT_TOKEN_USAGE = "gen_ai.client.token.usage";
+    public const string GEN_AI_TOKEN_TYPE = "gen_ai.token.type";
+    public const string GEN_AI_TOKEN_TYPE_INPUT = "input";
+    public const string GEN_AI_TOKEN_TYPE_OUTPUT = "output";
+    public const string GEN_AI_METRIC_CLIENT_OPERATION_DURATION = "gen_ai.client.operation.duration";
+}

dev-proxy-plugins/Inspection/LanguageModelPricingLoader.cs

@@ -0,0 +1,53 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using DevProxy.Abstractions;
+using DevProxy.Abstractions.LanguageModel;
+using Microsoft.Extensions.Logging;
+using System.Text.Json;
+
+namespace DevProxy.Plugins.Inspection;
+
+internal class LanguageModelPricesLoader(ILogger logger, LanguageModelPricesPluginConfiguration configuration, bool validateSchemas) : BaseLoader(logger, validateSchemas)
+{
+    private readonly ILogger _logger = logger ?? throw new ArgumentNullException(nameof(logger));
+    private readonly LanguageModelPricesPluginConfiguration _configuration = configuration ?? throw new ArgumentNullException(nameof(configuration));
+    protected override string FilePath => Path.Combine(Directory.GetCurrentDirectory(), _configuration.PricesFile ?? "");
+
+    protected override void LoadData(string fileContents)
+    {
+        try
+        {
+            // we need to deserialize manually because standard deserialization
+            // doesn't support nested dictionaries
+            using JsonDocument document = JsonDocument.Parse(fileContents);
+
+            if (document.RootElement.TryGetProperty("prices", out JsonElement pricesElement))
+            {
+                var pricesData = new PricesData();
+
+                foreach (JsonProperty modelProperty in pricesElement.EnumerateObject())
+                {
+                    var modelName = modelProperty.Name;
+                    if (modelProperty.Value.TryGetProperty("input", out JsonElement inputElement) &&
+                        modelProperty.Value.TryGetProperty("output", out JsonElement outputElement))
+                    {
+                        pricesData[modelName] = new()
+                        {
+                            Input = inputElement.GetDouble(),
+                            Output = outputElement.GetDouble()
+                        };
+                    }
+                }
+
+                _configuration.Prices = pricesData;
+                _logger.LogInformation("Language model prices data loaded from {PricesFile}", _configuration.PricesFile);
+            }
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(ex, "An error has occurred while reading {PricesFile}:", _configuration.PricesFile);
+        }
+    }
+}