Skip to content

Commit

Permalink
[AdvancedPaste]Improved telemetry (microsoft#33520)
Browse files Browse the repository at this point in the history
<!-- Enter a brief description/summary of your PR here. What does it
fix/what does it change/how was it tested (even manually, if necessary)?
-->
## Summary of the Pull Request

This PR improves advanced paste telemetry. Here's what's changed

- Added `AdvancedPasteClipboardItemClicked` event
- Changed `CustomFormatEvent` to only fire on successful completion and
include the number of tokens used, and the model name

Here are the goals of adding this telemtry:

- `AdvancedPasteClipboardItemClicked` helps us estimate the total number
of user who are using the clipboard history feature, which helps us
prioritize future investments and improvements in PowerToys. (This is
just regular feature usage data).
- `CustomFormatEvent` now includes number of tokens used, and the model
name. We are considering using alternative models to power Advanced
Paste (as we've heard feedback about this), and these different models
could have different token lengths. Understanding the average usage will
help us make good decisions that will benefit the most users. As well,
the model name is hard coded right now, but in the future if we do add
different AI models for completion then this will help us compare their
performance.

<!-- Please review the items on the PR checklist before submitting-->
## PR Checklist

- [X] Communication: I've discussed this with core contributors already.
If work hasn't been agreed, this work might be rejected


<!-- Provide a more detailed description of the PR, other things fixed
or any additional comments/features here -->
## Detailed Description of the Pull Request / Additional comments

The details above are detailed enough since this change is super small. 

<!-- Describe how you validated the behavior. Add automated tests
wherever possible, but list manual validation steps taken as well -->
## Validation Steps Performed

Ensured that PowerToys successfully built (Need help verifying the
events fire off correctly).
  • Loading branch information
craigloewen-msft committed Jun 27, 2024
1 parent 8b8c75b commit 3b7adbe
Show file tree
Hide file tree
Showing 5 changed files with 42 additions and 6 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -78,8 +78,6 @@ private void GenerateCustom()
{
Logger.LogTrace();

PowerToysTelemetry.Log.WriteEvent(new Telemetry.AdvancedPasteGenerateCustomFormatEvent());

VisualStateManager.GoToState(this, "LoadingState", true);
string inputInstructions = InputTxtBox.Text;
ViewModel.SaveQuery(inputInstructions);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -231,6 +231,7 @@ private async void ClipboardHistory_ItemClick(object sender, ItemClickEventArgs
var item = e.ClickedItem as ClipboardItem;
if (item is not null)
{
PowerToysTelemetry.Log.WriteEvent(new Telemetry.AdvancedPasteClipboardItemClicked());
if (!string.IsNullOrEmpty(item.Content))
{
ClipboardHelper.SetClipboardTextContent(item.Content);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@ public AICompletionsResponse(string response, int apiRequestStatus)

private string _openAIKey;

private string _modelName = "gpt-3.5-turbo-instruct";

public bool IsAIEnabled => !string.IsNullOrEmpty(this._openAIKey);

public AICompletionsHelper()
Expand Down Expand Up @@ -69,14 +71,14 @@ public static string LoadOpenAIKey()
return string.Empty;
}

public string GetAICompletion(string systemInstructions, string userMessage)
private Response<Completions> GetAICompletion(string systemInstructions, string userMessage)
{
OpenAIClient azureAIClient = new OpenAIClient(_openAIKey);

var response = azureAIClient.GetCompletions(
new CompletionsOptions()
{
DeploymentName = "gpt-3.5-turbo-instruct",
DeploymentName = _modelName,
Prompts =
{
systemInstructions + "\n\n" + userMessage,
Expand All @@ -90,7 +92,7 @@ public string GetAICompletion(string systemInstructions, string userMessage)
Console.WriteLine("Cut off due to length constraints");
}

return response.Value.Choices[0].Text;
return response;
}

public AICompletionsResponse AIFormatString(string inputInstructions, string inputString)
Expand All @@ -109,10 +111,16 @@ public AICompletionsResponse AIFormatString(string inputInstructions, string inp
";

string aiResponse = null;
Response<Completions> rawAIResponse = null;
int apiRequestStatus = (int)HttpStatusCode.OK;
try
{
aiResponse = this.GetAICompletion(systemInstructions, userMessage);
rawAIResponse = this.GetAICompletion(systemInstructions, userMessage);
aiResponse = rawAIResponse.Value.Choices[0].Text;

int promptTokens = rawAIResponse.Value.Usage.PromptTokens;
int completionTokens = rawAIResponse.Value.Usage.CompletionTokens;
PowerToysTelemetry.Log.WriteEvent(new Telemetry.AdvancedPasteGenerateCustomFormatEvent(promptTokens, completionTokens, _modelName));
}
catch (Azure.RequestFailedException error)
{
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
// Copyright (c) Microsoft Corporation
// The Microsoft Corporation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.

using System.Diagnostics.Tracing;
using Microsoft.PowerToys.Telemetry;
using Microsoft.PowerToys.Telemetry.Events;

namespace AdvancedPaste.Telemetry
{
[EventData]
public class AdvancedPasteClipboardItemClicked : EventBase, IEvent
{
public PartA_PrivTags PartA_PrivTags => PartA_PrivTags.ProductAndServiceUsage;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,19 @@ namespace AdvancedPaste.Telemetry
[EventData]
public class AdvancedPasteGenerateCustomFormatEvent : EventBase, IEvent
{
public int PromptTokens { get; set; }

public int CompletionTokens { get; set; }

public string ModelName { get; set; }

public AdvancedPasteGenerateCustomFormatEvent(int promptTokens, int completionTokens, string modelName)
{
this.PromptTokens = promptTokens;
this.CompletionTokens = completionTokens;
ModelName = modelName;
}

public PartA_PrivTags PartA_PrivTags => PartA_PrivTags.ProductAndServiceUsage;
}
}

0 comments on commit 3b7adbe

Please sign in to comment.