Discourse/discourse-ai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

FEATURE: AI Usage page (#964)

Browse Source 
- Added a new admin interface to track AI usage metrics, including tokens, features, and models.
- Introduced a new route `/admin/plugins/discourse-ai/ai-usage` and supporting API endpoint in `AiUsageController`.
- Implemented `AiUsageSerializer` for structuring AI usage data.
- Integrated CSS stylings for charts and tables under `stylesheets/modules/llms/common/usage.scss`.
- Enhanced backend with `AiApiAuditLog` model changes: added `cached_tokens` column  (implemented with OpenAI for now) with relevant DB migration and indexing.
- Created `Report` module for efficient aggregation and filtering of AI usage metrics.
- Updated AI Bot title generation logic to log correctly to user vs bot
- Extended test coverage for the new tracking features, ensuring data consistency and access controls.
This commit is contained in:
Sam
2024-11-29 06:26:48 +11:00
committed by GitHub
parent c980c34d77
commit bc0657f478
22 changed files with 1108 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
admin/assets/javascripts/discourse/routes/admin-plugins-show-discourse-ai-usage.js
+11
View File
@@ -0,0 +1,11 @@
import { service } from "@ember/service";
import { ajax } from "discourse/lib/ajax";
import DiscourseRoute from "discourse/routes/discourse";
export default class DiscourseAiUsageRoute extends DiscourseRoute {
@service store;
model() {
return ajax("/admin/plugins/discourse-ai/ai-usage.json");
}
}
admin/assets/javascripts/discourse/templates/admin-plugins/show/discourse-ai-usage.hbs
+1
View File
@@ -0,0 +1 @@
<AiUsage @model={{this.model}} />
app/controllers/discourse_ai/admin/ai_usage_controller.rb
+27
View File
@@ -0,0 +1,27 @@
# frozen_string_literal: true
module DiscourseAi
module Admin
class AiUsageController < ::Admin::AdminController
requires_plugin "discourse-ai"
def show
render json: AiUsageSerializer.new(create_report, root: false)
end
private
def create_report
report =
DiscourseAi::Completions::Report.new(
start_date: params[:start_date]&.to_date || 30.days.ago,
end_date: params[:end_date]&.to_date || Time.current,
)
report = report.filter_by_feature(params[:feature]) if params[:feature].present?
report = report.filter_by_model(params[:model]) if params[:model].present?
report
end
end
end
end
app/models/ai_api_audit_log.rb
+8
View File
@@ -3,6 +3,7 @@
class AiApiAuditLog < ActiveRecord::Base
belongs_to :post
belongs_to :topic
belongs_to :user
module Provider
OpenAI = 1
@@ -43,3 +44,10 @@ end
# feature_name :string(255)
# language_model :string(255)
# feature_context :jsonb
# cached_tokens :integer
#
# Indexes
#
# index_ai_api_audit_logs_on_created_at_and_feature_name (created_at,feature_name)
# index_ai_api_audit_logs_on_created_at_and_language_model (created_at,language_model)
#
app/serializers/ai_usage_serializer.rb
+69
View File
@@ -0,0 +1,69 @@
# frozen_string_literal: true
class AiUsageSerializer < ApplicationSerializer
attributes :data, :features, :models, :users, :summary, :period
def data
object.tokens_by_period.as_json(
only: %i[period total_tokens total_cached_tokens total_request_tokens total_response_tokens],
)
end
def period
object.guess_period
end
def features
object.feature_breakdown.as_json(
only: %i[
feature_name
usage_count
total_tokens
total_cached_tokens
total_request_tokens
total_response_tokens
],
)
end
def models
object.model_breakdown.as_json(
only: %i[
llm
usage_count
total_tokens
total_cached_tokens
total_request_tokens
total_response_tokens
],
)
end
def users
object.user_breakdown.map do |user|
{
avatar_template: User.avatar_template(user.username, user.uploaded_avatar_id),
username: user.username,
usage_count: user.usage_count,
total_tokens: user.total_tokens,
total_cached_tokens: user.total_cached_tokens,
total_request_tokens: user.total_request_tokens,
total_response_tokens: user.total_response_tokens,
}
end
end
def summary
{
total_tokens: object.total_tokens,
total_cached_tokens: object.total_cached_tokens,
total_request_tokens: object.total_request_tokens,
total_response_tokens: object.total_response_tokens,
total_requests: object.total_requests,
date_range: {
start: object.start_date,
end: object.end_date,
},
}
end
end
assets/javascripts/discourse/admin-discourse-ai-plugin-route-map.js
+1
View File
@@ -18,5 +18,6 @@ export default {
this.route("new");
this.route("show", { path: "/:id" });
});
this.route("discourse-ai-usage", { path: "ai-usage" });
},
};
assets/javascripts/discourse/components/ai-usage.gjs
+484
View File
File diff suppressed because it is too large Load Diff
assets/javascripts/initializers/admin-plugin-configuration-nav.js
+4
View File
@@ -24,6 +24,10 @@ export default {
label: "discourse_ai.tools.short_title",
route: "adminPlugins.show.discourse-ai-tools",
},
{
label: "discourse_ai.usage.short_title",
route: "adminPlugins.show.discourse-ai-usage",
},
]);
});
},
assets/stylesheets/modules/llms/common/usage.scss
+168
View File
@@ -0,0 +1,168 @@
.ai-usage {
--chart-response-color: rgba(75, 192, 192, 0.8);
--chart-request-color: rgba(153, 102, 255, 0.8);
--chart-cached-color: rgba(153, 102, 255, 0.4);
padding: 1em;
&__filters-dates {
display: flex;
flex-direction: column;
gap: 1em;
margin-bottom: 1em;
}
&__period-buttons {
display: flex;
gap: 0.5em;
align-items: center;
.btn {
padding: 0.5em 1em;
&.btn-primary {
background: var(--tertiary);
color: var(--secondary);
}
}
}
&__custom-date-pickers {
display: flex;
gap: 1em;
align-items: center;
margin-top: 0.5em;
}
&__filters {
margin-bottom: 2em;
}
&__filters-period {
display: flex;
align-items: center;
gap: 1em;
}
.d-date-time-input-range {
display: flex;
gap: 1em;
align-items: center;
}
.d-date-time-input-range .from {
margin: 0;
}
&__period-label {
font-weight: bold;
}
&__summary {
margin: 2em 0;
padding: 1.5em;
background: var(--primary-very-low);
border-radius: 0.5em;
}
&__summary-title {
margin-bottom: 1em;
color: var(--primary);
font-size: 1.2em;
}
&__summary-stats {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
gap: 1em;
}
&__summary-stat {
display: flex;
flex-direction: column;
padding: 1em;
background: var(--secondary);
border-radius: 0.25em;
.label {
color: var(--primary-medium);
font-size: 0.875em;
margin-bottom: 0.5em;
}
.value {
color: var(--primary);
font-size: 1.5em;
font-weight: bold;
}
}
&__charts {
margin-top: 2em;
}
&__chart {
position: relative;
}
&__chart-container {
margin-bottom: 2em;
}
&__chart-title {
margin-bottom: 1em;
}
&__breakdowns {
display: grid;
grid-template-columns: 1fr 1fr;
gap: 2em;
margin-top: 2em;
@media (max-width: 768px) {
grid-template-columns: 1fr;
}
}
&__features,
&__users,
&__models {
background: var(--primary-very-low);
padding: 1em;
border-radius: 0.5em;
}
&__features-title,
&__users-title,
&__models-title {
margin-bottom: 1em;
}
&__features-table,
&__users-table,
&__models-table {
width: 100%;
border-collapse: collapse;
th {
text-align: left;
padding: 0.5em;
border-bottom: 2px solid var(--primary-low);
}
}
&__features-row,
&__users-row,
&__models-row {
&:hover {
background: var(--primary-low);
}
}
&__features-cell,
&__users-cell,
&__models-cell {
padding: 0.5em;
border-bottom: 1px solid var(--primary-low);
}
}
config/locales/client.en.yml
+20
View File
@@ -126,6 +126,26 @@ en:
modals:
select_option: "Select an option..."
usage:
short_title: "Usage"
summary: "Summary"
total_tokens: "Total tokens"
tokens_over_time: "Tokens over time"
features_breakdown: "Usage per feature"
feature: "Feature"
usage_count: "Usage count"
model: "Model"
models_breakdown: "Usage per model"
users_breakdown: "Usage per user"
all_features: "All features"
all_models: "All models"
username: "Username"
total_requests: "Total requests"
request_tokens: "Request tokens"
response_tokens: "Response tokens"
cached_tokens: "Cached tokens"
ai_persona:
tool_strategies:
all: "Apply to all replies"
config/routes.rb
+2
View File
@@ -77,6 +77,8 @@ Discourse::Application.routes.draw do
get "/rag-document-fragments/files/status",
to: "discourse_ai/admin/rag_document_fragments#indexing_status_check"
get "/ai-usage", to: "discourse_ai/admin/ai_usage#show"
resources :ai_llms,
only: %i[index create show update destroy],
path: "ai-llms",
db/migrate/20241128010221_add_cached_tokens_to_ai_api_audit_log.rb
+9
View File
@@ -0,0 +1,9 @@
# frozen_string_literal: true
class AddCachedTokensToAiApiAuditLog < ActiveRecord::Migration[7.2]
def change
add_column :ai_api_audit_logs, :cached_tokens, :integer
add_index :ai_api_audit_logs, %i[created_at feature_name]
add_index :ai_api_audit_logs, %i[created_at language_model]
end
end
lib/ai_bot/bot.rb
+2 -2
View File
@@ -22,7 +22,7 @@ module DiscourseAi
attr_reader :bot_user
attr_accessor :persona
def get_updated_title(conversation_context, post)
def get_updated_title(conversation_context, post, user)
system_insts = <<~TEXT.strip
You are titlebot. Given a conversation, you will suggest a title.
@@ -61,7 +61,7 @@ module DiscourseAi
DiscourseAi::Completions::Llm
.proxy(model)
.generate(title_prompt, user: post.user, feature_name: "bot_title")
.generate(title_prompt, user: user, feature_name: "bot_title")
.strip
.split("\n")
.last
lib/ai_bot/playground.rb
+3 -3
View File
@@ -245,11 +245,11 @@ module DiscourseAi
builder.to_a
end
def title_playground(post)
def title_playground(post, user)
context = conversation_context(post)
bot
.get_updated_title(context, post)
.get_updated_title(context, post, user)
.tap do |new_title|
PostRevisor.new(post.topic.first_post, post.topic).revise!(
bot.bot_user,
@@ -544,7 +544,7 @@ module DiscourseAi
post_streamer&.finish(skip_callback: true)
publish_final_update(reply_post) if stream_reply
if reply_post && post.post_number == 1 && post.topic.private_message?
title_playground(reply_post)
title_playground(reply_post, post.user)
end
end
lib/completions/endpoints/open_ai.rb
+1
View File
@@ -97,6 +97,7 @@ module DiscourseAi
def final_log_update(log)
log.request_tokens = processor.prompt_tokens if processor.prompt_tokens
log.response_tokens = processor.completion_tokens if processor.completion_tokens
log.cached_tokens = processor.cached_tokens if processor.cached_tokens
end
def decode(response_raw)
lib/completions/open_ai_message_processor.rb
+3 -1
View File
@@ -1,13 +1,14 @@
# frozen_string_literal: true
module DiscourseAi::Completions
class OpenAiMessageProcessor
attr_reader :prompt_tokens, :completion_tokens
attr_reader :prompt_tokens, :completion_tokens, :cached_tokens
def initialize(partial_tool_calls: false)
@tool = nil
@tool_arguments = +""
@prompt_tokens = nil
@completion_tokens = nil
@cached_tokens = nil
@partial_tool_calls = partial_tool_calls
end
@@ -121,6 +122,7 @@ module DiscourseAi::Completions
def update_usage(json)
@prompt_tokens ||= json.dig(:usage, :prompt_tokens)
@completion_tokens ||= json.dig(:usage, :completion_tokens)
@cached_tokens ||= json.dig(:usage, :prompt_tokens_details, :cached_tokens)
end
end
end
lib/completions/report.rb
+148
View File
@@ -0,0 +1,148 @@
# frozen_string_literal: true
module DiscourseAi
module Completions
class Report
UNKNOWN_FEATURE = "unknown"
USER_LIMIT = 50
attr_reader :start_date, :end_date, :base_query
def initialize(start_date: 30.days.ago, end_date: Time.current)
@start_date = start_date.beginning_of_day
@end_date = end_date.end_of_day
@base_query = AiApiAuditLog.where(created_at: @start_date..@end_date)
end
def total_tokens
stats.total_tokens
end
def total_cached_tokens
stats.total_cached_tokens
end
def total_request_tokens
stats.total_request_tokens
end
def total_response_tokens
stats.total_response_tokens
end
def total_requests
stats.total_requests
end
def stats
@stats ||=
base_query.select(
"COUNT(*) as total_requests",
"SUM(request_tokens + response_tokens) as total_tokens",
"SUM(COALESCE(cached_tokens,0)) as total_cached_tokens",
"SUM(request_tokens) as total_request_tokens",
"SUM(response_tokens) as total_response_tokens",
)[
0
]
end
def guess_period(period = nil)
period = nil if %i[day month hour].include?(period)
period ||
case @end_date - @start_date
when 0..3.days
:hour
when 3.days..90.days
:day
else
:month
end
end
def tokens_by_period(period = nil)
period = guess_period(period)
base_query
.group("DATE_TRUNC('#{period}', created_at)")
.order("DATE_TRUNC('#{period}', created_at)")
.select(
"DATE_TRUNC('#{period}', created_at) as period",
"SUM(request_tokens + response_tokens) as total_tokens",
"SUM(COALESCE(cached_tokens,0)) as total_cached_tokens",
"SUM(request_tokens) as total_request_tokens",
"SUM(response_tokens) as total_response_tokens",
)
end
def user_breakdown
base_query
.joins(:user)
.group(:user_id, "users.username", "users.uploaded_avatar_id")
.order("usage_count DESC")
.limit(USER_LIMIT)
.select(
"users.username",
"users.uploaded_avatar_id",
"COUNT(*) as usage_count",
"SUM(request_tokens + response_tokens) as total_tokens",
"SUM(COALESCE(cached_tokens,0)) as total_cached_tokens",
"SUM(request_tokens) as total_request_tokens",
"SUM(response_tokens) as total_response_tokens",
)
end
def feature_breakdown
base_query
.group(:feature_name)
.order("usage_count DESC")
.select(
"case when coalesce(feature_name, '') = '' then '#{UNKNOWN_FEATURE}' else feature_name end as feature_name",
"COUNT(*) as usage_count",
"SUM(request_tokens + response_tokens) as total_tokens",
"SUM(COALESCE(cached_tokens,0)) as total_cached_tokens",
"SUM(request_tokens) as total_request_tokens",
"SUM(response_tokens) as total_response_tokens",
)
end
def model_breakdown
base_query
.group(:language_model)
.order("usage_count DESC")
.select(
"language_model as llm",
"COUNT(*) as usage_count",
"SUM(request_tokens + response_tokens) as total_tokens",
"SUM(COALESCE(cached_tokens,0)) as total_cached_tokens",
"SUM(request_tokens) as total_request_tokens",
"SUM(response_tokens) as total_response_tokens",
)
end
def tokens_per_hour
tokens_by_period(:hour)
end
def tokens_per_day
tokens_by_period(:day)
end
def tokens_per_month
tokens_by_period(:month)
end
def filter_by_feature(feature_name)
if feature_name == UNKNOWN_FEATURE
@base_query = base_query.where("coalesce(feature_name, '') = ''")
else
@base_query = base_query.where(feature_name: feature_name)
end
self
end
def filter_by_model(model_name)
@base_query = base_query.where(language_model: model_name)
self
end
end
end
end
plugin.rb
+2
View File
@@ -37,6 +37,8 @@ register_asset "stylesheets/modules/sentiment/common/dashboard.scss"
register_asset "stylesheets/modules/llms/common/ai-llms-editor.scss"
register_asset "stylesheets/modules/llms/common/usage.scss"
register_asset "stylesheets/modules/ai-bot/common/ai-tools.scss"
register_asset "stylesheets/modules/ai-bot/common/ai-artifact.scss"
spec/fixtures/bot/openai_artifact_call.txt
Vendored
+1 -1
View File
@@ -292,7 +292,7 @@ data: {"id":"chatcmpl-ATimVYagKnCWQ0VXY0Hn2SDjRuN6B","object":"chat.completion.c
data: {"id":"chatcmpl-ATimVYagKnCWQ0VXY0Hn2SDjRuN6B","object":"chat.completion.chunk","created":1731647015,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_45cf54deae","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"tool_calls"}],"usage":null}
data: {"id":"chatcmpl-ATimVYagKnCWQ0VXY0Hn2SDjRuN6B","object":"chat.completion.chunk","created":1731647015,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_45cf54deae","choices":[],"usage":{"prompt_tokens":735,"completion_tokens":156,"total_tokens":891,"prompt_tokens_details":{"cached_tokens":0,"audio_tokens":0},"completion_tokens_details":{"reasoning_tokens":0,"audio_tokens":0,"accepted_prediction_tokens":0,"rejected_prediction_tokens":0}}}
data: {"id":"chatcmpl-ATimVYagKnCWQ0VXY0Hn2SDjRuN6B","object":"chat.completion.chunk","created":1731647015,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_45cf54deae","choices":[],"usage":{"prompt_tokens":735,"completion_tokens":156,"total_tokens":891,"prompt_tokens_details":{"cached_tokens":33,"audio_tokens":0},"completion_tokens_details":{"reasoning_tokens":0,"audio_tokens":0,"accepted_prediction_tokens":0,"rejected_prediction_tokens":0}}}
data: [DONE]
spec/lib/completions/endpoints/open_ai_spec.rb
+3
View File
@@ -657,6 +657,9 @@ TEXT
end
end
end
audit_log = AiApiAuditLog.order("id desc").first
expect(audit_log.cached_tokens).to eq(33)
end
it "properly handles spaces in tools payload and partial tool calls" do
spec/lib/modules/ai_bot/playground_spec.rb
+1 -2
View File
@@ -829,8 +829,7 @@ RSpec.describe DiscourseAi::AiBot::Playground do
it "updates the title using bot suggestions" do
DiscourseAi::Completions::Llm.with_prepared_responses([expected_response]) do
playground.title_playground(third_post)
playground.title_playground(third_post, user)
expect(pm.reload.title).to eq(expected_response)
end
end
spec/requests/admin/ai_usage_controller_spec.rb
+140
View File
@@ -0,0 +1,140 @@
# frozen_string_literal: true
require "rails_helper"
RSpec.describe DiscourseAi::Admin::AiUsageController do
fab!(:admin)
fab!(:user)
let(:usage_path) { "/admin/plugins/discourse-ai/ai-usage.json" }
before { SiteSetting.discourse_ai_enabled = true }
context "when logged in as admin" do
before { sign_in(admin) }
describe "#show" do
fab!(:log1) do
AiApiAuditLog.create!(
provider_id: 1,
feature_name: "summarize",
language_model: "gpt-4",
request_tokens: 100,
response_tokens: 50,
created_at: 1.day.ago,
)
end
fab!(:log2) do
AiApiAuditLog.create!(
provider_id: 1,
feature_name: "translate",
language_model: "gpt-3.5",
request_tokens: 200,
response_tokens: 100,
created_at: 2.days.ago,
)
end
it "returns correct data structure" do
get usage_path
expect(response.status).to eq(200)
json = response.parsed_body
expect(json).to have_key("data")
expect(json).to have_key("features")
expect(json).to have_key("models")
expect(json).to have_key("summary")
end
it "respects date filters" do
get usage_path, params: { start_date: 3.days.ago.to_date, end_date: 1.day.ago.to_date }
json = response.parsed_body
expect(json["summary"]["total_tokens"]).to eq(450) # sum of all tokens
end
it "filters by feature" do
get usage_path, params: { feature: "summarize" }
json = response.parsed_body
features = json["features"]
expect(features.length).to eq(1)
expect(features.first["feature_name"]).to eq("summarize")
expect(features.first["total_tokens"]).to eq(150)
end
it "filters by model" do
get usage_path, params: { model: "gpt-3.5" }
json = response.parsed_body
models = json["models"]
expect(models.length).to eq(1)
expect(models.first["llm"]).to eq("gpt-3.5")
expect(models.first["total_tokens"]).to eq(300)
end
it "handles different period groupings" do
get usage_path, params: { period: "hour" }
expect(response.status).to eq(200)
get usage_path, params: { period: "month" }
expect(response.status).to eq(200)
end
end
# spec/requests/admin/ai_usage_controller_spec.rb
context "with hourly data" do
before do
freeze_time Time.parse("2021-02-01 00:00:00")
# Create data points across different hours
[23.hours.ago, 22.hours.ago, 21.hours.ago, 20.hours.ago].each do |time|
AiApiAuditLog.create!(
provider_id: 1,
feature_name: "summarize",
language_model: "gpt-4",
request_tokens: 100,
response_tokens: 50,
created_at: time,
)
end
end
it "returns hourly data when period is day" do
get usage_path, params: { start_date: 1.day.ago.to_date, end_date: Time.current.to_date }
expect(response.status).to eq(200)
json = response.parsed_body
expect(json["data"].length).to eq(4)
data_by_hour = json["data"].index_by { |d| Time.parse(d["period"]).hour }
expect(data_by_hour.keys.length).to eq(4)
expect(data_by_hour.first[1]["total_tokens"]).to eq(150)
end
end
end
context "when not admin" do
before { sign_in(user) }
it "blocks access" do
get usage_path
expect(response.status).to eq(404)
end
end
context "when plugin disabled" do
before do
SiteSetting.discourse_ai_enabled = false
sign_in(admin)
end
it "returns error" do
get usage_path
expect(response.status).to eq(404)
end
end
end