test added

assets/javascripts/discourse/admin/models/ai-tool.js

@@ -32,10 +32,8 @@ export default class AiTool extends RestModel {
 
         if (parameter.enum && parameter.enum.length) {
           parameter.enum = new TrackedArray(parameter.enum);
-          parameter.isEnum = true;
         } else {
           parameter.enum = null;
-          parameter.isEnum = false;
         }
 
         return parameter;

assets/javascripts/discourse/components/ai-tool-editor-form.gjs

@@ -28,13 +28,12 @@ export default class AiToolEditorForm extends Component {
   ];
 
   get formData() {
-    const parameters = this.args.editingModel.parameters || [];
     return {
       name: this.args.editingModel.name || "",
       tool_name: this.args.editingModel.tool_name || "",
       description: this.args.editingModel.description || "",
       summary: this.args.editingModel.summary || "",
-      parameters,
+      parameters: this.args.editingModel.parameters || [],
       script: this.args.editingModel.script || "",
       rag_uploads: this.args.editingModel.rag_uploads || [],
     };
@@ -110,10 +109,6 @@ export default class AiToolEditorForm extends Component {
     return data.parameters[index].type;
   }
 
-  parameterIsEnum(data, index) {
-    return data.parameters[index].isEnum;
-  }
-
   get ragUploadsDescription() {
     return this.siteSettings.rag_images_enabled
       ? i18n("discourse_ai.rag.uploads.description_with_images")
@@ -234,7 +229,7 @@ export default class AiToolEditorForm extends Component {
             </row.Col>
 
             <row.Col @size={{4}}>
-              <collection.Field @name="isEnum" @title="Enum" as |field|>
+              <collection.Field @name="enum" @title="Enum" as |field|>
                 <field.Checkbox />
               </collection.Field>
             </row.Col>
@@ -248,9 +243,6 @@ export default class AiToolEditorForm extends Component {
               />
             </row.Col>
           </form.Row>
-          {{#if (this.parameterIsEnum data index)}}
-            I am an enum
-          {{/if}}
         </div>
       </form.Collection>
 
@@ -260,9 +252,7 @@ export default class AiToolEditorForm extends Component {
         @action={{fn
           form.addItemToCollection
           "parameters"
-          (hash
-            name="" type="string" description="" required=false isEnum=false
-          )
+          (hash name="" type="string" description="" required=false enum=false)
         }}
       />
 
@@ -309,13 +299,6 @@ export default class AiToolEditorForm extends Component {
             class="ai-tool-editor__test-button"
           />
 
-        {{/unless}}
-
-        <form.Submit
-          @label="discourse_ai.tools.save"
-          class="ai-tool-editor__save"
-        />
-        {{#unless @isNew}}
           <form.Button
             @label="discourse_ai.tools.delete"
             @icon="trash-can"
@@ -323,6 +306,11 @@ export default class AiToolEditorForm extends Component {
             class="btn-danger ai-tool-editor__delete"
           />
         {{/unless}}
+
+        <form.Submit
+          @label="discourse_ai.tools.save"
+          class="ai-tool-editor__save"
+        />
       </form.Actions>
     </Form>
   </template>

assets/javascripts/discourse/components/modal/ai-summary-modal.gjs

@@ -1,6 +1,7 @@
 import Component from "@glimmer/component";
 import { tracked } from "@glimmer/tracking";
 import { array } from "@ember/helper";
+import { on } from "@ember/modifier";
 import { action } from "@ember/object";
 import didInsert from "@ember/render-modifiers/modifiers/did-insert";
 import didUpdate from "@ember/render-modifiers/modifiers/did-update";
@@ -29,6 +30,7 @@ export default class AiSummaryModal extends Component {
   @service currentUser;
   @service site;
   @service modal;
+  @service appEvents;
 
   @tracked text = "";
   @tracked summarizedOn = null;
@@ -208,6 +210,18 @@ export default class AiSummaryModal extends Component {
     this.args.closeModal();
   }
 
+  @action
+  onSummaryTextClick(event) {
+    // Check if the clicked element is an anchor tag
+    if (event.target.tagName === "A" && event.target.href) {
+      this.appEvents.trigger("ai_summary_link_clicked", {
+        topic_id: this.args.model.topic.id,
+        user_id: this.currentUser.id,
+        link: event.target.href,
+      });
+    }
+  }
+
   <template>
     <DModal
       @title={{i18n "discourse_ai.summarization.topic.title"}}
@@ -221,7 +235,12 @@ export default class AiSummaryModal extends Component {
     >
       <:body>
         {{htmlClass "scrollable-modal"}}
-        <div class="ai-summary-container" {{didInsert this.generateSummary}}>
+        <div
+          class="ai-summary-container"
+          {{didInsert this.generateSummary}}
+          {{on "click" this.onSummaryTextClick}}
+          role="button"
+        >
           <article
             class={{concatClass
               "ai-summary-box"

evals/lib/eval.rb

@@ -29,6 +29,7 @@ class DiscourseAi::Evals::Eval
     @id = @yaml[:id]
     @description = @yaml[:description]
     @vision = @yaml[:vision]
+    @args = @yaml[:args]&.symbolize_keys
     @type = @yaml[:type]
     @expected_output = @yaml[:expected_output]
     @expected_output_regex = @yaml[:expected_output_regex]
@@ -38,14 +39,10 @@ class DiscourseAi::Evals::Eval
     @expected_tool_call.symbolize_keys! if @expected_tool_call
     @judge = @yaml[:judge]
     @judge.symbolize_keys! if @judge
-    if @yaml[:args].is_a?(Array)
-      @args = @yaml[:args].map(&:symbolize_keys)
-    else
-      @args = @yaml[:args].symbolize_keys
-      @args.each do |key, value|
-        if (key.to_s.include?("_path") || key.to_s == "path") && value.is_a?(String)
-          @args[key] = File.expand_path(File.join(File.dirname(path), value))
-        end
+
+    @args.each do |key, value|
+      if (key.to_s.include?("_path") || key.to_s == "path") && value.is_a?(String)
+        @args[key] = File.expand_path(File.join(File.dirname(path), value))
       end
     end
   end
@@ -60,7 +57,7 @@ class DiscourseAi::Evals::Eval
       when "image_to_text"
         image_to_text(llm, **args)
       when "prompt"
-        DiscourseAi::Evals::PromptEvaluator.new(llm).prompt_call(args)
+        DiscourseAi::Evals::PromptEvaluator.new(llm).prompt_call(**args)
       when "edit_artifact"
         edit_artifact(llm, **args)
       when "summarization"

evals/lib/prompt_evaluator.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+class DiscourseAi::Evals::PromptEvaluator
+  def initialize(llm)
+    @llm = llm.llm_model.to_llm
+  end
+
+  def prompt_call(prompts:, messages: nil, temperature: nil, tools: nil, stream: false)
+    tools = symbolize_tools(tools)
+    total = prompts.size * messages.size
+    count = 0
+    puts ""
+
+    prompts.flat_map do |prompt|
+      messages.map do |content|
+        count += 1
+        print "\rProcessing #{count}/#{total}"
+
+        c_prompt =
+          DiscourseAi::Completions::Prompt.new(prompt, messages: [{ type: :user, content: }])
+        c_prompt.tools = tools if tools
+        result = { prompt:, message: content }
+        result[:result] = generate_result(c_prompt, temperature, stream)
+        result
+      end
+    end
+  ensure
+    print "\r\033[K"
+  end
+
+  private
+
+  def generate_result(c_prompt, temperature, stream)
+    if stream
+      stream_result = []
+      @llm.generate(c_prompt, user: Discourse.system_user, temperature: temperature) do |partial|
+        stream_result << partial
+      end
+      stream_result
+    else
+      @llm.generate(c_prompt, user: Discourse.system_user, temperature: temperature)
+    end
+  end
+
+  def symbolize_tools(tools)
+    return nil if tools.nil?
+
+    tools.map do |tool|
+      { name: tool["name"], parameters: tool["parameters"]&.transform_keys(&:to_sym) }.compact
+    end
+  end
+end

evals/lib/prompts/prompt_evaluator.rb

@@ -1,84 +0,0 @@
-# frozen_string_literal: true
-
-class DiscourseAi::Evals::PromptEvaluator
-  def initialize(llm)
-    @llm = llm.llm_model.to_llm
-  end
-
-  def prompt_call(args)
-    args = [args] if !args.is_a?(Array)
-    runner = DiscourseAi::Evals::PromptSingleTestRunner.new(@llm)
-
-    with_tests_progress(total: args.size) do |bump_progress|
-      args.flat_map do |test|
-        bump_progress.call
-
-        prompts, messages, followups, output_thinking, stream, temperature, tools =
-          symbolize_test_args(test)
-
-        prompts.flat_map do |prompt|
-          messages.map do |message|
-            runner.run_single_test(
-              prompt,
-              message,
-              followups,
-              output_thinking,
-              stream,
-              temperature,
-              tools,
-            )
-          end
-        end
-      end
-    end
-  end
-
-  private
-
-  def symbolize_test_args(args)
-    prompts = args[:prompts] || [args[:prompt]]
-    messages = args[:messages] || [args[:message]]
-    followups = symbolize_followups(args)
-    output_thinking = args[:output_thinking] || false
-    stream = args[:stream] || false
-    temperature = args[:temperature]
-    tools = symbolize_tools(args[:tools])
-    [prompts, messages, followups, output_thinking, stream, temperature, tools]
-  end
-
-  def symbolize_followups(args)
-    return nil if args[:followups].nil? && args[:followup].nil?
-    followups = args[:followups] || [args[:followup]]
-    followups.map do |followup|
-      followup = followup.dup.symbolize_keys!
-      message = followup[:message].dup.symbolize_keys!
-      message[:type] = message[:type].to_sym if message[:type]
-      followup[:message] = message
-      followup
-    end
-  end
-
-  def symbolize_tools(tools)
-    return nil if tools.nil?
-    tools.map do |tool|
-      tool.symbolize_keys!
-      tool.merge(
-        parameters: tool[:parameters]&.map { |param| param.transform_keys(&:to_sym) },
-      ).compact
-    end
-  end
-
-  def with_tests_progress(total:)
-    puts ""
-    count = 0
-    result =
-      yield(
-        -> do
-          count += 1
-          print "\rProcessing test #{count}/#{total}"
-        end
-      )
-    print "\r\033[K"
-    result
-  end
-end

evals/lib/prompts/single_test_runner.rb

@@ -1,76 +0,0 @@
-# frozen_string_literal: true
-
-class DiscourseAi::Evals::PromptSingleTestRunner
-  def initialize(llm)
-    @llm = llm
-  end
-
-  # Run a single test with a prompt and message, and some model settings
-  # @param prompt [String] the prompt to use
-  # @param message [String] the message to use
-  # @param followups [Array<Hash>] an array of followups (messages) to run after the initial prompt
-  # @param output_thinking [Boolean] whether to output the thinking state of the model
-  # @param stream [Boolean] whether to stream the output of the model
-  # @param temperature [Float] the temperature to use when generating completions
-  # @param tools [Array<Hash>] an array of tools to use when generating completions
-  # @return [Hash] the prompt, message, and result of the test
-  def run_single_test(prompt, message, followups, output_thinking, stream, temperature, tools)
-    @c_prompt =
-      DiscourseAi::Completions::Prompt.new(prompt, messages: [{ type: :user, content: message }])
-    @c_prompt.tools = tools if tools
-    generate_result(temperature, output_thinking, stream)
-
-    if followups
-      followups.each do |followup|
-        generate_followup(followup, output_thinking, stream, temperature)
-      end
-    end
-
-    { prompt:, message:, result: @result }
-  end
-
-  private
-
-  def generate_followup(followup, output_thinking, stream, temperature)
-    @c_prompt.push_model_response(@result)
-    followup_message = set_followup_tool(followup)
-    @c_prompt.push(**followup_message)
-    begin
-      generate_result(temperature, output_thinking, stream)
-    rescue => e
-      # should not happen but it helps debugging...
-      puts e
-      result = []
-    end
-  end
-
-  def set_followup_tool(followup)
-    @c_prompt.tools = followup[:tools] if followup[:tools]
-    followup_message = followup[:message]
-    %i[id name].each do |key|
-      if followup_message[key].is_a?(Array)
-        type, inner_key = followup_message[key]
-        # this allows us to dynamically set the id or name of the tool call
-        prev = @c_prompt.messages.reverse.find { |m| m[:type] == type.to_sym }
-        followup_message[key] = prev[inner_key.to_sym] if prev
-      end
-    end
-    followup_message
-  end
-
-  def generate_result(temperature, output_thinking, stream)
-    @result =
-      if stream
-        stream_result = []
-        @llm.generate(
-          @c_prompt,
-          user: Discourse.system_user,
-          temperature:,
-          output_thinking:,
-        ) { |partial| stream_result << partial }
-        stream_result
-      else
-        @llm.generate(@c_prompt, user: Discourse.system_user, temperature:, output_thinking:)
-      end
-  end
-end

evals/run

@@ -6,8 +6,7 @@ require_relative "lib/llm"
 require_relative "lib/cli"
 require_relative "lib/runner"
 require_relative "lib/eval"
-require_relative "lib/prompts/prompt_evaluator"
-require_relative "lib/prompts/single_test_runner"
+require_relative "lib/prompt_evaluator"
 
 options = DiscourseAi::Evals::Cli.parse_options!
 

test/javascripts/acceptance/topic-summary-test.js

@@ -1,3 +1,4 @@
+import { getOwner } from "@ember/owner";
 import { click, visit } from "@ember/test-helpers";
 import { test } from "qunit";
 import { cloneJSON } from "discourse/lib/object";
@@ -97,10 +98,19 @@ acceptance("Topic - Summary", function (needs) {
       },
     });
 
+    let appEventTriggered = 0;
+    const appEvents = getOwner(this).lookup("service:app-events");
+    appEvents.on("ai_summary_link_clicked", this, () => appEventTriggered++);
+
     await click(".generated-summary a");
     assert
       .dom(".ai-summary-box .generated-summary p")
       .hasText(finalSummaryResult, "Retains final summary after clicking link");
+    assert.equal(
+      appEventTriggered,
+      1,
+      "ai_summary_link_clicked appEvent triggered once"
+    );
   });
 });