DEV: Re-introduce validator

- add sleep function for tool polling with rate limits
- Support base64 encoding for HTTP requests and uploads
-  Enhance forum researcher with cost warnings and comprehensive planning
- Add cancellation support for research operations
- Include feature_name parameter for bot analytics
- richer research support (OR queries)

app/jobs/regular/create_ai_reply.rb

@@ -15,7 +15,7 @@ module ::Jobs
 
         bot = DiscourseAi::Personas::Bot.as(bot_user, persona: persona.new)
 
-        DiscourseAi::AiBot::Playground.new(bot).reply_to(post)
+        DiscourseAi::AiBot::Playground.new(bot).reply_to(post, feature_name: "bot")
       rescue DiscourseAi::Personas::Bot::BOT_NOT_FOUND
         Rails.logger.warn(
           "Bot not found for post #{post.id} - perhaps persona was deleted or bot was disabled",

app/jobs/regular/generate_inferred_concepts.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module Jobs
+  class GenerateInferredConcepts < ::Jobs::Base
+    sidekiq_options queue: "low"
+
+    # Process items to generate new concepts
+    #
+    # @param args [Hash] Contains job arguments
+    # @option args [String] :item_type Required - Type of items to process ('topics' or 'posts')
+    # @option args [Array<Integer>] :item_ids Required - List of item IDs to process
+    # @option args [Integer] :batch_size (100) Number of items to process in each batch
+    # @option args [Boolean] :match_only (false) Only match against existing concepts without generating new ones
+    def execute(args = {})
+      return if args[:item_ids].blank? || args[:item_type].blank?
+
+      if %w[topics posts].exclude?(args[:item_type])
+        Rails.logger.error("Invalid item_type for GenerateInferredConcepts: #{args[:item_type]}")
+        return
+      end
+
+      # Process items in smaller batches to avoid memory issues
+      batch_size = args[:batch_size] || 100
+
+      # Get the list of item IDs
+      item_ids = args[:item_ids]
+      match_only = args[:match_only] || false
+
+      # Process items in batches
+      item_ids.each_slice(batch_size) do |batch_item_ids|
+        process_batch(batch_item_ids, args[:item_type], match_only)
+      end
+    end
+
+    private
+
+    def process_batch(item_ids, item_type, match_only)
+      klass = item_type.singularize.classify.constantize
+      items = klass.where(id: item_ids)
+      manager = DiscourseAi::InferredConcepts::Manager.new
+
+      items.each do |item|
+        begin
+          process_item(item, item_type, match_only, manager)
+        rescue => e
+          Rails.logger.error(
+            "Error generating concepts from #{item_type.singularize} #{item.id}: #{e.message}\n#{e.backtrace.join("\n")}",
+          )
+        end
+      end
+    end
+
+    def process_item(item, item_type, match_only, manager)
+      # Use the Manager method that handles both identifying and creating concepts
+      if match_only
+        if item_type == "topics"
+          manager.match_topic_to_concepts(item)
+        else # posts
+          manager.match_post_to_concepts(item)
+        end
+      else
+        if item_type == "topics"
+          manager.generate_concepts_from_topic(item)
+        else # posts
+          manager.generate_concepts_from_post(item)
+        end
+      end
+    end
+  end
+end

app/jobs/scheduled/generate_concepts_from_popular_items.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+module Jobs
+  class GenerateConceptsFromPopularItems < ::Jobs::Scheduled
+    every 1.day
+
+    # This job runs daily and generates new concepts from popular topics and posts
+    # It selects items based on engagement metrics and generates concepts from their content
+    def execute(_args)
+      return unless SiteSetting.inferred_concepts_enabled
+
+      process_popular_topics
+      process_popular_posts
+    end
+
+    private
+
+    def process_popular_topics
+      # Find candidate topics that are popular and don't have concepts yet
+      manager = DiscourseAi::InferredConcepts::Manager.new
+      candidates =
+        manager.find_candidate_topics(
+          limit: SiteSetting.inferred_concepts_daily_topics_limit || 20,
+          min_posts: SiteSetting.inferred_concepts_min_posts || 5,
+          min_likes: SiteSetting.inferred_concepts_min_likes || 10,
+          min_views: SiteSetting.inferred_concepts_min_views || 100,
+          created_after: SiteSetting.inferred_concepts_lookback_days.days.ago,
+        )
+
+      return if candidates.blank?
+
+      # Process candidate topics - first generate concepts, then match
+      Jobs.enqueue(
+        :generate_inferred_concepts,
+        item_type: "topics",
+        item_ids: candidates.map(&:id),
+        batch_size: 10,
+      )
+
+      if SiteSetting.inferred_concepts_background_match
+        # Schedule a follow-up job to match existing concepts
+        Jobs.enqueue_in(
+          1.hour,
+          :generate_inferred_concepts,
+          item_type: "topics",
+          item_ids: candidates.map(&:id),
+          batch_size: 10,
+          match_only: true,
+        )
+      end
+    end
+
+    def process_popular_posts
+      # Find candidate posts that are popular and don't have concepts yet
+      manager = DiscourseAi::InferredConcepts::Manager.new
+      candidates =
+        manager.find_candidate_posts(
+          limit: SiteSetting.inferred_concepts_daily_posts_limit || 30,
+          min_likes: SiteSetting.inferred_concepts_post_min_likes || 5,
+          exclude_first_posts: true,
+          created_after: SiteSetting.inferred_concepts_lookback_days.days.ago,
+        )
+
+      return if candidates.blank?
+
+      # Process candidate posts - first generate concepts, then match
+      Jobs.enqueue(
+        :generate_inferred_concepts,
+        item_type: "posts",
+        item_ids: candidates.map(&:id),
+        batch_size: 10,
+      )
+
+      if SiteSetting.inferred_concepts_background_match
+        # Schedule a follow-up job to match against existing concepts
+        Jobs.enqueue_in(
+          1.hour,
+          :generate_inferred_concepts,
+          item_type: "posts",
+          item_ids: candidates.map(&:id),
+          batch_size: 10,
+          match_only: true,
+        )
+      end
+    end
+  end
+end

app/models/ai_tool.rb

@@ -22,6 +22,8 @@ class AiTool < ActiveRecord::Base
               message: I18n.t("discourse_ai.tools.name.characters"),
             }
 
+  validate :validate_parameters_enum
+
   def signature
     {
       name: function_call_name,
@@ -57,6 +59,30 @@ class AiTool < ActiveRecord::Base
     end
   end
 
+  def validate_parameters_enum
+    return unless parameters.is_a?(Array)
+
+    parameters.each_with_index do |param, index|
+      next if !param.is_a?(Hash) || !param.key?("enum")
+      enum_values = param["enum"]
+
+      if enum_values.empty?
+        errors.add(
+          :parameters,
+          "Parameter '#{param["name"]}' at index #{index}: enum cannot be empty",
+        )
+        next
+      end
+
+      if enum_values.uniq.length != enum_values.length
+        errors.add(
+          :parameters,
+          "Parameter '#{param["name"]}' at index #{index}: enum values must be unique",
+        )
+      end
+    end
+  end
+
   def self.preamble
     <<~JS
      /**
@@ -142,6 +168,7 @@ class AiTool < ActiveRecord::Base
      *      base_64_content (string): Base64 encoded content of the file.
      *    Returns: { id: number, url: string, short_url: string } - Details of the created upload record.
      *
+     *    upload.getUrl(shortUrl): Given a short URL, eg upload://12345, returns the full CDN friendly URL of the upload.
      * 5. chain
      *    Controls the execution flow.
      *

app/models/inferred_concept.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+class InferredConcept < ActiveRecord::Base
+  has_many :inferred_concept_topics
+  has_many :topics, through: :inferred_concept_topics
+
+  has_many :inferred_concept_posts
+  has_many :posts, through: :inferred_concept_posts
+
+  validates :name, presence: true, uniqueness: true
+end
+
+# == Schema Information
+#
+# Table name: inferred_concepts
+#
+#  id         :bigint           not null, primary key
+#  name       :string           not null
+#  created_at :datetime         not null
+#  updated_at :datetime         not null
+#
+# Indexes
+#
+#  index_inferred_concepts_on_name  (name) UNIQUE
+#

app/models/inferred_concept_post.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+class InferredConceptPost < ActiveRecord::Base
+  belongs_to :inferred_concept
+  belongs_to :post
+
+  validates :inferred_concept_id, presence: true
+  validates :post_id, presence: true
+  validates :inferred_concept_id, uniqueness: { scope: :post_id }
+end
+
+# == Schema Information
+#
+# Table name: inferred_concept_posts
+#
+#  inferred_concept_id :bigint
+#  post_id             :bigint
+#  created_at          :datetime         not null
+#  updated_at          :datetime         not null
+#
+# Indexes
+#
+#  index_inferred_concept_posts_on_inferred_concept_id  (inferred_concept_id)
+#  index_inferred_concept_posts_uniqueness              (post_id,inferred_concept_id) UNIQUE
+#

app/models/inferred_concept_topic.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+class InferredConceptTopic < ActiveRecord::Base
+  belongs_to :inferred_concept
+  belongs_to :topic
+
+  validates :inferred_concept_id, presence: true
+  validates :topic_id, presence: true
+  validates :inferred_concept_id, uniqueness: { scope: :topic_id }
+end
+
+# == Schema Information
+#
+# Table name: inferred_concept_topics
+#
+#  inferred_concept_id :bigint
+#  topic_id            :bigint
+#  created_at          :datetime         not null
+#  updated_at          :datetime         not null
+#
+# Indexes
+#
+#  index_inferred_concept_topics_on_inferred_concept_id  (inferred_concept_id)
+#  index_inferred_concept_topics_uniqueness              (topic_id,inferred_concept_id) UNIQUE
+#

app/serializers/ai_inferred_concept_post_serializer.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+class AiInferredConceptPostSerializer < ApplicationSerializer
+  attributes :id,
+             :post_number,
+             :topic_id,
+             :topic_title,
+             :username,
+             :avatar_template,
+             :created_at,
+             :updated_at,
+             :excerpt,
+             :truncated,
+             :inferred_concepts
+
+  def avatar_template
+    User.avatar_template(object.username, object.uploaded_avatar_id)
+  end
+
+  def excerpt
+    Post.excerpt(object.cooked)
+  end
+
+  def truncated
+    object.cooked.length > SiteSetting.post_excerpt_maxlength
+  end
+
+  def inferred_concepts
+    ActiveModel::ArraySerializer.new(
+      object.inferred_concepts,
+      each_serializer: InferredConceptSerializer,
+    )
+  end
+end

app/serializers/inferred_concept_serializer.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+class InferredConceptSerializer < ApplicationSerializer
+  attributes :id, :name, :created_at, :updated_at
+end

assets/javascripts/discourse/components/ai-tool-editor-form.gjs

@@ -30,11 +30,16 @@ export default class AiToolEditorForm extends Component {
 
   get formData() {
     const parameters = (this.args.editingModel.parameters ?? []).map(
-      (parameter) => ({
-        ...parameter,
-        isEnum: !!parameter.enum?.length,
-        enum: (parameter.enum ??= []),
-      })
+      (parameter) => {
+        const mappedParameter = {
+          ...parameter,
+        };
+        mappedParameter.isEnum = parameter.enum && parameter.enum.length > 0;
+        if (!mappedParameter.isEnum) {
+          delete mappedParameter.enum;
+        }
+        return mappedParameter;
+      }
     );
 
     return {
@@ -63,8 +68,19 @@ export default class AiToolEditorForm extends Component {
   async save(data) {
     this.isSaving = true;
 
+    // we injected a isEnum thing, we need to clean it up
+    const copiedData = JSON.parse(JSON.stringify(data));
+    if (copiedData.parameters) {
+      copiedData.parameters.forEach((parameter) => {
+        if (!parameter.isEnum) {
+          delete parameter.enum;
+        }
+        delete parameter.isEnum;
+      });
+    }
+
     try {
-      await this.args.model.save(data);
+      await this.args.model.save(copiedData);
 
       this.toasts.success({
         data: { message: i18n("discourse_ai.tools.saved") },

assets/javascripts/discourse/components/modal/ai-persona-response-format-editor.gjs

@@ -22,10 +22,20 @@ export default class AiPersonaResponseFormatEditor extends Component {
           type: "string",
         },
         type: {
+          type: "string",
+          enum: ["string", "integer", "boolean", "array"],
+        },
+        array_type: {
           type: "string",
           enum: ["string", "integer", "boolean"],
+          options: {
+            dependencies: {
+              type: "array",
+            },
+          },
         },
       },
+      required: ["key", "type"],
     },
   };
 
@@ -41,7 +51,11 @@ export default class AiPersonaResponseFormatEditor extends Component {
     const toDisplay = {};
 
     this.args.data.response_format.forEach((keyDesc) => {
-      toDisplay[keyDesc.key] = keyDesc.type;
+      if (keyDesc.type === "array") {
+        toDisplay[keyDesc.key] = `[${keyDesc.array_type}]`;
+      } else {
+        toDisplay[keyDesc.key] = keyDesc.type;
+      }
     });
 
     return prettyJSON(toDisplay);

assets/javascripts/discourse/components/modal/debug-ai-modal.gjs

@@ -144,11 +144,11 @@ export default class DebugAiModal extends Component {
             >{{i18n "discourse_ai.ai_bot.debug_ai_modal.response"}}</a></li>
         </ul>
         <div class="ai-debug-modal__tokens">
-          <span>
+          <span class="ai-debug-modal__tokens__count">
             {{i18n "discourse_ai.ai_bot.debug_ai_modal.request_tokens"}}
             {{this.info.request_tokens}}
           </span>
-          <span>
+          <span class="ai-debug-modal__tokens__count">
             {{i18n "discourse_ai.ai_bot.debug_ai_modal.response_tokens"}}
             {{this.info.response_tokens}}
           </span>

assets/javascripts/discourse/components/modal/diff-modal.gjs

@@ -5,6 +5,7 @@ import didInsert from "@ember/render-modifiers/modifiers/did-insert";
 import willDestroy from "@ember/render-modifiers/modifiers/will-destroy";
 import { service } from "@ember/service";
 import { htmlSafe } from "@ember/template";
+import { or } from "truth-helpers";
 import CookText from "discourse/components/cook-text";
 import DButton from "discourse/components/d-button";
 import DModal from "discourse/components/d-modal";
@@ -41,6 +42,10 @@ export default class ModalDiffModal extends Component {
   }
 
   get diffResult() {
+    if (this.loading) {
+      return this.escapedSelectedText;
+    }
+
     if (this.diffStreamer.diff?.length > 0) {
       return this.diffStreamer.diff;
     }
@@ -50,10 +55,22 @@ export default class ModalDiffModal extends Component {
     return this.escapedSelectedText;
   }
 
+  get smoothStreamerResult() {
+    if (this.loading) {
+      return this.escapedSelectedText;
+    }
+
+    return this.smoothStreamer.renderedText;
+  }
+
   get isStreaming() {
     // diffStreamer stops Streaming when it is finished with a chunk, looking at isDone is safe
     // it starts off not done
-    return !this.diffStreamer.isDone || this.smoothStreamer.isStreaming;
+    if (this.args.model.showResultAsDiff) {
+      return !this.diffStreamer.isDone;
+    }
+
+    return this.smoothStreamer.isStreaming;
   }
 
   get primaryBtnLabel() {
@@ -154,42 +171,37 @@ export default class ModalDiffModal extends Component {
           {{willDestroy this.cleanup}}
           class="text-preview"
         >
-          {{#if this.loading}}
-            <div class="composer-ai-helper-modal__loading">
-              {{[email protected]~}}
-            </div>
-          {{else}}
-            <div
-              class={{concatClass
-                "composer-ai-helper-modal__suggestion"
-                "streamable-content"
-                (if this.isStreaming "streaming")
-                (if @model.showResultAsDiff "inline-diff")
-                (if this.diffStreamer.isThinking "thinking")
-              }}
-            >
-              {{~#if @model.showResultAsDiff~}}
-                <span class="diff-inner">{{htmlSafe this.diffResult}}</span>
+          <div
+            class={{concatClass
+              "composer-ai-helper-modal__suggestion"
+              "streamable-content"
+              (if this.isStreaming "streaming")
+              (if @model.showResultAsDiff "inline-diff")
+              (if this.diffStreamer.isThinking "thinking")
+              (if this.loading "composer-ai-helper-modal__loading")
+            }}
+          >
+            {{~#if @model.showResultAsDiff~}}
+              <span class="diff-inner">{{htmlSafe this.diffResult}}</span>
+            {{else}}
+              {{#if (or this.loading this.smoothStreamer.isStreaming)}}
+                <CookText
+                  @rawText={{this.smoothStreamerResult}}
+                  class="cooked"
+                />
               {{else}}
-                {{#if this.smoothStreamer.isStreaming}}
+                <div class="composer-ai-helper-modal__old-value">
+                  {{~this.escapedSelectedText~}}
+                </div>
+                <div class="composer-ai-helper-modal__new-value">
                   <CookText
-                    @rawText={{this.smoothStreamer.renderedText}}
+                    @rawText={{this.smoothStreamerResult}}
                     class="cooked"
                   />
-                {{else}}
-                  <div class="composer-ai-helper-modal__old-value">
-                    {{@model.selectedText}}
-                  </div>
-                  <div class="composer-ai-helper-modal__new-value">
-                    <CookText
-                      @rawText={{this.smoothStreamer.renderedText}}
-                      class="cooked"
-                    />
-                  </div>
-                {{/if}}
+                </div>
               {{/if}}
-            </div>
-          {{/if}}
+            {{/if}}
+          </div>
         </div>
       </:body>
 

assets/javascripts/discourse/connectors/full-page-search-below-search-header/ai-full-page-discobot-discoveries.gjs

@@ -15,6 +15,7 @@ export default class AiFullPageDiscobotDiscoveries extends Component {
     );
   }
 
+  @service capabilities;
   @service discobotDiscoveries;
   @service site;
 

assets/stylesheets/common/streaming.scss

@@ -111,7 +111,7 @@ mark.highlight {
   animation-name: mark-blink;
 }
 
-.composer-ai-helper-modal__loading {
+.composer-ai-helper-modal__loading.inline-diff {
   white-space: pre-wrap;
 }
 

assets/stylesheets/modules/ai-bot/common/bot-replies.scss

@@ -139,7 +139,7 @@ span.onebox-ai-llm-title {
   }
 }
 
-.ai-debug-modal__tokens span {
+.ai-debug-modal__tokens__count {
   display: block;
 }
 

assets/stylesheets/modules/ai-helper/common/ai-helper.scss

@@ -1,11 +1,13 @@
 @use "lib/viewport";
 
 .composer-ai-helper-modal {
-  .text-preview,
   .inline-diff {
     font-family: var(--d-font-family--monospace);
     font-variant-ligatures: none;
+  }
 
+  .text-preview,
+  .inline-diff {
     ins {
       background-color: var(--success-low);
       text-decoration: none;
@@ -55,13 +57,16 @@
   }
 
   &__old-value {
-    background-color: var(--danger-low);
+    white-space: pre-wrap;
+    border-left: 2px solid var(--danger);
+    padding-left: 1rem;
     color: var(--danger);
     margin-bottom: 1rem;
   }
 
   &__new-value {
-    background-color: var(--success-low);
+    border-left: 2px solid var(--success);
+    padding-left: 1rem;
     color: var(--success);
   }
 
@@ -77,7 +82,6 @@
 }
 
 .ai-composer-helper-menu {
-  padding: 0.25rem;
   max-width: 25rem;
   list-style: none;
 
@@ -701,7 +705,7 @@
     width: 100%;
     border-radius: 0;
     margin: 0;
-    padding: 0.5em 1rem;
+    padding: 0.7rem 1rem;
 
     &:focus,
     &:hover {

config/locales/server.en.yml

@@ -330,6 +330,15 @@ en:
         short_summarizer:
           name: "Summarizer (short form)"
           description: "Default persona used to power AI short summaries for topic lists' items"
+        concept_finder:
+          name: "Concept Finder"
+          description: "AI Bot specialized in identifying concepts and themes in content"
+        concept_matcher:
+          name: "Concept Matcher"
+          description: "AI Bot specialized in matching content against existing concepts"
+        concept_deduplicator:
+          name: "Concept Deduplicator"
+          description: "AI Bot specialized in deduplicating concepts"
       topic_not_found: "Summary unavailable, topic not found!"
       summarizing: "Summarizing topic"
       searching: "Searching for: '%{query}'"
@@ -549,6 +558,9 @@ en:
       discord_search:
         name: "Discord Search"
         description: "Adds the ability to search Discord channels"
+      inferred_concepts:
+        name: "Inferred Concepts"
+        description: "Classifies topics and posts into areas of interest / labels."
 
     errors:
       quota_exceeded: "You have exceeded the quota for this model. Please try again in %{relative_time}."

config/settings.yml

@@ -417,3 +417,55 @@ discourse_ai:
     default: false
     client: false
     hidden: true
+
+  inferred_concepts_enabled:
+    default: false
+    client: true
+    area: "ai-features/inferred_concepts"
+  inferred_concepts_background_match:
+    default: false
+    client: false
+    area: "ai-features/inferred_concepts"
+  inferred_concepts_daily_topics_limit:
+    default: 20
+    client: false
+    area: "ai-features/inferred_concepts"
+  inferred_concepts_min_posts:
+    default: 5
+    client: false
+    area: "ai-features/inferred_concepts"
+  inferred_concepts_min_likes:
+    default: 10
+    client: false
+    area: "ai-features/inferred_concepts"
+  inferred_concepts_min_views:
+    default: 100
+    client: false
+    area: "ai-features/inferred_concepts"
+  inferred_concepts_lookback_days:
+    default: 30
+    client: false
+    area: "ai-features/inferred_concepts"
+  inferred_concepts_daily_posts_limit:
+    default: 30
+    client: false
+    area: "ai-features/inferred_concepts"
+  inferred_concepts_post_min_likes:
+    default: 5
+    client: false
+    area: "ai-features/inferred_concepts"
+  inferred_concepts_generate_persona:
+    default: "-15"
+    type: enum
+    enum: "DiscourseAi::Configuration::PersonaEnumerator"
+    area: "ai-features/inferred_concepts"
+  inferred_concepts_match_persona:
+    default: "-16"
+    type: enum
+    enum: "DiscourseAi::Configuration::PersonaEnumerator"
+    area: "ai-features/inferred_concepts"
+  inferred_concepts_deduplicate_persona:
+    default: "-17"
+    type: enum
+    enum: "DiscourseAi::Configuration::PersonaEnumerator"
+    area: "ai-features/inferred_concepts"

db/fixtures/personas/603_ai_personas.rb

@@ -72,9 +72,13 @@ DiscourseAi::Personas::Persona.system_personas.each do |persona_class, id|
 
   persona.tools = tools.map { |name, value| [name, value] }
 
-  persona.response_format = instance.response_format
+  # Only set response_format if it's not defined as a method in the persona class
+  if !instance.class.instance_methods.include?(:response_format)
+    persona.response_format = instance.response_format
+  end
 
-  persona.examples = instance.examples
+  # Only set examples if it's not defined as a method in the persona class
+  persona.examples = instance.examples if !instance.class.instance_methods.include?(:examples)
 
   persona.system_prompt = instance.system_prompt
   persona.top_p = instance.top_p

db/migrate/20250508182047_create_inferred_concepts_table.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+class CreateInferredConceptsTable < ActiveRecord::Migration[7.2]
+  def change
+    create_table :inferred_concepts do |t|
+      t.string :name, null: false
+      t.timestamps
+    end
+
+    add_index :inferred_concepts, :name, unique: true
+  end
+end

db/migrate/20250508183456_create_inferred_concept_topics.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+class CreateInferredConceptTopics < ActiveRecord::Migration[7.0]
+  def change
+    create_table :inferred_concept_topics, id: false do |t|
+      t.bigint :inferred_concept_id
+      t.bigint :topic_id
+      t.timestamps
+    end
+
+    add_index :inferred_concept_topics,
+              %i[topic_id inferred_concept_id],
+              unique: true,
+              name: "index_inferred_concept_topics_uniqueness"
+
+    add_index :inferred_concept_topics, :inferred_concept_id
+  end
+end

db/migrate/20250509000001_create_inferred_concept_posts.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+class CreateInferredConceptPosts < ActiveRecord::Migration[7.0]
+  def change
+    create_table :inferred_concept_posts, id: false do |t|
+      t.bigint :inferred_concept_id
+      t.bigint :post_id
+      t.timestamps
+    end
+
+    add_index :inferred_concept_posts,
+              %i[post_id inferred_concept_id],
+              unique: true,
+              name: "index_inferred_concept_posts_uniqueness"
+
+    add_index :inferred_concept_posts, :inferred_concept_id
+  end
+end

lib/completions/json_streaming_tracker.rb

@@ -9,6 +9,7 @@ module DiscourseAi
         @stream_consumer = stream_consumer
         @current_key = nil
         @current_value = nil
+        @tracking_array = false
         @parser = DiscourseAi::Completions::JsonStreamingParser.new
 
         @parser.key do |k|
@@ -16,12 +17,28 @@ module DiscourseAi
           @current_value = nil
         end
 
-        @parser.value do |v|
+        @parser.value do |value|
           if @current_key
-            stream_consumer.notify_progress(@current_key, v)
-            @current_key = nil
+            if @tracking_array
+              @current_value << value
+              stream_consumer.notify_progress(@current_key, @current_value)
+            else
+              stream_consumer.notify_progress(@current_key, value)
+              @current_key = nil
+            end
           end
         end
+
+        @parser.start_array do
+          @tracking_array = true
+          @current_value = []
+        end
+
+        @parser.end_array do
+          @tracking_array = false
+          @current_key = nil
+          @current_value = nil
+        end
       end
 
       def broken?
@@ -46,8 +63,9 @@ module DiscourseAi
         end
 
         if @parser.state == :start_string && @current_key
+          buffered = @tracking_array ? [@parser.buf] : @parser.buf
           # this is is worth notifying
-          stream_consumer.notify_progress(@current_key, @parser.buf)
+          stream_consumer.notify_progress(@current_key, buffered)
         end
 
         @current_key = nil if @parser.state == :end_value

lib/completions/structured_output.rb

@@ -45,7 +45,7 @@ module DiscourseAi
           @property_cursors[prop_name] = @tracked[prop_name].length
           unread
         else
-          # Ints and bools are always returned as is.
+          # Ints and bools, and arrays are always returned as is.
           @tracked[prop_name]
         end
       end

lib/configuration/spam_detection_validator.rb

@@ -10,6 +10,8 @@ module DiscourseAi
       def valid_value?(val)
         return true if Rails.env.test?
         return true if AiModerationSetting.spam
+        # only validate when enabling setting
+        return true if val == "f"
 
         false
       end

lib/discord/bot/persona_replier.rb

@@ -7,7 +7,7 @@ module DiscourseAi
         @persona =
           AiPersona
             .all_personas(enabled_only: false)
-            .find { |persona| persona.id == SiteSetting.ai_discord_search_persona.to_i }
+            .find { |p| p.id == SiteSetting.ai_discord_search_persona.to_i }
             .new
         @bot =
           DiscourseAi::Personas::Bot.as(

lib/features.rb

@@ -36,6 +36,14 @@ module DiscourseAi
           persona_setting_name: "ai_discord_search_persona",
           enable_setting_name: "ai_discord_search_enabled",
         },
+        {
+          id: 5,
+          name_ref: "inferred_concepts",
+          name_key: "discourse_ai.features.inferred_concepts.name",
+          description_key: "discourse_ai.features.inferred_concepts.description",
+          persona_setting_name: "inferred_concepts_generate_persona",
+          enable_setting_name: "inferred_concepts_enabled",
+        },
       ]
     end
 

lib/inferred_concepts/applier.rb

@@ -0,0 +1,135 @@
+# frozen_string_literal: true
+
+module DiscourseAi
+  module InferredConcepts
+    class Applier
+      # Associates the provided concepts with a topic
+      # topic: a Topic instance
+      # concepts: an array of InferredConcept instances
+      def apply_to_topic(topic, concepts)
+        return if topic.blank? || concepts.blank?
+
+        topic.inferred_concepts << concepts
+      end
+
+      # Associates the provided concepts with a post
+      # post: a Post instance
+      # concepts: an array of InferredConcept instances
+      def apply_to_post(post, concepts)
+        return if post.blank? || concepts.blank?
+
+        post.inferred_concepts << concepts
+      end
+
+      # Extracts content from a topic for concept analysis
+      # Returns a string with the topic title and first few posts
+      def topic_content_for_analysis(topic)
+        return "" if topic.blank?
+
+        # Combine title and first few posts for analysis
+        posts = Post.where(topic_id: topic.id).order(:post_number).limit(10)
+
+        content = "Title: #{topic.title}\n\n"
+        content += posts.map { |p| "#{p.post_number}) #{p.user.username}: #{p.raw}" }.join("\n\n")
+
+        content
+      end
+
+      # Extracts content from a post for concept analysis
+      # Returns a string with the post content
+      def post_content_for_analysis(post)
+        return "" if post.blank?
+
+        # Get the topic title for context
+        topic_title = post.topic&.title || ""
+
+        content = "Topic: #{topic_title}\n\n"
+        content += "Post by #{post.user.username}:\n#{post.raw}"
+
+        content
+      end
+
+      # Match a topic with existing concepts
+      def match_existing_concepts(topic)
+        return [] if topic.blank?
+
+        # Get content to analyze
+        content = topic_content_for_analysis(topic)
+
+        # Get all existing concepts
+        existing_concepts = DiscourseAi::InferredConcepts::Manager.new.list_concepts
+        return [] if existing_concepts.empty?
+
+        # Use the ConceptMatcher persona to match concepts
+        matched_concept_names = match_concepts_to_content(content, existing_concepts)
+
+        # Find concepts in the database
+        matched_concepts = InferredConcept.where(name: matched_concept_names)
+
+        # Apply concepts to the topic
+        apply_to_topic(topic, matched_concepts)
+
+        matched_concepts
+      end
+
+      # Match a post with existing concepts
+      def match_existing_concepts_for_post(post)
+        return [] if post.blank?
+
+        # Get content to analyze
+        content = post_content_for_analysis(post)
+
+        # Get all existing concepts
+        existing_concepts = DiscourseAi::InferredConcepts::Manager.new.list_concepts
+        return [] if existing_concepts.empty?
+
+        # Use the ConceptMatcher persona to match concepts
+        matched_concept_names = match_concepts_to_content(content, existing_concepts)
+
+        # Find concepts in the database
+        matched_concepts = InferredConcept.where(name: matched_concept_names)
+
+        # Apply concepts to the post
+        apply_to_post(post, matched_concepts)
+
+        matched_concepts
+      end
+
+      # Use ConceptMatcher persona to match content against provided concepts
+      def match_concepts_to_content(content, concept_list)
+        return [] if content.blank? || concept_list.blank?
+
+        # Prepare user message with only the content
+        user_message = content
+
+        # Use the ConceptMatcher persona to match concepts
+
+        persona =
+          AiPersona
+            .all_personas(enabled_only: false)
+            .find { |p| p.id == SiteSetting.inferred_concepts_match_persona.to_i }
+            .new
+
+        llm = LlmModel.find(persona.class.default_llm_id)
+
+        input = { type: :user, content: content }
+
+        context =
+          DiscourseAi::Personas::BotContext.new(
+            messages: [input],
+            user: Discourse.system_user,
+            inferred_concepts: concept_list,
+          )
+
+        bot = DiscourseAi::Personas::Bot.as(Discourse.system_user, persona: persona, model: llm)
+        structured_output = nil
+
+        bot.reply(context) do |partial, _, type|
+          structured_output = partial if type == :structured_output
+        end
+
+        structured_output&.read_buffered_property(:matching_concepts) || []
+      end
+    end
+  end
+end

lib/inferred_concepts/finder.rb

@@ -0,0 +1,176 @@
+# frozen_string_literal: true
+
+module DiscourseAi
+  module InferredConcepts
+    class Finder
+      # Identifies potential concepts from provided content
+      # Returns an array of concept names (strings)
+      def identify_concepts(content)
+        return [] if content.blank?
+
+        # Use the ConceptFinder persona to identify concepts
+        persona =
+          AiPersona
+            .all_personas(enabled_only: false)
+            .find { |p| p.id == SiteSetting.inferred_concepts_generate_persona.to_i }
+            .new
+
+        llm = LlmModel.find(persona.class.default_llm_id)
+        context =
+          DiscourseAi::Personas::BotContext.new(
+            messages: [{ type: :user, content: content }],
+            user: Discourse.system_user,
+            inferred_concepts: DiscourseAi::InferredConcepts::Manager.new.list_concepts,
+          )
+
+        bot = DiscourseAi::Personas::Bot.as(Discourse.system_user, persona: persona, model: llm)
+        structured_output = nil
+
+        bot.reply(context) do |partial, _, type|
+          structured_output = partial if type == :structured_output
+        end
+
+        structured_output&.read_buffered_property(:concepts) || []
+      end
+
+      # Creates or finds concepts in the database from provided names
+      # Returns an array of InferredConcept instances
+      def create_or_find_concepts(concept_names)
+        return [] if concept_names.blank?
+
+        concept_names.map { |name| InferredConcept.find_or_create_by(name: name) }
+      end
+
+      # Finds candidate topics to use for concept generation
+      #
+      # @param limit [Integer] Maximum number of topics to return
+      # @param min_posts [Integer] Minimum number of posts in topic
+      # @param min_likes [Integer] Minimum number of likes across all posts
+      # @param min_views [Integer] Minimum number of views
+      # @param exclude_topic_ids [Array<Integer>] Topic IDs to exclude
+      # @param category_ids [Array<Integer>] Only include topics from these categories (optional)
+      # @param created_after [DateTime] Only include topics created after this time (optional)
+      # @return [Array<Topic>] Array of Topic objects that are good candidates
+      def find_candidate_topics(
+        limit: 100,
+        min_posts: 5,
+        min_likes: 10,
+        min_views: 100,
+        exclude_topic_ids: [],
+        category_ids: nil,
+        created_after: 30.days.ago
+      )
+        query =
+          Topic.where(
+            "topics.posts_count >= ? AND topics.views >= ? AND topics.like_count >= ?",
+            min_posts,
+            min_views,
+            min_likes,
+          )
+
+        # Apply additional filters
+        query = query.where("topics.id NOT IN (?)", exclude_topic_ids) if exclude_topic_ids.present?
+        query = query.where("topics.category_id IN (?)", category_ids) if category_ids.present?
+        query = query.where("topics.created_at >= ?", created_after) if created_after.present?
+
+        # Exclude PM topics (if they exist in Discourse)
+        query = query.where(archetype: Archetype.default)
+
+        # Exclude topics that already have concepts
+        topics_with_concepts = <<~SQL
+          SELECT DISTINCT topic_id
+          FROM inferred_concept_topics
+        SQL
+
+        query = query.where("topics.id NOT IN (#{topics_with_concepts})")
+
+        # Score and order topics by engagement (combination of views, likes, and posts)
+        query =
+          query.select(
+            "topics.*,
+          (topics.like_count * 2 + topics.posts_count * 3 + topics.views * 0.1) AS engagement_score",
+          ).order("engagement_score DESC")
+
+        # Return limited number of topics
+        query.limit(limit)
+      end
+
+      # Find candidate posts that are good for concept generation
+      #
+      # @param limit [Integer] Maximum number of posts to return
+      # @param min_likes [Integer] Minimum number of likes
+      # @param exclude_first_posts [Boolean] Exclude first posts in topics
+      # @param exclude_post_ids [Array<Integer>] Post IDs to exclude
+      # @param category_ids [Array<Integer>] Only include posts from topics in these categories
+      # @param created_after [DateTime] Only include posts created after this time
+      # @return [Array<Post>] Array of Post objects that are good candidates
+      def find_candidate_posts(
+        limit: 100,
+        min_likes: 5,
+        exclude_first_posts: true,
+        exclude_post_ids: [],
+        category_ids: nil,
+        created_after: 30.days.ago
+      )
+        query = Post.where("posts.like_count >= ?", min_likes)
+
+        # Exclude first posts if specified
+        query = query.where("posts.post_number > 1") if exclude_first_posts
+
+        # Apply additional filters
+        query = query.where("posts.id NOT IN (?)", exclude_post_ids) if exclude_post_ids.present?
+        query = query.where("posts.created_at >= ?", created_after) if created_after.present?
+
+        # Filter by category if specified
+        if category_ids.present?
+          query = query.joins(:topic).where("topics.category_id IN (?)", category_ids)
+        end
+
+        # Exclude posts that already have concepts
+        posts_with_concepts = <<~SQL
+          SELECT DISTINCT post_id
+          FROM inferred_concept_posts
+        SQL
+
+        query = query.where("posts.id NOT IN (#{posts_with_concepts})")
+
+        # Order by engagement (likes)
+        query = query.order(like_count: :desc)
+
+        # Return limited number of posts
+        query.limit(limit)
+      end
+
+      # Deduplicate and standardize a list of concepts
+      # @param concept_names [Array<String>] List of concept names to deduplicate
+      # @return [Hash] Hash with deduplicated concepts and mapping
+      def deduplicate_concepts(concept_names)
+        return { deduplicated_concepts: [], mapping: {} } if concept_names.blank?
+
+        # Use the ConceptDeduplicator persona to deduplicate concepts
+        persona =
+          AiPersona
+            .all_personas(enabled_only: false)
+            .find { |p| p.id == SiteSetting.inferred_concepts_deduplicate_persona.to_i }
+            .new
+
+        llm = LlmModel.find(persona.class.default_llm_id)
+
+        # Create the input for the deduplicator
+        input = { type: :user, content: concept_names.join(", ") }
+
+        context =
+          DiscourseAi::Personas::BotContext.new(messages: [input], user: Discourse.system_user)
+
+        bot = DiscourseAi::Personas::Bot.as(Discourse.system_user, persona: persona, model: llm)
+        structured_output = nil
+
+        bot.reply(context) do |partial, _, type|
+          structured_output = partial if type == :structured_output
+        end
+
+        structured_output&.read_buffered_property(:streamlined_tags) || []
+      end
+    end
+  end
+end