Build(deps-dev): Bump rack from 3.1.16 to 3.1.18

Bumps [rack](https://github.com/rack/rack) from 3.1.16 to 3.1.18.
- [Release notes](https://github.com/rack/rack/releases)
- [Changelog](https://github.com/rack/rack/blob/main/CHANGELOG.md)
- [Commits](https://github.com/rack/rack/compare/v3.1.16...v3.1.18)

---
updated-dependencies:
- dependency-name: rack
  dependency-version: 3.1.18
  dependency-type: indirect
...

Signed-off-by: dependabot[bot] <[email protected]>

.discourse-compatibility

@@ -1,3 +1,4 @@
+< 3.5.0.beta8-dev: 5d80a34589d63e381d80273828072badc18955b1
 < 3.5.0.beta7-dev: cd14b0c0bee0cf63c59b64b6f7213e31a37f11a7
 < 3.5.0.beta6-dev: 3e74eea1e5e3143888d67a8d8a11206df214dc24
 < 3.5.0.beta3-dev: 09a68414804a1447f52e5d60691ba59742cda9ec

Gemfile.lock

@@ -34,7 +34,7 @@ GEM
       racc
     prettier_print (1.2.1)
     racc (1.8.1)
-    rack (3.1.16)
+    rack (3.1.18)
     rainbow (3.1.1)
     regexp_parser (2.10.0)
     rubocop (1.74.0)

README.md

@@ -1,15 +1,18 @@
 # **Discourse AI** Plugin
 
+> [!IMPORTANT]
+> This plugin has now been bundled into Discourse core. See: https://meta.discourse.org/t/bundling-more-popular-plugins-with-discourse-core/373574
+
 **Plugin Summary**
 
 For more information, please see: https://meta.discourse.org/t/discourse-ai/259214?u=falco
 
 ### Evals
 
-The directory `evals` contains AI evals for the Discourse AI plugin. 
+The directory `evals` contains AI evals for the Discourse AI plugin.
 You may create a local config by copying `config/eval-llms.yml` to `config/eval-llms.local.yml` and modifying the values.
 
-To run them use: 
+To run them use:
 
 cd evals
 ./run --help

app/jobs/regular/detect_translate_post.rb

@@ -40,7 +40,7 @@ module Jobs
       return if locales.blank?
 
       locales.each do |locale|
-        next if DiscourseAi::Translation::LocaleNormalizer.is_same?(locale, detected_locale)
+        next if LocaleNormalizer.is_same?(locale, detected_locale)
         regionless_locale = locale.split("_").first
         next if post.post_localizations.where("locale LIKE ?", "#{regionless_locale}%").exists?
 

app/jobs/regular/detect_translate_topic.rb

@@ -38,7 +38,7 @@ module Jobs
       return if locales.blank?
 
       locales.each do |locale|
-        next if DiscourseAi::Translation::LocaleNormalizer.is_same?(locale, detected_locale)
+        next if LocaleNormalizer.is_same?(locale, detected_locale)
         regionless_locale = locale.split("_").first
         next if topic.topic_localizations.where("locale LIKE ?", "#{regionless_locale}%").exists?
 

app/jobs/regular/localize_categories.rb

@@ -29,7 +29,7 @@ module Jobs
         missing_locales = locales - existing_locales - [category.locale]
         missing_locales.each do |locale|
           break if remaining_limit <= 0
-          next if DiscourseAi::Translation::LocaleNormalizer.is_same?(locale, category.locale)
+          next if LocaleNormalizer.is_same?(locale, category.locale)
 
           begin
             DiscourseAi::Translation::CategoryLocalizer.localize(category, locale)

config/locales/client.en.yml

@@ -180,6 +180,9 @@ en:
             temperature:
               label: "Temperature"
               description: "Temperature to use for the LLM. Increase to increase randomness (leave empty to use model default)"
+            max_output_tokens:
+              label: "Max output tokens"
+              description: "When specified, sets an upper bound to the maximum number of tokens the model can generate. Respects LLM's max output tokens limit"
 
     discourse_ai:
       title: "AI"

config/settings.yml

@@ -1,6 +1,6 @@
 discourse_ai:
   discourse_ai_enabled:
-    default: true
+    default: false
     client: true
   ai_artifact_security:
     client: true
@@ -527,6 +527,8 @@ discourse_ai:
     area: "ai-features/translation"
   ai_translation_backfill_max_age_days:
     default: 5
+    min: 0
+    max: 20000
     client: false
     area: "ai-features/translation"
   ai_translation_verbose_logs:

db/migrate/20250717075002_set_translation_backfill_max_age.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+class SetTranslationBackfillMaxAge < ActiveRecord::Migration[7.2]
+  def up
+    execute <<~SQL
+      UPDATE site_settings
+      SET value = '20000'
+      WHERE name = 'ai_translation_backfill_max_age_days'
+      AND value::integer > 20000;
+    SQL
+
+    execute <<~SQL
+      UPDATE site_settings
+      SET value = '0'
+      WHERE name = 'ai_translation_backfill_max_age_days'
+      AND value::integer < 0;
+    SQL
+  end
+
+  def down
+    raise ActiveRecord::IrreversibleMigration
+  end
+end

db/migrate/20250721192553_enable_ai_if_already_installed.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+class EnableAiIfAlreadyInstalled < ActiveRecord::Migration[7.2]
+  def up
+    installed_at = DB.query_single(<<~SQL)&.first
+      SELECT created_at FROM schema_migration_details WHERE version='20230224165056'
+    SQL
+
+    if installed_at && installed_at < 1.hour.ago
+      # The plugin was installed before we changed it to be disabled-by-default
+      # Therefore, if there is no existing database value, enable the plugin
+      execute <<~SQL
+        INSERT INTO site_settings(name, data_type, value, created_at, updated_at)
+        VALUES('discourse_ai_enabled', 5, 't', NOW(), NOW())
+        ON CONFLICT (name) DO NOTHING
+      SQL
+    end
+  end
+
+  def down
+    raise ActiveRecord::IrreversibleMigration
+  end
+end

db/migrate/20250722082515_add_index_to_ai_topics_embeddings.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class AddIndexToAiTopicsEmbeddings < ActiveRecord::Migration[7.2]
+  def up
+    add_index :ai_topics_embeddings, %i[topic_id model_id]
+  end
+
+  def down
+    raise ActiveRecord::IrreversibleMigration
+  end
+end

discourse_automation/llm_triage.rb

@@ -24,6 +24,7 @@ if defined?(DiscourseAutomation)
     field :max_post_tokens, component: :text
     field :stop_sequences, component: :text_list, required: false
     field :temperature, component: :text
+    field :max_output_tokens, component: :text
 
     # Actions
     field :category, component: :category
@@ -85,6 +86,9 @@ if defined?(DiscourseAutomation)
         temperature = temperature.to_f
       end
 
+      max_output_tokens = fields.dig("max_output_tokens", "value").to_i
+      max_output_tokens = nil if max_output_tokens <= 0
+
       max_post_tokens = nil if max_post_tokens <= 0
 
       stop_sequences = fields.dig("stop_sequences", "value")
@@ -122,6 +126,7 @@ if defined?(DiscourseAutomation)
           stop_sequences: stop_sequences,
           automation: self.automation,
           temperature: temperature,
+          max_output_tokens: max_output_tokens,
           action: context["action"],
         )
       rescue => e

lib/automation/llm_triage.rb

@@ -21,6 +21,7 @@ module DiscourseAi
         temperature: nil,
         whisper: nil,
         reply_persona_id: nil,
+        max_output_tokens: nil,
         action: nil
       )
         if category_id.blank? && tags.blank? && canned_reply.blank? && hide_topic.blank? &&
@@ -59,8 +60,8 @@ module DiscourseAi
         result =
           llm.generate(
             prompt,
+            max_tokens: max_output_tokens,
             temperature: temperature,
-            max_tokens: 700, # ~500 words
             user: Discourse.system_user,
             stop_sequences: stop_sequences,
             feature_name: "llm_triage",

lib/translation/base_translator.rb

@@ -46,12 +46,24 @@ module DiscourseAi
             topic: @topic,
             post: @post,
           )
+        max_tokens = get_max_tokens(text)
+        llm_args = { max_tokens: }
 
         result = +""
-        bot.reply(context) { |partial| result << partial }
+        bot.reply(context, llm_args:) { |partial| result << partial }
         result
       end
 
+      def get_max_tokens(text)
+        if text.length < 100
+          500
+        elsif text.length < 500
+          1000
+        else
+          text.length * 2
+        end
+      end
+
       def persona_setting
         raise NotImplementedError
       end

lib/translation/locale_normalizer.rb

@@ -1,50 +0,0 @@
-# frozen_string_literal: true
-
-module DiscourseAi
-  module Translation
-    class LocaleNormalizer
-      # Normalizes locale string, matching the list of I18n.locales where possible
-      # @param locale [String,Symbol] the locale to normalize
-      # @return [String] the normalized locale
-      def self.normalize_to_i18n(locale)
-        return nil if locale.blank?
-        locale = locale.to_s.gsub("-", "_")
-
-        i18n_pairs.each { |downcased, value| return value if locale.downcase == downcased }
-
-        locale
-      end
-
-      def self.is_same?(locale1, locale2)
-        return true if locale1 == locale2
-        locale1 = locale1.gsub("-", "_").downcase
-        locale2 = locale2.gsub("-", "_").downcase
-        locale1.split("_").first == locale2.split("_").first
-      end
-
-      private
-
-      def self.i18n_pairs
-        # they should look like this for the input to match against:
-        # {
-        #   "lowercased" => "actual",
-        #   "en" => "en",
-        #   "zh_cn" => "zh_CN",
-        #   "zh" => "zh_CN",
-        # }
-        @locale_map ||=
-          I18n
-            .available_locales
-            .reduce({}) do |output, sym|
-              locale = sym.to_s
-              output[locale.downcase] = locale
-              if locale.include?("_")
-                short = locale.split("_").first
-                output[short] = locale if output[short].blank?
-              end
-              output
-            end
-      end
-    end
-  end
-end

spec/configuration/feature_spec.rb

@@ -10,6 +10,8 @@ RSpec.describe DiscourseAi::Configuration::Feature do
     DiscourseAi::Completions::Llm.with_prepared_responses(["OK"]) { block.call }
   end
 
+  before { enable_current_plugin }
+
   describe "#llm_model" do
     context "when persona is not found" do
       it "returns nil when persona_id is invalid" do

spec/configuration/llm_enumerator_spec.rb

@@ -8,6 +8,8 @@ RSpec.describe DiscourseAi::Configuration::LlmEnumerator do
     Fabricate(:automation, script: "llm_report", name: "some automation", enabled: true)
   end
 
+  before { enable_current_plugin }
+
   describe "#values_for_serialization" do
     it "returns an array for that can be used for serialization" do
       fake_model.destroy!

spec/configuration/llm_validator_spec.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 RSpec.describe DiscourseAi::Configuration::LlmValidator do
+  before { enable_current_plugin }
+
   describe "#valid_value?" do
     context "when the parent module is enabled and we try to reset the selected model" do
       before do

spec/configuration/spam_detection_validator_spec.rb

@@ -3,6 +3,8 @@
 RSpec.describe DiscourseAi::Configuration::SpamDetectionValidator do
   let(:validator) { described_class.new }
 
+  before { enable_current_plugin }
+
   it "always returns true if setting the value to false" do
     expect(validator.valid_value?("f")).to eq(true)
   end

spec/db/migrate/20241031041242_migrate_sentiment_classification_result_format_spec.rb

@@ -8,11 +8,14 @@ require Rails.root.join(
 RSpec.describe MigrateSentimentClassificationResultFormat do
   let(:connection) { ActiveRecord::Base.connection }
 
-  before { connection.execute(<<~SQL) }
+  before do
+    enable_current_plugin
+    connection.execute(<<~SQL)
       INSERT INTO classification_results (model_used, classification, created_at, updated_at) VALUES
         ('sentiment', '{"neutral": 65, "negative": 20, "positive": 14}', NOW(), NOW()),
         ('emotion', '{"sadness": 10, "surprise": 15, "fear": 5, "anger": 20, "joy": 30, "disgust": 8, "neutral": 10}', NOW(), NOW());
     SQL
+  end
 
   after { connection.execute("DELETE FROM classification_results") }
 
@@ -21,7 +24,7 @@ RSpec.describe MigrateSentimentClassificationResultFormat do
 
     it "migrates sentiment classifications correctly" do
       sentiment_result = connection.execute(<<~SQL).first
-        SELECT * FROM classification_results 
+        SELECT * FROM classification_results
         WHERE model_used = 'cardiffnlp/twitter-roberta-base-sentiment-latest';
       SQL
 
@@ -32,7 +35,7 @@ RSpec.describe MigrateSentimentClassificationResultFormat do
 
     it "migrates emotion classifications correctly" do
       emotion_result = connection.execute(<<~SQL).first
-        SELECT * FROM classification_results 
+        SELECT * FROM classification_results
         WHERE model_used = 'j-hartmann/emotion-english-distilroberta-base';
       SQL
 

spec/db/migrate/20250125162658_fix_broken_open_ai_embeddings_config_spec.rb

@@ -21,6 +21,8 @@ RSpec.describe FixBrokenOpenAiEmbeddingsConfig do
     ).first
   end
 
+  before { enable_current_plugin }
+
   describe "#up" do
     context "when embeddings are already configured" do
       fab!(:embedding_definition)

spec/db/migrate/20250127145305_clean_unused_embedding_search_indexes_spec.rb

@@ -8,6 +8,8 @@ require Rails.root.join(
 RSpec.describe CleanUnusedEmbeddingSearchIndexes do
   let(:connection) { ActiveRecord::Base.connection }
 
+  before { enable_current_plugin }
+
   describe "#up" do
     before do
       # Copied from 20241008054440_create_binary_indexes_for_embeddings

spec/jobs/regular/detect_translate_post_spec.rb

@@ -7,7 +7,7 @@ describe Jobs::DetectTranslatePost do
   let(:locales) { %w[en ja] }
 
   before do
-    SiteSetting.discourse_ai_enabled = true
+    enable_current_plugin
     Fabricate(:fake_model).tap do |fake_llm|
       SiteSetting.public_send("ai_translation_model=", "custom:#{fake_llm.id}")
     end

spec/jobs/regular/detect_translate_topic_spec.rb

@@ -7,7 +7,7 @@ describe Jobs::DetectTranslateTopic do
   let(:locales) { %w[en ja] }
 
   before do
-    SiteSetting.discourse_ai_enabled = true
+    enable_current_plugin
     Fabricate(:fake_model).tap do |fake_llm|
       SiteSetting.public_send("ai_translation_model=", "custom:#{fake_llm.id}")
     end
@@ -32,7 +32,6 @@ describe Jobs::DetectTranslateTopic do
   end
 
   it "detects locale" do
-    SiteSetting.discourse_ai_enabled = true
     allow(DiscourseAi::Translation::TopicLocaleDetector).to receive(:detect_locale).with(
       topic,
     ).and_return("zh_CN")

spec/jobs/regular/digest_rag_upload_spec.rb

@@ -20,6 +20,8 @@ RSpec.describe Jobs::DigestRagUpload do
   end
 
   before do
+    enable_current_plugin
+
     SiteSetting.ai_embeddings_selected_model = cloudflare_embedding_def.id
     SiteSetting.ai_embeddings_enabled = true
     SiteSetting.authorized_extensions = "txt"

spec/jobs/regular/fast_track_topic_gist_spec.rb

@@ -7,6 +7,7 @@ RSpec.describe Jobs::FastTrackTopicGist do
     fab!(:post_2) { Fabricate(:post, topic: topic_1, post_number: 2) }
 
     before do
+      enable_current_plugin
       assign_fake_provider_to(:ai_summarization_model)
       SiteSetting.ai_summarization_enabled = true
       SiteSetting.ai_summary_gists_enabled = true

spec/jobs/regular/generate_inferred_concepts_spec.rb

@@ -5,7 +5,10 @@ RSpec.describe Jobs::GenerateInferredConcepts do
   fab!(:post)
   fab!(:concept) { Fabricate(:inferred_concept, name: "programming") }
 
-  before { SiteSetting.inferred_concepts_enabled = true }
+  before do
+    enable_current_plugin
+    SiteSetting.inferred_concepts_enabled = true
+  end
 
   describe "#execute" do
     it "does nothing with blank item_ids" do

spec/jobs/regular/generate_rag_embeddings_spec.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 RSpec.describe Jobs::GenerateRagEmbeddings do
+  before { enable_current_plugin }
+
   describe "#execute" do
     fab!(:vector_def) { Fabricate(:embedding_definition) }
 

spec/jobs/regular/localize_categories_spec.rb

@@ -10,7 +10,7 @@ describe Jobs::LocalizeCategories do
   end
 
   before do
-    SiteSetting.discourse_ai_enabled = true
+    enable_current_plugin
     Fabricate(:fake_model).tap do |fake_llm|
       SiteSetting.public_send("ai_translation_model=", "custom:#{fake_llm.id}")
     end

spec/jobs/regular/localize_posts_spec.rb

@@ -7,7 +7,7 @@ describe Jobs::LocalizePosts do
   let(:locales) { %w[en ja de] }
 
   before do
-    SiteSetting.discourse_ai_enabled = true
+    enable_current_plugin
     Fabricate(:fake_model).tap do |fake_llm|
       SiteSetting.public_send("ai_translation_model=", "custom:#{fake_llm.id}")
     end