DEV: Remove the short-lived prompt for locale detection

2024-11-28 20:18:55 +08:00
51 changed files with 252 additions and 1845 deletions
@@ -1,18 +1,18 @@
 import DiscourseRoute from "discourse/routes/discourse";

-export default class AdminPluginsShowDiscourseAiLlmsNew extends DiscourseRoute {
-  queryParams = {
+export default DiscourseRoute.extend({
+  queryParams: {
    llmTemplate: { refreshModel: true },
-  };
+  },

  async model() {
    const record = this.store.createRecord("ai-llm");
    record.provider_params = {};
    return record;
-  }
+  },

  setupController(controller, model) {
-    super.setupController(controller, model);
+    this._super(controller, model);
    controller.set(
      "allLlms",
      this.modelFor("adminPlugins.show.discourse-ai-llms")
@@ -21,5 +21,5 @@ export default class AdminPluginsShowDiscourseAiLlmsNew extends DiscourseRoute {
      "llmTemplate",
      this.paramsFor(this.routeName).llmTemplate || null
    );
-  }
-}
+  },
+});
@@ -1,19 +1,19 @@
 import DiscourseRoute from "discourse/routes/discourse";

-export default class AdminPluginsShowDiscourseAiLlmsShow extends DiscourseRoute {
+export default DiscourseRoute.extend({
  async model(params) {
    const allLlms = this.modelFor("adminPlugins.show.discourse-ai-llms");
    const id = parseInt(params.id, 10);
    const record = allLlms.findBy("id", id);
    record.provider_params = record.provider_params || {};
    return record;
-  }
+  },

  setupController(controller, model) {
-    super.setupController(controller, model);
+    this._super(controller, model);
    controller.set(
      "allLlms",
      this.modelFor("adminPlugins.show.discourse-ai-llms")
    );
-  }
-}
+  },
+});
@@ -1,7 +1,7 @@
 import { AUTO_GROUPS } from "discourse/lib/constants";
 import DiscourseRoute from "discourse/routes/discourse";

-export default class AdminPluginsShowDiscourseAiPersonasNew extends DiscourseRoute {
+export default DiscourseRoute.extend({
  async model() {
    const record = this.store.createRecord("ai-persona");
    record.set("allowed_group_ids", [AUTO_GROUPS.trust_level_0.id]);
@@ -13,13 +13,13 @@ export default class AdminPluginsShowDiscourseAiPersonasNew extends DiscourseRou
    record.set("allow_personal_messages", true);
    record.set("tool_details", false);
    return record;
-  }
+  },

  setupController(controller, model) {
-    super.setupController(controller, model);
+    this._super(controller, model);
    controller.set(
      "allPersonas",
      this.modelFor("adminPlugins.show.discourse-ai-personas")
    );
-  }
-}
+  },
+});
@@ -1,19 +1,19 @@
 import DiscourseRoute from "discourse/routes/discourse";

-export default class AdminPluginsShowDiscourseAiPersonasShow extends DiscourseRoute {
+export default DiscourseRoute.extend({
  async model(params) {
    const allPersonas = this.modelFor(
      "adminPlugins.show.discourse-ai-personas"
    );
    const id = parseInt(params.id, 10);
    return allPersonas.findBy("id", id);
-  }
+  },

  setupController(controller, model) {
-    super.setupController(controller, model);
+    this._super(controller, model);
    controller.set(
      "allPersonas",
      this.modelFor("adminPlugins.show.discourse-ai-personas")
    );
-  }
-}
+  },
+});
@@ -1,11 +0,0 @@
-import { service } from "@ember/service";
-import { ajax } from "discourse/lib/ajax";
-import DiscourseRoute from "discourse/routes/discourse";
-
-export default class DiscourseAiUsageRoute extends DiscourseRoute {
-  @service store;
-
-  model() {
-    return ajax("/admin/plugins/discourse-ai/ai-usage.json");
-  }
-}
@@ -1 +0,0 @@
-<AiUsage @model={{this.model}} />
@@ -1,27 +0,0 @@
-# frozen_string_literal: true
-
-module DiscourseAi
-  module Admin
-    class AiUsageController < ::Admin::AdminController
-      requires_plugin "discourse-ai"
-
-      def show
-        render json: AiUsageSerializer.new(create_report, root: false)
-      end
-
-      private
-
-      def create_report
-        report =
-          DiscourseAi::Completions::Report.new(
-            start_date: params[:start_date]&.to_date || 30.days.ago,
-            end_date: params[:end_date]&.to_date || Time.current,
-          )
-
-        report = report.filter_by_feature(params[:feature]) if params[:feature].present?
-        report = report.filter_by_model(params[:model]) if params[:model].present?
-        report
-      end
-    end
-  end
-end
@@ -9,7 +9,9 @@ module ::Jobs
      post = Post.find_by(id: post_id, post_type: Post.types[:regular])
      return if post&.raw.blank?

-      DiscourseAi::Sentiment::PostClassification.new.classify!(post)
+      DiscourseAi::PostClassificator.new(
+        DiscourseAi::Sentiment::SentimentClassification.new,
+      ).classify!(post)
    end
  end
 end
@@ -3,7 +3,6 @@
 class AiApiAuditLog < ActiveRecord::Base
  belongs_to :post
  belongs_to :topic
-  belongs_to :user

  module Provider
    OpenAI = 1
@@ -44,10 +43,3 @@ end
 #  feature_name         :string(255)
 #  language_model       :string(255)
 #  feature_context      :jsonb
-#  cached_tokens        :integer
-#
-# Indexes
-#
-#  index_ai_api_audit_logs_on_created_at_and_feature_name    (created_at,feature_name)
-#  index_ai_api_audit_logs_on_created_at_and_language_model  (created_at,language_model)
-#
@@ -9,23 +9,15 @@ class AiSummary < ActiveRecord::Base
  def self.store!(strategy, llm_model, summary, og_content, human:)
    content_ids = og_content.map { |c| c[:id] }

-    AiSummary
-      .upsert(
-        {
-          target_id: strategy.target.id,
-          target_type: strategy.target.class.name,
-          algorithm: llm_model.name,
-          content_range: (content_ids.first..content_ids.last),
-          summarized_text: summary,
-          original_content_sha: build_sha(content_ids.join),
-          summary_type: strategy.type,
-          origin: !!human ? origins[:human] : origins[:system],
-        },
-        unique_by: %i[target_id target_type summary_type],
-        update_only: %i[summarized_text original_content_sha algorithm origin content_range],
-      )
-      .first
-      .then { AiSummary.find_by(id: _1["id"]) }
+    AiSummary.create!(
+      target: strategy.target,
+      algorithm: llm_model.name,
+      content_range: (content_ids.first..content_ids.last),
+      summarized_text: summary,
+      original_content_sha: build_sha(content_ids.join),
+      summary_type: strategy.type,
+      origin: !!human ? origins[:human] : origins[:system],
+    )
  end

  def self.build_sha(joined_ids)
@@ -8,7 +8,6 @@ class CompletionPrompt < ActiveRecord::Base
  CUSTOM_PROMPT = -305
  EXPLAIN = -306
  ILLUSTRATE_POST = -308
-  DETECT_TEXT_LOCALE = -309

  enum :prompt_type, { text: 0, list: 1, diff: 2 }

@@ -1,69 +0,0 @@
-# frozen_string_literal: true
-
-class AiUsageSerializer < ApplicationSerializer
-  attributes :data, :features, :models, :users, :summary, :period
-
-  def data
-    object.tokens_by_period.as_json(
-      only: %i[period total_tokens total_cached_tokens total_request_tokens total_response_tokens],
-    )
-  end
-
-  def period
-    object.guess_period
-  end
-
-  def features
-    object.feature_breakdown.as_json(
-      only: %i[
-        feature_name
-        usage_count
-        total_tokens
-        total_cached_tokens
-        total_request_tokens
-        total_response_tokens
-      ],
-    )
-  end
-
-  def models
-    object.model_breakdown.as_json(
-      only: %i[
-        llm
-        usage_count
-        total_tokens
-        total_cached_tokens
-        total_request_tokens
-        total_response_tokens
-      ],
-    )
-  end
-
-  def users
-    object.user_breakdown.map do |user|
-      {
-        avatar_template: User.avatar_template(user.username, user.uploaded_avatar_id),
-        username: user.username,
-        usage_count: user.usage_count,
-        total_tokens: user.total_tokens,
-        total_cached_tokens: user.total_cached_tokens,
-        total_request_tokens: user.total_request_tokens,
-        total_response_tokens: user.total_response_tokens,
-      }
-    end
-  end
-
-  def summary
-    {
-      total_tokens: object.total_tokens,
-      total_cached_tokens: object.total_cached_tokens,
-      total_request_tokens: object.total_request_tokens,
-      total_response_tokens: object.total_response_tokens,
-      total_requests: object.total_requests,
-      date_range: {
-        start: object.start_date,
-        end: object.end_date,
-      },
-    }
-  end
-end
@@ -18,6 +18,5 @@ export default {
      this.route("new");
      this.route("show", { path: "/:id" });
    });
-    this.route("discourse-ai-usage", { path: "ai-usage" });
  },
 };
@@ -1,25 +1,22 @@
-import { computed } from "@ember/object";
-import { observes } from "@ember-decorators/object";
+import { computed, observer } from "@ember/object";
 import I18n from "discourse-i18n";
 import ComboBox from "select-kit/components/combo-box";
-import { selectKitOptions } from "select-kit/components/select-kit";

-@selectKitOptions({
-  filterable: true,
-})
-export default class AiLlmSelector extends ComboBox {
-  @observes("attrs.disabled")
-  _modelDisabledChanged() {
+export default ComboBox.extend({
+  _modelDisabledChanged: observer("attrs.disabled", function () {
    this.selectKit.options.set("disabled", this.get("attrs.disabled.value"));
-  }
+  }),

-  @computed
-  get content() {
+  content: computed(function () {
    return [
      {
        id: "blank",
        name: I18n.t("discourse_ai.ai_persona.no_llm_selected"),
      },
    ].concat(this.llms);
-  }
-}
+  }),
+
+  selectKitOptions: {
+    filterable: true,
+  },
+});
@@ -1,18 +1,17 @@
+import { observer } from "@ember/object";
 import { readOnly } from "@ember/object/computed";
-import { observes } from "@ember-decorators/object";
 import MultiSelectComponent from "select-kit/components/multi-select";
-import { selectKitOptions } from "select-kit/components/select-kit";

-@selectKitOptions({
-  filterable: true,
-})
-export default class AiToolSelector extends MultiSelectComponent {
-  @readOnly("tools") content;
-
-  value = "";
-
-  @observes("attrs.disabled")
-  _modelDisabledChanged() {
+export default MultiSelectComponent.extend({
+  _modelDisabledChanged: observer("attrs.disabled", function () {
    this.selectKit.options.set("disabled", this.get("attrs.disabled.value"));
-  }
-}
+  }),
+
+  content: readOnly("tools"),
+
+  value: "",
+
+  selectKitOptions: {
+    filterable: true,
+  },
+});
@@ -1,16 +1,10 @@
-import didInsert from "@ember/render-modifiers/modifiers/did-insert";
 import icon from "discourse-common/helpers/d-icon";
 import i18n from "discourse-common/helpers/i18n";

-function addResultClass(element) {
-  element.closest(".fps-result")?.classList.add("ai-result");
-}
-
 const SearchResultDecoration = <template>
  <div
    class="ai-result__icon"
    title={{i18n "discourse_ai.embeddings.ai_generated_result"}}
-    {{didInsert addResultClass}}
  >
    {{icon "discourse-sparkles"}}
  </div>
@@ -24,10 +24,6 @@ export default {
          label: "discourse_ai.tools.short_title",
          route: "adminPlugins.show.discourse-ai-tools",
        },
-        {
-          label: "discourse_ai.usage.short_title",
-          route: "adminPlugins.show.discourse-ai-usage",
-        },
      ]);
    });
  },
@@ -26,38 +26,34 @@ function attachHeaderIcon(api) {
 function initializeAIBotReplies(api) {
  initializePauseButton(api);

-  api.modifyClass(
-    "controller:topic",
-    (Superclass) =>
-      class extends Superclass {
-        onAIBotStreamedReply(data) {
-          streamPostText(this.model.postStream, data);
-        }
+  api.modifyClass("controller:topic", {
+    pluginId: "discourse-ai",

-        subscribe() {
-          super.subscribe();
+    onAIBotStreamedReply: function (data) {
+      streamPostText(this.model.postStream, data);
+    },
+    subscribe: function () {
+      this._super();

-          if (
-            this.model.isPrivateMessage &&
-            this.model.details.allowed_users &&
-            this.model.details.allowed_users.filter(isGPTBot).length >= 1
-          ) {
-            // we attempt to recover the last message in the bus
-            // so we subscribe at -2
-            this.messageBus.subscribe(
-              `discourse-ai/ai-bot/topic/${this.model.id}`,
-              this.onAIBotStreamedReply.bind(this),
-              -2
-            );
-          }
-        }
-
-        unsubscribe() {
-          this.messageBus.unsubscribe("discourse-ai/ai-bot/topic/*");
-          super.unsubscribe();
-        }
+      if (
+        this.model.isPrivateMessage &&
+        this.model.details.allowed_users &&
+        this.model.details.allowed_users.filter(isGPTBot).length >= 1
+      ) {
+        // we attempt to recover the last message in the bus
+        // so we subscribe at -2
+        this.messageBus.subscribe(
+          `discourse-ai/ai-bot/topic/${this.model.id}`,
+          this.onAIBotStreamedReply.bind(this),
+          -2
+        );
      }
-  );
+    },
+    unsubscribe: function () {
+      this.messageBus.unsubscribe("discourse-ai/ai-bot/topic/*");
+      this._super();
+    },
+  });
 }

 function initializePersonaDecorator(api) {
@@ -0,0 +1,9 @@
+import { apiInitializer } from "discourse/lib/api";
+
+export default apiInitializer("1.15.0", (api) => {
+  api.modifyClass("component:search-result-entry", {
+    pluginId: "discourse-ai",
+
+    classNameBindings: ["bulkSelectEnabled", "post.generatedByAi:ai-result"],
+  });
+});
@@ -47,16 +47,10 @@ export default {
        (Superclass) =>
          class extends Superclass {
            @tracked related_topics;
-            relatedTopicsCache = [];

            @cached
            get relatedTopics() {
-              // Used to keep related topics when a user scrolls up from the
-              // bottom of the topic and then scrolls back down
-              if (this.related_topics) {
-                this.relatedTopicsCache = this.related_topics;
-              }
-              return this.relatedTopicsCache?.map((topic) =>
+              return this.related_topics?.map((topic) =>
                this.store.createRecord("topic", topic)
              );
            }
@@ -272,15 +272,13 @@
 .edit-topic-title {
  .suggestion-button {
    margin: 0;
-    padding: 0.465rem;
+    padding: 0.45rem;
  }
 }

 #topic-title .edit-topic-title.showing-ai-suggestions {
  #edit-title {
    flex: 1 1 90%;
-    border-top-right-radius: 0;
-    border-bottom-right-radius: 0;
  }

  .suggest-titles-button {
@@ -42,30 +42,3 @@
    }
  }
 }
-
-#topic-title .edit-topic-title.showing-ai-suggestions {
-  .category-chooser {
-    flex: 1 1 90%;
-  }
-  .ai-category-suggester-trigger {
-    padding: 0.425em;
-  }
-  .ai-tag-suggester-trigger {
-    padding: 0.45em;
-  }
-}
-
-.ios-device #topic-title .edit-topic-title.showing-ai-suggestions button {
-  &.ai-title-suggester-trigger {
-    padding-top: 0.4em;
-    padding-bottom: 0.4em;
-  }
-  &.ai-category-suggester-trigger {
-    padding-top: 0.4em;
-    padding-bottom: 0.25em;
-  }
-  &.ai-tag-suggester-trigger {
-    padding-top: 0.4em;
-    padding-bottom: 0.3em;
-  }
-}
@@ -1,168 +0,0 @@
-.ai-usage {
-  --chart-response-color: rgba(75, 192, 192, 0.8);
-  --chart-request-color: rgba(153, 102, 255, 0.8);
-  --chart-cached-color: rgba(153, 102, 255, 0.4);
-
-  padding: 1em;
-
-  &__filters-dates {
-    display: flex;
-    flex-direction: column;
-    gap: 1em;
-    margin-bottom: 1em;
-  }
-
-  &__period-buttons {
-    display: flex;
-    gap: 0.5em;
-    align-items: center;
-
-    .btn {
-      padding: 0.5em 1em;
-
-      &.btn-primary {
-        background: var(--tertiary);
-        color: var(--secondary);
-      }
-    }
-  }
-
-  &__custom-date-pickers {
-    display: flex;
-    gap: 1em;
-    align-items: center;
-    margin-top: 0.5em;
-  }
-
-  &__filters {
-    margin-bottom: 2em;
-  }
-
-  &__filters-period {
-    display: flex;
-    align-items: center;
-    gap: 1em;
-  }
-
-  .d-date-time-input-range {
-    display: flex;
-    gap: 1em;
-    align-items: center;
-  }
-
-  .d-date-time-input-range .from {
-    margin: 0;
-  }
-
-  &__period-label {
-    font-weight: bold;
-  }
-
-  &__summary {
-    margin: 2em 0;
-    padding: 1.5em;
-    background: var(--primary-very-low);
-    border-radius: 0.5em;
-  }
-
-  &__summary-title {
-    margin-bottom: 1em;
-    color: var(--primary);
-    font-size: 1.2em;
-  }
-
-  &__summary-stats {
-    display: grid;
-    grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
-    gap: 1em;
-  }
-
-  &__summary-stat {
-    display: flex;
-    flex-direction: column;
-    padding: 1em;
-    background: var(--secondary);
-    border-radius: 0.25em;
-
-    .label {
-      color: var(--primary-medium);
-      font-size: 0.875em;
-      margin-bottom: 0.5em;
-    }
-
-    .value {
-      color: var(--primary);
-      font-size: 1.5em;
-      font-weight: bold;
-    }
-  }
-
-  &__charts {
-    margin-top: 2em;
-  }
-
-  &__chart {
-    position: relative;
-  }
-
-  &__chart-container {
-    margin-bottom: 2em;
-  }
-
-  &__chart-title {
-    margin-bottom: 1em;
-  }
-
-  &__breakdowns {
-    display: grid;
-    grid-template-columns: 1fr 1fr;
-    gap: 2em;
-    margin-top: 2em;
-
-    @media (max-width: 768px) {
-      grid-template-columns: 1fr;
-    }
-  }
-
-  &__features,
-  &__users,
-  &__models {
-    background: var(--primary-very-low);
-    padding: 1em;
-    border-radius: 0.5em;
-  }
-
-  &__features-title,
-  &__users-title,
-  &__models-title {
-    margin-bottom: 1em;
-  }
-
-  &__features-table,
-  &__users-table,
-  &__models-table {
-    width: 100%;
-    border-collapse: collapse;
-
-    th {
-      text-align: left;
-      padding: 0.5em;
-      border-bottom: 2px solid var(--primary-low);
-    }
-  }
-
-  &__features-row,
-  &__users-row,
-  &__models-row {
-    &:hover {
-      background: var(--primary-low);
-    }
-  }
-
-  &__features-cell,
-  &__users-cell,
-  &__models-cell {
-    padding: 0.5em;
-    border-bottom: 1px solid var(--primary-low);
-  }
-}
@@ -126,26 +126,6 @@ en:
      modals:
        select_option: "Select an option..."

-      usage:
-        short_title: "Usage"
-        summary: "Summary"
-        total_tokens: "Total tokens"
-        tokens_over_time: "Tokens over time"
-        features_breakdown: "Usage per feature"
-        feature: "Feature"
-        usage_count: "Usage count"
-        model: "Model"
-        models_breakdown: "Usage per model"
-        users_breakdown: "Usage per user"
-        all_features: "All features"
-        all_models: "All models"
-        username: "Username"
-        total_requests: "Total requests"
-        request_tokens: "Request tokens"
-        response_tokens: "Response tokens"
-        cached_tokens: "Cached tokens"
-
-
      ai_persona:
        tool_strategies:
          all: "Apply to all replies"
@@ -77,8 +77,6 @@ Discourse::Application.routes.draw do
    get "/rag-document-fragments/files/status",
        to: "discourse_ai/admin/rag_document_fragments#indexing_status_check"

-    get "/ai-usage", to: "discourse_ai/admin/ai_usage#show"
-
    resources :ai_llms,
              only: %i[index create show update destroy],
              path: "ai-llms",
@@ -200,20 +200,3 @@ CompletionPrompt.seed do |cp|
  cp.prompt_type = CompletionPrompt.prompt_types[:list]
  cp.messages = {}
 end
-
-CompletionPrompt.seed do |cp|
-  cp.id = -309
-  cp.name = "detect_text_locale"
-  cp.prompt_type = CompletionPrompt.prompt_types[:text]
-  cp.messages = {
-    insts: <<~TEXT,
-      I want you to act as a language expert, determining the locale for a set of text.
-      The locale is a language identifier, such as "en" for English, "de" for German, etc,
-      and can also include a region identifier, such as "en-GB" for British English, or "zh-Hans" for Simplified Chinese.
-      I will provide you with text, and you will determine the locale of the text.
-      You will find the text between <input></input> XML tags.
-      Include your locale between <output></output> XML tags.
-    TEXT
-    examples: [["<input>Hello my favourite colour is red</input>", "<output>en-GB</output>"]],
-  }
-end
@@ -1,9 +0,0 @@
-# frozen_string_literal: true
-
-class AddCachedTokensToAiApiAuditLog < ActiveRecord::Migration[7.2]
-  def change
-    add_column :ai_api_audit_logs, :cached_tokens, :integer
-    add_index :ai_api_audit_logs, %i[created_at feature_name]
-    add_index :ai_api_audit_logs, %i[created_at language_model]
-  end
-end
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class RemoveDetectTextLocalePrompt < ActiveRecord::Migration[7.0]
+  def up
+    execute "DELETE FROM completion_prompts WHERE id = -309"
+  end
+
+  def down
+    raise ActiveRecord::IrreversibleMigration
+  end
+end
@@ -1,37 +0,0 @@
-# frozen_string_literal: true
-
-class FixClassificationData < ActiveRecord::Migration[7.2]
-  def up
-    classifications = DB.query(<<~SQL)
-      SELECT id, classification
-      FROM classification_results
-      WHERE classification_type = 'sentiment'
-        AND SUBSTRING(LTRIM(classification::text), 1, 1) = '['
-    SQL
-
-    transformed =
-      classifications.reduce([]) do |memo, c|
-        hash_result = {}
-        c.classification.each { |r| hash_result[r["label"]] = r["score"] }
-
-        memo << { id: c.id, fixed_classification: hash_result }
-      end
-
-    transformed_json = transformed.to_json
-
-    DB.exec(<<~SQL, values: transformed_json)
-      UPDATE classification_results
-      SET classification = N.fixed_classification
-      FROM (
-        SELECT (value::jsonb->'id')::integer AS id, (value::jsonb->'fixed_classification')::jsonb AS fixed_classification
-        FROM jsonb_array_elements(:values::jsonb)
-      ) N
-      WHERE classification_results.id = N.id
-        AND classification_type = 'sentiment'
-    SQL
-  end
-
-  def down
-    raise ActiveRecord::IrreversibleMigration
-  end
-end
--- a/Show More
+++ b/Show More