tweaks to open llm implementation

testing mistral at the moment and did the minimum to get it going
tweaks to llama2 support in chat, fix message so it conforms to recommendation
2023-10-04 17:20:46 +11:00 · 2023-10-04 16:27:00 +11:00 · 2023-10-04 16:25:44 +11:00
888 changed files with 106202 additions and 1601588 deletions
@@ -1,7 +1,3 @@
-< 3.4.0.beta3-dev: decf1bb49d737ea15308400f22f89d1d1e71d13d
-< 3.4.0.beta1-dev: 9d887ad4ace8e33c3fe7dbb39237e882c08b4f0b
-< 3.3.0.beta5-dev: 4d8090002f6dcd8e34d41033606bf131fa221475
-< 3.3.0.beta2-dev: 61890b667c06299841ae88946f84a112f00060e1
-< 3.3.0.beta1-dev: d94234d89f50733678a5c62809445198a9fda74b
-3.2.999: d94234d89f50733678a5c62809445198a9fda74b
 3.1.999: c0415bb7eb878b1b7abf112d65cba981030df8df
+< 3.2.0.beta1-dev: 3e7c99de894b03e9161d9481da118fe893991a1d
+3.1.0.beta3: 057fbe1ce6
@@ -0,0 +1,3 @@
+{
+  "extends": "eslint-config-discourse"
+}
@@ -1,4 +1,3 @@
 node_modules
 /gems
 /auto_generated
-.env
@@ -1,2 +0,0 @@
-engine-strict = true
-auto-install-peers = false
@@ -1,21 +0,0 @@
-app/assets/stylesheets/vendor/
-documentation/
-package.json
-config/locales/**/*.yml
-!config/locales/**/*.en*.yml
-script/import_scripts/**/*.yml
-
-plugins/**/lib/javascripts/locale
-public/
-!/app/assets/javascripts/discourse/public
-vendor/
-app/assets/javascripts/discourse/tests/fixtures
-spec/
-node_modules/
-dist/
-tmp/
-
-**/*.rb
-**/*.html
-**/*.json
-**/*.md
@@ -0,0 +1 @@
+{}
@@ -1 +0,0 @@
-module.exports = require("@discourse/lint-configs/prettier");
@@ -1,4 +1,2 @@
 inherit_gem:
  rubocop-discourse: stree-compat.yml
-RSpec/NamedSubject:
-  Enabled: false
@@ -1,2 +1,2 @@
 --print-width=100
--plugins=plugin/trailing_comma,plugin/disable_auto_ternary
+--plugins=plugin/trailing_comma,disable_ternary
@@ -1 +0,0 @@
-module.exports = require("@discourse/lint-configs/template-lint");
@@ -0,0 +1,4 @@
+module.exports = {
+  plugins: ["ember-template-lint-plugin-discourse"],
+  extends: "discourse:recommended",
+};
@@ -5,4 +5,5 @@ source "https://rubygems.org"
 group :development do
  gem "rubocop-discourse"
  gem "syntax_tree"
+  gem "syntax_tree-disable_ternary"
 end
@@ -1,86 +1,50 @@
 GEM
  remote: https://rubygems.org/
  specs:
-    activesupport (7.1.3.4)
-      base64
-      bigdecimal
-      concurrent-ruby (~> 1.0, >= 1.0.2)
-      connection_pool (>= 2.2.5)
-      drb
-      i18n (>= 1.6, < 2)
-      minitest (>= 5.1)
-      mutex_m
-      tzinfo (~> 2.0)
    ast (2.4.2)
-    base64 (0.2.0)
-    bigdecimal (3.1.8)
-    concurrent-ruby (1.3.3)
-    connection_pool (2.4.1)
-    drb (2.2.1)
-    i18n (1.14.5)
-      concurrent-ruby (~> 1.0)
-    json (2.7.2)
-    language_server-protocol (3.17.0.3)
-    minitest (5.24.1)
-    mutex_m (0.2.0)
-    parallel (1.25.1)
-    parser (3.3.3.0)
+    json (2.6.2)
+    parallel (1.22.1)
+    parser (3.1.2.1)
      ast (~> 2.4.1)
-      racc
-    prettier_print (1.2.1)
-    racc (1.8.0)
-    rack (3.1.6)
+    prettier_print (1.2.0)
    rainbow (3.1.1)
-    regexp_parser (2.9.2)
-    rexml (3.3.9)
-    rubocop (1.64.1)
+    regexp_parser (2.6.0)
+    rexml (3.2.5)
+    rubocop (1.36.0)
      json (~> 2.3)
-      language_server-protocol (>= 3.17.0)
      parallel (~> 1.10)
-      parser (>= 3.3.0.2)
+      parser (>= 3.1.2.1)
      rainbow (>= 2.2.2, < 4.0)
      regexp_parser (>= 1.8, < 3.0)
      rexml (>= 3.2.5, < 4.0)
-      rubocop-ast (>= 1.31.1, < 2.0)
+      rubocop-ast (>= 1.20.1, < 2.0)
      ruby-progressbar (~> 1.7)
-      unicode-display_width (>= 2.4.0, < 3.0)
-    rubocop-ast (1.31.3)
-      parser (>= 3.3.1.0)
-    rubocop-capybara (2.21.0)
-      rubocop (~> 1.41)
-    rubocop-discourse (3.8.1)
-      activesupport (>= 6.1)
-      rubocop (>= 1.59.0)
-      rubocop-capybara (>= 2.0.0)
-      rubocop-factory_bot (>= 2.0.0)
-      rubocop-rails (>= 2.25.0)
-      rubocop-rspec (>= 3.0.1)
-      rubocop-rspec_rails (>= 2.30.0)
-    rubocop-factory_bot (2.26.1)
-      rubocop (~> 1.61)
-    rubocop-rails (2.25.1)
-      activesupport (>= 4.2.0)
-      rack (>= 1.1)
-      rubocop (>= 1.33.0, < 2.0)
-      rubocop-ast (>= 1.31.1, < 2.0)
-    rubocop-rspec (3.0.2)
-      rubocop (~> 1.61)
-    rubocop-rspec_rails (2.30.0)
-      rubocop (~> 1.61)
-      rubocop-rspec (~> 3, >= 3.0.1)
-    ruby-progressbar (1.13.0)
-    syntax_tree (6.2.0)
+      unicode-display_width (>= 1.4.0, < 3.0)
+    rubocop-ast (1.21.0)
+      parser (>= 3.1.1.0)
+    rubocop-discourse (3.0)
+      rubocop (>= 1.1.0)
+      rubocop-rspec (>= 2.0.0)
+    rubocop-rspec (2.13.2)
+      rubocop (~> 1.33)
+    ruby-progressbar (1.11.0)
+    syntax_tree (5.1.0)
      prettier_print (>= 1.2.0)
-    tzinfo (2.0.6)
-      concurrent-ruby (~> 1.0)
-    unicode-display_width (2.5.0)
+    syntax_tree-disable_ternary (1.0.0)
+    unicode-display_width (2.3.0)

 PLATFORMS
+  arm64-darwin-20
  ruby
+  x86_64-darwin-18
+  x86_64-darwin-19
+  x86_64-darwin-20
+  x86_64-linux

 DEPENDENCIES
  rubocop-discourse
  syntax_tree
+  syntax_tree-disable_ternary

 BUNDLED WITH
-   2.5.10
+   2.3.10
@@ -1,25 +0,0 @@
-import DiscourseRoute from "discourse/routes/discourse";
-
-export default class AdminPluginsShowDiscourseAiLlmsNew extends DiscourseRoute {
-  queryParams = {
-    llmTemplate: { refreshModel: true },
-  };
-
-  async model() {
-    const record = this.store.createRecord("ai-llm");
-    record.provider_params = {};
-    return record;
-  }
-
-  setupController(controller, model) {
-    super.setupController(controller, model);
-    controller.set(
-      "allLlms",
-      this.modelFor("adminPlugins.show.discourse-ai-llms")
-    );
-    controller.set(
-      "llmTemplate",
-      this.paramsFor(this.routeName).llmTemplate || null
-    );
-  }
-}
@@ -1,19 +0,0 @@
-import DiscourseRoute from "discourse/routes/discourse";
-
-export default class AdminPluginsShowDiscourseAiLlmsShow extends DiscourseRoute {
-  async model(params) {
-    const allLlms = this.modelFor("adminPlugins.show.discourse-ai-llms");
-    const id = parseInt(params.id, 10);
-    const record = allLlms.findBy("id", id);
-    record.provider_params = record.provider_params || {};
-    return record;
-  }
-
-  setupController(controller, model) {
-    super.setupController(controller, model);
-    controller.set(
-      "allLlms",
-      this.modelFor("adminPlugins.show.discourse-ai-llms")
-    );
-  }
-}
@@ -1,7 +0,0 @@
-import DiscourseRoute from "discourse/routes/discourse";
-
-export default class DiscourseAiAiLlmsRoute extends DiscourseRoute {
-  model() {
-    return this.store.findAll("ai-llm");
-  }
-}
@@ -1,25 +0,0 @@
-import { AUTO_GROUPS } from "discourse/lib/constants";
-import DiscourseRoute from "discourse/routes/discourse";
-
-export default class AdminPluginsShowDiscourseAiPersonasNew extends DiscourseRoute {
-  async model() {
-    const record = this.store.createRecord("ai-persona");
-    record.set("allowed_group_ids", [AUTO_GROUPS.trust_level_0.id]);
-    record.set("rag_uploads", []);
-    // these match the defaults on the table
-    record.set("rag_chunk_tokens", 374);
-    record.set("rag_chunk_overlap_tokens", 10);
-    record.set("rag_conversation_chunks", 10);
-    record.set("allow_personal_messages", true);
-    record.set("tool_details", false);
-    return record;
-  }
-
-  setupController(controller, model) {
-    super.setupController(controller, model);
-    controller.set(
-      "allPersonas",
-      this.modelFor("adminPlugins.show.discourse-ai-personas")
-    );
-  }
-}
@@ -1,19 +0,0 @@
-import DiscourseRoute from "discourse/routes/discourse";
-
-export default class AdminPluginsShowDiscourseAiPersonasShow extends DiscourseRoute {
-  async model(params) {
-    const allPersonas = this.modelFor(
-      "adminPlugins.show.discourse-ai-personas"
-    );
-    const id = parseInt(params.id, 10);
-    return allPersonas.findBy("id", id);
-  }
-
-  setupController(controller, model) {
-    super.setupController(controller, model);
-    controller.set(
-      "allPersonas",
-      this.modelFor("adminPlugins.show.discourse-ai-personas")
-    );
-  }
-}
@@ -1,7 +0,0 @@
-import DiscourseRoute from "discourse/routes/discourse";
-
-export default class DiscourseAiAiPersonasRoute extends DiscourseRoute {
-  model() {
-    return this.store.findAll("ai-persona");
-  }
-}
@@ -1,15 +0,0 @@
-import DiscourseRoute from "discourse/routes/discourse";
-
-export default class DiscourseAiToolsNewRoute extends DiscourseRoute {
-  async model() {
-    return this.store.createRecord("ai-tool");
-  }
-
-  setupController(controller) {
-    super.setupController(...arguments);
-    const toolsModel = this.modelFor("adminPlugins.show.discourse-ai-tools");
-
-    controller.set("allTools", toolsModel);
-    controller.set("presets", toolsModel.resultSetMeta.presets);
-  }
-}
@@ -1,18 +0,0 @@
-import DiscourseRoute from "discourse/routes/discourse";
-
-export default class DiscourseAiToolsShowRoute extends DiscourseRoute {
-  async model(params) {
-    const allTools = this.modelFor("adminPlugins.show.discourse-ai-tools");
-    const id = parseInt(params.id, 10);
-
-    return allTools.find((tool) => tool.id === id);
-  }
-
-  setupController(controller) {
-    super.setupController(...arguments);
-    const toolsModel = this.modelFor("adminPlugins.show.discourse-ai-tools");
-
-    controller.set("allTools", toolsModel);
-    controller.set("presets", toolsModel.resultSetMeta.presets);
-  }
-}
@@ -1,10 +0,0 @@
-import { service } from "@ember/service";
-import DiscourseRoute from "discourse/routes/discourse";
-
-export default class DiscourseAiToolsRoute extends DiscourseRoute {
-  @service store;
-
-  model() {
-    return this.store.findAll("ai-tool");
-  }
-}
@@ -1,11 +0,0 @@
-import { service } from "@ember/service";
-import { ajax } from "discourse/lib/ajax";
-import DiscourseRoute from "discourse/routes/discourse";
-
-export default class DiscourseAiUsageRoute extends DiscourseRoute {
-  @service store;
-
-  model() {
-    return ajax("/admin/plugins/discourse-ai/ai-usage.json");
-  }
-}
@@ -1 +0,0 @@
-<AiLlmsListEditor @llms={{this.model}} />
@@ -1,5 +0,0 @@
-<AiLlmsListEditor
-  @llms={{this.allLlms}}
-  @currentLlm={{this.model}}
-  @llmTemplate={{this.llmTemplate}}
-/>
@@ -1 +0,0 @@
-<AiLlmsListEditor @llms={{this.allLlms}} @currentLlm={{this.model}} />
@@ -1 +0,0 @@
-<AiPersonaListEditor @personas={{this.model}} />
@@ -1,4 +0,0 @@
-<AiPersonaListEditor
-  @personas={{this.allPersonas}}
-  @currentPersona={{this.model}}
-/>
@@ -1,4 +0,0 @@
-<AiPersonaListEditor
-  @personas={{this.allPersonas}}
-  @currentPersona={{this.model}}
-/>
@@ -1 +0,0 @@
-<AiToolListEditor @tools={{this.model}} />
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Sam Saffron	13fca098d1	tweaks to open llm implementation testing mistral at the moment and did the minimum to get it going	2023-10-04 17:20:46 +11:00
Sam Saffron	d76309fa17	tweaks to llama2 support in chat, fix message so it conforms to recommendation by facebook (add <s> )	2023-10-04 16:27:00 +11:00
Sam Saffron	facce51f98	FEATURE: add support for an open LLM implementation This is very much a work in progress, testing at the moment on: TheBloke/Llama-2-7B-chat-fp16	2023-10-04 16:25:44 +11:00
				`@@ -1 +0,0 @@`
				`module.exports = require("@discourse/lint-configs/prettier");`
				`@@ -1 +0,0 @@`
				`<AiLlmsListEditor @llms={{this.allLlms}} @currentLlm={{this.model}} />`
				`@@ -1 +0,0 @@`
				`<AiPersonaListEditor @personas={{this.model}} />`