Merge pull request #9 from sshivaditya2019/development

.cspell.json

@@ -22,7 +22,11 @@
     "knip",
     "mischeck",
     "commentbody",
-    "issuebody"
+    "issuebody",
+    "voyageai",
+    "vectordump",
+    "payloadobject",
+    "markdownit"
   ],
   "dictionaries": ["typescript", "node", "software-terms"],
   "import": ["@cspell/dict-typescript/cspell-ext.json", "@cspell/dict-node/cspell-ext.json", "@cspell/dict-software-terms"],

.dev.vars.example

@@ -1,3 +1,3 @@
 SUPABASE_URL=""
 SUPABASE_KEY=""
-OPENAI_API_KEY=""
+VOYAGEAI_API_KEY=""

.env.example

@@ -1,3 +1,3 @@
 SUPABASE_URL=
 SUPABASE_KEY=
-OPENAI_API_KEY=
+VOYAGEAI_API_KEY=

.github/workflows/compute.yml

@@ -24,7 +24,7 @@ jobs:
     env:
       SUPABASE_URL: ${{ secrets.SUPABASE_URL }}
       SUPABASE_KEY: ${{ secrets.SUPABASE_KEY }}
-      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+      VOYAGEAI_API_KEY: ${{secrets.VOYAGEAI_API_KEY}}
 
     steps:
       - uses: actions/checkout@v4
@@ -43,4 +43,4 @@ jobs:
         env:
           SUPABASE_URL: ${{ secrets.SUPABASE_URL }}
           SUPABASE_KEY: ${{ secrets.SUPABASE_KEY }}
-          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          VOYAGEAI_API_KEY: ${{secrets.VOYAGEAI_API_KEY}}

README.md

@@ -7,15 +7,16 @@ This is a plugin for [Ubiquibot](https://github.com/ubiquity/ubiquibot-kernel).
 To set up the `.dev.vars` file, you will need to provide the following variables:
 - `SUPABASE_URL`: The URL for your Supabase instance.
 - `SUPABASE_KEY`: The key for your Supabase instance.
-- `OPENAI_API_KEY`: The API key for OpenAI.
+- `VOYAGEAI_API_KEY`: The API key for Voyage.
 
 ## Usage
-- Add the following to your `.ubiquibot.config.yml` file with the appropriate URL:
+- Add the following to your `.ubiquibot-config.yml` file with the appropriate URL:
 ```javascript
   -plugin: http://127.0.0.1:4000
-      runsOn: [ "issue_comment.created", "issue_comment.edited", "issue_comment.deleted" ]
+      runsOn: [ "issue_comment.created", "issue_comment.edited", "issue_comment.deleted" , "issues.opened", "issues.edited", "issues.deleted"]
 ```
 
+
 ## Testing Locally
 - Run `yarn install` to install the dependencies.
 - Run `yarn worker` to start the server.

eslint.config.mjs

@@ -9,7 +9,7 @@ export default tsEslint.config({
     "@typescript-eslint": tsEslint.plugin,
     "check-file": checkFile,
   },
-  ignores: [".github/knip.ts", "src/types/database.ts"],
+  ignores: [".github/knip.ts", "src/types/database.ts", "src/adapters/utils/markdown-to-plaintext.ts"],
   extends: [eslint.configs.recommended, ...tsEslint.configs.recommended, sonarjs.configs.recommended],
   languageOptions: {
     parser: tsEslint.parser,

manifest.json

@@ -1,5 +1,5 @@
 {
   "name": "@ubiquity-os/comment-vector-embeddings",
   "description": "Issue comment plugin for Ubiquibot. It enables the storage, updating, and deletion of issue comment embeddings.",
-  "ubiquity:listeners": ["issue_comment.created", "issue_comment.edited", "issue_comment.deleted"]
+  "ubiquity:listeners": ["issue_comment.created", "issue_comment.edited", "issue_comment.deleted", "issues.opened", "issues.edited", "issues.deleted"]
 }

package.json

@@ -17,7 +17,7 @@
     "knip-ci": "knip --no-exit-code --reporter json --config .github/knip.ts",
     "prepare": "husky install",
     "test": "jest --setupFiles dotenv/config --coverage",
-    "worker": "wrangler dev --env dev --port 4000",
+    "worker": "wrangler dev --env dev --port 5000",
     "supabase:generate:local": "supabase gen types typescript --local > src/types/database.ts",
     "supabase:generate:remote": "cross-env-shell \"supabase gen types typescript --project-id $SUPABASE_PROJECT_ID --schema public > src/types/database.ts\""
   },
@@ -35,10 +35,13 @@
     "@octokit/webhooks": "13.2.7",
     "@sinclair/typebox": "0.32.33",
     "@supabase/supabase-js": "^2.45.2",
+    "@types/markdown-it": "^14.1.2",
     "@ubiquity-dao/ubiquibot-logger": "^1.3.0",
     "dotenv": "16.4.5",
-    "openai": "^4.56.0",
-    "typebox-validators": "0.3.5"
+    "markdown-it": "^14.1.0",
+    "markdown-it-plain-text": "^0.3.0",
+    "typebox-validators": "0.3.5",
+    "voyageai": "^0.0.1-5"
   },
   "devDependencies": {
     "@commitlint/cli": "19.3.0",

src/adapters/index.ts

@@ -2,19 +2,21 @@ import { SupabaseClient } from "@supabase/supabase-js";
 import { Context } from "../types";
 import { Comment } from "./supabase/helpers/comment";
 import { SuperSupabase } from "./supabase/helpers/supabase";
-import { SuperOpenAi } from "./openai/helpers/openai";
-import OpenAI from "openai";
-import { Embedding } from "./openai/helpers/embedding";
+import { Embedding as VoyageEmbedding } from "./voyage/helpers/embedding";
+import { SuperVoyage } from "./voyage/helpers/voyage";
+import { VoyageAIClient } from "voyageai";
+import { Issues } from "./supabase/helpers/issues";
 
-export function createAdapters(supabaseClient: SupabaseClient, openai: OpenAI, context: Context) {
+export function createAdapters(supabaseClient: SupabaseClient, voyage: VoyageAIClient, context: Context) {
   return {
     supabase: {
       comment: new Comment(supabaseClient, context),
+      issue: new Issues(supabaseClient, context),
       super: new SuperSupabase(supabaseClient, context),
     },
-    openai: {
-      embedding: new Embedding(openai, context),
-      super: new SuperOpenAi(openai, context),
+    voyage: {
+      embedding: new VoyageEmbedding(voyage, context),
+      super: new SuperVoyage(voyage, context),
     },
   };
 }

src/adapters/supabase/helpers/comment.ts

@@ -1,10 +1,14 @@
 import { SupabaseClient } from "@supabase/supabase-js";
 import { SuperSupabase } from "./supabase";
 import { Context } from "../../../types/context";
+import { markdownToPlainText } from "../../utils/markdown-to-plaintext";
 
 export interface CommentType {
-  id: number;
-  body: string;
+  id: string;
+  markdown?: string;
+  author_id: number;
+  created_at: string;
+  modified_at: string;
   embedding: number[];
 }
 
@@ -13,9 +17,16 @@ export class Comment extends SuperSupabase {
     super(supabase, context);
   }
 
-  async createComment(commentBody: string, commentId: number, issueBody: string) {
+  async createComment(
+    markdown: string | null,
+    commentNodeId: string,
+    authorId: number,
+    payload: Record<string, unknown> | null,
+    isPrivate: boolean,
+    issueId: string
+  ) {
     //First Check if the comment already exists
-    const { data, error } = await this.supabase.from("issue_comments").select("*").eq("id", commentId);
+    const { data, error } = await this.supabase.from("issue_comments").select("*").eq("id", commentNodeId);
     if (error) {
       this.context.logger.error("Error creating comment", error);
       return;
@@ -25,10 +36,16 @@ export class Comment extends SuperSupabase {
       return;
     } else {
       //Create the embedding for this comment
-      const embedding = await this.context.adapters.openai.embedding.createEmbedding(commentBody);
+      const embedding = await this.context.adapters.voyage.embedding.createEmbedding(markdown);
+      let plaintext: string | null = markdownToPlainText(markdown || "");
+      if (isPrivate) {
+        markdown = null as string | null;
+        payload = null as Record<string, unknown> | null;
+        plaintext = null as string | null;
+      }
       const { error } = await this.supabase
         .from("issue_comments")
-        .insert([{ id: commentId, commentbody: commentBody, issuebody: issueBody, embedding: embedding }]);
+        .insert([{ id: commentNodeId, markdown, plaintext, author_id: authorId, payload, embedding: embedding, issue_id: issueId }]);
       if (error) {
         this.context.logger.error("Error creating comment", error);
         return;
@@ -37,25 +54,34 @@ export class Comment extends SuperSupabase {
     this.context.logger.info("Comment created successfully");
   }
 
-  async updateComment(commentBody: string, commentId: number) {
+  async updateComment(markdown: string | null, commentNodeId: string, payload: Record<string, unknown> | null, isPrivate: boolean) {
     //Create the embedding for this comment
-    const embedding = Array.from(await this.context.adapters.openai.embedding.createEmbedding(commentBody));
-    const { error } = await this.supabase.from("issue_comments").update({ commentbody: commentBody, embedding: embedding }).eq("id", commentId);
+    const embedding = Array.from(await this.context.adapters.voyage.embedding.createEmbedding(markdown));
+    let plaintext: string | null = markdownToPlainText(markdown || "");
+    if (isPrivate) {
+      markdown = null as string | null;
+      payload = null as Record<string, unknown> | null;
+      plaintext = null as string | null;
+    }
+    const { error } = await this.supabase
+      .from("issue_comments")
+      .update({ markdown, plaintext, embedding: embedding, payload, modified_at: new Date() })
+      .eq("id", commentNodeId);
     if (error) {
       this.context.logger.error("Error updating comment", error);
     }
   }
 
-  async getComment(commentId: number): Promise<CommentType[] | null> {
-    const { data, error } = await this.supabase.from("issue_comments").select("*").eq("id", commentId);
+  async getComment(commentNodeId: string): Promise<CommentType[] | null> {
+    const { data, error } = await this.supabase.from("issue_comments").select("*").eq("id", commentNodeId);
     if (error) {
       this.context.logger.error("Error getting comment", error);
     }
     return data;
   }
 
-  async deleteComment(commentId: number) {
-    const { error } = await this.supabase.from("issue_comments").delete().eq("id", commentId);
+  async deleteComment(commentNodeId: string) {
+    const { error } = await this.supabase.from("issue_comments").delete().eq("id", commentNodeId);
     if (error) {
       this.context.logger.error("Error deleting comment", error);
     }

src/adapters/supabase/helpers/issues.ts

@@ -0,0 +1,88 @@
+import { SupabaseClient } from "@supabase/supabase-js";
+import { SuperSupabase } from "./supabase";
+import { Context } from "../../../types/context";
+import { markdownToPlainText } from "../../utils/markdown-to-plaintext";
+
+export interface IssueType {
+  id: string;
+  markdown?: string;
+  author_id: number;
+  created_at: string;
+  modified_at: string;
+  payloadObject: Record<string, unknown> | null;
+  embedding: number[];
+}
+
+export class Issues extends SuperSupabase {
+  constructor(supabase: SupabaseClient, context: Context) {
+    super(supabase, context);
+  }
+
+  async createIssue(issueNodeId: string, payload: Record<string, unknown> | null, isPrivate: boolean, markdown: string | null, authorId: number) {
+    //First Check if the issue already exists
+    const { data, error } = await this.supabase.from("issues").select("*").eq("id", issueNodeId);
+    if (error) {
+      this.context.logger.error("Error creating issue", error);
+      return;
+    }
+    if (data && data.length > 0) {
+      this.context.logger.info("Issue already exists");
+      return;
+    } else {
+      const embedding = await this.context.adapters.voyage.embedding.createEmbedding(markdown);
+      let plaintext: string | null = markdownToPlainText(markdown || "");
+      if (isPrivate) {
+        payload = null;
+        markdown = null;
+        plaintext = null;
+      }
+      const { error } = await this.supabase.from("issues").insert([{ id: issueNodeId, payload, markdown, plaintext, author_id: authorId, embedding }]);
+      if (error) {
+        this.context.logger.error("Error creating issue", error);
+        return;
+      }
+    }
+    this.context.logger.info("Issue created successfully");
+  }
+
+  async updateIssue(markdown: string | null, issueNodeId: string, payload: Record<string, unknown> | null, isPrivate: boolean) {
+    //Create the embedding for this comment
+    const embedding = Array.from(await this.context.adapters.voyage.embedding.createEmbedding(markdown));
+    let plaintext: string | null = markdownToPlainText(markdown || "");
+    if (isPrivate) {
+      markdown = null as string | null;
+      payload = null as Record<string, unknown> | null;
+      plaintext = null as string | null;
+    }
+    const { error } = await this.supabase
+      .from("issues")
+      .update({ markdown, plaintext, embedding: embedding, payload, modified_at: new Date() })
+      .eq("id", issueNodeId);
+    if (error) {
+      this.context.logger.error("Error updating comment", error);
+    }
+  }
+
+  async deleteIssue(issueNodeId: string) {
+    const { error } = await this.supabase.from("issues").delete().eq("id", issueNodeId);
+    if (error) {
+      this.context.logger.error("Error deleting comment", error);
+    }
+  }
+
+  async findSimilarIssues(markdown: string, threshold: number): Promise<IssueType[] | null> {
+    const embedding = await this.context.adapters.voyage.embedding.createEmbedding(markdown);
+    const { data, error } = await this.supabase
+      .from("issues")
+      .select("*")
+      .eq("type", "issue")
+      .textSearch("embedding", embedding.join(","))
+      .order("embedding", { foreignTable: "issues", ascending: false })
+      .lte("embedding", threshold);
+    if (error) {
+      this.context.logger.error("Error finding similar issues", error);
+      return [];
+    }
+    return data;
+  }
+}

src/adapters/utils/markdown-to-plaintext.ts

@@ -0,0 +1,18 @@
+import markdownit from "markdown-it";
+import plainTextPlugin from "markdown-it-plain-text";
+
+/**
+ * Converts a Markdown string to plain text.
+ * @param markdown
+ * @returns
+ */
+export function markdownToPlainText(markdown: string | null): string | null {
+  if (!markdown) {
+    return markdown;
+  }
+  const md = markdownit();
+  md.use(plainTextPlugin);
+  md.render(markdown);
+  //Package markdown-it-plain-text does not have types
+  return (md as any).plainText;
+}