Add Reviewer agent to summarize PR. (#54)

* Add reviewer agent for pull request

app/api/webhook/github/route.ts

@@ -1,5 +1,6 @@
 import {App} from '@octokit/app'
 import maige from '~/agents/maige'
+import reviewer from '~/agents/reviewer'
 import {GITHUB} from '~/constants'
 import prisma from '~/prisma'
 import {stripe} from '~/stripe'
@@ -153,6 +154,48 @@ export const POST = async (req: Request) => {
 	/**
 	 * Issue-related events. We care about new issues and comments.
 	 */
+	const {
+		comment,
+		issue,
+		sender: {login: sender},
+		installation: {id: instanceId}
+	} = payload
+
+	if (sender.includes('maige'))
+		return new Response('Comment by Maige', {status: 202})
+
+	if (comment && !comment.body.toLowerCase().includes('maige'))
+		return new Response('Irrelevant comment', {status: 202})
+
+	if (issue.pull_request) {
+		const {
+			pull_request: {diff_url: diffUrl},
+			node_id: pullId
+		} = issue
+
+		// Get GitHub app instance access token
+		const app = new App({
+			appId: process.env.GITHUB_APP_ID || '',
+			privateKey: process.env.GITHUB_PRIVATE_KEY || ''
+		})
+
+		const octokit = await app.getInstallationOctokit(instanceId)
+
+		const response = await fetch(diffUrl)
+
+		if (!response.ok) return new Response('Could not fetch diff', {status: 401})
+
+		const data = await response.text()
+
+		await reviewer({
+			octokit: octokit,
+			input: `Instruction: ${comment?.body}\n\nPR Diff:\n${data}`,
+			pullId
+		})
+
+		return new Response('Reviewed PR', {status: 200})
+	}
+
 	if (
 		!(
 			(action === 'opened' && payload?.issue) ||
@@ -173,18 +216,9 @@ export const POST = async (req: Request) => {
 			node_id: repoId,
 			name,
 			owner: {login: owner}
-		},
-		sender: {login: sender},
-		installation: {id: instanceId},
-		comment
+		}
 	} = payload
 
-	if (sender.includes('maige'))
-		return new Response('Comment by Maige', {status: 202})
-
-	if (comment && !comment.body.toLowerCase().includes('maige'))
-		return new Response('Irrelevant comment', {status: 202})
-
 	const existingLabelNames = existingLabels?.map((l: Label) => l.name)
 
 	const customer = await prisma.customer.findUnique({
@@ -344,7 +378,8 @@ ${isComment ? `Comment by @${comment.user.login}: ${comment?.body}.` : ''}
 			octokit,
 			prisma,
 			customerId,
-			repoName: name
+			repoName: `${owner}/${name}`,
+			allLabels
 		})
 
 		return new Response('ok', {status: 200})

lib/agents/engineer.ts

@@ -5,14 +5,13 @@ import {SerpAPI} from 'langchain/tools'
 import env from '~/env.mjs'
 import commentTool from '~/tools/comment'
 import execTool from '~/tools/exec'
-import githubTool from '~/tools/github'
 import updateInstructionsTool from '~/tools/updateInstructions'
 import {isDev} from '~/utils'
 
 const model = new ChatOpenAI({
 	modelName: 'gpt-4-1106-preview',
 	openAIApiKey: env.OPENAI_API_KEY,
-	temperature: 0.7
+	temperature: 0.3
 })
 
 export default async function engineer({
@@ -30,16 +29,34 @@ export default async function engineer({
 }) {
 	const shell = await Sandbox.create({
 		apiKey: env.E2B_API_KEY,
-		id: 'Nodejs',
 		onStderr: data => console.error(data.line),
 		onStdout: data => console.log(data.line)
 	})
 
+	function preCmdCallback(cmd: string) {
+		const tokenB64 = btoa(`pat:${env.GITHUB_ACCESS_TOKEN}`)
+		const authFlag = `-c http.extraHeader="AUTHORIZATION: basic ${tokenB64}"`
+
+		// Replace only first occurrence to avoid prompt injection
+		// Otherwise "git log && echo 'git '" would print the token
+		return cmd.replace('git ', `git ${authFlag} `)
+	}
+
+	const cloneName = `maige-${repoName.split('/')[1]}`
+
+	const repoSetup = preCmdCallback(
+		`git config --global user.email "${env.GITHUB_EMAIL}" && git config --global user.name "${env.GITHUB_USERNAME}" && git clone https://github.com/${repoName}.git ${cloneName} && cd ${cloneName} && git log -n 3`
+	)
+
+	const clone = await shell.process.start({
+		cmd: repoSetup
+	})
+	await clone.wait()
+
 	const tools = [
 		new SerpAPI(),
 		commentTool({octokit}),
 		updateInstructionsTool({octokit, prisma, customerId, repoName}),
-		githubTool({octokit}),
 		execTool({
 			name: 'shell',
 			description: 'Executes a shell command.',
@@ -48,7 +65,7 @@ export default async function engineer({
 		execTool({
 			name: 'git',
 			description:
-				'Executes a shell command with git logged in. Commands must begin with "git ".',
+				'Executes a shell command with git already logged in and configured. Commands must begin with "git ".',
 			setupCmd: `git config --global user.email "${env.GITHUB_EMAIL}" && git config --global user.name "${env.GITHUB_USERNAME}"`,
 			preCmdCallback: (cmd: string) => {
 				const tokenB64 = btoa(`pat:${env.GITHUB_ACCESS_TOKEN}`)
@@ -64,16 +81,19 @@ export default async function engineer({
 
 	const prefix = `You are a senior AI engineer.
 You use the internet, shell, and git to solve problems.
-You like to read the docs.
-Only use necessary tools.
+A shell has been initialized for your session and has a file system.
+The repo has already been cloned and you can use ls or cat to view files, touch, mkdir, echo, etc. 
+Your job is to write code, commit it to a new branch, and open a pull request.
+Always commit code before terminating.
+YOUR FIRST STEP SHOULD ALWAYS BE TO RUN ls
 {agent_scratchpad}
 `.replaceAll('\n', ' ')
 
 	const executor = await initializeAgentExecutorWithOptions(tools, model, {
 		agentType: 'openai-functions',
 		returnIntermediateSteps: isDev,
 		handleParsingErrors: true,
-		verbose: isDev,
+		verbose: false,
 		agentArgs: {
 			prefix
 		}

lib/agents/maige.ts

@@ -4,7 +4,7 @@ import {SerpAPI} from 'langchain/tools'
 import env from '~/env.mjs'
 import {codebaseSearch} from '~/tools/codeSearch'
 import commentTool from '~/tools/comment'
-import githubTool from '~/tools/github'
+import {labelTool} from '~/tools/label'
 import updateInstructionsTool from '~/tools/updateInstructions'
 import {isDev} from '~/utils'
 
@@ -19,20 +19,23 @@ export default async function maige({
 	octokit,
 	prisma,
 	customerId,
-	repoName
+	repoName,
+	allLabels
 }: {
 	input: string
 	octokit: any
 	prisma: any
 	customerId: string
 	repoName: string
+	allLabels: any[]
 }) {
 	const tools = [
 		new SerpAPI(),
 		commentTool({octokit}),
 		updateInstructionsTool({octokit, prisma, customerId, repoName}),
-		githubTool({octokit}),
+		labelTool({octokit, allLabels}),
 		codebaseSearch({customerId, repoName})
+		// dispatchEngineer({octokit, prisma, customerId, repoName})
 	]
 
 	const prefix = `
@@ -46,7 +49,7 @@ You also maintain a set of user instructions that can customize your behaviour;
 		agentType: 'openai-functions',
 		returnIntermediateSteps: isDev,
 		handleParsingErrors: true,
-		verbose: isDev,
+		verbose: false,
 		agentArgs: {
 			prefix
 		}

lib/agents/reviewer.tsx

@@ -0,0 +1,47 @@
+import {initializeAgentExecutorWithOptions} from 'langchain/agents'
+import {ChatOpenAI} from 'langchain/chat_models/openai'
+import {SerpAPI} from 'langchain/tools'
+import env from '~/env.mjs'
+import {prComment} from '~/tools/prComment'
+import {isDev} from '~/utils'
+
+const model = new ChatOpenAI({
+	modelName: 'gpt-4-1106-preview',
+	openAIApiKey: env.OPENAI_API_KEY,
+	temperature: 0.3
+})
+
+export default async function reviewer({
+	input,
+	octokit,
+	pullId
+}: {
+	input: string
+	octokit: any
+	pullId: string
+}) {
+	const tools = [new SerpAPI(), prComment({octokit, pullId})]
+
+	const prefix = `
+	You are senior engineer reviewing a Pull Request in GitHub made by a junior engineer.
+		You MUST leave a comment on the PR according to the user's instructions using the prComment function.
+    Format your answer beautifully using markdown suitable for GitHub.
+    DO NOT use any emojis or non-Ascii characters.
+    {agent_scratchpad}
+	`.replaceAll('\n', ' ')
+
+	const executor = await initializeAgentExecutorWithOptions(tools, model, {
+		agentType: 'openai-functions',
+		returnIntermediateSteps: isDev,
+		handleParsingErrors: true,
+		verbose: false,
+		agentArgs: {
+			prefix
+		}
+	})
+
+	const result = await executor.call({input})
+	const {output} = result
+
+	return output
+}

lib/tools/dispatchEngineer.ts

@@ -0,0 +1,39 @@
+import {DynamicStructuredTool} from 'langchain/tools'
+import {z} from 'zod'
+import engineer from '~/agents/engineer'
+/**
+ * Execute a shell command
+ */
+export default function dispatchEngineer({
+	octokit,
+	prisma,
+	customerId,
+	repoName
+}: {
+	octokit: any
+	prisma: any
+	customerId: string
+	repoName: string
+}) {
+	return new DynamicStructuredTool({
+		description: 'Dispatch an engineer to work on an issue',
+		func: async ({input}) => {
+			console.log('DISPATCHING ENGINEER')
+			engineer({
+				input,
+				octokit,
+				prisma,
+				customerId,
+				repoName
+			})
+			// await new Promise(resolve => setTimeout(resolve, 1000))
+			return 'dispatched'
+		},
+		name: 'dispatchEngineer',
+		schema: z.object({
+			input: z
+				.string()
+				.describe('specific, detailed instructions for the engineer.')
+		})
+	})
+}

lib/tools/github.ts

@@ -4,7 +4,7 @@ import {z} from 'zod'
 /**
  * Call the GitHub REST API
  */
-export default function github({octokit}: {octokit: any}) {
+export function githubTool({octokit}: {octokit: any}) {
 	return new DynamicStructuredTool({
 		description: 'GitHub REST API',
 		func: async ({path, body, method}) => {

lib/tools/label.ts

@@ -6,7 +6,7 @@ import {labelIssue} from '~/utils/github'
 /**
  * Label an issue using GitHub REST API tool
  */
-export default function label({
+export function labelTool({
 	octokit,
 	allLabels
 }: {

lib/tools/prComment.ts

@@ -0,0 +1,29 @@
+import {DynamicStructuredTool} from 'langchain/tools'
+import {z} from 'zod'
+import {addComment} from '~/utils/github'
+
+/**
+ * Comment on an issue
+ */
+export function prComment({octokit, pullId}: {octokit: any; pullId: string}) {
+	return new DynamicStructuredTool({
+		description: 'Adds a comment to a PR',
+		func: async ({comment, severity}) => {
+			const footer = `By [Maige](https://maige.app). How's my driving?`
+			const res = await addComment({
+				octokit,
+				issueId: pullId,
+				comment: `${comment}\nThis is a Level ${severity} problem.\n\n${footer}`
+			})
+			return JSON.stringify(res)
+		},
+		name: 'prComment',
+		schema: z.object({
+			comment: z.string().describe('The comment to add'),
+			severity: z
+				.number()
+				.optional()
+				.describe('The severity of the needed fix on a scale of 1-5')
+		})
+	})
+}

lib/tools/updateInstructions.ts

@@ -21,7 +21,7 @@ export default function updateInstructions({
 				where: {
 					customerId_name: {
 						customerId,
-						name: repoName
+						name: repoName.split('/')[1]
 					}
 				},
 				data: {