feat(generation): add handling for models without SpecificToolChoice

packages/amplify-graphql-api-construct-tests/src/__tests__/generations/generation-evaluate-code.test.ts

@@ -0,0 +1,4 @@
+
+describe('generation-evaluate-code', () => {
+
+});

packages/amplify-graphql-generation-transformer/src/__tests__/__snapshots__/amplify-graphql-generation-transformer.test.ts.snap

@@ -112,17 +112,22 @@ export function response(ctx) {
   }
 
   const body = JSON.parse(ctx.result.body);
-  const value = body?.output?.message?.content?.[0]?.toolUse?.input?.value;
+  let value = body?.output?.message?.content?.find((content) => !!content.toolUse)?.toolUse?.input?.value;
 
   if (!value) {
-    util.error('Invalid Bedrock response', 'InvalidResponseException');
+    util.error('Invalid foundation model response', 'InvalidResponseException');
   }
 
-  // Added for non-string scalar response types
-  // This catches the occasional stringified JSON response.
-  if (typeof value === 'string') {
-    return JSON.parse(value);
+  // Represents whether the return type of the generation route is a raw string or not.
+  // If the return type is \`String\`, \`ID\`, or \`AWSJSON\` (including required \`!\` ), the value is \`false\` and we don't attempt any fallback parsing.
+  // If the return type isn't on the the string based scalars above, the value is \`true\`.
+  const isScalarStringResponseType = false;
+  // When \`isScalarStringResponseType\` is \`false\` and the toolUse input is a \`string\`,
+  // the foundation model has returned stringified JSON, so we attempt to parse it into a valid object.
+  if (!isScalarStringResponseType && typeof value === 'string') {
+    return parseIncorrectlyStringifiedJSON(value);
   }
+
   return value;
 }
 
@@ -136,7 +141,41 @@ function createUserAgent(request) {
   }
   return userAgent;
 }
-",
+
+function parseIncorrectlyStringifiedJSON(input) {
+  // Try statements are not supported:
+  // \`@aws-appsync/no-try: Try statements are not supported\`
+
+  // This initial attempt covers the case where the tool input is valid stringified JSON
+  let value = JSON.parse(input);
+  // A failed parse attempt doesn't throw an error in resolver functions.
+  // It returns an empty string, so a truthiness check suffices.
+  if (value) return value;
+
+  // Since the tool input wasn't valid stringified JSON, we're assuming that
+  // it contains \`'\` where it should contain \`\\"\`. Some foundation models like to do this.
+  // This is our last fallback attempt and covers the cases observed in the wild.
+
+  // Regular expression is not supported in resolver functions:
+  // \`error @aws-appsync/no-regex: Regex literals are not supported\`
+  // However, raw string inputs are processed by the underlying Java runtime.
+  // So the patterns used are valid Java patterns, and not necessarily valid JavaScript patterns
+
+  // Replaces single quotes with double quotes, handling escaped single quotes.
+  value = input
+    // Replace any escaped single quotes with a marker.
+    .replaceAll("\\\\\\\\'", "___ESCAPED_QUOTE___")
+    // Replace all remaining single quotes with double quotes
+    .replaceAll("'", "\\"")
+    // Restore escaped single quotes
+    .replaceAll("___ESCAPED_QUOTE___", "'");
+
+  value = JSON.parse(value);
+  if (value) return value;
+
+  // Nothing more to do, time to bail.
+  util.error('Unable to parse foundation model response', 'InvalidResponseException')
+}",
 }
 `;
 
@@ -252,17 +291,22 @@ export function response(ctx) {
   }
 
   const body = JSON.parse(ctx.result.body);
-  const value = body?.output?.message?.content?.[0]?.toolUse?.input?.value;
+  let value = body?.output?.message?.content?.find((content) => !!content.toolUse)?.toolUse?.input?.value;
 
   if (!value) {
-    util.error('Invalid Bedrock response', 'InvalidResponseException');
+    util.error('Invalid foundation model response', 'InvalidResponseException');
   }
 
-  // Added for non-string scalar response types
-  // This catches the occasional stringified JSON response.
-  if (typeof value === 'string') {
-    return JSON.parse(value);
+  // Represents whether the return type of the generation route is a raw string or not.
+  // If the return type is \`String\`, \`ID\`, or \`AWSJSON\` (including required \`!\` ), the value is \`false\` and we don't attempt any fallback parsing.
+  // If the return type isn't on the the string based scalars above, the value is \`true\`.
+  const isScalarStringResponseType = false;
+  // When \`isScalarStringResponseType\` is \`false\` and the toolUse input is a \`string\`,
+  // the foundation model has returned stringified JSON, so we attempt to parse it into a valid object.
+  if (!isScalarStringResponseType && typeof value === 'string') {
+    return parseIncorrectlyStringifiedJSON(value);
   }
+
   return value;
 }
 
@@ -276,7 +320,41 @@ function createUserAgent(request) {
   }
   return userAgent;
 }
-",
+
+function parseIncorrectlyStringifiedJSON(input) {
+  // Try statements are not supported:
+  // \`@aws-appsync/no-try: Try statements are not supported\`
+
+  // This initial attempt covers the case where the tool input is valid stringified JSON
+  let value = JSON.parse(input);
+  // A failed parse attempt doesn't throw an error in resolver functions.
+  // It returns an empty string, so a truthiness check suffices.
+  if (value) return value;
+
+  // Since the tool input wasn't valid stringified JSON, we're assuming that
+  // it contains \`'\` where it should contain \`\\"\`. Some foundation models like to do this.
+  // This is our last fallback attempt and covers the cases observed in the wild.
+
+  // Regular expression is not supported in resolver functions:
+  // \`error @aws-appsync/no-regex: Regex literals are not supported\`
+  // However, raw string inputs are processed by the underlying Java runtime.
+  // So the patterns used are valid Java patterns, and not necessarily valid JavaScript patterns
+
+  // Replaces single quotes with double quotes, handling escaped single quotes.
+  value = input
+    // Replace any escaped single quotes with a marker.
+    .replaceAll("\\\\\\\\'", "___ESCAPED_QUOTE___")
+    // Replace all remaining single quotes with double quotes
+    .replaceAll("'", "\\"")
+    // Restore escaped single quotes
+    .replaceAll("___ESCAPED_QUOTE___", "'");
+
+  value = JSON.parse(value);
+  if (value) return value;
+
+  // Nothing more to do, time to bail.
+  util.error('Unable to parse foundation model response', 'InvalidResponseException')
+}",
 }
 `;
 
@@ -391,17 +469,22 @@ export function response(ctx) {
   }
 
   const body = JSON.parse(ctx.result.body);
-  const value = body?.output?.message?.content?.[0]?.toolUse?.input?.value;
+  let value = body?.output?.message?.content?.find((content) => !!content.toolUse)?.toolUse?.input?.value;
 
   if (!value) {
-    util.error('Invalid Bedrock response', 'InvalidResponseException');
+    util.error('Invalid foundation model response', 'InvalidResponseException');
   }
 
-  // Added for non-string scalar response types
-  // This catches the occasional stringified JSON response.
-  if (typeof value === 'string') {
-    return JSON.parse(value);
+  // Represents whether the return type of the generation route is a raw string or not.
+  // If the return type is \`String\`, \`ID\`, or \`AWSJSON\` (including required \`!\` ), the value is \`false\` and we don't attempt any fallback parsing.
+  // If the return type isn't on the the string based scalars above, the value is \`true\`.
+  const isScalarStringResponseType = false;
+  // When \`isScalarStringResponseType\` is \`false\` and the toolUse input is a \`string\`,
+  // the foundation model has returned stringified JSON, so we attempt to parse it into a valid object.
+  if (!isScalarStringResponseType && typeof value === 'string') {
+    return parseIncorrectlyStringifiedJSON(value);
   }
+
   return value;
 }
 
@@ -415,7 +498,41 @@ function createUserAgent(request) {
   }
   return userAgent;
 }
-"
+
+function parseIncorrectlyStringifiedJSON(input) {
+  // Try statements are not supported:
+  // \`@aws-appsync/no-try: Try statements are not supported\`
+
+  // This initial attempt covers the case where the tool input is valid stringified JSON
+  let value = JSON.parse(input);
+  // A failed parse attempt doesn't throw an error in resolver functions.
+  // It returns an empty string, so a truthiness check suffices.
+  if (value) return value;
+
+  // Since the tool input wasn't valid stringified JSON, we're assuming that
+  // it contains \`'\` where it should contain \`\\"\`. Some foundation models like to do this.
+  // This is our last fallback attempt and covers the cases observed in the wild.
+
+  // Regular expression is not supported in resolver functions:
+  // \`error @aws-appsync/no-regex: Regex literals are not supported\`
+  // However, raw string inputs are processed by the underlying Java runtime.
+  // So the patterns used are valid Java patterns, and not necessarily valid JavaScript patterns
+
+  // Replaces single quotes with double quotes, handling escaped single quotes.
+  value = input
+    // Replace any escaped single quotes with a marker.
+    .replaceAll("\\\\\\\\'", "___ESCAPED_QUOTE___")
+    // Replace all remaining single quotes with double quotes
+    .replaceAll("'", "\\"")
+    // Restore escaped single quotes
+    .replaceAll("___ESCAPED_QUOTE___", "'");
+
+  value = JSON.parse(value);
+  if (value) return value;
+
+  // Nothing more to do, time to bail.
+  util.error('Unable to parse foundation model response', 'InvalidResponseException')
+}"
 `;
 
 exports[`generation route scalar type 1`] = `
@@ -547,13 +664,22 @@ export function response(ctx) {
   }
 
   const body = JSON.parse(ctx.result.body);
-  const value = body?.output?.message?.content?.[0]?.toolUse?.input?.value;
+  let value = body?.output?.message?.content?.find((content) => !!content.toolUse)?.toolUse?.input?.value;
 
   if (!value) {
-    util.error('Invalid Bedrock response', 'InvalidResponseException');
+    util.error('Invalid foundation model response', 'InvalidResponseException');
+  }
+
+  // Represents whether the return type of the generation route is a raw string or not.
+  // If the return type is \`String\`, \`ID\`, or \`AWSJSON\` (including required \`!\` ), the value is \`false\` and we don't attempt any fallback parsing.
+  // If the return type isn't on the the string based scalars above, the value is \`true\`.
+  const isScalarStringResponseType = true;
+  // When \`isScalarStringResponseType\` is \`false\` and the toolUse input is a \`string\`,
+  // the foundation model has returned stringified JSON, so we attempt to parse it into a valid object.
+  if (!isScalarStringResponseType && typeof value === 'string') {
+    return parseIncorrectlyStringifiedJSON(value);
   }
 
-
   return value;
 }
 
@@ -567,7 +693,41 @@ function createUserAgent(request) {
   }
   return userAgent;
 }
-",
+
+function parseIncorrectlyStringifiedJSON(input) {
+  // Try statements are not supported:
+  // \`@aws-appsync/no-try: Try statements are not supported\`
+
+  // This initial attempt covers the case where the tool input is valid stringified JSON
+  let value = JSON.parse(input);
+  // A failed parse attempt doesn't throw an error in resolver functions.
+  // It returns an empty string, so a truthiness check suffices.
+  if (value) return value;
+
+  // Since the tool input wasn't valid stringified JSON, we're assuming that
+  // it contains \`'\` where it should contain \`\\"\`. Some foundation models like to do this.
+  // This is our last fallback attempt and covers the cases observed in the wild.
+
+  // Regular expression is not supported in resolver functions:
+  // \`error @aws-appsync/no-regex: Regex literals are not supported\`
+  // However, raw string inputs are processed by the underlying Java runtime.
+  // So the patterns used are valid Java patterns, and not necessarily valid JavaScript patterns
+
+  // Replaces single quotes with double quotes, handling escaped single quotes.
+  value = input
+    // Replace any escaped single quotes with a marker.
+    .replaceAll("\\\\\\\\'", "___ESCAPED_QUOTE___")
+    // Replace all remaining single quotes with double quotes
+    .replaceAll("'", "\\"")
+    // Restore escaped single quotes
+    .replaceAll("___ESCAPED_QUOTE___", "'");
+
+  value = JSON.parse(value);
+  if (value) return value;
+
+  // Nothing more to do, time to bail.
+  util.error('Unable to parse foundation model response', 'InvalidResponseException')
+}",
 }
 `;
 
@@ -628,13 +788,22 @@ export function response(ctx) {
   }
 
   const body = JSON.parse(ctx.result.body);
-  const value = body?.output?.message?.content?.[0]?.toolUse?.input?.value;
+  let value = body?.output?.message?.content?.find((content) => !!content.toolUse)?.toolUse?.input?.value;
 
   if (!value) {
-    util.error('Invalid Bedrock response', 'InvalidResponseException');
+    util.error('Invalid foundation model response', 'InvalidResponseException');
+  }
+
+  // Represents whether the return type of the generation route is a raw string or not.
+  // If the return type is \`String\`, \`ID\`, or \`AWSJSON\` (including required \`!\` ), the value is \`false\` and we don't attempt any fallback parsing.
+  // If the return type isn't on the the string based scalars above, the value is \`true\`.
+  const isScalarStringResponseType = true;
+  // When \`isScalarStringResponseType\` is \`false\` and the toolUse input is a \`string\`,
+  // the foundation model has returned stringified JSON, so we attempt to parse it into a valid object.
+  if (!isScalarStringResponseType && typeof value === 'string') {
+    return parseIncorrectlyStringifiedJSON(value);
   }
 
-
   return value;
 }
 
@@ -648,6 +817,40 @@ function createUserAgent(request) {
   }
   return userAgent;
 }
-",
+
+function parseIncorrectlyStringifiedJSON(input) {
+  // Try statements are not supported:
+  // \`@aws-appsync/no-try: Try statements are not supported\`
+
+  // This initial attempt covers the case where the tool input is valid stringified JSON
+  let value = JSON.parse(input);
+  // A failed parse attempt doesn't throw an error in resolver functions.
+  // It returns an empty string, so a truthiness check suffices.
+  if (value) return value;
+
+  // Since the tool input wasn't valid stringified JSON, we're assuming that
+  // it contains \`'\` where it should contain \`\\"\`. Some foundation models like to do this.
+  // This is our last fallback attempt and covers the cases observed in the wild.
+
+  // Regular expression is not supported in resolver functions:
+  // \`error @aws-appsync/no-regex: Regex literals are not supported\`
+  // However, raw string inputs are processed by the underlying Java runtime.
+  // So the patterns used are valid Java patterns, and not necessarily valid JavaScript patterns
+
+  // Replaces single quotes with double quotes, handling escaped single quotes.
+  value = input
+    // Replace any escaped single quotes with a marker.
+    .replaceAll("\\\\\\\\'", "___ESCAPED_QUOTE___")
+    // Replace all remaining single quotes with double quotes
+    .replaceAll("'", "\\"")
+    // Restore escaped single quotes
+    .replaceAll("___ESCAPED_QUOTE___", "'");
+
+  value = JSON.parse(value);
+  if (value) return value;
+
+  // Nothing more to do, time to bail.
+  util.error('Unable to parse foundation model response', 'InvalidResponseException')
+}",
 }
 `;