JS-160 Create Java based AST

its/plugin/tests/src/test/java/com/sonar/javascript/it/plugin/ConsumerPluginTest.java

@@ -19,9 +19,6 @@
  */
 package com.sonar.javascript.it.plugin;
 
-import static com.sonar.javascript.it.plugin.OrchestratorStarter.JAVASCRIPT_PLUGIN_LOCATION;
-import static org.assertj.core.api.Assertions.assertThat;
-
 import com.sonar.orchestrator.Orchestrator;
 import com.sonar.orchestrator.build.BuildResult;
 import com.sonar.orchestrator.build.SonarScanner;
@@ -32,6 +29,9 @@
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
 
+import static com.sonar.javascript.it.plugin.OrchestratorStarter.JAVASCRIPT_PLUGIN_LOCATION;
+import static org.assertj.core.api.Assertions.assertThat;
+
 class ConsumerPluginTest {
 
   private static final String PLUGIN_ARTIFACT_ID = "consumer-plugin";
@@ -97,6 +97,7 @@ void test() {
     var logMatch = ".*DEBUG: Registered JsAnalysisConsumers \\[org.sonar.samples.javascript.consumer.Consumer.*]";
     assertThat(buildResult.getLogsLines(l -> l.matches(logMatch))).hasSize(1);
 
-    assertThat(buildResult.getLogsLines(l -> l.matches(".*Processing file src/dir.*"))).hasSize(2);
+    // TS file is not processed yet.
+    assertThat(buildResult.getLogsLines(l -> l.matches(".*Processing file src/dir.*"))).hasSize(1);
   }
 }

packages/bridge/src/worker.js

@@ -50,15 +50,15 @@ exports.delegate = function (worker, type) {
         case 'success':
           if (message.format === 'multipart') {
             const fd = new formData();
-            const buf = Buffer.from(message.result.ast);
-            fd.append('ast', buf);
+            fd.append('ast', Buffer.from(message.result.ast));
             delete message.result.ast;
             fd.append('json', JSON.stringify(message.result));
             // this adds the boundary string that will be used to separate the parts
             response.set('Content-Type', fd.getHeaders()['content-type']);
             response.set('Content-Length', fd.getLengthSync());
             fd.pipe(response);
           } else {
+            delete message.result.ast;
             response.send(message.result);
           }
           break;
@@ -109,7 +109,7 @@ if (parentPort) {
           parentThread.postMessage({
             type: 'success',
             result: output,
-            format: 'multipart',
+            format: isSupported(output.ast) ? 'multipart' : 'json',
           });
           break;
         }
@@ -128,7 +128,7 @@ if (parentPort) {
           parentThread.postMessage({
             type: 'success',
             result: output,
-            format: 'multipart',
+            format: isSupported(output.ast) ? 'multipart' : 'json',
           });
           break;
         }
@@ -231,3 +231,7 @@ if (parentPort) {
     }
   }
 }
+
+function isSupported(ast) {
+  return ast !== 'not-supported';
+}

packages/bridge/tests/router.test.ts

@@ -119,10 +119,11 @@ describe('router', () => {
     const filePath = path.join(fixtures, 'file.js');
     const fileType = 'MAIN';
     const data = { filePath, fileType, tsConfigs: [] };
-    const response = (await request(server, '/analyze-js', 'POST', data, 'formdata')) as FormData;
+    const response = (await request(server, '/analyze-js', 'POST', data)) as FormData;
+    const responseData = JSON.parse(response.get('json') as string);
     const {
       issues: [issue],
-    } = JSON.parse(response.get('json') as string);
+    } = responseData;
     expect(issue).toEqual(
       expect.objectContaining({
         ruleId: 'prefer-regex-literals',
@@ -149,10 +150,10 @@ describe('router', () => {
     const fileType = 'MAIN';
     const tsConfig = path.join(fixtures, 'tsconfig.json');
     const data = { filePath, fileType, tsConfigs: [tsConfig] };
-    const response = (await request(server, '/analyze-ts', 'POST', data, 'formdata')) as FormData;
+    const response = (await request(server, '/analyze-ts', 'POST', data)) as string;
     const {
       issues: [issue],
-    } = JSON.parse(response.get('json') as string);
+    } = JSON.parse(response);
     expect(issue).toEqual(
       expect.objectContaining({
         ruleId: 'no-duplicate-in-composite',
@@ -163,9 +164,6 @@ describe('router', () => {
         message: `Remove this duplicated type or replace with another one.`,
       }),
     );
-    const ast = response.get('ast') as File;
-    const buffer = Buffer.from(await ast.arrayBuffer());
-    expect(buffer.toString()).toEqual('plop');
   });
 
   it('should route /analyze-with-program requests', async () => {
@@ -179,16 +177,10 @@ describe('router', () => {
       (await request(server, '/create-program', 'POST', { tsConfig })) as string,
     );
     const data = { filePath, fileType, programId };
-    const response = (await request(
-      server,
-      '/analyze-with-program',
-      'POST',
-      data,
-      'formdata',
-    )) as FormData;
+    const response = (await request(server, '/analyze-with-program', 'POST', data)) as string;
     const {
       issues: [issue],
-    } = JSON.parse(response.get('json') as string);
+    } = JSON.parse(response);
     expect(issue).toEqual(
       expect.objectContaining({
         ruleId: 'no-duplicate-in-composite',

packages/bridge/tests/server.test.ts

@@ -21,7 +21,7 @@ import { start } from '../src/server';
 import path from 'path';
 import { setContext } from '@sonar/shared';
 import { AddressInfo } from 'net';
-import { BridgeResponseType, request } from './tools';
+import { request } from './tools';
 import http from 'http';
 
 describe('server', () => {
@@ -75,7 +75,7 @@ describe('server', () => {
     });
 
     expect(await requestInitLinter(server, fileType, ruleId)).toBe('OK!');
-    const response = await requestAnalyzeJs(server, fileType, 'formdata');
+    const response = await requestAnalyzeJs(server, fileType);
     const {
       issues: [issue],
     } = JSON.parse(response.get('json'));
@@ -99,7 +99,7 @@ describe('server', () => {
     const fileType = 'MAIN';
 
     await requestInitLinter(server, fileType, ruleId);
-    const response = await requestAnalyzeJs(server, fileType, 'formdata');
+    const response = await requestAnalyzeJs(server, fileType);
 
     const {
       issues: [issue],
@@ -162,15 +162,11 @@ describe('server', () => {
   });
 });
 
-async function requestAnalyzeJs(
-  server: http.Server,
-  fileType: string,
-  format: BridgeResponseType = 'text',
-): Promise<any> {
+async function requestAnalyzeJs(server: http.Server, fileType: string): Promise<any> {
   const filePath = path.join(__dirname, 'fixtures', 'routing.js');
   const analysisInput = { filePath, fileType };
 
-  return await request(server, '/analyze-js', 'POST', analysisInput, format);
+  return await request(server, '/analyze-js', 'POST', analysisInput);
 }
 
 function requestInitLinter(server: http.Server, fileType: string, ruleId: string) {

packages/bridge/tests/tools/request.ts

@@ -25,13 +25,7 @@ export type BridgeResponseType = 'text' | 'formdata';
 /**
  * Sends an HTTP request to a server's endpoint running on localhost.
  */
-export async function request(
-  server: http.Server,
-  path: string,
-  method: string,
-  body: any = {},
-  format: BridgeResponseType = 'text',
-) {
+export async function request(server: http.Server, path: string, method: string, body: any = {}) {
   const res = await fetch(`http://127.0.0.1:${(server.address() as AddressInfo).port}${path}`, {
     headers: {
       'Content-Type': 'application/json',
@@ -40,12 +34,11 @@ export async function request(
     body: method !== 'GET' ? JSON.stringify(body) : undefined,
   });
 
-  switch (format) {
-    case 'text':
-      return res.text();
-    case 'formdata':
-      return res.formData();
-    default:
-      throw new Error(`Unsupported format: ${format}`);
+  const contentTypeHeader = res.headers.get('Content-Type');
+
+  if (contentTypeHeader?.includes('multipart/form-data')) {
+    return res.formData();
+  } else {
+    return res.text();
   }
 }

packages/jsts/src/analysis/analyzer.ts

@@ -17,7 +17,7 @@
  * along with this program; if not, write to the Free Software Foundation,
  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  */
-import { APIError, debug, getContext, JsTsLanguage } from '@sonar/shared';
+import { APIError, debug, info, getContext, JsTsLanguage } from '@sonar/shared';
 import { SourceCode } from 'eslint';
 import {
   computeMetrics,
@@ -85,6 +85,7 @@ function analyzeFile(
       highlightedSymbols,
       cognitiveComplexity,
     );
+
     return {
       issues,
       ucfgPaths,
@@ -105,10 +106,15 @@ function analyzeFile(
  * Remove this when we figure out how to serialize the TypeScript AST
  */
 function serializeAst(sourceCode: SourceCode, filePath: string) {
-  if (isSupported(filePath)) {
+  if (!isSupported(filePath)) {
+    return 'not-supported';
+  }
+
+  try {
     return serializeInProtobuf(sourceCode.ast);
-  } else {
-    return 'plop';
+  } catch (e) {
+    info(`Failed to serialize AST for file "${filePath}"`);
+    return 'not-supported';
   }
 
   function isSupported(filePath: string) {

packages/jsts/src/parsers/ast.ts

@@ -51,7 +51,8 @@ export function deserializeProtobuf(serialized: Uint8Array): any {
 
 export function visitNode(node: estree.BaseNodeWithoutComments | undefined | null): any {
   if (!node) {
-    return {};
+    // Null and undefined will be both serialized as "not set" in protobuf when the field is optional.
+    return undefined;
   }
 
   return {
@@ -362,8 +363,10 @@ export function visitNode(node: estree.BaseNodeWithoutComments | undefined | nul
   }
 
   function visitArrayPattern(node: estree.ArrayPattern) {
+    // If the elements ever contain null together with other nodes, protobuf will fail to serialize the array.
+    // It is unclear from the estree documentation if this is possible, but as it is from a type perspective, we prefer to be safe.
     return {
-      elements: node.elements.map(visitNode),
+      elements: node.elements.filter(e => e !== null).map(visitNode),
     };
   }
 
@@ -518,8 +521,10 @@ export function visitNode(node: estree.BaseNodeWithoutComments | undefined | nul
   }
 
   function visitArrayExpression(node: estree.ArrayExpression) {
+    // If the elements ever contain null together with other nodes, protobuf will fail to serialize the array.
+    // It is unclear from the estree documentation if this is possible, but as it is from a type perspective, we prefer to be safe.
     return {
-      elements: node.elements.map(visitNode),
+      elements: node.elements.filter(e => e !== null).map(visitNode),
     };
   }