Fix crash.json page in report. (#224)

Current reports fail to generate `crash.json` page because
`benchmark_json()` misuses its parameter `benchmark` (a `str`) as a
`Benchmark` class.

This PR fixes it by adding a new function to generate `Benchmark()` with
the `benchmark` string.

The PR also adds the missing index JSON, and removes outdated sorting
pages.

report/templates/benchmark.html

@@ -30,7 +30,7 @@ <h1>{{ benchmark }}</h1>
         <th>Builds</th>
         <th>Crashes</th>
         <th>Bug</th>
-        <th>Crash reason</th>
+        <th>Diagnosis</th>
         <th>Coverage</th>
         <th>Line coverage diff</th>
     </tr>

report/templates/benchmark.json → report/templates/crash.json

@@ -16,7 +16,7 @@
         "target_binary": "{{ sample.target_binary }}",
         "reproducer": "{{ sample.reproducer }}",
         "run_log": "{{ sample.run_log }}",
-        "source_code": {{ get_benchmark_final_target_code(sample.id) }},
+        "source_code": {{ get_benchmark_final_target_code(sample.id) | replace('\\n', '\\\\n')}},
         "model": "{{ model }}"
     }{% if not loop.last %},{% endif %}
 {% endfor %}

report/upload_report.sh

@@ -65,21 +65,25 @@ while true; do
   echo "Download results from localhost."
   wget2 --quiet --inet4-only --no-host-directories --http2-request-window 10 --recursive localhost:${WEB_PORT:?}/ 2>&1
 
-  # Also fetch the sorted reports.
-  wget2 --quiet --inet4-only localhost:${WEB_PORT:?}/sort_build -O sort/build 2>&1
-  wget2 --quiet --inet4-only localhost:${WEB_PORT:?}/sort_cov -O sort/cov 2>&1
-  wget2 --quiet --inet4-only localhost:${WEB_PORT:?}/sort_cov_diff -O sort/cov_diff 2>&1
-  wget2 --quiet --inet4-only localhost:${WEB_PORT:?}/sort_crash -O sort/crash 2>&1
-  wget2 --quiet --inet4-only localhost:${WEB_PORT:?}/sort_status -O sort/status 2>&1
+  # Also fetch index JSON.
+  wget2 --quiet --inet4-only localhost:${WEB_PORT:?}/json -O json 2>&1
 
   # Stop the server.
   kill -9 "$pid_web"
 
   # Upload the report to GCS.
   echo "Uploading the report."
+  BUCKET_PATH="gs://oss-fuzz-gcb-experiment-run-logs/Result-reports/${GCS_DIR:?}"
+  # Upload HTMLs.
   gsutil -q -m -h "Content-Type:text/html" \
          -h "Cache-Control:public, max-age=3600" \
-         cp -r . "gs://oss-fuzz-gcb-experiment-run-logs/Result-reports/${GCS_DIR:?}"
+         cp -r . "$BUCKET_PATH"
+  # Find all JSON files and upload them, removing the leading './'
+  find . -name '*json' | while read -r file; do
+    file_path="${file#./}"  # Remove the leading "./".
+    gsutil -q -m -h "Content-Type:application/json" \
+        -h "Cache-Control:public, max-age=3600" cp "$file" "$BUCKET_PATH/$file_path"
+  done
 
   cd ..
 

report/web.py

@@ -56,9 +56,9 @@ class Benchmark:
   def __post_init__(self):
     self.project = '-'.join(self.id.split('-')[1:-1])
     self.function = self.id.split('-')[-1]
-    self.signature = self.find_signature() or self.id
+    self.signature = self._find_signature() or self.id
 
-  def find_signature(self) -> str:
+  def _find_signature(self) -> str:
     """
     Finds the function signature by searching for its |benchmark_id| in
     BENCHMARK_DIR.
@@ -96,7 +96,7 @@ class Sample:
   """Result of a fuzz target sample of a benchmark."""
   id: str
   status: str
-  result: Optional[evaluator.Result] = None
+  result: evaluator.Result
 
   @property
   def stacktrace(self) -> str:
@@ -225,18 +225,18 @@ def list_benchmarks() -> List[Benchmark]:
   return benchmarks
 
 
-def sort_benchmarks(benchmarks: List[Benchmark],
-                    sort_by: str = 'cov_diff') -> List[Benchmark]:
-  """Keeps benchmarks with the highest line coverage diff on the top."""
-  sort_dict = {
-      'build': lambda b: b.result.build_success_rate,
-      'crash': lambda b: b.result.crash_rate,
-      'cov': lambda b: b.result.max_coverage,
-      'status': lambda b: b.status,
-      'cov_diff': lambda b: b.result.max_line_coverage_diff,
-  }
-  sorted_benchmarks = sorted(benchmarks, key=sort_dict[sort_by], reverse=True)
-  return sorted_benchmarks
+def match_benchmark(benchmark_id: str) -> Benchmark:
+  """Returns a benchmark class based on |benchmark_id|."""
+  results, targets = get_results(benchmark_id)
+  status = 'Done' if results and all(results) else 'Running'
+  filtered_results = [(i, stat) for i, stat in enumerate(results) if stat]
+
+  if filtered_results:
+    result = run_one_experiment.aggregate_results(filtered_results, targets)
+  else:
+    result = run_one_experiment.AggregatedResult()
+
+  return Benchmark(benchmark_id, status, result)
 
 
 def get_samples(benchmark: str) -> list[Sample]:
@@ -246,7 +246,7 @@ def get_samples(benchmark: str) -> list[Sample]:
 
   for i, sample_id in enumerate(sample_ids(get_generated_targets(benchmark))):
     status = 'Running'
-    result = None
+    result = evaluator.Result()
     if results[i]:
       status = 'Done'
       result = results[i]
@@ -264,7 +264,7 @@ def match_sample(benchmark: str, target_sample_id: str) -> Optional[Sample]:
     if sample_id != target_sample_id:
       continue
     status = 'Running'
-    result = None
+    result = evaluator.Result()
     if results[i]:
       status = 'Done'
       result = results[i]
@@ -377,63 +377,27 @@ def index():
 def index_json():
   return render_template('index.json',
                          benchmarks=list_benchmarks(),
-                         model=model)
-
-
-@app.route('/sort_build')
-def index_sort_build():
-  return render_template('index.html',
-                         benchmarks=sort_benchmarks(list_benchmarks(),
-                                                    sort_by='build'),
-                         model=model)
-
-
-@app.route('/sort_cov')
-def index_sort_cov():
-  return render_template('index.html',
-                         benchmarks=sort_benchmarks(list_benchmarks(),
-                                                    sort_by='cov'),
-                         model=model)
-
-
-@app.route('/sort_cov_diff')
-def index_sort():
-  return render_template('index.html',
-                         benchmarks=sort_benchmarks(list_benchmarks(),
-                                                    sort_by='cov_diff'),
-                         model=model)
-
-
-@app.route('/sort_crash')
-def index_sort_crash():
-  return render_template('index.html',
-                         benchmarks=sort_benchmarks(list_benchmarks(),
-                                                    sort_by='crash'),
-                         model=model)
-
-
-@app.route('/sort_status')
-def index_sort_stauts():
-  return render_template('index.html',
-                         benchmarks=sort_benchmarks(list_benchmarks(),
-                                                    sort_by='status'),
-                         model=model)
+                         model=model), 200, {
+                             'Content-Type': 'application/json'
+                         }
 
 
 @app.route('/benchmark/<benchmark>/crash.json')
-def benchmark_json(benchmark):
+def benchmark_json(benchmark: str):
   """Generates a JSON containing crash reproducing info."""
   if not _is_valid_benchmark_dir(benchmark):
     # TODO(dongge): This won't be needed after resolving the `lost+found` issue.
     abort(404)
 
   try:
-    return render_template('benchmark.json',
-                           benchmark=Benchmark.find_signature(benchmark),
+    return render_template('crash.json',
+                           benchmark=match_benchmark(benchmark).signature,
                            samples=get_samples(benchmark),
                            get_benchmark_final_target_code=partial(
                                get_final_target_code, benchmark),
-                           model=model)
+                           model=model), 200, {
+                               'Content-Type': 'application/json'
+                           }
   except Exception as e:
     logging.warning('Failed to render benchmark crash JSON: %s\n  %s',
                     benchmark, e)