bapdoc.ml

(** Builds BAP Annotated Reference .

    The program relies on argot.1.2-devel version, that is available
    from <https://github.com/ivg/argot#1.2-devel>.

    The code works only for 4.03, as ocamldoc is broken for older
    versions of OCaml.

*)

open Core_kernel.Std
open Bap_plugins.Std


let libraries = [
  "Core libraries", [
    "bap", "Bap.Std", "BAP standard library";
    "regular", "Regular.Std", "regular inductive types";
    "monads", "Monads.Std", "a missing monads library";
    "ogre", "Ogre", "a sexp-based NoSQL database";
    "bap-future", "Bap_future.Std", "coinductive types";
    "graphlib", "Graphlib.Std", "graph library";
    "bap-taint", "Bap_taint.Std", "The Taint Analysis Framework";
    "bap-primus", "Bap_primus.Std", "The Microexecution Framework";
    "bare", "Bare", "Binary Analysis Rule Engine";
  ];


  "Hardware Architectures",
  [
    "bap-arm", "ARM", "ARM architecture";
    "bap-x86-cpu", "X86_cpu", "x86 family";
  ];

  "Programming Languages",
  [
    "bap-api", "Bap_api", "interface for adding new languages";
    "bap-abi", "Bap_abi", "interface for adding new ABI";
    "bap-c", "Bap_c.Std", "support library for C language";
  ];

  "Auxiliary libraries",
  [
    "bap-bml", "Bap_bml", "an extensible DSL for term transformation";
    "bap-build", "Bap_build.Std", "BAP build system as an ocamlbuild plugin";
    "bap-byteweight", "Bap_byteweight", "an interface to byteweight implementation";
    "bap-demangle", "Bap_demangle.Std", "custom name demanglers";
    "bap-dwarf", "Bap_dwarf.Std", "native DWARF parser";
    "bap-ida", "Bap_ida.Std", "call IDA from OCaml";
    "bap-llvm", "Bap_llvm.Std", "setup LLVM backend";
    "bap-traces", "Bap_traces.Std", "loading execution traces";
    "bap-strings", "Bap_strings.Std", "text and string processing utilities";
    "text-tags", "Text_tags", "Use semantics tags to format your texts";
  ];
]

let std_libraries = [
  "Bap"; "Core_kernel"; "Regular"; "Graphlib"; "Future"; "Monads";
  "Bap_primus"
]

let frontends = [
  "bap", "bap main frontend";
  "bap-mc", "machine code playground";
  "bap-byteweight", "create, obtain and evaluate byteweight signatures";
  "bap-fsi-benchmark", "function start identification benchmark game";
]

let tools = [
  "baptop", "run BAP interactively";
  "bapbuild", "build BAP plugins";
  "bap-server", "call to BAP via JSON RPC";
  "bapbundle", "bundle BAP plugins";
]

let introduction = {|
{2 Introduction}

Binary Analysis Platform is a framework for writing program analysis
tools, that target binary files. The framework consists of a bunch
of libraries, plugins and frontends. The libraries provide code
reusability, plugins facilitate extensibility and frontends serve as
entry points.

The BAP Standard Library, also known just as bap library, is the core
component, around which the rest of the Platform is built. Start by
reading its manual. Frontends come with comprehensive manuals,
that can be accessed by using [--help] command line options, or via
the [man] command, if the manpath is configured correctly. Finally,
you can access a man page for a plugin using
[--<PLUGIN>-help] command line option of a frontend, e.g., [bap --map-terms-help].

The document is autogenerated from the library mli files, using our
[bapdoc] utility, that relies on the standard [ocamldoc] and enhanced html
generator [argot]. The reference part of the doc is optimized for using
from an IDE powered by [merlin]. Although it should be also readable
and searchable directly from the browser. The type manifest search is
capable of finding values by type signatures, by using fuzzy search
techniques and unification over different representation over
semantically same types. If you do not see the search frame to the
left, then {{:argot_index.html}click_here}.


|}

let packages =
  List.concat_map ~f:(fun (_,ps) -> List.map ~f:(fun (p,_,_) -> p) ps) libraries


let is_interface file =
  Filename.check_suffix file ".ml" || Filename.check_suffix file ".mli"

module Pkg = struct
  let deps pkg =
    pkg :: Findlib.package_ancestors ["byte"] pkg

  let dir pkg = Findlib.package_directory pkg

  let files pkg =
    let dir =  dir pkg in
    Sys.readdir dir |> Array.to_list |>
    List.filter ~f:is_interface |>
    List.map ~f:(Filename.concat dir)
end

let mkdir path =
  if not (Sys.file_exists path) then
    Unix.mkdir path 0o770

let run cmd =
  let res = Sys.command cmd in
  if res <> 0 then
    failwith ("Command: '" ^ cmd ^ "' failed")

let compile pkg =
  let out = Filename.temp_file "bapdoc" ".mldoc" in
  run @@ sprintf "ocamlfind ocamldoc -package %s -dump %s %s"
    (String.concat ~sep:"," (Pkg.deps pkg)) out
    (String.concat ~sep:" " (Pkg.files pkg));
  out

let debap name =
  try
    if String.sub name 0 4 = "bap-"
    then String.sub name 4 (String.length name - 4)
    else name
  with exn -> name

let enbap name =
  try
    if String.sub name 0 3 = "bap" then name else "bap-"^name
  with _ -> "bap-"^name


let generate_redirection lib entry =
  mkdir "man3";
  let redirection = sprintf {|
    <!doctype html>
    <html>
    <head>
    <meta http-equiv="refresh" content="0; url='../%s.html'" />
    </head>
    <body></body>
    </html> |} entry in
  let name = sprintf "man3/%s.3.html" lib in
  Out_channel.write_all name ~data:redirection

let render_entry (lib,entry,desc) =
  printf "compiling %s\n%!" lib;
  generate_redirection lib entry;
  sprintf "- {{!%s}%s} - %s" entry (debap lib) desc

let render_section (name,entries) =
  sprintf "{3 %s}\n%s" name
    (String.concat ~sep:"\n" @@ List.map ~f:render_entry entries)

let library_index =
  sprintf "\n{2 Libraries}\n%s"
    (List.map ~f:render_section libraries |> String.concat ~sep:"\n\n")

type info = {
  man : string -> string;
  help : string -> string;
}

let program = {
  man  = ident;
  help = sprintf "%s --help=groff"
}

let plugin = {
  man = sprintf "bap-plugin-%s";
  help = sprintf "bap --%s-help=groff";
}

let tool = {
  man = ident;
  help = fun _ -> "false"
}


let generate_manual {man; help} tool =
  mkdir "man1";
  match Sys.command @@ sprintf "%s >/dev/null" (help tool) with
  | 0 ->
    let output = sprintf "man1/%s.1.html" (man tool) in
    run @@ sprintf "%s | man2html -r > %s" (help tool) output;
    Some output
  | n ->
    eprintf "Warning: can't render manpage for %s\n" tool;
    eprintf "Called as %S, got:\n" (help tool);
    ignore(Sys.command @@ sprintf "%s >&2" (help tool));
    eprintf "\n%!";
    None


let render_entry (name,desc,manual) =
  printf "compiling %s\n%!" name;
  match manual with
  | None -> sprintf "- [%s] - %s" name desc
  | Some file -> sprintf "- {{:%s}%s} - %s" file name desc

let render_entries es =
  List.map ~f:render_entry es |> String.concat ~sep:"\n"

let render t descs =
  List.map descs ~f:(fun (name,desc) ->
      name, desc, generate_manual t name) |>
  render_entries

let frontends_index =
  sprintf "\n\n{2 Frontends}\n%s" @@ render program frontends

let tools_index =
  sprintf "\n\n{2 Tools}\n%s" @@ render tool tools

let by_plugin_name p1 p2 =
  String.compare (Plugin.name p1) (Plugin.name p2)

let plugins =
  Plugins.list () |> List.sort ~cmp:by_plugin_name |>
  List.map ~f:(fun p ->
      Plugin.name p, Plugin.desc p) |>
  render plugin

let plugins_index =
  sprintf "\n\n{2 Plugins}\n%s" plugins

let hides = String.concat ~sep:"," std_libraries
let argot = if Dynlink.is_native then "argot.cmxs" else "argot.cmo"

let render modules =
  let intro,out = Filename.open_temp_file "intro" ".txt" in
  output_string out introduction;
  output_string out library_index;
  output_string out frontends_index;
  output_string out plugins_index;
  output_string out tools_index;
  Out_channel.close out;
  let options = String.concat ~sep:" " [
      "-charset UTF-8";
      "-html";
      "-colorize-code";
      "-short-paths";
      "-i"; Pkg.dir "argot";
      "-g"; argot;
      "-short-functors";
      "-hide"; hides;
      "-passopt -search-frame";
      "-passopt -search";
      "-passopt -full-text";
      "-passopt -hide-undocumented-values";
      "-intro"; intro;
      "-t 'BAR: BAP Annotated Reference'"
    ] in
  printf "Linking\n%!";
  run @@ sprintf "ocamlfind ocamldoc %s %s"
    options
    (List.map ~f:(sprintf "-load %s") modules |> String.concat ~sep:" ");
  Sys.remove intro


let cleanup = List.iter ~f:Sys.remove

let generate () =
  let modules = packages |> List.map ~f:compile in
  try
    render modules;
    cleanup modules;
  with exn ->
    cleanup modules;
    raise exn

let () = generate ()