implement IDL generation through compilation

cli/src/lib.rs

@@ -6,7 +6,7 @@ use crate::config::{
 use anchor_client::Cluster;
 use anchor_lang::idl::{IdlAccount, IdlInstruction, ERASED_AUTHORITY};
 use anchor_lang::{AccountDeserialize, AnchorDeserialize, AnchorSerialize};
-use anchor_syn::idl::types::{EnumFields, Idl, IdlType, IdlTypeDefinitionTy};
+use anchor_syn::idl::types::{EnumFields, Idl, IdlConst, IdlErrorCode, IdlEvent, IdlType, IdlTypeDefinitionTy, IdlTypeDefinition};
 use anyhow::{anyhow, Context, Result};
 use clap::Parser;
 use flate2::read::GzDecoder;
@@ -391,6 +391,8 @@ pub enum IdlCommand {
         #[clap(long)]
         no_docs: bool,
     },
+    /// Generates the IDL for the program using the compilation method.
+    Build,
     /// Fetches an IDL for the given address from a cluster.
     /// The address can be a program, IDL account, or IDL buffer.
     Fetch {
@@ -1685,6 +1687,7 @@ fn idl(cfg_override: &ConfigOverride, subcmd: IdlCommand) -> Result<()> {
             out_ts,
             no_docs,
         } => idl_parse(cfg_override, file, out, out_ts, no_docs),
+        IdlCommand::Build => idl_build(),
         IdlCommand::Fetch { address, out } => idl_fetch(cfg_override, address, out),
     }
 }
@@ -2013,6 +2016,149 @@ fn idl_parse(
     Ok(())
 }
 
+fn idl_build() -> Result<()> {
+    let exit = std::process::Command::new("cargo")
+        .args([
+            "test",
+            "__anchor_private_print_idl",
+            "--",
+            "--show-output",
+            "--quiet",
+        ])
+        .stderr(Stdio::inherit())
+        .output()
+        .map_err(|e| anyhow::format_err!("{}", e.to_string()))?;
+    if !exit.status.success() {
+        std::process::exit(exit.status.code().unwrap_or(1));
+    }
+
+    enum State {
+        Pass,
+        ConstLines(Vec<String>),
+        EventLines(Vec<String>),
+        ErrorsLines(Vec<String>),
+        ProgramLines(Vec<String>),
+    }
+
+    #[derive(Serialize, Deserialize)]
+    struct IdlGenEventPrint {
+        event: IdlEvent,
+        defined_types: Vec<IdlTypeDefinition>,
+    }
+
+    let mut state = State::Pass;
+
+    let mut events: Vec<IdlEvent> = vec![];
+    let mut error_codes: Option<Vec<IdlErrorCode>> = None;
+    let mut constants: Vec<IdlConst> = vec![];
+    let mut defined_types: HashMap<String, IdlTypeDefinition> = HashMap::new();
+    let mut curr_idl: Option<Idl> = None;
+
+    let mut idls: Vec<Idl> = vec![];
+
+    let out = String::from_utf8_lossy(&exit.stdout);
+    for line in out.lines() {
+        match &mut state {
+            State::Pass => {
+                if line == "---- IDL begin const ----" {
+                    state = State::ConstLines(vec![]);
+                    continue;
+                } else if line == "---- IDL begin event ----" {
+                    state = State::EventLines(vec![]);
+                    continue;
+                } else if line == "---- IDL begin errors ----" {
+                    state = State::ErrorsLines(vec![]);
+                    continue;
+                } else if line == "---- IDL begin program ----" {
+                    state = State::ProgramLines(vec![]);
+                    continue;
+                } else if line.starts_with("test result: ok") {
+                    let events = std::mem::take(&mut events);
+                    let error_codes = error_codes.take();
+                    let constants = std::mem::take(&mut constants);
+                    let mut defined_types = std::mem::take(&mut defined_types);
+                    let curr_idl = curr_idl.take();
+
+                    let events = if !events.is_empty() { Some(events) } else { None };
+
+                    let mut idl = match curr_idl {
+                        Some(idl) => idl,
+                        None => continue,
+                    };
+
+                    idl.events = events;
+                    idl.errors = error_codes;
+                    idl.constants = constants;
+
+                    let prog_ty = std::mem::take(&mut idl.types);
+                    defined_types.extend(
+                        prog_ty
+                            .into_iter()
+                            .map(|ty| (ty.full_path.clone().unwrap(), ty)),
+                    );
+                    idl.types = defined_types.into_values().collect::<Vec<_>>();
+
+                    idls.push(idl);
+                    continue;
+                }
+            }
+            State::ConstLines(lines) => {
+                if line == "---- IDL end const ----" {
+                    let constant: IdlConst = serde_json::from_str(&lines.join("\n"))?;
+                    constants.push(constant);
+                    state = State::Pass;
+                    continue;
+                }
+                lines.push(line.to_string());
+            }
+            State::EventLines(lines) => {
+                if line == "---- IDL end event ----" {
+                    let event: IdlGenEventPrint = serde_json::from_str(&lines.join("\n"))?;
+                    events.push(event.event);
+                    defined_types.extend(
+                        event
+                            .defined_types
+                            .into_iter()
+                            .map(|ty| (ty.full_path.clone().unwrap(), ty)),
+                    );
+                    state = State::Pass;
+                    continue;
+                }
+                lines.push(line.to_string());
+            }
+            State::ErrorsLines(lines) => {
+                if line == "---- IDL end errors ----" {
+                    let errs: Vec<IdlErrorCode> = serde_json::from_str(&lines.join("\n"))?;
+                    error_codes = Some(errs);
+                    state = State::Pass;
+                    continue;
+                }
+                lines.push(line.to_string());
+            }
+            State::ProgramLines(lines) => {
+                if line == "---- IDL end program ----" {
+                    let idl: Idl = serde_json::from_str(&lines.join("\n"))?;
+                    curr_idl = Some(idl);
+                    state = State::Pass;
+                    continue;
+                }
+                lines.push(line.to_string());
+            }
+        }
+    }
+
+    if idls.len() == 1 {
+        println!(
+            "{}",
+            serde_json::to_string_pretty(&idls.first().unwrap()).unwrap()
+        );
+    } else if idls.len() >= 2 {
+        println!("{}", serde_json::to_string_pretty(&idls).unwrap());
+    };
+
+    Ok(())
+}
+
 fn idl_fetch(cfg_override: &ConfigOverride, address: Pubkey, out: Option<String>) -> Result<()> {
     let idl = fetch_idl(cfg_override, address)?;
     let out = match out {

lang/Cargo.toml

@@ -34,6 +34,9 @@ anchor-attribute-event = { path = "./attribute/event", version = "0.27.0" }
 anchor-derive-accounts = { path = "./derive/accounts", version = "0.27.0" }
 anchor-derive-space = { path = "./derive/space", version = "0.27.0" }
 anchor-derive-serde = { path = "./derive/serde", version = "0.27.0" }
+# anchor-syn can and should only be included only for idl-gen. It won't compile
+# for bpf due to proc-macro2 crate.
+anchor-syn = { path = "./syn", version = "0.27.0" }
 arrayref = "0.3.6"
 base64 = "0.13.0"
 borsh = "0.9"

lang/attribute/account/src/lib.rs

@@ -1,5 +1,6 @@
 extern crate proc_macro;
 
+use anchor_syn::idl::gen::*;
 use quote::quote;
 use syn::parse_macro_input;
 
@@ -392,12 +393,16 @@ pub fn zero_copy(
         quote! {#[derive(::bytemuck::Zeroable)]}
     };
 
+    let no_docs = false;
+    let idl_gen_impl = gen_idl_gen_impl_for_struct(&account_strct, no_docs);
+
     proc_macro::TokenStream::from(quote! {
         #[derive(anchor_lang::__private::ZeroCopyAccessor, Copy, Clone)]
         #repr
         #pod
         #zeroable
         #account_strct
+        #idl_gen_impl
     })
 }
 

lang/attribute/constant/Cargo.toml

@@ -18,3 +18,4 @@ anchor-debug = ["anchor-syn/anchor-debug"]
 proc-macro2 = "1.0"
 syn = { version = "1.0.60", features = ["full"] }
 anchor-syn = { path = "../../syn", version = "0.27.0" }
+quote = "1.0"

lang/attribute/constant/src/lib.rs

@@ -1,11 +1,27 @@
 extern crate proc_macro;
 
+use anchor_syn::idl::gen::gen_idl_print_function_for_constant;
+use quote::quote;
+
 /// A marker attribute used to mark const values that should be included in the
 /// generated IDL but functionally does nothing.
 #[proc_macro_attribute]
 pub fn constant(
     _attr: proc_macro::TokenStream,
     input: proc_macro::TokenStream,
 ) -> proc_macro::TokenStream {
-    input
+    let ts = match syn::parse(input).unwrap() {
+        syn::Item::Const(item) => {
+            let idl_print = gen_idl_print_function_for_constant(&item);
+            quote! {
+                #item
+                #idl_print
+            }
+        }
+        item => quote! {#item},
+    };
+
+    proc_macro::TokenStream::from(quote! {
+        #ts
+    })
 }

lang/attribute/event/src/lib.rs

@@ -27,6 +27,8 @@ pub fn event(
         format!("{discriminator:?}").parse().unwrap()
     };
 
+    let idl_gen = anchor_syn::idl::gen::gen_idl_print_function_for_event(&event_strct);
+
     proc_macro::TokenStream::from(quote! {
         #[derive(anchor_lang::__private::EventIndex, AnchorSerialize, AnchorDeserialize)]
         #event_strct
@@ -42,6 +44,8 @@ pub fn event(
         impl anchor_lang::Discriminator for #event_name {
             const DISCRIMINATOR: [u8; 8] = #discriminator;
         }
+
+        #idl_gen
     })
 }
 

lang/derive/serde/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "anchor-derive-serde"
-version = "0.26.0"
+version = "0.27.0"
 authors = ["Serum Foundation <foundation@projectserum.com>"]
 repository = "https://github.com/coral-xyz/anchor"
 license = "Apache-2.0"
@@ -17,4 +17,5 @@ proc-macro = true
 proc-macro2 = "1.0"
 borsh-derive-internal = "0.9"
 syn = { version = "1.0.60", features = ["full"] }
-
+anchor-syn = { path = "../../syn", version = "0.27.0" }
+quote = "1.0"

lang/derive/serde/src/lib.rs

@@ -1,12 +1,13 @@
 extern crate proc_macro;
 
+use anchor_syn::idl::gen::*;
 use borsh_derive_internal::*;
 use proc_macro::TokenStream;
-use proc_macro2::Span;
+use proc_macro2::{Span, TokenStream as TokenStream2};
+use quote::quote;
 use syn::{Ident, Item};
 
-#[proc_macro_derive(AnchorSerialize, attributes(borsh_skip))]
-pub fn anchor_serialize(input: TokenStream) -> TokenStream {
+fn gen_borsh_serialize(input: TokenStream) -> TokenStream2 {
     let cratename = Ident::new("borsh", Span::call_site());
 
     let item: Item = syn::parse(input).unwrap();
@@ -18,14 +19,36 @@ pub fn anchor_serialize(input: TokenStream) -> TokenStream {
         _ => unreachable!(),
     };
 
-    TokenStream::from(match res {
+    match res {
         Ok(res) => res,
         Err(err) => err.to_compile_error(),
+    }
+}
+
+#[proc_macro_derive(AnchorSerialize, attributes(borsh_skip))]
+pub fn anchor_serialize(input: TokenStream) -> TokenStream {
+    let borsh = gen_borsh_serialize(input.clone());
+
+    let no_docs = false; // TODO
+
+    let idl_gen_impl = match syn::parse(input).unwrap() {
+        Item::Struct(item) => gen_idl_gen_impl_for_struct(&item, no_docs),
+        Item::Enum(item) => gen_idl_gen_impl_for_enum(item, no_docs),
+        Item::Union(item) => {
+            // unions are not included in the IDL - TODO print a warning
+            idl_gen_impl_skeleton(quote! {None}, quote! {}, &item.ident, &item.generics)
+        }
+        // Derive macros can only be defined on structs, enums, and unions.
+        _ => unreachable!(),
+    };
+
+    TokenStream::from(quote! {
+        #borsh
+        #idl_gen_impl
     })
 }
 
-#[proc_macro_derive(AnchorDeserialize, attributes(borsh_skip, borsh_init))]
-pub fn borsh_deserialize(input: TokenStream) -> TokenStream {
+fn gen_borsh_deserialize(input: TokenStream) -> TokenStream2 {
     let cratename = Ident::new("borsh", Span::call_site());
 
     let item: Item = syn::parse(input).unwrap();
@@ -37,8 +60,13 @@ pub fn borsh_deserialize(input: TokenStream) -> TokenStream {
         _ => unreachable!(),
     };
 
-    TokenStream::from(match res {
+    match res {
         Ok(res) => res,
         Err(err) => err.to_compile_error(),
-    })
+    }
+}
+
+#[proc_macro_derive(AnchorDeserialize, attributes(borsh_skip, borsh_init))]
+pub fn borsh_deserialize(input: TokenStream) -> TokenStream {
+    TokenStream::from(gen_borsh_deserialize(input))
 }

lang/src/lib.rs

@@ -57,6 +57,9 @@ pub use borsh::de::BorshDeserialize as AnchorDeserialize;
 pub use borsh::ser::BorshSerialize as AnchorSerialize;
 pub use solana_program;
 
+// TODO - add this behind feature gate
+pub use anchor_syn;
+
 pub type Result<T> = std::result::Result<T, error::Error>;
 
 /// A data structure of validated accounts that can be deserialized from the