diff --git a/dev/.documenter-siteinfo.json b/dev/.documenter-siteinfo.json index d2de2fe2ea..1d19882d2c 100644 --- a/dev/.documenter-siteinfo.json +++ b/dev/.documenter-siteinfo.json @@ -1 +1 @@ -{"documenter":{"julia_version":"1.10.5","generation_timestamp":"2024-09-27T20:36:34","documenter_version":"1.7.0"}} \ No newline at end of file +{"documenter":{"julia_version":"1.10.5","generation_timestamp":"2024-09-27T21:54:41","documenter_version":"1.7.0"}} \ No newline at end of file diff --git a/dev/api/index.html b/dev/api/index.html index 7987c780c3..c50bc15c74 100644 --- a/dev/api/index.html +++ b/dev/api/index.html @@ -1,5 +1,5 @@ -API reference · Enzyme.jl

API reference

Types and constants

Functions and macros

Documentation

Enzyme.@import_frule — Macro
import_frule(::fn, tys...)

Automatically import a ChainRulesCore.fruleas a custom forward modeEnzymeRule. When called in batch mode, this will end up calling the primal multiple times, which may result in incorrect behavior if the function mutates, and slow code, always. Importing the rule fromChainRules` is also likely to be slower than writing your own rule, and may also be slower than not having a rule at all.

Use with caution.

Enzyme.@import_frule(typeof(Base.sort), Any);
+API reference · Enzyme.jl
API reference
Types and constants
Functions and macros
Documentation
Enzyme.@import_frule — Macro
import_frule(::fn, tys...)
Automatically import a ChainRulesCore.fruleas a custom forward modeEnzymeRule. When called in batch mode, this will end up calling the primal multiple times, which may result in incorrect behavior if the function mutates, and slow code, always. Importing the rule fromChainRules` is also likely to be slower than writing your own rule, and may also be slower than not having a rule at all.
Use with caution.
Enzyme.@import_frule(typeof(Base.sort), Any);
 
 x=[1.0, 2.0, 0.0]; dx=[0.1, 0.2, 0.3]; ddx = [0.01, 0.02, 0.03];
 
@@ -14,7 +14,7 @@
 (var"1" = (var"1" = [0.3, 0.1, 0.2], var"2" = [0.03, 0.01, 0.02]),)
 (var"1" = [0.3, 0.1, 0.2],)
 (var"1" = [0.0, 1.0, 2.0], var"2" = [0.3, 0.1, 0.2])
-
source
Enzyme.@import_rrule — Macro
import_rrule(::fn, tys...)
Automatically import a ChainRules.rrule as a custom reverse mode EnzymeRule. When called in batch mode, this will end up calling the primal multiple times which results in slower code. This macro assumes that the underlying function to be imported is read-only, and returns a Duplicated or Const object. This macro also assumes that the inputs permit a .+= operation and that the output has a valid Enzyme.make_zero function defined. It also assumes that overwritten(x) accurately describes if there is any non-preserved data from forward to reverse, not just the outermost data structure being overwritten as provided by the specification.
Finally, this macro falls back to almost always caching all of the inputs, even if it may not be needed for the derivative computation.
As a result, this auto importer is also likely to be slower than writing your own rule, and may also be slower than not having a rule at all.
Use with caution.
Enzyme.@import_rrule(typeof(Base.sort), Any);
source
Enzyme.gradient! — Method
gradient!(::ReverseMode, dx, f, x)
Compute the gradient of an array-input function f using reverse mode, storing the derivative result in an existing array dx. Both x and dx must be Arrays of the same type.
Example:
f(x) = x[1]*x[2]
+
source
Enzyme.@import_rrule — Macro
import_rrule(::fn, tys...)
Automatically import a ChainRules.rrule as a custom reverse mode EnzymeRule. When called in batch mode, this will end up calling the primal multiple times which results in slower code. This macro assumes that the underlying function to be imported is read-only, and returns a Duplicated or Const object. This macro also assumes that the inputs permit a .+= operation and that the output has a valid Enzyme.make_zero function defined. It also assumes that overwritten(x) accurately describes if there is any non-preserved data from forward to reverse, not just the outermost data structure being overwritten as provided by the specification.
Finally, this macro falls back to almost always caching all of the inputs, even if it may not be needed for the derivative computation.
As a result, this auto importer is also likely to be slower than writing your own rule, and may also be slower than not having a rule at all.
Use with caution.
Enzyme.@import_rrule(typeof(Base.sort), Any);
source
Enzyme.gradient! — Method
gradient!(::ReverseMode, dx, f, x)
Compute the gradient of an array-input function f using reverse mode, storing the derivative result in an existing array dx. Both x and dx must be Arrays of the same type.
Example:
f(x) = x[1]*x[2]
 
 dx = [0.0, 0.0]
 gradient!(Reverse, dx, f, [2.0, 3.0])
@@ -24,7 +24,7 @@
 gradient!(ReverseWithPrimal, dx, f, [2.0, 3.0])
 
 # output
-(derivs = ([3.0, 2.0],), val = 6.0)
source
Enzyme.gradient — Method
gradient(::ForwardMode, f, x; shadows=onehot(x), chunk=nothing)
Compute the gradient of an array-input function f using forward mode. The optional keyword argument shadow is a vector of one-hot vectors of type x which are used to forward-propagate into the return. For performance reasons, this should be computed once, outside the call to gradient, rather than within this call.
Example:
f(x) = x[1]*x[2]
+(derivs = ([3.0, 2.0],), val = 6.0)
source
Enzyme.gradient — Method
gradient(::ForwardMode, f, x; shadows=onehot(x), chunk=nothing)
Compute the gradient of an array-input function f using forward mode. The optional keyword argument shadow is a vector of one-hot vectors of type x which are used to forward-propagate into the return. For performance reasons, this should be computed once, outside the call to gradient, rather than within this call.
Example:
f(x) = x[1]*x[2]
 
 gradient(Forward, f, [2.0, 3.0])
 
@@ -45,7 +45,7 @@
 grad = gradient(Forward, f, [2.0, 3.0, 4.0])
 
 # output
-([3.0 2.0 0.0; 0.0 1.0 1.0],)
source
Enzyme.gradient — Method
gradient(::ReverseMode, f, args...)
Compute the gradient of a real-valued function f using reverse mode. For each differentiable argument, this function will allocate and return new derivative object, returning a tuple of derivatives for each argument. If an argument is not differentiable, the element of the returned tuple with be nothing.
In reverse mode (here), the derivatives will be the same type as the original argument.
This is a structure gradient. For a struct x it returns another instance of the same type, whose fields contain the components of the gradient. In the result, grad.a contains ∂f/∂x.a for any differential x.a, while grad.c == x.c for other types.
Examples:
f(x) = x[1]*x[2]
+([3.0 2.0 0.0; 0.0 1.0 1.0],)
source
Enzyme.gradient — Method
gradient(::ReverseMode, f, args...)
Compute the gradient of a real-valued function f using reverse mode. For each differentiable argument, this function will allocate and return new derivative object, returning a tuple of derivatives for each argument. If an argument is not differentiable, the element of the returned tuple with be nothing.
In reverse mode (here), the derivatives will be the same type as the original argument.
This is a structure gradient. For a struct x it returns another instance of the same type, whose fields contain the components of the gradient. In the result, grad.a contains ∂f/∂x.a for any differential x.a, while grad.c == x.c for other types.
Examples:
f(x) = x[1]*x[2]
 
 grad = gradient(Reverse, f, [2.0, 3.0])
 
@@ -74,7 +74,7 @@
 (derivs = ([3.0], [2.0]), val = 6.0)
grad = gradient(ReverseWithPrimal, mul, [2.0], Const([3.0]))
 
 # output
-(derivs = ([3.0], nothing), val = 6.0)
source
Enzyme.hvp! — Method
hvp!(res::X, f::F, x::X, v::X) where {F, X}
Compute an in-place Hessian-vector product of an array-input scalar-output function f, as evaluated at x times the vector v. The result will be stored into res. The function still allocates and zero's a buffer to store the intermediate gradient, which is not returned to the user.
In other words, compute res .= hessian(f)(x) * v
See hvp_and_gradient! for a function to compute both the hvp and the gradient in a single call.
Example:
f(x) = sin(x[1] * x[2])
+(derivs = ([3.0], nothing), val = 6.0)
source
Enzyme.hvp! — Method
hvp!(res::X, f::F, x::X, v::X) where {F, X}
Compute an in-place Hessian-vector product of an array-input scalar-output function f, as evaluated at x times the vector v. The result will be stored into res. The function still allocates and zero's a buffer to store the intermediate gradient, which is not returned to the user.
In other words, compute res .= hessian(f)(x) * v
See hvp_and_gradient! for a function to compute both the hvp and the gradient in a single call.
Example:
f(x) = sin(x[1] * x[2])
 
 res = Vector{Float64}(undef, 2)
 hvp!(res, f, [2.0, 3.0], [5.0, 2.7])
@@ -83,14 +83,14 @@
 # output
 2-element Vector{Float64}:
  19.6926882637302
- 16.201003759768003
source
Enzyme.hvp — Method
hvp(f::F, x::X, v::X) where {F, X}
Compute the Hessian-vector product of an array-input scalar-output function f, as evaluated at x times the vector v.
In other words, compute hessian(f)(x) * v
See hvp! for a version which stores the result in an existing buffer and also hvp_and_gradient! for a function to compute both the hvp and the gradient in a single call.
Example:
f(x) = sin(x[1] * x[2])
+ 16.201003759768003
source
Enzyme.hvp — Method
hvp(f::F, x::X, v::X) where {F, X}
Compute the Hessian-vector product of an array-input scalar-output function f, as evaluated at x times the vector v.
In other words, compute hessian(f)(x) * v
See hvp! for a version which stores the result in an existing buffer and also hvp_and_gradient! for a function to compute both the hvp and the gradient in a single call.
Example:
f(x) = sin(x[1] * x[2])
 
 hvp(f, [2.0, 3.0], [5.0, 2.7])
 
 # output
 2-element Vector{Float64}:
  19.6926882637302
- 16.201003759768003
source
Enzyme.hvp_and_gradient! — Method
hvp_and_gradient!(res::X, grad::X, f::F, x::X, v::X) where {F, X}
Compute an in-place Hessian-vector product of an array-input scalar-output function f, as evaluated at x times the vector v as well as the gradient, storing the gradient into grad. Both the hessian vector product and the gradient can be computed together more efficiently than computing them separately.
The result will be stored into res. The gradient will be stored into grad.
In other words, compute res .= hessian(f)(x) * v  and grad .= gradient(Reverse, f)(x)
Example:
f(x) = sin(x[1] * x[2])
+ 16.201003759768003
source
Enzyme.hvp_and_gradient! — Method
hvp_and_gradient!(res::X, grad::X, f::F, x::X, v::X) where {F, X}
Compute an in-place Hessian-vector product of an array-input scalar-output function f, as evaluated at x times the vector v as well as the gradient, storing the gradient into grad. Both the hessian vector product and the gradient can be computed together more efficiently than computing them separately.
The result will be stored into res. The gradient will be stored into grad.
In other words, compute res .= hessian(f)(x) * v  and grad .= gradient(Reverse, f)(x)
Example:
f(x) = sin(x[1] * x[2])
 
 res = Vector{Float64}(undef, 2)
 grad = Vector{Float64}(undef, 2)
@@ -101,7 +101,7 @@
 # output
 2-element Vector{Float64}:
  2.880510859951098
- 1.920340573300732
source
Enzyme.jacobian — Method
jacobian(::ForwardMode, args...; kwargs...)
Equivalent to gradient(::ForwardMode, args...; kwargs...)
source
Enzyme.jacobian — Method
jacobian(::ReverseMode, f, x; n_outs=nothing, chunk=nothing)
+ 1.920340573300732
source
Enzyme.jacobian — Method
jacobian(::ForwardMode, args...; kwargs...)
Equivalent to gradient(::ForwardMode, args...; kwargs...)
source
Enzyme.jacobian — Method
jacobian(::ReverseMode, f, x; n_outs=nothing, chunk=nothing)
 jacobian(::ReverseMode, f, x)
Compute the jacobian of a array-output function f using (potentially vector) reverse mode. The chunk argument optionally denotes the chunk size to use and n_outs optionally denotes the shape of the array returned by f (e.g size(f(x))).
Example:
f(x) = [ x[1] * x[2], x[2] + x[3] ]
 
 jacobian(Reverse, f, [2.0, 3.0, 4.0])
@@ -122,7 +122,7 @@
 grad = jacobian(ReverseWithPrimal, f, [2.0, 3.0, 4.0], n_outs=Val((2,)))
 
 # output
-(derivs = ([3.0 2.0 0.0; 0.0 1.0 1.0],), val = [6.0, 7.0])
This function will return an AbstractArray whose shape is (size(output)..., size(input)...). No guarantees are presently made about the type of the AbstractArray returned by this function (which may or may not be the same as the input AbstractArray if provided).
In the future, when this function is extended to handle non-array return types,  this function will retun an AbstractArray of shape size(output) of values of the input type.  ```
source
Enzyme.typetree — Function
function typetree(T, ctx, dl, seen=TypeTreeTable())
Construct a Enzyme typetree from a Julia type.
Warning
When using a memoized lookup by providing seen across multiple calls to typtree the user must call copy on the returned value before mutating it.
source
Enzyme.unsafe_to_pointer — Method
unsafe_to_pointer
Warning
Assumes that val is globally rooted and pointer to it can be leaked. Prefer pointer_from_objref. Only use inside Enzyme.jl should be for Types.
source
EnzymeCore.autodiff — Method
autodiff(::ForwardMode, f, Activity, args::Annotation...)
Auto-differentiate function f at arguments args using forward mode.
args may be numbers, arrays, structs of numbers, structs of arrays and so on. Enzyme will only differentiate in respect to arguments that are wrapped in a Duplicated or similar argument. Unlike reverse mode in autodiff, Active arguments are not allowed here, since all derivative results of immutable objects will be returned and should instead use Duplicated or variants like DuplicatedNoNeed.
Activity is the Activity of the return value, it may be:
  • Const if the return is not to be differentiated with respect to
  • Duplicated, if the return is being differentiated with respect to
  • BatchDuplicated, like Duplicated, but computing multiple derivatives at once. All batch sizes must be the same for all arguments.
Example returning both original return and derivative:
f(x) = x*x
+(derivs = ([3.0 2.0 0.0; 0.0 1.0 1.0],), val = [6.0, 7.0])
This function will return an AbstractArray whose shape is (size(output)..., size(input)...). No guarantees are presently made about the type of the AbstractArray returned by this function (which may or may not be the same as the input AbstractArray if provided).
In the future, when this function is extended to handle non-array return types,  this function will retun an AbstractArray of shape size(output) of values of the input type.  ```
source
Enzyme.typetree — Function
function typetree(T, ctx, dl, seen=TypeTreeTable())
Construct a Enzyme typetree from a Julia type.
Warning
When using a memoized lookup by providing seen across multiple calls to typtree the user must call copy on the returned value before mutating it.
source
Enzyme.unsafe_to_pointer — Method
unsafe_to_pointer
Warning
Assumes that val is globally rooted and pointer to it can be leaked. Prefer pointer_from_objref. Only use inside Enzyme.jl should be for Types.
source
EnzymeCore.autodiff — Method
autodiff(::ForwardMode, f, Activity, args::Annotation...)
Auto-differentiate function f at arguments args using forward mode.
args may be numbers, arrays, structs of numbers, structs of arrays and so on. Enzyme will only differentiate in respect to arguments that are wrapped in a Duplicated or similar argument. Unlike reverse mode in autodiff, Active arguments are not allowed here, since all derivative results of immutable objects will be returned and should instead use Duplicated or variants like DuplicatedNoNeed.
Activity is the Activity of the return value, it may be:
  • Const if the return is not to be differentiated with respect to
  • Duplicated, if the return is being differentiated with respect to
  • BatchDuplicated, like Duplicated, but computing multiple derivatives at once. All batch sizes must be the same for all arguments.
Example returning both original return and derivative:
f(x) = x*x
 res, ∂f_∂x = autodiff(ForwardWithPrimal, f, Duplicated, Duplicated(3.14, 1.0))
 
 # output
@@ -132,7 +132,7 @@
 
 # output
 
-(6.28,)
source
EnzymeCore.autodiff — Method
autodiff(::ReverseMode, f, Activity, args::Annotation...)
Auto-differentiate function f at arguments args using reverse mode.
Limitations:
  • f may only return a Real (of a built-in/primitive type) or nothing, not an array, struct, BigFloat, etc. To handle vector-valued return types, use a mutating f! that returns nothing and stores it's return value in one of the arguments, which must be wrapped in a Duplicated.
args may be numbers, arrays, structs of numbers, structs of arrays and so on. Enzyme will only differentiate in respect to arguments that are wrapped in an Active (for arguments whose derivative result must be returned rather than mutated in place, such as primitive types and structs thereof) or Duplicated (for mutable arguments like arrays, Refs and structs thereof).
Activity is the Activity of the return value, it may be Const or Active.
Example:
a = 4.2
+(6.28,)
source
EnzymeCore.autodiff — Method
autodiff(::ReverseMode, f, Activity, args::Annotation...)
Auto-differentiate function f at arguments args using reverse mode.
Limitations:
  • f may only return a Real (of a built-in/primitive type) or nothing, not an array, struct, BigFloat, etc. To handle vector-valued return types, use a mutating f! that returns nothing and stores it's return value in one of the arguments, which must be wrapped in a Duplicated.
args may be numbers, arrays, structs of numbers, structs of arrays and so on. Enzyme will only differentiate in respect to arguments that are wrapped in an Active (for arguments whose derivative result must be returned rather than mutated in place, such as primitive types and structs thereof) or Duplicated (for mutable arguments like arrays, Refs and structs thereof).
Activity is the Activity of the return value, it may be Const or Active.
Example:
a = 4.2
 b = [2.2, 3.3]; ∂f_∂b = zero(b)
 c = 55; d = 9
 
@@ -145,13 +145,13 @@
 
 # output
 
-((6.0,), 9.0)
Note
Enzyme gradients with respect to integer values are zero. Active will automatically convert plain integers to floating point values, but cannot do so for integer values in tuples and structs.
source
EnzymeCore.autodiff — Method
autodiff(::Function, ::Mode, args...)
Specialization of autodiff to handle do argument closures.

+((6.0,), 9.0)
Note
Enzyme gradients with respect to integer values are zero. Active will automatically convert plain integers to floating point values, but cannot do so for integer values in tuples and structs.
source
EnzymeCore.autodiff — Method
autodiff(::Function, ::Mode, args...)
Specialization of autodiff to handle do argument closures.

 autodiff(Reverse, Active(3.1)) do x
   return x*x
 end
 
 # output
-((6.2,),)
source
EnzymeCore.autodiff_deferred — Method
autodiff_deferred(::ForwardMode, f, Activity, args::Annotation...)
Same as autodiff(::ForwardMode, f, Activity, args...) but uses deferred compilation to support usage in GPU code, as well as high-order differentiation.
source
EnzymeCore.autodiff_deferred — Method
autodiff_deferred(::ReverseMode, f, Activity, args::Annotation...)
Same as autodiff but uses deferred compilation to support usage in GPU code, as well as high-order differentiation.
source
EnzymeCore.autodiff_deferred_thunk — Method
autodiff_deferred_thunk(::ReverseModeSplit, ftype, Activity, argtypes::Type{<:Annotation}...)
Provide the split forward and reverse pass functions for annotated function type ftype when called with args of type argtypes when using reverse mode.
Activity is the Activity of the return value, it may be Const, Active, or Duplicated (or its variants DuplicatedNoNeed, BatchDuplicated, and BatchDuplicatedNoNeed).
The forward function will return a tape, the primal (or nothing if not requested), and the shadow (or nothing if not a Duplicated variant), and tapes the corresponding type arguements provided.
The reverse function will return the derivative of Active arguments, updating the Duplicated arguments in place. The same arguments to the forward pass should be provided, followed by the adjoint of the return (if the return is active), and finally the tape from the forward pass.
Example:

+((6.2,),)
source
EnzymeCore.autodiff_deferred — Method
autodiff_deferred(::ForwardMode, f, Activity, args::Annotation...)
Same as autodiff(::ForwardMode, f, Activity, args...) but uses deferred compilation to support usage in GPU code, as well as high-order differentiation.
source
EnzymeCore.autodiff_deferred — Method
autodiff_deferred(::ReverseMode, f, Activity, args::Annotation...)
Same as autodiff but uses deferred compilation to support usage in GPU code, as well as high-order differentiation.
source
EnzymeCore.autodiff_deferred_thunk — Method
autodiff_deferred_thunk(::ReverseModeSplit, ftype, Activity, argtypes::Type{<:Annotation}...)
Provide the split forward and reverse pass functions for annotated function type ftype when called with args of type argtypes when using reverse mode.
Activity is the Activity of the return value, it may be Const, Active, or Duplicated (or its variants DuplicatedNoNeed, BatchDuplicated, and BatchDuplicatedNoNeed).
The forward function will return a tape, the primal (or nothing if not requested), and the shadow (or nothing if not a Duplicated variant), and tapes the corresponding type arguements provided.
The reverse function will return the derivative of Active arguments, updating the Duplicated arguments in place. The same arguments to the forward pass should be provided, followed by the adjoint of the return (if the return is active), and finally the tape from the forward pass.
Example:

 A = [2.2]; ∂A = zero(A)
 v = 3.3
 
@@ -171,7 +171,7 @@
 
 # output
 
-(7.26, 2.2, [3.3])
source
EnzymeCore.autodiff_thunk — Method
autodiff_thunk(::ForwardMode, ftype, Activity, argtypes::Type{<:Annotation}...)
Provide the thunk forward mode function for annotated function type ftype when called with args of type argtypes.
Activity is the Activity of the return value, it may be Const or Duplicated (or its variants DuplicatedNoNeed, BatchDuplicated, andBatchDuplicatedNoNeed).
The forward function will return the primal (if requested) and the shadow (or nothing if not a Duplicated variant).
Example returning both the return derivative and original return:
a = 4.2
+(7.26, 2.2, [3.3])
source
EnzymeCore.autodiff_thunk — Method
autodiff_thunk(::ForwardMode, ftype, Activity, argtypes::Type{<:Annotation}...)
Provide the thunk forward mode function for annotated function type ftype when called with args of type argtypes.
Activity is the Activity of the return value, it may be Const or Duplicated (or its variants DuplicatedNoNeed, BatchDuplicated, andBatchDuplicatedNoNeed).
The forward function will return the primal (if requested) and the shadow (or nothing if not a Duplicated variant).
Example returning both the return derivative and original return:
a = 4.2
 b = [2.2, 3.3]; ∂f_∂b = zero(b)
 c = 55; d = 9
 
@@ -191,7 +191,7 @@
 
 # output
 
-(6.28,)
source
EnzymeCore.autodiff_thunk — Method
autodiff_thunk(::ReverseModeSplit, ftype, Activity, argtypes::Type{<:Annotation}...)
Provide the split forward and reverse pass functions for annotated function type ftype when called with args of type argtypes when using reverse mode.
Activity is the Activity of the return value, it may be Const, Active, or Duplicated (or its variants DuplicatedNoNeed, BatchDuplicated, and BatchDuplicatedNoNeed).
The forward function will return a tape, the primal (or nothing if not requested), and the shadow (or nothing if not a Duplicated variant), and tapes the corresponding type arguements provided.
The reverse function will return the derivative of Active arguments, updating the Duplicated arguments in place. The same arguments to the forward pass should be provided, followed by the adjoint of the return (if the return is active), and finally the tape from the forward pass.
Example:

+(6.28,)
source
EnzymeCore.autodiff_thunk — Method
autodiff_thunk(::ReverseModeSplit, ftype, Activity, argtypes::Type{<:Annotation}...)
Provide the split forward and reverse pass functions for annotated function type ftype when called with args of type argtypes when using reverse mode.
Activity is the Activity of the return value, it may be Const, Active, or Duplicated (or its variants DuplicatedNoNeed, BatchDuplicated, and BatchDuplicatedNoNeed).
The forward function will return a tape, the primal (or nothing if not requested), and the shadow (or nothing if not a Duplicated variant), and tapes the corresponding type arguements provided.
The reverse function will return the derivative of Active arguments, updating the Duplicated arguments in place. The same arguments to the forward pass should be provided, followed by the adjoint of the return (if the return is active), and finally the tape from the forward pass.
Example:

 A = [2.2]; ∂A = zero(A)
 v = 3.3
 
@@ -210,24 +210,36 @@
 
 # output
 
-(7.26, 2.2, [3.3])
source
EnzymeCore.Active — Type
Active(x)
Mark a function argument x of autodiff as active, Enzyme will auto-differentiate in respect Active arguments.
Note
Enzyme gradients with respect to integer values are zero. Active will automatically convert plain integers to floating point values, but cannot do so for integer values in tuples and structs.
source
EnzymeCore.BatchDuplicated — Type
BatchDuplicated(x, ∂f_∂xs)
Like Duplicated, except contains several shadows to compute derivatives for all at once. Argument ∂f_∂xs should be a tuple of the several values of type x.
source
EnzymeCore.Const — Type
Const(x)
Mark a function argument x of autodiff as constant, Enzyme will not auto-differentiate in respect Const arguments.
source
EnzymeCore.Duplicated — Type
Duplicated(x, ∂f_∂x)
Mark a function argument x of autodiff as duplicated, Enzyme will auto-differentiate in respect to such arguments, with dx acting as an accumulator for gradients (so $\partial f / \partial x$ will be added to) ∂f_∂x.
source
EnzymeCore.DuplicatedNoNeed — Type
DuplicatedNoNeed(x, ∂f_∂x)
Like Duplicated, except also specifies that Enzyme may avoid computing the original result and only compute the derivative values.
This should only be used if x is a write-only variable. Otherwise, if the differentiated function stores values in x and reads them back in subsequent computations, using DuplicatedNoNeed may result in incorrect derivatives. In particular, DuplicatedNoNeed should not be used for preallocated workspace, even if the user might not care about its final value, as marking a variable as NoNeed means that reads from the variable are now undefined.
source
EnzymeCore.FFIABI — Type
struct FFIABI <: ABI
Foreign function call ABI. JIT the differentiated function, then inttoptr call the address.
source
EnzymeCore.ForwardMode — Type
struct Forward{ReturnPrimal, ABI, ErrIfFuncWritten,RuntimeActivity} <: Mode{ABI, ErrIfFuncWritten, RuntimeActivity}
Forward mode differentiation
source
EnzymeCore.MixedDuplicated — Type
MixedDuplicated(x, ∂f_∂x)
Like Duplicated, except x may contain both active [immutable] and duplicated [mutable] data which is differentiable. Only used within custom rules.
source
EnzymeCore.Mode — Type
abstract type Mode
Abstract type for what differentiation mode will be used.
source
EnzymeCore.ReverseMode — Type
struct ReverseMode{ReturnPrimal,RuntimeActivity,ABI,Holomorphic,ErrIfFuncWritten} <: Mode{ABI, ErrIfFuncWritten, RuntimeActivity}
Reverse mode differentiation.
  • ReturnPrimal: Should Enzyme return the primal return value from the augmented-forward.
  • RuntimeActivity: Should Enzyme enable runtime activity (default off)
  • ABI: What runtime ABI to use
  • Holomorphic: Whether the complex result function is holomorphic and we should compute d/dz
  • ErrIfFuncWritten: Should Enzyme err if the function differentiated is a closure and written to.
source
EnzymeCore.ReverseModeSplit — Type
struct ReverseModeSplit{ReturnPrimal,ReturnShadow,RuntimeActivity,Width,ModifiedBetween,ABI} <: Mode{ABI,ErrIfFuncWritten,RuntimeActivity}
Reverse mode differentiation.
  • ReturnPrimal: Should Enzyme return the primal return value from the augmented-forward.
  • ReturnShadow: Should Enzyme return the shadow return value from the augmented-forward.
  • RuntimeActivity: Should Enzyme differentiate with runtime activity on (default off).
  • Width: Batch Size (0 if to be automatically derived)
  • ModifiedBetween: Tuple of each argument's modified between state (true if to be automatically derived).
source
EnzymeCore.compiler_job_from_backend — Function
compiler_job_from_backend(::KernelAbstractions.Backend, F::Type, TT:Type)::GPUCompiler.CompilerJob
Returns a GPUCompiler CompilerJob from a backend as specified by the first argument to the function.
For example, in CUDA one would do:
function EnzymeCore.compiler_job_from_backend(::CUDABackend, @nospecialize(F::Type), @nospecialize(TT::Type))
+(7.26, 2.2, [3.3])
source
EnzymeCore.Active — Type
Active(x)
Mark a function argument x of autodiff as active, Enzyme will auto-differentiate in respect Active arguments.
Note
Enzyme gradients with respect to integer values are zero. Active will automatically convert plain integers to floating point values, but cannot do so for integer values in tuples and structs.
source
EnzymeCore.BatchDuplicated — Type
BatchDuplicated(x, ∂f_∂xs)
Like Duplicated, except contains several shadows to compute derivatives for all at once. Argument ∂f_∂xs should be a tuple of the several values of type x.
source
EnzymeCore.Const — Type
Const(x)
Mark a function argument x of autodiff as constant, Enzyme will not auto-differentiate in respect Const arguments.
source
EnzymeCore.Duplicated — Type
Duplicated(x, ∂f_∂x)
Mark a function argument x of autodiff as duplicated, Enzyme will auto-differentiate in respect to such arguments, with dx acting as an accumulator for gradients (so $\partial f / \partial x$ will be added to) ∂f_∂x.
source
EnzymeCore.DuplicatedNoNeed — Type
DuplicatedNoNeed(x, ∂f_∂x)
Like Duplicated, except also specifies that Enzyme may avoid computing the original result and only compute the derivative values. This creates opportunities for improved performance.

+function square_byref(out, v)
+    out[] = v * v
+    nothing
+end
+
+out = Ref(0.0)
+dout = Ref(1.0)
+Enzyme.autodiff(Reverse, square_byref, DuplicatedNoNeed(out, dout), Active(1.0))
+dout[]
+
+# output
+0.0
For example, marking the out variable as DuplicatedNoNeed instead of Duplicated allows Enzyme to avoid computing v * v (while still computing its derivative).
This should only be used if x is a write-only variable. Otherwise, if the differentiated function stores values in x and reads them back in subsequent computations, using DuplicatedNoNeed may result in incorrect derivatives. In particular, DuplicatedNoNeed should not be used for preallocated workspace, even if the user might not care about its final value, as marking a variable as NoNeed means that reads from the variable are now undefined.
source
EnzymeCore.FFIABI — Type
struct FFIABI <: ABI
Foreign function call ABI. JIT the differentiated function, then inttoptr call the address.
source
EnzymeCore.ForwardMode — Type
struct Forward{ReturnPrimal, ABI, ErrIfFuncWritten,RuntimeActivity} <: Mode{ABI, ErrIfFuncWritten, RuntimeActivity}
Forward mode differentiation
source
EnzymeCore.MixedDuplicated — Type
MixedDuplicated(x, ∂f_∂x)
Like Duplicated, except x may contain both active [immutable] and duplicated [mutable] data which is differentiable. Only used within custom rules.
source
EnzymeCore.Mode — Type
abstract type Mode
Abstract type for what differentiation mode will be used.
source
EnzymeCore.ReverseMode — Type
struct ReverseMode{ReturnPrimal,RuntimeActivity,ABI,Holomorphic,ErrIfFuncWritten} <: Mode{ABI, ErrIfFuncWritten, RuntimeActivity}
Reverse mode differentiation.
  • ReturnPrimal: Should Enzyme return the primal return value from the augmented-forward.
  • RuntimeActivity: Should Enzyme enable runtime activity (default off)
  • ABI: What runtime ABI to use
  • Holomorphic: Whether the complex result function is holomorphic and we should compute d/dz
  • ErrIfFuncWritten: Should Enzyme err if the function differentiated is a closure and written to.
source
EnzymeCore.ReverseModeSplit — Type
struct ReverseModeSplit{ReturnPrimal,ReturnShadow,RuntimeActivity,Width,ModifiedBetween,ABI} <: Mode{ABI,ErrIfFuncWritten,RuntimeActivity}
Reverse mode differentiation.
  • ReturnPrimal: Should Enzyme return the primal return value from the augmented-forward.
  • ReturnShadow: Should Enzyme return the shadow return value from the augmented-forward.
  • RuntimeActivity: Should Enzyme differentiate with runtime activity on (default off).
  • Width: Batch Size (0 if to be automatically derived)
  • ModifiedBetween: Tuple of each argument's modified between state (true if to be automatically derived).
source
EnzymeCore.compiler_job_from_backend — Function
compiler_job_from_backend(::KernelAbstractions.Backend, F::Type, TT:Type)::GPUCompiler.CompilerJob
Returns a GPUCompiler CompilerJob from a backend as specified by the first argument to the function.
For example, in CUDA one would do:
function EnzymeCore.compiler_job_from_backend(::CUDABackend, @nospecialize(F::Type), @nospecialize(TT::Type))
     mi = GPUCompiler.methodinstance(F, TT)
     return GPUCompiler.CompilerJob(mi, CUDA.compiler_config(CUDA.device()))
-end
source
EnzymeCore.make_zero — Function
make_zero(::Type{T}, seen::IdDict, prev::T, ::Val{copy_if_inactive}=Val(false))::T
 
 Recursively make a zero'd copy of the value `prev` of type `T`. The argument `copy_if_inactive` specifies
-what to do if the type `T` is guaranteed to be inactive, use the primal (the default) or still copy the value.
source
EnzymeCore.make_zero! — Function
make_zero!(val::T, seen::IdSet{Any}=IdSet())::Nothing
-
-Recursively set a variables differentiable fields to zero. Only applicable for mutable types `T`.
source
EnzymeCore.EnzymeRules.AugmentedReturn — Type
AugmentedReturn(primal, shadow, tape)
Augment the primal return value of a function with its shadow, as well as any additional information needed to correctly  compute the reverse pass, stored in tape.
Unless specified by the config that a variable is not overwritten, rules must assume any arrays/data structures/etc are  overwritten between the forward and the reverse pass. Any floats or variables passed by value are always preserved as is  (as are the arrays themselves, just not necessarily the values in the array).
See also augmented_primal.
source
EnzymeCore.EnzymeRules.FwdConfig — Type
FwdConfig{NeedsPrimal, NeedsShadow, Width, RuntimeActivity}
-FwdConfigWidth{Width} = FwdConfig{<:Any, <:Any, Width}
Configuration type to dispatch on in custom forward rules (see forward.
  • NeedsPrimal and NeedsShadow: boolean values specifying whether the primal and shadow (resp.) should be returned. 
  • Width: an integer that specifies the number of adjoints/shadows simultaneously being propagated.
  • RuntimeActivity: whether runtime activity is enabled.
Getters for the type parameters are provided by needs_primal, needs_shadow, width and runtime_activity.
source
EnzymeCore.EnzymeRules.RevConfig — Type
RevConfig{NeedsPrimal, NeedsShadow, Width, Overwritten, RuntimeActivity}
-RevConfigWidth{Width} = RevConfig{<:Any, <:Any, Width}
Configuration type to dispatch on in custom reverse rules (see augmented_primal and reverse).
  • NeedsPrimal and NeedsShadow: boolean values specifying whether the primal and shadow (resp.) should be returned. 
  • Width: an integer that specifies the number of adjoints/shadows simultaneously being propagated.
  • Overwritten: a tuple of booleans of whether each argument (including the function itself) is modified between the   forward and reverse pass (true if potentially modified between).
  • RuntimeActivity: whether runtime activity is enabled.
Getters for the four type parameters are provided by needs_primal, needs_shadow, width, overwritten, and runtime_activity.
source
EnzymeCore.EnzymeRules.augmented_primal — Function
augmented_primal(::RevConfig, func::Annotation{typeof(f)}, RT::Type{<:Annotation}, args::Annotation...)
Must return an AugmentedReturn type.
  • The primal must be the same type of the original return if needs_primal(config), otherwise nothing.
  • The shadow must be nothing if needs_shadow(config) is false. If width is 1, the shadow should be the same type of the original return. If the width is greater than 1, the shadow should be NTuple{original return, width}.
  • The tape can be any type (including Nothing) and is preserved for the reverse call.
source
EnzymeCore.EnzymeRules.forward — Function
forward(fwdconfig, func::Annotation{typeof(f)}, RT::Type{<:Annotation}, args::Annotation...)
Calculate the forward derivative. The first argument is a `FwdConfig object describing parameters of the differentiation. The second argument func is the callable for which the rule applies to. Either wrapped in a Const), or a Duplicated if it is a closure. The third argument is the return type annotation, and all other arguments are the annotated function arguments.
source
EnzymeCore.EnzymeRules.inactive — Function
inactive(func::typeof(f), args...)
Mark a particular function as always being inactive in both its return result and the function call itself.
source
EnzymeCore.EnzymeRules.inactive_noinl — Function
inactive_noinl(func::typeof(f), args...)
Mark a particular function as always being inactive in both its return result and the function call itself, but do not prevent inlining of the function.
source
EnzymeCore.EnzymeRules.noalias — Function
noalias(func::typeof(f), args...)
Mark a particular function as always being a fresh allocation which does not alias any other  accessible memory.
source
EnzymeCore.EnzymeRules.primal_type — Method
primal_type(::FwdConfig, ::Type{<:Annotation{RT}})
-primal_type(::RevConfig, ::Type{<:Annotation{RT}})
Compute the exepcted primal return type given a reverse mode config and return activity
source
EnzymeCore.EnzymeRules.reverse — Function
reverse(::RevConfig, func::Annotation{typeof(f)}, dret::Active, tape, args::Annotation...)
-reverse(::RevConfig, func::Annotation{typeof(f)}, ::Type{<:Annotation), tape, args::Annotation...)
Takes gradient of derivative, activity annotation, and tape. If there is an active return dret is passed as Active{T} with the derivative of the active return val. Otherwise dret is passed as Type{Duplicated{T}}, etc.
source
EnzymeCore.EnzymeRules.shadow_type — Method
shadow_type(::FwdConfig, ::Type{<:Annotation{RT}})
-shadow_type(::RevConfig, ::Type{<:Annotation{RT}})
Compute the exepcted shadow return type given a reverse mode config and return activity
source
EnzymeTestUtils.@test_msg — Macro
@test_msg msg condion kws...
This is per Test.@test condion kws... except that if it fails it also prints the msg. If msg=="" then this is just like @test, nothing is printed
Examles
julia> @test_msg "It is required that the total is under 10" sum(1:1000) < 10;
+what to do if the type `T` is guaranteed to be inactive, use the primal (the default) or still copy the value.
source
EnzymeCore.make_zero! — Function
make_zero!(val::T, seen::IdSet{Any}=IdSet())::Nothing
+
+Recursively set a variables differentiable fields to zero. Only applicable for mutable types `T`.
source
EnzymeCore.EnzymeRules.AugmentedReturn — Type
AugmentedReturn(primal, shadow, tape)
Augment the primal return value of a function with its shadow, as well as any additional information needed to correctly  compute the reverse pass, stored in tape.
Unless specified by the config that a variable is not overwritten, rules must assume any arrays/data structures/etc are  overwritten between the forward and the reverse pass. Any floats or variables passed by value are always preserved as is  (as are the arrays themselves, just not necessarily the values in the array).
See also augmented_primal.
source
EnzymeCore.EnzymeRules.FwdConfig — Type
FwdConfig{NeedsPrimal, NeedsShadow, Width, RuntimeActivity}
+FwdConfigWidth{Width} = FwdConfig{<:Any, <:Any, Width}
Configuration type to dispatch on in custom forward rules (see forward.
  • NeedsPrimal and NeedsShadow: boolean values specifying whether the primal and shadow (resp.) should be returned. 
  • Width: an integer that specifies the number of adjoints/shadows simultaneously being propagated.
  • RuntimeActivity: whether runtime activity is enabled.
Getters for the type parameters are provided by needs_primal, needs_shadow, width and runtime_activity.
source
EnzymeCore.EnzymeRules.RevConfig — Type
RevConfig{NeedsPrimal, NeedsShadow, Width, Overwritten, RuntimeActivity}
+RevConfigWidth{Width} = RevConfig{<:Any, <:Any, Width}
Configuration type to dispatch on in custom reverse rules (see augmented_primal and reverse).
  • NeedsPrimal and NeedsShadow: boolean values specifying whether the primal and shadow (resp.) should be returned. 
  • Width: an integer that specifies the number of adjoints/shadows simultaneously being propagated.
  • Overwritten: a tuple of booleans of whether each argument (including the function itself) is modified between the   forward and reverse pass (true if potentially modified between).
  • RuntimeActivity: whether runtime activity is enabled.
Getters for the four type parameters are provided by needs_primal, needs_shadow, width, overwritten, and runtime_activity.
source
EnzymeCore.EnzymeRules.augmented_primal — Function
augmented_primal(::RevConfig, func::Annotation{typeof(f)}, RT::Type{<:Annotation}, args::Annotation...)
Must return an AugmentedReturn type.
  • The primal must be the same type of the original return if needs_primal(config), otherwise nothing.
  • The shadow must be nothing if needs_shadow(config) is false. If width is 1, the shadow should be the same type of the original return. If the width is greater than 1, the shadow should be NTuple{original return, width}.
  • The tape can be any type (including Nothing) and is preserved for the reverse call.
source
EnzymeCore.EnzymeRules.forward — Function
forward(fwdconfig, func::Annotation{typeof(f)}, RT::Type{<:Annotation}, args::Annotation...)
Calculate the forward derivative. The first argument is a `FwdConfig object describing parameters of the differentiation. The second argument func is the callable for which the rule applies to. Either wrapped in a Const), or a Duplicated if it is a closure. The third argument is the return type annotation, and all other arguments are the annotated function arguments.
source
EnzymeCore.EnzymeRules.inactive — Function
inactive(func::typeof(f), args...)
Mark a particular function as always being inactive in both its return result and the function call itself.
source
EnzymeCore.EnzymeRules.inactive_noinl — Function
inactive_noinl(func::typeof(f), args...)
Mark a particular function as always being inactive in both its return result and the function call itself, but do not prevent inlining of the function.
source
EnzymeCore.EnzymeRules.noalias — Function
noalias(func::typeof(f), args...)
Mark a particular function as always being a fresh allocation which does not alias any other  accessible memory.
source
EnzymeCore.EnzymeRules.primal_type — Method
primal_type(::FwdConfig, ::Type{<:Annotation{RT}})
+primal_type(::RevConfig, ::Type{<:Annotation{RT}})
Compute the exepcted primal return type given a reverse mode config and return activity
source
EnzymeCore.EnzymeRules.reverse — Function
reverse(::RevConfig, func::Annotation{typeof(f)}, dret::Active, tape, args::Annotation...)
+reverse(::RevConfig, func::Annotation{typeof(f)}, ::Type{<:Annotation), tape, args::Annotation...)
Takes gradient of derivative, activity annotation, and tape. If there is an active return dret is passed as Active{T} with the derivative of the active return val. Otherwise dret is passed as Type{Duplicated{T}}, etc.
source
EnzymeCore.EnzymeRules.shadow_type — Method
shadow_type(::FwdConfig, ::Type{<:Annotation{RT}})
+shadow_type(::RevConfig, ::Type{<:Annotation{RT}})
Compute the exepcted shadow return type given a reverse mode config and return activity
source
EnzymeTestUtils.@test_msg — Macro
@test_msg msg condion kws...
This is per Test.@test condion kws... except that if it fails it also prints the msg. If msg=="" then this is just like @test, nothing is printed
Examles
julia> @test_msg "It is required that the total is under 10" sum(1:1000) < 10;
 Test Failed at REPL[1]:1
   Expression: sum(1:1000) < 10
   Problem: It is required that the total is under 10
@@ -249,7 +261,7 @@
   Test Failed at REPL[153]:1
     Expression: sum(1:1000) < 10
      Evaluated: 500500 < 10
-  ERROR: There was an error during testing
source
EnzymeTestUtils.test_forward — Method
test_forward(f, Activity, args...; kwargs...)
Test Enzyme.autodiff of f in Forward-mode against finite differences.
f has all constraints of the same argument passed to Enzyme.autodiff, with additional constraints:
  • If it mutates one of its arguments, it must return that argument.
Arguments
  • Activity: the activity of the return value of f
  • args: Each entry is either an argument to f, an activity type accepted by autodiff,   or a tuple of the form (arg, Activity), where Activity is the activity type of   arg. If the activity type specified requires a tangent, a random tangent will be   automatically generated.
Keywords
  • rng::AbstractRNG: The random number generator to use for generating random tangents.
  • fdm=FiniteDifferences.central_fdm(5, 1): The finite differences method to use.
  • fkwargs: Keyword arguments to pass to f.
  • rtol: Relative tolerance for isapprox.
  • atol: Absolute tolerance for isapprox.
  • testset_name: Name to use for a testset in which all tests are evaluated.
Examples
Here we test a rule for a function of scalars. Because we don't provide an activity annotation for y, it is assumed to be Const.
using Enzyme, EnzymeTestUtils
+  ERROR: There was an error during testing
source
EnzymeTestUtils.test_forward — Method
test_forward(f, Activity, args...; kwargs...)
Test Enzyme.autodiff of f in Forward-mode against finite differences.
f has all constraints of the same argument passed to Enzyme.autodiff, with additional constraints:
  • If it mutates one of its arguments, it must return that argument.
Arguments
  • Activity: the activity of the return value of f
  • args: Each entry is either an argument to f, an activity type accepted by autodiff,   or a tuple of the form (arg, Activity), where Activity is the activity type of   arg. If the activity type specified requires a tangent, a random tangent will be   automatically generated.
Keywords
  • rng::AbstractRNG: The random number generator to use for generating random tangents.
  • fdm=FiniteDifferences.central_fdm(5, 1): The finite differences method to use.
  • fkwargs: Keyword arguments to pass to f.
  • rtol: Relative tolerance for isapprox.
  • atol: Absolute tolerance for isapprox.
  • testset_name: Name to use for a testset in which all tests are evaluated.
Examples
Here we test a rule for a function of scalars. Because we don't provide an activity annotation for y, it is assumed to be Const.
using Enzyme, EnzymeTestUtils
 
 x, y = randn(2)
 for Tret in (Const, Duplicated, DuplicatedNoNeed), Tx in (Const, Duplicated)
@@ -261,7 +273,7 @@
     Ty in (Const, BatchDuplicated)
 
     test_forward(*, Tret, (x, Tx), (y, Ty))
-end
source
EnzymeTestUtils.test_reverse — Method
test_reverse(f, Activity, args...; kwargs...)
Test Enzyme.autodiff_thunk of f in ReverseSplitWithPrimal-mode against finite differences.
f has all constraints of the same argument passed to Enzyme.autodiff_thunk, with additional constraints:
  • If an Array{<:AbstractFloat} appears in the input/output, then a reshaped version of it   may not also appear in the input/output.
Arguments
  • Activity: the activity of the return value of f.
  • args: Each entry is either an argument to f, an activity type accepted by autodiff,   or a tuple of the form (arg, Activity), where Activity is the activity type of   arg. If the activity type specified requires a shadow, one will be automatically   generated.
Keywords
  • rng::AbstractRNG: The random number generator to use for generating random tangents.
  • fdm=FiniteDifferences.central_fdm(5, 1): The finite differences method to use.
  • fkwargs: Keyword arguments to pass to f.
  • rtol: Relative tolerance for isapprox.
  • atol: Absolute tolerance for isapprox.
  • testset_name: Name to use for a testset in which all tests are evaluated.
Examples
Here we test a rule for a function of scalars. Because we don't provide an activity annotation for y, it is assumed to be Const.
using Enzyme, EnzymeTestUtils
+end
source
EnzymeTestUtils.test_reverse — Method
test_reverse(f, Activity, args...; kwargs...)
Test Enzyme.autodiff_thunk of f in ReverseSplitWithPrimal-mode against finite differences.
f has all constraints of the same argument passed to Enzyme.autodiff_thunk, with additional constraints:
  • If an Array{<:AbstractFloat} appears in the input/output, then a reshaped version of it   may not also appear in the input/output.
Arguments
  • Activity: the activity of the return value of f.
  • args: Each entry is either an argument to f, an activity type accepted by autodiff,   or a tuple of the form (arg, Activity), where Activity is the activity type of   arg. If the activity type specified requires a shadow, one will be automatically   generated.
Keywords
  • rng::AbstractRNG: The random number generator to use for generating random tangents.
  • fdm=FiniteDifferences.central_fdm(5, 1): The finite differences method to use.
  • fkwargs: Keyword arguments to pass to f.
  • rtol: Relative tolerance for isapprox.
  • atol: Absolute tolerance for isapprox.
  • testset_name: Name to use for a testset in which all tests are evaluated.
Examples
Here we test a rule for a function of scalars. Because we don't provide an activity annotation for y, it is assumed to be Const.
using Enzyme, EnzymeTestUtils
 
 x = randn()
 y = randn()
@@ -270,4 +282,4 @@
 end
Here we test a rule for a function of an array in batch reverse-mode:
x = randn(3)
 for Tret in (Const, Active), Tx in (Const, BatchDuplicated)
     test_reverse(prod, Tret, (x, Tx))
-end
source
Enzyme.API.inlineall! — Method
inlineall!(val::Bool)
Whether to inline all (non-recursive) functions generated by Julia within a  single compilation unit. This may improve Enzyme's ability to successfully differentiate code and improve performance of the original and generated  derivative program. It often, however, comes with an increase in compile time. This is off by default.
source
Enzyme.API.instname! — Method
instname!(val::Bool)
Whether to add a name to all LLVM values. This may be helpful for debugging generated programs, both primal and derivative. Off by default.
source
Enzyme.API.looseTypeAnalysis! — Method
looseTypeAnalysis!(val::Bool)
Enzyme runs a type analysis to deduce the corresponding types of all values being differentiated. This is necessary to compute correct derivatives of various values. For example, a copy of Float32's requires a different derivative than a memcpy of Float64's, Ptr's, etc. In some cases Enzyme may not be able to deduce all the types necessary and throw an unknown type error. If this is the case, open an issue.  One can silence these issues by setting looseTypeAnalysis!(true) which tells  Enzyme to make its best guess. This will remove the error and allow differentiation to continue, however, it may produce incorrect results. Alternatively one can consider increasing the space of the evaluated type lattice which gives Enzyme more time to run a more thorough analysis through the use of maxtypeoffset!
source
Enzyme.API.maxtypedepth! — Method
maxtypedepth!(val::Int)
Enzyme runs a type analysis to deduce the corresponding types of all values being differentiated. This is necessary to compute correct derivatives of various values. To ensure this analysis temrinates, it operates on a finite lattice of possible states. This function sets the maximum depth into a type that Enzyme will consider. A smaller value will cause type analysis to run faster, but may result in some necessary types not being found and result in unknown type errors. A larger value may result in unknown type errors being resolved by searching a larger space, but may run longer. The default setting is 6.
source
Enzyme.API.maxtypeoffset! — Method
maxtypeoffset!(val::Int)
Enzyme runs a type analysis to deduce the corresponding types of all values being differentiated. This is necessary to compute correct derivatives of various values. To ensure this analysis temrinates, it operates on a finite lattice of possible states. This function sets the maximum offset into a type that Enzyme will consider. A smaller value will cause type analysis to run faster, but may result in some necessary types not being found and result in unknown type errors. A larger value may result in unknown type errors being resolved by searching a larger space, but may run longer. The default setting is 512.
source
Enzyme.API.printactivity! — Method
printactivity!(val::Bool)
An debugging option for developers of Enzyme. If one sets this flag prior to the first differentiation of a function, Enzyme will print (to stderr) a log of all decisions made during Activity Analysis (the analysis which determines what values/instructions are differentiated). This may be useful for debugging MixedActivity errors, correctness, and performance errors. Off by default
source
Enzyme.API.printall! — Method
printall!(val::Bool)
An debugging option for developers of Enzyme. If one sets this flag prior to the first differentiation of a function, Enzyme will print (to stderr) the LLVM function being differentiated, as well as all generated derivatives immediately after running Enzyme (but prior to any other optimizations). Off by default
source
Enzyme.API.printdiffuse! — Method
printdiffuse!(val::Bool)
An debugging option for developers of Enzyme. If one sets this flag prior to the first differentiation of a function, Enzyme will print (to stderr) information about each LLVM value – specifically whether it and its shadow is required for computing the derivative. In contrast to printunnecessary!, this flag prints debug log for the analysis which determines for each value and shadow value, whether it can find a user which would require it to be kept around (rather than being deleted). This is prior to any cache optimizations and a debug log of Differential Use Analysis. This may be helpful for debugging caching, phi node deletion, performance, and other errors. Off by default
source
Enzyme.API.printperf! — Method
printperf!(val::Bool)
An debugging option for developers of Enzyme. If one sets this flag prior to the first differentiation of a function, Enzyme will print (to stderr) performance information about generated derivative programs. It will provide debug information that warns why particular values are cached for the reverse pass, and thus require additional computation/storage. This is particularly helpful for debugging derivatives which OOM or otherwise run slow. ff by default
source
Enzyme.API.printtype! — Method
printtype!(val::Bool)
An debugging option for developers of Enzyme. If one sets this flag prior to the first differentiation of a function, Enzyme will print (to stderr) a log of all decisions made during Type Analysis (the analysis which Enzyme determines the type of all values in the program). This may be useful for debugging correctness errors, illegal type analysis errors, insufficient type information errors, correctness, and performance errors. Off by default
source
Enzyme.API.printunnecessary! — Method
printunnecessary!(val::Bool)
An debugging option for developers of Enzyme. If one sets this flag prior to the first differentiation of a function, Enzyme will print (to stderr) information about each LLVM value – specifically whether it and its shadow is required for computing the derivative. In contrast to printdiffuse!, this flag prints the final results after running cache optimizations such as minCut (see Recompute vs Cache Heuristics from this paper and slides 31-33 from this presentation) for a description of the caching algorithm. This may be helpful for debugging caching, phi node deletion, performance, and other errors. Off by default
source
Enzyme.API.strictAliasing! — Method
strictAliasing!(val::Bool)
Whether Enzyme's type analysis will assume strict aliasing semantics. When strict aliasing semantics are on (the default), Enzyme can propagate type information up through conditional branches. This may lead to illegal type errors when analyzing code with unions. Disabling strict aliasing will enable these union types to be correctly analyzed. However, it may lead to some errors that sufficient type information cannot be deduced. One can turn these insufficient type information errors into to warnings by calling looseTypeAnalysis!(true) which tells Enzyme to use its best guess in such scenarios.
source
Enzyme.API.strong_zero! — Method
strong_zero!(val::Bool)
Whether to enforce multiplication by zero as enforcing a zero result even if multiplying against a NaN or infinity. Necessary for some programs in which a value has a zero derivative since it is unused, even if it has an otherwise infinite or nan derivative.
source
Enzyme.API.typeWarning! — Method
typeWarning!(val::Bool)
Whether to print a warning when Type Analysis learns informatoin about a value's type which cannot be represented in the current size of the lattice. See maxtypeoffset! for more information. Off by default.
source

+end
source
Enzyme.API.inlineall! — Method
inlineall!(val::Bool)

Whether to inline all (non-recursive) functions generated by Julia within a single compilation unit. This may improve Enzyme's ability to successfully differentiate code and improve performance of the original and generated derivative program. It often, however, comes with an increase in compile time. This is off by default.

source
Enzyme.API.instname! — Method
instname!(val::Bool)

Whether to add a name to all LLVM values. This may be helpful for debugging generated programs, both primal and derivative. Off by default.

source
Enzyme.API.looseTypeAnalysis! — Method
looseTypeAnalysis!(val::Bool)

Enzyme runs a type analysis to deduce the corresponding types of all values being differentiated. This is necessary to compute correct derivatives of various values. For example, a copy of Float32's requires a different derivative than a memcpy of Float64's, Ptr's, etc. In some cases Enzyme may not be able to deduce all the types necessary and throw an unknown type error. If this is the case, open an issue. One can silence these issues by setting looseTypeAnalysis!(true) which tells Enzyme to make its best guess. This will remove the error and allow differentiation to continue, however, it may produce incorrect results. Alternatively one can consider increasing the space of the evaluated type lattice which gives Enzyme more time to run a more thorough analysis through the use of maxtypeoffset!

source
Enzyme.API.maxtypedepth! — Method
maxtypedepth!(val::Int)

Enzyme runs a type analysis to deduce the corresponding types of all values being differentiated. This is necessary to compute correct derivatives of various values. To ensure this analysis temrinates, it operates on a finite lattice of possible states. This function sets the maximum depth into a type that Enzyme will consider. A smaller value will cause type analysis to run faster, but may result in some necessary types not being found and result in unknown type errors. A larger value may result in unknown type errors being resolved by searching a larger space, but may run longer. The default setting is 6.

source
Enzyme.API.maxtypeoffset! — Method
maxtypeoffset!(val::Int)

Enzyme runs a type analysis to deduce the corresponding types of all values being differentiated. This is necessary to compute correct derivatives of various values. To ensure this analysis temrinates, it operates on a finite lattice of possible states. This function sets the maximum offset into a type that Enzyme will consider. A smaller value will cause type analysis to run faster, but may result in some necessary types not being found and result in unknown type errors. A larger value may result in unknown type errors being resolved by searching a larger space, but may run longer. The default setting is 512.

source
Enzyme.API.printactivity! — Method
printactivity!(val::Bool)

An debugging option for developers of Enzyme. If one sets this flag prior to the first differentiation of a function, Enzyme will print (to stderr) a log of all decisions made during Activity Analysis (the analysis which determines what values/instructions are differentiated). This may be useful for debugging MixedActivity errors, correctness, and performance errors. Off by default

source
Enzyme.API.printall! — Method
printall!(val::Bool)

An debugging option for developers of Enzyme. If one sets this flag prior to the first differentiation of a function, Enzyme will print (to stderr) the LLVM function being differentiated, as well as all generated derivatives immediately after running Enzyme (but prior to any other optimizations). Off by default

source
Enzyme.API.printdiffuse! — Method
printdiffuse!(val::Bool)

An debugging option for developers of Enzyme. If one sets this flag prior to the first differentiation of a function, Enzyme will print (to stderr) information about each LLVM value – specifically whether it and its shadow is required for computing the derivative. In contrast to printunnecessary!, this flag prints debug log for the analysis which determines for each value and shadow value, whether it can find a user which would require it to be kept around (rather than being deleted). This is prior to any cache optimizations and a debug log of Differential Use Analysis. This may be helpful for debugging caching, phi node deletion, performance, and other errors. Off by default

source
Enzyme.API.printperf! — Method
printperf!(val::Bool)

An debugging option for developers of Enzyme. If one sets this flag prior to the first differentiation of a function, Enzyme will print (to stderr) performance information about generated derivative programs. It will provide debug information that warns why particular values are cached for the reverse pass, and thus require additional computation/storage. This is particularly helpful for debugging derivatives which OOM or otherwise run slow. ff by default

source
Enzyme.API.printtype! — Method
printtype!(val::Bool)

An debugging option for developers of Enzyme. If one sets this flag prior to the first differentiation of a function, Enzyme will print (to stderr) a log of all decisions made during Type Analysis (the analysis which Enzyme determines the type of all values in the program). This may be useful for debugging correctness errors, illegal type analysis errors, insufficient type information errors, correctness, and performance errors. Off by default

source
Enzyme.API.printunnecessary! — Method
printunnecessary!(val::Bool)

An debugging option for developers of Enzyme. If one sets this flag prior to the first differentiation of a function, Enzyme will print (to stderr) information about each LLVM value – specifically whether it and its shadow is required for computing the derivative. In contrast to printdiffuse!, this flag prints the final results after running cache optimizations such as minCut (see Recompute vs Cache Heuristics from this paper and slides 31-33 from this presentation) for a description of the caching algorithm. This may be helpful for debugging caching, phi node deletion, performance, and other errors. Off by default

source
Enzyme.API.strictAliasing! — Method
strictAliasing!(val::Bool)

Whether Enzyme's type analysis will assume strict aliasing semantics. When strict aliasing semantics are on (the default), Enzyme can propagate type information up through conditional branches. This may lead to illegal type errors when analyzing code with unions. Disabling strict aliasing will enable these union types to be correctly analyzed. However, it may lead to some errors that sufficient type information cannot be deduced. One can turn these insufficient type information errors into to warnings by calling looseTypeAnalysis!(true) which tells Enzyme to use its best guess in such scenarios.

source
Enzyme.API.strong_zero! — Method
strong_zero!(val::Bool)

Whether to enforce multiplication by zero as enforcing a zero result even if multiplying against a NaN or infinity. Necessary for some programs in which a value has a zero derivative since it is unused, even if it has an otherwise infinite or nan derivative.

source
Enzyme.API.typeWarning! — Method
typeWarning!(val::Bool)

Whether to print a warning when Type Analysis learns informatoin about a value's type which cannot be represented in the current size of the lattice. See maxtypeoffset! for more information. Off by default.

source
diff --git a/dev/dev_docs/index.html b/dev/dev_docs/index.html index 197168f64e..ce93fcf4d7 100644 --- a/dev/dev_docs/index.html +++ b/dev/dev_docs/index.html @@ -14,4 +14,4 @@ julia -e "using Pkg; pkg\"add LLVM_full_jll@${LLVM_MAJOR_VER}\"" LLVM_DIR=`julia -e "using LLVM_full_jll; print(LLVM_full_jll.artifact_dir)"` echo "LLVM_DIR=$LLVM_DIR" -cmake ../enzyme/ -G Ninja -DENZYME_EXTERNAL_SHARED_LIB=ON -DLLVM_DIR=${LLVM_DIR} -DLLVM_EXTERNAL_LIT=${LLVM_DIR}/tools/lit/lit.py

Manual build of Julia

cmake ../enzyme/ -G Ninja -DENZYME_EXTERNAL_SHARED_LIB=ON -DLLVM_DIR=${PATH_TO_BUILDDIR_OF_JULIA}/usr/lib/cmake/llvm/
+cmake ../enzyme/ -G Ninja -DENZYME_EXTERNAL_SHARED_LIB=ON -DLLVM_DIR=${LLVM_DIR} -DLLVM_EXTERNAL_LIT=${LLVM_DIR}/tools/lit/lit.py

Manual build of Julia

cmake ../enzyme/ -G Ninja -DENZYME_EXTERNAL_SHARED_LIB=ON -DLLVM_DIR=${PATH_TO_BUILDDIR_OF_JULIA}/usr/lib/cmake/llvm/
diff --git a/dev/faq/index.html b/dev/faq/index.html index 5ea6cffd9a..f884a67e54 100644 --- a/dev/faq/index.html +++ b/dev/faq/index.html @@ -279,4 +279,4 @@ # output -ERROR: Type of ghost or constant type Duplicated{Val{1.0}} is marked as differentiable. +ERROR: Type of ghost or constant type Duplicated{Val{1.0}} is marked as differentiable. diff --git a/dev/generated/autodiff/index.html b/dev/generated/autodiff/index.html index ab7c56caac..f1cdec5e0b 100644 --- a/dev/generated/autodiff/index.html +++ b/dev/generated/autodiff/index.html @@ -66,4 +66,4 @@ hess[1][2] == 1.0
true

as well as the second row/column

hess[2][1] == 1.0
 
-hess[2][2] == 0.0
true

This page was generated using Literate.jl.

+hess[2][2] == 0.0
true

This page was generated using Literate.jl.

diff --git a/dev/generated/box/index.html b/dev/generated/box/index.html index b28fb92997..969b4a5620 100644 --- a/dev/generated/box/index.html +++ b/dev/generated/box/index.html @@ -362,4 +362,4 @@ 9.732210923954578e-5 0.0026401658789532625 0.015152571729925521 - 0.03212933056407103

and we get down to a percent difference on the order of $1e^{-5}$, showing Enzyme calculated the correct derivative. Success!

This page was generated using Literate.jl.

+ 0.03212933056407103

and we get down to a percent difference on the order of $1e^{-5}$, showing Enzyme calculated the correct derivative. Success!

This page was generated using Literate.jl.

diff --git a/dev/generated/custom_rule/index.html b/dev/generated/custom_rule/index.html index 0773770ee9..bb4494e713 100644 --- a/dev/generated/custom_rule/index.html +++ b/dev/generated/custom_rule/index.html @@ -167,4 +167,4 @@ test_reverse(fun, RT, (x, Tx), (y, Ty)) end end -end
Test.DefaultTestSet("f rules", Any[Test.DefaultTestSet("forward", Any[Test.DefaultTestSet("RT = Const, Tx = Const, Ty = Const", Any[Test.DefaultTestSet("test_forward: g with return activity Const on (::Vector{Float64}, Const), (::Vector{Float64}, Const)", Any[], 6, false, false, true, 1.727469364036102e9, 1.727469365817183e9, false, "/home/runner/work/Enzyme.jl/Enzyme.jl/lib/EnzymeTestUtils/src/test_forward.jl")], 0, false, false, true, 1.727469361374895e9, 1.727469365817191e9, false, "custom_rule.md"), Test.DefaultTestSet("RT = Const, Tx = Const, Ty = Duplicated", Any[Test.DefaultTestSet("test_forward: g with return activity Const on (::Vector{Float64}, Const), (::Vector{Float64}, Duplicated)", Any[], 6, false, false, true, 1.727469365818325e9, 1.727469366629072e9, false, "/home/runner/work/Enzyme.jl/Enzyme.jl/lib/EnzymeTestUtils/src/test_forward.jl")], 0, false, false, true, 1.727469365817233e9, 1.727469366629076e9, false, "custom_rule.md"), Test.DefaultTestSet("RT = Const, Tx = Duplicated, Ty = Const", Any[Test.DefaultTestSet("test_forward: g with return activity Const on (::Vector{Float64}, Duplicated), (::Vector{Float64}, Const)", Any[], 6, false, false, true, 1.727469366630157e9, 1.727469368146757e9, false, "/home/runner/work/Enzyme.jl/Enzyme.jl/lib/EnzymeTestUtils/src/test_forward.jl")], 0, false, false, true, 1.727469366629113e9, 1.727469368146761e9, false, "custom_rule.md"), Test.DefaultTestSet("RT = Const, Tx = Duplicated, Ty = Duplicated", Any[Test.DefaultTestSet("test_forward: g with return activity Const on (::Vector{Float64}, Duplicated), (::Vector{Float64}, Duplicated)", Any[], 6, false, false, true, 1.727469368148245e9, 1.727469369610356e9, false, "/home/runner/work/Enzyme.jl/Enzyme.jl/lib/EnzymeTestUtils/src/test_forward.jl")], 0, false, false, true, 1.727469368146806e9, 1.727469369610362e9, false, "custom_rule.md"), Test.DefaultTestSet("RT = DuplicatedNoNeed, Tx = Const, Ty = Const", Any[Test.DefaultTestSet("test_forward: g with return activity DuplicatedNoNeed on (::Vector{Float64}, Const), (::Vector{Float64}, Const)", Any[], 6, false, false, true, 1.727469369611512e9, 1.727469371100409e9, false, "/home/runner/work/Enzyme.jl/Enzyme.jl/lib/EnzymeTestUtils/src/test_forward.jl")], 0, false, false, true, 1.727469369610418e9, 1.727469371100414e9, false, "custom_rule.md"), Test.DefaultTestSet("RT = DuplicatedNoNeed, Tx = Const, Ty = Duplicated", Any[Test.DefaultTestSet("test_forward: g with return activity DuplicatedNoNeed on (::Vector{Float64}, Const), (::Vector{Float64}, Duplicated)", Any[], 6, false, false, true, 1.727469371101557e9, 1.727469373404117e9, false, "/home/runner/work/Enzyme.jl/Enzyme.jl/lib/EnzymeTestUtils/src/test_forward.jl")], 0, false, false, true, 1.72746937110047e9, 1.727469373404121e9, false, "custom_rule.md"), Test.DefaultTestSet("RT = DuplicatedNoNeed, Tx = Duplicated, Ty = Const", Any[Test.DefaultTestSet("test_forward: g with return activity DuplicatedNoNeed on (::Vector{Float64}, Duplicated), (::Vector{Float64}, Const)", Any[], 6, false, false, true, 1.727469373405228e9, 1.727469374924478e9, false, "/home/runner/work/Enzyme.jl/Enzyme.jl/lib/EnzymeTestUtils/src/test_forward.jl")], 0, false, false, true, 1.727469373404151e9, 1.727469374924482e9, false, "custom_rule.md"), Test.DefaultTestSet("RT = DuplicatedNoNeed, Tx = Duplicated, Ty = Duplicated", Any[Test.DefaultTestSet("test_forward: g with return activity DuplicatedNoNeed on (::Vector{Float64}, Duplicated), (::Vector{Float64}, Duplicated)", Any[], 6, false, false, true, 1.727469374925571e9, 1.727469376726121e9, false, "/home/runner/work/Enzyme.jl/Enzyme.jl/lib/EnzymeTestUtils/src/test_forward.jl")], 0, false, false, true, 1.727469374924511e9, 1.727469376726124e9, false, "custom_rule.md"), Test.DefaultTestSet("RT = Duplicated, Tx = Const, Ty = Const", Any[Test.DefaultTestSet("test_forward: g with return activity Duplicated on (::Vector{Float64}, Const), (::Vector{Float64}, Const)", Any[], 7, false, false, true, 1.727469376727217e9, 1.727469377829484e9, false, "/home/runner/work/Enzyme.jl/Enzyme.jl/lib/EnzymeTestUtils/src/test_forward.jl")], 0, false, false, true, 1.727469376726155e9, 1.727469377829488e9, false, "custom_rule.md"), Test.DefaultTestSet("RT = Duplicated, Tx = Const, Ty = Duplicated", Any[Test.DefaultTestSet("test_forward: g with return activity Duplicated on (::Vector{Float64}, Const), (::Vector{Float64}, Duplicated)", Any[], 7, false, false, true, 1.7274693778306e9, 1.72746937903096e9, false, "/home/runner/work/Enzyme.jl/Enzyme.jl/lib/EnzymeTestUtils/src/test_forward.jl")], 0, false, false, true, 1.727469377829517e9, 1.727469379030964e9, false, "custom_rule.md"), Test.DefaultTestSet("RT = Duplicated, Tx = Duplicated, Ty = Const", Any[Test.DefaultTestSet("test_forward: g with return activity Duplicated on (::Vector{Float64}, Duplicated), (::Vector{Float64}, Const)", Any[], 7, false, false, true, 1.727469379032051e9, 1.727469380251639e9, false, "/home/runner/work/Enzyme.jl/Enzyme.jl/lib/EnzymeTestUtils/src/test_forward.jl")], 0, false, false, true, 1.727469379030993e9, 1.727469380251643e9, false, "custom_rule.md"), Test.DefaultTestSet("RT = Duplicated, Tx = Duplicated, Ty = Duplicated", Any[Test.DefaultTestSet("test_forward: g with return activity Duplicated on (::Vector{Float64}, Duplicated), (::Vector{Float64}, Duplicated)", Any[], 7, false, false, true, 1.72746938025277e9, 1.727469381562051e9, false, "/home/runner/work/Enzyme.jl/Enzyme.jl/lib/EnzymeTestUtils/src/test_forward.jl")], 0, false, false, true, 1.727469380251669e9, 1.727469381562054e9, false, "custom_rule.md")], 0, false, false, true, 1.727469361374843e9, 1.727469381562055e9, false, "custom_rule.md"), Test.DefaultTestSet("reverse", Any[Test.DefaultTestSet("RT = Active, Tx = Duplicated, Ty = Duplicated, fun = g", Any[Test.DefaultTestSet("test_reverse: g with return activity Active on (::Vector{Float64}, Duplicated), (::Vector{Float64}, Duplicated)", Any[], 11, false, false, true, 1.727469381690329e9, 1.727469384447065e9, false, "/home/runner/work/Enzyme.jl/Enzyme.jl/lib/EnzymeTestUtils/src/test_reverse.jl")], 0, false, false, true, 1.727469381562112e9, 1.727469384447072e9, false, "custom_rule.md"), Test.DefaultTestSet("RT = Active, Tx = Duplicated, Ty = Duplicated, fun = h", Any[Test.DefaultTestSet("test_reverse: h with return activity Active on (::Vector{Float64}, Duplicated), (::Vector{Float64}, Duplicated)", Any[], 11, false, false, true, 1.727469384578993e9, 1.727469388179576e9, false, "/home/runner/work/Enzyme.jl/Enzyme.jl/lib/EnzymeTestUtils/src/test_reverse.jl")], 0, false, false, true, 1.727469384447112e9, 1.72746938817958e9, false, "custom_rule.md")], 0, false, false, true, 1.727469381562074e9, 1.727469388179581e9, false, "custom_rule.md")], 0, false, false, true, 1.727469361374792e9, 1.727469388179582e9, false, "custom_rule.md")

In any package that implements Enzyme rules using EnzymeRules, it is recommended to add EnzymeTestUtils as a test dependency to test the rules.

This page was generated using Literate.jl.

+end
Test.DefaultTestSet("f rules", Any[Test.DefaultTestSet("forward", Any[Test.DefaultTestSet("RT = Const, Tx = Const, Ty = Const", Any[Test.DefaultTestSet("test_forward: g with return activity Const on (::Vector{Float64}, Const), (::Vector{Float64}, Const)", Any[], 6, false, false, true, 1.72747405286606e9, 1.727474054592771e9, false, "/home/runner/work/Enzyme.jl/Enzyme.jl/lib/EnzymeTestUtils/src/test_forward.jl")], 0, false, false, true, 1.727474050249541e9, 1.727474054592778e9, false, "custom_rule.md"), Test.DefaultTestSet("RT = Const, Tx = Const, Ty = Duplicated", Any[Test.DefaultTestSet("test_forward: g with return activity Const on (::Vector{Float64}, Const), (::Vector{Float64}, Duplicated)", Any[], 6, false, false, true, 1.727474054593999e9, 1.727474055403816e9, false, "/home/runner/work/Enzyme.jl/Enzyme.jl/lib/EnzymeTestUtils/src/test_forward.jl")], 0, false, false, true, 1.727474054592826e9, 1.727474055403823e9, false, "custom_rule.md"), Test.DefaultTestSet("RT = Const, Tx = Duplicated, Ty = Const", Any[Test.DefaultTestSet("test_forward: g with return activity Const on (::Vector{Float64}, Duplicated), (::Vector{Float64}, Const)", Any[], 6, false, false, true, 1.727474055404971e9, 1.727474056721225e9, false, "/home/runner/work/Enzyme.jl/Enzyme.jl/lib/EnzymeTestUtils/src/test_forward.jl")], 0, false, false, true, 1.727474055403862e9, 1.72747405672123e9, false, "custom_rule.md"), Test.DefaultTestSet("RT = Const, Tx = Duplicated, Ty = Duplicated", Any[Test.DefaultTestSet("test_forward: g with return activity Const on (::Vector{Float64}, Duplicated), (::Vector{Float64}, Duplicated)", Any[], 6, false, false, true, 1.727474056722346e9, 1.727474058140066e9, false, "/home/runner/work/Enzyme.jl/Enzyme.jl/lib/EnzymeTestUtils/src/test_forward.jl")], 0, false, false, true, 1.727474056721265e9, 1.727474058140071e9, false, "custom_rule.md"), Test.DefaultTestSet("RT = DuplicatedNoNeed, Tx = Const, Ty = Const", Any[Test.DefaultTestSet("test_forward: g with return activity DuplicatedNoNeed on (::Vector{Float64}, Const), (::Vector{Float64}, Const)", Any[], 6, false, false, true, 1.72747405814122e9, 1.727474059594773e9, false, "/home/runner/work/Enzyme.jl/Enzyme.jl/lib/EnzymeTestUtils/src/test_forward.jl")], 0, false, false, true, 1.727474058140114e9, 1.727474059594777e9, false, "custom_rule.md"), Test.DefaultTestSet("RT = DuplicatedNoNeed, Tx = Const, Ty = Duplicated", Any[Test.DefaultTestSet("test_forward: g with return activity DuplicatedNoNeed on (::Vector{Float64}, Const), (::Vector{Float64}, Duplicated)", Any[], 6, false, false, true, 1.727474059595914e9, 1.727474061894701e9, false, "/home/runner/work/Enzyme.jl/Enzyme.jl/lib/EnzymeTestUtils/src/test_forward.jl")], 0, false, false, true, 1.727474059594817e9, 1.727474061894707e9, false, "custom_rule.md"), Test.DefaultTestSet("RT = DuplicatedNoNeed, Tx = Duplicated, Ty = Const", Any[Test.DefaultTestSet("test_forward: g with return activity DuplicatedNoNeed on (::Vector{Float64}, Duplicated), (::Vector{Float64}, Const)", Any[], 6, false, false, true, 1.727474061895823e9, 1.727474063388433e9, false, "/home/runner/work/Enzyme.jl/Enzyme.jl/lib/EnzymeTestUtils/src/test_forward.jl")], 0, false, false, true, 1.727474061894735e9, 1.727474063388437e9, false, "custom_rule.md"), Test.DefaultTestSet("RT = DuplicatedNoNeed, Tx = Duplicated, Ty = Duplicated", Any[Test.DefaultTestSet("test_forward: g with return activity DuplicatedNoNeed on (::Vector{Float64}, Duplicated), (::Vector{Float64}, Duplicated)", Any[], 6, false, false, true, 1.727474063389513e9, 1.727474065126013e9, false, "/home/runner/work/Enzyme.jl/Enzyme.jl/lib/EnzymeTestUtils/src/test_forward.jl")], 0, false, false, true, 1.727474063388463e9, 1.727474065126017e9, false, "custom_rule.md"), Test.DefaultTestSet("RT = Duplicated, Tx = Const, Ty = Const", Any[Test.DefaultTestSet("test_forward: g with return activity Duplicated on (::Vector{Float64}, Const), (::Vector{Float64}, Const)", Any[], 7, false, false, true, 1.727474065127114e9, 1.727474066206357e9, false, "/home/runner/work/Enzyme.jl/Enzyme.jl/lib/EnzymeTestUtils/src/test_forward.jl")], 0, false, false, true, 1.727474065126044e9, 1.727474066206361e9, false, "custom_rule.md"), Test.DefaultTestSet("RT = Duplicated, Tx = Const, Ty = Duplicated", Any[Test.DefaultTestSet("test_forward: g with return activity Duplicated on (::Vector{Float64}, Const), (::Vector{Float64}, Duplicated)", Any[], 7, false, false, true, 1.727474066207524e9, 1.72747406737145e9, false, "/home/runner/work/Enzyme.jl/Enzyme.jl/lib/EnzymeTestUtils/src/test_forward.jl")], 0, false, false, true, 1.727474066206419e9, 1.727474067371454e9, false, "custom_rule.md"), Test.DefaultTestSet("RT = Duplicated, Tx = Duplicated, Ty = Const", Any[Test.DefaultTestSet("test_forward: g with return activity Duplicated on (::Vector{Float64}, Duplicated), (::Vector{Float64}, Const)", Any[], 7, false, false, true, 1.727474067372608e9, 1.727474068574249e9, false, "/home/runner/work/Enzyme.jl/Enzyme.jl/lib/EnzymeTestUtils/src/test_forward.jl")], 0, false, false, true, 1.727474067371481e9, 1.727474068574252e9, false, "custom_rule.md"), Test.DefaultTestSet("RT = Duplicated, Tx = Duplicated, Ty = Duplicated", Any[Test.DefaultTestSet("test_forward: g with return activity Duplicated on (::Vector{Float64}, Duplicated), (::Vector{Float64}, Duplicated)", Any[], 7, false, false, true, 1.727474068575415e9, 1.727474069851383e9, false, "/home/runner/work/Enzyme.jl/Enzyme.jl/lib/EnzymeTestUtils/src/test_forward.jl")], 0, false, false, true, 1.727474068574276e9, 1.727474069851386e9, false, "custom_rule.md")], 0, false, false, true, 1.727474050249494e9, 1.727474069851387e9, false, "custom_rule.md"), Test.DefaultTestSet("reverse", Any[Test.DefaultTestSet("RT = Active, Tx = Duplicated, Ty = Duplicated, fun = g", Any[Test.DefaultTestSet("test_reverse: g with return activity Active on (::Vector{Float64}, Duplicated), (::Vector{Float64}, Duplicated)", Any[], 11, false, false, true, 1.727474069978408e9, 1.727474072681456e9, false, "/home/runner/work/Enzyme.jl/Enzyme.jl/lib/EnzymeTestUtils/src/test_reverse.jl")], 0, false, false, true, 1.727474069851461e9, 1.727474072681462e9, false, "custom_rule.md"), Test.DefaultTestSet("RT = Active, Tx = Duplicated, Ty = Duplicated, fun = h", Any[Test.DefaultTestSet("test_reverse: h with return activity Active on (::Vector{Float64}, Duplicated), (::Vector{Float64}, Duplicated)", Any[], 11, false, false, true, 1.727474072807259e9, 1.72747407631185e9, false, "/home/runner/work/Enzyme.jl/Enzyme.jl/lib/EnzymeTestUtils/src/test_reverse.jl")], 0, false, false, true, 1.727474072681498e9, 1.727474076311853e9, false, "custom_rule.md")], 0, false, false, true, 1.727474069851423e9, 1.727474076311854e9, false, "custom_rule.md")], 0, false, false, true, 1.727474050249443e9, 1.727474076311855e9, false, "custom_rule.md")

In any package that implements Enzyme rules using EnzymeRules, it is recommended to add EnzymeTestUtils as a test dependency to test the rules.

This page was generated using Literate.jl.

diff --git a/dev/index.html b/dev/index.html index 31e28e20e2..d219148153 100644 --- a/dev/index.html +++ b/dev/index.html @@ -134,4 +134,4 @@ julia> grad 2-element Vector{Float64}: 2.880510859951098 - 1.920340573300732 + 1.920340573300732 diff --git a/dev/internal_api/index.html b/dev/internal_api/index.html index 7ca00cd482..fca5e822ed 100644 --- a/dev/internal_api/index.html +++ b/dev/internal_api/index.html @@ -1,2 +1,2 @@ -Internal API · Enzyme.jl

Internal API

Note

This is the documentation of Enzymes's internal API. The internal API is not subject to semantic versioning and may change at any time and without deprecation.

+Internal API · Enzyme.jl

Internal API

Note

This is the documentation of Enzymes's internal API. The internal API is not subject to semantic versioning and may change at any time and without deprecation.

diff --git a/dev/search_index.js b/dev/search_index.js index b63d08d824..666ac9a431 100644 --- a/dev/search_index.js +++ b/dev/search_index.js @@ -1,3 +1,3 @@ var documenterSearchIndex = {"docs": -[{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"EditURL = \"../../../examples/autodiff.jl\"","category":"page"},{"location":"generated/autodiff/#Basics","page":"Basics","title":"Basics","text":"","category":"section"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"The goal of this tutorial is to give users already familiar with automatic differentiation (AD) an overview of the Enzyme differentiation API for the following differentiation modes","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"Reverse mode\nForward mode\nForward over reverse mode\nVector Forward over reverse mode","category":"page"},{"location":"generated/autodiff/#Defining-a-function","page":"Basics","title":"Defining a function","text":"","category":"section"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"Enzyme differentiates arbitrary multivariate vector functions as the most general case in automatic differentiation","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"f mathbbR^n rightarrow mathbbR^m y = f(x)","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"For simplicity we define a vector function with m=1. However, this tutorial can easily be applied to arbitrary m in mathbbN.","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"using Enzyme\n\nfunction f(x::Array{Float64}, y::Array{Float64})\n y[1] = x[1] * x[1] + x[2] * x[1]\n return nothing\nend;\nnothing #hide","category":"page"},{"location":"generated/autodiff/#Reverse-mode","page":"Basics","title":"Reverse mode","text":"","category":"section"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"The reverse model in AD is defined as","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"beginaligned\ny = f(x) \nbarx = bary cdot nabla f(x)\nendaligned","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"bar denotes an adjoint variable. Note that executing an AD in reverse mode computes both y and the adjoint barx.","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"x = [2.0, 2.0]\nbx = [0.0, 0.0]\ny = [0.0]\nby = [1.0];\nnothing #hide","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"Enzyme stores the value and adjoint of a variable in an object of type Duplicated where the first element represent the value and the second the adjoint. Evaluating the reverse model using Enzyme is done via the following call.","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"Enzyme.autodiff(Reverse, f, Duplicated(x, bx), Duplicated(y, by));\nnothing #hide","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"This yields the gradient of f in bx at point x = [2.0, 2.0]. by is called the seed and has to be set to 10 in order to compute the gradient. Let's save the gradient for later.","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"g = copy(bx)","category":"page"},{"location":"generated/autodiff/#Forward-mode","page":"Basics","title":"Forward mode","text":"","category":"section"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"The forward model in AD is defined as","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"beginaligned\ny = f(x) \ndoty = nabla f(x) cdot dotx\nendaligned","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"To obtain the first element of the gradient using the forward model we have to seed dotx with dotx = 1000","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"x = [2.0, 2.0]\ndx = [1.0, 0.0]\ny = [0.0]\ndy = [0.0];\nnothing #hide","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"In the forward mode the second element of Duplicated stores the tangent.","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"Enzyme.autodiff(Forward, f, Duplicated(x, dx), Duplicated(y, dy));\nnothing #hide","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"We can now verify that indeed the reverse mode and forward mode yield the same result for the first component of the gradient. Note that to acquire the full gradient one needs to execute the forward model a second time with the seed dx set to [0.0,1.0].","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"Let's verify whether the reverse and forward model agree.","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"g[1] == dy[1]","category":"page"},{"location":"generated/autodiff/#Forward-over-reverse","page":"Basics","title":"Forward over reverse","text":"","category":"section"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"The forward over reverse (FoR) model is obtained by applying the forward model to the reverse model using the chain rule for the product in the adjoint statement.","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"beginaligned\ny = f(x) \ndoty = nabla f(x) cdot dotx \nbarx = bary cdot nabla f(x) \ndotbarx = bary cdot nabla^2 f(x) cdot dotx + dotbary cdot nabla f(x)\nendaligned","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"To obtain the first column/row of the Hessian nabla^2 f(x) we have to seed dotbary with 00, bary with 10 and dotx with 10 00.","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"y = [0.0]\nx = [2.0, 2.0]\n\ndy = [0.0]\ndx = [1.0, 0.0]\n\nbx = [0.0, 0.0]\nby = [1.0]\ndbx = [0.0, 0.0]\ndby = [0.0]\n\nEnzyme.autodiff(\n Forward,\n (x,y) -> Enzyme.autodiff(Reverse, f, x, y),\n Duplicated(Duplicated(x, bx), Duplicated(dx, dbx)),\n Duplicated(Duplicated(y, by), Duplicated(dy, dby)),\n)","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"The FoR model also computes the forward model from before, giving us again the first component of the gradient.","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"g[1] == dy[1]","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"In addition we now have the first row/column of the Hessian.","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"dbx[1] == 2.0\ndbx[2] == 1.0","category":"page"},{"location":"generated/autodiff/#Vector-forward-over-reverse","page":"Basics","title":"Vector forward over reverse","text":"","category":"section"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"The vector FoR allows us to propagate several tangents at once through the second-order model by computing the derivative of the gradient at multiple points at once. We begin by defining a helper function for the gradient. Since we will not need the original results (stored in y), we can mark it DuplicatedNoNeed. Specifically, this will perform the following:","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"beginaligned\nbarx = barx + bary cdot nabla f(x) \nbary = 0\nendaligned","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"function grad(x, dx, y, dy)\n Enzyme.autodiff(Reverse, f, Duplicated(x, dx), DuplicatedNoNeed(y, dy))\n nothing\nend","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"To compute the conventional gradient, we would call this function with our given inputs, dy = [1.0], and dx = [0.0, 0.0]. Since y is not needed, we can just set it to an undef vector.","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"x = [2.0, 2.0]\ny = Vector{Float64}(undef, 1)\ndx = [0.0, 0.0]\ndy = [1.0]\n\ngrad(x, dx, y, dy)","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"dx now contains the gradient","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"@show dx","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"To compute the hessian, we need to take the dervative of this gradient function at every input. Following the same seeding strategy as before, we now seed both in the vx[1]=[1.0, 0.0] and vx[2]=[0.0, 1.0] direction. These tuples have to be put into a BatchDuplicated type. We then compute the forward mode derivative at all these points.","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"vx = ([1.0, 0.0], [0.0, 1.0])\nhess = ([0.0, 0.0], [0.0, 0.0])\ndx = [0.0, 0.0]\ndy = [1.0]\n\nEnzyme.autodiff(Enzyme.Forward, grad,\n Enzyme.BatchDuplicated(x, vx),\n Enzyme.BatchDuplicated(dx, hess),\n Const(y),\n Const(dy))","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"Again we obtain the first-order gradient. If we did not want to compute the gradient again, we could instead have used Enzyme.BatchDuplicatedNoNeed(dx, hess)","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"g[1] == dx[1]","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"We have now the first row/column of the Hessian","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"hess[1][1] == 2.0\n\nhess[1][2] == 1.0","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"as well as the second row/column","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"hess[2][1] == 1.0\n\nhess[2][2] == 0.0","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"This page was generated using Literate.jl.","category":"page"},{"location":"api/#API-reference","page":"API reference","title":"API reference","text":"","category":"section"},{"location":"api/#Types-and-constants","page":"API reference","title":"Types and constants","text":"","category":"section"},{"location":"api/","page":"API reference","title":"API reference","text":"Modules = [Enzyme, EnzymeCore, EnzymeCore.EnzymeRules, EnzymeTestUtils, Enzyme.API]\nOrder = [:type, :constant]","category":"page"},{"location":"api/#Functions-and-macros","page":"API reference","title":"Functions and macros","text":"","category":"section"},{"location":"api/","page":"API reference","title":"API reference","text":"Modules = [Enzyme, EnzymeCore, EnzymeCore.EnzymeRules, EnzymeTestUtils, Enzyme.API]\nOrder = [:macro, :function]","category":"page"},{"location":"api/#Documentation","page":"API reference","title":"Documentation","text":"","category":"section"},{"location":"api/","page":"API reference","title":"API reference","text":"Modules = [Enzyme, EnzymeCore, EnzymeCore.EnzymeRules, EnzymeTestUtils, Enzyme.API]\nOrder = [:module, :type, :constant, :macro, :function]","category":"page"},{"location":"api/#Enzyme.@import_frule-Tuple","page":"API reference","title":"Enzyme.@import_frule","text":"import_frule(::fn, tys...)\n\nAutomatically import a ChainRulesCore.fruleas a custom forward modeEnzymeRule. When called in batch mode, this will end up calling the primal multiple times, which may result in incorrect behavior if the function mutates, and slow code, always. Importing the rule fromChainRules` is also likely to be slower than writing your own rule, and may also be slower than not having a rule at all.\n\nUse with caution.\n\nEnzyme.@import_frule(typeof(Base.sort), Any);\n\nx=[1.0, 2.0, 0.0]; dx=[0.1, 0.2, 0.3]; ddx = [0.01, 0.02, 0.03];\n\nEnzyme.autodiff(Forward, sort, Duplicated, BatchDuplicated(x, (dx,ddx)))\nEnzyme.autodiff(Forward, sort, DuplicatedNoNeed, BatchDuplicated(x, (dx,ddx)))\nEnzyme.autodiff(Forward, sort, DuplicatedNoNeed, BatchDuplicated(x, (dx,)))\nEnzyme.autodiff(Forward, sort, Duplicated, BatchDuplicated(x, (dx,)))\n\n# output\n\n(var\"1\" = [0.0, 1.0, 2.0], var\"2\" = (var\"1\" = [0.3, 0.1, 0.2], var\"2\" = [0.03, 0.01, 0.02]))\n(var\"1\" = (var\"1\" = [0.3, 0.1, 0.2], var\"2\" = [0.03, 0.01, 0.02]),)\n(var\"1\" = [0.3, 0.1, 0.2],)\n(var\"1\" = [0.0, 1.0, 2.0], var\"2\" = [0.3, 0.1, 0.2])\n\n\n\n\n\n\n","category":"macro"},{"location":"api/#Enzyme.@import_rrule-Tuple","page":"API reference","title":"Enzyme.@import_rrule","text":"import_rrule(::fn, tys...)\n\nAutomatically import a ChainRules.rrule as a custom reverse mode EnzymeRule. When called in batch mode, this will end up calling the primal multiple times which results in slower code. This macro assumes that the underlying function to be imported is read-only, and returns a Duplicated or Const object. This macro also assumes that the inputs permit a .+= operation and that the output has a valid Enzyme.make_zero function defined. It also assumes that overwritten(x) accurately describes if there is any non-preserved data from forward to reverse, not just the outermost data structure being overwritten as provided by the specification.\n\nFinally, this macro falls back to almost always caching all of the inputs, even if it may not be needed for the derivative computation.\n\nAs a result, this auto importer is also likely to be slower than writing your own rule, and may also be slower than not having a rule at all.\n\nUse with caution.\n\nEnzyme.@import_rrule(typeof(Base.sort), Any);\n\n\n\n\n\n","category":"macro"},{"location":"api/#Enzyme.gradient!-Union{Tuple{ErrIfFuncWritten}, Tuple{Holomorphic}, Tuple{ABI}, Tuple{RuntimeActivity}, Tuple{ReturnPrimal}, Tuple{F}, Tuple{X}, Tuple{ReverseMode{ReturnPrimal, RuntimeActivity, ABI, Holomorphic, ErrIfFuncWritten}, X, F, X}} where {X<:Array, F, ReturnPrimal, RuntimeActivity, ABI, Holomorphic, ErrIfFuncWritten}","page":"API reference","title":"Enzyme.gradient!","text":"gradient!(::ReverseMode, dx, f, x)\n\nCompute the gradient of an array-input function f using reverse mode, storing the derivative result in an existing array dx. Both x and dx must be Arrays of the same type.\n\nExample:\n\nf(x) = x[1]*x[2]\n\ndx = [0.0, 0.0]\ngradient!(Reverse, dx, f, [2.0, 3.0])\n\n# output\n([3.0, 2.0],)\n\ndx = [0.0, 0.0]\ngradient!(ReverseWithPrimal, dx, f, [2.0, 3.0])\n\n# output\n(derivs = ([3.0, 2.0],), val = 6.0)\n\n\n\n\n\n","category":"method"},{"location":"api/#Enzyme.gradient-Union{Tuple{CS}, Tuple{RuntimeActivity}, Tuple{ErrIfFuncWritten}, Tuple{ABI}, Tuple{ReturnPrimal}, Tuple{ForwardMode{ReturnPrimal, ABI, ErrIfFuncWritten, RuntimeActivity}, Any, Any}} where {ReturnPrimal, ABI, ErrIfFuncWritten, RuntimeActivity, CS}","page":"API reference","title":"Enzyme.gradient","text":"gradient(::ForwardMode, f, x; shadows=onehot(x), chunk=nothing)\n\nCompute the gradient of an array-input function f using forward mode. The optional keyword argument shadow is a vector of one-hot vectors of type x which are used to forward-propagate into the return. For performance reasons, this should be computed once, outside the call to gradient, rather than within this call.\n\nExample:\n\nf(x) = x[1]*x[2]\n\ngradient(Forward, f, [2.0, 3.0])\n\n# output\n\n([3.0, 2.0],)\n\ngradient(ForwardWithPrimal, f, [2.0, 3.0])\n\n# output\n(derivs = ([3.0, 2.0],), val = 6.0)\n\ngradient(Forward, f, [2.0, 3.0]; chunk=Val(1))\n\n# output\n\n([3.0, 2.0],)\n\ngradient(ForwardWithPrimal, f, [2.0, 3.0]; chunk=Val(1))\n\n# output\n(derivs = ([3.0, 2.0],), val = 6.0)\n\nFor functions which return an AbstractArray or scalar, this function will return an AbstracttArray whose shape is (size(output)..., size(input)...). No guarantees are presently made about the type of the AbstractArray returned by this function (which may or may not be the same as the input AbstractArray if provided).\n\nFor functions who return other types, this function will retun an AbstractArray of shape size(input) of values of the output type. \n\nf(x) = [ x[1] * x[2], x[2] + x[3] ]\n\ngrad = gradient(Forward, f, [2.0, 3.0, 4.0])\n\n# output\n([3.0 2.0 0.0; 0.0 1.0 1.0],)\n\n\n\n\n\n","category":"method"},{"location":"api/#Enzyme.gradient-Union{Tuple{N}, Tuple{ErrIfFuncWritten}, Tuple{Holomorphic}, Tuple{ABI}, Tuple{RuntimeActivity}, Tuple{ReturnPrimal}, Tuple{ty_0}, Tuple{F}, Tuple{ReverseMode{ReturnPrimal, RuntimeActivity, ABI, Holomorphic, ErrIfFuncWritten}, F, ty_0, Vararg{Any, N}}} where {F, ty_0, ReturnPrimal, RuntimeActivity, ABI, Holomorphic, ErrIfFuncWritten, N}","page":"API reference","title":"Enzyme.gradient","text":"gradient(::ReverseMode, f, args...)\n\nCompute the gradient of a real-valued function f using reverse mode. For each differentiable argument, this function will allocate and return new derivative object, returning a tuple of derivatives for each argument. If an argument is not differentiable, the element of the returned tuple with be nothing.\n\nIn reverse mode (here), the derivatives will be the same type as the original argument.\n\nThis is a structure gradient. For a struct x it returns another instance of the same type, whose fields contain the components of the gradient. In the result, grad.a contains ∂f/∂x.a for any differential x.a, while grad.c == x.c for other types.\n\nExamples:\n\nf(x) = x[1]*x[2]\n\ngrad = gradient(Reverse, f, [2.0, 3.0])\n\n# output\n([3.0, 2.0],)\n\ngrad = gradient(Reverse, only ∘ f, (a = 2.0, b = [3.0], c = \"str\"))\n\n# output\n\n((a = 3.0, b = [2.0], c = \"str\"),)\n\nmul(x, y) = x[1]*y[1]\n\ngrad = gradient(Reverse, mul, [2.0], [3.0])\n\n# output\n([3.0], [2.0])\n\n\ngrad = gradient(Reverse, mul, [2.0], Const([3.0]))\n\n# output\n([3.0], nothing)\n\nIf passing a mode that returns the primal (e.g. ReverseWithPrimal), the return type will instead be a tuple where the first element contains the derivatives, and the second element contains the result of the original computation.\n\n\ngrad = gradient(ReverseWithPrimal, f, [2.0, 3.0])\n\n# output\n(derivs = ([3.0, 2.0],), val = 6.0)\n\n\ngrad = gradient(ReverseWithPrimal, mul, [2.0], [3.0])\n\n# output\n(derivs = ([3.0], [2.0]), val = 6.0)\n\ngrad = gradient(ReverseWithPrimal, mul, [2.0], Const([3.0]))\n\n# output\n(derivs = ([3.0], nothing), val = 6.0)\n\n\n\n\n\n","category":"method"},{"location":"api/#Enzyme.hvp!-Union{Tuple{X}, Tuple{F}, Tuple{X, F, X, X}} where {F, X}","page":"API reference","title":"Enzyme.hvp!","text":"hvp!(res::X, f::F, x::X, v::X) where {F, X}\n\nCompute an in-place Hessian-vector product of an array-input scalar-output function f, as evaluated at x times the vector v. The result will be stored into res. The function still allocates and zero's a buffer to store the intermediate gradient, which is not returned to the user.\n\nIn other words, compute res .= hessian(f)(x) * v\n\nSee hvp_and_gradient! for a function to compute both the hvp and the gradient in a single call.\n\nExample:\n\nf(x) = sin(x[1] * x[2])\n\nres = Vector{Float64}(undef, 2)\nhvp!(res, f, [2.0, 3.0], [5.0, 2.7])\n\nres\n# output\n2-element Vector{Float64}:\n 19.6926882637302\n 16.201003759768003\n\n\n\n\n\n","category":"method"},{"location":"api/#Enzyme.hvp-Union{Tuple{X}, Tuple{F}, Tuple{F, X, X}} where {F, X}","page":"API reference","title":"Enzyme.hvp","text":"hvp(f::F, x::X, v::X) where {F, X}\n\nCompute the Hessian-vector product of an array-input scalar-output function f, as evaluated at x times the vector v.\n\nIn other words, compute hessian(f)(x) * v\n\nSee hvp! for a version which stores the result in an existing buffer and also hvp_and_gradient! for a function to compute both the hvp and the gradient in a single call.\n\nExample:\n\nf(x) = sin(x[1] * x[2])\n\nhvp(f, [2.0, 3.0], [5.0, 2.7])\n\n# output\n2-element Vector{Float64}:\n 19.6926882637302\n 16.201003759768003\n\n\n\n\n\n","category":"method"},{"location":"api/#Enzyme.hvp_and_gradient!-Union{Tuple{X}, Tuple{F}, Tuple{X, X, F, X, X}} where {F, X}","page":"API reference","title":"Enzyme.hvp_and_gradient!","text":"hvp_and_gradient!(res::X, grad::X, f::F, x::X, v::X) where {F, X}\n\nCompute an in-place Hessian-vector product of an array-input scalar-output function f, as evaluated at x times the vector v as well as the gradient, storing the gradient into grad. Both the hessian vector product and the gradient can be computed together more efficiently than computing them separately.\n\nThe result will be stored into res. The gradient will be stored into grad.\n\nIn other words, compute res .= hessian(f)(x) * v and grad .= gradient(Reverse, f)(x)\n\nExample:\n\nf(x) = sin(x[1] * x[2])\n\nres = Vector{Float64}(undef, 2)\ngrad = Vector{Float64}(undef, 2)\nhvp_and_gradient!(res, grad, f, [2.0, 3.0], [5.0, 2.7])\n\nres\ngrad\n# output\n2-element Vector{Float64}:\n 2.880510859951098\n 1.920340573300732\n\n\n\n\n\n","category":"method"},{"location":"api/#Enzyme.jacobian-Tuple{ForwardMode, Vararg{Any}}","page":"API reference","title":"Enzyme.jacobian","text":"jacobian(::ForwardMode, args...; kwargs...)\n\nEquivalent to gradient(::ForwardMode, args...; kwargs...)\n\n\n\n\n\n","category":"method"},{"location":"api/#Enzyme.jacobian-Union{Tuple{Holomorphic}, Tuple{CT}, Tuple{OutType}, Tuple{RuntimeActivity}, Tuple{ErrIfFuncWritten}, Tuple{RABI}, Tuple{X}, Tuple{F}, Tuple{ReturnPrimal}, Tuple{ReverseMode{ReturnPrimal, RuntimeActivity, RABI, Holomorphic, ErrIfFuncWritten}, F, X}} where {ReturnPrimal, F, X, RABI<:EnzymeCore.ABI, ErrIfFuncWritten, RuntimeActivity, OutType, CT, Holomorphic}","page":"API reference","title":"Enzyme.jacobian","text":"jacobian(::ReverseMode, f, x; n_outs=nothing, chunk=nothing)\njacobian(::ReverseMode, f, x)\n\nCompute the jacobian of a array-output function f using (potentially vector) reverse mode. The chunk argument optionally denotes the chunk size to use and n_outs optionally denotes the shape of the array returned by f (e.g size(f(x))).\n\nExample:\n\nf(x) = [ x[1] * x[2], x[2] + x[3] ]\n\njacobian(Reverse, f, [2.0, 3.0, 4.0])\n\n# output\n([3.0 2.0 0.0; 0.0 1.0 1.0],)\n\nf(x) = [ x[1] * x[2], x[2] + x[3] ]\n\ngrad = jacobian(ReverseWithPrimal, f, [2.0, 3.0, 4.0])\n\n# output\n(derivs = ([3.0 2.0 0.0; 0.0 1.0 1.0],), val = [6.0, 7.0])\n\nf(x) = [ x[1] * x[2], x[2] + x[3] ]\n\ngrad = jacobian(Reverse, f, [2.0, 3.0, 4.0], n_outs=Val((2,)))\n\n# output\n([3.0 2.0 0.0; 0.0 1.0 1.0],)\n\nf(x) = [ x[1] * x[2], x[2] + x[3] ]\n\ngrad = jacobian(ReverseWithPrimal, f, [2.0, 3.0, 4.0], n_outs=Val((2,)))\n\n# output\n(derivs = ([3.0 2.0 0.0; 0.0 1.0 1.0],), val = [6.0, 7.0])\n\nThis function will return an AbstractArray whose shape is (size(output)..., size(input)...). No guarantees are presently made about the type of the AbstractArray returned by this function (which may or may not be the same as the input AbstractArray if provided).\n\nIn the future, when this function is extended to handle non-array return types, this function will retun an AbstractArray of shape size(output) of values of the input type. ```\n\n\n\n\n\n","category":"method"},{"location":"api/#Enzyme.typetree","page":"API reference","title":"Enzyme.typetree","text":"function typetree(T, ctx, dl, seen=TypeTreeTable())\n\nConstruct a Enzyme typetree from a Julia type.\n\nwarning: Warning\nWhen using a memoized lookup by providing seen across multiple calls to typtree the user must call copy on the returned value before mutating it.\n\n\n\n\n\n","category":"function"},{"location":"api/#Enzyme.unsafe_to_pointer-Union{Tuple{Type{T}}, Tuple{T}} where T","page":"API reference","title":"Enzyme.unsafe_to_pointer","text":"unsafe_to_pointer\n\nwarning: Warning\nAssumes that val is globally rooted and pointer to it can be leaked. Prefer pointer_from_objref. Only use inside Enzyme.jl should be for Types.\n\n\n\n\n\n","category":"method"},{"location":"api/#EnzymeCore.autodiff-Union{Tuple{A}, Tuple{FA}, Tuple{RuntimeActivity}, Tuple{ErrIfFuncWritten}, Tuple{Nargs}, Tuple{RABI}, Tuple{ReturnPrimal}, Tuple{ForwardMode{ReturnPrimal, RABI, ErrIfFuncWritten, RuntimeActivity}, FA, Type{A}, Vararg{Annotation, Nargs}}} where {ReturnPrimal, RABI<:EnzymeCore.ABI, Nargs, ErrIfFuncWritten, RuntimeActivity, FA<:Annotation, A<:Annotation}","page":"API reference","title":"EnzymeCore.autodiff","text":"autodiff(::ForwardMode, f, Activity, args::Annotation...)\n\nAuto-differentiate function f at arguments args using forward mode.\n\nargs may be numbers, arrays, structs of numbers, structs of arrays and so on. Enzyme will only differentiate in respect to arguments that are wrapped in a Duplicated or similar argument. Unlike reverse mode in autodiff, Active arguments are not allowed here, since all derivative results of immutable objects will be returned and should instead use Duplicated or variants like DuplicatedNoNeed.\n\nActivity is the Activity of the return value, it may be:\n\nConst if the return is not to be differentiated with respect to\nDuplicated, if the return is being differentiated with respect to\nBatchDuplicated, like Duplicated, but computing multiple derivatives at once. All batch sizes must be the same for all arguments.\n\nExample returning both original return and derivative:\n\nf(x) = x*x\nres, ∂f_∂x = autodiff(ForwardWithPrimal, f, Duplicated, Duplicated(3.14, 1.0))\n\n# output\n\n(6.28, 9.8596)\n\nExample returning just the derivative:\n\nf(x) = x*x\n∂f_∂x = autodiff(Forward, f, Duplicated, Duplicated(3.14, 1.0))\n\n# output\n\n(6.28,)\n\n\n\n\n\n","category":"method"},{"location":"api/#EnzymeCore.autodiff-Union{Tuple{ErrIfFuncWritten}, Tuple{Nargs}, Tuple{Holomorphic}, Tuple{RABI}, Tuple{RuntimeActivity}, Tuple{ReturnPrimal}, Tuple{A}, Tuple{FA}, Tuple{ReverseMode{ReturnPrimal, RuntimeActivity, RABI, Holomorphic, ErrIfFuncWritten}, FA, Type{A}, Vararg{Annotation, Nargs}}} where {FA<:Annotation, A<:Annotation, ReturnPrimal, RuntimeActivity, RABI<:EnzymeCore.ABI, Holomorphic, Nargs, ErrIfFuncWritten}","page":"API reference","title":"EnzymeCore.autodiff","text":"autodiff(::ReverseMode, f, Activity, args::Annotation...)\n\nAuto-differentiate function f at arguments args using reverse mode.\n\nLimitations:\n\nf may only return a Real (of a built-in/primitive type) or nothing, not an array, struct, BigFloat, etc. To handle vector-valued return types, use a mutating f! that returns nothing and stores it's return value in one of the arguments, which must be wrapped in a Duplicated.\n\nargs may be numbers, arrays, structs of numbers, structs of arrays and so on. Enzyme will only differentiate in respect to arguments that are wrapped in an Active (for arguments whose derivative result must be returned rather than mutated in place, such as primitive types and structs thereof) or Duplicated (for mutable arguments like arrays, Refs and structs thereof).\n\nActivity is the Activity of the return value, it may be Const or Active.\n\nExample:\n\na = 4.2\nb = [2.2, 3.3]; ∂f_∂b = zero(b)\nc = 55; d = 9\n\nf(a, b, c, d) = a * √(b[1]^2 + b[2]^2) + c^2 * d^2\n∂f_∂a, _, _, ∂f_∂d = autodiff(Reverse, f, Active, Active(a), Duplicated(b, ∂f_∂b), Const(c), Active(d))[1]\n\n# output\n\n(3.966106403010388, nothing, nothing, 54450.0)\n\nhere, autodiff returns a tuple (partial fpartial a partial fpartial d), while partial fpartial b will be added to ∂f_∂b (but not returned). c will be treated as Const(c).\n\nOne can also request the original returned value of the computation.\n\nExample:\n\nEnzyme.autodiff(ReverseWithPrimal, x->x*x, Active(3.0))\n\n# output\n\n((6.0,), 9.0)\n\nnote: Note\nEnzyme gradients with respect to integer values are zero. Active will automatically convert plain integers to floating point values, but cannot do so for integer values in tuples and structs.\n\n\n\n\n\n","category":"method"},{"location":"api/#EnzymeCore.autodiff-Union{Tuple{MMode}, Tuple{Nargs}, Tuple{A}, Tuple{Function, MMode, Type{A}, Vararg{Annotation, Nargs}}} where {A<:Annotation, Nargs, MMode<:EnzymeCore.Mode}","page":"API reference","title":"EnzymeCore.autodiff","text":"autodiff(::Function, ::Mode, args...)\n\nSpecialization of autodiff to handle do argument closures.\n\n\nautodiff(Reverse, Active(3.1)) do x\n return x*x\nend\n\n# output\n((6.2,),)\n\n\n\n\n\n","category":"method"},{"location":"api/#EnzymeCore.autodiff-Union{Tuple{Nargs}, Tuple{CMode}, Tuple{FA}, Tuple{CMode, FA, Vararg{Annotation, Nargs}}} where {FA<:Annotation, CMode<:EnzymeCore.Mode, Nargs}","page":"API reference","title":"EnzymeCore.autodiff","text":"autodiff(mode::Mode, f, args...)\n\nLike autodiff but will try to guess the activity of the return value.\n\n\n\n\n\n","category":"method"},{"location":"api/#EnzymeCore.autodiff-Union{Tuple{Nargs}, Tuple{CMode}, Tuple{F}, Tuple{CMode, F, Vararg{Annotation, Nargs}}} where {F, CMode<:EnzymeCore.Mode, Nargs}","page":"API reference","title":"EnzymeCore.autodiff","text":"autodiff(mode::Mode, f, ::Type{A}, args::Annotation...)\n\nLike autodiff but will try to extend f to an annotation, if needed.\n\n\n\n\n\n","category":"method"},{"location":"api/#EnzymeCore.autodiff_deferred-Union{Tuple{RuntimeActivity}, Tuple{ErrIfFuncWritten}, Tuple{ABI}, Tuple{Nargs}, Tuple{A}, Tuple{FA}, Tuple{ReturnPrimal}, Tuple{ForwardMode{ReturnPrimal, ABI, ErrIfFuncWritten, RuntimeActivity}, FA, Type{A}, Vararg{Annotation, Nargs}}} where {ReturnPrimal, FA<:Annotation, A<:Annotation, Nargs, ABI, ErrIfFuncWritten, RuntimeActivity}","page":"API reference","title":"EnzymeCore.autodiff_deferred","text":"autodiff_deferred(::ForwardMode, f, Activity, args::Annotation...)\n\nSame as autodiff(::ForwardMode, f, Activity, args...) but uses deferred compilation to support usage in GPU code, as well as high-order differentiation.\n\n\n\n\n\n","category":"method"},{"location":"api/#EnzymeCore.autodiff_deferred-Union{Tuple{RuntimeActivity}, Tuple{ErrIfFuncWritten}, Tuple{Holomorphic}, Tuple{ABI}, Tuple{Nargs}, Tuple{ReturnPrimal}, Tuple{A}, Tuple{FA}, Tuple{ReverseMode{ReturnPrimal, RuntimeActivity, ABI, Holomorphic, ErrIfFuncWritten}, FA, Type{A}, Vararg{Annotation, Nargs}}} where {FA<:Annotation, A<:Annotation, ReturnPrimal, Nargs, ABI, Holomorphic, ErrIfFuncWritten, RuntimeActivity}","page":"API reference","title":"EnzymeCore.autodiff_deferred","text":"autodiff_deferred(::ReverseMode, f, Activity, args::Annotation...)\n\nSame as autodiff but uses deferred compilation to support usage in GPU code, as well as high-order differentiation.\n\n\n\n\n\n","category":"method"},{"location":"api/#EnzymeCore.autodiff_deferred_thunk-Union{Tuple{RuntimeActivity}, Tuple{ErrIfFuncWritten}, Tuple{Nargs}, Tuple{RABI}, Tuple{ModifiedBetweenT}, Tuple{Width}, Tuple{ReturnShadow}, Tuple{ReturnPrimal}, Tuple{TapeType}, Tuple{A2}, Tuple{FA}, Tuple{EnzymeCore.ReverseModeSplit{ReturnPrimal, ReturnShadow, RuntimeActivity, Width, ModifiedBetweenT, RABI, ErrIfFuncWritten}, Type{TapeType}, Type{FA}, Type{A2}, Vararg{Type{<:Annotation}, Nargs}}} where {FA<:Annotation, A2<:Annotation, TapeType, ReturnPrimal, ReturnShadow, Width, ModifiedBetweenT, RABI<:EnzymeCore.ABI, Nargs, ErrIfFuncWritten, RuntimeActivity}","page":"API reference","title":"EnzymeCore.autodiff_deferred_thunk","text":"autodiff_deferred_thunk(::ReverseModeSplit, ftype, Activity, argtypes::Type{<:Annotation}...)\n\nProvide the split forward and reverse pass functions for annotated function type ftype when called with args of type argtypes when using reverse mode.\n\nActivity is the Activity of the return value, it may be Const, Active, or Duplicated (or its variants DuplicatedNoNeed, BatchDuplicated, and BatchDuplicatedNoNeed).\n\nThe forward function will return a tape, the primal (or nothing if not requested), and the shadow (or nothing if not a Duplicated variant), and tapes the corresponding type arguements provided.\n\nThe reverse function will return the derivative of Active arguments, updating the Duplicated arguments in place. The same arguments to the forward pass should be provided, followed by the adjoint of the return (if the return is active), and finally the tape from the forward pass.\n\nExample:\n\n\nA = [2.2]; ∂A = zero(A)\nv = 3.3\n\nfunction f(A, v)\n res = A[1] * v\n A[1] = 0\n res\nend\n\nTapeType = tape_type(ReverseSplitWithPrimal, Const{typeof(f)}, Active, Duplicated{typeof(A)}, Active{typeof(v)})\nforward, reverse = autodiff_deferred_thunk(ReverseSplitWithPrimal, TapeType, Const{typeof(f)}, Active{Float64}, Duplicated{typeof(A)}, Active{typeof(v)})\n\ntape, result, shadow_result = forward(Const(f), Duplicated(A, ∂A), Active(v))\n_, ∂v = reverse(Const(f), Duplicated(A, ∂A), Active(v), 1.0, tape)[1]\n\nresult, ∂v, ∂A \n\n# output\n\n(7.26, 2.2, [3.3])\n\n\n\n\n\n","category":"method"},{"location":"api/#EnzymeCore.autodiff_thunk-Union{Tuple{RuntimeActivity}, Tuple{ErrIfFuncWritten}, Tuple{Nargs}, Tuple{RABI}, Tuple{A}, Tuple{FA}, Tuple{ReturnPrimal}, Tuple{ForwardMode{ReturnPrimal, RABI, ErrIfFuncWritten, RuntimeActivity}, Type{FA}, Type{A}, Vararg{Type{<:Annotation}, Nargs}}} where {ReturnPrimal, FA<:Annotation, A<:Annotation, RABI<:EnzymeCore.ABI, Nargs, ErrIfFuncWritten, RuntimeActivity}","page":"API reference","title":"EnzymeCore.autodiff_thunk","text":"autodiff_thunk(::ForwardMode, ftype, Activity, argtypes::Type{<:Annotation}...)\n\nProvide the thunk forward mode function for annotated function type ftype when called with args of type argtypes.\n\nActivity is the Activity of the return value, it may be Const or Duplicated (or its variants DuplicatedNoNeed, BatchDuplicated, andBatchDuplicatedNoNeed).\n\nThe forward function will return the primal (if requested) and the shadow (or nothing if not a Duplicated variant).\n\nExample returning both the return derivative and original return:\n\na = 4.2\nb = [2.2, 3.3]; ∂f_∂b = zero(b)\nc = 55; d = 9\n\nf(x) = x*x\nforward = autodiff_thunk(ForwardWithPrimal, Const{typeof(f)}, Duplicated, Duplicated{Float64})\nres, ∂f_∂x = forward(Const(f), Duplicated(3.14, 1.0))\n\n# output\n\n(6.28, 9.8596)\n\nExample returning just the derivative:\n\na = 4.2\nb = [2.2, 3.3]; ∂f_∂b = zero(b)\nc = 55; d = 9\n\nf(x) = x*x\nforward = autodiff_thunk(Forward, Const{typeof(f)}, Duplicated, Duplicated{Float64})\n∂f_∂x = forward(Const(f), Duplicated(3.14, 1.0))\n\n# output\n\n(6.28,)\n\n\n\n\n\n","category":"method"},{"location":"api/#EnzymeCore.autodiff_thunk-Union{Tuple{RuntimeActivity}, Tuple{ErrIfFuncWritten}, Tuple{Nargs}, Tuple{RABI}, Tuple{ModifiedBetweenT}, Tuple{Width}, Tuple{ReturnShadow}, Tuple{ReturnPrimal}, Tuple{A}, Tuple{FA}, Tuple{EnzymeCore.ReverseModeSplit{ReturnPrimal, ReturnShadow, RuntimeActivity, Width, ModifiedBetweenT, RABI, ErrIfFuncWritten}, Type{FA}, Type{A}, Vararg{Type{<:Annotation}, Nargs}}} where {FA<:Annotation, A<:Annotation, ReturnPrimal, ReturnShadow, Width, ModifiedBetweenT, RABI<:EnzymeCore.ABI, Nargs, ErrIfFuncWritten, RuntimeActivity}","page":"API reference","title":"EnzymeCore.autodiff_thunk","text":"autodiff_thunk(::ReverseModeSplit, ftype, Activity, argtypes::Type{<:Annotation}...)\n\nProvide the split forward and reverse pass functions for annotated function type ftype when called with args of type argtypes when using reverse mode.\n\nActivity is the Activity of the return value, it may be Const, Active, or Duplicated (or its variants DuplicatedNoNeed, BatchDuplicated, and BatchDuplicatedNoNeed).\n\nThe forward function will return a tape, the primal (or nothing if not requested), and the shadow (or nothing if not a Duplicated variant), and tapes the corresponding type arguements provided.\n\nThe reverse function will return the derivative of Active arguments, updating the Duplicated arguments in place. The same arguments to the forward pass should be provided, followed by the adjoint of the return (if the return is active), and finally the tape from the forward pass.\n\nExample:\n\n\nA = [2.2]; ∂A = zero(A)\nv = 3.3\n\nfunction f(A, v)\n res = A[1] * v\n A[1] = 0\n res\nend\n\nforward, reverse = autodiff_thunk(ReverseSplitWithPrimal, Const{typeof(f)}, Active, Duplicated{typeof(A)}, Active{typeof(v)})\n\ntape, result, shadow_result = forward(Const(f), Duplicated(A, ∂A), Active(v))\n_, ∂v = reverse(Const(f), Duplicated(A, ∂A), Active(v), 1.0, tape)[1]\n\nresult, ∂v, ∂A \n\n# output\n\n(7.26, 2.2, [3.3])\n\n\n\n\n\n","category":"method"},{"location":"api/#EnzymeCore.within_autodiff-Tuple{}","page":"API reference","title":"EnzymeCore.within_autodiff","text":"within_autodiff()\n\nReturns true if within autodiff, otherwise false.\n\n\n\n\n\n","category":"method"},{"location":"api/#EnzymeCore.ABI","page":"API reference","title":"EnzymeCore.ABI","text":"abstract type ABI\n\nAbstract type for what ABI will be used.\n\n\n\n\n\n","category":"type"},{"location":"api/#EnzymeCore.Active","page":"API reference","title":"EnzymeCore.Active","text":"Active(x)\n\nMark a function argument x of autodiff as active, Enzyme will auto-differentiate in respect Active arguments.\n\nnote: Note\nEnzyme gradients with respect to integer values are zero. Active will automatically convert plain integers to floating point values, but cannot do so for integer values in tuples and structs.\n\n\n\n\n\n","category":"type"},{"location":"api/#EnzymeCore.Annotation","page":"API reference","title":"EnzymeCore.Annotation","text":"abstract type Annotation{T}\n\nAbstract type for autodiff function argument wrappers like Const, Active and Duplicated.\n\n\n\n\n\n","category":"type"},{"location":"api/#EnzymeCore.BatchDuplicated","page":"API reference","title":"EnzymeCore.BatchDuplicated","text":"BatchDuplicated(x, ∂f_∂xs)\n\nLike Duplicated, except contains several shadows to compute derivatives for all at once. Argument ∂f_∂xs should be a tuple of the several values of type x.\n\n\n\n\n\n","category":"type"},{"location":"api/#EnzymeCore.BatchDuplicatedNoNeed","page":"API reference","title":"EnzymeCore.BatchDuplicatedNoNeed","text":"BatchDuplicatedNoNeed(x, ∂f_∂xs)\n\nLike DuplicatedNoNeed, except contains several shadows to compute derivatives for all at once. Argument ∂f_∂xs should be a tuple of the several values of type x.\n\n\n\n\n\n","category":"type"},{"location":"api/#EnzymeCore.BatchMixedDuplicated","page":"API reference","title":"EnzymeCore.BatchMixedDuplicated","text":"BatchMixedDuplicated(x, ∂f_∂xs)\n\nLike MixedDuplicated, except contains several shadows to compute derivatives for all at once. Only used within custom rules.\n\n\n\n\n\n","category":"type"},{"location":"api/#EnzymeCore.Const","page":"API reference","title":"EnzymeCore.Const","text":"Const(x)\n\nMark a function argument x of autodiff as constant, Enzyme will not auto-differentiate in respect Const arguments.\n\n\n\n\n\n","category":"type"},{"location":"api/#EnzymeCore.Duplicated","page":"API reference","title":"EnzymeCore.Duplicated","text":"Duplicated(x, ∂f_∂x)\n\nMark a function argument x of autodiff as duplicated, Enzyme will auto-differentiate in respect to such arguments, with dx acting as an accumulator for gradients (so partial f partial x will be added to) ∂f_∂x.\n\n\n\n\n\n","category":"type"},{"location":"api/#EnzymeCore.DuplicatedNoNeed","page":"API reference","title":"EnzymeCore.DuplicatedNoNeed","text":"DuplicatedNoNeed(x, ∂f_∂x)\n\nLike Duplicated, except also specifies that Enzyme may avoid computing the original result and only compute the derivative values.\n\nThis should only be used if x is a write-only variable. Otherwise, if the differentiated function stores values in x and reads them back in subsequent computations, using DuplicatedNoNeed may result in incorrect derivatives. In particular, DuplicatedNoNeed should not be used for preallocated workspace, even if the user might not care about its final value, as marking a variable as NoNeed means that reads from the variable are now undefined.\n\n\n\n\n\n","category":"type"},{"location":"api/#EnzymeCore.FFIABI","page":"API reference","title":"EnzymeCore.FFIABI","text":"struct FFIABI <: ABI\n\nForeign function call ABI. JIT the differentiated function, then inttoptr call the address.\n\n\n\n\n\n","category":"type"},{"location":"api/#EnzymeCore.ForwardMode","page":"API reference","title":"EnzymeCore.ForwardMode","text":"struct Forward{ReturnPrimal, ABI, ErrIfFuncWritten,RuntimeActivity} <: Mode{ABI, ErrIfFuncWritten, RuntimeActivity}\n\nForward mode differentiation\n\n\n\n\n\n","category":"type"},{"location":"api/#EnzymeCore.InlineABI","page":"API reference","title":"EnzymeCore.InlineABI","text":"struct InlineABI <: ABI\n\nInlining function call ABI. \n\n\n\n\n\n","category":"type"},{"location":"api/#EnzymeCore.MixedDuplicated","page":"API reference","title":"EnzymeCore.MixedDuplicated","text":"MixedDuplicated(x, ∂f_∂x)\n\nLike Duplicated, except x may contain both active [immutable] and duplicated [mutable] data which is differentiable. Only used within custom rules.\n\n\n\n\n\n","category":"type"},{"location":"api/#EnzymeCore.Mode","page":"API reference","title":"EnzymeCore.Mode","text":"abstract type Mode\n\nAbstract type for what differentiation mode will be used.\n\n\n\n\n\n","category":"type"},{"location":"api/#EnzymeCore.NonGenABI","page":"API reference","title":"EnzymeCore.NonGenABI","text":"struct NonGenABI <: ABI\n\nNon-generated function ABI. \n\n\n\n\n\n","category":"type"},{"location":"api/#EnzymeCore.ReverseMode","page":"API reference","title":"EnzymeCore.ReverseMode","text":"struct ReverseMode{ReturnPrimal,RuntimeActivity,ABI,Holomorphic,ErrIfFuncWritten} <: Mode{ABI, ErrIfFuncWritten, RuntimeActivity}\n\nReverse mode differentiation.\n\nReturnPrimal: Should Enzyme return the primal return value from the augmented-forward.\nRuntimeActivity: Should Enzyme enable runtime activity (default off)\nABI: What runtime ABI to use\nHolomorphic: Whether the complex result function is holomorphic and we should compute d/dz\nErrIfFuncWritten: Should Enzyme err if the function differentiated is a closure and written to.\n\n\n\n\n\n","category":"type"},{"location":"api/#EnzymeCore.ReverseModeSplit","page":"API reference","title":"EnzymeCore.ReverseModeSplit","text":"struct ReverseModeSplit{ReturnPrimal,ReturnShadow,RuntimeActivity,Width,ModifiedBetween,ABI} <: Mode{ABI,ErrIfFuncWritten,RuntimeActivity}\n\nReverse mode differentiation.\n\nReturnPrimal: Should Enzyme return the primal return value from the augmented-forward.\nReturnShadow: Should Enzyme return the shadow return value from the augmented-forward.\nRuntimeActivity: Should Enzyme differentiate with runtime activity on (default off).\nWidth: Batch Size (0 if to be automatically derived)\nModifiedBetween: Tuple of each argument's modified between state (true if to be automatically derived).\n\n\n\n\n\n","category":"type"},{"location":"api/#EnzymeCore.NoPrimal-Union{Tuple{ReverseMode{ReturnPrimal, RuntimeActivity, ABI, Holomorphic, ErrIfFuncWritten}}, Tuple{ErrIfFuncWritten}, Tuple{Holomorphic}, Tuple{ABI}, Tuple{RuntimeActivity}, Tuple{ReturnPrimal}} where {ReturnPrimal, RuntimeActivity, ABI, Holomorphic, ErrIfFuncWritten}","page":"API reference","title":"EnzymeCore.NoPrimal","text":" NoPrimal(::Enzyme.Mode)\n\nModifies the mode to exclude the primal value.\n\n\n\n\n\n","category":"method"},{"location":"api/#EnzymeCore.WithPrimal-Union{Tuple{ReverseMode{ReturnPrimal, RuntimeActivity, ABI, Holomorphic, ErrIfFuncWritten}}, Tuple{ErrIfFuncWritten}, Tuple{Holomorphic}, Tuple{ABI}, Tuple{RuntimeActivity}, Tuple{ReturnPrimal}} where {ReturnPrimal, RuntimeActivity, ABI, Holomorphic, ErrIfFuncWritten}","page":"API reference","title":"EnzymeCore.WithPrimal","text":" WithPrimal(::Enzyme.Mode)\n\nModifies the mode to include the primal value.\n\n\n\n\n\n","category":"method"},{"location":"api/#EnzymeCore.compiler_job_from_backend","page":"API reference","title":"EnzymeCore.compiler_job_from_backend","text":"compiler_job_from_backend(::KernelAbstractions.Backend, F::Type, TT:Type)::GPUCompiler.CompilerJob\n\nReturns a GPUCompiler CompilerJob from a backend as specified by the first argument to the function.\n\nFor example, in CUDA one would do:\n\nfunction EnzymeCore.compiler_job_from_backend(::CUDABackend, @nospecialize(F::Type), @nospecialize(TT::Type))\n mi = GPUCompiler.methodinstance(F, TT)\n return GPUCompiler.CompilerJob(mi, CUDA.compiler_config(CUDA.device()))\nend\n\n\n\n\n\n","category":"function"},{"location":"api/#EnzymeCore.make_zero","page":"API reference","title":"EnzymeCore.make_zero","text":"make_zero(::Type{T}, seen::IdDict, prev::T, ::Val{copy_if_inactive}=Val(false))::T\n\nRecursively make a zero'd copy of the value `prev` of type `T`. The argument `copy_if_inactive` specifies\nwhat to do if the type `T` is guaranteed to be inactive, use the primal (the default) or still copy the value.\n\n\n\n\n\n","category":"function"},{"location":"api/#EnzymeCore.make_zero!","page":"API reference","title":"EnzymeCore.make_zero!","text":"make_zero!(val::T, seen::IdSet{Any}=IdSet())::Nothing\n\nRecursively set a variables differentiable fields to zero. Only applicable for mutable types `T`.\n\n\n\n\n\n","category":"function"},{"location":"api/#EnzymeCore.make_zero-Union{Tuple{copy_if_inactive}, Tuple{T}, Tuple{T, Val{copy_if_inactive}}} where {T, copy_if_inactive}","page":"API reference","title":"EnzymeCore.make_zero","text":"make_zero(prev::T)\n\nHelper function to recursively make zero.\n\n\n\n\n\n","category":"method"},{"location":"api/#EnzymeCore.within_autodiff","page":"API reference","title":"EnzymeCore.within_autodiff","text":"within_autodiff()\n\nReturns true if within autodiff, otherwise false.\n\n\n\n\n\n","category":"function"},{"location":"api/#EnzymeCore.EnzymeRules.AugmentedReturn","page":"API reference","title":"EnzymeCore.EnzymeRules.AugmentedReturn","text":"AugmentedReturn(primal, shadow, tape)\n\nAugment the primal return value of a function with its shadow, as well as any additional information needed to correctly compute the reverse pass, stored in tape.\n\nUnless specified by the config that a variable is not overwritten, rules must assume any arrays/data structures/etc are overwritten between the forward and the reverse pass. Any floats or variables passed by value are always preserved as is (as are the arrays themselves, just not necessarily the values in the array).\n\nSee also augmented_primal.\n\n\n\n\n\n","category":"type"},{"location":"api/#EnzymeCore.EnzymeRules.FwdConfig","page":"API reference","title":"EnzymeCore.EnzymeRules.FwdConfig","text":"FwdConfig{NeedsPrimal, NeedsShadow, Width, RuntimeActivity}\nFwdConfigWidth{Width} = FwdConfig{<:Any, <:Any, Width}\n\nConfiguration type to dispatch on in custom forward rules (see forward.\n\nNeedsPrimal and NeedsShadow: boolean values specifying whether the primal and shadow (resp.) should be returned. \nWidth: an integer that specifies the number of adjoints/shadows simultaneously being propagated.\nRuntimeActivity: whether runtime activity is enabled.\n\nGetters for the type parameters are provided by needs_primal, needs_shadow, width and runtime_activity.\n\n\n\n\n\n","category":"type"},{"location":"api/#EnzymeCore.EnzymeRules.RevConfig","page":"API reference","title":"EnzymeCore.EnzymeRules.RevConfig","text":"RevConfig{NeedsPrimal, NeedsShadow, Width, Overwritten, RuntimeActivity}\nRevConfigWidth{Width} = RevConfig{<:Any, <:Any, Width}\n\nConfiguration type to dispatch on in custom reverse rules (see augmented_primal and reverse).\n\nNeedsPrimal and NeedsShadow: boolean values specifying whether the primal and shadow (resp.) should be returned. \nWidth: an integer that specifies the number of adjoints/shadows simultaneously being propagated.\nOverwritten: a tuple of booleans of whether each argument (including the function itself) is modified between the forward and reverse pass (true if potentially modified between).\nRuntimeActivity: whether runtime activity is enabled.\n\nGetters for the four type parameters are provided by needs_primal, needs_shadow, width, overwritten, and runtime_activity.\n\n\n\n\n\n","category":"type"},{"location":"api/#EnzymeCore.EnzymeRules.augmented_primal","page":"API reference","title":"EnzymeCore.EnzymeRules.augmented_primal","text":"augmented_primal(::RevConfig, func::Annotation{typeof(f)}, RT::Type{<:Annotation}, args::Annotation...)\n\nMust return an AugmentedReturn type.\n\nThe primal must be the same type of the original return if needs_primal(config), otherwise nothing.\nThe shadow must be nothing if needs_shadow(config) is false. If width is 1, the shadow should be the same type of the original return. If the width is greater than 1, the shadow should be NTuple{original return, width}.\nThe tape can be any type (including Nothing) and is preserved for the reverse call.\n\n\n\n\n\n","category":"function"},{"location":"api/#EnzymeCore.EnzymeRules.forward","page":"API reference","title":"EnzymeCore.EnzymeRules.forward","text":"forward(fwdconfig, func::Annotation{typeof(f)}, RT::Type{<:Annotation}, args::Annotation...)\n\nCalculate the forward derivative. The first argument is a `FwdConfig object describing parameters of the differentiation. The second argument func is the callable for which the rule applies to. Either wrapped in a Const), or a Duplicated if it is a closure. The third argument is the return type annotation, and all other arguments are the annotated function arguments.\n\n\n\n\n\n","category":"function"},{"location":"api/#EnzymeCore.EnzymeRules.inactive","page":"API reference","title":"EnzymeCore.EnzymeRules.inactive","text":"inactive(func::typeof(f), args...)\n\nMark a particular function as always being inactive in both its return result and the function call itself.\n\n\n\n\n\n","category":"function"},{"location":"api/#EnzymeCore.EnzymeRules.inactive_noinl","page":"API reference","title":"EnzymeCore.EnzymeRules.inactive_noinl","text":"inactive_noinl(func::typeof(f), args...)\n\nMark a particular function as always being inactive in both its return result and the function call itself, but do not prevent inlining of the function.\n\n\n\n\n\n","category":"function"},{"location":"api/#EnzymeCore.EnzymeRules.inactive_type-Tuple{Type}","page":"API reference","title":"EnzymeCore.EnzymeRules.inactive_type","text":"inactive_type(::Type{Ty})\n\nMark a particular type Ty as always being inactive.\n\n\n\n\n\n","category":"method"},{"location":"api/#EnzymeCore.EnzymeRules.needs_primal-Union{Tuple{EnzymeCore.EnzymeRules.FwdConfigWidth{Width, NeedsPrimal, NeedsShadow} where {NeedsShadow, Width}}, Tuple{NeedsPrimal}} where NeedsPrimal","page":"API reference","title":"EnzymeCore.EnzymeRules.needs_primal","text":"needs_primal(::FwdConfig)\nneeds_primal(::RevConfig)\n\nWhether a custom rule should return the original result of the function.\n\n\n\n\n\n","category":"method"},{"location":"api/#EnzymeCore.EnzymeRules.needs_shadow-Union{Tuple{EnzymeCore.EnzymeRules.FwdConfigWidth{Width, var\"#s9\", NeedsShadow} where {var\"#s9\", Width}}, Tuple{NeedsShadow}} where NeedsShadow","page":"API reference","title":"EnzymeCore.EnzymeRules.needs_shadow","text":"needs_shadow(::FwdConfig)\nneeds_shadow(::RevConfig)\n\nWhether a custom rule should return the shadow (derivative) of the function result.\n\n\n\n\n\n","category":"method"},{"location":"api/#EnzymeCore.EnzymeRules.noalias","page":"API reference","title":"EnzymeCore.EnzymeRules.noalias","text":"noalias(func::typeof(f), args...)\n\nMark a particular function as always being a fresh allocation which does not alias any other accessible memory.\n\n\n\n\n\n","category":"function"},{"location":"api/#EnzymeCore.EnzymeRules.primal_type-Union{Tuple{RT}, Tuple{EnzymeCore.EnzymeRules.FwdConfig, Type{<:Annotation{RT}}}} where RT","page":"API reference","title":"EnzymeCore.EnzymeRules.primal_type","text":"primal_type(::FwdConfig, ::Type{<:Annotation{RT}})\nprimal_type(::RevConfig, ::Type{<:Annotation{RT}})\n\nCompute the exepcted primal return type given a reverse mode config and return activity\n\n\n\n\n\n","category":"method"},{"location":"api/#EnzymeCore.EnzymeRules.reverse","page":"API reference","title":"EnzymeCore.EnzymeRules.reverse","text":"reverse(::RevConfig, func::Annotation{typeof(f)}, dret::Active, tape, args::Annotation...)\nreverse(::RevConfig, func::Annotation{typeof(f)}, ::Type{<:Annotation), tape, args::Annotation...)\n\nTakes gradient of derivative, activity annotation, and tape. If there is an active return dret is passed as Active{T} with the derivative of the active return val. Otherwise dret is passed as Type{Duplicated{T}}, etc.\n\n\n\n\n\n","category":"function"},{"location":"api/#EnzymeCore.EnzymeRules.shadow_type-Union{Tuple{RT}, Tuple{EnzymeCore.EnzymeRules.FwdConfig, Type{<:Annotation{RT}}}} where RT","page":"API reference","title":"EnzymeCore.EnzymeRules.shadow_type","text":"shadow_type(::FwdConfig, ::Type{<:Annotation{RT}})\nshadow_type(::RevConfig, ::Type{<:Annotation{RT}})\n\nCompute the exepcted shadow return type given a reverse mode config and return activity\n\n\n\n\n\n","category":"method"},{"location":"api/#EnzymeTestUtils.ExprAndMsg","page":"API reference","title":"EnzymeTestUtils.ExprAndMsg","text":"A cunning hack to carry extra message along with the original expression in a test\n\n\n\n\n\n","category":"type"},{"location":"api/#EnzymeTestUtils.@test_msg-Tuple{Any, Any, Vararg{Any}}","page":"API reference","title":"EnzymeTestUtils.@test_msg","text":"@test_msg msg condion kws...\n\nThis is per Test.@test condion kws... except that if it fails it also prints the msg. If msg==\"\" then this is just like @test, nothing is printed\n\nExamles\n\njulia> @test_msg \"It is required that the total is under 10\" sum(1:1000) < 10;\nTest Failed at REPL[1]:1\n Expression: sum(1:1000) < 10\n Problem: It is required that the total is under 10\n Evaluated: 500500 < 10\nERROR: There was an error during testing\n\n\njulia> @test_msg \"It is required that the total is under 10\" error(\"not working at all\");\nError During Test at REPL[2]:1\n Test threw exception\n Expression: error(\"not working at all\")\n Problem: It is required that the total is under 10\n \"not working at all\"\n Stacktrace:\n\njulia> a = \"\";\n\njulia> @test_msg a sum(1:1000) < 10;\n Test Failed at REPL[153]:1\n Expression: sum(1:1000) < 10\n Evaluated: 500500 < 10\n ERROR: There was an error during testing\n\n\n\n\n\n","category":"macro"},{"location":"api/#EnzymeTestUtils.are_activities_compatible-Tuple{Any, Vararg{Any}}","page":"API reference","title":"EnzymeTestUtils.are_activities_compatible","text":"are_activities_compatible(Tret, activities...) -> Bool\n\nReturn true if return activity type Tret and activity types activities are compatible.\n\n\n\n\n\n","category":"method"},{"location":"api/#EnzymeTestUtils.test_forward-Tuple{Any, Any, Vararg{Any}}","page":"API reference","title":"EnzymeTestUtils.test_forward","text":"test_forward(f, Activity, args...; kwargs...)\n\nTest Enzyme.autodiff of f in Forward-mode against finite differences.\n\nf has all constraints of the same argument passed to Enzyme.autodiff, with additional constraints:\n\nIf it mutates one of its arguments, it must return that argument.\n\nArguments\n\nActivity: the activity of the return value of f\nargs: Each entry is either an argument to f, an activity type accepted by autodiff, or a tuple of the form (arg, Activity), where Activity is the activity type of arg. If the activity type specified requires a tangent, a random tangent will be automatically generated.\n\nKeywords\n\nrng::AbstractRNG: The random number generator to use for generating random tangents.\nfdm=FiniteDifferences.central_fdm(5, 1): The finite differences method to use.\nfkwargs: Keyword arguments to pass to f.\nrtol: Relative tolerance for isapprox.\natol: Absolute tolerance for isapprox.\ntestset_name: Name to use for a testset in which all tests are evaluated.\n\nExamples\n\nHere we test a rule for a function of scalars. Because we don't provide an activity annotation for y, it is assumed to be Const.\n\nusing Enzyme, EnzymeTestUtils\n\nx, y = randn(2)\nfor Tret in (Const, Duplicated, DuplicatedNoNeed), Tx in (Const, Duplicated)\n test_forward(*, Tret, (x, Tx), y)\nend\n\nHere we test a rule for a function of an array in batch forward-mode:\n\nx = randn(3)\ny = randn()\nfor Tret in (Const, BatchDuplicated, BatchDuplicatedNoNeed),\n Tx in (Const, BatchDuplicated),\n Ty in (Const, BatchDuplicated)\n\n test_forward(*, Tret, (x, Tx), (y, Ty))\nend\n\n\n\n\n\n","category":"method"},{"location":"api/#EnzymeTestUtils.test_reverse-Tuple{Any, Any, Vararg{Any}}","page":"API reference","title":"EnzymeTestUtils.test_reverse","text":"test_reverse(f, Activity, args...; kwargs...)\n\nTest Enzyme.autodiff_thunk of f in ReverseSplitWithPrimal-mode against finite differences.\n\nf has all constraints of the same argument passed to Enzyme.autodiff_thunk, with additional constraints:\n\nIf an Array{<:AbstractFloat} appears in the input/output, then a reshaped version of it may not also appear in the input/output.\n\nArguments\n\nActivity: the activity of the return value of f.\nargs: Each entry is either an argument to f, an activity type accepted by autodiff, or a tuple of the form (arg, Activity), where Activity is the activity type of arg. If the activity type specified requires a shadow, one will be automatically generated.\n\nKeywords\n\nrng::AbstractRNG: The random number generator to use for generating random tangents.\nfdm=FiniteDifferences.central_fdm(5, 1): The finite differences method to use.\nfkwargs: Keyword arguments to pass to f.\nrtol: Relative tolerance for isapprox.\natol: Absolute tolerance for isapprox.\ntestset_name: Name to use for a testset in which all tests are evaluated.\n\nExamples\n\nHere we test a rule for a function of scalars. Because we don't provide an activity annotation for y, it is assumed to be Const.\n\nusing Enzyme, EnzymeTestUtils\n\nx = randn()\ny = randn()\nfor Tret in (Const, Active), Tx in (Const, Active)\n test_reverse(*, Tret, (x, Tx), y)\nend\n\nHere we test a rule for a function of an array in batch reverse-mode:\n\nx = randn(3)\nfor Tret in (Const, Active), Tx in (Const, BatchDuplicated)\n test_reverse(prod, Tret, (x, Tx))\nend\n\n\n\n\n\n","category":"method"},{"location":"api/#Enzyme.API.fast_math!-Tuple{Any}","page":"API reference","title":"Enzyme.API.fast_math!","text":"fast_math!(val::Bool)\n\nWhether generated derivatives have fast math on or off, default on.\n\n\n\n\n\n","category":"method"},{"location":"api/#Enzyme.API.inlineall!-Tuple{Any}","page":"API reference","title":"Enzyme.API.inlineall!","text":"inlineall!(val::Bool)\n\nWhether to inline all (non-recursive) functions generated by Julia within a single compilation unit. This may improve Enzyme's ability to successfully differentiate code and improve performance of the original and generated derivative program. It often, however, comes with an increase in compile time. This is off by default.\n\n\n\n\n\n","category":"method"},{"location":"api/#Enzyme.API.instname!-Tuple{Any}","page":"API reference","title":"Enzyme.API.instname!","text":"instname!(val::Bool)\n\nWhether to add a name to all LLVM values. This may be helpful for debugging generated programs, both primal and derivative. Off by default.\n\n\n\n\n\n","category":"method"},{"location":"api/#Enzyme.API.looseTypeAnalysis!-Tuple{Any}","page":"API reference","title":"Enzyme.API.looseTypeAnalysis!","text":"looseTypeAnalysis!(val::Bool)\n\nEnzyme runs a type analysis to deduce the corresponding types of all values being differentiated. This is necessary to compute correct derivatives of various values. For example, a copy of Float32's requires a different derivative than a memcpy of Float64's, Ptr's, etc. In some cases Enzyme may not be able to deduce all the types necessary and throw an unknown type error. If this is the case, open an issue. One can silence these issues by setting looseTypeAnalysis!(true) which tells Enzyme to make its best guess. This will remove the error and allow differentiation to continue, however, it may produce incorrect results. Alternatively one can consider increasing the space of the evaluated type lattice which gives Enzyme more time to run a more thorough analysis through the use of maxtypeoffset!\n\n\n\n\n\n","category":"method"},{"location":"api/#Enzyme.API.maxtypedepth!-Tuple{Any}","page":"API reference","title":"Enzyme.API.maxtypedepth!","text":"maxtypedepth!(val::Int)\n\nEnzyme runs a type analysis to deduce the corresponding types of all values being differentiated. This is necessary to compute correct derivatives of various values. To ensure this analysis temrinates, it operates on a finite lattice of possible states. This function sets the maximum depth into a type that Enzyme will consider. A smaller value will cause type analysis to run faster, but may result in some necessary types not being found and result in unknown type errors. A larger value may result in unknown type errors being resolved by searching a larger space, but may run longer. The default setting is 6.\n\n\n\n\n\n","category":"method"},{"location":"api/#Enzyme.API.maxtypeoffset!-Tuple{Any}","page":"API reference","title":"Enzyme.API.maxtypeoffset!","text":"maxtypeoffset!(val::Int)\n\nEnzyme runs a type analysis to deduce the corresponding types of all values being differentiated. This is necessary to compute correct derivatives of various values. To ensure this analysis temrinates, it operates on a finite lattice of possible states. This function sets the maximum offset into a type that Enzyme will consider. A smaller value will cause type analysis to run faster, but may result in some necessary types not being found and result in unknown type errors. A larger value may result in unknown type errors being resolved by searching a larger space, but may run longer. The default setting is 512.\n\n\n\n\n\n","category":"method"},{"location":"api/#Enzyme.API.memmove_warning!-Tuple{Any}","page":"API reference","title":"Enzyme.API.memmove_warning!","text":"memmove_warning!(val::Bool)\n\nWhether to issue a warning when differentiating memmove. Off by default.\n\n\n\n\n\n","category":"method"},{"location":"api/#Enzyme.API.printactivity!-Tuple{Any}","page":"API reference","title":"Enzyme.API.printactivity!","text":"printactivity!(val::Bool)\n\nAn debugging option for developers of Enzyme. If one sets this flag prior to the first differentiation of a function, Enzyme will print (to stderr) a log of all decisions made during Activity Analysis (the analysis which determines what values/instructions are differentiated). This may be useful for debugging MixedActivity errors, correctness, and performance errors. Off by default\n\n\n\n\n\n","category":"method"},{"location":"api/#Enzyme.API.printall!-Tuple{Any}","page":"API reference","title":"Enzyme.API.printall!","text":"printall!(val::Bool)\n\nAn debugging option for developers of Enzyme. If one sets this flag prior to the first differentiation of a function, Enzyme will print (to stderr) the LLVM function being differentiated, as well as all generated derivatives immediately after running Enzyme (but prior to any other optimizations). Off by default\n\n\n\n\n\n","category":"method"},{"location":"api/#Enzyme.API.printdiffuse!-Tuple{Any}","page":"API reference","title":"Enzyme.API.printdiffuse!","text":"printdiffuse!(val::Bool)\n\nAn debugging option for developers of Enzyme. If one sets this flag prior to the first differentiation of a function, Enzyme will print (to stderr) information about each LLVM value – specifically whether it and its shadow is required for computing the derivative. In contrast to printunnecessary!, this flag prints debug log for the analysis which determines for each value and shadow value, whether it can find a user which would require it to be kept around (rather than being deleted). This is prior to any cache optimizations and a debug log of Differential Use Analysis. This may be helpful for debugging caching, phi node deletion, performance, and other errors. Off by default\n\n\n\n\n\n","category":"method"},{"location":"api/#Enzyme.API.printperf!-Tuple{Any}","page":"API reference","title":"Enzyme.API.printperf!","text":"printperf!(val::Bool)\n\nAn debugging option for developers of Enzyme. If one sets this flag prior to the first differentiation of a function, Enzyme will print (to stderr) performance information about generated derivative programs. It will provide debug information that warns why particular values are cached for the reverse pass, and thus require additional computation/storage. This is particularly helpful for debugging derivatives which OOM or otherwise run slow. ff by default\n\n\n\n\n\n","category":"method"},{"location":"api/#Enzyme.API.printtype!-Tuple{Any}","page":"API reference","title":"Enzyme.API.printtype!","text":"printtype!(val::Bool)\n\nAn debugging option for developers of Enzyme. If one sets this flag prior to the first differentiation of a function, Enzyme will print (to stderr) a log of all decisions made during Type Analysis (the analysis which Enzyme determines the type of all values in the program). This may be useful for debugging correctness errors, illegal type analysis errors, insufficient type information errors, correctness, and performance errors. Off by default\n\n\n\n\n\n","category":"method"},{"location":"api/#Enzyme.API.printunnecessary!-Tuple{Any}","page":"API reference","title":"Enzyme.API.printunnecessary!","text":"printunnecessary!(val::Bool)\n\nAn debugging option for developers of Enzyme. If one sets this flag prior to the first differentiation of a function, Enzyme will print (to stderr) information about each LLVM value – specifically whether it and its shadow is required for computing the derivative. In contrast to printdiffuse!, this flag prints the final results after running cache optimizations such as minCut (see Recompute vs Cache Heuristics from this paper and slides 31-33 from this presentation) for a description of the caching algorithm. This may be helpful for debugging caching, phi node deletion, performance, and other errors. Off by default\n\n\n\n\n\n","category":"method"},{"location":"api/#Enzyme.API.strictAliasing!-Tuple{Any}","page":"API reference","title":"Enzyme.API.strictAliasing!","text":"strictAliasing!(val::Bool)\n\nWhether Enzyme's type analysis will assume strict aliasing semantics. When strict aliasing semantics are on (the default), Enzyme can propagate type information up through conditional branches. This may lead to illegal type errors when analyzing code with unions. Disabling strict aliasing will enable these union types to be correctly analyzed. However, it may lead to some errors that sufficient type information cannot be deduced. One can turn these insufficient type information errors into to warnings by calling looseTypeAnalysis!(true) which tells Enzyme to use its best guess in such scenarios.\n\n\n\n\n\n","category":"method"},{"location":"api/#Enzyme.API.strong_zero!-Tuple{Any}","page":"API reference","title":"Enzyme.API.strong_zero!","text":"strong_zero!(val::Bool)\n\nWhether to enforce multiplication by zero as enforcing a zero result even if multiplying against a NaN or infinity. Necessary for some programs in which a value has a zero derivative since it is unused, even if it has an otherwise infinite or nan derivative.\n\n\n\n\n\n","category":"method"},{"location":"api/#Enzyme.API.typeWarning!-Tuple{Any}","page":"API reference","title":"Enzyme.API.typeWarning!","text":"typeWarning!(val::Bool)\n\nWhether to print a warning when Type Analysis learns informatoin about a value's type which cannot be represented in the current size of the lattice. See maxtypeoffset! for more information. Off by default.\n\n\n\n\n\n","category":"method"},{"location":"dev_docs/#Enzyme-developer-documentation","page":"For developers","title":"Enzyme developer documentation","text":"","category":"section"},{"location":"dev_docs/#Development-of-Enzyme-and-Enzyme.jl-together-(recommended)","page":"For developers","title":"Development of Enzyme and Enzyme.jl together (recommended)","text":"","category":"section"},{"location":"dev_docs/","page":"For developers","title":"For developers","text":"Normally Enzyme.jl downloads and installs Enzyme for the user automatically since Enzyme needs to be built against Julia bundeled LLVM. In case that you are making updates to Enzyme and want to test them against Enzyme.jl the instructions below should help you get started.","category":"page"},{"location":"dev_docs/","page":"For developers","title":"For developers","text":"Start Julia in your development copy of Enzyme.jl and initialize the deps project","category":"page"},{"location":"dev_docs/","page":"For developers","title":"For developers","text":"~/s/Enzyme.jl (master)> julia --project=deps","category":"page"},{"location":"dev_docs/","page":"For developers","title":"For developers","text":"julia> # Hit the `]` key to enter package repl.\n(deps) pkg> instantiate","category":"page"},{"location":"dev_docs/","page":"For developers","title":"For developers","text":"We can now build a custom version of Enzyme for use in Enzyme.jl. To build the latest commit on the main branch of Enzyme, run the following. It may take a few minutes to compile fully.","category":"page"},{"location":"dev_docs/","page":"For developers","title":"For developers","text":"~/s/Enzyme.jl (master)> julia --project=deps deps/build_local.jl","category":"page"},{"location":"dev_docs/","page":"For developers","title":"For developers","text":"You will now find a file LocalPrefernces.toml which has been generated and contains a path to the new Enzyme_jll binary you have built. To use your Enzyme_jll instead of the default shipped by Enzyme.jl, ensure that this file is at the root of any Julia project you wish to test it with and that the Julia project has Enzyme_jll as an explicit dependency. Note that an indirect dependency here is not sufficient (e.g. just because a project depends on Enzyme.jl, which depends on Enzyme_jll, does not mean that your project will pick up this file unless you also add a direct dependency to Enzyme_jll).","category":"page"},{"location":"dev_docs/","page":"For developers","title":"For developers","text":"To test whether your project found the custom version of Enzyme_jll, you can inspect the path of the Enzyme_jll library in use as follows.","category":"page"},{"location":"dev_docs/","page":"For developers","title":"For developers","text":"~/my/project.jl (master)> julia --project=.","category":"page"},{"location":"dev_docs/","page":"For developers","title":"For developers","text":"julia> using Enzyme_jll\njulia> Enzyme_jll.libEnzyme_path\n\"${JULIA_PKG_DEVDIR}/Enzyme_jll/override/lib/LLVMEnzyme-9.so\"","category":"page"},{"location":"dev_docs/","page":"For developers","title":"For developers","text":"This should correspond to the path in the LocalPreferences.toml you just generated.","category":"page"},{"location":"dev_docs/","page":"For developers","title":"For developers","text":"Note that your system can have only one custom built Enzyme_jll at a time. If you build one version for one version of Enzyme or Julia and later build a new version of Enzyme, it removes the old build. ","category":"page"},{"location":"dev_docs/","page":"For developers","title":"For developers","text":"Note that Julia versions are tightly coupled and you cannot use an Enzyme_jll built for one version of Julia for another version of Julia.","category":"page"},{"location":"dev_docs/","page":"For developers","title":"For developers","text":"The same script can also be used to build Enzyme_jll for a branch other than main as follows.","category":"page"},{"location":"dev_docs/","page":"For developers","title":"For developers","text":"~/s/Enzyme.jl (master)> julia --project=deps deps/build_local.jl --branch mybranch","category":"page"},{"location":"dev_docs/","page":"For developers","title":"For developers","text":"It can also be used to build Enzyme_jll from a local copy of Enzyme on your machine, which does not need to be committed to git.","category":"page"},{"location":"dev_docs/","page":"For developers","title":"For developers","text":"~/s/Enzyme.jl (master)> julia --project=deps deps/build_local.jl ../path/to/Enzyme","category":"page"},{"location":"dev_docs/#Development-of-Enzyme-and-Enzyme.jl-together-(manual)","page":"For developers","title":"Development of Enzyme and Enzyme.jl together (manual)","text":"","category":"section"},{"location":"dev_docs/","page":"For developers","title":"For developers","text":"Start Julia in your development copy of Enzyme.jl","category":"page"},{"location":"dev_docs/","page":"For developers","title":"For developers","text":"~/s/Enzyme.jl (master)> julia --project=.","category":"page"},{"location":"dev_docs/","page":"For developers","title":"For developers","text":"Then create a development copy of Enzyme_jll and activate it within.","category":"page"},{"location":"dev_docs/","page":"For developers","title":"For developers","text":"julia> using Enzyme_jll\njulia> Enzyme_jll.dev_jll()\n[ Info: Enzyme_jll dev'ed out to ${JULIA_PKG_DEVDIR}/Enzyme_jll with pre-populated override directory\n(Enzyme) pkg> dev Enzyme_jll\nPath `${JULIA_PKG_DEVDIR}/Enzyme_jll` exists and looks like the correct package. Using existing path.","category":"page"},{"location":"dev_docs/","page":"For developers","title":"For developers","text":"After restarting Julia:","category":"page"},{"location":"dev_docs/","page":"For developers","title":"For developers","text":"julia> Enzyme_jll.dev_jll()\njulia> Enzyme_jll.libEnzyme_path\n\"${JULIA_PKG_DEVDIR}/Enzyme_jll/override/lib/LLVMEnzyme-9.so\"","category":"page"},{"location":"dev_docs/","page":"For developers","title":"For developers","text":"On your machine ${JULIA_PKG_DEVDIR} most likely corresponds to ~/.julia/dev. Now we can inspect \"${JULIA_PKG_DEVDIR}/Enzyme_jll/override/lib and see that there is a copy of LLVMEnzyme-9.so, which we can replace with a symbolic link or a copy of a version of Enzyme.","category":"page"},{"location":"dev_docs/#Building-Enzyme-against-Julia's-LLVM.","page":"For developers","title":"Building Enzyme against Julia's LLVM.","text":"","category":"section"},{"location":"dev_docs/","page":"For developers","title":"For developers","text":"Depending on how you installed Julia the LLVM Julia is using will be different.","category":"page"},{"location":"dev_docs/","page":"For developers","title":"For developers","text":"Download from julialang.org (Recommended)\nManual build on your machine\nUses a pre-built Julia from your system vendor (Not recommended)","category":"page"},{"location":"dev_docs/","page":"For developers","title":"For developers","text":"To check what LLVM Julia is using use:","category":"page"},{"location":"dev_docs/","page":"For developers","title":"For developers","text":"julia> Base.libllvm_version_string\n\"9.0.1jl\"","category":"page"},{"location":"dev_docs/","page":"For developers","title":"For developers","text":"If the LLVM version ends in a jl you are likely using the private LLVM.","category":"page"},{"location":"dev_docs/","page":"For developers","title":"For developers","text":"In your source checkout of Enzyme:","category":"page"},{"location":"dev_docs/","page":"For developers","title":"For developers","text":"mkdir build-jl\ncd build-jl","category":"page"},{"location":"dev_docs/#Prebuilt-binary-from-julialang.org","page":"For developers","title":"Prebuilt binary from julialang.org","text":"","category":"section"},{"location":"dev_docs/","page":"For developers","title":"For developers","text":"LLVM_MAJOR_VER=`julia -e \"print(Base.libllvm_version.major)\"`\njulia -e \"using Pkg; pkg\\\"add LLVM_full_jll@${LLVM_MAJOR_VER}\\\"\"\nLLVM_DIR=`julia -e \"using LLVM_full_jll; print(LLVM_full_jll.artifact_dir)\"`\necho \"LLVM_DIR=$LLVM_DIR\"\ncmake ../enzyme/ -G Ninja -DENZYME_EXTERNAL_SHARED_LIB=ON -DLLVM_DIR=${LLVM_DIR} -DLLVM_EXTERNAL_LIT=${LLVM_DIR}/tools/lit/lit.py","category":"page"},{"location":"dev_docs/#Manual-build-of-Julia","page":"For developers","title":"Manual build of Julia","text":"","category":"section"},{"location":"dev_docs/","page":"For developers","title":"For developers","text":"cmake ../enzyme/ -G Ninja -DENZYME_EXTERNAL_SHARED_LIB=ON -DLLVM_DIR=${PATH_TO_BUILDDIR_OF_JULIA}/usr/lib/cmake/llvm/","category":"page"},{"location":"internal_api/#Internal-API","page":"Internal API","title":"Internal API","text":"","category":"section"},{"location":"internal_api/","page":"Internal API","title":"Internal API","text":"note: Note\nThis is the documentation of Enzymes's internal API. The internal API is not subject to semantic versioning and may change at any time and without deprecation.","category":"page"},{"location":"internal_api/","page":"Internal API","title":"Internal API","text":"Modules = [Enzyme.Compiler]\nOrder = [:module, :type, :constant, :macro, :function]","category":"page"},{"location":"internal_api/#Enzyme.Compiler.fspec","page":"Internal API","title":"Enzyme.Compiler.fspec","text":"Create the methodinstance pair, and lookup the primal return type.\n\n\n\n\n\n","category":"function"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"EditURL = \"../../../examples/box.jl\"","category":"page"},{"location":"generated/box/#Enzyme-for-adjoint-tutorial:-Stommel-three-box-ocean-model","page":"Box model","title":"Enzyme for adjoint tutorial: Stommel three-box ocean model","text":"","category":"section"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"The goal of this tutorial is to teach about a specific usage of Enzyme's automatic differentiation capabilities, and will be centered around the Stommel ocean model. This is a nice example to see how powerful Enzyme is, and the ability of it to take a derivative of a complicated function (namely one that has many parts and parameters). This tutorial will focus first on the computations and getting Enzyme running, for those interested a mathematical explanation of the model and what an adjoint variable is will be provided at the end.","category":"page"},{"location":"generated/box/#Brief-model-overview","page":"Box model","title":"Brief model overview","text":"","category":"section"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"The Stommel box model can be viewed as a watered down full ocean model. In our example, we have three boxes (Box One, Box Two, and Box Three) and we model the transport of fluid between them. The full equations of our system are given by:","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"beginaligned\n U = u_0 left rho_2 - left rho_1 + (1 - delta) rho_3 right right \n rho_i = -alpha T_i + beta S_i i = 1 2 3\nendaligned","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"for the transport U and densities rho, and then the time derivatives","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"beginaligned\n dotT_1 = U(T_3 - T_1)V_1 + gamma (T_1^* - T_1 ) dotS_1 = U(S_3 - S_1)V_1 + FW_1V_1 \n dotT_2 = U(T_1 - T_2)V_2 + gamma (T_2^* - T_2 ) dotS_2 = U(S_1 - S_2)V_2 + FW_2V_2 \n dotT_3 = U(T_2 - T_3)V_3 dotS_3 = U(S_2 - S_3)V_3\nendaligned","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"for positive transport, U 0, and","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"beginaligned\n dotT_1 = U(T_2 - T_1)V_1 + gamma (T_1^* - T_1) dotS_1 = U(S_2 - S_1)V_1 + FW_1V_1 \n dotT_2 = U(T_3 - T_2)V_2 + gamma (T_2^* - T_2 ) dotS_2 = U(S_3 - S_2)V_2 + FW_2V_2 \n dotT_3 = U(T_1 - T_3)V_3 dotS_3 = U(S_1 - S_3)V_3\nendaligned","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"for U leq 0. The only force driving our system is a density gradient generated via temperature and salinity differences between the boxes. This makes it a really easy model to play around with! With this in mind, the model is run forward with the steps:","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"Compute densities\nCompute transport\nCompute time derivatives of the box temperatures and salinities\nUpdate the state vector","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"We'll start by going through the model setup step by step, then providing a few test cases with Enzyme.","category":"page"},{"location":"generated/box/#Model-setup","page":"Box model","title":"Model setup","text":"","category":"section"},{"location":"generated/box/#Model-dependencies","page":"Box model","title":"Model dependencies","text":"","category":"section"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"Let's first add the necessary packages to run everything","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"using Enzyme","category":"page"},{"location":"generated/box/#Initialize-constants","page":"Box model","title":"Initialize constants","text":"","category":"section"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"The system equations have quite a few constants that appear, here we initialize them for later use. We'll do this in a Julia way: we have an empty structure that will hold all the parameters, and a function (we'll call this setup) that initializes them. This means that, so long as we don't need to change parameters, we only need to run setup once.","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"struct ModelParameters\n\n # handy to have constants\n day::Float64\n year::Float64\n\n # Information related to the boxes\n boxlength::Vector{Float64} ## Vector with north-south size of each box [cm]\n boxdepth::Vector{Float64} ## \" \" the depth of each box [cm]\n boxwidth::Float64 ## \" \" the width of each box [cm]\n boxarea::Vector{Float64} ## \" \" the area of each box [cm^2]\n boxvol::Vector{Float64} ## \" \" the volume of each box [cm^3]\n\n delta::Float64 ## Constant ratio depth(box1) / (depth(box1) + depth(box3))\n\n # Parameters that appear in the box model equations\n u0::Float64\n alpha::Float64\n beta::Float64\n gamma::Float64\n\n # Coefficient for the Robert filter smoother\n rf_coeff::Float64\n\n # Freshwater forcing\n FW::Vector{Float64}\n\n # Restoring atmospheric temperatures and salinities\n Tstar::Vector{Float64}\n Sstar::Vector{Float64}\n\nend\n\nfunction setup()\n\n blength = [5000.0e5; 1000.0e5; 5000.0e5]\n bdepth = [1.0e5; 5.0e5; 4.0e5]\n\n delta = bdepth[1]/(bdepth[1] + bdepth[3])\n\n bwidth = 4000.0*1e5 ## box width, centimeters\n\n # box areas\n barea = [blength[1]*bwidth;\n blength[2]*bwidth;\n blength[3]*bwidth]\n\n # box volumes\n bvolume = [barea[1]*bdepth[1];\n barea[2]*bdepth[2];\n barea[3]*bdepth[3]]\n\n # parameters that are used to ensure units are in CGS (cent-gram-sec)\n\n day = 3600.0*24.0\n year = day*365.0\n Sv = 1e12 ## one Sverdrup (a unit of ocean transport), 1e6 meters^3/second\n\n # parameters that appear in box model equations\n u0 = 16.0*Sv/0.0004\n alpha = 1668e-7\n beta = 0.7811e-3\n\n gamma = 1/(300*day)\n\n # robert filter coefficient for the smoother part of the timestep\n robert_filter_coeff = 0.25\n\n # freshwater forcing\n FW = [(100/year) * 35.0 * barea[1]; -(100/year) * 35.0 * barea[1]]\n\n # restoring atmospheric temperatures\n Tstar = [22.0; 0.0]\n Sstar = [36.0; 34.0]\n\n structure_with_parameters = ModelParameters(day,\n year,\n blength,\n bdepth,\n bwidth,\n barea,\n bvolume,\n delta,\n u0,\n alpha,\n beta,\n gamma,\n robert_filter_coeff,\n FW,\n Tstar,\n Sstar\n )\n\n return structure_with_parameters\n\nend","category":"page"},{"location":"generated/box/#Define-model-functions","page":"Box model","title":"Define model functions","text":"","category":"section"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"Here we define functions that will calculate quantities used in the forward steps.","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"# function to compute transport\n# Input: rho - the density vector\n# Output: U - transport value\n\nfunction compute_transport(rho, params)\n\n U = params.u0 * (rho[2] - (params.delta * rho[1] + (1 - params.delta)*rho[3]))\n return U\n\nend\n\n# function to compute density\n# Input: state = [T1; T2; T3; S1; S2; S3]\n# Output: rho\n\nfunction compute_density(state, params)\n\n rho = -params.alpha * state[1:3] + params.beta * state[4:6]\n return rho\n\nend\n\n# lastly, a function that takes one step forward\n# Input: state_now = [T1(t), T2(t), ..., S3(t)]\n# state_old = [T1(t-dt), ..., S3(t-dt)]\n# u = transport(t)\n# dt = time step\n# Output: state_new = [T1(t+dt), ..., S3(t+dt)]\n\nfunction compute_update(state_now, state_old, u, params, dt)\n\n dstate_now_dt = zeros(6)\n state_new = zeros(6)\n\n # first computing the time derivatives of the various temperatures and salinities\n if u > 0\n\n dstate_now_dt[1] = u * (state_now[3] - state_now[1]) / params.boxvol[1] + params.gamma * (params.Tstar[1] - state_now[1])\n dstate_now_dt[2] = u * (state_now[1] - state_now[2]) / params.boxvol[2] + params.gamma * (params.Tstar[2] - state_now[2])\n dstate_now_dt[3] = u * (state_now[2] - state_now[3]) / params.boxvol[3]\n\n dstate_now_dt[4] = u * (state_now[6] - state_now[4]) / params.boxvol[1] + params.FW[1] / params.boxvol[1]\n dstate_now_dt[5] = u * (state_now[4] - state_now[5]) / params.boxvol[2] + params.FW[2] / params.boxvol[2]\n dstate_now_dt[6] = u * (state_now[5] - state_now[6]) / params.boxvol[3]\n\n elseif u <= 0\n\n dstate_now_dt[1] = u * (state_now[2] - state_now[1]) / params.boxvol[1] + params.gamma * (params.Tstar[1] - state_now[1])\n dstate_now_dt[2] = u * (state_now[3] - state_now[2]) / params.boxvol[2] + params.gamma * (params.Tstar[2] - state_now[2])\n dstate_now_dt[3] = u * (state_now[1] - state_now[3]) / params.boxvol[3]\n\n dstate_now_dt[4] = u * (state_now[5] - state_now[4]) / params.boxvol[1] + params.FW[1] / params.boxvol[1]\n dstate_now_dt[5] = u * (state_now[6] - state_now[5]) / params.boxvol[2] + params.FW[2] / params.boxvol[2]\n dstate_now_dt[6] = u * (state_now[4] - state_now[6]) / params.boxvol[3]\n\n end\n\n # update fldnew using a version of Euler's method\n state_new .= state_old + 2.0 * dt * dstate_now_dt\n\n return state_new\nend","category":"page"},{"location":"generated/box/#Define-forward-functions","page":"Box model","title":"Define forward functions","text":"","category":"section"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"Finally, we create two functions, the first of which computes and stores all the states of the system, and the second will take just a single step forward.","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"Let's start with the standard forward function. This is just going to be used to store the states at every timestep:","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"function integrate(state_now, state_old, dt, M, parameters)\n\n # Because of the adjoint problem we're setting up, we need to store both the states before\n # and after the Robert filter smoother has been applied\n states_before = [state_old]\n states_after = [state_old]\n\n for t = 1:M\n\n rho = compute_density(state_now, parameters)\n u = compute_transport(rho, parameters)\n state_new = compute_update(state_now, state_old, u, parameters, dt)\n\n # Applying the Robert filter smoother (needed for stability)\n state_new_smoothed = state_now + parameters.rf_coeff * (state_new - 2.0 * state_now + state_old)\n\n push!(states_after, state_new_smoothed)\n push!(states_before, state_new)\n\n # cycle the \"now, new, old\" states\n state_old = state_new_smoothed\n state_now = state_new\n\n end\n\n return states_after, states_before\nend","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"Now, for the purposes of Enzyme, it would be convenient for us to have a function that runs a single step of the model forward rather than the whole integration. This would allow us to save as many of the adjoint variables as we wish when running the adjoint method, although for the example we'll discuss later we technically only need one of them","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"function one_step_forward(state_now, state_old, out_now, out_old, parameters, dt)\n\n state_new_smoothed = zeros(6)\n rho = compute_density(state_now, parameters) ## compute density\n u = compute_transport(rho, parameters) ## compute transport\n state_new = compute_update(state_now, state_old, u, parameters, dt) ## compute new state values\n\n # Robert filter smoother\n state_new_smoothed[:] = state_now + parameters.rf_coeff * (state_new - 2.0 * state_now + state_old)\n\n out_old[:] = state_new_smoothed\n out_now[:] = state_new\n\n return nothing\n\nend","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"One difference to note is that one_step_forward now returns nothing, but is rather a function of both its input and output. Since the output of the function is a vector, we need to have this return nothing for Enzyme to work. Now we can move on to some examples using Enzyme.","category":"page"},{"location":"generated/box/#Example-1:-Simply-using-Enzyme","page":"Box model","title":"Example 1: Simply using Enzyme","text":"","category":"section"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"For the first example let's just compute the gradient of our forward function and examine the output. We'll just run the model for one step, and take a dt of ten days. The initial conditions of the system are given as Tbar and Sbar. We run setup once here, and never have to run it again! (Unless we decide to change a parameter)","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"parameters = setup()\n\nTbar = [20.0; 1.0; 1.0] ## initial temperatures\nSbar = [35.5; 34.5; 34.5] ## initial salinities\n\n# Running the model one step forward\nstates_after_smoother, states_before_smoother = integrate(\n copy([Tbar; Sbar]),\n copy([Tbar; Sbar]),\n 10*parameters.day,\n 1,\n parameters\n)\n\n# Run Enzyme one time on `one_step_forward``\ndstate_now = zeros(6)\ndstate_old = zeros(6)\nout_now = zeros(6); dout_now = ones(6)\nout_old = zeros(6); dout_old = ones(6)\n\nautodiff(Reverse,\n one_step_forward,\n Duplicated([Tbar; Sbar], dstate_now),\n Duplicated([Tbar; Sbar], dstate_old),\n Duplicated(out_now, dout_now),\n Duplicated(out_old, dout_old),\n Const(parameters),\n Const(10*parameters.day)\n)","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"In order to run Enzyme on one_step_forward, we've needed to provide quite a few placeholders, and wrap everything in Duplicated as all components of our function are vectors, not scalars. Let's go through and see what Enzyme did with all of those placeholders.","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"First we can look at what happened to the zero vectors out_now and out_old:","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"@show out_now, out_old","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"Comparing to the results of forward func:","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"@show states_before_smoother[2], states_after_smoother[2]","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"we see that Enzyme has computed and stored exactly the output of the forward step. Next, let's look at dstate_now:","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"@show dstate_now","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"Just a few numbers, but this is what makes AD so nice: Enzyme has exactly computed the derivative of all outputs with respect to the input state_now, evaluated at state_now, and acted with this gradient on what we gave as dout_now (in our case, all ones). Using AD notation for reverse mode, this is","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"overlinetextstate_now = fracpartial textout_nowpartial textstate_nowright_textstate_now overlinetextout_now + fracpartial textout_oldpartial textstate_nowright_textstate_now overlinetextout_old","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"We note here that had we initialized dstate_now and dstate_old as something else, our results will change. Let's multiply them by two and see what happens.","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"dstate_now_new = zeros(6)\ndstate_old_new = zeros(6)\nout_now = zeros(6); dout_now = 2*ones(6)\nout_old = zeros(6); dout_old = 2*ones(6)\nautodiff(Reverse,\n one_step_forward,\n Duplicated([Tbar; Sbar], dstate_now_new),\n Duplicated([Tbar; Sbar], dstate_old_new),\n Duplicated(out_now, dout_now),\n Duplicated(out_old, dout_old),\n Const(parameters),\n Const(10*parameters.day)\n)","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"Now checking dstate_now and dstate_old we see","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"@show dstate_now_new","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"What happened? Enzyme is actually taking the computed gradient and acting on what we give as input to dout_now and dout_old. Checking this, we see","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"@show 2*dstate_now","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"and they match the new results. This exactly matches what we'd expect to happen since we scaled dout_now by two.","category":"page"},{"location":"generated/box/#Example-2:-Full-sensitivity-calculations","page":"Box model","title":"Example 2: Full sensitivity calculations","text":"","category":"section"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"Now we want to use Enzyme for a bit more than just a single derivative. Let's say we'd like to understand how sensitive the final temperature of Box One is to the initial salinity of Box Two. That is, given the function","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"J = (100000)^T cdot mathbfx(t_f)","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"we want Enzyme to calculate the derivative","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"fracpartial Jpartial mathbfx(0)","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"where x(t) is the state of the model at time t. If we think about x(t_f) as solely depending on the initial condition, then this derivative is really","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"fracpartial Jpartial mathbfx(0) = fracpartialpartial mathbfx(0) left( (100000)^T cdot L(ldots(L(mathbfx(0)))) right)","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"with L(x(t)) = x(t + dt), i.e. one forward step. One could expand this derivative with the chain rule (and it would be very complicated), but really this is where Enzyme comes in. Each run of autodiff on our forward function is one piece of this big chain rule done for us! We also note that the chain rule goes from the outside in, so we start with the derivative of the forward function at the final state, and work backwards until the initial state. To get Enzyme to do this, we complete the following steps:","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"Run the forward model and store outputs (in a real ocean model this wouldn't be feasible and we'd need to use checkpointing)\nCompute the initial derivative from the final state\nUse Enzyme to work backwards until we reach the desired derivative.","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"For simplicity we define a function that takes completes our AD steps","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"function compute_adjoint_values(states_before_smoother, states_after_smoother, M, parameters)\n\n dout_now = [0.0;0.0;0.0;0.0;0.0;0.0]\n dout_old = [1.0;0.0;0.0;0.0;0.0;0.0]\n\n for j = M:-1:1\n\n dstate_now = zeros(6)\n dstate_old = zeros(6)\n\n autodiff(Reverse,\n one_step_forward,\n Duplicated(states_before_smoother[j], dstate_now),\n Duplicated(states_after_smoother[j], dstate_old),\n Duplicated(zeros(6), dout_now),\n Duplicated(zeros(6), dout_old),\n Const(parameters),\n Const(10*parameters.day)\n )\n\n if j == 1\n return dstate_now, dstate_old\n end\n\n dout_now = copy(dstate_now)\n dout_old = copy(dstate_old)\n\n end\n\nend","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"First we integrate the model forward:","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"M = 10000 ## Total number of forward steps to take\nTbar = [20.0; 1.0; 1.0] ## initial temperatures\nSbar = [35.5; 34.5; 34.5] ## initial salinities\n\nstates_after_smoother, states_before_smoother = integrate(\n copy([Tbar; Sbar]),\n copy([Tbar; Sbar]),\n 10*parameters.day,\n M,\n parameters\n)","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"Next, we pass all of our states to the AD function to get back to the desired derivative:","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"dstate_now, dstate_old = compute_adjoint_values(\n states_before_smoother,\n states_after_smoother,\n M,\n parameters\n)","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"And we're done! We were interested in sensitivity to the initial salinity of box two, which will live in what we've called dstate_old. Checking this value we see","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"@show dstate_old[5]","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"As it stands this is just a number, but a good check that Enzyme has computed what we want is to approximate the derivative with a Taylor series. Specifically,","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"J(mathbfx(0) + varepsilon) approx J(mathbfx(0)) +\nvarepsilon fracpartial Jpartial mathbfx(0)","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"and a simple rearrangement yields","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"fracpartial Jpartial mathbfx(0) approx\nfracJ(mathbfx(0) + varepsilon) - J(mathbfx(0))varepsilon","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"Hopefully we see that the analytical values converge close to the one we found with Enzyme:","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"# unperturbed final state\nuse_to_check = states_after_smoother[M+1]\n\n# a loop to compute the perturbed final states\ndiffs = []\nstep_sizes = [1e-1, 1e-2, 1e-3, 1e-4, 1e-5, 1e-6, 1e-7, 1e-8, 1e-9, 1e-10]\nfor eps in step_sizes\n\n state_new_smoothed = zeros(6)\n\n initial_temperature = [20.0; 1.0; 1.0]\n perturbed_initial_salinity = [35.5; 34.5; 34.5] + [0.0; eps; 0.0]\n\n state_old = [initial_temperature; perturbed_initial_salinity]\n state_now = [20.0; 1.0; 1.0; 35.5; 34.5; 34.5]\n\n for t = 1:M\n\n rho = compute_density(state_now, parameters)\n u = compute_transport(rho, parameters)\n state_new = compute_update(state_now, state_old, u, parameters, 10*parameters.day)\n\n state_new_smoothed[:] = state_now + parameters.rf_coeff * (state_new - 2.0 * state_now + state_old)\n\n state_old = state_new_smoothed\n state_now = state_new\n\n end\n\n push!(diffs, (state_old[1] - use_to_check[1])/eps)\n\nend","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"Then checking what we found the derivative to be analytically:","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"@show diffs","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"which comes very close to our calculated value. We can go further and check the percent difference to see","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"@show abs.(diffs .- dstate_old[5])./dstate_old[5]","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"and we get down to a percent difference on the order of 1e^-5, showing Enzyme calculated the correct derivative. Success!","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"This page was generated using Literate.jl.","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"EditURL = \"../../../examples/custom_rule.jl\"","category":"page"},{"location":"generated/custom_rule/#Enzyme-custom-rules-tutorial","page":"Custom rules","title":"Enzyme custom rules tutorial","text":"","category":"section"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"note: More Examples\nThe tutorial below focuses on a simple setting to illustrate the basic concepts of writing custom rules. For more complex custom rules beyond the scope of this tutorial, you may take inspiration from the following in-the-wild examples:Enzyme internal rules\nKernelAbstractions.jl\nLinearSolve.jl\nNNlib.jl","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"The goal of this tutorial is to give a simple example of defining a custom rule with Enzyme. Specifically, our goal will be to write custom rules for the following function f:","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"function f(y, x)\n y .= x.^2\n return sum(y)\nend","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"Our function f populates its first input y with the element-wise square of x. In addition, it returns sum(y) as output. What a sneaky function!","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"In this case, Enzyme can differentiate through f automatically. For example, using forward mode:","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"using Enzyme\nx = [3.0, 1.0]\ndx = [1.0, 0.0]\ny = [0.0, 0.0]\ndy = [0.0, 0.0]\n\ng(y, x) = f(y, x)^2 # function to differentiate\n\n@show autodiff(Forward, g, Duplicated(y, dy), Duplicated(x, dx)) # derivative of g w.r.t. x[1]\n@show dy; # derivative of y w.r.t. x[1] when g is run\nnothing #hide","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"(See the AutoDiff API tutorial for more information on using autodiff.)","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"But there may be special cases where we need to write a custom rule to help Enzyme out. Let's see how to write a custom rule for f!","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"warning: Don't use custom rules unnecessarily!\nEnzyme can efficiently handle a wide range of constructs, and so a custom rule should only be required in certain special cases. For example, a function may make a foreign call that Enzyme cannot differentiate, or we may have higher-level mathematical knowledge that enables us to write a more efficient rule. Even in these cases, try to make your custom rule encapsulate the minimum possible construct that Enzyme cannot differentiate, rather than expanding the scope of the rule unnecessarily. For pedagogical purposes, we will disregard this principle here and go ahead and write a custom rule for f :)","category":"page"},{"location":"generated/custom_rule/#Defining-our-first-rule","page":"Custom rules","title":"Defining our first rule","text":"","category":"section"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"First, we import the functions EnzymeRules.forward, EnzymeRules.augmented_primal, and EnzymeRules.reverse. We need to overload forward in order to define a custom forward rule, and we need to overload augmented_primal and reverse in order to define a custom reverse rule.","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"import .EnzymeRules: forward, reverse, augmented_primal\nusing .EnzymeRules","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"In this section, we write a simple forward rule to start out:","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"function forward(config::FwdConfig, func::Const{typeof(f)}, ::Type{<:Duplicated}, y::Duplicated, x::Duplicated)\n println(\"Using custom rule!\")\n ret = func.val(y.val, x.val)\n y.dval .= 2 .* x.val .* x.dval\n return Duplicated(ret, sum(y.dval))\nend","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"In the signature of our rule, we have made use of Enzyme's activity annotations. Let's break down each one:","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"the EnzymeRules.FwdConfig configuration passes certain compile-time information about differentiation procedure (the width, and if we're using runtime activity),\nthe Const annotation on f indicates that we accept a function f that does not have a derivative component, which makes sense since f is not a closure with data that could be differentiated.\nthe Duplicated annotation given in the second argument annotates the return value of f. This means that our forward function should return an output of type Duplicated, containing the original output sum(y) and its derivative.\nthe Duplicated annotations for x and y mean that our forward function handles inputs x and y which have been marked as Duplicated. We should update their shadows with their derivative contributions.","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"In the logic of our forward function, we run the original function, populate y.dval (the shadow of y), and finally return a Duplicated for the output as promised. Let's see our rule in action! With the same setup as before:","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"x = [3.0, 1.0]\ndx = [1.0, 0.0]\ny = [0.0, 0.0]\ndy = [0.0, 0.0]\n\ng(y, x) = f(y, x)^2 # function to differentiate\n\n@show autodiff(Forward, g, Duplicated(y, dy), Duplicated(x, dx)) # derivative of g w.r.t. x[1]\n@show dy; # derivative of y w.r.t. x[1] when g is run\nnothing #hide","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"We see that our custom forward rule has been triggered and gives the same answer as before.","category":"page"},{"location":"generated/custom_rule/#Handling-more-activities","page":"Custom rules","title":"Handling more activities","text":"","category":"section"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"Our custom rule applies for the specific set of activities that are annotated for f in the above autodiff call. However, Enzyme has a number of other annotations. Let us consider a particular example, where the output has a DuplicatedNoNeed annotation. This means we are only interested in its derivative, not its value. To squeeze out the last drop of performance, the below rule avoids computing the output of the original function and just computes its derivative.","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"function forward(config, func::Const{typeof(f)}, ::Type{<:DuplicatedNoNeed}, y::Duplicated, x::Duplicated)\n println(\"Using custom rule with DuplicatedNoNeed output.\")\n y.val .= x.val.^2\n y.dval .= 2 .* x.val .* x.dval\n return sum(y.dval)\nend","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"Our rule is triggered, for example, when we call autodiff directly on f, as the return value's derivative isn't needed:","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"x = [3.0, 1.0]\ndx = [1.0, 0.0]\ny = [0.0, 0.0]\ndy = [0.0, 0.0]\n\n@show autodiff(Forward, f, Duplicated(y, dy), Duplicated(x, dx)) # derivative of f w.r.t. x[1]\n@show dy; # derivative of y w.r.t. x[1] when f is run\nnothing #hide","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"note: Custom rule dispatch\nWhen multiple custom rules for a function are defined, the correct rule is chosen using Julia's multiple dispatch. In particular, it is important to understand that the custom rule does not determine the activities of the inputs and the return value: rather, Enzyme decides the activity annotations independently, and then dispatches to the custom rule handling the activities, if one exists. If a custom rule is specified for the correct function/argument types, but not the correct activity annotation, a runtime error will be thrown alerting the user to the missing activity rule rather than silently ignoring the rule.\"","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"Finally, it may be that either x, y, or the return value are marked as Const, in which case we can simply return the original result. However, Enzyme also may determine the return is not differentiable and also not needed for other computations, in which case we should simply return nothing.","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"We can in fact handle this case, along with the previous two cases, all together in a single rule by leveraging utility functions EnzymeRules.needs_primal and EnzymeRules.needs_shadow, which return true if the original return or the derivative is needed to be returned, respectively:","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"Base.delete_method.(methods(forward, (Const{typeof(f)}, Vararg{Any}))) # delete our old rules\n\nfunction forward(config, func::Const{typeof(f)}, RT::Type{<:Union{Const, DuplicatedNoNeed, Duplicated}},\n y::Union{Const, Duplicated}, x::Union{Const, Duplicated})\n println(\"Using our general custom rule!\")\n y.val .= x.val.^2\n if !(x isa Const) && !(y isa Const)\n y.dval .= 2 .* x.val .* x.dval\n elseif !(y isa Const)\n make_zero!(y.dval)\n end\n dret = !(y isa Const) ? sum(y.dval) : zero(eltype(y.val))\n if needs_primal(config) && needs_shadow(config)\n return Duplicated(sum(y.val), dret)\n elseif needs_primal(config)\n return sum(y.val)\n elseif needs_shadow(config)\n return dret\n else\n return nothing\n end\nend","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"Let's try out our rule:","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"x = [3.0, 1.0]\ndx = [1.0, 0.0]\ny = [0.0, 0.0]\ndy = [0.0, 0.0]\n\ng(y, x) = f(y, x)^2 # function to differentiate\n\n@show autodiff(Forward, g, Duplicated(y, dy), Duplicated(x, dx)) # derivative of g w.r.t. x[1]\n@show autodiff(Forward, g, Const(y), Duplicated(x, dx)) # derivative of g w.r.t. x[1], with y annotated Const\n@show autodiff(Forward, g, Const(y), Const(x)); # derivative of g w.r.t. x[1], with x and y annotated Const\nnothing #hide","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"Note that there are also exist batched duplicated annotations for forward mode, namely BatchDuplicated and BatchDuplicatedNoNeed, which are not covered in this tutorial.","category":"page"},{"location":"generated/custom_rule/#Defining-a-reverse-mode-rule","page":"Custom rules","title":"Defining a reverse-mode rule","text":"","category":"section"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"Let's look at how to write a simple reverse-mode rule! First, we write a method for EnzymeRules.augmented_primal:","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"function augmented_primal(config::RevConfigWidth{1}, func::Const{typeof(f)}, ::Type{<:Active},\n y::Duplicated, x::Duplicated)\n println(\"In custom augmented primal rule.\")\n # Compute primal\n if needs_primal(config)\n primal = func.val(y.val, x.val)\n else\n y.val .= x.val.^2 # y still needs to be mutated even if primal not needed!\n primal = nothing\n end\n # Save x in tape if x will be overwritten\n if overwritten(config)[3]\n tape = copy(x.val)\n else\n tape = nothing\n end\n # Return an AugmentedReturn object with shadow = nothing\n return AugmentedReturn(primal, nothing, tape)\nend","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"Let's unpack our signature for augmented_primal :","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"We accepted a EnzymeRules.RevConfig object with a specified width of 1, which means that our rule does not support batched reverse mode.\nWe annotated f with Const as usual.\nWe dispatched on an Active annotation for the return value. This is a special annotation for scalar values, such as our return value, that indicates that that we care about the value's derivative but we need not explicitly allocate a mutable shadow since it is a scalar value.\nWe annotated x and y with Duplicated, similar to our first simple forward rule.","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"Now, let's unpack the body of our augmented_primal rule:","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"We checked if the config requires the primal. If not, we need not compute the return value, but we make sure to mutate y in all cases.\nWe checked if x could possibly be overwritten using the Overwritten attribute of EnzymeRules.RevConfig. If so, we save the elements of x on the tape of the returned EnzymeRules.AugmentedReturn object.\nWe return a shadow of nothing since the return value is Active and hence does not need a shadow.","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"Now, we write a method for EnzymeRules.reverse:","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"function reverse(config::RevConfigWidth{1}, func::Const{typeof(f)}, dret::Active, tape,\n y::Duplicated, x::Duplicated)\n println(\"In custom reverse rule.\")\n # retrieve x value, either from original x or from tape if x may have been overwritten.\n xval = overwritten(config)[3] ? tape : x.val\n # accumulate dret into x's shadow. don't assign!\n x.dval .+= 2 .* xval .* dret.val\n # also accumulate any derivative in y's shadow into x's shadow.\n x.dval .+= 2 .* xval .* y.dval\n make_zero!(y.dval)\n return (nothing, nothing)\nend","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"Let's make a few observations about our reverse rule:","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"The activities used in the signature correspond to what we used for augmented_primal.\nHowever, for Active return types such as in this case, we now receive an instance dret of Active for the return type, not just a type annotation, which stores the derivative value for ret (not the original return value!). For the other annotations (e.g. Duplicated), we still receive only the type. In that case, if necessary a reference to the shadow of the output should be placed on the tape in augmented_primal.\nUsing dret.val and y.dval, we accumulate the backpropagated derivatives for x into its shadow x.dval. Note that we have to accumulate from both y.dval and dret.val. This is because in reverse-mode AD we have to sum up the derivatives from all uses: if y was read after our function, we need to consider derivatives from that use as well.\nWe zero-out y's shadow. This is because y is overwritten within f, so there is no derivative w.r.t. to the y that was originally inputted.\nFinally, since all derivatives are accumulated in place (in the shadows of the Duplicated arguments), these derivatives must not be communicated via the return value. Hence, we return (nothing, nothing). If, instead, one of our arguments was annotated as Active, we would have to provide its derivative at the corresponding index in the tuple returned.","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"Finally, let's see our reverse rule in action!","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"x = [3.0, 1.0]\ndx = [0.0, 0.0]\ny = [0.0, 0.0]\ndy = [0.0, 0.0]\n\ng(y, x) = f(y, x)^2\n\nautodiff(Reverse, g, Duplicated(y, dy), Duplicated(x, dx))\n@show dx # derivative of g w.r.t. x\n@show dy; # derivative of g w.r.t. y\nnothing #hide","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"Let's also try a function which mutates x after running f, and also uses y directly rather than only ret after running f (but ultimately gives the same result as above):","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"function h(y, x)\n ret = f(y, x)\n x .= x.^2\n return ret * sum(y)\nend\n\nx = [3.0, 1.0]\ny = [0.0, 0.0]\nmake_zero!(dx)\nmake_zero!(dy)\n\nautodiff(Reverse, h, Duplicated(y, dy), Duplicated(x, dx))\n@show dx # derivative of h w.r.t. x\n@show dy; # derivative of h w.r.t. y\nnothing #hide","category":"page"},{"location":"generated/custom_rule/#Marking-functions-inactive","page":"Custom rules","title":"Marking functions inactive","text":"","category":"section"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"If we want to tell Enzyme that the function call does not affect the differentiation result in any form (i.e. not by side effects or through its return values), we can simply use EnzymeRules.inactive. So long as there exists a matching dispatch to EnzymeRules.inactive, the function will be considered inactive. For example:","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"printhi() = println(\"Hi!\")\nEnzymeRules.inactive(::typeof(printhi), args...) = nothing\n\nfunction k(x)\n printhi()\n return x^2\nend\n\nautodiff(Forward, k, Duplicated(2.0, 1.0))","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"Or for a case where we incorrectly mark a function inactive:","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"double(x) = 2*x\nEnzymeRules.inactive(::typeof(double), args...) = nothing\n\nautodiff(Forward, x -> x + double(x), Duplicated(2.0, 1.0)) # mathematically should be 3.0, inactive rule causes it to be 1.0","category":"page"},{"location":"generated/custom_rule/#Testing-our-rules","page":"Custom rules","title":"Testing our rules","text":"","category":"section"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"We can test our rules using finite differences using EnzymeTestUtils.test_forward and EnzymeTestUtils.test_reverse.","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"using EnzymeTestUtils, Test\n\n@testset \"f rules\" begin\n @testset \"forward\" begin\n @testset for RT in (Const, DuplicatedNoNeed, Duplicated),\n Tx in (Const, Duplicated),\n Ty in (Const, Duplicated)\n\n x = [3.0, 1.0]\n y = [0.0, 0.0]\n test_forward(g, RT, (x, Tx), (y, Ty))\n end\n end\n @testset \"reverse\" begin\n @testset for RT in (Active,),\n Tx in (Duplicated,),\n Ty in (Duplicated,),\n fun in (g, h)\n\n x = [3.0, 1.0]\n y = [0.0, 0.0]\n test_reverse(fun, RT, (x, Tx), (y, Ty))\n end\n end\nend","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"In any package that implements Enzyme rules using EnzymeRules, it is recommended to add EnzymeTestUtils as a test dependency to test the rules.","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"This page was generated using Literate.jl.","category":"page"},{"location":"","page":"Home","title":"Home","text":"CurrentModule = Enzyme\nDocTestSetup = quote\n using Enzyme\nend","category":"page"},{"location":"#Enzyme","page":"Home","title":"Enzyme","text":"","category":"section"},{"location":"","page":"Home","title":"Home","text":"Documentation for Enzyme.jl, the Julia bindings for Enzyme.","category":"page"},{"location":"","page":"Home","title":"Home","text":"Enzyme performs automatic differentiation (AD) of statically analyzable LLVM. It is highly-efficient and its ability to perform AD on optimized code allows Enzyme to meet or exceed the performance of state-of-the-art AD tools.","category":"page"},{"location":"#Getting-started","page":"Home","title":"Getting started","text":"","category":"section"},{"location":"","page":"Home","title":"Home","text":"Enzyme.jl can be installed in the usual way Julia packages are installed:","category":"page"},{"location":"","page":"Home","title":"Home","text":"] add Enzyme","category":"page"},{"location":"","page":"Home","title":"Home","text":"The Enzyme binary dependencies will be installed automatically via Julia's binary artifact system.","category":"page"},{"location":"","page":"Home","title":"Home","text":"The Enzyme.jl API revolves around the function autodiff. For some common operations, Enzyme additionally wraps autodiff in several convenience functions; e.g., gradient and jacobian.","category":"page"},{"location":"","page":"Home","title":"Home","text":"The tutorial below covers the basic usage of these functions. For a complete overview of Enzyme's functionality, see the API reference documentation. Also see Implementing pullbacks on how to implement back-propagation for functions with non-scalar results.","category":"page"},{"location":"","page":"Home","title":"Home","text":"We will try a few things with the following functions:","category":"page"},{"location":"","page":"Home","title":"Home","text":"julia> rosenbrock(x, y) = (1.0 - x)^2 + 100.0 * (y - x^2)^2\nrosenbrock (generic function with 1 method)\n\njulia> rosenbrock_inp(x) = (1.0 - x[1])^2 + 100.0 * (x[2] - x[1]^2)^2\nrosenbrock_inp (generic function with 1 method)","category":"page"},{"location":"#Reverse-mode","page":"Home","title":"Reverse mode","text":"","category":"section"},{"location":"","page":"Home","title":"Home","text":"The return value of reverse mode autodiff is a tuple that contains as a first value the derivative value of the active inputs and optionally the primal return value.","category":"page"},{"location":"","page":"Home","title":"Home","text":"julia> autodiff(Reverse, rosenbrock, Active, Active(1.0), Active(2.0))\n((-400.0, 200.0),)\n\njulia> autodiff(ReverseWithPrimal, rosenbrock, Active, Active(1.0), Active(2.0))\n((-400.0, 200.0), 100.0)","category":"page"},{"location":"","page":"Home","title":"Home","text":"julia> x = [1.0, 2.0]\n2-element Vector{Float64}:\n 1.0\n 2.0\n\njulia> dx = [0.0, 0.0]\n2-element Vector{Float64}:\n 0.0\n 0.0\n\njulia> autodiff(Reverse, rosenbrock_inp, Active, Duplicated(x, dx))\n((nothing,),)\n\njulia> dx\n2-element Vector{Float64}:\n -400.0\n 200.0","category":"page"},{"location":"","page":"Home","title":"Home","text":"Both the inplace and \"normal\" variant return the gradient. The difference is that with Active the gradient is returned and with Duplicated the gradient is accumulated in place.","category":"page"},{"location":"#Forward-mode","page":"Home","title":"Forward mode","text":"","category":"section"},{"location":"","page":"Home","title":"Home","text":"The return value when using ForwardWithPrimal is a tuple containing as the first value the derivative return value and as the second value the original value.","category":"page"},{"location":"","page":"Home","title":"Home","text":"The return value when using Forward is a single-element tuple containing the derivative.","category":"page"},{"location":"","page":"Home","title":"Home","text":"In forward mode Duplicated(x, 0.0) is equivalent to Const(x), except that we can perform more optimizations for Const.","category":"page"},{"location":"","page":"Home","title":"Home","text":"julia> autodiff(ForwardWithPrimal, rosenbrock, Const(1.0), Duplicated(3.0, 1.0))\n(400.0, 400.0)\n\njulia> autodiff(Forward, rosenbrock, Const(1.0), Duplicated(3.0, 1.0))\n(400.0,)\n\njulia> autodiff(ForwardWithPrimal, rosenbrock, Duplicated(1.0, 1.0), Const(3.0))\n(-800.0, 400.0)\n\njulia> autodiff(Forward, rosenbrock, Duplicated(1.0, 1.0), Const(3.0))\n(-800.0,)","category":"page"},{"location":"","page":"Home","title":"Home","text":"Of note, when we seed both arguments at once the tangent return is the sum of both.","category":"page"},{"location":"","page":"Home","title":"Home","text":"julia> autodiff(ForwardWithPrimal, rosenbrock, Duplicated(1.0, 1.0), Duplicated(3.0, 1.0))\n(-400.0, 400.0)\n\njulia> autodiff(Forward, rosenbrock, Duplicated(1.0, 1.0), Duplicated(3.0, 1.0))\n(-400.0,)","category":"page"},{"location":"","page":"Home","title":"Home","text":"We can also use forward mode with our inplace method.","category":"page"},{"location":"","page":"Home","title":"Home","text":"julia> x = [1.0, 3.0]\n2-element Vector{Float64}:\n 1.0\n 3.0\n\njulia> dx = [1.0, 1.0]\n2-element Vector{Float64}:\n 1.0\n 1.0\n\njulia> autodiff(ForwardWithPrimal, rosenbrock_inp, Duplicated, Duplicated(x, dx))\n(-400.0, 400.0)","category":"page"},{"location":"","page":"Home","title":"Home","text":"Note the seeding through dx.","category":"page"},{"location":"#Vector-forward-mode","page":"Home","title":"Vector forward mode","text":"","category":"section"},{"location":"","page":"Home","title":"Home","text":"We can also use vector mode to calculate both derivatives at once.","category":"page"},{"location":"","page":"Home","title":"Home","text":"julia> autodiff(ForwardWithPrimal, rosenbrock, BatchDuplicated(1.0, (1.0, 0.0)), BatchDuplicated(3.0, (0.0, 1.0)))\n((var\"1\" = -800.0, var\"2\" = 400.0), 400.0)\n\njulia> x = [1.0, 3.0]\n2-element Vector{Float64}:\n 1.0\n 3.0\n\njulia> dx_1 = [1.0, 0.0]; dx_2 = [0.0, 1.0];\n\njulia> autodiff(ForwardWithPrimal, rosenbrock_inp, BatchDuplicated(x, (dx_1, dx_2)))\n((var\"1\" = -800.0, var\"2\" = 400.0), 400.0)","category":"page"},{"location":"#Gradient-Convenience-functions","page":"Home","title":"Gradient Convenience functions","text":"","category":"section"},{"location":"","page":"Home","title":"Home","text":"note: Note\nWhile the convenience functions discussed below use autodiff internally, they are generally more limited in their functionality. Beyond that, these convenience functions may also come with performance penalties; especially if one makes a closure of a multi-argument function instead of calling the appropriate multi-argument autodiff function directly.","category":"page"},{"location":"","page":"Home","title":"Home","text":"Key convenience functions for common derivative computations are gradient (and its inplace variant gradient!). Like autodiff, the mode (forward or reverse) is determined by the first argument.","category":"page"},{"location":"","page":"Home","title":"Home","text":"The functions gradient and gradient! compute the gradient of function with vector input and scalar return.","category":"page"},{"location":"","page":"Home","title":"Home","text":"Gradient functions take a mode as the first argument. If the mode is Reverse or Forward, the return type is a tuple of gradients of each argument. If the mode is ReverseWithPrimal or ForwardWithPrimal, the return type is a named tuple containing both the derivatives and the original return result.","category":"page"},{"location":"","page":"Home","title":"Home","text":"julia> gradient(Reverse, rosenbrock_inp, [1.0, 2.0])\n([-400.0, 200.0],)\n\njulia> gradient(ReverseWithPrimal, rosenbrock_inp, [1.0, 2.0])\n(derivs = ([-400.0, 200.0],), val = 100.0)\n\njulia> # inplace variant\n dx = [0.0, 0.0];\n gradient!(Reverse, dx, rosenbrock_inp, [1.0, 2.0])\n([-400.0, 200.0],)\n\njulia> dx\n2-element Vector{Float64}:\n -400.0\n 200.0\n\njulia> gradient(Forward, rosenbrock_inp, [1.0, 2.0])\n([-400.0, 200.0],)\n\njulia> gradient(ForwardWithPrimal, rosenbrock_inp, [1.0, 2.0])\n(derivs = ([-400.0, 200.0],), val = 100.0)\n\njulia> # in forward mode, we can also optionally pass a chunk size\n # to specify the number of derivatives computed simulateneously\n # using vector forward mode\n gradient(Forward, rosenbrock_inp, [1.0, 2.0]; chunk=Val(1))\n([-400.0, 200.0],)","category":"page"},{"location":"#Jacobian-Convenience-functions","page":"Home","title":"Jacobian Convenience functions","text":"","category":"section"},{"location":"","page":"Home","title":"Home","text":"The function jacobian computes the Jacobian of a function vector input and vector return. Like autodiff and gradient, the mode (forward or reverse) is determined by the first argument.","category":"page"},{"location":"","page":"Home","title":"Home","text":"Again like gradient, if the mode is Reverse or Forward, the return type is a tuple of jacobians of each argument. If the mode is ReverseWithPrimal or ForwardWithPrimal, the return type is a named tuple containing both the derivatives and the original return result.","category":"page"},{"location":"","page":"Home","title":"Home","text":"Both forward and reverse modes take an optional chunk size to compute several derivatives simultaneously using vector mode, and reverse mode optionally takes n_outs which describes the shape of the output value.","category":"page"},{"location":"","page":"Home","title":"Home","text":"julia> foo(x) = [rosenbrock_inp(x), prod(x)];\n\njulia> jacobian(Reverse, foo, [1.0, 2.0]) \n([-400.0 200.0; 2.0 1.0],)\n\njulia> jacobian(ReverseWithPrimal, foo, [1.0, 2.0]) \n(derivs = ([-400.0 200.0; 2.0 1.0],), val = [100.0, 2.0])\n\njulia> jacobian(Reverse, foo, [1.0, 2.0]; chunk=Val(2)) \n([-400.0 200.0; 2.0 1.0],)\n\njulia> jacobian(Reverse, foo, [1.0, 2.0]; chunk=Val(2), n_outs=Val((2,)))\n([-400.0 200.0; 2.0 1.0],)\n\njulia> jacobian(Forward, foo, [1.0, 2.0])\n([-400.0 200.0; 2.0 1.0],)\n\njulia> jacobian(Forward, foo, [1.0, 2.0], chunk=Val(2))\n([-400.0 200.0; 2.0 1.0],)","category":"page"},{"location":"#Hessian-Vector-Product-Convenience-functions","page":"Home","title":"Hessian Vector Product Convenience functions","text":"","category":"section"},{"location":"","page":"Home","title":"Home","text":"Enzyme provides convenience functions for second-order derivative computations, like hvp to compute Hessian vector products. Mathematically, this computes H(x) v, where H is the hessian operator.","category":"page"},{"location":"","page":"Home","title":"Home","text":"Unlike autodiff and gradient, a mode is not specified. Here, Enzyme will choose to perform forward over reverse mode (generally the fastest for this type of operation).","category":"page"},{"location":"","page":"Home","title":"Home","text":"julia> f(x) = sin(x[1] * x[2]);\n\njulia> hvp(f, [2.0, 3.0], [5.0, 2.7])\n2-element Vector{Float64}:\n 19.69268826373025\n 16.201003759768003","category":"page"},{"location":"","page":"Home","title":"Home","text":"Enzyme also provides an in-place variant which will store the hessian vector product in a pre-allocated array (this will, however, still allocate another array for storing an intermediate gradient).","category":"page"},{"location":"","page":"Home","title":"Home","text":"julia> f(x) = sin(x[1] * x[2])\nf (generic function with 1 method)\n\njulia> res = Vector{Float64}(undef, 2);\n\njulia> hvp!(res, f, [2.0, 3.0], [5.0, 2.7]);\n\njulia> res\n2-element Vector{Float64}:\n 19.69268826373025\n 16.201003759768003","category":"page"},{"location":"","page":"Home","title":"Home","text":"Finally. Enzyme provides a second in-place variant which simultaneously computes both the hessian vector product, and the gradient. This function uses no additional allocation, and is much more efficient than separately computing the hvp and the gradient.","category":"page"},{"location":"","page":"Home","title":"Home","text":"julia> f(x) = sin(x[1] * x[2]);\n\njulia> res = Vector{Float64}(undef, 2);\n\njulia> grad = Vector{Float64}(undef, 2);\n\njulia> hvp_and_gradient!(res, grad, f, [2.0, 3.0], [5.0, 2.7])\n\njulia> res\n2-element Vector{Float64}:\n 19.69268826373025\n 16.201003759768003\n\njulia> grad\n2-element Vector{Float64}:\n 2.880510859951098\n 1.920340573300732","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"CurrentModule = Enzyme\nDocTestSetup = quote\n using Enzyme\nend","category":"page"},{"location":"faq/#Frequently-asked-questions","page":"FAQ","title":"Frequently asked questions","text":"","category":"section"},{"location":"faq/#Implementing-pullbacks","page":"FAQ","title":"Implementing pullbacks","text":"","category":"section"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"In combined reverse mode, Enzyme's autodiff function can only handle functions with scalar output (this is not true for split reverse mode, aka autodiff_thunk). To implement pullbacks (back-propagation of gradients/tangents) for array-valued functions, use a mutating function that returns nothing and stores its result in one of the arguments, which must be passed wrapped in a Duplicated. Regardless of AD mode, this mutating function will be much more efficient anyway than one which allocates the output.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Given a function mymul! that performs the equivalent of R = A * B for matrices A and B, and given a gradient (tangent) ∂z_∂R, we can compute ∂z_∂A and ∂z_∂B like this:","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"using Enzyme, Random\n\nfunction mymul!(R, A, B)\n @assert axes(A,2) == axes(B,1)\n @inbounds @simd for i in eachindex(R)\n R[i] = 0\n end\n @inbounds for j in axes(B, 2), i in axes(A, 1)\n @inbounds @simd for k in axes(A,2)\n R[i,j] += A[i,k] * B[k,j]\n end\n end\n nothing\nend\n\nRandom.seed!(1234)\nA = rand(5, 3)\nB = rand(3, 7)\n\nR = zeros(size(A,1), size(B,2))\n∂z_∂R = rand(size(R)...) # Some gradient/tangent passed to us\n∂z_∂R0 = copyto!(similar(∂z_∂R), ∂z_∂R) # exact copy for comparison\n\n∂z_∂A = zero(A)\n∂z_∂B = zero(B)\n\nEnzyme.autodiff(Reverse, mymul!, Const, Duplicated(R, ∂z_∂R), Duplicated(A, ∂z_∂A), Duplicated(B, ∂z_∂B))","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Now we have:","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"R ≈ A * B &&\n∂z_∂A ≈ ∂z_∂R0 * B' && # equivalent to Zygote.pullback(*, A, B)[2](∂z_∂R)[1]\n∂z_∂B ≈ A' * ∂z_∂R0 # equivalent to Zygote.pullback(*, A, B)[2](∂z_∂R)[2]","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Note that the result of the backpropagation is added to ∂z_∂A and ∂z_∂B, they act as accumulators for gradient information.","category":"page"},{"location":"faq/#Identical-types-in-Duplicated-/-Memory-Layout","page":"FAQ","title":"Identical types in Duplicated / Memory Layout","text":"","category":"section"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Enzyme checks that x and ∂f_∂x have the same types when constructing objects of type Duplicated, DuplicatedNoNeed, BatchDuplicated, etc. This is not a mathematical or practical requirement within Enzyme, but rather a guardrail to prevent user error. The memory locations of the shadow ∂f_∂x can only be accessed in the derivative function ∂f if the corresponding memory locations of the variable x are accessed by the function f. Imposing that the variable x and shadow ∂f_∂x have the same type is a heuristic way to ensure that they have the same data layout. This helps prevent some user errors, for instance when the provided shadow cannot be accessed at the relevant memory locations.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"In some ways, type equality is too strict: two different types can have the same data layout. For instance, a vector and a view of a matrix column are arranged identically in memory. But in other ways it is not strict enough. Suppose you have a function f(x) = x[7]. If you call Enzyme.autodiff(Reverse, f, Duplicated(ones(10), ones(1)), the type check alone will not be sufficient. Since the original code accesses x[7], the derivative code will try to set ∂f_∂x[7]. The length is not encoded in the type, so Julia cannot provide a high-level error before running autodiff, and the user may end up with a segfault (or other memory error) when running the generated derivative code. Another typical example is sparse arrays, for which the sparsity pattern of x and ∂f_∂x should be identical.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"To make sure that ∂f_∂x has the right data layout, create it with ∂f_∂x = Enzyme.make_zero(x).","category":"page"},{"location":"faq/#Circumventing-Duplicated-Restrictions-/-Advanced-Memory-Layout","page":"FAQ","title":"Circumventing Duplicated Restrictions / Advanced Memory Layout","text":"","category":"section"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Advanced users may leverage Enzyme's memory semantics (only touching locations in the shadow that were touched in the primal) for additional performance/memory savings, at the obvious cost of potential safety if used incorrectly.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Consider the following function that loads from offset 47 of a Ptr","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"function f(ptr)\n x = unsafe_load(ptr, 47)\n x * x\nend\n\nptr = Base.reinterpret(Ptr{Float64}, Libc.malloc(100*sizeof(Float64)))\nunsafe_store!(ptr, 3.14, 47)\n\nf(ptr)\n\n# output\n9.8596","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"The recommended (and guaranteed sound) way to differentiate this is to pass in a shadow pointer that is congruent with the primal. That is to say, its length (and recursively for any sub types) are equivalent to the primal.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"ptr = Base.reinterpret(Ptr{Float64}, Libc.malloc(100*sizeof(Float64)))\nunsafe_store!(ptr, 3.14, 47)\ndptr = Base.reinterpret(Ptr{Float64}, Libc.calloc(100*sizeof(Float64), 1))\n\nautodiff(Reverse, f, Duplicated(ptr, dptr))\n\nunsafe_load(dptr, 47)\n\n# output\n6.28","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"However, since we know the original function only reads from one float64, we could choose to only allocate a single float64 for the shadow, as long as we ensure that loading from offset 47 (the only location accessed) is in bounds.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"ptr = Base.reinterpret(Ptr{Float64}, Libc.malloc(100*sizeof(Float64)))\nunsafe_store!(ptr, 3.14, 47)\ndptr = Base.reinterpret(Ptr{Float64}, Libc.calloc(sizeof(Float64), 1))\n\n# offset the pointer to have unsafe_load(dptr, 47) access the 0th byte of dptr\n# since julia one indexes we subtract 46 * sizeof(Float64) here\nautodiff(Reverse, f, Duplicated(ptr, dptr - 46 * sizeof(Float64)))\n\n# represents the derivative of the 47'th elem of ptr, \nunsafe_load(dptr)\n\n# output\n6.28","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"However, this style of optimization is not specific to Enzyme, or AD, as one could have done the same thing on the primal code where it only passed in one float. The difference, here however, is that performing these memory-layout tricks safely in Enzyme requires understanding the access patterns of the generated derivative code – like discussed here.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"ptr = Base.reinterpret(Ptr{Float64}, Libc.calloc(sizeof(Float64), 1))\nunsafe_store!(ptr, 3.14)\n# offset the pointer to have unsafe_load(ptr, 47) access the 0th byte of dptr\n# again since julia one indexes we subtract 46 * sizeof(Float64) here\nf(ptr - 46 * sizeof(Float64))\n\n# output\n9.8596","category":"page"},{"location":"faq/#CUDA-support","page":"FAQ","title":"CUDA support","text":"","category":"section"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"CUDA.jl is only supported on Julia v1.7.0 and onwards. On v1.6, attempting to differentiate CUDA kernel functions will not use device overloads correctly and thus returns fundamentally wrong results.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Specifically, differentiating within device kernels is supported. See our cuda tests for some examples. ","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Differentiating through a heterogeneous (e.g. combined host and device) code presently requires defining a custom derivative that tells Enzyme that differentiating an @cuda call is done by performing @cuda of its generated derivative. For an example of this in Enzyme-C++ see here. Automating this for a better experience for CUDA.jl requires an update to CUDA.jl, and is now available for Kernel Abstractions.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Differentiating host-side code when accesses device memory (e.g. sum(CuArray)) is not yet supported, but in progress.","category":"page"},{"location":"faq/#Linear-Algebra","page":"FAQ","title":"Linear Algebra","text":"","category":"section"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Enzyme supports presently some, but not all of Julia's linear algebra library. This is because some of Julia's linear algebra library is not pure Julia code and calls external functions such as BLAS, LaPACK, CuBLAS, SuiteSparse, etc.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"For all BLAS functions, Enzyme will generate a correct derivative function. If it is a gemm (matmul), gemv (matvec), dot (dot product), axpy (vector add and scale), and a few others, Enzyme will generate a fast derivative using another corresponding BLAS call. For other BLAS functions, Enzyme will presently emit a warning Fallback BLAS [functionname] that indicates that Enzyme will differentiate this function by differentiating a serial implementation of BLAS. This will still work for all BLAS codes, but may be slower on a parallel platform.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Other libraries do not yet have derivatives (either fast or fallback) implemented within Enzyme. Supporting these is not a fundamental limitation, but requires implementing a rule in Enzyme describing how to differentiate them. Contributions welcome!","category":"page"},{"location":"faq/#Sparse-arrays","page":"FAQ","title":"Sparse arrays","text":"","category":"section"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Differentiating code using sparse arrays is supported, but care must be taken because backing arrays drop zeros in Julia (unless told not to).","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"using SparseArrays\na = sparse([2.0])\nda1 = sparse([0.0]) # Incorrect: SparseMatrixCSC drops explicit zeros\nEnzyme.autodiff(Reverse, sum, Active, Duplicated(a, da1))\nda1\n\n# output\n\n1-element SparseVector{Float64, Int64} with 0 stored entries","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"da2 = sparsevec([1], [0.0]) # Correct: Prevent SparseMatrixCSC from dropping zeros\nEnzyme.autodiff(Reverse, sum, Active, Duplicated(a, da2))\nda2\n\n# output\n\n1-element SparseVector{Float64, Int64} with 1 stored entry:\n [1] = 1.0","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Sometimes, determining how to perform this zeroing can be complicated. That is why Enzyme provides a helper function Enzyme.make_zero that does this automatically.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Enzyme.make_zero(a)\nEnzyme.gradient(Reverse, sum, a)[1] # This calls make_zero(a)\n\n# output\n\n1-element SparseVector{Float64, Int64} with 1 stored entry:\n [1] = 1.0","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Some Julia libraries sparse linear algebra libraries call out to external C code like SuiteSparse which we don't presently implement derivatives for (we have some but have yet to complete all). If that case happens, Enzyme will throw a \"no derivative found\" error at the callsite of that function. This isn't a fundamental limitation, and is easily resolvable by writing a custom rule or internal Enzyme support. Help is certainly welcome :).","category":"page"},{"location":"faq/#Advanced-Sparse-arrays","page":"FAQ","title":"Advanced Sparse arrays","text":"","category":"section"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Essentially the way Enzyme represents all data structures, including sparse data structures, is to have the shadow (aka derivative) memory be the same memory layout as the primal. Suppose you have an input data structure x. The derivative of x at byte offset 12 will be stored in the shadow dx at byte offset 12, etc.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"This has the nice property that the storage for the derivative, including all intermediate computations, is the same as that of the primal (ignoring caching requirements for reverse mode).","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"It also means that any arbitrary data structure can be differentiated with respect to, and we don’t have any special handling required to register every data structure one could create.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"This representation does have some caveats (e.g. see Identical types in Duplicated above).","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Sparse data structures are often represented with say a Vector{Float64} that holds the actual elements, and a Vector{Int} that specifies the index n the backing array that corresponds to the true location in the overall vector.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"We have no explicit special cases for sparse Data structures, so the layout semantics mentioned above is indeed what Enzyme uses.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Thus the derivative of a sparse array is to have a second backing array of the same size, and another Vector{Int} (of the same offsets).","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"As a concrete example, suppose we have the following: x = { 3 : 2.7, 10 : 3.14 }. In other words, a sparse data structure with two elements, one at index 3, another at index 10. This could be represented with the backing array being [2.7, 3.14] and the index array being [3, 10].","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"A correctly zero-initialized shadow data structure would be to have a backing array of size 2 with zero’s, and an index array again being [3, 10].","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"In this form the second element of the derivative backing array is used to store/represent the derivative of the second element of the original backing array, in other words the derivative at index 10.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Like mentioned above, a caveat here is that this correctly zero’d initializer is not the default produced by sparse([0.0]) as this drops the zero elements from the backing array. Enzyme.makezero recursively goes through your data structure to generate the shadows of the correct structure (and in this case would make a new backing array of appropriate size). The `makezero` function is not special cased to sparsity, but just comes out as a result.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Internally, when differentiating a function this is the type of data structure that Enzyme builds and uses to represent variables. However, at the Julia level that there’s a bit of a sharp edge.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Consider a function f(A(x)) where x is a scalar or dense input, A(x) returns a sparse array, and f(A(x)) returns a scalar loss. ","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"The derivative that Enzyme creates for A(x) would create both the backing/index arrays for the original result A, as well as the equal sized backing/index arrays for the derivative.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"For any program which generates sparse data structures internally, like the total program f(A(x)), this will always give you the answer you expect. Moreover, the memory requirements of the derivative will be the same as the primal (other AD tools will blow up the memory usage and construct dense derivatives where the primal was sparse).","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"The added caveat, however, comes when you differentiate a top level function that has a sparse array input. For example, consider the sparse sum function which adds up all elements. While in one definition, this function represents summing up all elements of the virtual sparse array (including the zero's which are never materialized), in a more literal sense this sum function will only add elements 3 and 10 of the input sparse array – the only two nonzero elements – or equivalently the sum of the whole backing array. Correspondingly Enzyme will update the sparse shadow data structure to mark both elements 3 and 10 as having a derivative of 1 (or more literally set all the elements of the backing array to derivative 1). These are the only variables that Enzyme needs to update, since they are the only variables read (and thus the only ones which have a non-zero derivative). Thus any function which may call this method and compose via the chain rule will only ever read the derivative of these two elements. This is why this memory-safe representation composes within Enzyme, though may produce counter-intuitive reuslts at the top level.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"If the name we gave to this data structure wasn’t \"SparseArray\" but instead \"MyStruct\" this is precisely the answer we would have desired. However, since the sparse array printer prints zeros for elements outside of the sparse backing array, this isn’t what one would expect. Making a nicer user conversion from Enzyme’s form of differential data structures, to the more natural \"Julia\" form where there is a semantic mismatch between what Julia intends a data structure to mean by name, and what is being discussed here.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"The benefit of this representation is that : (1) all of our rules compose correctly (you get the correct answer for f(A(x)), (2) without the need to special case any sparse code, and (3) with the same memory/performance expectations as the original code.","category":"page"},{"location":"faq/#Activity-of-temporary-storage","page":"FAQ","title":"Activity of temporary storage","text":"","category":"section"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"If you pass in any temporary storage which may be involved in an active computation to a function you want to differentiate, you must also pass in a duplicated temporary storage for use in computing the derivatives. For example, consider the following function which uses a temporary buffer to compute the result.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"function f(x, tmp, k, n)\n tmp[1] = 1.0\n for i in 1:n\n tmp[k] *= x\n end\n tmp[1]\nend\n\n# output\n\nf (generic function with 1 method)","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Marking the argument for tmp as Const (aka non-differentiable) means that Enzyme believes that all variables loaded from or stored into tmp must also be non-differentiable, since all values inside a non-differentiable variable must also by definition be non-differentiable.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Enzyme.autodiff(Reverse, f, Active(1.2), Const(Vector{Float64}(undef, 1)), Const(1), Const(5)) # Incorrect\n\n# output\n\n((0.0, nothing, nothing, nothing),)","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Passing in a duplicated (e.g. differentiable) variable for tmp now leads to the correct answer.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Enzyme.autodiff(Reverse, f, Active(1.2), Duplicated(Vector{Float64}(undef, 1), zeros(1)), Const(1), Const(5)) # Correct (returns 10.367999999999999 == 1.2^4 * 5)\n\n# output\n\n((10.367999999999999, nothing, nothing, nothing),)","category":"page"},{"location":"faq/#Runtime-Activity","page":"FAQ","title":"Runtime Activity","text":"","category":"section"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"When computing the derivative of mutable variables, Enzyme also needs additional temporary storage space for the corresponding derivative variables. If an argument tmp is marked as Const, Enzyme does not have any temporary storage space for the derivatives!","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Enzyme will error when they detect these latter types of situations, which we will refer to as activity unstable. This term is chosen to mirror the Julia notion of type-unstable code (e.g. where a type is not known at compile time). If an expression is activity unstable, it could either be constant, or active, depending on data not known at compile time. For example, consider the following:","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"function g(cond, active_var, constant_var)\n if cond\n return active_var\n else\n return constant_var\nend\n\nEnzyme.autodiff(Forward, g, Const(condition), Duplicated(x, dx), Const(y))","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"The returned value here could either by constant or duplicated, depending on the runtime-defined value of cond. If cond is true, Enzyme simply returns the shadow of active_var as the derivative. However, if cond is false, there is no derivative shadow for constant_var and Enzyme will throw a EnzymeRuntimeActivityError error. For some simple types, e.g. a float Enzyme can circumvent this issue, for example by returning the float 0. Similarly, for some types like the Symbol type, which are never differentiable, such a shadow value will never be used, and Enzyme can return the original \"primal\" value as its derivative. However, for arbitrary data structures, Enzyme presently has no generic mechanism to resolve this.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"For example consider a third function:","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"function h(cond, active_var, constant_var)\n return [g(cond, active_var, constant_var), g(cond, active_var, constant_var)]\nend\n\nEnzyme.autodiff(Forward, h, Const(condition), Duplicated(x, dx), Const(y))","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Enzyme provides a nice utility Enzyme.make_zero which takes a data structure and constructs a deepcopy of the data structure with all of the floats set to zero and non-differentiable types like Symbols set to their primal value. If Enzyme gets into such a \"Mismatched activity\" situation where it needs to return a differentiable data structure from a constant variable, it could try to resolve this situation by constructing a new shadow data structure, such as with Enzyme.make_zero. However, this still can lead to incorrect results. In the case of h above, suppose that active_var and consant_var are both arrays, which are mutable (aka in-place) data types. This means that the return of h is going to either be result = [active_var, active_var] or result = [constant_var, constant_var]. Thus an update to result[1][1] would also change result[2][1] since result[1] and result[2] are the same array. ","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"If one created a new zero'd copy of each return from g, this would mean that the derivative dresult would have one copy made for the first element, and a second copy made for the second element. This could lead to incorrect results, and is unfortunately not a general resolution. However, for non-mutable variables (e.g. like floats) or non-differrentiable types (e.g. like Symbols) this problem can never arise.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Instead, Enzyme has a special mode known as \"Runtime Activity\" which can handle these types of situations. It can come with a minor performance reduction, and is therefore off by default. It can be enabled with by setting runtime activity to true in a desired differentiation mode.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"The way Enzyme's runtime activity resolves this issue is to return the original primal variable as the derivative whenever it needs to denote the fact that a variable is a constant. As this issue can only arise with mutable variables, they must be represented in memory via a pointer. All addtional loads and stores will now be modified to first check if the primal pointer is the same as the shadow pointer, and if so, treat it as a constant. Note that this check is not saying that the same arrays contain the same values, but rather the same backing memory represents both the primal and the shadow (e.g. a === b or equivalently pointer(a) == pointer(b)). ","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Enabling runtime activity does therefore, come with a sharp edge, which is that if the computed derivative of a function is mutable, one must also check to see if the primal and shadow represent the same pointer, and if so the true derivative of the function is actually zero.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Generally, the preferred solution to these type of activity unstable codes should be to make your variables all activity-stable (e.g. always containing differentiable memory or always containing non-differentiable memory). However, with care, Enzyme does support \"Runtime Activity\" as a way to differentiate these programs without having to modify your code. One can enable runtime activity for your code by changing the mode, such as","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Enzyme.autodiff(set_runtime_activity(Forward), h, Const(condition), Duplicated(x, dx), Const(y))","category":"page"},{"location":"faq/#Mixed-activity","page":"FAQ","title":"Mixed activity","text":"","category":"section"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Sometimes in Reverse mode (but not forward mode), you may see an error Type T has mixed internal activity types for some type. This error arises when a variable in a computation cannot be fully represented as either a Duplicated or Active variable.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Active variables are used for immutable variables (like Float64), whereas Duplicated variables are used for mutable variables (like Vector{Float64}). Speciically, since Active variables are immutable, functions with Active inputs will return the adjoint of that variable. In contrast Duplicated variables will have their derivatives +='d in place.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"This error indicates that you have a type, like Tuple{Float, Vector{Float64}} that has immutable components and mutable components. Therefore neither Active nor Duplicated can be used for this type.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Internally, by virtue of working at the LLVM level, most Julia types are represented as pointers, and this issue does not tend to arise within code fully differentiated by Enzyme internally. However, when a program needs to interact with Julia API's (e.g. as arguments to a custom rule, a type unstable call, or the outermost function being differentiated), Enzyme must adhere to Julia's notion of immutability and will throw this error rather than risk an incorrect result.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"For example, consider the following code, which has a type unstable call to myfirst, passing in a mixed type Tuple{Float64, Vector{Float64}}.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"@noinline function myfirst(tup::T) where T\n return tup[1]\nend\n\nfunction f(x::Float64)\n vec = [x]\n tup = (x, vec)\n Base.inferencebarrier(myfirst)(tup)::Float64\nend\n\nEnzyme.autodiff(Reverse, f, Active, Active(3.1))","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"When this situation arises, it is often easiest to resolve it by adding a level of indirection to ensure the entire variable is mutable. For example, one could enclose this variable in a reference, such as Ref{Tuple{Float, Vector{Float64}}}, like as follows.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"@noinline function myfirst_ref(tup_ref::T) where T\n tup = tup_ref[]\n return tup[1]\nend\n\nfunction f2(x::Float64)\n vec = [x]\n tup = (x, vec)\n tup_ref = Ref(tup)\n Base.inferencebarrier(myfirst_ref)(tup_ref)::Float64\nend\n\nEnzyme.autodiff(Reverse, f2, Active, Active(3.1))","category":"page"},{"location":"faq/#Complex-numbers","page":"FAQ","title":"Complex numbers","text":"","category":"section"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Differentiation of a function which returns a complex number is ambiguous, because there are several different gradients which may be desired. Rather than assume a specific of these conventions and potentially result in user error when the resulting derivative is not the desired one, Enzyme forces users to specify the desired convention by returning a real number instead.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Consider the function f(z) = z*z. If we were to differentiate this and have real inputs and outputs, the derivative f'(z) would be unambiguously 2*z. However, consider breaking down a complex number down into real and imaginary parts. Suppose now we were to call f with the explicit real and imaginary components, z = x + i y. This means that f is a function that takes an input of two values and returns two values f(x, y) = u(x, y) + i v(x, y). In the case of z*z this means that u(x,y) = x*x-y*y and v(x,y) = 2*x*y.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"If we were to look at all first-order derivatives in total, we would end up with a 2x2 matrix (i.e. Jacobian), the derivative of each output wrt each input. Let's try to compute this, first by hand, then with Enzyme.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"grad u(x, y) = [d/dx u, d/dy u] = [d/dx x*x-y*y, d/dy x*x-y*y] = [2*x, -2*y];\ngrad v(x, y) = [d/dx v, d/dy v] = [d/dx 2*x*y, d/dy 2*x*y] = [2*y, 2*x];","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Reverse mode differentiation computes the derivative of all inputs with respect to a single output by propagating the derivative of the return to its inputs. Here, we can explicitly differentiate with respect to the real and imaginary results, respectively, to find this matrix.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"f(z) = z * z\n\n# a fixed input to use for testing\nz = 3.1 + 2.7im\n\ngrad_u = Enzyme.autodiff(Reverse, z->real(f(z)), Active, Active(z))[1][1]\ngrad_v = Enzyme.autodiff(Reverse, z->imag(f(z)), Active, Active(z))[1][1]\n\n(grad_u, grad_v)\n# output\n(6.2 - 5.4im, 5.4 + 6.2im)","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"This is somewhat inefficient, since we need to call the forward pass twice, once for the real part, once for the imaginary. We can solve this using batched derivatives in Enzyme, which computes several derivatives for the same function all in one go. To make it work, we're going to need to use split mode, which allows us to provide a custom derivative return value.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"fwd, rev = Enzyme.autodiff_thunk(ReverseSplitNoPrimal, Const{typeof(f)}, Active, Active{ComplexF64})\n\n# Compute the reverse pass seeded with a differntial return of 1.0 + 0.0im\ngrad_u = rev(Const(f), Active(z), 1.0 + 0.0im, fwd(Const(f), Active(z))[1])[1][1]\n# Compute the reverse pass seeded with a differntial return of 0.0 + 1.0im\ngrad_v = rev(Const(f), Active(z), 0.0 + 1.0im, fwd(Const(f), Active(z))[1])[1][1]\n\n(grad_u, grad_v)\n\n# output\n(6.2 - 5.4im, 5.4 + 6.2im)","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Now let's make this batched","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"fwd, rev = Enzyme.autodiff_thunk(ReverseSplitWidth(ReverseSplitNoPrimal, Val(2)), Const{typeof(f)}, Active, Active{ComplexF64})\n\n# Compute the reverse pass seeded with a differential return of 1.0 + 0.0im and 0.0 + 1.0im in one go!\nrev(Const(f), Active(z), (1.0 + 0.0im, 0.0 + 1.0im), fwd(Const(f), Active(z))[1])[1][1]\n\n# output\n(6.2 - 5.4im, 5.4 + 6.2im)","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"In contrast, Forward mode differentiation computes the derivative of all outputs with respect to a single input by providing a differential input. Thus we need to seed the shadow input with either 1.0 or 1.0im, respectively. This will compute the transpose of the matrix we found earlier.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"d/dx f(x, y) = d/dx [u(x,y), v(x,y)] = d/dx [x*x-y*y, 2*x*y] = [ 2*x, 2*y];\nd/dy f(x, y) = d/dy [u(x,y), v(x,y)] = d/dy [x*x-y*y, 2*x*y] = [-2*y, 2*x];","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"d_dx = Enzyme.autodiff(Forward, f, Duplicated(z, 1.0+0.0im))[1]\nd_dy = Enzyme.autodiff(Forward, f, Duplicated(z, 0.0+1.0im))[1]\n\n(d_dx, d_dy)\n\n# output\n(6.2 + 5.4im, -5.4 + 6.2im)","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Again, we can go ahead and batch this.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Enzyme.autodiff(Forward, f, BatchDuplicated(z, (1.0+0.0im, 0.0+1.0im)))[1]\n\n# output\n(var\"1\" = 6.2 + 5.4im, var\"2\" = -5.4 + 6.2im)","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Taking Jacobians with respect to the real and imaginary results is fine, but for a complex scalar function it would be really nice to have a single complex derivative. More concretely, in this case when differentiating z*z, it would be nice to simply return 2*z. However, there are four independent variables in the 2x2 jacobian, but only two in a complex number. ","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Complex differentiation is often viewed in the lens of directional derivatives. For example, what is the derivative of the function as the real input increases, or as the imaginary input increases. Consider the derivative along the real axis, textttlim_Delta x rightarrow 0 fracf(x+Delta x y)-f(x y)Delta x. This simplifies to textttlim_Delta x rightarrow 0 fracu(x+Delta x y)-u(x y) + i left v(x+Delta x y)-v(x y)rightDelta x = fracpartialpartial x u(xy) + ifracpartialpartial x v(xy). This is exactly what we computed by seeding forward mode with a shadow of 1.0 + 0.0im.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"For completeness, we can also consider the derivative along the imaginary axis textttlim_Delta y rightarrow 0 fracf(x y+Delta y)-f(x y)iDelta y. Here this simplifies to textttlim_u(x y+Delta y)-u(x y) + i left v(x y+Delta y)-v(x y)rightiDelta y = -ifracpartialpartial y u(xy) + fracpartialpartial y v(xy). Except for the i in the denominator of the limit, this is the same as the result of Forward mode, when seeding x with a shadow of 0.0 + 1.0im. We can thus compute the derivative along the real axis by multiplying our second Forward mode call by -im.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"d_real = Enzyme.autodiff(Forward, f, Duplicated(z, 1.0+0.0im))[1]\nd_im = -im * Enzyme.autodiff(Forward, f, Duplicated(z, 0.0+1.0im))[1]\n\n(d_real, d_im)\n\n# output\n(6.2 + 5.4im, 6.2 + 5.4im)","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Interestingly, the derivative of z*z is the same when computed in either axis. That is because this function is part of a special class of functions that are invariant to the input direction, called holomorphic. ","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Thus, for holomorphic functions, we can simply seed Forward-mode AD with a shadow of one for whatever input we are differenitating. This is nice since seeding the shadow with an input of one is exactly what we'd do for real-valued funtions as well.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Reverse-mode AD, however, is more tricky. This is because holomorphic functions are invariant to the direction of differentiation (aka the derivative inputs), not the direction of the differential return.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"However, if a function is holomorphic, the two derivative functions we computed above must be the same. As a result, fracpartialpartial x u = fracpartialpartial y v and fracpartialpartial y u = -fracpartialpartial x v. ","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"We saw earlier, that performing reverse-mode AD with a return seed of 1.0 + 0.0im yielded [d/dx u, d/dy u]. Thus, for a holomorphic function, a real-seeded Reverse-mode AD computes [d/dx u, -d/dx v], which is the complex conjugate of the derivative.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"conj(grad_u)\n\n# output\n\n6.2 + 5.4im","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"In the case of a scalar-input scalar-output function, that's sufficient. However, most of the time one uses reverse mode, it involves either several inputs or outputs, perhaps via memory. This case requires additional handling to properly sum all the partial derivatives from the use of each input and apply the conjugate operator at only the ones relevant to the differential return.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"For simplicity, Enzyme provides a helper utlity ReverseHolomorphic which performs Reverse mode properly here, assuming that the function is indeed holomorphic and thus has a well-defined single derivative.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Enzyme.autodiff(ReverseHolomorphic, f, Active, Active(z))[1][1]\n\n# output\n\n6.2 + 5.4im","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"For even non-holomorphic functions, complex analysis allows us to define fracpartialpartial z = frac12left(fracpartialpartial x - i fracpartialpartial y right). For non-holomorphic functions, this allows us to compute d/dz. Let's consider myabs2(z) = z * conj(z). We can compute the derivative wrt z of this in Forward mode as follows, which as one would expect results in a result of conj(z):","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"myabs2(z) = z * conj(z)\n\ndabs2_dx, dabs2_dy = Enzyme.autodiff(Forward, myabs2, BatchDuplicated(z, (1.0 + 0.0im, 0.0 + 1.0im)))[1]\n(dabs2_dx - im * dabs2_dy) / 2\n\n# output\n\n3.1 - 2.7im","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Similarly, we can compute d/d conj(z) = d/dx + i d/dy.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"(dabs2_dx + im * dabs2_dy) / 2\n\n# output\n\n3.1 + 2.7im","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Computing this in Reverse mode is more tricky. Let's expand f in terms of u and v. fracpartialpartial z f = frac12 left( u_x + i v_x - i u_y + i v_y right) = frac12 left( u_x + v_y + i v_x - u_y right). Thus d/dz = (conj(grad_u) + im * conj(grad_v))/2.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"abs2_fwd, abs2_rev = Enzyme.autodiff_thunk(ReverseSplitWidth(ReverseSplitNoPrimal, Val(2)), Const{typeof(myabs2)}, Active, Active{ComplexF64})\n\n# Compute the reverse pass seeded with a differential return of 1.0 + 0.0im and 0.0 + 1.0im in one go!\ngradabs2_u, gradabs2_v = abs2_rev(Const(myabs2), Active(z), (1.0 + 0.0im, 0.0 + 1.0im), abs2_fwd(Const(myabs2), Active(z))[1])[1][1]\n\n(conj(gradabs2_u) + im * conj(gradabs2_v)) / 2\n\n# output\n\n3.1 - 2.7im","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"For d/d conj(z), frac12 left( u_x + i v_x + i u_y + i v_y right) = frac12 left( u_x - v_y + i v_x + u_y right). Thus d/d conj(z) = (grad_u + im * grad_v)/2.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"(gradabs2_u + im * gradabs2_v) / 2\n\n# output\n\n3.1 + 2.7im","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Note: when writing rules for complex scalar functions, in reverse mode one needs to conjugate the differential return, and similarly the true result will be the conjugate of that value (in essence you can think of reverse-mode AD as working in the conjugate space).","category":"page"},{"location":"faq/#What-types-are-differentiable?","page":"FAQ","title":"What types are differentiable?","text":"","category":"section"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Enzyme tracks differentiable dataflow through values. Specifically Enzyme tracks differentiable data in base types like Float32, Float64, Float16, BFloat16, etc.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"As a simple example:","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"f(x) = x * x\nEnzyme.autodiff(Forward, f, Duplicated(3.0, 1.0))\n\n# output\n\n(6.0,)","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Enzyme also tracks differentiable data in any types containing these base types (e.g. floats). For example, consider a struct or array containing floats.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"struct Pair\n lhs::Float64\n rhs::Float64\nend\nf_pair(x) = x.lhs * x.rhs\nEnzyme.autodiff(Forward, f_pair, Duplicated(Pair(3.0, 2.0), Pair(1.0, 0.0)))\n\n# output\n\n(2.0,)","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Enzyme.autodiff(Forward, sum, Duplicated([1.0, 2.0, 3.0], [5.0, 0.0, 100.0]))\n\n\n# output\n\n(105.0,)","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"A differentiable data structure can be arbitrarily complex, such as a linked list.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"\nstruct LList\n prev::Union{Nothing, LList}\n value::Float64\nend\n\nfunction make_list(x::Vector)\n result = nothing\n for value in reverse(x)\n result = LList(result, value)\n end\n return result\nend\n\nfunction list_sum(list::Union{Nothing, LList})\n result = 0.0\n while list != nothing\n result += list.value\n list = list.prev\n end\n return result\nend\n\nlist = make_list([1.0, 2.0, 3.0])\ndlist = make_list([5.0, 0.0, 100.0])\n\nEnzyme.autodiff(Forward, list_sum, Duplicated(list, dlist))\n\n# output\n\n(105.0,)","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Presently Enzyme only considers floats as base types. As a result, Enzyme does not support differentiating data contained in Ints, Strings, or Vals. If it is desirable for Enzyme to add a base type, please open an issue.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"f_int(x) = x * x\nEnzyme.autodiff(Forward, f_int, Duplicated, Duplicated(3, 1))\n\n# output\n\nERROR: Return type `Int64` not marked Const, but type is guaranteed to be constant","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"f_str(x) = parse(Float64, x) * parse(Float64, x)\n\nautodiff(Forward, f_str, Duplicated(\"1.0\", \"1.0\"))\n\n# output\n\n(0.0,)","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"f_val(::Val{x}) where x = x * x\n\nautodiff(Forward, f_val, Duplicated(Val(1.0), Val(1.0)))\n\n# output\n\nERROR: Type of ghost or constant type Duplicated{Val{1.0}} is marked as differentiable.","category":"page"}] +[{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"EditURL = \"../../../examples/autodiff.jl\"","category":"page"},{"location":"generated/autodiff/#Basics","page":"Basics","title":"Basics","text":"","category":"section"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"The goal of this tutorial is to give users already familiar with automatic differentiation (AD) an overview of the Enzyme differentiation API for the following differentiation modes","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"Reverse mode\nForward mode\nForward over reverse mode\nVector Forward over reverse mode","category":"page"},{"location":"generated/autodiff/#Defining-a-function","page":"Basics","title":"Defining a function","text":"","category":"section"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"Enzyme differentiates arbitrary multivariate vector functions as the most general case in automatic differentiation","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"f mathbbR^n rightarrow mathbbR^m y = f(x)","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"For simplicity we define a vector function with m=1. However, this tutorial can easily be applied to arbitrary m in mathbbN.","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"using Enzyme\n\nfunction f(x::Array{Float64}, y::Array{Float64})\n y[1] = x[1] * x[1] + x[2] * x[1]\n return nothing\nend;\nnothing #hide","category":"page"},{"location":"generated/autodiff/#Reverse-mode","page":"Basics","title":"Reverse mode","text":"","category":"section"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"The reverse model in AD is defined as","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"beginaligned\ny = f(x) \nbarx = bary cdot nabla f(x)\nendaligned","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"bar denotes an adjoint variable. Note that executing an AD in reverse mode computes both y and the adjoint barx.","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"x = [2.0, 2.0]\nbx = [0.0, 0.0]\ny = [0.0]\nby = [1.0];\nnothing #hide","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"Enzyme stores the value and adjoint of a variable in an object of type Duplicated where the first element represent the value and the second the adjoint. Evaluating the reverse model using Enzyme is done via the following call.","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"Enzyme.autodiff(Reverse, f, Duplicated(x, bx), Duplicated(y, by));\nnothing #hide","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"This yields the gradient of f in bx at point x = [2.0, 2.0]. by is called the seed and has to be set to 10 in order to compute the gradient. Let's save the gradient for later.","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"g = copy(bx)","category":"page"},{"location":"generated/autodiff/#Forward-mode","page":"Basics","title":"Forward mode","text":"","category":"section"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"The forward model in AD is defined as","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"beginaligned\ny = f(x) \ndoty = nabla f(x) cdot dotx\nendaligned","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"To obtain the first element of the gradient using the forward model we have to seed dotx with dotx = 1000","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"x = [2.0, 2.0]\ndx = [1.0, 0.0]\ny = [0.0]\ndy = [0.0];\nnothing #hide","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"In the forward mode the second element of Duplicated stores the tangent.","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"Enzyme.autodiff(Forward, f, Duplicated(x, dx), Duplicated(y, dy));\nnothing #hide","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"We can now verify that indeed the reverse mode and forward mode yield the same result for the first component of the gradient. Note that to acquire the full gradient one needs to execute the forward model a second time with the seed dx set to [0.0,1.0].","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"Let's verify whether the reverse and forward model agree.","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"g[1] == dy[1]","category":"page"},{"location":"generated/autodiff/#Forward-over-reverse","page":"Basics","title":"Forward over reverse","text":"","category":"section"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"The forward over reverse (FoR) model is obtained by applying the forward model to the reverse model using the chain rule for the product in the adjoint statement.","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"beginaligned\ny = f(x) \ndoty = nabla f(x) cdot dotx \nbarx = bary cdot nabla f(x) \ndotbarx = bary cdot nabla^2 f(x) cdot dotx + dotbary cdot nabla f(x)\nendaligned","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"To obtain the first column/row of the Hessian nabla^2 f(x) we have to seed dotbary with 00, bary with 10 and dotx with 10 00.","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"y = [0.0]\nx = [2.0, 2.0]\n\ndy = [0.0]\ndx = [1.0, 0.0]\n\nbx = [0.0, 0.0]\nby = [1.0]\ndbx = [0.0, 0.0]\ndby = [0.0]\n\nEnzyme.autodiff(\n Forward,\n (x,y) -> Enzyme.autodiff(Reverse, f, x, y),\n Duplicated(Duplicated(x, bx), Duplicated(dx, dbx)),\n Duplicated(Duplicated(y, by), Duplicated(dy, dby)),\n)","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"The FoR model also computes the forward model from before, giving us again the first component of the gradient.","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"g[1] == dy[1]","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"In addition we now have the first row/column of the Hessian.","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"dbx[1] == 2.0\ndbx[2] == 1.0","category":"page"},{"location":"generated/autodiff/#Vector-forward-over-reverse","page":"Basics","title":"Vector forward over reverse","text":"","category":"section"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"The vector FoR allows us to propagate several tangents at once through the second-order model by computing the derivative of the gradient at multiple points at once. We begin by defining a helper function for the gradient. Since we will not need the original results (stored in y), we can mark it DuplicatedNoNeed. Specifically, this will perform the following:","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"beginaligned\nbarx = barx + bary cdot nabla f(x) \nbary = 0\nendaligned","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"function grad(x, dx, y, dy)\n Enzyme.autodiff(Reverse, f, Duplicated(x, dx), DuplicatedNoNeed(y, dy))\n nothing\nend","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"To compute the conventional gradient, we would call this function with our given inputs, dy = [1.0], and dx = [0.0, 0.0]. Since y is not needed, we can just set it to an undef vector.","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"x = [2.0, 2.0]\ny = Vector{Float64}(undef, 1)\ndx = [0.0, 0.0]\ndy = [1.0]\n\ngrad(x, dx, y, dy)","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"dx now contains the gradient","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"@show dx","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"To compute the hessian, we need to take the dervative of this gradient function at every input. Following the same seeding strategy as before, we now seed both in the vx[1]=[1.0, 0.0] and vx[2]=[0.0, 1.0] direction. These tuples have to be put into a BatchDuplicated type. We then compute the forward mode derivative at all these points.","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"vx = ([1.0, 0.0], [0.0, 1.0])\nhess = ([0.0, 0.0], [0.0, 0.0])\ndx = [0.0, 0.0]\ndy = [1.0]\n\nEnzyme.autodiff(Enzyme.Forward, grad,\n Enzyme.BatchDuplicated(x, vx),\n Enzyme.BatchDuplicated(dx, hess),\n Const(y),\n Const(dy))","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"Again we obtain the first-order gradient. If we did not want to compute the gradient again, we could instead have used Enzyme.BatchDuplicatedNoNeed(dx, hess)","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"g[1] == dx[1]","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"We have now the first row/column of the Hessian","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"hess[1][1] == 2.0\n\nhess[1][2] == 1.0","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"as well as the second row/column","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"hess[2][1] == 1.0\n\nhess[2][2] == 0.0","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"","category":"page"},{"location":"generated/autodiff/","page":"Basics","title":"Basics","text":"This page was generated using Literate.jl.","category":"page"},{"location":"api/#API-reference","page":"API reference","title":"API reference","text":"","category":"section"},{"location":"api/#Types-and-constants","page":"API reference","title":"Types and constants","text":"","category":"section"},{"location":"api/","page":"API reference","title":"API reference","text":"Modules = [Enzyme, EnzymeCore, EnzymeCore.EnzymeRules, EnzymeTestUtils, Enzyme.API]\nOrder = [:type, :constant]","category":"page"},{"location":"api/#Functions-and-macros","page":"API reference","title":"Functions and macros","text":"","category":"section"},{"location":"api/","page":"API reference","title":"API reference","text":"Modules = [Enzyme, EnzymeCore, EnzymeCore.EnzymeRules, EnzymeTestUtils, Enzyme.API]\nOrder = [:macro, :function]","category":"page"},{"location":"api/#Documentation","page":"API reference","title":"Documentation","text":"","category":"section"},{"location":"api/","page":"API reference","title":"API reference","text":"Modules = [Enzyme, EnzymeCore, EnzymeCore.EnzymeRules, EnzymeTestUtils, Enzyme.API]\nOrder = [:module, :type, :constant, :macro, :function]","category":"page"},{"location":"api/#Enzyme.@import_frule-Tuple","page":"API reference","title":"Enzyme.@import_frule","text":"import_frule(::fn, tys...)\n\nAutomatically import a ChainRulesCore.fruleas a custom forward modeEnzymeRule. When called in batch mode, this will end up calling the primal multiple times, which may result in incorrect behavior if the function mutates, and slow code, always. Importing the rule fromChainRules` is also likely to be slower than writing your own rule, and may also be slower than not having a rule at all.\n\nUse with caution.\n\nEnzyme.@import_frule(typeof(Base.sort), Any);\n\nx=[1.0, 2.0, 0.0]; dx=[0.1, 0.2, 0.3]; ddx = [0.01, 0.02, 0.03];\n\nEnzyme.autodiff(Forward, sort, Duplicated, BatchDuplicated(x, (dx,ddx)))\nEnzyme.autodiff(Forward, sort, DuplicatedNoNeed, BatchDuplicated(x, (dx,ddx)))\nEnzyme.autodiff(Forward, sort, DuplicatedNoNeed, BatchDuplicated(x, (dx,)))\nEnzyme.autodiff(Forward, sort, Duplicated, BatchDuplicated(x, (dx,)))\n\n# output\n\n(var\"1\" = [0.0, 1.0, 2.0], var\"2\" = (var\"1\" = [0.3, 0.1, 0.2], var\"2\" = [0.03, 0.01, 0.02]))\n(var\"1\" = (var\"1\" = [0.3, 0.1, 0.2], var\"2\" = [0.03, 0.01, 0.02]),)\n(var\"1\" = [0.3, 0.1, 0.2],)\n(var\"1\" = [0.0, 1.0, 2.0], var\"2\" = [0.3, 0.1, 0.2])\n\n\n\n\n\n\n","category":"macro"},{"location":"api/#Enzyme.@import_rrule-Tuple","page":"API reference","title":"Enzyme.@import_rrule","text":"import_rrule(::fn, tys...)\n\nAutomatically import a ChainRules.rrule as a custom reverse mode EnzymeRule. When called in batch mode, this will end up calling the primal multiple times which results in slower code. This macro assumes that the underlying function to be imported is read-only, and returns a Duplicated or Const object. This macro also assumes that the inputs permit a .+= operation and that the output has a valid Enzyme.make_zero function defined. It also assumes that overwritten(x) accurately describes if there is any non-preserved data from forward to reverse, not just the outermost data structure being overwritten as provided by the specification.\n\nFinally, this macro falls back to almost always caching all of the inputs, even if it may not be needed for the derivative computation.\n\nAs a result, this auto importer is also likely to be slower than writing your own rule, and may also be slower than not having a rule at all.\n\nUse with caution.\n\nEnzyme.@import_rrule(typeof(Base.sort), Any);\n\n\n\n\n\n","category":"macro"},{"location":"api/#Enzyme.gradient!-Union{Tuple{ErrIfFuncWritten}, Tuple{Holomorphic}, Tuple{ABI}, Tuple{RuntimeActivity}, Tuple{ReturnPrimal}, Tuple{F}, Tuple{X}, Tuple{ReverseMode{ReturnPrimal, RuntimeActivity, ABI, Holomorphic, ErrIfFuncWritten}, X, F, X}} where {X<:Array, F, ReturnPrimal, RuntimeActivity, ABI, Holomorphic, ErrIfFuncWritten}","page":"API reference","title":"Enzyme.gradient!","text":"gradient!(::ReverseMode, dx, f, x)\n\nCompute the gradient of an array-input function f using reverse mode, storing the derivative result in an existing array dx. Both x and dx must be Arrays of the same type.\n\nExample:\n\nf(x) = x[1]*x[2]\n\ndx = [0.0, 0.0]\ngradient!(Reverse, dx, f, [2.0, 3.0])\n\n# output\n([3.0, 2.0],)\n\ndx = [0.0, 0.0]\ngradient!(ReverseWithPrimal, dx, f, [2.0, 3.0])\n\n# output\n(derivs = ([3.0, 2.0],), val = 6.0)\n\n\n\n\n\n","category":"method"},{"location":"api/#Enzyme.gradient-Union{Tuple{CS}, Tuple{RuntimeActivity}, Tuple{ErrIfFuncWritten}, Tuple{ABI}, Tuple{ReturnPrimal}, Tuple{ForwardMode{ReturnPrimal, ABI, ErrIfFuncWritten, RuntimeActivity}, Any, Any}} where {ReturnPrimal, ABI, ErrIfFuncWritten, RuntimeActivity, CS}","page":"API reference","title":"Enzyme.gradient","text":"gradient(::ForwardMode, f, x; shadows=onehot(x), chunk=nothing)\n\nCompute the gradient of an array-input function f using forward mode. The optional keyword argument shadow is a vector of one-hot vectors of type x which are used to forward-propagate into the return. For performance reasons, this should be computed once, outside the call to gradient, rather than within this call.\n\nExample:\n\nf(x) = x[1]*x[2]\n\ngradient(Forward, f, [2.0, 3.0])\n\n# output\n\n([3.0, 2.0],)\n\ngradient(ForwardWithPrimal, f, [2.0, 3.0])\n\n# output\n(derivs = ([3.0, 2.0],), val = 6.0)\n\ngradient(Forward, f, [2.0, 3.0]; chunk=Val(1))\n\n# output\n\n([3.0, 2.0],)\n\ngradient(ForwardWithPrimal, f, [2.0, 3.0]; chunk=Val(1))\n\n# output\n(derivs = ([3.0, 2.0],), val = 6.0)\n\nFor functions which return an AbstractArray or scalar, this function will return an AbstracttArray whose shape is (size(output)..., size(input)...). No guarantees are presently made about the type of the AbstractArray returned by this function (which may or may not be the same as the input AbstractArray if provided).\n\nFor functions who return other types, this function will retun an AbstractArray of shape size(input) of values of the output type. \n\nf(x) = [ x[1] * x[2], x[2] + x[3] ]\n\ngrad = gradient(Forward, f, [2.0, 3.0, 4.0])\n\n# output\n([3.0 2.0 0.0; 0.0 1.0 1.0],)\n\n\n\n\n\n","category":"method"},{"location":"api/#Enzyme.gradient-Union{Tuple{N}, Tuple{ErrIfFuncWritten}, Tuple{Holomorphic}, Tuple{ABI}, Tuple{RuntimeActivity}, Tuple{ReturnPrimal}, Tuple{ty_0}, Tuple{F}, Tuple{ReverseMode{ReturnPrimal, RuntimeActivity, ABI, Holomorphic, ErrIfFuncWritten}, F, ty_0, Vararg{Any, N}}} where {F, ty_0, ReturnPrimal, RuntimeActivity, ABI, Holomorphic, ErrIfFuncWritten, N}","page":"API reference","title":"Enzyme.gradient","text":"gradient(::ReverseMode, f, args...)\n\nCompute the gradient of a real-valued function f using reverse mode. For each differentiable argument, this function will allocate and return new derivative object, returning a tuple of derivatives for each argument. If an argument is not differentiable, the element of the returned tuple with be nothing.\n\nIn reverse mode (here), the derivatives will be the same type as the original argument.\n\nThis is a structure gradient. For a struct x it returns another instance of the same type, whose fields contain the components of the gradient. In the result, grad.a contains ∂f/∂x.a for any differential x.a, while grad.c == x.c for other types.\n\nExamples:\n\nf(x) = x[1]*x[2]\n\ngrad = gradient(Reverse, f, [2.0, 3.0])\n\n# output\n([3.0, 2.0],)\n\ngrad = gradient(Reverse, only ∘ f, (a = 2.0, b = [3.0], c = \"str\"))\n\n# output\n\n((a = 3.0, b = [2.0], c = \"str\"),)\n\nmul(x, y) = x[1]*y[1]\n\ngrad = gradient(Reverse, mul, [2.0], [3.0])\n\n# output\n([3.0], [2.0])\n\n\ngrad = gradient(Reverse, mul, [2.0], Const([3.0]))\n\n# output\n([3.0], nothing)\n\nIf passing a mode that returns the primal (e.g. ReverseWithPrimal), the return type will instead be a tuple where the first element contains the derivatives, and the second element contains the result of the original computation.\n\n\ngrad = gradient(ReverseWithPrimal, f, [2.0, 3.0])\n\n# output\n(derivs = ([3.0, 2.0],), val = 6.0)\n\n\ngrad = gradient(ReverseWithPrimal, mul, [2.0], [3.0])\n\n# output\n(derivs = ([3.0], [2.0]), val = 6.0)\n\ngrad = gradient(ReverseWithPrimal, mul, [2.0], Const([3.0]))\n\n# output\n(derivs = ([3.0], nothing), val = 6.0)\n\n\n\n\n\n","category":"method"},{"location":"api/#Enzyme.hvp!-Union{Tuple{X}, Tuple{F}, Tuple{X, F, X, X}} where {F, X}","page":"API reference","title":"Enzyme.hvp!","text":"hvp!(res::X, f::F, x::X, v::X) where {F, X}\n\nCompute an in-place Hessian-vector product of an array-input scalar-output function f, as evaluated at x times the vector v. The result will be stored into res. The function still allocates and zero's a buffer to store the intermediate gradient, which is not returned to the user.\n\nIn other words, compute res .= hessian(f)(x) * v\n\nSee hvp_and_gradient! for a function to compute both the hvp and the gradient in a single call.\n\nExample:\n\nf(x) = sin(x[1] * x[2])\n\nres = Vector{Float64}(undef, 2)\nhvp!(res, f, [2.0, 3.0], [5.0, 2.7])\n\nres\n# output\n2-element Vector{Float64}:\n 19.6926882637302\n 16.201003759768003\n\n\n\n\n\n","category":"method"},{"location":"api/#Enzyme.hvp-Union{Tuple{X}, Tuple{F}, Tuple{F, X, X}} where {F, X}","page":"API reference","title":"Enzyme.hvp","text":"hvp(f::F, x::X, v::X) where {F, X}\n\nCompute the Hessian-vector product of an array-input scalar-output function f, as evaluated at x times the vector v.\n\nIn other words, compute hessian(f)(x) * v\n\nSee hvp! for a version which stores the result in an existing buffer and also hvp_and_gradient! for a function to compute both the hvp and the gradient in a single call.\n\nExample:\n\nf(x) = sin(x[1] * x[2])\n\nhvp(f, [2.0, 3.0], [5.0, 2.7])\n\n# output\n2-element Vector{Float64}:\n 19.6926882637302\n 16.201003759768003\n\n\n\n\n\n","category":"method"},{"location":"api/#Enzyme.hvp_and_gradient!-Union{Tuple{X}, Tuple{F}, Tuple{X, X, F, X, X}} where {F, X}","page":"API reference","title":"Enzyme.hvp_and_gradient!","text":"hvp_and_gradient!(res::X, grad::X, f::F, x::X, v::X) where {F, X}\n\nCompute an in-place Hessian-vector product of an array-input scalar-output function f, as evaluated at x times the vector v as well as the gradient, storing the gradient into grad. Both the hessian vector product and the gradient can be computed together more efficiently than computing them separately.\n\nThe result will be stored into res. The gradient will be stored into grad.\n\nIn other words, compute res .= hessian(f)(x) * v and grad .= gradient(Reverse, f)(x)\n\nExample:\n\nf(x) = sin(x[1] * x[2])\n\nres = Vector{Float64}(undef, 2)\ngrad = Vector{Float64}(undef, 2)\nhvp_and_gradient!(res, grad, f, [2.0, 3.0], [5.0, 2.7])\n\nres\ngrad\n# output\n2-element Vector{Float64}:\n 2.880510859951098\n 1.920340573300732\n\n\n\n\n\n","category":"method"},{"location":"api/#Enzyme.jacobian-Tuple{ForwardMode, Vararg{Any}}","page":"API reference","title":"Enzyme.jacobian","text":"jacobian(::ForwardMode, args...; kwargs...)\n\nEquivalent to gradient(::ForwardMode, args...; kwargs...)\n\n\n\n\n\n","category":"method"},{"location":"api/#Enzyme.jacobian-Union{Tuple{Holomorphic}, Tuple{CT}, Tuple{OutType}, Tuple{RuntimeActivity}, Tuple{ErrIfFuncWritten}, Tuple{RABI}, Tuple{X}, Tuple{F}, Tuple{ReturnPrimal}, Tuple{ReverseMode{ReturnPrimal, RuntimeActivity, RABI, Holomorphic, ErrIfFuncWritten}, F, X}} where {ReturnPrimal, F, X, RABI<:EnzymeCore.ABI, ErrIfFuncWritten, RuntimeActivity, OutType, CT, Holomorphic}","page":"API reference","title":"Enzyme.jacobian","text":"jacobian(::ReverseMode, f, x; n_outs=nothing, chunk=nothing)\njacobian(::ReverseMode, f, x)\n\nCompute the jacobian of a array-output function f using (potentially vector) reverse mode. The chunk argument optionally denotes the chunk size to use and n_outs optionally denotes the shape of the array returned by f (e.g size(f(x))).\n\nExample:\n\nf(x) = [ x[1] * x[2], x[2] + x[3] ]\n\njacobian(Reverse, f, [2.0, 3.0, 4.0])\n\n# output\n([3.0 2.0 0.0; 0.0 1.0 1.0],)\n\nf(x) = [ x[1] * x[2], x[2] + x[3] ]\n\ngrad = jacobian(ReverseWithPrimal, f, [2.0, 3.0, 4.0])\n\n# output\n(derivs = ([3.0 2.0 0.0; 0.0 1.0 1.0],), val = [6.0, 7.0])\n\nf(x) = [ x[1] * x[2], x[2] + x[3] ]\n\ngrad = jacobian(Reverse, f, [2.0, 3.0, 4.0], n_outs=Val((2,)))\n\n# output\n([3.0 2.0 0.0; 0.0 1.0 1.0],)\n\nf(x) = [ x[1] * x[2], x[2] + x[3] ]\n\ngrad = jacobian(ReverseWithPrimal, f, [2.0, 3.0, 4.0], n_outs=Val((2,)))\n\n# output\n(derivs = ([3.0 2.0 0.0; 0.0 1.0 1.0],), val = [6.0, 7.0])\n\nThis function will return an AbstractArray whose shape is (size(output)..., size(input)...). No guarantees are presently made about the type of the AbstractArray returned by this function (which may or may not be the same as the input AbstractArray if provided).\n\nIn the future, when this function is extended to handle non-array return types, this function will retun an AbstractArray of shape size(output) of values of the input type. ```\n\n\n\n\n\n","category":"method"},{"location":"api/#Enzyme.typetree","page":"API reference","title":"Enzyme.typetree","text":"function typetree(T, ctx, dl, seen=TypeTreeTable())\n\nConstruct a Enzyme typetree from a Julia type.\n\nwarning: Warning\nWhen using a memoized lookup by providing seen across multiple calls to typtree the user must call copy on the returned value before mutating it.\n\n\n\n\n\n","category":"function"},{"location":"api/#Enzyme.unsafe_to_pointer-Union{Tuple{Type{T}}, Tuple{T}} where T","page":"API reference","title":"Enzyme.unsafe_to_pointer","text":"unsafe_to_pointer\n\nwarning: Warning\nAssumes that val is globally rooted and pointer to it can be leaked. Prefer pointer_from_objref. Only use inside Enzyme.jl should be for Types.\n\n\n\n\n\n","category":"method"},{"location":"api/#EnzymeCore.autodiff-Union{Tuple{A}, Tuple{FA}, Tuple{RuntimeActivity}, Tuple{ErrIfFuncWritten}, Tuple{Nargs}, Tuple{RABI}, Tuple{ReturnPrimal}, Tuple{ForwardMode{ReturnPrimal, RABI, ErrIfFuncWritten, RuntimeActivity}, FA, Type{A}, Vararg{Annotation, Nargs}}} where {ReturnPrimal, RABI<:EnzymeCore.ABI, Nargs, ErrIfFuncWritten, RuntimeActivity, FA<:Annotation, A<:Annotation}","page":"API reference","title":"EnzymeCore.autodiff","text":"autodiff(::ForwardMode, f, Activity, args::Annotation...)\n\nAuto-differentiate function f at arguments args using forward mode.\n\nargs may be numbers, arrays, structs of numbers, structs of arrays and so on. Enzyme will only differentiate in respect to arguments that are wrapped in a Duplicated or similar argument. Unlike reverse mode in autodiff, Active arguments are not allowed here, since all derivative results of immutable objects will be returned and should instead use Duplicated or variants like DuplicatedNoNeed.\n\nActivity is the Activity of the return value, it may be:\n\nConst if the return is not to be differentiated with respect to\nDuplicated, if the return is being differentiated with respect to\nBatchDuplicated, like Duplicated, but computing multiple derivatives at once. All batch sizes must be the same for all arguments.\n\nExample returning both original return and derivative:\n\nf(x) = x*x\nres, ∂f_∂x = autodiff(ForwardWithPrimal, f, Duplicated, Duplicated(3.14, 1.0))\n\n# output\n\n(6.28, 9.8596)\n\nExample returning just the derivative:\n\nf(x) = x*x\n∂f_∂x = autodiff(Forward, f, Duplicated, Duplicated(3.14, 1.0))\n\n# output\n\n(6.28,)\n\n\n\n\n\n","category":"method"},{"location":"api/#EnzymeCore.autodiff-Union{Tuple{ErrIfFuncWritten}, Tuple{Nargs}, Tuple{Holomorphic}, Tuple{RABI}, Tuple{RuntimeActivity}, Tuple{ReturnPrimal}, Tuple{A}, Tuple{FA}, Tuple{ReverseMode{ReturnPrimal, RuntimeActivity, RABI, Holomorphic, ErrIfFuncWritten}, FA, Type{A}, Vararg{Annotation, Nargs}}} where {FA<:Annotation, A<:Annotation, ReturnPrimal, RuntimeActivity, RABI<:EnzymeCore.ABI, Holomorphic, Nargs, ErrIfFuncWritten}","page":"API reference","title":"EnzymeCore.autodiff","text":"autodiff(::ReverseMode, f, Activity, args::Annotation...)\n\nAuto-differentiate function f at arguments args using reverse mode.\n\nLimitations:\n\nf may only return a Real (of a built-in/primitive type) or nothing, not an array, struct, BigFloat, etc. To handle vector-valued return types, use a mutating f! that returns nothing and stores it's return value in one of the arguments, which must be wrapped in a Duplicated.\n\nargs may be numbers, arrays, structs of numbers, structs of arrays and so on. Enzyme will only differentiate in respect to arguments that are wrapped in an Active (for arguments whose derivative result must be returned rather than mutated in place, such as primitive types and structs thereof) or Duplicated (for mutable arguments like arrays, Refs and structs thereof).\n\nActivity is the Activity of the return value, it may be Const or Active.\n\nExample:\n\na = 4.2\nb = [2.2, 3.3]; ∂f_∂b = zero(b)\nc = 55; d = 9\n\nf(a, b, c, d) = a * √(b[1]^2 + b[2]^2) + c^2 * d^2\n∂f_∂a, _, _, ∂f_∂d = autodiff(Reverse, f, Active, Active(a), Duplicated(b, ∂f_∂b), Const(c), Active(d))[1]\n\n# output\n\n(3.966106403010388, nothing, nothing, 54450.0)\n\nhere, autodiff returns a tuple (partial fpartial a partial fpartial d), while partial fpartial b will be added to ∂f_∂b (but not returned). c will be treated as Const(c).\n\nOne can also request the original returned value of the computation.\n\nExample:\n\nEnzyme.autodiff(ReverseWithPrimal, x->x*x, Active(3.0))\n\n# output\n\n((6.0,), 9.0)\n\nnote: Note\nEnzyme gradients with respect to integer values are zero. Active will automatically convert plain integers to floating point values, but cannot do so for integer values in tuples and structs.\n\n\n\n\n\n","category":"method"},{"location":"api/#EnzymeCore.autodiff-Union{Tuple{MMode}, Tuple{Nargs}, Tuple{A}, Tuple{Function, MMode, Type{A}, Vararg{Annotation, Nargs}}} where {A<:Annotation, Nargs, MMode<:EnzymeCore.Mode}","page":"API reference","title":"EnzymeCore.autodiff","text":"autodiff(::Function, ::Mode, args...)\n\nSpecialization of autodiff to handle do argument closures.\n\n\nautodiff(Reverse, Active(3.1)) do x\n return x*x\nend\n\n# output\n((6.2,),)\n\n\n\n\n\n","category":"method"},{"location":"api/#EnzymeCore.autodiff-Union{Tuple{Nargs}, Tuple{CMode}, Tuple{FA}, Tuple{CMode, FA, Vararg{Annotation, Nargs}}} where {FA<:Annotation, CMode<:EnzymeCore.Mode, Nargs}","page":"API reference","title":"EnzymeCore.autodiff","text":"autodiff(mode::Mode, f, args...)\n\nLike autodiff but will try to guess the activity of the return value.\n\n\n\n\n\n","category":"method"},{"location":"api/#EnzymeCore.autodiff-Union{Tuple{Nargs}, Tuple{CMode}, Tuple{F}, Tuple{CMode, F, Vararg{Annotation, Nargs}}} where {F, CMode<:EnzymeCore.Mode, Nargs}","page":"API reference","title":"EnzymeCore.autodiff","text":"autodiff(mode::Mode, f, ::Type{A}, args::Annotation...)\n\nLike autodiff but will try to extend f to an annotation, if needed.\n\n\n\n\n\n","category":"method"},{"location":"api/#EnzymeCore.autodiff_deferred-Union{Tuple{RuntimeActivity}, Tuple{ErrIfFuncWritten}, Tuple{ABI}, Tuple{Nargs}, Tuple{A}, Tuple{FA}, Tuple{ReturnPrimal}, Tuple{ForwardMode{ReturnPrimal, ABI, ErrIfFuncWritten, RuntimeActivity}, FA, Type{A}, Vararg{Annotation, Nargs}}} where {ReturnPrimal, FA<:Annotation, A<:Annotation, Nargs, ABI, ErrIfFuncWritten, RuntimeActivity}","page":"API reference","title":"EnzymeCore.autodiff_deferred","text":"autodiff_deferred(::ForwardMode, f, Activity, args::Annotation...)\n\nSame as autodiff(::ForwardMode, f, Activity, args...) but uses deferred compilation to support usage in GPU code, as well as high-order differentiation.\n\n\n\n\n\n","category":"method"},{"location":"api/#EnzymeCore.autodiff_deferred-Union{Tuple{RuntimeActivity}, Tuple{ErrIfFuncWritten}, Tuple{Holomorphic}, Tuple{ABI}, Tuple{Nargs}, Tuple{ReturnPrimal}, Tuple{A}, Tuple{FA}, Tuple{ReverseMode{ReturnPrimal, RuntimeActivity, ABI, Holomorphic, ErrIfFuncWritten}, FA, Type{A}, Vararg{Annotation, Nargs}}} where {FA<:Annotation, A<:Annotation, ReturnPrimal, Nargs, ABI, Holomorphic, ErrIfFuncWritten, RuntimeActivity}","page":"API reference","title":"EnzymeCore.autodiff_deferred","text":"autodiff_deferred(::ReverseMode, f, Activity, args::Annotation...)\n\nSame as autodiff but uses deferred compilation to support usage in GPU code, as well as high-order differentiation.\n\n\n\n\n\n","category":"method"},{"location":"api/#EnzymeCore.autodiff_deferred_thunk-Union{Tuple{RuntimeActivity}, Tuple{ErrIfFuncWritten}, Tuple{Nargs}, Tuple{RABI}, Tuple{ModifiedBetweenT}, Tuple{Width}, Tuple{ReturnShadow}, Tuple{ReturnPrimal}, Tuple{TapeType}, Tuple{A2}, Tuple{FA}, Tuple{EnzymeCore.ReverseModeSplit{ReturnPrimal, ReturnShadow, RuntimeActivity, Width, ModifiedBetweenT, RABI, ErrIfFuncWritten}, Type{TapeType}, Type{FA}, Type{A2}, Vararg{Type{<:Annotation}, Nargs}}} where {FA<:Annotation, A2<:Annotation, TapeType, ReturnPrimal, ReturnShadow, Width, ModifiedBetweenT, RABI<:EnzymeCore.ABI, Nargs, ErrIfFuncWritten, RuntimeActivity}","page":"API reference","title":"EnzymeCore.autodiff_deferred_thunk","text":"autodiff_deferred_thunk(::ReverseModeSplit, ftype, Activity, argtypes::Type{<:Annotation}...)\n\nProvide the split forward and reverse pass functions for annotated function type ftype when called with args of type argtypes when using reverse mode.\n\nActivity is the Activity of the return value, it may be Const, Active, or Duplicated (or its variants DuplicatedNoNeed, BatchDuplicated, and BatchDuplicatedNoNeed).\n\nThe forward function will return a tape, the primal (or nothing if not requested), and the shadow (or nothing if not a Duplicated variant), and tapes the corresponding type arguements provided.\n\nThe reverse function will return the derivative of Active arguments, updating the Duplicated arguments in place. The same arguments to the forward pass should be provided, followed by the adjoint of the return (if the return is active), and finally the tape from the forward pass.\n\nExample:\n\n\nA = [2.2]; ∂A = zero(A)\nv = 3.3\n\nfunction f(A, v)\n res = A[1] * v\n A[1] = 0\n res\nend\n\nTapeType = tape_type(ReverseSplitWithPrimal, Const{typeof(f)}, Active, Duplicated{typeof(A)}, Active{typeof(v)})\nforward, reverse = autodiff_deferred_thunk(ReverseSplitWithPrimal, TapeType, Const{typeof(f)}, Active{Float64}, Duplicated{typeof(A)}, Active{typeof(v)})\n\ntape, result, shadow_result = forward(Const(f), Duplicated(A, ∂A), Active(v))\n_, ∂v = reverse(Const(f), Duplicated(A, ∂A), Active(v), 1.0, tape)[1]\n\nresult, ∂v, ∂A \n\n# output\n\n(7.26, 2.2, [3.3])\n\n\n\n\n\n","category":"method"},{"location":"api/#EnzymeCore.autodiff_thunk-Union{Tuple{RuntimeActivity}, Tuple{ErrIfFuncWritten}, Tuple{Nargs}, Tuple{RABI}, Tuple{A}, Tuple{FA}, Tuple{ReturnPrimal}, Tuple{ForwardMode{ReturnPrimal, RABI, ErrIfFuncWritten, RuntimeActivity}, Type{FA}, Type{A}, Vararg{Type{<:Annotation}, Nargs}}} where {ReturnPrimal, FA<:Annotation, A<:Annotation, RABI<:EnzymeCore.ABI, Nargs, ErrIfFuncWritten, RuntimeActivity}","page":"API reference","title":"EnzymeCore.autodiff_thunk","text":"autodiff_thunk(::ForwardMode, ftype, Activity, argtypes::Type{<:Annotation}...)\n\nProvide the thunk forward mode function for annotated function type ftype when called with args of type argtypes.\n\nActivity is the Activity of the return value, it may be Const or Duplicated (or its variants DuplicatedNoNeed, BatchDuplicated, andBatchDuplicatedNoNeed).\n\nThe forward function will return the primal (if requested) and the shadow (or nothing if not a Duplicated variant).\n\nExample returning both the return derivative and original return:\n\na = 4.2\nb = [2.2, 3.3]; ∂f_∂b = zero(b)\nc = 55; d = 9\n\nf(x) = x*x\nforward = autodiff_thunk(ForwardWithPrimal, Const{typeof(f)}, Duplicated, Duplicated{Float64})\nres, ∂f_∂x = forward(Const(f), Duplicated(3.14, 1.0))\n\n# output\n\n(6.28, 9.8596)\n\nExample returning just the derivative:\n\na = 4.2\nb = [2.2, 3.3]; ∂f_∂b = zero(b)\nc = 55; d = 9\n\nf(x) = x*x\nforward = autodiff_thunk(Forward, Const{typeof(f)}, Duplicated, Duplicated{Float64})\n∂f_∂x = forward(Const(f), Duplicated(3.14, 1.0))\n\n# output\n\n(6.28,)\n\n\n\n\n\n","category":"method"},{"location":"api/#EnzymeCore.autodiff_thunk-Union{Tuple{RuntimeActivity}, Tuple{ErrIfFuncWritten}, Tuple{Nargs}, Tuple{RABI}, Tuple{ModifiedBetweenT}, Tuple{Width}, Tuple{ReturnShadow}, Tuple{ReturnPrimal}, Tuple{A}, Tuple{FA}, Tuple{EnzymeCore.ReverseModeSplit{ReturnPrimal, ReturnShadow, RuntimeActivity, Width, ModifiedBetweenT, RABI, ErrIfFuncWritten}, Type{FA}, Type{A}, Vararg{Type{<:Annotation}, Nargs}}} where {FA<:Annotation, A<:Annotation, ReturnPrimal, ReturnShadow, Width, ModifiedBetweenT, RABI<:EnzymeCore.ABI, Nargs, ErrIfFuncWritten, RuntimeActivity}","page":"API reference","title":"EnzymeCore.autodiff_thunk","text":"autodiff_thunk(::ReverseModeSplit, ftype, Activity, argtypes::Type{<:Annotation}...)\n\nProvide the split forward and reverse pass functions for annotated function type ftype when called with args of type argtypes when using reverse mode.\n\nActivity is the Activity of the return value, it may be Const, Active, or Duplicated (or its variants DuplicatedNoNeed, BatchDuplicated, and BatchDuplicatedNoNeed).\n\nThe forward function will return a tape, the primal (or nothing if not requested), and the shadow (or nothing if not a Duplicated variant), and tapes the corresponding type arguements provided.\n\nThe reverse function will return the derivative of Active arguments, updating the Duplicated arguments in place. The same arguments to the forward pass should be provided, followed by the adjoint of the return (if the return is active), and finally the tape from the forward pass.\n\nExample:\n\n\nA = [2.2]; ∂A = zero(A)\nv = 3.3\n\nfunction f(A, v)\n res = A[1] * v\n A[1] = 0\n res\nend\n\nforward, reverse = autodiff_thunk(ReverseSplitWithPrimal, Const{typeof(f)}, Active, Duplicated{typeof(A)}, Active{typeof(v)})\n\ntape, result, shadow_result = forward(Const(f), Duplicated(A, ∂A), Active(v))\n_, ∂v = reverse(Const(f), Duplicated(A, ∂A), Active(v), 1.0, tape)[1]\n\nresult, ∂v, ∂A \n\n# output\n\n(7.26, 2.2, [3.3])\n\n\n\n\n\n","category":"method"},{"location":"api/#EnzymeCore.within_autodiff-Tuple{}","page":"API reference","title":"EnzymeCore.within_autodiff","text":"within_autodiff()\n\nReturns true if within autodiff, otherwise false.\n\n\n\n\n\n","category":"method"},{"location":"api/#EnzymeCore.ABI","page":"API reference","title":"EnzymeCore.ABI","text":"abstract type ABI\n\nAbstract type for what ABI will be used.\n\n\n\n\n\n","category":"type"},{"location":"api/#EnzymeCore.Active","page":"API reference","title":"EnzymeCore.Active","text":"Active(x)\n\nMark a function argument x of autodiff as active, Enzyme will auto-differentiate in respect Active arguments.\n\nnote: Note\nEnzyme gradients with respect to integer values are zero. Active will automatically convert plain integers to floating point values, but cannot do so for integer values in tuples and structs.\n\n\n\n\n\n","category":"type"},{"location":"api/#EnzymeCore.Annotation","page":"API reference","title":"EnzymeCore.Annotation","text":"abstract type Annotation{T}\n\nAbstract type for autodiff function argument wrappers like Const, Active and Duplicated.\n\n\n\n\n\n","category":"type"},{"location":"api/#EnzymeCore.BatchDuplicated","page":"API reference","title":"EnzymeCore.BatchDuplicated","text":"BatchDuplicated(x, ∂f_∂xs)\n\nLike Duplicated, except contains several shadows to compute derivatives for all at once. Argument ∂f_∂xs should be a tuple of the several values of type x.\n\n\n\n\n\n","category":"type"},{"location":"api/#EnzymeCore.BatchDuplicatedNoNeed","page":"API reference","title":"EnzymeCore.BatchDuplicatedNoNeed","text":"BatchDuplicatedNoNeed(x, ∂f_∂xs)\n\nLike DuplicatedNoNeed, except contains several shadows to compute derivatives for all at once. Argument ∂f_∂xs should be a tuple of the several values of type x.\n\n\n\n\n\n","category":"type"},{"location":"api/#EnzymeCore.BatchMixedDuplicated","page":"API reference","title":"EnzymeCore.BatchMixedDuplicated","text":"BatchMixedDuplicated(x, ∂f_∂xs)\n\nLike MixedDuplicated, except contains several shadows to compute derivatives for all at once. Only used within custom rules.\n\n\n\n\n\n","category":"type"},{"location":"api/#EnzymeCore.Const","page":"API reference","title":"EnzymeCore.Const","text":"Const(x)\n\nMark a function argument x of autodiff as constant, Enzyme will not auto-differentiate in respect Const arguments.\n\n\n\n\n\n","category":"type"},{"location":"api/#EnzymeCore.Duplicated","page":"API reference","title":"EnzymeCore.Duplicated","text":"Duplicated(x, ∂f_∂x)\n\nMark a function argument x of autodiff as duplicated, Enzyme will auto-differentiate in respect to such arguments, with dx acting as an accumulator for gradients (so partial f partial x will be added to) ∂f_∂x.\n\n\n\n\n\n","category":"type"},{"location":"api/#EnzymeCore.DuplicatedNoNeed","page":"API reference","title":"EnzymeCore.DuplicatedNoNeed","text":"DuplicatedNoNeed(x, ∂f_∂x)\n\nLike Duplicated, except also specifies that Enzyme may avoid computing the original result and only compute the derivative values. This creates opportunities for improved performance.\n\n\nfunction square_byref(out, v)\n out[] = v * v\n nothing\nend\n\nout = Ref(0.0)\ndout = Ref(1.0)\nEnzyme.autodiff(Reverse, square_byref, DuplicatedNoNeed(out, dout), Active(1.0))\ndout[]\n\n# output\n0.0\n\nFor example, marking the out variable as DuplicatedNoNeed instead of Duplicated allows Enzyme to avoid computing v * v (while still computing its derivative).\n\nThis should only be used if x is a write-only variable. Otherwise, if the differentiated function stores values in x and reads them back in subsequent computations, using DuplicatedNoNeed may result in incorrect derivatives. In particular, DuplicatedNoNeed should not be used for preallocated workspace, even if the user might not care about its final value, as marking a variable as NoNeed means that reads from the variable are now undefined.\n\n\n\n\n\n","category":"type"},{"location":"api/#EnzymeCore.FFIABI","page":"API reference","title":"EnzymeCore.FFIABI","text":"struct FFIABI <: ABI\n\nForeign function call ABI. JIT the differentiated function, then inttoptr call the address.\n\n\n\n\n\n","category":"type"},{"location":"api/#EnzymeCore.ForwardMode","page":"API reference","title":"EnzymeCore.ForwardMode","text":"struct Forward{ReturnPrimal, ABI, ErrIfFuncWritten,RuntimeActivity} <: Mode{ABI, ErrIfFuncWritten, RuntimeActivity}\n\nForward mode differentiation\n\n\n\n\n\n","category":"type"},{"location":"api/#EnzymeCore.InlineABI","page":"API reference","title":"EnzymeCore.InlineABI","text":"struct InlineABI <: ABI\n\nInlining function call ABI. \n\n\n\n\n\n","category":"type"},{"location":"api/#EnzymeCore.MixedDuplicated","page":"API reference","title":"EnzymeCore.MixedDuplicated","text":"MixedDuplicated(x, ∂f_∂x)\n\nLike Duplicated, except x may contain both active [immutable] and duplicated [mutable] data which is differentiable. Only used within custom rules.\n\n\n\n\n\n","category":"type"},{"location":"api/#EnzymeCore.Mode","page":"API reference","title":"EnzymeCore.Mode","text":"abstract type Mode\n\nAbstract type for what differentiation mode will be used.\n\n\n\n\n\n","category":"type"},{"location":"api/#EnzymeCore.NonGenABI","page":"API reference","title":"EnzymeCore.NonGenABI","text":"struct NonGenABI <: ABI\n\nNon-generated function ABI. \n\n\n\n\n\n","category":"type"},{"location":"api/#EnzymeCore.ReverseMode","page":"API reference","title":"EnzymeCore.ReverseMode","text":"struct ReverseMode{ReturnPrimal,RuntimeActivity,ABI,Holomorphic,ErrIfFuncWritten} <: Mode{ABI, ErrIfFuncWritten, RuntimeActivity}\n\nReverse mode differentiation.\n\nReturnPrimal: Should Enzyme return the primal return value from the augmented-forward.\nRuntimeActivity: Should Enzyme enable runtime activity (default off)\nABI: What runtime ABI to use\nHolomorphic: Whether the complex result function is holomorphic and we should compute d/dz\nErrIfFuncWritten: Should Enzyme err if the function differentiated is a closure and written to.\n\n\n\n\n\n","category":"type"},{"location":"api/#EnzymeCore.ReverseModeSplit","page":"API reference","title":"EnzymeCore.ReverseModeSplit","text":"struct ReverseModeSplit{ReturnPrimal,ReturnShadow,RuntimeActivity,Width,ModifiedBetween,ABI} <: Mode{ABI,ErrIfFuncWritten,RuntimeActivity}\n\nReverse mode differentiation.\n\nReturnPrimal: Should Enzyme return the primal return value from the augmented-forward.\nReturnShadow: Should Enzyme return the shadow return value from the augmented-forward.\nRuntimeActivity: Should Enzyme differentiate with runtime activity on (default off).\nWidth: Batch Size (0 if to be automatically derived)\nModifiedBetween: Tuple of each argument's modified between state (true if to be automatically derived).\n\n\n\n\n\n","category":"type"},{"location":"api/#EnzymeCore.NoPrimal-Union{Tuple{ReverseMode{ReturnPrimal, RuntimeActivity, ABI, Holomorphic, ErrIfFuncWritten}}, Tuple{ErrIfFuncWritten}, Tuple{Holomorphic}, Tuple{ABI}, Tuple{RuntimeActivity}, Tuple{ReturnPrimal}} where {ReturnPrimal, RuntimeActivity, ABI, Holomorphic, ErrIfFuncWritten}","page":"API reference","title":"EnzymeCore.NoPrimal","text":" NoPrimal(::Enzyme.Mode)\n\nModifies the mode to exclude the primal value.\n\n\n\n\n\n","category":"method"},{"location":"api/#EnzymeCore.WithPrimal-Union{Tuple{ReverseMode{ReturnPrimal, RuntimeActivity, ABI, Holomorphic, ErrIfFuncWritten}}, Tuple{ErrIfFuncWritten}, Tuple{Holomorphic}, Tuple{ABI}, Tuple{RuntimeActivity}, Tuple{ReturnPrimal}} where {ReturnPrimal, RuntimeActivity, ABI, Holomorphic, ErrIfFuncWritten}","page":"API reference","title":"EnzymeCore.WithPrimal","text":" WithPrimal(::Enzyme.Mode)\n\nModifies the mode to include the primal value.\n\n\n\n\n\n","category":"method"},{"location":"api/#EnzymeCore.compiler_job_from_backend","page":"API reference","title":"EnzymeCore.compiler_job_from_backend","text":"compiler_job_from_backend(::KernelAbstractions.Backend, F::Type, TT:Type)::GPUCompiler.CompilerJob\n\nReturns a GPUCompiler CompilerJob from a backend as specified by the first argument to the function.\n\nFor example, in CUDA one would do:\n\nfunction EnzymeCore.compiler_job_from_backend(::CUDABackend, @nospecialize(F::Type), @nospecialize(TT::Type))\n mi = GPUCompiler.methodinstance(F, TT)\n return GPUCompiler.CompilerJob(mi, CUDA.compiler_config(CUDA.device()))\nend\n\n\n\n\n\n","category":"function"},{"location":"api/#EnzymeCore.make_zero","page":"API reference","title":"EnzymeCore.make_zero","text":"make_zero(::Type{T}, seen::IdDict, prev::T, ::Val{copy_if_inactive}=Val(false))::T\n\nRecursively make a zero'd copy of the value `prev` of type `T`. The argument `copy_if_inactive` specifies\nwhat to do if the type `T` is guaranteed to be inactive, use the primal (the default) or still copy the value.\n\n\n\n\n\n","category":"function"},{"location":"api/#EnzymeCore.make_zero!","page":"API reference","title":"EnzymeCore.make_zero!","text":"make_zero!(val::T, seen::IdSet{Any}=IdSet())::Nothing\n\nRecursively set a variables differentiable fields to zero. Only applicable for mutable types `T`.\n\n\n\n\n\n","category":"function"},{"location":"api/#EnzymeCore.make_zero-Union{Tuple{copy_if_inactive}, Tuple{T}, Tuple{T, Val{copy_if_inactive}}} where {T, copy_if_inactive}","page":"API reference","title":"EnzymeCore.make_zero","text":"make_zero(prev::T)\n\nHelper function to recursively make zero.\n\n\n\n\n\n","category":"method"},{"location":"api/#EnzymeCore.within_autodiff","page":"API reference","title":"EnzymeCore.within_autodiff","text":"within_autodiff()\n\nReturns true if within autodiff, otherwise false.\n\n\n\n\n\n","category":"function"},{"location":"api/#EnzymeCore.EnzymeRules.AugmentedReturn","page":"API reference","title":"EnzymeCore.EnzymeRules.AugmentedReturn","text":"AugmentedReturn(primal, shadow, tape)\n\nAugment the primal return value of a function with its shadow, as well as any additional information needed to correctly compute the reverse pass, stored in tape.\n\nUnless specified by the config that a variable is not overwritten, rules must assume any arrays/data structures/etc are overwritten between the forward and the reverse pass. Any floats or variables passed by value are always preserved as is (as are the arrays themselves, just not necessarily the values in the array).\n\nSee also augmented_primal.\n\n\n\n\n\n","category":"type"},{"location":"api/#EnzymeCore.EnzymeRules.FwdConfig","page":"API reference","title":"EnzymeCore.EnzymeRules.FwdConfig","text":"FwdConfig{NeedsPrimal, NeedsShadow, Width, RuntimeActivity}\nFwdConfigWidth{Width} = FwdConfig{<:Any, <:Any, Width}\n\nConfiguration type to dispatch on in custom forward rules (see forward.\n\nNeedsPrimal and NeedsShadow: boolean values specifying whether the primal and shadow (resp.) should be returned. \nWidth: an integer that specifies the number of adjoints/shadows simultaneously being propagated.\nRuntimeActivity: whether runtime activity is enabled.\n\nGetters for the type parameters are provided by needs_primal, needs_shadow, width and runtime_activity.\n\n\n\n\n\n","category":"type"},{"location":"api/#EnzymeCore.EnzymeRules.RevConfig","page":"API reference","title":"EnzymeCore.EnzymeRules.RevConfig","text":"RevConfig{NeedsPrimal, NeedsShadow, Width, Overwritten, RuntimeActivity}\nRevConfigWidth{Width} = RevConfig{<:Any, <:Any, Width}\n\nConfiguration type to dispatch on in custom reverse rules (see augmented_primal and reverse).\n\nNeedsPrimal and NeedsShadow: boolean values specifying whether the primal and shadow (resp.) should be returned. \nWidth: an integer that specifies the number of adjoints/shadows simultaneously being propagated.\nOverwritten: a tuple of booleans of whether each argument (including the function itself) is modified between the forward and reverse pass (true if potentially modified between).\nRuntimeActivity: whether runtime activity is enabled.\n\nGetters for the four type parameters are provided by needs_primal, needs_shadow, width, overwritten, and runtime_activity.\n\n\n\n\n\n","category":"type"},{"location":"api/#EnzymeCore.EnzymeRules.augmented_primal","page":"API reference","title":"EnzymeCore.EnzymeRules.augmented_primal","text":"augmented_primal(::RevConfig, func::Annotation{typeof(f)}, RT::Type{<:Annotation}, args::Annotation...)\n\nMust return an AugmentedReturn type.\n\nThe primal must be the same type of the original return if needs_primal(config), otherwise nothing.\nThe shadow must be nothing if needs_shadow(config) is false. If width is 1, the shadow should be the same type of the original return. If the width is greater than 1, the shadow should be NTuple{original return, width}.\nThe tape can be any type (including Nothing) and is preserved for the reverse call.\n\n\n\n\n\n","category":"function"},{"location":"api/#EnzymeCore.EnzymeRules.forward","page":"API reference","title":"EnzymeCore.EnzymeRules.forward","text":"forward(fwdconfig, func::Annotation{typeof(f)}, RT::Type{<:Annotation}, args::Annotation...)\n\nCalculate the forward derivative. The first argument is a `FwdConfig object describing parameters of the differentiation. The second argument func is the callable for which the rule applies to. Either wrapped in a Const), or a Duplicated if it is a closure. The third argument is the return type annotation, and all other arguments are the annotated function arguments.\n\n\n\n\n\n","category":"function"},{"location":"api/#EnzymeCore.EnzymeRules.inactive","page":"API reference","title":"EnzymeCore.EnzymeRules.inactive","text":"inactive(func::typeof(f), args...)\n\nMark a particular function as always being inactive in both its return result and the function call itself.\n\n\n\n\n\n","category":"function"},{"location":"api/#EnzymeCore.EnzymeRules.inactive_noinl","page":"API reference","title":"EnzymeCore.EnzymeRules.inactive_noinl","text":"inactive_noinl(func::typeof(f), args...)\n\nMark a particular function as always being inactive in both its return result and the function call itself, but do not prevent inlining of the function.\n\n\n\n\n\n","category":"function"},{"location":"api/#EnzymeCore.EnzymeRules.inactive_type-Tuple{Type}","page":"API reference","title":"EnzymeCore.EnzymeRules.inactive_type","text":"inactive_type(::Type{Ty})\n\nMark a particular type Ty as always being inactive.\n\n\n\n\n\n","category":"method"},{"location":"api/#EnzymeCore.EnzymeRules.needs_primal-Union{Tuple{EnzymeCore.EnzymeRules.FwdConfigWidth{Width, NeedsPrimal, NeedsShadow} where {NeedsShadow, Width}}, Tuple{NeedsPrimal}} where NeedsPrimal","page":"API reference","title":"EnzymeCore.EnzymeRules.needs_primal","text":"needs_primal(::FwdConfig)\nneeds_primal(::RevConfig)\n\nWhether a custom rule should return the original result of the function.\n\n\n\n\n\n","category":"method"},{"location":"api/#EnzymeCore.EnzymeRules.needs_shadow-Union{Tuple{EnzymeCore.EnzymeRules.FwdConfigWidth{Width, var\"#s9\", NeedsShadow} where {var\"#s9\", Width}}, Tuple{NeedsShadow}} where NeedsShadow","page":"API reference","title":"EnzymeCore.EnzymeRules.needs_shadow","text":"needs_shadow(::FwdConfig)\nneeds_shadow(::RevConfig)\n\nWhether a custom rule should return the shadow (derivative) of the function result.\n\n\n\n\n\n","category":"method"},{"location":"api/#EnzymeCore.EnzymeRules.noalias","page":"API reference","title":"EnzymeCore.EnzymeRules.noalias","text":"noalias(func::typeof(f), args...)\n\nMark a particular function as always being a fresh allocation which does not alias any other accessible memory.\n\n\n\n\n\n","category":"function"},{"location":"api/#EnzymeCore.EnzymeRules.primal_type-Union{Tuple{RT}, Tuple{EnzymeCore.EnzymeRules.FwdConfig, Type{<:Annotation{RT}}}} where RT","page":"API reference","title":"EnzymeCore.EnzymeRules.primal_type","text":"primal_type(::FwdConfig, ::Type{<:Annotation{RT}})\nprimal_type(::RevConfig, ::Type{<:Annotation{RT}})\n\nCompute the exepcted primal return type given a reverse mode config and return activity\n\n\n\n\n\n","category":"method"},{"location":"api/#EnzymeCore.EnzymeRules.reverse","page":"API reference","title":"EnzymeCore.EnzymeRules.reverse","text":"reverse(::RevConfig, func::Annotation{typeof(f)}, dret::Active, tape, args::Annotation...)\nreverse(::RevConfig, func::Annotation{typeof(f)}, ::Type{<:Annotation), tape, args::Annotation...)\n\nTakes gradient of derivative, activity annotation, and tape. If there is an active return dret is passed as Active{T} with the derivative of the active return val. Otherwise dret is passed as Type{Duplicated{T}}, etc.\n\n\n\n\n\n","category":"function"},{"location":"api/#EnzymeCore.EnzymeRules.shadow_type-Union{Tuple{RT}, Tuple{EnzymeCore.EnzymeRules.FwdConfig, Type{<:Annotation{RT}}}} where RT","page":"API reference","title":"EnzymeCore.EnzymeRules.shadow_type","text":"shadow_type(::FwdConfig, ::Type{<:Annotation{RT}})\nshadow_type(::RevConfig, ::Type{<:Annotation{RT}})\n\nCompute the exepcted shadow return type given a reverse mode config and return activity\n\n\n\n\n\n","category":"method"},{"location":"api/#EnzymeTestUtils.ExprAndMsg","page":"API reference","title":"EnzymeTestUtils.ExprAndMsg","text":"A cunning hack to carry extra message along with the original expression in a test\n\n\n\n\n\n","category":"type"},{"location":"api/#EnzymeTestUtils.@test_msg-Tuple{Any, Any, Vararg{Any}}","page":"API reference","title":"EnzymeTestUtils.@test_msg","text":"@test_msg msg condion kws...\n\nThis is per Test.@test condion kws... except that if it fails it also prints the msg. If msg==\"\" then this is just like @test, nothing is printed\n\nExamles\n\njulia> @test_msg \"It is required that the total is under 10\" sum(1:1000) < 10;\nTest Failed at REPL[1]:1\n Expression: sum(1:1000) < 10\n Problem: It is required that the total is under 10\n Evaluated: 500500 < 10\nERROR: There was an error during testing\n\n\njulia> @test_msg \"It is required that the total is under 10\" error(\"not working at all\");\nError During Test at REPL[2]:1\n Test threw exception\n Expression: error(\"not working at all\")\n Problem: It is required that the total is under 10\n \"not working at all\"\n Stacktrace:\n\njulia> a = \"\";\n\njulia> @test_msg a sum(1:1000) < 10;\n Test Failed at REPL[153]:1\n Expression: sum(1:1000) < 10\n Evaluated: 500500 < 10\n ERROR: There was an error during testing\n\n\n\n\n\n","category":"macro"},{"location":"api/#EnzymeTestUtils.are_activities_compatible-Tuple{Any, Vararg{Any}}","page":"API reference","title":"EnzymeTestUtils.are_activities_compatible","text":"are_activities_compatible(Tret, activities...) -> Bool\n\nReturn true if return activity type Tret and activity types activities are compatible.\n\n\n\n\n\n","category":"method"},{"location":"api/#EnzymeTestUtils.test_forward-Tuple{Any, Any, Vararg{Any}}","page":"API reference","title":"EnzymeTestUtils.test_forward","text":"test_forward(f, Activity, args...; kwargs...)\n\nTest Enzyme.autodiff of f in Forward-mode against finite differences.\n\nf has all constraints of the same argument passed to Enzyme.autodiff, with additional constraints:\n\nIf it mutates one of its arguments, it must return that argument.\n\nArguments\n\nActivity: the activity of the return value of f\nargs: Each entry is either an argument to f, an activity type accepted by autodiff, or a tuple of the form (arg, Activity), where Activity is the activity type of arg. If the activity type specified requires a tangent, a random tangent will be automatically generated.\n\nKeywords\n\nrng::AbstractRNG: The random number generator to use for generating random tangents.\nfdm=FiniteDifferences.central_fdm(5, 1): The finite differences method to use.\nfkwargs: Keyword arguments to pass to f.\nrtol: Relative tolerance for isapprox.\natol: Absolute tolerance for isapprox.\ntestset_name: Name to use for a testset in which all tests are evaluated.\n\nExamples\n\nHere we test a rule for a function of scalars. Because we don't provide an activity annotation for y, it is assumed to be Const.\n\nusing Enzyme, EnzymeTestUtils\n\nx, y = randn(2)\nfor Tret in (Const, Duplicated, DuplicatedNoNeed), Tx in (Const, Duplicated)\n test_forward(*, Tret, (x, Tx), y)\nend\n\nHere we test a rule for a function of an array in batch forward-mode:\n\nx = randn(3)\ny = randn()\nfor Tret in (Const, BatchDuplicated, BatchDuplicatedNoNeed),\n Tx in (Const, BatchDuplicated),\n Ty in (Const, BatchDuplicated)\n\n test_forward(*, Tret, (x, Tx), (y, Ty))\nend\n\n\n\n\n\n","category":"method"},{"location":"api/#EnzymeTestUtils.test_reverse-Tuple{Any, Any, Vararg{Any}}","page":"API reference","title":"EnzymeTestUtils.test_reverse","text":"test_reverse(f, Activity, args...; kwargs...)\n\nTest Enzyme.autodiff_thunk of f in ReverseSplitWithPrimal-mode against finite differences.\n\nf has all constraints of the same argument passed to Enzyme.autodiff_thunk, with additional constraints:\n\nIf an Array{<:AbstractFloat} appears in the input/output, then a reshaped version of it may not also appear in the input/output.\n\nArguments\n\nActivity: the activity of the return value of f.\nargs: Each entry is either an argument to f, an activity type accepted by autodiff, or a tuple of the form (arg, Activity), where Activity is the activity type of arg. If the activity type specified requires a shadow, one will be automatically generated.\n\nKeywords\n\nrng::AbstractRNG: The random number generator to use for generating random tangents.\nfdm=FiniteDifferences.central_fdm(5, 1): The finite differences method to use.\nfkwargs: Keyword arguments to pass to f.\nrtol: Relative tolerance for isapprox.\natol: Absolute tolerance for isapprox.\ntestset_name: Name to use for a testset in which all tests are evaluated.\n\nExamples\n\nHere we test a rule for a function of scalars. Because we don't provide an activity annotation for y, it is assumed to be Const.\n\nusing Enzyme, EnzymeTestUtils\n\nx = randn()\ny = randn()\nfor Tret in (Const, Active), Tx in (Const, Active)\n test_reverse(*, Tret, (x, Tx), y)\nend\n\nHere we test a rule for a function of an array in batch reverse-mode:\n\nx = randn(3)\nfor Tret in (Const, Active), Tx in (Const, BatchDuplicated)\n test_reverse(prod, Tret, (x, Tx))\nend\n\n\n\n\n\n","category":"method"},{"location":"api/#Enzyme.API.fast_math!-Tuple{Any}","page":"API reference","title":"Enzyme.API.fast_math!","text":"fast_math!(val::Bool)\n\nWhether generated derivatives have fast math on or off, default on.\n\n\n\n\n\n","category":"method"},{"location":"api/#Enzyme.API.inlineall!-Tuple{Any}","page":"API reference","title":"Enzyme.API.inlineall!","text":"inlineall!(val::Bool)\n\nWhether to inline all (non-recursive) functions generated by Julia within a single compilation unit. This may improve Enzyme's ability to successfully differentiate code and improve performance of the original and generated derivative program. It often, however, comes with an increase in compile time. This is off by default.\n\n\n\n\n\n","category":"method"},{"location":"api/#Enzyme.API.instname!-Tuple{Any}","page":"API reference","title":"Enzyme.API.instname!","text":"instname!(val::Bool)\n\nWhether to add a name to all LLVM values. This may be helpful for debugging generated programs, both primal and derivative. Off by default.\n\n\n\n\n\n","category":"method"},{"location":"api/#Enzyme.API.looseTypeAnalysis!-Tuple{Any}","page":"API reference","title":"Enzyme.API.looseTypeAnalysis!","text":"looseTypeAnalysis!(val::Bool)\n\nEnzyme runs a type analysis to deduce the corresponding types of all values being differentiated. This is necessary to compute correct derivatives of various values. For example, a copy of Float32's requires a different derivative than a memcpy of Float64's, Ptr's, etc. In some cases Enzyme may not be able to deduce all the types necessary and throw an unknown type error. If this is the case, open an issue. One can silence these issues by setting looseTypeAnalysis!(true) which tells Enzyme to make its best guess. This will remove the error and allow differentiation to continue, however, it may produce incorrect results. Alternatively one can consider increasing the space of the evaluated type lattice which gives Enzyme more time to run a more thorough analysis through the use of maxtypeoffset!\n\n\n\n\n\n","category":"method"},{"location":"api/#Enzyme.API.maxtypedepth!-Tuple{Any}","page":"API reference","title":"Enzyme.API.maxtypedepth!","text":"maxtypedepth!(val::Int)\n\nEnzyme runs a type analysis to deduce the corresponding types of all values being differentiated. This is necessary to compute correct derivatives of various values. To ensure this analysis temrinates, it operates on a finite lattice of possible states. This function sets the maximum depth into a type that Enzyme will consider. A smaller value will cause type analysis to run faster, but may result in some necessary types not being found and result in unknown type errors. A larger value may result in unknown type errors being resolved by searching a larger space, but may run longer. The default setting is 6.\n\n\n\n\n\n","category":"method"},{"location":"api/#Enzyme.API.maxtypeoffset!-Tuple{Any}","page":"API reference","title":"Enzyme.API.maxtypeoffset!","text":"maxtypeoffset!(val::Int)\n\nEnzyme runs a type analysis to deduce the corresponding types of all values being differentiated. This is necessary to compute correct derivatives of various values. To ensure this analysis temrinates, it operates on a finite lattice of possible states. This function sets the maximum offset into a type that Enzyme will consider. A smaller value will cause type analysis to run faster, but may result in some necessary types not being found and result in unknown type errors. A larger value may result in unknown type errors being resolved by searching a larger space, but may run longer. The default setting is 512.\n\n\n\n\n\n","category":"method"},{"location":"api/#Enzyme.API.memmove_warning!-Tuple{Any}","page":"API reference","title":"Enzyme.API.memmove_warning!","text":"memmove_warning!(val::Bool)\n\nWhether to issue a warning when differentiating memmove. Off by default.\n\n\n\n\n\n","category":"method"},{"location":"api/#Enzyme.API.printactivity!-Tuple{Any}","page":"API reference","title":"Enzyme.API.printactivity!","text":"printactivity!(val::Bool)\n\nAn debugging option for developers of Enzyme. If one sets this flag prior to the first differentiation of a function, Enzyme will print (to stderr) a log of all decisions made during Activity Analysis (the analysis which determines what values/instructions are differentiated). This may be useful for debugging MixedActivity errors, correctness, and performance errors. Off by default\n\n\n\n\n\n","category":"method"},{"location":"api/#Enzyme.API.printall!-Tuple{Any}","page":"API reference","title":"Enzyme.API.printall!","text":"printall!(val::Bool)\n\nAn debugging option for developers of Enzyme. If one sets this flag prior to the first differentiation of a function, Enzyme will print (to stderr) the LLVM function being differentiated, as well as all generated derivatives immediately after running Enzyme (but prior to any other optimizations). Off by default\n\n\n\n\n\n","category":"method"},{"location":"api/#Enzyme.API.printdiffuse!-Tuple{Any}","page":"API reference","title":"Enzyme.API.printdiffuse!","text":"printdiffuse!(val::Bool)\n\nAn debugging option for developers of Enzyme. If one sets this flag prior to the first differentiation of a function, Enzyme will print (to stderr) information about each LLVM value – specifically whether it and its shadow is required for computing the derivative. In contrast to printunnecessary!, this flag prints debug log for the analysis which determines for each value and shadow value, whether it can find a user which would require it to be kept around (rather than being deleted). This is prior to any cache optimizations and a debug log of Differential Use Analysis. This may be helpful for debugging caching, phi node deletion, performance, and other errors. Off by default\n\n\n\n\n\n","category":"method"},{"location":"api/#Enzyme.API.printperf!-Tuple{Any}","page":"API reference","title":"Enzyme.API.printperf!","text":"printperf!(val::Bool)\n\nAn debugging option for developers of Enzyme. If one sets this flag prior to the first differentiation of a function, Enzyme will print (to stderr) performance information about generated derivative programs. It will provide debug information that warns why particular values are cached for the reverse pass, and thus require additional computation/storage. This is particularly helpful for debugging derivatives which OOM or otherwise run slow. ff by default\n\n\n\n\n\n","category":"method"},{"location":"api/#Enzyme.API.printtype!-Tuple{Any}","page":"API reference","title":"Enzyme.API.printtype!","text":"printtype!(val::Bool)\n\nAn debugging option for developers of Enzyme. If one sets this flag prior to the first differentiation of a function, Enzyme will print (to stderr) a log of all decisions made during Type Analysis (the analysis which Enzyme determines the type of all values in the program). This may be useful for debugging correctness errors, illegal type analysis errors, insufficient type information errors, correctness, and performance errors. Off by default\n\n\n\n\n\n","category":"method"},{"location":"api/#Enzyme.API.printunnecessary!-Tuple{Any}","page":"API reference","title":"Enzyme.API.printunnecessary!","text":"printunnecessary!(val::Bool)\n\nAn debugging option for developers of Enzyme. If one sets this flag prior to the first differentiation of a function, Enzyme will print (to stderr) information about each LLVM value – specifically whether it and its shadow is required for computing the derivative. In contrast to printdiffuse!, this flag prints the final results after running cache optimizations such as minCut (see Recompute vs Cache Heuristics from this paper and slides 31-33 from this presentation) for a description of the caching algorithm. This may be helpful for debugging caching, phi node deletion, performance, and other errors. Off by default\n\n\n\n\n\n","category":"method"},{"location":"api/#Enzyme.API.strictAliasing!-Tuple{Any}","page":"API reference","title":"Enzyme.API.strictAliasing!","text":"strictAliasing!(val::Bool)\n\nWhether Enzyme's type analysis will assume strict aliasing semantics. When strict aliasing semantics are on (the default), Enzyme can propagate type information up through conditional branches. This may lead to illegal type errors when analyzing code with unions. Disabling strict aliasing will enable these union types to be correctly analyzed. However, it may lead to some errors that sufficient type information cannot be deduced. One can turn these insufficient type information errors into to warnings by calling looseTypeAnalysis!(true) which tells Enzyme to use its best guess in such scenarios.\n\n\n\n\n\n","category":"method"},{"location":"api/#Enzyme.API.strong_zero!-Tuple{Any}","page":"API reference","title":"Enzyme.API.strong_zero!","text":"strong_zero!(val::Bool)\n\nWhether to enforce multiplication by zero as enforcing a zero result even if multiplying against a NaN or infinity. Necessary for some programs in which a value has a zero derivative since it is unused, even if it has an otherwise infinite or nan derivative.\n\n\n\n\n\n","category":"method"},{"location":"api/#Enzyme.API.typeWarning!-Tuple{Any}","page":"API reference","title":"Enzyme.API.typeWarning!","text":"typeWarning!(val::Bool)\n\nWhether to print a warning when Type Analysis learns informatoin about a value's type which cannot be represented in the current size of the lattice. See maxtypeoffset! for more information. Off by default.\n\n\n\n\n\n","category":"method"},{"location":"dev_docs/#Enzyme-developer-documentation","page":"For developers","title":"Enzyme developer documentation","text":"","category":"section"},{"location":"dev_docs/#Development-of-Enzyme-and-Enzyme.jl-together-(recommended)","page":"For developers","title":"Development of Enzyme and Enzyme.jl together (recommended)","text":"","category":"section"},{"location":"dev_docs/","page":"For developers","title":"For developers","text":"Normally Enzyme.jl downloads and installs Enzyme for the user automatically since Enzyme needs to be built against Julia bundeled LLVM. In case that you are making updates to Enzyme and want to test them against Enzyme.jl the instructions below should help you get started.","category":"page"},{"location":"dev_docs/","page":"For developers","title":"For developers","text":"Start Julia in your development copy of Enzyme.jl and initialize the deps project","category":"page"},{"location":"dev_docs/","page":"For developers","title":"For developers","text":"~/s/Enzyme.jl (master)> julia --project=deps","category":"page"},{"location":"dev_docs/","page":"For developers","title":"For developers","text":"julia> # Hit the `]` key to enter package repl.\n(deps) pkg> instantiate","category":"page"},{"location":"dev_docs/","page":"For developers","title":"For developers","text":"We can now build a custom version of Enzyme for use in Enzyme.jl. To build the latest commit on the main branch of Enzyme, run the following. It may take a few minutes to compile fully.","category":"page"},{"location":"dev_docs/","page":"For developers","title":"For developers","text":"~/s/Enzyme.jl (master)> julia --project=deps deps/build_local.jl","category":"page"},{"location":"dev_docs/","page":"For developers","title":"For developers","text":"You will now find a file LocalPrefernces.toml which has been generated and contains a path to the new Enzyme_jll binary you have built. To use your Enzyme_jll instead of the default shipped by Enzyme.jl, ensure that this file is at the root of any Julia project you wish to test it with and that the Julia project has Enzyme_jll as an explicit dependency. Note that an indirect dependency here is not sufficient (e.g. just because a project depends on Enzyme.jl, which depends on Enzyme_jll, does not mean that your project will pick up this file unless you also add a direct dependency to Enzyme_jll).","category":"page"},{"location":"dev_docs/","page":"For developers","title":"For developers","text":"To test whether your project found the custom version of Enzyme_jll, you can inspect the path of the Enzyme_jll library in use as follows.","category":"page"},{"location":"dev_docs/","page":"For developers","title":"For developers","text":"~/my/project.jl (master)> julia --project=.","category":"page"},{"location":"dev_docs/","page":"For developers","title":"For developers","text":"julia> using Enzyme_jll\njulia> Enzyme_jll.libEnzyme_path\n\"${JULIA_PKG_DEVDIR}/Enzyme_jll/override/lib/LLVMEnzyme-9.so\"","category":"page"},{"location":"dev_docs/","page":"For developers","title":"For developers","text":"This should correspond to the path in the LocalPreferences.toml you just generated.","category":"page"},{"location":"dev_docs/","page":"For developers","title":"For developers","text":"Note that your system can have only one custom built Enzyme_jll at a time. If you build one version for one version of Enzyme or Julia and later build a new version of Enzyme, it removes the old build. ","category":"page"},{"location":"dev_docs/","page":"For developers","title":"For developers","text":"Note that Julia versions are tightly coupled and you cannot use an Enzyme_jll built for one version of Julia for another version of Julia.","category":"page"},{"location":"dev_docs/","page":"For developers","title":"For developers","text":"The same script can also be used to build Enzyme_jll for a branch other than main as follows.","category":"page"},{"location":"dev_docs/","page":"For developers","title":"For developers","text":"~/s/Enzyme.jl (master)> julia --project=deps deps/build_local.jl --branch mybranch","category":"page"},{"location":"dev_docs/","page":"For developers","title":"For developers","text":"It can also be used to build Enzyme_jll from a local copy of Enzyme on your machine, which does not need to be committed to git.","category":"page"},{"location":"dev_docs/","page":"For developers","title":"For developers","text":"~/s/Enzyme.jl (master)> julia --project=deps deps/build_local.jl ../path/to/Enzyme","category":"page"},{"location":"dev_docs/#Development-of-Enzyme-and-Enzyme.jl-together-(manual)","page":"For developers","title":"Development of Enzyme and Enzyme.jl together (manual)","text":"","category":"section"},{"location":"dev_docs/","page":"For developers","title":"For developers","text":"Start Julia in your development copy of Enzyme.jl","category":"page"},{"location":"dev_docs/","page":"For developers","title":"For developers","text":"~/s/Enzyme.jl (master)> julia --project=.","category":"page"},{"location":"dev_docs/","page":"For developers","title":"For developers","text":"Then create a development copy of Enzyme_jll and activate it within.","category":"page"},{"location":"dev_docs/","page":"For developers","title":"For developers","text":"julia> using Enzyme_jll\njulia> Enzyme_jll.dev_jll()\n[ Info: Enzyme_jll dev'ed out to ${JULIA_PKG_DEVDIR}/Enzyme_jll with pre-populated override directory\n(Enzyme) pkg> dev Enzyme_jll\nPath `${JULIA_PKG_DEVDIR}/Enzyme_jll` exists and looks like the correct package. Using existing path.","category":"page"},{"location":"dev_docs/","page":"For developers","title":"For developers","text":"After restarting Julia:","category":"page"},{"location":"dev_docs/","page":"For developers","title":"For developers","text":"julia> Enzyme_jll.dev_jll()\njulia> Enzyme_jll.libEnzyme_path\n\"${JULIA_PKG_DEVDIR}/Enzyme_jll/override/lib/LLVMEnzyme-9.so\"","category":"page"},{"location":"dev_docs/","page":"For developers","title":"For developers","text":"On your machine ${JULIA_PKG_DEVDIR} most likely corresponds to ~/.julia/dev. Now we can inspect \"${JULIA_PKG_DEVDIR}/Enzyme_jll/override/lib and see that there is a copy of LLVMEnzyme-9.so, which we can replace with a symbolic link or a copy of a version of Enzyme.","category":"page"},{"location":"dev_docs/#Building-Enzyme-against-Julia's-LLVM.","page":"For developers","title":"Building Enzyme against Julia's LLVM.","text":"","category":"section"},{"location":"dev_docs/","page":"For developers","title":"For developers","text":"Depending on how you installed Julia the LLVM Julia is using will be different.","category":"page"},{"location":"dev_docs/","page":"For developers","title":"For developers","text":"Download from julialang.org (Recommended)\nManual build on your machine\nUses a pre-built Julia from your system vendor (Not recommended)","category":"page"},{"location":"dev_docs/","page":"For developers","title":"For developers","text":"To check what LLVM Julia is using use:","category":"page"},{"location":"dev_docs/","page":"For developers","title":"For developers","text":"julia> Base.libllvm_version_string\n\"9.0.1jl\"","category":"page"},{"location":"dev_docs/","page":"For developers","title":"For developers","text":"If the LLVM version ends in a jl you are likely using the private LLVM.","category":"page"},{"location":"dev_docs/","page":"For developers","title":"For developers","text":"In your source checkout of Enzyme:","category":"page"},{"location":"dev_docs/","page":"For developers","title":"For developers","text":"mkdir build-jl\ncd build-jl","category":"page"},{"location":"dev_docs/#Prebuilt-binary-from-julialang.org","page":"For developers","title":"Prebuilt binary from julialang.org","text":"","category":"section"},{"location":"dev_docs/","page":"For developers","title":"For developers","text":"LLVM_MAJOR_VER=`julia -e \"print(Base.libllvm_version.major)\"`\njulia -e \"using Pkg; pkg\\\"add LLVM_full_jll@${LLVM_MAJOR_VER}\\\"\"\nLLVM_DIR=`julia -e \"using LLVM_full_jll; print(LLVM_full_jll.artifact_dir)\"`\necho \"LLVM_DIR=$LLVM_DIR\"\ncmake ../enzyme/ -G Ninja -DENZYME_EXTERNAL_SHARED_LIB=ON -DLLVM_DIR=${LLVM_DIR} -DLLVM_EXTERNAL_LIT=${LLVM_DIR}/tools/lit/lit.py","category":"page"},{"location":"dev_docs/#Manual-build-of-Julia","page":"For developers","title":"Manual build of Julia","text":"","category":"section"},{"location":"dev_docs/","page":"For developers","title":"For developers","text":"cmake ../enzyme/ -G Ninja -DENZYME_EXTERNAL_SHARED_LIB=ON -DLLVM_DIR=${PATH_TO_BUILDDIR_OF_JULIA}/usr/lib/cmake/llvm/","category":"page"},{"location":"internal_api/#Internal-API","page":"Internal API","title":"Internal API","text":"","category":"section"},{"location":"internal_api/","page":"Internal API","title":"Internal API","text":"note: Note\nThis is the documentation of Enzymes's internal API. The internal API is not subject to semantic versioning and may change at any time and without deprecation.","category":"page"},{"location":"internal_api/","page":"Internal API","title":"Internal API","text":"Modules = [Enzyme.Compiler]\nOrder = [:module, :type, :constant, :macro, :function]","category":"page"},{"location":"internal_api/#Enzyme.Compiler.fspec","page":"Internal API","title":"Enzyme.Compiler.fspec","text":"Create the methodinstance pair, and lookup the primal return type.\n\n\n\n\n\n","category":"function"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"EditURL = \"../../../examples/box.jl\"","category":"page"},{"location":"generated/box/#Enzyme-for-adjoint-tutorial:-Stommel-three-box-ocean-model","page":"Box model","title":"Enzyme for adjoint tutorial: Stommel three-box ocean model","text":"","category":"section"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"The goal of this tutorial is to teach about a specific usage of Enzyme's automatic differentiation capabilities, and will be centered around the Stommel ocean model. This is a nice example to see how powerful Enzyme is, and the ability of it to take a derivative of a complicated function (namely one that has many parts and parameters). This tutorial will focus first on the computations and getting Enzyme running, for those interested a mathematical explanation of the model and what an adjoint variable is will be provided at the end.","category":"page"},{"location":"generated/box/#Brief-model-overview","page":"Box model","title":"Brief model overview","text":"","category":"section"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"The Stommel box model can be viewed as a watered down full ocean model. In our example, we have three boxes (Box One, Box Two, and Box Three) and we model the transport of fluid between them. The full equations of our system are given by:","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"beginaligned\n U = u_0 left rho_2 - left rho_1 + (1 - delta) rho_3 right right \n rho_i = -alpha T_i + beta S_i i = 1 2 3\nendaligned","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"for the transport U and densities rho, and then the time derivatives","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"beginaligned\n dotT_1 = U(T_3 - T_1)V_1 + gamma (T_1^* - T_1 ) dotS_1 = U(S_3 - S_1)V_1 + FW_1V_1 \n dotT_2 = U(T_1 - T_2)V_2 + gamma (T_2^* - T_2 ) dotS_2 = U(S_1 - S_2)V_2 + FW_2V_2 \n dotT_3 = U(T_2 - T_3)V_3 dotS_3 = U(S_2 - S_3)V_3\nendaligned","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"for positive transport, U 0, and","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"beginaligned\n dotT_1 = U(T_2 - T_1)V_1 + gamma (T_1^* - T_1) dotS_1 = U(S_2 - S_1)V_1 + FW_1V_1 \n dotT_2 = U(T_3 - T_2)V_2 + gamma (T_2^* - T_2 ) dotS_2 = U(S_3 - S_2)V_2 + FW_2V_2 \n dotT_3 = U(T_1 - T_3)V_3 dotS_3 = U(S_1 - S_3)V_3\nendaligned","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"for U leq 0. The only force driving our system is a density gradient generated via temperature and salinity differences between the boxes. This makes it a really easy model to play around with! With this in mind, the model is run forward with the steps:","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"Compute densities\nCompute transport\nCompute time derivatives of the box temperatures and salinities\nUpdate the state vector","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"We'll start by going through the model setup step by step, then providing a few test cases with Enzyme.","category":"page"},{"location":"generated/box/#Model-setup","page":"Box model","title":"Model setup","text":"","category":"section"},{"location":"generated/box/#Model-dependencies","page":"Box model","title":"Model dependencies","text":"","category":"section"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"Let's first add the necessary packages to run everything","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"using Enzyme","category":"page"},{"location":"generated/box/#Initialize-constants","page":"Box model","title":"Initialize constants","text":"","category":"section"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"The system equations have quite a few constants that appear, here we initialize them for later use. We'll do this in a Julia way: we have an empty structure that will hold all the parameters, and a function (we'll call this setup) that initializes them. This means that, so long as we don't need to change parameters, we only need to run setup once.","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"struct ModelParameters\n\n # handy to have constants\n day::Float64\n year::Float64\n\n # Information related to the boxes\n boxlength::Vector{Float64} ## Vector with north-south size of each box [cm]\n boxdepth::Vector{Float64} ## \" \" the depth of each box [cm]\n boxwidth::Float64 ## \" \" the width of each box [cm]\n boxarea::Vector{Float64} ## \" \" the area of each box [cm^2]\n boxvol::Vector{Float64} ## \" \" the volume of each box [cm^3]\n\n delta::Float64 ## Constant ratio depth(box1) / (depth(box1) + depth(box3))\n\n # Parameters that appear in the box model equations\n u0::Float64\n alpha::Float64\n beta::Float64\n gamma::Float64\n\n # Coefficient for the Robert filter smoother\n rf_coeff::Float64\n\n # Freshwater forcing\n FW::Vector{Float64}\n\n # Restoring atmospheric temperatures and salinities\n Tstar::Vector{Float64}\n Sstar::Vector{Float64}\n\nend\n\nfunction setup()\n\n blength = [5000.0e5; 1000.0e5; 5000.0e5]\n bdepth = [1.0e5; 5.0e5; 4.0e5]\n\n delta = bdepth[1]/(bdepth[1] + bdepth[3])\n\n bwidth = 4000.0*1e5 ## box width, centimeters\n\n # box areas\n barea = [blength[1]*bwidth;\n blength[2]*bwidth;\n blength[3]*bwidth]\n\n # box volumes\n bvolume = [barea[1]*bdepth[1];\n barea[2]*bdepth[2];\n barea[3]*bdepth[3]]\n\n # parameters that are used to ensure units are in CGS (cent-gram-sec)\n\n day = 3600.0*24.0\n year = day*365.0\n Sv = 1e12 ## one Sverdrup (a unit of ocean transport), 1e6 meters^3/second\n\n # parameters that appear in box model equations\n u0 = 16.0*Sv/0.0004\n alpha = 1668e-7\n beta = 0.7811e-3\n\n gamma = 1/(300*day)\n\n # robert filter coefficient for the smoother part of the timestep\n robert_filter_coeff = 0.25\n\n # freshwater forcing\n FW = [(100/year) * 35.0 * barea[1]; -(100/year) * 35.0 * barea[1]]\n\n # restoring atmospheric temperatures\n Tstar = [22.0; 0.0]\n Sstar = [36.0; 34.0]\n\n structure_with_parameters = ModelParameters(day,\n year,\n blength,\n bdepth,\n bwidth,\n barea,\n bvolume,\n delta,\n u0,\n alpha,\n beta,\n gamma,\n robert_filter_coeff,\n FW,\n Tstar,\n Sstar\n )\n\n return structure_with_parameters\n\nend","category":"page"},{"location":"generated/box/#Define-model-functions","page":"Box model","title":"Define model functions","text":"","category":"section"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"Here we define functions that will calculate quantities used in the forward steps.","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"# function to compute transport\n# Input: rho - the density vector\n# Output: U - transport value\n\nfunction compute_transport(rho, params)\n\n U = params.u0 * (rho[2] - (params.delta * rho[1] + (1 - params.delta)*rho[3]))\n return U\n\nend\n\n# function to compute density\n# Input: state = [T1; T2; T3; S1; S2; S3]\n# Output: rho\n\nfunction compute_density(state, params)\n\n rho = -params.alpha * state[1:3] + params.beta * state[4:6]\n return rho\n\nend\n\n# lastly, a function that takes one step forward\n# Input: state_now = [T1(t), T2(t), ..., S3(t)]\n# state_old = [T1(t-dt), ..., S3(t-dt)]\n# u = transport(t)\n# dt = time step\n# Output: state_new = [T1(t+dt), ..., S3(t+dt)]\n\nfunction compute_update(state_now, state_old, u, params, dt)\n\n dstate_now_dt = zeros(6)\n state_new = zeros(6)\n\n # first computing the time derivatives of the various temperatures and salinities\n if u > 0\n\n dstate_now_dt[1] = u * (state_now[3] - state_now[1]) / params.boxvol[1] + params.gamma * (params.Tstar[1] - state_now[1])\n dstate_now_dt[2] = u * (state_now[1] - state_now[2]) / params.boxvol[2] + params.gamma * (params.Tstar[2] - state_now[2])\n dstate_now_dt[3] = u * (state_now[2] - state_now[3]) / params.boxvol[3]\n\n dstate_now_dt[4] = u * (state_now[6] - state_now[4]) / params.boxvol[1] + params.FW[1] / params.boxvol[1]\n dstate_now_dt[5] = u * (state_now[4] - state_now[5]) / params.boxvol[2] + params.FW[2] / params.boxvol[2]\n dstate_now_dt[6] = u * (state_now[5] - state_now[6]) / params.boxvol[3]\n\n elseif u <= 0\n\n dstate_now_dt[1] = u * (state_now[2] - state_now[1]) / params.boxvol[1] + params.gamma * (params.Tstar[1] - state_now[1])\n dstate_now_dt[2] = u * (state_now[3] - state_now[2]) / params.boxvol[2] + params.gamma * (params.Tstar[2] - state_now[2])\n dstate_now_dt[3] = u * (state_now[1] - state_now[3]) / params.boxvol[3]\n\n dstate_now_dt[4] = u * (state_now[5] - state_now[4]) / params.boxvol[1] + params.FW[1] / params.boxvol[1]\n dstate_now_dt[5] = u * (state_now[6] - state_now[5]) / params.boxvol[2] + params.FW[2] / params.boxvol[2]\n dstate_now_dt[6] = u * (state_now[4] - state_now[6]) / params.boxvol[3]\n\n end\n\n # update fldnew using a version of Euler's method\n state_new .= state_old + 2.0 * dt * dstate_now_dt\n\n return state_new\nend","category":"page"},{"location":"generated/box/#Define-forward-functions","page":"Box model","title":"Define forward functions","text":"","category":"section"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"Finally, we create two functions, the first of which computes and stores all the states of the system, and the second will take just a single step forward.","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"Let's start with the standard forward function. This is just going to be used to store the states at every timestep:","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"function integrate(state_now, state_old, dt, M, parameters)\n\n # Because of the adjoint problem we're setting up, we need to store both the states before\n # and after the Robert filter smoother has been applied\n states_before = [state_old]\n states_after = [state_old]\n\n for t = 1:M\n\n rho = compute_density(state_now, parameters)\n u = compute_transport(rho, parameters)\n state_new = compute_update(state_now, state_old, u, parameters, dt)\n\n # Applying the Robert filter smoother (needed for stability)\n state_new_smoothed = state_now + parameters.rf_coeff * (state_new - 2.0 * state_now + state_old)\n\n push!(states_after, state_new_smoothed)\n push!(states_before, state_new)\n\n # cycle the \"now, new, old\" states\n state_old = state_new_smoothed\n state_now = state_new\n\n end\n\n return states_after, states_before\nend","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"Now, for the purposes of Enzyme, it would be convenient for us to have a function that runs a single step of the model forward rather than the whole integration. This would allow us to save as many of the adjoint variables as we wish when running the adjoint method, although for the example we'll discuss later we technically only need one of them","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"function one_step_forward(state_now, state_old, out_now, out_old, parameters, dt)\n\n state_new_smoothed = zeros(6)\n rho = compute_density(state_now, parameters) ## compute density\n u = compute_transport(rho, parameters) ## compute transport\n state_new = compute_update(state_now, state_old, u, parameters, dt) ## compute new state values\n\n # Robert filter smoother\n state_new_smoothed[:] = state_now + parameters.rf_coeff * (state_new - 2.0 * state_now + state_old)\n\n out_old[:] = state_new_smoothed\n out_now[:] = state_new\n\n return nothing\n\nend","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"One difference to note is that one_step_forward now returns nothing, but is rather a function of both its input and output. Since the output of the function is a vector, we need to have this return nothing for Enzyme to work. Now we can move on to some examples using Enzyme.","category":"page"},{"location":"generated/box/#Example-1:-Simply-using-Enzyme","page":"Box model","title":"Example 1: Simply using Enzyme","text":"","category":"section"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"For the first example let's just compute the gradient of our forward function and examine the output. We'll just run the model for one step, and take a dt of ten days. The initial conditions of the system are given as Tbar and Sbar. We run setup once here, and never have to run it again! (Unless we decide to change a parameter)","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"parameters = setup()\n\nTbar = [20.0; 1.0; 1.0] ## initial temperatures\nSbar = [35.5; 34.5; 34.5] ## initial salinities\n\n# Running the model one step forward\nstates_after_smoother, states_before_smoother = integrate(\n copy([Tbar; Sbar]),\n copy([Tbar; Sbar]),\n 10*parameters.day,\n 1,\n parameters\n)\n\n# Run Enzyme one time on `one_step_forward``\ndstate_now = zeros(6)\ndstate_old = zeros(6)\nout_now = zeros(6); dout_now = ones(6)\nout_old = zeros(6); dout_old = ones(6)\n\nautodiff(Reverse,\n one_step_forward,\n Duplicated([Tbar; Sbar], dstate_now),\n Duplicated([Tbar; Sbar], dstate_old),\n Duplicated(out_now, dout_now),\n Duplicated(out_old, dout_old),\n Const(parameters),\n Const(10*parameters.day)\n)","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"In order to run Enzyme on one_step_forward, we've needed to provide quite a few placeholders, and wrap everything in Duplicated as all components of our function are vectors, not scalars. Let's go through and see what Enzyme did with all of those placeholders.","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"First we can look at what happened to the zero vectors out_now and out_old:","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"@show out_now, out_old","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"Comparing to the results of forward func:","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"@show states_before_smoother[2], states_after_smoother[2]","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"we see that Enzyme has computed and stored exactly the output of the forward step. Next, let's look at dstate_now:","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"@show dstate_now","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"Just a few numbers, but this is what makes AD so nice: Enzyme has exactly computed the derivative of all outputs with respect to the input state_now, evaluated at state_now, and acted with this gradient on what we gave as dout_now (in our case, all ones). Using AD notation for reverse mode, this is","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"overlinetextstate_now = fracpartial textout_nowpartial textstate_nowright_textstate_now overlinetextout_now + fracpartial textout_oldpartial textstate_nowright_textstate_now overlinetextout_old","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"We note here that had we initialized dstate_now and dstate_old as something else, our results will change. Let's multiply them by two and see what happens.","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"dstate_now_new = zeros(6)\ndstate_old_new = zeros(6)\nout_now = zeros(6); dout_now = 2*ones(6)\nout_old = zeros(6); dout_old = 2*ones(6)\nautodiff(Reverse,\n one_step_forward,\n Duplicated([Tbar; Sbar], dstate_now_new),\n Duplicated([Tbar; Sbar], dstate_old_new),\n Duplicated(out_now, dout_now),\n Duplicated(out_old, dout_old),\n Const(parameters),\n Const(10*parameters.day)\n)","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"Now checking dstate_now and dstate_old we see","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"@show dstate_now_new","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"What happened? Enzyme is actually taking the computed gradient and acting on what we give as input to dout_now and dout_old. Checking this, we see","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"@show 2*dstate_now","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"and they match the new results. This exactly matches what we'd expect to happen since we scaled dout_now by two.","category":"page"},{"location":"generated/box/#Example-2:-Full-sensitivity-calculations","page":"Box model","title":"Example 2: Full sensitivity calculations","text":"","category":"section"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"Now we want to use Enzyme for a bit more than just a single derivative. Let's say we'd like to understand how sensitive the final temperature of Box One is to the initial salinity of Box Two. That is, given the function","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"J = (100000)^T cdot mathbfx(t_f)","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"we want Enzyme to calculate the derivative","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"fracpartial Jpartial mathbfx(0)","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"where x(t) is the state of the model at time t. If we think about x(t_f) as solely depending on the initial condition, then this derivative is really","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"fracpartial Jpartial mathbfx(0) = fracpartialpartial mathbfx(0) left( (100000)^T cdot L(ldots(L(mathbfx(0)))) right)","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"with L(x(t)) = x(t + dt), i.e. one forward step. One could expand this derivative with the chain rule (and it would be very complicated), but really this is where Enzyme comes in. Each run of autodiff on our forward function is one piece of this big chain rule done for us! We also note that the chain rule goes from the outside in, so we start with the derivative of the forward function at the final state, and work backwards until the initial state. To get Enzyme to do this, we complete the following steps:","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"Run the forward model and store outputs (in a real ocean model this wouldn't be feasible and we'd need to use checkpointing)\nCompute the initial derivative from the final state\nUse Enzyme to work backwards until we reach the desired derivative.","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"For simplicity we define a function that takes completes our AD steps","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"function compute_adjoint_values(states_before_smoother, states_after_smoother, M, parameters)\n\n dout_now = [0.0;0.0;0.0;0.0;0.0;0.0]\n dout_old = [1.0;0.0;0.0;0.0;0.0;0.0]\n\n for j = M:-1:1\n\n dstate_now = zeros(6)\n dstate_old = zeros(6)\n\n autodiff(Reverse,\n one_step_forward,\n Duplicated(states_before_smoother[j], dstate_now),\n Duplicated(states_after_smoother[j], dstate_old),\n Duplicated(zeros(6), dout_now),\n Duplicated(zeros(6), dout_old),\n Const(parameters),\n Const(10*parameters.day)\n )\n\n if j == 1\n return dstate_now, dstate_old\n end\n\n dout_now = copy(dstate_now)\n dout_old = copy(dstate_old)\n\n end\n\nend","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"First we integrate the model forward:","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"M = 10000 ## Total number of forward steps to take\nTbar = [20.0; 1.0; 1.0] ## initial temperatures\nSbar = [35.5; 34.5; 34.5] ## initial salinities\n\nstates_after_smoother, states_before_smoother = integrate(\n copy([Tbar; Sbar]),\n copy([Tbar; Sbar]),\n 10*parameters.day,\n M,\n parameters\n)","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"Next, we pass all of our states to the AD function to get back to the desired derivative:","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"dstate_now, dstate_old = compute_adjoint_values(\n states_before_smoother,\n states_after_smoother,\n M,\n parameters\n)","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"And we're done! We were interested in sensitivity to the initial salinity of box two, which will live in what we've called dstate_old. Checking this value we see","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"@show dstate_old[5]","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"As it stands this is just a number, but a good check that Enzyme has computed what we want is to approximate the derivative with a Taylor series. Specifically,","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"J(mathbfx(0) + varepsilon) approx J(mathbfx(0)) +\nvarepsilon fracpartial Jpartial mathbfx(0)","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"and a simple rearrangement yields","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"fracpartial Jpartial mathbfx(0) approx\nfracJ(mathbfx(0) + varepsilon) - J(mathbfx(0))varepsilon","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"Hopefully we see that the analytical values converge close to the one we found with Enzyme:","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"# unperturbed final state\nuse_to_check = states_after_smoother[M+1]\n\n# a loop to compute the perturbed final states\ndiffs = []\nstep_sizes = [1e-1, 1e-2, 1e-3, 1e-4, 1e-5, 1e-6, 1e-7, 1e-8, 1e-9, 1e-10]\nfor eps in step_sizes\n\n state_new_smoothed = zeros(6)\n\n initial_temperature = [20.0; 1.0; 1.0]\n perturbed_initial_salinity = [35.5; 34.5; 34.5] + [0.0; eps; 0.0]\n\n state_old = [initial_temperature; perturbed_initial_salinity]\n state_now = [20.0; 1.0; 1.0; 35.5; 34.5; 34.5]\n\n for t = 1:M\n\n rho = compute_density(state_now, parameters)\n u = compute_transport(rho, parameters)\n state_new = compute_update(state_now, state_old, u, parameters, 10*parameters.day)\n\n state_new_smoothed[:] = state_now + parameters.rf_coeff * (state_new - 2.0 * state_now + state_old)\n\n state_old = state_new_smoothed\n state_now = state_new\n\n end\n\n push!(diffs, (state_old[1] - use_to_check[1])/eps)\n\nend","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"Then checking what we found the derivative to be analytically:","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"@show diffs","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"which comes very close to our calculated value. We can go further and check the percent difference to see","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"@show abs.(diffs .- dstate_old[5])./dstate_old[5]","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"and we get down to a percent difference on the order of 1e^-5, showing Enzyme calculated the correct derivative. Success!","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"","category":"page"},{"location":"generated/box/","page":"Box model","title":"Box model","text":"This page was generated using Literate.jl.","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"EditURL = \"../../../examples/custom_rule.jl\"","category":"page"},{"location":"generated/custom_rule/#Enzyme-custom-rules-tutorial","page":"Custom rules","title":"Enzyme custom rules tutorial","text":"","category":"section"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"note: More Examples\nThe tutorial below focuses on a simple setting to illustrate the basic concepts of writing custom rules. For more complex custom rules beyond the scope of this tutorial, you may take inspiration from the following in-the-wild examples:Enzyme internal rules\nKernelAbstractions.jl\nLinearSolve.jl\nNNlib.jl","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"The goal of this tutorial is to give a simple example of defining a custom rule with Enzyme. Specifically, our goal will be to write custom rules for the following function f:","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"function f(y, x)\n y .= x.^2\n return sum(y)\nend","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"Our function f populates its first input y with the element-wise square of x. In addition, it returns sum(y) as output. What a sneaky function!","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"In this case, Enzyme can differentiate through f automatically. For example, using forward mode:","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"using Enzyme\nx = [3.0, 1.0]\ndx = [1.0, 0.0]\ny = [0.0, 0.0]\ndy = [0.0, 0.0]\n\ng(y, x) = f(y, x)^2 # function to differentiate\n\n@show autodiff(Forward, g, Duplicated(y, dy), Duplicated(x, dx)) # derivative of g w.r.t. x[1]\n@show dy; # derivative of y w.r.t. x[1] when g is run\nnothing #hide","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"(See the AutoDiff API tutorial for more information on using autodiff.)","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"But there may be special cases where we need to write a custom rule to help Enzyme out. Let's see how to write a custom rule for f!","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"warning: Don't use custom rules unnecessarily!\nEnzyme can efficiently handle a wide range of constructs, and so a custom rule should only be required in certain special cases. For example, a function may make a foreign call that Enzyme cannot differentiate, or we may have higher-level mathematical knowledge that enables us to write a more efficient rule. Even in these cases, try to make your custom rule encapsulate the minimum possible construct that Enzyme cannot differentiate, rather than expanding the scope of the rule unnecessarily. For pedagogical purposes, we will disregard this principle here and go ahead and write a custom rule for f :)","category":"page"},{"location":"generated/custom_rule/#Defining-our-first-rule","page":"Custom rules","title":"Defining our first rule","text":"","category":"section"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"First, we import the functions EnzymeRules.forward, EnzymeRules.augmented_primal, and EnzymeRules.reverse. We need to overload forward in order to define a custom forward rule, and we need to overload augmented_primal and reverse in order to define a custom reverse rule.","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"import .EnzymeRules: forward, reverse, augmented_primal\nusing .EnzymeRules","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"In this section, we write a simple forward rule to start out:","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"function forward(config::FwdConfig, func::Const{typeof(f)}, ::Type{<:Duplicated}, y::Duplicated, x::Duplicated)\n println(\"Using custom rule!\")\n ret = func.val(y.val, x.val)\n y.dval .= 2 .* x.val .* x.dval\n return Duplicated(ret, sum(y.dval))\nend","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"In the signature of our rule, we have made use of Enzyme's activity annotations. Let's break down each one:","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"the EnzymeRules.FwdConfig configuration passes certain compile-time information about differentiation procedure (the width, and if we're using runtime activity),\nthe Const annotation on f indicates that we accept a function f that does not have a derivative component, which makes sense since f is not a closure with data that could be differentiated.\nthe Duplicated annotation given in the second argument annotates the return value of f. This means that our forward function should return an output of type Duplicated, containing the original output sum(y) and its derivative.\nthe Duplicated annotations for x and y mean that our forward function handles inputs x and y which have been marked as Duplicated. We should update their shadows with their derivative contributions.","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"In the logic of our forward function, we run the original function, populate y.dval (the shadow of y), and finally return a Duplicated for the output as promised. Let's see our rule in action! With the same setup as before:","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"x = [3.0, 1.0]\ndx = [1.0, 0.0]\ny = [0.0, 0.0]\ndy = [0.0, 0.0]\n\ng(y, x) = f(y, x)^2 # function to differentiate\n\n@show autodiff(Forward, g, Duplicated(y, dy), Duplicated(x, dx)) # derivative of g w.r.t. x[1]\n@show dy; # derivative of y w.r.t. x[1] when g is run\nnothing #hide","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"We see that our custom forward rule has been triggered and gives the same answer as before.","category":"page"},{"location":"generated/custom_rule/#Handling-more-activities","page":"Custom rules","title":"Handling more activities","text":"","category":"section"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"Our custom rule applies for the specific set of activities that are annotated for f in the above autodiff call. However, Enzyme has a number of other annotations. Let us consider a particular example, where the output has a DuplicatedNoNeed annotation. This means we are only interested in its derivative, not its value. To squeeze out the last drop of performance, the below rule avoids computing the output of the original function and just computes its derivative.","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"function forward(config, func::Const{typeof(f)}, ::Type{<:DuplicatedNoNeed}, y::Duplicated, x::Duplicated)\n println(\"Using custom rule with DuplicatedNoNeed output.\")\n y.val .= x.val.^2\n y.dval .= 2 .* x.val .* x.dval\n return sum(y.dval)\nend","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"Our rule is triggered, for example, when we call autodiff directly on f, as the return value's derivative isn't needed:","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"x = [3.0, 1.0]\ndx = [1.0, 0.0]\ny = [0.0, 0.0]\ndy = [0.0, 0.0]\n\n@show autodiff(Forward, f, Duplicated(y, dy), Duplicated(x, dx)) # derivative of f w.r.t. x[1]\n@show dy; # derivative of y w.r.t. x[1] when f is run\nnothing #hide","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"note: Custom rule dispatch\nWhen multiple custom rules for a function are defined, the correct rule is chosen using Julia's multiple dispatch. In particular, it is important to understand that the custom rule does not determine the activities of the inputs and the return value: rather, Enzyme decides the activity annotations independently, and then dispatches to the custom rule handling the activities, if one exists. If a custom rule is specified for the correct function/argument types, but not the correct activity annotation, a runtime error will be thrown alerting the user to the missing activity rule rather than silently ignoring the rule.\"","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"Finally, it may be that either x, y, or the return value are marked as Const, in which case we can simply return the original result. However, Enzyme also may determine the return is not differentiable and also not needed for other computations, in which case we should simply return nothing.","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"We can in fact handle this case, along with the previous two cases, all together in a single rule by leveraging utility functions EnzymeRules.needs_primal and EnzymeRules.needs_shadow, which return true if the original return or the derivative is needed to be returned, respectively:","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"Base.delete_method.(methods(forward, (Const{typeof(f)}, Vararg{Any}))) # delete our old rules\n\nfunction forward(config, func::Const{typeof(f)}, RT::Type{<:Union{Const, DuplicatedNoNeed, Duplicated}},\n y::Union{Const, Duplicated}, x::Union{Const, Duplicated})\n println(\"Using our general custom rule!\")\n y.val .= x.val.^2\n if !(x isa Const) && !(y isa Const)\n y.dval .= 2 .* x.val .* x.dval\n elseif !(y isa Const)\n make_zero!(y.dval)\n end\n dret = !(y isa Const) ? sum(y.dval) : zero(eltype(y.val))\n if needs_primal(config) && needs_shadow(config)\n return Duplicated(sum(y.val), dret)\n elseif needs_primal(config)\n return sum(y.val)\n elseif needs_shadow(config)\n return dret\n else\n return nothing\n end\nend","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"Let's try out our rule:","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"x = [3.0, 1.0]\ndx = [1.0, 0.0]\ny = [0.0, 0.0]\ndy = [0.0, 0.0]\n\ng(y, x) = f(y, x)^2 # function to differentiate\n\n@show autodiff(Forward, g, Duplicated(y, dy), Duplicated(x, dx)) # derivative of g w.r.t. x[1]\n@show autodiff(Forward, g, Const(y), Duplicated(x, dx)) # derivative of g w.r.t. x[1], with y annotated Const\n@show autodiff(Forward, g, Const(y), Const(x)); # derivative of g w.r.t. x[1], with x and y annotated Const\nnothing #hide","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"Note that there are also exist batched duplicated annotations for forward mode, namely BatchDuplicated and BatchDuplicatedNoNeed, which are not covered in this tutorial.","category":"page"},{"location":"generated/custom_rule/#Defining-a-reverse-mode-rule","page":"Custom rules","title":"Defining a reverse-mode rule","text":"","category":"section"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"Let's look at how to write a simple reverse-mode rule! First, we write a method for EnzymeRules.augmented_primal:","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"function augmented_primal(config::RevConfigWidth{1}, func::Const{typeof(f)}, ::Type{<:Active},\n y::Duplicated, x::Duplicated)\n println(\"In custom augmented primal rule.\")\n # Compute primal\n if needs_primal(config)\n primal = func.val(y.val, x.val)\n else\n y.val .= x.val.^2 # y still needs to be mutated even if primal not needed!\n primal = nothing\n end\n # Save x in tape if x will be overwritten\n if overwritten(config)[3]\n tape = copy(x.val)\n else\n tape = nothing\n end\n # Return an AugmentedReturn object with shadow = nothing\n return AugmentedReturn(primal, nothing, tape)\nend","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"Let's unpack our signature for augmented_primal :","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"We accepted a EnzymeRules.RevConfig object with a specified width of 1, which means that our rule does not support batched reverse mode.\nWe annotated f with Const as usual.\nWe dispatched on an Active annotation for the return value. This is a special annotation for scalar values, such as our return value, that indicates that that we care about the value's derivative but we need not explicitly allocate a mutable shadow since it is a scalar value.\nWe annotated x and y with Duplicated, similar to our first simple forward rule.","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"Now, let's unpack the body of our augmented_primal rule:","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"We checked if the config requires the primal. If not, we need not compute the return value, but we make sure to mutate y in all cases.\nWe checked if x could possibly be overwritten using the Overwritten attribute of EnzymeRules.RevConfig. If so, we save the elements of x on the tape of the returned EnzymeRules.AugmentedReturn object.\nWe return a shadow of nothing since the return value is Active and hence does not need a shadow.","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"Now, we write a method for EnzymeRules.reverse:","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"function reverse(config::RevConfigWidth{1}, func::Const{typeof(f)}, dret::Active, tape,\n y::Duplicated, x::Duplicated)\n println(\"In custom reverse rule.\")\n # retrieve x value, either from original x or from tape if x may have been overwritten.\n xval = overwritten(config)[3] ? tape : x.val\n # accumulate dret into x's shadow. don't assign!\n x.dval .+= 2 .* xval .* dret.val\n # also accumulate any derivative in y's shadow into x's shadow.\n x.dval .+= 2 .* xval .* y.dval\n make_zero!(y.dval)\n return (nothing, nothing)\nend","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"Let's make a few observations about our reverse rule:","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"The activities used in the signature correspond to what we used for augmented_primal.\nHowever, for Active return types such as in this case, we now receive an instance dret of Active for the return type, not just a type annotation, which stores the derivative value for ret (not the original return value!). For the other annotations (e.g. Duplicated), we still receive only the type. In that case, if necessary a reference to the shadow of the output should be placed on the tape in augmented_primal.\nUsing dret.val and y.dval, we accumulate the backpropagated derivatives for x into its shadow x.dval. Note that we have to accumulate from both y.dval and dret.val. This is because in reverse-mode AD we have to sum up the derivatives from all uses: if y was read after our function, we need to consider derivatives from that use as well.\nWe zero-out y's shadow. This is because y is overwritten within f, so there is no derivative w.r.t. to the y that was originally inputted.\nFinally, since all derivatives are accumulated in place (in the shadows of the Duplicated arguments), these derivatives must not be communicated via the return value. Hence, we return (nothing, nothing). If, instead, one of our arguments was annotated as Active, we would have to provide its derivative at the corresponding index in the tuple returned.","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"Finally, let's see our reverse rule in action!","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"x = [3.0, 1.0]\ndx = [0.0, 0.0]\ny = [0.0, 0.0]\ndy = [0.0, 0.0]\n\ng(y, x) = f(y, x)^2\n\nautodiff(Reverse, g, Duplicated(y, dy), Duplicated(x, dx))\n@show dx # derivative of g w.r.t. x\n@show dy; # derivative of g w.r.t. y\nnothing #hide","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"Let's also try a function which mutates x after running f, and also uses y directly rather than only ret after running f (but ultimately gives the same result as above):","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"function h(y, x)\n ret = f(y, x)\n x .= x.^2\n return ret * sum(y)\nend\n\nx = [3.0, 1.0]\ny = [0.0, 0.0]\nmake_zero!(dx)\nmake_zero!(dy)\n\nautodiff(Reverse, h, Duplicated(y, dy), Duplicated(x, dx))\n@show dx # derivative of h w.r.t. x\n@show dy; # derivative of h w.r.t. y\nnothing #hide","category":"page"},{"location":"generated/custom_rule/#Marking-functions-inactive","page":"Custom rules","title":"Marking functions inactive","text":"","category":"section"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"If we want to tell Enzyme that the function call does not affect the differentiation result in any form (i.e. not by side effects or through its return values), we can simply use EnzymeRules.inactive. So long as there exists a matching dispatch to EnzymeRules.inactive, the function will be considered inactive. For example:","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"printhi() = println(\"Hi!\")\nEnzymeRules.inactive(::typeof(printhi), args...) = nothing\n\nfunction k(x)\n printhi()\n return x^2\nend\n\nautodiff(Forward, k, Duplicated(2.0, 1.0))","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"Or for a case where we incorrectly mark a function inactive:","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"double(x) = 2*x\nEnzymeRules.inactive(::typeof(double), args...) = nothing\n\nautodiff(Forward, x -> x + double(x), Duplicated(2.0, 1.0)) # mathematically should be 3.0, inactive rule causes it to be 1.0","category":"page"},{"location":"generated/custom_rule/#Testing-our-rules","page":"Custom rules","title":"Testing our rules","text":"","category":"section"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"We can test our rules using finite differences using EnzymeTestUtils.test_forward and EnzymeTestUtils.test_reverse.","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"using EnzymeTestUtils, Test\n\n@testset \"f rules\" begin\n @testset \"forward\" begin\n @testset for RT in (Const, DuplicatedNoNeed, Duplicated),\n Tx in (Const, Duplicated),\n Ty in (Const, Duplicated)\n\n x = [3.0, 1.0]\n y = [0.0, 0.0]\n test_forward(g, RT, (x, Tx), (y, Ty))\n end\n end\n @testset \"reverse\" begin\n @testset for RT in (Active,),\n Tx in (Duplicated,),\n Ty in (Duplicated,),\n fun in (g, h)\n\n x = [3.0, 1.0]\n y = [0.0, 0.0]\n test_reverse(fun, RT, (x, Tx), (y, Ty))\n end\n end\nend","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"In any package that implements Enzyme rules using EnzymeRules, it is recommended to add EnzymeTestUtils as a test dependency to test the rules.","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"","category":"page"},{"location":"generated/custom_rule/","page":"Custom rules","title":"Custom rules","text":"This page was generated using Literate.jl.","category":"page"},{"location":"","page":"Home","title":"Home","text":"CurrentModule = Enzyme\nDocTestSetup = quote\n using Enzyme\nend","category":"page"},{"location":"#Enzyme","page":"Home","title":"Enzyme","text":"","category":"section"},{"location":"","page":"Home","title":"Home","text":"Documentation for Enzyme.jl, the Julia bindings for Enzyme.","category":"page"},{"location":"","page":"Home","title":"Home","text":"Enzyme performs automatic differentiation (AD) of statically analyzable LLVM. It is highly-efficient and its ability to perform AD on optimized code allows Enzyme to meet or exceed the performance of state-of-the-art AD tools.","category":"page"},{"location":"#Getting-started","page":"Home","title":"Getting started","text":"","category":"section"},{"location":"","page":"Home","title":"Home","text":"Enzyme.jl can be installed in the usual way Julia packages are installed:","category":"page"},{"location":"","page":"Home","title":"Home","text":"] add Enzyme","category":"page"},{"location":"","page":"Home","title":"Home","text":"The Enzyme binary dependencies will be installed automatically via Julia's binary artifact system.","category":"page"},{"location":"","page":"Home","title":"Home","text":"The Enzyme.jl API revolves around the function autodiff. For some common operations, Enzyme additionally wraps autodiff in several convenience functions; e.g., gradient and jacobian.","category":"page"},{"location":"","page":"Home","title":"Home","text":"The tutorial below covers the basic usage of these functions. For a complete overview of Enzyme's functionality, see the API reference documentation. Also see Implementing pullbacks on how to implement back-propagation for functions with non-scalar results.","category":"page"},{"location":"","page":"Home","title":"Home","text":"We will try a few things with the following functions:","category":"page"},{"location":"","page":"Home","title":"Home","text":"julia> rosenbrock(x, y) = (1.0 - x)^2 + 100.0 * (y - x^2)^2\nrosenbrock (generic function with 1 method)\n\njulia> rosenbrock_inp(x) = (1.0 - x[1])^2 + 100.0 * (x[2] - x[1]^2)^2\nrosenbrock_inp (generic function with 1 method)","category":"page"},{"location":"#Reverse-mode","page":"Home","title":"Reverse mode","text":"","category":"section"},{"location":"","page":"Home","title":"Home","text":"The return value of reverse mode autodiff is a tuple that contains as a first value the derivative value of the active inputs and optionally the primal return value.","category":"page"},{"location":"","page":"Home","title":"Home","text":"julia> autodiff(Reverse, rosenbrock, Active, Active(1.0), Active(2.0))\n((-400.0, 200.0),)\n\njulia> autodiff(ReverseWithPrimal, rosenbrock, Active, Active(1.0), Active(2.0))\n((-400.0, 200.0), 100.0)","category":"page"},{"location":"","page":"Home","title":"Home","text":"julia> x = [1.0, 2.0]\n2-element Vector{Float64}:\n 1.0\n 2.0\n\njulia> dx = [0.0, 0.0]\n2-element Vector{Float64}:\n 0.0\n 0.0\n\njulia> autodiff(Reverse, rosenbrock_inp, Active, Duplicated(x, dx))\n((nothing,),)\n\njulia> dx\n2-element Vector{Float64}:\n -400.0\n 200.0","category":"page"},{"location":"","page":"Home","title":"Home","text":"Both the inplace and \"normal\" variant return the gradient. The difference is that with Active the gradient is returned and with Duplicated the gradient is accumulated in place.","category":"page"},{"location":"#Forward-mode","page":"Home","title":"Forward mode","text":"","category":"section"},{"location":"","page":"Home","title":"Home","text":"The return value when using ForwardWithPrimal is a tuple containing as the first value the derivative return value and as the second value the original value.","category":"page"},{"location":"","page":"Home","title":"Home","text":"The return value when using Forward is a single-element tuple containing the derivative.","category":"page"},{"location":"","page":"Home","title":"Home","text":"In forward mode Duplicated(x, 0.0) is equivalent to Const(x), except that we can perform more optimizations for Const.","category":"page"},{"location":"","page":"Home","title":"Home","text":"julia> autodiff(ForwardWithPrimal, rosenbrock, Const(1.0), Duplicated(3.0, 1.0))\n(400.0, 400.0)\n\njulia> autodiff(Forward, rosenbrock, Const(1.0), Duplicated(3.0, 1.0))\n(400.0,)\n\njulia> autodiff(ForwardWithPrimal, rosenbrock, Duplicated(1.0, 1.0), Const(3.0))\n(-800.0, 400.0)\n\njulia> autodiff(Forward, rosenbrock, Duplicated(1.0, 1.0), Const(3.0))\n(-800.0,)","category":"page"},{"location":"","page":"Home","title":"Home","text":"Of note, when we seed both arguments at once the tangent return is the sum of both.","category":"page"},{"location":"","page":"Home","title":"Home","text":"julia> autodiff(ForwardWithPrimal, rosenbrock, Duplicated(1.0, 1.0), Duplicated(3.0, 1.0))\n(-400.0, 400.0)\n\njulia> autodiff(Forward, rosenbrock, Duplicated(1.0, 1.0), Duplicated(3.0, 1.0))\n(-400.0,)","category":"page"},{"location":"","page":"Home","title":"Home","text":"We can also use forward mode with our inplace method.","category":"page"},{"location":"","page":"Home","title":"Home","text":"julia> x = [1.0, 3.0]\n2-element Vector{Float64}:\n 1.0\n 3.0\n\njulia> dx = [1.0, 1.0]\n2-element Vector{Float64}:\n 1.0\n 1.0\n\njulia> autodiff(ForwardWithPrimal, rosenbrock_inp, Duplicated, Duplicated(x, dx))\n(-400.0, 400.0)","category":"page"},{"location":"","page":"Home","title":"Home","text":"Note the seeding through dx.","category":"page"},{"location":"#Vector-forward-mode","page":"Home","title":"Vector forward mode","text":"","category":"section"},{"location":"","page":"Home","title":"Home","text":"We can also use vector mode to calculate both derivatives at once.","category":"page"},{"location":"","page":"Home","title":"Home","text":"julia> autodiff(ForwardWithPrimal, rosenbrock, BatchDuplicated(1.0, (1.0, 0.0)), BatchDuplicated(3.0, (0.0, 1.0)))\n((var\"1\" = -800.0, var\"2\" = 400.0), 400.0)\n\njulia> x = [1.0, 3.0]\n2-element Vector{Float64}:\n 1.0\n 3.0\n\njulia> dx_1 = [1.0, 0.0]; dx_2 = [0.0, 1.0];\n\njulia> autodiff(ForwardWithPrimal, rosenbrock_inp, BatchDuplicated(x, (dx_1, dx_2)))\n((var\"1\" = -800.0, var\"2\" = 400.0), 400.0)","category":"page"},{"location":"#Gradient-Convenience-functions","page":"Home","title":"Gradient Convenience functions","text":"","category":"section"},{"location":"","page":"Home","title":"Home","text":"note: Note\nWhile the convenience functions discussed below use autodiff internally, they are generally more limited in their functionality. Beyond that, these convenience functions may also come with performance penalties; especially if one makes a closure of a multi-argument function instead of calling the appropriate multi-argument autodiff function directly.","category":"page"},{"location":"","page":"Home","title":"Home","text":"Key convenience functions for common derivative computations are gradient (and its inplace variant gradient!). Like autodiff, the mode (forward or reverse) is determined by the first argument.","category":"page"},{"location":"","page":"Home","title":"Home","text":"The functions gradient and gradient! compute the gradient of function with vector input and scalar return.","category":"page"},{"location":"","page":"Home","title":"Home","text":"Gradient functions take a mode as the first argument. If the mode is Reverse or Forward, the return type is a tuple of gradients of each argument. If the mode is ReverseWithPrimal or ForwardWithPrimal, the return type is a named tuple containing both the derivatives and the original return result.","category":"page"},{"location":"","page":"Home","title":"Home","text":"julia> gradient(Reverse, rosenbrock_inp, [1.0, 2.0])\n([-400.0, 200.0],)\n\njulia> gradient(ReverseWithPrimal, rosenbrock_inp, [1.0, 2.0])\n(derivs = ([-400.0, 200.0],), val = 100.0)\n\njulia> # inplace variant\n dx = [0.0, 0.0];\n gradient!(Reverse, dx, rosenbrock_inp, [1.0, 2.0])\n([-400.0, 200.0],)\n\njulia> dx\n2-element Vector{Float64}:\n -400.0\n 200.0\n\njulia> gradient(Forward, rosenbrock_inp, [1.0, 2.0])\n([-400.0, 200.0],)\n\njulia> gradient(ForwardWithPrimal, rosenbrock_inp, [1.0, 2.0])\n(derivs = ([-400.0, 200.0],), val = 100.0)\n\njulia> # in forward mode, we can also optionally pass a chunk size\n # to specify the number of derivatives computed simulateneously\n # using vector forward mode\n gradient(Forward, rosenbrock_inp, [1.0, 2.0]; chunk=Val(1))\n([-400.0, 200.0],)","category":"page"},{"location":"#Jacobian-Convenience-functions","page":"Home","title":"Jacobian Convenience functions","text":"","category":"section"},{"location":"","page":"Home","title":"Home","text":"The function jacobian computes the Jacobian of a function vector input and vector return. Like autodiff and gradient, the mode (forward or reverse) is determined by the first argument.","category":"page"},{"location":"","page":"Home","title":"Home","text":"Again like gradient, if the mode is Reverse or Forward, the return type is a tuple of jacobians of each argument. If the mode is ReverseWithPrimal or ForwardWithPrimal, the return type is a named tuple containing both the derivatives and the original return result.","category":"page"},{"location":"","page":"Home","title":"Home","text":"Both forward and reverse modes take an optional chunk size to compute several derivatives simultaneously using vector mode, and reverse mode optionally takes n_outs which describes the shape of the output value.","category":"page"},{"location":"","page":"Home","title":"Home","text":"julia> foo(x) = [rosenbrock_inp(x), prod(x)];\n\njulia> jacobian(Reverse, foo, [1.0, 2.0]) \n([-400.0 200.0; 2.0 1.0],)\n\njulia> jacobian(ReverseWithPrimal, foo, [1.0, 2.0]) \n(derivs = ([-400.0 200.0; 2.0 1.0],), val = [100.0, 2.0])\n\njulia> jacobian(Reverse, foo, [1.0, 2.0]; chunk=Val(2)) \n([-400.0 200.0; 2.0 1.0],)\n\njulia> jacobian(Reverse, foo, [1.0, 2.0]; chunk=Val(2), n_outs=Val((2,)))\n([-400.0 200.0; 2.0 1.0],)\n\njulia> jacobian(Forward, foo, [1.0, 2.0])\n([-400.0 200.0; 2.0 1.0],)\n\njulia> jacobian(Forward, foo, [1.0, 2.0], chunk=Val(2))\n([-400.0 200.0; 2.0 1.0],)","category":"page"},{"location":"#Hessian-Vector-Product-Convenience-functions","page":"Home","title":"Hessian Vector Product Convenience functions","text":"","category":"section"},{"location":"","page":"Home","title":"Home","text":"Enzyme provides convenience functions for second-order derivative computations, like hvp to compute Hessian vector products. Mathematically, this computes H(x) v, where H is the hessian operator.","category":"page"},{"location":"","page":"Home","title":"Home","text":"Unlike autodiff and gradient, a mode is not specified. Here, Enzyme will choose to perform forward over reverse mode (generally the fastest for this type of operation).","category":"page"},{"location":"","page":"Home","title":"Home","text":"julia> f(x) = sin(x[1] * x[2]);\n\njulia> hvp(f, [2.0, 3.0], [5.0, 2.7])\n2-element Vector{Float64}:\n 19.69268826373025\n 16.201003759768003","category":"page"},{"location":"","page":"Home","title":"Home","text":"Enzyme also provides an in-place variant which will store the hessian vector product in a pre-allocated array (this will, however, still allocate another array for storing an intermediate gradient).","category":"page"},{"location":"","page":"Home","title":"Home","text":"julia> f(x) = sin(x[1] * x[2])\nf (generic function with 1 method)\n\njulia> res = Vector{Float64}(undef, 2);\n\njulia> hvp!(res, f, [2.0, 3.0], [5.0, 2.7]);\n\njulia> res\n2-element Vector{Float64}:\n 19.69268826373025\n 16.201003759768003","category":"page"},{"location":"","page":"Home","title":"Home","text":"Finally. Enzyme provides a second in-place variant which simultaneously computes both the hessian vector product, and the gradient. This function uses no additional allocation, and is much more efficient than separately computing the hvp and the gradient.","category":"page"},{"location":"","page":"Home","title":"Home","text":"julia> f(x) = sin(x[1] * x[2]);\n\njulia> res = Vector{Float64}(undef, 2);\n\njulia> grad = Vector{Float64}(undef, 2);\n\njulia> hvp_and_gradient!(res, grad, f, [2.0, 3.0], [5.0, 2.7])\n\njulia> res\n2-element Vector{Float64}:\n 19.69268826373025\n 16.201003759768003\n\njulia> grad\n2-element Vector{Float64}:\n 2.880510859951098\n 1.920340573300732","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"CurrentModule = Enzyme\nDocTestSetup = quote\n using Enzyme\nend","category":"page"},{"location":"faq/#Frequently-asked-questions","page":"FAQ","title":"Frequently asked questions","text":"","category":"section"},{"location":"faq/#Implementing-pullbacks","page":"FAQ","title":"Implementing pullbacks","text":"","category":"section"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"In combined reverse mode, Enzyme's autodiff function can only handle functions with scalar output (this is not true for split reverse mode, aka autodiff_thunk). To implement pullbacks (back-propagation of gradients/tangents) for array-valued functions, use a mutating function that returns nothing and stores its result in one of the arguments, which must be passed wrapped in a Duplicated. Regardless of AD mode, this mutating function will be much more efficient anyway than one which allocates the output.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Given a function mymul! that performs the equivalent of R = A * B for matrices A and B, and given a gradient (tangent) ∂z_∂R, we can compute ∂z_∂A and ∂z_∂B like this:","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"using Enzyme, Random\n\nfunction mymul!(R, A, B)\n @assert axes(A,2) == axes(B,1)\n @inbounds @simd for i in eachindex(R)\n R[i] = 0\n end\n @inbounds for j in axes(B, 2), i in axes(A, 1)\n @inbounds @simd for k in axes(A,2)\n R[i,j] += A[i,k] * B[k,j]\n end\n end\n nothing\nend\n\nRandom.seed!(1234)\nA = rand(5, 3)\nB = rand(3, 7)\n\nR = zeros(size(A,1), size(B,2))\n∂z_∂R = rand(size(R)...) # Some gradient/tangent passed to us\n∂z_∂R0 = copyto!(similar(∂z_∂R), ∂z_∂R) # exact copy for comparison\n\n∂z_∂A = zero(A)\n∂z_∂B = zero(B)\n\nEnzyme.autodiff(Reverse, mymul!, Const, Duplicated(R, ∂z_∂R), Duplicated(A, ∂z_∂A), Duplicated(B, ∂z_∂B))","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Now we have:","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"R ≈ A * B &&\n∂z_∂A ≈ ∂z_∂R0 * B' && # equivalent to Zygote.pullback(*, A, B)[2](∂z_∂R)[1]\n∂z_∂B ≈ A' * ∂z_∂R0 # equivalent to Zygote.pullback(*, A, B)[2](∂z_∂R)[2]","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Note that the result of the backpropagation is added to ∂z_∂A and ∂z_∂B, they act as accumulators for gradient information.","category":"page"},{"location":"faq/#Identical-types-in-Duplicated-/-Memory-Layout","page":"FAQ","title":"Identical types in Duplicated / Memory Layout","text":"","category":"section"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Enzyme checks that x and ∂f_∂x have the same types when constructing objects of type Duplicated, DuplicatedNoNeed, BatchDuplicated, etc. This is not a mathematical or practical requirement within Enzyme, but rather a guardrail to prevent user error. The memory locations of the shadow ∂f_∂x can only be accessed in the derivative function ∂f if the corresponding memory locations of the variable x are accessed by the function f. Imposing that the variable x and shadow ∂f_∂x have the same type is a heuristic way to ensure that they have the same data layout. This helps prevent some user errors, for instance when the provided shadow cannot be accessed at the relevant memory locations.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"In some ways, type equality is too strict: two different types can have the same data layout. For instance, a vector and a view of a matrix column are arranged identically in memory. But in other ways it is not strict enough. Suppose you have a function f(x) = x[7]. If you call Enzyme.autodiff(Reverse, f, Duplicated(ones(10), ones(1)), the type check alone will not be sufficient. Since the original code accesses x[7], the derivative code will try to set ∂f_∂x[7]. The length is not encoded in the type, so Julia cannot provide a high-level error before running autodiff, and the user may end up with a segfault (or other memory error) when running the generated derivative code. Another typical example is sparse arrays, for which the sparsity pattern of x and ∂f_∂x should be identical.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"To make sure that ∂f_∂x has the right data layout, create it with ∂f_∂x = Enzyme.make_zero(x).","category":"page"},{"location":"faq/#Circumventing-Duplicated-Restrictions-/-Advanced-Memory-Layout","page":"FAQ","title":"Circumventing Duplicated Restrictions / Advanced Memory Layout","text":"","category":"section"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Advanced users may leverage Enzyme's memory semantics (only touching locations in the shadow that were touched in the primal) for additional performance/memory savings, at the obvious cost of potential safety if used incorrectly.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Consider the following function that loads from offset 47 of a Ptr","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"function f(ptr)\n x = unsafe_load(ptr, 47)\n x * x\nend\n\nptr = Base.reinterpret(Ptr{Float64}, Libc.malloc(100*sizeof(Float64)))\nunsafe_store!(ptr, 3.14, 47)\n\nf(ptr)\n\n# output\n9.8596","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"The recommended (and guaranteed sound) way to differentiate this is to pass in a shadow pointer that is congruent with the primal. That is to say, its length (and recursively for any sub types) are equivalent to the primal.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"ptr = Base.reinterpret(Ptr{Float64}, Libc.malloc(100*sizeof(Float64)))\nunsafe_store!(ptr, 3.14, 47)\ndptr = Base.reinterpret(Ptr{Float64}, Libc.calloc(100*sizeof(Float64), 1))\n\nautodiff(Reverse, f, Duplicated(ptr, dptr))\n\nunsafe_load(dptr, 47)\n\n# output\n6.28","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"However, since we know the original function only reads from one float64, we could choose to only allocate a single float64 for the shadow, as long as we ensure that loading from offset 47 (the only location accessed) is in bounds.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"ptr = Base.reinterpret(Ptr{Float64}, Libc.malloc(100*sizeof(Float64)))\nunsafe_store!(ptr, 3.14, 47)\ndptr = Base.reinterpret(Ptr{Float64}, Libc.calloc(sizeof(Float64), 1))\n\n# offset the pointer to have unsafe_load(dptr, 47) access the 0th byte of dptr\n# since julia one indexes we subtract 46 * sizeof(Float64) here\nautodiff(Reverse, f, Duplicated(ptr, dptr - 46 * sizeof(Float64)))\n\n# represents the derivative of the 47'th elem of ptr, \nunsafe_load(dptr)\n\n# output\n6.28","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"However, this style of optimization is not specific to Enzyme, or AD, as one could have done the same thing on the primal code where it only passed in one float. The difference, here however, is that performing these memory-layout tricks safely in Enzyme requires understanding the access patterns of the generated derivative code – like discussed here.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"ptr = Base.reinterpret(Ptr{Float64}, Libc.calloc(sizeof(Float64), 1))\nunsafe_store!(ptr, 3.14)\n# offset the pointer to have unsafe_load(ptr, 47) access the 0th byte of dptr\n# again since julia one indexes we subtract 46 * sizeof(Float64) here\nf(ptr - 46 * sizeof(Float64))\n\n# output\n9.8596","category":"page"},{"location":"faq/#CUDA-support","page":"FAQ","title":"CUDA support","text":"","category":"section"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"CUDA.jl is only supported on Julia v1.7.0 and onwards. On v1.6, attempting to differentiate CUDA kernel functions will not use device overloads correctly and thus returns fundamentally wrong results.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Specifically, differentiating within device kernels is supported. See our cuda tests for some examples. ","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Differentiating through a heterogeneous (e.g. combined host and device) code presently requires defining a custom derivative that tells Enzyme that differentiating an @cuda call is done by performing @cuda of its generated derivative. For an example of this in Enzyme-C++ see here. Automating this for a better experience for CUDA.jl requires an update to CUDA.jl, and is now available for Kernel Abstractions.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Differentiating host-side code when accesses device memory (e.g. sum(CuArray)) is not yet supported, but in progress.","category":"page"},{"location":"faq/#Linear-Algebra","page":"FAQ","title":"Linear Algebra","text":"","category":"section"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Enzyme supports presently some, but not all of Julia's linear algebra library. This is because some of Julia's linear algebra library is not pure Julia code and calls external functions such as BLAS, LaPACK, CuBLAS, SuiteSparse, etc.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"For all BLAS functions, Enzyme will generate a correct derivative function. If it is a gemm (matmul), gemv (matvec), dot (dot product), axpy (vector add and scale), and a few others, Enzyme will generate a fast derivative using another corresponding BLAS call. For other BLAS functions, Enzyme will presently emit a warning Fallback BLAS [functionname] that indicates that Enzyme will differentiate this function by differentiating a serial implementation of BLAS. This will still work for all BLAS codes, but may be slower on a parallel platform.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Other libraries do not yet have derivatives (either fast or fallback) implemented within Enzyme. Supporting these is not a fundamental limitation, but requires implementing a rule in Enzyme describing how to differentiate them. Contributions welcome!","category":"page"},{"location":"faq/#Sparse-arrays","page":"FAQ","title":"Sparse arrays","text":"","category":"section"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Differentiating code using sparse arrays is supported, but care must be taken because backing arrays drop zeros in Julia (unless told not to).","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"using SparseArrays\na = sparse([2.0])\nda1 = sparse([0.0]) # Incorrect: SparseMatrixCSC drops explicit zeros\nEnzyme.autodiff(Reverse, sum, Active, Duplicated(a, da1))\nda1\n\n# output\n\n1-element SparseVector{Float64, Int64} with 0 stored entries","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"da2 = sparsevec([1], [0.0]) # Correct: Prevent SparseMatrixCSC from dropping zeros\nEnzyme.autodiff(Reverse, sum, Active, Duplicated(a, da2))\nda2\n\n# output\n\n1-element SparseVector{Float64, Int64} with 1 stored entry:\n [1] = 1.0","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Sometimes, determining how to perform this zeroing can be complicated. That is why Enzyme provides a helper function Enzyme.make_zero that does this automatically.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Enzyme.make_zero(a)\nEnzyme.gradient(Reverse, sum, a)[1] # This calls make_zero(a)\n\n# output\n\n1-element SparseVector{Float64, Int64} with 1 stored entry:\n [1] = 1.0","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Some Julia libraries sparse linear algebra libraries call out to external C code like SuiteSparse which we don't presently implement derivatives for (we have some but have yet to complete all). If that case happens, Enzyme will throw a \"no derivative found\" error at the callsite of that function. This isn't a fundamental limitation, and is easily resolvable by writing a custom rule or internal Enzyme support. Help is certainly welcome :).","category":"page"},{"location":"faq/#Advanced-Sparse-arrays","page":"FAQ","title":"Advanced Sparse arrays","text":"","category":"section"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Essentially the way Enzyme represents all data structures, including sparse data structures, is to have the shadow (aka derivative) memory be the same memory layout as the primal. Suppose you have an input data structure x. The derivative of x at byte offset 12 will be stored in the shadow dx at byte offset 12, etc.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"This has the nice property that the storage for the derivative, including all intermediate computations, is the same as that of the primal (ignoring caching requirements for reverse mode).","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"It also means that any arbitrary data structure can be differentiated with respect to, and we don’t have any special handling required to register every data structure one could create.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"This representation does have some caveats (e.g. see Identical types in Duplicated above).","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Sparse data structures are often represented with say a Vector{Float64} that holds the actual elements, and a Vector{Int} that specifies the index n the backing array that corresponds to the true location in the overall vector.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"We have no explicit special cases for sparse Data structures, so the layout semantics mentioned above is indeed what Enzyme uses.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Thus the derivative of a sparse array is to have a second backing array of the same size, and another Vector{Int} (of the same offsets).","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"As a concrete example, suppose we have the following: x = { 3 : 2.7, 10 : 3.14 }. In other words, a sparse data structure with two elements, one at index 3, another at index 10. This could be represented with the backing array being [2.7, 3.14] and the index array being [3, 10].","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"A correctly zero-initialized shadow data structure would be to have a backing array of size 2 with zero’s, and an index array again being [3, 10].","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"In this form the second element of the derivative backing array is used to store/represent the derivative of the second element of the original backing array, in other words the derivative at index 10.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Like mentioned above, a caveat here is that this correctly zero’d initializer is not the default produced by sparse([0.0]) as this drops the zero elements from the backing array. Enzyme.makezero recursively goes through your data structure to generate the shadows of the correct structure (and in this case would make a new backing array of appropriate size). The `makezero` function is not special cased to sparsity, but just comes out as a result.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Internally, when differentiating a function this is the type of data structure that Enzyme builds and uses to represent variables. However, at the Julia level that there’s a bit of a sharp edge.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Consider a function f(A(x)) where x is a scalar or dense input, A(x) returns a sparse array, and f(A(x)) returns a scalar loss. ","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"The derivative that Enzyme creates for A(x) would create both the backing/index arrays for the original result A, as well as the equal sized backing/index arrays for the derivative.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"For any program which generates sparse data structures internally, like the total program f(A(x)), this will always give you the answer you expect. Moreover, the memory requirements of the derivative will be the same as the primal (other AD tools will blow up the memory usage and construct dense derivatives where the primal was sparse).","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"The added caveat, however, comes when you differentiate a top level function that has a sparse array input. For example, consider the sparse sum function which adds up all elements. While in one definition, this function represents summing up all elements of the virtual sparse array (including the zero's which are never materialized), in a more literal sense this sum function will only add elements 3 and 10 of the input sparse array – the only two nonzero elements – or equivalently the sum of the whole backing array. Correspondingly Enzyme will update the sparse shadow data structure to mark both elements 3 and 10 as having a derivative of 1 (or more literally set all the elements of the backing array to derivative 1). These are the only variables that Enzyme needs to update, since they are the only variables read (and thus the only ones which have a non-zero derivative). Thus any function which may call this method and compose via the chain rule will only ever read the derivative of these two elements. This is why this memory-safe representation composes within Enzyme, though may produce counter-intuitive reuslts at the top level.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"If the name we gave to this data structure wasn’t \"SparseArray\" but instead \"MyStruct\" this is precisely the answer we would have desired. However, since the sparse array printer prints zeros for elements outside of the sparse backing array, this isn’t what one would expect. Making a nicer user conversion from Enzyme’s form of differential data structures, to the more natural \"Julia\" form where there is a semantic mismatch between what Julia intends a data structure to mean by name, and what is being discussed here.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"The benefit of this representation is that : (1) all of our rules compose correctly (you get the correct answer for f(A(x)), (2) without the need to special case any sparse code, and (3) with the same memory/performance expectations as the original code.","category":"page"},{"location":"faq/#Activity-of-temporary-storage","page":"FAQ","title":"Activity of temporary storage","text":"","category":"section"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"If you pass in any temporary storage which may be involved in an active computation to a function you want to differentiate, you must also pass in a duplicated temporary storage for use in computing the derivatives. For example, consider the following function which uses a temporary buffer to compute the result.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"function f(x, tmp, k, n)\n tmp[1] = 1.0\n for i in 1:n\n tmp[k] *= x\n end\n tmp[1]\nend\n\n# output\n\nf (generic function with 1 method)","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Marking the argument for tmp as Const (aka non-differentiable) means that Enzyme believes that all variables loaded from or stored into tmp must also be non-differentiable, since all values inside a non-differentiable variable must also by definition be non-differentiable.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Enzyme.autodiff(Reverse, f, Active(1.2), Const(Vector{Float64}(undef, 1)), Const(1), Const(5)) # Incorrect\n\n# output\n\n((0.0, nothing, nothing, nothing),)","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Passing in a duplicated (e.g. differentiable) variable for tmp now leads to the correct answer.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Enzyme.autodiff(Reverse, f, Active(1.2), Duplicated(Vector{Float64}(undef, 1), zeros(1)), Const(1), Const(5)) # Correct (returns 10.367999999999999 == 1.2^4 * 5)\n\n# output\n\n((10.367999999999999, nothing, nothing, nothing),)","category":"page"},{"location":"faq/#Runtime-Activity","page":"FAQ","title":"Runtime Activity","text":"","category":"section"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"When computing the derivative of mutable variables, Enzyme also needs additional temporary storage space for the corresponding derivative variables. If an argument tmp is marked as Const, Enzyme does not have any temporary storage space for the derivatives!","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Enzyme will error when they detect these latter types of situations, which we will refer to as activity unstable. This term is chosen to mirror the Julia notion of type-unstable code (e.g. where a type is not known at compile time). If an expression is activity unstable, it could either be constant, or active, depending on data not known at compile time. For example, consider the following:","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"function g(cond, active_var, constant_var)\n if cond\n return active_var\n else\n return constant_var\nend\n\nEnzyme.autodiff(Forward, g, Const(condition), Duplicated(x, dx), Const(y))","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"The returned value here could either by constant or duplicated, depending on the runtime-defined value of cond. If cond is true, Enzyme simply returns the shadow of active_var as the derivative. However, if cond is false, there is no derivative shadow for constant_var and Enzyme will throw a EnzymeRuntimeActivityError error. For some simple types, e.g. a float Enzyme can circumvent this issue, for example by returning the float 0. Similarly, for some types like the Symbol type, which are never differentiable, such a shadow value will never be used, and Enzyme can return the original \"primal\" value as its derivative. However, for arbitrary data structures, Enzyme presently has no generic mechanism to resolve this.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"For example consider a third function:","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"function h(cond, active_var, constant_var)\n return [g(cond, active_var, constant_var), g(cond, active_var, constant_var)]\nend\n\nEnzyme.autodiff(Forward, h, Const(condition), Duplicated(x, dx), Const(y))","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Enzyme provides a nice utility Enzyme.make_zero which takes a data structure and constructs a deepcopy of the data structure with all of the floats set to zero and non-differentiable types like Symbols set to their primal value. If Enzyme gets into such a \"Mismatched activity\" situation where it needs to return a differentiable data structure from a constant variable, it could try to resolve this situation by constructing a new shadow data structure, such as with Enzyme.make_zero. However, this still can lead to incorrect results. In the case of h above, suppose that active_var and consant_var are both arrays, which are mutable (aka in-place) data types. This means that the return of h is going to either be result = [active_var, active_var] or result = [constant_var, constant_var]. Thus an update to result[1][1] would also change result[2][1] since result[1] and result[2] are the same array. ","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"If one created a new zero'd copy of each return from g, this would mean that the derivative dresult would have one copy made for the first element, and a second copy made for the second element. This could lead to incorrect results, and is unfortunately not a general resolution. However, for non-mutable variables (e.g. like floats) or non-differrentiable types (e.g. like Symbols) this problem can never arise.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Instead, Enzyme has a special mode known as \"Runtime Activity\" which can handle these types of situations. It can come with a minor performance reduction, and is therefore off by default. It can be enabled with by setting runtime activity to true in a desired differentiation mode.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"The way Enzyme's runtime activity resolves this issue is to return the original primal variable as the derivative whenever it needs to denote the fact that a variable is a constant. As this issue can only arise with mutable variables, they must be represented in memory via a pointer. All addtional loads and stores will now be modified to first check if the primal pointer is the same as the shadow pointer, and if so, treat it as a constant. Note that this check is not saying that the same arrays contain the same values, but rather the same backing memory represents both the primal and the shadow (e.g. a === b or equivalently pointer(a) == pointer(b)). ","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Enabling runtime activity does therefore, come with a sharp edge, which is that if the computed derivative of a function is mutable, one must also check to see if the primal and shadow represent the same pointer, and if so the true derivative of the function is actually zero.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Generally, the preferred solution to these type of activity unstable codes should be to make your variables all activity-stable (e.g. always containing differentiable memory or always containing non-differentiable memory). However, with care, Enzyme does support \"Runtime Activity\" as a way to differentiate these programs without having to modify your code. One can enable runtime activity for your code by changing the mode, such as","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Enzyme.autodiff(set_runtime_activity(Forward), h, Const(condition), Duplicated(x, dx), Const(y))","category":"page"},{"location":"faq/#Mixed-activity","page":"FAQ","title":"Mixed activity","text":"","category":"section"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Sometimes in Reverse mode (but not forward mode), you may see an error Type T has mixed internal activity types for some type. This error arises when a variable in a computation cannot be fully represented as either a Duplicated or Active variable.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Active variables are used for immutable variables (like Float64), whereas Duplicated variables are used for mutable variables (like Vector{Float64}). Speciically, since Active variables are immutable, functions with Active inputs will return the adjoint of that variable. In contrast Duplicated variables will have their derivatives +='d in place.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"This error indicates that you have a type, like Tuple{Float, Vector{Float64}} that has immutable components and mutable components. Therefore neither Active nor Duplicated can be used for this type.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Internally, by virtue of working at the LLVM level, most Julia types are represented as pointers, and this issue does not tend to arise within code fully differentiated by Enzyme internally. However, when a program needs to interact with Julia API's (e.g. as arguments to a custom rule, a type unstable call, or the outermost function being differentiated), Enzyme must adhere to Julia's notion of immutability and will throw this error rather than risk an incorrect result.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"For example, consider the following code, which has a type unstable call to myfirst, passing in a mixed type Tuple{Float64, Vector{Float64}}.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"@noinline function myfirst(tup::T) where T\n return tup[1]\nend\n\nfunction f(x::Float64)\n vec = [x]\n tup = (x, vec)\n Base.inferencebarrier(myfirst)(tup)::Float64\nend\n\nEnzyme.autodiff(Reverse, f, Active, Active(3.1))","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"When this situation arises, it is often easiest to resolve it by adding a level of indirection to ensure the entire variable is mutable. For example, one could enclose this variable in a reference, such as Ref{Tuple{Float, Vector{Float64}}}, like as follows.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"@noinline function myfirst_ref(tup_ref::T) where T\n tup = tup_ref[]\n return tup[1]\nend\n\nfunction f2(x::Float64)\n vec = [x]\n tup = (x, vec)\n tup_ref = Ref(tup)\n Base.inferencebarrier(myfirst_ref)(tup_ref)::Float64\nend\n\nEnzyme.autodiff(Reverse, f2, Active, Active(3.1))","category":"page"},{"location":"faq/#Complex-numbers","page":"FAQ","title":"Complex numbers","text":"","category":"section"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Differentiation of a function which returns a complex number is ambiguous, because there are several different gradients which may be desired. Rather than assume a specific of these conventions and potentially result in user error when the resulting derivative is not the desired one, Enzyme forces users to specify the desired convention by returning a real number instead.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Consider the function f(z) = z*z. If we were to differentiate this and have real inputs and outputs, the derivative f'(z) would be unambiguously 2*z. However, consider breaking down a complex number down into real and imaginary parts. Suppose now we were to call f with the explicit real and imaginary components, z = x + i y. This means that f is a function that takes an input of two values and returns two values f(x, y) = u(x, y) + i v(x, y). In the case of z*z this means that u(x,y) = x*x-y*y and v(x,y) = 2*x*y.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"If we were to look at all first-order derivatives in total, we would end up with a 2x2 matrix (i.e. Jacobian), the derivative of each output wrt each input. Let's try to compute this, first by hand, then with Enzyme.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"grad u(x, y) = [d/dx u, d/dy u] = [d/dx x*x-y*y, d/dy x*x-y*y] = [2*x, -2*y];\ngrad v(x, y) = [d/dx v, d/dy v] = [d/dx 2*x*y, d/dy 2*x*y] = [2*y, 2*x];","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Reverse mode differentiation computes the derivative of all inputs with respect to a single output by propagating the derivative of the return to its inputs. Here, we can explicitly differentiate with respect to the real and imaginary results, respectively, to find this matrix.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"f(z) = z * z\n\n# a fixed input to use for testing\nz = 3.1 + 2.7im\n\ngrad_u = Enzyme.autodiff(Reverse, z->real(f(z)), Active, Active(z))[1][1]\ngrad_v = Enzyme.autodiff(Reverse, z->imag(f(z)), Active, Active(z))[1][1]\n\n(grad_u, grad_v)\n# output\n(6.2 - 5.4im, 5.4 + 6.2im)","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"This is somewhat inefficient, since we need to call the forward pass twice, once for the real part, once for the imaginary. We can solve this using batched derivatives in Enzyme, which computes several derivatives for the same function all in one go. To make it work, we're going to need to use split mode, which allows us to provide a custom derivative return value.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"fwd, rev = Enzyme.autodiff_thunk(ReverseSplitNoPrimal, Const{typeof(f)}, Active, Active{ComplexF64})\n\n# Compute the reverse pass seeded with a differntial return of 1.0 + 0.0im\ngrad_u = rev(Const(f), Active(z), 1.0 + 0.0im, fwd(Const(f), Active(z))[1])[1][1]\n# Compute the reverse pass seeded with a differntial return of 0.0 + 1.0im\ngrad_v = rev(Const(f), Active(z), 0.0 + 1.0im, fwd(Const(f), Active(z))[1])[1][1]\n\n(grad_u, grad_v)\n\n# output\n(6.2 - 5.4im, 5.4 + 6.2im)","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Now let's make this batched","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"fwd, rev = Enzyme.autodiff_thunk(ReverseSplitWidth(ReverseSplitNoPrimal, Val(2)), Const{typeof(f)}, Active, Active{ComplexF64})\n\n# Compute the reverse pass seeded with a differential return of 1.0 + 0.0im and 0.0 + 1.0im in one go!\nrev(Const(f), Active(z), (1.0 + 0.0im, 0.0 + 1.0im), fwd(Const(f), Active(z))[1])[1][1]\n\n# output\n(6.2 - 5.4im, 5.4 + 6.2im)","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"In contrast, Forward mode differentiation computes the derivative of all outputs with respect to a single input by providing a differential input. Thus we need to seed the shadow input with either 1.0 or 1.0im, respectively. This will compute the transpose of the matrix we found earlier.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"d/dx f(x, y) = d/dx [u(x,y), v(x,y)] = d/dx [x*x-y*y, 2*x*y] = [ 2*x, 2*y];\nd/dy f(x, y) = d/dy [u(x,y), v(x,y)] = d/dy [x*x-y*y, 2*x*y] = [-2*y, 2*x];","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"d_dx = Enzyme.autodiff(Forward, f, Duplicated(z, 1.0+0.0im))[1]\nd_dy = Enzyme.autodiff(Forward, f, Duplicated(z, 0.0+1.0im))[1]\n\n(d_dx, d_dy)\n\n# output\n(6.2 + 5.4im, -5.4 + 6.2im)","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Again, we can go ahead and batch this.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Enzyme.autodiff(Forward, f, BatchDuplicated(z, (1.0+0.0im, 0.0+1.0im)))[1]\n\n# output\n(var\"1\" = 6.2 + 5.4im, var\"2\" = -5.4 + 6.2im)","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Taking Jacobians with respect to the real and imaginary results is fine, but for a complex scalar function it would be really nice to have a single complex derivative. More concretely, in this case when differentiating z*z, it would be nice to simply return 2*z. However, there are four independent variables in the 2x2 jacobian, but only two in a complex number. ","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Complex differentiation is often viewed in the lens of directional derivatives. For example, what is the derivative of the function as the real input increases, or as the imaginary input increases. Consider the derivative along the real axis, textttlim_Delta x rightarrow 0 fracf(x+Delta x y)-f(x y)Delta x. This simplifies to textttlim_Delta x rightarrow 0 fracu(x+Delta x y)-u(x y) + i left v(x+Delta x y)-v(x y)rightDelta x = fracpartialpartial x u(xy) + ifracpartialpartial x v(xy). This is exactly what we computed by seeding forward mode with a shadow of 1.0 + 0.0im.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"For completeness, we can also consider the derivative along the imaginary axis textttlim_Delta y rightarrow 0 fracf(x y+Delta y)-f(x y)iDelta y. Here this simplifies to textttlim_u(x y+Delta y)-u(x y) + i left v(x y+Delta y)-v(x y)rightiDelta y = -ifracpartialpartial y u(xy) + fracpartialpartial y v(xy). Except for the i in the denominator of the limit, this is the same as the result of Forward mode, when seeding x with a shadow of 0.0 + 1.0im. We can thus compute the derivative along the real axis by multiplying our second Forward mode call by -im.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"d_real = Enzyme.autodiff(Forward, f, Duplicated(z, 1.0+0.0im))[1]\nd_im = -im * Enzyme.autodiff(Forward, f, Duplicated(z, 0.0+1.0im))[1]\n\n(d_real, d_im)\n\n# output\n(6.2 + 5.4im, 6.2 + 5.4im)","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Interestingly, the derivative of z*z is the same when computed in either axis. That is because this function is part of a special class of functions that are invariant to the input direction, called holomorphic. ","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Thus, for holomorphic functions, we can simply seed Forward-mode AD with a shadow of one for whatever input we are differenitating. This is nice since seeding the shadow with an input of one is exactly what we'd do for real-valued funtions as well.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Reverse-mode AD, however, is more tricky. This is because holomorphic functions are invariant to the direction of differentiation (aka the derivative inputs), not the direction of the differential return.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"However, if a function is holomorphic, the two derivative functions we computed above must be the same. As a result, fracpartialpartial x u = fracpartialpartial y v and fracpartialpartial y u = -fracpartialpartial x v. ","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"We saw earlier, that performing reverse-mode AD with a return seed of 1.0 + 0.0im yielded [d/dx u, d/dy u]. Thus, for a holomorphic function, a real-seeded Reverse-mode AD computes [d/dx u, -d/dx v], which is the complex conjugate of the derivative.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"conj(grad_u)\n\n# output\n\n6.2 + 5.4im","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"In the case of a scalar-input scalar-output function, that's sufficient. However, most of the time one uses reverse mode, it involves either several inputs or outputs, perhaps via memory. This case requires additional handling to properly sum all the partial derivatives from the use of each input and apply the conjugate operator at only the ones relevant to the differential return.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"For simplicity, Enzyme provides a helper utlity ReverseHolomorphic which performs Reverse mode properly here, assuming that the function is indeed holomorphic and thus has a well-defined single derivative.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Enzyme.autodiff(ReverseHolomorphic, f, Active, Active(z))[1][1]\n\n# output\n\n6.2 + 5.4im","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"For even non-holomorphic functions, complex analysis allows us to define fracpartialpartial z = frac12left(fracpartialpartial x - i fracpartialpartial y right). For non-holomorphic functions, this allows us to compute d/dz. Let's consider myabs2(z) = z * conj(z). We can compute the derivative wrt z of this in Forward mode as follows, which as one would expect results in a result of conj(z):","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"myabs2(z) = z * conj(z)\n\ndabs2_dx, dabs2_dy = Enzyme.autodiff(Forward, myabs2, BatchDuplicated(z, (1.0 + 0.0im, 0.0 + 1.0im)))[1]\n(dabs2_dx - im * dabs2_dy) / 2\n\n# output\n\n3.1 - 2.7im","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Similarly, we can compute d/d conj(z) = d/dx + i d/dy.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"(dabs2_dx + im * dabs2_dy) / 2\n\n# output\n\n3.1 + 2.7im","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Computing this in Reverse mode is more tricky. Let's expand f in terms of u and v. fracpartialpartial z f = frac12 left( u_x + i v_x - i u_y + i v_y right) = frac12 left( u_x + v_y + i v_x - u_y right). Thus d/dz = (conj(grad_u) + im * conj(grad_v))/2.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"abs2_fwd, abs2_rev = Enzyme.autodiff_thunk(ReverseSplitWidth(ReverseSplitNoPrimal, Val(2)), Const{typeof(myabs2)}, Active, Active{ComplexF64})\n\n# Compute the reverse pass seeded with a differential return of 1.0 + 0.0im and 0.0 + 1.0im in one go!\ngradabs2_u, gradabs2_v = abs2_rev(Const(myabs2), Active(z), (1.0 + 0.0im, 0.0 + 1.0im), abs2_fwd(Const(myabs2), Active(z))[1])[1][1]\n\n(conj(gradabs2_u) + im * conj(gradabs2_v)) / 2\n\n# output\n\n3.1 - 2.7im","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"For d/d conj(z), frac12 left( u_x + i v_x + i u_y + i v_y right) = frac12 left( u_x - v_y + i v_x + u_y right). Thus d/d conj(z) = (grad_u + im * grad_v)/2.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"(gradabs2_u + im * gradabs2_v) / 2\n\n# output\n\n3.1 + 2.7im","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Note: when writing rules for complex scalar functions, in reverse mode one needs to conjugate the differential return, and similarly the true result will be the conjugate of that value (in essence you can think of reverse-mode AD as working in the conjugate space).","category":"page"},{"location":"faq/#What-types-are-differentiable?","page":"FAQ","title":"What types are differentiable?","text":"","category":"section"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Enzyme tracks differentiable dataflow through values. Specifically Enzyme tracks differentiable data in base types like Float32, Float64, Float16, BFloat16, etc.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"As a simple example:","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"f(x) = x * x\nEnzyme.autodiff(Forward, f, Duplicated(3.0, 1.0))\n\n# output\n\n(6.0,)","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Enzyme also tracks differentiable data in any types containing these base types (e.g. floats). For example, consider a struct or array containing floats.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"struct Pair\n lhs::Float64\n rhs::Float64\nend\nf_pair(x) = x.lhs * x.rhs\nEnzyme.autodiff(Forward, f_pair, Duplicated(Pair(3.0, 2.0), Pair(1.0, 0.0)))\n\n# output\n\n(2.0,)","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Enzyme.autodiff(Forward, sum, Duplicated([1.0, 2.0, 3.0], [5.0, 0.0, 100.0]))\n\n\n# output\n\n(105.0,)","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"A differentiable data structure can be arbitrarily complex, such as a linked list.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"\nstruct LList\n prev::Union{Nothing, LList}\n value::Float64\nend\n\nfunction make_list(x::Vector)\n result = nothing\n for value in reverse(x)\n result = LList(result, value)\n end\n return result\nend\n\nfunction list_sum(list::Union{Nothing, LList})\n result = 0.0\n while list != nothing\n result += list.value\n list = list.prev\n end\n return result\nend\n\nlist = make_list([1.0, 2.0, 3.0])\ndlist = make_list([5.0, 0.0, 100.0])\n\nEnzyme.autodiff(Forward, list_sum, Duplicated(list, dlist))\n\n# output\n\n(105.0,)","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"Presently Enzyme only considers floats as base types. As a result, Enzyme does not support differentiating data contained in Ints, Strings, or Vals. If it is desirable for Enzyme to add a base type, please open an issue.","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"f_int(x) = x * x\nEnzyme.autodiff(Forward, f_int, Duplicated, Duplicated(3, 1))\n\n# output\n\nERROR: Return type `Int64` not marked Const, but type is guaranteed to be constant","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"f_str(x) = parse(Float64, x) * parse(Float64, x)\n\nautodiff(Forward, f_str, Duplicated(\"1.0\", \"1.0\"))\n\n# output\n\n(0.0,)","category":"page"},{"location":"faq/","page":"FAQ","title":"FAQ","text":"f_val(::Val{x}) where x = x * x\n\nautodiff(Forward, f_val, Duplicated(Val(1.0), Val(1.0)))\n\n# output\n\nERROR: Type of ghost or constant type Duplicated{Val{1.0}} is marked as differentiable.","category":"page"}] }