Merge in tip-of-tree changes

enzyme/Enzyme/ActivityAnalysis.cpp

@@ -484,27 +484,6 @@ bool ActivityAnalyzer::isFunctionArgumentConstant(CallInst *CI, Value *val) {
       CI->getArgOperand(0) != val && CI->getArgOperand(1) != val)
     return true;
 
-  // only the float arg input is potentially active
-  if (Name == "frexp" || Name == "frexpf" || Name == "frexpl") {
-    return val != CI->getOperand(0);
-  }
-
-  // The relerr argument is inactive
-  if (Name == "Faddeeva_erf" || Name == "Faddeeva_erfc" ||
-      Name == "Faddeeva_erfcx" || Name == "Faddeeva_erfi" ||
-      Name == "Faddeeva_dawson") {
-#if LLVM_VERSION_MAJOR >= 14
-    for (size_t i = 0; i < CI->arg_size() - 1; i++)
-#else
-    for (size_t i = 0; i < CI->getNumArgOperands() - 1; i++)
-#endif
-    {
-      if (val == CI->getOperand(i))
-        return false;
-    }
-    return true;
-  }
-
   // only the buffer is active for mpi send/recv
   if (Name == "MPI_Recv" || Name == "PMPI_Recv" || Name == "MPI_Send" ||
       Name == "PMPI_Send") {
@@ -550,23 +529,6 @@ static inline void propagateArgumentInformation(
     propagateFromOperand(CI.getArgOperand(0));
     return;
   }
-  if (Name == "frexp" || Name == "frexpf" || Name == "frexpl") {
-    propagateFromOperand(CI.getOperand(0));
-    return;
-  }
-  if (Name == "Faddeeva_erf" || Name == "Faddeeva_erfc" ||
-      Name == "Faddeeva_erfcx" || Name == "Faddeeva_erfi" ||
-      Name == "Faddeeva_dawson") {
-#if LLVM_VERSION_MAJOR >= 14
-    for (size_t i = 0; i < CI.arg_size() - 1; i++)
-#else
-    for (size_t i = 0; i < CI.getNumArgOperands() - 1; i++)
-#endif
-    {
-      propagateFromOperand(CI.getOperand(i));
-    }
-    return;
-  }
 
   if (Name == "julia.call" || Name == "julia.call2") {
 #if LLVM_VERSION_MAJOR >= 14

enzyme/Enzyme/BlasDerivatives.td

@@ -86,6 +86,9 @@ class Constant<string _value> {
 class Char<string _value> {
   string value = _value;
 }
+class Alloca<int _value> {
+  int value = _value;
+}
 
 class transpose<string _name> {
   string name = _name;
@@ -242,7 +245,7 @@ def gemm : CallBlasPattern<(Op $layout, $transa, $transb, $m, $n, $k, $alpha, $A
                                                     (Concat adj<"C">, $A, (ld $A, $transa, $lda, $m, $k))),
                                                   Constant<"1.0">, adj<"B">),
                   /* beta  */ (FrobInnerProd<""> $m, $n, adj<"C">, input<"C">),
-                  /* C     */ (b<"lascl"> $layout, Char<"G">, ConstantInt<0>, ConstantInt<0>, Constant<"1.0">, $beta, $m, $n, adj<"C">)
+                  /* C     */ (b<"lascl"> $layout, Char<"G">, ConstantInt<0>, ConstantInt<0>, Constant<"1.0">, $beta, $m, $n, adj<"C">, Alloca<1>)
                   ]
                   >;
 

enzyme/Enzyme/Clang/EnzymeClang.cpp

@@ -414,6 +414,114 @@ struct EnzymeInactiveAttrInfo : public ParsedAttrInfo {
 static ParsedAttrInfoRegistry::Add<EnzymeInactiveAttrInfo> X4("enzyme_inactive",
                                                               "");
 
+struct EnzymeNoFreeAttrInfo : public ParsedAttrInfo {
+  EnzymeNoFreeAttrInfo() {
+    OptArgs = 1;
+    // GNU-style __attribute__(("example")) and C++/C2x-style [[example]] and
+    // [[plugin::example]] supported.
+    static constexpr Spelling S[] = {
+      {ParsedAttr::AS_GNU, "enzyme_nofree"},
+#if LLVM_VERSION_MAJOR > 17
+      {ParsedAttr::AS_C23, "enzyme_nofree"},
+#else
+      {ParsedAttr::AS_C2x, "enzyme_nofree"},
+#endif
+      {ParsedAttr::AS_CXX11, "enzyme_nofree"},
+      {ParsedAttr::AS_CXX11, "enzyme::nofree"}
+    };
+    Spellings = S;
+  }
+
+  bool diagAppertainsToDecl(Sema &S, const ParsedAttr &Attr,
+                            const Decl *D) const override {
+    // This attribute appertains to functions only.
+    if (isa<FunctionDecl>(D))
+      return true;
+    if (auto VD = dyn_cast<VarDecl>(D)) {
+      if (VD->hasGlobalStorage())
+        return true;
+    }
+    S.Diag(Attr.getLoc(), diag::warn_attribute_wrong_decl_type_str)
+        << Attr << "functions and globals";
+    return false;
+  }
+
+  AttrHandling handleDeclAttribute(Sema &S, Decl *D,
+                                   const ParsedAttr &Attr) const override {
+    if (Attr.getNumArgs() != 0) {
+      unsigned ID = S.getDiagnostics().getCustomDiagID(
+          DiagnosticsEngine::Error,
+          "'enzyme_nofree' attribute requires zero arguments");
+      S.Diag(Attr.getLoc(), ID);
+      return AttributeNotApplied;
+    }
+
+    auto &AST = S.getASTContext();
+    DeclContext *declCtx = D->getDeclContext();
+    auto loc = D->getLocation();
+    RecordDecl *RD;
+    if (S.getLangOpts().CPlusPlus)
+      RD = CXXRecordDecl::Create(AST, clang::TagTypeKind::TTK_Struct, declCtx,
+                                 loc, loc, nullptr); // rId);
+    else
+      RD = RecordDecl::Create(AST, clang::TagTypeKind::TTK_Struct, declCtx, loc,
+                              loc, nullptr); // rId);
+    RD->setAnonymousStructOrUnion(true);
+    RD->setImplicit();
+    RD->startDefinition();
+    auto T = isa<FunctionDecl>(D) ? cast<FunctionDecl>(D)->getType()
+                                  : cast<VarDecl>(D)->getType();
+    auto Name = isa<FunctionDecl>(D) ? cast<FunctionDecl>(D)->getNameAsString()
+                                     : cast<VarDecl>(D)->getNameAsString();
+    auto FT = AST.getPointerType(T);
+    auto &Id = AST.Idents.get(
+        (StringRef("__enzyme_nofree") + "_autoreg_" + Name).str());
+    auto V = VarDecl::Create(AST, declCtx, loc, loc, &Id, FT, nullptr, SC_None);
+    V->setStorageClass(SC_PrivateExtern);
+    V->addAttr(clang::UsedAttr::CreateImplicit(AST));
+    TemplateArgumentListInfo *TemplateArgs = nullptr;
+    auto DR = DeclRefExpr::Create(
+        AST, NestedNameSpecifierLoc(), loc, cast<ValueDecl>(D), false, loc, T,
+        ExprValueKind::VK_LValue, cast<NamedDecl>(D), TemplateArgs);
+#if LLVM_VERSION_MAJOR >= 13
+    auto rval = ExprValueKind::VK_PRValue;
+#else
+    auto rval = ExprValueKind::VK_RValue;
+#endif
+    Expr *expr = nullptr;
+    if (isa<FunctionDecl>(D)) {
+#if LLVM_VERSION_MAJOR >= 12
+      expr =
+          ImplicitCastExpr::Create(AST, FT, CastKind::CK_FunctionToPointerDecay,
+                                   DR, nullptr, rval, FPOptionsOverride());
+#else
+      expr = ImplicitCastExpr::Create(
+          AST, FT, CastKind::CK_FunctionToPointerDecay, DR, nullptr, rval);
+#endif
+    } else {
+      expr =
+          UnaryOperator::Create(AST, DR, UnaryOperatorKind::UO_AddrOf, FT, rval,
+                                clang::ExprObjectKind ::OK_Ordinary, loc,
+                                /*canoverflow*/ false, FPOptionsOverride());
+    }
+
+    if (expr->isValueDependent()) {
+      unsigned ID = S.getDiagnostics().getCustomDiagID(
+          DiagnosticsEngine::Error, "use of attribute 'enzyme_nofree' "
+                                    "in a templated context not yet supported");
+      S.Diag(Attr.getLoc(), ID);
+      return AttributeNotApplied;
+    }
+    V->setInit(expr);
+    V->dump();
+    S.MarkVariableReferenced(loc, V);
+    S.getASTConsumer().HandleTopLevelDecl(DeclGroupRef(V));
+    return AttributeApplied;
+  }
+};
+
+static ParsedAttrInfoRegistry::Add<EnzymeNoFreeAttrInfo> X5("enzyme_nofree",
+                                                            "");
 } // namespace
 
 #endif

enzyme/Enzyme/EnzymeLogic.cpp

@@ -1740,6 +1740,18 @@ void clearFunctionAttributes(Function *f) {
     if (f->hasAttribute(llvm::AttributeList::ReturnIndex, attr)) {
       f->removeAttribute(llvm::AttributeList::ReturnIndex, attr);
     }
+#endif
+  }
+  for (auto attr : {"enzyme_inactive"}) {
+#if LLVM_VERSION_MAJOR >= 14
+    if (f->getAttributes().hasRetAttr(attr)) {
+      f->removeRetAttr(attr);
+    }
+#else
+    if (f->getAttributes().hasAttribute(llvm::AttributeList::ReturnIndex,
+                                        attr)) {
+      f->removeAttribute(llvm::AttributeList::ReturnIndex, attr);
+    }
 #endif
   }
 }
@@ -2564,6 +2576,18 @@ const AugmentedReturn &EnzymeLogic::CreateAugmentedPrimal(
     }
 #endif
   }
+  for (auto attr : {"enzyme_inactive"}) {
+#if LLVM_VERSION_MAJOR >= 14
+    if (gutils->newFunc->getAttributes().hasRetAttr(attr)) {
+      gutils->newFunc->removeRetAttr(attr);
+    }
+#else
+    if (gutils->newFunc->getAttributes().hasAttribute(
+            llvm::AttributeList::ReturnIndex, attr)) {
+      gutils->newFunc->removeAttribute(llvm::AttributeList::ReturnIndex, attr);
+    }
+#endif
+  }
 
   gutils->eraseFictiousPHIs();
 

enzyme/Enzyme/InstructionDerivatives.td

@@ -550,7 +550,7 @@ def : CallPattern<(Op $x, $tbd),
                   ["Faddeeva_erf"],
                   [
                     (ToStruct2 (CFMul (DiffeRet), (CFMul (ConstantCFP<"1.1283791670955125738961589031215451716881012586580","0"> $x), (CFExp (CFNeg (CFMul $x, $x)))))),
-                    (AssertingInactiveArg)
+                    (InactiveArg) // relerr
                   ],
                   (ForwardFromSummedReverse),
                   [ReadNone, NoUnwind]
@@ -560,7 +560,7 @@ def : CallPattern<(Op $x, $tbd),
                   ["Faddeeva_erfi"],
                   [
                     (ToStruct2 (CFMul (DiffeRet), (CFMul (ConstantCFP<"1.1283791670955125738961589031215451716881012586580","0"> $x), (CFExp (CFMul $x, $x))))),
-                    (AssertingInactiveArg)
+                    (InactiveArg) // relerr
                   ],
                   (ForwardFromSummedReverse),
                   [ReadNone, NoUnwind]
@@ -570,7 +570,7 @@ def : CallPattern<(Op $x, $tbd),
                   ["Faddeeva_erfc"],
                   [
                     (ToStruct2 (CFMul (DiffeRet), (CFMul (ConstantCFP<"-1.1283791670955125738961589031215451716881012586580","0"> $x), (CFExp (CFNeg (CFMul $x, $x)))))),
-                    (AssertingInactiveArg)
+                    (InactiveArg) // relerr
                   ],
                   (ForwardFromSummedReverse),
                   [ReadNone, NoUnwind]

enzyme/Enzyme/MustExitScalarEvolution.cpp

@@ -113,6 +113,31 @@ ScalarEvolution::ExitLimit MustExitScalarEvolution::computeExitLimit(
   return getCouldNotCompute();
 }
 
+ScalarEvolution::ExitLimit
+MustExitScalarEvolution::computeExitLimitFromSingleExitSwitch(
+    const Loop *L, SwitchInst *Switch, BasicBlock *ExitingBlock,
+    bool ControlsOnlyExit) {
+  assert(!L->contains(ExitingBlock) && "Not an exiting block!");
+
+  // Give up if the exit is the default dest of a switch.
+  if (Switch->getDefaultDest() == ExitingBlock)
+    return getCouldNotCompute();
+
+  ///! If we're guaranteed unreachable, the default dest does not matter.
+  if (!GuaranteedUnreachable.count(Switch->getDefaultDest()))
+    assert(L->contains(Switch->getDefaultDest()) &&
+           "Default case must not exit the loop!");
+  const SCEV *LHS = getSCEVAtScope(Switch->getCondition(), L);
+  const SCEV *RHS = getConstant(Switch->findCaseDest(ExitingBlock));
+
+  // while (X != Y) --> while (X-Y != 0)
+  ExitLimit EL = howFarToZero(getMinusSCEV(LHS, RHS), L, ControlsOnlyExit);
+  if (EL.hasAnyInfo())
+    return EL;
+
+  return getCouldNotCompute();
+}
+
 ScalarEvolution::ExitLimit
 MustExitScalarEvolution::computeExitLimitFromCondCached(
     ExitLimitCacheTy &Cache, const Loop *L, Value *ExitCond, bool ExitIfTrue,

enzyme/Enzyme/MustExitScalarEvolution.h

@@ -78,6 +78,10 @@ class MustExitScalarEvolution final : public llvm::ScalarEvolution {
                                               const llvm::Loop *L,
                                               bool IsSigned, bool ControlsExit,
                                               bool AllowPredicates);
+
+  ScalarEvolution::ExitLimit computeExitLimitFromSingleExitSwitch(
+      const llvm::Loop *L, llvm::SwitchInst *Switch,
+      llvm::BasicBlock *ExitingBB, bool IsSubExpr);
 };
 
 #endif

enzyme/Enzyme/PreserveNVVM.cpp

@@ -359,6 +359,7 @@ bool preserveNVVM(bool Begin, Function &F) {
     if (g.getName().contains(gradient_handler_name) ||
         g.getName().contains(derivative_handler_name) ||
         g.getName().contains(splitderivative_handler_name) ||
+        g.getName().contains("__enzyme_nofree") ||
         g.getName().contains("__enzyme_inactivefn") ||
         g.getName().contains("__enzyme_function_like") ||
         g.getName().contains("__enzyme_allocation_like")) {
@@ -455,6 +456,34 @@ bool preserveNVVM(bool Begin, Function &F) {
         }
       }
     }
+    if (g.getName().contains("__enzyme_nofree")) {
+      if (g.hasInitializer()) {
+        Value *V = g.getInitializer();
+        while (1) {
+          if (auto CE = dyn_cast<ConstantExpr>(V)) {
+            V = CE->getOperand(0);
+            continue;
+          }
+          if (auto CA = dyn_cast<ConstantAggregate>(V)) {
+            V = CA->getOperand(0);
+            continue;
+          }
+          break;
+        }
+        if (auto F = cast<Function>(V)) {
+          F->addAttribute(AttributeList::FunctionIndex,
+                          Attribute::get(g.getContext(), Attribute::NoFree));
+          toErase.push_back(&g);
+          changed = true;
+        } else {
+          llvm::errs() << "Param of __enzyme_nofree must be a "
+                          "constant function"
+                       << g << "\n"
+                       << *V << "\n";
+          llvm_unreachable("__enzyme_nofree");
+        }
+      }
+    }
     if (g.getName().contains("__enzyme_function_like")) {
       if (g.hasInitializer()) {
         auto CA = dyn_cast<ConstantAggregate>(g.getInitializer());

enzyme/Enzyme/Utils.cpp

@@ -309,6 +309,10 @@ Value *CreateAllocation(IRBuilder<> &Builder, llvm::Type *T, Value *Count,
       ZeroMem = nullptr;
     }
   } else {
+#if LLVM_VERSION_MAJOR > 17
+    res =
+        Builder.CreateMalloc(Count->getType(), T, Align, Count, nullptr, Name);
+#else
     if (Builder.GetInsertPoint() == Builder.GetInsertBlock()->end()) {
       res = CallInst::CreateMalloc(Builder.GetInsertBlock(), Count->getType(),
                                    T, Align, Count, nullptr, Name);
@@ -319,6 +323,7 @@ Value *CreateAllocation(IRBuilder<> &Builder, llvm::Type *T, Value *Count,
     }
     if (!cast<Instruction>(res)->getParent())
       Builder.Insert(cast<Instruction>(res));
+#endif
 
     malloccall = dyn_cast<CallInst>(res);
     if (malloccall == nullptr) {
@@ -408,6 +413,9 @@ CallInst *CreateDealloc(llvm::IRBuilder<> &Builder, llvm::Value *ToFree) {
 
     ToFree = Builder.CreatePointerCast(
         ToFree, Type::getInt8PtrTy(ToFree->getContext()));
+#if LLVM_VERSION_MAJOR > 17
+    res = cast<CallInst>(Builder.CreateFree(ToFree));
+#else
     if (Builder.GetInsertPoint() == Builder.GetInsertBlock()->end()) {
       res = cast<CallInst>(
           CallInst::CreateFree(ToFree, Builder.GetInsertBlock()));
@@ -418,6 +426,7 @@ CallInst *CreateDealloc(llvm::IRBuilder<> &Builder, llvm::Value *ToFree) {
     }
     if (!cast<Instruction>(res)->getParent())
       Builder.Insert(cast<Instruction>(res));
+#endif
 #if LLVM_VERSION_MAJOR >= 14
     res->addAttributeAtIndex(AttributeList::FirstArgIndex, Attribute::NonNull);
 #else

enzyme/test/Enzyme/ReverseMode/blas/cblas_ddot.ll

@@ -60,7 +60,7 @@ entry:
 
 ; COM: Can't check the attrs since number and order depends on llvm version
 ; COM: ; Function Attrs: argmemonly mustprogress nofree norecurse nosync nounwind readonly willreturn
-; CHECK: declare double @cblas_ddot(i32, double* nocapture readonly, i32, double* nocapture readonly, i32)
+; CHECK: declare double @cblas_ddot(i32 "enzyme_inactive", double* nocapture readonly, i32 "enzyme_inactive", double* nocapture readonly, i32 "enzyme_inactive")
 
 ; CHECK: define void @active
 ; CHECK-NEXT: entry
@@ -96,7 +96,7 @@ entry:
 
 ; COM: Can't check the attrs since number and order depends on llvm version
 ; COM: ; Function Attrs: argmemonly mustprogress nofree norecurse nosync nounwind willreturn
-; CHECK: declare void @cblas_daxpy(i32, double, double* nocapture readonly, i32, double* nocapture, i32)
+; CHECK: declare void @cblas_daxpy(i32 "enzyme_inactive", double, double* nocapture readonly, i32 "enzyme_inactive", double* nocapture, i32 "enzyme_inactive")
 
 ; CHECK: define internal void @[[inactiveFirst]](i32 %len, double* noalias %m, i32 %incm, double* noalias %n, double* %"n'", i32 %incn, double %differeturn)
 ; CHECK-NEXT: entry: