Renv Lockfiles

.gitmodules

@@ -150,3 +150,6 @@
 	path = Common/jaspColumnEncoder
 	url = https://github.com/jasp-stats/jaspColumnEncoder
 	branch = master
+[submodule "Engine/jaspModuleInstaller"]
+	path = Engine/jaspModuleInstaller
+	url = https://github.com/jasp-stats/jaspModuleInstaller.git

Desktop/main.cpp

@@ -52,7 +52,7 @@ const std::string	jaspExtension		= ".jasp",
 bool runJaspEngineJunctionFixer(int argc, char *argv[], bool removeJunctions = false, bool exitAfterwards = true)
 {
 	QApplication	*	app		= exitAfterwards ? new QApplication(argc, argv) : nullptr;
-	QProcessEnvironment env		= ProcessHelper::getProcessEnvironmentForJaspEngine();
+	QProcessEnvironment env		= ProcessHelper::getProcessEnvironmentForJaspEngine(true);
 	QString				workDir = QFileInfo( QCoreApplication::applicationFilePath() ).absoluteDir().absolutePath();
 
 	QProcess engine;
@@ -64,7 +64,8 @@ bool runJaspEngineJunctionFixer(int argc, char *argv[], bool removeJunctions = f
 
 	//remove any leftover ModuleDir 
 	QDir modulesDir(AppDirs::bundledModulesDir());
-	if(modulesDir.exists() && AppDirs::bundledModulesDir().contains("Modules", Qt::CaseInsensitive) && DynamicRuntimeInfo::getInstance()->getRuntimeEnvironment() != DynamicRuntimeInfo::ZIP)	{
+	if(modulesDir.exists() && AppDirs::bundledModulesDir().contains("Modules", Qt::CaseInsensitive) && DynamicRuntimeInfo::getInstance()->getRuntimeEnvironment() != DynamicRuntimeInfo::ZIP)
+	{
 		std::function<void(QDir)> removeDir = [&](QDir x) -> void {
 			for(const auto& entry : x.entryInfoList(QDir::NoDotAndDotDot | QDir::Dirs | QDir::Files))
 			{

Desktop/modules/dynamicmodule.cpp

@@ -392,14 +392,15 @@ Json::Value	DynamicModule::requestJsonForPackageLoadingRequest()
 
 	requestJson["moduleRequest"]	= moduleStatusToString(moduleStatus::loading);
 	requestJson["moduleName"]		= _name;
+	requestJson["moduleLibPaths"]   =  getLibPathsToUse();
 	requestJson["moduleCode"]		= generateModuleLoadingR();
 
 	return requestJson;
 }
 
 std::string DynamicModule::getLibPathsToUse() const
 {
-	return "c('" + AppDirs::rHome().toStdString() + "/library', '" + shortenWinPaths(moduleRLibrary()).toStdString()	+ "')";
+	return "c('" + shortenWinPaths(moduleRLibrary()).toStdString() + "', '" + AppDirs::rHome().toStdString() + "/library')";
 }
 
 ///It would probably be better to move all of this code to jasp-r-pkg or something, but for now this works fine.
@@ -427,17 +428,30 @@ std::string DynamicModule::generateModuleInstallingR(bool onlyModPkg)
 		return "stop('Something went wrong during intialization of the Description!\nMake sure it follows the standard set in https://github.com/jasp-stats/jasp-desktop/blob/development/Docs/development/jasp-adding-module.md#descriptionqml\n')";
 	}
 	setInstallLog("Installing module " + _name + ".\n");
-
-	return "options(\"renv.config.install.verbose\" = FALSE);jaspBase::installJaspModule(modulePkg='" + _modulePackage + "', libPathsToUse=" + getLibPathsToUse() + ", moduleLibrary='" + moduleRLibrary().toStdString() +
-		"', repos='" + Settings::value(Settings::CRAN_REPO_URL).toString().toStdString() + "', onlyModPkg=" + (onlyModPkg ? "TRUE" : "FALSE") +
-	 	", force=TRUE, cacheAble=FALSE, frameworkLibrary='"+fq(AppDirs::rHome())+"/library');";
+	return QString(
+	R"readableR(
+	tmp <- .libPaths();
+	.libPaths("%1");
+	Sys.setenv(MODULE_INSTALL_MODE="localizeModuleOnly");
+	options("renv.config.install.verbose" = TRUE, "PKGDEPENDS_LIBRARY"="%2");
+	result <- jaspModuleInstaller::installJaspModule(modulePkg='%3', moduleLibrary='%4', repos='%5', onlyModPkg=%6, force=TRUE, frameworkLibrary='%7');
+	.libPaths(tmp);
+	return(result);
+	)readableR")
+	.arg(AppDirs::bundledModulesDir() + "Tools/jaspModuleInstaller_library/")
+	.arg(AppDirs::bundledModulesDir() + "Tools/pkgdepends_library/")
+	.arg(tq(_modulePackage))
+	.arg(moduleRLibrary())
+	.arg(Settings::value(Settings::CRAN_REPO_URL).toString())
+	.arg(onlyModPkg ? "TRUE" : "FALSE")
+	.arg(AppDirs::rHome()+"/library")
+	.toStdString();
 }
 
 std::string DynamicModule::generateModuleLoadingR(bool shouldReturnSucces)
 {
 	std::stringstream R;
 
-	R << ".libPaths(" << getLibPathsToUse() <<");\n";
 	R << standardRIndent << "library('" << _name << "');\n";
 
 	if(shouldReturnSucces)

Desktop/utilities/processhelper.cpp

@@ -1,12 +1,9 @@
 #include "processhelper.h"
 #include "utilities/appdirs.h"
-
-#ifdef _WIN32
 #include "utilities/qutils.h"
 #include "log.h"
-#endif
 
-QProcessEnvironment ProcessHelper::getProcessEnvironmentForJaspEngine()
+QProcessEnvironment ProcessHelper::getProcessEnvironmentForJaspEngine(bool bootStrap)
 {
 	QDir				programDir	= AppDirs::programDir();
 	QString				engineExe	= programDir.absoluteFilePath("JASPEngine");
@@ -21,8 +18,8 @@ QProcessEnvironment ProcessHelper::getProcessEnvironmentForJaspEngine()
 	//Seems a bit weird but we need to tell this to jaspBase so it can tell renv to run it again because that will be running in a subprocess. 
 	//Which also means we have the following process -> subprocess structure while installing a dynamic module:
 	// jasp -> JASPEngine with R-embedded -> Separate R -> separate instances of JASPEngine...
-	env.insert("JASPENGINE_LOCATION",				engineExe); 
-	
+	env.insert("JASPENGINE_LOCATION",				engineExe);
+
 	QString TZDIR		= AppDirs::rHome() + "/share/zoneinfo";
 	QString rHomePath	= AppDirs::rHome();
 	QDir	rHome		( rHomePath );
@@ -45,7 +42,7 @@ QProcessEnvironment ProcessHelper::getProcessEnvironmentForJaspEngine()
 			R_HOME		= shortenWinPaths(rHome.absolutePath()),
 			JAGS_HOME	= shortenWinPaths(programDir.absoluteFilePath("R/opt/jags/"));
 			// JAGS_LIBDIR	= shortenWinPaths(programDir.absoluteFilePath("R/opt/jags/lib/"));
-	
+
 	Log::log() << "R_HOME set to " << R_HOME << std::endl;
 
 	env.insert("PATH",				PATH);
@@ -55,7 +52,10 @@ QProcessEnvironment ProcessHelper::getProcessEnvironmentForJaspEngine()
 
 #undef ARCH_SUBPATH
 
-	env.insert("R_LIBS",			 R_HOME + "/library");
+	if(bootStrap)
+		env.insert("R_LIBS",			programDir.absoluteFilePath("Modules/Tools/junction_bootstrap_library") + ";" + R_HOME + "/library");
+	else
+		env.insert("R_LIBS",			AppDirs::bundledModulesDir() + "Tools/R_cpp_includes_library" + ";" + R_HOME + "/library");
 
 	env.insert("R_ENVIRON",			"something-which-doesn't-exist");
 	env.insert("R_PROFILE",			"something-which-doesn't-exist");
@@ -73,7 +73,7 @@ QProcessEnvironment ProcessHelper::getProcessEnvironmentForJaspEngine()
 	env.insert("R_HOME",			rHome.absolutePath());
 	env.insert("RHOME",				rHome.absolutePath()); //For Rscript
 	env.insert("JASP_R_HOME",		rHome.absolutePath()); //Used by the modified R script in jasp-required-files/Framework/etc/bin to make sure we use the actual R of JASP! (https://github.com/jasp-stats/INTERNAL-jasp/issues/452)
-	env.insert("R_LIBS",			rHome.absoluteFilePath("library") + ":" + programDir.absoluteFilePath("R/library"));
+	env.insert("R_LIBS",			programDir.absoluteFilePath("../Modules/Tools/R_cpp_includes_library") + ":" + rHome.absoluteFilePath("library") + ":" + programDir.absoluteFilePath("R/library") + custom_R_library);
 	env.insert("JAGS_HOME",			rHome.absolutePath() + "/opt/jags/lib/JAGS/");
 	// env.insert("JAGS_LIBDIR",		rHome.absolutePath() + "/opt/jags/lib/");
 
@@ -84,11 +84,13 @@ QProcessEnvironment ProcessHelper::getProcessEnvironmentForJaspEngine()
 
 	env.insert("LC_CTYPE",			"UTF-8"); //This isn't really a locale but seems necessary to get proper output from gettext on mac
 	env.insert("TZDIR",				TZDIR);
-
+#elif FLATPAK_USED
+	env.insert("R_HOME",			rHome.absolutePath());
+	env.insert("R_LIBS",			"/app/Modules/Tools/R_cpp_includes_library:/app/lib64/R/library" + custom_R_library);
+	env.insert("LD_LIBRARY_PATH",	"/app/Modules/Tools/R_cpp_includes_library/RInside/lib/:/app/lib64/R/lib/");
 #else  // linux
-	env.insert("LD_LIBRARY_PATH",	rHome.absoluteFilePath("lib") + ":" + rHome.absoluteFilePath("library/RInside/lib") + ":" + rHome.absoluteFilePath("library/Rcpp/lib") + ":" + rHome.absoluteFilePath("site-library/RInside/lib") + ":" + rHome.absoluteFilePath("site-library/Rcpp/lib") + ":/app/lib/:/app/lib64/");
 	env.insert("R_HOME",			rHome.absolutePath());
-	env.insert("R_LIBS",			programDir.absoluteFilePath("R/library") + custom_R_library + ":" + rHome.absoluteFilePath("library") + ":" + rHome.absoluteFilePath("site-library"));
+	env.insert("R_LIBS",			programDir.absoluteFilePath("../Modules/Tools/R_cpp_includes_library") + ":" + programDir.absoluteFilePath("R/library") + custom_R_library);
 #endif
 
 	env.insert("R_LIBS_SITE",		"");
@@ -99,5 +101,10 @@ QProcessEnvironment ProcessHelper::getProcessEnvironmentForJaspEngine()
 	env.insert("R_LIBS_USER", (AppDirs::programDir().absolutePath().toStdString() + "/../R/library").c_str());
 #endif
 
+	Log::log() <<	"R_LIBS:"			<< env.value("R_LIBS")			<< "\n" <<
+					"R_LIBS_USER:"		<< env.value("R_LIBS_USER")		<< "\n" <<
+					"LD_LIBRARY_PATH:"	<< env.value("LD_LIBRARY_PATH") << "\n" <<
+					std::endl;
+
 	return(env);	
 }

Desktop/utilities/processhelper.h

@@ -10,7 +10,7 @@ class ProcessHelper
 {
 public:
 
-    static QProcessEnvironment getProcessEnvironmentForJaspEngine();
+    static QProcessEnvironment getProcessEnvironmentForJaspEngine(bool bootStrap = false);
 
 private:
 	ProcessHelper(){}

Docs/development/windows-installer.md

@@ -45,7 +45,7 @@ The generated MSIX for the Microsoft Store will be unsigned and must be provided
 The MSIXs can be generated after building JASP by running the following commands in your JASP build folder:
 
 ```
-cmake --build . --target windowsPreInstallHacks && cmake --build . --target install && cmake --build . --target collect-junctions && cmake --install . --component MSIX && cmake --build . --target msix
+cmake --build . --target install && cmake --build . --target collect-junctions && cmake --install . --component MSIX && cmake --build . --target msix
 ```
 
 This is subject to change, for an up to date sequence take a peak at the [buildbot-script](/Tools/windows/BuildBotScript.cmd).

Engine/engine.cpp

@@ -481,8 +481,15 @@ void Engine::receiveModuleRequestMessage(const Json::Value & jsonRequest)
 	std::string		moduleRequest	= jsonRequest["moduleRequest"].asString();
 	std::string		moduleCode		= jsonRequest["moduleCode"].asString();
 	std::string		moduleName		= jsonRequest["moduleName"].asString();
+	std::string		moduleLibPaths  = jsonRequest["moduleLibPaths"].asString();
 
-	Log::log() << "About to run module request for module '" << moduleName << "' and code to run:\n'" << moduleCode << "'" << std::endl; 
+	Log::log() << "About to run module request for module '" << moduleName << "' and code to run:\n'" << moduleCode << "'" << std::endl;
+
+	if(moduleStatusFromString((moduleRequest)) == moduleStatus::loading) {
+		//Some jaspModules use jaspBase calls in their .onload so we first we need to prepare jaspbase
+		jaspRCPP_evalRCode((".libPaths( " + moduleLibPaths +  " );").c_str(), false);
+		jaspRCPP_init_jaspBase();
+	}
 
 	std::string		result			= jaspRCPP_evalRCode(moduleCode.c_str(), false);
 	bool			succes			= result == "succes!"; //Defined in DynamicModule::succesResultString()

Modules/Rcpp_RInside.lock

@@ -0,0 +1,23 @@
+{
+  "R": {
+    "Version": "4.3.3",
+    "Repositories": [
+      {
+        "Name": "CRAN",
+        "URL": "https://cran.rstudio.com"
+      }
+    ]
+  },
+  "Packages": {
+    "RInside": {
+      "Package": "RInside",
+      "Version": "0.2.18",
+      "Source": "Repository"
+    },
+    "Rcpp": {
+      "Package": "Rcpp",
+      "Version": "1.0.12",
+      "Source": "Repository"
+    }
+  }
+}

Modules/install-jaspModuleInstaller.R.in

@@ -0,0 +1,105 @@
+# Generated from install-jaspBase.R.in
+#
+Sys.setenv(GITHUB_PAT          = "@GITHUB_PAT@")
+Sys.setenv(RENV_PATHS_ROOT     = "@MODULES_RENV_ROOT_PATH@")
+Sys.setenv(RENV_PATHS_CACHE    = "@MODULES_RENV_CACHE_PATH@")
+Sys.setenv(RENV_PATHS_SANDBOX  = "@RENV_SANDBOX@")
+Sys.setenv(JASPENGINE_LOCATION = "@JASP_ENGINE_PATH@/JASPEngine")
+
+if("@R_PKG_CELLAR_PATH@" != "")
+	Sys.setenv(RENV_PATHS_CELLAR   = "@R_PKG_CELLAR_PATH@")
+
+
+#Load the post-install fixes from jaspBase. Think Baron von Munchhausen ;)
+source("@PROJECT_SOURCE_DIR@/Engine/jaspModuleInstaller/R/renvOverrides.R")
+source("@PROJECT_SOURCE_DIR@/Engine/jaspModuleInstaller/R/checksums.R")
+source("@PROJECT_SOURCE_DIR@/Engine/jaspModuleInstaller/R/utils.R")
+source("@PROJECT_SOURCE_DIR@/Engine/jaspModuleInstaller/R/installModule.R")
+
+# The R_LIBRARY_PATH might already be there, but depending on the configuration
+# of the CMake, we might be installing in a different location, so, I just add
+# it anyway! It gets to if-y.
+
+
+.libPaths("@RENV_LIBRARY@")
+sandboxPaths <- renv:::renv_sandbox_activate()
+
+JASPMODULEINSTALLER_LIBRARY <- "@JASPMODULEINSTALLER_LIBRARY@"
+PKGDEPENDS_LIBRARY          <- "@PKGDEPENDS_LIBRARY@"
+RENV_LIBRARY                <- "@RENV_LIBRARY@"
+R_CPP_INCLUDES_LIBRARY      <- "@R_CPP_INCLUDES_LIBRARY@"
+JUNCTION_HANDLING_LIBRARY   <- "@JUNCTION_HANDLING_LIBRARY@"
+
+ENGINE                 <- file.path("@PROJECT_SOURCE_DIR@", "Engine")
+MODULES                <- file.path("@PROJECT_SOURCE_DIR@", "Modules")
+TOOLS                  <- file.path("@PROJECT_SOURCE_DIR@", "Tools")
+
+
+modulePkg     <- file.path("@PROJECT_SOURCE_DIR@", "Engine", "jaspModuleInstaller")
+
+cat("Restoring pkgdepends\n")
+setupRenv(PKGDEPENDS_LIBRARY)
+renv::restore(
+	library  = PKGDEPENDS_LIBRARY,
+	lockfile = file.path(MODULES, "pkgdepends.lock"),
+	clean    = TRUE
+)
+
+if (.Platform$OS.type == "windows") {
+  cat("Restoring junction handling library\n")
+  options(renv.config.cache.symlinks = FALSE)
+  setupRenv(JUNCTION_HANDLING_LIBRARY)
+  renv::restore(
+	library  = JUNCTION_HANDLING_LIBRARY,
+	lockfile = file.path(MODULES, "junctionBootstrap.lock"),
+	clean    = TRUE
+  )
+}
+
+cat("Restoring jaspModuleInstaller\n")
+options(renv.config.cache.symlinks = TRUE)
+setupRenv(JASPMODULEINSTALLER_LIBRARY)
+renv::restore(
+	library  = JASPMODULEINSTALLER_LIBRARY,
+	lockfile = file.path(ENGINE, "jaspModuleInstaller", "renv.lock"),
+	exclude  = "jaspModuleInstaller", # otherwise "clean" would remove jaspModuleInstaller
+	clean    = TRUE
+)
+
+print("Installing jaspModuleInstaller")
+.libPaths(JASPMODULEINSTALLER_LIBRARY)
+setupRenv("@R_LIBRARY_PATH@", modulePkg)
+
+record <- createLocalRecord(modulePkg, getModuleInfo(modulePkg), cacheAble = FALSE, addJaspToVersion = FALSE)
+lf <- renv::lockfile_read(file.path(modulePkg, "renv.lock"))
+lf <- renv::record(record, lockfile = lf)
+cat(".libPaths()", .libPaths(), sep = "\n")
+
+# remove the package if it is installed, otherwise renv doesn't realize we want to reinstall it
+pkgName <- basename(modulePkg)
+if (dir.exists(file.path(.libPaths()[1L], pkgName)))
+  utils::remove.packages(pkgs = pkgName, lib = .libPaths()[1L])
+
+renv::restore(lockfile = lf, library = .libPaths(), rebuild = pkgName)
+
+
+
+print("jaspModuleInstaller::writeModuleStatusObject(@PROJECT_SOURCE_DIR@")
+jaspModuleInstaller::writeModuleStatusObject("@PROJECT_SOURCE_DIR@")
+#This is necessary because of conflicts with Matrix dep of base R lib.
+#Can be removed when a new version of R updates its Matrix
+if (!@IS_FLATPAK_USED@) {
+
+  if (Sys.info()["sysname"] == "Darwin") { 
+    options(pkgType = "source")
+  }  
+
+	renv::install("Matrix", library = JASPMODULEINSTALLER_LIBRARY, prompt = FALSE)
+}
+
+# Converting the absolute symlinks to relative symlinks on macOS
+# Todo, I can do this using CMake like I do on Windows
+if (Sys.info()["sysname"] == "Darwin") {
+  source('@MODULES_BINARY_PATH@/Tools/symlinkTools.R')
+  convertAbsoluteSymlinksToRelative('@R_LIBRARY_PATH@', '@MODULES_RENV_CACHE_PATH@')
+}