JASP configuration file feature + remote option

.gitmodules

@@ -116,3 +116,12 @@
 [submodule "Modules/jaspPower"]
 	path = Modules/jaspPower
 	url = https://github.com/jasp-stats/jaspPower
+	branch = master
+[submodule "Modules/jaspPredictiveAnalytics"]
+	path = Modules/jaspPredictiveAnalytics
+	url = https://github.com/jasp-stats/jaspPredictiveAnalytics
+	branch = main
+[submodule "Modules/jaspLearnStats"]
+	path = Modules/jaspLearnStats
+	url = https://github.com/jasp-stats/jaspLearnStats
+	branch = master

Common/utils.cpp

@@ -36,14 +36,6 @@
 
 using namespace std;
 
-std::string Utils::doubleToString(double dbl, int precision)
-{
-	std::stringstream conv; //Use this instead of std::to_string to make sure there are no trailing zeroes (and to get full precision)
-	conv << std::setprecision(precision);
-	conv << dbl;
-	return conv.str();
-}
-
 Utils::FileType Utils::getTypeFromFileName(const std::string &path)
 {
 
@@ -229,7 +221,6 @@ void Utils::sleep(int ms)
 #endif
 }
 
-
 bool Utils::isEqual(const double a, const double b)
 {
 	if (isnan(a) || isnan(b)) return false;

Common/utils.h

@@ -22,6 +22,8 @@
 #include <vector>
 #include <limits>
 #include <filesystem>
+#include <set>
+#include <map>
 #include "timers.h"
 
 enum class FileTypeBase;
@@ -44,15 +46,12 @@ class Utils
 	static bool renameOverwrite(		const std::string &oldName, const std::string &newName);
 	static bool removeFile(				const std::string &path);
 
-	static std::string	doubleToString(double dbl, int precision = 10);
-
 	static std::filesystem::path osPath(const std::string &path);
 	static std::string osPath(const std::filesystem::path &path);
 
 	static void remove(std::vector<std::string> &target, const std::vector<std::string> &toRemove);
 	static void sleep(int ms);
 
-
 	static bool isEqual(const float a, const float b);
 	static bool isEqual(const double a, const double b);
 

CommonData/column.cpp

@@ -180,7 +180,7 @@ bool Column::_resetEmptyValuesForScale(std::map<int, string> &emptyValuesMap)
 			// This value is now considered as empty
 			*doubles = NAN;
 			hasChanged = true;
-			emptyValuesMap.insert(make_pair(row, Utils::doubleToString(doubleValue)));
+			emptyValuesMap.insert(make_pair(row, ColumnUtils::doubleToString(doubleValue)));
 		}
 		row++;
 	}
@@ -525,7 +525,7 @@ columnTypeChangeResult Column::_changeColumnToNominalOrOrdinal(enum columnType n
 
 			for (double doubleValue : AsDoubles)
 				if (std::isnan(doubleValue))	values.push_back("");
-				else							values.push_back(Utils::doubleToString(doubleValue));
+				else							values.push_back(ColumnUtils::doubleToString(doubleValue));
 
 			setColumnAsNominalText(values);
 			return columnTypeChangeResult::changed;
@@ -1019,7 +1019,7 @@ string Column::_getScaleValue(int row, bool forDisplay)
 	if (v > std::numeric_limits<double>::max())					return "∞";
 	else if (v < std::numeric_limits<double>::lowest())			return "-∞";
 	else if (ColumnUtils::isEmptyValue(v))							return forDisplay ? ColumnUtils::emptyValue : "";
-	else														return Utils::doubleToString(v);
+	else														return ColumnUtils::doubleToString(v);
 }
 
 string Column::getOriginalValue(int row)
@@ -1071,6 +1071,8 @@ void Column::append(int rows)
 	if (rows == 0)
 		return;
 
+	size_t originalRowCount = _rowCount;
+
 	BlockMap::reverse_iterator itr = _blocks.rbegin();
 
 	if (itr == _blocks.rend()) // no blocks
@@ -1092,39 +1094,64 @@ void Column::append(int rows)
 	{
 		block->insert(rows);
 		_rowCount += rows;
-		return;
+		goto finishingUp; //yeah yeah this is a goto, no problem right?
 	}
 
-	block->insert(room);
-	_rowCount += room;
-
-	int newBlocksRequired = rowsLeft / DataBlock::capacity();
-	if (rowsLeft % DataBlock::capacity())
-		newBlocksRequired++;
-
-	for (int i = 0; i < newBlocksRequired; i++)
+	//Extra scope to avoid var init between goto and label
 	{
-		try {
+		block->insert(room);
+		_rowCount += room;
 
-		DataBlock *newBlock = _mem->construct<DataBlock>(anonymous_instance)();
+		int newBlocksRequired = rowsLeft / DataBlock::capacity();
+		if (rowsLeft % DataBlock::capacity())
+			newBlocksRequired++;
 
-		int toInsert = min(rowsLeft, DataBlock::capacity());
-		newBlock->insert(toInsert);
-		rowsLeft -= toInsert;
+		for (int i = 0; i < newBlocksRequired; i++)
+		{
+			try {
+
+				DataBlock *newBlock = _mem->construct<DataBlock>(anonymous_instance)();
+
+				int toInsert = min(rowsLeft, DataBlock::capacity());
+				newBlock->insert(toInsert);
+				rowsLeft -= toInsert;
+
+				id += DataBlock::capacity();
+				_blocks.insert(BlockEntry(id, newBlock));
+
+				_rowCount += toInsert;
+			}
+			catch (boost::interprocess::bad_alloc &e)
+			{
+				std::cout << e.what() << " ";
+				std::cout << "append column " << name() << ", append: " << rows << ", rowCount: " << _rowCount << std::endl;
+				throw e;
+			}
+		}
+	}
 
-		id += DataBlock::capacity();
-		_blocks.insert(BlockEntry(id, newBlock));
+finishingUp:
 
-		_rowCount += toInsert;
+	//Make sure the new rows are empty looking
+	switch(_columnType)
+	{
+	case columnType::unknown:
+	case columnType::scale:
+		for(size_t r=originalRowCount; r<_rowCount; r++)
+			AsDoubles[r] = NAN;
+		break;
 
-		}
-		catch (boost::interprocess::bad_alloc &e)
-		{
-			std::cout << e.what() << " ";
-			std::cout << "append column " << name() << ", append: " << rows << ", rowCount: " << _rowCount << std::endl;
-			throw e;
-		}
+	case columnType::ordinal:
+	case columnType::nominal:
+		for(size_t r=originalRowCount; r<_rowCount; r++)
+			AsInts[r] = INT_MIN;
+		break;
+
+	case columnType::nominalText:
+		break;
 	}
+
+
 }
 
 void Column::truncate(int rows)
@@ -1138,6 +1165,8 @@ void Column::truncate(int rows)
 
 	if (rowsToDelete > _rowCount)
 		rowsToDelete = _rowCount;
+
+	std::vector<DataBlock*> destroyThese;
 
 	while (rowsToDelete > 0)
 	{
@@ -1152,7 +1181,7 @@ void Column::truncate(int rows)
 			rowsToDelete -= block->rowCount();
 			_rowCount -= block->rowCount();
 			block->erase(block->rowCount());
-			//_mem->destroy_ptr<DataBlock>(block);
+			destroyThese.push_back(block);
 			itr++;
 			if (itr == _blocks.rend())
 			{
@@ -1166,6 +1195,19 @@ void Column::truncate(int rows)
 			}
 		}
 	}
+
+	for(DataBlock * block : destroyThese)
+	{
+		_mem->destroy_ptr<DataBlock>(block);
+
+		for(const auto & keyval : _blocks)
+			if(keyval.second == block)
+			{
+				_blocks.erase(keyval.first);
+				break;
+			}
+	}
+
 }
 
 void Column::setColumnType(enum columnType columnType)

CommonData/columns.cpp

@@ -63,9 +63,15 @@ void Columns::setRowCount(size_t rowCount)
 
 void Columns::setColumnCount(size_t columnCount)
 {
-	_columnStore.reserve(columnCount);
-	for (size_t i = _columnStore.size(); i < columnCount; i++)
-		_columnStore.push_back(Column(_mem));
+	if(columnCount > _columnStore.size())
+	{
+		_columnStore.reserve(columnCount);
+		for (size_t i = _columnStore.size(); i < columnCount; i++)
+			_columnStore.push_back(Column(_mem));
+	}
+	else
+		while(_columnStore.size() > columnCount)
+			_columnStore.erase(--_columnStore.end());
 }
 
 
@@ -84,6 +90,13 @@ void Columns::removeColumn(std::string name)
 		}
 }
 
+void Columns::insertColumn(size_t index)
+{
+	auto c = Column(_mem);
+	c._setRowCount(maxRowCount());
+	_columnStore.insert(_columnStore.begin() + index, c);
+}
+
 
 void Columns::setSharedMemory(boost::interprocess::managed_shared_memory *mem)
 {

CommonData/columns.h

@@ -56,6 +56,7 @@ class Columns
 
 	void removeColumn(size_t index);
 	void removeColumn(std::string name);
+	void insertColumn(size_t index);
 
 	ColumnVector _columnStore;
 

CommonData/columnutils.cpp

@@ -31,10 +31,10 @@ void ColumnUtils::processEmptyValues()
 {
 	_currentDoubleEmptyValues.clear();
 
-	for (vector<string>::const_iterator it = _currentEmptyValues.begin(); it != _currentEmptyValues.end(); ++it)
+	for (const std::string & curEmptyVal : _currentEmptyValues)
 	{
 		double doubleValue;
-		if (ColumnUtils::getDoubleValue(*it, doubleValue))
+		if (ColumnUtils::getDoubleValue(curEmptyVal, doubleValue))
 			_currentDoubleEmptyValues.push_back(doubleValue);
 	}
 }
@@ -167,6 +167,72 @@ bool ColumnUtils::convertValueToDoubleForImport(const std::string &strValue, dou
 	return true;
 }
 
+std::string ColumnUtils::doubleToString(double dbl, int precision)
+{
+	std::stringstream conv; //Use this instead of std::to_string to make sure there are no trailing zeroes (and to get full precision)
+	conv << std::setprecision(precision);
+	conv << dbl;
+	return conv.str();
+}
+
+
+bool ColumnUtils::convertVecToInt(const std::vector<std::string> &values, std::vector<int> &intValues, std::set<int> &uniqueValues, std::map<int, std::string> &emptyValuesMap)
+{
+	emptyValuesMap.clear();
+	uniqueValues.clear();
+	intValues.clear();
+	intValues.reserve(values.size());
+
+	int row = 0;
+
+	for (const std::string &value : values)
+	{
+		int intValue = std::numeric_limits<int>::lowest();
+
+		if (ColumnUtils::convertValueToIntForImport(value, intValue))
+		{
+			if (intValue != std::numeric_limits<int>::lowest())	uniqueValues.insert(intValue);
+			else if (!value.empty())							emptyValuesMap.insert(make_pair(row, value));
+
+			intValues.push_back(intValue);
+		}
+		else
+			return false;
+
+		row++;
+	}
+
+	return true;
+}
+
+
+bool ColumnUtils::convertVecToDouble(const std::vector<std::string> &values, std::vector<double> &doubleValues, std::map<int, std::string> &emptyValuesMap)
+{
+	emptyValuesMap.clear();
+	doubleValues.clear();
+	doubleValues.reserve(values.size());
+
+	int row = 0;
+	for (const std::string &value : values)
+	{
+		double doubleValue = static_cast<double>(NAN);
+
+		if (ColumnUtils::convertValueToDoubleForImport(value, doubleValue))
+		{
+			doubleValues.push_back(doubleValue);
+
+			if (std::isnan(doubleValue) && value != ColumnUtils::emptyValue)
+				emptyValuesMap.insert(std::make_pair(row, value));
+		}
+		else
+			return false;
+
+		row++;
+	}
+
+	return true;
+}
+
 // hex should be 4 hexadecimals characters
 std::string ColumnUtils::_convertEscapedUnicodeToUTF8(std::string hex)
 {

CommonData/columnutils.h

@@ -3,7 +3,8 @@
 
 #include <string>
 #include <vector>
-
+#include <set>
+#include <map>
 
 class ColumnUtils
 {
@@ -28,6 +29,11 @@ class ColumnUtils
 	static bool			convertValueToDoubleForImport(	const std::string &strValue, double &doubleValue);
 	static void			convertEscapedUnicodeToUTF8(	std::string &inputStr);
 
+	static bool convertVecToInt(const std::vector<std::string> &values, std::vector<int> &intValues, std::set<int> &uniqueValues, std::map<int, std::string> &emptyValuesMap);
+	static bool convertVecToDouble(const std::vector<std::string> &values, std::vector<double> &doubleValues, std::map<int, std::string> &emptyValuesMap);
+
+	static std::string	doubleToString(double dbl, int precision = 10);
+
 private:
 	static std::string _deEuropeaniseForImport(			const std::string &value);
 	static std::string _convertEscapedUnicodeToUTF8(	std::string hex);

CommonData/labels.cpp

@@ -271,11 +271,11 @@ const Label &Labels::getLabelObjectFromKey(int index) const
 			return label;
 	}
 
-	Log::log() << "Cannot find entry " << index << std::endl;
+	/*Log::log() << "Cannot find entry " << index << std::endl;
 	for(const Label &label: _labels)
 	{
 		Log::log() << "Label Value: " << label.value() << ", Text: " << label.text() << std::endl;
-	}
+	}*/
 
 	throw labelNotFound("Cannot find this entry");
 }