diff --git a/pom.xml b/pom.xml index a59059eee4..ddfdf16b3f 100644 --- a/pom.xml +++ b/pom.xml @@ -101,7 +101,7 @@ org.rocksdb rocksdbjni - 5.17.2 + 6.2.2 @@ -396,7 +396,7 @@ commons-io:commons-io:2.5:jar:null:compile:2852e6e05fbb95076fc091f6d1780f1f8fe35e0f - org.rocksdb:rocksdbjni:5.17.2:jar:null:compile:bca52276cabe91a3b97cc18e50fa2eabc2986f58 + org.rocksdb:rocksdbjni:6.2.2:jar:null:compile:5337051b43477a3fb345889378abbba5bdb29830 com.google.code.gson:gson:2.8.1:jar:null:compile:02a8e0aa38a2e21cb39e2f5a7d6704cbdc941da0 diff --git a/src/main/java/com/iota/iri/Iota.java b/src/main/java/com/iota/iri/Iota.java index a996a93e17..26b288a226 100644 --- a/src/main/java/com/iota/iri/Iota.java +++ b/src/main/java/com/iota/iri/Iota.java @@ -1,5 +1,12 @@ package com.iota.iri; +import java.util.List; +import java.util.Map; + +import org.apache.commons.lang3.NotImplementedException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import com.iota.iri.conf.IotaConfig; import com.iota.iri.controllers.TipsViewModel; import com.iota.iri.controllers.TransactionViewModel; @@ -8,7 +15,11 @@ import com.iota.iri.network.TransactionRequester; import com.iota.iri.network.pipeline.TransactionProcessingPipeline; import com.iota.iri.service.ledger.LedgerService; -import com.iota.iri.service.milestone.*; +import com.iota.iri.service.milestone.LatestMilestoneTracker; +import com.iota.iri.service.milestone.LatestSolidMilestoneTracker; +import com.iota.iri.service.milestone.MilestoneService; +import com.iota.iri.service.milestone.MilestoneSolidifier; +import com.iota.iri.service.milestone.SeenMilestonesRetriever; import com.iota.iri.service.snapshot.LocalSnapshotManager; import com.iota.iri.service.snapshot.SnapshotException; import com.iota.iri.service.snapshot.SnapshotProvider; @@ -26,13 +37,6 @@ import com.iota.iri.utils.Pair; import com.iota.iri.zmq.ZmqMessageQueueProvider; -import java.util.List; -import java.util.Map; - -import org.apache.commons.lang3.NotImplementedException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - /** * * The main class of IRI. This will propagate transactions into and throughout the network. This data is stored as a @@ -259,6 +263,7 @@ private void initializeTangle() { tangle.addPersistenceProvider(createRocksDbProvider( configuration.getDbPath(), configuration.getDbLogPath(), + configuration.getDbConfigFile(), configuration.getDbCacheSize(), Tangle.COLUMN_FAMILIES, Tangle.METADATA_COLUMN_FAMILY) @@ -279,16 +284,17 @@ private void initializeTangle() { * * @param path The location where the database will be stored * @param log The location where the log files will be stored + * @param configFile The location where the RocksDB config is read from * @param cacheSize the size of the cache used by the database implementation * @param columnFamily A map of the names related to their Persistable class * @param metadata Map of metadata used by the Persistable class, can be null * @return A new Persistance provider */ - private PersistenceProvider createRocksDbProvider(String path, String log, int cacheSize, + private PersistenceProvider createRocksDbProvider(String path, String log, String configFile, int cacheSize, Map> columnFamily, Map.Entry> metadata) { return new RocksDBPersistenceProvider( - path, log, cacheSize, columnFamily, metadata); + path, log, configFile, cacheSize, columnFamily, metadata); } } \ No newline at end of file diff --git a/src/main/java/com/iota/iri/MainInjectionConfiguration.java b/src/main/java/com/iota/iri/MainInjectionConfiguration.java index 9eeea7cb80..0e5debb681 100644 --- a/src/main/java/com/iota/iri/MainInjectionConfiguration.java +++ b/src/main/java/com/iota/iri/MainInjectionConfiguration.java @@ -1,5 +1,10 @@ package com.iota.iri; +import java.security.SecureRandom; +import java.util.HashMap; + +import javax.annotation.Nullable; + import com.google.inject.AbstractModule; import com.google.inject.Provides; import com.google.inject.Singleton; @@ -50,10 +55,6 @@ import com.iota.iri.storage.Tangle; import com.iota.iri.storage.rocksDB.RocksDBPersistenceProvider; -import javax.annotation.Nullable; -import java.security.SecureRandom; -import java.util.HashMap; - /** * Guice module. Configuration class for dependency injection. */ @@ -82,6 +83,7 @@ SpentAddressesProvider provideSpentAddressesProvider() { PersistenceProvider persistenceProvider = new RocksDBPersistenceProvider( configuration.getSpentAddressesDbPath(), configuration.getSpentAddressesDbLogPath(), + configuration.getDbConfigFile(), 1000, new HashMap>(1) {{put("spent-addresses", SpentAddress.class);}}, null); diff --git a/src/main/java/com/iota/iri/conf/BaseIotaConfig.java b/src/main/java/com/iota/iri/conf/BaseIotaConfig.java index 693c4db327..0d11a83a13 100644 --- a/src/main/java/com/iota/iri/conf/BaseIotaConfig.java +++ b/src/main/java/com/iota/iri/conf/BaseIotaConfig.java @@ -1,23 +1,22 @@ package com.iota.iri.conf; -import com.iota.iri.crypto.SpongeFactory; -import com.iota.iri.model.Hash; -import com.iota.iri.model.HashFactory; -import com.iota.iri.utils.IotaUtils; - import java.net.InetAddress; import java.net.UnknownHostException; import java.util.Collections; import java.util.List; import java.util.stream.Collectors; +import org.apache.commons.lang3.ArrayUtils; + import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; import com.beust.jcommander.ParameterException; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; - -import org.apache.commons.lang3.ArrayUtils; +import com.iota.iri.crypto.SpongeFactory; +import com.iota.iri.model.Hash; +import com.iota.iri.model.HashFactory; +import com.iota.iri.utils.IotaUtils; /** Note: the fields in this class are being deserialized from Jackson so they must follow Java Bean convention. @@ -63,6 +62,7 @@ public abstract class BaseIotaConfig implements IotaConfig { //DB protected String dbPath = Defaults.DB_PATH; protected String dbLogPath = Defaults.DB_LOG_PATH; + protected String dbConfigFile = Defaults.DB_CONFIG_FILE; protected int dbCacheSize = Defaults.DB_CACHE_SIZE; //KB protected String mainDb = Defaults.MAIN_DB; protected boolean revalidate = Defaults.REVALIDATE; @@ -405,6 +405,17 @@ public String getDbLogPath() { protected void setDbLogPath(String dbLogPath) { this.dbLogPath = dbLogPath; } + + @Override + public String getDbConfigFile() { + return dbConfigFile; + } + + @JsonProperty + @Parameter(names = {"--db-config-file"}, description = DbConfig.Descriptions.DB_CONFIG_FILE) + protected void setDbConfigFile(String dbConfigFile) { + this.dbConfigFile = dbConfigFile; + } @Override public int getDbCacheSize() { @@ -864,6 +875,7 @@ public interface Defaults { //DB String DB_PATH = "mainnetdb"; String DB_LOG_PATH = "mainnet.log"; + String DB_CONFIG_FILE = "rocksdb-config.properties"; int DB_CACHE_SIZE = 100_000; String MAIN_DB = "rocksdb"; boolean REVALIDATE = false; diff --git a/src/main/java/com/iota/iri/conf/DbConfig.java b/src/main/java/com/iota/iri/conf/DbConfig.java index 7d0d6c897a..f95ffbeb82 100644 --- a/src/main/java/com/iota/iri/conf/DbConfig.java +++ b/src/main/java/com/iota/iri/conf/DbConfig.java @@ -18,6 +18,13 @@ public interface DbConfig extends Config { * @return {@value DbConfig.Descriptions#DB_LOG_PATH} */ String getDbLogPath(); + + /** + * Default Value: {@value BaseIotaConfig.Defaults#DB_CONFIG_FILE} + * + * @return {@value DbConfig.Descriptions#DB_CONFIG_FILE} + */ + String getDbConfigFile(); /** * Default Value: {@value BaseIotaConfig.Defaults#DB_CACHE_SIZE} @@ -56,5 +63,6 @@ interface Descriptions { String REVALIDATE = "Reload from the db data about confirmed transaction (milestones), state of the ledger, " + "and transaction metadata."; String RESCAN_DB = "Rescan all transaction metadata (Approvees, Bundles, and Tags)"; + String DB_CONFIG_FILE = "The location of the RocksDB configuration file"; } } diff --git a/src/main/java/com/iota/iri/conf/TestnetConfig.java b/src/main/java/com/iota/iri/conf/TestnetConfig.java index 4cc320bcf1..19ae8daf37 100644 --- a/src/main/java/com/iota/iri/conf/TestnetConfig.java +++ b/src/main/java/com/iota/iri/conf/TestnetConfig.java @@ -1,5 +1,7 @@ package com.iota.iri.conf; +import java.util.Objects; + import com.beust.jcommander.Parameter; import com.beust.jcommander.ParameterException; import com.fasterxml.jackson.annotation.JsonProperty; @@ -7,8 +9,6 @@ import com.iota.iri.model.Hash; import com.iota.iri.model.HashFactory; -import java.util.Objects; - public class TestnetConfig extends BaseIotaConfig { protected Hash coordinator = Defaults.COORDINATOR_ADDRESS; diff --git a/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProvider.java b/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProvider.java index e2731dfe35..917079d398 100644 --- a/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProvider.java +++ b/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProvider.java @@ -1,13 +1,9 @@ package com.iota.iri.storage.rocksDB; -import com.iota.iri.model.HashFactory; -import com.iota.iri.storage.Indexable; -import com.iota.iri.storage.Persistable; -import com.iota.iri.storage.PersistenceProvider; -import com.iota.iri.utils.IotaIOUtils; -import com.iota.iri.utils.Pair; - import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; import java.nio.file.Paths; import java.security.SecureRandom; import java.util.ArrayList; @@ -19,6 +15,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Properties; import java.util.Set; import org.apache.commons.collections4.CollectionUtils; @@ -28,12 +25,15 @@ import org.rocksdb.BackupableDBOptions; import org.rocksdb.BlockBasedTableConfig; import org.rocksdb.BloomFilter; +import org.rocksdb.Cache; import org.rocksdb.ColumnFamilyDescriptor; import org.rocksdb.ColumnFamilyHandle; import org.rocksdb.ColumnFamilyOptions; import org.rocksdb.DBOptions; import org.rocksdb.Env; +import org.rocksdb.LRUCache; import org.rocksdb.MergeOperator; +import org.rocksdb.Priority; import org.rocksdb.RestoreOptions; import org.rocksdb.RocksDB; import org.rocksdb.RocksDBException; @@ -46,6 +46,14 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.iota.iri.conf.BaseIotaConfig; +import com.iota.iri.model.HashFactory; +import com.iota.iri.storage.Indexable; +import com.iota.iri.storage.Persistable; +import com.iota.iri.storage.PersistenceProvider; +import com.iota.iri.utils.IotaIOUtils; +import com.iota.iri.utils.Pair; + public class RocksDBPersistenceProvider implements PersistenceProvider { private static final Logger log = LoggerFactory.getLogger(RocksDBPersistenceProvider.class); @@ -58,6 +66,8 @@ public class RocksDBPersistenceProvider implements PersistenceProvider { private final String dbPath; private final String logPath; + private String configPath; + private final int cacheSize; private final Map> columnFamilies; private final Map.Entry> metadataColumnFamily; @@ -70,8 +80,36 @@ public class RocksDBPersistenceProvider implements PersistenceProvider { private DBOptions options; private BloomFilter bloomFilter; private boolean available; - + + private Cache cache, compressedCache; + private ColumnFamilyOptions columnFamilyOptions; + + /** + * Creates a new RocksDB provider without reading from a configuration file + * + * @param dbPath The location where the database will be stored + * @param logPath The location where the log files will be stored + * @param cacheSize the size of the cache used by the database implementation + * @param columnFamilies A map of the names related to their Persistable class + * @param metadataColumnFamily Map of metadata used by the Persistable class, can be null + */ public RocksDBPersistenceProvider(String dbPath, String logPath, int cacheSize, + Map> columnFamilies, + Map.Entry> metadataColumnFamily) { + this(dbPath, logPath, null, cacheSize, columnFamilies, metadataColumnFamily); + } + + /** + * Creates a new RocksDB provider by reading the configuration to be used in this instance from a file + * + * @param dbPath The location where the database will be stored + * @param logPath The location where the log files will be stored + * @param configPath The location where the RocksDB config is read from + * @param cacheSize the size of the cache used by the database implementation + * @param columnFamilies A map of the names related to their Persistable class + * @param metadataColumnFamily Map of metadata used by the Persistable class, can be null + */ + public RocksDBPersistenceProvider(String dbPath, String logPath, String configPath, int cacheSize, Map> columnFamilies, Map.Entry> metadataColumnFamily) { this.dbPath = dbPath; @@ -79,13 +117,14 @@ public RocksDBPersistenceProvider(String dbPath, String logPath, int cacheSize, this.cacheSize = cacheSize; this.columnFamilies = columnFamilies; this.metadataColumnFamily = metadataColumnFamily; + this.configPath = configPath; } @Override public void init() throws Exception { log.info("Initializing Database on " + dbPath); - initDB(dbPath, logPath, columnFamilies); + initDB(dbPath, logPath, configPath, columnFamilies); available = true; log.info("RocksDB persistence provider initialized."); } @@ -101,7 +140,7 @@ public void shutdown() { for (final ColumnFamilyHandle columnFamilyHandle : columnFamilyHandles) { IotaIOUtils.closeQuietly(columnFamilyHandle); } - IotaIOUtils.closeQuietly(db, options, bloomFilter); + IotaIOUtils.closeQuietly(db, options, bloomFilter, cache, compressedCache, columnFamilyOptions); } @Override @@ -320,7 +359,13 @@ public boolean saveBatch(List> models) throws Excep public void deleteBatch(Collection>> models) throws Exception { if (CollectionUtils.isNotEmpty(models)) { - try (WriteBatch writeBatch = new WriteBatch()) { + try (WriteBatch writeBatch = new WriteBatch(); + WriteOptions writeOptions = new WriteOptions() + //We are explicit about what happens if the node reboots before a flush to the db + .setDisableWAL(false) + //We want to make sure deleted data was indeed deleted + .setSync(true)) { + for (Pair> entry : models) { Indexable indexable = entry.low; byte[] keyBytes = indexable.bytes(); @@ -332,11 +377,6 @@ public void deleteBatch(Collection> columnFamilies) throws Exception { + // options is closed in shutdown + @SuppressWarnings("resource") + private void initDB(String path, String logPath, String configFile, Map> columnFamilies) throws Exception { try { try { RocksDB.loadLibrary(); @@ -436,52 +478,29 @@ private void initDB(String path, String logPath, Map columnFamilyDescriptors = new ArrayList<>(); + + columnFamilyOptions = new ColumnFamilyOptions() .setMergeOperator(mergeOperator) .setTableFormatConfig(blockBasedTableConfig) .setMaxWriteBufferNumber(2) .setWriteBufferSize(2 * SizeUnit.MB); - - List columnFamilyDescriptors = new ArrayList<>(); + //Add default column family. Main motivation is to not change legacy code columnFamilyDescriptors.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, columnFamilyOptions)); for (String name : columnFamilies.keySet()) { @@ -494,14 +513,13 @@ private void initDB(String path, String logPath, Map(); } - db = RocksDB.open(options, path, columnFamilyDescriptors, columnFamilyHandles); db.enableFileDeletions(true); initClassTreeMap(columnFamilyDescriptors); } catch (Exception e) { - IotaIOUtils.closeQuietly(db); + IotaIOUtils.closeQuietly(db, options, bloomFilter, columnFamilyOptions, cache, compressedCache); throw e; } } @@ -513,7 +531,7 @@ private void initClassTreeMap(List columnFamilyDescripto int i = 1; for (; i < columnFamilyDescriptors.size(); i++) { - String name = new String(columnFamilyDescriptors.get(i).columnFamilyName()); + String name = new String(columnFamilyDescriptors.get(i).getName()); if (name.equals(mcfName)) { Map, ColumnFamilyHandle> metadataRef = new HashMap<>(); metadataRef.put(metadataColumnFamily.getValue(), columnFamilyHandles.get(i)); @@ -530,4 +548,54 @@ private void initClassTreeMap(List columnFamilyDescripto classTreeMap = MapUtils.unmodifiableMap(classMap); } + private DBOptions createOptions(String logPath, String configFile) throws IOException { + DBOptions options = null; + File pathToLogDir = Paths.get(logPath).toFile(); + if (!pathToLogDir.exists() || !pathToLogDir.isDirectory()) { + boolean success = pathToLogDir.mkdir(); + if (!success) { + log.warn("Unable to make directory: {}", pathToLogDir); + } + } + + int numThreads = Math.max(1, Runtime.getRuntime().availableProcessors() / 2); + RocksEnv.getDefault() + .setBackgroundThreads(numThreads, Priority.HIGH) + .setBackgroundThreads(numThreads, Priority.LOW); + + if (configFile != null) { + File config = Paths.get(configFile).toFile(); + if (config.exists() && config.isFile() && config.canRead()) { + Properties configProperties = new Properties(); + + try (InputStream stream = new FileInputStream(config)){ + configProperties.load(stream); + options = DBOptions.getDBOptionsFromProps(configProperties); + } catch (IllegalArgumentException e) { + log.warn("RocksDB configuration file is empty, falling back to default values"); + } + } + } + if (options == null) { + options = new DBOptions() + .setCreateIfMissing(true) + .setCreateMissingColumnFamilies(true) + .setMaxLogFileSize(SizeUnit.MB) + .setMaxManifestFileSize(SizeUnit.MB) + .setMaxOpenFiles(10000) + .setMaxBackgroundCompactions(1) + .setAllowConcurrentMemtableWrite(true) + .setMaxSubcompactions(Runtime.getRuntime().availableProcessors()); + } + + if (!BaseIotaConfig.Defaults.DB_LOG_PATH.equals(logPath) && logPath != null) { + if (!options.dbLogDir().equals("")) { + log.warn("Defined a db log path in config and commandline; Using the command line setting."); + } + + options.setDbLogDir(logPath); + } + + return options; + } } diff --git a/src/test/java/com/iota/iri/benchmarks/dbbenchmark/states/DbState.java b/src/test/java/com/iota/iri/benchmarks/dbbenchmark/states/DbState.java index 5f608b5495..229a56482e 100644 --- a/src/test/java/com/iota/iri/benchmarks/dbbenchmark/states/DbState.java +++ b/src/test/java/com/iota/iri/benchmarks/dbbenchmark/states/DbState.java @@ -1,5 +1,15 @@ package com.iota.iri.benchmarks.dbbenchmark.states; +import java.io.File; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import org.apache.commons.io.FileUtils; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.State; + import com.iota.iri.TransactionTestUtils; import com.iota.iri.conf.BaseIotaConfig; import com.iota.iri.conf.MainnetConfig; @@ -10,15 +20,6 @@ import com.iota.iri.storage.PersistenceProvider; import com.iota.iri.storage.Tangle; import com.iota.iri.storage.rocksDB.RocksDBPersistenceProvider; -import org.apache.commons.io.FileUtils; -import org.openjdk.jmh.annotations.Param; -import org.openjdk.jmh.annotations.Scope; -import org.openjdk.jmh.annotations.State; - -import java.io.File; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; @State(Scope.Benchmark) public abstract class DbState { @@ -42,7 +43,7 @@ public void setup() throws Exception { } logFolder.mkdirs(); PersistenceProvider dbProvider = new RocksDBPersistenceProvider( - dbFolder.getAbsolutePath(), logFolder.getAbsolutePath(), BaseIotaConfig.Defaults.DB_CACHE_SIZE, Tangle.COLUMN_FAMILIES, Tangle.METADATA_COLUMN_FAMILY); + dbFolder.getAbsolutePath(), logFolder.getAbsolutePath(), null, BaseIotaConfig.Defaults.DB_CACHE_SIZE, Tangle.COLUMN_FAMILIES, Tangle.METADATA_COLUMN_FAMILY); dbProvider.init(); tangle = new Tangle(); snapshotProvider = new SnapshotProviderImpl(new MainnetConfig());