Merge branch 'release/1.2.3'

leo-project · Jan 20, 2015 · 4f18419 · 4f18419
2 parents fff7aa7 + e1e83c2
commit 4f18419
Show file tree

Hide file tree

Showing 15 changed files with 1,160 additions and 203 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,6 +1,53 @@
 CHANGELOG
 =========
 
+1.2.3 (Jan 20, 2015)
+====================
+
+* Improvements
+    * [#285](https://github.com/leo-project/leofs/issues/285) ``leo_storage`` Implemented "cluster-watchdog" in order to keep best condition of a cluster
+    * [#286](https://github.com/leo-project/leofs/issues/286) ``leo_storage`` Unified name of configuration items for the mq and the compaction
+* Fixed Bugs
+    * [#288](https://github.com/leo-project/leofs/issues/288) ``leo_manager`` Fixed wrong mix/max values when issueing [the status &lt;storage-node&gt; command](http://leo-project.net/leofs/docs/admin_guide/admin_guide_2.html#status-command) with ``leofs-adm command``
+    * [#290](https://github.com/leo-project/leofs/issues/290) [the du detail &lt;storage-node&gt; command](http://leo-project.net/leofs/docs/admin_guide/admin_guide_5.html#du-detail) with ``leofs-adm command`` had broken 1.2.2
+    * [#292](https://github.com/leo-project/leofs/issues/292) ``leo_storage`` [proc_lib:start_link](http://erlang.org/doc/man/proc_lib.html#spawn_link-1) must be used instead of [erlang:spawn](http://erlang.org/doc/man/erlang.html#spawn-1)
+    * [#293](https://github.com/leo-project/leofs/issues/293) Fixed that remain objects after finished data-compaction, situation of which is combination auto-compaction and deletion of a bucket
+* Used libraries
+    * leo project
+        * [leo_backend-db v1.1.4](https://github.com/leo-project/leo_backend_db.git)
+        * [leo_cache v0.6.0](https://github.com/leo-project/leo_cache.git)
+        * [leo_commons v1.1.0](https://github.com/leo-project/leo_commons.git)
+        * [leo_dcerl v0.2.11](https://github.com/leo-project/leo_dcerl.git)
+        * [leo_logger v1.1.4](https://github.com/leo-project/leo_logger.git)
+        * [leo_mcerl v0.4.1](https://github.com/leo-project/leo_mcerl.git)
+        * [leo_mq v1.3.3](https://github.com/leo-project/leo_mq.git)
+        * [leo_object_storage v1.2.2](https://github.com/leo-project/leo_object_storage.git)
+        * [leo_ordning_reda v0.10.9](https://github.com/leo-project/leo_ordning_reda.git)
+        * [leo_redundant_manager 1.9.5](https://github.com/leo-project/leo_redundant_manager.git)
+        * [leo_rpc v0.8.10](https://github.com/leo-project/leo_rpc.git)
+        * [leo_pod v0.6.2](https://github.com/leo-project/leo_pod.git)
+        * [leo_s3_libs v1.1.5](https://github.com/leo-project/leo_s3_libs.git)
+        * [leo_statistics v1.0.8](https://github.com/leo-project/leo_statistics.git)
+        * [leo_watchdog v0.6.1](https://github.com/leo-project/leo_watchdog.git)
+        * [savanna_agent v0.4.4](https://github.com/leo-project/savanna_agent.git)
+        * [savanna_commons v0.8.6](https://github.com/leo-project/savanna_commons.git)
+        * [erpcgen v0.2.3](https://github.com/leo-project/erpcgen.git)
+        * [nfs_rpc_server v0.2.2](https://github.com/leo-project/nfs_rpc_server.git)
+        * [leo_gateway v1.2.3](https://github.com/leo-project/leo_gateway.git)
+        * [leo_manager v1.2.3](https://github.com/leo-project/leo_manager.git)
+        * [leo_storage v1.2.3](https://github.com/leo-project/leo_storage.git)
+    * others
+        * [bitcask v1.7.1](https://github.com/leo-project/bitcask.git)
+        * [cowboy v1.0.0](https://github.com/extend/cowboy.git)
+        * [cowlib v1.0.0](https://github.com/extend/cowboy.git)
+        * [elarm v0.3.0](https://github.com/leo-project/elarm.git)
+        * [eleveldb v1.4.10](https://github.com/basho/eleveldb.git)
+        * [folsom v0.8.1](https://github.com/boundary/folsom.git)
+        * [jiffy v0.8.5](https://github.com/davisp/jiffy.git)
+        * [lz4 v0.2.2](https://github.com/leo-project/erlang-lz4.git) - forked from [szktty/erlang-lz4](https://github.com/szktty/erlng-lz4)
+
+
+
 1.2.2 (Dec 17, 2014)
 ====================
 

diff --git a/leofs.mk b/leofs.mk
@@ -1 +1 @@
-LEOFS_VERSION=1.2.2
+LEOFS_VERSION=1.2.3
diff --git a/pkg/leo_storage/files/leo_storage.conf b/pkg/leo_storage/files/leo_storage.conf
@@ -145,6 +145,9 @@ watchdog.rex.threshold_mem_capacity = 33554432
 ## [compaction] number of parallel procs - default:1
 ## autonomic_op.compaction.parallel_procs = 1
 
+## [compaction] execution intarval time(sec) - default:300(sec)
+## autonomic_op.compaction.interval = 300
+
 ## [compaction] warning ratio of active size - default:70%
 ## autonomic_op.compaction.warn_active_size_ratio = 70
 
@@ -159,29 +162,29 @@ watchdog.rex.threshold_mem_capacity = 33554432
 ##compaction.limit_num_of_compaction_procs = 4
 
 ## Minimum value of compaction-proc waiting time/batch-proc(msec)
-##compaction.min_waiting_time = 100
+##compaction.waiting_time_min = 100
 
 ## Regular value of compaction-proc waiting time/batch-proc(msec)
-##compaction.regular_waiting_time = 300
+##compaction.waiting_time_regular = 300
 
 ## Maximum value of compaction-proc waiting time/batch-proc(msec)
-##compaction.max_waiting_time = 1000
+##compaction.waiting_time_max = 1000
 
 ## Step of compaction-proc waiting time(msec)
-##compaction.step_waiting_time = 100
+##compaction.waiting_time_step = 100
 
 
 ## Minimum compaction batch processes
-##compaction.min_batch_procs = 1000
+##compaction.batch_procs_min = 1000
 
 ## Regular compaction batch processes
-##compaction.regular_batch_procs = 10000
+##compaction.batch_procs_regular = 10000
 
 ## Maximum compaction batch processes
-##compaction.max_batch_procs = 100000
+##compaction.batch_procs_max = 100000
 
 ## Step compaction batch processes
-##compaction.step_batch_procs = 1000
+##compaction.batch_procs_step = 1000
 
 
 ## --------------------------------------------------------------------

diff --git a/priv/mdcr-test/c1/leo_storage_0.conf b/priv/mdcr-test/c1/leo_storage_0.conf
@@ -49,41 +49,170 @@ obj_containers.num_of_containers = [8]
 ## obj_containers.path = [/var/leofs/avs/1, /var/leofs/avs/2]
 ## obj_containers.num_of_containers = [32, 64]
 
+## Metadata Storage: [bitcask, leveldb] - default:leveldb
+obj_containers.metadata_storage = leveldb
 
 ## A number of virtual-nodes for the redundant-manager
 ## num_of_vnodes = 168
 
+## Enable strict check between checksum of a metadata and checksum of an object
+## - default:faluse
+## object_storage.is_strict_check = false
+
+
+## --------------------------------------------------------------------
+## STORAGE - Watchdog
+## --------------------------------------------------------------------
+##
+##  Watchdog.REX(RPC)
+##
+## rex - watch interval - default:5sec
+## watchdog.rex.interval = 5
+
+## Threshold memory capacity of binary for rex(rpc) - default:32MB
+watchdog.rex.threshold_mem_capacity = 33554432
+
+
+##
+##  Watchdog.CPU
+##
+## Is cpu-watchdog enabled - default:false
+watchdog.cpu.is_enabled = true
+
+## cpu - raised error times
+watchdog.cpu.raised_error_times = 3
+
+## cpu - watch interval - default:5sec
+watchdog.cpu.interval = 5
+
+## Threshold CPU load avg for 1min/5min
+watchdog.cpu.threshold_cpu_load_avg = 5.0
+
+## Threshold CPU load util - default:100 = "100%"
+watchdog.cpu.threshold_cpu_util = 100
+
+
+##
+##  Watchdog.IO
+##
+## Is io-watchdog enabled - default:false
+## watchdog.io.is_enabled = true
+
+## io - watch interval - default:1sec
+watchdog.io.interval = 1
+
+## Threshold input size/sec  - default:134217728(B) - 128MB/sec
+watchdog.io.threshold_input_per_sec = 134217728
+
+## Threshold output size/sec - default:134217728(B) - 128MB/sec
+watchdog.io.threshold_output_per_sec = 134217728
+
+
+##
+##  Watchdog.DISK
+##
+## Is disk-watchdog enabled - default:true
+watchdog.disk.is_enabled = true
+
+## disk - raised error times
+watchdog.disk.raised_error_times = 3
+
+## disk - watch interval - default:1sec
+watchdog.disk.interval = 5
+
+## Threshold disk use% - defalut:85%
+watchdog.disk.threshold_disk_use = 80
+
+## Threshold disk util% - defalut:100%
+watchdog.disk.threshold_disk_use = 100
+
+## Threshold disk read kb/sec - defalut:131072(KB)
+watchdog.disk.threshold_disk_rkb = 131072
+
+## Threshold disk write kb/sec - defalut:65536(KB)
+watchdog.disk.threshold_disk_wkb = 131072
+
+##
+##  Watchdog.Cluster
+##
+## Is cluster-watchdog enabled - default:false
+watchdog.cluster.is_enabled = true
+
+## cluster - watch interval - default:1sec
+watchdog.cluster.interval = 10
+
+
+## --------------------------------------------------------------------
+## STORAGE - Autonomic Operation
+## --------------------------------------------------------------------
+## [compaction] enabled compaction? - default:false
+autonomic_op.compaction.is_enabled = true
+
+## [compaction] number of parallel procs - default:1
+## autonomic_op.compaction.parallel_procs = 1
+
+## [compaction] warning ratio of active size - default:70%
+## autonomic_op.compaction.warn_active_size_ratio = 70
+
+## [compaction] threshold ratio of active size - default:60%
+## autonomic_op.compaction.threshold_active_size_ratio = 60
+
+
+## --------------------------------------------------------------------
+## STORAGE - Data Compcation
+## --------------------------------------------------------------------
+## Limit of a number of procs to execute data-compaction in parallel
+##compaction.limit_num_of_compaction_procs = 4
+
+## Minimum value of compaction-proc waiting time/batch-proc(msec)
+##compaction.waiting_time_min = 100
+
+## Regular value of compaction-proc waiting time/batch-proc(msec)
+##compaction.waiting_time_regular = 300
+
+## Maximum value of compaction-proc waiting time/batch-proc(msec)
+##compaction.waiting_time_max = 1000
+
+## Step of compaction-proc waiting time(msec)
+##compaction.waiting_time_step = 100
+
+
+## Minimum compaction batch processes
+##compaction.batch_procs_min = 1000
+
+## Regular compaction batch processes
+##compaction.batch_procs_regular = 10000
+
+## Maximum compaction batch processes
+##compaction.batch_procs_max = 100000
+
+## Step compaction batch processes
+##compaction.batch_procs_step = 1000
+
 
 ## --------------------------------------------------------------------
 ## STORAGE - MQ
 ## --------------------------------------------------------------------
-## A number of mq-server's processes
-mq.num_of_mq_procs = 8
+## MQ backend storage: [bitcask, leveldb] - default:bitcask
+## mq.backend_db = bitcask
 
-## MQ recover per_object
-mq.recover_per_object.num_of_batch_process = 1
-mq.recover_per_object.interval_min = 1
-mq.recover_per_object.interval_max = 1
+## A number of mq-server's processes
+## mq.num_of_mq_procs = 8
 
-## MQ synchronize objects by vnode-id
-mq.sync_by_vnode_id.num_of_batch_process = 1
-mq.sync_by_vnode_id.interval_min = 8
-mq.sync_by_vnode_id.interval_max = 32
+##
+## [Number of bach processes of message]
+##
+## Minimum number of bach processes of message
+mq.num_of_batch_process_min = 100
 
-## MQ rebalance objects
-mq.rebalance.num_of_batch_process = 1
-mq.rebalance.interval_min = 1
-mq.rebalance.interval_max = 1
+## Maxmim  number of bach processes of message
+mq.num_of_batch_process_max = 10000
 
-## MQ delete objects
-mq.delete_object.num_of_batch_process = 1
-mq.delete_object.interval_min = 1
-mq.delete_object.interval_max = 1
+## Maxmim  number of bach processes of message
+mq.num_of_batch_process_regular = 1000
 
-## MQ recover a node's object
-mq.recovery_node.num_of_batch_process = 1
-mq.recovery_node.interval_min = 8
-mq.recovery_node.interval_max = 32
+## Maxmim  number of bach processes of message
+mq.num_of_batch_process_step = 100
 
 
 ## --------------------------------------------------------------------
@@ -103,7 +232,7 @@ mq.recovery_node.interval_max = 32
 ## STORAGE - Log
 ## --------------------------------------------------------------------
 ## Log level: [0:debug, 1:info, 2:warn, 3:error]
-## log.log_level = 1
+log.log_level = 0
 
 ## Output log file(s) - Erlang's log
 ## log.erlang = ./log/erlang
@@ -117,6 +246,9 @@ mq.recovery_node.interval_max = 32
 ## Output log file(s) - ring
 ## log.ring_dir = ./log/ring
 
+## Output data-diagnosis log
+log.is_enable_diagnosis_log = true
+
 
 ## --------------------------------------------------------------------
 ## STORAGE - Other Directories
@@ -125,7 +257,7 @@ mq.recovery_node.interval_max = 32
 ## queue_dir  = ./work/queue
 
 ## Directory of SNMP agent configuration
-## snmp_agent = ./snmp/snmpa_storage_0/LEO-STORAGE
+snmp_agent = ./snmp/snmpa_storage_0/LEO-STORAGE
 
 
 ## --------------------------------------------------------------------
@@ -148,7 +280,7 @@ leo_object_storage.metadata_storage = leveldb
 ## RPC
 ## --------------------------------------------------------------------
 ## RPC-Server's acceptors
-rpc.server.acceptors = 128
+rpc.server.acceptors = 16
 
 ## RPC-Server's listening port number
 rpc.server.listen_port = 13077
@@ -157,10 +289,10 @@ rpc.server.listen_port = 13077
 rpc.server.listen_timeout = 30000
 
 ## RPC-Client's size of connection pool
-rpc.client.connection_pool_size = 4
+rpc.client.connection_pool_size = 16
 
 ## RPC-Client's size of connection buffer
-rpc.client.connection_buffer_size = 4
+rpc.client.connection_buffer_size = 16
 
 
 ## --------------------------------------------------------------------