Skip to content

Commit

Permalink
Align buffer blocks in saved state.
Browse files Browse the repository at this point in the history
This wastes some bytes for the rare irregular buffer, but it opens up the possibility of running block-based deduplication that reaches across multiple large system components. For example, we can detect when a block from disk has one or more copies in main memory.

In a small experiment I ran to choose an optimal block size for memory blocks, I found 512 was the best (balancing frequency of block repetition with the storage needed to store block ids). However, a block size of 256 was nearly as good. Further, this block size matches the one used in the Buffer system for caching lazily loaded disk blocks. As a result, it seems like the best overall choice.
  • Loading branch information
rndmcnlly committed Feb 1, 2025
1 parent 85ec802 commit 2bc50fc
Showing 1 changed file with 7 additions and 4 deletions.
11 changes: 7 additions & 4 deletions src/state.js
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@ var STATE_INDEX_INFO_LEN = 3;
/** @const */
var STATE_INFO_BLOCK_START = 16;

/** @const */
var STATE_BUFFER_ALIGN_BYTES = 256;

const ZSTD_MAGIC = 0xFD2FB528;

/** @constructor */
Expand Down Expand Up @@ -143,7 +146,7 @@ CPU.prototype.save_state = function()
total_buffer_size += len;

// align
total_buffer_size = total_buffer_size + 3 & ~3;
total_buffer_size = total_buffer_size + STATE_BUFFER_ALIGN_BYTES & ~STATE_BUFFER_ALIGN_BYTES;
}

var info_object = JSON.stringify({
Expand All @@ -153,7 +156,7 @@ CPU.prototype.save_state = function()
var info_block = new TextEncoder().encode(info_object);

var buffer_block_start = STATE_INFO_BLOCK_START + info_block.length;
buffer_block_start = buffer_block_start + 3 & ~3;
buffer_block_start = buffer_block_start + STATE_BUFFER_ALIGN_BYTES & ~STATE_BUFFER_ALIGN_BYTES;
var total_size = buffer_block_start + total_buffer_size;

//console.log("State: json_size=" + Math.ceil(buffer_block_start / 1024 / 1024) + "MB " +
Expand Down Expand Up @@ -257,7 +260,7 @@ CPU.prototype.restore_state = function(state)

for(const buffer_info of buffer_infos)
{
const front_padding = (position + 3 & ~3) - position;
const front_padding = (position + STATE_BUFFER_ALIGN_BYTES -1 & ~STATE_BUFFER_ALIGN_BYTES) - position;
const CHUNK_SIZE = 1 * 1024 * 1024;

if(buffer_info.length > CHUNK_SIZE)
Expand Down Expand Up @@ -312,7 +315,7 @@ CPU.prototype.restore_state = function(state)
let state_object = info_block_obj["state"];
const buffer_infos = info_block_obj["buffer_infos"];
let buffer_block_start = STATE_INFO_BLOCK_START + info_block_len;
buffer_block_start = buffer_block_start + 3 & ~3;
buffer_block_start = buffer_block_start + STATE_BUFFER_ALIGN_BYTES - 1 & ~STATE_BUFFER_ALIGN_BYTES;

const buffers = buffer_infos.map(buffer_info => {
const offset = buffer_block_start + buffer_info.offset;
Expand Down

0 comments on commit 2bc50fc

Please sign in to comment.