Support newer `merges` format in tokenizer.json files #952

	name: CI
	on:
	pull_request:
	push:
	branches:
	- 'main'
	- 'ci/**'
	jobs:
	ci:
	strategy:
	matrix:
	os: [ubuntu-latest, macos-14]
	runs-on: ${{ matrix.os }}
	steps:
	- name: Checkout
	uses: actions/checkout@v3
	- name: Install Rust wasm toolchain
	run: rustup target add wasm32-unknown-unknown
	if: ${{ matrix.os == 'ubuntu-latest' }}
	- name: Cache
	uses: actions/cache@v3
	with:
	path: \|
	~/.cargo/
	target/
	key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
	- name: Install wasm-bindgen
	# nb. wasm-bindgen-cli version must match `wasm-bindgen` version in Cargo.lock
	run: cargo install wasm-bindgen-cli --version 0.2.92
	if: ${{ matrix.os == 'ubuntu-latest' }}
	- name: Build
	run: cargo build
	- name: WASM build
	run: make wasm
	if: ${{ matrix.os == 'ubuntu-latest' }}
	- name: Test
	run: make test
	- name: Lint
	run: \|
	make checkformatting
	make lint
	- name: Docs
	run: \|
	make docs
	- name: Setup Python
	run: \|
	python -m venv .venv
	.venv/bin/pip install --upgrade pip
	- name: Python Lint
	run: \|
	source .venv/bin/activate
	cd rten-convert
	pip install -e .
	pip install -r requirements.dev.txt
	make check

Provide feedback