tokenizer-models.js

export const tokenizerModels = [
    {
      "model_name": "Xenova/all-MiniLM-L6-v2",
      "max_tokens": 512,
    },
    {
      "model_name": "Xenova/paraphrase-multilingual-MiniLM-L12-v2",
      "max_tokens": 512,
    },
    {
      "model_name": "Xenova/bert-base-uncased",
      "max_tokens": 512,
    },
    {
      "model_name": "Xenova/gpt2",
      "max_tokens": 512,
    },
    {
      "model_name": "Xenova/roberta-base",
      "max_tokens": 512,
    },
    {
      "model_name": "Xenova/all-distilroberta-v1",
      "max_tokens": 512,
    },
    {
      "model_name": "Xenova/multilingual-e5-large",
      "max_tokens": 512,
    },
    {
      "model_name": "Xenova/bert-base-multilingual-uncased",
      "max_tokens": 512,
    },
    {
      "model_name": "Xenova/xlm-roberta-base",
      "max_tokens": 512,
    },
    {
      "model_name": "BAAI/bge-base-en-v1.5",
      "max_tokens": 512,
    }
  ]